From 2de72c94848176ed319059b9d8469bf004cf7f08 Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:15:16 -0500 Subject: [PATCH 01/17] feat: enforce database runtime contract --- .github/workflows/ci.yml | 24 +++ Containerfile | 2 +- README.md | 7 + compose.yaml | 4 +- container/entrypoint.sh | 172 ++++++++++++++-- docs/CI.md | 5 +- docs/DEPLOYMENT.md | 407 +++++++++++++++++++++++++++++++++++++ docs/ROADMAP.md | 8 + docs/RUNTIME-SECURITY.md | 101 +++++++++ docs/STORAGE.md | 128 ++++++++++++ tests/minor-update.sh | 111 ++++++++++ tests/runtime-security.sh | 230 +++++++++++++++++++++ tests/smoke.sh | 20 +- tests/storage-lifecycle.sh | 240 ++++++++++++++++++++++ tests/tls.sh | 176 ++++++++++++++++ 15 files changed, 1619 insertions(+), 16 deletions(-) create mode 100644 docs/DEPLOYMENT.md create mode 100644 docs/RUNTIME-SECURITY.md create mode 100644 docs/STORAGE.md create mode 100644 tests/minor-update.sh create mode 100644 tests/runtime-security.sh create mode 100644 tests/storage-lifecycle.sh create mode 100644 tests/tls.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a051ba..78a1348 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -163,6 +163,30 @@ jobs: IMAGE: ${{ env.TEST_IMAGE }} run: bash tests/smoke.sh + - name: Test authentication and configuration security + env: + CONTAINER_RUNTIME: docker + IMAGE: ${{ env.TEST_IMAGE }} + run: bash tests/runtime-security.sh + + - name: Test durable storage and lifecycle + env: + CONTAINER_RUNTIME: docker + IMAGE: ${{ env.TEST_IMAGE }} + run: bash tests/storage-lifecycle.sh + + - name: Test a preserved-data PostgreSQL 18 minor update + env: + CONTAINER_RUNTIME: docker + IMAGE: ${{ env.TEST_IMAGE }} + run: bash tests/minor-update.sh + + - name: Test the TLS server profile + env: + CONTAINER_RUNTIME: docker + IMAGE: ${{ env.TEST_IMAGE }} + run: bash tests/tls.sh + - name: Scan image with Trivy uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 with: diff --git a/Containerfile b/Containerfile index 7b56471..8148d7a 100644 --- a/Containerfile +++ b/Containerfile @@ -81,7 +81,7 @@ WORKDIR /var/lib/pgsql EXPOSE 5432 HEALTHCHECK --interval=10s --timeout=5s --start-period=30s --retries=5 \ - CMD ["/usr/pgsql-18/bin/psql", "--quiet", "--host=/tmp", "--username=postgres", "--dbname=postgres", "--command=SELECT 1"] + CMD ["/usr/pgsql-18/bin/pg_isready", "--quiet", "--host=/tmp", "--timeout=3"] STOPSIGNAL SIGINT diff --git a/README.md b/README.md index d14b767..ac4fb00 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,13 @@ publisher keys, source RPMs, and digest-pinned UBI inputs. Artifact acquisition is separate from a network-disabled, pull-disabled container build; see the [artifact acquisition contract](docs/ARTIFACT-ACQUISITION.md). +Detailed procedures for choosing, deploying, verifying, operating, updating, +recovering, and safely removing each intended profile are in the +[deployment and operations guide](docs/DEPLOYMENT.md). The +[runtime security contract](docs/RUNTIME-SECURITY.md) and +[storage, backup, and upgrade guide](docs/STORAGE.md) define the associated +security and data-lifecycle boundaries. + ## Approved first-release boundary The first release is scoped to native AMD64 and ARM64, an exact rootless diff --git a/compose.yaml b/compose.yaml index 9d47ee7..1fb2830 100644 --- a/compose.yaml +++ b/compose.yaml @@ -4,7 +4,9 @@ services: environment: POSTGRES_PASSWORD_FILE: /run/secrets/postgres-password secrets: - - postgres-password + - source: postgres-password + target: postgres-password + mode: 0440 ports: - "127.0.0.1:5432:5432" read_only: true diff --git a/container/entrypoint.sh b/container/entrypoint.sh index 29932e4..81b172f 100644 --- a/container/entrypoint.sh +++ b/container/entrypoint.sh @@ -2,12 +2,74 @@ set -eu postgres_major=18 +initialization_lock= +password_file= fatal() { printf 'postgresql-entrypoint: %s\n' "$*" >&2 exit 1 } +cleanup_initialization() { + test -z "${password_file}" || rm -f -- "${password_file}" + test -z "${initialization_lock}" || rmdir -- "${initialization_lock}" 2>/dev/null || true +} + +on_initialization_signal() { + cleanup_initialization + trap - EXIT HUP INT TERM + exit 1 +} + +reject_line_breaks() { + value=$1 + label=$2 + newline=' +' + carriage_return=$(printf '\r') + case "${value}" in + *"${newline}"*|*"${carriage_return}"*) fatal "${label} must not contain line breaks" ;; + esac +} + +require_regular_file() { + path=$1 + label=$2 + test ! -L "${path}" || fatal "${label} must not be a symbolic link" + test -f "${path}" || fatal "${label} must be a regular file" + test -r "${path}" || fatal "${label} is not readable" +} + +require_secret_permissions() { + path=$1 + label=$2 + permissions=$(stat -c '%a' "${path}") + owner=$(stat -c '%u' "${path}") + group=$(stat -c '%g' "${path}") + + case "${permissions}" in + 400|440|600|640) ;; + *) fatal "${label} permissions must be 0400, 0440, 0600, or 0640" ;; + esac + current_uid=$(id -u) + test "${owner}" = 0 || test "${owner}" = "${current_uid}" || \ + fatal "${label} must be owned by UID 0 or the runtime UID" + case "${permissions}" in + 440|640) + test "${group}" = 0 || \ + fatal "${label} group-readable files must be owned by GID 0" + ;; + esac +} + +require_public_file_permissions() { + path=$1 + label=$2 + permissions=$(stat -c '%a' "${path}") + test $((0${permissions} & 07022)) -eq 0 || \ + fatal "${label} must not be group/other writable or have special permission bits" +} + configure_arbitrary_uid() { if id -un >/dev/null 2>&1; then return @@ -31,21 +93,44 @@ read_initial_password() { fi if test -n "${POSTGRES_PASSWORD_FILE:-}"; then - test -r "${POSTGRES_PASSWORD_FILE}" || \ - fatal "POSTGRES_PASSWORD_FILE is not readable" - POSTGRES_PASSWORD=$(cat "${POSTGRES_PASSWORD_FILE}") + require_regular_file "${POSTGRES_PASSWORD_FILE}" POSTGRES_PASSWORD_FILE + require_secret_permissions "${POSTGRES_PASSWORD_FILE}" POSTGRES_PASSWORD_FILE + password_lines=$(wc -l <"${POSTGRES_PASSWORD_FILE}") + test "${password_lines}" -eq 0 || \ + fatal "POSTGRES_PASSWORD_FILE must contain exactly one value without a line break" + POSTGRES_PASSWORD=$(cat -- "${POSTGRES_PASSWORD_FILE}") fi test -n "${POSTGRES_PASSWORD:-}" || \ fatal "initialization requires POSTGRES_PASSWORD or POSTGRES_PASSWORD_FILE" + reject_line_breaks "${POSTGRES_PASSWORD}" POSTGRES_PASSWORD +} + +write_host_authentication() { + umask 077 + { + printf '# Managed by postgresql-ubi; replaced on every start.\n' + printf 'local all all trust\n' + if test -n "${POSTGRESQL_TLS_CERT_FILE:-}"; then + printf 'hostssl all all 0.0.0.0/0 scram-sha-256\n' + printf 'hostssl all all ::/0 scram-sha-256\n' + else + printf 'host all all 0.0.0.0/0 scram-sha-256\n' + printf 'host all all ::/0 scram-sha-256\n' + fi + } >"${PGDATA}/pg_hba.conf" } initialize_database() { read_initial_password + initialization_lock="${PGDATA}.postgresql-ubi.init.lock" + mkdir -- "${initialization_lock}" 2>/dev/null || \ + fatal "initialization is already running or a stale initialization lock exists: ${initialization_lock}" umask 077 password_file="/tmp/postgresql-password.$$" printf '%s' "${POSTGRES_PASSWORD}" >"${password_file}" - trap 'rm -f "${password_file}"' EXIT HUP INT TERM + trap cleanup_initialization EXIT + trap on_initialization_signal HUP INT TERM initdb \ --pgdata="${PGDATA}" \ @@ -53,42 +138,105 @@ initialize_database() { --pwfile="${password_file}" \ --auth-host=scram-sha-256 \ --auth-local=trust \ + --data-checksums \ --encoding=UTF8 - printf "\nlisten_addresses = '*'\nunix_socket_directories = '/tmp'\nlogging_collector = off\npassword_encryption = 'scram-sha-256'\n" \ - >>"${PGDATA}/postgresql.conf" - printf '\nhost all all all scram-sha-256\n' >>"${PGDATA}/pg_hba.conf" + write_host_authentication pg_ctl --pgdata="${PGDATA}" \ - --options="-c listen_addresses='' -c unix_socket_directories=/tmp" \ + --options="-c listen_addresses='' -c unix_socket_directories=/tmp -c password_encryption=scram-sha-256 -c hba_file=${PGDATA}/pg_hba.conf -c logging_collector=off" \ --wait start database=${POSTGRES_DB:-${POSTGRES_USER}} + reject_line_breaks "${database}" POSTGRES_DB if test "${database}" != "${POSTGRES_USER}"; then createdb --host=/tmp --username="${POSTGRES_USER}" -- "${database}" fi pg_ctl --pgdata="${PGDATA}" --mode=fast --wait stop - rm -f "${password_file}" + cleanup_initialization + initialization_lock= + password_file= trap - EXIT HUP INT TERM unset POSTGRES_PASSWORD } -if test "${1:-}" = "postgres"; then +validate_runtime_files() { + if test -n "${POSTGRESQL_CONFIG_FILE:-}"; then + require_regular_file "${POSTGRESQL_CONFIG_FILE}" POSTGRESQL_CONFIG_FILE + require_public_file_permissions "${POSTGRESQL_CONFIG_FILE}" POSTGRESQL_CONFIG_FILE + fi + + if test -n "${POSTGRESQL_TLS_CERT_FILE:-}" || test -n "${POSTGRESQL_TLS_KEY_FILE:-}"; then + test -n "${POSTGRESQL_TLS_CERT_FILE:-}" && test -n "${POSTGRESQL_TLS_KEY_FILE:-}" || \ + fatal "set both POSTGRESQL_TLS_CERT_FILE and POSTGRESQL_TLS_KEY_FILE" + require_regular_file "${POSTGRESQL_TLS_CERT_FILE}" POSTGRESQL_TLS_CERT_FILE + require_public_file_permissions "${POSTGRESQL_TLS_CERT_FILE}" POSTGRESQL_TLS_CERT_FILE + require_regular_file "${POSTGRESQL_TLS_KEY_FILE}" POSTGRESQL_TLS_KEY_FILE + require_secret_permissions "${POSTGRESQL_TLS_KEY_FILE}" POSTGRESQL_TLS_KEY_FILE + fi +} + +if test "${1:-}" = postgres; then + # Supported by the UBI /bin/sh; POSIX leaves ulimit options unspecified. + # shellcheck disable=SC3045 + ulimit -c 0 || fatal "cannot disable core dumps" + reject_line_breaks "${POSTGRES_USER}" POSTGRES_USER configure_arbitrary_uid - mkdir -p "${PGDATA}" + test ! -L "${PGDATA}" || fatal "PGDATA must not be a symbolic link" + mkdir -p -- "${PGDATA}" 2>/dev/null || fatal "cannot create PGDATA: ${PGDATA}" + test -d "${PGDATA}" || fatal "PGDATA is not a directory: ${PGDATA}" test -w "${PGDATA}" || fatal "PGDATA is not writable: ${PGDATA}" + writability_probe="${PGDATA}/.postgresql-ubi-write-test.$$" + (umask 077 && : >"${writability_probe}") 2>/dev/null || \ + fatal "PGDATA cannot accept a validation write: ${PGDATA}" + rm -f -- "${writability_probe}" || \ + fatal "PGDATA validation file cannot be removed: ${PGDATA}" if test ! -s "${PGDATA}/PG_VERSION"; then if test -n "$(find "${PGDATA}" -mindepth 1 -maxdepth 1 -print -quit)"; then - fatal "PGDATA is non-empty but has no PG_VERSION file" + fatal "PGDATA is non-empty but has no PG_VERSION file; refusing automatic recovery or reinitialization" fi initialize_database else installed_major=$(cat "${PGDATA}/PG_VERSION") test "${installed_major}" = "${postgres_major}" || \ fatal "PGDATA major ${installed_major} is incompatible with PostgreSQL ${postgres_major}" + write_host_authentication + fi + + unset POSTGRES_PASSWORD POSTGRES_PASSWORD_FILE + validate_runtime_files + + if test -n "${POSTGRESQL_CONFIG_FILE:-}"; then + set -- "$@" -c "config_file=${POSTGRESQL_CONFIG_FILE}" + fi + if test -n "${POSTGRESQL_TLS_CERT_FILE:-}"; then + set -- "$@" \ + -c ssl=on \ + -c "ssl_cert_file=${POSTGRESQL_TLS_CERT_FILE}" \ + -c "ssl_key_file=${POSTGRESQL_TLS_KEY_FILE}" \ + -c ssl_min_protocol_version=TLSv1.2 \ + -c ssl_max_protocol_version=TLSv1.3 + else + set -- "$@" -c ssl=off fi + set -- "$@" \ + -c "data_directory=${PGDATA}" \ + -c listen_addresses='*' \ + -c port=5432 \ + -c password_encryption=scram-sha-256 \ + -c "hba_file=${PGDATA}/pg_hba.conf" \ + -c unix_socket_directories=/tmp \ + -c logging_collector=off \ + -c log_destination=stderr \ + -c log_statement=none \ + -c log_min_duration_statement=-1 \ + -c log_parameter_max_length=0 \ + -c log_parameter_max_length_on_error=0 \ + -c log_connections=on \ + -c log_disconnections=on \ + -c log_checkpoints=on fi exec "$@" diff --git a/docs/CI.md b/docs/CI.md index b195959..9279b21 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -2,7 +2,10 @@ CI runs repository validation, configuration scanning, native AMD64 and ARM64 image builds, restricted-runtime smoke tests, Trivy and Grype vulnerability -gates, and Syft SPDX SBOM generation. The aggregate `image` job fails unless +gates, and Syft SPDX SBOM generation. Runtime tests cover authentication and +configuration precedence, initialization failure modes, durable-state +lifecycle and crash recovery, logical backup/restoration, and the TLS profile. +The aggregate `image` job fails unless every native image job succeeds. Each image job acquires the architecture's committed lock, verifies the bundle and full key fingerprints, pre-pulls only the digest-pinned bases, and performs a clean build with network access and diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md new file mode 100644 index 0000000..55f48e0 --- /dev/null +++ b/docs/DEPLOYMENT.md @@ -0,0 +1,407 @@ +# Deployment and operations guide + +This guide describes every currently intended first-release deployment use +case. No supported image has been released yet, so examples use a locally +built development image. When releases exist, replace `IMAGE` with an +immutable `ghcr.io/datopsis/postgresql-ubi@sha256:` reference; never +substitute a mutable tag. + +## Choose a deployment profile + +| Use case | Runtime identity | Storage | Transport | Procedure | +| --- | --- | --- | --- | --- | +| Rootless Podman baseline | `26:0` | Named volume | TLS | Sections 1-4 and 7 | +| OpenShift-style identity evaluation | Arbitrary UID, GID `0` | Named volume or prepared bind mount | TLS | Sections 1-3, 5, and 7 | +| Host-managed database path | `26:0` or arbitrary UID/GID `0` | SELinux-labeled bind mount | TLS | Sections 1-3, 6, and 7 | +| Docker compatibility | `26:0` | Named volume | TLS | Sections 1-4 and 7, using Docker syntax | +| Compose development | `26:0` | Compose named volume | Isolated clear text or mounted TLS | Section 8 | +| Controlled network | Either qualified identity | Pre-provisioned storage | TLS | Sections 1-7 and 13 | + +The non-TLS profile permits SCRAM-authenticated but unencrypted TCP traffic. +Use it only when traffic is confined by a separately enforced local or private +network boundary. The TLS profile changes host rules to `hostssl`, so clear-text +TCP connections are rejected rather than opportunistically accepted. + +## 1. Establish prerequisites and ownership + +1. Select the exact image digest, CPU architecture, runtime, host, storage + implementation, network boundary, and TLS profile. Record these values in + the deployment change record. +2. Install rootless Podman on the qualified RHEL 9 baseline. Docker is a CI + compatibility target until a release support statement says otherwise. +3. Keep SELinux enforcing on RHEL. Do not disable labels to work around a mount + denial; prepare the label as described below. +4. Provision durable storage separately from the container lifecycle. The + volume must support PostgreSQL durability, locking, permissions, atomic file + operations, and sufficient space and inodes. A persistent volume is not a + backup. +5. Assign separate accountable owners for the database, storage, backups, + certificate lifecycle, network policy, logging/SIEM, and image updates. +6. Set a termination grace period of at least 30 seconds as an initial value. + Measure real shutdown and recovery behavior under peak workload before + selecting a production value. + +Set the runtime and immutable image reference for the remaining commands: + +```console +export RUNTIME=podman +export IMAGE=localhost/postgresql-ubi:development +``` + +For a published release, verify its digest, signature, attestation, SBOM, and +support statement before creating storage or credentials. Release verification +commands will be published with Package 4; development builds are not a +substitute for that gate. + +## 2. Prepare the initialization credential + +The preferred interface is a mounted file. Create it without a trailing line +break and make it readable only by its owner, or by group `0` when the runtime +uses an arbitrary UID: + +```console +umask 077 +printf '%s' 'replace-with-a-random-secret' > postgres-password +chmod 0400 postgres-password +``` + +The entrypoint accepts only modes `0400`, `0440`, `0600`, or `0640`, rejects +symbolic links, rejects line breaks and empty values, and requires ownership by +UID `0` or the runtime UID. Group-readable files must use GID `0`. A CSI secret +driver that exposes symlinks is not compatible with this interface; configure +the driver to present a regular file or copy the secret into a protected +ephemeral volume before startup. + +`POSTGRES_PASSWORD` is supported for bootstrap compatibility, but container +inspect APIs can retain its original value even after the process removes it +from the PostgreSQL environment. Use the file interface for deployments. Do +not place the value in shell history, command arguments, Compose YAML, Git, +logs, tickets, or qualification evidence. + +The bootstrap value is used only when `PGDATA` is empty. It does not rotate an +existing role password. + +## 3. Create and verify the network boundary + +1. Permit port 5432 only from intended application, administration, backup, and + monitoring sources. +2. Do not publish `0.0.0.0:5432` on a workstation. Bind a specific protected + address or use an orchestrator network policy. +3. For TLS, distribute the issuing CA to clients through the organization's + trust process and require `sslmode=verify-full` with the expected DNS name. +4. For the isolated non-TLS profile, prove that traffic cannot leave or enter + the protected boundary. SCRAM protects the password exchange but does not + encrypt query results or other session traffic. + +## 4. Deploy with a named volume and fixed UID + +Create the volume independently so replacing the container cannot delete it: + +```console +$RUNTIME volume create postgresql-data +``` + +Start the isolated non-TLS profile for evaluation: + +```console +$RUNTIME run --detach --name postgresql \ + --read-only \ + --tmpfs /tmp:rw,noexec,nosuid,nodev,size=64m,mode=1777 \ + --mount type=volume,src=postgresql-data,dst=/var/lib/pgsql \ + --mount type=bind,src="$PWD/postgres-password",dst=/run/secrets/postgres-password,readonly \ + --env POSTGRES_PASSWORD_FILE=/run/secrets/postgres-password \ + --publish 127.0.0.1:5432:5432 \ + --cap-drop ALL \ + --security-opt no-new-privileges \ + --stop-signal SIGINT \ + "$IMAGE" +``` + +Podman may require `--security-opt no-new-privileges`; Docker accepts +`--security-opt no-new-privileges:true`. Verify initialization and the effective +security boundary: + +```console +$RUNTIME logs postgresql +$RUNTIME exec postgresql pg_isready --host=/tmp --timeout=3 +$RUNTIME exec --env PGPASSWORD='replace-with-a-random-secret' postgresql \ + psql --host=127.0.0.1 --username=postgres --dbname=postgres \ + --command='SELECT version(), current_user;' +$RUNTIME exec postgresql sh -c \ + 'id; grep -E "^(NoNewPrivs|CapEff):" /proc/1/status; ulimit -c' +``` + +Expected results are UID `26`, GID `0`, `NoNewPrivs: 1`, an all-zero effective +capability mask, core limit `0`, PostgreSQL 18.6, and an authenticated query. +Inspect logs for errors and confirm that no credential value appears. + +## 5. Deploy with an arbitrary UID + +Use an allowed nonzero UID and supplemental/primary GID `0`. The named volume +inherits a group-writable setgid layout from the image: + +```console +$RUNTIME volume create postgresql-arbitrary-data +$RUNTIME run --detach --name postgresql-arbitrary \ + --user 10001:0 \ + --read-only \ + --tmpfs /tmp:rw,noexec,nosuid,nodev,size=64m,mode=1777 \ + --mount type=volume,src=postgresql-arbitrary-data,dst=/var/lib/pgsql \ + --mount type=bind,src="$PWD/postgres-password",dst=/run/secrets/postgres-password,readonly \ + --env POSTGRES_PASSWORD_FILE=/run/secrets/postgres-password \ + --cap-drop ALL \ + --security-opt no-new-privileges \ + "$IMAGE" +``` + +The entrypoint uses `nss_wrapper` only when the UID is absent from `/etc/passwd`. +It never changes privilege. Confirm `id -u` is `10001`, all PostgreSQL processes +use that UID, and the effective capability mask is zero. This is not an +OpenShift support claim; restricted-SCC qualification remains a separate gate. + +## 6. Deploy with a bind mount + +Stop before changing ownership. For fixed UID operation: + +```console +sudo install -d -o 26 -g 0 -m 2770 /srv/postgresql-ubi +sudo semanage fcontext -a -t container_file_t '/srv/postgresql-ubi(/.*)?' +sudo restorecon -Rv /srv/postgresql-ubi +``` + +If local policy permits Podman-managed private relabeling, `-v +/srv/postgresql-ubi:/var/lib/pgsql:Z` is an alternative. Do not use recursive +`chown` against an existing database without a reviewed backup and rollback +plan. For arbitrary UID `10001:0`, use owner `10001`, group `0`, and mode +`2770`, accounting for the rootless user-namespace mapping (`podman unshare +chown` may be required). + +NFS root-squash commonly prevents the runtime UID from establishing ownership +and can expose unsuitable locking or durability semantics. Have the storage +administrator pre-provision ownership and validate PostgreSQL behavior; do not +disable root-squash as a shortcut. For CSI/PVC storage, record the driver, +access mode, `fsGroup`/UID behavior, reclaim policy, snapshot semantics, and +tested failure behavior. Only one PostgreSQL server may mount a v1 data volume +for write access. + +Replace the named-volume mount in Section 4 with: + +```console +--mount type=bind,src=/srv/postgresql-ubi,dst=/var/lib/pgsql +``` + +If startup reports that `PGDATA` cannot be created or written, fix host/CSI +ownership, mapping, mount state, capacity, or SELinux policy. Never run the +database as root or grant broad capabilities to bypass the failure. + +## 7. Enable the TLS 1.2/1.3 profile + +Obtain a server key and certificate chain from the deployment PKI. The +certificate needs the client-visible DNS name in its SAN. Keep the unencrypted +private key outside the image and Git repository: + +```console +chmod 0444 server-chain.crt +chmod 0440 server.key +sudo chown root:0 server-chain.crt server.key +``` + +Add these mounts and variables to the Section 4 or 5 command: + +```console +--mount type=bind,src="$PWD/server-chain.crt",dst=/run/tls/server.crt,readonly \ +--mount type=bind,src="$PWD/server.key",dst=/run/tls/server.key,readonly \ +--env POSTGRESQL_TLS_CERT_FILE=/run/tls/server.crt \ +--env POSTGRESQL_TLS_KEY_FILE=/run/tls/server.key +``` + +Both variables are mandatory together. The key must meet the same strict file +rules as the password. The profile pins TLS 1.2 through 1.3 and replaces all +network HBA records with `hostssl ... scram-sha-256`. It does not enable client +certificate authentication or certificate-to-role mapping. + +Verify from a separate client boundary: + +```console +psql 'host=db.example.test dbname=postgres user=postgres sslmode=verify-full sslrootcert=/path/to/ca.crt' \ + --command="SELECT ssl, version FROM pg_stat_ssl WHERE pid=pg_backend_pid();" +``` + +Also prove that `sslmode=disable`, an incorrect hostname, and an untrusted CA +fail. Monitor certificate expiration. PostgreSQL reloads certificate files on +SIGHUP; stage complete replacement files atomically, send SIGHUP, establish a +new verified connection, and confirm the served serial. If validation fails, +restore the previous protected key/chain, send SIGHUP again, and reverify. Do +not remove the old material until the new connection is confirmed. + +PostgreSQL documents server key mode `0600`, or root ownership with group-read +mode `0640`; this image additionally accepts their read-only equivalents. See +[Secure TCP/IP connections](https://www.postgresql.org/docs/18/ssl-tcp.html) +and [libpq certificate verification](https://www.postgresql.org/docs/18/libpq-ssl.html). + +## 8. Deploy with Compose + +The repository `compose.yaml` is a local-development profile. Export the secret +without placing it in the YAML, then start and verify: + +```console +export POSTGRES_PASSWORD='replace-with-a-development-secret' +podman compose up --detach +podman compose ps +podman compose exec postgresql pg_isready --host=/tmp --timeout=3 +``` + +The service binds only `127.0.0.1`, uses a named volume, read-only root, +bounded `/tmp`, no capabilities, and no-new-privileges. For production, use a +deployment-specific secret provider and immutable image digest. Add the TLS +mounts and variables from Section 7 or keep the port inside a proven isolated +boundary. + +Stop without deleting data: + +```console +podman compose down +``` + +Delete the volume only after backup-retention and decommission approval: + +```console +podman compose down --volumes +``` + +## 9. Supply reviewed PostgreSQL configuration + +Create a regular, immutable file that is not group/other writable or marked +setuid/setgid/sticky. Mount it read-only and set: + +```console +--mount type=bind,src="$PWD/postgresql.conf",dst=/run/postgresql/postgresql.conf,readonly \ +--env POSTGRESQL_CONFIG_FILE=/run/postgresql/postgresql.conf +``` + +Validate in a disposable container against a copy of production-like data, +then start and inspect every effective value with `SHOW` or `pg_settings`. +Changes marked `pending_restart` require replacement/restart; reloadable values +can use `SELECT pg_reload_conf()`. Preserve the previous file and image digest +for rollback. + +The entrypoint applies these non-overridable values after mounted and +command-line configuration: SCRAM password storage, managed HBA path, `/tmp` +socket path, TLS profile state and protocol range, stderr-only logging, no +logging collector, no statement/duration/parameter logging, and connection, +disconnection, checkpoint, recovery, and shutdown event visibility. Attempts +to override them earlier on the command line are superseded. The managed HBA +file is replaced on every start. + +## 10. Create application roles and rotate credentials + +`POSTGRES_USER` creates the initial superuser; `POSTGRES_DB` optionally creates +one database. These variables are bootstrap inputs, not ongoing account +management. Connect through a protected administrative path and create a +separate least-privilege login: + +```sql +CREATE ROLE app LOGIN PASSWORD 'generated-secret' + NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION; +GRANT CONNECT ON DATABASE appdb TO app; +GRANT USAGE ON SCHEMA app TO app; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA app TO app; +ALTER DEFAULT PRIVILEGES IN SCHEMA app + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO app; +``` + +Tailor privileges to the application; the example is a starting point, not a +universal grant set. Rotate with an interactive protected administrator session +(`\password app`) or a secret-safe automation channel. Verify the new secret, +revoke the old one, update consumers, and avoid SQL text or command lines that +would retain the value. + +Init scripts under `/docker-entrypoint-initdb.d` are deliberately unsupported +for v1. Run reviewed schema migration tooling after readiness using a dedicated +role and transaction/retry policy. + +## 11. Configure probes, resources, and logs + +- The OCI healthcheck runs bounded `pg_isready` over the local socket. Treat it + as startup/readiness evidence, not proof that application transactions work. +- Use a TCP/TLS authenticated `SELECT 1` from outside the container for external + transaction monitoring with a dedicated low-privilege role. +- Do not use expensive SQL as a liveness probe; load-induced probe failures can + cause destructive restart loops. A liveness policy should distinguish a hung + process from saturation and allow recovery startup to finish. +- Start with `/tmp` at 64 MiB, `shm_size` at least 256 MiB, sufficient file + descriptors/PIDs for configured connections and workers, and a measured + memory limit above PostgreSQL shared memory plus per-session/workload demand. + These are safe test starting points, not universal production sizing values. +- Alert on volume bytes and inodes before exhaustion, connection saturation, + repeated authentication failures, checkpoints, recovery, OOM kills, and + abnormal shutdown. Size termination grace from observed checkpoint/shutdown + time and large-WAL crash recovery. + +Logs stay on stdout/stderr. Collect them with runtime/orchestrator metadata and +access controls. The immutable profile disables statement, duration, and bind +parameter logging because SQL and values can contain credentials, personal +data, regulated data, or application secrets. Database/user/application/client +identifiers can also be sensitive; choose non-personal identifiers and apply +retention/redaction controls. Broad statement logging is not a substitute for +an audit design. PostgreSQL audit extensions are not included in v1. + +## 12. Back up, restore, update, and recover + +Follow [Storage, backup, and upgrade operations](STORAGE.md). At minimum: + +1. define RPO/RTO, scope, owner, schedule, retention, immutability, encryption, + access, and deletion policy; +2. run `pg_dump` or `pg_dumpall` through a TLS-verified connection into a + separately protected destination; +3. record the image digest, PostgreSQL version, options, timestamps, and backup + digest without recording contents or credentials; +4. restore into an isolated empty database on a schedule; +5. validate schema, role handling, row counts, and application-specific content + digests, then record measured recovery time; and +6. retain the previous image digest and a verified pre-update backup. Do not + assume that starting older binaries on newer database files is a safe + rollback. + +## 13. Controlled-network deployment + +Use the verified bundle-transfer procedure in +[External artifact acquisition](ARTIFACT-ACQUISITION.md#controlled-network-transfer). +Transfer the immutable image by digest with its signature, attestations, SBOM, +scan results, lock, and verification instructions. Map the public digest to the +internal registry without rebuilding or changing bytes, configure internal CA +trust through the host/runtime trust mechanism, verify again inside the target +boundary, and refresh vulnerability databases through a separately controlled +process. Never embed repository credentials or private CAs in the image. + +## 14. Failure diagnosis and data-preserving removal + +- Missing/invalid credential: correct the protected secret only when the data + directory is empty. Credentials are not required for an existing cluster. +- Non-empty directory without `PG_VERSION`: stop and preserve it. Investigate + interrupted initialization or a wrong mount; never delete or reinitialize it + automatically. +- Initialization lock: confirm no initializer is running. Preserve partial + contents and obtain backup/storage-owner approval before removing a stale + `.postgresql-ubi.init.lock`. +- Wrong PostgreSQL major: do not edit `PG_VERSION`. Follow the major-upgrade + decision in `STORAGE.md`. +- Read-only, permission, SELinux, NFS, CSI, disk-full, or inode-full failure: + correct the underlying storage condition while stopped. Do not run as root. +- Crash/OOM/forced termination: preserve the volume, allow WAL recovery, review + logs, verify application data, and take a fresh backup after recovery. + +Remove only the container while preserving data: + +```console +$RUNTIME stop --time 30 postgresql +$RUNTIME inspect postgresql --format '{{.State.ExitCode}}' +$RUNTIME rm postgresql +$RUNTIME volume inspect postgresql-data +``` + +Destructive decommissioning requires a verified retained backup or approved +retention expiry, consumer shutdown, credential/certificate revocation, legal +and records approval, and storage-specific secure deletion. Only then remove +the named volume explicitly. Container removal alone is not evidence that +database blocks, backups, WAL, snapshots, logs, or secrets were destroyed. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index b7658c6..b3246e5 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -228,6 +228,14 @@ unexpected, wrong-base, and source-mismatch cases fail closed. ### Durable storage and lifecycle +- [ ] Publish detailed, executable deployment playbooks for every v1 use case: + fixed UID and arbitrary UID, rootless Podman and Docker/Compose, named volumes + and SELinux-labeled bind mounts, TLS and intentionally isolated non-TLS + profiles, mounted configuration, controlled-network transfer, backup/restore, + minor update, rollback, failure recovery, and teardown. Each playbook must + state prerequisites, trust and ownership boundaries, every deployment step, + expected verification evidence, security-sensitive alternatives, failure + diagnostics, and data-preserving removal steps. - [ ] Document named-volume and bind-mount ownership for UID `26:0` and arbitrary UID/group `0`, including SELinux labels, NFS root-squash, CSI/PVC behavior, filesystem permissions, and safe failure diagnostics. diff --git a/docs/RUNTIME-SECURITY.md b/docs/RUNTIME-SECURITY.md new file mode 100644 index 0000000..a6017c2 --- /dev/null +++ b/docs/RUNTIME-SECURITY.md @@ -0,0 +1,101 @@ +# Runtime security contract + +## Initialization and authentication + +Initialization is an all-or-stop operation for an empty `PGDATA`. An atomic +lock prevents concurrent initialization. A signal removes transient password +and lock files; any partial data remains deliberately blocked as a non-empty +directory without `PG_VERSION` and requires operator investigation. + +`POSTGRES_PASSWORD_FILE` is preferred and subject to regular-file, symlink, +ownership, mode, readability, non-empty, and no-line-break checks. +`POSTGRES_PASSWORD` remains a compatibility interface; the container runtime's +configuration metadata can retain it even after it is unset from the final +PostgreSQL process. Passwords are passed to `initdb` only through a protected +temporary file, never command arguments. Core dumps are disabled. + +The generated HBA file is short enough to audit completely and is replaced on +every start: + +```text +# Managed by postgresql-ubi; replaced on every start. +local all all trust +host all all 0.0.0.0/0 scram-sha-256 +host all all ::/0 scram-sha-256 +``` + +The TLS profile changes both `host` records to `hostssl`. Local trust is +limited to the container's `/tmp` Unix socket and is not a host/network escape +hatch. Network passwords and every newly stored role password use +SCRAM-SHA-256. The image exposes no `POSTGRES_HOST_AUTH_METHOD` option. + +`POSTGRES_USER` names the initial superuser and defaults to `postgres`; +`POSTGRES_DB` optionally creates one database. Neither variable changes an +existing cluster. Use PostgreSQL role administration for rotation and +least-privilege application roles as shown in `DEPLOYMENT.md`. + +## Configuration precedence + +Precedence from lowest to highest is PostgreSQL compiled defaults, the mounted +`POSTGRESQL_CONFIG_FILE`, user command-line options, and image-enforced final +options. The final options protect HBA location, SCRAM password storage, TLS +mode/protocol, Unix socket, stderr logging, statement/value exclusion, and core +operational events. `postgresql.auto.conf` cannot supersede command-line +settings. The server itself rejects malformed or unknown configuration before +accepting connections. + +Inspect `pg_settings.source`, `sourcefile`, `pending_restart`, and effective +values after every change. Reload only reloadable parameters; replace/restart +for postmaster parameters. Retain the prior config and image for rollback. + +## Logging and probes + +PostgreSQL writes to stderr for runtime collection. Statement, duration, and +bind-parameter logging are forced off to prevent broad SQL/value capture. +Connection, disconnection, authentication failure, checkpoint, recovery, and +shutdown events remain available. Identifiers and client addresses may still +be sensitive operational metadata and need access, retention, and redaction +controls. + +The OCI healthcheck is bounded `pg_isready` on the local socket. It indicates +that the server is accepting connections but does not authenticate or prove an +application transaction. External monitoring must use TLS verification and a +dedicated low-privilege role. Liveness must tolerate startup/recovery and avoid +restarting a merely saturated database. + +`pgaudit` and all other audit extensions are deferred. Adding one requires new +locked RPM/source provenance, configuration, performance, log-volume, +vulnerability, upgrade, privacy, and maintenance decisions. + +## Deliberately unsupported interfaces + +- `/docker-entrypoint-initdb.d` execution is not implemented. It creates + ordering, secret, retry, partial-execution, and ownership semantics that are + not safe to imply without a separate contract. +- Client-certificate authentication and certificate-to-role mapping are not in + the HBA policy. Server TLS does not imply mTLS. +- Remote trust authentication, root execution, privilege transition, added + capabilities, writable root filesystems, and automatic reinitialization are + not supported escape hatches. + +## Comparison with the Docker Official Image + +| Area | `postgresql-ubi` v1 contract | Docker Official `postgres` image | Compatibility consequence | +| --- | --- | --- | --- | +| Base/runtime | UBI 9 Micro final image; no package manager | Debian and Alpine variants | Package paths and utilities differ. | +| User | Fixed `26:0`; arbitrary UID with GID `0` | `postgres` user; mostly arbitrary UID behavior | Do not assume identical UID or ownership. | +| Data mount | `/var/lib/pgsql`, with `PGDATA=/var/lib/pgsql/data` | PostgreSQL 18 uses `/var/lib/postgresql`, with versioned `PGDATA` | Volume paths are not interchangeable. | +| Required secret | Non-empty password or strict regular `_FILE`; no line breaks/symlinks | `POSTGRES_PASSWORD` and `_FILE` supported | This image intentionally rejects more file layouts/modes. | +| Host auth | Remote SCRAM only; TLS profile requires `hostssl`; no trust override | Configurable `POSTGRES_HOST_AUTH_METHOD`, including documented trust option | Trust-based examples are incompatible by design. | +| Checksums | Always enabled | Optional through initdb arguments | Existing data characteristics can differ. | +| Init arguments/scripts | Fixed initdb policy; init scripts unsupported | `POSTGRES_INITDB_ARGS`, WAL-dir option, ordered shell/SQL init scripts | Move schema setup to explicit migration tooling. | +| Configuration | One mounted file plus args, with enforced final security settings | Standard PostgreSQL config/argument mechanisms | Security-critical overrides may be superseded here. | +| TLS | Tested mounted TLS 1.2/1.3 server profile | PostgreSQL mechanisms available; image-specific profile is operator-defined | Do not transfer TLS assumptions without review. | +| Health | Bounded local `pg_isready` | Variant/tag behavior must be inspected | Add external transaction monitoring separately. | +| Packages | Exact PGDG 18 server/client closure | Distribution-specific package set | Extensions/tools and CVE mapping differ. | +| Release identity | Planned immutable Datopsis tags/digests only | Official tag family includes mutable aliases | Pin the correct repository digest. | + +The official image documentation describes broader `_FILE`, init-script, +arbitrary-user, and version-specific data-path behavior at +[Docker Official Image documentation](https://github.com/docker-library/docs/blob/master/postgres/README.md). +This table is a compatibility analysis, not a security or quality comparison. diff --git a/docs/STORAGE.md b/docs/STORAGE.md new file mode 100644 index 0000000..bd50107 --- /dev/null +++ b/docs/STORAGE.md @@ -0,0 +1,128 @@ +# Storage, backup, recovery, and upgrade operations + +## Durable-state contract + +The only declared persistent mount is `/var/lib/pgsql`; `PGDATA` is +`/var/lib/pgsql/data`. One v1 PostgreSQL server owns one writable volume. +PostgreSQL clusters are initialized with data page checksums. The image writes +only to the persistent mount and bounded `/tmp` when its root filesystem is +read-only. + +The operator owns capacity, inode availability, latency, durability, filesystem +semantics, encryption at rest, snapshots, replication below the filesystem, +and recovery from node/storage loss. Monitor bytes and inodes independently. +Do not use NFS or a CSI driver until its locking, fsync, ownership, failure, +snapshot, and recovery semantics have been tested with the exact platform. + +Fast shutdown uses SIGINT and aborts active transactions before checkpointing. +Forced termination can require WAL replay; PostgreSQL warns that SIGKILL should +be reserved for emergencies. See +[Shutting down the server](https://www.postgresql.org/docs/18/server-shutdown.html). + +## Logical backup and restoration + +Logical backup/restore is the v1 qualified recovery interface. Decide whether +the unit is a database (`pg_dump`) or the cluster's globals plus databases +(`pg_dumpall` plus per-database dumps). A per-database dump does not include all +cluster roles and tablespaces. + +Example custom-format backup through a verified TLS connection: + +```console +umask 077 +export PGSSLMODE=verify-full +export PGSSLROOTCERT=/run/secrets/postgresql-ca.crt +pg_dump --host=db.example.test --username=backup --dbname=appdb \ + --format=custom --file=appdb.dump +sha256sum appdb.dump > appdb.dump.sha256 +``` + +Supply the password through a protected password file or secret broker, not a +command argument. Encrypt the backup before it leaves the protected execution +boundary, using an organizationally approved mechanism and separately managed +key. Restrict read/delete access, make retained copies immutable where policy +requires, replicate to the approved failure domain, and test key recovery. + +Restore only into an isolated empty target first: + +```console +sha256sum --check appdb.dump.sha256 +createdb --host=restore.example.test --username=restore_owner appdb_restore +pg_restore --host=restore.example.test --username=restore_owner \ + --dbname=appdb_restore --exit-on-error --no-owner appdb.dump +``` + +Review whether ownership must be preserved instead of using `--no-owner`. +Validate schema/migration versions, row counts, constraints, application-level +content digests, privileges, and an application transaction. Record start/end +UTC time and measured recovery time. Schedule isolated restoration at least +quarterly and after material schema, backup-tool, encryption, or version +changes. The database/backup owner defines the tighter schedule required by +RPO/RTO or regulation. + +PostgreSQL describes logical, filesystem, and continuous-archive approaches in +[Backup and Restore](https://www.postgresql.org/docs/18/backup.html). + +## Physical backup, WAL, PITR, and snapshots + +These are operator-owned and not qualified v1 features: + +- `pg_basebackup` requires a narrowly scoped replication role, HBA allowance, + `max_wal_senders`, complete protected output, and a restoration exercise. +- Continuous archiving needs an uninterrupted WAL sequence beginning before + the base backup, durable archive commands, monitoring, retention, timelines, + recovery targets, and tested restoration. Logical dumps cannot supply WAL + replay. +- Storage snapshots require a database-consistent procedure, crash-consistent + semantics or coordinated backup API, all volumes/WAL, encryption, ordering, + retention, and restoration testing. Copying a live data directory is not + automatically a valid backup. + +The presence of `pg_basebackup` does not establish an operational product. See +[pg_basebackup](https://www.postgresql.org/docs/18/app-pgbasebackup.html) and +[continuous archiving/PITR](https://www.postgresql.org/docs/18/continuous-archiving.html). + +## PostgreSQL 18 minor update + +Minor releases retain the major data format, but every update still requires +review and qualification: + +1. read all PostgreSQL release notes, PGDG packaging changes, UBI errata, and + known application/extension issues; +2. review both architecture lock diffs, sources, signing identities, complete + RPM closure, image filesystem, SBOM, and vulnerability results; +3. retain the old image digest and create/verify a pre-update logical backup; +4. clone representative protected data into an isolated environment; +5. start the new digest on the preserved PostgreSQL 18 data, allow recovery, + and test schema migrations, application transactions, query plans where + relevant, backup, and isolated restore; +6. stop the old deployment cleanly, snapshot/backup according to policy, then + replace only the container while preserving its volume; +7. verify version, checksum state, data counts/digests, roles, TLS, logs, + readiness, application transactions, and backup after deployment; and +8. record elapsed shutdown/start/recovery time and the exact old/new digests. + +Rollback means restoring the previous backup/snapshot with the previous image, +or applying a reviewed forward fix. Do not assume an older PostgreSQL binary +can safely open files after a newer minor has started them. The initial 18.6 +image has no earlier supported Datopsis digest. Native CI therefore uses the +immutable Docker Official PostgreSQL 18.4 architecture digests as +preserved-data compatibility fixtures, verifies their platform and version, +takes a pre-update logical backup, adapts only the documented mount layout, +starts 18.6, validates application data, and takes another dump. This is +development update evidence, not support for the official image or a +substitute for repeating the procedure between Datopsis release digests. + +## Major upgrade decision + +The entrypoint rejects a data directory whose `PG_VERSION` is not `18`; never +edit that file. + +Choose logical dump/restore when downtime and data volume permit it, when a +clean logical transformation is desired, or when extension/platform binary +compatibility is uncertain. Choose `pg_upgrade` only after both old and new +binaries, extensions, locales, collations, storage layout, link/copy mode, +rollback point, disk capacity, and the exact procedure have been qualified. +Either path requires verified backup, rehearsal, application validation, +cutover/rollback criteria, and retained evidence. No major upgrade is qualified +for v1. diff --git a/tests/minor-update.sh b/tests/minor-update.sh new file mode 100644 index 0000000..fdfd85c --- /dev/null +++ b/tests/minor-update.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +runtime=${CONTAINER_RUNTIME:-podman} +image=${IMAGE:-localhost/postgresql-ubi:development} +prefix="postgresql-ubi-update-${RANDOM}-$$" +password="update-${RANDOM}-OnlyForTesting" +volume="${prefix}-data" +backup_volume="${prefix}-backup" +old_container="${prefix}-18-4" +new_container="${prefix}-18-6" + +case "$(uname -m)" in + x86_64) + previous_image='docker.io/library/postgres@sha256:4cc13dede823cab4e05290c7fb3350fb4e599ecabd9b07e6706b5d5e8f5bc929' + expected_architecture=amd64 + ;; + aarch64|arm64) + previous_image='docker.io/library/postgres@sha256:0826e5f2996099babb925e09fb72bf2c6eb5d187cfcae20aa9291af1612307e4' + expected_architecture=arm64 + ;; + *) + echo "unsupported update-fixture architecture: $(uname -m)" >&2 + exit 1 + ;; +esac + +cleanup() { + "${runtime}" rm --force --volumes "${old_container}" "${new_container}" >/dev/null 2>&1 || true + "${runtime}" volume rm --force "${volume}" "${backup_volume}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +wait_ready() { + local name=$1 + local version=$2 + local _ + for _ in {1..120}; do + if test "$("${runtime}" exec --env "PGPASSWORD=${password}" "${name}" \ + psql -qAt --host=127.0.0.1 --username=postgres --dbname=postgres \ + --command='SHOW server_version;' 2>/dev/null || true)" = "${version}"; then + return + fi + sleep 1 + done + "${runtime}" logs "${name}" >&2 + return 1 +} + +"${runtime}" pull "${previous_image}" +test "$("${runtime}" image inspect --format '{{.Architecture}}' "${previous_image}")" = \ + "${expected_architecture}" +"${runtime}" volume create "${volume}" >/dev/null +"${runtime}" volume create "${backup_volume}" >/dev/null + +"${runtime}" run --detach --name "${old_container}" \ + --mount "type=volume,src=${volume},dst=/var/lib/postgresql" \ + --mount "type=volume,src=${backup_volume},dst=/backup" \ + --env "POSTGRES_PASSWORD=${password}" \ + "${previous_image}" >/dev/null +wait_ready "${old_container}" 18.4 +test "$("${runtime}" exec "${old_container}" id -u)" = 999 +# PGDATA expands inside the compatibility-fixture container. +# shellcheck disable=SC2016 +test "$("${runtime}" exec "${old_container}" sh -c 'printf %s "$PGDATA"')" = \ + /var/lib/postgresql/18/docker +"${runtime}" exec "${old_container}" psql --host=/var/run/postgresql \ + --username=postgres --set=ON_ERROR_STOP=1 \ + --command='CREATE TABLE update_fixture(id integer PRIMARY KEY, payload text NOT NULL);' \ + --command="INSERT INTO update_fixture SELECT i, md5(i::text) FROM generate_series(1, 1000) AS i;" >/dev/null +fixture_digest=$("${runtime}" exec "${old_container}" psql -qAt --host=/var/run/postgresql \ + --username=postgres \ + --command="SELECT md5(string_agg(id || ':' || payload, ',' ORDER BY id)) FROM update_fixture;") +"${runtime}" exec "${old_container}" pg_dump --host=/var/run/postgresql \ + --username=postgres --format=custom --file=/backup/pre-update.dump postgres +"${runtime}" exec "${old_container}" sh -c 'test -s /backup/pre-update.dump; chmod 0400 /backup/pre-update.dump' +"${runtime}" stop --time 30 "${old_container}" >/dev/null +"${runtime}" rm "${old_container}" >/dev/null + +# Adapt only the official image's versioned mount layout. +# shellcheck disable=SC2016 +"${runtime}" run --rm --user 0 \ + --mount "type=volume,src=${volume},dst=/var/lib/pgsql" \ + --entrypoint sh "${image}" -ceu ' + test "$(cat /var/lib/pgsql/18/docker/PG_VERSION)" = 18 + test ! -e /var/lib/pgsql/data + mv /var/lib/pgsql/18/docker /var/lib/pgsql/data + rmdir /var/lib/pgsql/18 + chown -R 26:0 /var/lib/pgsql/data + chmod 0700 /var/lib/pgsql/data + ' + +"${runtime}" run --detach --name "${new_container}" \ + --read-only \ + --tmpfs /tmp:rw,noexec,nosuid,nodev,size=64m,mode=1777 \ + --mount "type=volume,src=${volume},dst=/var/lib/pgsql" \ + --mount "type=volume,src=${backup_volume},dst=/backup,readonly" \ + --cap-drop ALL \ + --security-opt no-new-privileges:true \ + "${image}" >/dev/null +wait_ready "${new_container}" 18.6 +test "$("${runtime}" exec "${new_container}" psql -qAt --host=/tmp --username=postgres \ + --command='SELECT count(*) FROM update_fixture;')" = 1000 +test "$("${runtime}" exec "${new_container}" psql -qAt --host=/tmp --username=postgres \ + --command="SELECT md5(string_agg(id || ':' || payload, ',' ORDER BY id)) FROM update_fixture;")" = \ + "${fixture_digest}" +"${runtime}" exec "${new_container}" pg_dump --host=/tmp --username=postgres \ + --format=custom --file=/tmp/post-update.dump postgres +"${runtime}" exec "${new_container}" sh -c 'test -s /tmp/post-update.dump; test -s /backup/pre-update.dump' + +echo "PostgreSQL 18.4 to 18.6 preserved-data update passed for ${image}" diff --git a/tests/runtime-security.sh b/tests/runtime-security.sh new file mode 100644 index 0000000..bb24d6e --- /dev/null +++ b/tests/runtime-security.sh @@ -0,0 +1,230 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +runtime=${CONTAINER_RUNTIME:-podman} +image=${IMAGE:-localhost/postgresql-ubi:development} +prefix="postgresql-ubi-security-${RANDOM}-$$" +password='Odd !@#$%^&*()[]{}:;,.?=+_- value' +rotated='Rotated !@#$%^&*()[]{}:;,.?=+_- value' +data_volume="${prefix}-data" +secret_volume="${prefix}-secrets" +config_volume="${prefix}-config" +containers=() +volumes=("${data_volume}" "${secret_volume}" "${config_volume}") +no_new_privileges=no-new-privileges:true + +if grep -qi podman <<<"$("${runtime}" --version 2>&1)"; then + no_new_privileges=no-new-privileges +fi + +cleanup() { + if test "${#containers[@]}" -gt 0; then + "${runtime}" rm --force --volumes "${containers[@]}" >/dev/null 2>&1 || true + fi + "${runtime}" volume rm --force "${volumes[@]}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +new_container() { + containers+=("$1") +} + +run_restricted() { + local name=$1 + local volume=$2 + local options=() + shift 2 + while test "$#" -gt 0 && test "$1" != --container-command; do + options+=("$1") + shift + done + if test "${1:-}" = --container-command; then + shift + fi + new_container "${name}" + "${runtime}" run --detach --name "${name}" \ + --read-only \ + --tmpfs /tmp:rw,noexec,nosuid,nodev,size=64m,mode=1777 \ + --mount "type=volume,src=${volume},dst=/var/lib/pgsql" \ + --cap-drop ALL \ + --security-opt "${no_new_privileges}" \ + "${options[@]}" "${image}" "$@" >/dev/null +} + +wait_ready() { + local name=$1 + local credential=$2 + local _ + for _ in {1..90}; do + if "${runtime}" exec --env "PGPASSWORD=${credential}" "${name}" \ + psql -qAt --host=127.0.0.1 --username=postgres --dbname=postgres \ + --command='SELECT 1' >/dev/null 2>&1; then + return + fi + sleep 1 + done + "${runtime}" logs "${name}" >&2 + return 1 +} + +expect_failure() { + local name=$1 + local expected=$2 + shift 2 + new_container "${name}" + if "${runtime}" run --name "${name}" "$@" "${image}"; then + echo "expected ${name} to fail" >&2 + return 1 + fi + "${runtime}" logs "${name}" 2>&1 | grep -Fq "${expected}" +} + +seed_file() { + local volume=$1 + local path=$2 + local mode=$3 + local value=$4 + # Variables expand inside the seed container. + # shellcheck disable=SC2016 + printf '%s' "${value}" | "${runtime}" run --rm --interactive --user 0 \ + --mount "type=volume,src=${volume},dst=/seed" \ + --entrypoint sh "${image}" -ceu \ + 'umask 077; mkdir -p "$(dirname "/seed/$1")"; cat >"/seed/$1"; chmod "$2" "/seed/$1"; chown 0:0 "/seed/$1"' \ + sh "${path}" "${mode}" +} + +for volume in "${volumes[@]}"; do + "${runtime}" volume create "${volume}" >/dev/null +done + +seed_file "${secret_volume}" password 440 "${password}" +seed_file "${secret_volume}" empty 440 '' +seed_file "${secret_volume}" newline 440 $'value\nsecond' +seed_file "${secret_volume}" permissive 444 value +"${runtime}" run --rm --user 0 \ + --mount "type=volume,src=${secret_volume},dst=/seed" \ + --entrypoint sh "${image}" -ceu \ + 'ln -s password /seed/symlink; : >/seed/unreadable; chmod 000 /seed/unreadable' + +common_failure=(--read-only --tmpfs '/tmp:rw,noexec,nosuid,nodev,size=64m,mode=1777' + --cap-drop ALL --security-opt "${no_new_privileges}") + +expect_failure "${prefix}-empty-env" 'initialization requires POSTGRES_PASSWORD' \ + "${common_failure[@]}" --env POSTGRES_PASSWORD= +expect_failure "${prefix}-both" 'set only one of POSTGRES_PASSWORD or POSTGRES_PASSWORD_FILE' \ + "${common_failure[@]}" --env POSTGRES_PASSWORD=value \ + --env POSTGRES_PASSWORD_FILE=/run/secrets/password \ + --mount "type=volume,src=${secret_volume},dst=/run/secrets,readonly" +expect_failure "${prefix}-empty-file" 'initialization requires POSTGRES_PASSWORD' \ + "${common_failure[@]}" --env POSTGRES_PASSWORD_FILE=/run/secrets/empty \ + --mount "type=volume,src=${secret_volume},dst=/run/secrets,readonly" +expect_failure "${prefix}-newline" 'without a line break' \ + "${common_failure[@]}" --env POSTGRES_PASSWORD_FILE=/run/secrets/newline \ + --mount "type=volume,src=${secret_volume},dst=/run/secrets,readonly" +expect_failure "${prefix}-permissive" 'permissions must be' \ + "${common_failure[@]}" --env POSTGRES_PASSWORD_FILE=/run/secrets/permissive \ + --mount "type=volume,src=${secret_volume},dst=/run/secrets,readonly" +expect_failure "${prefix}-symlink" 'must not be a symbolic link' \ + "${common_failure[@]}" --env POSTGRES_PASSWORD_FILE=/run/secrets/symlink \ + --mount "type=volume,src=${secret_volume},dst=/run/secrets,readonly" +expect_failure "${prefix}-unreadable" 'is not readable' \ + "${common_failure[@]}" --env POSTGRES_PASSWORD_FILE=/run/secrets/unreadable \ + --mount "type=volume,src=${secret_volume},dst=/run/secrets,readonly" + +primary="${prefix}-primary" +run_restricted "${primary}" "${data_volume}" \ + --env POSTGRES_PASSWORD_FILE=/run/secrets/password \ + --mount "type=volume,src=${secret_volume},dst=/run/secrets,readonly" +wait_ready "${primary}" "${password}" + +"${runtime}" exec "${primary}" psql -qAt --host=/tmp --username=postgres \ + --command="SELECT current_setting('data_checksums'), current_setting('password_encryption');" \ + | grep -Fxq 'on|scram-sha-256' +# Variables expand inside the database container. +# shellcheck disable=SC2016 +"${runtime}" exec "${primary}" sh -ceu ' + test "$(ulimit -c)" = 0 + ! find /tmp -maxdepth 1 -name "postgresql-password.*" -print -quit | grep -q . + ! tr "\0" "\n" &2 + exit 1 +fi +if "${runtime}" logs "${primary}" 2>&1 | grep -Fq "${password}"; then + echo 'initialization credential exposed in logs' >&2 + exit 1 +fi + +"${runtime}" exec "${primary}" psql --host=/tmp --username=postgres \ + --set=ON_ERROR_STOP=1 --command="CREATE ROLE app LOGIN PASSWORD '${rotated}' NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;" \ + --command='CREATE TABLE role_probe(value integer);' \ + --command='GRANT SELECT ON role_probe TO app;' >/dev/null +test "$("${runtime}" exec --env "PGPASSWORD=${rotated}" "${primary}" \ + psql -qAt --host=127.0.0.1 --username=app --dbname=postgres \ + --command='SELECT count(*) FROM role_probe;')" = 0 +if "${runtime}" exec --env PGPASSWORD=wrong "${primary}" \ + psql -qAt --host=127.0.0.1 --username=app --dbname=postgres \ + --command='SELECT 1' >/dev/null 2>&1; then + echo 'wrong application password authenticated' >&2 + exit 1 +fi +if "${runtime}" exec --env "PGPASSWORD=${rotated}" "${primary}" \ + psql -qAt --host=127.0.0.1 --username=app --dbname=postgres \ + --command='CREATE TABLE forbidden(value integer)' >/dev/null 2>&1; then + echo 'least-privilege application role created an unauthorized table' >&2 + exit 1 +fi + +"${runtime}" exec "${primary}" psql --host=/tmp --username=postgres \ + --set=ON_ERROR_STOP=1 --command="ALTER ROLE app PASSWORD '${password}';" >/dev/null +if "${runtime}" exec --env "PGPASSWORD=${rotated}" "${primary}" \ + psql -qAt --host=127.0.0.1 --username=app --dbname=postgres \ + --command='SELECT 1' >/dev/null 2>&1; then + echo 'old application password remained valid after rotation' >&2 + exit 1 +fi +test "$("${runtime}" exec --env "PGPASSWORD=${password}" "${primary}" \ + psql -qAt --host=127.0.0.1 --username=app --dbname=postgres \ + --command='SELECT 1')" = 1 + +# Persisted attempts to weaken HBA are replaced at the next start. +# PGDATA expands inside the database container. +# shellcheck disable=SC2016 +"${runtime}" exec "${primary}" sh -c 'printf "host all all all trust\n" >"$PGDATA/pg_hba.conf"' +"${runtime}" stop --time 30 "${primary}" >/dev/null +"${runtime}" rm "${primary}" >/dev/null + +seed_file "${config_volume}" postgresql.conf 444 $'max_connections = 37\npassword_encryption = md5\nlogging_collector = on\nlog_statement = all\nhba_file = '\''/tmp/untrusted-hba'\''\nssl = on\n' +configured="${prefix}-configured" +run_restricted "${configured}" "${data_volume}" \ + --env POSTGRESQL_CONFIG_FILE=/run/config/postgresql.conf \ + --mount "type=volume,src=${config_volume},dst=/run/config,readonly" \ + --container-command postgres -c password_encryption=md5 -c log_statement=all +wait_ready "${configured}" "${password}" +for expectation in \ + 'max_connections|37' \ + 'password_encryption|scram-sha-256' \ + 'logging_collector|off' \ + 'log_destination|stderr' \ + 'log_statement|none' \ + 'ssl|off'; do + setting=${expectation%%|*} + value=${expectation#*|} + test "$("${runtime}" exec "${configured}" psql -qAt --host=/tmp --username=postgres \ + --command="SHOW ${setting};")" = "${value}" +done +# PGDATA expands inside the database container. +# shellcheck disable=SC2016 +test "$("${runtime}" exec "${configured}" sh -c 'cat "$PGDATA/pg_hba.conf"')" = \ +"# Managed by postgresql-ubi; replaced on every start. +local all all trust +host all all 0.0.0.0/0 scram-sha-256 +host all all ::/0 scram-sha-256" + +if "${runtime}" logs "${configured}" 2>&1 | grep -Fq "${rotated}"; then + echo 'SQL value exposed in database logs' >&2 + exit 1 +fi + +echo "PostgreSQL authentication and configuration contract passed for ${image}" diff --git a/tests/smoke.sh b/tests/smoke.sh index 2211a7c..c766f06 100644 --- a/tests/smoke.sh +++ b/tests/smoke.sh @@ -20,7 +20,7 @@ if grep -qi podman <<<"$("${runtime}" --version 2>&1)"; then fi cleanup() { - "${runtime}" rm --force \ + "${runtime}" rm --force --volumes \ "${primary}" "${restart}" "${arbitrary}" \ "${missing_password}" "${wrong_major}" >/dev/null 2>&1 || true "${runtime}" volume rm --force \ @@ -109,6 +109,19 @@ run_restricted "${primary}" "${primary_volume}" \ wait_for_postgresql "${primary}" test "$(sql "${primary}" 'SHOW server_version;')" = "18.6" test "$(sql "${primary}" 'SHOW password_encryption;')" = "scram-sha-256" +test "$(sql "${primary}" 'SHOW data_checksums;')" = on +test "$(sql "${primary}" 'SHOW logging_collector;')" = off +test "$(sql "${primary}" 'SHOW log_statement;')" = none +test "$(sql "${primary}" 'SHOW log_parameter_max_length;')" = 0 +test "$(sql "${primary}" 'SHOW log_parameter_max_length_on_error;')" = 0 +test "$(sql "${primary}" 'SHOW ssl;')" = off +# Variables expand inside the container. +# shellcheck disable=SC2016 +test "$("${runtime}" exec "${primary}" sh -c \ + 'cat "$PGDATA/pg_hba.conf"')" = "# Managed by postgresql-ubi; replaced on every start. +local all all trust +host all all 0.0.0.0/0 scram-sha-256 +host all all ::/0 scram-sha-256" sql "${primary}" \ 'CREATE TABLE persistence_probe (value text NOT NULL); INSERT INTO persistence_probe VALUES ('"'"'survives'"'"');' \ >/dev/null @@ -123,6 +136,11 @@ if grep -Fq "${password}" <<<"$("${runtime}" logs "${primary}" 2>&1)"; then echo "Initialization password was exposed in container logs" >&2 exit 1 fi +if "${runtime}" exec "${primary}" sh -c \ + 'tr "\0" "\n" &2 + exit 1 +fi "${runtime}" stop --time 30 "${primary}" >/dev/null test "$("${runtime}" inspect --format '{{.State.ExitCode}}' "${primary}")" = 0 diff --git a/tests/storage-lifecycle.sh b/tests/storage-lifecycle.sh new file mode 100644 index 0000000..bc4b244 --- /dev/null +++ b/tests/storage-lifecycle.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +runtime=${CONTAINER_RUNTIME:-podman} +image=${IMAGE:-localhost/postgresql-ubi:development} +prefix="postgresql-ubi-storage-${RANDOM}-$$" +password="storage-${RANDOM}-OnlyForTesting" +data_volume="${prefix}-data" +restore_volume="${prefix}-restore" +backup_volume="${prefix}-backup" +partial_volume="${prefix}-partial" +locked_volume="${prefix}-locked" +hook_volume="${prefix}-hook" +ownership_volume="${prefix}-ownership" +containers=() +volumes=("${data_volume}" "${restore_volume}" "${backup_volume}" \ + "${partial_volume}" "${locked_volume}" "${hook_volume}" "${ownership_volume}") +no_new_privileges=no-new-privileges:true + +if grep -qi podman <<<"$("${runtime}" --version 2>&1)"; then + no_new_privileges=no-new-privileges +fi + +cleanup() { + if test "${#containers[@]}" -gt 0; then + "${runtime}" rm --force --volumes "${containers[@]}" >/dev/null 2>&1 || true + fi + "${runtime}" volume rm --force "${volumes[@]}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +remember() { + containers+=("$1") +} + +run_database() { + local name=$1 + local volume=$2 + shift 2 + remember "${name}" + "${runtime}" run --detach --name "${name}" \ + --read-only \ + --tmpfs /tmp:rw,noexec,nosuid,nodev,size=64m,mode=1777 \ + --mount "type=volume,src=${volume},dst=/var/lib/pgsql" \ + --cap-drop ALL \ + --security-opt "${no_new_privileges}" \ + "$@" "${image}" >/dev/null +} + +wait_ready() { + local name=$1 + local database=${2:-postgres} + local _ + for _ in {1..120}; do + if "${runtime}" exec --env "PGPASSWORD=${password}" "${name}" \ + psql -qAt --host=127.0.0.1 --username=postgres --dbname="${database}" \ + --command='SELECT 1' >/dev/null 2>&1; then + return + fi + sleep 1 + done + "${runtime}" logs "${name}" >&2 + return 1 +} + +wait_failed() { + local name=$1 + local expected=$2 + local _ + for _ in {1..60}; do + if test "$("${runtime}" inspect --format '{{.State.Status}}' "${name}")" != running; then + test "$("${runtime}" inspect --format '{{.State.ExitCode}}' "${name}")" != 0 + "${runtime}" logs "${name}" 2>&1 | grep -Fq "${expected}" + return + fi + sleep 1 + done + "${runtime}" logs "${name}" >&2 + return 1 +} + +for volume in "${volumes[@]}"; do + "${runtime}" volume create "${volume}" >/dev/null +done + +primary="${prefix}-primary" +run_database "${primary}" "${data_volume}" --env "POSTGRES_PASSWORD=${password}" +wait_ready "${primary}" +"${runtime}" exec "${primary}" psql --host=/tmp --username=postgres \ + --set=ON_ERROR_STOP=1 \ + --command='CREATE TABLE durable(id bigint PRIMARY KEY, payload text NOT NULL);' \ + --command="INSERT INTO durable SELECT i, repeat(md5(i::text), 64) FROM generate_series(1, 30000) AS i;" \ + --command='CHECKPOINT;' >/dev/null +test "$("${runtime}" exec "${primary}" psql -qAt --host=/tmp --username=postgres \ + --command='SELECT count(*) FROM durable;')" = 30000 +test "$("${runtime}" exec "${primary}" pg_controldata /var/lib/pgsql/data \ + | sed -n 's/^Data page checksum version:[[:space:]]*//p')" = 1 + +# Bounded temporary storage fails without affecting durable data. +if "${runtime}" exec "${primary}" dd if=/dev/zero of=/tmp/space-probe \ + bs=1M count=80 status=none 2>/dev/null; then + echo 'bounded /tmp unexpectedly accepted an 80 MiB write' >&2 + exit 1 +fi +"${runtime}" exec "${primary}" rm -f /tmp/space-probe + +# A normal stop must checkpoint cleanly and remove transient PID/socket state. +"${runtime}" stop --time 30 "${primary}" >/dev/null +test "$("${runtime}" inspect --format '{{.State.ExitCode}}' "${primary}")" = 0 +"${runtime}" rm "${primary}" >/dev/null + +clean_restart="${prefix}-clean-restart" +run_database "${clean_restart}" "${data_volume}" +wait_ready "${clean_restart}" +test "$("${runtime}" exec "${clean_restart}" psql -qAt --host=/tmp --username=postgres \ + --command='SELECT count(*) FROM durable;')" = 30000 +# PGDATA expands inside the database container. +# shellcheck disable=SC2016 +"${runtime}" exec "${clean_restart}" sh -ceu 'test -s "$PGDATA/postmaster.pid"; test -S /tmp/.s.PGSQL.5432' + +# Force termination during writes. Committed rows must survive crash recovery. +"${runtime}" exec --detach "${clean_restart}" psql --host=/tmp --username=postgres \ + --command="INSERT INTO durable SELECT i, repeat(md5(i::text), 64) FROM generate_series(30001, 90000) AS i; SELECT pg_sleep(30);" +sleep 2 +"${runtime}" kill --signal KILL "${clean_restart}" >/dev/null +"${runtime}" rm "${clean_restart}" >/dev/null + +recovered="${prefix}-recovered" +run_database "${recovered}" "${data_volume}" +wait_ready "${recovered}" +row_count=$("${runtime}" exec "${recovered}" psql -qAt --host=/tmp --username=postgres \ + --command='SELECT count(*) FROM durable;') +test "${row_count}" -eq 30000 || test "${row_count}" -eq 90000 +test "$("${runtime}" exec "${recovered}" psql -qAt --host=/tmp --username=postgres \ + --command='SELECT pg_is_in_recovery();')" = f +"${runtime}" logs "${recovered}" 2>&1 | grep -Eq \ + 'database system was interrupted|database system was not properly shut down|redo starts at' + +# Logical backup and isolated restore validate schema, row count, and content. +"${runtime}" run --rm --user 0 \ + --mount "type=volume,src=${backup_volume},dst=/backup" \ + --entrypoint sh "${image}" -c 'chown 26:0 /backup; chmod 0770 /backup' +"${runtime}" exec "${recovered}" true +"${runtime}" stop --time 30 "${recovered}" >/dev/null +"${runtime}" rm "${recovered}" >/dev/null + +backup_source="${prefix}-backup-source" +run_database "${backup_source}" "${data_volume}" \ + --mount "type=volume,src=${backup_volume},dst=/backup" +wait_ready "${backup_source}" +source_digest=$("${runtime}" exec "${backup_source}" psql -qAt --host=/tmp --username=postgres \ + --command="SELECT md5(string_agg(id || ':' || payload, ',' ORDER BY id)) FROM durable;") +"${runtime}" exec "${backup_source}" pg_dump --host=/tmp --username=postgres \ + --format=custom --file=/backup/postgres.dump postgres +"${runtime}" exec "${backup_source}" sh -ceu 'test -s /backup/postgres.dump; chmod 0400 /backup/postgres.dump' +"${runtime}" stop --time 30 "${backup_source}" >/dev/null + +restore="${prefix}-restore" +run_database "${restore}" "${restore_volume}" \ + --env "POSTGRES_PASSWORD=${password}" \ + --mount "type=volume,src=${backup_volume},dst=/backup,readonly" +wait_ready "${restore}" +"${runtime}" exec "${restore}" pg_restore --host=/tmp --username=postgres \ + --dbname=postgres --clean --if-exists --exit-on-error /backup/postgres.dump +test "$("${runtime}" exec "${restore}" psql -qAt --host=/tmp --username=postgres \ + --command='SELECT count(*) FROM durable;')" = "${row_count}" +restore_digest=$("${runtime}" exec "${restore}" psql -qAt --host=/tmp --username=postgres \ + --command="SELECT md5(string_agg(id || ':' || payload, ',' ORDER BY id)) FROM durable;") +test "${source_digest}" = "${restore_digest}" + +# Partial initialization and stale/concurrent locks always fail closed. +"${runtime}" run --rm --user 26:0 \ + --mount "type=volume,src=${partial_volume},dst=/var/lib/pgsql" \ + --entrypoint sh "${image}" -c 'mkdir -p /var/lib/pgsql/data; : >/var/lib/pgsql/data/partial' +partial="${prefix}-partial" +run_database "${partial}" "${partial_volume}" --env "POSTGRES_PASSWORD=${password}" +wait_failed "${partial}" 'refusing automatic recovery or reinitialization' + +"${runtime}" run --rm --user 26:0 \ + --mount "type=volume,src=${locked_volume},dst=/var/lib/pgsql" \ + --entrypoint sh "${image}" -c 'mkdir /var/lib/pgsql/data.postgresql-ubi.init.lock' +locked="${prefix}-locked" +run_database "${locked}" "${locked_volume}" --env "POSTGRES_PASSWORD=${password}" +wait_failed "${locked}" 'stale initialization lock exists' + +"${runtime}" run --rm --user 0 \ + --mount "type=volume,src=${ownership_volume},dst=/var/lib/pgsql" \ + --entrypoint sh "${image}" -c 'chmod 0700 /var/lib/pgsql; chown 0:0 /var/lib/pgsql' +wrong_owner="${prefix}-wrong-owner" +run_database "${wrong_owner}" "${ownership_volume}" --env "POSTGRES_PASSWORD=${password}" +wait_failed "${wrong_owner}" 'cannot create PGDATA' + +readonly_data="${prefix}-readonly" +remember "${readonly_data}" +"${runtime}" run --detach --name "${readonly_data}" \ + --read-only \ + --tmpfs /tmp:rw,noexec,nosuid,nodev,size=64m,mode=1777 \ + --mount "type=volume,src=${data_volume},dst=/var/lib/pgsql,readonly" \ + --cap-drop ALL --security-opt "${no_new_privileges}" "${image}" >/dev/null +wait_failed "${readonly_data}" 'PGDATA cannot accept a validation write' + +full="${prefix}-full" +remember "${full}" +"${runtime}" run --detach --name "${full}" \ + --read-only \ + --tmpfs /tmp:rw,noexec,nosuid,nodev,size=64m,mode=1777 \ + --tmpfs /var/lib/pgsql:rw,nosuid,nodev,size=4m,uid=26,gid=0,mode=0770 \ + --cap-drop ALL --security-opt "${no_new_privileges}" \ + --env "POSTGRES_PASSWORD=${password}" "${image}" >/dev/null +wait_failed "${full}" 'No space left on device' + +# An intentionally delayed initdb makes interruption and concurrency deterministic. +"${runtime}" run --rm --user 0 \ + --mount "type=volume,src=${hook_volume},dst=/hook" \ + --entrypoint sh "${image}" -ceu ' + printf "%s\n" "#!/bin/sh" "sleep 30" "exec /usr/pgsql-18/bin/initdb \"\$@\"" >/hook/initdb + chmod 0755 /hook/initdb + chown 26:0 /hook/initdb + ' +concurrent_volume="${prefix}-concurrent" +volumes+=("${concurrent_volume}") +"${runtime}" volume create "${concurrent_volume}" >/dev/null +first="${prefix}-concurrent-one" +second="${prefix}-concurrent-two" +run_database "${first}" "${concurrent_volume}" \ + --env "POSTGRES_PASSWORD=${password}" --env PATH=/hook:/usr/pgsql-18/bin:/usr/bin:/bin \ + --mount "type=volume,src=${hook_volume},dst=/hook,readonly" +sleep 1 +run_database "${second}" "${concurrent_volume}" \ + --env "POSTGRES_PASSWORD=${password}" --env PATH=/hook:/usr/pgsql-18/bin:/usr/bin:/bin \ + --mount "type=volume,src=${hook_volume},dst=/hook,readonly" +wait_failed "${second}" 'initialization is already running' +"${runtime}" stop --time 5 "${first}" >/dev/null || true +"${runtime}" rm "${first}" >/dev/null + +interrupted_restart="${prefix}-interrupted-restart" +run_database "${interrupted_restart}" "${concurrent_volume}" --env "POSTGRES_PASSWORD=${password}" +wait_ready "${interrupted_restart}" + +echo "PostgreSQL durable-storage and lifecycle contract passed for ${image}" diff --git a/tests/tls.sh b/tests/tls.sh new file mode 100644 index 0000000..e3af53b --- /dev/null +++ b/tests/tls.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +runtime=${CONTAINER_RUNTIME:-podman} +image=${IMAGE:-localhost/postgresql-ubi:development} +prefix="postgresql-ubi-tls-${RANDOM}-$$" +password="tls-${RANDOM}-OnlyForTesting" +data_volume="${prefix}-data" +tls_volume="${prefix}-material" +container="${prefix}-server" +fixture=$(mktemp -d) +no_new_privileges=no-new-privileges:true + +if grep -qi podman <<<"$("${runtime}" --version 2>&1)"; then + no_new_privileges=no-new-privileges +fi + +cleanup() { + "${runtime}" rm --force --volumes "${container}" >/dev/null 2>&1 || true + "${runtime}" volume rm --force "${data_volume}" "${tls_volume}" >/dev/null 2>&1 || true + rm -rf -- "${fixture}" +} +trap cleanup EXIT + +make_ca() { + local stem=$1 + openssl req -x509 -newkey rsa:3072 -nodes -sha256 -days 2 \ + -subj "/CN=${stem} test CA" \ + -keyout "${fixture}/${stem}-ca.key" \ + -out "${fixture}/${stem}-ca.crt" >/dev/null 2>&1 +} + +make_server() { + local stem=$1 + local ca=$2 + local serial=$3 + openssl req -newkey rsa:3072 -nodes -sha256 \ + -subj '/CN=localhost' \ + -addext 'subjectAltName=DNS:localhost' \ + -keyout "${fixture}/${stem}.key" \ + -out "${fixture}/${stem}.csr" >/dev/null 2>&1 + printf 'subjectAltName=DNS:localhost\nextendedKeyUsage=serverAuth\n' >"${fixture}/${stem}.ext" + openssl x509 -req -sha256 -days 2 -set_serial "${serial}" \ + -in "${fixture}/${stem}.csr" \ + -CA "${fixture}/${ca}-ca.crt" \ + -CAkey "${fixture}/${ca}-ca.key" \ + -out "${fixture}/${stem}.crt" \ + -extfile "${fixture}/${stem}.ext" >/dev/null 2>&1 +} + +copy_material() { + local stem=$1 + # Positional parameters expand inside the copy container. + # shellcheck disable=SC2016 + "${runtime}" run --rm --user 0 \ + --mount "type=bind,src=${fixture},dst=/source,readonly" \ + --mount "type=volume,src=${tls_volume},dst=/tls" \ + --entrypoint sh "${image}" -ceu ' + cp "/source/$1.crt" /tls/server.crt + cp "/source/$1.key" /tls/server.key + cp /source/trusted-ca.crt /tls/ca.crt + cp /source/untrusted-ca.crt /tls/untrusted-ca.crt + chown 0:0 /tls/* + chmod 0444 /tls/*.crt + chmod 0440 /tls/server.key + ' sh "${stem}" +} + +wait_tls() { + local _ + for _ in {1..90}; do + if "${runtime}" exec --env "PGPASSWORD=${password}" "${container}" \ + psql -qAt 'host=localhost user=postgres dbname=postgres sslmode=verify-full sslrootcert=/run/tls/ca.crt connect_timeout=2' \ + --command='SELECT 1' >/dev/null 2>&1; then + return + fi + sleep 1 + done + "${runtime}" logs "${container}" >&2 + return 1 +} + +served_serial() { + "${runtime}" exec "${container}" sh -c \ + "openssl s_client -starttls postgres -connect localhost:5432 /dev/null | openssl x509 -noout -serial" \ + | tr -d '\r' +} + +make_ca trusted +make_ca untrusted +make_server server-one trusted 1001 +make_server server-two trusted 1002 + +cp "${fixture}/trusted-ca.crt" "${fixture}/trusted-ca-copy.crt" +cp "${fixture}/untrusted-ca.crt" "${fixture}/untrusted-ca-copy.crt" +mv "${fixture}/trusted-ca-copy.crt" "${fixture}/trusted-ca.crt" +mv "${fixture}/untrusted-ca-copy.crt" "${fixture}/untrusted-ca.crt" + +# The same valid fixture is rejected outside its validity interval. +not_before=$(date -u -d '1 day ago' +%s) +after_expiry=$(date -u -d '4 days' +%s) +if openssl verify -attime "${not_before}" -CAfile "${fixture}/trusted-ca.crt" \ + "${fixture}/server-one.crt" >/dev/null 2>&1; then + echo 'not-yet-valid certificate fixture was accepted' >&2 + exit 1 +fi +if openssl verify -attime "${after_expiry}" -CAfile "${fixture}/trusted-ca.crt" \ + "${fixture}/server-one.crt" >/dev/null 2>&1; then + echo 'expired certificate fixture was accepted' >&2 + exit 1 +fi + +"${runtime}" volume create "${data_volume}" >/dev/null +"${runtime}" volume create "${tls_volume}" >/dev/null +copy_material server-one + +"${runtime}" run --detach --name "${container}" \ + --read-only \ + --tmpfs /tmp:rw,noexec,nosuid,nodev,size=64m,mode=1777 \ + --mount "type=volume,src=${data_volume},dst=/var/lib/pgsql" \ + --mount "type=volume,src=${tls_volume},dst=/run/tls,readonly" \ + --cap-drop ALL \ + --security-opt "${no_new_privileges}" \ + --env "POSTGRES_PASSWORD=${password}" \ + --env POSTGRESQL_TLS_CERT_FILE=/run/tls/server.crt \ + --env POSTGRESQL_TLS_KEY_FILE=/run/tls/server.key \ + "${image}" >/dev/null +wait_tls + +test "$("${runtime}" exec --env "PGPASSWORD=${password}" "${container}" \ + psql -qAt 'host=localhost user=postgres dbname=postgres sslmode=verify-full sslrootcert=/run/tls/ca.crt' \ + --command="SELECT ssl || '|' || version FROM pg_stat_ssl WHERE pid=pg_backend_pid();")" = 'true|TLSv1.3' || \ +test "$("${runtime}" exec --env "PGPASSWORD=${password}" "${container}" \ + psql -qAt 'host=localhost user=postgres dbname=postgres sslmode=verify-full sslrootcert=/run/tls/ca.crt' \ + --command="SELECT ssl || '|' || version FROM pg_stat_ssl WHERE pid=pg_backend_pid();")" = 'true|TLSv1.2' + +if "${runtime}" exec --env "PGPASSWORD=${password}" "${container}" \ + psql -qAt 'host=localhost user=postgres dbname=postgres sslmode=disable connect_timeout=2' \ + --command='SELECT 1' >/dev/null 2>&1; then + echo 'clear-text network connection was accepted by the TLS profile' >&2 + exit 1 +fi +if "${runtime}" exec --env "PGPASSWORD=${password}" "${container}" \ + psql -qAt 'host=127.0.0.1 user=postgres dbname=postgres sslmode=verify-full sslrootcert=/run/tls/ca.crt connect_timeout=2' \ + --command='SELECT 1' >/dev/null 2>&1; then + echo 'certificate hostname mismatch was accepted' >&2 + exit 1 +fi +if "${runtime}" exec --env "PGPASSWORD=${password}" "${container}" \ + psql -qAt 'host=localhost user=postgres dbname=postgres sslmode=verify-full sslrootcert=/run/tls/untrusted-ca.crt connect_timeout=2' \ + --command='SELECT 1' >/dev/null 2>&1; then + echo 'untrusted certificate chain was accepted' >&2 + exit 1 +fi + +test "$(served_serial)" = serial=03E9 +copy_material server-two +"${runtime}" kill --signal HUP "${container}" >/dev/null +sleep 2 +test "$(served_serial)" = serial=03EA +wait_tls +copy_material server-one +"${runtime}" kill --signal HUP "${container}" >/dev/null +sleep 2 +test "$(served_serial)" = serial=03E9 +wait_tls + +# PGDATA expands inside the database container. +# shellcheck disable=SC2016 +test "$("${runtime}" exec "${container}" sh -c 'cat "$PGDATA/pg_hba.conf"')" = \ +"# Managed by postgresql-ubi; replaced on every start. +local all all trust +hostssl all all 0.0.0.0/0 scram-sha-256 +hostssl all all ::/0 scram-sha-256" + +echo "PostgreSQL TLS 1.2/1.3 server profile passed for ${image}" From 0af22db66120dd64158fc745b0aa04d66779fc60 Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:17:46 -0500 Subject: [PATCH 02/17] test: report runtime contract failures --- tests/minor-update.sh | 1 + tests/runtime-security.sh | 1 + tests/storage-lifecycle.sh | 1 + tests/tls.sh | 1 + 4 files changed, 4 insertions(+) diff --git a/tests/minor-update.sh b/tests/minor-update.sh index fdfd85c..1bbd604 100644 --- a/tests/minor-update.sh +++ b/tests/minor-update.sh @@ -30,6 +30,7 @@ cleanup() { "${runtime}" volume rm --force "${volume}" "${backup_volume}" >/dev/null 2>&1 || true } trap cleanup EXIT +trap 'status=$?; printf "minor-update failed at line %s\n" "${LINENO}" >&2; exit "${status}"' ERR wait_ready() { local name=$1 diff --git a/tests/runtime-security.sh b/tests/runtime-security.sh index bb24d6e..6b5229b 100644 --- a/tests/runtime-security.sh +++ b/tests/runtime-security.sh @@ -24,6 +24,7 @@ cleanup() { "${runtime}" volume rm --force "${volumes[@]}" >/dev/null 2>&1 || true } trap cleanup EXIT +trap 'status=$?; printf "runtime-security failed at line %s\n" "${LINENO}" >&2; exit "${status}"' ERR new_container() { containers+=("$1") diff --git a/tests/storage-lifecycle.sh b/tests/storage-lifecycle.sh index bc4b244..02dd5cf 100644 --- a/tests/storage-lifecycle.sh +++ b/tests/storage-lifecycle.sh @@ -28,6 +28,7 @@ cleanup() { "${runtime}" volume rm --force "${volumes[@]}" >/dev/null 2>&1 || true } trap cleanup EXIT +trap 'status=$?; printf "storage-lifecycle failed at line %s\n" "${LINENO}" >&2; exit "${status}"' ERR remember() { containers+=("$1") diff --git a/tests/tls.sh b/tests/tls.sh index e3af53b..ae2d931 100644 --- a/tests/tls.sh +++ b/tests/tls.sh @@ -21,6 +21,7 @@ cleanup() { rm -rf -- "${fixture}" } trap cleanup EXIT +trap 'status=$?; printf "tls test failed at line %s\n" "${LINENO}" >&2; exit "${status}"' ERR make_ca() { local stem=$1 From 76c13212ae51bde70f523096bf0366709d7f271e Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:20:11 -0500 Subject: [PATCH 03/17] fix: harden runtime test diagnostics --- tests/minor-update.sh | 7 ++++++- tests/runtime-security.sh | 12 ++++++++---- tests/storage-lifecycle.sh | 7 ++++++- tests/tls.sh | 7 ++++++- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/tests/minor-update.sh b/tests/minor-update.sh index 1bbd604..3e1d6e0 100644 --- a/tests/minor-update.sh +++ b/tests/minor-update.sh @@ -30,7 +30,12 @@ cleanup() { "${runtime}" volume rm --force "${volume}" "${backup_volume}" >/dev/null 2>&1 || true } trap cleanup EXIT -trap 'status=$?; printf "minor-update failed at line %s\n" "${LINENO}" >&2; exit "${status}"' ERR +report_error() { + failure_status=$? + printf 'minor-update failed at line %s\n' "${BASH_LINENO[0]}" >&2 + exit "${failure_status}" +} +trap report_error ERR wait_ready() { local name=$1 diff --git a/tests/runtime-security.sh b/tests/runtime-security.sh index 6b5229b..10d33d9 100644 --- a/tests/runtime-security.sh +++ b/tests/runtime-security.sh @@ -24,7 +24,12 @@ cleanup() { "${runtime}" volume rm --force "${volumes[@]}" >/dev/null 2>&1 || true } trap cleanup EXIT -trap 'status=$?; printf "runtime-security failed at line %s\n" "${LINENO}" >&2; exit "${status}"' ERR +report_error() { + failure_status=$? + printf 'runtime-security failed at line %s\n' "${BASH_LINENO[0]}" >&2 + exit "${failure_status}" +} +trap report_error ERR new_container() { containers+=("$1") @@ -138,9 +143,8 @@ run_restricted "${primary}" "${data_volume}" \ --mount "type=volume,src=${secret_volume},dst=/run/secrets,readonly" wait_ready "${primary}" "${password}" -"${runtime}" exec "${primary}" psql -qAt --host=/tmp --username=postgres \ - --command="SELECT current_setting('data_checksums'), current_setting('password_encryption');" \ - | grep -Fxq 'on|scram-sha-256' +test "$("${runtime}" exec "${primary}" psql -qAt --host=/tmp --username=postgres \ + --command='SHOW password_encryption;')" = scram-sha-256 # Variables expand inside the database container. # shellcheck disable=SC2016 "${runtime}" exec "${primary}" sh -ceu ' diff --git a/tests/storage-lifecycle.sh b/tests/storage-lifecycle.sh index 02dd5cf..48ed32d 100644 --- a/tests/storage-lifecycle.sh +++ b/tests/storage-lifecycle.sh @@ -28,7 +28,12 @@ cleanup() { "${runtime}" volume rm --force "${volumes[@]}" >/dev/null 2>&1 || true } trap cleanup EXIT -trap 'status=$?; printf "storage-lifecycle failed at line %s\n" "${LINENO}" >&2; exit "${status}"' ERR +report_error() { + failure_status=$? + printf 'storage-lifecycle failed at line %s\n' "${BASH_LINENO[0]}" >&2 + exit "${failure_status}" +} +trap report_error ERR remember() { containers+=("$1") diff --git a/tests/tls.sh b/tests/tls.sh index ae2d931..dda762f 100644 --- a/tests/tls.sh +++ b/tests/tls.sh @@ -21,7 +21,12 @@ cleanup() { rm -rf -- "${fixture}" } trap cleanup EXIT -trap 'status=$?; printf "tls test failed at line %s\n" "${LINENO}" >&2; exit "${status}"' ERR +report_error() { + failure_status=$? + printf 'tls test failed at line %s\n' "${BASH_LINENO[0]}" >&2 + exit "${failure_status}" +} +trap report_error ERR make_ca() { local stem=$1 From 59a6bee361a9ebc461e8a587adc3bc8529a71de0 Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:22:39 -0500 Subject: [PATCH 04/17] fix: inspect postgres core limit directly --- tests/runtime-security.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/runtime-security.sh b/tests/runtime-security.sh index 10d33d9..a67ada6 100644 --- a/tests/runtime-security.sh +++ b/tests/runtime-security.sh @@ -148,9 +148,18 @@ test "$("${runtime}" exec "${primary}" psql -qAt --host=/tmp --username=postgres # Variables expand inside the database container. # shellcheck disable=SC2016 "${runtime}" exec "${primary}" sh -ceu ' - test "$(ulimit -c)" = 0 - ! find /tmp -maxdepth 1 -name "postgresql-password.*" -print -quit | grep -q . - ! tr "\0" "\n" &2 + exit 1 + } + if find /tmp -maxdepth 1 -name "postgresql-password.*" -print -quit | grep -q .; then + echo "transient password file persisted after initialization" >&2 + exit 1 + fi + if tr "\0" "\n" &2 + exit 1 + fi ' if "${runtime}" top "${primary}" -eo args | grep -Fq "${password}"; then echo 'initialization credential exposed in process arguments' >&2 From 007f7c1325fee2c903a8b1bc8e4ea4f89ecd0835 Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:26:37 -0500 Subject: [PATCH 05/17] fix: validate mounted configuration syntax --- tests/runtime-security.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/runtime-security.sh b/tests/runtime-security.sh index a67ada6..bd0cecd 100644 --- a/tests/runtime-security.sh +++ b/tests/runtime-security.sh @@ -161,7 +161,7 @@ test "$("${runtime}" exec "${primary}" psql -qAt --host=/tmp --username=postgres exit 1 fi ' -if "${runtime}" top "${primary}" -eo args | grep -Fq "${password}"; then +if "${runtime}" top "${primary}" | grep -Fq "${password}"; then echo 'initialization credential exposed in process arguments' >&2 exit 1 fi @@ -209,12 +209,13 @@ test "$("${runtime}" exec --env "PGPASSWORD=${password}" "${primary}" \ "${runtime}" stop --time 30 "${primary}" >/dev/null "${runtime}" rm "${primary}" >/dev/null -seed_file "${config_volume}" postgresql.conf 444 $'max_connections = 37\npassword_encryption = md5\nlogging_collector = on\nlog_statement = all\nhba_file = '\''/tmp/untrusted-hba'\''\nssl = on\n' +seed_file "${config_volume}" postgresql.conf 444 $'max_connections = 37\npassword_encryption = md5\nlogging_collector = on\nlog_statement = all\nssl = on\n' configured="${prefix}-configured" run_restricted "${configured}" "${data_volume}" \ --env POSTGRESQL_CONFIG_FILE=/run/config/postgresql.conf \ --mount "type=volume,src=${config_volume},dst=/run/config,readonly" \ - --container-command postgres -c password_encryption=md5 -c log_statement=all + --container-command postgres -c password_encryption=md5 -c log_statement=all \ + -c hba_file=/tmp/untrusted-hba -c ssl=on wait_ready "${configured}" "${password}" for expectation in \ 'max_connections|37' \ From 7d3b00407ea6b78e0c6c56d0364200a101c038d7 Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:29:21 -0500 Subject: [PATCH 06/17] test: expose storage failure diagnostics --- tests/storage-lifecycle.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/storage-lifecycle.sh b/tests/storage-lifecycle.sh index 48ed32d..ea60af3 100644 --- a/tests/storage-lifecycle.sh +++ b/tests/storage-lifecycle.sh @@ -76,7 +76,11 @@ wait_failed() { for _ in {1..60}; do if test "$("${runtime}" inspect --format '{{.State.Status}}' "${name}")" != running; then test "$("${runtime}" inspect --format '{{.State.ExitCode}}' "${name}")" != 0 - "${runtime}" logs "${name}" 2>&1 | grep -Fq "${expected}" + if ! "${runtime}" logs "${name}" 2>&1 | grep -Fq "${expected}"; then + printf 'expected failure text not found: %s\n' "${expected}" >&2 + "${runtime}" logs "${name}" >&2 + return 1 + fi return fi sleep 1 From bf71a4b132f6b90236633252bb566ad87e86370c Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:31:52 -0500 Subject: [PATCH 07/17] fix: match read-only storage diagnostic --- tests/storage-lifecycle.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/storage-lifecycle.sh b/tests/storage-lifecycle.sh index ea60af3..359238f 100644 --- a/tests/storage-lifecycle.sh +++ b/tests/storage-lifecycle.sh @@ -207,7 +207,7 @@ remember "${readonly_data}" --tmpfs /tmp:rw,noexec,nosuid,nodev,size=64m,mode=1777 \ --mount "type=volume,src=${data_volume},dst=/var/lib/pgsql,readonly" \ --cap-drop ALL --security-opt "${no_new_privileges}" "${image}" >/dev/null -wait_failed "${readonly_data}" 'PGDATA cannot accept a validation write' +wait_failed "${readonly_data}" 'PGDATA is not writable' full="${prefix}-full" remember "${full}" From c63183aa0f6d8cbef0f0d2b9648d08608d7b0b76 Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:35:53 -0500 Subject: [PATCH 08/17] fix: stabilize lifecycle test diagnostics --- tests/runtime-security.sh | 12 ++++++++---- tests/storage-lifecycle.sh | 12 +++++++----- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/runtime-security.sh b/tests/runtime-security.sh index bd0cecd..779f422 100644 --- a/tests/runtime-security.sh +++ b/tests/runtime-security.sh @@ -82,7 +82,8 @@ expect_failure() { echo "expected ${name} to fail" >&2 return 1 fi - "${runtime}" logs "${name}" 2>&1 | grep -Fq "${expected}" + failure_logs=$("${runtime}" logs "${name}" 2>&1) + grep -Fq "${expected}" <<<"${failure_logs}" } seed_file() { @@ -161,11 +162,13 @@ test "$("${runtime}" exec "${primary}" psql -qAt --host=/tmp --username=postgres exit 1 fi ' -if "${runtime}" top "${primary}" | grep -Fq "${password}"; then +process_list=$("${runtime}" top "${primary}") +if grep -Fq "${password}" <<<"${process_list}"; then echo 'initialization credential exposed in process arguments' >&2 exit 1 fi -if "${runtime}" logs "${primary}" 2>&1 | grep -Fq "${password}"; then +primary_logs=$("${runtime}" logs "${primary}" 2>&1) +if grep -Fq "${password}" <<<"${primary_logs}"; then echo 'initialization credential exposed in logs' >&2 exit 1 fi @@ -237,7 +240,8 @@ local all all trust host all all 0.0.0.0/0 scram-sha-256 host all all ::/0 scram-sha-256" -if "${runtime}" logs "${configured}" 2>&1 | grep -Fq "${rotated}"; then +configured_logs=$("${runtime}" logs "${configured}" 2>&1) +if grep -Fq "${rotated}" <<<"${configured_logs}"; then echo 'SQL value exposed in database logs' >&2 exit 1 fi diff --git a/tests/storage-lifecycle.sh b/tests/storage-lifecycle.sh index 359238f..9417ad8 100644 --- a/tests/storage-lifecycle.sh +++ b/tests/storage-lifecycle.sh @@ -76,7 +76,8 @@ wait_failed() { for _ in {1..60}; do if test "$("${runtime}" inspect --format '{{.State.Status}}' "${name}")" != running; then test "$("${runtime}" inspect --format '{{.State.ExitCode}}' "${name}")" != 0 - if ! "${runtime}" logs "${name}" 2>&1 | grep -Fq "${expected}"; then + failure_logs=$("${runtime}" logs "${name}" 2>&1) + if ! grep -Fq "${expected}" <<<"${failure_logs}"; then printf 'expected failure text not found: %s\n' "${expected}" >&2 "${runtime}" logs "${name}" >&2 return 1 @@ -143,8 +144,9 @@ row_count=$("${runtime}" exec "${recovered}" psql -qAt --host=/tmp --username=po test "${row_count}" -eq 30000 || test "${row_count}" -eq 90000 test "$("${runtime}" exec "${recovered}" psql -qAt --host=/tmp --username=postgres \ --command='SELECT pg_is_in_recovery();')" = f -"${runtime}" logs "${recovered}" 2>&1 | grep -Eq \ - 'database system was interrupted|database system was not properly shut down|redo starts at' +recovery_logs=$("${runtime}" logs "${recovered}" 2>&1) +grep -Eq 'database system was interrupted|database system was not properly shut down|redo starts at' \ + <<<"${recovery_logs}" # Logical backup and isolated restore validate schema, row count, and content. "${runtime}" run --rm --user 0 \ @@ -233,11 +235,11 @@ volumes+=("${concurrent_volume}") first="${prefix}-concurrent-one" second="${prefix}-concurrent-two" run_database "${first}" "${concurrent_volume}" \ - --env "POSTGRES_PASSWORD=${password}" --env PATH=/hook:/usr/pgsql-18/bin:/usr/bin:/bin \ + --env "POSTGRES_PASSWORD=${password}" --env PATH=/hook:/usr/local/bin:/usr/pgsql-18/bin:/usr/bin:/bin \ --mount "type=volume,src=${hook_volume},dst=/hook,readonly" sleep 1 run_database "${second}" "${concurrent_volume}" \ - --env "POSTGRES_PASSWORD=${password}" --env PATH=/hook:/usr/pgsql-18/bin:/usr/bin:/bin \ + --env "POSTGRES_PASSWORD=${password}" --env PATH=/hook:/usr/local/bin:/usr/pgsql-18/bin:/usr/bin:/bin \ --mount "type=volume,src=${hook_volume},dst=/hook,readonly" wait_failed "${second}" 'initialization is already running' "${runtime}" stop --time 5 "${first}" >/dev/null || true From c285085aa87e52025a7edba6a567a17abb353856 Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:40:45 -0500 Subject: [PATCH 09/17] test: require explicit interrupted init recovery --- tests/storage-lifecycle.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/storage-lifecycle.sh b/tests/storage-lifecycle.sh index 9417ad8..fa29828 100644 --- a/tests/storage-lifecycle.sh +++ b/tests/storage-lifecycle.sh @@ -247,6 +247,18 @@ wait_failed "${second}" 'initialization is already running' interrupted_restart="${prefix}-interrupted-restart" run_database "${interrupted_restart}" "${concurrent_volume}" --env "POSTGRES_PASSWORD=${password}" -wait_ready "${interrupted_restart}" +wait_failed "${interrupted_restart}" 'stale initialization lock exists' +"${runtime}" rm "${interrupted_restart}" >/dev/null + +# Recovery is an explicit operator decision after confirming that initdb never +# began and the database directory is absent. The entrypoint never guesses. +"${runtime}" run --rm --user 26:0 \ + --mount "type=volume,src=${concurrent_volume},dst=/var/lib/pgsql" \ + --entrypoint sh "${image}" -ceu \ + 'test ! -e /var/lib/pgsql/data; rmdir /var/lib/pgsql/data.postgresql-ubi.init.lock' +recovered_initialization="${prefix}-recovered-initialization" +run_database "${recovered_initialization}" "${concurrent_volume}" \ + --env "POSTGRES_PASSWORD=${password}" +wait_ready "${recovered_initialization}" echo "PostgreSQL durable-storage and lifecycle contract passed for ${image}" From 4c1a48cc5edd6e7306afaebc9869f8d4f45b13e8 Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:45:59 -0500 Subject: [PATCH 10/17] test: qualify database resource limits --- .github/workflows/ci.yml | 6 ++ CHANGELOG.md | 6 ++ docs/CI.md | 3 + docs/DEPLOYMENT.md | 39 +++++++++ tests/resource-limits.sh | 161 +++++++++++++++++++++++++++++++++++++ tests/storage-lifecycle.sh | 6 +- 6 files changed, 219 insertions(+), 2 deletions(-) create mode 100755 tests/resource-limits.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 78a1348..f25a7e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,6 +175,12 @@ jobs: IMAGE: ${{ env.TEST_IMAGE }} run: bash tests/storage-lifecycle.sh + - name: Test resource exhaustion and recovery + env: + CONTAINER_RUNTIME: docker + IMAGE: ${{ env.TEST_IMAGE }} + run: bash tests/resource-limits.sh + - name: Test a preserved-data PostgreSQL 18 minor update env: CONTAINER_RUNTIME: docker diff --git a/CHANGELOG.md b/CHANGELOG.md index 1848783..20efc02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,3 +37,9 @@ but container releases use the upstream-derived format documented in - Added schema-defined AMD64 and ARM64 artifact locks for the complete runtime RPM and source closure, fail-closed acquisition and publisher verification, a manual lock-update workflow, and network-disabled offline assembly. +- Added the PostgreSQL authentication, initialization, immutable configuration, + storage, crash-recovery, backup/restore, minor-update, TLS, resource-limit, + logging, and probe runtime contract with native architecture tests. +- Added executable deployment guidance for fixed and arbitrary identities, + Podman, Docker and Compose, named and bind-mounted storage, TLS and isolated + non-TLS networks, controlled transfer, operations, recovery, and teardown. diff --git a/docs/CI.md b/docs/CI.md index 9279b21..6a8991a 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -5,6 +5,9 @@ image builds, restricted-runtime smoke tests, Trivy and Grype vulnerability gates, and Syft SPDX SBOM generation. Runtime tests cover authentication and configuration precedence, initialization failure modes, durable-state lifecycle and crash recovery, logical backup/restoration, and the TLS profile. +Resource tests exercise inode, shared-memory, connection, and cgroup-memory +exhaustion under explicit file-descriptor and PID ceilings, then validate +durable-data recovery. The aggregate `image` job fails unless every native image job succeeds. Each image job acquires the architecture's committed lock, verifies the bundle and full key fingerprints, pre-pulls only diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 55f48e0..f3f4624 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -338,6 +338,45 @@ role and transaction/retry policy. abnormal shutdown. Size termination grace from observed checkpoint/shutdown time and large-WAL crash recovery. +Apply explicit runtime ceilings to the start command. This test baseline is a +starting point only; replace it with limits measured against the deployment's +connection count, query shapes, extensions, maintenance work, backup activity, +and recovery workload: + +```console +$RUNTIME run --detach --name postgresql \ + --memory=2g --memory-swap=2g --shm-size=256m \ + --ulimit nofile=4096:4096 --pids-limit=512 \ + ...the identity, storage, secret, TLS, and hardening options above... \ + $IMAGE@$DIGEST +``` + +After starting, record the applied runtime limits, wait for local readiness, +and run the external TLS-verified transaction probe. With Docker, inspect +`.HostConfig.Memory`, `.HostConfig.MemorySwap`, `.HostConfig.ShmSize`, +`.HostConfig.Ulimits`, and `.HostConfig.PidsLimit`; use the equivalent Podman +inspection fields. Also record `SHOW max_connections`, `SHOW shared_buffers`, +and the workload-specific settings that affect per-session memory. A runtime +limit displayed in a manifest but absent from inspection is not applied +evidence. + +Exercise the following failures in a disposable qualification environment, +then repeat after every material workload or configuration change: + +| Condition | Expected signal | Required response | +| --- | --- | --- | +| Data bytes or inodes exhausted | Write/checkpoint/init fails and storage reports exhaustion | Stop new workload, preserve the volume, add capacity or free only approved disposable files, then verify WAL recovery and application data. Never delete PostgreSQL files manually. | +| `/tmp` or shared memory exhausted | Operation fails with an allocation/space diagnostic | Remove only the identified disposable file or increase the measured limit; confirm readiness and an external transaction before returning traffic. | +| File descriptor or PID ceiling | Connection/background-worker creation fails | Reject excess work, inspect actual limits and active sessions/workers, increase a justified ceiling or reduce concurrency, and verify reserved operator access. | +| Connection slots exhausted | Application receives a bounded refusal while reserved superuser slots remain | Shed/retry at the client, find leaked/long sessions, and tune pools; do not grant the application superuser or consume reserved slots for monitoring. | +| Memory pressure or OOM kill | Runtime memory event, killed backend/container, or abnormal shutdown | Preserve storage, identify the allocating query/process, adjust workload/settings or the measured limit, restart once, allow WAL recovery, validate data, and take a fresh backup. | +| Startup recovery exceeds probe window | Recovery log progress while application probe is not ready | Keep traffic blocked and extend startup allowance; do not let a liveness loop repeatedly kill a recovering database. Escalate if progress stops. | + +For each exercise, retain timestamps, immutable image digest, applied limits, +runtime events, redacted PostgreSQL logs, probe results, recovery duration, and +data-validation result. Never place SQL values, passwords, private keys, or +personal data in the evidence bundle. + Logs stay on stdout/stderr. Collect them with runtime/orchestrator metadata and access controls. The immutable profile disables statement, duration, and bind parameter logging because SQL and values can contain credentials, personal diff --git a/tests/resource-limits.sh b/tests/resource-limits.sh new file mode 100755 index 0000000..ecb537a --- /dev/null +++ b/tests/resource-limits.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +runtime=${CONTAINER_RUNTIME:-podman} +image=${IMAGE:-localhost/postgresql-ubi:development} +prefix="postgresql-ubi-resources-${RANDOM}-$$" +password="resources-${RANDOM}-OnlyForTesting" +application_password="application-${RANDOM}-OnlyForTesting" +data_volume="${prefix}-data" +containers=() +no_new_privileges=no-new-privileges:true + +if grep -qi podman <<<"$("${runtime}" --version 2>&1)"; then + no_new_privileges=no-new-privileges +fi + +cleanup() { + if test "${#containers[@]}" -gt 0; then + "${runtime}" rm --force --volumes "${containers[@]}" >/dev/null 2>&1 || true + fi + "${runtime}" volume rm --force "${data_volume}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +report_error() { + local status=$? + printf 'resource-limits failed at line %s\n' "${BASH_LINENO[0]}" >&2 + exit "${status}" +} +trap report_error ERR + +remember() { + containers+=("$1") +} + +run_limited() { + local name=$1 + shift + remember "${name}" + "${runtime}" run --detach --name "${name}" \ + --read-only \ + --tmpfs /tmp:rw,noexec,nosuid,nodev,size=768m,mode=1777 \ + --shm-size=256m \ + --memory=384m --memory-swap=384m \ + --ulimit nofile=256:256 --pids-limit=128 \ + --mount "type=volume,src=${data_volume},dst=/var/lib/pgsql" \ + --cap-drop ALL --security-opt "${no_new_privileges}" \ + "$@" "${image}" \ + -c max_connections=12 -c superuser_reserved_connections=3 >/dev/null +} + +wait_ready() { + local name=$1 + local _ + for _ in {1..120}; do + if "${runtime}" exec --env "PGPASSWORD=${password}" "${name}" \ + psql -qAt --host=127.0.0.1 --username=postgres --dbname=postgres \ + --command='SELECT 1' >/dev/null 2>&1; then + return + fi + sleep 1 + done + "${runtime}" inspect --format '{{json .State}}' "${name}" >&2 || true + "${runtime}" logs "${name}" >&2 || true + return 1 +} + +"${runtime}" volume create "${data_volume}" >/dev/null +database="${prefix}-database" +run_limited "${database}" --env "POSTGRES_PASSWORD=${password}" +wait_ready "${database}" + +"${runtime}" exec "${database}" psql --host=/tmp --username=postgres \ + --set=ON_ERROR_STOP=1 \ + --command='CREATE TABLE resource_probe(value integer PRIMARY KEY);' \ + --command='INSERT INTO resource_probe VALUES (1);' \ + --command="CREATE ROLE application LOGIN PASSWORD '${application_password}' NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;" >/dev/null + +# Fill every non-reserved connection slot with a low-privilege role. The next +# application connection must fail while a local superuser diagnostic remains +# available through the explicitly reserved slots. +for _ in {1..9}; do + "${runtime}" exec --detach --env "PGPASSWORD=${application_password}" "${database}" \ + psql --host=127.0.0.1 --username=application --dbname=postgres \ + --command='SELECT pg_sleep(120);' +done +for _ in {1..30}; do + connection_count=$("${runtime}" exec "${database}" psql -qAt \ + --host=/tmp --username=postgres \ + --command="SELECT count(*) FROM pg_stat_activity WHERE usename='application';") + test "${connection_count}" -eq 9 && break + sleep 1 +done +test "${connection_count}" -eq 9 +if "${runtime}" exec --env "PGPASSWORD=${application_password}" "${database}" \ + psql --host=127.0.0.1 --username=application --dbname=postgres \ + --command='SELECT 1' >/dev/null 2>&1; then + echo 'application connection was accepted after non-reserved slots were exhausted' >&2 + exit 1 +fi +test "$("${runtime}" exec "${database}" psql -qAt --host=/tmp --username=postgres \ + --command='SELECT value FROM resource_probe;')" = 1 +"${runtime}" exec "${database}" psql -qAt --host=/tmp --username=postgres \ + --command="SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE usename='application';" >/dev/null + +# Exhaust the deliberately bounded shared-memory filesystem without touching +# durable storage. PostgreSQL must remain available after the failed write. +if "${runtime}" exec "${database}" dd if=/dev/zero of=/dev/shm/exhaustion \ + bs=1M count=320 status=none 2>/dev/null; then + echo 'bounded shared memory unexpectedly accepted a 320 MiB write' >&2 + exit 1 +fi +"${runtime}" exec "${database}" rm -f /dev/shm/exhaustion +wait_ready "${database}" + +# Force a cgroup-contained OOM with a disposable tmpfs write. The database may +# survive or be killed; either outcome must preserve and recover durable data. +# The awk fields expand inside the database container. +# shellcheck disable=SC2016 +oom_before=$("${runtime}" exec "${database}" awk '$1 == "oom" { print $2 }' \ + /sys/fs/cgroup/memory.events) +"${runtime}" exec "${database}" dd if=/dev/zero of=/tmp/memory-pressure \ + bs=1M count=640 status=none >/dev/null 2>&1 || true +if test "$("${runtime}" inspect --format '{{.State.Running}}' "${database}")" = true; then + # shellcheck disable=SC2016 + oom_after=$("${runtime}" exec "${database}" awk '$1 == "oom" { print $2 }' \ + /sys/fs/cgroup/memory.events) + test "${oom_after}" -gt "${oom_before}" + "${runtime}" exec "${database}" rm -f /tmp/memory-pressure || true + wait_ready "${database}" + "${runtime}" stop --time 30 "${database}" >/dev/null +else + test "$("${runtime}" inspect --format '{{.State.OOMKilled}}' "${database}")" = true +fi +"${runtime}" rm "${database}" >/dev/null + +recovered="${prefix}-recovered" +run_limited "${recovered}" +wait_ready "${recovered}" +test "$("${runtime}" exec "${recovered}" psql -qAt --host=/tmp --username=postgres \ + --command='SELECT value FROM resource_probe;')" = 1 + +# A first initialization with too few inodes must fail closed with no fallback +# to another data path. +inode_limited="${prefix}-inode-limited" +remember "${inode_limited}" +"${runtime}" run --detach --name "${inode_limited}" \ + --read-only \ + --tmpfs /tmp:rw,noexec,nosuid,nodev,size=64m,mode=1777 \ + --tmpfs /var/lib/pgsql:rw,nosuid,nodev,size=256m,nr_inodes=64,uid=26,gid=0,mode=0770 \ + --cap-drop ALL --security-opt "${no_new_privileges}" \ + --env "POSTGRES_PASSWORD=${password}" "${image}" >/dev/null +for _ in {1..60}; do + test "$("${runtime}" inspect --format '{{.State.Running}}' "${inode_limited}")" = false && break + sleep 1 +done +test "$("${runtime}" inspect --format '{{.State.ExitCode}}' "${inode_limited}")" != 0 +inode_logs=$("${runtime}" logs "${inode_limited}" 2>&1) +grep -Fq 'No space left on device' <<<"${inode_logs}" + +echo "PostgreSQL resource-limit and recovery contract passed for ${image}" diff --git a/tests/storage-lifecycle.sh b/tests/storage-lifecycle.sh index fa29828..ed9ae6a 100644 --- a/tests/storage-lifecycle.sh +++ b/tests/storage-lifecycle.sh @@ -251,11 +251,13 @@ wait_failed "${interrupted_restart}" 'stale initialization lock exists' "${runtime}" rm "${interrupted_restart}" >/dev/null # Recovery is an explicit operator decision after confirming that initdb never -# began and the database directory is absent. The entrypoint never guesses. +# began and the database directory is empty. The entrypoint never guesses. +# Command substitution expands inside the inspection container. +# shellcheck disable=SC2016 "${runtime}" run --rm --user 26:0 \ --mount "type=volume,src=${concurrent_volume},dst=/var/lib/pgsql" \ --entrypoint sh "${image}" -ceu \ - 'test ! -e /var/lib/pgsql/data; rmdir /var/lib/pgsql/data.postgresql-ubi.init.lock' + 'test -d /var/lib/pgsql/data; test -z "$(find /var/lib/pgsql/data -mindepth 1 -print -quit)"; rmdir /var/lib/pgsql/data.postgresql-ubi.init.lock' recovered_initialization="${prefix}-recovered-initialization" run_database "${recovered_initialization}" "${concurrent_volume}" \ --env "POSTGRES_PASSWORD=${password}" From ec261982de70163d912e7892a2165a075fc60d09 Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:50:40 -0500 Subject: [PATCH 11/17] fix: invoke postgres in resource tests --- tests/resource-limits.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/resource-limits.sh b/tests/resource-limits.sh index ecb537a..d973fd1 100755 --- a/tests/resource-limits.sh +++ b/tests/resource-limits.sh @@ -45,7 +45,7 @@ run_limited() { --ulimit nofile=256:256 --pids-limit=128 \ --mount "type=volume,src=${data_volume},dst=/var/lib/pgsql" \ --cap-drop ALL --security-opt "${no_new_privileges}" \ - "$@" "${image}" \ + "$@" "${image}" postgres \ -c max_connections=12 -c superuser_reserved_connections=3 >/dev/null } @@ -130,7 +130,10 @@ if test "$("${runtime}" inspect --format '{{.State.Running}}' "${database}")" = wait_ready "${database}" "${runtime}" stop --time 30 "${database}" >/dev/null else - test "$("${runtime}" inspect --format '{{.State.OOMKilled}}' "${database}")" = true + if test "$("${runtime}" inspect --format '{{.State.OOMKilled}}' "${database}")" != true; then + database_logs=$("${runtime}" logs "${database}" 2>&1) + grep -Eq 'terminated by signal 9|out of memory|oom-kill' <<<"${database_logs}" + fi fi "${runtime}" rm "${database}" >/dev/null From bdb5df94cb156d91938304481b757cbe8f2e17f0 Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:53:04 -0500 Subject: [PATCH 12/17] test: harden resource and log evidence --- tests/resource-limits.sh | 26 ++++++++++++++++---------- tests/runtime-security.sh | 16 ++++++++++++++-- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/tests/resource-limits.sh b/tests/resource-limits.sh index d973fd1..e11bbb0 100755 --- a/tests/resource-limits.sh +++ b/tests/resource-limits.sh @@ -69,6 +69,9 @@ wait_ready() { database="${prefix}-database" run_limited "${database}" --env "POSTGRES_PASSWORD=${password}" wait_ready "${database}" +test "$("${runtime}" exec "${database}" sh -c 'ulimit -n')" = 256 +test "$("${runtime}" exec "${database}" cat /sys/fs/cgroup/pids.max)" = 128 +test "$("${runtime}" exec "${database}" cat /sys/fs/cgroup/memory.max)" = 402653184 "${runtime}" exec "${database}" psql --host=/tmp --username=postgres \ --set=ON_ERROR_STOP=1 \ @@ -121,20 +124,23 @@ oom_before=$("${runtime}" exec "${database}" awk '$1 == "oom" { print $2 }' \ /sys/fs/cgroup/memory.events) "${runtime}" exec "${database}" dd if=/dev/zero of=/tmp/memory-pressure \ bs=1M count=640 status=none >/dev/null 2>&1 || true -if test "$("${runtime}" inspect --format '{{.State.Running}}' "${database}")" = true; then - # shellcheck disable=SC2016 - oom_after=$("${runtime}" exec "${database}" awk '$1 == "oom" { print $2 }' \ - /sys/fs/cgroup/memory.events) +# The postmaster can begin its safety shutdown between an inspect and exec, so +# collect in-cgroup evidence opportunistically and use stopped-state/log +# evidence when the cgroup has already gone away. +# shellcheck disable=SC2016 +oom_after=$("${runtime}" exec "${database}" awk '$1 == "oom" { print $2 }' \ + /sys/fs/cgroup/memory.events 2>/dev/null || true) +if [[ "${oom_after}" =~ ^[0-9]+$ ]]; then test "${oom_after}" -gt "${oom_before}" +else + database_logs=$("${runtime}" logs "${database}" 2>&1) + grep -Eq 'terminated by signal 9|out of memory|oom-kill' <<<"${database_logs}" +fi +if test "$("${runtime}" inspect --format '{{.State.Running}}' "${database}")" = true; then "${runtime}" exec "${database}" rm -f /tmp/memory-pressure || true wait_ready "${database}" - "${runtime}" stop --time 30 "${database}" >/dev/null -else - if test "$("${runtime}" inspect --format '{{.State.OOMKilled}}' "${database}")" != true; then - database_logs=$("${runtime}" logs "${database}" 2>&1) - grep -Eq 'terminated by signal 9|out of memory|oom-kill' <<<"${database_logs}" - fi fi +"${runtime}" stop --time 30 "${database}" >/dev/null 2>&1 || true "${runtime}" rm "${database}" >/dev/null recovered="${prefix}-recovered" diff --git a/tests/runtime-security.sh b/tests/runtime-security.sh index 779f422..12ba7d1 100644 --- a/tests/runtime-security.sh +++ b/tests/runtime-security.sh @@ -6,6 +6,7 @@ image=${IMAGE:-localhost/postgresql-ubi:development} prefix="postgresql-ubi-security-${RANDOM}-$$" password='Odd !@#$%^&*()[]{}:;,.?=+_- value' rotated='Rotated !@#$%^&*()[]{}:;,.?=+_- value' +sensitive_value='person-for-log-test@example.invalid' data_volume="${prefix}-data" secret_volume="${prefix}-secrets" config_volume="${prefix}-config" @@ -177,6 +178,8 @@ fi --set=ON_ERROR_STOP=1 --command="CREATE ROLE app LOGIN PASSWORD '${rotated}' NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;" \ --command='CREATE TABLE role_probe(value integer);' \ --command='GRANT SELECT ON role_probe TO app;' >/dev/null +"${runtime}" exec "${primary}" psql --host=/tmp --username=postgres \ + --command="SELECT '${sensitive_value}';" >/dev/null test "$("${runtime}" exec --env "PGPASSWORD=${rotated}" "${primary}" \ psql -qAt --host=127.0.0.1 --username=app --dbname=postgres \ --command='SELECT count(*) FROM role_probe;')" = 0 @@ -204,6 +207,12 @@ fi test "$("${runtime}" exec --env "PGPASSWORD=${password}" "${primary}" \ psql -qAt --host=127.0.0.1 --username=app --dbname=postgres \ --command='SELECT 1')" = 1 +primary_logs=$("${runtime}" logs "${primary}" 2>&1) +if grep -Fq "${rotated}" <<<"${primary_logs}" || \ + grep -Fq "${sensitive_value}" <<<"${primary_logs}"; then + echo 'credential or personal-data fixture exposed in database logs' >&2 + exit 1 +fi # Persisted attempts to weaken HBA are replaced at the next start. # PGDATA expands inside the database container. @@ -220,6 +229,8 @@ run_restricted "${configured}" "${data_volume}" \ --container-command postgres -c password_encryption=md5 -c log_statement=all \ -c hba_file=/tmp/untrusted-hba -c ssl=on wait_ready "${configured}" "${password}" +"${runtime}" exec "${configured}" psql --host=/tmp --username=postgres \ + --command="SELECT '${sensitive_value}';" >/dev/null for expectation in \ 'max_connections|37' \ 'password_encryption|scram-sha-256' \ @@ -241,8 +252,9 @@ host all all 0.0.0.0/0 scram-sha-256 host all all ::/0 scram-sha-256" configured_logs=$("${runtime}" logs "${configured}" 2>&1) -if grep -Fq "${rotated}" <<<"${configured_logs}"; then - echo 'SQL value exposed in database logs' >&2 +if grep -Fq "${rotated}" <<<"${configured_logs}" || \ + grep -Fq "${sensitive_value}" <<<"${configured_logs}"; then + echo 'credential or personal-data fixture exposed in database logs' >&2 exit 1 fi From af9f26a105d4515af72d0413d6daaef522526606 Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:57:16 -0500 Subject: [PATCH 13/17] test: stabilize cgroup OOM evidence --- tests/resource-limits.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/resource-limits.sh b/tests/resource-limits.sh index e11bbb0..2d0aa85 100755 --- a/tests/resource-limits.sh +++ b/tests/resource-limits.sh @@ -124,6 +124,7 @@ oom_before=$("${runtime}" exec "${database}" awk '$1 == "oom" { print $2 }' \ /sys/fs/cgroup/memory.events) "${runtime}" exec "${database}" dd if=/dev/zero of=/tmp/memory-pressure \ bs=1M count=640 status=none >/dev/null 2>&1 || true +sleep 2 # The postmaster can begin its safety shutdown between an inspect and exec, so # collect in-cgroup evidence opportunistically and use stopped-state/log # evidence when the cgroup has already gone away. @@ -133,8 +134,10 @@ oom_after=$("${runtime}" exec "${database}" awk '$1 == "oom" { print $2 }' \ if [[ "${oom_after}" =~ ^[0-9]+$ ]]; then test "${oom_after}" -gt "${oom_before}" else - database_logs=$("${runtime}" logs "${database}" 2>&1) - grep -Eq 'terminated by signal 9|out of memory|oom-kill' <<<"${database_logs}" + if test "$("${runtime}" inspect --format '{{.State.OOMKilled}}' "${database}")" != true; then + database_logs=$("${runtime}" logs "${database}" 2>&1) + grep -Eq 'terminated by signal 9|out of memory|oom-kill' <<<"${database_logs}" + fi fi if test "$("${runtime}" inspect --format '{{.State.Running}}' "${database}")" = true; then "${runtime}" exec "${database}" rm -f /tmp/memory-pressure || true From ce3d072fe7b17759e9996c066f1df6aea73d2416 Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:02:28 -0500 Subject: [PATCH 14/17] fix: compare numeric postgres update versions --- tests/minor-update.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/minor-update.sh b/tests/minor-update.sh index 3e1d6e0..180dd95 100644 --- a/tests/minor-update.sh +++ b/tests/minor-update.sh @@ -44,7 +44,7 @@ wait_ready() { for _ in {1..120}; do if test "$("${runtime}" exec --env "PGPASSWORD=${password}" "${name}" \ psql -qAt --host=127.0.0.1 --username=postgres --dbname=postgres \ - --command='SHOW server_version;' 2>/dev/null || true)" = "${version}"; then + --command='SHOW server_version_num;' 2>/dev/null || true)" = "${version}"; then return fi sleep 1 @@ -64,7 +64,7 @@ test "$("${runtime}" image inspect --format '{{.Architecture}}' "${previous_imag --mount "type=volume,src=${backup_volume},dst=/backup" \ --env "POSTGRES_PASSWORD=${password}" \ "${previous_image}" >/dev/null -wait_ready "${old_container}" 18.4 +wait_ready "${old_container}" 180004 test "$("${runtime}" exec "${old_container}" id -u)" = 999 # PGDATA expands inside the compatibility-fixture container. # shellcheck disable=SC2016 @@ -104,7 +104,7 @@ fixture_digest=$("${runtime}" exec "${old_container}" psql -qAt --host=/var/run/ --cap-drop ALL \ --security-opt no-new-privileges:true \ "${image}" >/dev/null -wait_ready "${new_container}" 18.6 +wait_ready "${new_container}" 180006 test "$("${runtime}" exec "${new_container}" psql -qAt --host=/tmp --username=postgres \ --command='SELECT count(*) FROM update_fixture;')" = 1000 test "$("${runtime}" exec "${new_container}" psql -qAt --host=/tmp --username=postgres \ From eb8894c48a5c73d62c39535699aefac4bb1f3a5b Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:03:45 -0500 Subject: [PATCH 15/17] test: align update fixture identities --- tests/minor-update.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/minor-update.sh b/tests/minor-update.sh index 180dd95..870b8a5 100644 --- a/tests/minor-update.sh +++ b/tests/minor-update.sh @@ -65,7 +65,12 @@ test "$("${runtime}" image inspect --format '{{.Architecture}}' "${previous_imag --env "POSTGRES_PASSWORD=${password}" \ "${previous_image}" >/dev/null wait_ready "${old_container}" 180004 -test "$("${runtime}" exec "${old_container}" id -u)" = 999 +# Docker exec defaults to root in the official fixture; inspect PostgreSQL PID +# 1 to verify the actual server identity. +# The awk fields expand inside the fixture container. +# shellcheck disable=SC2016 +test "$("${runtime}" exec "${old_container}" awk '/^Uid:/ { print $2 }' \ + /proc/1/status)" = 999 # PGDATA expands inside the compatibility-fixture container. # shellcheck disable=SC2016 test "$("${runtime}" exec "${old_container}" sh -c 'printf %s "$PGDATA"')" = \ @@ -77,9 +82,12 @@ test "$("${runtime}" exec "${old_container}" sh -c 'printf %s "$PGDATA"')" = \ fixture_digest=$("${runtime}" exec "${old_container}" psql -qAt --host=/var/run/postgresql \ --username=postgres \ --command="SELECT md5(string_agg(id || ':' || payload, ',' ORDER BY id)) FROM update_fixture;") +"${runtime}" exec --user 0 "${old_container}" sh -c \ + 'chown 999:0 /backup; chmod 0770 /backup' "${runtime}" exec "${old_container}" pg_dump --host=/var/run/postgresql \ --username=postgres --format=custom --file=/backup/pre-update.dump postgres -"${runtime}" exec "${old_container}" sh -c 'test -s /backup/pre-update.dump; chmod 0400 /backup/pre-update.dump' +"${runtime}" exec --user 0 "${old_container}" sh -c \ + 'test -s /backup/pre-update.dump; chown 999:0 /backup/pre-update.dump; chmod 0440 /backup/pre-update.dump' "${runtime}" stop --time 30 "${old_container}" >/dev/null "${runtime}" rm "${old_container}" >/dev/null From 9827bd94ce41c35796b7fa56bd2cae35b7347aa3 Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:09:32 -0500 Subject: [PATCH 16/17] test: use portable locale for minor update --- docs/STORAGE.md | 10 ++++++++++ tests/minor-update.sh | 1 + 2 files changed, 11 insertions(+) diff --git a/docs/STORAGE.md b/docs/STORAGE.md index bd50107..448d955 100644 --- a/docs/STORAGE.md +++ b/docs/STORAGE.md @@ -102,6 +102,16 @@ review and qualification: readiness, application transactions, and backup after deployment; and 8. record elapsed shutdown/start/recovery time and the exact old/new digests. +The target image must provide every libc locale recorded in the cluster's +configuration and databases. Check `SHOW lc_collate`, `SHOW lc_ctype`, the +`datcollate`/`datctype` catalog values, and any ICU locale/provider settings +before changing the base distribution. A cluster initialized with a locale +available only in the source image must not be started here until that locale +dependency is provided and qualified or the data is migrated through a +supported logical procedure. The cross-image CI fixture deliberately uses +UTF-8 encoding with the portable `C` locale; it does not claim compatibility +for untested operating-system locale definitions. + Rollback means restoring the previous backup/snapshot with the previous image, or applying a reviewed forward fix. Do not assume an older PostgreSQL binary can safely open files after a newer minor has started them. The initial 18.6 diff --git a/tests/minor-update.sh b/tests/minor-update.sh index 870b8a5..0935546 100644 --- a/tests/minor-update.sh +++ b/tests/minor-update.sh @@ -63,6 +63,7 @@ test "$("${runtime}" image inspect --format '{{.Architecture}}' "${previous_imag --mount "type=volume,src=${volume},dst=/var/lib/postgresql" \ --mount "type=volume,src=${backup_volume},dst=/backup" \ --env "POSTGRES_PASSWORD=${password}" \ + --env 'POSTGRES_INITDB_ARGS=--locale=C --encoding=UTF8 --data-checksums' \ "${previous_image}" >/dev/null wait_ready "${old_container}" 180004 # Docker exec defaults to root in the official fixture; inspect PostgreSQL PID From c8c74cd2f65ff5515e4d8c6eae618caef1e142c1 Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Fri, 11 Sep 2026 16:14:42 -0500 Subject: [PATCH 17/17] docs: complete runtime contract roadmap --- README.md | 3 ++- docs/ROADMAP.md | 49 ++++++++++++++++++++++++++++--------------------- docs/SUPPORT.md | 4 ++-- 3 files changed, 32 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index ac4fb00..e365fba 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,8 @@ The current development image provides: authentication; - a persistent data volume with read-only-root compatibility; - dropped-capability and `no-new-privileges` operation; and -- stateful smoke tests for initialization, authentication, persistence, +- native stateful tests for secure initialization and configuration, TLS, + resource exhaustion/recovery, persistence, backup/restore, minor updates, shutdown, arbitrary UIDs, and incompatible data directories. Architecture-specific locks now cover the complete binary dependency closure, diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index b3246e5..390ad32 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -79,7 +79,7 @@ Work proceeds in this dependency order: 1. **Complete:** approve the release/support boundary and evidence schema. 2. **Complete:** implement the complete artifact lock, verified out-of-build acquisition, and network-disabled assembly. -3. Close the database security, storage, lifecycle, TLS, logging, backup, +3. **Complete:** close the database security, storage, lifecycle, TLS, logging, backup, restore, and upgrade test matrix. 4. Add the release pipeline and remaining supply-chain controls. 5. Complete the cybersecurity requirement analysis, threat model, control @@ -205,30 +205,30 @@ unexpected, wrong-base, and source-mismatch cases fail closed. ### Initialization, identity, and authentication -- [ ] Threat-test both password interfaces for empty values, simultaneous +- [x] Threat-test both password interfaces for empty values, simultaneous variables, unreadable files, symlinks, permissions, unusual characters, command-line exposure, environment inspection, logs, errors, core dumps, and persistence after initialization. Document the remaining environment-variable exposure and prefer secret-file mounts. -- [ ] Test interrupted and concurrent first initialization, a non-empty +- [x] Test interrupted and concurrent first initialization, a non-empty directory without `PG_VERSION`, partial initialization residue, wrong ownership, read-only storage, full storage, and restart after each failure. Fail closed with actionable diagnostics and never silently reinitialize data. -- [ ] Review the generated `pg_hba.conf` and `postgresql.conf` line by line. +- [x] Review the generated `pg_hba.conf` and `postgresql.conf` line by line. Prove remote password authentication is SCRAM, local trust is limited to the container-local socket boundary, password encryption remains SCRAM, and configuration precedence cannot silently weaken these defaults. -- [ ] Document role and database creation, password rotation after +- [x] Document role and database creation, password rotation after initialization, superuser ownership, least-privilege application roles, and why initialization environment variables are not a general account-management interface. -- [ ] Decide whether init scripts are deliberately unsupported for v1 or add a +- [x] Decide whether init scripts are deliberately unsupported for v1 or add a narrowly specified, ordered, failure-safe interface with tests for ownership, secrets, retries, and partial execution. ### Durable storage and lifecycle -- [ ] Publish detailed, executable deployment playbooks for every v1 use case: +- [x] Publish detailed, executable deployment playbooks for every v1 use case: fixed UID and arbitrary UID, rootless Podman and Docker/Compose, named volumes and SELinux-labeled bind mounts, TLS and intentionally isolated non-TLS profiles, mounted configuration, controlled-network transfer, backup/restore, @@ -236,63 +236,70 @@ unexpected, wrong-base, and source-mismatch cases fail closed. state prerequisites, trust and ownership boundaries, every deployment step, expected verification evidence, security-sensitive alternatives, failure diagnostics, and data-preserving removal steps. -- [ ] Document named-volume and bind-mount ownership for UID `26:0` and +- [x] Document named-volume and bind-mount ownership for UID `26:0` and arbitrary UID/group `0`, including SELinux labels, NFS root-squash, CSI/PVC behavior, filesystem permissions, and safe failure diagnostics. -- [ ] Prove data checksums are enabled and test clean shutdown, termination +- [x] Prove data checksums are enabled and test clean shutdown, termination during writes, forced termination, crash recovery, restart with a large WAL, PID/socket cleanup, and container replacement without data loss. -- [ ] Test disk-full, inode-full, bounded `/tmp`, insufficient shared memory, +- [x] Test disk-full, inode-full, bounded `/tmp`, insufficient shared memory, low file-descriptor/PID limits, memory pressure/OOM, connection exhaustion, and startup under recovery. Document safe resource, `shm_size`, timeout, and termination-grace guidance without claiming universal sizing values. -- [ ] Define logical `pg_dump`/`pg_restore` backup and restore procedures, +- [x] Define logical `pg_dump`/`pg_restore` backup and restore procedures, encryption and access requirements, retention/immutability ownership, and a scheduled isolated restoration test with measured recovery time and data validation. -- [ ] Document ownership and safe starting points for physical backup, +- [x] Document ownership and safe starting points for physical backup, `pg_basebackup`, WAL archiving, point-in-time recovery, and storage snapshots. Do not imply these are complete merely because the binaries are present. -- [ ] Test a PostgreSQL 18 minor update against preserved data, application +- [x] Test a PostgreSQL 18 minor update against preserved data, application compatibility fixtures, backup/restore, and rollback constraints. Retain the previous digest and explain that downgrading database files is not assumed safe. -- [ ] Keep other-major data directories rejected and publish a major-upgrade +- [x] Keep other-major data directories rejected and publish a major-upgrade decision tree for `pg_upgrade` versus logical dump/restore without claiming a major upgrade has been qualified. ### TLS, configuration, logging, and observability -- [ ] Provide a tested TLS 1.2/1.3 profile using operator-mounted server key, +- [x] Provide a tested TLS 1.2/1.3 profile using operator-mounted server key, certificate chain, and trust store. Enforce key ownership and permissions; test correct trust, hostname failure, untrusted chain, expired/not-yet-valid certificates, clear-text policy, renewal, rotation, and rollback using an ephemeral CA with no committed private material. -- [ ] Document and test the v1 exclusion of client-certificate authentication +- [x] Document and test the v1 exclusion of client-certificate authentication and certificate-to-role mapping. Do not imply mTLS support from a server-TLS test; qualifying this later requires a new support decision and profile. -- [ ] Define the supported configuration interface, validation command, +- [x] Define the supported configuration interface, validation command, precedence, reload/restart behavior, immutable defaults, rollback, and diagnostics. Test mounted configuration and command-line overrides for both valid and security-weakening cases. -- [ ] Keep database logs on stdout/stderr and provide structured collection +- [x] Keep database logs on stdout/stderr and provide structured collection guidance for connection, authentication, checkpoint, recovery, and shutdown events. Test secret, SQL-value, and personally identifiable information exclusion; explain why broad statement logging can itself expose sensitive data. -- [ ] Document the v1 deferral of PostgreSQL audit extensions. Adding one later +- [x] Document the v1 deferral of PostgreSQL audit extensions. Adding one later requires new RPM provenance, configuration, performance, log-volume, vulnerability-lifecycle, and support decisions. -- [ ] Distinguish startup, readiness, liveness, and external transaction +- [x] Distinguish startup, readiness, liveness, and external transaction monitoring. Keep probes low privilege, bounded, non-sensitive, and resistant to load-induced restart loops. -- [ ] Compare the candidate with the matching official PostgreSQL image: +- [x] Compare the candidate with the matching official PostgreSQL image: entrypoint behavior, environment interface, users, storage paths, packages, ports, health semantics, image size, and documented compatibility gaps. **Exit evidence:** positive and negative AMD64/ARM64 tests cover every claimed database interface, security default, storage transition, and lifecycle path. +**Completed 2026-09-11:** native AMD64 and ARM64 CI run +[34647961464](https://github.com/datopsis/postgresql-ubi/actions/runs/34647961464) +passed the restricted runtime, authentication/configuration, durable storage, +resource exhaustion/recovery, PostgreSQL 18.4-to-18.6 preserved-data update, +TLS rotation/negative, vulnerability, SBOM, and repository gates at revision +`9827bd94ce41c35796b7fa56bd2cae35b7347aa3`. + ## Package 4: CI, updates, and release supply chain - [ ] Add tests for release-tag syntax, real UTC dates and sequences, diff --git a/docs/SUPPORT.md b/docs/SUPPORT.md index 844c9ca..0775fb7 100644 --- a/docs/SUPPORT.md +++ b/docs/SUPPORT.md @@ -31,8 +31,8 @@ not a supported release. | Rootless Podman | Preview/unqualified | It is the primary workflow, but exact RHEL/Podman/SELinux qualification remains open. | | Docker | Compatible for CI behavior | Native Docker CI passes; this does not establish production equivalence with the planned Podman baseline. | | OpenShift arbitrary UID | Preview/unqualified | The entrypoint is tested with an arbitrary UID in group 0; restricted-SCC deployment qualification remains open. | -| TLS | Preview/unqualified | PostgreSQL provides TLS capabilities, but the v1 mounted-certificate profile and negative tests remain open. | -| Logical backup and restore | Preview/unqualified | PostgreSQL tools are present; the documented and rehearsed v1 procedure remains open. | +| TLS | Compatible in native Docker CI | The mounted-certificate TLS 1.2/1.3 profile, trust/hostname/time failures, clear-text rejection, and rotation/rollback pass on AMD64 and ARM64; target Podman/platform qualification remains open. | +| Logical backup and restore | Compatible in native Docker CI | Custom-format dump, isolated restore, row/content validation, and an 18.4-to-18.6 update fixture pass on AMD64 and ARM64; scheduled target-platform restoration remains open. | | Physical backup, WAL archive, and PITR products | Unsupported for v1 | Operators retain ownership; product-specific qualification is deferred. | | PostgreSQL major-version upgrades | Unsupported for v1 | Other-major data directories are rejected; operators must plan pg_upgrade or logical dump/restore. | | Replication, pooling, and high availability | Unsupported for v1 | These require separate topology, availability, and recovery qualification. |