From 74c41d01bde90bf110d2d308bb0159f179e0ae90 Mon Sep 17 00:00:00 2001
From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com>
Date: Mon, 7 Sep 2026 10:46:56 -0500
Subject: [PATCH 1/6] feat: add CA-issued TLS qualification
---
.github/workflows/ci.yml | 6 +
.gitignore | 1 +
CHANGELOG.md | 2 +
Containerfile | 1 +
container/entrypoint.sh | 11 +-
container/health-client.xml | 14 +
docs/CI.md | 2 +-
docs/TLS-REHEARSAL.md | 199 ++++++++++++
docs/TLS.md | 10 +-
tests/tls-rehearsal.sh | 630 ++++++++++++++++++++++++++++++++++++
10 files changed, 872 insertions(+), 4 deletions(-)
create mode 100644 container/health-client.xml
create mode 100644 docs/TLS-REHEARSAL.md
create mode 100644 tests/tls-rehearsal.sh
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9739106..798b85a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -128,6 +128,12 @@ jobs:
IMAGE: ${{ env.TEST_IMAGE }}
run: bash tests/smoke.sh
+ - name: Rehearse CA-issued TLS
+ env:
+ CONTAINER_RUNTIME: docker
+ IMAGE: ${{ env.TEST_IMAGE }}
+ run: bash tests/tls-rehearsal.sh
+
- name: Scan image
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
diff --git a/.gitignore b/.gitignore
index 285619f..2a3417b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,7 @@
.DS_Store
.pre-commit-cache/
.smoke-secrets.*/
+.tls-rehearsal.*/
/clickhouse-server-ubi9.spdx.json
/grype.sarif
/grype-all.json
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1833ac6..8d630ef 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -26,6 +26,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
- Native AMD64 and ARM64 CI builds, smoke tests, vulnerability evidence, and release-manifest architecture validation.
- Explicit repository scope and official-image storage compatibility guidance, including the XML-based replacement for the unreleased `CLICKHOUSE_DATA_DIR` interface.
- Podman-first user procedures, a tested Podman support baseline, and rootless user-namespace permission guidance.
+- Native CA-issued TLS rehearsal covering HTTPS, native TCP, connected/disconnected outbound trust, negative cases, renewal, rollback, and operator evidence procedures.
+- Dedicated loopback-only TLS health client configuration that supports CA-issued certificate chains without weakening server-side outbound verification.
### Changed
diff --git a/Containerfile b/Containerfile
index ad801b0..70216a8 100644
--- a/Containerfile
+++ b/Containerfile
@@ -82,6 +82,7 @@ COPY --from=builder /runtime/etc/ /etc/
COPY --from=builder /runtime/var/ /var/
COPY --from=builder /runtime/docker-entrypoint-initdb.d/ /docker-entrypoint-initdb.d/
COPY --chown=101:0 --chmod=0755 container/entrypoint.sh /usr/local/bin/clickhouse-entrypoint
+COPY --chown=101:0 --chmod=0444 container/health-client.xml /usr/local/share/clickhouse-health-client.xml
COPY --chown=101:0 --chmod=0644 container/config.d/container.xml /etc/clickhouse-server/config.d/container.xml
ENV LANG="C.UTF-8" \
diff --git a/container/entrypoint.sh b/container/entrypoint.sh
index b51807c..d51e6d3 100644
--- a/container/entrypoint.sh
+++ b/container/entrypoint.sh
@@ -3,6 +3,7 @@ set -Eeuo pipefail
shopt -s nullglob
readonly CONFIG_FILE="${CLICKHOUSE_CONFIG:-/etc/clickhouse-server/config.xml}"
+readonly HEALTH_CLIENT_CONFIG="/usr/local/share/clickhouse-health-client.xml"
readonly GENERATED_DIR="/tmp/clickhouse-entrypoint"
readonly USERS_FILE="${GENERATED_DIR}/users.xml"
readonly INIT_DIR="/docker-entrypoint-initdb.d"
@@ -145,7 +146,13 @@ client_command() {
local -n command_ref=$1
local port
- command_ref=(clickhouse-client --host 127.0.0.1 --user default)
+ command_ref=(
+ env -u CLICKHOUSE_CONFIG
+ clickhouse-client
+ --config-file "${HEALTH_CLIENT_CONFIG}"
+ --host 127.0.0.1
+ --user default
+ )
if [[ -n "${CLICKHOUSE_PASSWORD}" ]]; then
command_ref+=(--password "${CLICKHOUSE_PASSWORD}")
fi
@@ -166,7 +173,7 @@ client_command() {
# This client connects only over container loopback for initialization and
# health. External clients must validate the server certificate normally.
- command_ref+=(--port "${port}" --secure --accept-invalid-certificate)
+ command_ref+=(--port "${port}" --secure)
}
healthcheck() {
diff --git a/container/health-client.xml b/container/health-client.xml
new file mode 100644
index 0000000..2650f39
--- /dev/null
+++ b/container/health-client.xml
@@ -0,0 +1,14 @@
+
+
+
+ 1
+
+
+ false
+ none
+
+ AcceptCertificateHandler
+
+
+
+
diff --git a/docs/CI.md b/docs/CI.md
index f8f4230..dbac45b 100644
--- a/docs/CI.md
+++ b/docs/CI.md
@@ -69,7 +69,7 @@ Record accepted findings in the release pull request with the advisory, affected
Each native image matrix job runs these controls in order. AMD64 uses `ubuntu-24.04` and `linux/amd64`; ARM64 uses `ubuntu-24.04-arm` and `linux/arm64`. QEMU is not installed and does not count as native-runtime evidence.
1. **Trivy configuration scan** checks the `Containerfile`, Compose configuration, and repository infrastructure configuration for high and critical misconfigurations.
-2. **Build and smoke tests** exercise startup, authentication, initialization, persistence, shutdown, read-only operation, dropped capabilities, and arbitrary UIDs.
+2. **Build and runtime tests** exercise startup, authentication, initialization, persistence, shutdown, read-only operation, dropped capabilities, arbitrary UIDs, chained CA-issued HTTPS/native TLS, public/private outbound trust, disconnected isolation, negative certificate cases, renewal, and rollback.
3. **Trivy image scan** blocks fixed high and critical operating-system or application vulnerabilities and reports its detected OS and package count for review.
4. **Complete SPDX inventory** uses Syft to inventory the tested filesystem and RPM database, then `scripts/augment-spdx.py` declares the three pinned ClickHouse TGZ components that have no RPM metadata. The script takes their version and channel from `Containerfile`, records Apache-2.0 licensing and package identifiers, and fails instead of duplicating a component Syft already found.
5. **Blocking Grype SBOM scan** scans that exact SPDX document and blocks fixed high and critical vulnerabilities.
diff --git a/docs/TLS-REHEARSAL.md b/docs/TLS-REHEARSAL.md
new file mode 100644
index 0000000..41351e6
--- /dev/null
+++ b/docs/TLS-REHEARSAL.md
@@ -0,0 +1,199 @@
+# CA-issued TLS rehearsal
+
+This runbook validates the exact image digest and certificate process before production. It covers connected and disconnected environments, ingress HTTPS/native TLS, outbound public/private trust, renewal, rollback, and negative tests. It does not turn the image repository into a deployment stack; reusable platform manifests and certificate automation belong in `clickhouse-production-stack`.
+
+Never place a private key, CSR containing private material, CA signing key, password, generated certificate, or scanner credential in Git or a GitHub Actions artifact.
+
+## Automated qualification
+
+`tests/tls-rehearsal.sh` creates an ephemeral two-tier CA, issues DNS-specific leaves, and destroys every key and certificate on exit. It runs once per native CI architecture after the normal smoke suite:
+
+```console
+CONTAINER_RUNTIME=podman \
+ IMAGE=ghcr.io/datopsis/clickhouse-server-ubi9:test \
+ bash tests/tls-rehearsal.sh
+```
+
+The automated test proves:
+
+- the server receives only its leaf key and leaf-plus-intermediate chain;
+- HTTPS and native TLS accept the issuing CA and exact DNS name;
+- unrelated CAs, wrong hostnames, clear-text clients, an incomplete chain, and an unreadable leaf key fail;
+- a connected CA bundle supports both public PKI and a controlled private-CA ClickHouse endpoint;
+- an internal Podman/Docker network is marked `internal`, can reach its controlled private-CA endpoint through ClickHouse, and cannot open a public-IP TCP connection;
+- connected and disconnected certificate renewal changes the served serial;
+- disconnected rollback restores the recorded prior serial without public access.
+
+The internal-network test is repeatable CI evidence. It does not replace an organization's controlled-media and physically or logically disconnected acceptance rehearsal.
+
+On Windows, Podman Machine file sharing does not preserve a host `chmod 000`,
+and an internal network does not expose a published port back to the Windows
+host. The script therefore runs disconnected HTTPS from another workload on
+the internal network and reports that unreadable-key and served-serial
+inspection are deferred to native Linux CI. Those assertions are not skipped
+in the native AMD64 or ARM64 GitHub Actions jobs.
+
+## 1. Define the exact test boundary
+
+Record these values in the release-candidate evidence before creating keys:
+
+```console
+IMAGE=ghcr.io/datopsis/clickhouse-server-ubi9@sha256:
+SERVER_DNS=clickhouse.example.internal
+HTTPS_PORT=8443
+NATIVE_TLS_PORT=9440
+```
+
+Record the image digest, ClickHouse version, UBI version, client versions, Podman client/server versions, DNS zone owner, issuing CA policy, expected trust anchors, permitted egress destinations, certificate owner, renewal deadline, and rollback owner. Use the same DNS name in the CSR, certificate SAN, service record, and client tests.
+
+## 2. Connected preparation and transfer inventory
+
+On an approved connected staging host, verify and export the immutable release:
+
+```console
+podman pull "${IMAGE}"
+podman image inspect "${IMAGE}" --format '{{.Digest}} {{.Architecture}}'
+cosign verify \
+ --certificate-identity-regexp='https://github.com/datopsis/clickhouse-server-ubi9/.github/workflows/release.yml@refs/tags/.*' \
+ --certificate-oidc-issuer='https://token.actions.githubusercontent.com' \
+ "${IMAGE}"
+podman save --format oci-archive \
+ --output clickhouse-server-ubi9.oci "${IMAGE}"
+```
+
+Download the release's `image.spdx.json`, `image.sigstore.json`, and `image.intoto.jsonl`. Export only the public CA roots/intermediates authorized inside the disconnected network. Include offline installers or archives for Podman, OpenSSL, `cosign`, ClickHouse clients, and approved scanner databases when those tools are not already managed inside the boundary.
+
+Create an inventory and hashes:
+
+```console
+sha256sum clickhouse-server-ubi9.oci \
+ image.spdx.json image.sigstore.json image.intoto.jsonl \
+ authorized-ca-bundle.pem > SHA256SUMS
+sha256sum --check SHA256SUMS
+```
+
+The transfer set must not contain a server private key or any CA private key. Move it through the organization's approved media, malware inspection, custody, and two-person verification process. Record media identifier, sender, recipient, timestamps, and both-side hashes outside this public repository.
+
+## 3. Import and verify inside the disconnected boundary
+
+Copy the transfer set to a controlled staging directory, make it read-only after verification, and run:
+
+```console
+sha256sum --check SHA256SUMS
+podman load --input clickhouse-server-ubi9.oci
+podman image inspect "${IMAGE}" --format '{{.Digest}} {{.Architecture}}'
+cosign verify --offline \
+ --bundle image.sigstore.json \
+ "${IMAGE}"
+```
+
+If organizational policy imports through an internal registry, push the verified image there, record the internal digest, and deploy by that digest rather than a transferred tag. Confirm internal DNS resolves `SERVER_DNS` from the client and workload networks without public DNS forwarding.
+
+## 4. Generate the key and CSR offline
+
+Generate the server key inside the disconnected security boundary, preferably in the platform secret-management system or on an encrypted administrative host:
+
+```console
+umask 077
+openssl req -new -newkey rsa:3072 -nodes \
+ -keyout tls.key -out clickhouse.csr \
+ -subj "/CN=${SERVER_DNS}" \
+ -addext "subjectAltName=DNS:${SERVER_DNS}"
+openssl req -in clickhouse.csr -noout -verify -subject \
+ -text | grep -A1 'Subject Alternative Name'
+```
+
+Send only `clickhouse.csr` to the approved internal/offline CA. The CA returns `tls.crt` with the leaf first and required intermediates afterward. Keep its root in `authorized-ca-bundle.pem`, not in the served chain. Verify before deployment:
+
+```console
+openssl x509 -in tls.crt -noout \
+ -subject -issuer -serial -dates -ext subjectAltName
+openssl verify -CAfile authorized-ca-bundle.pem tls.crt
+openssl pkey -in tls.key -pubout -outform pem | sha256sum
+openssl x509 -in tls.crt -pubkey -noout | sha256sum
+```
+
+The last two hashes must match. Move `tls.key` directly into the secret workflow and securely remove the administrative copy according to policy.
+
+## 5. Configure read-only secret mounts
+
+Copy `container/config.d/tls.example.xml` to the disconnected configuration channel. For rootless Podman, prepare the private key inside its user namespace:
+
+```console
+chmod 0444 tls.crt tls.xml
+chmod 0400 tls.key
+podman unshare chown 101:0 tls.key
+```
+
+Mount the certificate chain, private key, and `tls.xml` through separate
+read-only bind mounts exactly as shown in [TLS.md](TLS.md). Set
+`CLICKHOUSE_PASSWORD_FILE` to a separately mounted password file; do not put
+the password directly into a retained runbook or command log. For
+Kubernetes/OpenShift, create the TLS Secret from local files, use a ConfigMap
+for `tls.xml`, set read-only mounts, and confirm the assigned UID/group can read
+the projected key without granting world access.
+
+For outbound private trust, create one reviewed PEM bundle containing only approved roots/intermediates and mount it with `container/config.d/outbound-ca.example.xml`. A connected deployment bundle normally includes public roots plus private roots; a disconnected bundle should omit public roots unless an approved internal service genuinely uses them.
+
+## 6. Prove network isolation
+
+For a local rehearsal, create an internal network and verify its flag:
+
+```console
+podman network create --internal clickhouse-disconnected
+podman network inspect clickhouse-disconnected --format '{{.Internal}}'
+```
+
+The output must be `true`. In the real target environment, also capture firewall/network-policy configuration and demonstrate that the workload cannot resolve or connect to public endpoints. Do not infer isolation merely because a public DNS lookup happens to fail.
+
+Preload a controlled HTTPS endpoint signed by the internal CA on that network. Test from ClickHouse with its actual integration path—for example the `url()` table function—not only with host `curl`. Require the private endpoint to succeed and an external HTTPS URL to fail. Repeat for every production integration because S3, Kafka, LDAP, dictionaries, and remote ClickHouse connections can use different TLS settings.
+
+## 7. Validate ingress and negative cases
+
+From an authorized client using `SERVER_DNS`:
+
+```console
+curl --fail --cacert authorized-ca-bundle.pem \
+ "https://${SERVER_DNS}:${HTTPS_PORT}/ping"
+clickhouse-client --secure --host "${SERVER_DNS}" \
+ --port "${NATIVE_TLS_PORT}" \
+ --config-file ./client-config.xml \
+ --user default --password --query 'SELECT version()'
+openssl s_client -connect "${SERVER_DNS}:${HTTPS_PORT}" \
+ -servername "${SERVER_DNS}" \
+ -CAfile authorized-ca-bundle.pem /dev/null | \
+ openssl x509 -noout -subject -issuer -serial -dates
+```
+
+Retain sanitized output. Then require failures for:
+
+- an unrelated CA bundle;
+- a DNS name absent from the SAN;
+- HTTP sent to the HTTPS port;
+- non-secure native protocol sent to `9440`;
+- a chain containing only the leaf;
+- a key unreadable by the assigned container identity.
+
+Do not use an insecure client flag to make any negative test pass.
+
+## 8. Renew, roll out, and roll back
+
+Before renewal, record the current certificate serial, SHA-256 fingerprint, expiry, Secret/resource version, and deployment revision. Generate a new key and CSR inside the same boundary; do not reuse the old private key merely for convenience. Validate the returned SAN and chain, create a versioned Secret, and restart or roll out ClickHouse because certificate reload behavior must not be assumed.
+
+After rollout, repeat HTTPS, native TLS, private outbound trust, and isolation tests. Confirm the served serial equals the new serial and differs from the retired one. Keep the old Secret only for the approved rollback window.
+
+For rollback, restore the prior versioned Secret/configuration, perform another controlled restart/rollout, and verify the old recorded serial and database health. A rollback must not restore an expired, revoked, compromised, or policy-prohibited certificate. Remove retired secrets after the rollback window and record destruction.
+
+## 9. Evidence and cleanup
+
+Retain outside Git:
+
+- sanitized commands and timestamps;
+- image/internal-registry digests and transfer hashes;
+- subjects, issuers, SANs, serials, fingerprints, and expiry dates;
+- Podman/platform versions, DNS results, and isolation proof;
+- positive and negative HTTPS/native/outbound results;
+- renewal and rollback revisions and serials;
+- confirmation that no private key entered Git, CI artifacts, logs, or transfer media.
+
+Delete test containers, internal networks, temporary volumes, CSRs, and expired certificate copies. Preserve durable database volumes only according to the test-data retention decision. Never delete an unidentified volume with a wildcard cleanup command.
diff --git a/docs/TLS.md b/docs/TLS.md
index e8d7254..bb1748d 100644
--- a/docs/TLS.md
+++ b/docs/TLS.md
@@ -7,6 +7,8 @@ TLS has two independent directions in this image:
Do not use a server certificate as an outbound trust anchor, disable certificate verification, put a private key in an image layer, or store it in Git.
+Use the step-by-step [CA-issued TLS rehearsal](TLS-REHEARSAL.md) to qualify connected and disconnected deployments, negative cases, renewal, and rollback for an exact image digest.
+
## Recommended production boundary
Prefer TLS termination at the platform ingress, load balancer, or service mesh when it supports every protocol in use and the network from that proxy to ClickHouse is trusted. This centralizes certificate issuance and rotation. HTTP ingress products commonly cover HTTPS only; the ClickHouse native protocol needs a TCP-capable load balancer, TLS passthrough, or ClickHouse's `tcp_port_secure` listener.
@@ -73,7 +75,13 @@ Copy [`container/config.d/tls.example.xml`](../container/config.d/tls.example.xm
`verificationMode` controls whether the server requires client certificates; `none` still provides server-authenticated TLS. Use a reviewed mutual-TLS configuration if client-certificate authentication is required.
-The entrypoint detects that the clear-text native port was removed and uses `tcp_port_secure` for initialization and its local health check. Certificate validation is skipped only for that loopback-only internal client, where the server certificate commonly does not identify `127.0.0.1`; external clients must validate the certificate and hostname normally.
+The entrypoint detects that the clear-text native port was removed and uses
+`tcp_port_secure` for initialization and its local health check. A dedicated,
+immutable client configuration skips certificate validation only for that
+loopback-only internal query, where a CA-issued server certificate commonly
+does not identify `127.0.0.1`. It is not merged into ClickHouse Server's
+outbound TLS configuration. External clients and server-side integrations must
+validate the certificate, chain, and hostname normally.
For rootless Podman on Linux, map the files to the image identity inside Podman's user namespace and keep the private key unreadable to other container users:
diff --git a/tests/tls-rehearsal.sh b/tests/tls-rehearsal.sh
new file mode 100644
index 0000000..47638c8
--- /dev/null
+++ b/tests/tls-rehearsal.sh
@@ -0,0 +1,630 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+runtime="${CONTAINER_RUNTIME:-podman}"
+image="${IMAGE:-ghcr.io/datopsis/clickhouse-server-ubi9:test}"
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+run_id="${RANDOM}-$$"
+prefix="clickhouse-ubi9-tls-${run_id}"
+connected_network="${prefix}-connected"
+disconnected_network="${prefix}-disconnected"
+secret_dir="$(mktemp -d "${repo_root}/.tls-rehearsal.XXXXXX")"
+password="tls-rehearsal-password"
+endpoint_password="tls-endpoint-password"
+
+runtime_path() {
+ if command -v cygpath >/dev/null 2>&1; then
+ cygpath -m "$1"
+ else
+ printf '%s\n' "$1"
+ fi
+}
+
+runtime_call() {
+ # Prevent Git Bash from rewriting container paths. Host mount sources are
+ # converted explicitly with runtime_path before reaching this wrapper.
+ MSYS2_ARG_CONV_EXCL='*' "${runtime}" "$@"
+}
+
+runtime_repo_root="$(runtime_path "${repo_root}")"
+runtime_secret_dir="$(runtime_path "${secret_dir}")"
+windows_podman_machine=false
+if command -v cygpath >/dev/null 2>&1; then
+ windows_podman_machine=true
+fi
+
+connected_server="${prefix}-connected-server"
+connected_endpoint="${prefix}-connected-endpoint"
+incomplete_server="${prefix}-incomplete-chain"
+unreadable_server="${prefix}-unreadable-key"
+disconnected_server="${prefix}-disconnected-server"
+disconnected_endpoint="${prefix}-disconnected-endpoint"
+
+connected_volume="${prefix}-connected-data"
+connected_endpoint_volume="${prefix}-connected-endpoint-data"
+incomplete_volume="${prefix}-incomplete-data"
+unreadable_volume="${prefix}-unreadable-data"
+disconnected_volume="${prefix}-disconnected-data"
+disconnected_endpoint_volume="${prefix}-disconnected-endpoint-data"
+
+cleanup() {
+ runtime_call rm -f \
+ "${connected_server}" "${connected_endpoint}" \
+ "${incomplete_server}" "${unreadable_server}" \
+ "${disconnected_server}" "${disconnected_endpoint}" \
+ >/dev/null 2>&1 || true
+ runtime_call network rm \
+ "${connected_network}" "${disconnected_network}" \
+ >/dev/null 2>&1 || true
+ runtime_call volume rm \
+ "${connected_volume}" "${connected_endpoint_volume}" \
+ "${incomplete_volume}" "${unreadable_volume}" \
+ "${disconnected_volume}" "${disconnected_endpoint_volume}" \
+ >/dev/null 2>&1 || true
+ rm -rf "${secret_dir}"
+}
+trap cleanup EXIT
+
+fail() {
+ echo "TLS rehearsal failed: $*" >&2
+ exit 1
+}
+
+cat > "${secret_dir}/intermediate.ext" <<'EOF'
+basicConstraints=critical,CA:TRUE,pathlen:0
+keyUsage=critical,keyCertSign,cRLSign
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid:always
+EOF
+
+cat > "${secret_dir}/root.ext" <<'EOF'
+basicConstraints=critical,CA:TRUE,pathlen:1
+keyUsage=critical,keyCertSign,cRLSign
+subjectKeyIdentifier=hash
+EOF
+
+cat > "${secret_dir}/server.ext.template" <<'EOF'
+basicConstraints=critical,CA:FALSE
+keyUsage=critical,digitalSignature,keyEncipherment
+extendedKeyUsage=serverAuth
+subjectKeyIdentifier=hash
+authorityKeyIdentifier=keyid:always
+EOF
+
+env -u MSYS_NO_PATHCONV MSYS2_ARG_CONV_EXCL='/CN=' \
+ openssl req -new -newkey rsa:2048 -sha256 -nodes \
+ -keyout "${secret_dir}/root.key" \
+ -out "${secret_dir}/root.csr" \
+ -subj '/CN=ClickHouse TLS Rehearsal Root' >/dev/null 2>&1
+
+openssl x509 -req -sha256 -days 2 \
+ -in "${secret_dir}/root.csr" \
+ -signkey "${secret_dir}/root.key" \
+ -extfile "${secret_dir}/root.ext" \
+ -out "${secret_dir}/root.crt" >/dev/null 2>&1
+
+env -u MSYS_NO_PATHCONV MSYS2_ARG_CONV_EXCL='/CN=' \
+ openssl req -new -newkey rsa:2048 -sha256 -nodes \
+ -keyout "${secret_dir}/intermediate.key" \
+ -out "${secret_dir}/intermediate.csr" \
+ -subj '/CN=ClickHouse TLS Rehearsal Intermediate' >/dev/null 2>&1
+
+openssl x509 -req -sha256 -days 2 \
+ -in "${secret_dir}/intermediate.csr" \
+ -CA "${secret_dir}/root.crt" \
+ -CAkey "${secret_dir}/root.key" \
+ -CAcreateserial \
+ -extfile "${secret_dir}/intermediate.ext" \
+ -out "${secret_dir}/intermediate.crt" >/dev/null 2>&1
+
+env -u MSYS_NO_PATHCONV MSYS2_ARG_CONV_EXCL='/CN=' \
+ openssl req -x509 -newkey rsa:2048 -sha256 -nodes -days 2 \
+ -keyout "${secret_dir}/unrelated-root.key" \
+ -out "${secret_dir}/unrelated-root.crt" \
+ -subj '/CN=Unrelated TLS Rehearsal Root' \
+ -addext 'basicConstraints=critical,CA:TRUE' \
+ -addext 'keyUsage=critical,keyCertSign,cRLSign' >/dev/null 2>&1
+
+issue_leaf() {
+ local name=$1
+ local dns_name=$2
+
+ env -u MSYS_NO_PATHCONV MSYS2_ARG_CONV_EXCL='/CN=' \
+ openssl req -new -newkey rsa:2048 -sha256 -nodes \
+ -keyout "${secret_dir}/${name}.key" \
+ -out "${secret_dir}/${name}.csr" \
+ -subj "/CN=${dns_name}" >/dev/null 2>&1
+ {
+ cat "${secret_dir}/server.ext.template"
+ printf 'subjectAltName=DNS:%s\n' "${dns_name}"
+ } > "${secret_dir}/${name}.ext"
+ openssl x509 -req -sha256 -days 2 \
+ -in "${secret_dir}/${name}.csr" \
+ -CA "${secret_dir}/intermediate.crt" \
+ -CAkey "${secret_dir}/intermediate.key" \
+ -CAcreateserial \
+ -extfile "${secret_dir}/${name}.ext" \
+ -out "${secret_dir}/${name}.crt" >/dev/null 2>&1
+ cat "${secret_dir}/${name}.crt" \
+ "${secret_dir}/intermediate.crt" \
+ > "${secret_dir}/${name}.chain.crt"
+}
+
+issue_leaf connected-v1 clickhouse.connected.test
+issue_leaf connected-v2 clickhouse.connected.test
+issue_leaf connected-endpoint private.connected.test
+issue_leaf disconnected-v1 clickhouse.disconnected.test
+issue_leaf disconnected-v2 clickhouse.disconnected.test
+issue_leaf disconnected-endpoint private.disconnected.test
+
+if ! (cd "${secret_dir}" && \
+ openssl verify -CAfile root.crt \
+ -untrusted intermediate.crt connected-v1.crt); then
+ openssl x509 -in "${secret_dir}/root.crt" \
+ -noout -subject -issuer -serial
+ openssl x509 -in "${secret_dir}/intermediate.crt" \
+ -noout -subject -issuer -serial
+ fail 'generated certificate chain did not validate'
+fi
+openssl x509 -in "${secret_dir}/connected-v1.crt" -noout \
+ -subject -issuer -serial -dates -ext subjectAltName
+
+runtime_call run --rm --entrypoint cat "${image}" \
+ /etc/pki/tls/certs/ca-bundle.crt \
+ > "${secret_dir}/public-ca-bundle.pem"
+cat "${secret_dir}/public-ca-bundle.pem" "${secret_dir}/root.crt" \
+ "${secret_dir}/intermediate.crt" \
+ > "${secret_dir}/connected-ca-bundle.pem"
+cat "${secret_dir}/root.crt" "${secret_dir}/intermediate.crt" \
+ > "${secret_dir}/disconnected-ca-bundle.pem"
+
+cat > "${secret_dir}/client.xml" <<'EOF'
+
+
+
+
+ false
+ /tls/ca-bundle.pem
+ strict
+
+ RejectCertificateHandler
+
+
+
+
+EOF
+
+chmod 0444 "${secret_dir}"/*.crt "${secret_dir}"/*.pem \
+ "${secret_dir}"/*.xml
+chmod 0400 "${secret_dir}"/*.key
+
+runtime_call network create "${connected_network}" >/dev/null
+runtime_call network create --internal "${disconnected_network}" >/dev/null
+test "$(runtime_call network inspect --format '{{.Internal}}' \
+ "${disconnected_network}")" = true
+
+for volume in \
+ "${connected_volume}" "${connected_endpoint_volume}" \
+ "${incomplete_volume}" "${unreadable_volume}" \
+ "${disconnected_volume}" "${disconnected_endpoint_volume}"; do
+ runtime_call volume create "${volume}" >/dev/null
+done
+
+run_tls_server() {
+ local name=$1
+ local network=$2
+ local alias=$3
+ local volume=$4
+ local certificate=$5
+ local key=$6
+ local runtime_certificate
+ local runtime_key
+ shift 6
+
+ runtime_certificate="$(runtime_path "${certificate}")"
+ runtime_key="$(runtime_path "${key}")"
+
+ runtime_call run --detach \
+ --name "${name}" \
+ --network "${network}" \
+ --network-alias "${alias}" \
+ --read-only \
+ --tmpfs /tmp:size=256m,mode=1777 \
+ --cap-drop ALL \
+ --security-opt no-new-privileges:true \
+ --ulimit nofile=262144:262144 \
+ --env CLICKHOUSE_PASSWORD="${password}" \
+ --volume "${volume}:/var/lib/clickhouse" \
+ --volume "${runtime_repo_root}/container/config.d/tls.example.xml:/etc/clickhouse-server/config.d/tls.xml:ro" \
+ --volume "${runtime_certificate}:/etc/clickhouse-server/certs/tls.crt:ro" \
+ --volume "${runtime_key}:/etc/clickhouse-server/certs/tls.key:ro" \
+ "$@" \
+ "${image}" >/dev/null
+}
+
+wait_healthy() {
+ local name=$1
+ local status
+
+ for _ in {1..90}; do
+ status="$(runtime_call inspect --format \
+ '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
+ "${name}")"
+ if [[ "${status}" == healthy ]]; then
+ return
+ fi
+ if [[ "${status}" == unhealthy ]]; then
+ runtime_call logs "${name}"
+ return 1
+ fi
+ sleep 1
+ done
+
+ runtime_call logs "${name}"
+ fail "${name} did not become healthy"
+}
+
+published_https_port() {
+ local name=$1
+ local mapping
+
+ mapping="$(runtime_call port "${name}" 8443/tcp | tail -n 1)"
+ printf '%s\n' "${mapping##*:}"
+}
+
+query_local() {
+ local name=$1
+ local query=$2
+ local query_password=${3:-${password}}
+
+ runtime_call exec "${name}" env -u CLICKHOUSE_CONFIG \
+ clickhouse-client \
+ --config-file /usr/local/share/clickhouse-health-client.xml \
+ --host 127.0.0.1 --port 9440 --secure \
+ --user default --password "${query_password}" --query "${query}"
+}
+
+native_tls_query() {
+ local network=$1
+ local host=$2
+ local ca_file=${3:-${secret_dir}/disconnected-ca-bundle.pem}
+ local runtime_ca_file
+
+ runtime_ca_file="$(runtime_path "${ca_file}")"
+
+ runtime_call run --rm \
+ --network "${network}" \
+ --read-only \
+ --tmpfs /tmp:size=64m,mode=1777 \
+ --cap-drop ALL \
+ --entrypoint env \
+ --volume "${runtime_ca_file}:/tls/ca-bundle.pem:ro" \
+ --volume "${runtime_secret_dir}/client.xml:/tls/client.xml:ro" \
+ "${image}" \
+ -u CLICKHOUSE_CONFIG clickhouse-client \
+ --config-file /tls/client.xml \
+ --secure --host "${host}" --port 9440 \
+ --user default --password "${password}" \
+ --connect_timeout 5 --query 'SELECT 1'
+}
+
+https_query() {
+ local host=$1
+ local port=$2
+ local ca_file=$3
+
+ curl --silent --show-error --fail --max-time 10 \
+ --noproxy '*' \
+ --cacert "${ca_file}" \
+ --resolve "${host}:${port}:127.0.0.1" \
+ --user "default:${password}" \
+ "https://${host}:${port}/?query=SELECT%201"
+}
+
+url_query() {
+ local name=$1
+ local url=$2
+
+ query_local "${name}" \
+ "SELECT length(data) > 0 FROM url('${url}', 'RawBLOB', 'data String') SETTINGS max_execution_time=10, http_max_tries=1, http_connection_timeout=5, http_receive_timeout=5, http_send_timeout=5"
+}
+
+# A controlled private-CA HTTPS service for connected outbound validation.
+run_tls_server "${connected_endpoint}" "${connected_network}" \
+ private.connected.test "${connected_endpoint_volume}" \
+ "${secret_dir}/connected-endpoint.chain.crt" \
+ "${secret_dir}/connected-endpoint.key" \
+ --volume "${runtime_repo_root}/container/config.d/outbound-ca.example.xml:/etc/clickhouse-server/config.d/outbound-ca.xml:ro" \
+ --volume "${runtime_secret_dir}/connected-ca-bundle.pem:/etc/clickhouse-server/certs/outbound-ca-bundle.pem:ro" \
+ --env CLICKHOUSE_PASSWORD="${endpoint_password}"
+wait_healthy "${connected_endpoint}"
+echo 'Connected private-CA endpoint is healthy'
+
+cp "${secret_dir}/connected-v1.chain.crt" "${secret_dir}/active.chain.crt"
+cp "${secret_dir}/connected-v1.key" "${secret_dir}/active.key"
+chmod 0444 "${secret_dir}/active.chain.crt" "${secret_dir}/active.key"
+
+run_tls_server "${connected_server}" "${connected_network}" \
+ clickhouse.connected.test "${connected_volume}" \
+ "${secret_dir}/active.chain.crt" "${secret_dir}/active.key" \
+ --publish 127.0.0.1::8443 \
+ --network-alias wrong.connected.test \
+ --volume "${runtime_repo_root}/container/config.d/outbound-ca.example.xml:/etc/clickhouse-server/config.d/outbound-ca.xml:ro" \
+ --volume "${runtime_secret_dir}/connected-ca-bundle.pem:/etc/clickhouse-server/certs/outbound-ca-bundle.pem:ro"
+wait_healthy "${connected_server}"
+echo 'Connected ClickHouse server is healthy'
+
+connected_port="$(published_https_port "${connected_server}")"
+test "$(https_query clickhouse.connected.test "${connected_port}" \
+ "${secret_dir}/root.crt")" = 1
+echo 'Connected HTTPS validation passed'
+test "$(native_tls_query "${connected_network}" clickhouse.connected.test)" = 1
+echo 'Connected native TLS validation passed'
+
+if https_query clickhouse.connected.test "${connected_port}" \
+ "${secret_dir}/unrelated-root.crt" >/dev/null 2>&1; then
+ fail 'HTTPS accepted an unrelated CA'
+fi
+if https_query wrong.connected.test "${connected_port}" \
+ "${secret_dir}/root.crt" >/dev/null 2>&1; then
+ fail 'HTTPS accepted the wrong hostname'
+fi
+if native_tls_query "${connected_network}" wrong.connected.test \
+ >/dev/null 2>&1; then
+ fail 'native TLS accepted the wrong hostname'
+fi
+if curl --silent --fail --max-time 5 \
+ "http://127.0.0.1:${connected_port}/ping" >/dev/null 2>&1; then
+ fail 'clear-text HTTP succeeded on the HTTPS listener'
+fi
+if runtime_call run --rm --network "${connected_network}" \
+ --entrypoint env "${image}" \
+ -u CLICKHOUSE_CONFIG clickhouse-client \
+ --host clickhouse.connected.test --port 9440 \
+ --user default --password "${password}" \
+ --connect_timeout 5 --query 'SELECT 1' >/dev/null 2>&1; then
+ fail 'clear-text native protocol succeeded on the TLS listener'
+fi
+
+# The connected bundle must support both public PKI and the controlled private CA.
+echo 'Testing connected public-PKI outbound TLS'
+test "$(url_query "${connected_server}" 'https://example.com/')" = 1
+echo 'Connected public-PKI outbound TLS passed'
+echo 'Testing connected private-CA outbound TLS'
+test "$(query_local "${connected_server}" \
+ "SELECT length(data) > 0 FROM url('https://private.connected.test:8443/?query=SELECT%201', 'RawBLOB', 'data String', headers('X-ClickHouse-User'='default', 'X-ClickHouse-Key'='${endpoint_password}'))")" = 1
+echo 'Connected private-CA outbound TLS passed'
+
+old_serial="$(openssl x509 -in "${secret_dir}/connected-v1.crt" \
+ -noout -serial | cut -d= -f2)"
+served_serial="$(openssl s_client \
+ -connect "127.0.0.1:${connected_port}" \
+ -servername clickhouse.connected.test \
+ -CAfile "${secret_dir}/root.crt" /dev/null | \
+ openssl x509 -noout -serial | cut -d= -f2)"
+test "${served_serial}" = "${old_serial}"
+echo 'Connected certificate serial matched'
+
+# A leaf without its intermediate must not validate to the trusted root.
+run_tls_server "${incomplete_server}" "${connected_network}" \
+ incomplete.connected.test "${incomplete_volume}" \
+ "${secret_dir}/connected-v1.crt" "${secret_dir}/connected-v1.key" \
+ --volume "${runtime_repo_root}/container/config.d/outbound-ca.example.xml:/etc/clickhouse-server/config.d/outbound-ca.xml:ro" \
+ --volume "${runtime_secret_dir}/connected-ca-bundle.pem:/etc/clickhouse-server/certs/outbound-ca-bundle.pem:ro" \
+ --publish 127.0.0.1::8443
+wait_healthy "${incomplete_server}"
+echo 'Incomplete-chain server is ready for the rejection test'
+incomplete_port="$(published_https_port "${incomplete_server}")"
+if https_query clickhouse.connected.test "${incomplete_port}" \
+ "${secret_dir}/root.crt" >/dev/null 2>&1; then
+ fail 'HTTPS accepted an incomplete certificate chain'
+fi
+runtime_call rm -f "${incomplete_server}" >/dev/null
+echo 'Incomplete-chain rejection passed'
+
+# An unreadable private key must prevent startup. Windows Podman Machine file
+# sharing does not preserve a host chmod 000, so native Linux CI owns this
+# assertion and the Windows reproduction reports the explicit limitation.
+if command -v cygpath >/dev/null 2>&1; then
+ echo 'Unreadable-key rejection skipped on Windows; native Linux CI runs it'
+else
+ cp "${secret_dir}/connected-v1.key" "${secret_dir}/unreadable.key"
+ chmod 000 "${secret_dir}/unreadable.key"
+ if run_tls_server "${unreadable_server}" "${connected_network}" \
+ unreadable.connected.test "${unreadable_volume}" \
+ "${secret_dir}/connected-v1.chain.crt" \
+ "${secret_dir}/unreadable.key"; then
+ for _ in {1..30}; do
+ if [[ "$(runtime_call inspect --format '{{.State.Running}}' \
+ "${unreadable_server}")" == false ]]; then
+ break
+ fi
+ sleep 1
+ done
+ test "$(runtime_call inspect --format '{{.State.Running}}' \
+ "${unreadable_server}")" = false
+ fi
+ chmod 0400 "${secret_dir}/unreadable.key"
+ echo 'Unreadable-key rejection passed'
+fi
+
+# Rotate by replacing the mounted leaf/key and recreating the workload. The
+# persistent data volume remains; the old serial must no longer be served.
+runtime_call rm -f "${connected_server}" >/dev/null
+chmod 0600 "${secret_dir}/active.chain.crt" "${secret_dir}/active.key"
+cp "${secret_dir}/connected-v2.chain.crt" "${secret_dir}/active.chain.crt"
+cp "${secret_dir}/connected-v2.key" "${secret_dir}/active.key"
+chmod 0444 "${secret_dir}/active.chain.crt" "${secret_dir}/active.key"
+run_tls_server "${connected_server}" "${connected_network}" \
+ clickhouse.connected.test "${connected_volume}" \
+ "${secret_dir}/active.chain.crt" "${secret_dir}/active.key" \
+ --publish 127.0.0.1::8443 \
+ --network-alias wrong.connected.test \
+ --volume "${runtime_repo_root}/container/config.d/outbound-ca.example.xml:/etc/clickhouse-server/config.d/outbound-ca.xml:ro" \
+ --volume "${runtime_secret_dir}/connected-ca-bundle.pem:/etc/clickhouse-server/certs/outbound-ca-bundle.pem:ro"
+wait_healthy "${connected_server}"
+echo 'Connected server is healthy after rotation'
+connected_port="$(published_https_port "${connected_server}")"
+new_serial="$(openssl x509 -in "${secret_dir}/connected-v2.crt" \
+ -noout -serial | cut -d= -f2)"
+served_serial="$(openssl s_client \
+ -connect "127.0.0.1:${connected_port}" \
+ -servername clickhouse.connected.test \
+ -CAfile "${secret_dir}/root.crt" /dev/null | \
+ openssl x509 -noout -serial | cut -d= -f2)"
+test "${new_serial}" != "${old_serial}"
+test "${served_serial}" = "${new_serial}"
+test "$(https_query clickhouse.connected.test "${connected_port}" \
+ "${secret_dir}/root.crt")" = 1
+echo 'Connected certificate rotation passed'
+
+# Disconnected phase: only the controlled endpoint and internal CA are present.
+run_tls_server "${disconnected_endpoint}" "${disconnected_network}" \
+ private.disconnected.test "${disconnected_endpoint_volume}" \
+ "${secret_dir}/disconnected-endpoint.chain.crt" \
+ "${secret_dir}/disconnected-endpoint.key" \
+ --volume "${runtime_repo_root}/container/config.d/outbound-ca.example.xml:/etc/clickhouse-server/config.d/outbound-ca.xml:ro" \
+ --volume "${runtime_secret_dir}/disconnected-ca-bundle.pem:/etc/clickhouse-server/certs/outbound-ca-bundle.pem:ro" \
+ --env CLICKHOUSE_PASSWORD="${endpoint_password}"
+wait_healthy "${disconnected_endpoint}"
+echo 'Disconnected private-CA endpoint is healthy'
+cp "${secret_dir}/disconnected-v1.chain.crt" \
+ "${secret_dir}/disconnected-active.chain.crt"
+cp "${secret_dir}/disconnected-v1.key" \
+ "${secret_dir}/disconnected-active.key"
+chmod 0444 "${secret_dir}/disconnected-active.chain.crt" \
+ "${secret_dir}/disconnected-active.key"
+run_tls_server "${disconnected_server}" "${disconnected_network}" \
+ clickhouse.disconnected.test "${disconnected_volume}" \
+ "${secret_dir}/disconnected-active.chain.crt" \
+ "${secret_dir}/disconnected-active.key" \
+ --publish 127.0.0.1::8443 \
+ --network-alias wrong.disconnected.test \
+ --volume "${runtime_repo_root}/container/config.d/outbound-ca.example.xml:/etc/clickhouse-server/config.d/outbound-ca.xml:ro" \
+ --volume "${runtime_secret_dir}/disconnected-ca-bundle.pem:/etc/clickhouse-server/certs/outbound-ca-bundle.pem:ro"
+wait_healthy "${disconnected_server}"
+echo 'Disconnected ClickHouse server is healthy'
+disconnected_port="$(published_https_port "${disconnected_server}")"
+if [[ "${windows_podman_machine}" == true ]]; then
+ test "$(query_local "${disconnected_endpoint}" \
+ "SELECT length(data) > 0 FROM url('https://clickhouse.disconnected.test:8443/?query=SELECT%201', 'RawBLOB', 'data String', headers('X-ClickHouse-User'='default', 'X-ClickHouse-Key'='${password}'))" \
+ "${endpoint_password}")" = 1
+ echo 'Disconnected HTTPS passed inside the isolated Podman network'
+else
+ test "$(https_query clickhouse.disconnected.test "${disconnected_port}" \
+ "${secret_dir}/root.crt")" = 1
+ if https_query clickhouse.disconnected.test "${disconnected_port}" \
+ "${secret_dir}/unrelated-root.crt" >/dev/null 2>&1; then
+ fail 'disconnected HTTPS accepted an unrelated CA'
+ fi
+ if https_query wrong.disconnected.test "${disconnected_port}" \
+ "${secret_dir}/root.crt" >/dev/null 2>&1; then
+ fail 'disconnected HTTPS accepted the wrong hostname'
+ fi
+fi
+test "$(native_tls_query "${disconnected_network}" \
+ clickhouse.disconnected.test)" = 1
+if native_tls_query "${disconnected_network}" \
+ clickhouse.disconnected.test "${secret_dir}/unrelated-root.crt" \
+ >/dev/null 2>&1; then
+ fail 'disconnected native TLS accepted an unrelated CA'
+fi
+if native_tls_query "${disconnected_network}" wrong.disconnected.test \
+ >/dev/null 2>&1; then
+ fail 'disconnected native TLS accepted the wrong hostname'
+fi
+echo 'Disconnected native TLS positive and negative validation passed'
+test "$(query_local "${disconnected_server}" \
+ "SELECT length(data) > 0 FROM url('https://private.disconnected.test:8443/?query=SELECT%201', 'RawBLOB', 'data String', headers('X-ClickHouse-User'='default', 'X-ClickHouse-Key'='${endpoint_password}'))")" = 1
+echo 'Disconnected private-CA outbound TLS passed'
+if runtime_call exec "${disconnected_server}" timeout 5 bash -c \
+ 'exec 3<>/dev/tcp/1.1.1.1/443' >/dev/null 2>&1; then
+ fail 'disconnected container unexpectedly reached a public IP'
+fi
+echo 'Disconnected public-IP egress rejection passed'
+
+disconnected_old_serial="$(openssl x509 \
+ -in "${secret_dir}/disconnected-v1.crt" \
+ -noout -serial | cut -d= -f2)"
+runtime_call rm -f "${disconnected_server}" >/dev/null
+chmod 0600 "${secret_dir}/disconnected-active.chain.crt" \
+ "${secret_dir}/disconnected-active.key"
+cp "${secret_dir}/disconnected-v2.chain.crt" \
+ "${secret_dir}/disconnected-active.chain.crt"
+cp "${secret_dir}/disconnected-v2.key" \
+ "${secret_dir}/disconnected-active.key"
+chmod 0444 "${secret_dir}/disconnected-active.chain.crt" \
+ "${secret_dir}/disconnected-active.key"
+run_tls_server "${disconnected_server}" "${disconnected_network}" \
+ clickhouse.disconnected.test "${disconnected_volume}" \
+ "${secret_dir}/disconnected-active.chain.crt" \
+ "${secret_dir}/disconnected-active.key" \
+ --publish 127.0.0.1::8443 \
+ --network-alias wrong.disconnected.test \
+ --volume "${runtime_repo_root}/container/config.d/outbound-ca.example.xml:/etc/clickhouse-server/config.d/outbound-ca.xml:ro" \
+ --volume "${runtime_secret_dir}/disconnected-ca-bundle.pem:/etc/clickhouse-server/certs/outbound-ca-bundle.pem:ro"
+wait_healthy "${disconnected_server}"
+disconnected_port="$(published_https_port "${disconnected_server}")"
+disconnected_new_serial="$(openssl x509 \
+ -in "${secret_dir}/disconnected-v2.crt" \
+ -noout -serial | cut -d= -f2)"
+if [[ "${windows_podman_machine}" == true ]]; then
+ test "$(query_local "${disconnected_endpoint}" \
+ "SELECT length(data) > 0 FROM url('https://clickhouse.disconnected.test:8443/?query=SELECT%201', 'RawBLOB', 'data String', headers('X-ClickHouse-User'='default', 'X-ClickHouse-Key'='${password}'))" \
+ "${endpoint_password}")" = 1
+ disconnected_served_serial="${disconnected_new_serial}"
+ echo 'Served renewal serial inspection deferred to native Linux CI'
+else
+ disconnected_served_serial="$(openssl s_client \
+ -connect "127.0.0.1:${disconnected_port}" \
+ -servername clickhouse.disconnected.test \
+ -CAfile "${secret_dir}/root.crt" /dev/null | \
+ openssl x509 -noout -serial | cut -d= -f2)"
+fi
+test "${disconnected_new_serial}" != "${disconnected_old_serial}"
+test "${disconnected_served_serial}" = "${disconnected_new_serial}"
+
+# Roll back to the retained prior certificate without requiring public access.
+runtime_call rm -f "${disconnected_server}" >/dev/null
+chmod 0600 "${secret_dir}/disconnected-active.chain.crt" \
+ "${secret_dir}/disconnected-active.key"
+cp "${secret_dir}/disconnected-v1.chain.crt" \
+ "${secret_dir}/disconnected-active.chain.crt"
+cp "${secret_dir}/disconnected-v1.key" \
+ "${secret_dir}/disconnected-active.key"
+chmod 0444 "${secret_dir}/disconnected-active.chain.crt" \
+ "${secret_dir}/disconnected-active.key"
+run_tls_server "${disconnected_server}" "${disconnected_network}" \
+ clickhouse.disconnected.test "${disconnected_volume}" \
+ "${secret_dir}/disconnected-active.chain.crt" \
+ "${secret_dir}/disconnected-active.key" \
+ --publish 127.0.0.1::8443 \
+ --network-alias wrong.disconnected.test \
+ --volume "${runtime_repo_root}/container/config.d/outbound-ca.example.xml:/etc/clickhouse-server/config.d/outbound-ca.xml:ro" \
+ --volume "${runtime_secret_dir}/disconnected-ca-bundle.pem:/etc/clickhouse-server/certs/outbound-ca-bundle.pem:ro"
+wait_healthy "${disconnected_server}"
+disconnected_port="$(published_https_port "${disconnected_server}")"
+if [[ "${windows_podman_machine}" == true ]]; then
+ test "$(query_local "${disconnected_endpoint}" \
+ "SELECT length(data) > 0 FROM url('https://clickhouse.disconnected.test:8443/?query=SELECT%201', 'RawBLOB', 'data String', headers('X-ClickHouse-User'='default', 'X-ClickHouse-Key'='${password}'))" \
+ "${endpoint_password}")" = 1
+ disconnected_served_serial="${disconnected_old_serial}"
+ echo 'Served rollback serial inspection deferred to native Linux CI'
+else
+ disconnected_served_serial="$(openssl s_client \
+ -connect "127.0.0.1:${disconnected_port}" \
+ -servername clickhouse.disconnected.test \
+ -CAfile "${secret_dir}/root.crt" /dev/null | \
+ openssl x509 -noout -serial | cut -d= -f2)"
+fi
+test "${disconnected_served_serial}" = "${disconnected_old_serial}"
+if [[ "${windows_podman_machine}" == false ]]; then
+ test "$(https_query clickhouse.disconnected.test "${disconnected_port}" \
+ "${secret_dir}/root.crt")" = 1
+fi
+
+echo "Connected and disconnected CA-issued TLS rehearsal passed"
+echo "Retired certificate serial: ${old_serial}"
+echo "Active certificate serial: ${new_serial}"
+echo "Disconnected renewal serial: ${disconnected_new_serial}"
+echo "Disconnected rollback serial: ${disconnected_old_serial}"
From 73f32b4df07f3850eeeab67f5231b1eafc8065f3 Mon Sep 17 00:00:00 2001
From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com>
Date: Mon, 7 Sep 2026 10:47:11 -0500
Subject: [PATCH 2/6] docs: plan tailored SCAP assurance
---
README.md | 6 +--
docs/ROADMAP.md | 31 ++++++++++-
docs/SCAP.md | 133 ++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 165 insertions(+), 5 deletions(-)
create mode 100644 docs/SCAP.md
diff --git a/README.md b/README.md
index c6bbc7b..598ce28 100644
--- a/README.md
+++ b/README.md
@@ -109,7 +109,7 @@ Build-time arguments are `CLICKHOUSE_VERSION`, `CLICKHOUSE_CHANNEL`, `UBI_MINIMA
The supported Podman baseline is version 5.3 or newer because 5.3.1 is the oldest engine on which the full smoke suite has been recorded. This is a tested support floor, not a claim that older versions cannot run the image. Docker Engine remains compatible and is used by GitHub Actions for its native architecture jobs and release Buildx workflow. See [Podman compatibility and version support](docs/PODMAN.md) for tested versions, rootless bind mounts, remote clients, and Compose behavior.
-The smoke suite verifies startup with a read-only root filesystem and no capabilities, package-manager absence, authenticated local and network queries, first-start initialization, persistent-data restarts, password-file support, the passwordless network restriction, TLS-only native initialization and health, graceful shutdown, and operation under an arbitrary OpenShift-style UID. It requires `openssl` on the test host to create an ephemeral TLS fixture.
+The smoke suite verifies startup with a read-only root filesystem and no capabilities, package-manager absence, authenticated local and network queries, first-start initialization, persistent-data restarts, password-file support, the passwordless network restriction, TLS-only native initialization and health, graceful shutdown, and operation under an arbitrary OpenShift-style UID. The separate CA-issued rehearsal validates chained certificates, HTTPS/native TLS, connected and disconnected outbound trust, negative cases, rotation, and rollback. Both require `openssl` on the test host; the TLS rehearsal also requires `curl`.
## Release process
@@ -139,9 +139,9 @@ ClickHouse commonly benefits from `nofile=262144:262144`. Optional capabilities
Treat the effective ClickHouse data path (default `/var/lib/clickhouse`) as durable state, back it up according to your ClickHouse topology, and pin production deployments to an image digest rather than a mutable tag.
-Before a production rollout, follow the [production deployment guide](docs/PRODUCTION.md). Configure inbound encryption and public/private outbound trust with the [TLS guide](docs/TLS.md). The secure ClickHouse ports (`8443`, `9440`, and `9010`) are configuration choices and are not enabled by default.
+Before a production rollout, follow the [production deployment guide](docs/PRODUCTION.md). Configure inbound encryption and public/private outbound trust with the [TLS guide](docs/TLS.md), then execute the [CA-issued TLS rehearsal](docs/TLS-REHEARSAL.md). The secure ClickHouse ports (`8443`, `9440`, and `9010`) are configuration choices and are not enabled by default.
-See [SECURITY.md](SECURITY.md) for vulnerability reporting and the support policy. Contributor references include [Podman compatibility](docs/PODMAN.md), [rootless storage and permissions](docs/ROOTLESS.md), [qualification evidence](docs/QUALIFICATION.md), the [vulnerability-management process](docs/VULNERABILITY-MANAGEMENT.md), [official-image comparison](docs/IMAGE-COMPARISON.md), [versioning and release standard](docs/VERSION.md), [first-release roadmap](docs/ROADMAP.md), [CI and security process](docs/CI.md), [Endor Labs posture](docs/ENDOR.md), [badge policy](docs/BADGING.md), and [OpenSSF Scorecard controls](docs/OPENSSF_SCORECARD.md).
+See [SECURITY.md](SECURITY.md) for vulnerability reporting and the support policy. Contributor references include [Podman compatibility](docs/PODMAN.md), [rootless storage and permissions](docs/ROOTLESS.md), [qualification evidence](docs/QUALIFICATION.md), the [vulnerability-management process](docs/VULNERABILITY-MANAGEMENT.md), [official-image comparison](docs/IMAGE-COMPARISON.md), [versioning and release standard](docs/VERSION.md), [first-release roadmap](docs/ROADMAP.md), [CI and security process](docs/CI.md), [SCAP compliance scanning](docs/SCAP.md), [Endor Labs posture](docs/ENDOR.md), [badge policy](docs/BADGING.md), and [OpenSSF Scorecard controls](docs/OPENSSF_SCORECARD.md).
## License
diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md
index 303c193..7b694fe 100644
--- a/docs/ROADMAP.md
+++ b/docs/ROADMAP.md
@@ -10,7 +10,8 @@ This roadmap is the release gate for the first supported image. A checked item m
2. Validate [TLS.md](TLS.md) with CA-issued certificates for HTTPS and native TCP, including rotation and a fully disconnected rehearsal.
3. Execute [PRODUCTION.md](PRODUCTION.md) on native `amd64`, native `arm64`, and OpenShift, recording resource, storage, backup/restore, shutdown, and recovery evidence.
4. Complete the ClickHouse/UBI notice and SBOM review described in [THIRD_PARTY_NOTICES.md](../THIRD_PARTY_NOTICES.md).
-5. Publish the first signed GHCR release only after every blocker below is complete. Reconsider Docker Hub and paid security services after that release has real consumer demand.
+5. Establish the tailored container SCAP baseline in [SCAP.md](SCAP.md), retain both architecture reports, and review every applicability decision before enforcing selected rules.
+6. Publish the first signed GHCR release only after every blocker below is complete. Reconsider Docker Hub and paid security services after that release has real consumer demand.
### Incremental delivery plan
@@ -94,7 +95,29 @@ This repository owns image-specific behavior, basic usage, and minimal platform
- [ ] If no real OpenShift environment can be obtained, run a restricted Kubernetes proxy test and explicitly mark OpenShift as unvalidated and unsupported in the first release. A Kind/K3s test does not satisfy or replace the OpenShift checklist above.
-#### 5. Final candidate and signed release
+#### 5. Tailored SCAP image-compliance baseline
+
+**Profile discovery and tailoring**
+
+- [ ] Pin a UBI 9 OpenSCAP scanner image by digest and pin the OpenSCAP and ComplianceAsCode content versions. Record the RHEL 9 data-stream SHA-256 and reject an unexpected stream.
+- [ ] Run the upstream RHEL 9 Standard profile in report-only discovery mode against the exported image filesystem. Inventory every pass, failure, error, not-applicable, and not-checked result without claiming host or deployment compliance.
+- [ ] Create a reviewed XCCDF tailoring profile containing only rules that are applicable to and controlled by this image. Commit a rule-rationale matrix and document every host/platform exclusion.
+- [ ] Exclude kernel, boot-loader, partition, mount-layout, systemd, audit, host-networking, sysctl, SELinux-mode, and FIPS-mode controls unless the image later gains direct ownership of one. Do not use automatic remediation.
+
+**Safe CI integration**
+
+- [ ] Export the stopped, already-tested image's merged filesystem into an ephemeral directory and mount that directory read-only into the scanner. Never execute target-image content to prepare the scan.
+- [ ] Run `oscap-chroot` in a digest-pinned scanner with no Docker/Podman socket, no host namespace, no workflow secrets, and no evaluation-time network. Prove the minimum chroot-related capability; do not use `--privileged`, Podman-in-Podman, or broad host mounts.
+- [ ] Generate architecture-specific ARF XML, XCCDF XML, and HTML reports containing the image digest, architecture, scanner/content versions, data-stream hash, and tailoring hash. Retain them with the other image-security evidence.
+- [ ] Run report-only on native AMD64 and ARM64 for at least three scheduled or `main` executions. Evaluation errors fail immediately; selected-rule findings become blocking only after the baseline is stable and reviewed.
+- [ ] Cross-check one exact image digest with `oscap-podman` on a disposable RHEL 9 host. Reconcile platform/applicability differences before enforcement; do not grant routine hosted CI root or engine access merely to match that command.
+
+**Exit evidence**
+
+- [ ] Retain the discovery report, final tailoring, rule-rationale/exclusion review, three stable two-architecture runs, capability inspection, and `oscap-chroot` versus `oscap-podman` comparison.
+- [ ] State precisely that the result covers selected image-filesystem controls and is not CIS/STIG certification of the host, OpenShift cluster, or production deployment.
+
+#### 6. Final candidate and signed release
- [ ] Refresh and review the UBI Minimal and Micro manifest-list digests together. Confirm both architectures resolve, rebuild from scratch, and retain the old/new digest and vulnerability comparison.
- [ ] Confirm the selected ClickHouse release/channel and archive checksums, run both native CI jobs, and complete the current unfixed-finding triage with owner, rationale, compensating controls, and review expiry.
@@ -124,6 +147,10 @@ This repository owns image-specific behavior, basic usage, and minimal platform
- [ ] Review workflow permissions, immutable action pins, secret-scanning alerts, and CodeQL/Scorecard findings.
- [ ] Test private vulnerability reporting and confirm that `SECURITY.md` names a monitored response path.
- [ ] Complete license, redistribution, trademark, and upstream-notice review for ClickHouse and Red Hat UBI content.
+- [ ] Complete an image threat model covering build inputs, CI trust, registry/release publication, runtime identity, storage, ingress/egress TLS, secrets, and the boundary with `clickhouse-production-stack`.
+- [ ] Define and exercise a UBI rebuild cadence and response SLA for exploitable critical/high findings, including unfixed findings that later receive a vendor fix.
+- [ ] Compare final SBOM and filesystem/package inventories against the reviewed baseline and investigate unexpected additions, removals, setuid/setgid files, or world-writable paths.
+- [ ] Exercise password/key/certificate rotation and failure paths while confirming logs and retained CI evidence do not expose secret material.
### Release mechanics and documentation
diff --git a/docs/SCAP.md b/docs/SCAP.md
new file mode 100644
index 0000000..e9c2444
--- /dev/null
+++ b/docs/SCAP.md
@@ -0,0 +1,133 @@
+# SCAP compliance scanning
+
+SCAP is useful as a configuration-compliance control for this image, but an
+unmodified RHEL host profile is not an appropriate release gate for a minimal
+container. Many RHEL rules govern the kernel, boot loader, partitions, systemd,
+audit daemon, host networking, or machine-wide security policy. Those controls
+belong to the container host or deployment platform and are outside this
+image's control.
+
+The project will therefore establish a measured baseline first, then maintain
+a tailored UBI 9 Micro container profile containing only applicable,
+image-owned rules. Passing that profile means the inspected image filesystem
+meets the documented rules; it is not a claim that the image, host, OpenShift
+cluster, or complete ClickHouse deployment is CIS- or STIG-certified.
+
+## Recommended CI architecture
+
+Do not run Podman inside Podman and do not mount a Docker or Podman socket into
+the scanner. Nested container engines commonly require additional namespace,
+device, seccomp, and capability allowances. A mounted engine socket also gives
+the job control over the host engine. Either design makes the scanner a larger
+privileged attack surface than the artifact being inspected warrants.
+
+Use this flow instead:
+
+1. Build and smoke-test the architecture-specific image in the existing native
+ CI job.
+2. Create a stopped container and export its merged filesystem into an
+ ephemeral staging directory. Exporting does not execute image content.
+3. Run a digest-pinned UBI 9 OpenSCAP tool image with the exported filesystem
+ mounted read-only and a separate results directory mounted read-write.
+4. Run `oscap-chroot` against the read-only root and a pinned RHEL 9 data
+ stream. Give the scanner no engine socket, host namespaces, secrets, or
+ network access during evaluation. Grant only the minimum chroot-related
+ capability proven necessary by the qualification test.
+5. Upload the ARF XML, XCCDF results, HTML report, tool/content versions, data
+ stream hash, image digest, architecture, and tailoring hash as CI evidence.
+6. Delete the exported root filesystem and stopped container after the job.
+
+The scanner process may run as UID 0 *inside its isolated scanner container* so
+it can inspect the mounted tree. That is different from granting the workflow a
+privileged container, host root, an engine socket, or broad host mounts. The
+target filesystem remains read-only and the ClickHouse image is never started
+as root.
+
+The implementation must first prove whether `CAP_SYS_CHROOT` alone is needed.
+If the selected runner cannot operate with that narrow allowance, stop and
+review the design rather than adding `--privileged` or broad capabilities.
+
+## Profile-development method
+
+The first implementation pull request should:
+
+- pin the OpenSCAP engine, ComplianceAsCode content, and scanner-image digest;
+- verify the downloaded or packaged RHEL 9 data stream and record its SHA-256;
+- run the upstream RHEL 9 Standard profile in discovery/report-only mode;
+- classify every result as applicable, not applicable, inherited from the
+ platform, pass, fail, error, or not checked;
+- commit an XCCDF tailoring file with a Datopsis-specific profile identifier;
+- commit a rationale table mapping each selected rule to the image-owned file,
+ package, account, or permission it evaluates;
+- explicitly exclude host-only rules for the kernel, boot loader, partitions,
+ mount layout, system services, audit subsystem, host firewall, host sysctls,
+ SELinux enforcement mode, and FIPS mode;
+- avoid automatic remediation, because remediation can silently mutate the
+ image after its tested build steps and invalidate other evidence.
+
+Candidate rule families include RPM/package integrity and signatures, account
+and password-file permissions, empty-password checks, unexpected setuid/setgid
+or world-writable files, service-account shells, and relevant crypto-policy
+files. Exact rule IDs must come from the pinned data stream after discovery;
+they must not be guessed from another RHEL content version.
+
+## Enforcement rollout
+
+SCAP integration should be incremental:
+
+1. **Discovery:** publish complete reports without blocking while the tailoring
+ and applicability decisions are reviewed.
+2. **Stabilization:** require the scan to execute successfully on native AMD64
+ and ARM64 for at least three scheduled or `main` runs. Evaluation errors
+ always fail; selected-rule findings remain visible but temporarily
+ non-blocking.
+3. **Enforcement:** make failure of a selected, image-owned rule blocking. Keep
+ exclusions and any accepted exception documented with owner, rationale, and
+ review date.
+4. **Drift review:** re-run discovery whenever the UBI major version, OpenSCAP
+ engine, ComplianceAsCode data stream, or tailoring changes.
+
+Before enforcement, compare one image digest's `oscap-chroot` result with
+`oscap-podman` on a disposable RHEL 9 host. Investigate differences in platform
+facts, applicability, and rule results. This is a qualification cross-check,
+not a reason to give routine GitHub-hosted CI root access.
+
+## Local Red Hat reproduction
+
+On a disposable RHEL 9 test host with the OpenSCAP container tooling installed,
+the reference scan remains:
+
+```console
+sudo oscap-podman xccdf eval \
+ --profile \
+ --tailoring-file datopsis-ubi9-micro-tailoring.xml \
+ --results-arf results.arf.xml \
+ --report report.html \
+ /usr/share/xml/scap/ssg/content/ssg-rhel9-ds.xml
+```
+
+`oscap-podman` requires root because it integrates with local container storage.
+Use only a dedicated test host containing no unrelated workloads or secrets.
+Record the exact image digest, host version, Podman/OpenSCAP versions, content
+package version, tailoring hash, command, and sanitized results.
+
+## Security boundaries beyond SCAP
+
+SCAP complements rather than replaces vulnerability scanning, behavioral
+tests, supply-chain verification, and deployment controls. Before the first
+release, also complete a concise image threat model, review SBOM/package drift,
+exercise secret and certificate rotation without log disclosure, and define a
+base-image rebuild and vulnerability-response SLA.
+
+The separate `clickhouse-production-stack` repository should own admission
+policy for digest/signature/attestation verification, NetworkPolicy and egress
+allowlists, platform TLS automation, secret-store integration, backup
+encryption and immutable/off-site copies, restore tests, audit-log routing,
+monitoring, and deployment-level incident response. Those controls cannot be
+proven by scanning this image filesystem.
+
+Authoritative references:
+
+- [Red Hat RHEL 9 Security hardening: scanning container and container images](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/security_hardening/scanning-the-system-for-configuration-compliance-and-vulnerabilities_security-hardening)
+- [OpenSCAP User Manual: scanning an arbitrary filesystem](https://static.open-scap.org/openscap-1.3/oscap_user_manual.html)
+- [ComplianceAsCode content and container applicability](https://github.com/ComplianceAsCode/content)
From a9bd221d467c00f10d657a6da4a6e4739132c640 Mon Sep 17 00:00:00 2001
From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com>
Date: Mon, 7 Sep 2026 11:12:22 -0500
Subject: [PATCH 3/6] test: fix native TLS fixture permissions
---
tests/tls-rehearsal.sh | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/tests/tls-rehearsal.sh b/tests/tls-rehearsal.sh
index 47638c8..a74f637 100644
--- a/tests/tls-rehearsal.sh
+++ b/tests/tls-rehearsal.sh
@@ -197,6 +197,17 @@ EOF
chmod 0444 "${secret_dir}"/*.crt "${secret_dir}"/*.pem \
"${secret_dir}"/*.xml
chmod 0400 "${secret_dir}"/*.key
+# These disposable leaf keys are owned by the CI runner, while the image runs
+# as UID 101. Make only leaf fixtures readable across that bind-mount boundary;
+# CA signing keys remain 0400 and are never mounted. Production uses the
+# namespace-aware ownership procedure in docs/TLS.md instead.
+chmod 0444 \
+ "${secret_dir}/connected-v1.key" \
+ "${secret_dir}/connected-v2.key" \
+ "${secret_dir}/connected-endpoint.key" \
+ "${secret_dir}/disconnected-v1.key" \
+ "${secret_dir}/disconnected-v2.key" \
+ "${secret_dir}/disconnected-endpoint.key"
runtime_call network create "${connected_network}" >/dev/null
runtime_call network create --internal "${disconnected_network}" >/dev/null
From c9aebcad96d4ead3ec3451e2319e3c5f37094741 Mon Sep 17 00:00:00 2001
From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com>
Date: Mon, 7 Sep 2026 11:18:02 -0500
Subject: [PATCH 4/6] test: verify native TLS hostname independently
---
docs/TLS-REHEARSAL.md | 2 +-
docs/TLS.md | 11 ++++++++++
tests/tls-rehearsal.sh | 48 ++++++++++++++++++++++++++++++++----------
3 files changed, 49 insertions(+), 12 deletions(-)
diff --git a/docs/TLS-REHEARSAL.md b/docs/TLS-REHEARSAL.md
index 41351e6..6d700ac 100644
--- a/docs/TLS-REHEARSAL.md
+++ b/docs/TLS-REHEARSAL.md
@@ -17,7 +17,7 @@ CONTAINER_RUNTIME=podman \
The automated test proves:
- the server receives only its leaf key and leaf-plus-intermediate chain;
-- HTTPS and native TLS accept the issuing CA and exact DNS name;
+- HTTPS accepts the issuing CA and exact DNS name; native TLS accepts the CA and protocol query, while OpenSSL independently verifies the listener certificate's DNS name;
- unrelated CAs, wrong hostnames, clear-text clients, an incomplete chain, and an unreadable leaf key fail;
- a connected CA bundle supports both public PKI and a controlled private-CA ClickHouse endpoint;
- an internal Podman/Docker network is marked `internal`, can reach its controlled private-CA endpoint through ClickHouse, and cannot open a public-IP TCP connection;
diff --git a/docs/TLS.md b/docs/TLS.md
index bb1748d..6598ce9 100644
--- a/docs/TLS.md
+++ b/docs/TLS.md
@@ -123,6 +123,17 @@ clickhouse-client --secure --host clickhouse.example.internal --port 19440 \
Configure the client's CA according to that client or driver; never use an `insecure` or `skip verification` option as the production solution.
+Client behavior must be qualified, not inferred from `verificationMode` alone.
+In native-protocol testing with ClickHouse 26.8.2.7, `clickhouse-client` strict
+mode rejected an unrelated CA but did not reject the same trusted certificate
+when the connection used a different DNS alias. The CI rehearsal therefore
+uses `clickhouse-client` to prove native protocol and CA validation, and
+OpenSSL's `-verify_hostname` to prove the certificate presented by the native
+listener has the expected identity. Confirm that each production driver
+performs hostname verification. If a required client does not, tightly scope
+CA issuance and network access and treat the limitation as accepted risk, or
+terminate native TLS at a proxy that enforces the expected identity.
+
## Kubernetes and OpenShift secret mount
Create the Secret locally without putting key material in YAML or shell history, then apply it through the approved cluster-management channel:
diff --git a/tests/tls-rehearsal.sh b/tests/tls-rehearsal.sh
index a74f637..839c072 100644
--- a/tests/tls-rehearsal.sh
+++ b/tests/tls-rehearsal.sh
@@ -283,6 +283,26 @@ published_https_port() {
printf '%s\n' "${mapping##*:}"
}
+published_native_port() {
+ local name=$1
+ local mapping
+
+ mapping="$(runtime_call port "${name}" 9440/tcp | tail -n 1)"
+ printf '%s\n' "${mapping##*:}"
+}
+
+verify_native_hostname() {
+ local host=$1
+ local port=$2
+
+ openssl s_client \
+ -connect "127.0.0.1:${port}" \
+ -servername "${host}" \
+ -CAfile "${secret_dir}/root.crt" \
+ -verify_hostname "${host}" \
+ -verify_return_error /dev/null 2>&1
+}
+
query_local() {
local name=$1
local query=$2
@@ -359,18 +379,20 @@ run_tls_server "${connected_server}" "${connected_network}" \
clickhouse.connected.test "${connected_volume}" \
"${secret_dir}/active.chain.crt" "${secret_dir}/active.key" \
--publish 127.0.0.1::8443 \
- --network-alias wrong.connected.test \
+ --publish 127.0.0.1::9440 \
--volume "${runtime_repo_root}/container/config.d/outbound-ca.example.xml:/etc/clickhouse-server/config.d/outbound-ca.xml:ro" \
--volume "${runtime_secret_dir}/connected-ca-bundle.pem:/etc/clickhouse-server/certs/outbound-ca-bundle.pem:ro"
wait_healthy "${connected_server}"
echo 'Connected ClickHouse server is healthy'
connected_port="$(published_https_port "${connected_server}")"
+connected_native_port="$(published_native_port "${connected_server}")"
test "$(https_query clickhouse.connected.test "${connected_port}" \
"${secret_dir}/root.crt")" = 1
echo 'Connected HTTPS validation passed'
test "$(native_tls_query "${connected_network}" clickhouse.connected.test)" = 1
echo 'Connected native TLS validation passed'
+verify_native_hostname clickhouse.connected.test "${connected_native_port}"
if https_query clickhouse.connected.test "${connected_port}" \
"${secret_dir}/unrelated-root.crt" >/dev/null 2>&1; then
@@ -380,9 +402,8 @@ if https_query wrong.connected.test "${connected_port}" \
"${secret_dir}/root.crt" >/dev/null 2>&1; then
fail 'HTTPS accepted the wrong hostname'
fi
-if native_tls_query "${connected_network}" wrong.connected.test \
- >/dev/null 2>&1; then
- fail 'native TLS accepted the wrong hostname'
+if verify_native_hostname wrong.connected.test "${connected_native_port}"; then
+ fail 'native TLS certificate accepted the wrong hostname'
fi
if curl --silent --fail --max-time 5 \
"http://127.0.0.1:${connected_port}/ping" >/dev/null 2>&1; then
@@ -470,7 +491,7 @@ run_tls_server "${connected_server}" "${connected_network}" \
clickhouse.connected.test "${connected_volume}" \
"${secret_dir}/active.chain.crt" "${secret_dir}/active.key" \
--publish 127.0.0.1::8443 \
- --network-alias wrong.connected.test \
+ --publish 127.0.0.1::9440 \
--volume "${runtime_repo_root}/container/config.d/outbound-ca.example.xml:/etc/clickhouse-server/config.d/outbound-ca.xml:ro" \
--volume "${runtime_secret_dir}/connected-ca-bundle.pem:/etc/clickhouse-server/certs/outbound-ca-bundle.pem:ro"
wait_healthy "${connected_server}"
@@ -510,12 +531,13 @@ run_tls_server "${disconnected_server}" "${disconnected_network}" \
"${secret_dir}/disconnected-active.chain.crt" \
"${secret_dir}/disconnected-active.key" \
--publish 127.0.0.1::8443 \
- --network-alias wrong.disconnected.test \
+ --publish 127.0.0.1::9440 \
--volume "${runtime_repo_root}/container/config.d/outbound-ca.example.xml:/etc/clickhouse-server/config.d/outbound-ca.xml:ro" \
--volume "${runtime_secret_dir}/disconnected-ca-bundle.pem:/etc/clickhouse-server/certs/outbound-ca-bundle.pem:ro"
wait_healthy "${disconnected_server}"
echo 'Disconnected ClickHouse server is healthy'
disconnected_port="$(published_https_port "${disconnected_server}")"
+disconnected_native_port="$(published_native_port "${disconnected_server}")"
if [[ "${windows_podman_machine}" == true ]]; then
test "$(query_local "${disconnected_endpoint}" \
"SELECT length(data) > 0 FROM url('https://clickhouse.disconnected.test:8443/?query=SELECT%201', 'RawBLOB', 'data String', headers('X-ClickHouse-User'='default', 'X-ClickHouse-Key'='${password}'))" \
@@ -540,9 +562,13 @@ if native_tls_query "${disconnected_network}" \
>/dev/null 2>&1; then
fail 'disconnected native TLS accepted an unrelated CA'
fi
-if native_tls_query "${disconnected_network}" wrong.disconnected.test \
- >/dev/null 2>&1; then
- fail 'disconnected native TLS accepted the wrong hostname'
+if [[ "${windows_podman_machine}" == false ]]; then
+ verify_native_hostname clickhouse.disconnected.test \
+ "${disconnected_native_port}"
+ if verify_native_hostname wrong.disconnected.test \
+ "${disconnected_native_port}"; then
+ fail 'disconnected native TLS certificate accepted the wrong hostname'
+ fi
fi
echo 'Disconnected native TLS positive and negative validation passed'
test "$(query_local "${disconnected_server}" \
@@ -571,7 +597,7 @@ run_tls_server "${disconnected_server}" "${disconnected_network}" \
"${secret_dir}/disconnected-active.chain.crt" \
"${secret_dir}/disconnected-active.key" \
--publish 127.0.0.1::8443 \
- --network-alias wrong.disconnected.test \
+ --publish 127.0.0.1::9440 \
--volume "${runtime_repo_root}/container/config.d/outbound-ca.example.xml:/etc/clickhouse-server/config.d/outbound-ca.xml:ro" \
--volume "${runtime_secret_dir}/disconnected-ca-bundle.pem:/etc/clickhouse-server/certs/outbound-ca-bundle.pem:ro"
wait_healthy "${disconnected_server}"
@@ -610,7 +636,7 @@ run_tls_server "${disconnected_server}" "${disconnected_network}" \
"${secret_dir}/disconnected-active.chain.crt" \
"${secret_dir}/disconnected-active.key" \
--publish 127.0.0.1::8443 \
- --network-alias wrong.disconnected.test \
+ --publish 127.0.0.1::9440 \
--volume "${runtime_repo_root}/container/config.d/outbound-ca.example.xml:/etc/clickhouse-server/config.d/outbound-ca.xml:ro" \
--volume "${runtime_secret_dir}/disconnected-ca-bundle.pem:/etc/clickhouse-server/certs/outbound-ca-bundle.pem:ro"
wait_healthy "${disconnected_server}"
From a767cee1f968e5058e8eca143f6f8ec6d0a18665 Mon Sep 17 00:00:00 2001
From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com>
Date: Mon, 7 Sep 2026 11:26:41 -0500
Subject: [PATCH 5/6] test: harden cross-architecture TLS checks
---
docs/TLS-REHEARSAL.md | 5 +++--
docs/TLS.md | 6 ++++++
tests/smoke.sh | 9 ++++++---
tests/tls-rehearsal.sh | 44 +++++++++++++++++++++++++++++++-----------
4 files changed, 48 insertions(+), 16 deletions(-)
diff --git a/docs/TLS-REHEARSAL.md b/docs/TLS-REHEARSAL.md
index 6d700ac..e883edd 100644
--- a/docs/TLS-REHEARSAL.md
+++ b/docs/TLS-REHEARSAL.md
@@ -18,7 +18,8 @@ The automated test proves:
- the server receives only its leaf key and leaf-plus-intermediate chain;
- HTTPS accepts the issuing CA and exact DNS name; native TLS accepts the CA and protocol query, while OpenSSL independently verifies the listener certificate's DNS name;
-- unrelated CAs, wrong hostnames, clear-text clients, an incomplete chain, and an unreadable leaf key fail;
+- unrelated CAs, wrong hostnames, clear-text clients, and an incomplete chain fail;
+- an unreadable leaf key produces a permission error and leaves both secure listeners unavailable, even if the ClickHouse process remains alive;
- a connected CA bundle supports both public PKI and a controlled private-CA ClickHouse endpoint;
- an internal Podman/Docker network is marked `internal`, can reach its controlled private-CA endpoint through ClickHouse, and cannot open a public-IP TCP connection;
- connected and disconnected certificate renewal changes the served serial;
@@ -172,7 +173,7 @@ Retain sanitized output. Then require failures for:
- HTTP sent to the HTTPS port;
- non-secure native protocol sent to `9440`;
- a chain containing only the leaf;
-- a key unreadable by the assigned container identity.
+- a key unreadable by the assigned container identity; treat logged key-load errors or unavailable secure listeners as a failed deployment even if the ClickHouse process remains alive.
Do not use an insecure client flag to make any negative test pass.
diff --git a/docs/TLS.md b/docs/TLS.md
index 6598ce9..a8a1107 100644
--- a/docs/TLS.md
+++ b/docs/TLS.md
@@ -93,6 +93,12 @@ podman unshare chown 101:0 tls.key
The key may display subordinate host IDs afterward; `podman unshare ls -l tls.key` shows its container-visible ownership. For rootful Podman, use `sudo chown 101:0 tls.key` instead. Do not make the private key world-readable to bypass a mapping problem.
+An unreadable key does not necessarily terminate the ClickHouse process. It can
+leave the process running while HTTPS and secure native listeners are absent.
+Readiness must therefore test a required TLS listener, and operators must alert
+on certificate/key loading errors instead of treating process liveness as proof
+that TLS is available.
+
Run the image with separate read-only mounts. Add `:Z` to bind mounts on SELinux hosts:
```console
diff --git a/tests/smoke.sh b/tests/smoke.sh
index 54073a2..947452d 100644
--- a/tests/smoke.sh
+++ b/tests/smoke.sh
@@ -83,12 +83,14 @@ wait_healthy() {
wait_failed_with() {
local server_name=$1
local expected_message=$2
+ local logs
local running
for _ in {1..30}; do
running="$("${runtime}" inspect --format '{{.State.Running}}' "${server_name}")"
if [[ "${running}" == false ]]; then
- "${runtime}" logs "${server_name}" 2>&1 | grep -Fq "${expected_message}"
+ logs="$("${runtime}" logs "${server_name}" 2>&1)"
+ grep -Fq "${expected_message}" <<< "${logs}"
return
fi
sleep 1
@@ -137,8 +139,9 @@ actual_version="$(query_server "${primary}" "${password}" 'SELECT version()')"
expected_version="$("${runtime}" inspect --format '{{index .Config.Labels "org.opencontainers.image.version"}}' "${image}")"
test "${actual_version}" = "${expected_version}"
test "$("${runtime}" inspect --format '{{.Config.User}}' "${image}")" = "101:0"
-if "${runtime}" inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "${image}" | \
- grep -q '^CLICKHOUSE_DATA_DIR='; then
+image_environment="$("${runtime}" inspect --format \
+ '{{range .Config.Env}}{{println .}}{{end}}' "${image}")"
+if grep -q '^CLICKHOUSE_DATA_DIR=' <<< "${image_environment}"; then
echo "Image metadata unexpectedly contains CLICKHOUSE_DATA_DIR" >&2
exit 1
fi
diff --git a/tests/tls-rehearsal.sh b/tests/tls-rehearsal.sh
index 839c072..523eb0e 100644
--- a/tests/tls-rehearsal.sh
+++ b/tests/tls-rehearsal.sh
@@ -303,6 +303,23 @@ verify_native_hostname() {
-verify_return_error /dev/null 2>&1
}
+wait_for_log() {
+ local name=$1
+ local expected=$2
+ local logs
+
+ for _ in {1..30}; do
+ logs="$(runtime_call logs "${name}" 2>&1 || true)"
+ if grep -Fq "${expected}" <<< "${logs}"; then
+ return
+ fi
+ sleep 1
+ done
+
+ runtime_call logs "${name}" || true
+ fail "${name} did not log the expected TLS key error"
+}
+
query_local() {
local name=$1
local query=$2
@@ -454,27 +471,32 @@ fi
runtime_call rm -f "${incomplete_server}" >/dev/null
echo 'Incomplete-chain rejection passed'
-# An unreadable private key must prevent startup. Windows Podman Machine file
-# sharing does not preserve a host chmod 000, so native Linux CI owns this
-# assertion and the Windows reproduction reports the explicit limitation.
+# ClickHouse can keep its process alive after failing to load an unreadable key,
+# but it must report the permission failure and leave both secure listeners
+# unavailable. Windows Podman Machine file sharing does not preserve a host
+# chmod 000, so native Linux CI owns this assertion.
if command -v cygpath >/dev/null 2>&1; then
echo 'Unreadable-key rejection skipped on Windows; native Linux CI runs it'
else
cp "${secret_dir}/connected-v1.key" "${secret_dir}/unreadable.key"
chmod 000 "${secret_dir}/unreadable.key"
- if run_tls_server "${unreadable_server}" "${connected_network}" \
+ if ! run_tls_server "${unreadable_server}" "${connected_network}" \
unreadable.connected.test "${unreadable_volume}" \
"${secret_dir}/connected-v1.chain.crt" \
"${secret_dir}/unreadable.key"; then
- for _ in {1..30}; do
- if [[ "$(runtime_call inspect --format '{{.State.Running}}' \
- "${unreadable_server}")" == false ]]; then
- break
+ fail 'container runtime rejected the unreadable-key fixture unexpectedly'
+ fi
+ wait_for_log "${unreadable_server}" 'Error loading private key'
+ wait_for_log "${unreadable_server}" 'Permission denied'
+ if [[ "$(runtime_call inspect --format '{{.State.Running}}' \
+ "${unreadable_server}")" == true ]]; then
+ for secure_port in 8443 9440; do
+ if runtime_call exec "${unreadable_server}" timeout 2 bash -c \
+ "exec 3<>/dev/tcp/127.0.0.1/${secure_port}" \
+ >/dev/null 2>&1; then
+ fail "unreadable key left secure port ${secure_port} available"
fi
- sleep 1
done
- test "$(runtime_call inspect --format '{{.State.Running}}' \
- "${unreadable_server}")" = false
fi
chmod 0400 "${secret_dir}/unreadable.key"
echo 'Unreadable-key rejection passed'
From 0a9436749ab5688e0d0301339ed93b6208efe265 Mon Sep 17 00:00:00 2001
From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com>
Date: Mon, 7 Sep 2026 11:32:54 -0500
Subject: [PATCH 6/6] test: support TLS checks on isolated networks
---
docs/TLS-REHEARSAL.md | 6 ++++++
tests/tls-rehearsal.sh | 48 ++++++++++++++++++++++++------------------
2 files changed, 33 insertions(+), 21 deletions(-)
diff --git a/docs/TLS-REHEARSAL.md b/docs/TLS-REHEARSAL.md
index e883edd..bc453d6 100644
--- a/docs/TLS-REHEARSAL.md
+++ b/docs/TLS-REHEARSAL.md
@@ -27,6 +27,12 @@ The automated test proves:
The internal-network test is repeatable CI evidence. It does not replace an organization's controlled-media and physically or logically disconnected acceptance rehearsal.
+Docker does not publish host ports from an `--internal` network. Native Linux
+CI therefore performs external hostname, CA, and certificate-serial checks
+against the container's bridge address while in-network ClickHouse requests
+prove application connectivity. The server remains attached only to the
+internal network throughout the disconnected phase.
+
On Windows, Podman Machine file sharing does not preserve a host `chmod 000`,
and an internal network does not expose a published port back to the Windows
host. The script therefore runs disconnected HTTPS from another workload on
diff --git a/tests/tls-rehearsal.sh b/tests/tls-rehearsal.sh
index 523eb0e..89e6fda 100644
--- a/tests/tls-rehearsal.sh
+++ b/tests/tls-rehearsal.sh
@@ -294,15 +294,23 @@ published_native_port() {
verify_native_hostname() {
local host=$1
local port=$2
+ local address=${3:-127.0.0.1}
openssl s_client \
- -connect "127.0.0.1:${port}" \
+ -connect "${address}:${port}" \
-servername "${host}" \
-CAfile "${secret_dir}/root.crt" \
-verify_hostname "${host}" \
-verify_return_error /dev/null 2>&1
}
+container_ipv4() {
+ local name=$1
+
+ runtime_call inspect --format \
+ '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "${name}"
+}
+
wait_for_log() {
local name=$1
local expected=$2
@@ -360,11 +368,12 @@ https_query() {
local host=$1
local port=$2
local ca_file=$3
+ local address=${4:-127.0.0.1}
curl --silent --show-error --fail --max-time 10 \
--noproxy '*' \
--cacert "${ca_file}" \
- --resolve "${host}:${port}:127.0.0.1" \
+ --resolve "${host}:${port}:${address}" \
--user "default:${password}" \
"https://${host}:${port}/?query=SELECT%201"
}
@@ -552,28 +561,29 @@ run_tls_server "${disconnected_server}" "${disconnected_network}" \
clickhouse.disconnected.test "${disconnected_volume}" \
"${secret_dir}/disconnected-active.chain.crt" \
"${secret_dir}/disconnected-active.key" \
- --publish 127.0.0.1::8443 \
- --publish 127.0.0.1::9440 \
--volume "${runtime_repo_root}/container/config.d/outbound-ca.example.xml:/etc/clickhouse-server/config.d/outbound-ca.xml:ro" \
--volume "${runtime_secret_dir}/disconnected-ca-bundle.pem:/etc/clickhouse-server/certs/outbound-ca-bundle.pem:ro"
wait_healthy "${disconnected_server}"
echo 'Disconnected ClickHouse server is healthy'
-disconnected_port="$(published_https_port "${disconnected_server}")"
-disconnected_native_port="$(published_native_port "${disconnected_server}")"
if [[ "${windows_podman_machine}" == true ]]; then
test "$(query_local "${disconnected_endpoint}" \
"SELECT length(data) > 0 FROM url('https://clickhouse.disconnected.test:8443/?query=SELECT%201', 'RawBLOB', 'data String', headers('X-ClickHouse-User'='default', 'X-ClickHouse-Key'='${password}'))" \
"${endpoint_password}")" = 1
echo 'Disconnected HTTPS passed inside the isolated Podman network'
else
+ disconnected_address="$(container_ipv4 "${disconnected_server}")"
+ disconnected_port=8443
+ disconnected_native_port=9440
test "$(https_query clickhouse.disconnected.test "${disconnected_port}" \
- "${secret_dir}/root.crt")" = 1
+ "${secret_dir}/root.crt" "${disconnected_address}")" = 1
if https_query clickhouse.disconnected.test "${disconnected_port}" \
- "${secret_dir}/unrelated-root.crt" >/dev/null 2>&1; then
+ "${secret_dir}/unrelated-root.crt" "${disconnected_address}" \
+ >/dev/null 2>&1; then
fail 'disconnected HTTPS accepted an unrelated CA'
fi
if https_query wrong.disconnected.test "${disconnected_port}" \
- "${secret_dir}/root.crt" >/dev/null 2>&1; then
+ "${secret_dir}/root.crt" "${disconnected_address}" \
+ >/dev/null 2>&1; then
fail 'disconnected HTTPS accepted the wrong hostname'
fi
fi
@@ -586,9 +596,9 @@ if native_tls_query "${disconnected_network}" \
fi
if [[ "${windows_podman_machine}" == false ]]; then
verify_native_hostname clickhouse.disconnected.test \
- "${disconnected_native_port}"
+ "${disconnected_native_port}" "${disconnected_address}"
if verify_native_hostname wrong.disconnected.test \
- "${disconnected_native_port}"; then
+ "${disconnected_native_port}" "${disconnected_address}"; then
fail 'disconnected native TLS certificate accepted the wrong hostname'
fi
fi
@@ -618,12 +628,9 @@ run_tls_server "${disconnected_server}" "${disconnected_network}" \
clickhouse.disconnected.test "${disconnected_volume}" \
"${secret_dir}/disconnected-active.chain.crt" \
"${secret_dir}/disconnected-active.key" \
- --publish 127.0.0.1::8443 \
- --publish 127.0.0.1::9440 \
--volume "${runtime_repo_root}/container/config.d/outbound-ca.example.xml:/etc/clickhouse-server/config.d/outbound-ca.xml:ro" \
--volume "${runtime_secret_dir}/disconnected-ca-bundle.pem:/etc/clickhouse-server/certs/outbound-ca-bundle.pem:ro"
wait_healthy "${disconnected_server}"
-disconnected_port="$(published_https_port "${disconnected_server}")"
disconnected_new_serial="$(openssl x509 \
-in "${secret_dir}/disconnected-v2.crt" \
-noout -serial | cut -d= -f2)"
@@ -634,8 +641,9 @@ if [[ "${windows_podman_machine}" == true ]]; then
disconnected_served_serial="${disconnected_new_serial}"
echo 'Served renewal serial inspection deferred to native Linux CI'
else
+ disconnected_address="$(container_ipv4 "${disconnected_server}")"
disconnected_served_serial="$(openssl s_client \
- -connect "127.0.0.1:${disconnected_port}" \
+ -connect "${disconnected_address}:8443" \
-servername clickhouse.disconnected.test \
-CAfile "${secret_dir}/root.crt" /dev/null | \
openssl x509 -noout -serial | cut -d= -f2)"
@@ -657,12 +665,9 @@ run_tls_server "${disconnected_server}" "${disconnected_network}" \
clickhouse.disconnected.test "${disconnected_volume}" \
"${secret_dir}/disconnected-active.chain.crt" \
"${secret_dir}/disconnected-active.key" \
- --publish 127.0.0.1::8443 \
- --publish 127.0.0.1::9440 \
--volume "${runtime_repo_root}/container/config.d/outbound-ca.example.xml:/etc/clickhouse-server/config.d/outbound-ca.xml:ro" \
--volume "${runtime_secret_dir}/disconnected-ca-bundle.pem:/etc/clickhouse-server/certs/outbound-ca-bundle.pem:ro"
wait_healthy "${disconnected_server}"
-disconnected_port="$(published_https_port "${disconnected_server}")"
if [[ "${windows_podman_machine}" == true ]]; then
test "$(query_local "${disconnected_endpoint}" \
"SELECT length(data) > 0 FROM url('https://clickhouse.disconnected.test:8443/?query=SELECT%201', 'RawBLOB', 'data String', headers('X-ClickHouse-User'='default', 'X-ClickHouse-Key'='${password}'))" \
@@ -670,16 +675,17 @@ if [[ "${windows_podman_machine}" == true ]]; then
disconnected_served_serial="${disconnected_old_serial}"
echo 'Served rollback serial inspection deferred to native Linux CI'
else
+ disconnected_address="$(container_ipv4 "${disconnected_server}")"
disconnected_served_serial="$(openssl s_client \
- -connect "127.0.0.1:${disconnected_port}" \
+ -connect "${disconnected_address}:8443" \
-servername clickhouse.disconnected.test \
-CAfile "${secret_dir}/root.crt" /dev/null | \
openssl x509 -noout -serial | cut -d= -f2)"
fi
test "${disconnected_served_serial}" = "${disconnected_old_serial}"
if [[ "${windows_podman_machine}" == false ]]; then
- test "$(https_query clickhouse.disconnected.test "${disconnected_port}" \
- "${secret_dir}/root.crt")" = 1
+ test "$(https_query clickhouse.disconnected.test 8443 \
+ "${secret_dir}/root.crt" "${disconnected_address}")" = 1
fi
echo "Connected and disconnected CA-issued TLS rehearsal passed"