joey-huckabee · Sep 7, 2026 · Sep 7, 2026 · Sep 7, 2026 · Sep 7, 2026 · Sep 7, 2026
@@ -80,6 +80,8 @@ jobs:
       SBOM_FILE: clickhouse-server-ubi9-${{ matrix.architecture }}.spdx.json
       GRYPE_SARIF: grype-${{ matrix.architecture }}.sarif
       GRYPE_ALL: grype-all-${{ matrix.architecture }}.json
+      SCAP_SCANNER_IMAGE: localhost/datopsis-openscap:0.1.82-${{ matrix.architecture }}
+      SCAP_RESULTS_DIR: scap-results-${{ matrix.architecture }}
     steps:
       - name: Check out repository
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -134,6 +136,30 @@ jobs:
           IMAGE: ${{ env.TEST_IMAGE }}
         run: bash tests/tls-rehearsal.sh
 
+      - name: Build pinned OpenSCAP tool image
+        id: build-scap
+        continue-on-error: true
+        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: .
+          file: Containerfile.scap
+          platforms: ${{ matrix.platform }}
+          load: true
+          push: false
+          tags: ${{ env.SCAP_SCANNER_IMAGE }}
+          cache-from: type=gha,scope=scap-${{ matrix.architecture }}
+          cache-to: type=gha,mode=max,scope=scap-${{ matrix.architecture }}
+
+      - name: Run SCAP discovery scan
+        id: scan-scap
+        if: ${{ steps.build-scap.outcome == 'success' }}
+        continue-on-error: true
+        env:
+          ARCHITECTURE: ${{ matrix.architecture }}
+          CONTAINER_RUNTIME: docker
+          IMAGE: ${{ env.TEST_IMAGE }}
+        run: bash scripts/scap-scan.sh
+
       - name: Scan image
         uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
         with:
@@ -194,6 +220,7 @@ jobs:
             ${{ env.SBOM_FILE }}
             ${{ env.GRYPE_SARIF }}
             ${{ env.GRYPE_ALL }}
+            ${{ env.SCAP_RESULTS_DIR }}/
           if-no-files-found: warn
           retention-days: 14
 
@@ -204,6 +231,15 @@ jobs:
           sarif_file: ${{ env.GRYPE_SARIF }}
           category: grype-image-${{ matrix.architecture }}
 
+      - name: Require successful SCAP evaluation
+        if: ${{ always() }}
+        env:
+          SCAP_BUILD_OUTCOME: ${{ steps.build-scap.outcome }}
+          SCAP_SCAN_OUTCOME: ${{ steps.scan-scap.outcome }}
+        run: |
+          test "${SCAP_BUILD_OUTCOME}" = success
+          test "${SCAP_SCAN_OUTCOME}" = success
+
   image-result:
     name: image
     if: ${{ always() }}

@@ -0,0 +1,45 @@
+# syntax=docker/dockerfile:1.7
+
+ARG UBI_MINIMAL_IMAGE="registry.access.redhat.com/ubi9/ubi-minimal:9.8@sha256:7fbeae18dc9476399f565e68255f602a3374ea8614ba3d14843565131a13ff93"
+
+FROM ${UBI_MINIMAL_IMAGE}
+
+ARG OPENSCAP_NEVRA="1.3.14-1.el9_8"
+ARG SSG_VERSION="0.1.82"
+ARG SSG_ARCHIVE_SHA256="765e84bdce7f9055f9b9c2dd0ee2b713d4255f8eec94eac6d35ea4973c28919c"
+ARG SSG_RHEL9_DATASTREAM_SHA256="92204daafbf4f38011671ef034fae4cffb48f708516186710346a9ec702a1f8f"
+
+# This image is a CI tool, not a runtime layer of the ClickHouse image. The
+# OpenSCAP RPM version and upstream content are deliberately pinned so a
+# repository or content update cannot silently change compliance evidence.
+# hadolint ignore=DL3041
+RUN microdnf install -y \
+        "openscap-scanner-${OPENSCAP_NEVRA}" \
+        tar \
+        unzip \
+    && mkdir -p /opt/scap \
+    && archive="/tmp/scap-security-guide-${SSG_VERSION}.zip" \
+    && curl --fail --location --proto '=https' --tlsv1.2 \
+         --retry 5 --retry-delay 2 --retry-all-errors \
+         --output "${archive}" \
+         "https://github.com/ComplianceAsCode/content/releases/download/v${SSG_VERSION}/scap-security-guide-${SSG_VERSION}.zip" \
+    && printf '%s  %s\n' "${SSG_ARCHIVE_SHA256}" "${archive}" | sha256sum --check --strict \
+    && unzip -j "${archive}" \
+         "scap-security-guide-${SSG_VERSION}/ssg-rhel9-ds.xml" \
+         -d /opt/scap \
+    && printf '%s  %s\n' \
+         "${SSG_RHEL9_DATASTREAM_SHA256}" \
+         /opt/scap/ssg-rhel9-ds.xml | sha256sum --check --strict \
+    && rm -f "${archive}" \
+    && microdnf clean all \
+    && rm -rf /var/cache/yum /var/log/dnf* /var/log/yum.*
+
+COPY --chmod=0755 scripts/scap-container.sh /usr/local/bin/scap-container
+
+LABEL org.opencontainers.image.title="Datopsis OpenSCAP offline scanner" \
+      org.opencontainers.image.description="Pinned, isolated CI scanner for clickhouse-server-ubi9 exported filesystems" \
+      org.opencontainers.image.source="https://github.com/datopsis/clickhouse-server-ubi9"
+
+USER 65534:65534
+
+ENTRYPOINT ["/usr/local/bin/scap-container"]
@@ -6,7 +6,7 @@ This repository treats the built image as the primary deliverable. CI therefore
 
 | Workflow | Triggers | Purpose |
 | --- | --- | --- |
-| `CI` | Pull requests, pushes to `main`, weekly schedule, manual dispatch | Lint and workflow audit, followed by native AMD64 and ARM64 image builds, smoke tests, Trivy scans, Syft SBOMs, and Grype scans. |
+| `CI` | Pull requests, pushes to `main`, weekly schedule, manual dispatch | Lint and workflow audit, followed by native AMD64 and ARM64 image builds, smoke tests, isolated SCAP discovery, Trivy scans, Syft SBOMs, and Grype scans. |
 | `CodeQL` | Workflow changes, weekly schedule, manual dispatch | Static analysis of GitHub Actions with the security-extended query suite. |
 | `OpenSSF Scorecard` | Pushes to `main`, ruleset changes, weekly schedule, manual dispatch | Supply-chain posture analysis, SARIF upload, and public Scorecard publication. |
 | `Release image` | Tags matching `v*` | Tag/input/changelog validation, multi-architecture publish, digest scans, evidence generation, keyless signing, and GitHub release creation. |
@@ -24,6 +24,7 @@ Workflow-level permissions default to read-only. Write scopes are applied only t
 | Runtime behavior | Native GitHub-hosted AMD64 and ARM64 runners, Buildx, and `tests/smoke.sh` | Each architecture's exact test image starts and stops correctly under production-oriented restrictions and supports documented initialization/authentication behavior. Runner and loaded-image assertions prevent emulation or a mislabeled image from being treated as native evidence. | Hosted-runner tests do not replace OpenShift qualification or application-specific performance testing. |
 | Image vulnerabilities | Trivy image scan | No fixed high/critical findings according to Trivy's current databases and vendor severity selection. | `ignore-unfixed` intentionally leaves unfixed risk for human release review. |
 | Independent inventory and scan | Syft plus Grype | SPDX inventory of the tested image and a second vulnerability matcher/database; fixed high/critical findings block. | Overlap is intentional, but scanner agreement is not proof of absence. |
+| Filesystem compliance discovery | Pinned OpenSCAP engine and ComplianceAsCode RHEL 9 STIG profile | Complete architecture-specific inventory of upstream rule results against a root-owner-preserving export; scanner errors block. | STIG is a broad discovery source, not wholesale adoption. Findings are non-blocking until applicability is reviewed and a container-specific tailoring is approved. |
 | Supply-chain posture | OpenSSF Scorecard | Repository and build-pipeline practice signals published independently. | Historical and popularity signals improve only through genuine project operation. |
 | Release integrity | BuildKit attestations, Cosign, GHCR, and GitHub Releases | Digest-bound multi-architecture artifact, SBOM/provenance evidence, keyless signature, and durable release assets. | The tag workflow publishes before post-build scans; a failed candidate must be quarantined or removed. |
 
@@ -57,7 +58,7 @@ Audit these settings before each release and after organization policy changes.
 For every required run, verify the event and head SHA first. Then review the following evidence:
 
 - `lint`: every hook and the release-tag test ran, Zizmor audited every workflow, and there are no warnings or annotations hidden behind a successful wrapper.
-- `image (amd64)` and `image (arm64)`: the native runner assertion, loaded-image architecture assertion, configuration scan count, ClickHouse version printed by the smoke suite, Trivy target/OS/package count and result count, SBOM package count, and both Grype's blocking fixed-findings result and full finding inventory. The aggregate `image` job is only the merge gate; inspect the two jobs that produced the evidence.
+- `image (amd64)` and `image (arm64)`: the native runner assertion, loaded-image architecture assertion, configuration scan count, ClickHouse version printed by the smoke suite, SCAP execution outcome and result counts, Trivy target/OS/package count and result count, SBOM package count, and both Grype's blocking fixed-findings result and full finding inventory. The aggregate `image` job is only the merge gate; inspect the two jobs that produced the evidence.
 - `CodeQL` and Scorecard: analysis covered the intended files, SARIF processing completed, and the Security tab has no new open alert. A successful upload is not the same as zero findings.
 - skipped steps: PR SARIF publication is intentionally skipped to avoid permission failures from untrusted forks; it runs on `main`. A skipped build, smoke test, or scanner is not acceptable.
 - warnings: Trivy may use another vendor's severity when Red Hat data is absent. Grype's `only-fixed` option can ignore real but currently unfixable findings. Review both against Red Hat and ClickHouse advisories before a release.
@@ -70,11 +71,12 @@ Each native image matrix job runs these controls in order. AMD64 uses `ubuntu-24
 
 1. **Trivy configuration scan** checks the `Containerfile`, Compose configuration, and repository infrastructure configuration for high and critical misconfigurations.
 2. **Build and runtime tests** exercise startup, authentication, initialization, persistence, shutdown, read-only operation, dropped capabilities, arbitrary UIDs, chained CA-issued HTTPS/native TLS, public/private outbound trust, disconnected isolation, negative certificate cases, renewal, and rollback.
-3. **Trivy image scan** blocks fixed high and critical operating-system or application vulnerabilities and reports its detected OS and package count for review.
-4. **Complete SPDX inventory** uses Syft to inventory the tested filesystem and RPM database, then `scripts/augment-spdx.py` declares the three pinned ClickHouse TGZ components that have no RPM metadata. The script takes their version and channel from `Containerfile`, records Apache-2.0 licensing and package identifiers, and fails instead of duplicating a component Syft already found.
-5. **Blocking Grype SBOM scan** scans that exact SPDX document and blocks fixed high and critical vulnerabilities.
-6. **Full Grype inventory** performs a non-blocking scan of the same SBOM without filtering unfixed matches and retains `grype-all.json`. Non-blocking means “record for triage,” not “accepted risk.”
-7. **Artifact and SARIF publication** retains the inventory and results for investigation and publishes fixed Grype findings from non-PR runs to GitHub code scanning.
+3. **OpenSCAP discovery** builds a pinned-input UBI scanner, exports but never executes the stopped target, preserves filesystem ownership inside an isolated tmpfs, and evaluates the pinned RHEL 9 STIG profile without network or an engine socket. The profile is an analysis source rather than wholesale control adoption. Findings remain report-only; execution errors block.
+4. **Trivy image scan** blocks fixed high and critical operating-system or application vulnerabilities and reports its detected OS and package count for review.
+5. **Complete SPDX inventory** uses Syft to inventory the tested filesystem and RPM database, then `scripts/augment-spdx.py` declares the three pinned ClickHouse TGZ components that have no RPM metadata. The script takes their version and channel from `Containerfile`, records Apache-2.0 licensing and package identifiers, and fails instead of duplicating a component Syft already found.
+6. **Blocking Grype SBOM scan** scans that exact SPDX document and blocks fixed high and critical vulnerabilities.
+7. **Full Grype inventory** performs a non-blocking scan of the same SBOM without filtering unfixed matches and retains `grype-all.json`. Non-blocking means “record for triage,” not “accepted risk.”
+8. **Artifact and SARIF publication** retains the inventory and results for investigation and publishes fixed Grype findings from non-PR runs to GitHub code scanning.
 
 Trivy and Grype deliberately overlap. They use different databases and matching logic, so a clean result from one does not replace the other. Both gates ignore vulnerabilities without an upstream fix; unfixed findings still require periodic review before release. Scanner disagreements should be investigated against the vendor advisory and documented if accepted.
 
@@ -85,6 +87,7 @@ Trivy and Grype deliberately overlap. They use different databases and matching
 | `clickhouse-server-ubi9-<architecture>.spdx.json` | CI artifact `image-security-<commit>-<architecture>` | 14 days | Package inventory for the exact native AMD64 or ARM64 test image. |
 | `grype-<architecture>.sarif` | Same architecture-specific CI artifact and GitHub code scanning on non-PR runs | 14 days for the downloadable artifact | Machine-readable findings and architecture-specific review evidence. |
 | `grype-all-<architecture>.json` | Architecture-specific CI artifact | 14 days | Complete point-in-time inventory including unfixed Low and Medium matches for human triage. The release workflow separately retains `grype-all.json` for 30 days. |
+| `scap-results-<architecture>/` | Architecture-specific CI artifact | 14 days | Discovery ARF/XCCDF/HTML, full JSON rule inventory, exit code, data-stream hash, scanner version, and RPM versions for the exact target/scanner image IDs. |
 | `image.spdx.json` | Tag-run artifact and GitHub release asset | 30-day Actions copy; release asset retained with the release | Downloadable inventory for the published digest. |
 | Release `grype.sarif` | Tag-run artifact and GitHub code scanning | 30 days for the downloadable artifact | Point-in-time scan evidence; not attached to the release because vulnerability data ages rapidly. |
 | BuildKit SBOM/provenance and complete SPDX attestation | OCI registry attestations; downloaded together as `image.intoto.jsonl` | Lifetime of the package/release | Registry-native build evidence plus the keyless, digest-bound copy of `image.spdx.json`. |
@@ -110,6 +113,10 @@ CONTAINER_RUNTIME=podman \
   IMAGE="clickhouse-server-ubi9:test-${ARCHITECTURE}" bash tests/smoke.sh
 ```
 
+Build and run the isolated SCAP discovery scanner with the Podman procedure in
+[SCAP.md](SCAP.md). It intentionally creates a second tooling image and does
+not alter the ClickHouse deliverable.
+
 Running an ARM64 image under emulation on an AMD64 workstation can help diagnose portable build failures, but it does not reproduce the native ARM64 qualification. GitHub's `ubuntu-24.04-arm` runner supplies that evidence using Docker Engine and Buildx. To reproduce both jobs faithfully, run the procedure once on each native architecture and retain separate results. Podman and Docker exercise the same image contract but remain distinct runtime implementations, so first-release evidence records both the native CI results and the separately tested Podman version.
 
 For the scanner examples below, keep using the architecture-specific image name:

@@ -102,22 +102,22 @@ This repository owns image-specific behavior, basic usage, and minimal platform
 
 **Profile discovery and tailoring**
 
-- [ ] Pin a UBI 9 OpenSCAP scanner image by digest and pin the OpenSCAP and ComplianceAsCode content versions. Record the RHEL 9 data-stream SHA-256 and reject an unexpected stream.
-- [ ] Run the upstream RHEL 9 Standard profile in report-only discovery mode against the exported image filesystem. Inventory every pass, failure, error, not-applicable, and not-checked result without claiming host or deployment compliance.
+- [x] Build the scanner from the same digest-pinned UBI 9 base, pin OpenSCAP `1.3.14-1.el9_8` and ComplianceAsCode `0.1.82`, verify the release archive, and verify/record the RHEL 9 data-stream SHA-256. Retain the produced scanner image ID for every run; use a manifest digest if the tool image is later published for reuse.
+- [x] Add upstream RHEL 9 STIG-profile report-only discovery against an ownership-preserving exported image filesystem; ComplianceAsCode `0.1.82` does not contain a RHEL 9 Standard profile. Treat STIG as an analysis source rather than wholesale adoption, inventory every result, and keep evaluation errors blocking without claiming host or deployment compliance.
 - [ ] Create a reviewed XCCDF tailoring profile containing only rules that are applicable to and controlled by this image. Commit a rule-rationale matrix and document every host/platform exclusion.
 - [ ] Exclude kernel, boot-loader, partition, mount-layout, systemd, audit, host-networking, sysctl, SELinux-mode, and FIPS-mode controls unless the image later gains direct ownership of one. Do not use automatic remediation.
 
 **Safe CI integration**
 
-- [ ] Export the stopped, already-tested image's merged filesystem into an ephemeral directory and mount that directory read-only into the scanner. Never execute target-image content to prepare the scan.
-- [ ] Run `oscap-chroot` in a digest-pinned scanner with no Docker/Podman socket, no host namespace, no workflow secrets, and no evaluation-time network. Prove the minimum chroot-related capability; do not use `--privileged`, Podman-in-Podman, or broad host mounts.
-- [ ] Generate architecture-specific ARF XML, XCCDF XML, and HTML reports containing the image digest, architecture, scanner/content versions, data-stream hash, and tailoring hash. Retain them with the other image-security evidence.
+- [x] Export the stopped, already-tested image without executing it, mount the archive read-only, and extract as namespaced root into the scanner's disposable tmpfs so numeric ownership evidence is preserved.
+- [x] Configure OpenSCAP offline mode directly with `OSCAP_PROBE_ROOT` because UBI AppStream does not ship the `oscap-chroot` wrapper. Run with no Docker/Podman socket, host namespace, workflow secrets, or evaluation-time network; use a read-only scanner root, `no-new-privileges`, drop all capabilities, and add only `CHOWN`, `FOWNER`, `DAC_OVERRIDE`, and `SYS_CHROOT` for metadata preservation, restrictive-file inspection/results output, and offline probes.
+- [x] Generate and retain architecture-specific ARF XML, XCCDF XML, HTML, full JSON rule inventory, target/scanner image IDs, architecture, scanner/content versions, data-stream hash, and exit code with the other image-security evidence. Add the tailoring hash when the reviewed tailoring exists.
 - [ ] Run report-only on native AMD64 and ARM64 for at least three scheduled or `main` executions. Evaluation errors fail immediately; selected-rule findings become blocking only after the baseline is stable and reviewed.
 - [ ] Cross-check one exact image digest with `oscap-podman` on a disposable RHEL 9 host. Reconcile platform/applicability differences before enforcement; do not grant routine hosted CI root or engine access merely to match that command.
 
 **Exit evidence**
 
-- [ ] Retain the discovery report, final tailoring, rule-rationale/exclusion review, three stable two-architecture runs, capability inspection, and `oscap-chroot` versus `oscap-podman` comparison.
+- [ ] Retain the discovery report, final tailoring, rule-rationale/exclusion review, three stable two-architecture runs, capability inspection, and `OSCAP_PROBE_ROOT` versus `oscap-podman` comparison.
 - [ ] State precisely that the result covers selected image-filesystem controls and is not CIS/STIG certification of the host, OpenShift cluster, or production deployment.
 
 #### 6. Security-control provenance, STIG analysis, and SCTM export