diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5c08eaf..e6ee6cb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -277,6 +277,14 @@ jobs: # replaces reached healthy in about two minutes; the bound is for the step # that stops returning, not for a healthy run. timeout-minutes: 25 + env: + # The images the release matrix publishes, named once for the whole job: the + # pull below and the digest guard after it read this list, so a connector + # added to the matrix is one edit here rather than two that can drift. It is + # the matrix that is authoritative; this is the copy the job can iterate. + RELEASE_IMAGES: >- + backend frontend connector-dnstwist connector-shodan connector-hibp + connector-dns-security connector-intelligence connector-exposure-endpoints permissions: contents: read # The images are private: reading them back is the whole exercise, and the @@ -321,10 +329,20 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Pull every image the release published - # Every image the matrix names, including the connectors this job does not - # start: a tag that resolves to a missing manifest is a release an - # operator cannot install, and `pull` is the cheapest place to find out. - run: docker compose pull + # Compose is the list: `config --images` resolves the image of every + # service, and `--profile '*'` turns on the profiles the connectors sit + # behind (`shodan`, `hibp`, `netlas`, `netlas-endpoints`, + # `shodan-endpoints`). Without that wildcard `docker compose pull` fetched + # only the six images of the active profiles and said nothing, and the + # guard below reported four published images as missing — how this job + # failed on its first real run, at 0.2.4. Reading the set from Compose also + # means a new connector is covered by the Compose file alone, rather than + # by a second list in this workflow that has to be kept in step. + run: | + set -euo pipefail + docker compose --profile '*' config --images \ + | sort -u \ + | xargs -r -n1 docker pull - name: Refuse an image that was built instead of pulled # The smoke has to test the artifact and not the source tree. A locally @@ -339,16 +357,7 @@ jobs: version = sys.argv[1] prefix = os.environ["OPENDRP_IMAGE_PREFIX"] - images = [ - "backend", - "frontend", - "connector-dnstwist", - "connector-shodan", - "connector-hibp", - "connector-dns-security", - "connector-intelligence", - "connector-exposure-endpoints", - ] + images = os.environ["RELEASE_IMAGES"].split() problems = [] for image in images: reference = f"{prefix}/{image}:{version}" diff --git a/CHANGELOG.md b/CHANGELOG.md index 83b9201..960ad2b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,30 @@ with it. `/api/v1/ready` to be ready and the frontend to serve the SPA mount point. The release notes are published only after it passes. +### Fixed + +- The connector images upgrade their Debian base layer before installing their + own packages. All six build `FROM python:3.12-slim`, and the snapshot that + image carried at 0.2.4 shipped a `zlib1g` (`1:1.3.dfsg+really1.3.1-1+b1`) with + two HIGH findings whose fix was already published as `+deb13u1`. The release + workflow fails an image that carries a fixable HIGH, so the first tagged run of + 0.2.4 stopped at the image scan of every connector while backend and frontend + passed — the backend image had upgraded its base layer for exactly this reason + and the connectors never did. The upgrade runs before each image installs its + own packages, so a pinned package such as `dnstwist=0~20250130-1` is still what + apt applies last and a rebuild cannot move it. +- The release smoke job pulls every image the release publishes, not the six + Compose fetches by default. Four of the six connectors sit behind profiles + (`shodan`, `hibp`, `netlas`, `netlas-endpoints`, `shodan-endpoints`), and + `docker compose pull` skips an inactive profile without saying so: the job + fetched six of the eight images, and the digest guard after it — which checks + all eight — then reported four images the release had published as missing. + The pull now takes its list from `docker compose --profile '*' config + --images`, so a new connector is covered by the Compose file alone rather than + by a second list in the workflow, and `scripts/check_release_images.py` fails + CI if the list the guard reads stops matching the build matrix, so the two + copies that failed here cannot drift apart again. + ## [0.2.3] - 2026-11-05 ### Security diff --git a/backend/tests/test_release_image_matrix.py b/backend/tests/test_release_image_matrix.py index 2806407..3025748 100644 --- a/backend/tests/test_release_image_matrix.py +++ b/backend/tests/test_release_image_matrix.py @@ -210,3 +210,75 @@ def test_dockerfile_outside_repository_is_reported_from_config() -> None: assert any("connector-intelligence" in failure and "context/dockerfile" in failure for failure in failures) +def test_smoke_job_pulls_every_image_the_matrix_builds() -> None: + assert _GATE.validate_published_image_list(_release()) == [] + + +def test_smoke_image_list_missing_a_matrix_image_is_reported() -> None: + source = _release().replace( + "connector-dnstwist connector-shodan connector-hibp", + "connector-dnstwist connector-shodan", + ) + + failures = _GATE.validate_published_image_list(source) + + assert any("connector-hibp" in failure and "missing" in failure for failure in failures) + + +def test_smoke_image_list_naming_an_unpublished_image_is_reported() -> None: + source = _release().replace( + "backend frontend connector-dnstwist", + "backend frontend connector-otherwise-unpublished", + ) + + failures = _GATE.validate_published_image_list(source) + + assert any("connector-otherwise-unpublished" in failure for failure in failures) + + +def test_smoke_image_list_repeating_an_image_is_reported() -> None: + source = _release().replace( + "backend frontend connector-dnstwist", + "backend frontend backend connector-dnstwist", + ) + + failures = _GATE.validate_published_image_list(source) + + assert any("repeats" in failure and "backend" in failure for failure in failures) + + +def test_smoke_job_without_the_shared_list_is_reported() -> None: + source = _release().replace(" RELEASE_IMAGES: >-\n", " SOMETHING_ELSE: >-\n") + + failures = _GATE.validate_published_image_list(source) + + assert any("RELEASE_IMAGES" in failure for failure in failures) + + +def test_smoke_job_whose_list_no_step_reads_is_reported() -> None: + source = _release().replace( + 'images = os.environ["RELEASE_IMAGES"].split()', 'images = ["backend"]' + ) + + failures = _GATE.validate_published_image_list(source) + + assert any("no step reads it" in failure for failure in failures) + + +def test_smoke_pull_asks_compose_for_every_profile() -> None: + """Plain `docker compose pull` skips the profiled connectors, which is the + failure this step exists to prevent.""" + smoke = "\n".join(_GATE._job_block(_release(), "smoke-published-images")) + + assert "--profile '*'" in smoke + assert "config --images" in smoke + + +def test_smoke_job_missing_entirely_is_reported() -> None: + source = _release().replace(" smoke-published-images:\n", " start-images:\n") + + failures = _GATE.validate_published_image_list(source) + + assert any("no smoke-published-images job" in failure for failure in failures) + + diff --git a/connectors/dns_security/Dockerfile b/connectors/dns_security/Dockerfile index 710a870..d0a787f 100644 --- a/connectors/dns_security/Dockerfile +++ b/connectors/dns_security/Dockerfile @@ -11,7 +11,11 @@ RUN pip install --no-cache-dir --require-hashes -r /app/requirements.txt # against the system store is the point of the authoritative pass, and `%{certs}` # needs curl 7.88 (the worker records its version and refuses to probe on an older # build rather than reporting hosts as certificate-less). +# The base layer is upgraded before the probe's packages are installed; the +# reason is in `connectors/hibp/Dockerfile`. It keeps whether this image passes +# the release scan from depending on when Docker last rebuilt the base. RUN apt-get update \ + && apt-get upgrade -y --no-install-recommends \ && apt-get install -y --no-install-recommends curl ca-certificates \ && rm -rf /var/lib/apt/lists/* COPY connectors/dns_security/main.py connectors/dns_security/tls_probe.py /app/ diff --git a/connectors/dnstwist/Dockerfile b/connectors/dnstwist/Dockerfile index 51d9c76..1f6d075 100644 --- a/connectors/dnstwist/Dockerfile +++ b/connectors/dnstwist/Dockerfile @@ -14,7 +14,12 @@ FROM python:3.12-slim # for the same domain — a change that would otherwise arrive silently in a rebuild. # The pin is a Debian version; a base-image bump whose archive serves another one # fails the build rather than the scan. +# The base layer is upgraded before the pin is installed (reason: +# `connectors/hibp/Dockerfile`), so the pinned install is the last thing apt +# does here and the upgrade cannot move dnstwist off the release that +# `_EXPECTED_DNSTWIST_VERSION` names. RUN apt-get update \ + && apt-get upgrade -y --no-install-recommends \ && apt-get install -y --no-install-recommends dnstwist=0~20250130-1 python3-dnspython \ && rm -rf /var/lib/apt/lists/* diff --git a/connectors/exposure_endpoints/Dockerfile b/connectors/exposure_endpoints/Dockerfile index 1d36974..d8f9c71 100644 --- a/connectors/exposure_endpoints/Dockerfile +++ b/connectors/exposure_endpoints/Dockerfile @@ -1,4 +1,17 @@ FROM python:3.12-slim + +# The base layer is upgraded, not just the packages this image asks for. The +# official `python:3.12-slim` carries a Debian trixie snapshot, and its +# point-release security fixes reach the archive only after Docker rebuilds that +# image: at 0.2.4 the snapshot shipped a `zlib1g` that Trivy reports as HIGH +# (CVE-2026-76844, CVE-2026-85091) with a fix already published as `+deb13u1`. +# The release workflow fails on a fixable HIGH, so nothing ships until this +# layer moves. `upgrade`, not `dist-upgrade`: only packages the base image +# already installed are moved, and no new package is added. +RUN apt-get update \ + && apt-get upgrade -y --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* + WORKDIR /app COPY connectors/base /opt/sdk RUN pip install --no-cache-dir /opt/sdk diff --git a/connectors/hibp/Dockerfile b/connectors/hibp/Dockerfile index 07935cd..0693e30 100644 --- a/connectors/hibp/Dockerfile +++ b/connectors/hibp/Dockerfile @@ -1,5 +1,17 @@ FROM python:3.12-slim +# The base layer is upgraded, not just the packages this image asks for. The +# official `python:3.12-slim` carries a Debian trixie snapshot, and its +# point-release security fixes reach the archive only after Docker rebuilds that +# image: at 0.2.4 the snapshot shipped a `zlib1g` that Trivy reports as HIGH +# (CVE-2026-76844, CVE-2026-85091) with a fix already published as `+deb13u1`. +# The release workflow fails on a fixable HIGH, so nothing ships until this +# layer moves. `upgrade`, not `dist-upgrade`: only packages the base image +# already installed are moved, and no new package is added. +RUN apt-get update \ + && apt-get upgrade -y --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* + WORKDIR /app # Shared SDK first (cached layer). diff --git a/connectors/intelligence/Dockerfile b/connectors/intelligence/Dockerfile index 34a386d..dab1a8b 100644 --- a/connectors/intelligence/Dockerfile +++ b/connectors/intelligence/Dockerfile @@ -1,4 +1,17 @@ FROM python:3.12-slim + +# The base layer is upgraded, not just the packages this image asks for. The +# official `python:3.12-slim` carries a Debian trixie snapshot, and its +# point-release security fixes reach the archive only after Docker rebuilds that +# image: at 0.2.4 the snapshot shipped a `zlib1g` that Trivy reports as HIGH +# (CVE-2026-76844, CVE-2026-85091) with a fix already published as `+deb13u1`. +# The release workflow fails on a fixable HIGH, so nothing ships until this +# layer moves. `upgrade`, not `dist-upgrade`: only packages the base image +# already installed are moved, and no new package is added. +RUN apt-get update \ + && apt-get upgrade -y --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* + WORKDIR /app COPY connectors/base /opt/sdk RUN pip install --no-cache-dir /opt/sdk diff --git a/connectors/shodan/Dockerfile b/connectors/shodan/Dockerfile index 1de0c4f..43e84f1 100644 --- a/connectors/shodan/Dockerfile +++ b/connectors/shodan/Dockerfile @@ -1,5 +1,17 @@ FROM python:3.12-slim +# The base layer is upgraded, not just the packages this image asks for. The +# official `python:3.12-slim` carries a Debian trixie snapshot, and its +# point-release security fixes reach the archive only after Docker rebuilds that +# image: at 0.2.4 the snapshot shipped a `zlib1g` that Trivy reports as HIGH +# (CVE-2026-76844, CVE-2026-85091) with a fix already published as `+deb13u1`. +# The release workflow fails on a fixable HIGH, so nothing ships until this +# layer moves. `upgrade`, not `dist-upgrade`: only packages the base image +# already installed are moved, and no new package is added. +RUN apt-get update \ + && apt-get upgrade -y --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* + WORKDIR /app # Shared SDK first (cached layer). diff --git a/scripts/check_release_images.py b/scripts/check_release_images.py index 6dc410a..eab490f 100644 --- a/scripts/check_release_images.py +++ b/scripts/check_release_images.py @@ -84,6 +84,51 @@ def finish() -> None: return entries, failures +_FOLDED_SCALAR = {">", ">-", ">+", "|", "|-", "|+"} + + +def _job_block(source: str, job: str) -> list[str]: + """The lines of one job, from its `key:` line up to the next job.""" + block: list[str] = [] + in_job = False + for line in source.splitlines(): + stripped = line.strip() + indent = len(line) - len(line.lstrip()) + if indent == 2 and stripped.endswith(":") and not stripped.startswith("#"): + in_job = stripped[:-1] == job + continue + if in_job: + block.append(line) + return block + + +def _job_env_list(block: list[str], key: str) -> tuple[list[str], bool]: + """Read a whitespace-separated list from a job's `env:` block.""" + values: list[str] = [] + found = False + in_env = False + collecting = False + for line in block: + stripped = line.strip() + indent = len(line) - len(line.lstrip()) + if collecting: + if stripped and indent > 6 and not stripped.startswith("#"): + values.extend(stripped.split()) + continue + collecting = False + if indent == 4 and stripped and not stripped.startswith("#"): + in_env = stripped == "env:" + continue + if in_env and indent == 6 and stripped.startswith(f"{key}:"): + found = True + value = stripped.split(":", 1)[1].strip() + if value in _FOLDED_SCALAR: + collecting = True + else: + values = value.split() + return values, found + + def _repo_path(path: str, root: str | Path = ROOT) -> str: """Normalize POSIX or Windows paths to repo-relative POSIX form.""" root_path = str(root) @@ -214,6 +259,56 @@ def validate_release_coverage(release_source: str, compose_config: dict) -> list return failures +def validate_published_image_list(release_source: str) -> list[str]: + """Fail when the smoke job's image list drifts from the build matrix. + + The smoke job pulls the images it starts by name, and the digest guard after + it reads the same list. Both used to keep their own copy of the matrix's + images, which is how the job's first real run at 0.2.4 asked the runner for + four images nobody had pulled. The matrix stays the authority; this checks + that the list the job iterates is exactly the matrix, with nothing missing + and nothing invented. + """ + matrix, failures = _release_matrix(release_source) + block = _job_block(release_source, "smoke-published-images") + if not block: + failures.append( + "release workflow has no smoke-published-images job, so nothing pulls " + "the images the release publishes" + ) + return failures + listed, found = _job_env_list(block, "RELEASE_IMAGES") + if not found or not listed: + failures.append("the smoke job names no images: RELEASE_IMAGES is missing or empty") + return failures + duplicates = sorted({image for image in listed if listed.count(image) > 1}) + if duplicates: + failures.append("RELEASE_IMAGES repeats images: " + ", ".join(duplicates)) + unknown = sorted(set(listed) - set(matrix)) + if unknown: + failures.append( + "RELEASE_IMAGES names images the build matrix does not produce: " + + ", ".join(unknown) + ) + missing = sorted(set(matrix) - set(listed)) + if missing: + failures.append( + "RELEASE_IMAGES is missing images the build matrix publishes: " + + ", ".join(missing) + ) + uses = [ + line + for line in block + if "RELEASE_IMAGES" in line and not line.strip().startswith("RELEASE_IMAGES:") + ] + if not uses: + failures.append( + "RELEASE_IMAGES is declared but no step reads it, so the pull is not " + "the list this gate checked" + ) + return failures + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("compose_json", type=Path, help="resolved `docker compose config --format json` output") @@ -225,12 +320,16 @@ def main(argv: list[str] | None = None) -> int: print(f"release image gate: FAILED — {exc}") return 2 failures = validate_release_coverage(release, compose) + failures += validate_published_image_list(release) if failures: print("release image gate: FAILED") for failure in failures: print(f" {failure}") return 1 - print("release image gate: OK — every resolved Compose image has a matching release build") + print( + "release image gate: OK — every resolved Compose image has a matching " + "release build, and the smoke job pulls exactly that set" + ) return 0