Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 23 additions & 14 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -277,6 +277,14 @@ jobs:
# replaces reached healthy in about two minutes; the bound is for the step
# that stops returning, not for a healthy run.
timeout-minutes: 25
env:
# The images the release matrix publishes, named once for the whole job: the
# pull below and the digest guard after it read this list, so a connector
# added to the matrix is one edit here rather than two that can drift. It is
# the matrix that is authoritative; this is the copy the job can iterate.
RELEASE_IMAGES: >-
backend frontend connector-dnstwist connector-shodan connector-hibp
connector-dns-security connector-intelligence connector-exposure-endpoints
permissions:
contents: read
# The images are private: reading them back is the whole exercise, and the
Expand Down Expand Up @@ -321,10 +329,20 @@ jobs:
password: ${{ secrets.GITHUB_TOKEN }}

- name: Pull every image the release published
# Every image the matrix names, including the connectors this job does not
# start: a tag that resolves to a missing manifest is a release an
# operator cannot install, and `pull` is the cheapest place to find out.
run: docker compose pull
# Compose is the list: `config --images` resolves the image of every
# service, and `--profile '*'` turns on the profiles the connectors sit
# behind (`shodan`, `hibp`, `netlas`, `netlas-endpoints`,
# `shodan-endpoints`). Without that wildcard `docker compose pull` fetched
# only the six images of the active profiles and said nothing, and the
# guard below reported four published images as missing — how this job
# failed on its first real run, at 0.2.4. Reading the set from Compose also
# means a new connector is covered by the Compose file alone, rather than
# by a second list in this workflow that has to be kept in step.
run: |
set -euo pipefail
docker compose --profile '*' config --images \
| sort -u \
| xargs -r -n1 docker pull

- name: Refuse an image that was built instead of pulled
# The smoke has to test the artifact and not the source tree. A locally
Expand All @@ -339,16 +357,7 @@ jobs:

version = sys.argv[1]
prefix = os.environ["OPENDRP_IMAGE_PREFIX"]
images = [
"backend",
"frontend",
"connector-dnstwist",
"connector-shodan",
"connector-hibp",
"connector-dns-security",
"connector-intelligence",
"connector-exposure-endpoints",
]
images = os.environ["RELEASE_IMAGES"].split()
problems = []
for image in images:
reference = f"{prefix}/{image}:{version}"
Expand Down
24 changes: 24 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,30 @@ with it.
`/api/v1/ready` to be ready and the frontend to serve the SPA mount point. The
release notes are published only after it passes.

### Fixed

- The connector images upgrade their Debian base layer before installing their
own packages. All six build `FROM python:3.12-slim`, and the snapshot that
image carried at 0.2.4 shipped a `zlib1g` (`1:1.3.dfsg+really1.3.1-1+b1`) with
two HIGH findings whose fix was already published as `+deb13u1`. The release
workflow fails an image that carries a fixable HIGH, so the first tagged run of
0.2.4 stopped at the image scan of every connector while backend and frontend
passed — the backend image had upgraded its base layer for exactly this reason
and the connectors never did. The upgrade runs before each image installs its
own packages, so a pinned package such as `dnstwist=0~20250130-1` is still what
apt applies last and a rebuild cannot move it.
- The release smoke job pulls every image the release publishes, not the six
Compose fetches by default. Four of the six connectors sit behind profiles
(`shodan`, `hibp`, `netlas`, `netlas-endpoints`, `shodan-endpoints`), and
`docker compose pull` skips an inactive profile without saying so: the job
fetched six of the eight images, and the digest guard after it — which checks
all eight — then reported four images the release had published as missing.
The pull now takes its list from `docker compose --profile '*' config
--images`, so a new connector is covered by the Compose file alone rather than
by a second list in the workflow, and `scripts/check_release_images.py` fails
CI if the list the guard reads stops matching the build matrix, so the two
copies that failed here cannot drift apart again.

## [0.2.3] - 2026-11-05

### Security
Expand Down
72 changes: 72 additions & 0 deletions backend/tests/test_release_image_matrix.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,3 +210,75 @@ def test_dockerfile_outside_repository_is_reported_from_config() -> None:
assert any("connector-intelligence" in failure and "context/dockerfile" in failure for failure in failures)


def test_smoke_job_pulls_every_image_the_matrix_builds() -> None:
assert _GATE.validate_published_image_list(_release()) == []


def test_smoke_image_list_missing_a_matrix_image_is_reported() -> None:
source = _release().replace(
"connector-dnstwist connector-shodan connector-hibp",
"connector-dnstwist connector-shodan",
)

failures = _GATE.validate_published_image_list(source)

assert any("connector-hibp" in failure and "missing" in failure for failure in failures)


def test_smoke_image_list_naming_an_unpublished_image_is_reported() -> None:
source = _release().replace(
"backend frontend connector-dnstwist",
"backend frontend connector-otherwise-unpublished",
)

failures = _GATE.validate_published_image_list(source)

assert any("connector-otherwise-unpublished" in failure for failure in failures)


def test_smoke_image_list_repeating_an_image_is_reported() -> None:
source = _release().replace(
"backend frontend connector-dnstwist",
"backend frontend backend connector-dnstwist",
)

failures = _GATE.validate_published_image_list(source)

assert any("repeats" in failure and "backend" in failure for failure in failures)


def test_smoke_job_without_the_shared_list_is_reported() -> None:
source = _release().replace(" RELEASE_IMAGES: >-\n", " SOMETHING_ELSE: >-\n")

failures = _GATE.validate_published_image_list(source)

assert any("RELEASE_IMAGES" in failure for failure in failures)


def test_smoke_job_whose_list_no_step_reads_is_reported() -> None:
source = _release().replace(
'images = os.environ["RELEASE_IMAGES"].split()', 'images = ["backend"]'
)

failures = _GATE.validate_published_image_list(source)

assert any("no step reads it" in failure for failure in failures)


def test_smoke_pull_asks_compose_for_every_profile() -> None:
"""Plain `docker compose pull` skips the profiled connectors, which is the
failure this step exists to prevent."""
smoke = "\n".join(_GATE._job_block(_release(), "smoke-published-images"))

assert "--profile '*'" in smoke
assert "config --images" in smoke


def test_smoke_job_missing_entirely_is_reported() -> None:
source = _release().replace(" smoke-published-images:\n", " start-images:\n")

failures = _GATE.validate_published_image_list(source)

assert any("no smoke-published-images job" in failure for failure in failures)


4 changes: 4 additions & 0 deletions connectors/dns_security/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,11 @@ RUN pip install --no-cache-dir --require-hashes -r /app/requirements.txt
# against the system store is the point of the authoritative pass, and `%{certs}`
# needs curl 7.88 (the worker records its version and refuses to probe on an older
# build rather than reporting hosts as certificate-less).
# The base layer is upgraded before the probe's packages are installed; the
# reason is in `connectors/hibp/Dockerfile`. It keeps whether this image passes
# the release scan from depending on when Docker last rebuilt the base.
RUN apt-get update \
&& apt-get upgrade -y --no-install-recommends \
&& apt-get install -y --no-install-recommends curl ca-certificates \
&& rm -rf /var/lib/apt/lists/*
COPY connectors/dns_security/main.py connectors/dns_security/tls_probe.py /app/
Expand Down
5 changes: 5 additions & 0 deletions connectors/dnstwist/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,12 @@ FROM python:3.12-slim
# for the same domain — a change that would otherwise arrive silently in a rebuild.
# The pin is a Debian version; a base-image bump whose archive serves another one
# fails the build rather than the scan.
# The base layer is upgraded before the pin is installed (reason:
# `connectors/hibp/Dockerfile`), so the pinned install is the last thing apt
# does here and the upgrade cannot move dnstwist off the release that
# `_EXPECTED_DNSTWIST_VERSION` names.
RUN apt-get update \
&& apt-get upgrade -y --no-install-recommends \
&& apt-get install -y --no-install-recommends dnstwist=0~20250130-1 python3-dnspython \
&& rm -rf /var/lib/apt/lists/*

Expand Down
13 changes: 13 additions & 0 deletions connectors/exposure_endpoints/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,17 @@
FROM python:3.12-slim

# The base layer is upgraded, not just the packages this image asks for. The
# official `python:3.12-slim` carries a Debian trixie snapshot, and its
# point-release security fixes reach the archive only after Docker rebuilds that
# image: at 0.2.4 the snapshot shipped a `zlib1g` that Trivy reports as HIGH
# (CVE-2026-76844, CVE-2026-85091) with a fix already published as `+deb13u1`.
# The release workflow fails on a fixable HIGH, so nothing ships until this
# layer moves. `upgrade`, not `dist-upgrade`: only packages the base image
# already installed are moved, and no new package is added.
RUN apt-get update \
&& apt-get upgrade -y --no-install-recommends \
&& rm -rf /var/lib/apt/lists/*

WORKDIR /app
COPY connectors/base /opt/sdk
RUN pip install --no-cache-dir /opt/sdk
Expand Down
12 changes: 12 additions & 0 deletions connectors/hibp/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,17 @@
FROM python:3.12-slim

# The base layer is upgraded, not just the packages this image asks for. The
# official `python:3.12-slim` carries a Debian trixie snapshot, and its
# point-release security fixes reach the archive only after Docker rebuilds that
# image: at 0.2.4 the snapshot shipped a `zlib1g` that Trivy reports as HIGH
# (CVE-2026-76844, CVE-2026-85091) with a fix already published as `+deb13u1`.
# The release workflow fails on a fixable HIGH, so nothing ships until this
# layer moves. `upgrade`, not `dist-upgrade`: only packages the base image
# already installed are moved, and no new package is added.
RUN apt-get update \
&& apt-get upgrade -y --no-install-recommends \
&& rm -rf /var/lib/apt/lists/*

WORKDIR /app

# Shared SDK first (cached layer).
Expand Down
13 changes: 13 additions & 0 deletions connectors/intelligence/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,17 @@
FROM python:3.12-slim

# The base layer is upgraded, not just the packages this image asks for. The
# official `python:3.12-slim` carries a Debian trixie snapshot, and its
# point-release security fixes reach the archive only after Docker rebuilds that
# image: at 0.2.4 the snapshot shipped a `zlib1g` that Trivy reports as HIGH
# (CVE-2026-76844, CVE-2026-85091) with a fix already published as `+deb13u1`.
# The release workflow fails on a fixable HIGH, so nothing ships until this
# layer moves. `upgrade`, not `dist-upgrade`: only packages the base image
# already installed are moved, and no new package is added.
RUN apt-get update \
&& apt-get upgrade -y --no-install-recommends \
&& rm -rf /var/lib/apt/lists/*

WORKDIR /app
COPY connectors/base /opt/sdk
RUN pip install --no-cache-dir /opt/sdk
Expand Down
12 changes: 12 additions & 0 deletions connectors/shodan/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,17 @@
FROM python:3.12-slim

# The base layer is upgraded, not just the packages this image asks for. The
# official `python:3.12-slim` carries a Debian trixie snapshot, and its
# point-release security fixes reach the archive only after Docker rebuilds that
# image: at 0.2.4 the snapshot shipped a `zlib1g` that Trivy reports as HIGH
# (CVE-2026-76844, CVE-2026-85091) with a fix already published as `+deb13u1`.
# The release workflow fails on a fixable HIGH, so nothing ships until this
# layer moves. `upgrade`, not `dist-upgrade`: only packages the base image
# already installed are moved, and no new package is added.
RUN apt-get update \
&& apt-get upgrade -y --no-install-recommends \
&& rm -rf /var/lib/apt/lists/*

WORKDIR /app

# Shared SDK first (cached layer).
Expand Down
101 changes: 100 additions & 1 deletion scripts/check_release_images.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,51 @@ def finish() -> None:
return entries, failures


_FOLDED_SCALAR = {">", ">-", ">+", "|", "|-", "|+"}


def _job_block(source: str, job: str) -> list[str]:
"""The lines of one job, from its `key:` line up to the next job."""
block: list[str] = []
in_job = False
for line in source.splitlines():
stripped = line.strip()
indent = len(line) - len(line.lstrip())
if indent == 2 and stripped.endswith(":") and not stripped.startswith("#"):
in_job = stripped[:-1] == job
continue
if in_job:
block.append(line)
return block


def _job_env_list(block: list[str], key: str) -> tuple[list[str], bool]:
"""Read a whitespace-separated list from a job's `env:` block."""
values: list[str] = []
found = False
in_env = False
collecting = False
for line in block:
stripped = line.strip()
indent = len(line) - len(line.lstrip())
if collecting:
if stripped and indent > 6 and not stripped.startswith("#"):
values.extend(stripped.split())
continue
collecting = False
if indent == 4 and stripped and not stripped.startswith("#"):
in_env = stripped == "env:"
continue
if in_env and indent == 6 and stripped.startswith(f"{key}:"):
found = True
value = stripped.split(":", 1)[1].strip()
if value in _FOLDED_SCALAR:
collecting = True
else:
values = value.split()
return values, found


def _repo_path(path: str, root: str | Path = ROOT) -> str:
"""Normalize POSIX or Windows paths to repo-relative POSIX form."""
root_path = str(root)
Expand Down Expand Up @@ -214,6 +259,56 @@ def validate_release_coverage(release_source: str, compose_config: dict) -> list
return failures


def validate_published_image_list(release_source: str) -> list[str]:
"""Fail when the smoke job's image list drifts from the build matrix.

The smoke job pulls the images it starts by name, and the digest guard after
it reads the same list. Both used to keep their own copy of the matrix's
images, which is how the job's first real run at 0.2.4 asked the runner for
four images nobody had pulled. The matrix stays the authority; this checks
that the list the job iterates is exactly the matrix, with nothing missing
and nothing invented.
"""
matrix, failures = _release_matrix(release_source)
block = _job_block(release_source, "smoke-published-images")
if not block:
failures.append(
"release workflow has no smoke-published-images job, so nothing pulls "
"the images the release publishes"
)
return failures
listed, found = _job_env_list(block, "RELEASE_IMAGES")
if not found or not listed:
failures.append("the smoke job names no images: RELEASE_IMAGES is missing or empty")
return failures
duplicates = sorted({image for image in listed if listed.count(image) > 1})
if duplicates:
failures.append("RELEASE_IMAGES repeats images: " + ", ".join(duplicates))
unknown = sorted(set(listed) - set(matrix))
if unknown:
failures.append(
"RELEASE_IMAGES names images the build matrix does not produce: "
+ ", ".join(unknown)
)
missing = sorted(set(matrix) - set(listed))
if missing:
failures.append(
"RELEASE_IMAGES is missing images the build matrix publishes: "
+ ", ".join(missing)
)
uses = [
line
for line in block
if "RELEASE_IMAGES" in line and not line.strip().startswith("RELEASE_IMAGES:")
]
if not uses:
failures.append(
"RELEASE_IMAGES is declared but no step reads it, so the pull is not "
"the list this gate checked"
)
return failures


def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("compose_json", type=Path, help="resolved `docker compose config --format json` output")
Expand All @@ -225,12 +320,16 @@ def main(argv: list[str] | None = None) -> int:
print(f"release image gate: FAILED — {exc}")
return 2
failures = validate_release_coverage(release, compose)
failures += validate_published_image_list(release)
if failures:
print("release image gate: FAILED")
for failure in failures:
print(f" {failure}")
return 1
print("release image gate: OK — every resolved Compose image has a matching release build")
print(
"release image gate: OK — every resolved Compose image has a matching "
"release build, and the smoke job pulls exactly that set"
)
return 0


Expand Down
Loading