From a0219bf7a53c40ab68a935bda04f9adc8ce8f439 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 06:14:36 +1100 Subject: [PATCH 1/3] fix(compose): give the engine the image digests the stack runs The engine names the stack by matching the digests it is told about against the published stack manifests. Compose passed only the build's stack name. This names the wrong stack when two stacks share the same app digests. DFE_ENGINE_STACK_IMAGES lists the ten DFE app images Compose runs. It uses the same interpolations as the image lines, so the list matches those images. DFE_ENGINE_STACK_VERSION stays as the offline fallback. check_compose checks that the list holds exactly those images, each by digest. --- docker-compose.yml | 1 + docs/configuration.md | 1 + scripts/check_compose.py | 82 +++++++++++++++++++++++++ scripts/tests/test_check_compose.py | 92 +++++++++++++++++++++++++++++ 4 files changed, 176 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 94f18bd4..d6c7e558 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -426,6 +426,7 @@ services: DFE_ENGINE_KAFKA_BOOTSTRAP_SERVERS: ${DFE_KAFKA_BOOTSTRAP_SERVERS:-} DFE_ENGINE_PROFILE: ${DFE_STACK_PROFILE:-docker-slim} DFE_ENGINE_SECRETS_PATH: /app/config/.secrets + DFE_ENGINE_STACK_IMAGES: ${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-archiver:${DFE_ARCHIVER_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-engine:${DFE_ENGINE_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-fetcher:${DFE_FETCHER_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-hyperdx:${DFE_HYPERDX_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-loader:${DFE_LOADER_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-receiver:${DFE_RECEIVER_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-transform-elastic:${DFE_TRANSFORM_ELASTIC_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-transform-vector:${DFE_TRANSFORM_VECTOR_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-transform-vrl:${DFE_TRANSFORM_VRL_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-ui:${DFE_UI_VERSION} DFE_ENGINE_STACK_VERSION: ${DFE_STACK_VERSION:-} DFE_ENGINE_TRANSPORT_BUS_PRESENT: ${DFE_TRANSPORT_BUS_PRESENT:-false} DFE_ENGINE_TRANSPORT_DEFAULT: ${DFE_TRANSPORT_DEFAULT:-direct} diff --git a/docs/configuration.md b/docs/configuration.md index 803a639a..ea79df2e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -358,6 +358,7 @@ Compose sets these on the engine itself: | `DFE_ENGINE_HYPERDX_BASE_URL`, `DFE_ENGINE_HYPERDX_ENABLED` | Follow the footprint, like the dfe-hyperdx containers | | `DFE_ENGINE_PROFILE` | The tier name, which seeds its default app instances | | `DFE_ENGINE_SECRETS_PATH` | On the config volume, so the JWT key survives a recreate | +| `DFE_ENGINE_STACK_IMAGES` | Every DFE app image the stack runs, by digest. The stack the console shows is the published one these match, with `DFE_STACK_VERSION` as the offline fallback | | `DFE_ENGINE_STACK_VERSION`, `DFE_ENGINE_UI_VERSION` | The certified stack and the console version, for the console footer. The second is the UI pin without its digest | | `DFE_ENGINE_TRANSPORT_BUS_PRESENT`, `DFE_ENGINE_TRANSPORT_DEFAULT` | From the profile. Their defaults are slim-grpc's | diff --git a/scripts/check_compose.py b/scripts/check_compose.py index 8f8f3d7f..692916e1 100644 --- a/scripts/check_compose.py +++ b/scripts/check_compose.py @@ -77,11 +77,14 @@ - Every path a pinned image declares as a VOLUME must be mounted from a named volume or a bind, and wherever the daemon holds the image, `_IMAGE_VOLUMES` must list exactly the paths it declares. See `_image_volume_drift`. - No committed service may carry a name a generated per-source instance can take. See `_instance_name_collisions`. +- The engine's DFE_ENGINE_STACK_IMAGES must list every DFE app image the stack runs, + each pinned by digest, and no other image. See `_stack_images_mismatches`. - Every service dfe-engine renders, and every instance the generator declares, must read the custom env file the engine writes for it, last; no service may read one the engine does not write. See `_custom_env_mismatches`. """ +import hashlib import json import os import re @@ -1232,6 +1235,75 @@ def _version_check_failures(*, env: dict[str, str]) -> tuple[list[str], int]: return failures, made +# The engine names the stack it runs from these, so each entry is an app image a stack manifest lists +_STACK_IMAGES_KEY = "DFE_ENGINE_STACK_IMAGES" +_STACK_IMAGE_PREFIX = "dfe-" +_DIGEST_REF = re.compile(r"@sha256:[0-9a-f]{64}$") + + +def _image_name(*, reference: str) -> str: + """Return the name of `registry/path/name[:tag][@sha256:digest]`.""" + return reference.split("@", 1)[0].rsplit("/", 1)[-1].split(":", 1)[0] + + +def _stack_images_mismatches(*, config: dict) -> list[str]: + """Return one message per way the engine's DFE_ENGINE_STACK_IMAGES differs from the DFE app images the model runs. + + Every service counts, so the hunt runner's copy of the engine image adds no second entry and no image a service runs is left out. An entry with no digest cannot match a stack manifest, and an image that is not a DFE app matches none. + """ + services = config.get("services", {}) + engine = services.get(_ENGINE_SERVICE) + if engine is None: + return [f"{_ENGINE_SERVICE}: not in the stack"] + value = str((engine.get("environment") or {}).get(_STACK_IMAGES_KEY) or "") + listed = [entry.strip() for entry in value.split(",") if entry.strip()] + running = { + str(service["image"]) + for service in services.values() + if _image_name(reference=str(service.get("image", ""))).startswith( + _STACK_IMAGE_PREFIX + ) + } + failures = [ + f"{_STACK_IMAGES_KEY} lists {entry!r}, which carries no sha256 digest" + for entry in listed + if not _DIGEST_REF.search(entry) + ] + failures.extend( + f"{_STACK_IMAGES_KEY} lists {entry!r}, which no service runs as a DFE app" + for entry in sorted(set(listed) - running) + ) + failures.extend( + f"{_STACK_IMAGES_KEY} leaves out {image!r}, which a service runs" + for image in sorted(running - set(listed)) + ) + failures.extend( + f"{_STACK_IMAGES_KEY} lists {entry!r} {listed.count(entry)} times" + for entry in sorted({entry for entry in listed if listed.count(entry) > 1}) + ) + return failures + + +def _stack_images_failures(*, env: dict[str, str]) -> tuple[list[str], int]: + """Return (messages, assertions made) for DFE_ENGINE_STACK_IMAGES on the registry path. + + Each mandatory key gets a digest-shaped value of its own, so a pin that never reaches the list shows as a missing entry. The checkout's .env is not loaded. + """ + pinned = { + **env, + **{ + name: f"v0.0.0@sha256:{hashlib.sha256(name.encode()).hexdigest()}" + for name in _required_compose_vars() + }, + } + config = _config_json( + env=pinned, files=[COMPOSE_FILE.name], env_file=_empty_env_file() + ) + if config is None: + return (["docker-compose.yml did not resolve with every pin digested"], 0) + return _stack_images_mismatches(config=config), 1 + + def _proxy_tls_mismatches(*, config: dict, tls: dict[str, str] | None) -> list[str]: """Return one message per way the proxies, the network and the engine differ from the console TLS side they are on. @@ -2111,6 +2183,16 @@ def _checks() -> int: f"to true unless .env says otherwise ({version_made} assertions)" ) + stack_images_failures, stack_images_made = _stack_images_failures(env=env) + for message in stack_images_failures: + _print(msg=f"FAIL {message}") + if stack_images_failures: + return 1 + _print( + msg=f"{_ENGINE_SERVICE} names the stack from every DFE app image the stack " + f"runs, each by digest, and no other ({stack_images_made} assertion)" + ) + tls_failures, tls_made = _proxy_tls_failures(env=env) for message in tls_failures: _print(msg=f"FAIL {message}") diff --git a/scripts/tests/test_check_compose.py b/scripts/tests/test_check_compose.py index 1404f7c8..19c7c7cf 100644 --- a/scripts/tests/test_check_compose.py +++ b/scripts/tests/test_check_compose.py @@ -817,3 +817,95 @@ def test_a_dfe_app_gets_the_stop_grace_its_contract_asks_for( resolved_model, service, grace ): assert resolved_model["services"][service]["stop_grace_period"] == grace + + +def _digest(name: str) -> str: + return f"sha256:{name.encode().hex().ljust(64, '0')[:64]}" + + +def _stack_model(*, listed: list[str], running: list[str]) -> dict: + """A resolved model whose services run `running` and whose engine lists `listed`.""" + services = { + f"service-{index}": {"image": image} for index, image in enumerate(running) + } + services["dfe-engine"] = { + "image": "ghcr.io/hyperi-io/dfe-engine:v1@" + _digest("engine"), + "environment": {"DFE_ENGINE_STACK_IMAGES": ",".join(listed)}, + } + return {"services": services} + + +_ENGINE_REF = "ghcr.io/hyperi-io/dfe-engine:v1@" + _digest("engine") +_LOADER_REF = "ghcr.io/hyperi-io/dfe-loader:v2@" + _digest("loader") +_BARE_LOADER_REF = "ghcr.io/hyperi-io/dfe-loader:v2" +_CLICKHOUSE_REF = "clickhouse/clickhouse-server:26.3@" + _digest("clickhouse") + + +@pytest.mark.parametrize( + ("listed", "running", "expected"), + [ + pytest.param( + [_ENGINE_REF, _LOADER_REF], [_LOADER_REF, _ENGINE_REF], [], id="exact" + ), + pytest.param( + [_ENGINE_REF, _LOADER_REF], + [_LOADER_REF, _ENGINE_REF, _CLICKHOUSE_REF], + [], + id="a_third_party_image_is_not_listed", + ), + pytest.param( + [_ENGINE_REF], + [_LOADER_REF], + ["leaves out"], + id="a_running_app_left_out", + ), + pytest.param( + [_ENGINE_REF, _LOADER_REF, _CLICKHOUSE_REF], + [_LOADER_REF], + ["no service runs"], + id="a_third_party_image_listed", + ), + pytest.param( + [_ENGINE_REF, _BARE_LOADER_REF], + [_BARE_LOADER_REF], + ["no sha256 digest"], + id="an_entry_without_a_digest", + ), + pytest.param( + [_ENGINE_REF, _LOADER_REF, _LOADER_REF], + [_LOADER_REF], + ["2 times"], + id="an_entry_twice", + ), + pytest.param( + [], [_LOADER_REF], ["leaves out", "leaves out"], id="an_empty_list" + ), + ], +) +def test_the_engine_lists_the_dfe_app_images_the_stack_runs( + listed: list[str], running: list[str], expected: list[str] +) -> None: + model = _stack_model(listed=listed, running=running) + + failures = check_compose._stack_images_mismatches(config=model) + + assert len(failures) == len(expected), failures + for fragment, failure in zip(expected, failures, strict=True): + assert fragment in failure + + +def test_a_stack_with_no_engine_fails_rather_than_passing() -> None: + failures = check_compose._stack_images_mismatches(config={"services": {}}) + + assert failures == ["dfe-engine: not in the stack"] + + +def test_the_committed_compose_lists_every_dfe_app_image_it_runs_by_digest() -> None: + if shutil.which("docker") is None: + pytest.skip("`docker compose config` renders the model this asserts on") + env, _ = check_compose._check_env() + + failures, made = check_compose._stack_images_failures(env=env) + + assert failures == [] + assert made == 1 From 087549fbea2c977d88369bb5c53d11492ee39234 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 06:21:03 +1100 Subject: [PATCH 2/3] fix(compose): leave the engine's stack images empty on a local build The dev overlay from make dev and LOCAL= repoints images to :local. The engine still received the registry digests in DFE_ENGINE_STACK_IMAGES, so it reported a published stack while running local builds. When the overlay repoints any image, it clears DFE_ENGINE_STACK_IMAGES on dfe-engine. The engine then reports DFE_ENGINE_STACK_VERSION. The overlay is unchanged when nothing is built. --- scripts/build_dev_images.py | 14 ++++++++-- scripts/check_compose.py | 9 +++++++ scripts/tests/test_build_dev_images.py | 37 ++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 2 deletions(-) diff --git a/scripts/build_dev_images.py b/scripts/build_dev_images.py index 37d6991a..d5f4fe8e 100644 --- a/scripts/build_dev_images.py +++ b/scripts/build_dev_images.py @@ -67,6 +67,9 @@ ), "hyperdx-documentdb-tls": ("hyperdx-documentdb",), } +# The engine names its stack from the registry digests this lists, which no local build carries +ENGINE_SERVICE = "dfe-engine" +STACK_IMAGES_KEY = "DFE_ENGINE_STACK_IMAGES" RUST_COMPONENTS = [ "dfe-archiver", "dfe-fetcher", @@ -374,9 +377,16 @@ def overlay_text(components: list[str]) -> str: services = sorted(local_image_services(components=components).items()) if not (services): return f"{header}\nservices: {{}}\n" + local = dict(services) lines = [header, "services:"] - for service, image in services: - lines += [f" {service}:", f" image: {image}", ""] + for service in sorted({*local, ENGINE_SERVICE}): + lines.append(f" {service}:") + if service in local: + lines.append(f" image: {local[service]}") + if service == ENGINE_SERVICE: + # A local build is no published stack, so the engine reports DFE_ENGINE_STACK_VERSION instead + lines += [" environment:", f' {STACK_IMAGES_KEY}: ""'] + lines.append("") return "\n".join(lines).rstrip("\n") + "\n" diff --git a/scripts/check_compose.py b/scripts/check_compose.py index 692916e1..3c886315 100644 --- a/scripts/check_compose.py +++ b/scripts/check_compose.py @@ -1745,6 +1745,15 @@ def _local_overlay_failures(*, env: dict[str, str]) -> tuple[list[str], int]: f"{_LOCAL_RENDER.name}: {service} runs {got}, not {image} -- a " "LOCAL build leaves a follower on the registry image" ) + made += 1 + stack_images = (services.get(_ENGINE_SERVICE, {}).get("environment") or {}).get( + _STACK_IMAGES_KEY + ) + if stack_images != "": + failures.append( + f"{_LOCAL_RENDER.name}: {_ENGINE_SERVICE} gets {_STACK_IMAGES_KEY}=" + f"{stack_images!r}, not empty -- a LOCAL build is no published stack" + ) for service, service_config in sorted(services.items()): if service in want: continue diff --git a/scripts/tests/test_build_dev_images.py b/scripts/tests/test_build_dev_images.py index 23d8b700..c23249a1 100644 --- a/scripts/tests/test_build_dev_images.py +++ b/scripts/tests/test_build_dev_images.py @@ -18,6 +18,8 @@ import re from pathlib import Path +import pytest + import build_dev_images from _common import COMPOSE_FILE, COMPOSE_ONE_SHOTS_FILE @@ -147,3 +149,38 @@ def test_dev_images_are_packaged_with_no_build_arg_the_release_does_not_set(): assert "--build-arg" not in command, component assert command[-1] == "ctx" assert f"{component}:local" in command + + +def _engine_block(*, components: list[str]) -> list[str]: + """The lines of the dfe-engine service in the overlay built for `components`.""" + lines = build_dev_images.overlay_text(components=components).splitlines() + start = lines.index(" dfe-engine:") + block = [] + for line in lines[start + 1 :]: + if not line.startswith(" "): + break + block.append(line) + return block + + +@pytest.mark.parametrize( + ("components", "engine_image"), + [ + pytest.param(["dfe-engine"], True, id="the_engine_itself"), + pytest.param(["dfe-loader"], False, id="another_app"), + pytest.param(["dfe-ui", "dfe-fetcher"], False, id="several_apps"), + ], +) +def test_a_local_build_empties_the_engines_stack_images( + components: list[str], engine_image: bool +) -> None: + block = _engine_block(components=components) + + assert ' DFE_ENGINE_STACK_IMAGES: ""' in block + assert (" image: dfe-engine:local" in block) is engine_image + + +def test_an_overlay_of_nothing_built_leaves_the_stack_images_alone() -> None: + text = build_dev_images.overlay_text(components=[]) + + assert "DFE_ENGINE_STACK_IMAGES" not in text From 476c84f35e5d628bf6c0340dd3e60cfcb5174c81 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 06:25:19 +1100 Subject: [PATCH 3/3] docs(developing): say the dev overlay empties the engine's stack images The overlay from make dev and LOCAL= sets DFE_ENGINE_STACK_IMAGES to empty on the engine. This makes the console show DFE_STACK_VERSION on a local build. --- docs/developing.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/developing.md b/docs/developing.md index c2a405cf..53c1058a 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -116,7 +116,7 @@ make dev # build :local images from local component source, then start make ci # pull the pinned GHCR images, then start ``` -`make dev` builds the `:local` images, then starts with the overlay the build writes, `.tmp/docker-compose.local.yml`, which repoints each component it built at `:local`. `make ci` names no overlay, so it runs registry images only. +`make dev` builds the `:local` images, then starts with the overlay the build writes, `.tmp/docker-compose.local.yml`, which repoints each component it built at `:local`. It also empties the engine's `DFE_ENGINE_STACK_IMAGES`, so the console shows `DFE_STACK_VERSION`. `make ci` names no overlay, so it runs registry images only. Both start targets run `make env-files` first: it asserts every `env.example/.env` has a counterpart in `env/`, runs `make init` for the @@ -174,7 +174,7 @@ Two of them can start without their component (`IMPLICIT_CONSUMERS`, same file): make dev LOCAL="dfe-engine dfe-ui" # build these two; everything else from GHCR ``` -The build writes `.tmp/docker-compose.local.yml` repointing only the named components and the services sharing their image at `:local`. `SERVICES` still filters what starts and a name outside the resolved stack is a hard error. +The build writes `.tmp/docker-compose.local.yml` repointing only the named components and the services sharing their image at `:local`, and empties the engine's `DFE_ENGINE_STACK_IMAGES`, so the console shows `DFE_STACK_VERSION`. `SERVICES` still filters what starts and a name outside the resolved stack is a hard error. ### Where it looks for your source