diff --git a/docker-compose.yml b/docker-compose.yml index 94f18bd..d6c7e55 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -426,6 +426,7 @@ services: DFE_ENGINE_KAFKA_BOOTSTRAP_SERVERS: ${DFE_KAFKA_BOOTSTRAP_SERVERS:-} DFE_ENGINE_PROFILE: ${DFE_STACK_PROFILE:-docker-slim} DFE_ENGINE_SECRETS_PATH: /app/config/.secrets + DFE_ENGINE_STACK_IMAGES: ${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-archiver:${DFE_ARCHIVER_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-engine:${DFE_ENGINE_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-fetcher:${DFE_FETCHER_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-hyperdx:${DFE_HYPERDX_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-loader:${DFE_LOADER_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-receiver:${DFE_RECEIVER_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-transform-elastic:${DFE_TRANSFORM_ELASTIC_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-transform-vector:${DFE_TRANSFORM_VECTOR_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-transform-vrl:${DFE_TRANSFORM_VRL_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-ui:${DFE_UI_VERSION} DFE_ENGINE_STACK_VERSION: ${DFE_STACK_VERSION:-} DFE_ENGINE_TRANSPORT_BUS_PRESENT: ${DFE_TRANSPORT_BUS_PRESENT:-false} DFE_ENGINE_TRANSPORT_DEFAULT: ${DFE_TRANSPORT_DEFAULT:-direct} diff --git a/docs/configuration.md b/docs/configuration.md index 803a639..ea79df2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -358,6 +358,7 @@ Compose sets these on the engine itself: | `DFE_ENGINE_HYPERDX_BASE_URL`, `DFE_ENGINE_HYPERDX_ENABLED` | Follow the footprint, like the dfe-hyperdx containers | | `DFE_ENGINE_PROFILE` | The tier name, which seeds its default app instances | | `DFE_ENGINE_SECRETS_PATH` | On the config volume, so the JWT key survives a recreate | +| `DFE_ENGINE_STACK_IMAGES` | Every DFE app image the stack runs, by digest. The stack the console shows is the published one these match, with `DFE_STACK_VERSION` as the offline fallback | | `DFE_ENGINE_STACK_VERSION`, `DFE_ENGINE_UI_VERSION` | The certified stack and the console version, for the console footer. The second is the UI pin without its digest | | `DFE_ENGINE_TRANSPORT_BUS_PRESENT`, `DFE_ENGINE_TRANSPORT_DEFAULT` | From the profile. Their defaults are slim-grpc's | diff --git a/docs/developing.md b/docs/developing.md index c2a405c..53c1058 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -116,7 +116,7 @@ make dev # build :local images from local component source, then start make ci # pull the pinned GHCR images, then start ``` -`make dev` builds the `:local` images, then starts with the overlay the build writes, `.tmp/docker-compose.local.yml`, which repoints each component it built at `:local`. `make ci` names no overlay, so it runs registry images only. +`make dev` builds the `:local` images, then starts with the overlay the build writes, `.tmp/docker-compose.local.yml`, which repoints each component it built at `:local`. It also empties the engine's `DFE_ENGINE_STACK_IMAGES`, so the console shows `DFE_STACK_VERSION`. `make ci` names no overlay, so it runs registry images only. Both start targets run `make env-files` first: it asserts every `env.example/.env` has a counterpart in `env/`, runs `make init` for the @@ -174,7 +174,7 @@ Two of them can start without their component (`IMPLICIT_CONSUMERS`, same file): make dev LOCAL="dfe-engine dfe-ui" # build these two; everything else from GHCR ``` -The build writes `.tmp/docker-compose.local.yml` repointing only the named components and the services sharing their image at `:local`. `SERVICES` still filters what starts and a name outside the resolved stack is a hard error. +The build writes `.tmp/docker-compose.local.yml` repointing only the named components and the services sharing their image at `:local`, and empties the engine's `DFE_ENGINE_STACK_IMAGES`, so the console shows `DFE_STACK_VERSION`. `SERVICES` still filters what starts and a name outside the resolved stack is a hard error. ### Where it looks for your source diff --git a/scripts/build_dev_images.py b/scripts/build_dev_images.py index 37d6991..d5f4fe8 100644 --- a/scripts/build_dev_images.py +++ b/scripts/build_dev_images.py @@ -67,6 +67,9 @@ ), "hyperdx-documentdb-tls": ("hyperdx-documentdb",), } +# The engine names its stack from the registry digests this lists, which no local build carries +ENGINE_SERVICE = "dfe-engine" +STACK_IMAGES_KEY = "DFE_ENGINE_STACK_IMAGES" RUST_COMPONENTS = [ "dfe-archiver", "dfe-fetcher", @@ -374,9 +377,16 @@ def overlay_text(components: list[str]) -> str: services = sorted(local_image_services(components=components).items()) if not (services): return f"{header}\nservices: {{}}\n" + local = dict(services) lines = [header, "services:"] - for service, image in services: - lines += [f" {service}:", f" image: {image}", ""] + for service in sorted({*local, ENGINE_SERVICE}): + lines.append(f" {service}:") + if service in local: + lines.append(f" image: {local[service]}") + if service == ENGINE_SERVICE: + # A local build is no published stack, so the engine reports DFE_ENGINE_STACK_VERSION instead + lines += [" environment:", f' {STACK_IMAGES_KEY}: ""'] + lines.append("") return "\n".join(lines).rstrip("\n") + "\n" diff --git a/scripts/check_compose.py b/scripts/check_compose.py index 8f8f3d7..3c88631 100644 --- a/scripts/check_compose.py +++ b/scripts/check_compose.py @@ -77,11 +77,14 @@ - Every path a pinned image declares as a VOLUME must be mounted from a named volume or a bind, and wherever the daemon holds the image, `_IMAGE_VOLUMES` must list exactly the paths it declares. See `_image_volume_drift`. - No committed service may carry a name a generated per-source instance can take. See `_instance_name_collisions`. +- The engine's DFE_ENGINE_STACK_IMAGES must list every DFE app image the stack runs, + each pinned by digest, and no other image. See `_stack_images_mismatches`. - Every service dfe-engine renders, and every instance the generator declares, must read the custom env file the engine writes for it, last; no service may read one the engine does not write. See `_custom_env_mismatches`. """ +import hashlib import json import os import re @@ -1232,6 +1235,75 @@ def _version_check_failures(*, env: dict[str, str]) -> tuple[list[str], int]: return failures, made +# The engine names the stack it runs from these, so each entry is an app image a stack manifest lists +_STACK_IMAGES_KEY = "DFE_ENGINE_STACK_IMAGES" +_STACK_IMAGE_PREFIX = "dfe-" +_DIGEST_REF = re.compile(r"@sha256:[0-9a-f]{64}$") + + +def _image_name(*, reference: str) -> str: + """Return the name of `registry/path/name[:tag][@sha256:digest]`.""" + return reference.split("@", 1)[0].rsplit("/", 1)[-1].split(":", 1)[0] + + +def _stack_images_mismatches(*, config: dict) -> list[str]: + """Return one message per way the engine's DFE_ENGINE_STACK_IMAGES differs from the DFE app images the model runs. + + Every service counts, so the hunt runner's copy of the engine image adds no second entry and no image a service runs is left out. An entry with no digest cannot match a stack manifest, and an image that is not a DFE app matches none. + """ + services = config.get("services", {}) + engine = services.get(_ENGINE_SERVICE) + if engine is None: + return [f"{_ENGINE_SERVICE}: not in the stack"] + value = str((engine.get("environment") or {}).get(_STACK_IMAGES_KEY) or "") + listed = [entry.strip() for entry in value.split(",") if entry.strip()] + running = { + str(service["image"]) + for service in services.values() + if _image_name(reference=str(service.get("image", ""))).startswith( + _STACK_IMAGE_PREFIX + ) + } + failures = [ + f"{_STACK_IMAGES_KEY} lists {entry!r}, which carries no sha256 digest" + for entry in listed + if not _DIGEST_REF.search(entry) + ] + failures.extend( + f"{_STACK_IMAGES_KEY} lists {entry!r}, which no service runs as a DFE app" + for entry in sorted(set(listed) - running) + ) + failures.extend( + f"{_STACK_IMAGES_KEY} leaves out {image!r}, which a service runs" + for image in sorted(running - set(listed)) + ) + failures.extend( + f"{_STACK_IMAGES_KEY} lists {entry!r} {listed.count(entry)} times" + for entry in sorted({entry for entry in listed if listed.count(entry) > 1}) + ) + return failures + + +def _stack_images_failures(*, env: dict[str, str]) -> tuple[list[str], int]: + """Return (messages, assertions made) for DFE_ENGINE_STACK_IMAGES on the registry path. + + Each mandatory key gets a digest-shaped value of its own, so a pin that never reaches the list shows as a missing entry. The checkout's .env is not loaded. + """ + pinned = { + **env, + **{ + name: f"v0.0.0@sha256:{hashlib.sha256(name.encode()).hexdigest()}" + for name in _required_compose_vars() + }, + } + config = _config_json( + env=pinned, files=[COMPOSE_FILE.name], env_file=_empty_env_file() + ) + if config is None: + return (["docker-compose.yml did not resolve with every pin digested"], 0) + return _stack_images_mismatches(config=config), 1 + + def _proxy_tls_mismatches(*, config: dict, tls: dict[str, str] | None) -> list[str]: """Return one message per way the proxies, the network and the engine differ from the console TLS side they are on. @@ -1673,6 +1745,15 @@ def _local_overlay_failures(*, env: dict[str, str]) -> tuple[list[str], int]: f"{_LOCAL_RENDER.name}: {service} runs {got}, not {image} -- a " "LOCAL build leaves a follower on the registry image" ) + made += 1 + stack_images = (services.get(_ENGINE_SERVICE, {}).get("environment") or {}).get( + _STACK_IMAGES_KEY + ) + if stack_images != "": + failures.append( + f"{_LOCAL_RENDER.name}: {_ENGINE_SERVICE} gets {_STACK_IMAGES_KEY}=" + f"{stack_images!r}, not empty -- a LOCAL build is no published stack" + ) for service, service_config in sorted(services.items()): if service in want: continue @@ -2111,6 +2192,16 @@ def _checks() -> int: f"to true unless .env says otherwise ({version_made} assertions)" ) + stack_images_failures, stack_images_made = _stack_images_failures(env=env) + for message in stack_images_failures: + _print(msg=f"FAIL {message}") + if stack_images_failures: + return 1 + _print( + msg=f"{_ENGINE_SERVICE} names the stack from every DFE app image the stack " + f"runs, each by digest, and no other ({stack_images_made} assertion)" + ) + tls_failures, tls_made = _proxy_tls_failures(env=env) for message in tls_failures: _print(msg=f"FAIL {message}") diff --git a/scripts/tests/test_build_dev_images.py b/scripts/tests/test_build_dev_images.py index 23d8b70..c23249a 100644 --- a/scripts/tests/test_build_dev_images.py +++ b/scripts/tests/test_build_dev_images.py @@ -18,6 +18,8 @@ import re from pathlib import Path +import pytest + import build_dev_images from _common import COMPOSE_FILE, COMPOSE_ONE_SHOTS_FILE @@ -147,3 +149,38 @@ def test_dev_images_are_packaged_with_no_build_arg_the_release_does_not_set(): assert "--build-arg" not in command, component assert command[-1] == "ctx" assert f"{component}:local" in command + + +def _engine_block(*, components: list[str]) -> list[str]: + """The lines of the dfe-engine service in the overlay built for `components`.""" + lines = build_dev_images.overlay_text(components=components).splitlines() + start = lines.index(" dfe-engine:") + block = [] + for line in lines[start + 1 :]: + if not line.startswith(" "): + break + block.append(line) + return block + + +@pytest.mark.parametrize( + ("components", "engine_image"), + [ + pytest.param(["dfe-engine"], True, id="the_engine_itself"), + pytest.param(["dfe-loader"], False, id="another_app"), + pytest.param(["dfe-ui", "dfe-fetcher"], False, id="several_apps"), + ], +) +def test_a_local_build_empties_the_engines_stack_images( + components: list[str], engine_image: bool +) -> None: + block = _engine_block(components=components) + + assert ' DFE_ENGINE_STACK_IMAGES: ""' in block + assert (" image: dfe-engine:local" in block) is engine_image + + +def test_an_overlay_of_nothing_built_leaves_the_stack_images_alone() -> None: + text = build_dev_images.overlay_text(components=[]) + + assert "DFE_ENGINE_STACK_IMAGES" not in text diff --git a/scripts/tests/test_check_compose.py b/scripts/tests/test_check_compose.py index 1404f7c..19c7c7c 100644 --- a/scripts/tests/test_check_compose.py +++ b/scripts/tests/test_check_compose.py @@ -817,3 +817,95 @@ def test_a_dfe_app_gets_the_stop_grace_its_contract_asks_for( resolved_model, service, grace ): assert resolved_model["services"][service]["stop_grace_period"] == grace + + +def _digest(name: str) -> str: + return f"sha256:{name.encode().hex().ljust(64, '0')[:64]}" + + +def _stack_model(*, listed: list[str], running: list[str]) -> dict: + """A resolved model whose services run `running` and whose engine lists `listed`.""" + services = { + f"service-{index}": {"image": image} for index, image in enumerate(running) + } + services["dfe-engine"] = { + "image": "ghcr.io/hyperi-io/dfe-engine:v1@" + _digest("engine"), + "environment": {"DFE_ENGINE_STACK_IMAGES": ",".join(listed)}, + } + return {"services": services} + + +_ENGINE_REF = "ghcr.io/hyperi-io/dfe-engine:v1@" + _digest("engine") +_LOADER_REF = "ghcr.io/hyperi-io/dfe-loader:v2@" + _digest("loader") +_BARE_LOADER_REF = "ghcr.io/hyperi-io/dfe-loader:v2" +_CLICKHOUSE_REF = "clickhouse/clickhouse-server:26.3@" + _digest("clickhouse") + + +@pytest.mark.parametrize( + ("listed", "running", "expected"), + [ + pytest.param( + [_ENGINE_REF, _LOADER_REF], [_LOADER_REF, _ENGINE_REF], [], id="exact" + ), + pytest.param( + [_ENGINE_REF, _LOADER_REF], + [_LOADER_REF, _ENGINE_REF, _CLICKHOUSE_REF], + [], + id="a_third_party_image_is_not_listed", + ), + pytest.param( + [_ENGINE_REF], + [_LOADER_REF], + ["leaves out"], + id="a_running_app_left_out", + ), + pytest.param( + [_ENGINE_REF, _LOADER_REF, _CLICKHOUSE_REF], + [_LOADER_REF], + ["no service runs"], + id="a_third_party_image_listed", + ), + pytest.param( + [_ENGINE_REF, _BARE_LOADER_REF], + [_BARE_LOADER_REF], + ["no sha256 digest"], + id="an_entry_without_a_digest", + ), + pytest.param( + [_ENGINE_REF, _LOADER_REF, _LOADER_REF], + [_LOADER_REF], + ["2 times"], + id="an_entry_twice", + ), + pytest.param( + [], [_LOADER_REF], ["leaves out", "leaves out"], id="an_empty_list" + ), + ], +) +def test_the_engine_lists_the_dfe_app_images_the_stack_runs( + listed: list[str], running: list[str], expected: list[str] +) -> None: + model = _stack_model(listed=listed, running=running) + + failures = check_compose._stack_images_mismatches(config=model) + + assert len(failures) == len(expected), failures + for fragment, failure in zip(expected, failures, strict=True): + assert fragment in failure + + +def test_a_stack_with_no_engine_fails_rather_than_passing() -> None: + failures = check_compose._stack_images_mismatches(config={"services": {}}) + + assert failures == ["dfe-engine: not in the stack"] + + +def test_the_committed_compose_lists_every_dfe_app_image_it_runs_by_digest() -> None: + if shutil.which("docker") is None: + pytest.skip("`docker compose config` renders the model this asserts on") + env, _ = check_compose._check_env() + + failures, made = check_compose._stack_images_failures(env=env) + + assert failures == [] + assert made == 1