Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -426,6 +426,7 @@ services:
DFE_ENGINE_KAFKA_BOOTSTRAP_SERVERS: ${DFE_KAFKA_BOOTSTRAP_SERVERS:-}
DFE_ENGINE_PROFILE: ${DFE_STACK_PROFILE:-docker-slim}
DFE_ENGINE_SECRETS_PATH: /app/config/.secrets
DFE_ENGINE_STACK_IMAGES: ${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-archiver:${DFE_ARCHIVER_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-engine:${DFE_ENGINE_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-fetcher:${DFE_FETCHER_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-hyperdx:${DFE_HYPERDX_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-loader:${DFE_LOADER_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-receiver:${DFE_RECEIVER_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-transform-elastic:${DFE_TRANSFORM_ELASTIC_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-transform-vector:${DFE_TRANSFORM_VECTOR_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-transform-vrl:${DFE_TRANSFORM_VRL_VERSION},${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-ui:${DFE_UI_VERSION}
DFE_ENGINE_STACK_VERSION: ${DFE_STACK_VERSION:-}
DFE_ENGINE_TRANSPORT_BUS_PRESENT: ${DFE_TRANSPORT_BUS_PRESENT:-false}
DFE_ENGINE_TRANSPORT_DEFAULT: ${DFE_TRANSPORT_DEFAULT:-direct}
Expand Down
1 change: 1 addition & 0 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,7 @@ Compose sets these on the engine itself:
| `DFE_ENGINE_HYPERDX_BASE_URL`, `DFE_ENGINE_HYPERDX_ENABLED` | Follow the footprint, like the dfe-hyperdx containers |
| `DFE_ENGINE_PROFILE` | The tier name, which seeds its default app instances |
| `DFE_ENGINE_SECRETS_PATH` | On the config volume, so the JWT key survives a recreate |
| `DFE_ENGINE_STACK_IMAGES` | Every DFE app image the stack runs, by digest. The stack the console shows is the published one these match, with `DFE_STACK_VERSION` as the offline fallback |
| `DFE_ENGINE_STACK_VERSION`, `DFE_ENGINE_UI_VERSION` | The certified stack and the console version, for the console footer. The second is the UI pin without its digest |
| `DFE_ENGINE_TRANSPORT_BUS_PRESENT`, `DFE_ENGINE_TRANSPORT_DEFAULT` | From the profile. Their defaults are slim-grpc's |

Expand Down
4 changes: 2 additions & 2 deletions docs/developing.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ make dev # build :local images from local component source, then start
make ci # pull the pinned GHCR images, then start
```

`make dev` builds the `:local` images, then starts with the overlay the build writes, `.tmp/docker-compose.local.yml`, which repoints each component it built at `<service>:local`. `make ci` names no overlay, so it runs registry images only.
`make dev` builds the `:local` images, then starts with the overlay the build writes, `.tmp/docker-compose.local.yml`, which repoints each component it built at `<service>:local`. It also empties the engine's `DFE_ENGINE_STACK_IMAGES`, so the console shows `DFE_STACK_VERSION`. `make ci` names no overlay, so it runs registry images only.

Both start targets run `make env-files` first: it asserts every
`env.example/<service>.env` has a counterpart in `env/`, runs `make init` for the
Expand Down Expand Up @@ -174,7 +174,7 @@ Two of them can start without their component (`IMPLICIT_CONSUMERS`, same file):
make dev LOCAL="dfe-engine dfe-ui" # build these two; everything else from GHCR
```

The build writes `.tmp/docker-compose.local.yml` repointing only the named components and the services sharing their image at `:local`. `SERVICES` still filters what starts and a name outside the resolved stack is a hard error.
The build writes `.tmp/docker-compose.local.yml` repointing only the named components and the services sharing their image at `:local`, and empties the engine's `DFE_ENGINE_STACK_IMAGES`, so the console shows `DFE_STACK_VERSION`. `SERVICES` still filters what starts and a name outside the resolved stack is a hard error.

### Where it looks for your source

Expand Down
14 changes: 12 additions & 2 deletions scripts/build_dev_images.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,9 @@
),
"hyperdx-documentdb-tls": ("hyperdx-documentdb",),
}
# The engine names its stack from the registry digests this lists, which no local build carries
ENGINE_SERVICE = "dfe-engine"
STACK_IMAGES_KEY = "DFE_ENGINE_STACK_IMAGES"
RUST_COMPONENTS = [
"dfe-archiver",
"dfe-fetcher",
Expand Down Expand Up @@ -374,9 +377,16 @@ def overlay_text(components: list[str]) -> str:
services = sorted(local_image_services(components=components).items())
if not (services):
return f"{header}\nservices: {{}}\n"
local = dict(services)
lines = [header, "services:"]
for service, image in services:
lines += [f" {service}:", f" image: {image}", ""]
for service in sorted({*local, ENGINE_SERVICE}):
lines.append(f" {service}:")
if service in local:
lines.append(f" image: {local[service]}")
if service == ENGINE_SERVICE:
# A local build is no published stack, so the engine reports DFE_ENGINE_STACK_VERSION instead
lines += [" environment:", f' {STACK_IMAGES_KEY}: ""']
lines.append("")
return "\n".join(lines).rstrip("\n") + "\n"


Expand Down
91 changes: 91 additions & 0 deletions scripts/check_compose.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,11 +77,14 @@
- Every path a pinned image declares as a VOLUME must be mounted from a named volume or a bind, and wherever the daemon holds the image, `_IMAGE_VOLUMES` must list exactly the paths it declares. See `_image_volume_drift`.
- No committed service may carry a name a generated per-source instance can take.
See `_instance_name_collisions`.
- The engine's DFE_ENGINE_STACK_IMAGES must list every DFE app image the stack runs,
each pinned by digest, and no other image. See `_stack_images_mismatches`.
- Every service dfe-engine renders, and every instance the generator declares,
must read the custom env file the engine writes for it, last; no service may
read one the engine does not write. See `_custom_env_mismatches`.
"""

import hashlib
import json
import os
import re
Expand Down Expand Up @@ -1232,6 +1235,75 @@ def _version_check_failures(*, env: dict[str, str]) -> tuple[list[str], int]:
return failures, made


# The engine names the stack it runs from these, so each entry is an app image a stack manifest lists
_STACK_IMAGES_KEY = "DFE_ENGINE_STACK_IMAGES"
_STACK_IMAGE_PREFIX = "dfe-"
_DIGEST_REF = re.compile(r"@sha256:[0-9a-f]{64}$")


def _image_name(*, reference: str) -> str:
"""Return the name of `registry/path/name[:tag][@sha256:digest]`."""
return reference.split("@", 1)[0].rsplit("/", 1)[-1].split(":", 1)[0]


def _stack_images_mismatches(*, config: dict) -> list[str]:
"""Return one message per way the engine's DFE_ENGINE_STACK_IMAGES differs from the DFE app images the model runs.

Every service counts, so the hunt runner's copy of the engine image adds no second entry and no image a service runs is left out. An entry with no digest cannot match a stack manifest, and an image that is not a DFE app matches none.
"""
services = config.get("services", {})
engine = services.get(_ENGINE_SERVICE)
if engine is None:
return [f"{_ENGINE_SERVICE}: not in the stack"]
value = str((engine.get("environment") or {}).get(_STACK_IMAGES_KEY) or "")
listed = [entry.strip() for entry in value.split(",") if entry.strip()]
running = {
str(service["image"])
for service in services.values()
if _image_name(reference=str(service.get("image", ""))).startswith(
_STACK_IMAGE_PREFIX
)
}
failures = [
f"{_STACK_IMAGES_KEY} lists {entry!r}, which carries no sha256 digest"
for entry in listed
if not _DIGEST_REF.search(entry)
]
failures.extend(
f"{_STACK_IMAGES_KEY} lists {entry!r}, which no service runs as a DFE app"
for entry in sorted(set(listed) - running)
)
failures.extend(
f"{_STACK_IMAGES_KEY} leaves out {image!r}, which a service runs"
for image in sorted(running - set(listed))
)
failures.extend(
f"{_STACK_IMAGES_KEY} lists {entry!r} {listed.count(entry)} times"
for entry in sorted({entry for entry in listed if listed.count(entry) > 1})
)
return failures


def _stack_images_failures(*, env: dict[str, str]) -> tuple[list[str], int]:
"""Return (messages, assertions made) for DFE_ENGINE_STACK_IMAGES on the registry path.

Each mandatory key gets a digest-shaped value of its own, so a pin that never reaches the list shows as a missing entry. The checkout's .env is not loaded.
"""
pinned = {
**env,
**{
name: f"v0.0.0@sha256:{hashlib.sha256(name.encode()).hexdigest()}"
for name in _required_compose_vars()
},
}
config = _config_json(
env=pinned, files=[COMPOSE_FILE.name], env_file=_empty_env_file()
)
if config is None:
return (["docker-compose.yml did not resolve with every pin digested"], 0)
return _stack_images_mismatches(config=config), 1


def _proxy_tls_mismatches(*, config: dict, tls: dict[str, str] | None) -> list[str]:
"""Return one message per way the proxies, the network and the engine differ from the console TLS side they are on.

Expand Down Expand Up @@ -1673,6 +1745,15 @@ def _local_overlay_failures(*, env: dict[str, str]) -> tuple[list[str], int]:
f"{_LOCAL_RENDER.name}: {service} runs {got}, not {image} -- a "
"LOCAL build leaves a follower on the registry image"
)
made += 1
stack_images = (services.get(_ENGINE_SERVICE, {}).get("environment") or {}).get(
_STACK_IMAGES_KEY
)
if stack_images != "":
failures.append(
f"{_LOCAL_RENDER.name}: {_ENGINE_SERVICE} gets {_STACK_IMAGES_KEY}="
f"{stack_images!r}, not empty -- a LOCAL build is no published stack"
)
for service, service_config in sorted(services.items()):
if service in want:
continue
Expand Down Expand Up @@ -2111,6 +2192,16 @@ def _checks() -> int:
f"to true unless .env says otherwise ({version_made} assertions)"
)

stack_images_failures, stack_images_made = _stack_images_failures(env=env)
for message in stack_images_failures:
_print(msg=f"FAIL {message}")
if stack_images_failures:
return 1
_print(
msg=f"{_ENGINE_SERVICE} names the stack from every DFE app image the stack "
f"runs, each by digest, and no other ({stack_images_made} assertion)"
)

tls_failures, tls_made = _proxy_tls_failures(env=env)
for message in tls_failures:
_print(msg=f"FAIL {message}")
Expand Down
37 changes: 37 additions & 0 deletions scripts/tests/test_build_dev_images.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
import re
from pathlib import Path

import pytest

import build_dev_images
from _common import COMPOSE_FILE, COMPOSE_ONE_SHOTS_FILE

Expand Down Expand Up @@ -147,3 +149,38 @@ def test_dev_images_are_packaged_with_no_build_arg_the_release_does_not_set():
assert "--build-arg" not in command, component
assert command[-1] == "ctx"
assert f"{component}:local" in command


def _engine_block(*, components: list[str]) -> list[str]:
"""The lines of the dfe-engine service in the overlay built for `components`."""
lines = build_dev_images.overlay_text(components=components).splitlines()
start = lines.index(" dfe-engine:")
block = []
for line in lines[start + 1 :]:
if not line.startswith(" "):
break
block.append(line)
return block


@pytest.mark.parametrize(
("components", "engine_image"),
[
pytest.param(["dfe-engine"], True, id="the_engine_itself"),
pytest.param(["dfe-loader"], False, id="another_app"),
pytest.param(["dfe-ui", "dfe-fetcher"], False, id="several_apps"),
],
)
def test_a_local_build_empties_the_engines_stack_images(
components: list[str], engine_image: bool
) -> None:
block = _engine_block(components=components)

assert ' DFE_ENGINE_STACK_IMAGES: ""' in block
assert (" image: dfe-engine:local" in block) is engine_image


def test_an_overlay_of_nothing_built_leaves_the_stack_images_alone() -> None:
text = build_dev_images.overlay_text(components=[])

assert "DFE_ENGINE_STACK_IMAGES" not in text
92 changes: 92 additions & 0 deletions scripts/tests/test_check_compose.py
Original file line number Diff line number Diff line change
Expand Up @@ -817,3 +817,95 @@ def test_a_dfe_app_gets_the_stop_grace_its_contract_asks_for(
resolved_model, service, grace
):
assert resolved_model["services"][service]["stop_grace_period"] == grace


def _digest(name: str) -> str:
return f"sha256:{name.encode().hex().ljust(64, '0')[:64]}"


def _stack_model(*, listed: list[str], running: list[str]) -> dict:
"""A resolved model whose services run `running` and whose engine lists `listed`."""
services = {
f"service-{index}": {"image": image} for index, image in enumerate(running)
}
services["dfe-engine"] = {
"image": "ghcr.io/hyperi-io/dfe-engine:v1@" + _digest("engine"),
"environment": {"DFE_ENGINE_STACK_IMAGES": ",".join(listed)},
}
return {"services": services}


_ENGINE_REF = "ghcr.io/hyperi-io/dfe-engine:v1@" + _digest("engine")
_LOADER_REF = "ghcr.io/hyperi-io/dfe-loader:v2@" + _digest("loader")
_BARE_LOADER_REF = "ghcr.io/hyperi-io/dfe-loader:v2"
_CLICKHOUSE_REF = "clickhouse/clickhouse-server:26.3@" + _digest("clickhouse")


@pytest.mark.parametrize(
("listed", "running", "expected"),
[
pytest.param(
[_ENGINE_REF, _LOADER_REF], [_LOADER_REF, _ENGINE_REF], [], id="exact"
),
pytest.param(
[_ENGINE_REF, _LOADER_REF],
[_LOADER_REF, _ENGINE_REF, _CLICKHOUSE_REF],
[],
id="a_third_party_image_is_not_listed",
),
pytest.param(
[_ENGINE_REF],
[_LOADER_REF],
["leaves out"],
id="a_running_app_left_out",
),
pytest.param(
[_ENGINE_REF, _LOADER_REF, _CLICKHOUSE_REF],
[_LOADER_REF],
["no service runs"],
id="a_third_party_image_listed",
),
pytest.param(
[_ENGINE_REF, _BARE_LOADER_REF],
[_BARE_LOADER_REF],
["no sha256 digest"],
id="an_entry_without_a_digest",
),
pytest.param(
[_ENGINE_REF, _LOADER_REF, _LOADER_REF],
[_LOADER_REF],
["2 times"],
id="an_entry_twice",
),
pytest.param(
[], [_LOADER_REF], ["leaves out", "leaves out"], id="an_empty_list"
),
],
)
def test_the_engine_lists_the_dfe_app_images_the_stack_runs(
listed: list[str], running: list[str], expected: list[str]
) -> None:
model = _stack_model(listed=listed, running=running)

failures = check_compose._stack_images_mismatches(config=model)

assert len(failures) == len(expected), failures
for fragment, failure in zip(expected, failures, strict=True):
assert fragment in failure


def test_a_stack_with_no_engine_fails_rather_than_passing() -> None:
failures = check_compose._stack_images_mismatches(config={"services": {}})

assert failures == ["dfe-engine: not in the stack"]


def test_the_committed_compose_lists_every_dfe_app_image_it_runs_by_digest() -> None:
if shutil.which("docker") is None:
pytest.skip("`docker compose config` renders the model this asserts on")
env, _ = check_compose._check_env()

failures, made = check_compose._stack_images_failures(env=env)

assert failures == []
assert made == 1
Loading