From 26dafaa02da9b7ee8a10ae09b2192ca0f62cee2d Mon Sep 17 00:00:00 2001 From: Derek Date: Sun, 27 Sep 2026 15:58:51 +1000 Subject: [PATCH] fix: make apply exists, and a fresh kafka stack passes its self test Two things stopped a public 2.2.0-rc.14 single stack at first use. The engine tells a Compose operator to run make apply SERVICES= after a source deploy, and this repo had no apply target. A new per-source transform therefore never got a container, and the only way to start one was make up, which takes the whole stack down first. make apply re-resolves the profile, starts what it adds or changes without a down, and restarts the services it is given so a rewritten mounted config is read. The SERVICES name check moves into a services-valid target, because the parse that ran it could still hold the profile from before the engine declared the instance. make up on the single tier failed its self test on every fresh stack. The subscription claim waits for the loader's per-topic lag metric, which librdkafka reports as -1 until the group commits an offset, and scalo exports only a lag it can state, so a stack that has carried no traffic never shows one. Each attempt that sees no topic now sends one probe record through the ingest edge first. Both are the same change dfe-docker #143 carries, taken on their own because that branch conflicts with main. Signed-off-by: Derek --- Makefile | 54 +++++++++++++++++++++++++--------- README.md | 1 + docs/troubleshooting.md | 3 +- scripts/post.py | 25 +++++++++++++--- scripts/tests/test_post.py | 60 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 124 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 29108ca7..f46b4359 100644 --- a/Makefile +++ b/Makefile @@ -60,15 +60,10 @@ ifneq (,$(filter-out $(BOOTSTRAP_GOALS),$(or $(MAKECMDGOALS),help))) ifeq ($(strip $(SERVICES)),) ACTIVE_SERVICES := $(DFE_SERVICES) else - # On a fresh checkout make parses once before .profile.mk exists, then - # remakes it and parses again; the name check only means anything on - # the second pass, when DFE_SERVICES is populated. - ifneq ($(strip $(DFE_SERVICES)),) - INVALID_SERVICES := $(filter-out $(DFE_SERVICES),$(SERVICES)) - ifneq ($(INVALID_SERVICES),) - $(error 'SERVICES' contains names not in the resolved stack: $(INVALID_SERVICES). Available: $(DFE_SERVICES)) - endif - endif + # Checked by the services-valid target, not here: this parse can hold the + # .profile.mk from before the engine declared a new per-source instance, + # and make regenerates it and parses again before any recipe runs. + INVALID_SERVICES := $(filter-out $(DFE_SERVICES),$(SERVICES)) ACTIVE_SERVICES := $(filter $(SERVICES),$(DFE_SERVICES)) endif # `make dev LOCAL="dfe-engine dfe-ui"` builds only those and keeps the rest @@ -255,6 +250,15 @@ endif .PHONY: FORCE FORCE: +# A prerequisite of every target that narrows to SERVICES. Recipes run only on +# the pass that read the regenerated .profile.mk, so a per-source instance the +# engine declared since the last run is a valid name here. +.PHONY: services-valid +services-valid: +ifneq ($(strip $(INVALID_SERVICES)),) + $(error 'SERVICES' contains names not in the resolved stack: $(INVALID_SERVICES). Available: $(DFE_SERVICES)) +endif + .env: @python3 scripts/init.py @@ -356,7 +360,7 @@ endif # which stay private until GA, so both are HyperI-internal until then; `make ci` # runs the same stack from the public images. .PHONY: dev -dev: env-files dev-posture down storage-dirs ## Build local DFE images from source and start the dev stack (LOCAL="svc ..." builds only those; AUTH=real for a minted login), then print the access summary. HyperI-internal: clones private repos +dev: services-valid env-files dev-posture down storage-dirs ## Build local DFE images from source and start the dev stack (LOCAL="svc ..." builds only those; AUTH=real for a minted login), then print the access summary. HyperI-internal: clones private repos docker compose $(DEV_PULL_FLAGS) $(PROFILE_FLAGS) pull python3 scripts/build_dev_images.py $(DEV_OVERLAY_ARG) $(DEV_BUILD) docker compose $(DEV_FLAGS) $(PROFILE_FLAGS) up -d $(ACTIVE_SERVICES) @@ -364,7 +368,7 @@ dev: env-files dev-posture down storage-dirs ## Build local DFE images from sour @$(MAKE) --no-print-directory creds .PHONY: dev-build -dev-build: ## Build local DFE images from source (no start; honours LOCAL). HyperI-internal: clones private repos +dev-build: services-valid ## Build local DFE images from source (no start; honours LOCAL). HyperI-internal: clones private repos docker compose $(DEV_PULL_FLAGS) $(PROFILE_FLAGS) pull python3 scripts/build_dev_images.py $(DEV_OVERLAY_ARG) $(DEV_BUILD) @@ -373,7 +377,7 @@ dev-build: ## Build local DFE images from source (no start; honours LOCAL). Hype # --------------------------------------------------------------------------- .PHONY: ci -ci: login env-files down storage-dirs ## Pull and start infra and registry DFE images. Prints no credentials -- run `make creds` for those +ci: services-valid login env-files down storage-dirs ## Pull and start infra and registry DFE images. Prints no credentials -- run `make creds` for those docker compose -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) $(PROFILE_FLAGS) pull docker compose -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) $(PROFILE_FLAGS) pull $(ACTIVE_SERVICES) docker compose -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) $(PROFILE_FLAGS) up -d $(ACTIVE_SERVICES) @@ -386,8 +390,32 @@ ci: login env-files down storage-dirs ## Pull and start infra and registry DFE up: ci ## Start the stack from the pinned registry images, then print the access summary (password on a TTY only) and write access-summary.md @python3 scripts/creds.py --write +# The profile is re-resolved first, as for every goal outside BOOTSTRAP_GOALS, so +# a per-source instance the engine rendered since the stack started is a service +# by now. No `down` here: compose recreates only a service whose resolved +# definition changed, and leaves every volume alone. DEV=1 uses the file chain +# `make dev` starts with rather than the registry one. +# +# A config file the engine rewrote under a running container changes no service +# definition, so `up` leaves that container as it is. Naming services therefore +# restarts them as well, which is the engine's hint: one command creates, +# recreates or restarts whatever the write needs. With no SERVICES nothing +# unchanged is touched. +ifneq ($(strip $(DEV)),) + APPLY_FLAGS = $(DEV_FLAGS) +else + APPLY_FLAGS = -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) +endif + +.PHONY: apply +apply: services-valid storage-dirs ## Start what the re-resolved profile adds or changes on a running stack, a new per-source instance included; SERVICES="..." also restarts those, so a rewritten mounted config is read; with no SERVICES unchanged containers are left alone. Volumes are never touched (DEV=1 for a `make dev` stack) + docker compose $(APPLY_FLAGS) $(PROFILE_FLAGS) up -d $(ACTIVE_SERVICES) +ifneq ($(strip $(SERVICES)),) + docker compose $(APPLY_FLAGS) $(PROFILE_FLAGS) restart --no-deps $(ACTIVE_SERVICES) +endif + .PHONY: ci-pull -ci-pull: login ## Pull infra and registry DFE images +ci-pull: services-valid login ## Pull infra and registry DFE images docker compose -f docker-compose.yml $(PROFILE_FLAGS) pull docker compose -f docker-compose.yml $(PROFILE_FLAGS) pull $(ACTIVE_SERVICES) diff --git a/README.md b/README.md index fd2b6b59..ed02e01a 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,7 @@ In `dev` mode, they build from each repo's own Dockerfile (not the shared Rust b | `make env-files` | Assert every `env/.env` exists, creating any the templates have gained | | `make creds` | Print the access summary -- console URL, admin login, where the break-glass password lives. The password prints on a TTY only; a pipe, a file or `DFE_CREDS_SHOW=0` gets the `.env` key instead | | `make up` | Start the pinned stack and print the access summary | +| `make apply` | Start what the re-resolved profile adds or changes on a running stack, a new per-source instance included, recreating nothing unchanged and touching no volume. `SERVICES="..."` also restarts those services, so a mounted config the engine rewrote is read. It is the command the engine's restart hints name (`DEV=1` for a `make dev` stack) | | `make dev` | Build local DFE images from source and start the stack (`LOCAL="..."` builds only those, rest pinned) | | `make dev-build` | Build local DFE images from source (no start) | | `make ci` | Pull and start infra and registry DFE images. Prints no credentials -- `make up` is the same start plus `make creds` | diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index c11629f4..a2a1ccd3 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -148,8 +148,7 @@ The e2e harness now does this for itself: `clean_topics` deletes the `_load` sibling of every expected `_land` topic, so a run depends on the test definition rather than on the broker's history. -`make post` asserts the loader is fetching a topic before it injects anything, so -an empty subscription is reported as one. +`make post` asserts the loader is fetching a topic before it injects anything, so an empty subscription is reported as one. The loader names no topic until its group has committed an offset, so on a stack that has carried no traffic the self test sends one probe record per attempt, tagged `post---probe`, and those rows stay in the table. ## Events are accepted but nothing lands diff --git a/scripts/post.py b/scripts/post.py index 7afd4646..896d6b5c 100644 --- a/scripts/post.py +++ b/scripts/post.py @@ -999,13 +999,19 @@ def _metric_label_values(body: str, name: str, label: str) -> set[str]: return found -def _verify_loader_subscription() -> Claim: +def _verify_loader_subscription(*, ingest_url: str, table: str) -> Claim: """Prove the loader is fetching a topic, on a tier that has a bus. The loader discovers its topics from the broker, so a topic that never appears -- or one scalo suppresses in favour of a `_load` sibling -- leaves it ready and healthy with nothing to read. Asserted before the injection, because after it the same fault is indistinguishable from a broken loader or a missing table. + + librdkafka reports a partition's lag as -1 until the group commits an offset + on it, and scalo exports only a lag it can state, so a fresh stack shows no + topic at all however well subscribed it is. Each attempt that sees none sends + one probe record through the ingest edge first: a subscribed loader loads and + commits it and names its topic, and one subscribed to nothing never does. """ if _profile_mk_value(key=TRANSPORT_BUS_KEY)[:1] != ["true"]: _print(msg="SKIP this tier has no bus -- the loader is handed its events") @@ -1022,14 +1028,24 @@ def _verify_loader_subscription() -> Claim: or LOADER_PROMETHEUS_PORT ) base = f"http://{bind}:{port}" - _print(msg=f"Waiting for {LOADER_SERVICE} to be fetching a topic at {base}") + probe = f"{_run_id()}-probe" + _print( + msg=f"Waiting for {LOADER_SERVICE} to be fetching a topic at {base}, " + f"sending probe records tagged {probe}" + ) def _fetching(): try: body = http_get(f"{base}/metrics", timeout=5) except Exception: # noqa: BLE001 - unreachable and not-yet-serving are one answer return set() - return _metric_label_values(body, LOADER_TOPIC_METRIC, LOADER_TOPIC_LABEL) + topics = _metric_label_values(body, LOADER_TOPIC_METRIC, LOADER_TOPIC_LABEL) + if not (topics): + try: + http_post(ingest_url, marked_event(marker=probe, source=table)) + except Exception: # noqa: BLE001 - a refused probe is another attempt + pass + return topics def _report(attempt, result): _print( @@ -1052,6 +1068,7 @@ def _report(attempt, result): ) return Claim(asserted=True, failed=1) _print(msg=f"PASS {LOADER_SERVICE} is fetching {', '.join(sorted(topics))}") + _print(msg=f" note: the probe row(s) stay in {table} tagged {probe}") return HELD @@ -2022,7 +2039,7 @@ def main() -> int: # Fails fast rather than joining the tally below: injecting into a pipeline # whose consumer is not attached proves nothing about the pipeline. - subscription = _verify_loader_subscription() + subscription = _verify_loader_subscription(ingest_url=ingest_url, table=table) if subscription.failed: return 1 diff --git a/scripts/tests/test_post.py b/scripts/tests/test_post.py index 57385a80..ddaaf8d5 100644 --- a/scripts/tests/test_post.py +++ b/scripts/tests/test_post.py @@ -513,3 +513,63 @@ def test_an_unresolvable_service_falls_back_to_its_own_name( post._container_name.cache_clear() assert post._container_name("dfe-engine") == "dfe-engine" + + +_LAG_SAMPLE = ( + 'rdkafka_topic_partition_consumer_lag{topic="main_land",partition="3"} 0\n' +) + + +def _stub_subscription( + monkeypatch: pytest.MonkeyPatch, scrapes: list[str] +) -> list[str]: + """A bus tier running the loader, whose /metrics answers each scrape in turn.""" + posted: list[str] = [] + monkeypatch.setattr(post, "_profile_mk_value", lambda key: ["true"]) + monkeypatch.setattr(post, "_resolved_services", lambda: [post.LOADER_SERVICE]) + monkeypatch.setattr(post, "LOADER_TOPIC_INTERVAL_SECONDS", 0.0) + monkeypatch.setattr(post, "http_get", lambda url, timeout=5: scrapes.pop(0)) + monkeypatch.setattr(post, "http_post", lambda url, body: posted.append(body) or 202) + return posted + + +def test_a_fresh_loader_is_sent_a_probe_until_it_names_a_topic( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """No lag metric exists before the group's first commit, so a quiet stack needs a record.""" + posted = _stub_subscription(monkeypatch, ["", "", _LAG_SAMPLE]) + + claim = post._verify_loader_subscription( + ingest_url="http://localhost:8080/ingest", table="main" + ) + + assert claim == post.HELD + assert len(posted) == 2 + assert all('"marker":"post-' in body and "-probe" in body for body in posted) + + +def test_a_loader_already_fetching_is_sent_no_probe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + posted = _stub_subscription(monkeypatch, [_LAG_SAMPLE]) + + claim = post._verify_loader_subscription( + ingest_url="http://localhost:8080/ingest", table="main" + ) + + assert claim == post.HELD + assert posted == [] + + +def test_a_loader_that_never_fetches_the_probe_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + posted = _stub_subscription(monkeypatch, [""] * 50) + monkeypatch.setattr(post, "LOADER_TOPIC_TIMEOUT_SECONDS", 0.0) + + claim = post._verify_loader_subscription( + ingest_url="http://localhost:8080/ingest", table="main" + ) + + assert claim.failed == 1 + assert posted