diff --git a/Makefile b/Makefile index 29108ca7..f46b4359 100644 --- a/Makefile +++ b/Makefile @@ -60,15 +60,10 @@ ifneq (,$(filter-out $(BOOTSTRAP_GOALS),$(or $(MAKECMDGOALS),help))) ifeq ($(strip $(SERVICES)),) ACTIVE_SERVICES := $(DFE_SERVICES) else - # On a fresh checkout make parses once before .profile.mk exists, then - # remakes it and parses again; the name check only means anything on - # the second pass, when DFE_SERVICES is populated. - ifneq ($(strip $(DFE_SERVICES)),) - INVALID_SERVICES := $(filter-out $(DFE_SERVICES),$(SERVICES)) - ifneq ($(INVALID_SERVICES),) - $(error 'SERVICES' contains names not in the resolved stack: $(INVALID_SERVICES). Available: $(DFE_SERVICES)) - endif - endif + # Checked by the services-valid target, not here: this parse can hold the + # .profile.mk from before the engine declared a new per-source instance, + # and make regenerates it and parses again before any recipe runs. + INVALID_SERVICES := $(filter-out $(DFE_SERVICES),$(SERVICES)) ACTIVE_SERVICES := $(filter $(SERVICES),$(DFE_SERVICES)) endif # `make dev LOCAL="dfe-engine dfe-ui"` builds only those and keeps the rest @@ -255,6 +250,15 @@ endif .PHONY: FORCE FORCE: +# A prerequisite of every target that narrows to SERVICES. Recipes run only on +# the pass that read the regenerated .profile.mk, so a per-source instance the +# engine declared since the last run is a valid name here. +.PHONY: services-valid +services-valid: +ifneq ($(strip $(INVALID_SERVICES)),) + $(error 'SERVICES' contains names not in the resolved stack: $(INVALID_SERVICES). Available: $(DFE_SERVICES)) +endif + .env: @python3 scripts/init.py @@ -356,7 +360,7 @@ endif # which stay private until GA, so both are HyperI-internal until then; `make ci` # runs the same stack from the public images. .PHONY: dev -dev: env-files dev-posture down storage-dirs ## Build local DFE images from source and start the dev stack (LOCAL="svc ..." builds only those; AUTH=real for a minted login), then print the access summary. HyperI-internal: clones private repos +dev: services-valid env-files dev-posture down storage-dirs ## Build local DFE images from source and start the dev stack (LOCAL="svc ..." builds only those; AUTH=real for a minted login), then print the access summary. HyperI-internal: clones private repos docker compose $(DEV_PULL_FLAGS) $(PROFILE_FLAGS) pull python3 scripts/build_dev_images.py $(DEV_OVERLAY_ARG) $(DEV_BUILD) docker compose $(DEV_FLAGS) $(PROFILE_FLAGS) up -d $(ACTIVE_SERVICES) @@ -364,7 +368,7 @@ dev: env-files dev-posture down storage-dirs ## Build local DFE images from sour @$(MAKE) --no-print-directory creds .PHONY: dev-build -dev-build: ## Build local DFE images from source (no start; honours LOCAL). HyperI-internal: clones private repos +dev-build: services-valid ## Build local DFE images from source (no start; honours LOCAL). HyperI-internal: clones private repos docker compose $(DEV_PULL_FLAGS) $(PROFILE_FLAGS) pull python3 scripts/build_dev_images.py $(DEV_OVERLAY_ARG) $(DEV_BUILD) @@ -373,7 +377,7 @@ dev-build: ## Build local DFE images from source (no start; honours LOCAL). Hype # --------------------------------------------------------------------------- .PHONY: ci -ci: login env-files down storage-dirs ## Pull and start infra and registry DFE images. Prints no credentials -- run `make creds` for those +ci: services-valid login env-files down storage-dirs ## Pull and start infra and registry DFE images. Prints no credentials -- run `make creds` for those docker compose -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) $(PROFILE_FLAGS) pull docker compose -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) $(PROFILE_FLAGS) pull $(ACTIVE_SERVICES) docker compose -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) $(PROFILE_FLAGS) up -d $(ACTIVE_SERVICES) @@ -386,8 +390,32 @@ ci: login env-files down storage-dirs ## Pull and start infra and registry DFE up: ci ## Start the stack from the pinned registry images, then print the access summary (password on a TTY only) and write access-summary.md @python3 scripts/creds.py --write +# The profile is re-resolved first, as for every goal outside BOOTSTRAP_GOALS, so +# a per-source instance the engine rendered since the stack started is a service +# by now. No `down` here: compose recreates only a service whose resolved +# definition changed, and leaves every volume alone. DEV=1 uses the file chain +# `make dev` starts with rather than the registry one. +# +# A config file the engine rewrote under a running container changes no service +# definition, so `up` leaves that container as it is. Naming services therefore +# restarts them as well, which is the engine's hint: one command creates, +# recreates or restarts whatever the write needs. With no SERVICES nothing +# unchanged is touched. +ifneq ($(strip $(DEV)),) + APPLY_FLAGS = $(DEV_FLAGS) +else + APPLY_FLAGS = -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) +endif + +.PHONY: apply +apply: services-valid storage-dirs ## Start what the re-resolved profile adds or changes on a running stack, a new per-source instance included; SERVICES="..." also restarts those, so a rewritten mounted config is read; with no SERVICES unchanged containers are left alone. Volumes are never touched (DEV=1 for a `make dev` stack) + docker compose $(APPLY_FLAGS) $(PROFILE_FLAGS) up -d $(ACTIVE_SERVICES) +ifneq ($(strip $(SERVICES)),) + docker compose $(APPLY_FLAGS) $(PROFILE_FLAGS) restart --no-deps $(ACTIVE_SERVICES) +endif + .PHONY: ci-pull -ci-pull: login ## Pull infra and registry DFE images +ci-pull: services-valid login ## Pull infra and registry DFE images docker compose -f docker-compose.yml $(PROFILE_FLAGS) pull docker compose -f docker-compose.yml $(PROFILE_FLAGS) pull $(ACTIVE_SERVICES) diff --git a/README.md b/README.md index fd2b6b59..ed02e01a 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,7 @@ In `dev` mode, they build from each repo's own Dockerfile (not the shared Rust b | `make env-files` | Assert every `env/.env` exists, creating any the templates have gained | | `make creds` | Print the access summary -- console URL, admin login, where the break-glass password lives. The password prints on a TTY only; a pipe, a file or `DFE_CREDS_SHOW=0` gets the `.env` key instead | | `make up` | Start the pinned stack and print the access summary | +| `make apply` | Start what the re-resolved profile adds or changes on a running stack, a new per-source instance included, recreating nothing unchanged and touching no volume. `SERVICES="..."` also restarts those services, so a mounted config the engine rewrote is read. It is the command the engine's restart hints name (`DEV=1` for a `make dev` stack) | | `make dev` | Build local DFE images from source and start the stack (`LOCAL="..."` builds only those, rest pinned) | | `make dev-build` | Build local DFE images from source (no start) | | `make ci` | Pull and start infra and registry DFE images. Prints no credentials -- `make up` is the same start plus `make creds` | diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index c11629f4..a2a1ccd3 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -148,8 +148,7 @@ The e2e harness now does this for itself: `clean_topics` deletes the `_load` sibling of every expected `_land` topic, so a run depends on the test definition rather than on the broker's history. -`make post` asserts the loader is fetching a topic before it injects anything, so -an empty subscription is reported as one. +`make post` asserts the loader is fetching a topic before it injects anything, so an empty subscription is reported as one. The loader names no topic until its group has committed an offset, so on a stack that has carried no traffic the self test sends one probe record per attempt, tagged `post---probe`, and those rows stay in the table. ## Events are accepted but nothing lands diff --git a/scripts/post.py b/scripts/post.py index 7afd4646..896d6b5c 100644 --- a/scripts/post.py +++ b/scripts/post.py @@ -999,13 +999,19 @@ def _metric_label_values(body: str, name: str, label: str) -> set[str]: return found -def _verify_loader_subscription() -> Claim: +def _verify_loader_subscription(*, ingest_url: str, table: str) -> Claim: """Prove the loader is fetching a topic, on a tier that has a bus. The loader discovers its topics from the broker, so a topic that never appears -- or one scalo suppresses in favour of a `_load` sibling -- leaves it ready and healthy with nothing to read. Asserted before the injection, because after it the same fault is indistinguishable from a broken loader or a missing table. + + librdkafka reports a partition's lag as -1 until the group commits an offset + on it, and scalo exports only a lag it can state, so a fresh stack shows no + topic at all however well subscribed it is. Each attempt that sees none sends + one probe record through the ingest edge first: a subscribed loader loads and + commits it and names its topic, and one subscribed to nothing never does. """ if _profile_mk_value(key=TRANSPORT_BUS_KEY)[:1] != ["true"]: _print(msg="SKIP this tier has no bus -- the loader is handed its events") @@ -1022,14 +1028,24 @@ def _verify_loader_subscription() -> Claim: or LOADER_PROMETHEUS_PORT ) base = f"http://{bind}:{port}" - _print(msg=f"Waiting for {LOADER_SERVICE} to be fetching a topic at {base}") + probe = f"{_run_id()}-probe" + _print( + msg=f"Waiting for {LOADER_SERVICE} to be fetching a topic at {base}, " + f"sending probe records tagged {probe}" + ) def _fetching(): try: body = http_get(f"{base}/metrics", timeout=5) except Exception: # noqa: BLE001 - unreachable and not-yet-serving are one answer return set() - return _metric_label_values(body, LOADER_TOPIC_METRIC, LOADER_TOPIC_LABEL) + topics = _metric_label_values(body, LOADER_TOPIC_METRIC, LOADER_TOPIC_LABEL) + if not (topics): + try: + http_post(ingest_url, marked_event(marker=probe, source=table)) + except Exception: # noqa: BLE001 - a refused probe is another attempt + pass + return topics def _report(attempt, result): _print( @@ -1052,6 +1068,7 @@ def _report(attempt, result): ) return Claim(asserted=True, failed=1) _print(msg=f"PASS {LOADER_SERVICE} is fetching {', '.join(sorted(topics))}") + _print(msg=f" note: the probe row(s) stay in {table} tagged {probe}") return HELD @@ -2022,7 +2039,7 @@ def main() -> int: # Fails fast rather than joining the tally below: injecting into a pipeline # whose consumer is not attached proves nothing about the pipeline. - subscription = _verify_loader_subscription() + subscription = _verify_loader_subscription(ingest_url=ingest_url, table=table) if subscription.failed: return 1 diff --git a/scripts/tests/test_post.py b/scripts/tests/test_post.py index 57385a80..ddaaf8d5 100644 --- a/scripts/tests/test_post.py +++ b/scripts/tests/test_post.py @@ -513,3 +513,63 @@ def test_an_unresolvable_service_falls_back_to_its_own_name( post._container_name.cache_clear() assert post._container_name("dfe-engine") == "dfe-engine" + + +_LAG_SAMPLE = ( + 'rdkafka_topic_partition_consumer_lag{topic="main_land",partition="3"} 0\n' +) + + +def _stub_subscription( + monkeypatch: pytest.MonkeyPatch, scrapes: list[str] +) -> list[str]: + """A bus tier running the loader, whose /metrics answers each scrape in turn.""" + posted: list[str] = [] + monkeypatch.setattr(post, "_profile_mk_value", lambda key: ["true"]) + monkeypatch.setattr(post, "_resolved_services", lambda: [post.LOADER_SERVICE]) + monkeypatch.setattr(post, "LOADER_TOPIC_INTERVAL_SECONDS", 0.0) + monkeypatch.setattr(post, "http_get", lambda url, timeout=5: scrapes.pop(0)) + monkeypatch.setattr(post, "http_post", lambda url, body: posted.append(body) or 202) + return posted + + +def test_a_fresh_loader_is_sent_a_probe_until_it_names_a_topic( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """No lag metric exists before the group's first commit, so a quiet stack needs a record.""" + posted = _stub_subscription(monkeypatch, ["", "", _LAG_SAMPLE]) + + claim = post._verify_loader_subscription( + ingest_url="http://localhost:8080/ingest", table="main" + ) + + assert claim == post.HELD + assert len(posted) == 2 + assert all('"marker":"post-' in body and "-probe" in body for body in posted) + + +def test_a_loader_already_fetching_is_sent_no_probe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + posted = _stub_subscription(monkeypatch, [_LAG_SAMPLE]) + + claim = post._verify_loader_subscription( + ingest_url="http://localhost:8080/ingest", table="main" + ) + + assert claim == post.HELD + assert posted == [] + + +def test_a_loader_that_never_fetches_the_probe_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + posted = _stub_subscription(monkeypatch, [""] * 50) + monkeypatch.setattr(post, "LOADER_TOPIC_TIMEOUT_SECONDS", 0.0) + + claim = post._verify_loader_subscription( + ingest_url="http://localhost:8080/ingest", table="main" + ) + + assert claim.failed == 1 + assert posted