Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 41 additions & 13 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -60,15 +60,10 @@ ifneq (,$(filter-out $(BOOTSTRAP_GOALS),$(or $(MAKECMDGOALS),help)))
ifeq ($(strip $(SERVICES)),)
ACTIVE_SERVICES := $(DFE_SERVICES)
else
# On a fresh checkout make parses once before .profile.mk exists, then
# remakes it and parses again; the name check only means anything on
# the second pass, when DFE_SERVICES is populated.
ifneq ($(strip $(DFE_SERVICES)),)
INVALID_SERVICES := $(filter-out $(DFE_SERVICES),$(SERVICES))
ifneq ($(INVALID_SERVICES),)
$(error 'SERVICES' contains names not in the resolved stack: $(INVALID_SERVICES). Available: $(DFE_SERVICES))
endif
endif
# Checked by the services-valid target, not here: this parse can hold the
# .profile.mk from before the engine declared a new per-source instance,
# and make regenerates it and parses again before any recipe runs.
INVALID_SERVICES := $(filter-out $(DFE_SERVICES),$(SERVICES))
ACTIVE_SERVICES := $(filter $(SERVICES),$(DFE_SERVICES))
endif
# `make dev LOCAL="dfe-engine dfe-ui"` builds only those and keeps the rest
Expand Down Expand Up @@ -255,6 +250,15 @@ endif
.PHONY: FORCE
FORCE:

# A prerequisite of every target that narrows to SERVICES. Recipes run only on
# the pass that read the regenerated .profile.mk, so a per-source instance the
# engine declared since the last run is a valid name here.
.PHONY: services-valid
services-valid:
ifneq ($(strip $(INVALID_SERVICES)),)
$(error 'SERVICES' contains names not in the resolved stack: $(INVALID_SERVICES). Available: $(DFE_SERVICES))
endif

.env:
@python3 scripts/init.py

Expand Down Expand Up @@ -356,15 +360,15 @@ endif
# which stay private until GA, so both are HyperI-internal until then; `make ci`
# runs the same stack from the public images.
.PHONY: dev
dev: env-files dev-posture down storage-dirs ## Build local DFE images from source and start the dev stack (LOCAL="svc ..." builds only those; AUTH=real for a minted login), then print the access summary. HyperI-internal: clones private repos
dev: services-valid env-files dev-posture down storage-dirs ## Build local DFE images from source and start the dev stack (LOCAL="svc ..." builds only those; AUTH=real for a minted login), then print the access summary. HyperI-internal: clones private repos
docker compose $(DEV_PULL_FLAGS) $(PROFILE_FLAGS) pull
python3 scripts/build_dev_images.py $(DEV_OVERLAY_ARG) $(DEV_BUILD)
docker compose $(DEV_FLAGS) $(PROFILE_FLAGS) up -d $(ACTIVE_SERVICES)
@$(MAKE) --no-print-directory post || { echo "post: SELF TEST FAILED -- the stack is up, but it did not prove it moves data. Run 'make post' for detail."; exit 1; }
@$(MAKE) --no-print-directory creds

.PHONY: dev-build
dev-build: ## Build local DFE images from source (no start; honours LOCAL). HyperI-internal: clones private repos
dev-build: services-valid ## Build local DFE images from source (no start; honours LOCAL). HyperI-internal: clones private repos
docker compose $(DEV_PULL_FLAGS) $(PROFILE_FLAGS) pull
python3 scripts/build_dev_images.py $(DEV_OVERLAY_ARG) $(DEV_BUILD)

Expand All @@ -373,7 +377,7 @@ dev-build: ## Build local DFE images from source (no start; honours LOCAL). Hype
# ---------------------------------------------------------------------------

.PHONY: ci
ci: login env-files down storage-dirs ## Pull and start infra and registry DFE images. Prints no credentials -- run `make creds` for those
ci: services-valid login env-files down storage-dirs ## Pull and start infra and registry DFE images. Prints no credentials -- run `make creds` for those
docker compose -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) $(PROFILE_FLAGS) pull
docker compose -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) $(PROFILE_FLAGS) pull $(ACTIVE_SERVICES)
docker compose -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) $(PROFILE_FLAGS) up -d $(ACTIVE_SERVICES)
Expand All @@ -386,8 +390,32 @@ ci: login env-files down storage-dirs ## Pull and start infra and registry DFE
up: ci ## Start the stack from the pinned registry images, then print the access summary (password on a TTY only) and write access-summary.md
@python3 scripts/creds.py --write

# The profile is re-resolved first, as for every goal outside BOOTSTRAP_GOALS, so
# a per-source instance the engine rendered since the stack started is a service
# by now. No `down` here: compose recreates only a service whose resolved
# definition changed, and leaves every volume alone. DEV=1 uses the file chain
# `make dev` starts with rather than the registry one.
#
# A config file the engine rewrote under a running container changes no service
# definition, so `up` leaves that container as it is. Naming services therefore
# restarts them as well, which is the engine's hint: one command creates,
# recreates or restarts whatever the write needs. With no SERVICES nothing
# unchanged is touched.
ifneq ($(strip $(DEV)),)
APPLY_FLAGS = $(DEV_FLAGS)
else
APPLY_FLAGS = -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS)
endif

.PHONY: apply
apply: services-valid storage-dirs ## Start what the re-resolved profile adds or changes on a running stack, a new per-source instance included; SERVICES="..." also restarts those, so a rewritten mounted config is read; with no SERVICES unchanged containers are left alone. Volumes are never touched (DEV=1 for a `make dev` stack)
docker compose $(APPLY_FLAGS) $(PROFILE_FLAGS) up -d $(ACTIVE_SERVICES)
ifneq ($(strip $(SERVICES)),)
docker compose $(APPLY_FLAGS) $(PROFILE_FLAGS) restart --no-deps $(ACTIVE_SERVICES)
endif

.PHONY: ci-pull
ci-pull: login ## Pull infra and registry DFE images
ci-pull: services-valid login ## Pull infra and registry DFE images
docker compose -f docker-compose.yml $(PROFILE_FLAGS) pull
docker compose -f docker-compose.yml $(PROFILE_FLAGS) pull $(ACTIVE_SERVICES)

Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ In `dev` mode, they build from each repo's own Dockerfile (not the shared Rust b
| `make env-files` | Assert every `env/<service>.env` exists, creating any the templates have gained |
| `make creds` | Print the access summary -- console URL, admin login, where the break-glass password lives. The password prints on a TTY only; a pipe, a file or `DFE_CREDS_SHOW=0` gets the `.env` key instead |
| `make up` | Start the pinned stack and print the access summary |
| `make apply` | Start what the re-resolved profile adds or changes on a running stack, a new per-source instance included, recreating nothing unchanged and touching no volume. `SERVICES="..."` also restarts those services, so a mounted config the engine rewrote is read. It is the command the engine's restart hints name (`DEV=1` for a `make dev` stack) |
| `make dev` | Build local DFE images from source and start the stack (`LOCAL="..."` builds only those, rest pinned) |
| `make dev-build` | Build local DFE images from source (no start) |
| `make ci` | Pull and start infra and registry DFE images. Prints no credentials -- `make up` is the same start plus `make creds` |
Expand Down
3 changes: 1 addition & 2 deletions docs/troubleshooting.md
Original file line number Diff line number Diff line change
Expand Up @@ -148,8 +148,7 @@ The e2e harness now does this for itself: `clean_topics` deletes the `_load`
sibling of every expected `_land` topic, so a run depends on the test definition
rather than on the broker's history.

`make post` asserts the loader is fetching a topic before it injects anything, so
an empty subscription is reported as one.
`make post` asserts the loader is fetching a topic before it injects anything, so an empty subscription is reported as one. The loader names no topic until its group has committed an offset, so on a stack that has carried no traffic the self test sends one probe record per attempt, tagged `post-<pid>-<hex>-probe`, and those rows stay in the table.

## Events are accepted but nothing lands

Expand Down
25 changes: 21 additions & 4 deletions scripts/post.py
Original file line number Diff line number Diff line change
Expand Up @@ -999,13 +999,19 @@ def _metric_label_values(body: str, name: str, label: str) -> set[str]:
return found


def _verify_loader_subscription() -> Claim:
def _verify_loader_subscription(*, ingest_url: str, table: str) -> Claim:
"""Prove the loader is fetching a topic, on a tier that has a bus.

The loader discovers its topics from the broker, so a topic that never appears
-- or one scalo suppresses in favour of a `_load` sibling -- leaves it ready
and healthy with nothing to read. Asserted before the injection, because after
it the same fault is indistinguishable from a broken loader or a missing table.

librdkafka reports a partition's lag as -1 until the group commits an offset
on it, and scalo exports only a lag it can state, so a fresh stack shows no
topic at all however well subscribed it is. Each attempt that sees none sends
one probe record through the ingest edge first: a subscribed loader loads and
commits it and names its topic, and one subscribed to nothing never does.
"""
if _profile_mk_value(key=TRANSPORT_BUS_KEY)[:1] != ["true"]:
_print(msg="SKIP this tier has no bus -- the loader is handed its events")
Expand All @@ -1022,14 +1028,24 @@ def _verify_loader_subscription() -> Claim:
or LOADER_PROMETHEUS_PORT
)
base = f"http://{bind}:{port}"
_print(msg=f"Waiting for {LOADER_SERVICE} to be fetching a topic at {base}")
probe = f"{_run_id()}-probe"
_print(
msg=f"Waiting for {LOADER_SERVICE} to be fetching a topic at {base}, "
f"sending probe records tagged {probe}"
)

def _fetching():
try:
body = http_get(f"{base}/metrics", timeout=5)
except Exception: # noqa: BLE001 - unreachable and not-yet-serving are one answer
return set()
return _metric_label_values(body, LOADER_TOPIC_METRIC, LOADER_TOPIC_LABEL)
topics = _metric_label_values(body, LOADER_TOPIC_METRIC, LOADER_TOPIC_LABEL)
if not (topics):
try:
http_post(ingest_url, marked_event(marker=probe, source=table))
except Exception: # noqa: BLE001 - a refused probe is another attempt
pass
return topics

def _report(attempt, result):
_print(
Expand All @@ -1052,6 +1068,7 @@ def _report(attempt, result):
)
return Claim(asserted=True, failed=1)
_print(msg=f"PASS {LOADER_SERVICE} is fetching {', '.join(sorted(topics))}")
_print(msg=f" note: the probe row(s) stay in {table} tagged {probe}")
return HELD


Expand Down Expand Up @@ -2022,7 +2039,7 @@ def main() -> int:

# Fails fast rather than joining the tally below: injecting into a pipeline
# whose consumer is not attached proves nothing about the pipeline.
subscription = _verify_loader_subscription()
subscription = _verify_loader_subscription(ingest_url=ingest_url, table=table)
if subscription.failed:
return 1

Expand Down
60 changes: 60 additions & 0 deletions scripts/tests/test_post.py
Original file line number Diff line number Diff line change
Expand Up @@ -513,3 +513,63 @@ def test_an_unresolvable_service_falls_back_to_its_own_name(
post._container_name.cache_clear()

assert post._container_name("dfe-engine") == "dfe-engine"


_LAG_SAMPLE = (
'rdkafka_topic_partition_consumer_lag{topic="main_land",partition="3"} 0\n'
)


def _stub_subscription(
monkeypatch: pytest.MonkeyPatch, scrapes: list[str]
) -> list[str]:
"""A bus tier running the loader, whose /metrics answers each scrape in turn."""
posted: list[str] = []
monkeypatch.setattr(post, "_profile_mk_value", lambda key: ["true"])
monkeypatch.setattr(post, "_resolved_services", lambda: [post.LOADER_SERVICE])
monkeypatch.setattr(post, "LOADER_TOPIC_INTERVAL_SECONDS", 0.0)
monkeypatch.setattr(post, "http_get", lambda url, timeout=5: scrapes.pop(0))
monkeypatch.setattr(post, "http_post", lambda url, body: posted.append(body) or 202)
return posted


def test_a_fresh_loader_is_sent_a_probe_until_it_names_a_topic(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""No lag metric exists before the group's first commit, so a quiet stack needs a record."""
posted = _stub_subscription(monkeypatch, ["", "", _LAG_SAMPLE])

claim = post._verify_loader_subscription(
ingest_url="http://localhost:8080/ingest", table="main"
)

assert claim == post.HELD
assert len(posted) == 2
assert all('"marker":"post-' in body and "-probe" in body for body in posted)


def test_a_loader_already_fetching_is_sent_no_probe(
monkeypatch: pytest.MonkeyPatch,
) -> None:
posted = _stub_subscription(monkeypatch, [_LAG_SAMPLE])

claim = post._verify_loader_subscription(
ingest_url="http://localhost:8080/ingest", table="main"
)

assert claim == post.HELD
assert posted == []


def test_a_loader_that_never_fetches_the_probe_fails(
monkeypatch: pytest.MonkeyPatch,
) -> None:
posted = _stub_subscription(monkeypatch, [""] * 50)
monkeypatch.setattr(post, "LOADER_TOPIC_TIMEOUT_SECONDS", 0.0)

claim = post._verify_loader_subscription(
ingest_url="http://localhost:8080/ingest", table="main"
)

assert claim.failed == 1
assert posted
Loading