From d5d92bb3d8d9b43c6c28fc860e5156e71b8fddde Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 07:32:13 +1100 Subject: [PATCH 1/3] fix(post): check dfe-ui's metrics and correct stale comments The self test checks dfe-ui's metrics like the other pushers. dfe-ui pushes metrics over OTLP/HTTP under the ServiceName dfe-ui. On the live dfe-docker stack running dfe-ui v1.7.45, dfe.otel_metrics_gauge held 70 dfe-ui rows from the last ten minutes. The no-tables failure names dfe-engine because it creates the otel tables. The collector runs with create_schema false. The CPU floor comment in check_compose.py drops the synchronous Kafka poll explanation. scalo-rs polls on tokio's blocking pool. The CPU floor stays unchanged without data to support lowering it. deployment.example.yaml sets hyperdx_enabled and otel_enabled to true, matching every profile in service_profiles.yaml. Its prose calls the target a platform. PROVENANCE.md names the repo that holds tests/fixtures/unencumbered/. --- deployment.example.yaml | 8 ++++---- docs/observability.md | 2 +- resources/demo-cyber/events/PROVENANCE.md | 2 +- scripts/check_compose.py | 10 ++++++---- scripts/post.py | 5 +++-- scripts/tests/test_post.py | 1 + 6 files changed, 16 insertions(+), 12 deletions(-) diff --git a/deployment.example.yaml b/deployment.example.yaml index 20953f7f..61a8b00a 100644 --- a/deployment.example.yaml +++ b/deployment.example.yaml @@ -21,8 +21,8 @@ ## estate endpoints stay blank here and are injected by the operator's own ## tooling at deploy time. -## Which substrate this deployment targets. This renderer requires docker-vm. -## (docker-vm | k8s -- the k8s substrate renders via dfe-infra into a DeployContext.) +## Which platform this deployment targets. This renderer requires docker-vm. +## (docker-vm | k8s -- a k8s deployment renders via dfe-infra into a DeployContext.) substrate: docker-vm ## Image registry hosting the published dfe-* images, pulled anonymously. @@ -58,11 +58,11 @@ docker: kafka_backend: redpanda # apache | redpanda kafbat_enabled: "true" core_enabled: "true" - hyperdx_enabled: "false" + hyperdx_enabled: "true" ## Self-monitoring. otel_enabled starts the bundled collector; setting ## otel_exporter_endpoint instead pushes to an OTLP backend of your own. ## Blank endpoint exports nothing. - otel_enabled: "false" + otel_enabled: "true" otel_exporter_endpoint: "" post_enabled: "true" ## Host exposure. bind_host is the OPERATOR surface (127.0.0.1 on a laptop, diff --git a/docs/observability.md b/docs/observability.md index 38c45679..4ef50e43 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -146,7 +146,7 @@ What reports: | the seven Rust services | metrics and traces | scalo's `metrics` feature pulls `otel-metrics` and `otel-tracing`, so OTLP export is on by default (scalo-rs#30) | | dfe-engine | metrics and traces, on the `opentelemetry` backend `env/engine.env` ships | scalo-py's exporter, whose CLI defaults the backend to `opentelemetry` (scalo-py#11) | | dfe-hunt-runner | metrics, as `dfe_hunt_*` | it has no listener to scrape and its default `opentelemetry` backend pushes on `OTEL_EXPORTER_OTLP_ENDPOINT` alone | -| dfe-ui | metrics, over OTLP/HTTP | `@opentelemetry/sdk-metrics` from its `instrumentation.ts`, on `DFE_OTEL_EXPORTER_HTTP_ENDPOINT`. It also serves `/metrics` on `:3000`. The self test does not require its rows (`OTEL_PUSHER_PREFIXES` in `scripts/post.py`) | +| dfe-ui | metrics, over OTLP/HTTP | `@opentelemetry/sdk-metrics` from its `instrumentation.ts`, on `DFE_OTEL_EXPORTER_HTTP_ENDPOINT`. It also serves `/metrics` on `:3000`. The self test requires its rows, like every pusher here (`OTEL_PUSHER_PREFIXES` in `scripts/post.py`) | | every DFE service | LOGS, via Docker's log driver | nothing in the stack exports logs over OTLP, so stdout is shipped instead -- see below | `OTEL_EXPORTER_OTLP_ENDPOINT` is set on every service regardless, so a rebuilt diff --git a/resources/demo-cyber/events/PROVENANCE.md b/resources/demo-cyber/events/PROVENANCE.md index c03487b9..1cf13c31 100644 --- a/resources/demo-cyber/events/PROVENANCE.md +++ b/resources/demo-cyber/events/PROVENANCE.md @@ -8,6 +8,6 @@ Every `/demo.log` here is written by [`generate.py`](generate.py), seede | [splunk/botsv3](https://github.com/splunk/botsv3) | CC0 1.0, [text](LICENSES/CC0-1.0-botsv3.txt) | `o365/` from its Microsoft 365 management activity records | | [Spamhaus DROP](https://www.spamhaus.org/drop/) | The Spamhaus Project's terms for the list | Eight network ranges from the list of 2026-11-13. Two hosts on each appear as outside addresses on the firewalls and as failed sign-ins | -The samples were read from dfe-transform-elastic's `tests/fixtures/unencumbered/`, which records the same upstreams. Their domains are moved onto `example.com`, `example.net` or `example.org` (RFC 2606). Their people are moved onto the demo's own staff and mailboxes, one name per person across okta, o365 and crowdstrike, and each o365 record names one person in `UserId`, `Actor` and `UserKey`. Staff addresses sit in RFC 5737 space. The DROP hosts are real public addresses on listed networks, used only as sample values. +The samples were read from `tests/fixtures/unencumbered/` in the dfe-transform-elastic repo, which records the same upstreams. Their domains are moved onto `example.com`, `example.net` or `example.org` (RFC 2606). Their people are moved onto the demo's own staff and mailboxes, one name per person across okta, o365 and crowdstrike, and each o365 record names one person in `UserId`, `Actor` and `UserKey`. Staff addresses sit in RFC 5737 space. The DROP hosts are real public addresses on listed networks, used only as sample values. CC0 waives attribution. The Apache-2.0 material requires it, and this table with its licence text is the record. diff --git a/scripts/check_compose.py b/scripts/check_compose.py index 3c886315..441b09bc 100644 --- a/scripts/check_compose.py +++ b/scripts/check_compose.py @@ -164,10 +164,12 @@ # Tokio sizes its worker pool from `available_parallelism()`, which reads the # cgroup CPU quota and FLOORS it. A ceiling under 2.0 therefore leaves a scalo -# service with a single worker thread, and its synchronous Kafka poll loop owns -# that thread -- so the operator surface (/livez, /readyz, /metrics) accepts -# connections and answers none of them while the data path keeps working. -# Observed on dfe-archiver at 1.5 (docs/operating.md#dfe_service_cpus-has-a-floor-of-20-and-it-is-not-about-speed). +# service with a single worker thread, and dfe-archiver at 1.5 stopped answering +# /livez, /readyz and /metrics while its data path kept working +# (docs/operating.md#dfe_service_cpus-has-a-floor-of-20-and-it-is-not-about-speed). +# A Kafka loop holding that thread was the cause. scalo-rs now polls on tokio's +# blocking pool, and nobody has measured whether one worker still starves the +# operator surface at 1.5, so the floor stays until a healthcheck run says so. # # Keyed off the health path rather than a service list so a service added later is # covered without anyone remembering this file exists. Both `*z` paths count: the diff --git a/scripts/post.py b/scripts/post.py index 261910d1..a7d169f8 100644 --- a/scripts/post.py +++ b/scripts/post.py @@ -199,7 +199,7 @@ # The services whose OWN metrics have to reach the otel database, as name prefixes # so a per-source instance (dfe-transform-vrl-filebeat) is covered by its app. -# dfe-ui carries the OTel API and no SDK, so it does not push. +# dfe-ui pushes over OTLP/HTTP under its own ServiceName, so it is checked like the rest. OTEL_PUSHER_PREFIXES = ( "dfe-archiver", "dfe-engine", @@ -210,6 +210,7 @@ "dfe-transform-elastic", "dfe-transform-vector", "dfe-transform-vrl", + "dfe-ui", ) # Metrics only: traces are sampled at 0.05, so a service missing from them is a # sampling outcome rather than one that is not exporting. @@ -872,7 +873,7 @@ def _done(result): if counts is None: _print( msg=f"FAIL no {database} table could be read within " - f"{OTEL_TIMEOUT_SECONDS:.0f}s -- the collector never wrote its schema" + f"{OTEL_TIMEOUT_SECONDS:.0f}s -- dfe-engine has not created the otel tables" ) return Claim(asserted=True, failed=1) landed = {table: count for table, count in counts.items() if count} diff --git a/scripts/tests/test_post.py b/scripts/tests/test_post.py index 362b922d..c1993e8e 100644 --- a/scripts/tests/test_post.py +++ b/scripts/tests/test_post.py @@ -195,6 +195,7 @@ def test_only_the_services_that_push_their_own_metrics_are_expected() -> None: "dfe-loader", "dfe-transform-elastic", "dfe-transform-vrl-filebeat", + "dfe-ui", ] From a7842a18227af9087f1989083a0b936c684335c7 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 07:39:23 +1100 Subject: [PATCH 2/3] fix(dial): rename the deployment dial's target key to platform scripts/render_dial.py reads the dial's top-level `platform` key (docker-vm | k8s). The example and test dial have `platform`. The rejection message says platform. The old key is not read. There is no fallback. A deployment.yaml with the old key is rejected with the same message as any other wrong value. A test covers the retired key, a k8s platform and a missing key. dfe-infra's dial reader and example move to the same key in a paired change. --- deployment.example.yaml | 2 +- scripts/render_dial.py | 8 ++++---- scripts/tests/test_render_dial.py | 20 +++++++++++++++++++- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/deployment.example.yaml b/deployment.example.yaml index 61a8b00a..ae321f78 100644 --- a/deployment.example.yaml +++ b/deployment.example.yaml @@ -23,7 +23,7 @@ ## Which platform this deployment targets. This renderer requires docker-vm. ## (docker-vm | k8s -- a k8s deployment renders via dfe-infra into a DeployContext.) -substrate: docker-vm +platform: docker-vm ## Image registry hosting the published dfe-* images, pulled anonymously. registry: ghcr.io/hyperi-io diff --git a/scripts/render_dial.py b/scripts/render_dial.py index f4da3a26..3869eb3a 100755 --- a/scripts/render_dial.py +++ b/scripts/render_dial.py @@ -151,11 +151,11 @@ def main() -> int: dial = _parse_yaml_subset( text=DEPLOYMENT_DIAL.read_text(encoding="utf-8", errors="replace") ) - substrate = _scalar(dial, ("substrate",)) - if substrate != "docker-vm": + platform = _scalar(dial, ("platform",)) + if platform != "docker-vm": raise SystemExit( - f"render_dial: this renderer handles substrate 'docker-vm', the dial " - f"says {substrate!r} - the k8s substrate renders via dfe-infra" + f"render_dial: this renderer handles platform 'docker-vm', the dial " + f"says {platform!r} - a k8s deployment renders via dfe-infra" ) targets = [ diff --git a/scripts/tests/test_render_dial.py b/scripts/tests/test_render_dial.py index 94dd84f2..811d87d2 100644 --- a/scripts/tests/test_render_dial.py +++ b/scripts/tests/test_render_dial.py @@ -19,7 +19,7 @@ import render_dial _DIAL = """\ -substrate: docker-vm +platform: docker-vm profile: slim-grpc retention: default_ttl_days: 30 @@ -56,3 +56,21 @@ def test_a_retention_with_no_engine_env_refuses_before_writing_anything( assert render_dial.main() == 1 assert "env/engine.env" in capsys.readouterr().err assert dotenv.read_text(encoding="utf-8") == "# DFE_PROFILE=\n" + + +@pytest.mark.parametrize( + "first_line", + ["substrate: docker-vm", "platform: k8s", ""], + ids=["retired key", "other platform", "no platform"], +) +def test_a_dial_without_platform_docker_vm_is_refused( + dial: Path, dotenv: Path, first_line: str +) -> None: + dotenv.write_text("# DFE_PROFILE=\n", encoding="utf-8", newline="\n") + dial.write_text( + f"{first_line}\nprofile: slim-grpc\n", encoding="utf-8", newline="\n" + ) + + with pytest.raises(SystemExit, match="handles platform 'docker-vm'"): + render_dial.main() + assert dotenv.read_text(encoding="utf-8") == "# DFE_PROFILE=\n" From 4ef6a2d962fd1fe3c95d433ccad2def4abc0ad1c Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 08:34:37 +1100 Subject: [PATCH 3/3] fix(post): expect dfe-ui metrics from v1.7.40, explain dial errors dfe-ui pushes metrics from v1.7.40. `make post` expects dfe-ui rows only when DFE_UI_VERSION is v1.7.40 or later. A 2.2.1 stack pins v1.7.34 and no longer fails the check. An unset or local version is still checked. A dial with `substrate:` fails with a message explaining the rename to `platform:`. A dial missing `platform:` reports the missing key. Both cases are in the render_dial tests. The e2e otel message names dfe-engine, which creates those tables. --- scripts/post.py | 27 +++++++++++++++++++++++--- scripts/render_dial.py | 10 ++++++++++ scripts/test_e2e.py | 2 +- scripts/tests/test_post.py | 32 +++++++++++++++++++++++++++++++ scripts/tests/test_render_dial.py | 13 +++++++++---- 5 files changed, 76 insertions(+), 8 deletions(-) diff --git a/scripts/post.py b/scripts/post.py index a7d169f8..9c68ad0e 100644 --- a/scripts/post.py +++ b/scripts/post.py @@ -212,6 +212,9 @@ "dfe-transform-vrl", "dfe-ui", ) +# dfe-ui pushes metrics from this release on, so an older pin is not expected to. +UI_VERSION_KEY = "DFE_UI_VERSION" +UI_PUSHES_METRICS_FROM = (1, 7, 40) # Metrics only: traces are sampled at 0.05, so a service missing from them is a # sampling outcome rather than one that is not exporting. OTEL_METRICS_TABLES = ( @@ -896,12 +899,26 @@ def _done(result): return Claim(asserted=True, failed=failed) -def _otel_expected_services(*, services: list[str], unapplied: set[str]) -> list[str]: +def _ui_pushes_metrics(*, ui_version: str) -> bool: + """Return False only for a dfe-ui release tag older than UI_PUSHES_METRICS_FROM, so a local or unparsed build is still checked.""" + tag = ui_version.split("@", 1)[0].strip().removeprefix("v") + parts = tag.split(".") + if len(parts) != 3 or not all(part.isdigit() for part in parts): + return True + return tuple(int(part) for part in parts) >= UI_PUSHES_METRICS_FROM + + +def _otel_expected_services( + *, services: list[str], unapplied: set[str], ui_version: str = "" +) -> list[str]: """The services whose own metrics have to be in the otel database, less the instances `make apply` has not created.""" + skip_ui = not _ui_pushes_metrics(ui_version=ui_version) return sorted( service for service in services - if service.startswith(OTEL_PUSHER_PREFIXES) and service not in unapplied + if service.startswith(OTEL_PUSHER_PREFIXES) + and service not in unapplied + and not (skip_ui and service == "dfe-ui") ) @@ -978,7 +995,11 @@ def _verify_service_metrics(*, database: str) -> int: f'expected here; `make apply SERVICES="{names}"` creates them, with DEV=1 ' "on a `make dev` stack" ) - expected = _otel_expected_services(services=services, unapplied=unapplied) + expected = _otel_expected_services( + services=services, + unapplied=unapplied, + ui_version=os.environ.get(UI_VERSION_KEY, ""), + ) if not (expected): _print( msg=" note: this profile runs no service that pushes its own metrics" diff --git a/scripts/render_dial.py b/scripts/render_dial.py index 3869eb3a..0767e153 100755 --- a/scripts/render_dial.py +++ b/scripts/render_dial.py @@ -152,6 +152,16 @@ def main() -> int: text=DEPLOYMENT_DIAL.read_text(encoding="utf-8", errors="replace") ) platform = _scalar(dial, ("platform",)) + if platform is None and _scalar(dial, ("substrate",)) is not None: + raise SystemExit( + "render_dial: the dial key `substrate` was renamed to `platform` - " + f"change `substrate:` to `platform:` in {_rel_path(path=DEPLOYMENT_DIAL)}" + ) + if platform is None: + raise SystemExit( + f"render_dial: {_rel_path(path=DEPLOYMENT_DIAL)} has no `platform:` " + "key - set `platform: docker-vm`" + ) if platform != "docker-vm": raise SystemExit( f"render_dial: this renderer handles platform 'docker-vm', the dial " diff --git a/scripts/test_e2e.py b/scripts/test_e2e.py index 344719c2..9febe86a 100755 --- a/scripts/test_e2e.py +++ b/scripts/test_e2e.py @@ -2083,7 +2083,7 @@ def _report(attempt, result): mark_fail( ctx, f"[{test_name}] no '{database}' table readable within " - f"{OTEL_TIMEOUT_SECONDS:.0f}s - the collector never wrote its schema", + f"{OTEL_TIMEOUT_SECONDS:.0f}s - dfe-engine has not created the otel tables", ) return landed = {table: count for table, count in counts.items() if count} diff --git a/scripts/tests/test_post.py b/scripts/tests/test_post.py index c1993e8e..5673c5fe 100644 --- a/scripts/tests/test_post.py +++ b/scripts/tests/test_post.py @@ -199,6 +199,38 @@ def test_only_the_services_that_push_their_own_metrics_are_expected() -> None: ] +@pytest.mark.parametrize( + ("ui_version", "expects_ui"), + [ + ("v1.7.34@sha256:" + "0" * 64, False), + ("v1.7.39", False), + ("v1.7.40", True), + ("1.7.44@sha256:" + "0" * 64, True), + ("v2.0.0", True), + ("", True), + ("local", True), + ], + ids=[ + "2.2.1 pin", + "just before", + "first pusher", + "unprefixed", + "major", + "unset", + "local build", + ], +) +def test_dfe_ui_is_expected_only_from_the_release_that_pushes_metrics( + ui_version: str, expects_ui: bool +) -> None: + expected = post._otel_expected_services( + services=["dfe-engine", "dfe-ui"], unapplied=set(), ui_version=ui_version + ) + + assert ("dfe-ui" in expected) is expects_ui + assert "dfe-engine" in expected + + def test_an_instance_make_apply_has_not_created_is_not_expected() -> None: services = [ "dfe-fetcher-aws-cloudtrail", diff --git a/scripts/tests/test_render_dial.py b/scripts/tests/test_render_dial.py index 811d87d2..eff78620 100644 --- a/scripts/tests/test_render_dial.py +++ b/scripts/tests/test_render_dial.py @@ -12,6 +12,7 @@ the file compose hands the engine. """ +import re from pathlib import Path import pytest @@ -59,18 +60,22 @@ def test_a_retention_with_no_engine_env_refuses_before_writing_anything( @pytest.mark.parametrize( - "first_line", - ["substrate: docker-vm", "platform: k8s", ""], + ("first_line", "message"), + [ + ("substrate: docker-vm", "the dial key `substrate` was renamed to `platform`"), + ("platform: k8s", "handles platform 'docker-vm', the dial says 'k8s'"), + ("", "has no `platform:` key - set `platform: docker-vm`"), + ], ids=["retired key", "other platform", "no platform"], ) def test_a_dial_without_platform_docker_vm_is_refused( - dial: Path, dotenv: Path, first_line: str + dial: Path, dotenv: Path, first_line: str, message: str ) -> None: dotenv.write_text("# DFE_PROFILE=\n", encoding="utf-8", newline="\n") dial.write_text( f"{first_line}\nprofile: slim-grpc\n", encoding="utf-8", newline="\n" ) - with pytest.raises(SystemExit, match="handles platform 'docker-vm'"): + with pytest.raises(SystemExit, match=re.escape(message)): render_dial.main() assert dotenv.read_text(encoding="utf-8") == "# DFE_PROFILE=\n"