From 5a1cf3906979bd8d513af701e4ef2c471d8bbc2b Mon Sep 17 00:00:00 2001 From: Derek Date: Sat, 26 Sep 2026 21:17:20 +1000 Subject: [PATCH] fix: env/ group access and e2e fixture names The engine writes env/.custom.env into the operator's checkout as UID 1000. On a host where the operator is anyone else it could not write there, and Compose could not read what it wrote. dfe-engine now runs with the checkout's group (group_add, DFE_DEV_GID), and make init plus every start target make env/ group-writable and setgid, so each new file keeps the group Compose reads it as. dfe-transform-vector-filebeat and dfe-transform-elastic-cisco-ios were exactly the names the engine gives a real instance for a source called filebeat or cisco-ios, and Compose merges two services sharing a name. They are now dfe-transform-e2e-vector-filebeat and dfe-transform-e2e-elastic-cisco-ios, with their env templates renamed to match. The DFE_TRANSFORM_*_FILEBEAT and *_CISCO_IOS variables keep their names so an existing .env still applies. --- Makefile | 3 +- docker-compose.container-logs.yml | 8 +- docker-compose.override.yml | 4 +- docker-compose.yml | 32 ++++--- docs/architecture.md | 6 +- docs/configuration.md | 8 +- ...nv => transform-e2e-elastic-cisco-ios.env} | 0 ....env => transform-e2e-vector-filebeat.env} | 0 scripts/build_dev_images.py | 4 +- scripts/check_compose.py | 4 +- scripts/env_files.py | 9 +- scripts/init.py | 36 +++++++- scripts/post.py | 7 +- scripts/resolve_profile.py | 8 +- scripts/test_e2e.py | 4 +- .../tests/test_engine_only_schema_control.py | 2 +- scripts/tests/test_env_dir.py | 85 +++++++++++++++++++ service_profiles.yaml | 6 +- 18 files changed, 184 insertions(+), 42 deletions(-) rename env.example/{transform-elastic-cisco-ios.env => transform-e2e-elastic-cisco-ios.env} (100%) rename env.example/{transform-vector-filebeat.env => transform-e2e-vector-filebeat.env} (100%) create mode 100644 scripts/tests/test_env_dir.py diff --git a/Makefile b/Makefile index ff9489ae..7d8146cf 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,8 @@ export CLICKHOUSE_USERNAME # Host UID/GID passed to live-mode containers (docker-compose.live.yml) that # write to bind-mounted host dirs (dfe-engine config/schemas), so files are owned # by the host user rather than the image user and writes don't hit permission -# errors. +# errors. The GID is also the group every stack adds to dfe-engine, which is how +# it writes env/.custom.env into the checkout. export DFE_DEV_UID := $(shell id -u) export DFE_DEV_GID := $(shell id -g) diff --git a/docker-compose.container-logs.yml b/docker-compose.container-logs.yml index 8346ef90..dd0798a7 100644 --- a/docker-compose.container-logs.yml +++ b/docker-compose.container-logs.yml @@ -52,13 +52,13 @@ services: logging: *container-logs dfe-receiver: logging: *container-logs - dfe-transform-elastic: + dfe-transform-e2e-elastic-cisco-ios: logging: *container-logs - dfe-transform-elastic-cisco-ios: + dfe-transform-e2e-vector-filebeat: logging: *container-logs - dfe-transform-vector: + dfe-transform-elastic: logging: *container-logs - dfe-transform-vector-filebeat: + dfe-transform-vector: logging: *container-logs dfe-transform-vrl: logging: *container-logs diff --git a/docker-compose.override.yml b/docker-compose.override.yml index 394019c3..95e7626a 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -45,13 +45,13 @@ services: dfe-transform-elastic: image: dfe-transform-elastic:local - dfe-transform-elastic-cisco-ios: + dfe-transform-e2e-elastic-cisco-ios: image: dfe-transform-elastic:local dfe-transform-vector: image: dfe-transform-vector:local - dfe-transform-vector-filebeat: + dfe-transform-e2e-vector-filebeat: image: dfe-transform-vector:local dfe-transform-vrl: diff --git a/docker-compose.yml b/docker-compose.yml index eefd48b5..e9480203 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -750,7 +750,7 @@ services: retries: 5 # =========================================================================== - # dfe-transform-elastic-cisco-ios + # dfe-transform-e2e-elastic-cisco-ios # A SECOND instance of the same image, for the cisco-ios source alone -- the # per-source shape dfe-transform-vrl-filebeat carries on the VRL side. The # receiver routes _source=cisco-ios to cisco-ios_land, this instance is the @@ -759,24 +759,27 @@ services: # `source.name` in the config is the whole difference between two instances of # this app: the transform is compiled in, so a second source adds a block like # this one and nothing else. + # + # Named off the `-` shape because dfe-engine names a real + # per-source instance that way, and Compose merges two services sharing a name. # =========================================================================== - dfe-transform-elastic-cisco-ios: + dfe-transform-e2e-elastic-cisco-ios: image: ${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-transform-elastic:${DFE_TRANSFORM_ELASTIC_VERSION:?run 'make stack VERSION=X.Y.Z' to pin DFE_TRANSFORM_ELASTIC_VERSION} - container_name: dfe-transform-elastic-cisco-ios + container_name: dfe-transform-e2e-elastic-cisco-ios profiles: ["dfe"] restart: unless-stopped deploy: *limits-service ports: - ${DFE_BIND_HOST:-127.0.0.1}:${DFE_TRANSFORM_ELASTIC_CISCO_IOS_PROMETHEUS_PORT:-9089}:9090 env_file: - - path: env/transform-elastic-cisco-ios.env + - path: env/transform-e2e-elastic-cisco-ios.env required: false environment: LOG_LEVEL: ${LOG_LEVEL:-info} LOG_FORMAT: ${LOG_FORMAT:-text} OTEL_EXPORTER_OTLP_ENDPOINT: ${DFE_OTEL_EXPORTER_ENDPOINT:-} - OTEL_SERVICE_NAME: dfe-transform-elastic-cisco-ios + OTEL_SERVICE_NAME: dfe-transform-e2e-elastic-cisco-ios depends_on: # No dlq-init here: this service does not mount dlq-spool. dfe-engine # creates the topics it reads and writes, so it waits on that instead. @@ -849,7 +852,7 @@ services: retries: 5 # =========================================================================== - # dfe-transform-vector-filebeat + # dfe-transform-e2e-vector-filebeat # A SECOND instance of the same image, for the filebeat source alone -- the # dfe-transform-vector half of the shape dfe-transform-vrl-filebeat carries # below. The receiver routes _source=filebeat-vector to filebeat-vector_land, @@ -861,11 +864,14 @@ services: # filebeat program and the timezone table it looks up is mounted from the # filebeat pipeline's own data directory, shared with the VRL instance rather # than copied. + # + # Named off the `-` shape because dfe-engine names a real + # per-source instance that way, and Compose merges two services sharing a name. # =========================================================================== - dfe-transform-vector-filebeat: + dfe-transform-e2e-vector-filebeat: image: ${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-transform-vector:${DFE_TRANSFORM_VECTOR_VERSION:?run 'make stack VERSION=X.Y.Z' to pin DFE_TRANSFORM_VECTOR_VERSION} - container_name: dfe-transform-vector-filebeat + container_name: dfe-transform-e2e-vector-filebeat profiles: ["dfe"] restart: unless-stopped deploy: *limits-service @@ -873,17 +879,17 @@ services: - ${DFE_BIND_HOST:-127.0.0.1}:${DFE_TRANSFORM_VECTOR_FILEBEAT_PROMETHEUS_PORT:-9098}:9090 - ${DFE_BIND_HOST:-127.0.0.1}:${DFE_TRANSFORM_VECTOR_FILEBEAT_API_PORT:-8687}:8686 env_file: - - path: env/transform-vector-filebeat.env + - path: env/transform-e2e-vector-filebeat.env required: false # The custom keys the engine writes for this app; an up -d applies them, # a restart keeps the environment the container already has. - - path: env/transform-vector-filebeat.custom.env + - path: env/transform-e2e-vector-filebeat.custom.env required: false environment: LOG_LEVEL: ${LOG_LEVEL:-info} LOG_FORMAT: ${LOG_FORMAT:-text} OTEL_EXPORTER_OTLP_ENDPOINT: ${DFE_OTEL_EXPORTER_ENDPOINT:-} - OTEL_SERVICE_NAME: dfe-transform-vector-filebeat + OTEL_SERVICE_NAME: dfe-transform-e2e-vector-filebeat depends_on: # No dlq-init here: this service does not mount dlq-spool. # The topic authority: dfe-engine creates the declared topic set at its own @@ -1011,6 +1017,10 @@ services: profiles: ["core"] restart: unless-stopped deploy: *limits-service + # The engine runs as UID 1000 and ./env belongs to the operator, so it writes + # there through the operator's group; `make` exports DFE_DEV_GID. + group_add: + - "${DFE_DEV_GID:-1000}" ports: # dfe-proxy reaches the engine over the docker network; this host mapping # is for direct API work. PRODUCT class -- it is the API the UI consumes -- diff --git a/docs/architecture.md b/docs/architecture.md index c77df021..0ada338f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -123,18 +123,20 @@ mode, profile resolution and the checks working. | Piece | `kafka-filebeat`, on dfe-transform-vrl | `kafka-filebeat-vector`, on dfe-transform-vector | |---|---|---| -| Compose service | `dfe-transform-vrl-filebeat`, the same image, its own metrics port | `dfe-transform-vector-filebeat`, likewise | +| Compose service | `dfe-transform-vrl-filebeat`, the same image, its own metrics port | `dfe-transform-e2e-vector-filebeat`, likewise | | Config | `config/transform-vrl/filebeat.yaml` -- `filebeat_land` in, `filebeat_load` out | `config/transform-vector/filebeat.yaml` -- `dfe_source: filebeat-vector` derives both topics | | Program | `config/transform-vrl/transforms-filebeat/`, vendored from dfe-transform-vrl | `config/transform-vector/transforms-filebeat/`, the same VRL inside a Vector `remap`, vendored from dfe-transform-vector | | Loader | `config/loader/kafka-load-filebeat.yaml` lists `filebeat_load` alongside `main_load` | `config/loader/kafka-load-filebeat-vector.yaml` lists `filebeat-vector_load` | | Table | `dfe.filebeat`, from the `_load` topic name -- the loader strips the suffix and routes on it | ``dfe.`filebeat-vector` ``, the same way | | Enrichment data | `config/transform-vrl/data/`, mounted beside the program dir -- the transform scans that dir for programs | the same directory, mounted at `/etc/dfe-transform-vector/data` -- one table, not a copy per app | | Dev images | an override block plus `IMAGE_CONSUMERS` in `scripts/build_dev_images.py`, or `make dev` leaves it on the registry pin | the same | -| Env file | `env.example/transform-vrl-filebeat.env` for its own overrides; `make init` copies it into `env/` | `env.example/transform-vector-filebeat.env` | +| Env file | `env.example/transform-vrl-filebeat.env` for its own overrides; `make init` copies it into `env/` | `env.example/transform-e2e-vector-filebeat.env` | | Profile resolution | `SERVICES` and `SERVICE_TO_CONFIG_VAR` in `scripts/resolve_profile.py`, so a profile can run it without the passthrough instance | the same | | Check coverage | `_DFE_OWNED_SERVICES` in `scripts/check_compose.py`, which holds it to the `/livez` health surface | the same | | Metrics ports | a free host port (`DFE_TRANSFORM_VRL_FILEBEAT_PROMETHEUS_PORT`, 9097): every instance serves 9090 and two cannot publish one | the same (`DFE_TRANSFORM_VECTOR_FILEBEAT_PROMETHEUS_PORT`, 9098) | +A static instance puts `e2e-` after `dfe-transform-` because dfe-engine names a real per-source instance `-`, and Compose merges two services sharing a name. Its `DFE_TRANSFORM_*` variables keep their names, so an existing `.env` still applies. + One port carries everything. The wrapper scrapes Vector's `internal_metrics` off its `prometheus_exporter` on loopback and registers the samples on scalo's own recorder (dfe-transform-vector#70), so `vector_*` reads on the published port AND diff --git a/docs/configuration.md b/docs/configuration.md index 11e1a879..b1f438a1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -281,6 +281,8 @@ container, the sibling of `DFE_DEPLOYMENT_APP_CONFIG_DIR`) and it is a read-writ bind mount of `./env`, so the deployment's own working tree is what changes when an operator saves a setting. +The engine runs as its image's user, not yours, so the two meet on a group. `make init` (and every start target) makes `env/` group-writable and setgid, the engine container gets the checkout's group through `group_add` (`DFE_DEV_GID`, which the Makefile sets to `id -g`), and each custom file is written `0640` in that group. Compose run by anyone in that group reads it, and nobody else on the host does. A bare `docker compose` without make adds group `1000`. When the engine cannot give a file the directory's group, its log says so and names the fix. + ## Self-monitoring (opt-in) Off by default. `DFE_OTEL_ENABLED=true` (or a profile declaring `otel: true`, as `single` does) starts a collector that takes the stack's own telemetry over OTLP and writes the `dfe.otel_*` tables, which HyperDX reads. Nothing is scraped, and the collector's OTLP ports stay on the Compose network. Full picture, including which services report today: [observability.md](observability.md). @@ -336,9 +338,9 @@ The same toggle points the engine at HyperDX: with it on, the engine receives `D | 8090 | hyperdx | App UI | | 8123 | ClickHouse | HTTP API | | 8686 | dfe-transform-vector | Vector API | -| 8687 | dfe-transform-vector-filebeat | Vector API | +| 8687 | dfe-transform-e2e-vector-filebeat | Vector API | | 9000 | ClickHouse | Native protocol | -| 9089 | dfe-transform-elastic-cisco-ios | Prometheus metrics | +| 9089 | dfe-transform-e2e-elastic-cisco-ios | Prometheus metrics | | 9090 | dfe-receiver | Prometheus metrics | | 9091 | dfe-loader | Prometheus metrics | | 9092 | Kafka (any backend) | Plaintext | @@ -347,7 +349,7 @@ The same toggle points the engine at HyperDX: with it on, the engine receives `D | 9095 | dfe-transform-vector | Prometheus metrics | | 9096 | dfe-transform-vrl | Prometheus metrics | | 9097 | dfe-transform-vrl-filebeat | Prometheus metrics | -| 9098 | dfe-transform-vector-filebeat | Prometheus metrics | +| 9098 | dfe-transform-e2e-vector-filebeat | Prometheus metrics | | 9099 | dfe-transform-elastic | Prometheus metrics | | 13133 | otel-collector | health_check | | 19092 | Kafka (any backend) | Plaintext host | diff --git a/env.example/transform-elastic-cisco-ios.env b/env.example/transform-e2e-elastic-cisco-ios.env similarity index 100% rename from env.example/transform-elastic-cisco-ios.env rename to env.example/transform-e2e-elastic-cisco-ios.env diff --git a/env.example/transform-vector-filebeat.env b/env.example/transform-e2e-vector-filebeat.env similarity index 100% rename from env.example/transform-vector-filebeat.env rename to env.example/transform-e2e-vector-filebeat.env diff --git a/scripts/build_dev_images.py b/scripts/build_dev_images.py index 26d0acb8..8031c366 100644 --- a/scripts/build_dev_images.py +++ b/scripts/build_dev_images.py @@ -42,8 +42,8 @@ IMAGE_CONSUMERS: dict[str, tuple[str, ...]] = { "dfe-archiver": ("dlq-init",), "dfe-engine": ("dfe-hunt-runner",), - "dfe-transform-elastic": ("dfe-transform-elastic-cisco-ios",), - "dfe-transform-vector": ("dfe-transform-vector-filebeat",), + "dfe-transform-elastic": ("dfe-transform-e2e-elastic-cisco-ios",), + "dfe-transform-vector": ("dfe-transform-e2e-vector-filebeat",), "dfe-transform-vrl": ("dfe-transform-vrl-filebeat",), } # The IMAGE_CONSUMERS no profile names, mapped to the services whose diff --git a/scripts/check_compose.py b/scripts/check_compose.py index 5b1f432f..b375ed41 100644 --- a/scripts/check_compose.py +++ b/scripts/check_compose.py @@ -264,10 +264,10 @@ "dfe-fetcher", "dfe-loader", "dfe-receiver", + "dfe-transform-e2e-elastic-cisco-ios", + "dfe-transform-e2e-vector-filebeat", "dfe-transform-elastic", - "dfe-transform-elastic-cisco-ios", "dfe-transform-vector", - "dfe-transform-vector-filebeat", "dfe-transform-vrl", "dfe-transform-vrl-filebeat", "dfe-ui", diff --git a/scripts/env_files.py b/scripts/env_files.py index faee6022..02847989 100644 --- a/scripts/env_files.py +++ b/scripts/env_files.py @@ -21,7 +21,10 @@ warning. It stays a warning: an operator's env file is theirs, and this gate starts stacks rather than editing their config. -Silent when nothing is missing and nothing has drifted. +It also widens env/ to group-writable and setgid where `make init` predates +that, because the engine writes each app's custom env file there. + +Silent when nothing is missing, nothing has drifted and env/ is already shared. """ from __future__ import annotations @@ -34,7 +37,7 @@ _print, _rel_path, ) -from init import drift_keys +from init import drift_keys, share_env_dir from init import main as init_main @@ -84,6 +87,8 @@ def main() -> int: f"file in {_rel_path(path=ENV_TEMPLATE_DIR)}/" ) return 1 + # A checkout initialised before the engine wrote here has env/ owner-only. + share_env_dir(env_dir=ENV_DIR) _report_drift(include_dotenv=not missing) return 0 diff --git a/scripts/init.py b/scripts/init.py index 5a95116e..c243f2cb 100755 --- a/scripts/init.py +++ b/scripts/init.py @@ -11,13 +11,15 @@ Copies .env.example to .env and each env.example/.env to env/.env. Existing files are left untouched (reported as skipped) so the target is safe to re-run. Errors go to stderr with a non-zero exit. -Two things happen beyond a plain copy: +Beyond a plain copy: Generated secrets. The keys in GENERATED_SECRETS have deterministic, committed defaults in docker-compose.yml, which means a stack that never ran this script is using a signing key and a database password anyone with the repo already knows. This script mints a random value for each - into a new .env, and topped up into an existing .env that predates the key. Compose cannot hard-fail on them (its interpolation is not profile-gated, so a `:?` would abort `make down` too, for services the operator may not even run), so the enforcement lives in the power-on self test: scripts/post.py refuses to pass while a default is still in place. Drift report. A re-run reports template keys that never reached .env. The copy is one-shot, so an operator who ran `make init` months ago otherwise never learns that .env.example grew a setting. Reporting is all it does - editing an operator's .env is theirs to do, not ours. The retention question. A NEW .env is asked, once and on a TTY only, for the default TTL every time-series table gets, and the answer lands as a live DFE_CLICKHOUSE_DEFAULT_TTL_DAYS line. The environment pre-answers it; a non-interactive run keeps the template's commented 90; an existing .env is never re-asked. + +The env/ directory. The engine writes env/.custom.env into it, so it is made group-writable and setgid: the engine writes through the group compose adds it to, and every file it creates keeps the group Compose reads it as. """ from __future__ import annotations @@ -26,6 +28,7 @@ import re import secrets import shutil +import stat import string import sys from pathlib import Path @@ -95,6 +98,9 @@ # random assignment. _SETTING_RE = re.compile(r"^[ \t]*(?:#[ \t]?)?(?P[A-Z][A-Z0-9_]*)[ \t]*=") +ENV_DIR_GROUP_BITS = stat.S_IRWXG | stat.S_ISGID +"""Group read, write and search on env/, plus setgid so new files inherit its group.""" + # The retention every time-series table gets unless a source or a dfe-schemas # definition sets its own. Whole days; 0 disables the default TTL. RETENTION_KEY = "DFE_CLICKHOUSE_DEFAULT_TTL_DAYS" @@ -297,6 +303,33 @@ def _report_drift(*, dotenv_path: Path, template_path: Path) -> None: ) +def share_env_dir(*, env_dir: Path) -> None: + """Let the engine write env/ through its group, and make every new file inherit that group. + + The engine writes env/.custom.env as its own user with this directory's + group added (compose `group_add`), writes it 0640, and Compose reads it back + as the operator, so the group has to write here and the setgid bit has to + hand each new file the group rather than the engine's own. Only ever widens. + """ + if not (env_dir.is_dir()): + return + mode = stat.S_IMODE(env_dir.stat().st_mode) + wanted = mode | ENV_DIR_GROUP_BITS + if mode == wanted: + return + try: + env_dir.chmod(wanted) + except PermissionError as exc: + _print( + header=f"{_rel_path(path=env_dir)}/", + msg=f"WARNING: cannot add group write and setgid ({exc}) -- the engine " + f"cannot write custom env files here until its owner runs " + f"`chmod g+rwxs {_rel_path(path=env_dir)}`", + ) + return + _print(header=f"{_rel_path(path=env_dir)}/", msg="Group-writable, setgid") + + def _copy_if_absent(*, dst_path: Path, src_path: Path) -> None: """Copy src_path to dst_path unless dst_path already exists - report the outcome.""" if dst_path.exists(): @@ -323,6 +356,7 @@ def main() -> int: _ask_retention(dotenv_path=DOTENV_FILE) ENV_DIR.mkdir(exist_ok=True, parents=True) + share_env_dir(env_dir=ENV_DIR) templates = sorted(ENV_TEMPLATE_DIR.glob("*.env")) if not (templates): diff --git a/scripts/post.py b/scripts/post.py index 24edf517..5a86c7c2 100644 --- a/scripts/post.py +++ b/scripts/post.py @@ -153,14 +153,17 @@ # The services whose OWN metrics have to reach the otel database, as name prefixes # so a per-source instance (dfe-transform-vrl-filebeat) is covered by its app. -# dfe-ui carries the OTel API and no SDK, and dfe-hunt-runner keeps scalo-py's -# prometheus backend, so neither pushes. +# dfe-transform-e2e-vector-filebeat is a static extra instance named off that +# shape, so no app prefix covers it and it is named in full. dfe-ui carries the +# OTel API and no SDK, and dfe-hunt-runner keeps scalo-py's prometheus backend, +# so neither pushes. OTEL_PUSHER_PREFIXES = ( "dfe-archiver", "dfe-engine", "dfe-fetcher", "dfe-loader", "dfe-receiver", + "dfe-transform-e2e-vector-filebeat", "dfe-transform-vector", "dfe-transform-vrl", ) diff --git a/scripts/resolve_profile.py b/scripts/resolve_profile.py index 8d101f15..46848a71 100755 --- a/scripts/resolve_profile.py +++ b/scripts/resolve_profile.py @@ -143,10 +143,10 @@ "dfe-fetcher": "DFE_FETCHER_CONFIG", "dfe-loader": "DFE_LOADER_CONFIG", "dfe-receiver": "DFE_RECEIVER_CONFIG", + "dfe-transform-e2e-elastic-cisco-ios": "DFE_TRANSFORM_ELASTIC_CISCO_IOS_CONFIG", + "dfe-transform-e2e-vector-filebeat": "DFE_TRANSFORM_VECTOR_FILEBEAT_CONFIG", "dfe-transform-elastic": "DFE_TRANSFORM_ELASTIC_CONFIG", - "dfe-transform-elastic-cisco-ios": "DFE_TRANSFORM_ELASTIC_CISCO_IOS_CONFIG", "dfe-transform-vector": "DFE_TRANSFORM_VECTOR_CONFIG", - "dfe-transform-vector-filebeat": "DFE_TRANSFORM_VECTOR_FILEBEAT_CONFIG", "dfe-transform-vrl": "DFE_TRANSFORM_VRL_CONFIG", "dfe-transform-vrl-filebeat": "DFE_TRANSFORM_VRL_FILEBEAT_CONFIG", } @@ -181,10 +181,10 @@ "dfe-fetcher", "dfe-loader", "dfe-receiver", + "dfe-transform-e2e-elastic-cisco-ios", + "dfe-transform-e2e-vector-filebeat", "dfe-transform-elastic", - "dfe-transform-elastic-cisco-ios", "dfe-transform-vector", - "dfe-transform-vector-filebeat", "dfe-transform-vrl", "dfe-transform-vrl-filebeat", ] diff --git a/scripts/test_e2e.py b/scripts/test_e2e.py index cb93cc9c..e130f59d 100755 --- a/scripts/test_e2e.py +++ b/scripts/test_e2e.py @@ -91,10 +91,10 @@ "dfe-fetcher": "/etc/dfe/fetcher.yaml", "dfe-loader": "/etc/dfe/loader.yaml", "dfe-receiver": "/etc/dfe-receiver/config.yaml", + "dfe-transform-e2e-elastic-cisco-ios": "/etc/dfe-transform-elastic/config.yaml", + "dfe-transform-e2e-vector-filebeat": "/etc/dfe-transform-vector/config.yaml", "dfe-transform-elastic": "/etc/dfe-transform-elastic/config.yaml", - "dfe-transform-elastic-cisco-ios": "/etc/dfe-transform-elastic/config.yaml", "dfe-transform-vector": "/etc/dfe-transform-vector/config.yaml", - "dfe-transform-vector-filebeat": "/etc/dfe-transform-vector/config.yaml", "dfe-transform-vrl": "/etc/dfe-transform-vrl/config.yaml", "dfe-transform-vrl-filebeat": "/etc/dfe-transform-vrl/config.yaml", } diff --git a/scripts/tests/test_engine_only_schema_control.py b/scripts/tests/test_engine_only_schema_control.py index 78f9f997..e9d03a57 100644 --- a/scripts/tests/test_engine_only_schema_control.py +++ b/scripts/tests/test_engine_only_schema_control.py @@ -98,8 +98,8 @@ "dfe-fetcher", "dfe-loader", "dfe-receiver", + "dfe-transform-e2e-vector-filebeat", "dfe-transform-vector", - "dfe-transform-vector-filebeat", "dfe-transform-vrl", "dfe-transform-vrl-filebeat", "otel-collector", diff --git a/scripts/tests/test_env_dir.py b/scripts/tests/test_env_dir.py new file mode 100644 index 00000000..934a0851 --- /dev/null +++ b/scripts/tests/test_env_dir.py @@ -0,0 +1,85 @@ +# Project: dfe-docker +# File: tests/test_env_dir.py +# Purpose: Assert env/ is left group-writable and setgid for the engine's custom env files +# Language: Python +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +"""The mode `make init` and every start target leave on env/. + +The engine writes env/.custom.env through the directory's group and Compose +reads it back as the operator, so env/ needs group write and the setgid bit. +""" + +import stat +from pathlib import Path + +import pytest + +import env_files +import init + + +def _mode(path: Path) -> int: + return stat.S_IMODE(path.stat().st_mode) + + +@pytest.fixture +def env_dir(dotenv: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + """Point init and env_files at a throwaway tree with one template; return its env/.""" + (dotenv.parent / ".env.example").write_text( + "# CLICKHOUSE_SECURE=false\n", encoding="utf-8" + ) + templates = dotenv.parent / "env.example" + templates.mkdir() + (templates / "engine.env").write_text("# DFE_API_PORT=8003\n", encoding="utf-8") + target = dotenv.parent / "env" + monkeypatch.setattr(init, "DOTENV_TEMPLATE", dotenv.parent / ".env.example") + for module in (init, env_files): + monkeypatch.setattr(module, "ENV_TEMPLATE_DIR", templates) + monkeypatch.setattr(module, "ENV_DIR", target) + monkeypatch.setattr(env_files, "DOTENV_FILE", dotenv) + monkeypatch.setattr(env_files, "DOTENV_TEMPLATE", dotenv.parent / ".env.example") + monkeypatch.delenv(init.RETENTION_KEY, raising=False) + monkeypatch.setattr("sys.stdin.isatty", lambda: False) + return target + + +def test_init_leaves_env_group_writable_and_setgid(env_dir: Path) -> None: + assert init.main() == 0 + + assert _mode(env_dir) & init.ENV_DIR_GROUP_BITS == init.ENV_DIR_GROUP_BITS + + +def test_a_start_target_heals_an_env_dir_an_older_init_made(env_dir: Path) -> None: + env_dir.mkdir() + (env_dir / "engine.env").write_text("", encoding="utf-8") + env_dir.chmod(0o700) + + assert env_files.main() == 0 + + assert _mode(env_dir) == 0o2770 + + +def test_only_group_bits_are_added_and_nothing_is_taken_away(dotenv: Path) -> None: + target = dotenv.parent / "env" + target.mkdir() + target.chmod(0o755) + + init.share_env_dir(env_dir=target) + + assert _mode(target) == 0o2775 + + +def test_an_already_shared_dir_is_left_alone( + dotenv: Path, capsys: pytest.CaptureFixture[str] +) -> None: + target = dotenv.parent / "env" + target.mkdir() + target.chmod(0o2770) + + init.share_env_dir(env_dir=target) + + assert _mode(target) == 0o2770 + assert capsys.readouterr().err == "" diff --git a/service_profiles.yaml b/service_profiles.yaml index a76b48ec..cc106f6c 100644 --- a/service_profiles.yaml +++ b/service_profiles.yaml @@ -186,7 +186,7 @@ profiles: config_path: receiver/kafka.yaml dfe-transform-vector: config_path: transform-vector/kafka.yaml - dfe-transform-vector-filebeat: + dfe-transform-e2e-vector-filebeat: config_path: transform-vector/filebeat.yaml # The per-source shape on dfe-transform-elastic: the source is cisco-ios rather @@ -202,7 +202,7 @@ profiles: config_path: loader/kafka.yaml dfe-receiver: config_path: receiver/kafka.yaml - dfe-transform-elastic-cisco-ios: + dfe-transform-e2e-elastic-cisco-ios: config_path: transform-elastic/cisco-ios.yaml kafka-receiver-archiver: @@ -227,7 +227,7 @@ profiles: config_path: loader/kafka-resilience.yaml dfe-receiver: config_path: receiver/kafka.yaml - dfe-transform-elastic-cisco-ios: + dfe-transform-e2e-elastic-cisco-ios: config_path: transform-elastic/cisco-ios.yaml dfe-transform-vector: config_path: transform-vector/resilience.yaml