Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ export CLICKHOUSE_USERNAME
# Host UID/GID passed to live-mode containers (docker-compose.live.yml) that
# write to bind-mounted host dirs (dfe-engine config/schemas), so files are owned
# by the host user rather than the image user and writes don't hit permission
# errors.
# errors. The GID is also the group every stack adds to dfe-engine, which is how
# it writes env/<app>.custom.env into the checkout.
export DFE_DEV_UID := $(shell id -u)
export DFE_DEV_GID := $(shell id -g)

Expand Down
8 changes: 4 additions & 4 deletions docker-compose.container-logs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,13 @@ services:
logging: *container-logs
dfe-receiver:
logging: *container-logs
dfe-transform-elastic:
dfe-transform-e2e-elastic-cisco-ios:
logging: *container-logs
dfe-transform-elastic-cisco-ios:
dfe-transform-e2e-vector-filebeat:
logging: *container-logs
dfe-transform-vector:
dfe-transform-elastic:
logging: *container-logs
dfe-transform-vector-filebeat:
dfe-transform-vector:
logging: *container-logs
dfe-transform-vrl:
logging: *container-logs
Expand Down
4 changes: 2 additions & 2 deletions docker-compose.override.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,13 @@ services:
dfe-transform-elastic:
image: dfe-transform-elastic:local

dfe-transform-elastic-cisco-ios:
dfe-transform-e2e-elastic-cisco-ios:
image: dfe-transform-elastic:local

dfe-transform-vector:
image: dfe-transform-vector:local

dfe-transform-vector-filebeat:
dfe-transform-e2e-vector-filebeat:
image: dfe-transform-vector:local

dfe-transform-vrl:
Expand Down
32 changes: 21 additions & 11 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -750,7 +750,7 @@ services:
retries: 5

# ===========================================================================
# dfe-transform-elastic-cisco-ios
# dfe-transform-e2e-elastic-cisco-ios
# A SECOND instance of the same image, for the cisco-ios source alone -- the
# per-source shape dfe-transform-vrl-filebeat carries on the VRL side. The
# receiver routes _source=cisco-ios to cisco-ios_land, this instance is the
Expand All @@ -759,24 +759,27 @@ services:
# `source.name` in the config is the whole difference between two instances of
# this app: the transform is compiled in, so a second source adds a block like
# this one and nothing else.
#
# Named off the `<app>-<source>` shape because dfe-engine names a real
# per-source instance that way, and Compose merges two services sharing a name.
# ===========================================================================

dfe-transform-elastic-cisco-ios:
dfe-transform-e2e-elastic-cisco-ios:
image: ${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-transform-elastic:${DFE_TRANSFORM_ELASTIC_VERSION:?run 'make stack VERSION=X.Y.Z' to pin DFE_TRANSFORM_ELASTIC_VERSION}
container_name: dfe-transform-elastic-cisco-ios
container_name: dfe-transform-e2e-elastic-cisco-ios
profiles: ["dfe"]
restart: unless-stopped
deploy: *limits-service
ports:
- ${DFE_BIND_HOST:-127.0.0.1}:${DFE_TRANSFORM_ELASTIC_CISCO_IOS_PROMETHEUS_PORT:-9089}:9090
env_file:
- path: env/transform-elastic-cisco-ios.env
- path: env/transform-e2e-elastic-cisco-ios.env
required: false
environment:
LOG_LEVEL: ${LOG_LEVEL:-info}
LOG_FORMAT: ${LOG_FORMAT:-text}
OTEL_EXPORTER_OTLP_ENDPOINT: ${DFE_OTEL_EXPORTER_ENDPOINT:-}
OTEL_SERVICE_NAME: dfe-transform-elastic-cisco-ios
OTEL_SERVICE_NAME: dfe-transform-e2e-elastic-cisco-ios
depends_on:
# No dlq-init here: this service does not mount dlq-spool. dfe-engine
# creates the topics it reads and writes, so it waits on that instead.
Expand Down Expand Up @@ -849,7 +852,7 @@ services:
retries: 5

# ===========================================================================
# dfe-transform-vector-filebeat
# dfe-transform-e2e-vector-filebeat
# A SECOND instance of the same image, for the filebeat source alone -- the
# dfe-transform-vector half of the shape dfe-transform-vrl-filebeat carries
# below. The receiver routes _source=filebeat-vector to filebeat-vector_land,
Expand All @@ -861,29 +864,32 @@ services:
# filebeat program and the timezone table it looks up is mounted from the
# filebeat pipeline's own data directory, shared with the VRL instance rather
# than copied.
#
# Named off the `<app>-<source>` shape because dfe-engine names a real
# per-source instance that way, and Compose merges two services sharing a name.
# ===========================================================================

dfe-transform-vector-filebeat:
dfe-transform-e2e-vector-filebeat:
image: ${IMAGE_REGISTRY:-ghcr.io/hyperi-io}/dfe-transform-vector:${DFE_TRANSFORM_VECTOR_VERSION:?run 'make stack VERSION=X.Y.Z' to pin DFE_TRANSFORM_VECTOR_VERSION}
container_name: dfe-transform-vector-filebeat
container_name: dfe-transform-e2e-vector-filebeat
profiles: ["dfe"]
restart: unless-stopped
deploy: *limits-service
ports:
- ${DFE_BIND_HOST:-127.0.0.1}:${DFE_TRANSFORM_VECTOR_FILEBEAT_PROMETHEUS_PORT:-9098}:9090
- ${DFE_BIND_HOST:-127.0.0.1}:${DFE_TRANSFORM_VECTOR_FILEBEAT_API_PORT:-8687}:8686
env_file:
- path: env/transform-vector-filebeat.env
- path: env/transform-e2e-vector-filebeat.env
required: false
# The custom keys the engine writes for this app; an up -d applies them,
# a restart keeps the environment the container already has.
- path: env/transform-vector-filebeat.custom.env
- path: env/transform-e2e-vector-filebeat.custom.env
required: false
environment:
LOG_LEVEL: ${LOG_LEVEL:-info}
LOG_FORMAT: ${LOG_FORMAT:-text}
OTEL_EXPORTER_OTLP_ENDPOINT: ${DFE_OTEL_EXPORTER_ENDPOINT:-}
OTEL_SERVICE_NAME: dfe-transform-vector-filebeat
OTEL_SERVICE_NAME: dfe-transform-e2e-vector-filebeat
depends_on:
# No dlq-init here: this service does not mount dlq-spool.
# The topic authority: dfe-engine creates the declared topic set at its own
Expand Down Expand Up @@ -1011,6 +1017,10 @@ services:
profiles: ["core"]
restart: unless-stopped
deploy: *limits-service
# The engine runs as UID 1000 and ./env belongs to the operator, so it writes
# there through the operator's group; `make` exports DFE_DEV_GID.
group_add:
- "${DFE_DEV_GID:-1000}"
ports:
# dfe-proxy reaches the engine over the docker network; this host mapping
# is for direct API work. PRODUCT class -- it is the API the UI consumes --
Expand Down
6 changes: 4 additions & 2 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -123,18 +123,20 @@ mode, profile resolution and the checks working.

| Piece | `kafka-filebeat`, on dfe-transform-vrl | `kafka-filebeat-vector`, on dfe-transform-vector |
|---|---|---|
| Compose service | `dfe-transform-vrl-filebeat`, the same image, its own metrics port | `dfe-transform-vector-filebeat`, likewise |
| Compose service | `dfe-transform-vrl-filebeat`, the same image, its own metrics port | `dfe-transform-e2e-vector-filebeat`, likewise |
| Config | `config/transform-vrl/filebeat.yaml` -- `filebeat_land` in, `filebeat_load` out | `config/transform-vector/filebeat.yaml` -- `dfe_source: filebeat-vector` derives both topics |
| Program | `config/transform-vrl/transforms-filebeat/`, vendored from dfe-transform-vrl | `config/transform-vector/transforms-filebeat/`, the same VRL inside a Vector `remap`, vendored from dfe-transform-vector |
| Loader | `config/loader/kafka-load-filebeat.yaml` lists `filebeat_load` alongside `main_load` | `config/loader/kafka-load-filebeat-vector.yaml` lists `filebeat-vector_load` |
| Table | `dfe.filebeat`, from the `_load` topic name -- the loader strips the suffix and routes on it | ``dfe.`filebeat-vector` ``, the same way |
| Enrichment data | `config/transform-vrl/data/`, mounted beside the program dir -- the transform scans that dir for programs | the same directory, mounted at `/etc/dfe-transform-vector/data` -- one table, not a copy per app |
| Dev images | an override block plus `IMAGE_CONSUMERS` in `scripts/build_dev_images.py`, or `make dev` leaves it on the registry pin | the same |
| Env file | `env.example/transform-vrl-filebeat.env` for its own overrides; `make init` copies it into `env/` | `env.example/transform-vector-filebeat.env` |
| Env file | `env.example/transform-vrl-filebeat.env` for its own overrides; `make init` copies it into `env/` | `env.example/transform-e2e-vector-filebeat.env` |
| Profile resolution | `SERVICES` and `SERVICE_TO_CONFIG_VAR` in `scripts/resolve_profile.py`, so a profile can run it without the passthrough instance | the same |
| Check coverage | `_DFE_OWNED_SERVICES` in `scripts/check_compose.py`, which holds it to the `/livez` health surface | the same |
| Metrics ports | a free host port (`DFE_TRANSFORM_VRL_FILEBEAT_PROMETHEUS_PORT`, 9097): every instance serves 9090 and two cannot publish one | the same (`DFE_TRANSFORM_VECTOR_FILEBEAT_PROMETHEUS_PORT`, 9098) |

A static instance puts `e2e-` after `dfe-transform-` because dfe-engine names a real per-source instance `<app>-<source>`, and Compose merges two services sharing a name. Its `DFE_TRANSFORM_*` variables keep their names, so an existing `.env` still applies.

One port carries everything. The wrapper scrapes Vector's `internal_metrics` off
its `prometheus_exporter` on loopback and registers the samples on scalo's own
recorder (dfe-transform-vector#70), so `vector_*` reads on the published port AND
Expand Down
8 changes: 5 additions & 3 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,8 @@ container, the sibling of `DFE_DEPLOYMENT_APP_CONFIG_DIR`) and it is a read-writ
bind mount of `./env`, so the deployment's own working tree is what changes when
an operator saves a setting.

The engine runs as its image's user, not yours, so the two meet on a group. `make init` (and every start target) makes `env/` group-writable and setgid, the engine container gets the checkout's group through `group_add` (`DFE_DEV_GID`, which the Makefile sets to `id -g`), and each custom file is written `0640` in that group. Compose run by anyone in that group reads it, and nobody else on the host does. A bare `docker compose` without make adds group `1000`. When the engine cannot give a file the directory's group, its log says so and names the fix.

## Self-monitoring (opt-in)

Off by default. `DFE_OTEL_ENABLED=true` (or a profile declaring `otel: true`, as `single` does) starts a collector that takes the stack's own telemetry over OTLP and writes the `dfe.otel_*` tables, which HyperDX reads. Nothing is scraped, and the collector's OTLP ports stay on the Compose network. Full picture, including which services report today: [observability.md](observability.md).
Expand Down Expand Up @@ -336,9 +338,9 @@ The same toggle points the engine at HyperDX: with it on, the engine receives `D
| 8090 | hyperdx | App UI |
| 8123 | ClickHouse | HTTP API |
| 8686 | dfe-transform-vector | Vector API |
| 8687 | dfe-transform-vector-filebeat | Vector API |
| 8687 | dfe-transform-e2e-vector-filebeat | Vector API |
| 9000 | ClickHouse | Native protocol |
| 9089 | dfe-transform-elastic-cisco-ios | Prometheus metrics |
| 9089 | dfe-transform-e2e-elastic-cisco-ios | Prometheus metrics |
| 9090 | dfe-receiver | Prometheus metrics |
| 9091 | dfe-loader | Prometheus metrics |
| 9092 | Kafka (any backend) | Plaintext |
Expand All @@ -347,7 +349,7 @@ The same toggle points the engine at HyperDX: with it on, the engine receives `D
| 9095 | dfe-transform-vector | Prometheus metrics |
| 9096 | dfe-transform-vrl | Prometheus metrics |
| 9097 | dfe-transform-vrl-filebeat | Prometheus metrics |
| 9098 | dfe-transform-vector-filebeat | Prometheus metrics |
| 9098 | dfe-transform-e2e-vector-filebeat | Prometheus metrics |
| 9099 | dfe-transform-elastic | Prometheus metrics |
| 13133 | otel-collector | health_check |
| 19092 | Kafka (any backend) | Plaintext host |
Expand Down
4 changes: 2 additions & 2 deletions scripts/build_dev_images.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@
IMAGE_CONSUMERS: dict[str, tuple[str, ...]] = {
"dfe-archiver": ("dlq-init",),
"dfe-engine": ("dfe-hunt-runner",),
"dfe-transform-elastic": ("dfe-transform-elastic-cisco-ios",),
"dfe-transform-vector": ("dfe-transform-vector-filebeat",),
"dfe-transform-elastic": ("dfe-transform-e2e-elastic-cisco-ios",),
"dfe-transform-vector": ("dfe-transform-e2e-vector-filebeat",),
"dfe-transform-vrl": ("dfe-transform-vrl-filebeat",),
}
# The IMAGE_CONSUMERS no profile names, mapped to the services whose
Expand Down
4 changes: 2 additions & 2 deletions scripts/check_compose.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,10 +264,10 @@
"dfe-fetcher",
"dfe-loader",
"dfe-receiver",
"dfe-transform-e2e-elastic-cisco-ios",
"dfe-transform-e2e-vector-filebeat",
"dfe-transform-elastic",
"dfe-transform-elastic-cisco-ios",
"dfe-transform-vector",
"dfe-transform-vector-filebeat",
"dfe-transform-vrl",
"dfe-transform-vrl-filebeat",
"dfe-ui",
Expand Down
9 changes: 7 additions & 2 deletions scripts/env_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,10 @@
warning. It stays a warning: an operator's env file is theirs, and this gate
starts stacks rather than editing their config.

Silent when nothing is missing and nothing has drifted.
It also widens env/ to group-writable and setgid where `make init` predates
that, because the engine writes each app's custom env file there.

Silent when nothing is missing, nothing has drifted and env/ is already shared.
"""

from __future__ import annotations
Expand All @@ -34,7 +37,7 @@
_print,
_rel_path,
)
from init import drift_keys
from init import drift_keys, share_env_dir
from init import main as init_main


Expand Down Expand Up @@ -84,6 +87,8 @@ def main() -> int:
f"file in {_rel_path(path=ENV_TEMPLATE_DIR)}/"
)
return 1
# A checkout initialised before the engine wrote here has env/ owner-only.
share_env_dir(env_dir=ENV_DIR)
_report_drift(include_dotenv=not missing)
return 0

Expand Down
36 changes: 35 additions & 1 deletion scripts/init.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,15 @@

Copies .env.example to .env and each env.example/<service>.env to env/<service>.env. Existing files are left untouched (reported as skipped) so the target is safe to re-run. Errors go to stderr with a non-zero exit.

Two things happen beyond a plain copy:
Beyond a plain copy:

Generated secrets. The keys in GENERATED_SECRETS have deterministic, committed defaults in docker-compose.yml, which means a stack that never ran this script is using a signing key and a database password anyone with the repo already knows. This script mints a random value for each - into a new .env, and topped up into an existing .env that predates the key. Compose cannot hard-fail on them (its interpolation is not profile-gated, so a `:?` would abort `make down` too, for services the operator may not even run), so the enforcement lives in the power-on self test: scripts/post.py refuses to pass while a default is still in place.

Drift report. A re-run reports template keys that never reached .env. The copy is one-shot, so an operator who ran `make init` months ago otherwise never learns that .env.example grew a setting. Reporting is all it does - editing an operator's .env is theirs to do, not ours.

The retention question. A NEW .env is asked, once and on a TTY only, for the default TTL every time-series table gets, and the answer lands as a live DFE_CLICKHOUSE_DEFAULT_TTL_DAYS line. The environment pre-answers it; a non-interactive run keeps the template's commented 90; an existing .env is never re-asked.

The env/ directory. The engine writes env/<app>.custom.env into it, so it is made group-writable and setgid: the engine writes through the group compose adds it to, and every file it creates keeps the group Compose reads it as.
"""

from __future__ import annotations
Expand All @@ -26,6 +28,7 @@
import re
import secrets
import shutil
import stat
import string
import sys
from pathlib import Path
Expand Down Expand Up @@ -95,6 +98,9 @@
# random assignment.
_SETTING_RE = re.compile(r"^[ \t]*(?:#[ \t]?)?(?P<key>[A-Z][A-Z0-9_]*)[ \t]*=")

ENV_DIR_GROUP_BITS = stat.S_IRWXG | stat.S_ISGID
"""Group read, write and search on env/, plus setgid so new files inherit its group."""

# The retention every time-series table gets unless a source or a dfe-schemas
# definition sets its own. Whole days; 0 disables the default TTL.
RETENTION_KEY = "DFE_CLICKHOUSE_DEFAULT_TTL_DAYS"
Expand Down Expand Up @@ -297,6 +303,33 @@ def _report_drift(*, dotenv_path: Path, template_path: Path) -> None:
)


def share_env_dir(*, env_dir: Path) -> None:
"""Let the engine write env/ through its group, and make every new file inherit that group.

The engine writes env/<app>.custom.env as its own user with this directory's
group added (compose `group_add`), writes it 0640, and Compose reads it back
as the operator, so the group has to write here and the setgid bit has to
hand each new file the group rather than the engine's own. Only ever widens.
"""
if not (env_dir.is_dir()):
return
mode = stat.S_IMODE(env_dir.stat().st_mode)
wanted = mode | ENV_DIR_GROUP_BITS
if mode == wanted:
return
try:
env_dir.chmod(wanted)
except PermissionError as exc:
_print(
header=f"{_rel_path(path=env_dir)}/",
msg=f"WARNING: cannot add group write and setgid ({exc}) -- the engine "
f"cannot write custom env files here until its owner runs "
f"`chmod g+rwxs {_rel_path(path=env_dir)}`",
)
return
_print(header=f"{_rel_path(path=env_dir)}/", msg="Group-writable, setgid")


def _copy_if_absent(*, dst_path: Path, src_path: Path) -> None:
"""Copy src_path to dst_path unless dst_path already exists - report the outcome."""
if dst_path.exists():
Expand All @@ -323,6 +356,7 @@ def main() -> int:
_ask_retention(dotenv_path=DOTENV_FILE)

ENV_DIR.mkdir(exist_ok=True, parents=True)
share_env_dir(env_dir=ENV_DIR)

templates = sorted(ENV_TEMPLATE_DIR.glob("*.env"))
if not (templates):
Expand Down
7 changes: 5 additions & 2 deletions scripts/post.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,14 +153,17 @@

# The services whose OWN metrics have to reach the otel database, as name prefixes
# so a per-source instance (dfe-transform-vrl-filebeat) is covered by its app.
# dfe-ui carries the OTel API and no SDK, and dfe-hunt-runner keeps scalo-py's
# prometheus backend, so neither pushes.
# dfe-transform-e2e-vector-filebeat is a static extra instance named off that
# shape, so no app prefix covers it and it is named in full. dfe-ui carries the
# OTel API and no SDK, and dfe-hunt-runner keeps scalo-py's prometheus backend,
# so neither pushes.
OTEL_PUSHER_PREFIXES = (
"dfe-archiver",
"dfe-engine",
"dfe-fetcher",
"dfe-loader",
"dfe-receiver",
"dfe-transform-e2e-vector-filebeat",
"dfe-transform-vector",
"dfe-transform-vrl",
)
Expand Down
8 changes: 4 additions & 4 deletions scripts/resolve_profile.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,10 +143,10 @@
"dfe-fetcher": "DFE_FETCHER_CONFIG",
"dfe-loader": "DFE_LOADER_CONFIG",
"dfe-receiver": "DFE_RECEIVER_CONFIG",
"dfe-transform-e2e-elastic-cisco-ios": "DFE_TRANSFORM_ELASTIC_CISCO_IOS_CONFIG",
"dfe-transform-e2e-vector-filebeat": "DFE_TRANSFORM_VECTOR_FILEBEAT_CONFIG",
"dfe-transform-elastic": "DFE_TRANSFORM_ELASTIC_CONFIG",
"dfe-transform-elastic-cisco-ios": "DFE_TRANSFORM_ELASTIC_CISCO_IOS_CONFIG",
"dfe-transform-vector": "DFE_TRANSFORM_VECTOR_CONFIG",
"dfe-transform-vector-filebeat": "DFE_TRANSFORM_VECTOR_FILEBEAT_CONFIG",
"dfe-transform-vrl": "DFE_TRANSFORM_VRL_CONFIG",
"dfe-transform-vrl-filebeat": "DFE_TRANSFORM_VRL_FILEBEAT_CONFIG",
}
Expand Down Expand Up @@ -181,10 +181,10 @@
"dfe-fetcher",
"dfe-loader",
"dfe-receiver",
"dfe-transform-e2e-elastic-cisco-ios",
"dfe-transform-e2e-vector-filebeat",
"dfe-transform-elastic",
"dfe-transform-elastic-cisco-ios",
"dfe-transform-vector",
"dfe-transform-vector-filebeat",
"dfe-transform-vrl",
"dfe-transform-vrl-filebeat",
]
Expand Down
4 changes: 2 additions & 2 deletions scripts/test_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,10 @@
"dfe-fetcher": "/etc/dfe/fetcher.yaml",
"dfe-loader": "/etc/dfe/loader.yaml",
"dfe-receiver": "/etc/dfe-receiver/config.yaml",
"dfe-transform-e2e-elastic-cisco-ios": "/etc/dfe-transform-elastic/config.yaml",
"dfe-transform-e2e-vector-filebeat": "/etc/dfe-transform-vector/config.yaml",
"dfe-transform-elastic": "/etc/dfe-transform-elastic/config.yaml",
"dfe-transform-elastic-cisco-ios": "/etc/dfe-transform-elastic/config.yaml",
"dfe-transform-vector": "/etc/dfe-transform-vector/config.yaml",
"dfe-transform-vector-filebeat": "/etc/dfe-transform-vector/config.yaml",
"dfe-transform-vrl": "/etc/dfe-transform-vrl/config.yaml",
"dfe-transform-vrl-filebeat": "/etc/dfe-transform-vrl/config.yaml",
}
Expand Down
Loading
Loading