diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e817138..a5d5333 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,22 +5,9 @@ # License: BUSL-1.1 - HYPERI PTY LIMITED # Copyright: (c) 2026 HYPERI PTY LIMITED # -# Hand-written because hyperi-ci has no reusable workflow for a compose-packaging -# repo. It follows the house conventions so it can collapse into a `uses:` line. -# -# The runner image is named (ubuntu-26.04) rather than ubuntu-latest, so a move -# of that label cannot swap the tools these jobs run on. -# -# Renovate automerges patch and digest updates in this repo "when CI passes", and -# before this workflow existed there was no CI to pass -- the rule was asserting a -# condition that could not be evaluated. This makes it mean what it says. -# -# Scope it honestly, though: the service image pins moved out of .env.example into -# the private stack SSoT, so they now live in a gitignored .env where Renovate -# cannot see them. What Renovate still manages HERE is RUST_IMAGE in the builder -# Dockerfile and the actions pinned below. And note check_compose.py substitutes -# placeholders for the mandatory keys, so this validates compose STRUCTURE, not -# that any particular digest resolves. +# Hand-written: hyperi-ci has no reusable workflow for a compose-packaging repo. +# ubuntu-26.04 is named, not ubuntu-latest, so a label move cannot swap the tools. +# Renovate automerges patch and digest updates once this workflow passes. name: CI @@ -57,8 +44,7 @@ concurrency: permissions: contents: read -# The house Python floor, one patch the pinned setup-python can install; the -# runner's own python3 is older and would run every helper script on it. +# The house Python floor. The runner's own python3 is older. env: PYTHON_VERSION: "3.14.7" # The hyperi-ci CLI the Release job runs. The HYPERCI_INSTALL_OVERRIDE variable replaces it, as in hyperi-ci's own workflows. @@ -84,49 +70,30 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} - # Every step below runs a `make` target rather than the command directly, - # so `make check` locally and CI cannot drift apart. - # - # Resolves docker-compose.yml on both the registry and dev-override paths, - # against both Kafka backends. Hermetic: it reads the mandatory keys out of - # the compose file and supplies placeholders, so it reads neither the - # stack SSoT nor the registry. + # Each step runs the make target `make check` runs locally (docs/checks.md). - name: Validate compose run: make check-compose - # The stack must refuse to start unpinned rather than silently pull - # `latest`. Without that, an operator cannot answer "which image am I - # running?" on a production deploy target, so it gets asserted rather than - # assumed. + # The stack must refuse to start unpinned rather than pull `latest`. - name: Assert a fresh checkout hard-fails run: make check-hardfail - # Five audience-scoped documents plus a README whose whole job is to route - # between them. A deleted or renamed file leaves a link that still looks - # like a link, and the reader finds out rather than the author -- which is - # exactly what retiring docs/SCOPE.md did to six live links. - # Stdlib only, so it needs nothing installed. + # Every relative link in the docs resolves. Stdlib only. - name: Docs run: make check-docs - # The pin lives in the Makefile (RUFF_VERSION), so a local `make check` - # lints with the same version this does. Installed here rather than run - # through uvx to keep the runner off the network mid-job. + # RUFF_VERSION in the Makefile pins it. Installed here to keep the runner + # off the network mid-job. - name: Install ruff run: pipx install --python "${{ steps.python.outputs.python-path }}" "ruff==$(make -s print-ruff-version)" - # Every check above is driven by a Python helper script, so a bug in one of - # them shows up as a check that passes when it should not. They get linted - # like anything else. Config in ruff.toml. + # Lints the helper scripts every check above runs. Config in ruff.toml. - name: Python run: make check-python RUFF=ruff - # The credential helpers rewrite .env and decide whether a minted password - # reaches stdout, so they carry unit tests. They run on a tmp_path .env: - # nothing here starts a stack or touches the runner's own files. - # pipx builds each venv on its own interpreter unless told otherwise, and - # pytest runs the tests on whatever its venv holds. - # PyYAML is injected for the seed tests, which parse resources/seed. + # Unit tests over the helper scripts, on a tmp_path .env. pipx builds the + # venv on the interpreter it is given, and PyYAML is injected for the seed + # tests, which parse resources/seed. - name: Install pytest run: | pipx install --python "${{ steps.python.outputs.python-path }}" "pytest==$(make -s print-pytest-version)" @@ -155,14 +122,9 @@ jobs: # .git/config where any later step could read it. persist-credentials: false - # hadolint GATES: a decade old, stable rule IDs, and it lints the shell - # inside RUN via ShellCheck -- a correctness check droast structurally - # does not have. - # stdin form, per the standard, with no config file, so the exact same - # command is runnable locally -- no mount, no config discovery. DL3008 (pin - # apt versions) fires on both Dockerfiles and is deliberately left as a - # visible warning, not suppressed: `--failure-threshold error` lets a - # warning inform without blocking, and the reason sits in the Dockerfile. + # hadolint gates on error severity and lints the shell inside RUN through + # ShellCheck. Warnings such as DL3008 print without failing, and the reason + # sits in the Dockerfile. - name: Hadolint run: make check-dockerfile @@ -180,10 +142,8 @@ jobs: - name: Proxy configs run: make check-proxy - # What a desktop with only Docker does: the image sets each profile up and the - # default one starts, serves the console and runs the demo, every step with a - # PATH that holds no make and no python. The default install goes to the - # Windows job, which reads it with the Windows compose. +# Only Docker on PATH: each profile is set up, and the default one starts, serves +# the console and runs the demo. The Windows job runs the default install. image: name: Image runs-on: ubuntu-26.04 @@ -502,12 +462,8 @@ jobs: } Write-Output "Windows compose starts the same services: $got" - # A push to main whose HEAD commit carries `Release: true`, or a `hyperi-ci - # release` dispatch, tags the version semantic-release computes and publishes - # a GitHub release. It is published with the release App's token because a - # release GITHUB_TOKEN publishes starts no workflow, and image.yml builds the - # image on `release: published`. It waits on the image jobs, so an image that - # fails CI never gets a tag. +# Tags and publishes a release on `Release: true` or a `hyperi-ci release` dispatch, +# after the image jobs. The App's token publishes it so image.yml starts on it. release: name: Release needs: [compose, dockerfile, image, image-windows, published] @@ -618,25 +574,5 @@ jobs: RELEASE_VERSION: ${{ steps.plan.outputs.version }} run: ${{ env.HYPERCI_INSTALL }} release-commit --branch "$RELEASE_BRANCH" "$RELEASE_VERSION" -# droast is DELIBERATELY NOT RUN HERE. That matches hyperi-ci's own posture -- -# quality/droast.py resolves it with `find_tool(recommended=False)` and info-skips -# when absent, so hyperi-ci never installs it either -- but the reason here is -# measured rather than inherited. -# -# Run against docker/dfe-rust-builder.Dockerfile on 2026-07-20 (droast 0.1.x, -# `--no-fail`): 6 findings, 0 true positives. -# -# DF036 no CMD/ENTRYPOINT - false. The final stage is `FROM scratch AS export`, -# an artefact export consumed via `--output`. -# DF001 unpinned image tag - false. ARG RUST_IMAGE is `rust:1.97-trixie@sha256:...`; -# droast does not resolve the ARG default. hadolint does. -# DF057 pipe, no pipefail - false. `SHELL ["/bin/bash", "-o", "pipefail", "-c"]` -# is set; droast ignores the SHELL directive. -# DF063 relative COPY dest - false. The destination is `"/${BINARY}"`, absolute. -# DF005 unpinned apt - duplicate of hadolint DL3008. -# DF033 no .dockerignore - wrong context (assumes the Dockerfile's own dir). -# -# Two of those tell us to fix things this Dockerfile already fixes correctly, so -# wiring it in would train people to ignore the advisory. droast.toml is kept so -# `droast .` works locally for anyone who wants the cache-ordering rules hadolint -# has no equivalent for. Revisit when its parser handles ARG defaults and SHELL. +# droast is not run here: it misreads ARG defaults and the SHELL directive, so its +# findings on these Dockerfiles are false. droast.toml keeps `droast .` working by hand. diff --git a/.gitignore b/.gitignore index ec9632d..1e8e3dc 100644 --- a/.gitignore +++ b/.gitignore @@ -72,8 +72,6 @@ env/ # Local stash of the per-service env files. .env-dir-stashed/ -# Predecessor CI clone and an inert submodule declaration, both untracked and -# left in the tree. Ignored so `git add -A` cannot pull a dead CI system (100+ -# files) or a broken submodule pointer into this repo. Safe to delete outright. +# A predecessor CI clone and an inert submodule declaration, both untracked. Safe to delete. ci/ .gitmodules diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 93e3990..a5f0822 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -199,7 +199,7 @@ dfe-docker stands the whole DFE suite up as one Docker Compose stack on a single | `docker/dfe-docker.Dockerfile` + `scripts/installer.py` | The `ghcr.io/hyperi-io/dfe-docker` image and its entrypoint, which copy this tree into a folder and run its scripts there. `.dockerignore` lists what the image carries. `.github/workflows/image.yml` builds it on each release and attaches `quickstart.json` from `scripts/quickstart_json.py` | | `scripts/test_e2e.py` + `tests/e2e/e2e-tests.yaml` | The declarative e2e suite, one stack per test | | `scripts/check_*.py` | The gates `make check` runs | -| `scripts/tests/` | Unit tests over the helper scripts - the only tests CI runs | +| `scripts/tests/` | Unit tests over the helper scripts, which CI runs beside the static checks | | `config//` | The config files services mount, one directory per component | | `.env.example` + `env.example/` | Committed templates. `make init` copies them to `.env` and `env/`, both gitignored | | `ops/daemon-update/` | The systemd timer that keeps a single-VM deployment on the newest certified stack | @@ -221,8 +221,8 @@ make down # stop and remove containers across EVERY profile Two ways a green run says less than it looks: -- **`make check` does not start the stack.** It resolves compose, lints the helpers and unit-tests them, and `check-proxy` runs temporary Envoy containers to validate the proxy config. `make dev`, `make ci`, `make post` and `make test-e2e` are the only things that prove the stack moves data, and CI runs none of them. -- **`check-compose` validates compose STRUCTURE, not digests.** It substitutes placeholders for the mandatory keys so it reads neither the stack SSoT nor the registry, which means no particular pin is proved to resolve. +- `make check` does not start the stack. It resolves compose, lints the helpers and unit-tests them, and `check-proxy` runs temporary Envoy containers to validate the proxy config. CI's image and published-compose jobs start the stack from the dfe-docker image and check each demo source lands rows, but CI runs neither `make post` nor `make test-e2e`, so run those for a change to the data path. +- `check-compose` validates compose STRUCTURE, not digests. It substitutes placeholders for the mandatory keys so it reads neither the stack SSoT nor the registry, which means no particular pin is proved to resolve. `check-hardfail` resolves compose with a scrubbed environment and an empty `--env-file` so a developer's pinned `.env` cannot mask it, requires the run to fail on a missing pin, then reads every `image:` line and requires each to resolve to an `@sha256:` digest. diff --git a/Makefile b/Makefile index e0a4760..51719fd 100644 --- a/Makefile +++ b/Makefile @@ -39,14 +39,8 @@ ifneq ($(strip $(DFE_DATA_ROOT)),) STORAGE_FLAGS := -f compose/storage.yml endif -# Goals that work without a resolved service profile. The check-* targets belong -# here because their whole point is running on a fresh checkout -- resolving a -# profile first would make them fail for the reason they exist to detect. -# -# down/clean belong here too: neither references PROFILE_FLAGS or ACTIVE_SERVICES -# any more (both sweep every profile), and requiring a resolvable profile to STOP -# a stack is the same lockout the compose secret comments argue against -- set -# DFE_PROFILE to something that does not exist and you could not tear down. +# Goals that run without a resolved profile, so a fresh checkout can be checked +# and a stack on a broken profile can still be stopped. BOOTSTRAP_GOALS := init env-files creds certs up demo dev-posture e2e-posture e2e-posture-off e2e-posture-check help stack dial modes down clean demo-stop limits check check-compose check-hardfail check-dockerfile check-docs check-locks check-proxy check-python check-tests print-default-profile print-demo-stream publish-compose # Resolve the active profile only when a goal actually needs the compose stack @@ -106,19 +100,8 @@ ifneq (,$(filter-out $(BOOTSTRAP_GOALS),$(or $(MAKECMDGOALS),help))) endif # --------------------------------------------------------------------------- -# Web UI exposure -# -# Two dials, because "external" means different things on the two boxes compose -# deploys to. DFE_BIND_SCOPE=localhost (the default, a developer workstation) -# publishes every UI on 127.0.0.1 so the host sees them and the LAN does not; -# DFE_BIND_SCOPE=all publishes on 0.0.0.0 for a small/VM deploy. It resolves to -# DFE_UI_BIND_HOST, the address compose interpolates into every UI port mapping. -# Ingest (DFE_INGRESS_BIND_HOST) and the operator ports (DFE_BIND_HOST) keep -# their own audiences and are untouched by it. -# -# Every UI publishes by default. An opt-out drops that service's ports mapping, which compose merging cannot do by override, so make writes a `!reset` entry per unpublished service into one fragment and chains it the way STORAGE_CHAIN chains the storage overlay. DFE_INFRA_UIS_EXTERNAL=false is the kill switch: it unpublishes every INFRA-class UI at once and beats their individual flags. dfe-ui and the engine API are PRODUCT class and answer only to their own. -# -# Below the profile block because DFE_AUTH_RESOLVED comes from .tmp/profile.mk, so the profile key and the env var can never disagree about whether the auth proxies are in play. The bootstrap goals skip that include and read it empty, which is the auth-off branch, the safe direction for a target that only stops a stack or checks a file. +# Web UI exposure: docs/operating.md, "Web UIs: one scope dial, one kill switch". +# Below the profile block, because DFE_AUTH_RESOLVED comes from .tmp/profile.mk. # --------------------------------------------------------------------------- DFE_BIND_SCOPE ?= localhost ifeq ($(strip $(DFE_BIND_SCOPE)),localhost) @@ -129,15 +112,8 @@ else $(error DFE_BIND_SCOPE must be `localhost` or `all`, got '$(strip $(DFE_BIND_SCOPE))') endif -# `all` publishes the UIs on addresses this box cannot infer, and two services -# build absolute URLs from DFE_EXTERNAL_ORIGIN -- dfe-proxy's frame-ancestors -# policy for the embedded dfe-hyperdx views, and next-auth's redirects. At loopback -# both name the browser's own machine, so the console comes up with its -# observability views blocked. -# -# Only the goals that start or self-test the stack ask for it. The bootstrap -# goals are exempt for the reason BOOTSTRAP_GOALS gives: needing a key to stop a -# stack, or to check a file, is a lockout. +# DFE_BIND_SCOPE=all needs DFE_EXTERNAL_ORIGIN on the goals that start or self-test +# the stack: docs/configuration.md, "External origin". ORIGIN_GOALS := dev ci up apply infra post test-source test-flows print-compose-env # The e2e suites start a stack per test and probe the console at DFE_EXTERNAL_ORIGIN's host under console TLS, so there they need what a start goal needs. E2E_GOALS := test-e2e test-resilience @@ -300,7 +276,7 @@ ifeq ($(strip $(DFE_OTEL_RESOLVED))-$(strip $(DFE_CONTAINER_LOGS_ENABLED)),true- $(eval $(call chain_fragment,compose/container-logs.yml)) endif -# Live console edits: `make dev LIVE_UI=1` runs dfe-ui as `next dev` over DFE_SRC_ROOT/dfe-ui (compose/live-ui.yml). It rides the dev paths only, so the registry goals (ci, up, apply without DEV) never run a dev server. It chains after the container-logs fragment because it puts dfe-ui back on the default log driver. +# LIVE_UI=1 runs dfe-ui as `next dev` on the dev goals only (docs/live-console.md). It chains after the container-logs fragment because it puts dfe-ui back on the default log driver. # The Node image comes from the checkout's own Dockerfile rather than a second pin here, read without a shell so a path with a quote in it stays a path. LIVE_UI_GOALS := dev dev-build apply apply-services ifneq ($(strip $(LIVE_UI)),) @@ -322,11 +298,8 @@ ifneq ($(strip $(LIVE_UI)),) endif # --------------------------------------------------------------------------- -# Console TLS (opt-in) -# -# DFE_PROXY_TLS=true chains compose/tls.yml, which serves the console, the dfe-hyperdx embed and both dfe-hyperdx origins over TLS with one certificate. Outside a dev posture the engine marks its OIDC cookies Secure, so a sign-in on any hostname but localhost needs it. -# -# The engine trusts X-Forwarded-Proto from dfe-proxy's address alone, so that address is fixed: the last host of DFE_NETWORK_SUBNET, with every other container allocated from the first half. Make cannot do the address arithmetic, so scripts/proxy_tls.py does. +# Console TLS (opt-in): docs/configuration.md, "Console TLS (opt-in)". +# scripts/proxy_tls.py does the subnet arithmetic make cannot. # --------------------------------------------------------------------------- # Exactly `true` or `false`, so a `True` or a `1` fails here rather than quietly serving plain http. Empty reads as false. DFE_PROXY_TLS ?= false @@ -372,12 +345,12 @@ ifneq ($(strip $(DFE_RECEIVER_HTTPS_PORT)),) endif endif -# These goals recreate services on the network that already exists with no `make down` first; the e2e suites keep ClickHouse, and so its network, between tests. Docker cannot re-address a live network, so a stack whose network predates a DFE_PROXY_TLS flip would leave every recreated service Exited. +# These goals recreate services on the network that exists, with no `make down` first, and the e2e suites keep ClickHouse and its network between tests. Docker cannot re-address a live network, so a stack whose network predates a DFE_PROXY_TLS flip would leave every recreated service Exited. NETWORK_REUSE_GOALS := apply apply-services infra $(E2E_GOALS) PROXY_NETWORK_REUSE := $(filter $(NETWORK_REUSE_GOALS),$(or $(MAKECMDGOALS),help)) # Compose names the default network after the project, which is COMPOSE_PROJECT_NAME or this directory. PROXY_PROJECT := $(or $(strip $(COMPOSE_PROJECT_NAME)),$(notdir $(CURDIR))) -# The certificate pair, the subnet and the network as Docker holds it now; the helper prints the first problem and nothing when there is none. +# The certificate pair, the subnet and the network as Docker holds it now. The helper prints the first problem, and nothing when there is none. # A helper that exits non-zero printed no problem because it checked nothing, so that exit becomes the problem and the start is refused. ifneq ($(PROXY_TLS_START)$(PROXY_NETWORK_REUSE),) $(CERT_DIR_REFUSAL) @@ -433,32 +406,19 @@ init: ## Create .env and per-service env/.env files from templates, and @python3 scripts/init.py @python3 scripts/creds.py --write -# Start targets require every per-service env file (dfe-engine reads its secrets -# from env/engine.env). Compose marks them optional so `make down` never needs them. -# The guard creates what is missing (init is non-destructive) rather than refusing, -# so a release that adds a template does not stop an initialised deployment. The -# .env prerequisite mints the secrets before a fresh checkout's first start. +# Start targets need every env/.env, so this creates any a release +# added. Compose marks them optional so `make down` never needs them. .PHONY: env-files env-files: .env ## Assert every env/.env exists, creating any the templates have gained @python3 scripts/env_files.py -# `make init` mints the admin and break-glass passwords and prints neither, so -# this is the hand-over. `make up` and `make dev` end with it; `make ci` does not -# call it, because its stdout is a build log. `make init` and `make up` pass -# --write, which also leaves access-summary.md (0600, gitignored) for the operator. +# The hand-over of the minted logins: docs/developing.md, "Logging in the first time". .PHONY: creds creds: ## Print the access summary. The admin password prints on a TTY only -- a pipe, a file, a CI log or DFE_CREDS_SHOW=0 gets the env/engine.env key instead @python3 scripts/creds.py -# A dev tyre-kick logs in without looking anything up, so `make dev` writes the -# KNOWN default password and DFE_ENGINE_ENV=dev, the one posture the engine accepts it in. -# A .env already on a dev posture keeps a non-default password, the replacement -# `make post` recorded after the engine made the admin change the default. -# It refuses with exit 2 on any other DFE_ENGINE_ENV and copies each file it rewrites to -# .bak- first, so a minted password in env/engine.env survives. -# -# `AUTH=real` is the other half: local images running the same authentication flow -# a deployment gets, so it mints a password and writes a non-dev posture instead. +# `make dev` writes the known default password and DFE_ENGINE_ENV=dev, and +# AUTH=real a minted one: docs/developing.md, "Logging in the first time". AUTH ?= ifeq ($(strip $(AUTH)),real) DEV_POSTURE_ARG := --real @@ -482,32 +442,21 @@ e2e-posture-off: ## Put back the .env and env/engine.env from before e2e-posture e2e-posture-check: ## Exit 1 naming what is missing unless the env files are in the e2e posture, which the Playwright run checks first @python3 scripts/e2e_posture.py check -# `make stack VERSION=X.Y.Z` pins the certified set. `make dial` writes the dial's -# version.pin to DFE_STACK_VERSION in .env, so `make dial && make stack` pins -# straight from the deployment dial. An explicit VERSION= on the command line wins. -# VERSION=latest (rc to include pre-releases) instead pins the newest published -# stack, read anonymously from the OCI registry. DFE_STACK_REPIN_IMAGES=1 also -# repins every ghcr.io/hyperi-io image at its component's newest GitHub release, -# through gh, or anonymously where it is not installed. Nobody certified that mix. -# Still digest-pinned either way. +# Pinned, latest and the GitHub-release repin: docs/deploying.md, "Staying +# current". An explicit VERSION= on the command line wins over the dial's pin. VERSION ?= $(DFE_STACK_VERSION) .PHONY: stack stack: .env ## Pin image versions into .env from the DFE stack SSoT (VERSION=X.Y.Z[-rc.N], or latest|rc for the newest published stack) @python3 scripts/stack.py $(VERSION) -# The deployment dial (deployment.yaml) is the single SSoT a deployment turns. -# This renders its docker-vm slice into .env; `make init` still mints the secrets, -# the dial merges the deploy-controlled keys over them. Canonical superset lives -# in dfe-infra; a lone dfe-docker clone deploys from deployment.yaml alone. +# The deployment dial: docs/deploying.md, "The deployment dial". .PHONY: dial dial: .env ## Render deployment.yaml (the deployment dial) into .env and env/engine.env -- docker-vm slice @python3 scripts/render_dial.py -# The two deploy-currency modes, stated in one place: PINNED (make stack + make -# ci) vs TRACK-LATEST (the ops/daemon-update timer). Reads local files only, so -# it is safe on a fresh checkout; the live latest-vs-applied number comes from -# `self_update.py --dry-run`, which this points at. +# Pinned against track-latest: docs/deploying.md, "Staying current". Reads local +# files only, so it is safe on a fresh checkout. .PHONY: modes modes: ## Show the deploy modes -- pinned vs track-latest -- and which one this checkout uses @python3 scripts/show_modes.py @@ -523,9 +472,7 @@ ifneq ($(strip $(DFE_DATA_ROOT)),) @mkdir -p $(addprefix $(DFE_DATA_ROOT)/,clickhouse kafka-apache kafka-redpanda archiver dlq-spool fetcher-cursors transform-elastic-geoip engine-config engine-schemas hyperdx-documentdb) endif -# `dev` and `dev-build` clone the component repos from github.com/hyperi-io, -# over public https with no token. `make ci` runs the same stack from the -# published images. +# `dev` and `dev-build` clone the public component repos over https with no token. .PHONY: dev dev: env-files dev-posture down storage-dirs ## Build local DFE images from source and start the dev stack (LOCAL="svc ..." builds only those; AUTH=real for a minted login; LIVE_UI=1 runs the console as `next dev` from DFE_SRC_ROOT/dfe-ui), then print the access summary. Clones the public component repos python3 scripts/port_check.py -- $(DEV_FLAGS) $(PROFILE_FLAGS) @@ -568,11 +515,8 @@ up: ## Start the stack from the pinned registry images (PROFILE=slim-grpc and VE @$(MAKE) --no-print-directory ci @python3 scripts/creds.py --write -# The command dfe-engine's restart and recreate hints name after a config write: -# resolving the profile regenerates the instances fragment, so it creates a -# per-source container the engine has just declared, recreates one that exists, -# and removes one whose source the engine has deleted. -# DEV=1 recreates from the local images a `make dev` stack runs. +# The command dfe-engine's restart and recreate hints name: docs/architecture.md, +# "The engine renders each app's config". DEV=1 recreates from the local images. ifneq ($(strip $(DEV)),) APPLY_FLAGS = $(DEV_FLAGS) else @@ -616,15 +560,7 @@ infra: .env storage-dirs ## Start infrastructure services docker compose $(PROFILE_FLAGS) up -d # --------------------------------------------------------------------------- -# Validation -# The same commands CI runs, so a green local run means a green pipeline. -# -# Safe on a fresh checkout: no stack SSoT and no credentials needed, and none of -# them writes a .env. One honest caveat: check-dockerfile pulls the pinned -# hadolint image while check-tests resolves the pinned pytest, so those two want -# a network the first time; the rest need none. -# -# check-proxy reads the stack SSoT, anonymously through oras, when DFE_PROXY_VERSION is unset, then pulls that Envoy and needs openssl on the host for its throwaway certificate. +# Validation: the static checks CI runs, docs/checks.md. # --------------------------------------------------------------------------- .PHONY: check @@ -634,9 +570,7 @@ check: check-compose check-hardfail check-dockerfile check-docs check-proxy chec limits: ## Show the resource limits and totals, computed from the resolved config @python3 scripts/show_limits.py -# Pinned so a new ruff release cannot turn this red on an unrelated change, and -# so a local run lints with the SAME version CI does. CI installs the pin itself -# and sets RUFF=ruff; override the same way if you have no uvx. +# Pinned so a local run lints with the ruff CI runs. CI installs the pin and sets RUFF=ruff. RUFF_VERSION := 0.15.22 RUFF ?= uvx ruff@$(RUFF_VERSION) @@ -649,9 +583,7 @@ check-python: ## Lint the helper scripts (config in ruff.toml) $(RUFF) check scripts/ ops/ resources/demo/ resources/demo-*/ $(RUFF) format --check scripts/ ops/ resources/demo/ resources/demo-*/ -# Pinned for the same reason RUFF is, and installed the same way in CI -# (PYTEST=pytest). Unit tests over the helper scripts: they -# touch a tmp_path .env and start nothing, so they belong with the static checks. +# Pinned like RUFF, and CI sets PYTEST=pytest. PYTEST_VERSION := 9.2.0 # The seed tests parse resources/seed with the PyYAML scripts/seed.py pins. scripts/tests/test_script_locks.py holds the two pins equal PYYAML_VERSION := 6.0.3 @@ -732,15 +664,8 @@ check-hardfail: ## Assert an unpinned stack refuses to start instead of pulling check-docs: ## Assert every relative link in the docs set still resolves @python3 scripts/check_docs.py -# `--failure-threshold error` matches the house gate (hyperi-ci's -# quality/hadolint.py gates on error severity only). Warning/info/style still -# print, they just do not fail -- which is the point: the estate's routine noise -# (DL3008 apt-pin, DL4006 pipefail) is all warning-tier, so a threshold of -# `info` turns every one of those into a blocked build and pushes people towards -# blanket ignores. Suppressed findings tell you nothing; visible ones do. -# -# hadolint version tracks tools.hadolint in hyperi-ci's config/versions.yaml, -# and the digest is pinned here as well. +# Gates on error severity, as hyperi-ci's hadolint step does, so warnings print +# without failing. The version tracks tools.hadolint in hyperi-ci's config/versions.yaml. HADOLINT_IMAGE := hadolint/hadolint:v2.15.1@sha256:32dac94127fd60b7b7e3fbfc65e1383b9b5e25c9bfd7b8536de7a539fe68a12d .PHONY: check-dockerfile @@ -748,11 +673,8 @@ check-dockerfile: ## Lint the dev builder and dfe-docker image Dockerfiles (hado docker run --rm -i $(HADOLINT_IMAGE) hadolint --failure-threshold error - < docker/dfe-rust-builder.Dockerfile docker run --rm -i $(HADOLINT_IMAGE) hadolint --failure-threshold error - < docker/dfe-docker.Dockerfile -# Validate mode loads every listener, cluster and TLS context, so a config that would crash-loop either proxy fails here instead. -# -# The Envoy is the one the stack runs: DFE_PROXY_VERSION when set, else the pin scripts/stack.py reads from the stack manifest at VERSION, which falls back to deployment.example.yaml's version.pin. -# -# The TLS variants read their certificate at validation, so a throwaway P-384 pair is minted into a scratch directory. Envoy runs as the make user, as both proxies do under TLS, because the image's own user cannot read a 0600 key. +# Validates both proxy configs on the Envoy the stack runs, with a throwaway +# P-384 pair for the TLS variants: docs/checks.md. ENVOY_REPO := envoyproxy/envoy # Run in the recipe, since $(shell) drops the exit status and a failed lookup would validate on an empty tag PROXY_VERSION_COMMAND = $(if $(strip $(DFE_PROXY_VERSION)),echo '$(strip $(DFE_PROXY_VERSION))',python3 scripts/stack.py --print DFE_PROXY_VERSION $(VERSION)) @@ -777,7 +699,7 @@ check-proxy: ## Validate the Envoy proxy configs, plain and TLS, on the Envoy th # Testing # --------------------------------------------------------------------------- -# The suite starts each test's stack from its own -f list. Under console TLS that stack needs the TLS fragment and the network it pins, so the suite takes every fragment `make ci` chains, the exposure opt-outs and container logs included; with TLS off it keeps its own list. DFE_E2E_COMPOSE_FILES follows either way, and the suite skips the empty entries the colons leave. +# The suite starts each test's stack from its own -f list. Under console TLS that stack needs the TLS fragment and the network it pins, so the suite takes every fragment `make ci` chains, the exposure opt-outs and container logs included. With TLS off it keeps its own list. DFE_E2E_COMPOSE_FILES follows either way, and the suite skips the empty entries the colons leave. E2E_COMPOSE_FILES = $(if $(filter true,$(strip $(DFE_PROXY_TLS))),$(UI_CHAIN)):$(strip $(DFE_E2E_COMPOSE_FILES)) # The scripts that import packages beyond the standard library declare them inline (PEP 723) and lock them beside the script, so uv installs exactly the hashed set and a host needs only uv. @@ -797,9 +719,7 @@ test-resilience: ## Outage tests: stop a service under load and prove the rest s test-flows: ## Flow shapes against a running stack (needs DFE_ENGINE_REPO; FLOW_ARGS passes flags) @$(UV) run --locked --script ./scripts/test_flows.py $(FLOW_ARGS) -# The post-deploy source test: add a source through the console and prove every -# hop. The runner is dfe-infra's, the same one `dfe-ops acceptance --suite -# source` calls, so both targets test one definition of the steps. +# The post-deploy source test, on dfe-infra's runner: docs/checks.md. .PHONY: test-source test-source: ## Post-deploy source test against a running stack (needs DFE_INFRA_DIR + DFE_ENGINE_REPO; SOURCE_ARGS passes flags) @python3 ./scripts/test_source.py $(SOURCE_ARGS) @@ -811,15 +731,7 @@ proof: ## Unattended acceptance: test-e2e, restore the stack (make ci), then tes @python3 ./scripts/proof.py # --------------------------------------------------------------------------- -# Power-on self test -# Runs automatically after `make dev` / `make ci`. Opt OUT with -# DFE_POST_ENABLED=false -- a self test you have to remember to run is not one. -# The ingest claim is transport-agnostic: it injects at the ingest edge and reads -# ClickHouse, so it behaves identically on the kafka and kafka-less (grpc) -# profiles. -# -# `make post` exits non-zero on failure and the auto-run after dev/ci aborts the -# target with it, so a stack that cannot move data is never handed over as ready. +# Power-on self test: docs/self-test.md # --------------------------------------------------------------------------- .PHONY: post @@ -827,8 +739,7 @@ post: ## Prove the RUNNING stack moves data end to end (DFE_POST_ENABLED=false t @env DFE_POST_SERVICES='$(strip $(SERVICES))' python3 ./scripts/post.py # --------------------------------------------------------------------------- -# Seed -# The starter set in resources/seed, created through the engine API on a RUNNING stack, plus fake events for its hunt to find. hunts also creates the orgs and rules in the same seed.py run, because the engine refuses a hunt naming an org or rule it does not hold. accounts creates the orgs first, because the engine refuses an account naming an org it does not hold. One run means a login asked for at the terminal is asked for once. Every target but events creates only what the engine lacks. events sends its records again on every run. A volume wipe loses every object, so run the targets again after one. +# Seed: docs/seeding.md # --------------------------------------------------------------------------- .PHONY: orgs @@ -856,8 +767,7 @@ events: ## Send the fake resources/seed events through the receiver, after `make @$(UV) run --locked --script ./scripts/seed.py events # --------------------------------------------------------------------------- -# Demo -# Raw firewall syslog through the RUNNING stack, each line's vendor detected by dfe-transform-elastic in agentless mode and landed in that vendor's table +# Demo: docs/evaluating.md, "Raw syslog, parsed per device" # --------------------------------------------------------------------------- # The replay script inside the dfe-transform-elastic checkout, fetched at the tag DFE_TRANSFORM_ELASTIC_VERSION pins or, with DEV=1, read from DFE_SRC_ROOT/dfe-transform-elastic @@ -896,9 +806,7 @@ ps: ## Show running containers, and warn about a deleted source's container stil docker compose ps @python3 scripts/instances.py --stale -- $(APPLY_FLAGS) $(PROFILE_FLAGS) -# Tears down across EVERY profile, so a switched profile leaves no container -# holding host ports or answering probes. Volumes survive; `make clean` is the -# one that deletes data. +# Tears down every profile and keeps volumes. `make clean` deletes data. .PHONY: down down: demo-stop ## Stop and remove containers across all profiles and the `make demo` pack (keeps volumes) docker compose -f docker-compose.yml --profile "*" down --remove-orphans diff --git a/README.md b/README.md index 574ca38..7be5a3f 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ The whole DFE stack on one host with Docker Compose, from pinned release images ## What it does - Runs DFE on one box, an edge site, a partner deployment or a developer's test rig. Compose is a supported production target for small environments. Kubernetes stays the primary path, so where both would work, choose Kubernetes. -- Pins every image by digest from a certified DFE stack release with `make stack`. With a pin unset the stack refuses to start rather than pull `latest`. +- Pins every image by digest from a certified DFE stack release with `make stack`. With a pin unset the stack will not start rather than pull `latest`. - Starts from the registry images with `make up`, or builds chosen components from source with `make dev`. - Runs Kafka or Redpanda as the broker, with opt-in overlays for TLS, storage and an oauth2-proxy `auth` profile. - Checks that the running stack moves data from the ingest edge to a ClickHouse row with `make post`, which fails if an enabled run asserts nothing. `DFE_POST_ENABLED=false` skips it. @@ -34,7 +34,7 @@ docker compose -f oci://ghcr.io/hyperi-io/dfe-docker:demo up -d docker compose -f oci://ghcr.io/hyperi-io/dfe-docker:demo run --rm secrets-init login ``` -`down` with the same `-f` stops either one, and `down -v` deletes its data too. +`down` with the same `-f` stops either one, and `down -v` deletes its data. From a clone, on a host with Docker Compose v2, make, git and Python 3: @@ -71,7 +71,7 @@ The suite's published docs are at , and this reposit ## Help and contributing -How to work in this repo, the commands that prove a change and what tends to bite are in [CONTRIBUTING.md](CONTRIBUTING.md). A question that is not a bug goes in a new issue. +How to work in this repo, the commands that test a change and what tends to bite are in [CONTRIBUTING.md](CONTRIBUTING.md). A question that is not a bug goes in a new issue. Open bugs: [this repo](https://github.com/hyperi-io/dfe-docker/issues?q=is%3Aopen+is%3Aissue+type%3ABug+-label%3Abacklog+-label%3Anext-minor) | [whole DFE suite](https://github.com/search?type=issues&q=is%3Aopen+is%3Aissue+type%3ABug+-label%3Abacklog+-label%3Anext-minor+repo%3Ahyperi-io%2Fdfe-infra+repo%3Ahyperi-io%2Fdfe-docker+repo%3Ahyperi-io%2Fdfe-deploy+repo%3Ahyperi-io%2Fdfe-engine+repo%3Ahyperi-io%2Fdfe-loader+repo%3Ahyperi-io%2Fdfe-receiver+repo%3Ahyperi-io%2Fdfe-fetcher+repo%3Ahyperi-io%2Fdfe-archiver+repo%3Ahyperi-io%2Fdfe-transform-vrl+repo%3Ahyperi-io%2Fdfe-transform-vector+repo%3Ahyperi-io%2Fdfe-transform-elastic+repo%3Ahyperi-io%2Fdfe-ui+repo%3Ahyperi-io%2Fdfe-hyperdx+repo%3Ahyperi-io%2Fscalo-rs+repo%3Ahyperi-io%2Fscalo-py+repo%3Ahyperi-io%2Flogreducer+repo%3Ahyperi-io%2Fdfe-schemas+repo%3Ahyperi-io%2Fvector-vrl+repo%3Ahyperi-io%2Fculvert+repo%3Ahyperi-io%2Fdfe-clickhouse-rs+repo%3Ahyperi-io%2Ffactbook-rs) diff --git a/docker-compose.yml b/docker-compose.yml index d6c7e55..d7cd49c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,17 +6,7 @@ # License: BUSL-1.1 # Copyright: (c) 2026 HYPERI PTY LIMITED # -# Profile | Services -# ---------------|---------------------------------------------- -# clickhouse | clickhouse -# kafka-apache | kafka-apache, the Apache-2.0 broker -# kafka-redpanda | kafka-redpanda, the default broker -# kafka-ui | kafka-ui -# otel | otel-collector -# auth | the three oauth2-proxy-* gates -# dfe | the data-plane apps and their init one-shots -# core | engine, hunt runner, UI, proxy and contracts -# hyperdx | dfe-hyperdx and its store, proxy and dashboards +# Compose profiles and the services each starts: docs/architecture.md. include: - path: compose/one-shots.yml diff --git a/docs/README.md b/docs/README.md index eb25688..d4c141c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -5,15 +5,18 @@ Running DFE on a single host with Docker Compose: a first look, deploying and up - [How the Compose stack fits together](architecture.md) - [Gating the infra UIs with OIDC](auth-profile.md) - [The Kafka broker](broker.md) +- [Outage, source and static checks](checks.md) - [Common failures](common-failures.md) - [Configuration reference](configuration.md) - [Deploying, and moving to a newer stack](deploying.md) - [Developing against the stack](developing.md) - [Taking DFE for a first look](evaluating.md) - [Known issues](known-issues.md) +- [Console edits without a rebuild](live-console.md) - [How the stack reports on itself](observability.md) - [Operating the DFE Compose stack](operating.md) - [Profiles](profiles.md) +- [Proving the pipeline with the e2e suite](testing.md) - [Secrets and storage](secrets-and-storage.md) - [Seeding a starter set](seeding.md) - [The power-on self test](self-test.md) diff --git a/docs/architecture.md b/docs/architecture.md index f4f5bcb..628c20c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -22,62 +22,66 @@ repo, not here. ## Events arrive two ways and leave one Pushed to `dfe-receiver` or pulled by `dfe-fetcher`, and out through `dfe-loader` -into ClickHouse. Everything between is optional and profile-selected. +into ClickHouse. Everything between is optional and profile-selected. This diagram shows the running parts of the data path on a Kafka profile. ```mermaid flowchart LR - classDef ext fill:#E69F00,stroke:#333333,color:#000000 - classDef dfe fill:#56B4E9,stroke:#333333,color:#000000 - classDef store fill:#009E73,stroke:#333333,color:#000000 - classDef broker fill:#F0E442,stroke:#333333,color:#000000 - - push(["Push clients: HTTP, Vector, Beats, OTLP, HEC, syslog, flow"]):::ext - api(["REST APIs, databases, files"]):::ext - - subgraph ingest [Ingest] - receiver["dfe-receiver"]:::dfe - fetcher["dfe-fetcher"]:::dfe - end - - land[(Kafka land topics)]:::broker - - subgraph xform [Transform - optional] - vrl["dfe-transform-vrl"]:::dfe - vector["dfe-transform-vector"]:::dfe - elastic["dfe-transform-elastic"]:::dfe + accTitle: L2 Services -- DFE Compose stack + accDescr: Push clients send events to dfe-receiver and dfe-fetcher pulls them from APIs, databases and files. Both produce to Kafka topics. dfe-loader reads the topics and writes ClickHouse, dfe-archiver writes each source's raw events to its archive, and a transform reads a source's topic and writes the transformed topic the loader then reads. + + push["Push clients"] + pull["APIs, databases,
files"] + + subgraph stack["DFE Compose stack"] + subgraph ingest["Ingest"] + receiver["dfe-receiver"] + fetcher["dfe-fetcher"] + end + topics[("Kafka topics")] + loader[["dfe-loader"]] + transforms["Transforms,
one per source"] + archiver["dfe-archiver"] + ch[("ClickHouse")] + archive[("Archive")] end - load[(Kafka main_load)]:::broker - archiver["dfe-archiver"]:::dfe - loader["dfe-loader"]:::dfe - ch[(ClickHouse)]:::store - - push --> receiver - api --> fetcher - receiver -->|kafka transport| land - fetcher -->|kafka transport| land - receiver -->|grpc transport| loader - fetcher -->|grpc transport| loader - land --> archiver - land --> loader - land --> xform - xform --> load - load --> loader + push -->|events| receiver + pull -->|events| fetcher + ingest --> topics + topics --> loader + topics --> transforms + topics --> archiver loader --> ch + archiver --> archive + transforms -.-> topics + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef store fill:#DDF3EC,stroke:#00785A,stroke-width:1px,color:#0B3327 + classDef async fill:#F6E6EF,stroke:#A8517F,stroke-width:1px,color:#3D1529 + classDef ext fill:#EEF0F2,stroke:#6B7280,stroke-width:1px,stroke-dasharray:4 3,color:#1F2937 + classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 + class push,pull ext + class topics async + class ch,archive store + class stack,ingest group ``` +*Figure 1. L2 Services -- DFE Compose stack. The dotted arrow is a transform writing `_load` back to the broker.* Up: top of dfe-docker. Down: [L4 Detail -- Two listeners on port 6000](#component-repos-and-their-grpc-listeners), [L4 Detail -- Loader topic choice](known-issues.md#a-_load-topic-changes-what-the-loader-reads), [L4 Detail -- Loader and schema checks](troubleshooting.md#events-are-accepted-but-nothing-lands), [L4 Detail -- Topic and dead-letter checks](troubleshooting.md#events-are-accepted-but-nothing-lands). + +Push clients are HTTP, Vector, Beats, OTLP, HEC, syslog and flow senders. dfe-engine, the console and Search run beside this path, and Figure 3 shows the console's side. + Transport is a per-profile choice, not a global one. On a `*-kafka` profile the ingest components produce to a `*_land` topic derived from the event's `_source`. `config/loader/kafka.yaml` names no topics, so the loader auto-discovers every topic ending in one of its `routing.topic_suffixes`, `_land` and `_load` by -default. On a `*-grpc` profile there is no broker: the ingest components dial +default. On a `*-grpc` profile there is no broker: the receiver dials `dfe-loader:6000` directly. -Each transform reads a topic and writes a topic on Kafka but pushes to the loader on gRPC. `dfe-archiver` archives each source from its landing topic on Kafka. No gRPC profile runs it, so archiving needs a Kafka profile. Once a transform writes `_load`, the loader reads it instead of `_land`: scalo drops `_land` from its subscription once `_load` exists. +A transform reads `_land` and writes `_load`, and runs on a Kafka profile only. `dfe-archiver` archives each source from its landing topic, and no gRPC profile runs it, so archiving needs a Kafka profile. Once a transform writes `_load`, the loader reads it instead of `_land`: scalo drops `_land` from its subscription once `_load` exists. dfe-engine creates a transform's table and nothing here provisions it. Create the source before sending it data: a loader that meets rows for a table it has no schema for buffers them, then dead-letters them. -The source NAME propagates character for character. The engine requires a Kubernetes DNS-1123 label (hyphens, never underscores, because a source-bound app deploys one instance named for its source), the receiver substitutes it into `<_source>_land`, and the loader strips `_load` back off for the table. So `filebeat-vector` gives `filebeat-vector_land`, `filebeat-vector_load` and a table name that needs backticks in SQL. +The source NAME propagates character for character. The engine needs a Kubernetes DNS-1123 label (hyphens, never underscores, because a source-bound app deploys one instance called after its source), the receiver substitutes it into `<_source>_land`, and the loader strips `_load` back off for the table. So `filebeat-vector` gives `filebeat-vector_land`, `filebeat-vector_load` and a table name that needs backticks in SQL. `slim-*` is the core data path alone: receiver, loader, engine, console and dfe-hyperdx. `single-kafka` adds the per-source apps its keys enable ([profiles.md](profiles.md#per-source-apps)) and one archiver. The `-kafka` profiles add the broker ([broker.md](broker.md)). dfe-fetcher runs idle until a fetcher source exists, and no transform starts until a source created through the engine binds one. @@ -87,29 +91,38 @@ On Kubernetes an app's chart turns its values overlay into a ConfigMap the pod m The committed file stays the DEPLOYMENT's (brokers, warehouse credentials, the spool) and the overlay wins wherever the two name the same key, so nothing deployment-specific is written into the deploy repo. `resolve_profile.py` turns this on wherever the core footprint runs. dfe-fetcher and the three transforms have no committed file: the overlay is their whole config, and their broker comes from `DFE_KAFKA_BOOTSTRAP_SERVERS` in `.env`. -Nothing restarts a container. A change the running app cannot take in place is reported by the API as `restart required: make apply SERVICES=` and one to its env file as `recreate required:` with the same command. `make apply` creates a per-source container the engine has only just declared, recreates one that exists and removes the container of a source the engine has deleted, which it finds by the `dfe.instance-of` label only generated services carry. A delete names no command, so `make apply` with no `SERVICES` does that removal alone and starts nothing. +Nothing restarts a container. A change the running app cannot take in place is reported by the API as `restart required: make apply SERVICES=` and one to its env file as `recreate required:` with the same command. `make apply` creates a per-source container the engine has only just declared, recreates one that exists and removes the container of a source the engine has deleted, which it finds by the `dfe.instance-of` label only generated services have. A delete names no command, so `make apply` with no `SERVICES` does that removal alone and starts nothing. -A source with a transform stops sharing `main_land`: the receiver routes it to `_land`, and only a transform reading THAT topic sees it. So a transform is deployed once per source, as the Kubernetes tier deploys one per Source definition, and its input topic, program directory and output topic all carry the name. dfe-engine renders the instance's config into `dfe-app-config`, and `make apply` starts a service that extends the committed `dfe-transform-vrl`, `dfe-transform-vector` or `dfe-transform-elastic`, which carry the broker env and nothing of a source's own. +A source with a transform stops sharing `main_land`: the receiver routes it to `_land`, and only a transform reading THAT topic sees it. So a transform is deployed once per source, as the Kubernetes tier deploys one per Source definition, and its input topic, program directory and output topic all have the name. dfe-engine renders the instance's config into `dfe-app-config`, and `make apply` starts a service that extends the committed `dfe-transform-vrl`, `dfe-transform-vector` or `dfe-transform-elastic`, which have the broker env and nothing of a source's own. ## Two independent selectors decide what runs -Compose profiles select INFRASTRUCTURE. `service_profiles.yaml` selects DFE SERVICES. `scripts/resolve_profile.py` reads the second and derives the first, writing `.tmp/profile.mk` for the Makefile. +Compose profiles select INFRASTRUCTURE. `service_profiles.yaml` selects DFE SERVICES. `scripts/resolve_profile.py` reads the second and derives the first, writing `.tmp/profile.mk` for the Makefile. This diagram shows where the resolver reads from and what reads its output. ```mermaid -flowchart TD - classDef cfg fill:#CC79A7,stroke:#333333,color:#000000 - classDef out fill:#56B4E9,stroke:#333333,color:#000000 +flowchart LR + accTitle: L4 Detail -- Profile resolution + accDescr: resolve_profile.py reads the active profile from service_profiles.yaml and the DFE_PROFILE and footprint flags from the environment. It writes profile.mk, which the Makefile includes to pass the Compose profiles and DFE services to the targets that start or check a stack. + - sp["service_profiles.yaml - active_profile, DFE_PROFILE overrides"]:::cfg - resolve["scripts/resolve_profile.py"]:::cfg - mk[".tmp/profile.mk - PROFILE_FLAGS, DFE_SERVICES, config paths"]:::out - make["make dev / make ci"]:::out + sp["service_profiles.yaml"] + env[".env flags"] + resolve["resolve_profile.py"] + mk[".tmp/profile.mk"] + make["make targets"] - sp --> resolve - resolve --> mk + sp -->|profile| resolve + env -->|overrides| resolve + resolve -->|writes| mk mk --> make + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef core fill:#DCEBF7,stroke:#0072B2,stroke-width:2px,color:#0B2A3F + class resolve core ``` +*Figure 2. L4 Detail -- Profile resolution. The flags are `DFE_PROFILE` and the footprint toggles such as `DFE_CORE_ENABLED`.* Up: [dfe-docker documentation](README.md). + | Compose profile | Services | |---|---| | `clickhouse` | `clickhouse` | @@ -122,28 +135,37 @@ flowchart TD | `otel` | `otel-collector` | | `auth` | the three `oauth2-proxy-*` gates ([auth-profile.md](auth-profile.md)) | -A profile declares its whole footprint through the optional `clickhouse`, `core`, `kafbat`, `hyperdx`, `otel` and `auth` keys, each overridable by the matching `.env` flag. Each service entry also names the config file it mounts, which is how one service gets a Kafka config and another a gRPC one without a second compose file. +A profile declares its whole footprint through the optional `clickhouse`, `core`, `kafbat`, `hyperdx`, `otel` and `auth` keys, each overridable by the matching `.env` flag. Each service entry names the config file it mounts, which is how one service gets a Kafka config and another a gRPC one without a second compose file. -The `core` footprint is the management plane, independent of transport. `dfe-proxy` gives the console and the engine API one origin, because the console's client calls the engine with a relative `/api/v1/...` base URL. +The `core` footprint is the management plane, independent of transport. `dfe-proxy` gives the console and the engine API one origin, because the console's client calls the engine with a relative `/api/v1/...` base URL. This diagram shows how dfe-proxy splits that origin. ```mermaid flowchart LR - classDef ext fill:#E69F00,stroke:#333333,color:#000000 - classDef dfe fill:#56B4E9,stroke:#333333,color:#000000 - classDef store fill:#009E73,stroke:#333333,color:#000000 - - user(["Browser"]):::ext - proxy["dfe-proxy - envoy :3000"]:::dfe - ui["dfe-ui :3000"]:::dfe - engine["dfe-engine :8000"]:::dfe - ch[(ClickHouse)]:::store - - user --> proxy - proxy -->|api v1 paths| engine - proxy -->|everything else| ui + accTitle: L4 Detail -- One console origin + accDescr: A browser reaches dfe-proxy on host port 3000. dfe-proxy sends paths under /api/v1 to dfe-engine and every other path to dfe-ui. dfe-engine reads and writes ClickHouse. + + browser["Browser"] + proxy["dfe-proxy"] + engine["dfe-engine :8000"] + ui["dfe-ui :3000"] + ch[("ClickHouse")] + + browser -->|host :3000| proxy + proxy -->|/api/v1| engine + proxy -->|other paths| ui engine --> ch + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef core fill:#DCEBF7,stroke:#0072B2,stroke-width:2px,color:#0B2A3F + classDef store fill:#DDF3EC,stroke:#00785A,stroke-width:1px,color:#0B3327 + classDef ext fill:#EEF0F2,stroke:#6B7280,stroke-width:1px,stroke-dasharray:4 3,color:#1F2937 + class browser ext + class proxy core + class ch store ``` +*Figure 3. L4 Detail -- One console origin. The ports in the boxes are container ports.* Up: [dfe-docker documentation](README.md). + ## dfe-engine owns the schema and the topics, and starts first Nothing here provisions a ClickHouse table or a Kafka topic. `dfe-engine` ships `/app/schemas` inside its own image, applies every object and every bootstrap topic from that manifest at startup, and reports healthy only once that pass converged. `dfe-schemas` rides inside the engine image, so there is no separate schema pin here and a schema change is a `dfe-engine` release. The ClickHouse service mounts only server settings from `config/clickhouse/`. A DDL file or a topic-creating step appearing in this repo is wrong by construction, and `scripts/tests/test_engine_only_schema_control.py` fails the build on one. @@ -152,13 +174,13 @@ Nothing here provisions a ClickHouse table or a Kafka topic. `dfe-engine` ships The apps then wait on `depends_on: dfe-engine: service_healthy`, because healthy means their tables and topics exist. A loader started first holds messages pending-schema and dead-letters them, which reads as data loss and is really start ordering. -Each limit tier extends `base` in `compose/base.yml`, whose `dns_search: ["."]` clears the search domain the host passes into every container. A bare name with no alias (`kafka` on a stack without a broker) then fails to resolve instead of reaching a host on that domain. +Each limit tier extends `base` in `compose/base.yml`, and its `dns_search: ["."]` clears the search domain the host passes into every container. A bare name with no alias (`kafka` on a stack without a broker) then fails to resolve instead of reaching a host on that domain. ## Component repos and their gRPC listeners Dev builds fetch source into a managed git cache, or from your own checkouts when `DFE_SRC_ROOT` is set -- see -[developing.md](developing.md#where-it-looks-for-your-source). +[developing.md](developing.md#some-from-source-the-rest-pinned). | Repo | Language | Role | Profile | |---|---|---|---| @@ -173,27 +195,40 @@ Dev builds fetch source into a managed git cache, or from your own checkouts whe | `dfe-ui` | TypeScript | The DFE console | `core` | | `dfe-hyperdx` | TypeScript | The console's Search, a fork of HyperDX. Repo and published image are both `dfe-hyperdx` | `hyperdx` | -dfe-transform-vector mounts `resources/enrichment/` at `/etc/dfe-transform-vector/data`, where its bundled filebeat pipeline reads `timezones.csv`. dfe-transform-elastic compiles its transforms in, and the engine uploads the enrichment tables a dfe-transform-vrl program reads, so neither mounts them. `make test-source` drives the filebeat, Vector and Elastic cases through every hop. dfe-transform-vector's one port carries everything: the wrapper scrapes Vector's `internal_metrics` off its `prometheus_exporter` on loopback and registers the samples on scalo's own recorder (dfe-transform-vector#70), so `vector_*` reads on its metrics port and rides the OTLP push. The exporter's 9598 is not published. +dfe-transform-vector mounts `resources/enrichment/` at `/etc/dfe-transform-vector/data`, where its bundled filebeat pipeline reads `timezones.csv`. dfe-transform-elastic compiles its transforms in, and the engine uploads the enrichment tables a dfe-transform-vrl program reads, so neither mounts them. `make test-source` drives the filebeat, Vector and Elastic cases through every hop. dfe-transform-vector's one port serves everything: the wrapper scrapes Vector's `internal_metrics` off its `prometheus_exporter` on loopback and registers the samples on scalo's own recorder (dfe-transform-vector#70), so `vector_*` reads on its metrics port and rides the OTLP push. The exporter's 9598 is not published. -Two gRPC listeners share port 6000, and they are not the same thing. +Two gRPC listeners share port 6000, and they are not the same thing. This diagram shows both on a `*-grpc` profile. ```mermaid flowchart LR - classDef ext fill:#E69F00,stroke:#333333,color:#000000 - classDef dfe fill:#56B4E9,stroke:#333333,color:#000000 + accTitle: L4 Detail -- Two listeners on port 6000 + accDescr: An external Vector client can push to dfe-receiver's Vector protocol listener on port 6000 once it is enabled. dfe-receiver pushes to dfe-loader's internal Push listener, also on port 6000, over the compose network. - client(["External client - Vector"]):::ext - receiver["dfe-receiver"]:::dfe - loader["dfe-loader"]:::dfe + client["Vector client"] - client -->|external - Vector protocol :6000, opt-in| receiver - receiver -->|internal - scalo Transport Push dfe-loader:6000| loader + subgraph net["Compose network"] + receiver["dfe-receiver"] + loader[["dfe-loader"]] + end + + client -.->|Vector :6000| receiver + receiver -->|Push :6000| loader + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef core fill:#DCEBF7,stroke:#0072B2,stroke-width:2px,color:#0B2A3F + classDef ext fill:#EEF0F2,stroke:#6B7280,stroke-width:1px,stroke-dasharray:4 3,color:#1F2937 + classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 + class client ext + class loader core + class net group ``` +*Figure 4. L4 Detail -- Two listeners on port 6000. The dotted arrow is off until `config.grpc.enabled` is set.* Up: [L2 Services -- DFE Compose stack](#events-arrive-two-ways-and-leave-one). + | Container port | Host port | Owner | Protocol | Audience | Binds | |---|---|---|---|---|---| | 6000 | none, until `DFE_RECEIVER_GRPC_PORT` is set | dfe-receiver | Vector protocol, an ingest source that listens only once `config.grpc.enabled` is set through the engine | external clients | `DFE_INGRESS_BIND_HOST` (`0.0.0.0`) | -| 6000 | none | dfe-loader | `scalo.transport.v1.Transport/Push`, internal transport | receiver, fetcher, the transforms | the compose network | +| 6000 | none | dfe-loader | `scalo.transport.v1.Transport/Push`, internal transport | dfe-receiver, on a `*-grpc` profile | the compose network | Every Push listener in the suite answers on 6000 and dfe-engine compiles each sender's endpoint against that port. The receiver's listener is meant to be reachable from outside the stack. The loader's is not, so it publishes no host port. @@ -210,7 +245,7 @@ from the same stack SSoT, so both run identical digests. Three deliberate asymmetries: - An operator applies a new source. Both tiers run a fetcher and a transform per source. On Kubernetes Argo creates the new Deployment. Here the engine declares the instance and reports `make apply`, which starts it. After deleting a source, run `make apply` with no `SERVICES` to stop its transform, because the delete names no command and the container runs until an apply or a restart removes it. `make ps` warns about any it finds. -- No edge authentication. Envoy fronts both tiers, but its OIDC filters stay unconfigured here, because Docker mode can never assume an issuer exists. The engine API still requires a login per route, which the proxy neither adds nor removes. The opt-in `auth` profile gates the infra UIs alone ([auth-profile.md](auth-profile.md)). +- No edge authentication. Envoy fronts both tiers, but its OIDC filters stay unconfigured here, because Docker mode can never assume an issuer exists. The engine API still needs a login per route, which the proxy neither adds nor removes. The opt-in `auth` profile gates the infra UIs alone ([auth-profile.md](auth-profile.md)). - Self-monitoring collects differently. Container stdout reaches the collector through Docker's `fluentd` log driver rather than a DaemonSet filelog receiver, and of the host's own metrics only its CPUs are collected. See [observability.md](observability.md). ## Related diff --git a/docs/auth-profile.md b/docs/auth-profile.md index 861424d..419873e 100644 --- a/docs/auth-profile.md +++ b/docs/auth-profile.md @@ -10,6 +10,8 @@ Copyright: (c) 2026 HYPERI PTY LIMITED # Gating the infra UIs with OIDC +Status: opt-in and off by default (`DFE_AUTH_ENABLED=false`). `make check-compose` checks its wiring, and no e2e test signs in through it. + `DFE_AUTH_ENABLED=true` arms the `auth` profile, which puts an oauth2-proxy in front of each infra UI. Reaching Kafbat, or dfe-hyperdx on `:8090` and `:8000`, then needs an OIDC sign-in **and** membership of one of `DFE_OIDC_ALLOWED_GROUPS` @@ -42,13 +44,13 @@ proxies as well as the UIs. ## Three limits -It gates page access only. Neither UI learns from the proxy who signed in, because dfe-hyperdx identifies callers by the engine token, and neither gains roles or an audit trail from it. +It gates page access only. Neither UI learns from the proxy who signed in, because dfe-hyperdx identifies callers by the engine token, and neither gets roles or an audit trail from it. An expired session on the API origin fails opaquely. The browser blocks a cross-origin XHR that gets a redirect to the IdP, so the app shows failed requests rather than a login prompt. Reload the app page to sign back in. Nothing else moves behind it. Ingest, ClickHouse, Kafka, the metrics ports and the product UIs are untouched. -Off by default, and it must stay possible to run every other profile without an +It must stay possible to run every other profile without an issuer. Arming it with a setting missing stops `make` and names the key rather than starting a proxy that redirects nowhere. diff --git a/docs/broker.md b/docs/broker.md index a8e96d5..11669e5 100644 --- a/docs/broker.md +++ b/docs/broker.md @@ -26,4 +26,4 @@ The default broker runs `--mode=dev-container --smp=1` with `REDPANDA_MEMORY=1G` For a small-environment deploy, set `REDPANDA_MODE=production` with real `REDPANDA_SMP` and `REDPANDA_MEMORY` values, and raise `DFE_BROKER_MEMORY` **above** `REDPANDA_MEMORY`. Seastar reserves its `--memory` up front, so a container limit that does not exceed it gets the broker OOM-killed rather than backpressured. -Redpanda subtracts the host's `vm.min_free_kbytes` from the container limit before it checks `--memory`, so the usable figure is lower than the limit you set and varies by host. A 92 GiB host reserving 1.84 GiB left the 2560M default 675M against a 1G request. `redpanda/start.sh` reads both at start and passes the smaller of `REDPANDA_MEMORY` and what the limit leaves, with a WARN naming the `DFE_BROKER_MEMORY` that restores the full amount. Under 512M it refuses to start and names the value that would fit. Check a host with `sysctl vm.min_free_kbytes`. +Redpanda subtracts the host's `vm.min_free_kbytes` from the container limit before it checks `--memory`, so the usable figure is lower than the limit you set and varies by host. A 92 GiB host reserving 1.84 GiB left the 2560M default 675M against a 1G request. `redpanda/start.sh` reads both at start and passes the smaller of `REDPANDA_MEMORY` and what the limit leaves, with a WARN naming the `DFE_BROKER_MEMORY` that restores the full amount. Under 512M it will not start and names the value that would fit. Check a host with `sysctl vm.min_free_kbytes`. diff --git a/docs/checks.md b/docs/checks.md new file mode 100644 index 0000000..ca4b2e0 --- /dev/null +++ b/docs/checks.md @@ -0,0 +1,86 @@ + + +# Outage, source and static checks + +The outage tests stop a service under load, the source test drives a new source through the console, and the static checks are what CI runs. The e2e suite they build on is in [testing.md](testing.md). + +## Outage tests stop a service under load + +```bash +make test-resilience +make test-resilience E2E_TESTS="clickhouse-outage loader-outage-grpc" +``` + +Opt-in: `make test-e2e` never runs these, and rejects one listed in `E2E_TESTS`. +An `outage:` test sends steady load and, once its first records land, stops the +named service for `seconds`, starts it again in a `finally`, and keeps sending +for 30 s. It then asserts every other DFE service ran straight through (same +pid, no restart, no die or OOM event), the receiver answered every request (a +refusal such as 503 passes, silence fails), every accepted record landed, and on Kafka +every consumer group is back at zero lag. They stop services, so they never run +in `make post` either. + +An outage's `method` is `stop` by default, a graceful stop that lets the service drain what it holds. `method: kill` sends SIGKILL and a crash gets no drain, so a record the service held in memory survives only if its acknowledgement was held until the next hop had it. The `*-kill-*` tests kill the loader, the receiver and the VRL transform on Kafka, and the loader on direct gRPC. They pass only when every record answered 2xx lands, duplicates counted but allowed. Where the killed service is the receiver itself, a request in flight while it is down may go unanswered but silence outside that window still fails. + +A kill that catches nothing in flight proves nothing about a crash, so a kill test fails on it. For the receiver or a gRPC hop that means no request out at the instant the container exited. For a Kafka consumer it means nothing its group had left uncommitted just before the kill. The kill tests raise the load with `workers` and `interval` (the default is 4 workers each waiting 0.5 s) so that a kill lands with records in flight. + +## The post-deploy source test is what an operator does first + +```bash +DFE_INFRA_DIR=../dfe-infra make test-source +DFE_INFRA_DIR=../dfe-infra make test-source SOURCE_ARGS="--case cloudwatch --aws-service cloudtrail" +DFE_INFRA_DIR=../dfe-infra make test-source SOURCE_ARGS="--case elastic" +DFE_INFRA_DIR=../dfe-infra make test-source SOURCE_ARGS="--case vector" +``` + +Above, sources come through the API. This one creates one in the console, attaches a transform, deploys it, feeds it data and reads the rows back, driving a browser to do it. The runner is dfe-infra's (`scripts/acceptance/source/run.py`), the same one `dfe-ops acceptance --suite source` calls for Kubernetes. + +`scripts/test_source.py` supplies what this repo owns: the published ports, the archiver container, the admin login from `env/engine.env` and `make apply` for each service the engine says must restart, which is what creates the source's own transform container. Run `make test-source DEV=1` against a `make dev` stack so that apply keeps the local images. `SOURCE_ARGS` reaches the runner untouched (`--keep`, `--per-module`, `--headed`). + +Beyond a running stack it needs `DFE_INFRA_DIR` (the runner), `DFE_ENGINE_REPO` and `DFE_TRANSFORM_VRL_REPO` (the corpus wrapper, the bundled pipeline), a dfe-transform-vector checkout for the vector case (`DFE_TRANSFORM_VECTOR_REPO`, else the one beside the engine repo) and an interpreter carrying Playwright (`DFE_ACCEPTANCE_PYTHON` where the system one is externally managed). The cloudwatch case polls a real AWS upstream and writes nothing to it: put the two `AWS_*` credentials in `env/fetcher.env` and `DFE_AWS_REGION` in `.env`. + +Every step is a report row and a screenshot under `--shots-dir` (`.tmp/source`). +A step the console cannot do falls back to the engine API and says so, because +that is a finding about the console rather than about the pipeline. + +`make proof` runs the filebeat and vector cases on every stack version. Each transform checkout is cloned at an explicit ref override when one is set, else at the tag the stack pins, else at its default branch with a WARN, and only the pinned tag proves that pin. + +## Static checks run what CI runs + +`make check` runs every static check CI runs, and CI calls each through the same make target. + +| Target | What it does | +|---|---| +| `make check-compose` | `docker compose config` on the registry, dev, `LIVE_UI=1` and `LOCAL=` paths, against both Kafka backends, plus the CPU floor and dev-overlay coverage assertions | +| `make check-hardfail` | asserts an unpinned checkout fails to resolve rather than pull `latest` | + +| `make check-dockerfile` | hadolint on `docker/dfe-rust-builder.Dockerfile` and `docker/dfe-docker.Dockerfile` | +| `make check-docs` | asserts every relative link across the README and every page under `docs/` resolves | +| `make check-proxy` | `envoy --mode validate` on both proxy configs and their console TLS variants, on the Envoy the stack pins | +| `make check-python` | `ruff check` and `ruff format --check` on `scripts/`, `ops/` and the demo packs | +| `make check-tests` | `pytest scripts/tests` -- unit tests over the helper scripts | +| `make check-locks` | `uv lock --check` on each script that declares packages inline, so a pin moved without `uv lock --script