diff --git a/.dockerignore b/.dockerignore index 733ccb05..d25ccb29 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,54 +1,37 @@ # Project: dfe-docker # File: .dockerignore -# Purpose: Build-context exclusions for any image built with this repo as context +# Purpose: Build-context allowlist for docker/dfe-docker.Dockerfile, the one build with this repo root as its context # Language: dockerignore # # License: BUSL-1.1 # Copyright: (c) 2026 HYPERI PTY LIMITED # -# Nothing in this repo currently builds with the repo root as its context: -# scripts/build_dev_images.py stages each component's source into a temp dir with -# its own STAGE_EXCLUDES, and the runtime image comes from the component repo's -# own Dockerfile. So today this file is a guard, not a fix. +# The dfe-docker image copies its whole context into the folder `init` sets up, +# so this names what that folder holds and nothing more. An allowlist, because +# a file nobody listed then stays out: .env, env/, certs/ and access-summary.md +# hold secrets and can never reach an image layer. # -# It earns its keep the moment someone adds a root-context build -- which is -# exactly when nobody remembers to write one, and secrets (.env, env/) get -# shipped into an image layer. That is the failure this prevents. +# scripts/build_dev_images.py stages each component's source into its own temp +# context, so this file does not reach those builds. -# Secrets and local config -- the reason this file exists at all. -.env -.env.* -!.env.example -env/ -certs/ - -# Version control and CI metadata -.git/ -.github/ -.gitignore -.gitleaks.toml +* -# Agent and editor artefacts -.claude/ -.hyperi-ai/ -.mcp.json -.vscode/ -.idea/ +!.env.example +!Makefile +!VERSION +!LICENSE +!compose/ +!config/ +!deployment.example.yaml +!docker-compose.yml +!docker/dfe-docker.requirements.txt +!env.example/ +!redpanda/ +!resources/ +!scripts/ +!service_profiles.yaml -# Local build and runtime state -.profile.mk -.tmp/ -tmp/ -**/target/ +# Inside the kept directories: tests and bytecode never run in a deployment +scripts/tests/ **/__pycache__/ **/*.pyc - -# Docs and packaging that never belong in a runtime image -docs/ -*.md -LICENSE -robots.txt - -# Compose definitions themselves -- an image built from this context does not -# need the stack that runs it. -docker-compose*.yml diff --git a/.env.example b/.env.example index 6689135c..b1202193 100644 --- a/.env.example +++ b/.env.example @@ -6,641 +6,251 @@ ## License: BUSL-1.1 ## Copyright: (c) 2026 HYPERI PTY LIMITED -## ============================================================================ +## Write values unquoted: `make` includes this file + +## ============================================================= ## Image Registry -## ============================================================================ -## OCI registry hosting the published dfe-* images. Default is GHCR. -## For cross-arch builds, set DOCKER_DEFAULT_PLATFORM. -# +## ============================================================= +# DFE_STACK_VERIFY=true +# DFE_STACK_VERSION= # DOCKER_DEFAULT_PLATFORM= # IMAGE_REGISTRY=ghcr.io/hyperi-io -# -## Registry pull credentials. `make login` (a prerequisite of `make ci` and -## `make stack`) reads these and logs docker + oras in, so an unattended pull of -## the private dfe-* images and the signed stack-manifest needs nothing typed. -## DFE_GHCR_TOKEN is a PAT (or GITHUB_TOKEN) with read:packages; on the devex -## fleet hyperi-infra injects both from OpenBao (secret/dfe) at deploy time. -## Leave BOTH blank if the daemon is already authenticated out of band (an -## existing docker login / ~/.docker/config.json) - `make login` then no-ops. -## Never commit real values here. -# -# DFE_GHCR_USERNAME= -# DFE_GHCR_TOKEN= - -## ============================================================================ -## Host port binding -## ============================================================================ -## Compose is a supported production deploy target for small environments, so -## host exposure is split by AUDIENCE rather than left open: -## -## DFE_INGRESS_BIND_HOST - what USERS reach. Receiver ingest (6000/8080) and -## fetcher ingest (8082). -## Defaults to 0.0.0.0 because an ingest port nobody can reach is useless. -## DFE_BIND_HOST - what OPERATORS reach. ClickHouse (8123/9000), Kafka -## (9092/19092), every /metrics port (9090-9096), the loader's internal -## gRPC (50051), the Vector API (8686) and the collector's health port -## (13133). -## Defaults to 127.0.0.1: on a laptop that costs nothing, and on a partner -## box it is the difference between an internal port and an open database. -## The WEB UIS are the third audience and have their own section below. -## -## Remote dev hosts (a shared box you reach over the network) set -## DFE_BIND_HOST=0.0.0.0. Do that knowingly - ClickHouse ships no password by -## default, so an open 8123 is an open database. -# -# DFE_BIND_HOST=127.0.0.1 -# DFE_INGRESS_BIND_HOST=0.0.0.0 +## ============================================================= +## Container Names +## ============================================================= +# DFE_CONTAINER_PREFIX= +## ============================================================= +## Profile +## ============================================================= +# DFE_CLICKHOUSE_ENABLED=true +# DFE_CORE_ENABLED=true +# DFE_HYPERDX_ENABLED=true +# DFE_OTEL_ENABLED=true +# DFE_PROFILE= +# KAFBAT_ENABLED=true -## ============================================================================ -## Web UI exposure -## ============================================================================ -## Every web UI publishes to the host by DEFAULT. Two things control that: WHERE -## it publishes (the bind scope) and WHETHER it publishes at all (the flags). -## -## DFE_BIND_SCOPE is one dial with two modes, because "external" means something -## different on the two boxes compose deploys to: -## -## localhost (the DEFAULT) - a developer workstation. Every UI publishes on -## 127.0.0.1, so the host sees them and the LAN does not. -## all - a small or VM deploy. Every UI publishes on 0.0.0.0. -## -## It governs the UI ports ONLY: the UI proxy (3000), the engine API (8003), -## Kafbat (8081) and the HyperDX app plus its API (8090/8000). Ingest and the -## backing services keep their own audience above and never move with it. -## -## The Makefile resolves the scope into DFE_UI_BIND_HOST, the address compose -## interpolates. Do not set that by hand -- like DFE_DATA_ROOT, the dial takes -## effect through `make`, and a raw `docker compose` gets the loopback default. -## -## UIs carry a CLASS. PRODUCT is the DFE UI and the engine API the UI consumes; -## INFRA is the ops consoles. DFE_INFRA_UIS_EXTERNAL=false is the kill switch: it -## unpublishes every infra UI at once and beats their individual flags. The -## product UIs are exempt and answer only to their own flag, so the DFE UI never -## goes dark as a side effect of locking the ops surfaces down. -## -## Unpublishing leaves the container running and reachable on the compose -## network. It is not access control - the stack still authenticates nobody. And -## nothing here closes the engine API: dfe-proxy reverse-proxies /api/v1/* on -## DFE_UI_PORT whatever DFE_ENGINE_API_EXTERNAL says. -## -## HyperDX at DFE_BIND_SCOPE=all is the sharp edge. Next.js inlines -## CLICKHOUSE_PASSWORD into its client bundle and local mode has no login, so -## every browser on the LAN that loads that page reads the credential. Set -## DFE_HYPERDX_UI_EXTERNAL=false, or the kill switch, unless you mean it. -# +## ============================================================= +## Host Exposure +## ============================================================= +# DFE_BIND_HOST=127.0.0.1 # DFE_BIND_SCOPE=localhost -# DFE_INFRA_UIS_EXTERNAL=true -# DFE_UI_EXTERNAL=true # DFE_ENGINE_API_EXTERNAL=true -# DFE_KAFBAT_UI_EXTERNAL=true # DFE_HYPERDX_UI_EXTERNAL=true +# DFE_INFRA_UIS_EXTERNAL=true +# DFE_INGRESS_BIND_HOST=0.0.0.0 +# DFE_KAFBAT_UI_EXTERNAL=true +# DFE_UI_EXTERNAL=true - -## ============================================================================ -## External origin -## ============================================================================ -## The scheme and host a BROWSER reaches this box on, with no port and no -## trailing slash -- every service appends its own port. Set it to a hostname on -## anything but a workstation. -## -## Three surfaces build absolute URLs from it and each gets the redirect wrong -## when it says localhost and the browser did not: the DFE UI's post-logout -## redirect, the oauth2-proxy callbacks the IdP must have registered, and -## HyperDX's link-backs into the DFE UI. -## -## DFE_OAUTH2_PROXY_EXTERNAL_ORIGIN and DFE_HYPERDX_APP_URL still override it -## per-surface, for a deployment that genuinely serves them on different names. -## Unset, they all follow this one. -## -## It defaults to localhost, and localhost is WRONG the moment a browser reaches -## this box by any other name -- the DFE UI has no fallback that infers the real -## one. Reaching the stack on a hostname and not setting this is what sends -## every logout to http://localhost:3000. -# +## ============================================================= +## Console Origin and TLS +## ============================================================= # DFE_EXTERNAL_ORIGIN=http://localhost - - -## ============================================================================ -## Infra UI authentication (opt-in `auth` profile) -## ============================================================================ -## DFE_AUTH_ENABLED=true puts an oauth2-proxy in front of each infra UI, so -## reaching Kafbat or HyperDX needs an OIDC sign-in AND membership of one of -## DFE_OIDC_ALLOWED_GROUPS. That group check is the point - authentication alone -## is not the gate. It is the docker half of the same seam k8s serves at the -## Envoy edge. -## -## OFF by default, and it must stay possible to run every other profile without -## an issuer: no dfe-docker profile may require an IdP to exist. Arming it with a -## setting missing stops `make` and names the key, rather than starting a proxy -## that redirects nowhere. -## -## Three proxies, one per ORIGIN: Kafbat (8081), the HyperDX app (8090) and the -## HyperDX API (8000), which the browser calls directly. They take over those -## host ports and the UIs stop publishing their own. One cookie secret across all -## three, and cookies ignore port, so one sign-in covers the lot. -## -## The auth profile gates PAGE ACCESS ONLY - neither UI learns who signed in, and -## neither gains a session, roles or an audit trail of its own. -## -## It does NOT contain the HyperDX credential leak: Next.js inlines -## CLICKHOUSE_PASSWORD into the client bundle, so anyone who passes the group -## check still reads it out of the JavaScript. -## -## Register all three redirect URIs with the IdP: -## :{8081,8090,8000}/oauth2/callback -## and make sure it emits a groups claim, or every sign-in is refused. -## -## DFE_OAUTH2_PROXY_COOKIE_SECRET is generated by `make init` (32 bytes - -## oauth2-proxy accepts exactly 16, 24 or 32). DFE_OAUTH2_PROXY_VERSION is NOT -## SSoT-derived: oauth2-proxy is outside the DFE stack manifest, so `make stack` -## cannot pin it and docker-compose.yml carries the digest as its default. Set -## this only to override, and keep the tag@sha256 form. -## -## Set DFE_OAUTH2_PROXY_COOKIE_SECURE=true behind real TLS. On plain HTTP a -## Secure cookie is never sent back and every sign-in loops. -# -# DFE_AUTH_ENABLED=false -# DFE_OIDC_ISSUER_URL= -# DFE_OIDC_CLIENT_ID= -# DFE_OIDC_CLIENT_SECRET= -# DFE_OIDC_ALLOWED_GROUPS=dfe-infra,dfe-admin -# DFE_OIDC_EMAIL_DOMAINS=* -# DFE_OAUTH2_PROXY_COOKIE_SECRET= -# DFE_OAUTH2_PROXY_COOKIE_DOMAIN= -# DFE_OAUTH2_PROXY_COOKIE_SECURE=false -## Per-surface override of DFE_EXTERNAL_ORIGIN, which it follows when unset. -# DFE_OAUTH2_PROXY_EXTERNAL_ORIGIN= -# DFE_OAUTH2_PROXY_VERSION=v7.15.4@sha256:b1b2021fe8f4004573e8d690dec6c7bb29cc44364572cf8510a05bf3a0ae2ded - - -## ============================================================================ -## OIDC test-fixture login -## ============================================================================ -## The shared throwaway login every fixture account carries, so a tester who -## cannot reach the secret store can still sign in to the console through an -## external provider. Nothing in the stack reads these: they name a HUMAN test -## identity, not a service credential, and they are neither the oauth2-proxy -## client above nor the engine's own client in env/engine.env. -## -## `make creds` prints the user and names the key the password lives in. It never -## prints the password. The console acceptance specs resolve the same names. -## -## A per-provider pair wins over the generic one, so a provider needing a -## different login does not split the rest. is the provider name -## uppercased as /api/v1/auth/setup-status reports it -- DEX, OKTA, ENTRA and -## GOOGLE today. Each name resolves per-provider, then generic, then unset, and a -## provider with no password is SKIPPED by the specs rather than failed. -## -## Throwaway identities on accounts that exist for testing. Never put a login -## that works anywhere real here, and never commit a value. -# -# DFE_OIDC_FIXTURE_USER=dfe-test@dfe-oidc.test -# DFE_OIDC_FIXTURE_PASSWORD= -# -## Per-provider overrides, one pair per provider that needs its own login -# DFE_OIDC_OKTA_FIXTURE_USER= -# DFE_OIDC_OKTA_FIXTURE_PASSWORD= - - -## ============================================================================ -## Resource limits -## ============================================================================ -## Every service declares deploy.resources.limits; `docker compose up` honours -## cpus and memory outside Swarm. An unlimited service on a single box is how -## one runaway container takes the host down - ClickHouse especially. -## -## Defaults are sized so the DEFAULT profile fits a 16 GB box that is ALSO running -## an operating system and whatever else lives there - 16 GB of RAM is not 16 GB of -## headroom. Enabling every profile at once wants retuning, not these defaults. -## Run `make limits` for the actual per-service values and totals -- computed from -## the resolved config, so it cannot go stale the way a written-down total does. -## -## Limits are ceilings, not reservations - nothing is pre-allocated, so a service -## sitting idle costs nothing. -## -## Four tiers: clickhouse; broker (kafka); service (the DFE components, engine, UI, -## kafka-ui, HyperDX, its Postgres and FerretDB); sidecar (kafka-init, dlq-init, -## dfe-proxy). kafka-ui is on the SERVICE tier, not sidecar - kafbat is a Spring -## Boot JVM that sizes its heap off the container limit, so 512M OOM-kills it into -## a restart loop. -## -## If you raise REDPANDA_MEMORY, raise DFE_BROKER_MEMORY above it. Redpanda -## subtracts the HOST's vm.min_free_kbytes from the container limit before it -## checks --memory, so redpanda/start.sh passes the smaller of REDPANDA_MEMORY and -## what that leaves, logs a WARN naming the DFE_BROKER_MEMORY that restores the -## full amount, and refuses to start under 512M. -## -## Docker allows an equal amount of SWAP alongside a memory limit, so a 3G limit -## can become 3G RAM + 3G swap on a swap-enabled host. Set memswap_limit equal to -## the memory limit per service if you need a genuinely hard cap. -## -## DFE_SERVICE_CPUS below 2.0 is not "a bit slower" - it is broken. Tokio sizes -## its worker pool from the cgroup quota and floors it, so a ceiling under 2.0 -## gives a scalo service ONE worker thread, which its Kafka poll loop then owns. -## dfe-archiver at 1.5 accepted connections on :9090 and answered none of them, -## so its healthcheck never passed while the data path kept working. See the -## x-limits-service comment in docker-compose.yml for the proof. -# -# DFE_CLICKHOUSE_CPUS=2.0 -# DFE_CLICKHOUSE_MEMORY=3G +# DFE_NETWORK_SUBNET=10.207.0.0/24 +# DFE_PROXY_CA_BUNDLE= +# DFE_PROXY_CERT_DIR=./certs +# DFE_PROXY_TLS=false + +## ============================================================= +## Resource Limits +## ============================================================= # DFE_BROKER_CPUS=1.5 # DFE_BROKER_MEMORY=2560M +# DFE_CLICKHOUSE_CPUS=2.0 +# DFE_CLICKHOUSE_MEMORY=3G +# DFE_HYPERDX_MEMORY=2G # DFE_SERVICE_CPUS=2.0 # DFE_SERVICE_MEMORY=768M # DFE_SIDECAR_CPUS=0.5 # DFE_SIDECAR_MEMORY=512M +## ============================================================= +## All DFE Apps +## ============================================================= +# DFE_OTEL_EXPORTER_ENDPOINT= +# DFE_VERSION_CHECK_ENABLED=true +# DFE_VERSION_CHECK_SEND_INSTANCE_ID=true +# LOG_FORMAT=text +# LOG_LEVEL=info -## ============================================================================ -## Generated secrets -## ============================================================================ -## `make init` mints a random value for each into .env - including topping up an -## existing .env that predates the key - so a fresh checkout still just works. -## Set them by hand only if you need a specific value. -## -## These do NOT hard-fail, unlike the image pins. Compose interpolates every -## service before profiles filter anything, so a `${VAR:?}` here would abort -## `make down` and `make ps` for someone who never enabled the service the secret -## belongs to. They carry a loud sentinel default instead, and `make post` fails -## on it. So `make init` is NOT optional: copy this file by hand and you get a -## stack running a publicly committed signing key with nothing stopping you. -## -## DFE_UI_NEXTAUTH_SECRET - dfe-ui session signing key. Previously defaulted -## to the literal string "secret", and because -## .env.example shipped it commented out, that -## default is what actually ran. -## DFE_AUTH_LOCAL_ADMIN_PASSWORD - the `admin` login for dfe-ui and the engine -## API, reasserted from here on every engine boot. -## The engine REFUSES TO START when it is empty or -## "changeme" unless DFE_ENV is a dev posture. -## DFE_AUTH_BREAKGLASS_PASSWORD - the `breakglass` recovery admin. The engine -## hashes it into the deploy repo on its first boot -## and ignores it after that, so it survives losing -## the engine, the UI and this file. `make creds` -## points at it rather than printing it. -## DFE_API_JWT_SECRET - keys the engine's OIDC session cookie. The SECOND -## thing the engine REFUSES TO START on outside a dev -## posture: it ships a known placeholder, and running -## on that lets anyone forge a session. -## HYPERDX_POSTGRES_PASSWORD - HyperDX metadata store. Not published to the -## host, so smaller blast radius, but a password -## whose value is also its name is not one. -## CLICKHOUSE_PASSWORD - ClickHouse default user, which has full admin -## (DEFAULT_ACCESS_MANAGEMENT=1). Was blank. BREAKING -## on upgrade - a data volume created with the blank -## password does not re-auth against a generated one; -## see docs/operating.md for the migration. Skipped -## when CLICKHOUSE_HOST points at an external instance. -## -## Most are covered again in their own sections below. - +## ============================================================= +## Dev Source +## ============================================================= +# DFE_SRC_CACHE=~/.cache/dfe-docker/src +# DFE_SRC_REF=main +# DFE_SRC_REMOTE=https://github.com/hyperi-io +# DFE_SRC_ROOT= -## ============================================================================ -## Power-on self test (POST) -## ============================================================================ -## After `make dev` / `make ci` bring the stack up, a small number of uniquely -## marked events are injected at the ingest edge and traced through to -## ClickHouse. It proves the stack MOVES DATA, not merely that containers -## started and ports answer. -## -## It is opt-OUT. A self test you have to remember to run is not a self test. -## -## Transport-agnostic: it posts to the ingest edge and reads ClickHouse, so it -## behaves the same on the kafka and the kafka-less (grpc) profiles. On a profile -## with no ingest component at all (loader-only) it skips with a stated reason -## rather than failing - there is nothing to prove end to end. -## -## Run it by hand any time against a running stack: `make post` -# +## ============================================================= +## Power-On Self Test +## ============================================================= +# DFE_POST_DATABASE=dfe # DFE_POST_ENABLED=true # DFE_POST_HOST=localhost -# DFE_POST_DATABASE=dfe -# DFE_POST_TABLE=default - +# DFE_POST_TABLE=main -## ============================================================================ -## Dev source (make dev / make dev-build) -## ============================================================================ -## Component source comes via git: each repo is cloned into a managed cache and -## built at DFE_SRC_REF. Set DFE_SRC_ROOT to build your own local checkouts -## instead (the directory holding dfe-receiver, dfe-loader, ...). DFE_SRC_ROOT -## is also what `make dev LIVE=1` bind-mounts dfe-engine config/schemas from. -# -# DFE_SRC_ROOT= -# DFE_SRC_REMOTE=https://github.com/hyperi-io -# DFE_SRC_REF=main -# DFE_SRC_CACHE=~/.cache/dfe-docker/src +## ============================================================= +## OIDC Test-Fixture Login +## ============================================================= +# DFE_OIDC_FIXTURE_PASSWORD= +# DFE_OIDC_FIXTURE_USER=dfe-test@dfe-oidc.test +# DFE_OIDC_OKTA_FIXTURE_PASSWORD= +# DFE_OIDC_OKTA_FIXTURE_USER= +## ============================================================= +## ClickHouse +## ============================================================= +# CLICKHOUSE_DB=default +# CLICKHOUSE_EXTERNAL_HTTP_PORT=8123 +# CLICKHOUSE_EXTERNAL_NATIVE_PORT=9000 +# CLICKHOUSE_HOST= +# CLICKHOUSE_HTTP_PORT=8123 +# CLICKHOUSE_KEEPER_PASSWORD= +# CLICKHOUSE_NATIVE_PORT=9000 +# CLICKHOUSE_PASSWORD= +# CLICKHOUSE_SECURE=false +# CLICKHOUSE_USERNAME=default +# CLICKHOUSE_VERSION= -## ============================================================================ -## DFE Profile (overrides active_profile in service_profiles.yaml) -## ============================================================================ -## Available profiles are defined in service_profiles.yaml at repo root. -# -# DFE_PROFILE= +## ============================================================= +## Kafka +## ============================================================= +# APACHE_KAFKA_VERSION= +# DFE_KAFKA_BOOTSTRAP_SERVERS=kafka:9092 +# KAFKA_ADVERTISED_HOST=localhost +# KAFKA_BACKEND=redpanda +# KAFKA_PLAINTEXT_HOST_PORT=19092 +# REDPANDA_MEMORY=1G +# REDPANDA_MODE=dev-container +# REDPANDA_SMP=1 +# REDPANDA_VERSION= +## ============================================================= +## Kafka UI +## ============================================================= +# KAFBAT_PORT=8081 +# KAFBAT_VERSION= -## ============================================================================ -## DFE Components -## ============================================================================ -## Image version pins (every *_VERSION below) are SSoT-derived PLACEHOLDERS, -## set by `make stack VERSION=X.Y.Z[-rc.N]` from the DFE stack manifest -## (dfe-infra/versions.yaml). docker-compose.yml hard-fails on an unset pin -- -## there is no silent `latest` fallback -- so run `make stack` before any -## compose command. Renovate no longer manages these; the stack SSoT does. -## -## ---------------------------------------------------------------------------- -## General -## ---------------------------------------------------------------------------- -# LOG_LEVEL=info -# LOG_FORMAT=text -# -## ---------------------------------------------------------------------------- +## ============================================================= ## DFE Archiver -## ---------------------------------------------------------------------------- -# DFE_ARCHIVER_VERSION= +## ============================================================= # DFE_ARCHIVER_PROMETHEUS_PORT=9093 -# -## ---------------------------------------------------------------------------- +# DFE_ARCHIVER_VERSION= + +## ============================================================= ## DFE Fetcher -## ---------------------------------------------------------------------------- +## ============================================================= +# DFE_FETCHER_INPUT_DIR= # DFE_FETCHER_VERSION= -# DFE_FETCHER_INGEST_PORT=8082 -# DFE_FETCHER_PROMETHEUS_PORT=9094 -# -## ---------------------------------------------------------------------------- + +## ============================================================= ## DFE Loader -## ---------------------------------------------------------------------------- -# DFE_LOADER_VERSION= +## ============================================================= +# DFE_LOADER_CLICKHOUSE_PASSWORD= # DFE_LOADER_PROMETHEUS_PORT=9091 -# DFE_LOADER_GRPC_PORT=50051 -# -## ---------------------------------------------------------------------------- +# DFE_LOADER_VERSION= + +## ============================================================= ## DFE Receiver -## ---------------------------------------------------------------------------- -# DFE_RECEIVER_VERSION= -# DFE_RECEIVER_BEATS_PORT=5044 -# DFE_RECEIVER_GRPC_PORT=6000 -# DFE_RECEIVER_HEC_PORT=8088 +## ============================================================= +# DFE_RECEIVER_BEATS_PORT= +# DFE_RECEIVER_GRPC_PORT= +# DFE_RECEIVER_HEC_PORT= # DFE_RECEIVER_HTTP_PORT=8080 -# DFE_RECEIVER_OTLP_GRPC_PORT=4317 -# DFE_RECEIVER_OTLP_HTTP_PORT=4318 +# DFE_RECEIVER_HTTPS_PORT= +# DFE_RECEIVER_IPFIX_PORT= +# DFE_RECEIVER_MAX_BODY_SIZE=8000000 +# DFE_RECEIVER_NETFLOW_PORT= +# DFE_RECEIVER_OTLP_GRPC_PORT= +# DFE_RECEIVER_OTLP_HTTP_PORT= # DFE_RECEIVER_PROMETHEUS_PORT=9090 -# -## ---------------------------------------------------------------------------- +# DFE_RECEIVER_SFLOW_PORT= +# DFE_RECEIVER_SYSLOG_TCP_PORT= +# DFE_RECEIVER_SYSLOG_TLS_PORT= +# DFE_RECEIVER_SYSLOG_UDP_PORT= +# DFE_RECEIVER_VERSION= + +## ============================================================= ## DFE Transform Elastic -## ---------------------------------------------------------------------------- -## Selects one compiled-in transform per instance (`source.name` in -## config/transform-elastic/*.yaml), so a second source is a second instance. -## The cisco-ios instance runs the same image and needs its own metrics port. +## ============================================================= # DFE_TRANSFORM_ELASTIC_VERSION= -# DFE_TRANSFORM_ELASTIC_PROMETHEUS_PORT=9099 -# DFE_TRANSFORM_ELASTIC_CISCO_IOS_PROMETHEUS_PORT=9089 -# -## ---------------------------------------------------------------------------- + +## ============================================================= ## DFE Transform Vector -## ---------------------------------------------------------------------------- -## 9095 is the whole metrics surface: the wrapper merges Vector's own -## internal_metrics into the scalo registry (dfe-transform-vector#70), so the -## `vector_*` series arrive on it and the exporter's 9598 stays in the container. +## ============================================================= # DFE_TRANSFORM_VECTOR_VERSION= -# DFE_TRANSFORM_VECTOR_PROMETHEUS_PORT=9095 -# DFE_TRANSFORM_VECTOR_API_PORT=8686 -# -## ---------------------------------------------------------------------------- -## DFE Transform Vector - the filebeat source's own instance (kafka-filebeat-vector) -## ---------------------------------------------------------------------------- -# DFE_TRANSFORM_VECTOR_FILEBEAT_PROMETHEUS_PORT=9098 -# DFE_TRANSFORM_VECTOR_FILEBEAT_API_PORT=8687 -# -## ---------------------------------------------------------------------------- + +## ============================================================= ## DFE Transform VRL -## ---------------------------------------------------------------------------- +## ============================================================= # DFE_TRANSFORM_VRL_VERSION= +## ============================================================= +## DFE Engine +## ============================================================= +# DFE_ENGINE_API_JWT_SECRET= +# DFE_ENGINE_ENV= +# DFE_ENGINE_GITOPS_ENABLED=true +# DFE_ENGINE_GITOPS_MODE=solo +# DFE_ENGINE_PORT=8003 +# DFE_ENGINE_VERSION= -## ============================================================================ -## ClickHouse -## ============================================================================ -## CLICKHOUSE_VERSION: SSoT-derived placeholder, set by `make stack VERSION=...` -## as tag@digest. REQUIRED by docker-compose.yml (hard-fail, no silent `latest`). -## Leave CLICKHOUSE_HOST blank to use the Docker ClickHouse container. -## Set to an external host to skip the Docker container. -## -## CLICKHOUSE_PASSWORD is generated by `make init` (see "Generated secrets"), NOT -## left blank - a blank password on DEFAULT_ACCESS_MANAGEMENT=1 is a full-admin -## database anyone on the box can reach. Deliberately not listed as a placeholder -## here, for the same reason as the other generated secrets. With CLICKHOUSE_HOST -## set (external instance) the credential is yours, not ours - set it by hand. -## -## DFE_CLICKHOUSE_ENABLED=false drops the `clickhouse` compose profile so no -## warehouse container starts - the external-instance posture. Setting -## CLICKHOUSE_HOST alone does NOT do this, and does not move the loader either: -## its ClickHouse host is a literal in config/loader/*.yaml. -# -# DFE_CLICKHOUSE_ENABLED=true -# CLICKHOUSE_VERSION= -# CLICKHOUSE_HOST= -# CLICKHOUSE_HTTP_PORT=8123 -# CLICKHOUSE_NATIVE_PORT=9000 -# CLICKHOUSE_DB=default -# CLICKHOUSE_USERNAME=default -# CLICKHOUSE_SECURE=false -# -## Days every time-series table keeps rows, the OTel tables included; 0 disables -## the default TTL, and a source or a dfe-schemas definition with its own TTL -## keeps it. `make init` asks once on a TTY and writes the answer here live. -# -# DFE_CLICKHOUSE_DEFAULT_TTL_DAYS=90 - - -## ============================================================================ -## Kafka -## ============================================================================ -## Selects which Kafka-API broker the `make` commands run. -## Options: -## - redpanda (default): Lightweight, fits 4 GB CI runners. Source-available -## under the BSL, NOT OSS - fine for local dev and CI under its Additional -## Use Grant, a per-deployment licence check for a partner or edge box. -## - apache (opt-in): Real-Kafka testing, fallback, and the Apache-2.0 answer -## when that licence check does not come back clean. Each backend now brings -## its own topic-init service, so this path pulls no BSL artefact at all. -## Image pins (APACHE_KAFKA_VERSION / REDPANDA_VERSION) are set by -## `make stack VERSION=...` -- see the DFE Components note above. -## -## Shared dev hosts: the broker publishes to host ports 9092 and 19092. If the -## host already runs a Kafka/Redpanda on 9092 (e.g. an always-on dev daemon), -## set KAFKA_PLAINTEXT_PORT (and KAFKA_PLAINTEXT_HOST_PORT) to free ports. The -## stack and the e2e harness reach the broker over the docker network alias -## kafka:9092, so remapping the host publish does not affect them. -# -## KAFKA_ADVERTISED_HOST is what EXTERNAL clients are told to connect to on the -## host-published listener. Leave it as localhost unless the broker is reached -## from another machine, in which case set it to this host's reachable name/IP -## (and set DFE_BIND_HOST=0.0.0.0, or nothing can reach it). -# -# KAFKA_ADVERTISED_HOST=localhost -# KAFKA_BACKEND=redpanda -# KAFKA_PLAINTEXT_HOST_PORT=19092 -# KAFKA_PLAINTEXT_PORT=9092 -# -## ---------------------------------------------------------------------------- -## Apache Kafka (opt-in) -## ---------------------------------------------------------------------------- -## APACHE_KAFKA_VERSION: SSoT-derived placeholder, set by `make stack VERSION=...`. -# -# APACHE_KAFKA_VERSION= -# KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:19092 -# KAFKA_AUTO_CREATE_TOPICS_ENABLE=true -# KAFKA_CLUSTER_ID=dfe-docker-dev-cluster-01 -# KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER -# KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafka:29092 -# KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 -# KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT -# KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT -# KAFKA_LISTENERS=PLAINTEXT://:9092,PLAINTEXT_HOST://:19092,CONTROLLER://:29092 -# KAFKA_NODE_ID=1 -# KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 -# KAFKA_PROCESS_ROLES=broker,controller -# KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1 -# KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1 -# -## ---------------------------------------------------------------------------- -## Redpanda (default) -## ---------------------------------------------------------------------------- -## REDPANDA_VERSION: SSoT-derived placeholder, set by `make stack VERSION=...`. -## -## REDPANDA_MODE defaults to dev-container: Redpanda's own DEVELOPER mode -## (relaxed fsync, single core, 1 GB) chosen so the stack fits a 4 GB CI runner. -## That is not a production configuration. A small-environment deploy sets -## REDPANDA_MODE=production with real REDPANDA_SMP / REDPANDA_MEMORY, and must -## raise DFE_BROKER_MEMORY above REDPANDA_MEMORY or the broker is OOM-killed. -# -# REDPANDA_VERSION= -# REDPANDA_MODE=dev-container -# REDPANDA_SMP=1 -# REDPANDA_MEMORY=1G - - -## ============================================================================ -## Kafka UI (Kafbat) -## ============================================================================ -## KAFBAT_VERSION: SSoT-derived placeholder, set by `make stack VERSION=...`. -## Enabling this (on by default) will start Kafka UI whenever Kafka is created. -# -# KAFBAT_ENABLED=true -# KAFBAT_VERSION= -# KAFBAT_DYNAMIC_CONFIG_ENABLED=true -# KAFBAT_KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka:9092 -# KAFBAT_KAFKA_CLUSTERS_0_NAME=dfe-local -# KAFBAT_PORT=8081 - +## ============================================================= +## DFE Hunt Runner +## ============================================================= +# DFE_HUNT_RUNNER_CLICKHOUSE_PASSWORD= -## ============================================================================ -## Core DFE Components -## ============================================================================ -## Image pins set by `make stack VERSION=...` -- see the DFE Components note above. -## -## ---------------------------------------------------------------------------- -## General -## ---------------------------------------------------------------------------- -## DFE_CORE_ENABLED starts dfe-engine, dfe-ui and dfe-proxy. It is one of five -## footprint flags -- with DFE_CLICKHOUSE_ENABLED, KAFBAT_ENABLED, -## DFE_HYPERDX_ENABLED and DFE_OTEL_ENABLED -- that OVERRIDE whatever the active -## profile in service_profiles.yaml declares. Leave them commented and the -## profile decides. -# DFE_CORE_ENABLED=true -# -## ---------------------------------------------------------------------------- -## dfe-engine -## ---------------------------------------------------------------------------- -## DFE_GITOPS_ENABLED gives the engine a deploy repo, which is where the -## break-glass hash is committed -- set it false and no recovery admin survives a -## rebuild. Local-only here: no repo_url, so nothing is pushed anywhere. -## DFE_GITOPS_LOCAL_PATH is where that repo lives, derived in docker-compose.yml -## as /deploy-repo so it sits inside the config volume and -## is owned by the engine's own UID rather than root. Not settable from here. -# DFE_GITOPS_ENABLED=true -# DFE_ENGINE_VERSION= -# DFE_ENGINE_PORT=8003 -# DFE_ENGINE_CONFIG_DIR=/app/config -# DFE_ENGINE_SCHEMAS_DIR=/app/schemas -# -## ---------------------------------------------------------------------------- -## dfe-ui -## ---------------------------------------------------------------------------- -## DFE_UI_NEXTAUTH_SECRET is generated by `make init` - see "Generated secrets" -## above. Deliberately absent here: a commented placeholder in this file is how -## the old literal "secret" default came to be what actually ran. -# -# DFE_UI_VERSION= +## ============================================================= +## DFE UI +## ============================================================= # DFE_UI_PORT=3000 -# DFE_UI_NODE_ENV=production -# -## ---------------------------------------------------------------------------- -## dfe-proxy (envoy) -## ---------------------------------------------------------------------------- -## DFE_PROXY_VERSION: SSoT-derived placeholder, set by `make stack VERSION=...`. -## It is the tag for envoyproxy/envoy -- an nginx-shaped value pins a manifest -## that does not exist, so re-run `make stack` rather than hand-editing it. -# DFE_PROXY_VERSION= +# DFE_UI_VERSION= +## ============================================================= +## DFE Proxy +## ============================================================= +# DFE_PROXY_VERSION= -## ============================================================================ -## Self-monitoring (OpenTelemetry) -## ============================================================================ -## The stack's own telemetry, kept off the ingest path. DFE_OTEL_ENABLED starts -## the bundled collector; DFE_OTEL_EXPORTER_ENDPOINT says where services push, and -## empty exports NOTHING. Enabling the profile points them at the bundled -## collector unless you name an endpoint yourself, which is how you push to an -## external OTLP backend instead. -## -## Every DFE service pushes on the endpoint alone, except dfe-engine, which also -## needs DFE_ENGINE_METRICS_BACKEND at `opentelemetry` -- the profile sets it. -## They all still serve /metrics for anything that wants to scrape them. -## See docs/operating.md#self-monitoring. -## -## DFE_OTEL_COLLECTOR_VERSION: SSoT-derived placeholder, set by -## `make stack VERSION=...`. -# -# DFE_OTEL_ENABLED=false +## ============================================================= +## OTel Collector +## ============================================================= +# DFE_CONTAINER_LOG_ADDRESS= +# DFE_CONTAINER_LOGS_ENABLED=true +# DFE_OTEL_COLLECTOR_LOG_LEVEL=warn # DFE_OTEL_COLLECTOR_VERSION= -# DFE_OTEL_EXPORTER_ENDPOINT= # DFE_OTEL_DATABASE=dfe +# DFE_OTEL_FLUENT_PORT=24224 # DFE_OTEL_HEALTH_PORT=13133 -# DFE_OTEL_COLLECTOR_LOG_LEVEL=warn -# DFE_ENGINE_METRICS_BACKEND=prometheus - +# DFE_OTEL_WRITER_CLICKHOUSE_PASSWORD= -## ============================================================================ +## ============================================================= ## HyperDX -## ============================================================================ -## DFE_HYPERDX_VERSION / HYPERDX_FERRETDB_VERSION / HYPERDX_POSTGRES_VERSION: -## SSoT-derived placeholders, set by `make stack VERSION=...` -- the fork image -## is pinned from versions.yaml (content.dfe-hyperdx) like every other image. -# -# DFE_HYPERDX_ENABLED=false -# DFE_HYPERDX_VERSION= -# HYPERDX_FERRETDB_VERSION= -# HYPERDX_POSTGRES_VERSION= +## ============================================================= # DFE_HYPERDX_API_PORT=8000 # DFE_HYPERDX_APP_PORT=8090 -## The port dfe-proxy re-serves HyperDX on, with frame-ancestors rewritten so -## the DFE UI can iframe it. This is what the UI points at, not the app port. -# DFE_HYPERDX_EMBED_PORT=8091 -## Per-surface override of DFE_EXTERNAL_ORIGIN, which it follows when unset. # DFE_HYPERDX_APP_URL= -## HyperDX verifies the engine's token (`oidc-proxy`), so the console login is -## the HyperDX login. `header-dev` takes the identity off request headers -## instead, for a HyperDX running without an engine. -# DFE_HYPERDX_AUTH_MODE=oidc-proxy -## Where the ENGINE reaches HyperDX. The in-stack service unless this names an -## external one; the profile decides whether the engine is pointed at it at all. -# DFE_HYPERDX_BASE_URL=http://hyperdx:8000 -# HYPERDX_THEME=dfe -# HYPERDX_POSTGRES_USER=hyperdx -# -## HYPERDX_POSTGRES_PASSWORD is generated by `make init` - see "Generated -## secrets" above. Deliberately not given a placeholder here. +# DFE_HYPERDX_BASE_URL=http://dfe-hyperdx:8000 +# DFE_HYPERDX_EMBED_PORT=8091 +# DFE_HYPERDX_TEAM=dfe +# DFE_HYPERDX_VERSION= +# DFE_QUERY_READER_CLICKHOUSE_PASSWORD= +# HYPERDX_DOCUMENTDB_PASSWORD= +# HYPERDX_DOCUMENTDB_USER=hyperdx +# HYPERDX_DOCUMENTDB_VERSION= +# USAGE_STATS_ENABLED=false + +## ============================================================= +## Infra UI Authentication +## ============================================================= +# DFE_AUTH_ENABLED=false +# DFE_OAUTH2_PROXY_COOKIE_DOMAIN= +# DFE_OAUTH2_PROXY_COOKIE_SECRET= +# DFE_OAUTH2_PROXY_COOKIE_SECURE=false +# DFE_OAUTH2_PROXY_EXTERNAL_ORIGIN= +# DFE_OAUTH2_PROXY_VERSION= +# DFE_OIDC_ALLOWED_GROUPS=dfe-infra,dfe-admin +# DFE_OIDC_CLIENT_ID= +# DFE_OIDC_CLIENT_SECRET= +# DFE_OIDC_EMAIL_DOMAINS=* +# DFE_OIDC_ISSUER_URL= diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6889a149..a5d53338 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,29 +1,13 @@ # Project: dfe-docker # File: .github/workflows/ci.yml -# Purpose: Validate the compose stack and lint the dev builder Dockerfile +# Purpose: Validate the compose stack, lint the Dockerfiles, validate the proxy configs, prove the dfe-docker image starts the stack with only Docker and cut releases # # License: BUSL-1.1 - HYPERI PTY LIMITED # Copyright: (c) 2026 HYPERI PTY LIMITED # -# Why this is hand-written rather than a `uses: hyperi-io/hyperi-ci/...` call: -# hyperi-ci's reusable workflows are per-language (python / rust / go / ts) and -# its language detection returns nothing for a compose-packaging repo, so there -# is no reusable workflow to call for this shape. The house conventions are -# followed deliberately -- least-privilege permissions, SHA-pinned third-party -# actions with a version comment, Title Case job names, ubuntu-latest -- so that -# when hyperi-ci grows the container/compose gating it is currently missing, -# this collapses into a `uses:` line rather than a rewrite. -# -# Renovate automerges patch and digest updates in this repo "when CI passes", and -# before this workflow existed there was no CI to pass -- the rule was asserting a -# condition that could not be evaluated. This makes it mean what it says. -# -# Scope it honestly, though: the service image pins moved out of .env.example into -# the private stack SSoT, so they now live in a gitignored .env where Renovate -# cannot see them. What Renovate still manages HERE is RUST_IMAGE in the builder -# Dockerfile and the actions pinned below. And note check_compose.py substitutes -# placeholders for the mandatory keys, so this validates compose STRUCTURE, not -# that any particular digest resolves. +# Hand-written: hyperi-ci has no reusable workflow for a compose-packaging repo. +# ubuntu-26.04 is named, not ubuntu-latest, so a label move cannot swap the tools. +# Renovate automerges patch and digest updates once this workflow passes. name: CI @@ -32,179 +16,563 @@ on: push: branches: - main + # The inputs `hyperi-ci release` dispatches with. A bare dispatch only validates. workflow_dispatch: + inputs: + tag: + type: string + required: false + default: "" + description: "Existing tag to publish a GitHub release for when it has none." + from-head: + type: string + required: false + default: "" + description: "'true' to release the current HEAD. The CI creates the tag." + bump: + type: string + required: false + default: "auto" + description: "Version for from-head: auto | patch | minor (forced) | X.Y.Z (exact)." +# A release run has its own group and is never cancelled, because a cancel after the tag push leaves a tag with no GitHub release. concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + group: "${{ github.workflow }}-${{ (inputs.tag != '' || inputs.from-head == 'true' || github.event_name == 'push' && (contains(github.event.head_commit.message, 'Release: true') || contains(github.event.head_commit.message, 'Release:true') || contains(github.event.head_commit.message, 'Publish: true') || contains(github.event.head_commit.message, 'Publish:true'))) && 'release-' || '' }}${{ github.ref }}" + cancel-in-progress: "${{ !(inputs.tag != '' || inputs.from-head == 'true' || github.event_name == 'push' && (contains(github.event.head_commit.message, 'Release: true') || contains(github.event.head_commit.message, 'Release:true') || contains(github.event.head_commit.message, 'Publish: true') || contains(github.event.head_commit.message, 'Publish:true'))) }}" # Least-privilege GITHUB_TOKEN default for every job. permissions: contents: read +# The house Python floor. The runner's own python3 is older. +env: + PYTHON_VERSION: "3.14.7" + # The hyperi-ci CLI the Release job runs. The HYPERCI_INSTALL_OVERRIDE variable replaces it, as in hyperi-ci's own workflows. + HYPERCI_INSTALL: ${{ vars.HYPERCI_INSTALL_OVERRIDE || 'uvx --python 3.14 --no-cache --refresh hyperi-ci' }} + jobs: compose: name: Compose - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # No job here pushes, so the token has no reason to survive in # .git/config where any later step could read it. persist-credentials: false - # Every step below runs a `make` target rather than the command directly, - # so `make check` locally and CI cannot drift apart. - # - # Resolves docker-compose.yml on both the registry and dev-override paths, - # against both Kafka backends. Hermetic: it reads the mandatory keys out of - # the compose file and supplies placeholders, so it needs neither the - # private stack SSoT nor registry credentials. + # Every check below runs a Python helper script, and the pipx steps build + # ruff's and pytest's venvs on this interpreter too. + - name: Set up Python + id: python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + # Each step runs the make target `make check` runs locally (docs/checks.md). - name: Validate compose run: make check-compose - # The stack must refuse to start unpinned rather than silently pull - # `latest`. Without that, an operator cannot answer "which image am I - # running?" on a production deploy target, so it gets asserted rather than - # assumed. + # The stack must refuse to start unpinned rather than pull `latest`. - name: Assert a fresh checkout hard-fails run: make check-hardfail - # Five audience-scoped documents plus a README whose whole job is to route - # between them. A deleted or renamed file leaves a link that still looks - # like a link, and the reader finds out rather than the author -- which is - # exactly what retiring docs/SCOPE.md did to six live links. - # Stdlib only, so it needs nothing installed. + # Every relative link in the docs resolves. Stdlib only. - name: Docs run: make check-docs - # The pin lives in the Makefile (RUFF_VERSION), so a local `make check` - # lints with the same version this does. Installed here rather than run - # through uvx to keep the runner off the network mid-job. + # RUFF_VERSION in the Makefile pins it. Installed here to keep the runner + # off the network mid-job. - name: Install ruff - run: pipx install "ruff==$(make -s print-ruff-version)" + run: pipx install --python "${{ steps.python.outputs.python-path }}" "ruff==$(make -s print-ruff-version)" - # Every check above is driven by a Python helper script, so a bug in one of - # them shows up as a check that passes when it should not. They get linted - # like anything else. Config in ruff.toml. + # Lints the helper scripts every check above runs. Config in ruff.toml. - name: Python run: make check-python RUFF=ruff - # The credential helpers rewrite .env and decide whether a minted password - # reaches stdout, so they carry unit tests. Stdlib only, on a tmp_path .env: - # nothing here starts a stack or touches the runner's own files. + # Unit tests over the helper scripts, on a tmp_path .env. pipx builds the + # venv on the interpreter it is given, and PyYAML is injected for the seed + # tests, which parse resources/seed. - name: Install pytest - run: pipx install "pytest==$(make -s print-pytest-version)" + run: | + pipx install --python "${{ steps.python.outputs.python-path }}" "pytest==$(make -s print-pytest-version)" + pipx inject pytest "pyyaml==$(make -s print-pyyaml-version)" - name: Tests run: make check-tests PYTEST=pytest - # The slim and single profiles are projections of the Kubernetes tiers of the - # same name, so they go stale when those tiers move and nothing here notices. - # dfe-infra is PRIVATE, so the checkout reads it with a short-lived GitHub App - # token scoped to contents:read on that one repo; a run that cannot mint one - # reports SKIPPED rather than a pass, the same posture dfe-infra's own - # cross-repo drift guard takes. - profile-projection-precondition: - name: profile projection (can this run check it) - runs-on: ubuntu-latest - outputs: - can_check: ${{ steps.gate.outputs.can_check }} + # `make test-e2e` runs its scripts with `uv run --locked`, and Renovate's + # pep723 manager can bump a script's pin but has no step that rewrites its + # lock, so a pin and lock that disagree are caught here, not mid-suite. + - name: Install uv + run: pipx install --python "${{ steps.python.outputs.python-path }}" "uv==$(make -s print-uv-version)" + + - name: Script locks + run: make check-locks LOCK_UV=uv + + dockerfile: + name: Dockerfile + runs-on: ubuntu-26.04 steps: - - name: Can this run reach dfe-infra - id: gate - env: - # A pull_request from a fork is handed no secrets at all, so a skip - # there is the event's doing rather than a gap in this repo. - EVENT_CARRIES_SECRETS: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} - HAS_APP_KEY: ${{ secrets.GH_APP_PRIVATE_KEY != '' && vars.GH_APP_CLIENT_ID != '' }} - run: | - if [[ "$HAS_APP_KEY" == "true" ]]; then - echo "can_check=true" >> "$GITHUB_OUTPUT" - echo "The CI app key is present -- the projection check runs." - exit 0 + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # No job here pushes, so the token has no reason to survive in + # .git/config where any later step could read it. + persist-credentials: false + + # hadolint gates on error severity and lints the shell inside RUN through + # ShellCheck. Warnings such as DL3008 print without failing, and the reason + # sits in the Dockerfile. + - name: Hadolint + run: make check-dockerfile + + # check-proxy runs scripts/stack.py, so it gets the same interpreter as every other helper script + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + # oras: setup-oras pinned by commit, which checks the release tarball against the SHA256 it ships + - name: Install oras + uses: oras-project/setup-oras@5e2f2482b30b0cb3e14e67f89a80c88c659a7c63 # v2.0.3 + + # Envoy's validate mode loads every listener, cluster and TLS context, so a proxy config that would crash-loop dfe-proxy or dfe-hyperdx-proxy fails here. With no .env, the Envoy is the one deployment.example.yaml's pinned stack carries, read anonymously from the stack manifest through oras. The TLS variants get a throwaway certificate from the runner's openssl. + - name: Proxy configs + run: make check-proxy + +# Only Docker on PATH: each profile is set up, and the default one starts, serves +# the console and runs the demo. The Windows job runs the default install. + image: + name: Image + runs-on: ubuntu-26.04 + timeout-minutes: 50 + env: + IMAGE: dfe-docker:ci + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + # The stack deployment.example.yaml pins, as image.yml bakes into the : image + - name: Build the image + run: docker build --file docker/dfe-docker.Dockerfile --build-arg "IMAGE=$IMAGE" --build-arg "STACK=$(python3 scripts/quickstart_json.py --print-stack)" --tag "$IMAGE" . + + # docker, curl for the console check and the few tools the checks below use. No make, no python + - name: Put only Docker on PATH + run: | + bin="$RUNNER_TEMP/docker-only" + mkdir -p "$bin" "$RUNNER_TEMP/dfe" + for tool in docker curl grep mkdir sha256sum; do + ln -s "$(command -v "$tool")" "$bin/$tool" + done + echo "DOCKER_ONLY=$bin" >> "$GITHUB_ENV" + echo "WORK=$RUNNER_TEMP/dfe" >> "$GITHUB_ENV" + + - name: Set up every profile + run: | + export PATH="$DOCKER_ONLY" + if command -v make || command -v python3 || command -v python; then + echo "::error::make or python is still on PATH" + exit 1 + fi + for profile in slim-grpc slim-kafka single-grpc single-kafka; do + mkdir "$WORK/$profile" + cd "$WORK/$profile" + docker run --rm -v "${PWD}:/dfe" "$IMAGE" init --profile "$profile" + docker compose config -q + done + + - name: Set up the default profile + run: | + export PATH="$DOCKER_ONLY" + mkdir "$WORK/default" + cd "$WORK/default" + said="$(docker run --rm -v "${PWD}:/dfe" "$IMAGE" init)" + printf '%s\n' "$said" + if [[ "$said" != *" $IMAGE login"* ]]; then + echo "::error::init's printed commands do not name $IMAGE, the image that ran" + exit 1 fi - echo "can_check=false" >> "$GITHUB_OUTPUT" - if [[ "$EVENT_CARRIES_SECRETS" == "true" ]]; then - echo "::notice::profile projection not checked -- this repo cannot use the GH_APP_PRIVATE_KEY org secret (or GH_APP_CLIENT_ID is unset), so no token for hyperi-io/dfe-infra can be minted. The check is skipped rather than reported as a pass." - else - echo "::notice::profile projection not checked -- a fork pull request is handed no secrets. The push to main checks the same commit." + docker compose config -q + docker compose config --services > "$WORK/default.services" + + # A second init, a bad profile and an upgrade, each against what it must leave alone + - name: Refuse and upgrade + run: | + export PATH="$DOCKER_ONLY" + cd "$WORK/default" + if docker run --rm -v "${PWD}:/dfe" "$IMAGE" init; then + echo "::error::init ran over an existing install" + exit 1 + fi + secrets() { grep -h -E '^[A-Z0-9_]*(PASSWORD|SECRET|KEY)=' .env env/*.env | sha256sum; } + before="$(secrets)" + docker run --rm -v "${PWD}:/dfe" "$IMAGE" init --upgrade + if [ "$(secrets)" != "$before" ]; then + echo "::error::init --upgrade changed a minted secret" + exit 1 + fi + docker compose config -q + mkdir "$WORK/bad" + cd "$WORK/bad" + if docker run --rm -v "${PWD}:/dfe" "$IMAGE" init --profile no-such-profile; then + echo "::error::init took a profile service_profiles.yaml does not declare" + exit 1 fi + shopt -s dotglob nullglob + left=(*) + if [ "${#left[@]}" -ne 0 ]; then + echo "::error::a refused init left ${left[*]} behind" + exit 1 + fi + + # A throwaway install whose stack is torn down below, so its minted secrets protect nothing + - name: Hand the default install to the Windows job + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: default-install + path: | + ${{ env.WORK }}/default + ${{ env.WORK }}/default.services + !${{ env.WORK }}/default/access-summary.md + include-hidden-files: true + if-no-files-found: error + retention-days: 1 - profile-projection: - name: Profile projection - needs: profile-projection-precondition - if: needs.profile-projection-precondition.outputs.can_check == 'true' - runs-on: ubuntu-latest + - name: Start the default profile and reach the console + timeout-minutes: 20 + run: | + export PATH="$DOCKER_ONLY" + cd "$WORK/default" + docker compose up -d --wait --wait-timeout 900 + curl -fsSL --retry 5 --retry-all-errors -o /dev/null -w 'console answered %{http_code} at %{url_effective}\n' http://localhost:3000/ + + - name: Run the demo and take it out again + timeout-minutes: 15 + run: | + export PATH="$DOCKER_ONLY" + cd "$WORK/default" + docker run --rm -v "${PWD}:${PWD}" -w "${PWD}" -v /var/run/docker.sock:/var/run/docker.sock "$IMAGE" demo + docker run --rm -v "${PWD}:${PWD}" -w "${PWD}" -v /var/run/docker.sock:/var/run/docker.sock "$IMAGE" demo-stop + docker compose config -q + + - name: Show what did not come up + if: failure() + working-directory: ${{ env.WORK }}/default + run: | + docker compose ps --all + docker compose logs --no-color --tail 100 + + - name: Tear the stack down + if: always() + working-directory: ${{ env.WORK }}/default + run: | + if [ -f .env ]; then + docker compose --profile '*' down --volumes --remove-orphans + fi + + # One Published compose leg per demo pack: resources/demo, then each resources/demo- directory in byte order, as scripts/_common.py demo_pack_names lists them + demo-packs: + name: Demo Packs + runs-on: ubuntu-26.04 + timeout-minutes: 5 + outputs: + dirs: ${{ steps.list.outputs.dirs }} steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - name: Mint a read-only dfe-infra token - id: infra-token - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 - with: - client-id: ${{ vars.GH_APP_CLIENT_ID }} - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - owner: ${{ github.repository_owner }} - repositories: dfe-infra - permission-contents: read + - name: List the packs + id: list + run: | + dirs="$({ echo demo; find resources -mindepth 1 -maxdepth 1 -type d -name 'demo-*' -printf '%f\n' | LC_ALL=C sort; } | jq -Rnc '[inputs]')" + echo "dirs=$dirs" | tee -a "$GITHUB_OUTPUT" - - name: Checkout dfe-infra - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + # The files a release publishes as OCI artifacts, generated from this commit and run from a folder holding nothing else, as `-f oci://` runs them. Each demo file holds the whole stack plus its pack, so it starts and has to land the data its pack replays. The demo leg also writes the stack file, which is read and checked + published: + name: Published compose (${{ matrix.pack }}) + needs: demo-packs + runs-on: ubuntu-26.04 + timeout-minutes: 50 + strategy: + fail-fast: false + matrix: + pack: ${{ fromJSON(needs.demo-packs.outputs.dirs) }} + env: + IMAGE: dfe-docker:ci + PACK_DIR: resources/${{ matrix.pack }} + FILE: dfe-${{ matrix.pack }}.yaml + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - repository: hyperi-io/dfe-infra - token: ${{ steps.infra-token.outputs.token }} - path: .dfe-infra persist-credentials: false - - name: Projected profiles match the Kubernetes ones - run: make check-profiles DFE_INFRA_DIR="$PWD/.dfe-infra" + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: ${{ env.PYTHON_VERSION }} - dockerfile: - name: Dockerfile - runs-on: ubuntu-latest + # scripts/stack.py reads the stack's image pins through oras + - name: Install oras + uses: oras-project/setup-oras@5e2f2482b30b0cb3e14e67f89a80c88c659a7c63 # v2.0.3 + + - name: Build the image + run: docker build --file docker/dfe-docker.Dockerfile --build-arg "IMAGE=$IMAGE" --tag "$IMAGE" . + + # The image is local here, so its digest is the one finding allowed + - name: Generate and check the published files + env: + PACK: ${{ matrix.pack }} + run: | + docker compose version + pack=() + if [ "$PACK" != demo ]; then + pack=(--pack "${PACK#demo-}") + fi + python3 scripts/publish_compose.py --image "$IMAGE" --out-dir "$RUNNER_TEMP/compose" "${pack[@]}" + echo "COMPOSE_DIR=$RUNNER_TEMP/compose" >> "$GITHUB_ENV" + + - name: Start the demo and reach the console + timeout-minutes: 25 + working-directory: ${{ env.COMPOSE_DIR }} + run: | + docker compose -f "$FILE" up -d --wait --wait-timeout 1200 + curl -fsSL --retry 5 --retry-all-errors -o /dev/null -w 'console answered %{http_code} at %{url_effective}\n' http://localhost:3000/ + + # Every source with events the pack replays, which need no egress, so a pack whose demo.json lists no sources checks none. Fetcher tables wait on their upstream and are not checked + - name: Land the pack's data + timeout-minutes: 10 + working-directory: ${{ env.COMPOSE_DIR }} + run: | + tables=() + for source in $(jq -r '.sources[]' "$GITHUB_WORKSPACE/$PACK_DIR/demo.json"); do + if [ -d "$GITHUB_WORKSPACE/$PACK_DIR/events/$source" ]; then + tables+=("$source") + fi + done + if [ "${#tables[@]}" -eq 0 ]; then + echo "$PACK_DIR replays no source, so no table is checked" + fi + for table in "${tables[@]}"; do + for attempt in $(seq 60); do + rows="$(docker compose -f "$FILE" exec -T clickhouse sh -c ". /run/dfe-secrets/env && clickhouse-client --password \"\$CLICKHOUSE_PASSWORD\" --query 'SELECT count() FROM dfe.\`$table\`'" || echo 0)" + if [ "${rows:-0}" -gt 0 ]; then + echo "dfe.$table holds $rows rows" + break + fi + if [ "$attempt" -eq 60 ]; then + echo "::error::no rows reached dfe.$table" + exit 1 + fi + sleep 5 + done + done + + # The secrets are this runner's own: no service holds one in its environment, ClickHouse refuses its empty default password, and the login the file prints is held for a change + - name: Run on secrets minted here + working-directory: ${{ env.COMPOSE_DIR }} + run: | + for svc in clickhouse demo-console demo-sources dfe-engine dfe-hunt-runner dfe-hyperdx dfe-loader dfe-ui hyperdx-documentdb otel-collector; do + held="$(docker inspect "$(docker compose -f "$FILE" ps -aq "$svc")" --format '{{range .Config.Env}}{{println .}}{{end}}' | cut -d= -f1 | grep -E '(PASSWORD|SECRET|TOKEN|KEY|MONGO_URI|IDENTITY)$' || true)" + if [ -n "$held" ]; then + echo "::error::$svc holds $held in its environment" + exit 1 + fi + done + # secrets-init declares the admin's chosen name and password, which are empty on a run that chose none + init_env="$(docker inspect "$(docker compose -f "$FILE" ps -aq secrets-init)" --format '{{range .Config.Env}}{{println .}}{{end}}')" + held="$(printf '%s\n' "$init_env" | cut -d= -f1 | grep -E '(PASSWORD|SECRET|TOKEN|KEY|MONGO_URI|IDENTITY)$' | grep -vx DFE_ADMIN_PASSWORD || true)" + if [ -n "$held" ] || printf '%s\n' "$init_env" | grep -Eq '^DFE_ADMIN_(NAME|PASSWORD)=.'; then + echo "::error::secrets-init holds $held or a chosen admin login in its environment" + exit 1 + fi + if docker compose -f "$FILE" exec -T clickhouse clickhouse-client --password '' --query 'SELECT 1'; then + echo "::error::ClickHouse took the empty default password" + exit 1 + fi + docker compose -f "$FILE" exec -T dfe-engine sh -c '. /run/dfe-secrets/env && test ${#DFE_ENGINE_API_JWT_SECRET} -eq 64 && test ${#DFE_ENGINE_CLICKHOUSE_PASSWORD} -eq 32' + login="$(docker compose -f "$FILE" run --rm secrets-init login)" + DFE_LOGIN_NAME="$(printf '%s\n' "$login" | sed -n 's/^Console login: \(.*\) \/ .*$/\1/p')" + DFE_LOGIN_PASSWORD="$(printf '%s\n' "$login" | sed -n 's/^Console login: .* \/ \(.*\)$/\1/p')" + echo "::add-mask::$DFE_LOGIN_PASSWORD" + test "${#DFE_LOGIN_PASSWORD}" -eq 32 + export DFE_LOGIN_NAME DFE_LOGIN_PASSWORD + docker compose -f "$FILE" exec -T -e DFE_LOGIN_NAME -e DFE_LOGIN_PASSWORD dfe-engine python -c 'import json, os, urllib.request + request = urllib.request.Request("http://localhost:8000/api/v1/auth/login", data=json.dumps({"username": os.environ["DFE_LOGIN_NAME"], "password": os.environ["DFE_LOGIN_PASSWORD"]}).encode(), headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(request, timeout=30) as answer: + assert json.load(answer).get("password_change_required") is True, "the first login is not held for a change"' + + - name: Show what did not come up + if: failure() + working-directory: ${{ env.COMPOSE_DIR }} + run: | + docker compose -f "$FILE" ps --all + # The helpers and data path this pack's file runs, since a pack with no syslog has no stream or transform instance + services=() + for svc in $(docker compose -f "$FILE" config --services); do + case "$svc" in + secrets-init | demo-* | dfe-receiver | dfe-transform-elastic-*) services+=("$svc") ;; + esac + done + docker compose -f "$FILE" logs --no-color --tail 100 "${services[@]}" + + - name: Tear the demo down + if: always() + run: | + if [ -f "${COMPOSE_DIR:-}/$FILE" ]; then + docker compose -f "$COMPOSE_DIR/$FILE" down --volumes --remove-orphans + fi + + # Compose on Windows splits COMPOSE_FILE on `;` unless COMPOSE_PATH_SEPARATOR in .env says otherwise + image-windows: + name: Image (Windows compose) + needs: image + runs-on: windows-2025 + timeout-minutes: 10 + steps: + - name: Fetch the default install + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: default-install + path: ${{ runner.temp }}/dfe + + - name: Read it with the Windows compose + shell: pwsh + run: | + Set-Location (Join-Path $env:RUNNER_TEMP 'dfe/default') + docker compose version + $listed = docker compose config --services + if ($LASTEXITCODE -ne 0) { exit 1 } + $got = ($listed | Sort-Object) -join ',' + $want = (Get-Content ../default.services | Sort-Object) -join ',' + if ($got -ne $want) { + Write-Output "::error::Windows compose read $got where Linux read $want" + exit 1 + } + Write-Output "Windows compose starts the same services: $got" + +# Tags and publishes a release on `Release: true` or a `hyperi-ci release` dispatch, +# after the image jobs. The App's token publishes it so image.yml starts on it. + release: + name: Release + needs: [compose, dockerfile, image, image-windows, published] + if: github.event_name != 'pull_request' + runs-on: ubuntu-26.04 + timeout-minutes: 20 + # semantic-release's dry run checks push access, which needs write + permissions: + contents: write steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - # No job here pushes, so the token has no reason to survive in - # .git/config where any later step could read it. + ref: ${{ inputs.tag || github.sha }} + # semantic-release reads every commit since the last tag + fetch-depth: 0 persist-credentials: false - # hadolint GATES: a decade old, stable rule IDs, and it lints the shell - # inside RUN via ShellCheck -- a correctness check droast structurally - # does not have. - # stdin form, per the standard, with no config file, so the exact same - # command is runnable locally -- no mount, no config discovery. DL3008 (pin - # apt versions) fires on the Rust builder and is deliberately left as a - # visible warning, not suppressed: `--failure-threshold error` lets a - # warning inform without blocking, and the reason sits in the Dockerfile. - - name: Hadolint - run: make check-dockerfile + # The trailer gate, the classification check and the version, as every hyperi-ci pipeline decides them + - name: Plan the release + id: plan + uses: hyperi-io/hyperi-ci/.github/actions/predict-version@main + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ inputs.tag }} + from-head: ${{ inputs.from-head }} + bump: ${{ inputs.bump || 'auto' }} -# droast is DELIBERATELY NOT RUN HERE. That matches hyperi-ci's own posture -- -# quality/droast.py resolves it with `find_tool(recommended=False)` and info-skips -# when absent, so hyperi-ci never installs it either -- but the reason here is -# measured rather than inherited. -# -# Run against docker/dfe-rust-builder.Dockerfile on 2026-07-20 (droast 0.1.x, -# `--no-fail`): 6 findings, 0 true positives. -# -# DF036 no CMD/ENTRYPOINT - false. The final stage is `FROM scratch AS export`, -# an artefact export consumed via `--output`. -# DF001 unpinned image tag - false. ARG RUST_IMAGE is `rust:1.97-trixie@sha256:...`; -# droast does not resolve the ARG default. hadolint does. -# DF057 pipe, no pipefail - false. `SHELL ["/bin/bash", "-o", "pipefail", "-c"]` -# is set; droast ignores the SHELL directive. -# DF063 relative COPY dest - false. The destination is `"/${BINARY}"`, absolute. -# DF005 unpinned apt - duplicate of hadolint DL3008. -# DF033 no .dockerignore - wrong context (assumes the Dockerfile's own dir). -# -# Two of those tell us to fix things this Dockerfile already fixes correctly, so -# wiring it in would train people to ignore the advisory. droast.toml is kept so -# `droast .` works locally for anyone who wants the cache-ordering rules hadolint -# has no equivalent for. Revisit when its parser handles ARG defaults and SHELL. + - name: Install uv + if: steps.plan.outputs.will-release == 'true' + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: + enable-cache: false + + - name: Mint the release token + id: bot + if: steps.plan.outputs.will-release == 'true' + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + client-id: ${{ vars.GH_APP_CLIENT_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ github.event.repository.name }} + permission-contents: write + + # Plan installed semantic-release and the central config in this job + - name: Tag (semantic-release) + if: steps.plan.outputs.will-release == 'true' && (github.event_name == 'push' || inputs.from-head == 'true' && inputs.bump == 'auto') + env: + GITHUB_TOKEN: ${{ steps.bot.outputs.token }} + run: semantic-release --repository-url "https://github.com/${GITHUB_REPOSITORY}" + + - name: Tag HEAD (forced bump or exact version) + if: steps.plan.outputs.will-release == 'true' && inputs.from-head == 'true' && inputs.bump != 'auto' && inputs.bump != '' + env: + GH_TOKEN: ${{ steps.bot.outputs.token }} + RELEASE_VERSION: ${{ steps.plan.outputs.version }} + run: ${{ env.HYPERCI_INSTALL }} tag-head --bump "$RELEASE_VERSION" + + # semantic-release tags nothing when this checkout is behind main, and gh release create would then tag main's tip + - name: Check the tag is on this commit + if: steps.plan.outputs.will-release == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_VERSION: ${{ steps.plan.outputs.version }} + run: | + tag="v${RELEASE_VERSION}" + head=$(git rev-parse "HEAD^{commit}") + if ! ref=$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/tags/${tag}" --jq '.object.type + " " + .object.sha'); then + echo "::error::${tag} does not exist, so no GitHub release is published. Release the current head with 'hyperi-ci release'." + exit 1 + fi + read -r kind sha <<< "$ref" + if [ "$kind" = "tag" ]; then + sha=$(gh api "repos/${GITHUB_REPOSITORY}/git/tags/${sha}" --jq '.object.sha') + fi + if [ "$sha" != "$head" ]; then + echo "::error::${tag} is at ${sha} but this run checked out ${head}." + exit 1 + fi + + - name: Publish the GitHub release + if: steps.plan.outputs.will-release == 'true' + env: + GH_TOKEN: ${{ steps.bot.outputs.token }} + PRERELEASE: ${{ steps.plan.outputs.prerelease }} + RELEASE_VERSION: ${{ steps.plan.outputs.version }} + run: | + tag="v${RELEASE_VERSION}" + if gh release view "$tag" --repo "$GITHUB_REPOSITORY" > /dev/null 2>&1; then + echo "::notice::${tag} already has a GitHub release, so image.yml does not start again. Rebuild with: gh workflow run image.yml -f tag=${tag}" + exit 0 + fi + kind=(--latest) + if [ "$PRERELEASE" = "true" ]; then + kind=(--prerelease --latest=false) + fi + gh release create "$tag" --repo "$GITHUB_REPOSITORY" --verify-tag --title "$tag" --generate-notes "${kind[@]}" + + # Non-fatal: the release is already published. A tag dispatch checks out an old tag, so it commits nothing back + - name: Commit VERSION and CHANGELOG.md back + if: steps.plan.outputs.will-release == 'true' && inputs.tag == '' + continue-on-error: true + env: + GH_TOKEN: ${{ steps.bot.outputs.token }} + HYPERCI_RELEASE_IDENTITY: app + RELEASE_BRANCH: ${{ github.ref_name }} + RELEASE_VERSION: ${{ steps.plan.outputs.version }} + run: ${{ env.HYPERCI_INSTALL }} release-commit --branch "$RELEASE_BRANCH" "$RELEASE_VERSION" + +# droast is not run here: it misreads ARG defaults and the SHELL directive, so its +# findings on these Dockerfiles are false. droast.toml keeps `droast .` working by hand. diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml new file mode 100644 index 00000000..a2881d54 --- /dev/null +++ b/.github/workflows/image.yml @@ -0,0 +1,210 @@ +# Project: dfe-docker +# File: .github/workflows/image.yml +# Purpose: Build the dfe-docker image for amd64 and arm64, and on a release push it and attach quickstart.json +# +# License: BUSL-1.1 - HYPERI PTY LIMITED +# Copyright: (c) 2026 HYPERI PTY LIMITED +# +# Two images per release, built from one Dockerfile on each architecture's own +# runner. `:` installs the stack deployment.example.yaml pins at the +# release tag, so it installs the same stack every time. `:latest` installs the +# newest published stack when init runs, and moves only on a release that is not +# a pre-release. A pull request that touches the image builds both on both +# architectures and pushes nothing. + +name: Image + +on: + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: The release tag to build and attach quickstart.json to + required: true + pull_request: + paths: + - .dockerignore + - .github/workflows/image.yml + - Makefile + - docker/dfe-docker.Dockerfile + - scripts/** + +permissions: + contents: read + +env: + PYTHON_VERSION: "3.14.7" + REPOSITORY: ghcr.io/hyperi-io/dfe-docker + TAG: ${{ github.event.release.tag_name || inputs.tag }} + +jobs: + build: + name: Build (${{ matrix.arch }}) + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-26.04 + - arch: arm64 + runner: ubuntu-26.04-arm + # Each architecture on its own runner, never emulated + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.TAG || github.sha }} + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Name the release + id: release + run: | + version="${TAG#v}" + # Assigned on its own line so a prerelease pin fails this step, not the publish. + stack="$(python3 scripts/quickstart_json.py --print-stack)" + { + echo "version=${version:-0.0.0-pr}" + echo "stack=$stack" + echo "push=${{ github.event_name != 'pull_request' }}" + } >> "$GITHUB_OUTPUT" + + - name: GHCR login + if: steps.release.outputs.push == 'true' + env: + GH_TOKEN: ${{ github.token }} + run: printf '%s' "$GH_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + + # Pushed by digest alone, so the publish job tags both architectures together + - name: Build both images + env: + PUSH: ${{ steps.release.outputs.push }} + STACK: ${{ steps.release.outputs.stack }} + VERSION: ${{ steps.release.outputs.version }} + run: | + mkdir -p "$RUNNER_TEMP/digests" + for variant in latest pinned; do + if [ "$variant" = latest ]; then + stack=latest + image="$REPOSITORY" + else + stack="$STACK" + image="$REPOSITORY:$VERSION" + fi + docker buildx build \ + --platform "linux/${{ matrix.arch }}" \ + --file docker/dfe-docker.Dockerfile \ + --build-arg "IMAGE=$image" \ + --build-arg "STACK=$stack" \ + --build-arg "VERSION=$VERSION" \ + --output "type=image,name=$REPOSITORY,push-by-digest=true,name-canonical=true,push=$PUSH" \ + --metadata-file "$RUNNER_TEMP/$variant.json" \ + . + jq -r '."containerimage.digest"' "$RUNNER_TEMP/$variant.json" > "$RUNNER_TEMP/digests/$variant-${{ matrix.arch }}" + done + + - name: Hand the digests to the publish job + if: steps.release.outputs.push == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: digests-${{ matrix.arch }} + path: ${{ runner.temp }}/digests/ + if-no-files-found: error + retention-days: 1 + + publish: + name: Publish + needs: build + if: github.event_name != 'pull_request' + runs-on: ubuntu-26.04 + timeout-minutes: 15 + permissions: + contents: write + packages: write + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ env.TAG }} + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Fetch the digests + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: digests-* + path: ${{ runner.temp }}/digests + merge-multiple: true + + - name: GHCR login + env: + GH_TOKEN: ${{ github.token }} + run: printf '%s' "$GH_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + + - name: Tag both architectures + env: + GH_TOKEN: ${{ github.token }} + run: | + version="${TAG#v}" + digests="$RUNNER_TEMP/digests" + docker buildx imagetools create --tag "$REPOSITORY:$version" \ + "$REPOSITORY@$(cat "$digests/pinned-amd64")" "$REPOSITORY@$(cat "$digests/pinned-arm64")" + if [ "$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json isPrerelease --jq .isPrerelease)" = false ]; then + docker buildx imagetools create --tag "$REPOSITORY:latest" \ + "$REPOSITORY@$(cat "$digests/latest-amd64")" "$REPOSITORY@$(cat "$digests/latest-arm64")" + fi + docker buildx imagetools inspect "$REPOSITORY:$version" + + # scripts/stack.py reads the stack's image pins through oras + - name: Install oras + uses: oras-project/setup-oras@5e2f2482b30b0cb3e14e67f89a80c88c659a7c63 # v2.0.3 + + # The compose files go into the same package under their own tags, so the image tags above stay as they are: the stack and demo, then each resources/demo- as demo-. The demo's helpers run the image just tagged, by digest + - name: Publish the compose files + env: + GH_TOKEN: ${{ github.token }} + run: | + version="${TAG#v}" + docker compose version + digest="$(docker buildx imagetools inspect "$REPOSITORY:$version" --format '{{.Manifest.Digest}}')" + moving=() + if [ "$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json isPrerelease --jq .isPrerelease)" = false ]; then + moving=(--moving) + fi + stack="$(python3 scripts/quickstart_json.py --print-stack)" + python3 scripts/publish_compose.py --stack "$stack" \ + --image "$REPOSITORY@$digest" --out-dir "$RUNNER_TEMP/compose" \ + --publish "$REPOSITORY" --release "$version" "${moving[@]}" + for dir in $(find resources -mindepth 1 -maxdepth 1 -type d -name 'demo-*' -printf '%f\n' | LC_ALL=C sort); do + python3 scripts/publish_compose.py --stack "$stack" --pack "${dir#demo-}" \ + --image "$REPOSITORY@$digest" --out-dir "$RUNNER_TEMP/compose" \ + --publish "$REPOSITORY" --release "$version" "${moving[@]}" + done + + - name: Attach quickstart.json + env: + GH_TOKEN: ${{ github.token }} + run: | + python3 scripts/quickstart_json.py --version "${TAG#v}" --out "$RUNNER_TEMP/quickstart.json" + cat "$RUNNER_TEMP/quickstart.json" + gh release upload "$TAG" "$RUNNER_TEMP/quickstart.json" --repo "$GITHUB_REPOSITORY" --clobber diff --git a/.gitignore b/.gitignore index 9aae17db..1e8e3dcf 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,14 @@ access-summary.md deployment.yaml # Written by the track-latest daemon: the stack version it last applied. .dfe-stack-applied +# Written by make proof: the stack version it last proved end to end. +.dfe-proof-passed +# Held by make proof and the daemon updater while either has the stack down. +.dfe-stack.lock *.log +# The demo and its packs replay these sample events +!resources/demo/events/**/*.log +!resources/demo-*/events/**/*.log dist/ dist-bin/ bin/ @@ -24,11 +31,6 @@ out/ test-results/ coverage/ .tmp/ -.profile.mk -# Written by `make dev LOCAL=...`: the built components repointed at :local. -docker-compose.local.yml -# Written with the profile: one service per app instance the engine has rendered. -docker-compose.instances.yml certs/ # Python bytecode (scripts/ generated by make dev / resolve_profile) @@ -42,18 +44,34 @@ docs/superpowers/ # Local working files (not committed) .claude/ -hyperi-ai/ +.codex/ +.cursor/ +.gemini/ +.superpowers/ +.playwright-mcp/ .mcp.json +.hyperi-ai +.worktrees/ +.kube/ +.secrets/ +.env.* +!.env*.example +hyperi-ai/ STATE.md CLAUDE.md TODO.md .ai-version +.benchmarks/ +# Written at the repo root by older checkouts, now under .tmp/ +/.profile.mk +/docker-compose.instances.yml +/docker-compose.local.yml # Per-service env files: env/ env/ +# Local stash of the per-service env files. +.env-dir-stashed/ -# Predecessor CI clone and an inert submodule declaration, both untracked and -# left in the tree. Ignored so `git add -A` cannot pull a dead CI system (100+ -# files) or a broken submodule pointer into this repo. Safe to delete outright. +# A predecessor CI clone and an inert submodule declaration, both untracked. Safe to delete. ci/ .gitmodules diff --git a/.pytest.ini b/.pytest.ini new file mode 100644 index 00000000..21659b3a --- /dev/null +++ b/.pytest.ini @@ -0,0 +1,11 @@ +# Project: dfe-docker +# File: .pytest.ini +# Purpose: Python pytest configuration file (hide temporary files) +# Language: Ini +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +[pytest] +cache_dir = .tmp/pytest +addopts = -p no:benchmark diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..14b18732 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,53 @@ +# Changelog + +Rendered by CI and committed back at the end of a release -- do not edit by +hand. Release notes also appear on the GitHub Releases page, one per tag. + +## [1.0.8](https://github.com/hyperi-io/dfe-docker/compare/v1.0.7...v1.0.8) (2026-11-16) + +### Bug Fixes + +* **compose:** mint the admin login and print it with one command ([#396](https://github.com/hyperi-io/dfe-docker/issues/396)) ([e6563d8](https://github.com/hyperi-io/dfe-docker/commit/e6563d868a7e820a55f5b0792bf7706fcda46289)) +* **compose:** pass the engine the images the stack runs ([#402](https://github.com/hyperi-io/dfe-docker/issues/402)) ([2c5cc48](https://github.com/hyperi-io/dfe-docker/commit/2c5cc48b5d3dd4e022f533b30c0c0d54503ae67b)) +* **compose:** publish syslog and flow listeners through port keys ([#400](https://github.com/hyperi-io/dfe-docker/issues/400)) ([0339b29](https://github.com/hyperi-io/dfe-docker/commit/0339b294b7e57320739cefbb48f158cf757e4035)) +* **compose:** TLS ingest via dfe-proxy and fetcher input mount ([#401](https://github.com/hyperi-io/dfe-docker/issues/401)) ([cd82ccf](https://github.com/hyperi-io/dfe-docker/commit/cd82ccfc9909e87d2fbb12d2cf39d6b5411fc400)) +* **make:** block ci while a demo pack is streaming ([#394](https://github.com/hyperi-io/dfe-docker/issues/394)) ([43d1659](https://github.com/hyperi-io/dfe-docker/commit/43d165951b0d05a57f7f35dd14c58faa3e86fa30)) +* **post:** check dfe-ui's metrics and correct stale comments ([#404](https://github.com/hyperi-io/dfe-docker/issues/404)) ([e4464ea](https://github.com/hyperi-io/dfe-docker/commit/e4464eaa263e9b238e72c605ed53dde9e1e9f4d8)), closes [#1099](https://github.com/hyperi-io/dfe-docker/issues/1099) +* **secrets:** mint 32 and 64 character alphanumeric secrets ([#395](https://github.com/hyperi-io/dfe-docker/issues/395)) ([b82f579](https://github.com/hyperi-io/dfe-docker/commit/b82f579dce19597e8be05a1ca014a904f4f313da)) +* **stack:** refuse a prerelease stack pin on a release ([#391](https://github.com/hyperi-io/dfe-docker/issues/391)) ([20f139a](https://github.com/hyperi-io/dfe-docker/commit/20f139a315ebecc1c6fd80fbb0b0556b62d578c7)) + +## [1.0.7](https://github.com/hyperi-io/dfe-docker/compare/v1.0.6...v1.0.7) (2026-11-14) + +### Bug Fixes + +* **stack:** pin the stack dial at 2.2.2 ([#377](https://github.com/hyperi-io/dfe-docker/issues/377)) ([c7aac00](https://github.com/hyperi-io/dfe-docker/commit/c7aac00918de5c368093105ea8b4361ed063acfd)) + +## [1.0.6](https://github.com/hyperi-io/dfe-docker/compare/v1.0.5...v1.0.6) (2026-11-13) + +### Bug Fixes + +* **demo:** add the cyber demo pack's sources, events and console ([#384](https://github.com/hyperi-io/dfe-docker/issues/384)) ([0eae396](https://github.com/hyperi-io/dfe-docker/commit/0eae396bd8955bb16891fa7cb2ef3d39c344e368)) +* **demo:** add the telco demo pack ([#389](https://github.com/hyperi-io/dfe-docker/issues/389)) ([7a45cd3](https://github.com/hyperi-io/dfe-docker/commit/7a45cd37440652cc2eed4bb499e6f1db6659c8fc)) +* **demo:** add the theatre and battlespace demo pack ([#386](https://github.com/hyperi-io/dfe-docker/issues/386)) ([863c6bc](https://github.com/hyperi-io/dfe-docker/commit/863c6bcd09c32dc6fb7cbec89f07d4486741aa08)) +* **demo:** publish one compose file per demo pack ([#383](https://github.com/hyperi-io/dfe-docker/issues/383)) ([e729085](https://github.com/hyperi-io/dfe-docker/commit/e7290857ade15242d1174d26d07fb6387ffc3bbe)) +* **demo:** replay a pack's Beats sources and host their transforms ([#385](https://github.com/hyperi-io/dfe-docker/issues/385)) ([be05917](https://github.com/hyperi-io/dfe-docker/commit/be059173adff716d18d35bef7516cdad149d359c)) +* **otel:** log HyperDX under dfe-hyperdx ([#371](https://github.com/hyperi-io/dfe-docker/issues/371)) ([b28bdc6](https://github.com/hyperi-io/dfe-docker/commit/b28bdc642f3273c00636fb3f260fd1d8c002c584)) + +## [1.0.5](https://github.com/hyperi-io/dfe-docker/compare/v1.0.4...v1.0.5) (2026-11-12) + +### Bug Fixes + +* **compose:** read alert destinations from the engine's secrets backend ([a291819](https://github.com/hyperi-io/dfe-docker/commit/a2918192afa1957bbab3b1ddd57ee319a4da991e)) +* **compose:** run one fetcher for every fetcher source ([cf2e8d3](https://github.com/hyperi-io/dfe-docker/commit/cf2e8d35ab9f493c59c572ed5f23b0045a0b322e)), closes [hyperi-io/dfe-fetcher#289](https://github.com/hyperi-io/dfe-fetcher/issues/289) +* **demo:** complete the engine's first-run setup ([ee86c18](https://github.com/hyperi-io/dfe-docker/commit/ee86c18073fca5e8f01c14287d86c5e4596e23e6)) +* **install:** publish one-line compose installs ([40169b4](https://github.com/hyperi-io/dfe-docker/commit/40169b43430fcdcca6ca7dbe63d5505724a4dd7a)) +* **renovate:** keep hyperi-ci references on main ([8f78312](https://github.com/hyperi-io/dfe-docker/commit/8f783127da1afa5faa113e3f8e760527f1e10bac)) +* **stack:** pin stack to 2.2.2 ([52f72ca](https://github.com/hyperi-io/dfe-docker/commit/52f72ca3573675a26480fd51e8af3d615b2f3c60)) +* **stack:** set stack version to 2.2.2-rc.4 ([2b9f2e5](https://github.com/hyperi-io/dfe-docker/commit/2b9f2e57a6a90d270e99b0c805fb5bd1d5d1126d)), closes [dfe-infra#116](https://github.com/hyperi-io/dfe-infra/issues/116) + +## [1.0.4](https://github.com/hyperi-io/dfe-docker/compare/v1.0.3...v1.0.4) (2026-11-10) + +### Bug Fixes + +* **compose:** name the HyperDX service dfe-hyperdx ([47f2a20](https://github.com/hyperi-io/dfe-docker/commit/47f2a205708180e46acf87c634f7b64240989eaa)) +* **docs:** name the search console dfe-hyperdx ([139024c](https://github.com/hyperi-io/dfe-docker/commit/139024c78a4459ee03f628cff0f5401da8dd5a2f)) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 51022ceb..a5f08227 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -160,12 +160,12 @@ hyperi-ci check it once with `uv tool install hyperi-ci` (or `pipx install hyperi-ci`), or run it ad hoc with `uvx hyperi-ci check`. It runs the project's full local validation -- the same suite CI runs -- and reports what to fix. Run it before every push and before opening a -pull request; it is the single best way to give your change the best chance of +pull request. It is the single best way to give your change the best chance of surviving CI. ## CI/CD Workflow -When your pull request is merged to `main`: +When a merge to `main` carries `Release: true` as the last line of its commit body: 1. **semantic-release** analyses commit messages since the last release 2. Determines the next version number based on commit types @@ -173,8 +173,82 @@ When your pull request is merged to `main`: 4. Creates a new GitHub release with release notes 5. Publishes the package (if applicable) -This happens automatically - no manual intervention required. +A merge without the trailer runs CI and releases nothing. `hyperi-ci release` releases the current `main` without a new commit. ## Questions If you have questions about contributing, please open an issue or contact us. + +## Context + +### What this is + +dfe-docker stands the whole DFE suite up as one Docker Compose stack on a single host, and it is the definition-of-done target for the suite's docker path: a change is proven when it runs on the deployment target, not because it ran on someone's laptop. It packages the stack: its release builds the dfe-docker image that sets a folder up from this tree, publishes it with the Compose artefacts and `quickstart.json`, and builds no component image (`make dev` builds those locally, for a test rig). It provisions no ClickHouse table and no Kafka topic. dfe-engine is the only schema and topic controller, and `scripts/tests/test_engine_only_schema_control.py` fails the build on a DDL file or a topic-creating step appearing here. When the data path is wrong the fix is nearly always in a component repo. It is not the Kubernetes path either: Kubernetes stays primary and dfe-infra owns it, which is why there is deliberately no `scale` profile - Compose cannot run an HA broker or a ClickHouse cluster. `main` is protected, so every change arrives as a pull request, with Conventional Commits and a DCO sign-off ([CONTRIBUTING.md](CONTRIBUTING.md)). + +### Where things live + +| Path | What it holds | +|---|---| +| `docker-compose.yml` | Every service and every infrastructure profile. The one file `make ci` names explicitly | +| `compose/` | The opt-in overlays the Makefile chains: `live-ui`, `storage`, `tls` and `container-logs`. It writes the exposure opt-outs and the dev build's local images into `.tmp/` beside them | +| `service_profiles.yaml` | Which DFE services start, the transport and the config each one mounts. Ships `active_profile: single-kafka`, which `make ci` and `make dev` run. `make up` and `make demo` default to the same profile on a clone with no image pins | +| `Makefile` | Every entry point, plus the profile, overlay and UI-exposure resolution that decides which compose files a target passes | +| `scripts/resolve_profile.py` | Reads `service_profiles.yaml` and writes the generated `.tmp/profile.mk` the Makefile includes | +| `scripts/stack.py` | `make stack` - writes `tag@sha256` pins into `.env` from a local dfe-infra checkout (`DFE_INFRA_DIR`) or the signed OCI stack manifest | +| `scripts/post.py` | The power-on self test `make dev` and `make ci` end with | +| `docker/dfe-docker.Dockerfile` + `scripts/installer.py` | The `ghcr.io/hyperi-io/dfe-docker` image and its entrypoint, which copy this tree into a folder and run its scripts there. `.dockerignore` lists what the image carries. `.github/workflows/image.yml` builds it on each release and attaches `quickstart.json` from `scripts/quickstart_json.py` | +| `scripts/test_e2e.py` + `tests/e2e/e2e-tests.yaml` | The declarative e2e suite, one stack per test | +| `scripts/check_*.py` | The gates `make check` runs | +| `scripts/tests/` | Unit tests over the helper scripts, which CI runs beside the static checks | +| `config//` | The config files services mount, one directory per component | +| `.env.example` + `env.example/` | Committed templates. `make init` copies them to `.env` and `env/`, both gitignored | +| `ops/daemon-update/` | The systemd timer that keeps a single-VM deployment on the newest certified stack | +| `docs/` | The guides for evaluating, operating, deploying, developing and troubleshooting the stack, plus profiles, configuration and observability. Read [docs/architecture.md](docs/architecture.md) first | + +### Commands that prove a change + +```bash +make check # every static gate CI runs +make check-tests # pytest over scripts/tests only +make dev # build from source, start, self-test +make ci # pull the pinned registry images, start, self-test +make post # re-run the self test against an already-running stack +make test-e2e # the declarative suite, one stack per test +make down # stop and remove containers across EVERY profile +``` + +`make check` is `check-compose check-hardfail check-dockerfile check-docs check-proxy check-python check-tests check-locks`, and CI runs each one through the same make target so the two cannot drift. + +Two ways a green run says less than it looks: + +- `make check` does not start the stack. It resolves compose, lints the helpers and unit-tests them, and `check-proxy` runs temporary Envoy containers to validate the proxy config. CI's image and published-compose jobs start the stack from the dfe-docker image and check each demo source lands rows, but CI runs neither `make post` nor `make test-e2e`, so run those for a change to the data path. +- `check-compose` validates compose STRUCTURE, not digests. It substitutes placeholders for the mandatory keys so it reads neither the stack SSoT nor the registry, which means no particular pin is proved to resolve. + +`check-hardfail` resolves compose with a scrubbed environment and an empty `--env-file` so a developer's pinned `.env` cannot mask it, requires the run to fail on a missing pin, then reads every `image:` line and requires each to resolve to an `@sha256:` digest. + +### What tends to bite + +| Don't | Do | Why | +|---|---|---| +| Leave containers running when you are finished | `make down`, on every host you touched, the same session | `down` removes containers across every profile, not only the active one, so a previous profile's containers cannot stay up holding host ports and answering health probes (`Makefile`, the `down` target) | +| Assume the deployment target is idle | Check what is already running before you start anything | The docker definition-of-done target carries a long-lived stack refreshed on a 6-hourly systemd timer (`ops/daemon-update/`), not a box you get to yourself | +| Assume events go through Kafka | Read the profile's `transport` first | A `-grpc` profile is `transport: grpc`: the receiver dials `dfe-loader:6000` directly and no broker starts. The `-kafka` profiles, the shipped `single-kafka` among them, start one. The core data path is otherwise identical. `post` and `test-e2e` assert the same landing row either way | +| Add a DDL file or a topic-creating step here | Change the schema in dfe-engine | dfe-engine ships the schemas in its own image and reports healthy only once it has applied them. `scripts/tests/test_engine_only_schema_control.py` checks the deployment trees for the schema and topic bootstrap patterns it knows, with exceptions for the self-test and the e2e cleanup (#118) | +| Give an external ClickHouse's port through `CLICKHOUSE_HTTP_PORT` or `CLICKHOUSE_NATIVE_PORT` | Set `CLICKHOUSE_EXTERNAL_HTTP_PORT` and `CLICKHOUSE_EXTERNAL_NATIVE_PORT` | The first pair is the bundled container's host publish and nothing else, so it is safe to move for a second stack. Services on the network dial 8123/9000 unless the EXTERNAL pair says otherwise (#75) | +| Move the loader's `grpc.listen` off 6000, or confuse it with the receiver's 6000 | Keep the loader on container port 6000 | dfe-engine compiles every sender's loader endpoint as `dfe-loader:6000`, so a loader listening anywhere else drops every record behind an HTTP 202 with nothing logged. The receiver's 6000 is the external Vector protocol, off until `config.grpc.enabled` is set through the engine, because under `--config` the receiver takes only its flat env keys, and unpublished until `DFE_RECEIVER_GRPC_PORT` is set. The loader's is the internal `scalo.transport.v1.Transport/Push` and publishes no host port. `scripts/tests/test_loader_grpc_port.py` fails the build on a mismatch | +| Publish the UIs beyond loopback and leave `DFE_EXTERNAL_ORIGIN` alone | Set it to the address browsers actually use | `DFE_BIND_SCOPE=all` with a loopback origin builds Search's frame-ancestors policy and every next-auth redirect from the wrong host, so the console comes up with its observability views blocked and sends logins to the wrong machine. The Makefile now refuses the combination (#108) | +| Raise `REDPANDA_MEMORY` on its own | Raise `DFE_BROKER_MEMORY` above it | The broker gets at most the container limit less the host's `vm.min_free_kbytes`: `redpanda/start.sh` lowers `--memory` to that and logs a WARN naming the `DFE_BROKER_MEMORY` that restores it (`.env.example:229`, `.env.example:519`) | +| Publish the broker's in-network listener `9092` to the host | Use the host listener on `19092` (`KAFKA_PLAINTEXT_HOST_PORT`) | `9092` advertises `kafka:9092`, a name a host client can't resolve. It also collides with an always-on dev Kafka (#36) | +| Set `DFE_UPDATE_WIPE_STATE=1` on a box that also sets `DFE_DATA_ROOT` | Leave `DFE_DATA_ROOT` unset if you rely on the wipe | `make clean` does not chain the storage overlay, so it removes the volume objects while the bind directories keep their contents and the wipe silently becomes a no-op (`ops/daemon-update/README.md`) | +| Take a local, gitignored note as current | Read the tracked file you are asking about | A local note is nobody's to maintain, and the ones seen so far named repos and directories that no longer exist | + +### Where this sits + +The suite graph in `dfe-infra/suite.yaml` records exactly one edge on this repo and no outbound edges at all. Read it with `dfe-stack suite --consumer dfe-docker` and `--producer dfe-docker`. + +| Repo | Direction | How they interact | +|---|---|---| +| dfe-infra | inbound, `derived-pins` | dfe-docker holds no copy of the pins. `scripts/stack.py` renders them at `make stack` time, from a local dfe-infra checkout when `DFE_INFRA_DIR` names one, else from the signed OCI stack manifest pulled with `oras`. `ops/daemon-update/self_update.py` is the second consumer of that manifest. The edge's declared check is "nothing" - there is no second copy that can drift, and the graph records the edge so the suite tooling knows to walk past it. Do not go hunting for a pin file here to update | +| dfe-receiver, dfe-fetcher, dfe-loader, dfe-archiver, dfe-transform-elastic, dfe-transform-vector, dfe-transform-vrl, dfe-engine, dfe-ui, dfe-hyperdx | inbound | This stack runs their published GHCR images at the digests `make stack` pins, and `make dev` builds the same components from source into local images. Nothing flows the other way | + +Nothing in the suite graph depends on dfe-docker, so a change here breaks a deployment rather than another repo's build. `dfe-transform-splack` and `dfe-transform-wasm` have repos but no service in this stack. diff --git a/Makefile b/Makefile index ff9489ae..51719fd4 100644 --- a/Makefile +++ b/Makefile @@ -7,9 +7,13 @@ # Copyright: (c) 2026 HYPERI PTY LIMITED .DEFAULT_GOAL := help +export PYTHONPYCACHEPREFIX := $(CURDIR)/.tmp/pycache +# The parse writes generated compose files into .tmp/, which a fresh checkout lacks +$(shell mkdir -p .tmp) -# Non-fatal: init creates .env, so it must not exist on a fresh checkout --include .env +# Read only when present, because make builds a missing include before any goal +# and `make help` must not mint secrets. Goals that need .env depend on it. +-include $(wildcard .env) # `include` makes these make-variables, not environment ones, so the helper # scripts (post, test-flows, test-source, test_e2e) read an empty password and @@ -20,73 +24,57 @@ export CLICKHOUSE_PASSWORD export CLICKHOUSE_URL export CLICKHOUSE_USERNAME -# Host UID/GID passed to live-mode containers (docker-compose.live.yml) that -# write to bind-mounted host dirs (dfe-engine config/schemas), so files are owned -# by the host user rather than the image user and writes don't hit permission -# errors. +# Strips the spaces an inline comment leaves before the `#` in a pin, which compose rejects as "invalid reference format" +$(foreach pin,$(filter DFE_%_VERSION DFE_%_VERSION_TAG,$(.VARIABLES)),$(eval $(pin) := $(strip $($(pin))))) + +# The host user, which dfe-ui runs as under LIVE_UI=1, both proxies under console TLS and check-proxy's Envoy always, so what they write or read in the checkout stays its own. Its GID is also the group every stack adds to dfe-engine, which is how it writes env/.custom.env into the checkout export DFE_DEV_UID := $(shell id -u) export DFE_DEV_GID := $(shell id -g) -# Live-edit mode: `make dev LIVE=1` layers docker-compose.live.yml, which -# bind-mounts dfe-engine config/schemas from DFE_SRC_ROOT so edits land without -# a rebuild. COMPOSE_FILE chains the overlay for every plain `docker compose` -# call this make run spawns; the targets that pass -f explicitly (ci, down, -# clean) ignore COMPOSE_FILE by design, so the registry path stays untouched. -# External data location: DFE_DATA_ROOT (env, else .env) rehomes every stateful -# volume onto that path via docker-compose.storage.yml. STORAGE_FLAGS carries it -# onto the explicit `-f` targets (ci, ci-pull), which must relocate storage too. +# DFE_DATA_ROOT (env, else .env) rehomes every stateful volume onto that path via compose/storage.yml. STORAGE_FLAGS carries it onto the targets that pass -f themselves DFE_DATA_ROOT ?= $(shell sed -n 's/^DFE_DATA_ROOT=//p' .env 2>/dev/null) ifneq ($(strip $(DFE_DATA_ROOT)),) export DFE_DATA_ROOT - STORAGE_CHAIN := :docker-compose.storage.yml - STORAGE_FLAGS := -f docker-compose.storage.yml + STORAGE_CHAIN := :compose/storage.yml + STORAGE_FLAGS := -f compose/storage.yml endif -# Goals that work without a resolved service profile. The check-* targets belong -# here because their whole point is running on a fresh checkout -- resolving a -# profile first would make them fail for the reason they exist to detect. -# -# down/clean belong here too: neither references PROFILE_FLAGS or ACTIVE_SERVICES -# any more (both sweep every profile), and requiring a resolvable profile to STOP -# a stack is the same lockout the compose secret comments argue against -- set -# DFE_PROFILE to something that does not exist and you could not tear down. -BOOTSTRAP_GOALS := init env-files creds dev-posture help login stack dial modes down clean limits check check-compose check-hardfail check-dockerfile check-docs check-python check-tests +# Goals that run without a resolved profile, so a fresh checkout can be checked +# and a stack on a broken profile can still be stopped. +BOOTSTRAP_GOALS := init env-files creds certs up demo dev-posture e2e-posture e2e-posture-off e2e-posture-check help stack dial modes down clean demo-stop limits check check-compose check-hardfail check-dockerfile check-docs check-locks check-proxy check-python check-tests print-default-profile print-demo-stream publish-compose # Resolve the active profile only when a goal actually needs the compose stack ifneq (,$(filter-out $(BOOTSTRAP_GOALS),$(or $(MAKECMDGOALS),help))) - include .profile.mk + include .tmp/profile.mk SERVICES ?= ifeq ($(strip $(SERVICES)),) ACTIVE_SERVICES := $(DFE_SERVICES) else - # On a fresh checkout make parses once before .profile.mk exists, then - # remakes it and parses again; the name check only means anything on - # the second pass, when DFE_SERVICES is populated. + # On a fresh checkout make parses once before .tmp/profile.mk exists, then remakes it and parses again, so the name check only means anything on the second pass, when DFE_SERVICES is populated. `apply` skips it: it names instances declared since .tmp/profile.mk was last written, so the make it starts checks them against the re-resolved list instead ifneq ($(strip $(DFE_SERVICES)),) - INVALID_SERVICES := $(filter-out $(DFE_SERVICES),$(SERVICES)) - ifneq ($(INVALID_SERVICES),) - $(error 'SERVICES' contains names not in the resolved stack: $(INVALID_SERVICES). Available: $(DFE_SERVICES)) + ifeq ($(filter apply,$(MAKECMDGOALS)),) + INVALID_SERVICES := $(filter-out $(DFE_SERVICES),$(SERVICES)) + ifneq ($(INVALID_SERVICES),) + $(error 'SERVICES' contains names not in the resolved stack: $(INVALID_SERVICES). Available: $(DFE_SERVICES)) + endif endif endif ACTIVE_SERVICES := $(filter $(SERVICES),$(DFE_SERVICES)) endif - # `make dev LOCAL="dfe-engine dfe-ui"` builds only those and keeps the rest - # on the registry pins: the build writes docker-compose.local.yml for the - # built components and the compose call names its files explicitly, which - # keeps the auto-loaded all-local override out. Empty LOCAL is plain dev. + # The overlay every dev compose call names, which the build writes to put what it built on `:local`. LOCAL="dfe-engine dfe-ui" builds only those LOCAL ?= - LOCAL_OVERLAY := docker-compose.local.yml - # The pull runs before the build writes any overlay, so it names none either - # way and resolves every image to its registry pin. The auto-loaded override - # otherwise sends it after a `:local` tag nothing has built yet, reached - # through an image consumer (dfe-hunt-runner runs dfe-engine's image), so it - # bites even when the profile holds no DFE service of its own. - # DEV_PULL_FLAGS and DEV_FLAGS expand at use time: UI_FLAGS is filled below. - DEV_PULL_FLAGS = -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) + LOCAL_OVERLAY := .tmp/docker-compose.local.yml + # Empty until a build writes it, so no dev chain names a missing file + $(if $(wildcard $(LOCAL_OVERLAY)),,$(file > $(LOCAL_OVERLAY),services: {})) + # Expanded at use time, since UI_FLAGS is filled below. The pull runs before the build, so it names no overlay and every image resolves to its registry pin + DEV_PULL_FLAGS = -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) $(LIVE_UI_FLAGS) + DEV_FLAGS = -f docker-compose.yml -f $(LOCAL_OVERLAY) $(STORAGE_FLAGS) $(UI_FLAGS) $(LIVE_UI_FLAGS) + # `LIVE_UI=1` runs dfe-ui from the checkout, so there is no dfe-ui image to build. + LIVE_UI_SKIP := $(if $(strip $(LIVE_UI)),dfe-ui) ifeq ($(strip $(LOCAL)),) - DEV_BUILD := $(ACTIVE_SERVICES) - DEV_FLAGS := - DEV_OVERLAY_ARG := + # DFE_SERVICES names a per-source app's instances and never the app, so the apps the profile runs per source are built beside it. The build records what it made in the overlay, which `make apply` reads to keep an instance on its local image + DEV_BUILD := $(filter-out $(LIVE_UI_SKIP),$(ACTIVE_SERVICES)) $(DFE_PER_SOURCE_APPS) + DEV_OVERLAY_ARG := --record $(LOCAL_OVERLAY) else # Validated against the buildable set rather than the resolved stack: a # name the builder skips builds nothing and leaves nothing to overlay. @@ -96,48 +84,24 @@ ifneq (,$(filter-out $(BOOTSTRAP_GOALS),$(or $(MAKECMDGOALS),help))) $(error 'LOCAL' contains names this repo cannot build from source: $(UNBUILDABLE_LOCAL). Buildable: $(DFE_BUILDABLE_SERVICES)) endif endif + # DFE_SERVICES names a per-source app's instances and never the app, so the apps the profile runs per source are named beside it ifneq ($(strip $(DFE_SERVICES)),) - INVALID_LOCAL := $(filter-out $(DFE_SERVICES),$(LOCAL)) + INVALID_LOCAL := $(filter-out $(DFE_SERVICES) $(DFE_PER_SOURCE_APPS),$(LOCAL)) ifneq ($(INVALID_LOCAL),) - $(error 'LOCAL' contains names not in the resolved stack: $(INVALID_LOCAL). Available: $(DFE_SERVICES)) + $(error 'LOCAL' contains names not in the resolved stack: $(INVALID_LOCAL). Available: $(DFE_SERVICES) $(DFE_PER_SOURCE_APPS)) endif endif - DEV_BUILD := $(LOCAL) - # The local overlay takes the committed override's slot, keeping the - # default chain's relative order so `LIVE=1` resolves the engine volumes - # the same way with and without LOCAL. - DEV_FLAGS = -f docker-compose.yml -f $(LOCAL_OVERLAY) - ifneq ($(strip $(LIVE)),) - DEV_FLAGS += -f docker-compose.live.yml + ifneq ($(filter $(LIVE_UI_SKIP),$(LOCAL)),) + $(error LIVE_UI=1 runs dfe-ui from your checkout with `next dev`, so there is no dfe-ui image to build. Drop dfe-ui from LOCAL) endif - DEV_FLAGS += $(STORAGE_FLAGS) $(UI_FLAGS) + DEV_BUILD := $(LOCAL) DEV_OVERLAY_ARG := --overlay $(LOCAL_OVERLAY) endif endif # --------------------------------------------------------------------------- -# Web UI exposure -# -# Two dials, because "external" means different things on the two boxes compose -# deploys to. DFE_BIND_SCOPE=localhost (the default, a developer workstation) -# publishes every UI on 127.0.0.1 so the host sees them and the LAN does not; -# DFE_BIND_SCOPE=all publishes on 0.0.0.0 for a small/VM deploy. It resolves to -# DFE_UI_BIND_HOST, the address compose interpolates into every UI port mapping. -# Ingest (DFE_INGRESS_BIND_HOST) and the operator ports (DFE_BIND_HOST) keep -# their own audiences and are untouched by it. -# -# Every UI publishes by default. An opt-out drops that service's ports mapping, -# which compose merging cannot do by override -- hence a `!reset` fragment per -# UI, chained here the same way STORAGE_CHAIN chains the storage overlay. -# DFE_INFRA_UIS_EXTERNAL=false is the kill switch: it unpublishes every -# INFRA-class UI at once and beats their individual flags. dfe-ui and the engine -# API are PRODUCT class and answer only to their own. -# -# Below the profile block because DFE_AUTH_RESOLVED comes from .profile.mk, so -# the profile key and the env var can never disagree about whether the auth -# proxies are in play. The bootstrap goals skip that include and read it empty, -# which is the auth-off branch -- the safe direction for a target that only -# stops a stack or checks a file. +# Web UI exposure: docs/operating.md, "Web UIs: one scope dial, one kill switch". +# Below the profile block, because DFE_AUTH_RESOLVED comes from .tmp/profile.mk. # --------------------------------------------------------------------------- DFE_BIND_SCOPE ?= localhost ifeq ($(strip $(DFE_BIND_SCOPE)),localhost) @@ -148,28 +112,66 @@ else $(error DFE_BIND_SCOPE must be `localhost` or `all`, got '$(strip $(DFE_BIND_SCOPE))') endif -# `all` publishes the UIs on addresses this box cannot infer, and two services -# build absolute URLs from DFE_EXTERNAL_ORIGIN -- dfe-proxy's frame-ancestors -# policy for the embedded HyperDX views, and next-auth's redirects. At loopback -# both name the browser's own machine, so the console comes up with its -# observability views blocked. -# -# Only the goals that start or self-test the stack ask for it. The bootstrap -# goals are exempt for the reason BOOTSTRAP_GOALS gives: needing a key to stop a -# stack, or to check a file, is a lockout. -ORIGIN_GOALS := dev ci up infra post test-source test-flows -# Origins a browser on another machine cannot use; scripts/_common.py keeps the -# same set for the helpers that read the key. -LOOPBACK_ORIGINS := http://localhost https://localhost http://127.0.0.1 https://127.0.0.1 +# DFE_BIND_SCOPE=all needs DFE_EXTERNAL_ORIGIN on the goals that start or self-test +# the stack: docs/configuration.md, "External origin". +ORIGIN_GOALS := dev ci up apply infra post test-source test-flows print-compose-env +# The e2e suites start a stack per test and probe the console at DFE_EXTERNAL_ORIGIN's host under console TLS, so there they need what a start goal needs. +E2E_GOALS := test-e2e test-resilience +ifeq ($(strip $(DFE_PROXY_TLS)),true) + ORIGIN_GOALS += $(E2E_GOALS) +endif +# Origins a browser on another machine cannot use, the wildcard bind address included. +# scripts/_common.py keeps the same set for the helpers that read the key. +LOOPBACK_ORIGINS := http://localhost https://localhost http://127.0.0.1 https://127.0.0.1 http://[::1] https://[::1] http://0.0.0.0 https://0.0.0.0 ifeq ($(strip $(DFE_BIND_SCOPE)),all) ifneq (,$(filter $(ORIGIN_GOALS),$(or $(MAKECMDGOALS),help))) ORIGIN_UNUSABLE := $(if $(strip $(DFE_EXTERNAL_ORIGIN)),$(filter $(strip $(DFE_EXTERNAL_ORIGIN)),$(LOOPBACK_ORIGINS)),unset) ifneq ($(ORIGIN_UNUSABLE),) - $(error DFE_BIND_SCOPE=all publishes the UIs beyond loopback, so DFE_EXTERNAL_ORIGIN must name the address browsers use -- it is $(if $(strip $(DFE_EXTERNAL_ORIGIN)),'$(strip $(DFE_EXTERNAL_ORIGIN))',unset). HyperDX's frame-ancestors policy and every next-auth redirect are built from it, so the console loads with its observability views blocked and sends logins back to the wrong host. Set DFE_EXTERNAL_ORIGIN=http:// in .env or the environment, or leave DFE_BIND_SCOPE=localhost) + $(error DFE_BIND_SCOPE=all publishes the UIs beyond loopback, so DFE_EXTERNAL_ORIGIN must name the address browsers use -- it is $(if $(strip $(DFE_EXTERNAL_ORIGIN)),'$(strip $(DFE_EXTERNAL_ORIGIN))',unset). dfe-hyperdx's frame-ancestors policy and every next-auth redirect are built from it, so the console loads with its observability views blocked and sends logins back to the wrong host. Set DFE_EXTERNAL_ORIGIN=http:// in .env or the environment, or leave DFE_BIND_SCOPE=localhost) endif endif endif +# Compose appends `:` to each of these (next-auth's NEXTAUTH_URL, dfe-hyperdx's FRONTEND_URL, the oauth2-proxy callbacks), so a value that already carries a port, a path or a query builds a URL no browser can parse: dfe-ui answers 500 on `https://host:3000:3000`. DFE_EXTERNAL_ORIGIN and DFE_HYPERDX_APP_URL also feed dfe-proxy's frame-ancestors source list, where a `;` in the value starts a new directive. Each must be a scheme and a host only, on every stack and with TLS on or off. +APPENDED_ORIGINS := DFE_EXTERNAL_ORIGIN DFE_HYPERDX_APP_URL DFE_OAUTH2_PROXY_EXTERNAL_ORIGIN +# The value without the surrounding quotes compose strips and make keeps. +origin_value = $(patsubst "%",%,$(patsubst '%',%,$(strip $($(1))))) +# What follows the scheme. +origin_rest = $(patsubst https://%,%,$(patsubst http://%,%,$(call origin_value,$(1)))) +# The characters a host is made of: letters, digits, dot, hyphen and underscore in a name or an IPv4 address, hex digits, colon and dot inside the brackets of an IPv6 one. +ORIGIN_NAME_CHARS := a b c d e f g h i j k l m n o p q r s t u v w x y z A B C D E F G H I J K L M N O P Q R S T U V W X Y Z 0 1 2 3 4 5 6 7 8 9 . - _ +ORIGIN_V6_CHARS := a b c d e f A B C D E F 0 1 2 3 4 5 6 7 8 9 : . +# The text with every character in the list removed, so what is left is what the list does not allow. +origin_strip = $(if $(2),$(call origin_strip,$(subst $(firstword $(2)),,$(1)),$(wordlist 2,$(words $(2)),$(2))),$(1)) +# The characters of one host that no host holds, empty when it is all host. A bracketed host must hold a colon, as every IPv6 address does. +origin_bad = $(if $(filter [%],$(1)),$(if $(findstring :,$(1)),$(call origin_strip,$(patsubst [%],%,$(1)),$(ORIGIN_V6_CHARS)),$(1)),$(call origin_strip,$(1),$(ORIGIN_NAME_CHARS))) +# The same for everything after the scheme, once the checks for a path, a query and a port have passed. +origin_stray = $(call origin_bad,$(call origin_rest,$(1))) +# What is wrong with one origin, empty when it is a scheme and a host or unset. A bracketed IPv6 host keeps its colons. +origin_fault = $(if $(call origin_value,$(1)),$(strip \ + $(if $(word 2,$(call origin_value,$(1))),contains a space,\ + $(if $(filter http://% https://%,$(call origin_value,$(1))),\ + $(if $(call origin_rest,$(1)),\ + $(if $(findstring /,$(call origin_rest,$(1))),carries a path (a trailing slash counts),\ + $(if $(findstring ?,$(call origin_rest,$(1))),carries a query,\ + $(if $(and $(findstring :,$(call origin_rest,$(1))),$(if $(filter [%],$(call origin_rest,$(1))),,colon)),carries a port,\ + $(if $(call origin_stray,$(1)),carries characters a host cannot hold: `$(call origin_stray,$(1))`)))),\ + names no host),\ + has no http:// or https:// scheme)))) +# The host the value names, a bracketed IPv6 one whole. +origin_host = $(if $(filter [%,$(call origin_rest,$(1))),$(firstword $(subst ], ,$(call origin_rest,$(1))))],$(firstword $(subst /, ,$(subst ?, ,$(subst :, ,$(call origin_rest,$(1))))))) +# The scheme and host the value meant: its own scheme, else https under console TLS and http without. A host with a character no host holds is not repeated back. +origin_fix = $(if $(filter http://%,$(call origin_value,$(1))),http,$(if $(or $(filter https://%,$(call origin_value,$(1))),$(filter true,$(strip $(DFE_PROXY_TLS)))),https,http))://$(or $(if $(call origin_bad,$(call origin_host,$(1))),,$(call origin_host,$(1))),) +ifneq (,$(filter $(ORIGIN_GOALS),$(or $(MAKECMDGOALS),help))) + ORIGIN_MALFORMED := $(firstword $(foreach key,$(APPENDED_ORIGINS),$(if $(call origin_fault,$(key)),$(key)))) + # Why a fault matters: a port or path breaks the URLs compose builds, a stray character can rewrite the frame-ancestors policy dfe-proxy builds from the same value. + ORIGIN_WHY_PORT = compose appends the port itself (DFE_UI_PORT for the console), so this value builds $(call origin_value,$(ORIGIN_MALFORMED)):$(or $(strip $(DFE_UI_PORT)),3000) and the console answers 500 + ORIGIN_WHY_STRAY = compose writes it into URLs and into dfe-proxy's frame-ancestors source list, where a `;` starts a new directive, so a character no host holds can rewrite the policy + ifneq ($(ORIGIN_MALFORMED),) + $(error $(ORIGIN_MALFORMED)='$(call origin_value,$(ORIGIN_MALFORMED))' $(call origin_fault,$(ORIGIN_MALFORMED)). It must be a scheme and a host only, no port: $(if $(findstring carries characters,$(call origin_fault,$(ORIGIN_MALFORMED))),$(ORIGIN_WHY_STRAY),$(ORIGIN_WHY_PORT)). Set $(ORIGIN_MALFORMED)=$(call origin_fix,$(ORIGIN_MALFORMED))) + endif +endif + DFE_INFRA_UIS_EXTERNAL ?= true DFE_UI_EXTERNAL ?= true DFE_ENGINE_API_EXTERNAL ?= true @@ -197,59 +199,197 @@ endef UI_CHAIN := UI_FLAGS := + +# The services whose host ports the exposure dials take away. dfe-proxy carries the console and dfe-hyperdx-proxy both dfe-hyperdx origins +UNPUBLISHED := ifeq ($(strip $(DFE_UI_EXTERNAL)),false) - $(eval $(call chain_fragment,docker-compose.unpublish-dfe-ui.yml)) + UNPUBLISHED += dfe-proxy endif ifeq ($(strip $(DFE_ENGINE_API_EXTERNAL)),false) - $(eval $(call chain_fragment,docker-compose.unpublish-engine-api.yml)) + UNPUBLISHED += dfe-engine endif # Infra class. With the auth profile armed each infra UI moves BEHIND its # oauth2-proxy, so its own port is unpublished either way and the gate lands on # the proxy instead. ifeq ($(strip $(DFE_AUTH_RESOLVED)),true) - $(eval $(call chain_fragment,docker-compose.unpublish-kafbat.yml)) - $(eval $(call chain_fragment,docker-compose.unpublish-hyperdx.yml)) + UNPUBLISHED += kafka-ui dfe-hyperdx-proxy ifneq ($(KAFBAT_GATED),) - $(eval $(call chain_fragment,docker-compose.unpublish-auth-kafbat.yml)) + UNPUBLISHED += oauth2-proxy-kafbat endif ifneq ($(HYPERDX_GATED),) - $(eval $(call chain_fragment,docker-compose.unpublish-auth-hyperdx.yml)) + UNPUBLISHED += oauth2-proxy-hyperdx oauth2-proxy-hyperdx-api endif else ifneq ($(KAFBAT_GATED),) - $(eval $(call chain_fragment,docker-compose.unpublish-kafbat.yml)) + UNPUBLISHED += kafka-ui endif ifneq ($(HYPERDX_GATED),) - $(eval $(call chain_fragment,docker-compose.unpublish-hyperdx.yml)) + UNPUBLISHED += dfe-hyperdx-proxy endif endif +# The line break $(file) writes between a fragment's lines +define newline + + +endef + +# The console's dfe-hyperdx embed goes with dfe-hyperdx, auth or not, keeping the console's line. dfe-proxy's own reset drops both and a second key would be duplicate YAML +EMBED_UNPUBLISHED := $(if $(filter dfe-proxy,$(UNPUBLISHED)),,$(HYPERDX_GATED)) + +ifneq ($(strip $(UNPUBLISHED))$(EMBED_UNPUBLISHED),) + $(file > .tmp/docker-compose.unpublish.yml,# Generated by make$(newline)services:) + $(foreach service,$(UNPUBLISHED),$(file >> .tmp/docker-compose.unpublish.yml, $(service):$(newline) ports: !reset [])) + ifneq ($(EMBED_UNPUBLISHED),) + $(file >> .tmp/docker-compose.unpublish.yml, dfe-proxy:$(newline) ports: !override$(newline) - $${DFE_UI_BIND_HOST:-127.0.0.1}:$${DFE_UI_PORT:-3000}:8080) + endif + $(eval $(call chain_fragment,.tmp/docker-compose.unpublish.yml)) +endif + +# A receiver listener's host port publishes once its key is set. The listener itself is enabled in the receiver's config +RECEIVER_LISTENERS := BEATS:5044 GRPC:6000 HEC:8088 OTLP_GRPC:4317 OTLP_HTTP:4318 SYSLOG_UDP:514/udp SYSLOG_TCP:514 SYSLOG_TLS:6514 NETFLOW:2055/udp IPFIX:4739/udp SFLOW:6343/udp +RECEIVER_PORTS_SET := $(strip $(foreach listener,$(RECEIVER_LISTENERS),$(if $(strip $(DFE_RECEIVER_$(firstword $(subst :, ,$(listener)))_PORT)),$(listener)))) + +ifneq ($(RECEIVER_PORTS_SET),) + $(file > .tmp/docker-compose.receiver-ports.yml,# Generated by make$(newline)services:$(newline) dfe-receiver:$(newline) ports:) + $(foreach listener,$(RECEIVER_PORTS_SET),$(file >> .tmp/docker-compose.receiver-ports.yml, - $${DFE_INGRESS_BIND_HOST:-0.0.0.0}:$${DFE_RECEIVER_$(firstword $(subst :, ,$(listener)))_PORT}:$(lastword $(subst :, ,$(listener))))) + $(eval $(call chain_fragment,.tmp/docker-compose.receiver-ports.yml)) +endif + +# A host directory mounts read only into dfe-fetcher once DFE_FETCHER_INPUT_DIR is set, for a `sources.file` source to read. `create_host_path: false` makes a mistyped path fail the start instead of mounting an empty directory Docker made +ifneq ($(strip $(DFE_FETCHER_INPUT_DIR)),) + $(file > .tmp/docker-compose.fetcher-input.yml,# Generated by make$(newline)services:$(newline) dfe-fetcher:$(newline) volumes:$(newline) - type: bind$(newline) source: $${DFE_FETCHER_INPUT_DIR}$(newline) target: /mnt/input$(newline) read_only: true$(newline) bind:$(newline) create_host_path: false) + $(eval $(call chain_fragment,.tmp/docker-compose.fetcher-input.yml)) +endif + # One service per app instance dfe-engine has rendered a config for, generated # with the profile (scripts/instances.py). Empty for the bootstrap goals, which # reads as off -- the safe direction for a target that only stops a stack. ifeq ($(strip $(DFE_INSTANCES_RESOLVED)),true) - $(eval $(call chain_fragment,docker-compose.instances.yml)) + $(eval $(call chain_fragment,.tmp/docker-compose.instances.yml)) endif -# Container stdout to the collector. Off wherever the collector is, because the -# fluentd driver cannot be read back by `docker compose logs` and would cost an -# operator that with nothing collecting at the other end. Empty for the bootstrap -# goals, which reads as off -- the safe direction for a target that only stops a -# stack or checks a file. +# Container stdout to the collector only where one runs, since the driver would queue lines for an endpoint nothing listens on (the bootstrap goals resolve empty, which reads as off) DFE_CONTAINER_LOGS_ENABLED ?= true ifeq ($(strip $(DFE_OTEL_RESOLVED))-$(strip $(DFE_CONTAINER_LOGS_ENABLED)),true-true) - $(eval $(call chain_fragment,docker-compose.container-logs.yml)) + $(eval $(call chain_fragment,compose/container-logs.yml)) +endif + +# LIVE_UI=1 runs dfe-ui as `next dev` on the dev goals only (docs/live-console.md). It chains after the container-logs fragment because it puts dfe-ui back on the default log driver. +# The Node image comes from the checkout's own Dockerfile rather than a second pin here, read without a shell so a path with a quote in it stays a path. +LIVE_UI_GOALS := dev dev-build apply apply-services +ifneq ($(strip $(LIVE_UI)),) + LIVE_UI_DOCKERFILE := $(firstword $(wildcard $(call origin_value,DFE_SRC_ROOT)/dfe-ui/Dockerfile)) + DFE_UI_NODE_IMAGE := $(if $(LIVE_UI_DOCKERFILE),$(firstword $(patsubst NODE_IMAGE=%,%,$(filter NODE_IMAGE=%,$(file < $(LIVE_UI_DOCKERFILE)))))) + export DFE_UI_NODE_IMAGE + # Next lets hot reload through only from localhost or the host it serves on. It can only serve on a name, so an IP or loopback origin keeps the all-interfaces bind. + LIVE_UI_ORIGIN_HOST := $(call origin_host,DFE_EXTERNAL_ORIGIN) + LIVE_UI_ORIGIN_NAME := $(and $(filter-out localhost [%,$(LIVE_UI_ORIGIN_HOST)),$(subst .,,$(subst 0,,$(subst 1,,$(subst 2,,$(subst 3,,$(subst 4,,$(subst 5,,$(subst 6,,$(subst 7,,$(subst 8,,$(subst 9,,$(LIVE_UI_ORIGIN_HOST))))))))))))) + export DFE_UI_DEV_HOST := $(if $(LIVE_UI_ORIGIN_NAME),$(LIVE_UI_ORIGIN_HOST),0.0.0.0) + LIVE_UI_CHAIN := :compose/live-ui.yml + LIVE_UI_FLAGS := -f compose/live-ui.yml + # Refused here rather than at compose interpolation, which `make dev` reaches only after `down` has stopped the stack. + ifneq (,$(filter $(LIVE_UI_GOALS),$(MAKECMDGOALS))) + ifeq ($(strip $(DFE_UI_NODE_IMAGE)),) + $(error LIVE_UI=1 reads the Node image from DFE_SRC_ROOT/dfe-ui/Dockerfile, but $(if $(LIVE_UI_DOCKERFILE),$(LIVE_UI_DOCKERFILE) declares no ARG NODE_IMAGE,DFE_SRC_ROOT='$(call origin_value,DFE_SRC_ROOT)' holds no dfe-ui/Dockerfile). Set DFE_SRC_ROOT to the directory holding your dfe-ui checkout) + endif + endif +endif + +# --------------------------------------------------------------------------- +# Console TLS (opt-in): docs/configuration.md, "Console TLS (opt-in)". +# scripts/proxy_tls.py does the subnet arithmetic make cannot. +# --------------------------------------------------------------------------- +# Exactly `true` or `false`, so a `True` or a `1` fails here rather than quietly serving plain http. Empty reads as false. +DFE_PROXY_TLS ?= false +ifneq ($(filter-out true false,$(strip $(DFE_PROXY_TLS)))$(word 2,$(DFE_PROXY_TLS)),) + $(error DFE_PROXY_TLS must be `true` or `false`, got '$(strip $(DFE_PROXY_TLS))') +endif + +# Set before the dial so `make certs` writes where a later DFE_PROXY_TLS=true start reads +DFE_PROXY_CERT_DIR ?= ./certs +# The directory goes into a quoted shell argument, so one holding a quote is refused before the shell sees it +CERT_DIR_REFUSAL = $(if $(findstring ',$(DFE_PROXY_CERT_DIR)),$(error DFE_PROXY_CERT_DIR='$(DFE_PROXY_CERT_DIR)' holds a single quote, which ends the quoted shell argument it is passed in. Set DFE_PROXY_CERT_DIR to a path without one)) +ifeq ($(strip $(DFE_PROXY_TLS)),true) + # Private space outside Docker's stock pools (172.17-31/16, 192.168/20) and the fleet's 172.16.0.0/12 pool. + DFE_NETWORK_SUBNET ?= 10.207.0.0/24 + export DFE_NETWORK_SUBNET + export DFE_PROXY_CERT_DIR + PROXY_NETWORK := $(shell python3 scripts/proxy_tls.py network '$(strip $(DFE_NETWORK_SUBNET))') + export DFE_PROXY_IP := $(word 1,$(PROXY_NETWORK)) + export DFE_NETWORK_IP_RANGE := $(word 2,$(PROXY_NETWORK)) + $(eval $(call chain_fragment,compose/tls.yml)) +endif + +# Only the start goals need the certificate, for the reason ORIGIN_GOALS gives. The auth refusal is in the .tmp/profile.mk recipe instead, because it reads DFE_AUTH_RESOLVED and this pass may still hold the previous run's +PROXY_TLS_START := $(and $(filter true,$(strip $(DFE_PROXY_TLS))),$(filter $(ORIGIN_GOALS),$(or $(MAKECMDGOALS),help))) +ifneq ($(PROXY_TLS_START),) + ifeq ($(filter https://%,$(call origin_value,DFE_EXTERNAL_ORIGIN)),) + $(error DFE_PROXY_TLS=true serves the console over https only, so DFE_EXTERNAL_ORIGIN must start with https:// and it is $(if $(strip $(DFE_EXTERNAL_ORIGIN)),'$(strip $(DFE_EXTERNAL_ORIGIN))',unset). next-auth builds every redirect from it and the proxy no longer answers plain http. Set DFE_EXTERNAL_ORIGIN=https://) + endif + ifneq ($(call origin_value,DFE_HYPERDX_APP_URL),) + ifeq ($(filter https://%,$(call origin_value,DFE_HYPERDX_APP_URL)),) + $(error DFE_PROXY_TLS=true serves dfe-hyperdx over https, so DFE_HYPERDX_APP_URL must start with https:// or be unset. It is '$(strip $(DFE_HYPERDX_APP_URL))') + endif + endif +endif + +# dfe-proxy's TLS ingest listener (config/proxy/envoy.tls.yaml) publishes once its key is set. The plain config has no such listener, so the key without the dial is refused on the goals that start a stack +ifneq ($(strip $(DFE_RECEIVER_HTTPS_PORT)),) + ifeq ($(strip $(DFE_PROXY_TLS)),true) + $(file > .tmp/docker-compose.receiver-https.yml,# Generated by make$(newline)services:$(newline) dfe-proxy:$(newline) ports:$(newline) - $${DFE_INGRESS_BIND_HOST:-0.0.0.0}:$${DFE_RECEIVER_HTTPS_PORT}:8443) + $(eval $(call chain_fragment,.tmp/docker-compose.receiver-https.yml)) + else ifneq (,$(filter $(ORIGIN_GOALS),$(MAKECMDGOALS))) + $(error DFE_RECEIVER_HTTPS_PORT='$(strip $(DFE_RECEIVER_HTTPS_PORT))' publishes dfe-proxy's TLS ingest listener, which only exists under DFE_PROXY_TLS=true. Set DFE_PROXY_TLS=true and run `make certs`, or unset DFE_RECEIVER_HTTPS_PORT. Plain http ingest stays on DFE_RECEIVER_HTTP_PORT) + endif endif -ifneq ($(strip $(LIVE)),) - export COMPOSE_FILE := docker-compose.yml:docker-compose.override.yml:docker-compose.live.yml$(STORAGE_CHAIN)$(UI_CHAIN) -else ifneq ($(strip $(STORAGE_CHAIN))$(strip $(UI_CHAIN)),) - export COMPOSE_FILE := docker-compose.yml:docker-compose.override.yml$(STORAGE_CHAIN)$(UI_CHAIN) +# These goals recreate services on the network that exists, with no `make down` first, and the e2e suites keep ClickHouse and its network between tests. Docker cannot re-address a live network, so a stack whose network predates a DFE_PROXY_TLS flip would leave every recreated service Exited. +NETWORK_REUSE_GOALS := apply apply-services infra $(E2E_GOALS) +PROXY_NETWORK_REUSE := $(filter $(NETWORK_REUSE_GOALS),$(or $(MAKECMDGOALS),help)) +# Compose names the default network after the project, which is COMPOSE_PROJECT_NAME or this directory. +PROXY_PROJECT := $(or $(strip $(COMPOSE_PROJECT_NAME)),$(notdir $(CURDIR))) +# The certificate pair, the subnet and the network as Docker holds it now. The helper prints the first problem, and nothing when there is none. +# A helper that exits non-zero printed no problem because it checked nothing, so that exit becomes the problem and the start is refused. +ifneq ($(PROXY_TLS_START)$(PROXY_NETWORK_REUSE),) + $(CERT_DIR_REFUSAL) + PROXY_TLS_PROBLEM := $(shell python3 scripts/proxy_tls.py precheck $(if $(PROXY_TLS_START),--certs --overlap) $(if $(PROXY_NETWORK_REUSE),--toggle) --cert-dir '$(DFE_PROXY_CERT_DIR)' --project '$(PROXY_PROJECT)' --subnet '$(strip $(DFE_NETWORK_SUBNET))' --tls $(if $(filter true,$(strip $(DFE_PROXY_TLS))),true,false) || echo "scripts/proxy_tls.py precheck failed before it could check the console TLS setup, so the start is refused rather than assumed safe. Its error is printed above") + ifneq ($(strip $(PROXY_TLS_PROBLEM)),) + $(error $(PROXY_TLS_PROBLEM)) + endif endif -.profile.mk: FORCE - @python3 scripts/resolve_profile.py +# The host goes into a quoted shell argument, so one holding a character no host holds is refused before the shell sees it +.PHONY: certs +certs: ## Mint a console TLS certificate for DFE_EXTERNAL_ORIGIN's host, localhost and 127.0.0.1 into DFE_PROXY_CERT_DIR, with a single-use CA to trust once. Never replaces a file already there + @$(if $(call origin_bad,$(call origin_host,DFE_EXTERNAL_ORIGIN)),$(error DFE_EXTERNAL_ORIGIN='$(call origin_value,DFE_EXTERNAL_ORIGIN)' carries characters a host cannot hold: `$(call origin_bad,$(call origin_host,DFE_EXTERNAL_ORIGIN))`. Set DFE_EXTERNAL_ORIGIN=$(call origin_fix,DFE_EXTERNAL_ORIGIN))) + @$(CERT_DIR_REFUSAL) + @python3 scripts/proxy_tls.py mint --cert-dir '$(DFE_PROXY_CERT_DIR)' --host '$(or $(call origin_host,DFE_EXTERNAL_ORIGIN),localhost)' + +# What every plain `docker compose` call this make run spawns reads. The targets that pass -f themselves ignore it +ifneq ($(strip $(STORAGE_CHAIN))$(strip $(UI_CHAIN))$(strip $(LIVE_UI_CHAIN)),) + export COMPOSE_FILE := docker-compose.yml$(STORAGE_CHAIN)$(UI_CHAIN)$(LIVE_UI_CHAIN) +endif + +# The gated infra UIs go to the resolver, so the engine's admin links name only +# what the host publishes. The per-source apps built locally go too, because the overlay never reaches an instance's `extends`: what LOCAL names, else every one the profile runs on `make dev`, else what the last dev build recorded on any goal but `make ci`. +# +# Console TLS refuses the auth profile here, after the resolver has written this run's answer: the oauth2-proxy ports serve no TLS, so an https origin gives them callbacks they cannot answer. +ifneq ($(strip $(LOCAL)),) + RESOLVE_LOCAL_ARGS = $(foreach app,$(LOCAL),--local $(app)) +else ifneq ($(filter dev dev-build,$(MAKECMDGOALS)),) + RESOLVE_LOCAL_ARGS = --local-per-source +else ifeq ($(filter ci,$(MAKECMDGOALS)),) + RESOLVE_LOCAL_ARGS = --local-overlay $(LOCAL_OVERLAY) +endif +# The goals that start or recreate services, or hand a plain `docker compose` what they would, the only ones that ask docker compose for the CPU limits the collector reports as cores deployed +USAGE_GOALS := dev ci up infra apply apply-services print-compose-env +.tmp/profile.mk: FORCE + @python3 scripts/resolve_profile.py $(if $(KAFBAT_GATED),--unpublished kafbat) $(if $(HYPERDX_GATED),--unpublished hyperdx) $(RESOLVE_LOCAL_ARGS) $(if $(filter $(USAGE_GOALS),$(MAKECMDGOALS)),--usage) +ifneq ($(PROXY_TLS_START),) + @if grep -qx 'export DFE_AUTH_RESOLVED := true' $@; then echo "DFE_PROXY_TLS=true cannot start with the auth profile: oauth2-proxy serves no TLS here, so the https callbacks it registers with the IdP never answer. Set DFE_AUTH_ENABLED=false (or drop auth from the profile) to keep TLS. Set DFE_PROXY_TLS=false to keep the auth profile" >&2; exit 1; fi +endif .PHONY: FORCE FORCE: @@ -266,29 +406,19 @@ init: ## Create .env and per-service env/.env files from templates, and @python3 scripts/init.py @python3 scripts/creds.py --write -# Start targets require every per-service env file (dfe-ui reads INTERNAL_API_URL -# from env/ui.env). Compose marks them optional so `make down` never needs them. -# The guard creates what is missing (init is non-destructive) rather than refusing, -# so a release that adds a template does not stop an initialised deployment. +# Start targets need every env/.env, so this creates any a release +# added. Compose marks them optional so `make down` never needs them. .PHONY: env-files -env-files: ## Assert every env/.env exists, creating any the templates have gained +env-files: .env ## Assert every env/.env exists, creating any the templates have gained @python3 scripts/env_files.py -# `make init` mints the admin and break-glass passwords and prints neither, so -# this is the hand-over. `make up` and `make dev` end with it; `make ci` does not -# call it, because its stdout is a build log. `make init` and `make up` pass -# --write, which also leaves access-summary.md (0600, gitignored) for the operator. +# The hand-over of the minted logins: docs/developing.md, "Logging in the first time". .PHONY: creds -creds: ## Print the access summary. The admin password prints on a TTY only -- a pipe, a file, a CI log or DFE_CREDS_SHOW=0 gets the .env key instead +creds: ## Print the access summary. The admin password prints on a TTY only -- a pipe, a file, a CI log or DFE_CREDS_SHOW=0 gets the env/engine.env key instead @python3 scripts/creds.py -# A dev tyre-kick logs in without looking anything up, so `make dev` writes the -# KNOWN default password and DFE_ENV=dev, the one posture the engine accepts it in. -# It refuses with exit 2 on any other DFE_ENV, and copies .env to .env.bak- -# before overwriting a minted password. -# -# `AUTH=real` is the other half: local images running the same authentication flow -# a deployment gets, so it mints a password and writes a non-dev posture instead. +# `make dev` writes the known default password and DFE_ENGINE_ENV=dev, and +# AUTH=real a minted one: docs/developing.md, "Logging in the first time". AUTH ?= ifeq ($(strip $(AUTH)),real) DEV_POSTURE_ARG := --real @@ -297,89 +427,125 @@ else endif .PHONY: dev-posture -dev-posture: .env ## Put .env into the dev posture (known admin password, DFE_ENV=dev); AUTH=real mints one and writes a non-dev posture instead +dev-posture: env-files ## Put .env and env/engine.env into the dev posture (DFE_ENGINE_ENV=dev, known admin password until the stack replaces it); AUTH=real mints one and writes a non-dev posture instead @python3 scripts/dev_posture.py $(DEV_POSTURE_ARG) -# GHCR auth for the private dfe-* images and the signed stack-manifest. A no-op -# when DFE_GHCR_* are unset (a daemon authed out of band), so it is safe as an -# unconditional prerequisite. The helper reads .env itself and pipes the token on -# stdin -- it never reaches a make variable, so it stays out of the process list. -.PHONY: login -login: ## Authenticate docker + oras to the image registry from .env (DFE_GHCR_USERNAME/TOKEN) - @python3 scripts/ghcr_login.py - -# `make stack VERSION=X.Y.Z` pins the certified set. `make dial` writes the dial's -# version.pin to DFE_STACK_VERSION in .env, so `make dial && make stack` pins -# straight from the deployment dial. An explicit VERSION= on the command line wins. -# VERSION=latest (rc to include pre-releases) instead takes the newest certified -# stack and repins every ghcr.io/hyperi-io image at its own newest published tag -# -- development currency, not a deployment. Still digest-pinned either way. +.PHONY: e2e-posture +e2e-posture: env-files ## Put .env and env/engine.env into the dfe-ui Playwright suite's posture (DFE_ENGINE_ENV=test, the engine's e2e routes on), keeping both originals for e2e-posture-off + @python3 scripts/e2e_posture.py on + +.PHONY: e2e-posture-off +e2e-posture-off: ## Put back the .env and env/engine.env from before e2e-posture + @python3 scripts/e2e_posture.py off + +.PHONY: e2e-posture-check +e2e-posture-check: ## Exit 1 naming what is missing unless the env files are in the e2e posture, which the Playwright run checks first + @python3 scripts/e2e_posture.py check + +# Pinned, latest and the GitHub-release repin: docs/deploying.md, "Staying +# current". An explicit VERSION= on the command line wins over the dial's pin. VERSION ?= $(DFE_STACK_VERSION) .PHONY: stack -stack: .env login ## Pin image versions into .env from the DFE stack SSoT (VERSION=X.Y.Z[-rc.N], or latest|rc for newest DFE images) +stack: .env ## Pin image versions into .env from the DFE stack SSoT (VERSION=X.Y.Z[-rc.N], or latest|rc for the newest published stack) @python3 scripts/stack.py $(VERSION) -# The deployment dial (deployment.yaml) is the single SSoT a deployment turns. -# This renders its docker-vm slice into .env; `make init` still mints the secrets, -# the dial merges the deploy-controlled keys over them. Canonical superset lives -# in dfe-infra; a lone dfe-docker clone deploys from deployment.yaml alone. +# The deployment dial: docs/deploying.md, "The deployment dial". .PHONY: dial -dial: .env ## Render deployment.yaml (the deployment dial) into .env -- docker-vm slice +dial: .env ## Render deployment.yaml (the deployment dial) into .env and env/engine.env -- docker-vm slice @python3 scripts/render_dial.py -# The two deploy-currency modes, stated in one place: PINNED (make stack + make -# ci) vs TRACK-LATEST (the ops/daemon-update timer). Reads local files only, so -# it is safe on a fresh checkout; the live latest-vs-applied number comes from -# `self_update.py --dry-run`, which this points at. +# Pinned against track-latest: docs/deploying.md, "Staying current". Reads local +# files only, so it is safe on a fresh checkout. .PHONY: modes modes: ## Show the deploy modes -- pinned vs track-latest -- and which one this checkout uses @python3 scripts/show_modes.py # --------------------------------------------------------------------------- -# Dev (builds from local source via docker-compose.override.yml) +# Dev (builds local images from source) # --------------------------------------------------------------------------- # The bind directories must exist before the local volume driver mounts them. .PHONY: storage-dirs storage-dirs: ifneq ($(strip $(DFE_DATA_ROOT)),) - @mkdir -p $(addprefix $(DFE_DATA_ROOT)/,clickhouse kafka-redpanda kafka-apache archiver dlq-spool engine-config engine-schemas hyperdx-pg) + @mkdir -p $(addprefix $(DFE_DATA_ROOT)/,clickhouse kafka-apache kafka-redpanda archiver dlq-spool fetcher-cursors transform-elastic-geoip engine-config engine-schemas hyperdx-documentdb) endif +# `dev` and `dev-build` clone the public component repos over https with no token. .PHONY: dev -dev: env-files dev-posture down storage-dirs ## Build local DFE images from source and start the dev stack (LOCAL="svc ..." builds only those; AUTH=real for a minted login), then print the access summary +dev: env-files dev-posture down storage-dirs ## Build local DFE images from source and start the dev stack (LOCAL="svc ..." builds only those; AUTH=real for a minted login; LIVE_UI=1 runs the console as `next dev` from DFE_SRC_ROOT/dfe-ui), then print the access summary. Clones the public component repos + python3 scripts/port_check.py -- $(DEV_FLAGS) $(PROFILE_FLAGS) docker compose $(DEV_PULL_FLAGS) $(PROFILE_FLAGS) pull - python3 scripts/build_dev_images.py $(DEV_OVERLAY_ARG) $(DEV_BUILD) + $(if $(strip $(DEV_BUILD)),python3 scripts/build_dev_images.py $(DEV_OVERLAY_ARG) $(DEV_BUILD)) docker compose $(DEV_FLAGS) $(PROFILE_FLAGS) up -d $(ACTIVE_SERVICES) @$(MAKE) --no-print-directory post || { echo "post: SELF TEST FAILED -- the stack is up, but it did not prove it moves data. Run 'make post' for detail."; exit 1; } @$(MAKE) --no-print-directory creds .PHONY: dev-build -dev-build: ## Build local DFE images from source (no start; honours LOCAL) +dev-build: ## Build local DFE images from source (no start; honours LOCAL). Clones the public component repos docker compose $(DEV_PULL_FLAGS) $(PROFILE_FLAGS) pull - python3 scripts/build_dev_images.py $(DEV_OVERLAY_ARG) $(DEV_BUILD) + $(if $(strip $(DEV_BUILD)),python3 scripts/build_dev_images.py $(DEV_OVERLAY_ARG) $(DEV_BUILD)) # --------------------------------------------------------------------------- -# CI / registry images (skips docker-compose.override.yml) +# CI / registry images # --------------------------------------------------------------------------- +# A registry stack runs no local image, so the record of the last dev build goes, or `make apply` would put an instance back on it. +# Pull before down, so a failed pull leaves the running stack up. .PHONY: ci -ci: login env-files down storage-dirs ## Pull and start infra and registry DFE images. Prints no credentials -- run `make creds` for those +ci: env-files storage-dirs ## Pull and start infra and registry DFE images. Prints no credentials -- run `make creds` for those + @python3 scripts/quickstart.py over-demo --container '$(DEMO_CONTAINER)' + @rm -f $(LOCAL_OVERLAY) docker compose -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) $(PROFILE_FLAGS) pull docker compose -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) $(PROFILE_FLAGS) pull $(ACTIVE_SERVICES) + @$(MAKE) --no-print-directory down + python3 scripts/port_check.py -- -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) $(PROFILE_FLAGS) docker compose -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) $(PROFILE_FLAGS) up -d $(ACTIVE_SERVICES) @$(MAKE) --no-print-directory post || { echo "post: SELF TEST FAILED -- the stack is up, but it did not prove it moves data. Run 'make post' for detail."; exit 1; } -# The operator-facing name for the registry start, and the one the engine prints -# when it tells someone how to apply a rotated password. It is `ci` plus the -# hand-over, which is the whole difference between the two. +# `make up` is a clone's one command: scripts/quickstart.py makes .env, picks the profile and pins the newest published stack, then the sub-make starts it. A sub-make because .tmp/profile.mk is resolved when make parses, before any recipe could write the profile into .env. PROFILE= and VERSION= count only from the make line, so a PROFILE in the shell's environment never moves the stack +DEFAULT_PROFILE := single-kafka +QUICKSTART_ARGS = --default-profile '$(DEFAULT_PROFILE)' --profile '$(if $(filter command line,$(origin PROFILE)),$(PROFILE))' --version '$(if $(filter command line,$(origin VERSION)),$(VERSION))' + +# The operator-facing name for the registry start and the one the engine prints when it tells someone how to apply a rotated password. It is `ci` with the quickstart before it and the hand-over after it .PHONY: up -up: ci ## Start the stack from the pinned registry images, then print the access summary (password on a TTY only) and write access-summary.md +up: ## Start the stack from the pinned registry images (PROFILE=slim-grpc and VERSION=X.Y.Z override the defaults), then print the access summary (password on a TTY only) and write access-summary.md + @python3 scripts/quickstart.py prepare up $(QUICKSTART_ARGS) + @$(MAKE) --no-print-directory ci @python3 scripts/creds.py --write +# The command dfe-engine's restart and recreate hints name: docs/architecture.md, +# "The engine renders each app's config". DEV=1 recreates from the local images. +ifneq ($(strip $(DEV)),) + APPLY_FLAGS = $(DEV_FLAGS) +else + APPLY_FLAGS = -f docker-compose.yml $(STORAGE_FLAGS) $(UI_FLAGS) +endif + +# The apps with a config-check- in compose/one-shots.yml, which a --no-deps recreate would skip +CONFIG_CHECKED_APPS := dfe-archiver dfe-loader dfe-receiver + +.PHONY: apply +apply: env-files storage-dirs ## Create or recreate the named services after a config write, removing the containers of deleted sources (SERVICES="svc ..."; DEV=1 for a `make dev` stack), the command dfe-engine's hints name. With no SERVICES it only removes, for after a source is deleted + @$(MAKE) --no-print-directory apply-services + +# A second make, because it reads the .tmp/profile.mk this one just re-resolved and so checks SERVICES against the instances the engine has declared. A source the engine deleted has left that fragment, so its container is removed first +.PHONY: apply-services +apply-services: + @python3 scripts/instances.py --prune -- $(APPLY_FLAGS) $(PROFILE_FLAGS) +# The engine names no command after a delete, and an empty SERVICES would recreate the whole stack. +ifneq ($(strip $(SERVICES)),) + @for check in $(addprefix config-check-,$(filter $(CONFIG_CHECKED_APPS),$(ACTIVE_SERVICES))); do docker compose $(APPLY_FLAGS) $(PROFILE_FLAGS) run --rm --no-deps "$$check" || exit 1; done + docker compose $(APPLY_FLAGS) $(PROFILE_FLAGS) up -d --force-recreate --no-deps $(ACTIVE_SERVICES) +endif +# An added or removed instance moves the CPU limits the collector reports and compose recreates it only when they did +ifeq ($(strip $(DFE_OTEL_RESOLVED)),true) + docker compose $(APPLY_FLAGS) $(PROFILE_FLAGS) up -d --no-deps otel-collector +endif + .PHONY: ci-pull -ci-pull: login ## Pull infra and registry DFE images +ci-pull: ## Pull infra and registry DFE images docker compose -f docker-compose.yml $(PROFILE_FLAGS) pull docker compose -f docker-compose.yml $(PROFILE_FLAGS) pull $(ACTIVE_SERVICES) @@ -387,33 +553,24 @@ ci-pull: login ## Pull infra and registry DFE images # Infrastructure only (Kafka + ClickHouse) # --------------------------------------------------------------------------- +# .env first: ClickHouse keeps the password its volume was created with. .PHONY: infra -infra: storage-dirs ## Start infrastructure services +infra: .env storage-dirs ## Start infrastructure services docker compose $(PROFILE_FLAGS) pull docker compose $(PROFILE_FLAGS) up -d # --------------------------------------------------------------------------- -# Validation -# The same commands CI runs, so a green local run means a green pipeline. -# -# Safe on a fresh checkout: no stack SSoT and no credentials needed. Two honest -# caveats. Make remakes the `-include .env` above before any target, so a fresh -# checkout gets a generated .env as a side effect of running these -- CI therefore -# leaves one on the runner. And check-dockerfile pulls the pinned hadolint image -# while check-tests resolves the pinned pytest, so those two want a network the -# first time; the rest need none. +# Validation: the static checks CI runs, docs/checks.md. # --------------------------------------------------------------------------- .PHONY: check -check: check-compose check-hardfail check-dockerfile check-docs check-python check-tests ## Run every static check CI runs +check: check-compose check-hardfail check-dockerfile check-docs check-proxy check-python check-tests check-locks ## Run every static check CI runs .PHONY: limits limits: ## Show the resource limits and totals, computed from the resolved config @python3 scripts/show_limits.py -# Pinned so a new ruff release cannot turn this red on an unrelated change, and -# so a local run lints with the SAME version CI does. CI installs the pin itself -# and sets RUFF=ruff; override the same way if you have no uvx. +# Pinned so a local run lints with the ruff CI runs. CI installs the pin and sets RUFF=ruff. RUFF_VERSION := 0.15.22 RUFF ?= uvx ruff@$(RUFF_VERSION) @@ -423,46 +580,80 @@ print-ruff-version: .PHONY: check-python check-python: ## Lint the helper scripts (config in ruff.toml) - $(RUFF) check scripts/ ops/ - $(RUFF) format --check scripts/ ops/ + $(RUFF) check scripts/ ops/ resources/demo/ resources/demo-*/ + $(RUFF) format --check scripts/ ops/ resources/demo/ resources/demo-*/ -# Pinned for the same reason RUFF is, and installed the same way in CI -# (PYTEST=pytest). Unit tests over the credential-handling helpers only: they -# touch a tmp_path .env and start nothing, so they belong with the static checks. -PYTEST_VERSION := 9.1.1 -PYTEST ?= uvx pytest@$(PYTEST_VERSION) +# Pinned like RUFF, and CI sets PYTEST=pytest. +PYTEST_VERSION := 9.2.0 +# The seed tests parse resources/seed with the PyYAML scripts/seed.py pins. scripts/tests/test_script_locks.py holds the two pins equal +PYYAML_VERSION := 6.0.3 +PYTEST ?= uvx --with pyyaml==$(PYYAML_VERSION) pytest@$(PYTEST_VERSION) .PHONY: print-pytest-version print-pytest-version: @echo $(PYTEST_VERSION) +.PHONY: print-pyyaml-version +print-pyyaml-version: + @echo $(PYYAML_VERSION) + .PHONY: check-tests check-tests: ## Run the helper-script unit tests (scripts/tests) $(PYTEST) -q scripts/tests -# `slim` and `single` are projections of the Kubernetes tiers of the same name, -# so both targets need DFE_INFRA_DIR pointed at a dfe-infra checkout. Unset, -# check-profiles reports SKIPPED rather than passing: it cannot read the master. -.PHONY: render-profiles -render-profiles: ## Re-render the projected slim/single profiles from dfe-infra (DFE_INFRA_DIR=...) - @python3 scripts/render_profiles.py - -.PHONY: check-profiles -check-profiles: ## Assert the projected profiles match the Kubernetes ones (DFE_INFRA_DIR=...) -ifeq ($(strip $(DFE_INFRA_DIR)),) - @echo "check-profiles: DFE_INFRA_DIR unset -- projection NOT checked (not a pass)" -else - @python3 scripts/render_profiles.py --check -endif +# Each inline-metadata script runs with `uv run --locked`, and Renovate's pep723 manager rewrites a pin but never its lock, so this fails a pin and lock that disagree. +# Pinned and installed in CI the way RUFF is (LOCK_UV=uv). +UV_VERSION := 0.13.6 +LOCK_UV ?= uvx uv@$(UV_VERSION) -# The resolved chain, for check-compose to compare the `make dev LOCAL=...` file -# list against. Not a bootstrap goal: the chain depends on DFE_AUTH_RESOLVED. +.PHONY: print-uv-version +print-uv-version: + @echo $(UV_VERSION) + +.PHONY: check-locks +check-locks: ## Assert each inline-metadata script's lock matches its pins (relock with `uv lock --script