diff --git a/.cargo/config.toml b/.cargo/config.toml index 298a960..a7257cf 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,5 +1,3 @@ -# Per-target CPU optimisations. -# # Under [target.], never [build]: cargo drops build.rustflags when a # CARGO_TARGET__RUSTFLAGS is set, as CI runners do. diff --git a/.gitignore b/.gitignore index 246d071..746710d 100644 --- a/.gitignore +++ b/.gitignore @@ -62,8 +62,5 @@ GEMINI.md tarpaulin-report.json cobertura.xml -# Local scratch / tmp (CORE.md: scratch goes here, gitignored, NOT -# /tmp). Bare patterns (no leading slash) match a tmp dir at ANY -# depth, so these cover top-level .tmp/ and tmp/ as well as nested -# ones like tests/tmp/. Mirrored in attach.py _DEFAULT_EXCLUDES. +# Scratch directories at any depth. tmp/ diff --git a/.hyperi-ci.yaml b/.hyperi-ci.yaml index eddd11b..2d2e3ad 100644 --- a/.hyperi-ci.yaml +++ b/.hyperi-ci.yaml @@ -8,36 +8,30 @@ classification: product language: rust -# Quality checks quality: enabled: true - # Advisory ignores fed to cargo-audit and osv-scanner. ignore: - tool: osv-scanner id: GHSA-w9wp-h8wv-79jx reason: "CVE-2026-48504, opentelemetry_sdk 0.31.0 via scalo: unreachable, no Baggage propagator registered in dfe-transform-vector or scalo. Not reachable per dfe-infra docs/THREAT-MODEL.md accepted risks. Blocked on metrics-exporter-opentelemetry 0.2.1 requiring opentelemetry ^0.31; tracked in hyperi-io/scalo-rs#281." -# Testing test: enabled: true coverage: false rust: # The metrics tests need a process each: the global recorder installs once per process. nextest: true - # A release runs the full tier: the Vector validator cases and the opt-in - # Kafka pipelines. tests/TESTING.md says which tests are which. + # tests/TESTING.md says which tests the full tier adds. full: required_for_release: true -# Building build: enabled: true strategies: - native rust: glibc_max: "2.41" - # Any C library a release links comes from hyperi-ci's static prefix, so - # the image installs no C library packages. + # C libraries come from hyperi-ci's static prefix, so the image installs none. static_libs: true targets: - x86_64-unknown-linux-gnu @@ -49,14 +43,11 @@ build: workload_cmd: "bash scripts/pgo-workload.sh" # Built before the workload clock starts: an uncached arm64 runner takes longer than the workload timeout to compile it. workload_setup_cmd: "cargo build --release --features pgo-driver --bin pgo-driver" - # 300s is the floor a stable profile needs: a short workload biases - # the compiler toward startup paths and turns PGO NEGATIVE. + # A shorter workload biases the profile toward startup paths. duration_secs: 300 bolt: enabled: true -# Release (where the artefacts go). The thin chart is assembled from the -# emitted contract and ships beside the image. release: enabled: true # Keep `library` at the scalo version in Cargo.toml: the library renders only the contract version its scalo release writes. @@ -66,8 +57,8 @@ release: library: "2.14.30" container: enabled: true + # Generated by `dfe-transform-vector emit-dockerfile`, see docs/internals.md. dockerfile: Dockerfile platforms: - linux/amd64 - linux/arm64 - # `src/deployment.rs::emit_dockerfile()` splices the Vector install into the Dockerfile. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3318a3e..16b6145 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,7 +44,7 @@ feat!: remove deprecated API endpoints BREAKING CHANGE: The /v1/users endpoint has been removed. Use /v2/users instead. ``` -### Examples +Examples: ```text feat(auth): add OAuth2 support for Google login @@ -58,8 +58,6 @@ refactor(core)!: restructure module exports BREAKING CHANGE: Named exports are now used instead of default exports. ``` -### Scope - Scope is optional but recommended. Use it to indicate the area of the codebase affected (e.g., `api`, `auth`, `core`, `cli`, `docs`). @@ -90,7 +88,7 @@ anniversary of its release. 3. Run the tests that cover your change 4. Submit a pull request with a clear description of what you've done -### Pull Request Checklist +Before you open the pull request: - [ ] Commits follow the conventional commit format - [ ] Tests pass (if applicable) @@ -108,7 +106,7 @@ If you have questions about contributing, please open an issue or contact us. ### What this is -A Rust supervisor that runs Vector as a child process so a Vector pipeline behaves like every other DFE app: one big-dial config, one ops port, an OTLP push, and a chart assembled from its deployment contract. It is NOT the transform engine and it does not build Vector. The image downloads the upstream release binary at the version pinned by `VECTOR_VERSION` in `src/deployment.rs`, and cargo compiles only the supervisor. It is not a singleton either: dfe-infra's `apps.yaml` declares it `multiplicity: per_config` with `scale_deployed: true` and `default_in: [docker-single]`, so one deployment exists per source config, and outside `docker-single` only when a deployment asks for it. +A Rust supervisor that runs Vector as a child process so a Vector pipeline behaves like every other DFE app: one big-dial config, one ops port, an OTLP push, and a chart assembled from its deployment contract. It is NOT the transform engine and it does not build Vector. The image downloads the upstream release binary at the version pinned by `VECTOR_VERSION` in `src/deployment.rs`, and cargo compiles only the supervisor. It is not a singleton either: dfe-infra's `apps.yaml` declares it `multiplicity: per_config` with `scale_deployed: true` and `default_in: [docker-single]`, so one deployment exists per source config, and outside `docker-single` when a deployment asks for it. ### Where things live @@ -116,12 +114,12 @@ A Rust supervisor that runs Vector as a child process so a Vector pipeline behav |---|---| | `src/main.rs` | CLI entry. Fills scalo's `CommonArgs` from the loaded config before `run_app`, which is what makes `metrics.address` and `logging.*` take effect | | `src/config/` | `loader` (the config and its env layers) and the Vector config it becomes: `generate`, `wiring`, `transforms`, `secrets`, `assembler`, `validate`, plus `reload`, `staging` and `kafka_defaults` | -| `src/vector/` | `process` spawn and backoff, `lifecycle` states, and `binary` -- version selection kept pure and NOT on the run path, which its own header explains | +| `src/vector/` | `process` spawn and backoff, `lifecycle` states, and `binary` -- version selection kept pure and NOT on the run path, which its header explains | | `src/bridge.rs` | The direct-transport translation between scalo `Transport/Push` and Vector `PushEvents`. Does nothing on `bus` | | `src/health.rs`, `src/metrics.rs`, `src/metrics/scrape.rs` | Readiness publishing, and the scrape that merges `vector_*` into scalo's registry | | `src/deployment.rs`, `src/vector-layer.dockerfile` | `contract()`, the source the Dockerfile and the released chart are generated from, and the two-binary image override spliced onto scalo's generated Dockerfile | | `templates/bus.yaml`, `templates/direct.yaml` | Whole runnable Vector topologies, one per transport, commented per field | -| `pipelines/filebeat/` | The bundled filebeat pipeline. The image does not carry it | +| `pipelines/filebeat/` | The bundled filebeat pipeline. The image does not ship it | | `docs/` | `architecture.md` for why the service is built this way and the invariants, `internals.md` for field-by-field depth, `migration.md`, `librdkafka.md`, the generated `config-schema.*` and `capability-catalog.*` | | `tests/` | `integration`, `e2e`, `smoke`, and `TESTING.md` for how the broker and Vector binary are resolved | | `scripts/fetch-vector.sh`, `scripts/pgo-workload.sh` | Downloads the pinned Vector for tests, and drives the PGO/BOLT workload | @@ -137,11 +135,11 @@ cargo nextest run --test e2e # needs Docker or a live broker, an cargo nextest run --run-ignored all # everything, opt-in cases included ``` -What green does NOT mean: +What a passing run does NOT mean: -- The default run leaves out the `#[ignore]` cases in `tests/integration/vector_validate.rs` and `tests/e2e/kafka.rs`. The `vector_validate` ones run Vector's own validator over each buffer, SASL and TLS variant of an assembled config, so until you pass `--run-ignored all` those variants are unchecked. -- Locally, a missing broker or Vector binary SKIPS rather than fails. In CI that absence is an assertion failure for the cases that run by default, so CI is stricter than a green laptop run. -- `e2e::filebeat_kafka` runs by default and needs a dfe-transform-vrl checkout, for its cases and the bundled pipeline: beside this repo, or named by `DFE_TRANSFORM_VRL_DIR`. Without one it skips locally and fails under CI, which checks the repo out beside this one. +- The default run leaves out the `#[ignore]` cases in `tests/integration/vector_validate.rs` and `tests/e2e/kafka.rs`. The `vector_validate` ones run Vector's validator over each buffer, SASL and TLS variant of an assembled config, so until you pass `--run-ignored all` those variants are unchecked. +- Locally, a missing broker or Vector binary SKIPS rather than fails. In CI that absence is an assertion failure for the cases that run by default, so CI is stricter than a passing laptop run. +- `e2e::filebeat_kafka` runs by default and needs a dfe-transform-vrl checkout, for its cases and the bundled pipeline: beside this repo, or at `DFE_TRANSFORM_VRL_DIR`. Without one it skips locally and fails under CI, which checks the repo out beside this one. - A docs-only push runs no CI at all. `paths-ignore` for `docs/**` and `**.md` is on the `push` trigger but not on `pull_request`, so the PR is the only place a docs change is checked. A silent push is not a pass. ### What tends to bite @@ -150,14 +148,14 @@ What green does NOT mean: |---|---|---| | Hand-edit `Dockerfile`, or commit a chart | Fix `src/deployment.rs::contract()` and regenerate | The Dockerfile is generator output, so `checked_in_dockerfile_matches_emit_dockerfile` fails on a hand edit, and the release assembles the chart from the contract | | Bump scalo and leave `release.helm.library` behind | Move `release.helm.library` in `.hyperi-ci.yaml` to the same scalo version | A scalo-service release renders only the contract version its scalo release writes | -| Turn the Kafka lag trigger back on in `contract()` | Leave `KafkaLagTrigger::disabled()` and scale on CPU, adding a pressure trigger through the chart's `keda.extraTriggers` where a deployment wants one | Lag rises when a downstream stage breaks, so a lag trigger adds pods that wait on the same broken stage. The library's trigger also reads `config.kafka.*`, which this app does not have, and refuses to render | -| Set `vector.version` in a deployment's config and forget it | Leave it unset, so the default reads `VECTOR_VERSION` | A pinned version that differs from the binary the image bakes refuses to start under `strict`, the default, and runs only where `vector.version_check: warn` lets it | -| Mount a whole `templates/*.yaml` as a transform file | Copy only its `transforms:` block | The supervisor generates `sources:` and `sinks:` from the big dials and refuses a transform file declaring them | +| Turn the Kafka lag trigger back on in `contract()` | Leave `KafkaLagTrigger::disabled()` and scale on CPU, adding a pressure trigger through the chart's `keda.extraTriggers` where a deployment wants one | Lag rises when a downstream stage breaks, so a lag trigger adds pods that wait on the same broken stage. The library's trigger reads `config.kafka.*`, which this app does not have, and will not render | +| Set `vector.version` in a deployment's config and forget it | Leave it unset, so the default reads `VECTOR_VERSION` | A pinned version that differs from the binary the image bakes does not start under `strict`, the default, and runs only where `vector.version_check: warn` lets it | +| Mount a whole `templates/*.yaml` as a transform file | Copy only its `transforms:` block | The supervisor generates `sources:` and `sinks:` from the big dials and rejects a transform file declaring them | | Change `password` to a `SensitiveString` to look safer | Leave it a `String` | `SensitiveString` serialises as `***REDACTED***` and the figment serialize-merge-deserialize round trip in `apply_figment_env()` destroys the value. Masking happens in logs and `Debug`, and Vector reads the credential from files through its `directory` secret backend, so it never lands in the Vector config | | Put a `${VAR}` placeholder in a credential or a transform | Mount the secret and set `sasl.secret_dir`, or give the credential through `DFE_TRANSFORM_VECTOR_{SOURCE,SINK}_SASL_*`, and read the environment in VRL with `get_env_var` | Vector 0.57+ expands no `${VAR}` without `--dangerously-allow-env-var-interpolation` or `VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION`, and the supervisor sets neither. A credential holding `${` is refused at load rather than reaching the broker as the password. Vector runs without the `DFE_TRANSFORM_VECTOR_*` variables, so `get_env_var` cannot read those | -| Assume a config key you added is wired because it parses and validates | Prove it changes behaviour, and keep `every_committed_config_file_loads_and_validates` passing over each committed config file | A key can parse, validate and reach nothing, so a deployment that moves a port on it keeps the old port and loses every probe | +| Assume a config key you added is wired because it parses and validates | Test that it changes behaviour, and keep `every_committed_config_file_loads_and_validates` passing over each committed config file | A key can parse, validate and reach nothing, so a deployment that moves a port on it keeps the old port and loses every probe | | Set `enable.auto.commit: false` here, as the shared DFE consumer baseline does | Leave auto-commit on | Vector's kafka source only stores offsets and leaves librdkafka's commit timer to flush them, and that timer is armed only when auto-commit is on, so with it off the stored offsets reach the broker only on a rebalance or a clean shutdown | -| Read `/readyz` as "carrying traffic" | Read it as "Vector survived 500 ms after spawn, and the sink is not stalled" | A pod whose broker is unreachable, with nothing held, stays Ready with no record moving | +| Read `/readyz` as "carrying traffic" | Read it as "Vector survived 500 ms after spawn, and the sink is not stalled" | A pod with an unreachable broker and nothing held stays Ready with no record moving | | Swap the reload poller for inotify | Keep polling | S3-backed mounts -- s3fs, goofys, Mountpoint for S3 -- generate no filesystem notification events, so inotify works on every laptop and stops reloading in production with nothing logged | | Add or rename a credential field without a schema marker | Schema it as `scalo::SensitiveString` with `#[schemars(with = ...)]`, as `SaslConfig::password` is | That puts `x-scalo-secret` and `writeOnly` on the field in the emitted config-schema, which is what tells a console to mask it. Without it only dfe-engine's name-based fallback can mask it, so a name that fallback does not read as a credential returns an operator's Kafka password in the clear | @@ -169,14 +167,14 @@ and `--producer dfe-transform-vector`. Inbound -- what this repo depends on: -- scalo-rs, `cargo-dep`. `Cargo.toml` declares the `scalo` crate by range, with a second range covering the dev dependency. A scalo release reaches this repo, so widen the range if it does not admit the new version, then rebuild. -- scalo-rs, `generated-file`, lockstep. The committed `Dockerfile` is written by `scalo::deployment::generate_dockerfile()` -- its own header names the generator and the schema version -- and this repo splices the Vector layer onto it. When the generator or its schema moves, regenerate with `dfe-transform-vector emit-dockerfile > Dockerfile` and commit the diff. The released chart is assembled on the scalo-service library chart at `release.helm.library`, which moves with the scalo version in `Cargo.toml`. +- scalo-rs, `cargo-dep`. `Cargo.toml` declares the `scalo` crate by range, with a second range covering the dev dependency. A scalo release reaches this repo, so widen the range if it does not admit the release, then rebuild. +- scalo-rs, `generated-file`, lockstep. The committed `Dockerfile` is written by `scalo::deployment::generate_dockerfile()` -- its header names the generator and the schema version -- and this repo splices the Vector layer onto it. When the generator or its schema moves, regenerate with `dfe-transform-vector emit-dockerfile > Dockerfile` and commit the diff. The released chart is assembled on the scalo-service library chart at `release.helm.library`, which moves with the scalo version in `Cargo.toml`. Outbound -- what depends on this repo: - dfe-infra, `image-pin`, lockstep. `dfe-infra/versions.yaml` pins this repo's container image as a tag plus the digest that makes the tag immutable, and the thin chart released here, which Argo deploys. Move the tag with `dfe-stack bump-app`, which repins the chart, then re-resolve the image digest. -Two more relationships that are real but are NOT declared edges in the suite graph, so no gate enforces them: +Two more relationships that exist but are NOT declared edges in the suite graph, so no gate enforces them: - dfe-infra `apps.yaml` is the SSoT for how this app is deployed -- multiplicity, scaling, transports, endpoints and which compiler derives its routing. dfe-engine reads that manifest, so adding or changing the app is a manifest edit, never an engine release. - dfe-transform-vrl holds the filebeat cases (`tests/fixtures/filebeat/cases.yaml`) and the bundled pipeline that `e2e::filebeat_kafka` grades against. The dependency is a test-time checkout only, which CI makes and a laptop may not. diff --git a/Cargo.toml b/Cargo.toml index 39e0db8..bd98b5a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,6 @@ path = "src/bin/pgo_driver.rs" required-features = ["pgo-driver"] [dependencies] -# Async runtime tokio = { version = ">=1.54.1, <2", features = ["full"] } tokio-util = { version = ">=0.7.20, <0.8", features = ["rt"] } @@ -60,7 +59,6 @@ scalo = { version = ">=2.14.30, <3", features = [ "worker-batch", ] } -# Configuration figment = { version = ">=0.10.19, <0.11", features = ["env"] } clap = { version = ">=4.6.7, <5", features = ["derive", "env"] } @@ -68,29 +66,24 @@ clap = { version = ">=4.6.7, <5", features = ["derive", "env"] } # than copying them between the two gRPC legs. bytes = ">=1.12.1, <2" -# Serialisation serde = { version = ">=1.0.229, <2", features = ["derive"] } serde_json = ">=1.0.154, <2" # Config-schema derivation. Same range as scalo so cargo unifies to one schemars. schemars = { version = ">=1.2.3, <2", default-features = false, features = ["std", "derive", "preserve_order"] } -# Error handling thiserror = ">=2.0.21, <3" anyhow = ">=1.0.104, <2" -# Logging and tracing tracing = ">=0.1.44, <0.2" tracing-subscriber = { version = ">=0.3.23, <0.4", features = ["env-filter", "json"] } -# Metrics metrics = ">=0.24.6, <0.25" # Unix signals (sending SIGTERM/SIGHUP to child process), and renameat2 to # swap a reload's staged config in whole nix = { version = ">=0.31.3, <0.32", features = ["signal", "process", "fs"] } -# Allocator tikv-jemallocator = { version = ">=0.7.0, <0.8", optional = true } prometheus-parse = ">=0.2.5, <0.3" @@ -108,9 +101,9 @@ scalo = { version = ">=2.14.30, <3", features = [ ] } # The repo's own .env for the live-broker tests. The binary's .env is read by scalo's cascade. dotenvy = ">=0.15.7, <0.16" -# A stand-in Vector source for the bridge tests; scalo already builds this tonic. +# A stand-in Vector source for the bridge tests, on the tonic scalo builds. tonic = { version = ">=0.14.9, <0.15", features = ["gzip"] } -# Reads back what the app counted; the Prometheus exporter already builds it. +# Reads back what the app counted, on the crate the Prometheus exporter builds. metrics-util = { version = ">=0.20.4, <0.21", default-features = false, features = [ "debugging", ] } diff --git a/Makefile b/Makefile index 9274714..bbeaa21 100644 --- a/Makefile +++ b/Makefile @@ -10,13 +10,13 @@ check: ## Pre-push validation (quality + test) hyperi-ci check -quality: ## Run all quality checks +quality: hyperi-ci run quality -test: ## Run tests +test: hyperi-ci run test -build: ## Build artifacts +build: hyperi-ci run build ci: quality test build ## Run full CI pipeline locally diff --git a/README.md b/README.md index b55d90a..fdc09cd 100644 --- a/README.md +++ b/README.md @@ -7,12 +7,58 @@ Runs a Vector pipeline as a supervised transform stage in a DFE deployment. ## What it does -- Vector does the transform, and the supervisor supplies the rest a stage needs: config, credentials, health, metrics and restarts. Reach for it when you need one of Vector's own transforms beyond `remap`. For VRL alone, [dfe-transform-vrl](https://github.com/hyperi-io/dfe-transform-vrl) runs it in-process with no Vector binary. +- Vector does the transform, and the supervisor supplies the rest a stage needs: config, credentials, health, metrics and restarts. Reach for it when you need one of Vector's transforms beyond `remap`. For VRL alone, [dfe-transform-vrl](https://github.com/hyperi-io/dfe-transform-vrl) runs it in-process with no Vector binary. - One small YAML (pipeline name, brokers, topics, SASL and TLS) generates Vector's source, sink and observability config, so you write only the `transforms:` and `enrichment_tables:` blocks. -- Kafka in and out (`bus`), or a direct gRPC push with no broker (`direct`). The transform you author is the same on both. +- Kafka (`bus`) or a direct gRPC push with no broker (`direct`), chosen for each end. The transform you author is the same on both. - A crashed Vector is restarted with exponential backoff, and readiness follows it. -- With reload enabled, a transform change is validated, sent to Vector with a SIGHUP, and read back from Vector's own counters as applied, refused (the running config is put back) or unconfirmed. One that fails validation is logged and never sent. -- Vector's `vector_*` metrics are merged into the supervisor's, so one endpoint and one OTLP push carry both. +- With reload enabled, a transform change is validated, sent to Vector with a SIGHUP, and read back from Vector's reload counters as applied, refused (the running config is put back) or unconfirmed. One that fails validation is logged and never sent. +- Vector's `vector_*` metrics are merged into the supervisor's, so one endpoint and one OTLP push serve both. + +The config engine and the process manager run Vector inside the pod, the bridge moves records on the direct transport, and the metrics merge serves Vector's metrics beside the supervisor's. + +```mermaid +flowchart TB + accTitle: L3 Components -- dfe-transform-vector + accDescr: The previous stage sends records into the pod and the config and transform files configure it. Inside, the config engine writes the Vector config that the process manager starts Vector on, and on the direct transport the bridge intake hands records to Vector and the bridge return takes them back. Vector's metrics are merged into the supervisor's and served on the ops port. Records leave for the next stage and metrics go to the collector. + + prev["Previous stage"] + files[("Config files")] + + subgraph pod["dfe-transform-vector"] + direction LR + cfg[["Config engine"]] + proc["Process manager"] + intake[["Bridge intake"]] + vec[["Vector"]] + ret[["Bridge return"]] + merge["Metrics merge"] + cfg --> proc + proc --> vec + intake --> vec + vec --> ret + vec --> merge + end + + next["Next stage"] + collector["Metrics collector"] + + prev -->|records| pod + files -->|config| pod + pod -->|records| next + pod -->|metrics| collector + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef store fill:#DDF3EC,stroke:#00785A,stroke-width:1px,color:#0B3327 + classDef ext fill:#EEF0F2,stroke:#6B7280,stroke-width:1px,stroke-dasharray:4 3,color:#1F2937 + classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 + class prev,next,collector ext + class files store + class pod group +``` + +*Figure 1. L3 Components -- dfe-transform-vector. Dashed boxes are external.* Up: top of dfe-transform-vector. Down: [L4 Detail -- Config assembly](docs/architecture.md#config-assembly), [L4 Detail -- Direct transport bridge](docs/architecture.md#where-the-transports-differ), [L4 Detail -- Vector lifecycle](docs/internals.md#the-vector-subprocess). + +On `bus` Vector reads and writes Kafka itself and the bridge does nothing. On `direct` the bridge takes records from the previous stage's Push client and hands Vector's output to the next stage's Push listener. ## Quick start @@ -24,7 +70,7 @@ scripts/fetch-vector.sh # caches the pinned Vector under .tmp/ and prints `--config` goes before the subcommand, and `run` is the default when you leave the subcommand out. To run it off the image, point `vector.binary` at the path `scripts/fetch-vector.sh` printed and `vector.config_dir` and `vector.data_dir` at directories you can write, then `./target/release/dfe-transform-vector --config config.example.yaml`. -[config.example.yaml](config.example.yaml) is the annotated reference. [templates/bus.yaml](templates/bus.yaml) and [templates/direct.yaml](templates/direct.yaml) are the whole Vector topology for each transport, commented per field and runnable as they stand: copy one, replace the `dfe_transform` step, and you have a working pipeline. In a deployment the supervisor generates `sources:` and `sinks:`, and refuses any other top-level key in a transform file. +[config.example.yaml](config.example.yaml) is the annotated reference. [templates/bus.yaml](templates/bus.yaml) and [templates/direct.yaml](templates/direct.yaml) are the whole Vector topology for each transport, commented per field and runnable as they stand: copy one, replace the `dfe_transform` step, and you have a working pipeline. In a deployment the supervisor generates `sources:` and `sinks:`, and rejects any other top-level key in a transform file. | | `bus` | `direct` | |---|---|---| @@ -32,13 +78,13 @@ scripts/fetch-vector.sh # caches the pinned Vector under .tmp/ and prints | Out | Kafka `_load` | a scalo Push client to `sink.endpoint` | | Needs a broker | yes | no | -One port, 9090 by default, carries the ops surface: `/livez`, `/readyz` and `/metrics`. `/readyz` answers 503 until Vector has survived its first 500 ms, and again while it is down or its sink has held records for `metrics.sink_stall_secs` without delivering any. A bus config with no topics idles instead: Ready, with no Vector running, until the config file names a topic. Vector's own exporter defaults to `127.0.0.1:9598`. Keep `metrics.vector_metrics_address` on loopback, because nothing checks it. +One port, 9090 by default, serves the ops surface: `/livez`, `/readyz` and `/metrics`. `/readyz` answers 503 until Vector has survived its first 500 ms, and again when it is down or its sink has held or refused records for `metrics.sink_stall_secs` without delivering any. A bus config with no topics idles instead: Ready, with no Vector running, until the config file names a topic. Vector's exporter defaults to `127.0.0.1:9598`. Keep `metrics.vector_metrics_address` on loopback, because nothing checks it. ## Configuration -The config file (`--config`, else `config.yaml` or `config.yml` in the working directory) sits over the defaults, and `DFE_TRANSFORM_VECTOR_*` environment variables sit over the file. The working directory's `.env` is read into the environment first, and a variable already set wins. The flat names take a single underscore and a comma list (`DFE_TRANSFORM_VECTOR_SOURCE_BROKERS=a:9092,b:9092`, `_SINK_TOPIC`, `_TRANSFORMS_DIR`, the SASL pairs) and beat the double-underscore form. That form reaches any key and takes a list in brackets (`DFE_TRANSFORM_VECTOR_SOURCE__BROKERS='[a:9092,b:9092]'`), and a bare comma list there fails to load. `--log-level`, `--log-format` and `--metrics-addr`, or `LOG_LEVEL`, `LOG_FORMAT` and `METRICS_ADDR`, outrank `logging.*` and `metrics.address`. [docs/config-schema.yaml](docs/config-schema.yaml) lists the app's own keys with their types and defaults. +The config file (`--config`, else `config.yaml` or `config.yml` in the working directory) sits over the defaults, and `DFE_TRANSFORM_VECTOR_*` environment variables sit over the file. The working directory's `.env` is read into the environment first, and a variable set before it wins. The flat names take a single underscore and a comma list (`DFE_TRANSFORM_VECTOR_SOURCE_BROKERS=a:9092,b:9092`, `_SINK_TOPIC`, `_TRANSFORMS_DIR`, the SASL pairs) and beat the double-underscore form. That form reaches any key and takes a list in brackets (`DFE_TRANSFORM_VECTOR_SOURCE__BROKERS='[a:9092,b:9092]'`), and a bare comma list there fails to load. `--log-level`, `--log-format` and `--metrics-addr`, or `LOG_LEVEL`, `LOG_FORMAT` and `METRICS_ADDR`, outrank `logging.*` and `metrics.address`. [docs/config-schema.yaml](docs/config-schema.yaml) lists the app's keys with their types and defaults. -With reload on, the default, a transform file change applies once it validates, with no restart. Any other change to the app's own keys, `dfe_source`, `pipeline.name` and `scaling.pressure_threshold` included, is logged as needing a pod restart, and no later transform change applies until the pod restarts or the edit is reverted. scalo's own sections in the file, such as `version_check`, `scaling.enabled` and `metrics.namespace`, are read once at startup: an edit there needs a restart too, and the watcher logs a successful reload without applying it. In a chart deployment a config or file set change rolls the pods anyway, because the chart hashes both into the pod template. The reload path, the readiness rules, the metrics merge and the Kafka settings are in [docs/internals.md](docs/internals.md) and [docs/librdkafka.md](docs/librdkafka.md). +With reload on, the default, a transform file change applies once it validates, with no restart. Any other change to the app's keys, `dfe_source`, `pipeline.name` and `scaling.pressure_threshold` included, is logged as needing a pod restart, and no later transform change applies until the pod restarts or the edit is reverted. scalo's sections in the file, such as `version_check`, `scaling.enabled` and `metrics.namespace`, are read once at startup, so an edit there needs a restart, and the watcher logs a successful reload without applying it. In a chart deployment a config or file set change rolls the pods anyway, because the chart hashes both into the pod template. The reload path, the readiness rules, the metrics merge and the Kafka settings are in [docs/internals.md](docs/internals.md) and [docs/librdkafka.md](docs/librdkafka.md). ## Documentation @@ -52,7 +98,7 @@ The suite's published docs are at , and this reposit ## Help and contributing -How to work in this repo, the commands that prove a change and what tends to bite are in [CONTRIBUTING.md](CONTRIBUTING.md). A question that is not a bug goes in a new issue. +How to work in this repo, the commands that test a change and what tends to bite are in [CONTRIBUTING.md](CONTRIBUTING.md). A question that is not a bug goes in an issue. Open bugs: [this repo](https://github.com/hyperi-io/dfe-transform-vector/issues?q=is%3Aopen+is%3Aissue+type%3ABug+-label%3Abacklog+-label%3Anext-minor) | [whole DFE suite](https://github.com/search?type=issues&q=is%3Aopen+is%3Aissue+type%3ABug+-label%3Abacklog+-label%3Anext-minor+repo%3Ahyperi-io%2Fdfe-infra+repo%3Ahyperi-io%2Fdfe-docker+repo%3Ahyperi-io%2Fdfe-deploy+repo%3Ahyperi-io%2Fdfe-engine+repo%3Ahyperi-io%2Fdfe-loader+repo%3Ahyperi-io%2Fdfe-receiver+repo%3Ahyperi-io%2Fdfe-fetcher+repo%3Ahyperi-io%2Fdfe-archiver+repo%3Ahyperi-io%2Fdfe-transform-vrl+repo%3Ahyperi-io%2Fdfe-transform-vector+repo%3Ahyperi-io%2Fdfe-transform-elastic+repo%3Ahyperi-io%2Fdfe-ui+repo%3Ahyperi-io%2Fdfe-hyperdx+repo%3Ahyperi-io%2Fscalo-rs+repo%3Ahyperi-io%2Fscalo-py+repo%3Ahyperi-io%2Flogreducer+repo%3Ahyperi-io%2Fdfe-schemas+repo%3Ahyperi-io%2Fvector-vrl+repo%3Ahyperi-io%2Fculvert+repo%3Ahyperi-io%2Fdfe-clickhouse-rs+repo%3Ahyperi-io%2Ffactbook-rs) diff --git a/clippy.toml b/clippy.toml index a363772..29a8f97 100644 --- a/clippy.toml +++ b/clippy.toml @@ -13,6 +13,5 @@ cognitive-complexity-threshold = 25 type-complexity-threshold = 350 too-many-lines-threshold = 150 -# unwrap_used / expect_used stay denied in shipped code, not in #[test] or #[cfg(test)] allow-unwrap-in-tests = true allow-expect-in-tests = true diff --git a/deny.toml b/deny.toml index 942fd36..e23a35e 100644 --- a/deny.toml +++ b/deny.toml @@ -1,4 +1,3 @@ -# cargo-deny v0.19+ configuration # Run: cargo deny check [advisories] @@ -21,9 +20,7 @@ allow = [ "CC0-1.0", "Unlicense", "OpenSSL", - # webpki-root-certs/webpki-roots (via reqwest -> rustls-platform-verifier) - # relicensed MPL-2.0 -> CDLA-Permissive-2.0; the CCADB cert data is a - # permissive data licence. Matches scalo's own deny.toml allowance. + # The CCADB certificate data in webpki-roots, a permissive data licence. "CDLA-Permissive-2.0", ] diff --git a/docs/architecture.md b/docs/architecture.md index 7d034ee..682aefc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,20 +1,20 @@ # Architecture -Why this repo exists, what owns what, and the invariants you cannot read off a single file. For the field-by-field design -- every config key, every generated file, every state transition -- read [internals.md](internals.md). +Why this repo exists, what owns what, and the invariants you cannot read off a single file. Field-level detail is in [internals.md](internals.md). ## The problem -DFE needs a transform stage between its sources and its loader, and Vector already does the per-record transform work well. What it lacks is the rest of a DFE app. Vector has its own config format, its own metrics endpoint, its own process lifecycle and its own idea of what "started" means. Every DFE app presents one control surface instead: a big-dial YAML, one ops port carrying `/livez`, `/readyz` and `/metrics`, an OTLP push, KEDA scaling and a chart assembled from its deployment contract. +DFE needs a transform stage between its sources and its loader, and Vector does the per-record transform work well. What it lacks is the rest of a DFE app. Vector has its own config format, metrics endpoint, process lifecycle and idea of what "started" means. Every DFE app presents one control surface instead: a big-dial YAML, one ops port carrying `/livez`, `/readyz` and `/metrics`, an OTLP push, KEDA scaling and a chart assembled from its deployment contract. So Vector is kept, and wrapped. The Rust supervisor is PID 1 and Vector is its child. The supervisor supplies the production half -- config, credentials, health, metrics, restarts -- and Vector does the transform. ## Why a subprocess -Vector is not designed as a library and has no stable public API for embedding, so the image downloads the upstream release binary at a pinned version and the supervisor spawns it with `--config-dir`. A Vector upgrade is then a version bump and a new image, where a compiled-in Vector would mean rebuilding against every Vector release. +Vector is not designed as a library and has no stable public API for embedding, so the image downloads the upstream release binary at a pinned version and the supervisor spawns it with `--config-dir`. A Vector upgrade is then a version bump and an image rebuild, where a compiled-in Vector would mean rebuilding against every Vector release. -That choice has one visible consequence. This is the only DFE Rust app whose runtime image carries two binaries -- its own, which scalo's deployment contract builds, and `vector`, which has no slot in that contract. The Dockerfile is therefore generated by scalo and then spliced: `src/deployment.rs::emit_dockerfile()` puts `src/vector-layer.dockerfile` in before the generated `USER` line, with the version and the two archive checksums substituted. The other DFE Rust apps are single-binary images and need no such override. +This DFE Rust app alone ships two binaries in its runtime image: the supervisor, which scalo's deployment contract builds, and `vector`, which has no slot in that contract. The Dockerfile is therefore generated by scalo and then spliced: `src/deployment.rs::emit_dockerfile()` puts `src/vector-layer.dockerfile` in before the generated `USER` line, with the version and the two archive checksums substituted. -The fragment is a real Dockerfile rather than a Rust string literal so hadolint and shellcheck can read it. Shell embedded in `format!` is checked by nothing until the image build runs. +The fragment is a Dockerfile rather than a Rust string literal so hadolint and shellcheck can read it. ## Who owns what @@ -22,69 +22,112 @@ Most confusion here is a question asked of the wrong layer. | Layer | Owns | |---|---| -| Vector (the child) | Per-record transform work, the Kafka client on the `bus` transport, its own internal metrics | +| Vector (the child) | Per-record transform work, the Kafka client on the `bus` transport, its internal metrics | | This supervisor | Config assembly, process lifecycle and crash recovery, health reporting, the metrics merge, the direct-transport bridge, the deployment artefacts | | dfe-engine | What the app IS -- it reads dfe-infra's `apps.yaml`, compiles the app's routing into its chart values and holds the config in its registry | | dfe-infra | The image pin and the GitOps deployment | -Nothing about how this app is deployed is coded into dfe-engine. dfe-infra's `apps.yaml` declares it `per_config` with `scale_deployed: true` -- one deployment per source config, not a singleton, and KEDA drives the replica count. Its `default_in` list names only `docker-single`, so every other deploy composition gets it only when a deployment asks for it. +Nothing about how this app is deployed is coded into dfe-engine. dfe-infra's `apps.yaml` declares it `per_config` with `scale_deployed: true` -- one deployment per source config, not a singleton, and KEDA drives the replica count. Its `default_in` list names `docker-single` alone, so any other deploy composition gets it when a deployment asks for it. ## The pieces +The pod's parts are in [Figure 1 of the README](../README.md). The config engine turns a handful of big dials into a Vector config directory: a global file, a source file, the user's transform files, a sink file and an observability file. The user authors only the `transforms:` and `enrichment_tables:` blocks. `wiring` parses the component labels and `inputs` references in those files and joins `dfe_source` to `dfe_sink` through them, so a transform author does not hand-wire the topology. + +### Config assembly + +The supervisor loads and validates the config, writes the secret files and the source, then wires the transforms, writes the sink and runs `vector validate` before Vector reads the directory. + ```mermaid -flowchart LR - subgraph CFG["Config engine"] - L["loader"] --> V["validate"] --> G["generate"] --> W["wiring"] --> A["assembler"] +flowchart TB + accTitle: L4 Detail -- Config assembly + accDescr: The config is loaded through its cascade and validated, the secret files are written and the source is generated. The transform files are read and wired into a checked graph, the sink is generated, and vector validate gates the directory before Vector is started or sent a SIGHUP. + + subgraph s1["1 Prepare"] + direction LR + load["Load cascade"] --> check["Validate config"] --> secrets["Write secret files"] --> source["Generate source"] + end + subgraph s2["2 Wire and gate"] + direction LR + read["Read transforms"] --> wire["Wire and check"] --> sink["Generate sink"] --> gate["vector validate"] end - A -->|config dir| VEC["Vector subprocess"] - PM["process + lifecycle"] --> VEC - RLD["reload (poll)"] --> A - VEC -->|exporter on loopback| MM["metrics merge"] - PM --> H["health registry"] - MM --> OPS["ops port 9090"] - H --> OPS + s1 --> s2 + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 + class s1,s2 group ``` -The config engine turns a handful of big dials into a Vector config directory: a global file, a source file, the user's transform files, a sink file and an observability file. The user authors only the `transforms:` and `enrichment_tables:` blocks. `wiring` parses the component labels and `inputs` references in those files and joins `dfe_source` to `dfe_sink` through them, so a transform author does not hand-wire the topology. +*Figure 2. L4 Detail -- Config assembly. Read each stage left to right, then down.* Up: [L3 Components -- dfe-transform-vector](../README.md). -The process manager spawns Vector, forwards SIGTERM at shutdown, and restarts it after a crash with exponential backoff from one to sixty seconds. +`run` assembles into `vector.config_dir`, and a [reload](internals.md#hot-reload) into `.next`. The wiring check rejects an undefined input, a transform `dfe_source` cannot reach and a cycle. -The bridge exists on the `direct` transport only. With no broker in the path the two protocols do not meet: the rest of DFE speaks scalo's `Transport/Push` and Vector speaks `vector.Vector/PushEvents`. The supervisor translates in both directions, and both inner legs stay on loopback inside the one pod. On `bus` the bridge does nothing and Vector talks to Kafka itself. +The process manager spawns Vector, forwards SIGTERM at shutdown, and restarts it after a crash with exponential backoff from one to sixty seconds. ## Runtime invariants **Reload polls, and must.** The watcher polls rather than using inotify because S3-backed mounts -- s3fs, goofys, Mountpoint for S3 -- generate no filesystem notification events. Inotify would work on every laptop and stop reloading in production with nothing logged. -**Only transforms hot-reload.** A transform change is validated, SIGHUPed into Vector, and reported from Vector's reload counters, since the signal succeeds even when Vector refuses it. Everything else is set at startup -- the consumer, the Push listener, the bridge legs, the group id, the ops listener -- and needs a pod restart. A reload compares the app's own config with the running one, so a restart-only change in the file blocks every later transform change until a restart or a revert. An edit to scalo's sections in the file is not compared: it logs a successful reload and waits for a restart. +**Only transforms hot-reload.** A transform change is validated, SIGHUPed into Vector, and reported from Vector's reload counters, since the signal succeeds even when Vector rejects it. Everything else is set at startup -- the consumer, the Push listener, the bridge legs, the group id, the ops listener -- and needs a pod restart. A reload compares the app's config with the running one, so a restart-only change in the file blocks every later transform change until a restart or a revert. An edit to scalo's sections in the file is not compared: it logs a successful reload and waits for a restart. -**A push on the direct path is answered only once the next hop has it.** Both bridge listeners are built armed, so a push is held until the pipeline releases its records. A hop that refuses is retried until the hold runs out, and then the push is answered `Unavailable` and its sender retries. The holds nest, 18 s at the intake, 16.5 s on the leg back from Vector and 15 s per send onward, so an inner hop is answered while the outer one can still answer its sender. A record no resend could deliver is not resent forever: one over the next stage's message-size ceiling goes to the DLQ, or is dropped and counted with the DLQ off, and one Vector refused for good is released dropped and counted. +**A push on the direct path is answered only once the next hop has it.** Both bridge listeners are built armed, so a push is held until the pipeline releases its records. A hop that rejects is retried until the hold runs out, and then the push is answered `Unavailable` and its sender retries. The holds nest, 18 s at the intake, 16.5 s on the leg back from Vector and 15 s per send onward, so an inner hop times out first and the outer one can still answer its sender. A record no resend could deliver is not resent forever: one over the next stage's message-size ceiling goes to the DLQ, or is dropped and counted with the DLQ off, and one Vector refused for good is released dropped and counted. **Only the intake sheds under pressure.** The leg back from Vector drains everything the intake holds, so shedding it would stall the stage until the holds expired into duplicates. Both listeners lease held bytes on the runtime's memory guard. **A failed bridge leg stops the process.** Its senders would otherwise retry for ever against a live pod, so the service exits non-zero. -**Transforms cannot reach the Kafka credentials.** Vector runs whatever a transform file declares, with read access to the credentials. So a transform file declares only `transforms` (never `lua`) and `enrichment_tables`, each table resolves inside `transforms.data_dir` after `..` and symlinks and outside every credential path (the config directory, a mounted `sasl.secret_dir`, a TLS key), and a `SECRET[...]` naming an assembler backend is refused. Vector also runs without the `DFE_TRANSFORM_VECTOR_*` variables. +**Transforms cannot reach the Kafka credentials.** Vector runs whatever a transform file declares, with read access to the credentials. So a transform file declares only `transforms` (never `lua`) and `enrichment_tables`, each table resolves inside `transforms.data_dir` after `..` and symlinks and outside every credential path (the config directory, a mounted `sasl.secret_dir`, a TLS key), and a `SECRET[...]` naming an assembler backend is refused. Vector runs without the `DFE_TRANSFORM_VECTOR_*` variables. ## Health and metrics invariants -**Port 9598 is a loopback debug surface, not a scrape target.** Vector's own `prometheus_exporter` binds it. The supervisor GETs it on scalo's metrics interval and registers every sample on the registry scalo already serves and pushes over OTLP, so one endpoint carries both. No second Prometheus target and no PodMonitor selector are involved. The merge is also what makes the app's own throughput counters non-zero, since Vector owns the Kafka client and the supervisor never sees a record on the bus transport. +**Port 9598 is a loopback debug surface, not a scrape target.** Vector's `prometheus_exporter` binds it. The supervisor GETs it on scalo's metrics interval and registers every sample on the registry scalo serves and pushes over OTLP, so one endpoint serves both. The merge is what makes the app's throughput counters non-zero, since Vector owns the Kafka client and the supervisor never sees a record on the bus transport. -**`/readyz` proves Vector runs and its sink moves, not that records flow.** The gate is that the child was still alive 500 ms after spawn, which catches bad argv, an unreadable config and a pod in crash-recovery backoff, and that the sink has not held records for `metrics.sink_stall_secs` without delivering any. A large program can take Vector longer than 500 ms to build, so the pod can report ready while Vector is still coming up. A pod whose broker is unreachable, with nothing held, stays Ready. So does a bus config with no topics, which idles with no Vector running. +**`/readyz` checks that Vector runs and its sink moves.** The gate is that the child was still alive 500 ms after spawn, which catches bad argv, an unreadable config and a pod in crash-recovery backoff, and that the sink has not held or refused records for `metrics.sink_stall_secs` without delivering any. A large program can take Vector longer than 500 ms to build, so the pod can report ready before Vector has finished starting. A pod with an unreachable broker and nothing held stays Ready. So does a bus config with no topics, which idles with no Vector running. ## Build, chart and licence invariants -**The Vector version has one source.** The constant `VECTOR_VERSION` in `src/deployment.rs` is substituted into the Dockerfile's `ARG VECTOR_VERSION`, and both `VectorConfig::default()` and the contract's `default_config` read it for `vector.version`. `vector.version_check: strict` refuses to start a pod when the config's version and the binary's disagree, so a deployment that sets `vector.version` itself has to move it with the image. dfe-infra's `versions.yaml` records the same release as `services.vector`, and dfe-fetcher's container Vector extractor runs that image. +**The Vector version has one source.** The constant `VECTOR_VERSION` in `src/deployment.rs` is substituted into the Dockerfile's `ARG VECTOR_VERSION`, and both `VectorConfig::default()` and the contract's `default_config` read it for `vector.version`. `vector.version_check: strict` stops a pod starting when the config's version and the binary's disagree, so a deployment that sets `vector.version` itself has to move it with the image. dfe-infra's `versions.yaml` records the same release as `services.vector`, and dfe-fetcher's container Vector extractor runs that image. **`password` is a `String` on purpose, not a `SensitiveString`.** The config goes through a figment serialize-merge-deserialize round trip in `apply_figment_env()`, and `SensitiveString` serialises as `***REDACTED***`, which destroys the value in transit. The protection is elsewhere: the env var is masked in logs, `Debug` redacts the field, and no credential is written into the Vector config. The generated components name it `SECRET[.]` and Vector's `directory` secret backend reads it from files: a mounted secret at `sasl.secret_dir`, or owner-only files the assembler writes under `/.secrets` from a credential given as text. The released chart takes the second path: its Kafka Secret arrives in `DFE_TRANSFORM_VECTOR_{SOURCE,SINK}_SASL_*`, and `` sits on the `run` writable path. -**Vector expands no `${VAR}`, and `vector validate` resolves no `SECRET[...]`.** Since 0.57 interpolation needs `--dangerously-allow-env-var-interpolation` or `VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION`, and the supervisor sets neither. `validate()` refuses a `${...}` in a SASL credential. Validate would hand the broker the literal `SECRET[...]` reference, so the supervisor runs it with `--no-environment`, and a missing secret file fails only when Vector starts -- which is why the assembler checks `secret_dir` itself. +**Vector expands no `${VAR}`, and `vector validate` resolves no `SECRET[...]`.** Since 0.57 interpolation needs `--dangerously-allow-env-var-interpolation` or `VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION`. The supervisor passes neither, but Vector inherits the pod's environment, so keep that variable unset. `validate()` rejects a `${...}` in a SASL credential. Validate would hand the broker the literal `SECRET[...]` reference, so the supervisor runs it with `--no-environment`, and a missing secret file fails when Vector starts, not at validate -- which is why the assembler checks `secret_dir` itself. -Vector is redistributed unmodified under MPL-2.0, which carries two obligations -- ship the licence text and tell recipients where the source is. Both are discharged inside the image from the release archive's own `LICENSE`, `NOTICE` and `licenses` tree, so the attribution always matches the exact binary shipped. Do not replace that with a separate download. +Vector is redistributed unmodified under MPL-2.0, which has two obligations -- ship the licence text and tell recipients where the source is. Both are discharged inside the image from the release archive's `LICENSE`, `NOTICE` and `licenses` tree, so the attribution always matches the exact binary shipped. Do not replace that with a separate download. -**No chart is committed.** At release, hyperi-ci emits the contract with `generate-artefacts` and assembles a thin chart from it on the scalo-service library chart at `release.helm.library`. `emit-contract` prints what the contract fixes: the startup budget, `kubernetes.io/h2c` on the Push port, the Kafka Secret mounted into the per-endpoint SASL names, the writable paths `data`, `run` and `dlq` under a read-only root, the requests and limits, and the grace period. +**No chart is committed.** At release, hyperi-ci emits the contract with `generate-artefacts` and assembles a thin chart from it on the scalo-service library chart at `release.helm.library`. [internals.md](internals.md#the-chart) lists what the contract fixes. -**The ScaledObject scales on CPU alone.** `contract()` sets `KafkaLagTrigger::disabled()`: consumer-group lag rises when a downstream stage breaks, and the library's lag trigger reads `config.kafka.*`, a block this app does not have, so the library refuses to render it. A deployment adds a trigger with the library's `keda.extraTriggers` value. +**The ScaledObject scales on CPU alone.** `contract()` sets `KafkaLagTrigger::disabled()`: consumer-group lag rises when a downstream stage breaks, and the library's lag trigger reads `config.kafka.*`, a block this app does not have, so the library will not render it. A deployment adds a trigger with the library's `keda.extraTriggers` value. ## Where the transports differ -One deployment runs one transport and the transform you author is identical on both. What changes is who hands the record over: on `bus` it is a Kafka topic at each end and Vector uses its own `kafka` source and sink, on `direct` it is a scalo Push listener in and a Push client out with Vector on `vector` components over loopback. `templates/bus.yaml` and `templates/direct.yaml` are the whole topology for each, and `vector validate` passes on both as they stand. In a DFE deployment the supervisor generates the `sources:` and `sinks:` blocks, so a whole template mounted as a transform file is refused. +`source.transport` and `sink.transport` each pick `bus` or `direct`, and the transform you author is identical on both. What changes is who hands the record over: on `bus` it is a Kafka topic and Vector uses its `kafka` source and sink, on `direct` there is no broker, the rest of DFE speaks scalo's `Transport/Push` and Vector speaks `vector.Vector/PushEvents`, so the supervisor's bridge translates in both directions over loopback. `templates/bus.yaml` and `templates/direct.yaml` are the whole topology for each, and `vector validate` passes on both as they stand. In a DFE deployment the supervisor generates the `sources:` and `sinks:` blocks, so a whole template mounted as a transform file is refused. + +On direct a record enters at the intake listener, crosses Vector over loopback and leaves from the return listener, which sends a record over the next stage's size ceiling to the DLQ. + +```mermaid +flowchart LR + accTitle: L4 Detail -- Direct transport bridge + accDescr: The previous stage pushes records to the intake listener, which hands them to Vector over loopback. Vector's output comes back to the return listener, which pushes it to the next stage, or sends a record over the size ceiling to the DLQ. + + prev["Previous
stage"] + intake["Intake
listener"] + vec["Vector"] + ret["Return
listener"] + next["Next
stage"] + dlq[("DLQ")] + + prev -->|Push| intake + intake --> vec + vec --> ret + ret -->|Push| next + ret -.->|too large| dlq + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef error fill:#FBE3D6,stroke:#C2410C,stroke-width:1px,color:#4A1A05 + classDef ext fill:#EEF0F2,stroke:#6B7280,stroke-width:1px,stroke-dasharray:4 3,color:#1F2937 + class prev,next ext + class dlq error +``` + +*Figure 3. L4 Detail -- Direct transport bridge. The dotted arrow is the dead letter path.* Up: [L3 Components -- dfe-transform-vector](../README.md). + +The intake listens on `source.listen`, Vector's `vector` source on `bridge.to_vector`, the return listener on `bridge.from_vector`, and the onward client dials `sink.endpoint`. A stage with one end on `direct` runs that half alone. diff --git a/docs/internals.md b/docs/internals.md index 8f3a6f7..71d2a67 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -4,16 +4,18 @@ Field-level depth under [architecture.md](architecture.md): how a config becomes ## The config -The app's own config layers in this order, each over the one before: +The app's config layers in this order, each over the one before: 1. The defaults, which [config-schema.yaml](config-schema.yaml) lists. 2. The config file: `--config`, else `config.yaml` or `config.yml` in the working directory. 3. `DFE_TRANSFORM_VECTOR_
__`, which reaches any key. A list takes brackets: `DFE_TRANSFORM_VECTOR_SOURCE__BROKERS='[a:9092,b:9092]'`. 4. The flat names, such as `DFE_TRANSFORM_VECTOR_SOURCE_BROKERS=a:9092,b:9092`, `_SINK_TOPIC`, `_TRANSFORMS_DIR`, `_VECTOR_BINARY` and the SASL pairs. -The working directory's `.env` is read into the environment before the env layers, and a variable already set wins. `--log-level`, `--log-format` and `--metrics-addr`, or `LOG_LEVEL`, `LOG_FORMAT` and `METRICS_ADDR`, outrank `logging.*` and `metrics.address`. Nothing reads `DFE_TRANSFORM_VECTOR_CONFIG`: name the file with `--config`, before any subcommand. +The working directory's `.env` is read into the environment before the env layers, and a variable set before it wins. `--log-level`, `--log-format` and `--metrics-addr`, or `LOG_LEVEL`, `LOG_FORMAT` and `METRICS_ADDR`, outrank `logging.*` and `metrics.address`. Nothing reads `DFE_TRANSFORM_VECTOR_CONFIG`: name the file with `--config`, before any subcommand. -The same file seeds scalo's own cascade, so scalo's sections in it apply too: `version_check`, `scaling.enabled`, `scaling.memory_gate_threshold` and `metrics.namespace`. `dfe_source` fills in `_land`, `_load` and the consumer group, `dfe-transform-vector-` or `dfe-transform-vector-` without a pipeline name, wherever the config leaves them at their defaults. +Two flat names cover both endpoints. A `DFE_TRANSFORM_VECTOR_KAFKA_SECURITY_PROTOCOL` naming SSL, such as `SASL_SSL`, turns on `source.tls.enabled` and `sink.tls.enabled`, and any other value leaves them as they were. `DFE_TRANSFORM_VECTOR_KAFKA_SASL_MECHANISM` sets both SASL mechanisms, and the per-endpoint `_SOURCE_SASL_MECHANISM` and `_SINK_SASL_MECHANISM` beat it. `source.transport` and `sink.transport` take `kafka` for `bus` and `grpc` for `direct`, the names the other DFE stages use. + +The same file seeds scalo's cascade as well, so scalo's sections in it apply: `version_check`, `scaling.enabled`, `scaling.memory_gate_threshold` and `metrics.namespace`. `dfe_source` fills in `_land`, `_load` and the consumer group, `dfe-transform-vector-` or `dfe-transform-vector-` without a pipeline name, wherever the config leaves them at their defaults. ```yaml pipeline: @@ -57,9 +59,9 @@ transforms: | `99_observability.yaml` | `internal_metrics` into a `prometheus_exporter` on `metrics.vector_metrics_address` | | `.secrets/` | Owner-only files for a SASL credential given as text | -Credentials appear only as references: `SECRET[dfe_source_sasl.username]` for a `secret_dir`, `SECRET[dfe_credentials.source_sasl_username]` for a credential given as text. The assembler refuses a `secret_dir` that lacks its `username` or `password` file. +Credentials appear only as references: `SECRET[dfe_source_sasl.username]` for a `secret_dir`, `SECRET[dfe_credentials.source_sasl_username]` for a credential given as text. The assembler rejects a `secret_dir` that lacks its `username` or `password` file. -`dfe_size_cap` drops a record whose encoded length is over the producer's `message.max.bytes` less 128 bytes for the key, headers and framing. `message.max.bytes` is scalo's pipeline-wide record ceiling of 16 MiB unless the central profile or `sink.librdkafka_options` sets it. librdkafka's own default of 1,000,000 bytes would refuse records the rest of the chain carries. +`dfe_size_cap` drops a record with an encoded length over the producer's `message.max.bytes` less 128 bytes for the key, headers and framing. `message.max.bytes` is scalo's pipeline-wide record ceiling of 16 MiB unless the central profile or `sink.librdkafka_options` sets it. librdkafka's default of 1,000,000 bytes would refuse records the rest of the chain accepts. The wiring follows these rules: @@ -76,76 +78,97 @@ The wiring follows these rules: The watcher polls the config file and the `.yaml` and `.yml` files of `transforms.dir`, or the listed `transforms.files`, every `reload.poll_interval_secs` (30 by default). A SIGHUP to the supervisor starts the same reload. Enrichment table files are not watched, and with `reload.enabled: false` nothing is, and a SIGHUP is logged and ignored. 1. The config is re-read and validated. -2. A change to the app's own keys outside the `transforms` block is refused. The watcher logs `non-transform config changed -- requires pod restart`, counts `rejected`, and keeps the running config. The change stays in the file, so every later transform change is refused too until the pod restarts or the edit is reverted, which lets the next transform change reload. scalo's sections in the file are read once at startup and not compared: an edit to one needs a restart, and the watcher reloads the transforms and counts `success` without applying it. +2. A change to the app's keys outside the `transforms` block is refused. The watcher logs `non-transform config changed -- requires pod restart`, counts `rejected`, and keeps the running config. The change stays in the file, so every later transform change is refused until the pod restarts or the edit is reverted, which lets the next transform change reload. scalo's sections in the file are read once at startup and not compared: an edit to one needs a restart, and the watcher reloads the transforms and counts `success` without applying it. 3. A transform change is assembled into `.next`, its table files checked and `vector validate` run there. A failure is logged and counted `error`, and the directory Vector reads is untouched. -4. The two directories swap in one `renameat2(RENAME_EXCHANGE)`, so a Vector restarted mid-reload reads the old one or the new one. A filesystem that cannot swap directories gets the files copied in place, which is not atomic. +4. The two directories swap in one `renameat2(RENAME_EXCHANGE)`, so a Vector restarted mid-reload reads the old directory or the staged one. A filesystem that cannot swap directories gets the files copied in place, which is not atomic. 5. Vector gets a SIGHUP, and its exporter is read for the outcome: `vector_reloaded_total` counts `success`, a refusal in `vector_component_errors_total` counts `error` and swaps the old directory back, and no answer within 60 s counts `unconfirmed`. -A failed attempt is retried on the same files after one poll interval, doubling to five minutes. `transform_vector_config_reloads_total{result}` carries `success`, `error`, `rejected` and `unconfirmed`. scalo's own `config_reloads_total{result}` counts `success` and `error`, a rejection included. +A failed attempt is retried on the same files after one poll interval, doubling to five minutes. `transform_vector_config_reloads_total{result}` counts `success`, `error`, `rejected` and `unconfirmed`. scalo's `config_reloads_total{result}` counts `success` and `error`, a rejection included. In a chart deployment an edit to the config or a file set rolls the pods anyway, because the chart hashes both into the pod template. ## The Vector subprocess +Vector is `Running` once it is alive 500 ms after spawn, and an exit nobody asked for is a crash, followed by a restart after the backoff. + ```mermaid stateDiagram-v2 + direction LR + accTitle: L4 Detail -- Vector lifecycle + accDescr: The supervisor initialises and validates, then starts Vector. Vector is running once it is alive 500 ms after spawn, and a reload returns to running. A crash from starting or running waits out the backoff and starts again. Shutdown ends the lifecycle. + state "Shutting down" as ShuttingDown [*] --> Initialising Initialising --> Validating Validating --> Starting - Starting --> Running: alive 500 ms after spawn - Running --> Reloading - Reloading --> Running - Starting --> Crashed - Running --> Crashed - Crashed --> Starting: after backoff - Running --> ShuttingDown + Starting --> Running: settled + Running --> Reloading: change + Reloading --> Running: done + Starting --> Crashed: exit + Running --> Crashed: exit + Crashed --> Starting: backoff + Running --> ShuttingDown: SIGTERM ShuttingDown --> [*] + classDef error fill:#FBE3D6,stroke:#C2410C,stroke-width:1px,color:#4A1A05 + class Crashed error ``` +*Figure 4. L4 Detail -- Vector lifecycle. Orange is the crash state.* Up: [L3 Components -- dfe-transform-vector](../README.md). + +Of these states `/readyz` can answer 200 in `Running` and `Reloading` alone, and the sink and idle rules under Health and metrics apply on top. + Startup runs in this order: the config loads and validates, the runtime binds the ops listener, `/readyz` starts reporting the lifecycle, `vector --version` is checked against `vector.version`, the directory is assembled, `vector validate` passes, the bridge binds its listeners on direct, and Vector is spawned. A failure before the spawn stops the process with its error. Any exit Vector was not asked for is a crash, exit 0 included. `crashes_total` counts it, the lifecycle reads `Crashed`, and Vector is respawned after a backoff that starts at 1 s, doubles to 60 s and resets once a run has lasted five minutes. There is no crash limit, and `restarts_total` counts each respawn. One Vector runs per pod, and replicas scale out. On SIGTERM or SIGINT the intake stops, Vector gets a SIGTERM and 55 s to exit before a SIGKILL, the bridge drains for up to 15 s, and the process exits 0. A failed bridge leg runs the same shutdown and exits non-zero. -At startup the supervisor also runs `vector --version` and compares the result with `vector.version`. `strict` and `warn` emit a security data-quality event on a mismatch, and no metric records it. +On a version mismatch `strict` and `warn` emit a security data-quality event, and no metric records it. | `vector.version_check` | On a mismatch | |---|---| -| `strict` (the default) | Logs it and refuses to start | -| `warn` | Logs a warning and carries on | +| `strict` (the default) | Logs it and exits | +| `warn` | Logs a warning and continues | | `disabled` | Runs no check, not even `vector --version` | ## Health and metrics | Path | Answers | |---|---| -| `/livez` | 200 while the ops listener answers | +| `/livez` | 200 whenever the ops listener answers | | `/readyz` | 200 or 503, below | | `/metrics` | The supervisor's metrics and every merged `vector_*` | | `/scaling/pressure` | scalo's scaling pressure | -All of them are on `metrics.address`, 9090 by default. `/readyz` answers 503 `{"status":"starting"}` until the runtime hands the service over. A bus config with no topics then idles: 200, `pipeline_idle` 1, no Vector, until the config file names a topic. Once running, it answers 503 unless the lifecycle is `Running` or `Reloading`, and while the sink has held records for `metrics.sink_stall_secs` (60 by default) without delivering any. On direct the bridge's gRPC transports report there too. `Reloading` is entered only from `Running`, so a change landing mid crash-backoff cannot turn it to 200. +All of them are on `metrics.address`, 9090 by default. `/readyz` answers 503 `{"status":"starting"}` until the runtime hands the service over. A bus config with no topics then idles: 200, `pipeline_idle` 1, no Vector, until the config file names a topic. Once running, it answers 503 unless the lifecycle is `Running` or `Reloading`, and when the sink has held or refused records for `metrics.sink_stall_secs` (60 by default) without delivering any. On direct the bridge's gRPC transports report there as well. `Reloading` is entered only from `Running`, so a change landing mid crash-backoff cannot turn it to 200. -Metric names are bare unless `metrics.namespace` is set, and `dfe-transform-vector metrics-manifest` prints the catalogue. The supervisor's own: +Metric names are bare unless `metrics.namespace` is set, and `dfe-transform-vector metrics-manifest` prints the catalogue. The supervisor's metrics: - `transform_vector_lifecycle_state{state}`, 1 on the current state, and `pipeline_ready`. - `crashes_total`, `restarts_total` and `uptime_seconds`, since the service started. - `transform_vector_config_reloads_total{result}`, `config_reloads_total{result}` and `config_validation_errors_total`. - `transform_vector_scrape_failures_total` and `transform_vector_sink_errors_total`. -- `records_received_total`, `records_processed_total`, `records_delivered_total` and `records_error_total`, derived from Vector's component counters on `dfe_source`, `dfe_sink` and `dfe_size_cap`. +- The record counters below, derived from Vector's component counters, since Vector and not the supervisor owns the Kafka client. -The scrape reads Vector's exporter on scalo's metrics interval and re-registers every sample. A counter is set with `absolute` and carried across Vector restarts: a total that falls is a new Vector, and the old count becomes a base. A histogram arrives pre-aggregated, so each `_bucket` registers as a counter with its `le` label. A gauge Vector stops reporting is zeroed after `metrics.vector_metrics_expiry_ticks` scrapes, a counter is forgotten after 240, and at most 50,000 series merge. +| Supervisor metric | Vector counter it reads | +|---|---| +| `records_received_total` | `vector_component_received_events_total{component_id="dfe_source"}` | +| `records_processed_total`, `records_delivered_total` | `vector_component_sent_events_total{component_id="dfe_sink"}` | +| `records_error_total` | `vector_component_discarded_events_total` on `dfe_sink` and on `dfe_size_cap` | +| `transform_vector_sink_errors_total` | `vector_component_errors_total{component_id="dfe_sink"}` | +| `pipeline_dead_letters_dropped_total{reason="too_large"}` | the growth of `vector_component_discarded_events_total` on `dfe_size_cap` | +| `pipeline_dead_letters_dropped_total{reason="rejected"}` | the growth of `vector_component_discarded_events_total` on `dfe_sink` | + +The scrape reads Vector's exporter on scalo's metrics interval and re-registers every sample. A failed scrape is not fatal: it counts `transform_vector_scrape_failures_total`, warns at most once every five minutes, and leaves readiness as it was. A response over 4 MiB is a failed scrape, not a partial merge. A counter is set with `absolute` and carried across Vector restarts: a total that falls is a restarted Vector, and the old count becomes a base. A histogram arrives pre-aggregated, so each `_bucket` registers as a counter with its `le` label. A gauge Vector stops reporting is zeroed after `metrics.vector_metrics_expiry_ticks` scrapes, a counter is forgotten after 240, and at most 50,000 series merge. -The contract's ScaledObject scales on CPU and has no Kafka lag trigger. dfe-infra's values set a CPU target and, on its single, scale and mesh profiles, add a pressure trigger through `keda.extraTriggers`. The `scaling_pressure` this app publishes stays 0 while Vector runs, since nothing feeds it a component or a memory reading, and `spawn_circuit_gate_task` pins it to 0 while Vector is down. +dfe-infra's values set a CPU target and, on its single, scale and mesh profiles, add a pressure trigger through `keda.extraTriggers`. The `scaling_pressure` this app publishes stays 0 when Vector runs, since nothing feeds it a component or a memory reading, and `spawn_circuit_gate_task` pins it to 0 when Vector is down. ## Delivery and dead letters -With `source.acknowledgements.enabled`, the default, delivery is at-least-once on both transports. On bus the sink's acknowledgements hold Vector's source offsets until delivery, so a crash redelivers from the last committed offset. On direct a push is answered once the next hop has it. With acknowledgements off, both happen at receipt. +With `source.acknowledgements.enabled`, the default, delivery is at-least-once on both transports. On bus the sink's acknowledgements hold Vector's source offsets until delivery, so a crash redelivers from the last committed offset. On direct a push is answered once the next hop has it. With acknowledgements off, both answer at receipt. -On direct a record over the next stage's message-size ceiling goes to scalo's file DLQ, NDJSON under `/var/spool/scalo/dlq` on the contract's `dlq` writable path. With `dlq.enabled: false` it is dropped and counted in `pipeline_dead_letters_dropped_total{reason="too_large"}`. A record Vector refuses for good is dropped and counted under `reason="rejected"`, whatever `dlq` says. On bus no DLQ runs: the size cap's drops count under `too_large` and the Kafka sink's under `rejected`. Off the image the DLQ directory is not writable, so a direct run logs `DLQ enabled but no backend started` and leaves dead letters unacknowledged at their source: point `dlq.file.path` at a directory you can write. +On direct a record over the next stage's message-size ceiling goes to scalo's file DLQ, NDJSON under `/var/spool/scalo/dlq` on the contract's `dlq` writable path. With `dlq.enabled: false` it is dropped and counted in `pipeline_dead_letters_dropped_total{reason="too_large"}`. A record Vector rejects for good is dropped and counted under `reason="rejected"`, whatever `dlq` says. On bus no DLQ runs: the size cap's drops count under `too_large` and the Kafka sink's under `rejected`. When no enabled DLQ backend can start, such as a file DLQ with no writable directory off the image, a direct run logs `DLQ enabled but no backend started` and leaves dead letters unacknowledged at their source. Point `dlq.file.path` at a directory you can write. -On direct both bridge listeners lease the bytes they hold on scalo's memory guard. On bus Vector holds the records, bounded by its `buffer` settings and the container's memory limit. +On bus Vector holds the records, bounded by its `buffer` settings and the container's memory limit. `vector.data_dir` sits on the contract's `data` writable path, an emptyDir unless `writablePaths.data.persistence.enabled` gives it a claim, so a disk buffer outlives a Vector restart but not a pod. The supervisor logs `DISK BUFFER HAS NO DURABLE STORAGE` at startup when a disk buffer sits on ephemeral storage. @@ -154,16 +177,14 @@ On direct both bridge listeners lease the bytes they hold on scalo's memory guar `VECTOR_VERSION` and the two archive checksums in `src/deployment.rs` are the only pin. The Dockerfile, the default `vector.version`, the contract's `default_config` and `scripts/fetch-vector.sh` all read the constant. 1. Change `VECTOR_VERSION`, and `VECTOR_SHA256_AMD64` and `VECTOR_SHA256_ARM64` from Vector's published `SHA256SUMS` for that release. -2. Regenerate the Dockerfile: `cargo run --bin dfe-transform-vector -- emit-dockerfile > Dockerfile`. `checked_in_dockerfile_matches_emit_dockerfile` fails while it drifts. +2. Regenerate the Dockerfile: `cargo run --bin dfe-transform-vector -- emit-dockerfile > Dockerfile`. `checked_in_dockerfile_matches_emit_dockerfile` fails when it drifts. 3. Move `services.vector` in dfe-infra's `versions.yaml` to the same release, tagged `-alpine`. dfe-fetcher's container Vector extractor runs that image. -4. Release this repo. The full test tier, nightly and before a release, runs the `vector_validate` cases against the new Vector. +4. Release this repo. The full test tier, nightly and before a release, runs the `vector_validate` cases against the bumped Vector. ## The chart -No chart is committed. At release hyperi-ci runs `generate-artefacts` for the deployment contract and assembles a thin chart on the scalo-service library chart, at the version `release.helm.library` names in `.hyperi-ci.yaml`. The library renders the Deployment, Service, ConfigMap, ServiceAccount and KEDA ScaledObject, and its README documents the values. - -`dfe-transform-vector emit-contract` prints what the contract fixes: the probes on the metrics port and their startup budget, the Push port while `config.source.transport` is `direct` or `grpc`, the Kafka Secret in `DFE_TRANSFORM_VECTOR_{SOURCE,SINK}_SASL_{USERNAME,PASSWORD}` and `DFE_TRANSFORM_VECTOR_KAFKA_SASL_MECHANISM`, the writable paths `data`, `run` and `dlq` beside the library's `/tmp` under a read-only root, the requests and limits, the grace period, and KEDA on CPU with no Kafka lag trigger. +No chart is committed. At release hyperi-ci runs `generate-artefacts` for the deployment contract and assembles a thin chart on the scalo-service library chart, at the version `release.helm.library` names in `.hyperi-ci.yaml`. The library renders the Deployment, Service, ConfigMap, ServiceAccount and KEDA ScaledObject, and its README documents the values. To see the chart a release would ship, build the binary and run `hyperi-ci chart assemble --binary target/debug/dfe-transform-vector --image ghcr.io/hyperi-io/dfe-transform-vector:@sha256:`, which prints the directory it wrote. -The mounted config is the chart's `config` with `configOverrides` merged over it. The contract's `default_config` is never merged in, so the app's own defaults apply to anything a deployment leaves unset, `vector.version_check: strict` included. The ConfigMap mounts over the config file's whole directory unless `configMount.subPath` is set, which hides the image's `/etc/dfe-transform-vector/transforms`: deliver transform files with a `fileSets` entry and point `transforms.dir` at its `mountPath`. +`dfe-transform-vector emit-contract` prints what the contract fixes: the probes on the metrics port and their startup budget, the Push port when `config.source.transport` is `direct` or `grpc`, the Kafka Secret in `DFE_TRANSFORM_VECTOR_{SOURCE,SINK}_SASL_{USERNAME,PASSWORD}` and `DFE_TRANSFORM_VECTOR_KAFKA_SASL_MECHANISM`, the writable paths `data`, `run` and `dlq` beside the library's `/tmp` under a read-only root, the requests and limits, the grace period, and KEDA on CPU with no Kafka lag trigger. -dfe-infra's `apps.yaml` declares the app `per_config`, so each source config gets its own Deployment, consumer group and ScaledObject. The app is in the default composition of `docker-single` only. +The mounted config is the chart's `config` with `configOverrides` merged over it. The contract's `default_config` is never merged in, so the app's defaults apply to anything a deployment leaves unset, `vector.version_check: strict` included. The ConfigMap mounts over the config file's whole directory unless `configMount.subPath` is set, which hides the image's `/etc/dfe-transform-vector/transforms`: deliver transform files with a `fileSets` entry and point `transforms.dir` at its `mountPath`. diff --git a/docs/librdkafka.md b/docs/librdkafka.md index 6480acf..7c22e1d 100644 --- a/docs/librdkafka.md +++ b/docs/librdkafka.md @@ -19,7 +19,7 @@ Vector applies `librdkafka_options` over the settings it writes itself from the | librdkafka key | Written by Vector from | |---|---| | `session.timeout.ms`, `auto.commit.interval.ms`, `auto.offset.reset` | `source.session_timeout_ms` (30000), `source.commit_interval_ms` (5000), `source.auto_offset_reset` (`largest`) | -| `enable.auto.commit`, `enable.auto.offset.store`, `statistics.interval.ms` | Vector's own source, as `true`, `false` and `1000` | +| `enable.auto.commit`, `enable.auto.offset.store`, `statistics.interval.ms` | Vector's kafka source code, as `true`, `false` and `1000` | | `message.timeout.ms`, `socket.timeout.ms` | `sink.message_timeout_ms` (0, no limit), `sink.socket_timeout_ms` (60000) | | `queue.buffering.max.ms`, `batch.num.messages`, `batch.size` | `sink.batch.timeout_secs`, `.max_events` and, where set, `.max_bytes` | @@ -34,7 +34,7 @@ A few keys are derived from the big dials and written after the merge, so `librd | `sasl.mechanism` | `{source,sink}.sasl.mechanism` | SASL is on | | `enable.ssl.certificate.verification` | `{source,sink}.tls.skip_verify` | TLS is on and verification is skipped | -`config-check` refuses a config that sets one of these in `librdkafka_options`, and names the big dial that owns the key. The conditions are part of the rule: with SASL off nothing derives `sasl.mechanism`, so setting it by hand there is allowed. Any other key can be set per instance: +`config-check` rejects a config that sets one of these in `librdkafka_options`, and names the big dial that owns the key. With SASL off nothing derives `sasl.mechanism`, so setting it by hand there is allowed. Any other key can be set per instance, bar two checks. `config-check` rejects `enable.auto.offset.store: true` in any layer of the source consumer's options, the central profile included, because librdkafka would then store offsets at fetch, before the sink acknowledges. The client config built from each endpoint's brokers, SASL, TLS and `librdkafka_options` must pass scalo's Kafka security floor. ```yaml source: @@ -46,9 +46,7 @@ sink: queue.buffering.max.kbytes: "524288" # 512 MiB ``` -## Example - -For a producer with no central file, `sink.compression: lz4` and `sink.librdkafka_options: {linger.ms: "20"}`, the supervisor writes `batch.timeout_secs: 0.02` and `compression: lz4` on the sink and, in `librdkafka_options`, `queue.buffering.max.kbytes=262144`, `enable.idempotence=true`, `socket.nagle.disable=true`, `statistics.interval.ms=1000` and `message.max.bytes=16777216`, the last because no layer set it. +For example, for a producer with no central file, `sink.compression: lz4` and `sink.librdkafka_options: {linger.ms: "20"}`, the supervisor writes `batch.timeout_secs: 0.02` and `compression: lz4` on the sink and, in `librdkafka_options`, `queue.buffering.max.kbytes=262144`, `enable.idempotence=true`, `socket.nagle.disable=true`, `statistics.interval.ms=1000` and `message.max.bytes=16777216`, the last because no layer set it. Add a central `producer.production` naming only `linger.ms: "50"` and `socket.nagle.disable` and `statistics.interval.ms` drop out of `librdkafka_options`, because the central profile replaced the constants rather than merging over them. Vector still writes `statistics.interval.ms=1000` itself. The delay stays 20 ms: the per-instance option outranks the central file. Drop that option and it becomes 50 ms. @@ -71,7 +69,7 @@ This app reads `CONSUMER_PRODUCTION` for the source and `PRODUCER_PRODUCTION` fo |---------|-------|--------------------|---------------| | `linger.ms` | `100` | `5` | Larger batches -- 100 ms of latency for higher throughput. This is the delay the sink uses, in place of `sink.batch.timeout_secs`. See Layers. | | `compression.type` | `zstd` | `none` | Best compression ratio with acceptable CPU cost. `sink.compression` replaces it on every sink. | -| `socket.nagle.disable` | `true` | `true` | Already librdkafka's default. The profile states it. | +| `socket.nagle.disable` | `true` | `true` | librdkafka's default as well. The profile states it. | | `statistics.interval.ms` | `1000` | `0` | librdkafka metrics. | Left at librdkafka's default, unless a layer or a Vector field above writes them: @@ -83,7 +81,7 @@ Left at librdkafka's default, unless a layer or a Vector field above writes them | `heartbeat.interval.ms` | `3000` | | `max.poll.interval.ms` | `300000` | | `fetch.max.bytes` | `52428800` (50 MiB) | -| `acks` | `all` (-1), which idempotence requires anyway | +| `acks` | `all` (-1), which idempotence needs anyway | | `retries` | `2147483647` | ## Service overrides @@ -93,7 +91,7 @@ Left at librdkafka's default, unless a layer or a Vector field above writes them | `queue.buffering.max.kbytes` | producer | `262144` (256 MiB) | librdkafka's default 1 GiB producer queue is too large for a transform-vector pod. | | `enable.idempotence` | producer | `true` | The sink retries without limit, and a retry on a non-idempotent producer can reorder and duplicate records within a partition. | | `enable.auto.commit` | consumer | `true` | Vector's kafka source stores offsets on delivery and leaves the periodic flush to librdkafka's commit timer, which the baseline's `false` never arms. Vector still commits on rebalance and clean shutdown, so the baseline left reported lag stuck between those points rather than losing offsets. | -| `message.max.bytes` | producer | `16777216` (16 MiB), scalo's `MESSAGE_MAX_BYTES` | The record ceiling every other stage's producer and every topic scalo creates carry, where librdkafka's own `1000000` refuses a larger record. Written only where neither layer 1 nor `sink.librdkafka_options` sets the key. The size cap in front of the sink reads the same value. | +| `message.max.bytes` | producer | `16777216` (16 MiB), scalo's `MESSAGE_MAX_BYTES` | The record ceiling every other stage's producer and every topic scalo creates accept, where librdkafka's default of `1000000` rejects a larger record. Written only where neither layer 1 nor `sink.librdkafka_options` sets the key. The size cap in front of the sink reads the same value. | ## Central config file @@ -117,7 +115,7 @@ producer: Only the `production` profile of each side is read. A profile in the file replaces the scalo constant set of that name, and a missing one leaves the constants in force. So does an empty one, or one holding no string, number or boolean value: it is skipped. -## Source code references +The code and the references behind this page: - Profile constants: [scalo-rs src/kafka_config.rs](https://github.com/hyperi-io/scalo-rs/blob/main/src/kafka_config.rs) - Central file loader: `src/config/kafka_defaults.rs` (`consumer_profile()`, `producer_profile()`, the `OnceLock` cache, `merge_layers()`) diff --git a/docs/migration.md b/docs/migration.md index 9adca0d..fd26e99 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -1,10 +1,6 @@ # Moving off the Vector chart -This guide covers moving an existing Vector pipeline from the official Vector Helm chart, served from `helm.vector.dev`, to dfe-transform-vector. - -## Pipeline values - -A values file for the official Vector chart carries the whole Vector config. dfe-transform-vector's chart takes the big-dial config instead, one values file per pipeline, and both releases can run side by side while you move. +A values file for the official Vector Helm chart, served from `helm.vector.dev`, holds the whole Vector config. dfe-transform-vector's chart takes the big-dial config instead, one values file per pipeline, and both releases can run side by side during the move. Before, the official Vector chart's `vector-syslog.yaml`: @@ -98,14 +94,14 @@ keda: Key differences: -- No inline VRL. Transforms are YAML files the app loads from `config.transforms.dir`, or lists in `config.transforms.files`. Deliver them with a `fileSets` entry or an image of your own: the chart's ConfigMap mounts over `/etc/dfe-transform-vector/` and hides the image's `transforms` directory there. +- No inline VRL. Transforms are YAML files the app loads from `config.transforms.dir`, or lists in `config.transforms.files`. The chart's ConfigMap mounts over `/etc/dfe-transform-vector/` and hides the image's `transforms` directory there. Deliver transforms with a `fileSets` entry, bake them into an image outside `/etc/dfe-transform-vector/` and point `transforms.dir` there, or set `configMount.subPath`. - No persistence by default. The workload is a Deployment, and `vector.data_dir` is an emptyDir unless `writablePaths.data.persistence.enabled` gives it a claim, so records not yet acknowledged are read again from Kafka after a pod restart. -- SASL and TLS can sit in one values file every pipeline layers over, so each pipeline file carries only its own values. +- SASL and TLS can sit in one values file every pipeline layers over, so a pipeline file holds the values that differ. - A `${VAR}` in the config is not expanded: neither the app nor Vector 0.57+ interpolates environment variables. Brokers and topics are big dials, and in a DFE deployment the `DFE_TRANSFORM_VECTOR_SOURCE_BROKERS` and `DFE_TRANSFORM_VECTOR_SINK_BROKERS` variables set the brokers. - SASL credentials come from the Kafka Secret through `DFE_TRANSFORM_VECTOR_{SOURCE,SINK}_SASL_{USERNAME,PASSWORD}`, and the app hands them to Vector's directory secret backend as owner-only files. -- `replicaCount` applies only while nothing autoscales the workload. KEDA scales on CPU, between `keda.minReplicaCount` and `keda.maxReplicaCount`, with no Kafka lag trigger and no TriggerAuthentication. +- `replicaCount` applies when nothing autoscales the workload. KEDA scales on CPU, between `keda.minReplicaCount` and `keda.maxReplicaCount`, with no Kafka lag trigger and no TriggerAuthentication. -Two things stay as they were. The chart creates no Secret, so the Kafka SASL secret comes from wherever your deployment already supplies it. Node placement carries over through the chart's `nodeSelector` and `tolerations`. +The chart creates no Secret, so the Kafka SASL secret comes from wherever your deployment supplies it. Node placement moves across through the chart's `nodeSelector` and `tolerations`. | Element | Before | After | |---------|--------|-------| @@ -113,9 +109,9 @@ Two things stay as they were. The chart creates no Secret, so the Kafka SASL sec | Image | `timberio/vector` | `ghcr.io/hyperi-io/dfe-transform-vector` | | Workload type | StatefulSet, the `Aggregator` role | Deployment | | Health endpoint | `/health` on Vector's API, 8686 by default | `/livez`, `/readyz` on 9090, beside `/metrics` | -| Metrics | Vector's own exporter | The app's and Vector's, merged on 9090 | +| Metrics | Vector's exporter | The app's and Vector's, merged on 9090 | | Config format | Full Vector YAML (`customConfig`) | Big-dial config (source/sink/transforms) | -| Transform delivery | Inline VRL in `customConfig` | YAML files in `transforms.dir`, from a `fileSets` entry or an image of your own | +| Transform delivery | Inline VRL in `customConfig` | YAML files in `transforms.dir`, from a `fileSets` entry or an image path outside the config directory | | DAG wiring | Manual (inline in values) | Auto-wired by the app | | Reload | Vector's `--watch-config`, where the args set it | A transform file changed in place reloads. A chart change rolls the pods | | Crash recovery | K8s pod restart only | The app restarts Vector with backoff, inside the pod | @@ -123,7 +119,7 @@ Two things stay as they were. The chart creates no Secret, so the Kafka SASL sec ## Cutover procedure 1. For each existing Vector values file, write a dfe-transform-vector values file in the big-dial format above. -2. Run both releases at once, the new one with a `sink.topic` of its own. The new pipeline uses a different consumer group ID, so it reads the same source topic independently of the existing pipeline, and its own output topic keeps the two from writing every record twice into one topic. A new group starts at the newest offset, because `source.auto_offset_reset` defaults to `largest`, so its output holds only what arrives after it starts. Set `smallest` to have it read the topic's backlog too. +2. Run both releases at once, the new one with a separate `sink.topic`. The new pipeline uses a different consumer group ID, so it reads the same source topic independently of the existing pipeline, and its separate output topic keeps the two from writing every record twice into one topic. A group with no committed offset starts at the newest one, because `source.auto_offset_reset` defaults to `largest`, so its output holds only what arrives after it starts. Set `smallest` to have it read the topic's backlog as well. 3. Validate: compare the output topics of the old and new pipelines, watch throughput, latency and error rates, check the health endpoints, and check KEDA scales. 4. Scale the old pipeline to 0 replicas and let downstream consumers drain its output topic, then point them at the new output topic. Records both pipelines carried during the overlap are in both topics, so a consumer that starts the new topic at its beginning reads those twice. Start it past the last record it took from the old topic, or deduplicate. 5. Monitor for 24-48 hours, then remove the old Vector release and its values file. diff --git a/pipelines/filebeat/README.md b/pipelines/filebeat/README.md index 7f16b7a..28ee76c 100644 --- a/pipelines/filebeat/README.md +++ b/pipelines/filebeat/README.md @@ -12,9 +12,7 @@ Copyright: (c) 2026 HYPERI PTY LIMITED A pre-canned transform file that keeps DFE/XDR 2.0 and 2.1 filebeat feeds working on dfe-transform-vector: the same VRL dfe-transform-vrl runs, wrapped in a Vector `remap` and carrying the enrichment table it needs. Point `transforms.dir` at this directory, mount the table, and the pipeline runs -- there is nothing to author. The modules, the routing rules, the departures from DFE 2.1 and the known limitations are documented once, in dfe-transform-vrl `pipelines/filebeat/README.md`, and apply here unchanged. -> **LEGACY.** Backwards compatibility for DFE/XDR 2.0 and 2.1 filebeat feeds, frozen at the Cisco IOS, Meraki and Umbrella modules. It takes no new modules and fetches nothing from Elastic at build, test or run time. New feeds use `dfe-transform-elastic`, and 2.0 and 2.1 feeds should plan to move there. - -## Using it +> **LEGACY.** Backwards compatibility for DFE/XDR 2.0 and 2.1 filebeat feeds, frozen at the Cisco IOS, Meraki and Umbrella modules. It takes no further modules and fetches nothing from Elastic at build, test or run time. Other feeds use `dfe-transform-elastic`, and 2.0 and 2.1 feeds should plan to move there. ```yaml # config.yaml @@ -27,8 +25,6 @@ Mount `filebeat.yaml` into that directory and `timezones.csv` at the path the pi The wrapper writes each transform file into Vector's config dir with its wired `inputs` added, and a transform file may declare a top-level `enrichment_tables` key, so Vector merges the declaration and the VRL's timezone lookups resolve. `vector validate` runs before Vector starts, so a pipeline that cannot compile stops the app rather than passing nothing on. -Events carry the DFE 2.1 Kafka fields (`{message, tags, timestamp}`), and the optional `._conf.tz_offset` / `._conf.tz_map` fields drive the ios and meraki timezone handling. - -## Provenance +Events hold the DFE 2.1 Kafka fields (`{message, tags, timestamp}`), and the optional `._conf.tz_offset` / `._conf.tz_map` fields drive the ios and meraki timezone handling. -`filebeat.yaml` embeds dfe-transform-vrl `pipelines/filebeat/filebeat.vrl` @ `e0b7087`, the same program bar two lines of its header comment, and `timezones.csv` is that repo's table copied. Refresh by copying both again rather than editing here -- the two transform apps grade against the same cases (`tests/e2e/filebeat_kafka.rs` in each), and that only means something while the program is the same one. +`filebeat.yaml` embeds dfe-transform-vrl `pipelines/filebeat/filebeat.vrl` @ `e0b7087`, the same program bar two lines of its header comment, and `timezones.csv` is that repo's table copied. Refresh by copying both again rather than editing here -- the two transform apps grade against the same cases (`tests/e2e/filebeat_kafka.rs` in each), and that means something as long as the program is the same one. diff --git a/tests/TESTING.md b/tests/TESTING.md index 41119f6..ba4c2c3 100644 --- a/tests/TESTING.md +++ b/tests/TESTING.md @@ -7,16 +7,35 @@ Which tier runs what, and how the tests find a broker and a Vector binary. - Core runs on every push and PR: `cargo nextest run --all-features`. Unit, integration, smoke, and the e2e cases that are not `#[ignore]`. - Full runs nightly and before every release (`test.full.required_for_release` in `.hyperi-ci.yaml`): core plus every `#[ignore]` case, `cargo nextest run --all-features --run-ignored all --ignore-default-filter`. Those are the `vector_validate` cases, the opt-in `e2e::kafka` pipelines and one timing measurement in `src/metrics/scrape.rs`. +```bash +# Unit tests -- no infrastructure +cargo nextest run --lib + +# Config assembly, wiring, fixtures, lifecycle, metrics +cargo nextest run --test integration + +# CLI surface +cargo nextest run --test smoke + +# The e2e tests -- need Docker or a live cluster, plus a Vector binary +cargo nextest run --test e2e + +# Everything, opt-in cases included +cargo nextest run --run-ignored all +``` + +Opt-in (`#[ignore]`) cases are the full tier: `vector_validate` runs Vector's validator over each buffer, SASL and TLS variant of an assembled config, and two `e2e::kafka` pipelines drive a bare Vector process against a broker. + ## Brokers and the Vector binary -There is no mode switch. `KafkaFixture::acquire` resolves a broker on its own: +There is no mode switch. `KafkaFixture::acquire` resolves a broker by itself: -1. A live cluster, if `KAFKA_BROKERS` (and any SASL settings) in `.env` authenticate against it. Faster, and a real multi-broker cluster. +1. A live cluster, if `KAFKA_BROKERS` (and any SASL settings) in `.env` authenticate against it. Faster, and a multi-broker cluster. 2. Otherwise the run's shared broker: one single-node Apache Kafka, scalo's `KAFKA_IMAGE`, that scalo's `shared_service` starts for the whole run and every test process joins. The first test that needs it starts it on the local Docker daemon, locally and in CI alike, and a watcher removes it when the run's last test process exits. 3. A skip, when neither is available. In CI that is an assertion failure instead: a test that skips when its environment disappears is not a gate. `KafkaFixture::hermetic` skips step 1 outright and takes the shared broker. -Anything using the real DFE topic names takes that path, because +Anything using the production DFE topic names takes that path, because `filebeat_land` on a shared live broker is somebody else's data. `KafkaFixture::scram` starts a SCRAM-only broker the test holds alone. @@ -30,28 +49,7 @@ docker rm -f $(docker ps -aq --filter label=io.hyperi.test.suite=dfe-transform-v Tests that spawn Vector resolve it through `scripts/fetch-vector.sh`, which downloads the pinned build once into `.tmp/`, and fall back to `vector` on `PATH`. Without either they skip locally, and the cases that run by default fail under CI. -## Suites - -```bash -# Unit tests -- no infrastructure -cargo nextest run --lib - -# Config assembly, wiring, fixtures, lifecycle, metrics -cargo nextest run --test integration - -# CLI surface -cargo nextest run --test smoke - -# The e2e tests -- need Docker or a live cluster, plus a Vector binary -cargo nextest run --test e2e - -# Everything, opt-in cases included -cargo nextest run --run-ignored all -``` - -Opt-in (`#[ignore]`) cases are the full tier: `vector_validate` runs the real validator over each buffer, SASL and TLS variant of an assembled config, and two `e2e::kafka` pipelines drive a bare Vector process against a broker. - -## The WS21 acceptance case +## The filebeat acceptance case `e2e::filebeat_kafka` runs in core, on the shared broker, which no live cluster's data shares. It seeds the synthetic filebeat cases onto `filebeat_land`, runs the shipped @@ -59,4 +57,4 @@ binary over them, and grades `filebeat_load` against the fields each case names. The cases (`tests/fixtures/filebeat/cases.yaml`) and the bundled pipeline belong to dfe-transform-vrl, so the case needs that repo checked out beside this one, -or `DFE_TRANSFORM_VRL_DIR` pointing at it. Without one it prints a skip. +or `DFE_TRANSFORM_VRL_DIR` pointing at it. Without one it skips locally and fails under CI, which checks the repo out beside this one.