Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions .cargo/config.toml
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
# Per-target CPU optimisations.
#
# Under [target.<triple>], never [build]: cargo drops build.rustflags when a
# CARGO_TARGET_<triple>_RUSTFLAGS is set, as CI runners do.

Expand Down
5 changes: 1 addition & 4 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,5 @@ GEMINI.md
tarpaulin-report.json
cobertura.xml

# Local scratch / tmp (CORE.md: scratch goes here, gitignored, NOT
# /tmp). Bare patterns (no leading slash) match a tmp dir at ANY
# depth, so these cover top-level .tmp/ and tmp/ as well as nested
# ones like tests/tmp/. Mirrored in attach.py _DEFAULT_EXCLUDES.
# Scratch directories at any depth.
tmp/
17 changes: 4 additions & 13 deletions .hyperi-ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,36 +8,30 @@ classification: product

language: rust

# Quality checks
quality:
enabled: true
# Advisory ignores fed to cargo-audit and osv-scanner.
ignore:
- tool: osv-scanner
id: GHSA-w9wp-h8wv-79jx
reason: "CVE-2026-48504, opentelemetry_sdk 0.31.0 via scalo: unreachable, no Baggage propagator registered in dfe-transform-vector or scalo. Not reachable per dfe-infra docs/THREAT-MODEL.md accepted risks. Blocked on metrics-exporter-opentelemetry 0.2.1 requiring opentelemetry ^0.31; tracked in hyperi-io/scalo-rs#281."

# Testing
test:
enabled: true
coverage: false
rust:
# The metrics tests need a process each: the global recorder installs once per process.
nextest: true
# A release runs the full tier: the Vector validator cases and the opt-in
# Kafka pipelines. tests/TESTING.md says which tests are which.
# tests/TESTING.md says which tests the full tier adds.
full:
required_for_release: true

# Building
build:
enabled: true
strategies:
- native
rust:
glibc_max: "2.41"
# Any C library a release links comes from hyperi-ci's static prefix, so
# the image installs no C library packages.
# C libraries come from hyperi-ci's static prefix, so the image installs none.
static_libs: true
targets:
- x86_64-unknown-linux-gnu
Expand All @@ -49,14 +43,11 @@ build:
workload_cmd: "bash scripts/pgo-workload.sh"
# Built before the workload clock starts: an uncached arm64 runner takes longer than the workload timeout to compile it.
workload_setup_cmd: "cargo build --release --features pgo-driver --bin pgo-driver"
# 300s is the floor a stable profile needs: a short workload biases
# the compiler toward startup paths and turns PGO NEGATIVE.
# A shorter workload biases the profile toward startup paths.
duration_secs: 300
bolt:
enabled: true

# Release (where the artefacts go). The thin chart is assembled from the
# emitted contract and ships beside the image.
release:
enabled: true
# Keep `library` at the scalo version in Cargo.toml: the library renders only the contract version its scalo release writes.
Expand All @@ -66,8 +57,8 @@ release:
library: "2.14.30"
container:
enabled: true
# Generated by `dfe-transform-vector emit-dockerfile`, see docs/internals.md.
dockerfile: Dockerfile
platforms:
- linux/amd64
- linux/arm64
# `src/deployment.rs::emit_dockerfile()` splices the Vector install into the Dockerfile.
36 changes: 17 additions & 19 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ feat!: remove deprecated API endpoints
BREAKING CHANGE: The /v1/users endpoint has been removed. Use /v2/users instead.
```

### Examples
Examples:

```text
feat(auth): add OAuth2 support for Google login
Expand All @@ -58,8 +58,6 @@ refactor(core)!: restructure module exports
BREAKING CHANGE: Named exports are now used instead of default exports.
```

### Scope

Scope is optional but recommended. Use it to indicate the area of the codebase
affected (e.g., `api`, `auth`, `core`, `cli`, `docs`).

Expand Down Expand Up @@ -90,7 +88,7 @@ anniversary of its release.
3. Run the tests that cover your change
4. Submit a pull request with a clear description of what you've done

### Pull Request Checklist
Before you open the pull request:

- [ ] Commits follow the conventional commit format
- [ ] Tests pass (if applicable)
Expand All @@ -108,20 +106,20 @@ If you have questions about contributing, please open an issue or contact us.

### What this is

A Rust supervisor that runs Vector as a child process so a Vector pipeline behaves like every other DFE app: one big-dial config, one ops port, an OTLP push, and a chart assembled from its deployment contract. It is NOT the transform engine and it does not build Vector. The image downloads the upstream release binary at the version pinned by `VECTOR_VERSION` in `src/deployment.rs`, and cargo compiles only the supervisor. It is not a singleton either: dfe-infra's `apps.yaml` declares it `multiplicity: per_config` with `scale_deployed: true` and `default_in: [docker-single]`, so one deployment exists per source config, and outside `docker-single` only when a deployment asks for it.
A Rust supervisor that runs Vector as a child process so a Vector pipeline behaves like every other DFE app: one big-dial config, one ops port, an OTLP push, and a chart assembled from its deployment contract. It is NOT the transform engine and it does not build Vector. The image downloads the upstream release binary at the version pinned by `VECTOR_VERSION` in `src/deployment.rs`, and cargo compiles only the supervisor. It is not a singleton either: dfe-infra's `apps.yaml` declares it `multiplicity: per_config` with `scale_deployed: true` and `default_in: [docker-single]`, so one deployment exists per source config, and outside `docker-single` when a deployment asks for it.

### Where things live

| Path | What it holds |
|---|---|
| `src/main.rs` | CLI entry. Fills scalo's `CommonArgs` from the loaded config before `run_app`, which is what makes `metrics.address` and `logging.*` take effect |
| `src/config/` | `loader` (the config and its env layers) and the Vector config it becomes: `generate`, `wiring`, `transforms`, `secrets`, `assembler`, `validate`, plus `reload`, `staging` and `kafka_defaults` |
| `src/vector/` | `process` spawn and backoff, `lifecycle` states, and `binary` -- version selection kept pure and NOT on the run path, which its own header explains |
| `src/vector/` | `process` spawn and backoff, `lifecycle` states, and `binary` -- version selection kept pure and NOT on the run path, which its header explains |
| `src/bridge.rs` | The direct-transport translation between scalo `Transport/Push` and Vector `PushEvents`. Does nothing on `bus` |
| `src/health.rs`, `src/metrics.rs`, `src/metrics/scrape.rs` | Readiness publishing, and the scrape that merges `vector_*` into scalo's registry |
| `src/deployment.rs`, `src/vector-layer.dockerfile` | `contract()`, the source the Dockerfile and the released chart are generated from, and the two-binary image override spliced onto scalo's generated Dockerfile |
| `templates/bus.yaml`, `templates/direct.yaml` | Whole runnable Vector topologies, one per transport, commented per field |
| `pipelines/filebeat/` | The bundled filebeat pipeline. The image does not carry it |
| `pipelines/filebeat/` | The bundled filebeat pipeline. The image does not ship it |
| `docs/` | `architecture.md` for why the service is built this way and the invariants, `internals.md` for field-by-field depth, `migration.md`, `librdkafka.md`, the generated `config-schema.*` and `capability-catalog.*` |
| `tests/` | `integration`, `e2e`, `smoke`, and `TESTING.md` for how the broker and Vector binary are resolved |
| `scripts/fetch-vector.sh`, `scripts/pgo-workload.sh` | Downloads the pinned Vector for tests, and drives the PGO/BOLT workload |
Expand All @@ -137,11 +135,11 @@ cargo nextest run --test e2e # needs Docker or a live broker, an
cargo nextest run --run-ignored all # everything, opt-in cases included
```

What green does NOT mean:
What a passing run does NOT mean:

- The default run leaves out the `#[ignore]` cases in `tests/integration/vector_validate.rs` and `tests/e2e/kafka.rs`. The `vector_validate` ones run Vector's own validator over each buffer, SASL and TLS variant of an assembled config, so until you pass `--run-ignored all` those variants are unchecked.
- Locally, a missing broker or Vector binary SKIPS rather than fails. In CI that absence is an assertion failure for the cases that run by default, so CI is stricter than a green laptop run.
- `e2e::filebeat_kafka` runs by default and needs a dfe-transform-vrl checkout, for its cases and the bundled pipeline: beside this repo, or named by `DFE_TRANSFORM_VRL_DIR`. Without one it skips locally and fails under CI, which checks the repo out beside this one.
- The default run leaves out the `#[ignore]` cases in `tests/integration/vector_validate.rs` and `tests/e2e/kafka.rs`. The `vector_validate` ones run Vector's validator over each buffer, SASL and TLS variant of an assembled config, so until you pass `--run-ignored all` those variants are unchecked.
- Locally, a missing broker or Vector binary SKIPS rather than fails. In CI that absence is an assertion failure for the cases that run by default, so CI is stricter than a passing laptop run.
- `e2e::filebeat_kafka` runs by default and needs a dfe-transform-vrl checkout, for its cases and the bundled pipeline: beside this repo, or at `DFE_TRANSFORM_VRL_DIR`. Without one it skips locally and fails under CI, which checks the repo out beside this one.
- A docs-only push runs no CI at all. `paths-ignore` for `docs/**` and `**.md` is on the `push` trigger but not on `pull_request`, so the PR is the only place a docs change is checked. A silent push is not a pass.

### What tends to bite
Expand All @@ -150,14 +148,14 @@ What green does NOT mean:
|---|---|---|
| Hand-edit `Dockerfile`, or commit a chart | Fix `src/deployment.rs::contract()` and regenerate | The Dockerfile is generator output, so `checked_in_dockerfile_matches_emit_dockerfile` fails on a hand edit, and the release assembles the chart from the contract |
| Bump scalo and leave `release.helm.library` behind | Move `release.helm.library` in `.hyperi-ci.yaml` to the same scalo version | A scalo-service release renders only the contract version its scalo release writes |
| Turn the Kafka lag trigger back on in `contract()` | Leave `KafkaLagTrigger::disabled()` and scale on CPU, adding a pressure trigger through the chart's `keda.extraTriggers` where a deployment wants one | Lag rises when a downstream stage breaks, so a lag trigger adds pods that wait on the same broken stage. The library's trigger also reads `config.kafka.*`, which this app does not have, and refuses to render |
| Set `vector.version` in a deployment's config and forget it | Leave it unset, so the default reads `VECTOR_VERSION` | A pinned version that differs from the binary the image bakes refuses to start under `strict`, the default, and runs only where `vector.version_check: warn` lets it |
| Mount a whole `templates/*.yaml` as a transform file | Copy only its `transforms:` block | The supervisor generates `sources:` and `sinks:` from the big dials and refuses a transform file declaring them |
| Turn the Kafka lag trigger back on in `contract()` | Leave `KafkaLagTrigger::disabled()` and scale on CPU, adding a pressure trigger through the chart's `keda.extraTriggers` where a deployment wants one | Lag rises when a downstream stage breaks, so a lag trigger adds pods that wait on the same broken stage. The library's trigger reads `config.kafka.*`, which this app does not have, and will not render |
| Set `vector.version` in a deployment's config and forget it | Leave it unset, so the default reads `VECTOR_VERSION` | A pinned version that differs from the binary the image bakes does not start under `strict`, the default, and runs only where `vector.version_check: warn` lets it |
| Mount a whole `templates/*.yaml` as a transform file | Copy only its `transforms:` block | The supervisor generates `sources:` and `sinks:` from the big dials and rejects a transform file declaring them |
| Change `password` to a `SensitiveString` to look safer | Leave it a `String` | `SensitiveString` serialises as `***REDACTED***` and the figment serialize-merge-deserialize round trip in `apply_figment_env()` destroys the value. Masking happens in logs and `Debug`, and Vector reads the credential from files through its `directory` secret backend, so it never lands in the Vector config |
| Put a `${VAR}` placeholder in a credential or a transform | Mount the secret and set `sasl.secret_dir`, or give the credential through `DFE_TRANSFORM_VECTOR_{SOURCE,SINK}_SASL_*`, and read the environment in VRL with `get_env_var` | Vector 0.57+ expands no `${VAR}` without `--dangerously-allow-env-var-interpolation` or `VECTOR_DANGEROUSLY_ALLOW_ENV_VAR_INTERPOLATION`, and the supervisor sets neither. A credential holding `${` is refused at load rather than reaching the broker as the password. Vector runs without the `DFE_TRANSFORM_VECTOR_*` variables, so `get_env_var` cannot read those |
| Assume a config key you added is wired because it parses and validates | Prove it changes behaviour, and keep `every_committed_config_file_loads_and_validates` passing over each committed config file | A key can parse, validate and reach nothing, so a deployment that moves a port on it keeps the old port and loses every probe |
| Assume a config key you added is wired because it parses and validates | Test that it changes behaviour, and keep `every_committed_config_file_loads_and_validates` passing over each committed config file | A key can parse, validate and reach nothing, so a deployment that moves a port on it keeps the old port and loses every probe |
| Set `enable.auto.commit: false` here, as the shared DFE consumer baseline does | Leave auto-commit on | Vector's kafka source only stores offsets and leaves librdkafka's commit timer to flush them, and that timer is armed only when auto-commit is on, so with it off the stored offsets reach the broker only on a rebalance or a clean shutdown |
| Read `/readyz` as "carrying traffic" | Read it as "Vector survived 500 ms after spawn, and the sink is not stalled" | A pod whose broker is unreachable, with nothing held, stays Ready with no record moving |
| Read `/readyz` as "carrying traffic" | Read it as "Vector survived 500 ms after spawn, and the sink is not stalled" | A pod with an unreachable broker and nothing held stays Ready with no record moving |
| Swap the reload poller for inotify | Keep polling | S3-backed mounts -- s3fs, goofys, Mountpoint for S3 -- generate no filesystem notification events, so inotify works on every laptop and stops reloading in production with nothing logged |
| Add or rename a credential field without a schema marker | Schema it as `scalo::SensitiveString` with `#[schemars(with = ...)]`, as `SaslConfig::password` is | That puts `x-scalo-secret` and `writeOnly` on the field in the emitted config-schema, which is what tells a console to mask it. Without it only dfe-engine's name-based fallback can mask it, so a name that fallback does not read as a credential returns an operator's Kafka password in the clear |

Expand All @@ -169,14 +167,14 @@ and `--producer dfe-transform-vector`.

Inbound -- what this repo depends on:

- scalo-rs, `cargo-dep`. `Cargo.toml` declares the `scalo` crate by range, with a second range covering the dev dependency. A scalo release reaches this repo, so widen the range if it does not admit the new version, then rebuild.
- scalo-rs, `generated-file`, lockstep. The committed `Dockerfile` is written by `scalo::deployment::generate_dockerfile()` -- its own header names the generator and the schema version -- and this repo splices the Vector layer onto it. When the generator or its schema moves, regenerate with `dfe-transform-vector emit-dockerfile > Dockerfile` and commit the diff. The released chart is assembled on the scalo-service library chart at `release.helm.library`, which moves with the scalo version in `Cargo.toml`.
- scalo-rs, `cargo-dep`. `Cargo.toml` declares the `scalo` crate by range, with a second range covering the dev dependency. A scalo release reaches this repo, so widen the range if it does not admit the release, then rebuild.
- scalo-rs, `generated-file`, lockstep. The committed `Dockerfile` is written by `scalo::deployment::generate_dockerfile()` -- its header names the generator and the schema version -- and this repo splices the Vector layer onto it. When the generator or its schema moves, regenerate with `dfe-transform-vector emit-dockerfile > Dockerfile` and commit the diff. The released chart is assembled on the scalo-service library chart at `release.helm.library`, which moves with the scalo version in `Cargo.toml`.

Outbound -- what depends on this repo:

- dfe-infra, `image-pin`, lockstep. `dfe-infra/versions.yaml` pins this repo's container image as a tag plus the digest that makes the tag immutable, and the thin chart released here, which Argo deploys. Move the tag with `dfe-stack bump-app`, which repins the chart, then re-resolve the image digest.

Two more relationships that are real but are NOT declared edges in the suite graph, so no gate enforces them:
Two more relationships that exist but are NOT declared edges in the suite graph, so no gate enforces them:

- dfe-infra `apps.yaml` is the SSoT for how this app is deployed -- multiplicity, scaling, transports, endpoints and which compiler derives its routing. dfe-engine reads that manifest, so adding or changing the app is a manifest edit, never an engine release.
- dfe-transform-vrl holds the filebeat cases (`tests/fixtures/filebeat/cases.yaml`) and the bundled pipeline that `e2e::filebeat_kafka` grades against. The dependency is a test-time checkout only, which CI makes and a laptop may not.
11 changes: 2 additions & 9 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ path = "src/bin/pgo_driver.rs"
required-features = ["pgo-driver"]

[dependencies]
# Async runtime
tokio = { version = ">=1.54.1, <2", features = ["full"] }
tokio-util = { version = ">=0.7.20, <0.8", features = ["rt"] }

Expand Down Expand Up @@ -60,37 +59,31 @@ scalo = { version = ">=2.14.30, <3", features = [
"worker-batch",
] }

# Configuration
figment = { version = ">=0.10.19, <0.11", features = ["env"] }
clap = { version = ">=4.6.7, <5", features = ["derive", "env"] }

# Record payloads across the direct-transport bridge, which moves bytes rather
# than copying them between the two gRPC legs.
bytes = ">=1.12.1, <2"

# Serialisation
serde = { version = ">=1.0.229, <2", features = ["derive"] }
serde_json = ">=1.0.154, <2"

# Config-schema derivation. Same range as scalo so cargo unifies to one schemars.
schemars = { version = ">=1.2.3, <2", default-features = false, features = ["std", "derive", "preserve_order"] }

# Error handling
thiserror = ">=2.0.21, <3"
anyhow = ">=1.0.104, <2"

# Logging and tracing
tracing = ">=0.1.44, <0.2"
tracing-subscriber = { version = ">=0.3.23, <0.4", features = ["env-filter", "json"] }

# Metrics
metrics = ">=0.24.6, <0.25"

# Unix signals (sending SIGTERM/SIGHUP to child process), and renameat2 to
# swap a reload's staged config in whole
nix = { version = ">=0.31.3, <0.32", features = ["signal", "process", "fs"] }

# Allocator
tikv-jemallocator = { version = ">=0.7.0, <0.8", optional = true }
prometheus-parse = ">=0.2.5, <0.3"

Expand All @@ -108,9 +101,9 @@ scalo = { version = ">=2.14.30, <3", features = [
] }
# The repo's own .env for the live-broker tests. The binary's .env is read by scalo's cascade.
dotenvy = ">=0.15.7, <0.16"
# A stand-in Vector source for the bridge tests; scalo already builds this tonic.
# A stand-in Vector source for the bridge tests, on the tonic scalo builds.
tonic = { version = ">=0.14.9, <0.15", features = ["gzip"] }
# Reads back what the app counted; the Prometheus exporter already builds it.
# Reads back what the app counted, on the crate the Prometheus exporter builds.
metrics-util = { version = ">=0.20.4, <0.21", default-features = false, features = [
"debugging",
] }
Expand Down
6 changes: 3 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@
check: ## Pre-push validation (quality + test)
hyperi-ci check

quality: ## Run all quality checks
quality:
hyperi-ci run quality

test: ## Run tests
test:
hyperi-ci run test

build: ## Build artifacts
build:
hyperi-ci run build

ci: quality test build ## Run full CI pipeline locally
Loading
Loading