From cfbe9a89cd218f9c68337d13c2c73dc280d2a011 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 11:53:45 +1100 Subject: [PATCH 1/3] docs(archiver): redraw diagrams to the standard and fix two claims The four diagrams fail the diagrams standard: no title, description or caption, vertical chains with five-line node labels, and skip arrows on the CI chart. Each is redrawn from the code. The pipeline becomes the repo's L3 component diagram, with a table naming the code behind each step. The release flow becomes its L4 child. Fix the README claim that --config is the only command-line setting. scalo's CLI takes --log-level, --log-format, --metrics-addr, -v and -q. The restore example named a file the writer cannot produce. Restore the work_config health component to the idle note. internals.md says GCS uploads have no CI test. Config comments that restated a key or labelled a group are cut from the Cargo manifests, .hyperi-ci.yaml, docker-compose.dev.yaml and _typos.toml. No value changes. --- .hyperi-ci.yaml | 5 --- Cargo.toml | 12 ------- README.md | 2 +- _typos.toml | 37 +++----------------- crates/archiver/Cargo.toml | 5 --- crates/core/Cargo.toml | 6 ---- crates/io/Cargo.toml | 3 -- docker-compose.dev.yaml | 23 +++--------- docs/architecture.md | 24 ++++++++++--- docs/archiving.md | 2 +- docs/delivery.md | 38 ++++++++++++++------ docs/design.md | 48 +++++++++++++++++++------ docs/internals.md | 72 ++++++++++++++++++++++++++------------ 13 files changed, 146 insertions(+), 131 deletions(-) diff --git a/.hyperi-ci.yaml b/.hyperi-ci.yaml index e42f8de..703df27 100644 --- a/.hyperi-ci.yaml +++ b/.hyperi-ci.yaml @@ -9,7 +9,6 @@ classification: product language: rust -# Quality checks quality: enabled: true # cargo-audit and osv-scanner do not read deny.toml, so its accepted advisories repeat here. @@ -29,7 +28,6 @@ quality: - "default" - "jemalloc" -# Testing test: enabled: true coverage: true @@ -54,7 +52,6 @@ test: - "default" - "jemalloc" -# Building build: enabled: true strategies: @@ -70,11 +67,9 @@ build: targets: - x86_64-unknown-linux-gnu - aarch64-unknown-linux-gnu - # Feature sets to build (production builds use jemalloc) features: - "jemalloc" -# Releasing release: enabled: true # Keep `library` at the scalo version in Cargo.toml: the library renders only the contract version its scalo release writes. diff --git a/Cargo.toml b/Cargo.toml index b901914..c51b0ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,6 @@ must_use_candidate = "allow" cast_precision_loss = "allow" [workspace.dependencies] -# Internal crates dfe-archiver-core = { path = "crates/core" } dfe-archiver-io = { path = "crates/io" } @@ -52,26 +51,21 @@ scalo = { version = "2.14.30" } # Config-schema derivation. Same range as scalo so cargo unifies to one schemars. schemars = { version = ">=1.2.3, <2", default-features = false, features = ["std", "derive", "preserve_order"] } -# Async runtime tokio = { version = ">=1.54.1, <2", features = ["full", "parking_lot"] } tokio-util = { version = ">=0.7.20, <0.8", features = ["time"] } -# Serialization serde = { version = ">=1.0.229, <2", features = ["derive"] } serde_json = ">=1.0.154, <2" sonic-rs = ">=0.5.10, <0.6" -# Error handling thiserror = ">=2.0.21, <3" anyhow = ">=1.0.104" -# Compression zstd = ">=0.14.3, <0.15" lz4_flex = ">=0.14.0, <0.15" snap = ">=1.1.2, <2" flate2 = ">=1.1.10, <2" -# Cloud storage backends object_store = { version = ">=0.14.2, <0.15", features = ["aws", "gcp", "azure"] } # The e2e MinIO fixture's client only. Default features off drops the legacy # `rustls` feature and its unmaintained rustls-webpki, so the rest are re-listed. @@ -82,7 +76,6 @@ aws-sdk-s3 = { version = ">=1.157.0, <2", default-features = false, features = [ "sigv4a", ] } -# Utilities bytes = ">=1.12.1, <2" chrono = { version = ">=0.4.45, <0.5", features = ["serde"] } uuid = { version = ">=1.29.0, <2", features = ["v4", "serde"] } @@ -98,25 +91,20 @@ lru = ">=0.19.0, <0.20" metrics = ">=0.24.4, <0.25" percent-encoding = ">=2.3.2, <3" -# CLI clap = { version = ">=4.6.7, <5", features = ["derive", "env"] } -# Tracing tracing = ">=0.1.44, <0.2" tracing-subscriber = { version = ">=0.3.23, <0.4", features = ["env-filter"] } -# Async utilities async-trait = ">=0.1.92, <0.2" futures = ">=0.3.34, <0.4" -# Config dotenvy = ">=0.15.7, <0.16" # The -ctl half gives scalo's memory guard its live heap bytes. tikv-jemallocator = ">=0.7.0, <0.8" tikv-jemalloc-ctl = { version = ">=0.7.0, <0.8", features = ["stats"] } -# Dev dependencies tempfile = ">=3.27, <4" # Scoped env-var set/restore without the workspace-forbidden `unsafe`. temp-env = ">=0.3.6, <0.4" diff --git a/README.md b/README.md index 5749047..2f7b39d 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Without a writable spool directory the archiver refuses to start, and the defaul ## Configuration -Settings load through a cascade: `DFE_ARCHIVER_*` environment variables, then `.env`, then the config file, then defaults. The only command-line setting is `--config`, which names the file. The pipeline reads its config at startup, so a change to any section below but `kafka.batch_size` takes a restart, and a reload logs the sections that need one. The archiver's own keys take a single underscore (`DFE_ARCHIVER_KAFKA_BROKERS`), and the scalo runtime sections (`version_check`, `metrics`, `logger`, `self_regulation`, `scaling`) take two (`DFE_ARCHIVER_VERSION_CHECK__ENABLED`). +Settings load through a cascade: `DFE_ARCHIVER_*` environment variables, then `.env`, then the config file, then defaults. `--config` names the file, and the other flags set logging and the metrics address ([docs/configuration.md](docs/configuration.md#scalo-runtime-variables)). The pipeline reads its config at startup, so a change to any section below but `kafka.batch_size` takes a restart, and a reload logs the sections that need one. The archiver's own keys take a single underscore (`DFE_ARCHIVER_KAFKA_BROKERS`), and the scalo runtime sections (`version_check`, `metrics`, `logger`, `self_regulation`, `scaling`) take two (`DFE_ARCHIVER_VERSION_CHECK__ENABLED`). | Key | What it sets | Default | |-----|--------------|---------| diff --git a/_typos.toml b/_typos.toml index 1624ebf..70e1717 100644 --- a/_typos.toml +++ b/_typos.toml @@ -4,31 +4,16 @@ # License: BUSL-1.1 # Copyright: (c) 2026 HYPERI PTY LIMITED # -# Based on: ci/templates/config/typos.toml # Reference: https://github.com/crate-ci/typos # -# Spelling policy (from ai/standards): -# Code: American English (serialize, color, initialize) -# Docs/comments: Australian English (serialise, colour, initialise) -# -# typos cannot distinguish code from docs, so we use en-au locale -# and allowlist the American spellings that appear in code. +# Prose is Australian English and code American, and typos cannot tell them +# apart, so the American spellings code uses are allowlisted. [default] -# Use Australian English — flags American spellings in prose locale = "en-au" [default.extend-words] -# --------------------------------------------------------------------------- -# American spellings allowed in code (programming convention) -# --------------------------------------------------------------------------- -# typos uses case-insensitive matching for extend-words, so "serialize" -# covers Serialize, SERIALIZE, serialize, etc. -# -# Format: american_spelling = "american_spelling" (identity = don't flag) -# --------------------------------------------------------------------------- - -# -ize/-ise variants (most common in code) +# Matching is case-insensitive, so each lowercase entry covers every casing. serialize = "serialize" serializes = "serializes" serialized = "serialized" @@ -209,7 +194,6 @@ monetized = "monetized" monetizing = "monetizing" monetization = "monetization" -# -our/-or variants color = "color" colors = "colors" colored = "colored" @@ -225,7 +209,6 @@ neighbor = "neighbor" neighbors = "neighbors" neighboring = "neighboring" -# -ence/-ense variants defense = "defense" defensive = "defensive" offense = "offense" @@ -235,20 +218,17 @@ licensed = "licensed" licenses = "licenses" licensing = "licensing" -# -re/-er variants (less common in code but appear in libs) center = "center" centered = "centered" centering = "centering" fiber = "fiber" meter = "meter" -# -log/-logue variants catalog = "catalog" catalogs = "catalogs" dialog = "dialog" dialogs = "dialogs" -# Other American spellings common in code canceled = "canceled" canceling = "canceling" labeled = "labeled" @@ -263,23 +243,17 @@ gray = "gray" artifact = "artifact" artifacts = "artifacts" -# --------------------------------------------------------------------------- -# Project-specific: Australian/British spellings in legal templates -# --------------------------------------------------------------------------- -# The BUSL-1.1 LICENSE file uses British "licence" (noun form). -# Allow both licence and license since both appear in the project. +# Both licence and license appear in the project. licence = "licence" # Crate name flate2 contains "flate" which is not a typo flate = "flate" [default.extend-identifiers] -# Rust crate names and identifiers flate2 = "flate2" lz4_flex = "lz4_flex" [files] -# Files/directories to ignore extend-exclude = [ # Lock files contain hashes "*.lock", @@ -288,7 +262,6 @@ extend-exclude = [ "package-lock.json", "pnpm-lock.yaml", - # Binary/minified assets "*.min.js", "*.min.css", "*.svg", @@ -297,7 +270,6 @@ extend-exclude = [ "*.ttf", "*.eot", - # Build/cache directories ".git/", ".venv/", "node_modules/", @@ -312,7 +284,6 @@ extend-exclude = [ "CHANGELOG.md", "**/CHANGELOG.md", - # Git submodules (contain their own changelogs/commit hashes) "ci/", "ai/", ] diff --git a/crates/archiver/Cargo.toml b/crates/archiver/Cargo.toml index 9f202e9..86a8354 100644 --- a/crates/archiver/Cargo.toml +++ b/crates/archiver/Cargo.toml @@ -46,23 +46,18 @@ scalo = { workspace = true, features = [ "scaling", "governor", "expression", "deployment", "config-schema", "cli-service", "memory", "health", "shutdown", "dlq", "dlq-kafka", "version-check", ] } -# Async tokio = { workspace = true } tokio-util = { workspace = true } futures = { workspace = true } -# Serialization serde = { workspace = true } serde_json = { workspace = true } -# Config clap = { workspace = true } dotenvy = { workspace = true } -# Tracing tracing = { workspace = true } -# Utilities chrono = { workspace = true } compact_str = { workspace = true } lru = { workspace = true } diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index b730099..06fb943 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -27,21 +27,17 @@ scalo = { workspace = true, features = ["transport-kafka", "scaling", "dlq", "dl schemars = { workspace = true } -# Serialization serde = { workspace = true } serde_json = { workspace = true } sonic-rs = { workspace = true } -# Error handling thiserror = { workspace = true } -# Compression zstd = { workspace = true } lz4_flex = { workspace = true } snap = { workspace = true } flate2 = { workspace = true } -# Utilities bytes = { workspace = true } chrono = { workspace = true } compact_str = { workspace = true } @@ -50,11 +46,9 @@ parking_lot = { workspace = true } smallvec = { workspace = true } indexmap = { workspace = true } -# Async tokio = { workspace = true } async-trait = { workspace = true } -# Tracing tracing = { workspace = true } [dev-dependencies] diff --git a/crates/io/Cargo.toml b/crates/io/Cargo.toml index cade3ad..e1370cc 100644 --- a/crates/io/Cargo.toml +++ b/crates/io/Cargo.toml @@ -26,15 +26,12 @@ dfe-archiver-core = { workspace = true } # kafka and grpc are the only transports `validate_config` accepts. scalo = { workspace = true, features = ["transport-kafka", "transport-grpc", "metrics", "governor"] } -# Cloud storage object_store = { workspace = true } -# Async tokio = { workspace = true } async-trait = { workspace = true } futures = { workspace = true } -# Utilities compact_str = { workspace = true } parking_lot = { workspace = true } tracing = { workspace = true } diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index fa2e6b6..98cd85a 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -11,9 +11,6 @@ # docker compose -f docker-compose.dev.yaml down -v services: - # ========================================================================= - # Redpanda (Kafka-compatible, lighter weight) - # ========================================================================= redpanda: # Matches KAFKA_TAG in scripts/pgo-workload.sh, so Renovate tracks one line. # renovate: datasource=docker depName=redpandadata/redpanda @@ -35,9 +32,9 @@ services: - --reserve-memory 0M - --default-log-level=warn ports: - - "19092:19092" # Kafka external - - "18082:18082" # Pandaproxy - - "18081:18081" # Schema registry + - "19092:19092" + - "18082:18082" + - "18081:18081" volumes: - redpanda-data:/var/lib/redpanda/data healthcheck: @@ -46,7 +43,6 @@ services: timeout: 5s retries: 5 - # Redpanda Console (Kafka UI) redpanda-console: # renovate: datasource=docker depName=redpandadata/console image: redpandadata/console:v3.13.1@sha256:d72608f64c65a5631adca6c99b44d157cefb9edb0473ed58451f91affcfcfa1b @@ -61,10 +57,7 @@ services: KAFKA_SCHEMAREGISTRY_ENABLED: "true" KAFKA_SCHEMAREGISTRY_URLS: http://redpanda:8081 - # ========================================================================= - # S3-compatible object storage (LocalStack), for the minio:// and s3:// - # destinations. It accepts any credentials, minioadmin included. - # ========================================================================= + # Serves the minio:// and s3:// destinations, and accepts any credentials, minioadmin included. s3: # Same pin as LOCALSTACK_TAG in scripts/pull-test-images.sh. Stay on # the semver line: the CalVer tags need a licence token to start. @@ -86,10 +79,7 @@ services: timeout: 5s retries: 5 - # ========================================================================= - # Local file storage (for file:// backend testing) - # ========================================================================= - # No container needed: config.dev.yaml writes under ./data/archive on the host. + # file:// needs no container: config.dev.yaml writes under ./data/archive on the host. volumes: redpanda-data: @@ -101,9 +91,6 @@ configs: awslocal s3 mb s3://archive awslocal s3 mb s3://archive-test -# ========================================================================= -# Network -# ========================================================================= networks: default: name: archiver-network diff --git a/docs/architecture.md b/docs/architecture.md index 76b663c..e53f134 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -14,13 +14,29 @@ Where things live and which way the dependencies point. The design itself -- the ## Codemap +This diagram shows which way the crate dependencies point. + ```mermaid -graph LR - archiver["crates/archiver
binary + pipeline"] --> io["crates/io
transports + object stores"] - archiver --> core["crates/core
archive logic"] - io --> core +flowchart LR + accTitle: L4 Detail -- Crate dependencies + accDescr: The archiver crate depends on both library crates, io and core. The io crate depends on core alone. + + archiver["archiver"] + + subgraph libs["Libraries"] + direction LR + io["io"] --> core["core"] + end + + archiver --> libs + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 + class libs group ``` +*Figure 1. L4 Detail -- Crate dependencies. An arrow points at what a crate depends on, so archiver depends on io and core alike.* Up: [dfe-archiver documentation](README.md). + | Crate | Package | What it holds | |---|---|---| | `crates/core` | `dfe-archiver-core` | The archive logic: config types and the flat environment names (`config.rs`), the rolling archive writer (`archive/`), the hot buffers (`buffer/`), destination routing (`routing/`), the compression codecs (`compression/`), and the storage trait with the sink probe (`storage.rs`). Its only I/O is creating the spool directory | diff --git a/docs/archiving.md b/docs/archiving.md index 00f9c43..fe3068c 100644 --- a/docs/archiving.md +++ b/docs/archiving.md @@ -55,7 +55,7 @@ The archiver never deletes an archived file, so retention belongs to the store. There is no restore command. A file is the records, so you decompress it and put the records back where they came from. 1. List the keys for the destination and window you need, for example `aws s3 ls --recursive s3://acme-archive/archives/events/acme/2026/10/29/`. -2. Fetch the files and decompress them: `zstd -dc 03-0001-3f9a.jsonl.zst > records.jsonl`. +2. Fetch the files and decompress them: `zstd -dc 03-0001-9c41d2e07b3a5f68.jsonl.zst > records.jsonl`. 3. On `kafka` the first part of the key is the topic. Produce the lines back to it, for example `kcat -P -b kafka:9092 -t events -l records.jsonl`. They then take the normal path, and a payload that held a newline goes back as two messages. On `grpc` the key holds no topic, so send the records to the receiver as new data. Check two things first: diff --git a/docs/delivery.md b/docs/delivery.md index ea67a9b..7009ca1 100644 --- a/docs/delivery.md +++ b/docs/delivery.md @@ -14,20 +14,36 @@ On `kafka` with `kafka.acknowledgements.enabled` (the default) no record is lost ## A record is released when its file is durable +This diagram shows when a record's offset is released. + ```mermaid -flowchart TB - A["Receive a batch
the consumer records every offset it hands out"] - B["Buffer per destination"] - C["Write each flushed batch into the destination's open file
retry a write that can clear, and hold its offsets on that file"] - D["The file closes on a roll, an age close, an eviction or shutdown"] - E["Local file synced"] - F["Upload task
retries until the store takes the file"] - G["Release the file's offsets
the consumer commits each partition up to its lowest offset not yet released"] - A --> B --> C --> D - D -->|local path| E --> G - D -->|object store| F --> G +flowchart LR + accTitle: L4 Detail -- Release on a complete file + accDescr: A batch is received, buffered and written into its destination's open file. When the file closes, a local file is synced or a staged file is uploaded, and only then are the file's offsets released. + + receive["Receive"] + buffer["Buffer"] + write["Write"] + sync["Sync local file"] + upload["Upload staged file"] + release["Release offsets"] + + receive --> buffer + buffer --> write + write --> sync + write --> upload + sync -->|synced| release + upload -->|stored| release + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef core fill:#DCEBF7,stroke:#0072B2,stroke-width:2px,color:#0B2A3F + class release core ``` +*Figure 1. L4 Detail -- Release on a complete file. The blue box is the release.* Up: [L3 Components -- dfe-archiver](internals.md#the-pipeline). + +The consumer records every offset it hands out, and the writer holds each record's offset on the file the record went into. The file closes on a roll, an age close, an eviction or shutdown. + Default routing on `_org_id` sends one partition's records to several files that complete at different times, so a partition commits only up to its lowest offset not yet released. `kafka.acknowledgements.enabled: false` commits each batch at receipt instead, and a kill then loses what the buffers and open files held. An object-store file is written to `/uploads` and uploaded whole once it closes, in a background task the loop collects each cycle, so the loop never waits on the store. A failed upload keeps the staged file and its held offsets and tries again. The waits double from 0.5 s to a 60 s ceiling, each drawn from the upper half of its step so uploads that failed together spread out. Every attempt is a fresh multipart upload from the staged copy, so it signs its requests afresh however long the store has been down. A failed attempt aborts its upload. diff --git a/docs/design.md b/docs/design.md index 594101c..e16c3c9 100644 --- a/docs/design.md +++ b/docs/design.md @@ -31,20 +31,48 @@ That is the committed profile. hyperi-ci switches LTO to `fat` on the beta and r `.github/workflows/ci.yml` calls hyperi-ci's reusable `rust-ci.yml` workflow, configured by `.hyperi-ci.yaml`. +This diagram shows the jobs one CI run takes. + ```mermaid flowchart LR - E["push, pull request, schedule or dispatch"] --> P["Plan
predicts the version and which jobs run"] - E --> CM["Commit messages"] - P --> Q["Quality
fmt, clippy, audit, deny"] - P --> T["Test
with coverage"] - Q --> B["Build
x86_64 and aarch64, each on its own arch
PGO then BOLT on a full release"] - T --> B - B -->|"a release"| RT["Release tail
tag, GitHub release, ghcr image,
binaries to R2, Helm chart"] - Q --> G["Gate"] - T --> G - B --> G + accTitle: L4 Detail -- CI pipeline + accDescr: Every trigger starts the plan, and a push to main or a pull request starts the commit message check. The plan starts quality and test, both feed the build, and the build feeds the release tail on a release and the gate on every run. + + trigger(["Push, PR,
schedule or dispatch"]) + plan["Plan"] + commits["Commit messages"] + + subgraph checks["Checks"] + quality["Quality"] + test["Test"] + end + + build["Build"] + tail["Release tail"] + gate["Gate"] + + trigger -->|every run| plan + trigger -->|main or PR| commits + plan --> checks + checks --> build + build -->|release| tail + build -->|always| gate + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 + class checks group ``` +*Figure 1. L4 Detail -- CI pipeline. The gate checks plan, quality, test and build, and the commit check and release tail report on their own.* Up: [dfe-archiver documentation](README.md). + +| Job | What it does | +|---|---| +| Plan | Predicts the version and which jobs run | +| Quality | fmt, clippy, audit and deny | +| Test | The test tier, with coverage | +| Build | x86_64 and aarch64, each on a runner of its own arch, with PGO then BOLT on a full release | +| Release tail | The tag, the GitHub release, the ghcr image, the binaries to R2 and the Helm chart | + A schedule run tests and publishes nothing. A release needs a `Release: true` trailer on the commit pushed to `main`, or a dispatch, as [CONTRIBUTING.md](../CONTRIBUTING.md#cicd-workflow) says. No crate is published, because all three set `publish = false`. | Artefact | Where it goes | diff --git a/docs/internals.md b/docs/internals.md index d34ed41..9060408 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -10,30 +10,56 @@ # How the archiver works -A record arrives on Kafka or the scalo Push listener, is routed to a destination, buffered in memory, compressed into that destination's open file, and released once the file is durable. When a record is released and what can drop one is [delivery.md](delivery.md). Every key and variable is in [configuration.md](configuration.md), every metric in [metrics.md](metrics.md), and the crates behind it in [architecture.md](architecture.md). +A record arrives on Kafka or the scalo Push listener, is routed to a destination, buffered in memory, compressed into that destination's open file, and released once the file is durable. Delivery is in [delivery.md](delivery.md), every key in [configuration.md](configuration.md), every metric in [metrics.md](metrics.md) and the crates in [architecture.md](architecture.md). ## The pipeline +This diagram shows the five steps inside the archiver and what it talks to. + ```mermaid flowchart TB - K["Kafka consumer group (scalo)
batches of kafka.batch_size, 10K by default
tracks every offset it hands out"] - P["scalo Push listener (grpc)
batches of kafka.batch_size
each push answered once queued"] - R["Router
topic, then JSON fields
depth check before every parse, past 64 levels to the DLQ
every path segment encoded"] - H["Hot buffers, one per destination, in memory
at most 64 at once, the least recently used flushed for another
each flushes at flush_bytes, flush_records or flush_age_secs
together at most a quarter of the memory limit, largest first"] - W["Archive writers, one per destination
up to archive.max_writers, the least recently used closed past it
each flush appends one compressed frame
a failed append is cut back and retried, its batch and offsets held
a file rolls at roll_size_bytes compressed, or at the roll interval"] - L["Local path
the file and its directories synced to disk"] - S["Object store
staged under buffer.spool_dir/uploads, uploaded whole in the background
writer_parallelism uploads at once, retried until the store takes it"] - C["Release
kafka: each partition commits up to its lowest offset not yet released
grpc: nothing to release, the push was answered at enqueue"] - K --> R - P --> R - R --> H - H -->|flush| W - W -->|file closes| L - W -->|file closes| S - L -->|synced| C - S -->|store confirms| C + accTitle: L3 Components -- dfe-archiver + accDescr: Records arrive from Kafka topics or are pushed by the previous stage. Inside, they are received, routed, buffered, written to files and released. Files land in a local path or a bucket, and records refused for good go to the DLQ topic when the DLQ is on. + + topics[("Kafka topics")] + stage["Previous stage"] + + subgraph archiver["dfe-archiver"] + direction LR + receive["Receive"] --> route["Route"] --> buffer["Buffer"] --> write["Write"] --> release[["Complete
and release"]] + end + + store[("Local path
or bucket")] + dlq[("DLQ topic")] + + topics -->|records| archiver + stage -->|pushes| archiver + archiver -->|files| store + archiver -.->|refused| dlq + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef store fill:#DDF3EC,stroke:#00785A,stroke-width:1px,color:#0B3327 + classDef async fill:#F6E6EF,stroke:#A8517F,stroke-width:1px,color:#3D1529 + classDef error fill:#FBE3D6,stroke:#C2410C,stroke-width:1px,color:#4A1A05 + classDef ext fill:#EEF0F2,stroke:#6B7280,stroke-width:1px,stroke-dasharray:4 3,color:#1F2937 + classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 + class topics async + class stage ext + class store store + class dlq error + class archiver group ``` +*Figure 1. L3 Components -- dfe-archiver. The dashed box is external and the dotted arrow carries refused records when the DLQ is on.* Up: top of dfe-archiver. Down: [L4 Detail -- Release on a complete file](delivery.md#a-record-is-released-when-its-file-is-durable). + +| Step | Code | What it does | +|---|---|---| +| Receive | `crates/io/src/transport.rs` | Takes up to `kafka.batch_size` records a call, 10,000 by default | +| Route | `crates/core/src/routing/` | Picks each record's destination ([Routing](#routing)) | +| Buffer | `crates/core/src/buffer/` | Holds records in memory per destination ([Buffers and writers](#buffers-and-writers)) | +| Write | `crates/core/src/archive/`, `crates/core/src/compression/` | Compresses each flush into the open file, and rolls it | +| Complete and release | `crates/archiver/src/archiver.rs`, `crates/io/src/storage.rs` | Syncs a local file or uploads a staged one, then releases its records ([delivery.md](delivery.md)) | + ## Two transports, one pipeline `transport` picks how records arrive, and nothing after the receive knows which one delivered them. @@ -47,11 +73,11 @@ On `kafka` an empty `kafka.topics` turns on discovery, re-read every `kafka.topi `grpc` lets a deployment with no broker still archive. The previous stage pushes each record to the listener on `grpc.listen`, by default `0.0.0.0:6000`, the contract's push port. The listener answers a push once its records are queued, because a record is released only when its file completes, long after any sender's deadline. So the archive copy on `grpc` is at-most-once, and `pipeline_delivery_guarantee` reports `best_effort` with reason `sink_cannot_confirm`, or `acks_disabled` under `grpc.acknowledgements.enabled: false`. There is no broker to dead-letter to, so the DLQ is off. -A DFE deployment feeds the archiver from a broker or from the previous stage, so the binary compiles in scalo's `transport-kafka` and `transport-grpc` and no other transport, and `validate_config` refuses any other `transport` at startup. +The binary compiles in scalo's `transport-kafka` and `transport-grpc` alone, and `validate_config` refuses any other `transport`. ### Idle until configured -An archiver with no `archive.destination` starts anyway. It stays Ready, serves `/livez`, `/readyz` and `/metrics`, opens no broker connection, binds no listener, logs the reason once and holds `pipeline_idle` at 1. A change to the `--config` file, or a SIGHUP, reloads the config, and one that names a destination starts the pipeline with no restart. The gate is scalo's (`scalo::lifecycle`), and the predicate is `Config::idle_reason`. +An archiver with no `archive.destination` starts anyway. It stays Ready, serves `/livez`, `/readyz` and `/metrics`, opens no broker connection, binds no listener, logs the reason once, reports the `work_config` health component `Degraded` and holds `pipeline_idle` at 1. A change to the `--config` file, or a SIGHUP, reloads the config, and one that names a destination starts the pipeline with no restart. The gate is scalo's (`scalo::lifecycle`), and the predicate is `Config::idle_reason`. Structural faults still refuse at startup: an unknown transport, `grpc` with no listen address, `kafka` with no brokers, and a codec or routing mode that does not exist. @@ -127,7 +153,7 @@ Expression routing checks a record's nesting before it parses it, because `sonic ## Compression -Each flush appends one complete frame, framed stream or gzip member, so a file of many flushes is one concatenated stream the codec's own tools read whole. Dead-lettering a refused file reads it back the same way, a flush at a time. +Each flush appends one complete frame, framed stream or gzip member, so a file of many flushes is one concatenated stream the codec's own tools read whole. | `compression.codec` | Extension | Format | |---|---|---| @@ -154,10 +180,12 @@ A store's block (`archive.s3`, `archive.minio`, `archive.gcs`, `archive.azure`) | GCS | `archive.gcs.service_account_key` (the key's JSON) or `archive.gcs.credentials_path`, never both, because the client refuses the pair. Then `GOOGLE_APPLICATION_CREDENTIALS`, gcloud's application-default credentials file, and GCE instance metadata | | Azure | a bearer token (`AZURE_STORAGE_TOKEN`), then `archive.azure.account_key` or `AZURE_STORAGE_ACCOUNT_KEY`, workload identity, a client secret, `archive.azure.sas_token`, and managed identity. `archive.azure.account_name` names the account and, when set, replaces the account name in the environment | -An S3 endpoint over plain `http://` needs `archive.s3.allow_http: true`, and without it the client refuses every request. `DFE_ARCHIVER_S3_ACCESS_KEY_ID` and `DFE_ARCHIVER_S3_SECRET_ACCESS_KEY` set the S3 keys from a Secret. The config file is read as written, so `${VAR}` in it is a literal string, never an expansion. +CI tests GCS listing and deletion on an emulator but no upload, which needs a live bucket ([testing.md](testing.md)). + +An S3 endpoint over plain `http://` needs `archive.s3.allow_http: true`, and without it the client refuses every request. Keys from a Secret go in as environment variables ([configuration.md](configuration.md#environment-variables)). ## Scaling -The deployment contract's ScaledObject scales on CPU alone and declares no raw Kafka lag trigger. dfe-infra sets the CPU target to 70% in every profile. The single, scale and mesh profiles add the `scaling_pressure` trigger, read through dfe-keda-shim with a target of 70, and slim runs on CPU alone. The engine's [KEDA scaling page](https://github.com/hyperi-io/dfe-engine/blob/main/docs/deployment/keda-scaling.md) covers the triggers for every app. +The deployment contract's ScaledObject scales on CPU alone and declares no raw Kafka lag trigger. dfe-infra adds the `scaling_pressure` trigger in the profiles that want it, and the engine's [KEDA scaling page](https://github.com/hyperi-io/dfe-engine/blob/main/docs/deployment/keda-scaling.md) lists every app's triggers. `scaling_pressure` is scalo's weighted composite of the three terms `scaling_components()` registers in `crates/archiver/src/lib.rs`, served at `/scaling/pressure`. `kafka_lag` is the consumer's position lag, records past its read position, because the committed-offset lag grows by up to a roll interval of intake while open files hold their commit. On `grpc` nothing sets it, so it stays at zero. `buffer_depth` is the hot-buffer count and `memory` the guard's pressure ratio. An open sink circuit zeroes the composite. scalo's memory gate (`scaling.memory_gate_threshold`) never fires here, because nothing gives the engine a memory reading. From 398619a140fe89cb3eadc7cf018cc4cc6bb489fd Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 12:16:57 +1100 Subject: [PATCH 2/3] docs(archiver): use plain wording and fold short sections Replace 103 of the 106 words flagged by house style in README.md and docs/ with plain wording. Examples: own, also, only, while, refuses, whose. The 3 left are part of a term or express a comparison. Fold 7 headings into paragraphs: three in architecture.md, two in design.md and two in archiving.md. Each covered one short paragraph or table. README.md keeps its headings because a test splits it on "## Storage Backends". Join the two-line comment in _typos.toml into one line. --- README.md | 22 +++++++++++----------- _typos.toml | 3 +-- docs/architecture.md | 17 ++++++----------- docs/archiving.md | 12 ++++-------- docs/configuration.md | 18 +++++++++--------- docs/delivery.md | 38 +++++++++++++++++++------------------- docs/design.md | 12 +++++------- docs/internals.md | 34 +++++++++++++++++----------------- docs/metrics.md | 2 +- docs/testing.md | 6 +++--- 10 files changed, 76 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 2f7b39d..7c41425 100644 --- a/README.md +++ b/README.md @@ -7,17 +7,17 @@ Writes a DFE deployment's streams to compressed files in a local path or object ## What it does -- Reads every Kafka topic discovery keeps, the landing topic where a source also has a transformed one, or takes a gRPC push on a deployment with no broker. Each record's payload is written as one line, exactly as it arrived. -- Writes to a local path, MinIO, S3, GCS or Azure Blob, in zstd (default), LZ4, Snappy or gzip frames, so the codec's own tools read a file whole. +- Reads every Kafka topic discovery keeps, the landing topic where a source has both a landing and a transformed one, or takes a gRPC push on a deployment with no broker. Each record's payload is written as one line, exactly as it arrived. +- Writes to a local path, MinIO, S3, GCS or Azure Blob, in zstd (default), LZ4, Snappy or gzip frames, so the codec's standard tools read a file whole. - Files each record under its topic and a JSON field (`_org_id` by default), or under its topic alone, and rolls a file at 1 GiB compressed or at the roll interval. -- On Kafka, commits an offset only once the file holding its record is durable, so a kill, a failed upload or a store outage costs duplicates or lag, never records, while the topic's retention still holds them. +- On Kafka, commits an offset once the file holding its record is durable and never before, so a kill, a failed upload or a store outage costs duplicates or lag, never records, as long as the topic's retention holds them. - Holds every destination's buffers inside a quarter of the memory limit, and pauses intake as staged uploads near the spool volume's cap, so an outage never fills memory or disk. Kafka keys, offsets and timestamps are not kept. A replay gets the payloads back, not their Kafka metadata. What a file holds, where it lands and how to restore from one: [docs/archiving.md](docs/archiving.md). ## Quick start -On Linux the build needs `protoc`, a C toolchain, `pkg-config` and librdkafka. scalo compiles its gRPC `.proto` files, `aws-lc-sys` compiles C, and `rdkafka-sys` links librdkafka through pkg-config. It refuses a librdkafka older than the version after the `+` in its `Cargo.lock` entry, and a Debian or Ubuntu package can be older than that, so there [build librdkafka from source](https://github.com/confluentinc/librdkafka#build-from-source) first. +On Linux the build needs `protoc`, a C toolchain, `pkg-config` and librdkafka. scalo compiles its gRPC `.proto` files, `aws-lc-sys` compiles C, and `rdkafka-sys` links librdkafka through pkg-config. It rejects a librdkafka older than the version after the `+` in its `Cargo.lock` entry, and a Debian or Ubuntu package can be older than that, so there [build librdkafka from source](https://github.com/confluentinc/librdkafka#build-from-source) first. ```bash sudo apt-get install -y protobuf-compiler build-essential pkg-config @@ -25,17 +25,17 @@ cargo build --release ./target/release/dfe-archiver --config config.dev.yaml ``` -[config.dev.yaml](config.dev.yaml) reads `test-events` from the Redpanda in [docker-compose.dev.yaml](docker-compose.dev.yaml) on `localhost:19092`, writes under `./data/archive` and stages under `./data/spool`. The essentials go in as environment variables too: +[config.dev.yaml](config.dev.yaml) reads `test-events` from the Redpanda in [docker-compose.dev.yaml](docker-compose.dev.yaml) on `localhost:19092`, writes under `./data/archive` and stages under `./data/spool`. The essentials can be set as environment variables instead: ```bash env DFE_ARCHIVER_KAFKA_BROKERS=localhost:19092 DFE_ARCHIVER_KAFKA_TOPICS=test-events DFE_ARCHIVER_DESTINATION=file://./data/archive DFE_ARCHIVER_SPOOL_DIR=./data/spool ./target/release/dfe-archiver ``` -Without a writable spool directory the archiver refuses to start, and the default, `/var/spool/dfe/archiver`, is one the image creates. Metrics and health answer on port 9090 (`/metrics`, `/livez`, `/readyz`), and `curl -s localhost:9090/readyz` prints `{"status":"ready"}` once the pipeline runs. With no destination set the archiver starts, passes readiness and idles until a config change names one. +Without a writable spool directory the archiver does not start, and the default, `/var/spool/dfe/archiver`, is one the image creates. Metrics and health answer on port 9090 (`/metrics`, `/livez`, `/readyz`), and `curl -s localhost:9090/readyz` prints `{"status":"ready"}` once the pipeline runs. With no destination set the archiver starts, passes readiness and idles until a config change names one. ## Configuration -Settings load through a cascade: `DFE_ARCHIVER_*` environment variables, then `.env`, then the config file, then defaults. `--config` names the file, and the other flags set logging and the metrics address ([docs/configuration.md](docs/configuration.md#scalo-runtime-variables)). The pipeline reads its config at startup, so a change to any section below but `kafka.batch_size` takes a restart, and a reload logs the sections that need one. The archiver's own keys take a single underscore (`DFE_ARCHIVER_KAFKA_BROKERS`), and the scalo runtime sections (`version_check`, `metrics`, `logger`, `self_regulation`, `scaling`) take two (`DFE_ARCHIVER_VERSION_CHECK__ENABLED`). +Settings load through a cascade: `DFE_ARCHIVER_*` environment variables, then `.env`, then the config file, then defaults. `--config` names the file, and the other flags set logging and the metrics address ([docs/configuration.md](docs/configuration.md#scalo-runtime-variables)). The pipeline reads its config at startup, so a change to any section below but `kafka.batch_size` takes a restart, and a reload logs the sections that need one. The archiver's keys take a single underscore (`DFE_ARCHIVER_KAFKA_BROKERS`), and the scalo runtime sections (`version_check`, `metrics`, `logger`, `self_regulation`, `scaling`) take two (`DFE_ARCHIVER_VERSION_CHECK__ENABLED`). | Key | What it sets | Default | |-----|--------------|---------| @@ -43,14 +43,14 @@ Settings load through a cascade: `DFE_ARCHIVER_*` environment variables, then `. | `kafka.brokers`, `kafka.topics` | Brokers, and the topics to read. Empty topics discover every topic, narrowed by `topic_include` and `topic_exclude` | `localhost:9092`, discover | | `archive.destination` | `file:///path`, `s3://bucket/prefix`, `minio://bucket/prefix`, `gs://bucket/prefix` or `az://container/prefix` | none, idles | | `compression.codec` | `zstd`, `lz4`, `snappy`, `gzip` or `none` | `zstd` | -| `archive.roll_size_bytes`, `archive.roll_interval_secs` | When a file closes | 1 GiB, and 300 s while Kafka offsets are held or on `grpc`, else 3600 | +| `archive.roll_size_bytes`, `archive.roll_interval_secs` | When a file closes | 1 GiB, and 300 s when Kafka offsets are held or on `grpc`, else 3600 | | `buffer.spool_dir` and its three caps | Where staged uploads wait, and how much disk they may take | `/var/spool/dfe/archiver`, and 80%, 10% and 10% of the volume's declared size limit | -Each store takes its own block (`archive.s3`, `archive.minio`, `archive.gcs`, `archive.azure`) for the endpoint and credentials. [docs/config-schema.yaml](docs/config-schema.yaml) lists every archiver key with its type and default, and [docs/configuration.md](docs/configuration.md) the cascade and every environment variable. +Each store takes a block (`archive.s3`, `archive.minio`, `archive.gcs`, `archive.azure`) for the endpoint and credentials. [docs/config-schema.yaml](docs/config-schema.yaml) lists every archiver key with its type and default, and [docs/configuration.md](docs/configuration.md) the cascade and every environment variable. ## Storage Backends -Each destination is one URL plus its store's block. These four load and validate as written, and a test keeps them that way. The file is read as written, so `${VAR}` in it stays a literal string. Set store keys in the environment instead: `DFE_ARCHIVER_S3_ACCESS_KEY_ID` and `DFE_ARCHIVER_S3_SECRET_ACCESS_KEY` for S3, `AZURE_STORAGE_ACCOUNT_KEY` for Azure, or each SDK's own credential chain ([docs/internals.md](docs/internals.md#storage-backends)). +Each destination is one URL plus its store's block. These four load and validate as written, and a test keeps them that way. The file is read as written, so `${VAR}` in it stays a literal string. Set store keys in the environment instead: `DFE_ARCHIVER_S3_ACCESS_KEY_ID` and `DFE_ARCHIVER_S3_SECRET_ACCESS_KEY` for S3, `AZURE_STORAGE_ACCOUNT_KEY` for Azure, or each SDK's credential chain ([docs/internals.md](docs/internals.md#storage-backends)). ```yaml archive: @@ -95,7 +95,7 @@ The suite's published docs are at , and this reposit ## Help and contributing -How to work in this repo, the commands that prove a change and what tends to bite are in [CONTRIBUTING.md](CONTRIBUTING.md). A question that is not a bug goes in a new issue. +How to work in this repo, the commands that test a change and what tends to bite are in [CONTRIBUTING.md](CONTRIBUTING.md). A question that is not a bug goes in an issue. Open bugs: [this repo](https://github.com/hyperi-io/dfe-archiver/issues?q=is%3Aopen+is%3Aissue+type%3ABug+-label%3Abacklog+-label%3Anext-minor) | [whole DFE suite](https://github.com/search?type=issues&q=is%3Aopen+is%3Aissue+type%3ABug+-label%3Abacklog+-label%3Anext-minor+repo%3Ahyperi-io%2Fdfe-infra+repo%3Ahyperi-io%2Fdfe-docker+repo%3Ahyperi-io%2Fdfe-deploy+repo%3Ahyperi-io%2Fdfe-engine+repo%3Ahyperi-io%2Fdfe-loader+repo%3Ahyperi-io%2Fdfe-receiver+repo%3Ahyperi-io%2Fdfe-fetcher+repo%3Ahyperi-io%2Fdfe-archiver+repo%3Ahyperi-io%2Fdfe-transform-vrl+repo%3Ahyperi-io%2Fdfe-transform-vector+repo%3Ahyperi-io%2Fdfe-transform-elastic+repo%3Ahyperi-io%2Fdfe-ui+repo%3Ahyperi-io%2Fdfe-hyperdx+repo%3Ahyperi-io%2Fscalo-rs+repo%3Ahyperi-io%2Fscalo-py+repo%3Ahyperi-io%2Flogreducer+repo%3Ahyperi-io%2Fdfe-schemas+repo%3Ahyperi-io%2Fvector-vrl+repo%3Ahyperi-io%2Fculvert+repo%3Ahyperi-io%2Fdfe-clickhouse-rs+repo%3Ahyperi-io%2Ffactbook-rs) diff --git a/_typos.toml b/_typos.toml index 70e1717..c50ab79 100644 --- a/_typos.toml +++ b/_typos.toml @@ -6,8 +6,7 @@ # # Reference: https://github.com/crate-ci/typos # -# Prose is Australian English and code American, and typos cannot tell them -# apart, so the American spellings code uses are allowlisted. +# Prose is Australian English and code American, so the American spellings code uses are allowlisted. [default] locale = "en-au" diff --git a/docs/architecture.md b/docs/architecture.md index e53f134..8759f4f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -12,8 +12,6 @@ Where things live and which way the dependencies point. The design itself -- the pipeline, the rolling model, the storage backends -- is [internals.md](internals.md), and the delivery guarantee [delivery.md](delivery.md). -## Codemap - This diagram shows which way the crate dependencies point. ```mermaid @@ -39,25 +37,22 @@ flowchart LR | Crate | Package | What it holds | |---|---|---| -| `crates/core` | `dfe-archiver-core` | The archive logic: config types and the flat environment names (`config.rs`), the rolling archive writer (`archive/`), the hot buffers (`buffer/`), destination routing (`routing/`), the compression codecs (`compression/`), and the storage trait with the sink probe (`storage.rs`). Its only I/O is creating the spool directory | +| `crates/core` | `dfe-archiver-core` | The archive logic: config types and the flat environment names (`config.rs`), the rolling archive writer (`archive/`), the hot buffers (`buffer/`), destination routing (`routing/`), the compression codecs (`compression/`), and the storage trait with the sink probe (`storage.rs`). The one I/O it does is creating the spool directory | | `crates/io` | `dfe-archiver-io` | Everything that talks to something else: the inbound transports, Kafka (`kafka.rs`) or the Push listener (`grpc.rs`), chosen by `transport.rs`, and the staging directory and the backends behind `create_backend` (`storage.rs`): file, S3, GCS, Azure Blob and MinIO | | `crates/archiver` | `dfe-archiver` | The binary and the pipeline that joins the other two. `archiver.rs` is the loop (receive, route, buffer, write, hand closed object-store files to background upload tasks, and release once a file is durable), `main.rs` the scalo `ServiceApp` wiring, CLI and config reloader, `config/loader.rs` the file loading and `validate_config`, `metrics.rs` the Prometheus surface, and `contract.rs` the deployment contract the checked-in `Dockerfile` and the Helm chart are generated from | -## Negative invariants +The rules between the crates, which nothing may break: - `core` depends on no other workspace crate. - Nothing in `core` records a metric: it counts what it did and the caller records it, which is why `ArchiveWriter::take_files_opened` exists. - Releasing an offset and uploading a file are never `core`'s. The writer settles the offsets held on a file as it closes, or hands them to the file's pending upload, and the caller drains both with `ArchiveWriter::take_settled`, uploads, and releases through the transport. -- `io` depends on `core` only, never on `archiver`. -- `archiver` is the only crate with a binary and the only reader of the - deployment contract, so the Dockerfile and the chart have one source. +- `io` depends on `core` and never on `archiver`. +- No crate but `archiver` has a binary or reads the deployment contract, so the Dockerfile and the chart have one source. - No crate is publishable: all three set `publish = false`, and their version is inherited from `[workspace.package]` so the release stamp reaches every one of them. -## Build graph - -The cold build is spent on dependencies: `object_store` with its aws, gcp and azure features, `aws-lc-sys`, which compiles C, and `rdkafka-sys` through scalo, which on Linux finds librdkafka through pkg-config (its `dynamic-linking` feature). The release build points pkg-config at a static-only prefix, so the shipped binary carries librdkafka statically and needs only libc, libm, libgcc_s and the dynamic loader at run time. +The cold build is spent on dependencies: `object_store` with its aws, gcp and azure features, `aws-lc-sys`, which compiles C, and `rdkafka-sys` through scalo, which on Linux finds librdkafka through pkg-config (its `dynamic-linking` feature). The release build points pkg-config at a static-only prefix, so the shipped binary links librdkafka statically and needs libc, libm, libgcc_s and the dynamic loader alone at run time. Change propagation follows the arrows: a `core` edit rebuilds all three, an `io` -edit rebuilds `io` and `archiver`, and an `archiver` edit rebuilds only itself. +edit rebuilds `io` and `archiver`, and an `archiver` edit rebuilds itself alone. diff --git a/docs/archiving.md b/docs/archiving.md index fe3068c..565c438 100644 --- a/docs/archiving.md +++ b/docs/archiving.md @@ -12,9 +12,7 @@ The archiver keeps a compressed copy of every record it receives, in files you can read with standard tools. This page is for the person running it. The design behind it is [internals.md](internals.md), and what can drop or repeat a record is [delivery.md](delivery.md). -## What it writes - -One record per line, exactly as it arrived, then a newline. A payload holding a newline of its own spans two lines. The archiver parses a record only to route it and never changes it, and it keeps no Kafka key, offset or timestamp. Lines are compressed, `zstd` unless `compression.codec` says otherwise: +It writes one record per line, exactly as it arrived, then a newline. A payload that holds a newline spans two lines. The archiver parses a record to route it and never changes it, and it keeps no Kafka key, offset or timestamp. Lines are compressed, `zstd` unless `compression.codec` says otherwise: | Codec | File ends in | |-------|--------------| @@ -28,7 +26,7 @@ One record per line, exactly as it arrived, then a newline. A payload holding a Each flush appends one complete frame, so a file is a standard concatenated stream. `zstd -dc`, `lz4 -dc` or `gzip -dc` reads it whole. -A file closes when it reaches `archive.roll_size_bytes` compressed (1 GiB) or `archive.roll_interval_secs` old. Unset, the interval is 300 seconds while offsets are held and on `grpc`, otherwise 3600. Expect many small files on a quiet destination and few large ones on a busy one. +A file closes when it reaches `archive.roll_size_bytes` compressed (1 GiB) or `archive.roll_interval_secs` old. Unset, the interval is 300 seconds when offsets are held and on `grpc`, otherwise 3600. Expect many small files on a quiet destination and few large ones on a busy one. ## Where it goes @@ -44,9 +42,7 @@ The routed part is the Kafka topic then the routing fields (`events/acme`). By d An object-store file is staged under `buffer.spool_dir/uploads` and uploaded whole once it closes. Nothing is visible in the bucket until then. -In a DFE deployment, dfe-infra's values set `DFE_ARCHIVER_DESTINATION` only from `archive.localPath`, as a `file://` path, and mount an `emptyDir` there, so files on that path last only as long as the pod. With no local path the variable is empty, the archiver reads it as unset, and the `archive.destination` in its config file stands. An object store also takes `s3.endpoint` and `s3.bucket` from the values. Store credentials come from the optional `dfe-archiver-s3` Secret, or from the store SDK's own credential chain where the Secret does not exist. - -## How long it stays +In a DFE deployment, dfe-infra's values set `DFE_ARCHIVER_DESTINATION` from `archive.localPath` alone, as a `file://` path, and mount an `emptyDir` there, so files on that path last as long as the pod and no longer. With no local path the variable is empty, the archiver reads it as unset, and the `archive.destination` in its config file stands. The values set an object store's `s3.endpoint` and `s3.bucket`. Store credentials come from the optional `dfe-archiver-s3` Secret, or from the store SDK's credential chain where the Secret does not exist. The archiver never deletes an archived file, so retention belongs to the store. Set a lifecycle rule on the bucket or container, or clean the directory yourself on a `file://` path. Add a rule for incomplete multipart uploads as well. A failed attempt aborts its upload, and an abort that itself fails leaves parts behind until that rule removes them. @@ -56,7 +52,7 @@ There is no restore command. A file is the records, so you decompress it and put 1. List the keys for the destination and window you need, for example `aws s3 ls --recursive s3://acme-archive/archives/events/acme/2026/10/29/`. 2. Fetch the files and decompress them: `zstd -dc 03-0001-9c41d2e07b3a5f68.jsonl.zst > records.jsonl`. -3. On `kafka` the first part of the key is the topic. Produce the lines back to it, for example `kcat -P -b kafka:9092 -t events -l records.jsonl`. They then take the normal path, and a payload that held a newline goes back as two messages. On `grpc` the key holds no topic, so send the records to the receiver as new data. +3. On `kafka` the first part of the key is the topic. Produce the lines back to it, for example `kcat -P -b kafka:9092 -t events -l records.jsonl`. They then take the normal path, and a payload that held a newline goes back as two messages. On `grpc` the key holds no topic, so send the records through the receiver again. Check two things first: diff --git a/docs/configuration.md b/docs/configuration.md index 5d88e53..deb9da8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -10,31 +10,31 @@ # Configuration reference -The archiver reads two sets of settings from two cascades: its own sections from its config file and the flat `DFE_ARCHIVER_*` variables, and scalo's runtime sections from scalo's cascade. Every key of the archiver's own sections, with its type and default, is in [config-schema.yaml](config-schema.yaml), generated from `Config`. +The archiver reads two sets of settings from two cascades: the archiver's sections from its config file and the flat `DFE_ARCHIVER_*` variables, and scalo's runtime sections from scalo's cascade. Every key of the archiver's sections, with its type and default, is in [config-schema.yaml](config-schema.yaml), generated from `Config`. -## The archiver's own sections +## The archiver's sections `transport`, `kafka`, `grpc`, `archive`, `buffer`, `routing`, `compression` and `dlq` resolve in this order, highest first: 1. The flat `DFE_ARCHIVER_*` variables below. An empty value reads as unset. -2. `.env` in the working directory, loaded into the environment without replacing a variable already set. +2. `.env` in the working directory, loaded into the environment without replacing a variable set before it. 3. The config file: `--config`, else `DFE_ARCHIVER_CONFIG`, else the first of `config.yaml`, `config/settings.yaml` and `/etc/dfe-archiver/config.yaml` that exists. 4. The defaults. -These sections take the flat names only. `DFE_ARCHIVER_KAFKA__BROKERS` reaches nothing, and neither does an unprefixed `KAFKA_BROKERS` or `ARCHIVER_DESTINATION`, with no warning. The file is read as written, so `${VAR}` in it stays a literal string. +These sections take the flat names and no others. `DFE_ARCHIVER_KAFKA__BROKERS` reaches nothing, and neither does an unprefixed `KAFKA_BROKERS` or `ARCHIVER_DESTINATION`, with no warning. The file is read as written, so `${VAR}` in it stays a literal string. The running pipeline reads its config once, at startup. The reloader watches the config file and SIGHUP, and a change to any section but `kafka.batch_size` logs that it needs a restart. An idle archiver is the exception: the change that gives it a destination starts the pipeline ([internals.md](internals.md#idle-until-configured)). ## scalo's sections -`version_check`, `metrics`, `logger`, `self_regulation`, `scaling` and `memory` belong to scalo's runtime, which reads them from its own cascade, highest first: +`version_check`, `metrics`, `logger`, `self_regulation`, `scaling` and `memory` belong to scalo's runtime, which reads them from scalo's cascade, highest first: 1. The CLI flags. -2. `DFE_ARCHIVER_
__`, so `DFE_ARCHIVER_SCALING__ENABLED` sets `scaling.enabled`. The memory guard also takes the flat `DFE_ARCHIVER_MEMORY_*` names below. +2. `DFE_ARCHIVER_
__`, so `DFE_ARCHIVER_SCALING__ENABLED` sets `scaling.enabled`. The flat `DFE_ARCHIVER_MEMORY_*` names below set the memory guard. 3. `settings..yaml`, then `settings.yaml` with the `--config` file over it, then `defaults.yaml`, each looked for in the working directory, `config/` and `/config/`. `` is `APP_ENV`, `ENVIRONMENT` or `ENV`, and `development` when none is set. 4. The defaults. -The archiver resolves its config file (`--config`, else `DFE_ARCHIVER_CONFIG`, else a default path) before scalo sets this cascade up, so the one file feeds both the archiver's sections and scalo's, however it was named. +The archiver resolves its config file (`--config`, else `DFE_ARCHIVER_CONFIG`, else a default path) before scalo sets this cascade up, so the one file feeds both the archiver's sections and scalo's, however it was given. ## Environment variables @@ -65,13 +65,13 @@ The archiver resolves its config file (`--config`, else `DFE_ARCHIVER_CONFIG`, e | `DFE_ARCHIVER_MULTIPART_CHUNK_SIZE` | `archive.multipart_chunk_size` | `8388608` | | `DFE_ARCHIVER_SPOOL_DIR` | `buffer.spool_dir` | `/var/spool/dfe/archiver` | | `DFE_ARCHIVER_S3_BUCKET` | `archive.s3.bucket` | the destination URL's | -| `DFE_ARCHIVER_S3_REGION` | `archive.s3.region` | the AWS SDK's own | +| `DFE_ARCHIVER_S3_REGION` | `archive.s3.region` | the AWS SDK's | | `DFE_ARCHIVER_S3_ACCESS_KEY_ID` | `archive.s3.access_key_id` | the AWS credential chain | | `DFE_ARCHIVER_S3_SECRET_ACCESS_KEY` | `archive.s3.secret_access_key` | the AWS credential chain | | `DFE_ARCHIVER_S3_ENDPOINT` | `archive.s3.endpoint` | AWS | | `DFE_ARCHIVER_S3_ALLOW_HTTP` | `archive.s3.allow_http` (`true`, `1` or `yes`) | `false` | -The store SDKs read their own variables too, `AWS_*`, `GOOGLE_*` and `AZURE_*` ([internals.md](internals.md#storage-backends)). +The store SDKs read the `AWS_*`, `GOOGLE_*` and `AZURE_*` variables ([internals.md](internals.md#storage-backends)). ## scalo runtime variables diff --git a/docs/delivery.md b/docs/delivery.md index 7009ca1..54e5a28 100644 --- a/docs/delivery.md +++ b/docs/delivery.md @@ -10,7 +10,7 @@ # Delivery -On `kafka` with `kafka.acknowledgements.enabled` (the default) no record is lost while the topic still holds it. A kill, a failed upload, a failing local disk or a store outage costs duplicates or consumer lag. Kafka's retention deletes records whether or not the archiver has read them, and a commit that falls below the log start resets to the earliest record left, so size retention for the longest outage you will ride out. Only a record refused for good, by the store, the codec or the nesting check, can be dropped, when no DLQ can take it, and it is counted. On `grpc` the archive copy is at-most-once, because the listener answers each push once its records are queued ([internals.md](internals.md#two-transports-one-pipeline)). How the pipeline gets a record into a file is [internals.md](internals.md). +On `kafka` with `kafka.acknowledgements.enabled` (the default) no record is lost as long as the topic holds it. A kill, a failed upload, a failing local disk or a store outage costs duplicates or consumer lag. Kafka's retention deletes records whether or not the archiver has read them, and a commit that falls below the log start resets to the earliest record left, so size retention for the longest outage you will ride out. One case drops a record: it is refused for good, by the store, the codec or the nesting check, and no DLQ can take it. That record is counted. On `grpc` the archive copy is at-most-once, because the listener answers each push once its records are queued ([internals.md](internals.md#two-transports-one-pipeline)). How the pipeline gets a record into a file is [internals.md](internals.md). ## A record is released when its file is durable @@ -44,7 +44,7 @@ flowchart LR The consumer records every offset it hands out, and the writer holds each record's offset on the file the record went into. The file closes on a roll, an age close, an eviction or shutdown. -Default routing on `_org_id` sends one partition's records to several files that complete at different times, so a partition commits only up to its lowest offset not yet released. `kafka.acknowledgements.enabled: false` commits each batch at receipt instead, and a kill then loses what the buffers and open files held. +Default routing on `_org_id` sends one partition's records to several files that complete at different times, so a partition commits up to its lowest offset not yet released. `kafka.acknowledgements.enabled: false` commits each batch at receipt instead, and a kill then loses what the buffers and open files held. An object-store file is written to `/uploads` and uploaded whole once it closes, in a background task the loop collects each cycle, so the loop never waits on the store. A failed upload keeps the staged file and its held offsets and tries again. The waits double from 0.5 s to a 60 s ceiling, each drawn from the upper half of its step so uploads that failed together spread out. Every attempt is a fresh multipart upload from the staged copy, so it signs its requests afresh however long the store has been down. A failed attempt aborts its upload. @@ -52,17 +52,17 @@ A staged copy that cannot be read back, because it is missing, unreadable by per ### Holding costs memory -A held record costs about 32 bytes until its file is durable. At 10,000 records a second that is 96 MB over a 300 s roll and 1.15 GB over an hour, so while offsets are held an unset `archive.roll_interval_secs` is 300 s rather than 3600 s. A configured value always wins, and one above 900 logs a startup warning with the cost. +A held record costs about 32 bytes until its file is durable. At 10,000 records a second that is 96 MB over a 300 s roll and 1.15 GB over an hour, so when offsets are held an unset `archive.roll_interval_secs` is 300 s rather than 3600 s. A configured value always wins, and one above 900 logs a startup warning with the cost. -While uploads fail, held records and staged bytes grow, so both are pressure sources on the self-regulation latch. Held records read against a quarter of the memory limit at 32 bytes each, and staged bytes against `buffer.staged_bytes_cap`. Past the latch's `pause_above` (0.8 by default) the consumer pauses its partitions and the Push listener sheds pushes with `Unavailable`, so a store outage turns into lag rather than memory or disk growth. With self-regulation off nothing pauses, and the archiver logs that at startup. +As uploads fail, held records and staged bytes grow, so both are pressure sources on the self-regulation latch. Held records read against a quarter of the memory limit at 32 bytes each, and staged bytes against `buffer.staged_bytes_cap`. Past the latch's `pause_above` (0.8 by default) the consumer pauses its partitions and the Push listener sheds pushes with `Unavailable`, so a store outage turns into lag rather than memory or disk growth. With self-regulation off nothing pauses, and the archiver logs that at startup. ## Refused for good goes to the DLQ -Only a refusal of the object itself is permanent. That is an invalid path, a key past 1024 bytes, checked before any record is staged, a name the local filesystem cannot hold, or an `EntityTooLarge`, `KeyTooLongError` or `InvalidObjectName` answer. Anything else, credentials and a missing bucket included, is retried, because retrying costs lag where dropping costs records. +A refusal of the object itself is the one permanent failure. That is an invalid path, a key past 1024 bytes, checked before any record is staged, a name the local filesystem cannot hold, or an `EntityTooLarge`, `KeyTooLongError` or `InvalidObjectName` answer. Anything else, credentials and a missing bucket included, is retried, because retrying costs lag where dropping costs records. -A file the store refuses is read back a block at a time, each block one flush decompressed on its own, and its records go to the DLQ one entry per line with the routed destination. A payload holding a newline of its own spans lines, so when a file holds more lines than records each block goes whole, as one entry saying so, and no record is split across entries. The file's offsets are released `Rejected` once the DLQ confirms them. +A file the store rejects is read back a block at a time, each block one flush, decompressed separately, and its records go to the DLQ one entry per line with the routed destination. A payload that holds a newline spans lines, so when a file holds more lines than records each block goes whole, as one entry saying so, and no record is split across entries. The file's offsets are released `Rejected` once the DLQ confirms them. -A batch the codec refuses as it is written goes to the DLQ through scalo's confirming write, one entry per record, and only a confirmed write releases a record's offset `Rejected`. Before any write, `Dlq::refusal` screens each entry. Only the Kafka backend has a ceiling, the producer's `message.max.bytes`, against the entry as written: a JSON object goes in as itself and anything else as text or base64. A file backend beside Kafka, as under the default `dlq.mode: cascade`, takes what Kafka refuses, so only a Kafka-only DLQ refuses an entry. A record no backend can hold is released `Dropped`, counted and logged, and the rest still go. A broker or topic ceiling below the producer's is not seen there, and fails the write instead. +A batch the codec rejects as it is written goes to the DLQ through scalo's confirming write, one entry per record, and a record's offset is released `Rejected` once the DLQ confirms its write. Before any write, `Dlq::refusal` screens each entry. The Kafka backend is the one with a ceiling, the producer's `message.max.bytes`, against the entry as written: a JSON object goes in as itself and anything else as text or base64. A file backend beside Kafka, as under the default `dlq.mode: cascade`, takes what Kafka rejects, so a DLQ rejects an entry when Kafka is its one backend. A record no backend can hold is released `Dropped`, counted and logged, and the rest still go. A broker or topic ceiling below the producer's is not seen there, and fails the write instead. With the DLQ off, as it always is on `grpc`, a refused batch or file is released `Dropped` with its reason. A failed DLQ write is released `Errored`, because it can clear. Inbound-filter dead letters are screened the same way, and one no backend can hold is dropped, because the filter routed it out of the archive by policy. @@ -70,11 +70,11 @@ Every dropped record is counted in `messages_dropped_total` under one `reason`: ## A local write that fails is held, never dropped -A local write that fails in a way that can clear, a full or failing disk or staging that cannot be created, is never dead-lettered. The failed append is cut back to the file's last whole block, and the batch and its offsets are held while the write is retried, the waits doubling from 0.1 s to a 5 s ceiling. The loop receives nothing until the write lands, so nothing later is read or committed past it, and a disk that stays broken shows as lag and one warning. +A local write that fails in a way that can clear, a full or failing disk or staging that cannot be created, is never dead-lettered. The failed append is cut back to the file's last whole block, and the batch and its offsets are held as the write is retried, the waits doubling from 0.1 s to a 5 s ceiling. The loop receives nothing until the write lands, so nothing later is read or committed past it, and a disk that stays broken shows as lag and one warning. -While it waits, a file the write rolled uploads at once, and another destination's file closes at its roll interval and uploads, because a full staging disk frees only as the store takes files. Once shutdown begins, a write still failing gets 10 s more, then its batch is released `Errored`. A file that cannot be cut back, or cannot complete because its sync or staging failed, is given up, and its offsets are `Errored` too. +As it waits, a file the write rolled uploads at once, and another destination's file closes at its roll interval and uploads, because nothing but the store taking files frees a full staging disk. Once shutdown begins, a write still failing gets 10 s more, then its batch is released `Errored`. A file that cannot be cut back, or cannot complete because its sync or staging failed, is given up, with its offsets `Errored`. -Nothing reads an `Errored` span again short of a restart or rebalance, and scalo's consumer has no seek back to the commit floor, so the loop ends with `Error::Withheld`. The process drains, exits non-zero, and the restart reads again from the committed offset. A source that cannot deliver the records again, `grpc` or `kafka` with acknowledgements off, has already let them go, so they are counted in `messages_dropped_total{reason="unreplayable"}` instead. +Nothing reads an `Errored` span again short of a restart or rebalance, and scalo's consumer has no seek back to the commit floor, so the loop ends with `Error::Withheld`. The process drains, exits non-zero, and the restart reads again from the committed offset. A source that cannot deliver the records again, `grpc` or `kafka` with acknowledgements off, let them go at receipt, so they are counted in `messages_dropped_total{reason="unreplayable"}` instead. ## Failures and what they cost @@ -83,33 +83,33 @@ Nothing reads an `Errored` span again short of a restart or rebalance, and scalo | Kill before the file is durable | The restart reads every record the file held again, and removes the staged files it left | Duplicates, up to one roll interval plus pending uploads | | Upload fails | File and offsets kept, upload aborted and retried, intake paused near the caps | Delayed, never lost | | Staged copy cannot be read back | File quarantined, offsets released `Errored`, the process exits and restarts | Duplicates | -| Store refuses the object for good, DLQ confirms | Records read back into the DLQ, offsets released | In the DLQ | -| Store refuses the object for good, DLQ off | Offsets released `Dropped`, counted, reason logged | Dropped | +| Store rejects the object for good, DLQ confirms | Records read back into the DLQ, offsets released | In the DLQ | +| Store rejects the object for good, DLQ off | Offsets released `Dropped`, counted, reason logged | Dropped | | Local write fails in a way that can clear | Append cut back, batch and offsets held, write retried, intake held | Delayed, never lost | | Local write still failing 10 s into shutdown | Offsets released `Errored`, the restart reads them again | Duplicates | | Local file cannot complete or be cut back | Offsets released `Errored`, the process exits and restarts | Duplicates | -| Codec refuses a batch, DLQ confirms | Offsets released `Rejected` | In the DLQ | +| Codec rejects a batch, DLQ confirms | Offsets released `Rejected` | In the DLQ | | Refused for good, no DLQ backend can hold one record | That record's offset released `Dropped`, counted, reason logged, the rest to the DLQ | That record dropped | | Refused for good, DLQ write fails | Offsets released `Errored`, the process exits and restarts | Duplicates | ## Shutdown and restart -The run loop stops first. The drain then closes the source, so the Push listener stops answering and the Kafka consumer stops fetching but can still commit, and receives until the source reports it is empty, so every push already answered is written. A source that returns nothing for 5 s without reporting empty ends the drain. Buffers flush into their files, evicted writers finish closing, every open file completes, and the uploads get 20 s. A file still uploading then keeps its offsets unreleased and its staged copy on disk. +The run loop stops first. The drain then closes the source, so the Push listener stops answering and the Kafka consumer stops fetching but can still commit, and receives until the source reports it is empty, so every push answered before the drain is written. A source that returns nothing for 5 s without reporting empty ends the drain. Buffers flush into their files, evicted writers finish closing, every open file completes, and the uploads get 20 s. A file still uploading then keeps its offsets unreleased and its staged copy on disk. -A completed staged file carries a manifest beside it, `.part.json`, with its object path and the length of each block. At startup the staging directory is settled before anything new is staged: +A completed staged file has a manifest beside it, `.part.json`, with its object path and the length of each block. At startup the staging directory is settled before anything further is staged: - On `kafka` with acknowledgements on, every staged file is removed, because its records are read again from the committed offset. -- On `grpc`, or with acknowledgements off, nothing delivers those records again. A file whose manifest matches its size is uploaded. One never completed, or whose manifest cannot be read, moves to `/uploads/quarantine` for an operator. +- On `grpc`, or with acknowledgements off, nothing delivers those records again. A file with a manifest that matches its size is uploaded. One never completed, or with an unreadable manifest, moves to `/uploads/quarantine` for an operator. Quarantine holds at most `buffer.quarantine_bytes_cap`, and a file past it is removed. Each outcome is counted in `staged_files_recovered_total`, `staged_files_quarantined_total{reason}` and `staged_files_removed_total{reason}`. ## Disk stays under the spool volume's limit -Only archive files sit on local disk: the open file of a local destination, and under `/uploads` the staged copy of each object-store file until the store takes it, with the quarantine beside them. +Archive files are all that sit on local disk: the open file of a local destination, and under `/uploads` the staged copy of each object-store file until the store takes it, with the quarantine beside them. -1. At startup the archiver refuses to start with less than `buffer.min_free_spool_bytes` free on the staging volume, read once staging is settled. When it kept complete files to upload it starts anyway and warns, because only a running archiver uploads them. +1. At startup the archiver does not start with less than `buffer.min_free_spool_bytes` free on the staging volume, read once staging is settled. When it kept complete files to upload it starts anyway and warns, because nothing but a running archiver uploads them. 2. Staged bytes pause intake at the latch's `pause_above` share of `buffer.staged_bytes_cap`, which sits under the volume's `emptyDir` limit, past which kubelet evicts the pod. 3. Quarantine holds at most `buffer.quarantine_bytes_cap`, and a file past it is removed and counted. -4. A local write that finds the disk full is held and retried, never dropped, while uploads free the staging disk. +4. A local write that finds the disk full is held and retried, never dropped, as uploads free the staging disk. `staged_bytes` and `uploads_pending` show what is waiting for the store ([metrics.md](metrics.md)). diff --git a/docs/design.md b/docs/design.md index e16c3c9..d134753 100644 --- a/docs/design.md +++ b/docs/design.md @@ -12,11 +12,9 @@ How the archiver is built and released, and the design decisions behind it. The operator-facing design is [internals.md](internals.md). -## Building and releasing - The build needs the tools in the README's [quick start](../README.md#quick-start). `cargo build` writes a debug binary to `target/debug/dfe-archiver`, and `cargo build --release --features jemalloc` a release one, with the allocator CI uses, to `target/release/dfe-archiver`. jemalloc is off by default, so a plain `cargo build` uses the system allocator. -CI builds each target on a runner of its own architecture, never cross-compiling, with `--target`, so its binaries land under `target/x86_64-unknown-linux-gnu/release/` and `target/aarch64-unknown-linux-gnu/release/`. `rdkafka` takes its `dynamic-linking` feature, which finds librdkafka through pkg-config, and the release build points pkg-config at a static-only prefix (scalo-rs `docs/deployment/native-deps.md`), so the shipped binary carries librdkafka statically. +CI builds each target on a runner of the same architecture, never cross-compiling, with `--target`, so its binaries land under `target/x86_64-unknown-linux-gnu/release/` and `target/aarch64-unknown-linux-gnu/release/`. `rdkafka` takes its `dynamic-linking` feature, which finds librdkafka through pkg-config, and the release build points pkg-config at a static-only prefix (scalo-rs `docs/deployment/native-deps.md`), so the shipped binary links librdkafka statically. ```toml [profile.release] @@ -63,14 +61,14 @@ flowchart LR class checks group ``` -*Figure 1. L4 Detail -- CI pipeline. The gate checks plan, quality, test and build, and the commit check and release tail report on their own.* Up: [dfe-archiver documentation](README.md). +*Figure 1. L4 Detail -- CI pipeline. The gate checks plan, quality, test and build, and the commit check and release tail report separately.* Up: [dfe-archiver documentation](README.md). | Job | What it does | |---|---| | Plan | Predicts the version and which jobs run | | Quality | fmt, clippy, audit and deny | | Test | The test tier, with coverage | -| Build | x86_64 and aarch64, each on a runner of its own arch, with PGO then BOLT on a full release | +| Build | x86_64 and aarch64, each on a runner of that arch, with PGO then BOLT on a full release | | Release tail | The tag, the GitHub release, the ghcr image, the binaries to R2 and the Helm chart | A schedule run tests and publishes nothing. A release needs a `Release: true` trailer on the commit pushed to `main`, or a dispatch, as [CONTRIBUTING.md](../CONTRIBUTING.md#cicd-workflow) says. No crate is published, because all three set `publish = false`. @@ -84,11 +82,11 @@ A schedule run tests and publishes nothing. A release needs a `Release: true` tr The binaries are `dfe-archiver-linux-amd64` and `dfe-archiver-linux-arm64`. Each has a `.sha256` beside it, in the format `sha256sum -c` reads. -## Key design decisions +The design decisions behind it: | Decision | Why | Rejected | |---|---|---| -| scalo for config, logging, metrics, the transports, the DLQ and scaling | The same patterns as the other DFE apps, and less code here | Direct rdkafka, a config system of its own | +| scalo for config, logging, metrics, the transports, the DLQ and scaling | The same patterns as the other DFE apps, and less code here | Direct rdkafka, a bespoke config system | | zstd at level 3 by default, with LZ4, Snappy and gzip | zstd gives the best ratio at an acceptable speed, LZ4 suits speed-critical work and gzip compatibility | brotli and lzma, both too slow | | Roll on compressed file size, 1 GiB by default | Inbound size does not predict the size a file reaches in the store | Inbound size, record count | | Hot buffers in memory under a ceiling per destination and a cap on all of them, then the writers, with nothing spilled to disk ahead of them | Memory and open files stay bounded however many destinations routing produces | One large shared buffer, an unbounded file per destination | diff --git a/docs/internals.md b/docs/internals.md index 9060408..688b0e7 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -50,7 +50,7 @@ flowchart TB class archiver group ``` -*Figure 1. L3 Components -- dfe-archiver. The dashed box is external and the dotted arrow carries refused records when the DLQ is on.* Up: top of dfe-archiver. Down: [L4 Detail -- Release on a complete file](delivery.md#a-record-is-released-when-its-file-is-durable). +*Figure 1. L3 Components -- dfe-archiver. The dashed box is external and the dotted arrow takes refused records to the DLQ when it is on.* Up: top of dfe-archiver. Down: [L4 Detail -- Release on a complete file](delivery.md#a-record-is-released-when-its-file-is-durable). | Step | Code | What it does | |---|---|---| @@ -71,9 +71,9 @@ flowchart TB On `kafka` an empty `kafka.topics` turns on discovery, re-read every `kafka.topic_refresh_secs`, so a source added later is archived without a restart. It takes every topic `kafka.topic_exclude` keeps, by default all but Kafka's internal and `_dlq` topics, narrowed by `kafka.topic_include` when set. Where a source has both `_land` and `_load`, it keeps `_land`, the record as it arrived, where dfe-loader keeps `_load`. -`grpc` lets a deployment with no broker still archive. The previous stage pushes each record to the listener on `grpc.listen`, by default `0.0.0.0:6000`, the contract's push port. The listener answers a push once its records are queued, because a record is released only when its file completes, long after any sender's deadline. So the archive copy on `grpc` is at-most-once, and `pipeline_delivery_guarantee` reports `best_effort` with reason `sink_cannot_confirm`, or `acks_disabled` under `grpc.acknowledgements.enabled: false`. There is no broker to dead-letter to, so the DLQ is off. +`grpc` lets a deployment with no broker still archive. The previous stage pushes each record to the listener on `grpc.listen`, by default `0.0.0.0:6000`, the contract's push port. The listener answers a push once its records are queued, because a record is released when its file completes, long after any sender's deadline. So the archive copy on `grpc` is at-most-once, and `pipeline_delivery_guarantee` reports `best_effort` with reason `sink_cannot_confirm`, or `acks_disabled` under `grpc.acknowledgements.enabled: false`. There is no broker to dead-letter to, so the DLQ is off. -The binary compiles in scalo's `transport-kafka` and `transport-grpc` alone, and `validate_config` refuses any other `transport`. +The binary compiles in scalo's `transport-kafka` and `transport-grpc` alone, and `validate_config` rejects any other `transport`. ### Idle until configured @@ -86,10 +86,10 @@ Structural faults still refuse at startup: an unknown transport, `grpc` with no Each destination gets a hot buffer in memory and an archive writer with one open file. Nothing spills to disk ahead of the writers. - At most 64 destinations buffer at once. A record for another first flushes the buffer that has gone longest without one, counted in `hot_buffer_evictions_total`. -- A buffer flushes at `buffer.flush_bytes` plus one record, at `buffer.flush_records`, or once `buffer.flush_age_secs` has passed since its last flush, checked on each record and on a timer. +- A buffer flushes at `buffer.flush_bytes` plus one record, at `buffer.flush_records`, or once `buffer.flush_age_secs` has gone by since its last flush, checked on each record and on a timer. - All buffers together hold at most a quarter of the memory guard's limit. Past it the largest flush first, so `flush_bytes` is a ceiling per destination, never multiplied by the destination count. - A buffer allocates nothing until a record arrives and hands its memory to the batch it flushes. The cap counts bytes buffered, and a growing buffer's allocation can run to twice that. -- Past `archive.max_writers` writers, the one that has gone longest without a write closes in the background and its file completes as on a roll (`writer_evictions_total`). It keeps its slot while it closes, so a write for a new destination waits. +- Past `archive.max_writers` writers, the one that has gone longest without a write closes in the background and its file completes as on a roll (`writer_evictions_total`). It keeps its slot until it closes, so a write for another destination waits. - A writer hands its data to the compressor at each flush, so an idle writer holds no buffer. | Key | Default | What it sets | @@ -112,7 +112,7 @@ The spool budget is 90% of the size limit declared for the spool volume, which t A file closes when its compressed size reaches `archive.roll_size_bytes` (1 GiB) or its age reaches the roll interval, and `archive_roll_total{trigger}` counts each by `size` or `age`. Size counts compressed bytes in the file, never inbound or buffered bytes. A timer closes an aged file even when nothing arrives, and opens no empty successor. -The roll interval is `archive.roll_interval_secs`. Unset, it is 300 s while offsets are held (`kafka` with acknowledgements on) or on `grpc`, otherwise 3600 s, because held offsets cost memory ([delivery.md](delivery.md#holding-costs-memory)). +The roll interval is `archive.roll_interval_secs`. Unset, it is 300 s when offsets are held (`kafka` with acknowledgements on) or on `grpc`, otherwise 3600 s, because held offsets cost memory ([delivery.md](delivery.md#holding-costs-memory)). A file's key is the routed destination, then `archive.path_template` expanded, then `--`, then `.jsonl` and the codec's extension: @@ -125,13 +125,13 @@ archive: path_template: "{year}/{month}/{day}/{hour}" ``` -The template carries no topic placeholder, because the routed destination already starts with the topic. It takes `{year}`, `{month}`, `{day}`, `{hour}` and `{minute}` in UTC, `{timestamp}` as Unix seconds and `{seq}`, the file's four-digit sequence number in that window. Validation refuses any other `{...}` token rather than let it reach the key as literal braces. +The template has no topic placeholder, because the routed destination starts with the topic. It takes `{year}`, `{month}`, `{day}`, `{hour}` and `{minute}` in UTC, `{timestamp}` as Unix seconds and `{seq}`, the file's four-digit sequence number in that window. Validation rejects any other `{...}` token rather than let it reach the key as literal braces. The writer id is 16 hex digits no other writer shares. A staged file is never checked against the store, so without it two replicas, or an evicted writer closing beside its replacement, could pick one key and the later upload would replace the earlier object. ## Routing -`routing.mode: expression`, the default, files each record under its topic and then the value of each field in `routing.expression_fields`, in dot notation for a nested field. `routing.mode: topic` files it under the topic alone and parses nothing. Unset, the fields are `_org_id` on `kafka`, where each source has its own topic, and `_source` then `_org_id` on `grpc`. +`routing.mode: expression`, the default, files each record under its topic and then the value of each field in `routing.expression_fields`, in dot notation for a nested field. `routing.mode: topic` files it under the topic alone and parses nothing. Unset, the fields are `_org_id` on `kafka`, where each source has a separate topic, and `_source` then `_org_id` on `grpc`. ```yaml routing: @@ -145,15 +145,15 @@ With those fields, `{"_org_id": "acme", "event_type": "login", "data": {}}` on t On `grpc` the topic is the push's routing key, and dfe-receiver sends none, so there is no topic folder. With the default fields `{"_source": "cisco_ios.log", "_org_id": "acme"}` lands under `cisco_ios.log/acme`. Under `mode: topic` a record with no key lands under `routing.default_segment`. -A field the record does not carry, or carries null or blank, takes `routing.default_segment`, which is `default`, the id dfe-receiver gives the default org. Each is counted in `routing_fallback_total{field}` and warned at most once a minute. A record that is not JSON goes under its topic, counted in `routing_errors_total`. +A field the record lacks, or holds as null or blank, takes `routing.default_segment`, which is `default`, the id dfe-receiver gives the default org. Each is counted in `routing_fallback_total{field}` and warned at most once a minute. A record that is not JSON goes under its topic, counted in `routing_errors_total`. -Every segment, the topic and each value, is encoded before it becomes part of a path, so a value never names another directory. Letters, digits, `-`, `_` and `.` pass through. Every other byte is written `=XX` in upper-case hex, `=` itself included, so `a/b` becomes `a=2Fb` and two values never share a name. The empty value is written `=`, and `.` and `..` become `=2E` and `=2E=2E`. Every backend refuses an absolute path, or one with a `.` or `..` segment, for good. +Every segment, the topic and each value, is encoded before it becomes part of a path, so a value never names another directory. Letters, digits, `-`, `_` and `.` pass through. Every other byte is written `=XX` in upper-case hex, `=` itself included, so `a/b` becomes `a=2Fb` and two values never share a name. The empty value is written `=`, and `.` and `..` become `=2E` and `=2E=2E`. Every backend rejects an absolute path, or one with a `.` or `..` segment, for good. Expression routing checks a record's nesting before it parses it, because `sonic-rs` recurses once per level with no limit. A record nested past scalo's maximum parse depth of 64 is never parsed. It goes to the DLQ as it arrived, under its topic with the reason `payload nesting exceeds the maximum parse depth of 64`, and its offset is released `Rejected`. With the DLQ off, as on `grpc`, or when no DLQ backend can hold it, the record is dropped and counted in `messages_dropped_total{reason}`. ## Compression -Each flush appends one complete frame, framed stream or gzip member, so a file of many flushes is one concatenated stream the codec's own tools read whole. +Each flush appends one complete frame, framed stream or gzip member, so a file of many flushes is one concatenated stream the codec's standard tools read whole. | `compression.codec` | Extension | Format | |---|---|---| @@ -167,25 +167,25 @@ Each flush appends one complete frame, framed stream or gzip member, so a file o ## Storage backends -`file:///path`, or a bare path, writes under that directory, which should be a mounted volume, never the root filesystem. Closing a file syncs it, and every directory above it up to the destination, before its offsets are released, so a node crash after the commit cannot lose it. A directory the archiver creates has its entry synced into the nearest parent that already existed. +`file:///path`, or a bare path, writes under that directory, which should be a mounted volume, never the root filesystem. Closing a file syncs it, and every directory above it up to the destination, before its offsets are released, so a node crash after the commit cannot lose it. A directory the archiver creates has its entry synced into the nearest parent that existed before. `s3://`, `minio://`, `gs://` (or `gcs://`) and `az://` (or `azure://`) go through the `object_store` crate. A file is staged under `buffer.spool_dir/uploads` and uploaded whole once it closes, as one multipart upload read from the staged copy. Parts are `archive.multipart_chunk_size`, 8 MiB by default and at least 5 MiB, S3's smallest part. Four are in flight per upload and `buffer.writer_parallelism` uploads run at once, so upload memory stays near 64 MiB by default whatever the file size. A failed attempt aborts its upload, and an abort that fails leaves its parts until the bucket's lifecycle rule for incomplete uploads removes them. -A store's block (`archive.s3`, `archive.minio`, `archive.gcs`, `archive.azure`) that names no bucket or container takes the destination URL's. One that names its own wins over the URL. The S3, GCS and Azure clients start from their SDK's environment and the block's keys apply over it, an empty key leaving the environment's value alone, and MinIO reads its block only: +A store's block (`archive.s3`, `archive.minio`, `archive.gcs`, `archive.azure`) that names no bucket or container takes the destination URL's. A bucket or container set in the block wins over the URL's. The S3, GCS and Azure clients start from their SDK's environment and the block's keys apply over it, an empty key leaving the environment's value alone, and MinIO reads nothing but its block: | Store | Credentials, first match wins | |---|---| | S3 | `archive.s3.access_key_id` and `secret_access_key`, or `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN`. Then web identity (`AWS_ROLE_ARN` with `AWS_WEB_IDENTITY_TOKEN_FILE`, which EKS IRSA injects), ECS container credentials, EKS Pod Identity, and EC2 instance metadata | -| MinIO | the `archive.minio` block only, which names `endpoint`, `access_key`, `secret_key`, `bucket` and `use_ssl`, all five | -| GCS | `archive.gcs.service_account_key` (the key's JSON) or `archive.gcs.credentials_path`, never both, because the client refuses the pair. Then `GOOGLE_APPLICATION_CREDENTIALS`, gcloud's application-default credentials file, and GCE instance metadata | +| MinIO | the `archive.minio` block alone, which names `endpoint`, `access_key`, `secret_key`, `bucket` and `use_ssl`, all five | +| GCS | `archive.gcs.service_account_key` (the key's JSON) or `archive.gcs.credentials_path`, never both, because the client rejects the pair. Then `GOOGLE_APPLICATION_CREDENTIALS`, gcloud's application-default credentials file, and GCE instance metadata | | Azure | a bearer token (`AZURE_STORAGE_TOKEN`), then `archive.azure.account_key` or `AZURE_STORAGE_ACCOUNT_KEY`, workload identity, a client secret, `archive.azure.sas_token`, and managed identity. `archive.azure.account_name` names the account and, when set, replaces the account name in the environment | CI tests GCS listing and deletion on an emulator but no upload, which needs a live bucket ([testing.md](testing.md)). -An S3 endpoint over plain `http://` needs `archive.s3.allow_http: true`, and without it the client refuses every request. Keys from a Secret go in as environment variables ([configuration.md](configuration.md#environment-variables)). +An S3 endpoint over plain `http://` needs `archive.s3.allow_http: true`, and without it the client rejects every request. Keys from a Secret go in as environment variables ([configuration.md](configuration.md#environment-variables)). ## Scaling The deployment contract's ScaledObject scales on CPU alone and declares no raw Kafka lag trigger. dfe-infra adds the `scaling_pressure` trigger in the profiles that want it, and the engine's [KEDA scaling page](https://github.com/hyperi-io/dfe-engine/blob/main/docs/deployment/keda-scaling.md) lists every app's triggers. -`scaling_pressure` is scalo's weighted composite of the three terms `scaling_components()` registers in `crates/archiver/src/lib.rs`, served at `/scaling/pressure`. `kafka_lag` is the consumer's position lag, records past its read position, because the committed-offset lag grows by up to a roll interval of intake while open files hold their commit. On `grpc` nothing sets it, so it stays at zero. `buffer_depth` is the hot-buffer count and `memory` the guard's pressure ratio. An open sink circuit zeroes the composite. scalo's memory gate (`scaling.memory_gate_threshold`) never fires here, because nothing gives the engine a memory reading. +`scaling_pressure` is scalo's weighted composite of the three terms `scaling_components()` registers in `crates/archiver/src/lib.rs`, served at `/scaling/pressure`. `kafka_lag` is the consumer's position lag, records past its read position, because the committed-offset lag grows by up to a roll interval of intake as long as open files hold their commit. On `grpc` nothing sets it, so it stays at zero. `buffer_depth` is the hot-buffer count and `memory` the guard's pressure ratio. An open sink circuit zeroes the composite. scalo's memory gate (`scaling.memory_gate_threshold`) never fires here, because nothing gives the engine a memory reading. diff --git a/docs/metrics.md b/docs/metrics.md index c28d984..be78090 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -32,7 +32,7 @@ The metrics listener, `0.0.0.0:9090` by default, serves `/metrics`, `/livez`, `/ | `archive_errors_total` | Failed write, close and upload attempts, each counted | | `kafka_commit_errors_total` | Offset releases the transport failed | | `routing_errors_total` | Records routing could not read, archived under their topic | -| `routing_fallback_total{field}` | Expression-routing fields a record did not carry, by field path | +| `routing_fallback_total{field}` | Expression-routing fields missing from a record, by field path | | `staged_files_recovered_total` | Complete staged files a previous process left, uploaded by this one | | `staged_files_quarantined_total{reason}` | Staged files moved to quarantine: `unreadable`, `incomplete`, `corrupt` or `no_store` | | `staged_files_removed_total{reason}` | Staged files removed at startup: `replayed` or `quarantine_full` | diff --git a/docs/testing.md b/docs/testing.md index 9c1b245..6c1a1c8 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -19,7 +19,7 @@ Which of dfe-archiver's tests run where, what they need, and the containers they | core | locally, on every pull request, and on every push that touches more than `docs/**` and Markdown | `cargo nextest run` | | full | nightly (`schedule` in `.github/workflows/ci.yml`) and before every release (`test.full.required_for_release: true`) | `cargo nextest run --run-ignored all` | -Only nextest runs the suite (`test.rust.nextest: true`). The metric tests read a process-global recorder, so under `cargo test`, where every test in a binary shares one process, they fail. +nextest is the one runner for the suite (`test.rust.nextest: true`). The metric tests read a process-global recorder, so under `cargo test`, where every test in a binary shares one process, they fail. Core is everything not `#[ignore]`d: the unit tests under `crates/`, and in `crates/archiver/tests/` the smoke tests (`smoke.rs`), the integration tests (`integration/`) and the e2e tests (`e2e/`) against containers each test starts. That covers at-least-once delivery against Kafka and an S3 store, the disk-full writes (capped with `prlimit`, from util-linux), Azure on Azurite, S3 on LocalStack, the GCS reads on fake-gcs-server, and the Kafka consumer over TLS. Docker is required: a missing daemon fails the run in CI and skips with the reason locally. @@ -31,7 +31,7 @@ Full adds the ignored tests, and in CI it skips them all again (`test.full.rust. Run them by hand with those in place. A Kafka test with no broker it can reach skips and says so. -There is no shared service container. Each e2e test starts the containers it needs, named `dfe-archiver-test-integration--`, labelled `io.hyperi.test.suite=dfe-archiver-integration` with the owning process, and removed when the test drops them. A dead leftover from a killed run is removed by the next test that needs the name (`reap_stale`), and a running one is left alone. +There is no shared service container. Each e2e test starts the containers it needs, called `dfe-archiver-test-integration--`, labelled `io.hyperi.test.suite=dfe-archiver-integration` with the owning process, and removed when the test drops them. A dead leftover from a killed run is removed by the next test that needs the name (`reap_stale`), and a running one is left alone. | Service | Image | |---|---| @@ -43,4 +43,4 @@ There is no shared service container. Each e2e test starts the containers it nee The tags, and the digests for Kafka and OpenSSL, live in `scripts/pull-test-images.sh`. nextest runs it as a setup script before the e2e binary, so no test spends its timeout on a pull, and in CI a fixture fails when its image was not pulled first. -The pull script and the harness's own `docker` calls use the docker CLI, which follows `DOCKER_HOST` or the current docker context. testcontainers starts the containers on `DOCKER_HOST`, or on the local socket when it is unset, and never reads the docker context. So to run the suite against another daemon, set `DOCKER_HOST`. +The pull script and the harness's `docker` calls use the docker CLI, which follows `DOCKER_HOST` or the current docker context. testcontainers starts the containers on `DOCKER_HOST`, or on the local socket when it is unset, and never reads the docker context. So to run the suite against another daemon, set `DOCKER_HOST`. From f8289ef97a18aea5a3d09ac6740aba07b15d97ed Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 12:46:14 +1100 Subject: [PATCH 3/3] docs(archiver): name the file close and the merge queue trigger delivery.md says the sync or upload starts when the file closes. The close step is in the paragraph under the diagram. A close node would make a sixth column, past the diagram limit of five. The CI diagram labels the commit-check arrow "landing". A line under the caption lists its triggers from hyperi-ci rust-ci.yml: a push to main, a pull request and a merge queue group. --- docs/delivery.md | 2 +- docs/design.md | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/delivery.md b/docs/delivery.md index 54e5a28..1ff7d65 100644 --- a/docs/delivery.md +++ b/docs/delivery.md @@ -42,7 +42,7 @@ flowchart LR *Figure 1. L4 Detail -- Release on a complete file. The blue box is the release.* Up: [L3 Components -- dfe-archiver](internals.md#the-pipeline). -The consumer records every offset it hands out, and the writer holds each record's offset on the file the record went into. The file closes on a roll, an age close, an eviction or shutdown. +The consumer records every offset it hands out, and the writer holds each record's offset on the file the record went into. The sync or upload starts when the file closes, on a roll, an age close, an eviction or shutdown. Default routing on `_org_id` sends one partition's records to several files that complete at different times, so a partition commits up to its lowest offset not yet released. `kafka.acknowledgements.enabled: false` commits each batch at receipt instead, and a kill then loses what the buffers and open files held. diff --git a/docs/design.md b/docs/design.md index d134753..d9d072d 100644 --- a/docs/design.md +++ b/docs/design.md @@ -34,7 +34,7 @@ This diagram shows the jobs one CI run takes. ```mermaid flowchart LR accTitle: L4 Detail -- CI pipeline - accDescr: Every trigger starts the plan, and a push to main or a pull request starts the commit message check. The plan starts quality and test, both feed the build, and the build feeds the release tail on a release and the gate on every run. + accDescr: Every trigger starts the plan, and a push to main, a pull request or the merge queue starts the commit message check. The plan starts quality and test, both feed the build, and the build feeds the release tail on a release and the gate on every run. trigger(["Push, PR,
schedule or dispatch"]) plan["Plan"] @@ -50,7 +50,7 @@ flowchart LR gate["Gate"] trigger -->|every run| plan - trigger -->|main or PR| commits + trigger -->|landing| commits plan --> checks checks --> build build -->|release| tail @@ -63,6 +63,8 @@ flowchart LR *Figure 1. L4 Detail -- CI pipeline. The gate checks plan, quality, test and build, and the commit check and release tail report separately.* Up: [dfe-archiver documentation](README.md). +The commit check runs on landing: a push to main, a pull request or a merge queue group. + | Job | What it does | |---|---| | Plan | Predicts the version and which jobs run |