Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions .hyperi-ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ classification: product

language: rust

# Quality checks
quality:
enabled: true
# cargo-audit and osv-scanner do not read deny.toml, so its accepted advisories repeat here.
Expand All @@ -29,7 +28,6 @@ quality:
- "default"
- "jemalloc"

# Testing
test:
enabled: true
coverage: true
Expand All @@ -54,7 +52,6 @@ test:
- "default"
- "jemalloc"

# Building
build:
enabled: true
strategies:
Expand All @@ -70,11 +67,9 @@ build:
targets:
- x86_64-unknown-linux-gnu
- aarch64-unknown-linux-gnu
# Feature sets to build (production builds use jemalloc)
features:
- "jemalloc"

# Releasing
release:
enabled: true
# Keep `library` at the scalo version in Cargo.toml: the library renders only the contract version its scalo release writes.
Expand Down
12 changes: 0 additions & 12 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ must_use_candidate = "allow"
cast_precision_loss = "allow"

[workspace.dependencies]
# Internal crates
dfe-archiver-core = { path = "crates/core" }
dfe-archiver-io = { path = "crates/io" }

Expand All @@ -52,26 +51,21 @@ scalo = { version = "2.14.30" }
# Config-schema derivation. Same range as scalo so cargo unifies to one schemars.
schemars = { version = ">=1.2.3, <2", default-features = false, features = ["std", "derive", "preserve_order"] }

# Async runtime
tokio = { version = ">=1.54.1, <2", features = ["full", "parking_lot"] }
tokio-util = { version = ">=0.7.20, <0.8", features = ["time"] }

# Serialization
serde = { version = ">=1.0.229, <2", features = ["derive"] }
serde_json = ">=1.0.154, <2"
sonic-rs = ">=0.5.10, <0.6"

# Error handling
thiserror = ">=2.0.21, <3"
anyhow = ">=1.0.104"

# Compression
zstd = ">=0.14.3, <0.15"
lz4_flex = ">=0.14.0, <0.15"
snap = ">=1.1.2, <2"
flate2 = ">=1.1.10, <2"

# Cloud storage backends
object_store = { version = ">=0.14.2, <0.15", features = ["aws", "gcp", "azure"] }
# The e2e MinIO fixture's client only. Default features off drops the legacy
# `rustls` feature and its unmaintained rustls-webpki, so the rest are re-listed.
Expand All @@ -82,7 +76,6 @@ aws-sdk-s3 = { version = ">=1.157.0, <2", default-features = false, features = [
"sigv4a",
] }

# Utilities
bytes = ">=1.12.1, <2"
chrono = { version = ">=0.4.45, <0.5", features = ["serde"] }
uuid = { version = ">=1.29.0, <2", features = ["v4", "serde"] }
Expand All @@ -98,25 +91,20 @@ lru = ">=0.19.0, <0.20"
metrics = ">=0.24.4, <0.25"
percent-encoding = ">=2.3.2, <3"

# CLI
clap = { version = ">=4.6.7, <5", features = ["derive", "env"] }

# Tracing
tracing = ">=0.1.44, <0.2"
tracing-subscriber = { version = ">=0.3.23, <0.4", features = ["env-filter"] }

# Async utilities
async-trait = ">=0.1.92, <0.2"
futures = ">=0.3.34, <0.4"

# Config
dotenvy = ">=0.15.7, <0.16"

# The -ctl half gives scalo's memory guard its live heap bytes.
tikv-jemallocator = ">=0.7.0, <0.8"
tikv-jemalloc-ctl = { version = ">=0.7.0, <0.8", features = ["stats"] }

# Dev dependencies
tempfile = ">=3.27, <4"
# Scoped env-var set/restore without the workspace-forbidden `unsafe`.
temp-env = ">=0.3.6, <0.4"
Expand Down
22 changes: 11 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,50 +7,50 @@ Writes a DFE deployment's streams to compressed files in a local path or object

## What it does

- Reads every Kafka topic discovery keeps, the landing topic where a source also has a transformed one, or takes a gRPC push on a deployment with no broker. Each record's payload is written as one line, exactly as it arrived.
- Writes to a local path, MinIO, S3, GCS or Azure Blob, in zstd (default), LZ4, Snappy or gzip frames, so the codec's own tools read a file whole.
- Reads every Kafka topic discovery keeps, the landing topic where a source has both a landing and a transformed one, or takes a gRPC push on a deployment with no broker. Each record's payload is written as one line, exactly as it arrived.
- Writes to a local path, MinIO, S3, GCS or Azure Blob, in zstd (default), LZ4, Snappy or gzip frames, so the codec's standard tools read a file whole.
- Files each record under its topic and a JSON field (`_org_id` by default), or under its topic alone, and rolls a file at 1 GiB compressed or at the roll interval.
- On Kafka, commits an offset only once the file holding its record is durable, so a kill, a failed upload or a store outage costs duplicates or lag, never records, while the topic's retention still holds them.
- On Kafka, commits an offset once the file holding its record is durable and never before, so a kill, a failed upload or a store outage costs duplicates or lag, never records, as long as the topic's retention holds them.
- Holds every destination's buffers inside a quarter of the memory limit, and pauses intake as staged uploads near the spool volume's cap, so an outage never fills memory or disk.

Kafka keys, offsets and timestamps are not kept. A replay gets the payloads back, not their Kafka metadata. What a file holds, where it lands and how to restore from one: [docs/archiving.md](docs/archiving.md).

## Quick start

On Linux the build needs `protoc`, a C toolchain, `pkg-config` and librdkafka. scalo compiles its gRPC `.proto` files, `aws-lc-sys` compiles C, and `rdkafka-sys` links librdkafka through pkg-config. It refuses a librdkafka older than the version after the `+` in its `Cargo.lock` entry, and a Debian or Ubuntu package can be older than that, so there [build librdkafka from source](https://github.com/confluentinc/librdkafka#build-from-source) first.
On Linux the build needs `protoc`, a C toolchain, `pkg-config` and librdkafka. scalo compiles its gRPC `.proto` files, `aws-lc-sys` compiles C, and `rdkafka-sys` links librdkafka through pkg-config. It rejects a librdkafka older than the version after the `+` in its `Cargo.lock` entry, and a Debian or Ubuntu package can be older than that, so there [build librdkafka from source](https://github.com/confluentinc/librdkafka#build-from-source) first.

```bash
sudo apt-get install -y protobuf-compiler build-essential pkg-config
cargo build --release
./target/release/dfe-archiver --config config.dev.yaml
```

[config.dev.yaml](config.dev.yaml) reads `test-events` from the Redpanda in [docker-compose.dev.yaml](docker-compose.dev.yaml) on `localhost:19092`, writes under `./data/archive` and stages under `./data/spool`. The essentials go in as environment variables too:
[config.dev.yaml](config.dev.yaml) reads `test-events` from the Redpanda in [docker-compose.dev.yaml](docker-compose.dev.yaml) on `localhost:19092`, writes under `./data/archive` and stages under `./data/spool`. The essentials can be set as environment variables instead:

```bash
env DFE_ARCHIVER_KAFKA_BROKERS=localhost:19092 DFE_ARCHIVER_KAFKA_TOPICS=test-events DFE_ARCHIVER_DESTINATION=file://./data/archive DFE_ARCHIVER_SPOOL_DIR=./data/spool ./target/release/dfe-archiver
```

Without a writable spool directory the archiver refuses to start, and the default, `/var/spool/dfe/archiver`, is one the image creates. Metrics and health answer on port 9090 (`/metrics`, `/livez`, `/readyz`), and `curl -s localhost:9090/readyz` prints `{"status":"ready"}` once the pipeline runs. With no destination set the archiver starts, passes readiness and idles until a config change names one.
Without a writable spool directory the archiver does not start, and the default, `/var/spool/dfe/archiver`, is one the image creates. Metrics and health answer on port 9090 (`/metrics`, `/livez`, `/readyz`), and `curl -s localhost:9090/readyz` prints `{"status":"ready"}` once the pipeline runs. With no destination set the archiver starts, passes readiness and idles until a config change names one.

## Configuration

Settings load through a cascade: `DFE_ARCHIVER_*` environment variables, then `.env`, then the config file, then defaults. The only command-line setting is `--config`, which names the file. The pipeline reads its config at startup, so a change to any section below but `kafka.batch_size` takes a restart, and a reload logs the sections that need one. The archiver's own keys take a single underscore (`DFE_ARCHIVER_KAFKA_BROKERS`), and the scalo runtime sections (`version_check`, `metrics`, `logger`, `self_regulation`, `scaling`) take two (`DFE_ARCHIVER_VERSION_CHECK__ENABLED`).
Settings load through a cascade: `DFE_ARCHIVER_*` environment variables, then `.env`, then the config file, then defaults. `--config` names the file, and the other flags set logging and the metrics address ([docs/configuration.md](docs/configuration.md#scalo-runtime-variables)). The pipeline reads its config at startup, so a change to any section below but `kafka.batch_size` takes a restart, and a reload logs the sections that need one. The archiver's keys take a single underscore (`DFE_ARCHIVER_KAFKA_BROKERS`), and the scalo runtime sections (`version_check`, `metrics`, `logger`, `self_regulation`, `scaling`) take two (`DFE_ARCHIVER_VERSION_CHECK__ENABLED`).

| Key | What it sets | Default |
|-----|--------------|---------|
| `transport` | `kafka` (a consumer group) or `grpc` (the Push listener on `0.0.0.0:6000`) | `kafka` |
| `kafka.brokers`, `kafka.topics` | Brokers, and the topics to read. Empty topics discover every topic, narrowed by `topic_include` and `topic_exclude` | `localhost:9092`, discover |
| `archive.destination` | `file:///path`, `s3://bucket/prefix`, `minio://bucket/prefix`, `gs://bucket/prefix` or `az://container/prefix` | none, idles |
| `compression.codec` | `zstd`, `lz4`, `snappy`, `gzip` or `none` | `zstd` |
| `archive.roll_size_bytes`, `archive.roll_interval_secs` | When a file closes | 1 GiB, and 300 s while Kafka offsets are held or on `grpc`, else 3600 |
| `archive.roll_size_bytes`, `archive.roll_interval_secs` | When a file closes | 1 GiB, and 300 s when Kafka offsets are held or on `grpc`, else 3600 |
| `buffer.spool_dir` and its three caps | Where staged uploads wait, and how much disk they may take | `/var/spool/dfe/archiver`, and 80%, 10% and 10% of the volume's declared size limit |

Each store takes its own block (`archive.s3`, `archive.minio`, `archive.gcs`, `archive.azure`) for the endpoint and credentials. [docs/config-schema.yaml](docs/config-schema.yaml) lists every archiver key with its type and default, and [docs/configuration.md](docs/configuration.md) the cascade and every environment variable.
Each store takes a block (`archive.s3`, `archive.minio`, `archive.gcs`, `archive.azure`) for the endpoint and credentials. [docs/config-schema.yaml](docs/config-schema.yaml) lists every archiver key with its type and default, and [docs/configuration.md](docs/configuration.md) the cascade and every environment variable.

## Storage Backends

Each destination is one URL plus its store's block. These four load and validate as written, and a test keeps them that way. The file is read as written, so `${VAR}` in it stays a literal string. Set store keys in the environment instead: `DFE_ARCHIVER_S3_ACCESS_KEY_ID` and `DFE_ARCHIVER_S3_SECRET_ACCESS_KEY` for S3, `AZURE_STORAGE_ACCOUNT_KEY` for Azure, or each SDK's own credential chain ([docs/internals.md](docs/internals.md#storage-backends)).
Each destination is one URL plus its store's block. These four load and validate as written, and a test keeps them that way. The file is read as written, so `${VAR}` in it stays a literal string. Set store keys in the environment instead: `DFE_ARCHIVER_S3_ACCESS_KEY_ID` and `DFE_ARCHIVER_S3_SECRET_ACCESS_KEY` for S3, `AZURE_STORAGE_ACCOUNT_KEY` for Azure, or each SDK's credential chain ([docs/internals.md](docs/internals.md#storage-backends)).

```yaml
archive:
Expand Down Expand Up @@ -95,7 +95,7 @@ The suite's published docs are at <https://dfe.docs.hyperi.io>, and this reposit

## Help and contributing

How to work in this repo, the commands that prove a change and what tends to bite are in [CONTRIBUTING.md](CONTRIBUTING.md). A question that is not a bug goes in a new issue.
How to work in this repo, the commands that test a change and what tends to bite are in [CONTRIBUTING.md](CONTRIBUTING.md). A question that is not a bug goes in an issue.

Open bugs: [this repo](https://github.com/hyperi-io/dfe-archiver/issues?q=is%3Aopen+is%3Aissue+type%3ABug+-label%3Abacklog+-label%3Anext-minor) | [whole DFE suite](https://github.com/search?type=issues&q=is%3Aopen+is%3Aissue+type%3ABug+-label%3Abacklog+-label%3Anext-minor+repo%3Ahyperi-io%2Fdfe-infra+repo%3Ahyperi-io%2Fdfe-docker+repo%3Ahyperi-io%2Fdfe-deploy+repo%3Ahyperi-io%2Fdfe-engine+repo%3Ahyperi-io%2Fdfe-loader+repo%3Ahyperi-io%2Fdfe-receiver+repo%3Ahyperi-io%2Fdfe-fetcher+repo%3Ahyperi-io%2Fdfe-archiver+repo%3Ahyperi-io%2Fdfe-transform-vrl+repo%3Ahyperi-io%2Fdfe-transform-vector+repo%3Ahyperi-io%2Fdfe-transform-elastic+repo%3Ahyperi-io%2Fdfe-ui+repo%3Ahyperi-io%2Fdfe-hyperdx+repo%3Ahyperi-io%2Fscalo-rs+repo%3Ahyperi-io%2Fscalo-py+repo%3Ahyperi-io%2Flogreducer+repo%3Ahyperi-io%2Fdfe-schemas+repo%3Ahyperi-io%2Fvector-vrl+repo%3Ahyperi-io%2Fculvert+repo%3Ahyperi-io%2Fdfe-clickhouse-rs+repo%3Ahyperi-io%2Ffactbook-rs)

Expand Down
36 changes: 3 additions & 33 deletions _typos.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,31 +4,15 @@
# License: BUSL-1.1
# Copyright: (c) 2026 HYPERI PTY LIMITED
#
# Based on: ci/templates/config/typos.toml
# Reference: https://github.com/crate-ci/typos
#
# Spelling policy (from ai/standards):
# Code: American English (serialize, color, initialize)
# Docs/comments: Australian English (serialise, colour, initialise)
#
# typos cannot distinguish code from docs, so we use en-au locale
# and allowlist the American spellings that appear in code.
# Prose is Australian English and code American, so the American spellings code uses are allowlisted.

[default]
# Use Australian English — flags American spellings in prose
locale = "en-au"

[default.extend-words]
# ---------------------------------------------------------------------------
# American spellings allowed in code (programming convention)
# ---------------------------------------------------------------------------
# typos uses case-insensitive matching for extend-words, so "serialize"
# covers Serialize, SERIALIZE, serialize, etc.
#
# Format: american_spelling = "american_spelling" (identity = don't flag)
# ---------------------------------------------------------------------------

# -ize/-ise variants (most common in code)
# Matching is case-insensitive, so each lowercase entry covers every casing.
serialize = "serialize"
serializes = "serializes"
serialized = "serialized"
Expand Down Expand Up @@ -209,7 +193,6 @@ monetized = "monetized"
monetizing = "monetizing"
monetization = "monetization"

# -our/-or variants
color = "color"
colors = "colors"
colored = "colored"
Expand All @@ -225,7 +208,6 @@ neighbor = "neighbor"
neighbors = "neighbors"
neighboring = "neighboring"

# -ence/-ense variants
defense = "defense"
defensive = "defensive"
offense = "offense"
Expand All @@ -235,20 +217,17 @@ licensed = "licensed"
licenses = "licenses"
licensing = "licensing"

# -re/-er variants (less common in code but appear in libs)
center = "center"
centered = "centered"
centering = "centering"
fiber = "fiber"
meter = "meter"

# -log/-logue variants
catalog = "catalog"
catalogs = "catalogs"
dialog = "dialog"
dialogs = "dialogs"

# Other American spellings common in code
canceled = "canceled"
canceling = "canceling"
labeled = "labeled"
Expand All @@ -263,23 +242,17 @@ gray = "gray"
artifact = "artifact"
artifacts = "artifacts"

# ---------------------------------------------------------------------------
# Project-specific: Australian/British spellings in legal templates
# ---------------------------------------------------------------------------
# The BUSL-1.1 LICENSE file uses British "licence" (noun form).
# Allow both licence and license since both appear in the project.
# Both licence and license appear in the project.
licence = "licence"

# Crate name flate2 contains "flate" which is not a typo
flate = "flate"

[default.extend-identifiers]
# Rust crate names and identifiers
flate2 = "flate2"
lz4_flex = "lz4_flex"

[files]
# Files/directories to ignore
extend-exclude = [
# Lock files contain hashes
"*.lock",
Expand All @@ -288,7 +261,6 @@ extend-exclude = [
"package-lock.json",
"pnpm-lock.yaml",

# Binary/minified assets
"*.min.js",
"*.min.css",
"*.svg",
Expand All @@ -297,7 +269,6 @@ extend-exclude = [
"*.ttf",
"*.eot",

# Build/cache directories
".git/",
".venv/",
"node_modules/",
Expand All @@ -312,7 +283,6 @@ extend-exclude = [
"CHANGELOG.md",
"**/CHANGELOG.md",

# Git submodules (contain their own changelogs/commit hashes)
"ci/",
"ai/",
]
5 changes: 0 additions & 5 deletions crates/archiver/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -46,23 +46,18 @@ scalo = { workspace = true, features = [
"scaling", "governor", "expression", "deployment", "config-schema", "cli-service", "memory", "health", "shutdown", "dlq", "dlq-kafka", "version-check",
] }

# Async
tokio = { workspace = true }
tokio-util = { workspace = true }
futures = { workspace = true }

# Serialization
serde = { workspace = true }
serde_json = { workspace = true }

# Config
clap = { workspace = true }
dotenvy = { workspace = true }

# Tracing
tracing = { workspace = true }

# Utilities
chrono = { workspace = true }
compact_str = { workspace = true }
lru = { workspace = true }
Expand Down
6 changes: 0 additions & 6 deletions crates/core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,17 @@ scalo = { workspace = true, features = ["transport-kafka", "scaling", "dlq", "dl

schemars = { workspace = true }

# Serialization
serde = { workspace = true }
serde_json = { workspace = true }
sonic-rs = { workspace = true }

# Error handling
thiserror = { workspace = true }

# Compression
zstd = { workspace = true }
lz4_flex = { workspace = true }
snap = { workspace = true }
flate2 = { workspace = true }

# Utilities
bytes = { workspace = true }
chrono = { workspace = true }
compact_str = { workspace = true }
Expand All @@ -50,11 +46,9 @@ parking_lot = { workspace = true }
smallvec = { workspace = true }
indexmap = { workspace = true }

# Async
tokio = { workspace = true }
async-trait = { workspace = true }

# Tracing
tracing = { workspace = true }

[dev-dependencies]
Expand Down
3 changes: 0 additions & 3 deletions crates/io/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,12 @@ dfe-archiver-core = { workspace = true }
# kafka and grpc are the only transports `validate_config` accepts.
scalo = { workspace = true, features = ["transport-kafka", "transport-grpc", "metrics", "governor"] }

# Cloud storage
object_store = { workspace = true }

# Async
tokio = { workspace = true }
async-trait = { workspace = true }
futures = { workspace = true }

# Utilities
compact_str = { workspace = true }
parking_lot = { workspace = true }
tracing = { workspace = true }
Expand Down
Loading
Loading