From d2758e8778b61ad0edc260c8228939375e554e46 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 13:40:24 +1100 Subject: [PATCH 1/2] docs(elastic): fix docs to match the code Wrong claims in README.md, CONTRIBUTING.md and docs/ are fixed or cut. The nine Mermaid diagrams follow the diagrams standard. All nine failed the checker before and pass it after the changes. - The architecture page drew a TransformChain the service never runs. It shows the four steps, the crate graph, one batch as a sequence, and what a failed processor does. - Pages describing the parity corpus test, its baseline and its fixture format are cut to what this repository holds: compare-policy.yaml and the known limits. - Docs mark layers 2 and 3 of dfe-parse, the dfe-parsers crate and schema generation as built but unused, or as designs. The dfe-parsers section is cut. - configuration.md documents the bad-input counters, the metrics-manifest subcommand and /scaling/pressure. - Config comments are cut to a pointer or a one-line reason in 15 files. Every parsed value is identical before and after. - src/envelope.rs: doc comments only. Envelope detection runs for each event in a batch. --- .cargo/config.toml | 15 +- .github/workflows/ci.yml | 27 +- .github/workflows/sync-elastic-variants.yml | 10 +- .gitleaks.toml | 15 +- .hyperi-ci.yaml | 45 +- .prose-exemptions.yaml | 19 + CONTRIBUTING.md | 103 +++-- Cargo.toml | 113 ++--- README.md | 27 +- crates/dfe-core/Cargo.toml | 13 +- crates/dfe-painless/Cargo.toml | 4 +- crates/dfe-runtime/Cargo.toml | 21 +- crates/dfe-transforms/Cargo.toml | 20 +- crates/pgo-driver/Cargo.toml | 8 +- deny.toml | 16 +- docs/README.md | 32 +- docs/architecture.md | 437 ++++++++------------ docs/compat-policy.md | 164 +------- docs/compat.md | 60 +-- docs/configuration.md | 58 ++- docs/enrichment.md | 87 ++-- docs/parity.md | 111 ++--- docs/parsers.md | 255 ++++-------- docs/performance.md | 117 ++---- docs/schema-generation.md | 149 +------ docs/shared-crates.md | 57 +-- docs/testing.md | 82 ++-- scripts/dfe-schemas.toml | 4 +- sources.yaml | 45 +- src/envelope.rs | 14 +- tests/compare-policy.yaml | 47 +-- 31 files changed, 668 insertions(+), 1507 deletions(-) create mode 100644 .prose-exemptions.yaml diff --git a/.cargo/config.toml b/.cargo/config.toml index f2fc0de6d..da7539cf5 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,25 +1,16 @@ # Cargo build configuration for dfe-transform-elastic # -# Target platforms: Linux x86_64 and aarch64 only. -# SIMD matters here: simd-json, memchr and aho-corasick all dispatch on it. +# Why these targets: docs/performance.md. # x86_64: AVX2, BMI1/2, FMA -- Haswell and later. [target.x86_64-unknown-linux-gnu] rustflags = ["-C", "target-cpu=x86-64-v3"] -# aarch64: NEON is baseline, so generic already has it. -# -# mold rather than the default bfd, because a PGO-instrumented build of this -# binary overflows AArch64's 128 MB direct branch and bfd cannot reach past it -# (hyperi-io/hyperi-ci#142). mold inserts the range-extension thunks. -# -# Not lld, which hyperi-ci installs only under its version-suffixed name, so -# gcc's `-fuse-ld=lld` finds no `ld.lld` and collect2 fails the link outright. +# mold: a PGO build overflows AArch64's direct branch range under bfd, and lld is not on the runners (hyperi-io/hyperi-ci#142). [target.aarch64-unknown-linux-gnu] linker = "aarch64-linux-gnu-gcc" rustflags = ["-C", "target-cpu=generic", "-C", "link-arg=-fuse-ld=mold"] -# Untargeted local builds get the build host's own features. The per-target -# sections above win whenever `--target` is passed, which is what CI does. +# Untargeted local builds. CI passes `--target`, so the sections above win there. [build] rustflags = ["-C", "target-cpu=native"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86f42ebab..806ff493a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,8 +8,7 @@ name: CI on: - # No nightly schedule: no test here is #[ignore]d, so the full tier is the core - # tier every push already runs (docs/testing.md). + # No nightly schedule: no test is #[ignore]d, so core is the full tier (docs/testing.md). push: branches: ["**"] paths-ignore: @@ -70,10 +69,7 @@ jobs: ci: uses: hyperi-io/hyperi-ci/.github/workflows/rust-ci.yml@main with: - # No runner-quality: one dfe-transforms rustc peaks near 14 GiB, which a - # 16 GiB 8x16 runner kills, so Quality stays on the org's GH_RUNNER_RUST. - # `both` resolves to the release channel; `internal` is the spike - # channel and would demote a release build silently. + # No runner-quality: a 16 GiB runner kills a 14 GiB rustc. `internal` would demote a release to the spike channel. publish-target: both tag: ${{ inputs.tag || '' }} from-head: ${{ inputs.from-head || '' }} @@ -81,14 +77,10 @@ jobs: skip-optimize: ${{ inputs.skip-optimize || '' }} release-unoptimized: ${{ inputs.release-unoptimized || '' }} optimize-tier: ${{ inputs.optimize-tier || '' }} - # A PGO+BOLT release builds the binary three times, ending in a single-threaded LTO pass; one build took over three hours. + # A PGO and BOLT release builds the binary three times. build-timeout-minutes: 360 test-tier: ${{ inputs.test-tier || 'core' }} - # Least privilege: the org carries 31+ secrets and this pipeline uses seven. - # Only names declared in rust-ci.yml's `workflow_call.secrets` can be - # passed, so an addition there is needed before one can be added here. - # No crates.io token: the package has binary targets and publishes no - # crate, so the preflight never asks for one. + # Least privilege: only the secrets this pipeline uses. No crates.io token, nothing is published there. secrets: GITLEAKS_GH_ACTIONS_KEY: ${{ secrets.GITLEAKS_GH_ACTIONS_KEY }} DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} @@ -98,15 +90,8 @@ jobs: R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} - # The engine offers `sources.yaml` as its source catalogue and content-pins it - # the way it pins dfe-schemas, so a release has to carry the file (issue #18). - # hyperi-ci has no release-asset stanza and this needs no change there: the - # upload lives in this repo's own workflow. - # - # NOT triggered on `release: published` -- semantic-release creates the - # release with a token whose events cannot start another workflow run, so - # that trigger would never fire. This gates on the same events that can - # publish and then proves a release for THIS commit exists before uploading. + # The engine pins `sources.yaml` from the release, so a release carries it (issue #18). + # Not on `release: published`: semantic-release's token cannot start a workflow run. catalogue-asset: name: Source catalogue asset needs: ci diff --git a/.github/workflows/sync-elastic-variants.yml b/.github/workflows/sync-elastic-variants.yml index a26f40673..50c45d7e5 100644 --- a/.github/workflows/sync-elastic-variants.yml +++ b/.github/workflows/sync-elastic-variants.yml @@ -5,13 +5,7 @@ # License: BUSL-1.1 # Copyright: (c) 2026 HYPERI PTY LIMITED # -# dfe-schemas checks every elastic catalogue entry against that file, and the contract this -# service publishes names its variants but not the intakes each one accepts. -# -# Auth: a GitHub App installation token (vars.GH_APP_CLIENT_ID + secrets.GH_APP_PRIVATE_KEY), -# scoped to dfe-schemas with contents + pull-requests write, which covers the branch push -# and opening, editing or closing the PR. -# Optional var: DFE_SCHEMAS_REPO - repo as owner/name (default: hyperi-io/dfe-schemas) +# Why: CONTRIBUTING.md, "The catalogue and the detect tables". Optional var DFE_SCHEMAS_REPO (owner/name). name: Sync elastic variants @@ -36,7 +30,7 @@ concurrency: jobs: push-to-dfe-schemas: name: Open or update the dfe-schemas sync PR - # scripts/sources.py reads with PyYAML, which the native runner image's python3 carries and the vanilla default may not. + # The native runner's python3 has PyYAML, which scripts/sources.py needs. runs-on: ${{ vars.GH_RUNNER_PYTHON || 'ubuntu-latest' }} timeout-minutes: 10 steps: diff --git a/.gitleaks.toml b/.gitleaks.toml index 0c960f98c..3c2195119 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -6,11 +6,6 @@ # License: BUSL-1.1 # Copyright: (c) 2026 HYPERI PTY LIMITED # -# Extends the CI base configuration (ci/.gitleaks.toml) -# -# The CI runs gitleaks to detect secrets in your code. This file lets you -# exclude false positives specific to your project. -# # Documentation: https://github.com/gitleaks/gitleaks#configuration [extend] @@ -19,23 +14,17 @@ useDefault = true [allowlist] description = "Project-specific false positive allowlist" -# Paths to exclude from scanning (regex patterns) paths = [ - # Git submodules (have their own scanning) '''(^|/)ci/''', '''(^|/)ai/''', - # Vendor sample logs -- synthetic ids that trip the entropy rules. + # Sample logs: invented ids trip the entropy rules. '''(^|/)tests/fixtures/''', '''(^|/)tests/pipelines/''', ] -# Regex patterns for known false positives regexes = [ # Meraki event-type literal carried through from the Elastic pipeline. '''8021x_eap_failure''', - # Confluent's PUBLIC signing-key fingerprint, pinned by the generated - # Dockerfile and carried into docs/deployment-contract.json. Publishing it - # is the security control -- the build verifies the downloaded key against - # it before trusting the apt repo. + # Confluent's public signing-key fingerprint, which the Dockerfile verifies. '''CBBB821E8FAF364F79835C438B1DA6120C2BF624''', ] diff --git a/.hyperi-ci.yaml b/.hyperi-ci.yaml index 281a35a4b..29f957adf 100644 --- a/.hyperi-ci.yaml +++ b/.hyperi-ci.yaml @@ -5,81 +5,54 @@ # License: BUSL-1.1 # Copyright: (c) 2026 HYPERI PTY LIMITED -# Repo category. Matches the `classification` custom property on the org. -# Values: internal, product, fork, general-oss classification: product -# Project language (auto-detected if not specified) -# Values: python, typescript, golang, rust, bash, none language: rust -# Quality checks quality: enabled: true ignore: - # GHSA-w9wp-h8wv-79jx / CVE-2026-48504: opentelemetry_sdk 0.31.0 is in the - # lock through scalo, in the affected range (<= 0.32.0, fixed in 0.32.1). - # The flaw is in BaggagePropagator::extract_with_context, which parses an - # inbound W3C Baggage header before enforcing its size limits. Neither this - # repo nor scalo registers a Baggage or other text map propagator, so it is - # never called. Recorded as not reachable in the DFE threat model (dfe-infra - # docs/THREAT-MODEL.md, accepted risks). It is a GitHub advisory with no - # RustSec entry, so only osv-scanner sees it. scalo holds 0.31 because - # metrics-exporter-opentelemetry 0.2.1 requires opentelemetry ^0.31; - # tracked in hyperi-io/scalo-rs#281. - tool: osv-scanner id: GHSA-w9wp-h8wv-79jx reason: "CVE-2026-48504, opentelemetry_sdk 0.31.0 via scalo: unreachable, no Baggage propagator registered in dfe-transform-elastic or scalo. Not reachable per dfe-infra docs/THREAT-MODEL.md accepted risks. Blocked on metrics-exporter-opentelemetry 0.2.1 requiring opentelemetry ^0.31; tracked in hyperi-io/scalo-rs#281." -# Testing test: enabled: true coverage: true rust: - # Required, not auto: the memory and cascade tests read process-wide state, and only nextest gives each test a process of its own. + # Required: the memory and cascade tests need a process each (docs/testing.md). nextest: true -# Building build: enabled: true strategies: - native rust: glibc_max: "2.41" - # librdkafka and OpenSSL link statically from hyperi-ci's prefix, so the - # image installs no C library packages. + # librdkafka and OpenSSL link statically, so the image installs no C libraries. static_libs: true - # The arm64 leg needs a larger runner for MEMORY, not cores. `dfe-transforms` - # carries one module per data stream and holds 13.15 GiB in a single rustc at - # opt-level 1; the hosted `ubuntu-24.04-arm` is 2 vCPU and 8 GiB on a private - # repo, so the build was OOM-killed after 12 minutes. A repo-level - # `GH_RUNNER_ARM64` points this repo at a 16-core, 64 GiB runner (issue #27). + # arm64 builds on the larger runner `GH_RUNNER_ARM64` names, for memory (issue #27). targets: - x86_64-unknown-linux-gnu - aarch64-unknown-linux-gnu - # One rustc here peaks near 14 GiB, and clippy --all-targets builds the lib and its test target at once, so a 24 GiB runner fits one job. + # One rustc here peaks near 14 GiB. memory_per_job_gib: 14 - # The cost lands on a RELEASE and never on a PR: the build half only runs - # behind the `Publish: true` trailer, a `hyperi-ci publish` dispatch, or a - # dispatch with `optimize-tier: release`, so the day-to-day loop is untouched. + # Release builds only, never a PR (CONTRIBUTING.md, "CI is slow here"). optimize: pgo: enabled: true - # 600s of load, 300s per source. The prefix holds it there over the duration_secs hyperi-ci exports. + # The prefix holds the load at 600 s over the duration_secs hyperi-ci exports. workload_cmd: "env PGO_WORKLOAD_DURATION_SECS=600 bash scripts/pgo-workload.sh" - # Built before the workload clock starts, so the script's own cargo build finds it fresh. + # Built before the workload clock starts. workload_setup_cmd: "cargo build --release -p pgo-driver --features driver" - # hyperi-ci's timeout is this plus 600s, which covers the broker pull as well as the load. + # hyperi-ci's timeout is this plus 600 s, which covers the broker pull. duration_secs: 900 bolt: enabled: true -# Publishing (on release). The stack pins every app by GHCR image digest, so -# a release must land an image, and the thin chart assembled from the emitted -# contract ships beside it. release: enabled: true - # Keep `library` at the scalo version in Cargo.toml: the library renders only the contract version its scalo release writes. + # Keep `library` at the scalo version in Cargo.toml (CONTRIBUTING.md, "Generated artefacts"). helm: enabled: true contract: emit diff --git a/.prose-exemptions.yaml b/.prose-exemptions.yaml new file mode 100644 index 000000000..569428a59 --- /dev/null +++ b/.prose-exemptions.yaml @@ -0,0 +1,19 @@ +# Project: dfe-transform-elastic +# File: .prose-exemptions.yaml +# Purpose: Reviewed spans of text the prose rules leave alone. +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +exemptions: + - path_glob: CONTRIBUTING.md + rules: [punct.semicolon] + why: the Developer Certificate of Origin, quoted verbatim + text: |- + 1. The contribution was created in whole or in part by you and you have the + right to submit it under the license indicated in the file; or + + 2. The contribution is based upon previous work that, to the best of your + knowledge, is covered under an appropriate open source license and you + have the right under that license to submit that work with modifications; + or diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9deea30ee..9495f12e1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,7 +17,7 @@ versioning and changelog generation. All commits must follow this format: [optional footer(s)] ``` -### Types +The type sets the version bump: | Type | Description | Version Bump | |------|-------------|--------------| @@ -33,8 +33,6 @@ versioning and changelog generation. All commits must follow this format: | `chore` | Other changes that don't modify src or test files | None | | `revert` | Reverts a previous commit | Varies | -### Breaking Changes - For breaking changes that require a major version bump, add `!` after the type or include `BREAKING CHANGE:` in the footer: @@ -44,7 +42,7 @@ feat!: remove deprecated API endpoints BREAKING CHANGE: The /v1/users endpoint has been removed. Use /v2/users instead. ``` -### Examples +Examples: ```text feat(auth): add OAuth2 support for Google login @@ -58,26 +56,22 @@ refactor(core)!: restructure module exports BREAKING CHANGE: Named exports are now used instead of default exports. ``` -### Scope - Scope is optional but recommended. Use it to indicate the area of the codebase affected (e.g., `api`, `auth`, `core`, `cli`, `docs`). -## Semantic Versioning - This project follows [Semantic Versioning 2.0.0](https://semver.org/): -- **MAJOR** (X.0.0): Breaking changes that require users to modify their code -- **MINOR** (0.X.0): New features that are backwards-compatible -- **PATCH** (0.0.X): Bug fixes and minor improvements +- MAJOR (X.0.0): breaking changes that need users to modify their code +- MINOR (0.X.0): backwards-compatible features +- PATCH (0.0.X): bug fixes and small improvements Versions are automatically determined by semantic-release based on commit messages. Do not manually update version numbers. ## Developer Certificate of Origin -This project uses the Developer Certificate of Origin (DCO) to ensure that -contributors have the right to submit their contributions. +This project uses the Developer Certificate of Origin (DCO) as the record that +each contributor has the right to submit their contribution. By making a contribution to this project, you certify that: @@ -98,8 +92,6 @@ By making a contribution to this project, you certify that: and may be redistributed consistent with this project or the license(s) involved. -## How to Sign Off Your Commits - You must sign off each commit to indicate your acceptance of the DCO. Combine the signoff with your conventional commit message: @@ -122,8 +114,6 @@ git config --global user.name "Your Name" git config --global user.email "your.email@example.com" ``` -## License for Contributions - All contributions to this project are licensed under the Business Source License 1.1 (BUSL-1.1), the same license that covers the project. @@ -133,13 +123,15 @@ anniversary of its release. ## How to Contribute -1. **Fork the repository** and create your branch from `main` -2. **Make your changes** following the commit message format above -3. **Sign off your commits** with the DCO -4. **Test your changes** to ensure they work as expected -5. **Submit a pull request** with a clear description of what you've done +1. Fork the repository and create your branch from `main`. +2. Make your changes, following the commit message format above. +3. Sign off your commits with the DCO. +4. Run the tests below. +5. Open a pull request that says what the change does. + +A question goes in an issue. -### Pull Request Checklist +Before you open it, check: - [ ] Commits follow the conventional commit format - [ ] All commits are signed off (DCO) @@ -148,22 +140,14 @@ anniversary of its release. ## CI/CD Workflow -When your pull request is merged to `main`: - -1. **semantic-release** analyses commit messages since the last release -2. Determines the next version number based on commit types -3. Generates/updates the CHANGELOG -4. Creates a new GitHub release with release notes -5. Publishes the package (if applicable) - -This happens automatically - no manual intervention required. +A pull request runs the quality and test jobs. Merging to `main` does not release. A release is a merge with a `Publish: true` trailer in its commit, or a `hyperi-ci publish` dispatch, and semantic-release then picks the version from the commit types since the last release, updates the CHANGELOG, and creates the GitHub release with the image. ## CI is slow here, so iterate off it **A push to find out what CI says costs the better part of an hour.** Treat CI as the gate that proves a finished change, never as the type checker. -Measured on this repo, per release run: +Per release run on this repo: | Stage | Elapsed | |---|---| @@ -171,26 +155,24 @@ Measured on this repo, per release run: | Build, linux-amd64 | ~30 min plain, 93 min with PGO | | Build, linux-arm64 | ~6-16% longer again than amd64 | -The PGO figure is measured rather than projected: instrument build, then 300 +The PGO figure is from a run, not a projection: instrument build, then 300 seconds of load against each of two sources, then the optimise build. The cause is this repo's size rather than a misconfiguration. `dfe-transforms` -carries one module per data stream, over a thousand of them across about 2,200 -source files, so it compiles as a single `rustc` holding about 13 GiB whatever -`-j` says, and the binary then links under fat LTO in one thread. Neither step -spreads across cores, which is why a bigger runner buys very little. - -**PGO and BOLT are configured for the release build and switched on together -at GA release prep.** PGO alone is two cargo passes around 600 -seconds of load, which measured 93 minutes. BOLT adds two more passes and +has one module per data stream, over a thousand of them across about 2,200 +source files, split into crates by package so each `rustc` peaks near 14 GiB +whatever `-j` says, and the binary then links under thin LTO with one codegen +unit. Neither step spreads well across cores, which is why a bigger runner buys +very little. + +**PGO and BOLT are both on for the release build** (`.hyperi-ci.yaml`). PGO alone is two cargo passes around 600 +seconds of load. BOLT adds two more passes and another workload, and retries the pair on a failure, so the two together are roughly two hours per architecture and three with a retry. -That cost falls only on a release, never on a PR, because the build half runs +That cost falls on a release, never on a PR, because the build half runs behind the `Publish: true` trailer or a `hyperi-ci publish` dispatch. -### What to run instead - Everything below runs locally. `hyperi-ci check` takes about an hour and runs the same gates CI does, and the rest answer in seconds: @@ -202,15 +184,15 @@ hyperi-ci check cargo test -p dfe-transforms --test integration --test untested_sources --test unencumbered ``` -Keep a `bacon` session running while you edit -- it recompiles on save and gives -verdicts without you spending a CI run on them. The first build of the workspace +Keep a `bacon` session running as you edit. It recompiles on save and gives +verdicts without spending a CI run on them. The first build of the workspace is cold and takes around twenty minutes, and every one after it is incremental. Two traps worth knowing before you burn a run on them: - **Never set `CARGO_BUILD_JOBS` by hand.** A hand-set value beats the project's - own pinning and any build governor on the host. -- **The compat corpus is not here.** It is Elastic-licensed and lives in the private dfe-transform-elastic-dev repository, which runs the parity tests. Never copy it into `tests/fixtures/`: `unencumbered.rs` fails a tree that holds anything but `unencumbered/` and `synthetic/`. + pinning and any build governor on the host. +- **The parity corpus is not here, and must never be added.** `unencumbered.rs` fails when `tests/fixtures/` holds anything but `unencumbered/`, `synthetic/` and the provenance, or any file naming Elastic's source repositories or licence. ## Working in this repo @@ -221,7 +203,7 @@ hyperi-ci check `librdkafka` 2.12.1 or later is needed for the `kafka` feature, which is on by default. `cargo build --no-default-features` builds without it. -Non-English text is a tested case, not an edge case. `tests/unicode.rs` runs every registered transform against seventeen scripts and a set of degenerate inputs. +Non-English text is a tested case, not an edge case. `tests/unicode.rs` runs every registered transform against seventeen samples of scripts, combining marks, emoji and degenerate inputs. ### Generated artefacts @@ -239,7 +221,7 @@ The deployed chart is not committed here. At release, hyperi-ci runs `generate-a **Not every artefact is pinned against a fresh regen.** `committed_config_artefacts_do_not_drift` in `src/deployment.rs` compares `config-schema.{json,yaml}` and `capability-catalog.{json,yaml}`, and its sibling tests do the same for the committed `deployment-contract.json`, `Dockerfile` and `config.example.yaml`. Nothing compares `container-manifest.json` or `Dockerfile.runtime`, so those can and do fall behind -- re-run the command when `src/deployment.rs` changes rather than assuming a test caught it. -`metrics-manifest.json` is compared by metric NAME SET in `src/metrics.rs`, not byte for byte, because it also records the version and commit of the build that wrote it. +`metrics-manifest.json` is compared by metric NAME SET in `src/metrics.rs`, not byte for byte, because it records the version and commit of the build that wrote it. The image expects the release binary in the build context: @@ -251,10 +233,21 @@ docker build -t dfe-transform-elastic . ### The catalogue and the detect tables -`scripts/emit_variants.py` prints every registered variant with the intakes it accepts. On a push to `main` that changes it, `.github/workflows/sync-elastic-variants.yml` opens or updates a `chore(elastic-variants)` PR on dfe-schemas, whose catalogue test checks every elastic entry against that file. +`sources.yaml` is the source catalogue: every source this service transforms, and what its name does not say. The registry is asserted against it, and `dfe-transform-elastic emit-catalogue` prints the copy compiled into the binary, which dfe-engine offers as its source catalogue. Per source: -The other direction is generated code. `src/detect/labels.rs` (each variant's `_source` label) and `src/detect/markers.rs` (the agentless marker table) come from the dfe-schemas wheel `scripts/dfe-schemas.toml` pins by version and sha256. To move the pin, set both values from `https://pypi.org/pypi/dfe-schemas//json` and run `scripts/gen_detect_tables.py`. `tests/scripts_suite.rs` runs it with `--check`, so a stale file fails the test job. +| Key | Holds | +|---|---| +| `package` | The integration package directory | +| `data_stream` | The data stream directory within that package | +| `pipelines` | Where the source's pipelines sit in the pipeline tree | +| `fixtures` | The source's fixture directory | +| `beats` | The Beats module and fileset for the same source, if any | +| `intakes` | Every way the payload can arrive: `beats` always, `receiver` when a device pushes it, `fetcher` when Elastic's agent input is a pure transport such as `httpjson`, `cel` or `aws-s3` | +| `framing` | Receiver intakes only: `line` if the pipeline groks the syslog header out of `message`, `body` if a header would corrupt its first field | +| `transforms` | The generated transforms wired into the registry, as `filebeat..` | + +None of it is derivable from the name: fortinet's package is `fortinet_fortigate`, and one azure package is four of our modules. -## Questions +`scripts/emit_variants.py` prints every registered variant with the intakes it accepts. On a push to `main` that changes it, `.github/workflows/sync-elastic-variants.yml` opens or updates a `chore(elastic-variants)` PR on dfe-schemas, and its catalogue test checks every elastic entry against that file. -If you have questions about contributing, please open an issue or contact us. +The other direction is generated code. `src/detect/labels.rs` (each variant's `_source` label) and `src/detect/markers.rs` (the agentless marker table) come from the dfe-schemas wheel `scripts/dfe-schemas.toml` pins by version and sha256. To move the pin, set both values from `https://pypi.org/pypi/dfe-schemas//json` and run `scripts/gen_detect_tables.py`. `tests/scripts_suite.rs` runs it with `--check`, so a stale file fails the test job. diff --git a/Cargo.toml b/Cargo.toml index 1b0a7352e..6f9dc6383 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,16 +46,11 @@ tikv-jemallocator = { workspace = true, optional = true } parking_lot = { workspace = true } [dev-dependencies] -# `tests/unicode.rs` runs its sweeps through `dfe_runtime::testutil`, which is -# feature-gated. The dependency above asks for no features, so the module is -# configured out whenever this package is built ALONE -- a whole-workspace -# build only worked because dfe-transforms happens to ask for it. +# `tests/unicode.rs` needs `testutil`, which a package-alone build leaves off. dfe-runtime = { workspace = true, features = ["testutil"] } -# `start_paused` for the loop tests: the pipeline's retry backoff is fast-forwarded -# by a paused clock instead of waited out. +# `start_paused` fast-forwards the loop tests' retry backoff. tokio = { workspace = true, features = ["test-util"] } -# An in-process source, so the loop's hold is tested without a broker, and -# the run's shared Kafka broker for the tests that need one. +# An in-process source for the loop tests, and the run's shared Kafka broker. scalo = { workspace = true, features = [ "transport-memory", "deployment-test-support", @@ -80,17 +75,11 @@ harness = false [features] default = ["kafka", "grpc"] -# hyperi-ci enables this on spike/alpha/beta/release. Local dev stays on the -# system allocator so `cargo build` is not paying for it. +# hyperi-ci enables this on release builds. Local builds use the system allocator. jemalloc = ["dep:tikv-jemallocator"] -# The Kafka service loop. Separable so the crate still builds and tests on a -# host without librdkafka-dev; needs the LIBRDKAFKA C library at 2.12.1 or -# later when enabled, which is unrelated to the scalo version below. +# Needs librdkafka 2.12.1 or later. Off, the crate builds without librdkafka-dev. kafka = ["scalo/transport-kafka"] -# The direct transport: a scalo Push listener inbound, a gRPC client outbound. -# Additive to `kafka` rather than an alternative, and in `default` because the -# release build takes no other feature list and the deployment contract -# declares the listener port the shipped binary must be able to bind. +# In `default`: the release build takes no other feature list, and the contract declares the push port. grpc = ["scalo/transport-grpc"] [lints.rust] @@ -107,7 +96,6 @@ dbg_macro = "deny" module_name_repetitions = "allow" # #[must_use] goes where dropping the value is a bug, not on every pure function. must_use_candidate = "allow" -# About 38 public Result and panicking fns have no # Errors or # Panics section, 8 of them in dfe-core. missing_errors_doc = "allow" missing_panics_doc = "allow" @@ -135,13 +123,9 @@ members = [ "crates/dfe-transforms-zscaler", "crates/pgo-driver", ] -# Edition 2024's resolver. Feature unification is per-target, so a dev- or -# build-dependency's features no longer leak into the normal build. resolver = "3" -# The member crates ARE the hot path, so they carry the same panic guards the -# root package does. `missing_docs` is deliberately absent: it is a library -# lint, and these crates are internal to a black-box service. +# The member crates are the hot path, so they share the root's panic guards. No `missing_docs`: they are internal. [workspace.lints.rust] unsafe_code = "forbid" # Set by the private repository's CI, which supplies the script-based test modules. @@ -157,22 +141,19 @@ dbg_macro = "deny" module_name_repetitions = "allow" # #[must_use] goes where dropping the value is a bug, not on every pure function. must_use_candidate = "allow" -# About 38 public Result and panicking fns have no # Errors or # Panics section, 8 of them in dfe-core. missing_errors_doc = "allow" missing_panics_doc = "allow" [workspace.package] version = "0.1.0" edition = "2024" -# A black-box service, not a library. Nothing here goes to crates.io, and -# saying so stops an accidental publish of BUSL-1.1 code. +# Stops an accidental crates.io publish of BUSL-1.1 code. publish = false license = "BUSL-1.1" repository = "https://github.com/hyperi-io/dfe-transform-elastic" rust-version = "1.95" [workspace.dependencies] -# Workspace crates dfe-core = { path = "crates/dfe-core" } dfe-painless = { path = "crates/dfe-painless" } dfe-parse = { path = "crates/dfe-parse" } @@ -194,10 +175,7 @@ dfe-transforms-sysdig-traefik = { path = "crates/dfe-transforms-sysdig-traefik" dfe-transforms-trellix-zoom = { path = "crates/dfe-transforms-trellix-zoom" } dfe-transforms-zscaler = { path = "crates/dfe-transforms-zscaler" } -# Batteries-included runtime (crates.io) — NEVER a path dependency. -# Features are added when their API is actually referenced, not speculatively. -# The floor moves with `release.helm.library` in .hyperi-ci.yaml: the library -# renders only the contract version its scalo release writes. +# crates.io, never a path dependency. The floor moves with `release.helm.library` in .hyperi-ci.yaml. scalo = { version = ">=2.14.30, <3", features = [ "cli-service", "config", @@ -213,116 +191,79 @@ scalo = { version = ">=2.14.30, <3", features = [ "metrics", "scaling", "service-metrics", - # The transport types the config and the loop name whichever transport is - # compiled in, AcknowledgementsConfig and Record among them. + # The transport types the config and loop name, whichever transport is compiled in. "transport", "version-check", # BatchEngine and its pipeline builder, which runs the service loop. "worker-batch", ] } -# The metric handle types (Counter/Gauge/Histogram) that MetricsManager hands -# back. scalo re-exports the manager, not the facade crate. +# The handle types MetricsManager returns. scalo re-exports the manager, not this crate. metrics = ">=0.24.6, <0.25" -# JSON Schema derivation for the reflectable config contract. Same range as -# scalo's own so cargo unifies to one schemars. +# Same range as scalo's, so cargo unifies to one schemars. schemars = { version = ">=1.2.3, <2", default-features = false, features = [ "std", "derive", "preserve_order", ] } -# Async runtime tokio = { version = ">=1.54.1", features = ["full"] } tokio-util = ">=0.7.20, <0.8" -# Zero-copy payload bytes -- the currency handed to the transport sender. bytes = ">=1.12.1, <2" -# Global allocator for the binary; behind the `jemalloc` feature. tikv-jemallocator = ">=0.7.0, <0.8" -# CLI surface for the service binary. clap = { version = ">=4.6.7", features = ["derive"] } -# JSON parsing (SIMD-accelerated) +# Dev-dependency of dfe-runtime's json_parse bench: docs/architecture.md#event-model. simd-json = { version = ">=0.19.0, <0.20", features = ["known-key"] } serde = { version = ">=1.0.229", features = ["derive"] } -# preserve_order keeps object keys in DOCUMENT order, which is what -# Elasticsearch's LinkedHashMap-backed ingest documents hold: a Painless -# `toString` of a map renders in that order, and sorted keys diverged on -# every rendered copy aws's cloudtrail writes. -# float_roundtrip parses every float to the nearest double, as Elasticsearch's -# Java parser does; without it a long literal such as 0.10617876052856445 reads -# one ulp low and is written back as ...444. +# preserve_order keeps Elasticsearch's document key order. float_roundtrip parses floats as Java does. serde_json = { version = ">=1.0.154", features = ["preserve_order", "float_roundtrip"] } -# SIMD byte search memchr = ">=2.8.3" -# One pass over a line for every marker the agentless detector knows. aho-corasick = ">=1.1.5" -# Test only: dfe-receiver's syslog parser, which decides what the detector is -# handed when the receiver keeps no raw line. +# Test only: dfe-receiver's syslog parser. syslog_loose = ">=0.23.0, <0.24" -# The lock in front of the GeoIP cache. std's RwLock carries poisoning and a -# slower uncontended path, and this one is taken on the per-event path. Already -# in the lockfile through -# scalo, so this adds no supply chain. +# std's RwLock poisons and is slower uncontended, and the GeoIP cache takes this per event. parking_lot = ">=0.13.0, <0.14" -# Pre-compiled DFA regex (serialisable, zero-cost startup) regex-automata = ">=0.4.18, <0.5" regex = ">=1.13.1" -# Backtracking fallback for the vendor patterns `regex` rejects by design -- -# lookaround. Only reached when `regex` fails to compile, so the hot path is -# unchanged; without it those patterns silently match nothing. Bounded because -# 0.x minors break, and the floor is the line the tests actually run. +# Fallback for the lookaround `regex` rejects. Bounded because 0.x minors break. fancy-regex = ">=0.20.0, <0.21" -# Mozilla's Public Suffix List, compiled in. Elasticsearch's registered_domain -# processor is a PSL lookup, and the last-two-labels heuristic it replaced was -# wrong in both directions: `a.co.uk` came out registered to `co.uk`, and -# `host.tld` got a registered domain Elasticsearch writes nothing for. +# Elasticsearch's registered_domain is a Public Suffix List lookup. psl = ">=2.1.246" -# Timestamp handling. chrono-tz carries the IANA database, which chrono -# deliberately does not ship: a pipeline configured with `America/Denver` -# needs the offset AT THE EVENT, and that moves with daylight saving. +# chrono-tz has the IANA database chrono lacks, for a zone's offset at the event. chrono = { version = ">=0.4.45, <0.5", features = ["serde"] } chrono-tz = { version = ">=0.10.4, <0.11" } -# GeoIP enrichment, memory-mapped for a long-running service. Upper bound -# because 0.x minors break. +# Bounded because 0.x minors break. maxminddb = { version = ">=0.32.0, <0.33", features = ["mmap"] } -# Error handling thiserror = ">=2.0.21" -# Logging tracing = ">=0.1.44, <0.2" -# Hashing. Upper bound because 0.x minors break: 0.11 returns a -# `hybrid_array::Array` from `finalize()`, which has no `LowerHex`. +# Bounded because 0.x minors break. sha1 = ">=0.11.0, <0.12" sha2 = ">=0.11.1, <0.12" -# Bounded for the same reason: 0.x minors break, and 0.23 is what the tests run. base64 = ">=0.23.1, <0.24" -# CSV parsing csv = ">=1.4.0" -# Testing (used as dev-dependencies in member crates) criterion = { version = ">=0.8.2, <0.9", features = ["html_reports"] } tempfile = ">=3.27.0, <4" -# Scores the compat corpus across cores. A test-only dependency: the service -# transforms each block on one thread and scales by replica, and nothing in -# the shipped path uses a work-stealing pool. +# Test only. The shipped path runs one transform thread per block. rayon = ">=1.12.0, <2" [profile.release] @@ -332,10 +273,7 @@ panic = "abort" strip = true opt-level = 3 -# The generated transforms build at opt-level 1: it leaves the three transform -# benches unchanged, because thin LTO re-optimises this code at the binary's own -# opt-level anyway, and it holds a rustc's memory down. Splitting the modules -# across crates by package is what keeps each rustc inside a 16 GiB runner. +# Why opt-level 1: docs/performance.md. [profile.release.package.dfe-transforms-1password-authentik] opt-level = 1 codegen-units = 16 @@ -399,10 +337,7 @@ codegen-units = 16 [profile.bench] inherits = "release" debug = true -# `release` strips, which contradicts the `debug` above: the debuginfo is -# emitted and then the symbol table it needs is thrown away, so every frame of -# a backtrace resolves to `` and the allocation-site profiler in -# `dfe-transforms/benches/allocations.rs` has nothing to attribute to. +# `release` strips, which leaves the allocation profiler no symbols to attribute to. strip = false [profile.profiling] diff --git a/README.md b/README.md index 3e254b33f..2f7969a30 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Turns Beats and Elastic Agent events into DFE-normalised events. ## What it does - Applies each source's Elastic ingest-pipeline logic, compiled in as a Rust module, with no interpreter or plugin on the hot path. In a DFE deployment it feeds dfe-loader, so an existing Elastic shipper lands in DFE without changing. -- Filebeat, Winlogbeat, Metricbeat, Auditbeat, Heartbeat, Packetbeat and Elastic Agent data. `dfe-transform-elastic sources` lists every source a build carries, and a config with work for one it does not carry is refused at startup. +- Filebeat, Winlogbeat, Metricbeat, Auditbeat, Heartbeat, Packetbeat and Elastic Agent data. `dfe-transform-elastic sources` lists every source a build has, and a config naming one it lacks fails at startup. - Events wrapped by Beats, by dfe-receiver or by dfe-fetcher, the wrapper read off each event. - Kafka in and out, or a direct gRPC push with no broker. - At-least-once for what it can deliver: the offset commit, or the answer to a push, is held until the sink takes the block. `source.acknowledgements.enabled: false` acknowledges at receipt instead. @@ -35,36 +35,37 @@ source: sink: topic: normalised_events - max_message_bytes: 8000000 # per-record ceiling, values below 4096 are clamped + max_message_bytes: 8000000 # per-record ceiling, 4096 to 16 MiB ``` Health and metrics answer on the metrics listener, port 9090 by default: `/livez`, `/readyz`, `/metrics`, `/metrics/manifest`, and `/scaling/pressure` for KEDA. ## Configuration -[config.example.yaml](config.example.yaml) is rendered from the deployment contract with illustrative values for the source and sink. Set those for your deployment (`source.name`, `source.topics`, `source.brokers`, `source.group_id` and `sink.topic` on the Kafka transport). A test pins the file against drift. `dfe-transform-elastic emit-config` reprints it. Settings load from an explicit `--config` path, or from scalo's config cascade under the `DFE_TRANSFORM_ELASTIC` prefix. [docs/configuration.md](docs/configuration.md) says which env spelling reaches a `--config` file and which sections that file cannot carry. +[config.example.yaml](config.example.yaml) is rendered from the deployment contract with illustrative values for the source and sink. Set those for your deployment (`source.name`, `source.topics`, `source.brokers`, `source.group_id` and `sink.topic` on the Kafka transport). A test pins the file against drift. `dfe-transform-elastic emit-config` reprints it. Settings load from an explicit `--config` path, or from scalo's config cascade under the `DFE_TRANSFORM_ELASTIC` prefix. [docs/configuration.md](docs/configuration.md) says which env spelling reaches a `--config` file and which sections that file cannot hold. -Every transformed event goes out as its own record, because dfe-loader parses one JSON document per message. Keep `sink.max_message_bytes` below the broker's `message.max.bytes`. +Each transformed event goes out as a separate record, because dfe-loader parses one JSON document per message. Keep `sink.max_message_bytes` below the broker's `message.max.bytes`. -`source.envelope` defaults to `auto`, which reads the producer's wrapper (`beats`, `receiver` or `fetcher`) off each event. The producer's own field names are stripped once lifted onto ECS, except the markers dfe-loader routes on: the receiver's `_source` and the fetcher's `_source_fetcher`. +`source.envelope` defaults to `auto`, which reads the producer's wrapper (`beats`, `receiver` or `fetcher`) off each event. The producer's field names are stripped once lifted onto ECS, except two markers: the receiver's `_source`, which dfe-loader routes on, and the fetcher's `_source_fetcher`, which dfe-engine's routing rule matches. -Each side also takes a `transport`: `bus` (Kafka, the default) or `direct` (a scalo Push listener in, a gRPC push out). +Each side takes a `transport`: `bus` (Kafka, the default) or `direct` (a scalo Push listener in, a gRPC push out). -The source catalogue, `sources.yaml`, is compiled into the binary and `dfe-transform-elastic emit-catalogue` prints it, so a deployment takes the catalogue from the image it already pulls. +The source catalogue, `sources.yaml`, is compiled into the binary and `dfe-transform-elastic emit-catalogue` prints it, so a deployment takes the catalogue from the image it runs. ## Documentation -- [docs/architecture.md](docs/architecture.md): the code map, the envelopes and the invariants. -- [docs/parity.md](docs/parity.md): what the service promises against Elastic's own output. -- [docs/compat.md](docs/compat.md) and [docs/compat-policy.md](docs/compat-policy.md): working a source towards parity. -- [docs/configuration.md](docs/configuration.md): the cascade, the env spellings and the sections a file cannot carry. -- [docs/enrichment.md](docs/enrichment.md): the enrichment tables. +- [docs/architecture.md](docs/architecture.md): the code map, the envelopes and delivery. +- [docs/parity.md](docs/parity.md): what the service promises against Elastic's output. +- [docs/compat.md](docs/compat.md) and [docs/compat-policy.md](docs/compat-policy.md): how parity is scored, and what counts as a difference. +- [docs/configuration.md](docs/configuration.md): the cascade, the env spellings and the sections a file cannot hold. +- [docs/enrichment.md](docs/enrichment.md): GeoIP, user agent and community ID. +- [docs/README.md](docs/README.md): every page, including parsers, performance and testing. The suite's published docs are at , and this repository's pages at . ## Help and contributing -How to work in this repo, the generated artefacts, the test tiers and what tends to bite are in [CONTRIBUTING.md](CONTRIBUTING.md). A question that is not a bug goes in a new issue. +How to work in this repo, the generated artefacts, the test tiers and what tends to bite are in [CONTRIBUTING.md](CONTRIBUTING.md). A question that is not a bug goes in an issue. Open bugs: [this repo](https://github.com/hyperi-io/dfe-transform-elastic/issues?q=is%3Aopen+is%3Aissue+type%3ABug+-label%3Abacklog+-label%3Anext-minor) | [whole DFE suite](https://github.com/search?type=issues&q=is%3Aopen+is%3Aissue+type%3ABug+-label%3Abacklog+-label%3Anext-minor+repo%3Ahyperi-io%2Fdfe-infra+repo%3Ahyperi-io%2Fdfe-docker+repo%3Ahyperi-io%2Fdfe-deploy+repo%3Ahyperi-io%2Fdfe-engine+repo%3Ahyperi-io%2Fdfe-loader+repo%3Ahyperi-io%2Fdfe-receiver+repo%3Ahyperi-io%2Fdfe-fetcher+repo%3Ahyperi-io%2Fdfe-archiver+repo%3Ahyperi-io%2Fdfe-transform-vrl+repo%3Ahyperi-io%2Fdfe-transform-vector+repo%3Ahyperi-io%2Fdfe-transform-elastic+repo%3Ahyperi-io%2Fdfe-ui+repo%3Ahyperi-io%2Fdfe-hyperdx+repo%3Ahyperi-io%2Fscalo-rs+repo%3Ahyperi-io%2Fscalo-py+repo%3Ahyperi-io%2Flogreducer+repo%3Ahyperi-io%2Fdfe-schemas+repo%3Ahyperi-io%2Fvector-vrl+repo%3Ahyperi-io%2Fculvert+repo%3Ahyperi-io%2Fdfe-clickhouse-rs+repo%3Ahyperi-io%2Ffactbook-rs) diff --git a/crates/dfe-core/Cargo.toml b/crates/dfe-core/Cargo.toml index 7323fc04b..0090ee3ba 100644 --- a/crates/dfe-core/Cargo.toml +++ b/crates/dfe-core/Cargo.toml @@ -31,25 +31,20 @@ fancy-regex = { workspace = true } [dev-dependencies] criterion = { workspace = true } -# `benches/hashing.rs` only. serde_json's preserve_order map is an IndexMap -# with the std hasher, and that bench measures what the hasher costs. +# `benches/hashing.rs` only. indexmap = { version = "2.14.2", default-features = false, features = ["std"] } -# The dotted-path accessors. Field plumbing is the hot path -- rename, set and -# append outnumber script and grok by an order of magnitude in the vendored -# pipelines -- so one path walk is the number that multiplies out to a batch. +# The dotted-path accessors, the hottest path. [[bench]] name = "event" harness = false -# What the map's hasher costs, against the lookup it is part of. A measurement, -# not a proposal: serde_json::Map does not let a caller choose the hasher. +# What the map's hasher costs. serde_json::Map does not let a caller choose it. [[bench]] name = "hashing" harness = false -# What a `date` processor costs per event, and how much of that is re-derived -# from a format literal the call site already owns. +# What a `date` processor costs per event. [[bench]] name = "date_formats" harness = false diff --git a/crates/dfe-painless/Cargo.toml b/crates/dfe-painless/Cargo.toml index 2e725f29f..f665452f7 100644 --- a/crates/dfe-painless/Cargo.toml +++ b/crates/dfe-painless/Cargo.toml @@ -20,9 +20,7 @@ publish.workspace = true workspace = true [features] -# The generator resolves a script to its runner call at generation time by -# asking this crate, so the two paths cannot diverge. The service never -# enables it and never compiles the emitters. +# Generator-time only: the service never enables it. codegen = [] [dependencies] diff --git a/crates/dfe-runtime/Cargo.toml b/crates/dfe-runtime/Cargo.toml index 3aa27d9e3..788ed509b 100644 --- a/crates/dfe-runtime/Cargo.toml +++ b/crates/dfe-runtime/Cargo.toml @@ -21,9 +21,7 @@ workspace = true [features] testutil = [] -# The generator resolves a script to its runner call at generation time by -# asking dfe-painless, so the two paths cannot diverge. The service never -# enables it and never compiles the emitters. +# Generator-time only: the service never enables it. codegen = ["dfe-painless/codegen"] [dependencies] @@ -47,29 +45,20 @@ psl = { workspace = true } [dev-dependencies] tempfile = { workspace = true } criterion = { workspace = true } -# `benches/json_parse.rs` only, which is what demoted it from a runtime -# dependency: both hot-path uses measured slower than serde_json. +# `benches/json_parse.rs` only (docs/architecture.md#event-model). simd-json = { workspace = true } -# The grok call site, native path against the compiled regex. Here rather than -# in dfe-transforms because it needs only grok_cache and dfe-parse, and -# benching it there meant an opt-level=3, codegen-units=1 build of a crate -# holding one module per data stream. +# Here, not in dfe-transforms, so the bench builds in minutes. [[bench]] name = "grok" harness = false -# Reading the vendor payload, which every JSON-carrying source does first. -# Tests the "simd-json is 2-3x faster" premise against this workload. Here -# rather than in dfe-core because it benches `codegen_api::parse_json_str` -# alongside `Event::from_bytes`. +# Here, not in dfe-core, because it benches `codegen_api::parse_json_str` too. [[bench]] name = "json_parse" harness = false -# The GeoIP cache hit, on one thread and on eight. 14 of the 60 source -# pipelines carry a geoip processor, several of them four or more times per -# event, and the cache is process-global, shared by every thread that transforms. +# The GeoIP cache hit, on one thread and on eight. [[bench]] name = "geoip" harness = false diff --git a/crates/dfe-transforms/Cargo.toml b/crates/dfe-transforms/Cargo.toml index bc839e8fb..bcd246328 100644 --- a/crates/dfe-transforms/Cargo.toml +++ b/crates/dfe-transforms/Cargo.toml @@ -16,9 +16,7 @@ repository.workspace = true rust-version.workspace = true publish.workspace = true -# Not `workspace = true`. The crate is ~200 GENERATED modules, and the workspace -# set flags the generator's own output rather than a defect, so adopting it means -# changing codegen first. An own table keeps cargo from hinting on every build. +# Its own table: the workspace set flags generated code rather than defects. [lints.clippy] all = "warn" @@ -47,28 +45,22 @@ csv = { workspace = true } [dev-dependencies] criterion = { workspace = true } -# `codegen` so `painless_binding` can ask a plan whether the generator could -# emit it directly, rather than a second copy of the allowlist drifting here. +# `codegen` lets `painless_binding` ask a plan whether it could be emitted directly. dfe-runtime = { workspace = true, features = ["testutil", "codegen"] } -# `grok_compiles` proves every pattern reaches one of the two engines the -# runtime dispatches over, so it has to be able to try both itself. +# `grok_compiles` tries both engines the runtime dispatches over. fancy-regex = { workspace = true } rayon = { workspace = true } -# A capture's `dynamic_fields` patterns are Go `regexp` in the file upstream -# writes them to, so the linear-time engine is the matching one -- and none of -# them needs lookaround. +# `dynamic_fields` patterns are Go regexp, so the linear-time engine matches them. regex = { workspace = true } # Fixture and pipeline YAML parses with the rules the service reads YAML by. scalo = { workspace = true } -# Per-event transform throughput. A 20k batch multiplies whatever one event -# costs, so anything done per event has to be measured, not reasoned about. +# Per-event transform throughput. [[bench]] name = "transforms" harness = false -# Where the allocations go. Timing says a transform is slow; a per-event -# allocation count says why, and it is exact rather than sampled. +# Allocations per event, counted exactly. [[bench]] name = "allocations" harness = false diff --git a/crates/pgo-driver/Cargo.toml b/crates/pgo-driver/Cargo.toml index de2480278..02ea9591c 100644 --- a/crates/pgo-driver/Cargo.toml +++ b/crates/pgo-driver/Cargo.toml @@ -20,9 +20,7 @@ publish.workspace = true workspace = true [features] -# hyperi-ci packages every non-gated bin in the workspace, and its -# `_detect_binary_names` skips a target carrying `required-features`, so the -# gate is what keeps this driver out of a release payload. +# hyperi-ci skips a bin with `required-features`, which keeps this driver out of a release. driver = [] [[bin]] @@ -31,9 +29,7 @@ path = "src/main.rs" required-features = ["driver"] [dependencies] -# Declared here rather than inherited: the workspace entry carries the -# service's full feature list, and this crate must build with the Kafka -# transport alone so the workload's driver build takes minutes. +# The Kafka transport alone, so the driver builds in minutes. scalo = { version = "2.14.30", default-features = false, features = [ "transport-kafka", ] } diff --git a/deny.toml b/deny.toml index 509ebd470..1843de312 100644 --- a/deny.toml +++ b/deny.toml @@ -5,17 +5,10 @@ # License: BUSL-1.1 # Copyright: (c) 2026 HYPERI PTY LIMITED # -# Run: cargo deny check -# -# Written against cargo-deny 0.20. The 0.14 spelling of this file -- -# `vulnerability`, `notice`, `unlicensed`, `copyleft`, `default`, and -# `unmaintained` as a lint level -- was rejected outright, so the check -# reported "issues found" while never actually running. +# Run: cargo deny check. Written for cargo-deny 0.20, which rejects the 0.14 keys. [advisories] db-urls = ["https://github.com/rustsec/advisory-db"] -# Advisories are errors by default in 0.20; only the scope of the -# unmaintained check is still configurable. unmaintained = "workspace" yanked = "warn" @@ -31,9 +24,7 @@ allow = [ "Zlib", "MPL-2.0", "Unicode-3.0", - # webpki-root-certs, reached through scalo -> reqwest: the Mozilla root CA - # store carried as DATA, under a licence that puts no obligation on what - # reads it. scalo-rs and dfe-loader allow it for the same dependency. + # webpki-root-certs via scalo: the Mozilla root CA store as data. "CDLA-Permissive-2.0", ] confidence-threshold = 0.8 @@ -41,8 +32,7 @@ confidence-threshold = 0.8 [bans] multiple-versions = "warn" wildcards = "deny" -# The workspace crates depend on each other by path and so carry no version -# requirement; without this every one of them reads as a wildcard. +# Path dependencies between workspace crates have no version. allow-wildcard-paths = true highlight = "all" diff --git a/docs/README.md b/docs/README.md index 95186fd18..a35366dbe 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,25 +1,19 @@ # docs -Everything deeper than the [README](../README.md). Start at -[architecture.md](architecture.md) if you are new to the service, and at -[compat.md](compat.md) if you are working a source towards parity. +Everything deeper than the [README](../README.md). Start at [architecture.md](architecture.md) for the code map, and at [parity.md](parity.md) to see what it promises against Elastic's output. | Page | What it is for | |---|---| -| [architecture.md](architecture.md) | The code map: system context, the crate graph, what the service binary does, and what an `Event` is on the way through | -| [configuration.md](configuration.md) | Where the values come from, which env spelling reaches a `--config` file, the `bus`/`direct` transport selector, and the sections the file cannot carry | -| [parity.md](parity.md) | What the service promises against Elastic's own output, what it leaves alone, and how the claim is measured | -| [parsers.md](parsers.md) | The three parser layers over grok and regex, every ingest processor the transforms use, and how far the Painless matchers reach | -| [enrichment.md](enrichment.md) | GeoIP, user agent and community ID -- the three runtime enrichers and the cache in front of the MMDB | -| [performance.md](performance.md) | SIMD, zero-copy, buffer reuse, the release profile, the native parser targets, and the key dependencies | -| [testing.md](testing.md) | The three test tracks, the match modes, and the non-English sweep | -| [compat.md](compat.md) | The confirmed documents from the real Elastic engine: what they are, what each fixture's `-expected.json` holds, and the known limits | -| [compat-policy.md](compat-policy.md) | What counts as a difference: the ratchet, the `compare-policy.yaml` sections, and the divergences kept on purpose | -| [schema-generation.md](schema-generation.md) | Publishing the Elastic-side field knowledge to `dfe-schemas` as ClickHouse schemas | -| [shared-crates.md](shared-crates.md) | What dfe-transform-splack took a copy of, and what it would take to extract the common half | +| [architecture.md](architecture.md) | The code map: the service's steps, the crates and modules, the envelopes, delivery, and what an `Event` is | +| [configuration.md](configuration.md) | Where the values come from, which env spelling reaches a `--config` file, the `bus`/`direct` transport, agentless mode, and the sections the file cannot hold | +| [parity.md](parity.md) | What the service promises against Elastic's output, what it leaves alone, and how parity is checked | +| [parsers.md](parsers.md) | The grok dispatch, the parsers in `dfe-parse`, every ingest processor the transforms use, and how far the Painless matchers reach | +| [enrichment.md](enrichment.md) | GeoIP, user agent and community ID, the three runtime enrichers | +| [performance.md](performance.md) | SIMD, borrowed values, the release profile, the benchmarks and the key dependencies | +| [testing.md](testing.md) | The floors, the scanners and the shared Kafka broker | +| [compat.md](compat.md) | How a transform is scored against the real Elastic engine, and the known limits | +| [compat-policy.md](compat-policy.md) | What counts as a difference: the `compare-policy.yaml` sections and the divergences kept on purpose | +| [schema-generation.md](schema-generation.md) | The design for publishing the Elastic-side field knowledge to dfe-schemas | +| [shared-crates.md](shared-crates.md) | What dfe-transform-splack took a copy of, and what extracting the common half would take | -Everything else in this directory is generated by -`dfe-transform-elastic generate-artefacts` -- `metrics-manifest.json`, -`deployment-contract.json`, `container-manifest.json`, `Dockerfile.runtime`, -`config-schema.{json,yaml}` and `capability-catalog.{json,yaml}`. Do not -hand-edit them; re-run the command. +`dfe-transform-elastic generate-artefacts --output-dir docs` writes everything else in this directory: `metrics-manifest.json`, `deployment-contract.json`, `container-manifest.json`, `Dockerfile.runtime`, `config-schema.{json,yaml}` and `capability-catalog.{json,yaml}`. Re-run the command rather than editing them. diff --git a/docs/architecture.md b/docs/architecture.md index 5451c013b..1e6597298 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,377 +1,286 @@ # Architecture -**Project:** dfe-transform-elastic -**Purpose:** Rust-optimised transform service for Elastic Stack data (Beats + Elastic Agent) ingested over Kafka or a direct gRPC push +dfe-transform-elastic applies a source's Elastic ingest pipeline to each event as compiled Rust. The service resolves the configured source name to one generated transform at startup, so there is no interpreter, scripting VM or plugin system, and no lookup per event. -Each transform reimplements one source's ingest pipeline from the ground up, as optimised parser -tree objects, with grok and regex kept for the patterns the parsers do not cover. Where a source has fixtures, [compat](compat.md) -scores its output against the real Elastic engine's. One module per source. There is no interpreter, no scripting VM and no -plugin system: the service resolves a source name to a compiled transform at startup, and the -hot path carries no dynamic dispatch per event. +This page is the code map: the parts of the service, the crates, the modules, the envelopes, delivery, and what an event is on the way through. [parity.md](parity.md), [parsers.md](parsers.md), [enrichment.md](enrichment.md), [testing.md](testing.md) and [performance.md](performance.md) cover the rest. -This page is the code map: where the service sits, what the crates are, what the binary does, -and what an event is on the way through. The neighbouring pages carry the rest -- -[parity.md](parity.md), [parsers.md](parsers.md), [enrichment.md](enrichment.md), -[testing.md](testing.md) and [performance.md](performance.md). +## Four steps between the producers and the sink ---- - -## System context - -Where dfe-transform-elastic fits in the Data Fusion Engine pipeline: +Each batch is parsed, unwrapped, transformed and serialised in that order, and dead letters leave for the DLQ. ```mermaid -flowchart LR - subgraph Sources - FB[Filebeat] - WB[Winlogbeat] - AB[Auditbeat] - MB[Metricbeat] - HB[Heartbeat] - PB[Packetbeat] - EA[Elastic Agent] - RX[dfe-receiver
pushed transports] - FX[dfe-fetcher
pulled APIs] - end +flowchart TB + accTitle: L3 Components -- dfe-transform-elastic + accDescr: Producers send events over Kafka or a direct push. Inside the service each batch is parsed, unwrapped, transformed and serialised in that order. Events go to Kafka or dfe-loader, and dead letters go to the DLQ. - K[Kafka
JSON events] + producers["Beats, Agent,
receiver, fetcher"] - subgraph dfe-transform-elastic - D[serde_json
Deserialise] - T[Rust Transform
Pipeline] - E[Enrichment
GeoIP / UA / CID] + subgraph svc["dfe-transform-elastic"] + direction LR + parse["Parse NDJSON"] --> unwrap["Unwrap envelope"] --> transform[["Transform"]] --> serialise["Serialise"] end - O[Output
Kafka, or a gRPC push
to dfe-loader] + sink["Kafka or
dfe-loader"] + dlq[("DLQ")] - FB & WB & AB & MB & HB & PB & EA & RX & FX --> K - RX & FX -.->|direct gRPC push| D - K --> D --> T --> O - T <--> E + producers -->|events| svc + svc -->|events| sink + svc -.->|dead letters| dlq + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef error fill:#FBE3D6,stroke:#C2410C,stroke-width:1px,color:#4A1A05 + classDef ext fill:#EEF0F2,stroke:#6B7280,stroke-width:1px,stroke-dasharray:4 3,color:#1F2937 + classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 + class producers,sink ext + class dlq error + class svc group ``` -JSON events arrive from Kafka or over a direct gRPC push, in any of the three producer -envelopes, and leave the same way as normalised events. Between the two, Elastic's ingest -pipeline logic runs as native Rust. +*Figure 1. L3 Components -- dfe-transform-elastic. Dashed boxes are external and the dotted arrow is the dead-letter path.* Up: top of dfe-transform-elastic. Down: [L4 Detail -- A failed processor](#a-failed-processor-ends-in-one-of-four-places). + +Events arrive from Kafka or over a direct gRPC push, in any of three producer envelopes, and leave the same way. Input that is not JSON, and a record the sink transport would reject, go to the DLQ. In agentless mode a detector picks the transform per event ([configuration.md](configuration.md#agentless-mode-serves-one-syslog-stream-of-many-device-types)). ---- +## Crates and modules -## Crate dependency graph +The workspace holds five library crates, the generated transform crates, the PGO workload driver and the root service package. -The workspace is five library crates, the generated transform crates, and the root service package. +Every generated transform crate builds on `dfe-runtime`, which builds on the three base crates. ```mermaid -flowchart TD - service[dfe-transform-elastic
Service binary: cli, config,
envelope, pipeline, registry
] - transforms[dfe-transforms
Re-exports the generated
modules as filebeat
] - generated[dfe-transforms-*
Generated transform modules,
split by package
] - runtime[dfe-runtime
Grok cache, enrichment,
codegen_api, Transform trait
] - painless[dfe-painless
Painless matchers, bespoke
runners, plan, params
] - core[dfe-core
Event, errors, date formats,
syslog priority, regex cache
] - parse[dfe-parse
High-performance parsers
replacing grok/regex
] +flowchart LR + accTitle: L4 Detail -- Crate dependencies + accDescr: The service depends on dfe-transforms, which re-exports the generated transform crates. They depend on dfe-runtime, which depends on dfe-painless, dfe-parse and dfe-core. + + service["dfe-transform-elastic"] + transforms["dfe-transforms"] + generated["dfe-transforms-*"] + runtime["dfe-runtime"] + + subgraph base["Base crates"] + painless["dfe-painless"] + parse["dfe-parse"] + core["dfe-core"] + end service --> transforms - service --> runtime transforms --> generated - transforms --> runtime generated --> runtime - transforms --> parse - runtime --> painless - runtime --> core - runtime --> parse - painless --> core -``` + runtime --> base -| Crate | Type | Purpose | -|-------|------|---------| -| `dfe-transform-elastic` | Binary + Library | The service, module by module in the table below | -| `dfe-transforms` | Library | Transform modules per data source (`filebeat::::`), re-exported from the generated crates | -| `dfe-transforms--` | Library | The generated filebeat modules for a contiguous run of packages, compiled in place from `dfe-transforms/src/filebeat` so no rustc holds the whole tree | -| `dfe-runtime` | Library | Grok caching, enrichment, `codegen_api`, the Transform trait, and the prelude | -| `dfe-painless` | Library | Painless pattern matching: the `known_patterns` ladder, `plan`, `params`, `patterns.lock`, and `bespoke/` -- one module per source holding hand-written equivalents of its scripts, resolved by script hash ahead of every matcher | -| `dfe-core` | Library | `Event`, error types, date formats, syslog priority, the regex cache | -| `dfe-parse` | Library | Zero-copy parsers replacing grok/regex patterns | + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 + class base group +``` -`dfe-transforms` has no direct edge to `dfe-painless`. It reaches the matchers through -`dfe-runtime`, which re-exports every moved module at its old `painless_*` path -- which -is why the thousand generated modules did not change when the split landed. +*Figure 2. L4 Detail -- Crate dependencies.* Up: [L3 Components -- dfe-transform-elastic](#four-steps-between-the-producers-and-the-sink). -`dfe-parse` is reached, but only for two whole-pattern forms: `grok_cache` dispatches -`^%{IPV4:f}$` and `^%{IPV4:a}:%{PORT:p}$` to it and falls back to the compiled regex for -everything else. Each call site holds that regex in its own `OnceLock` rather than a shared -map, for the reason [performance.md](performance.md) measures. +The service and `dfe-transforms` depend on `dfe-runtime` directly as well, and `dfe-painless` depends on `dfe-core`. The diagram leaves those edges out to keep each arrow one column long. -The third-party libraries underneath every crate are -[Key dependencies](performance.md#key-dependencies). +| Crate | Purpose | +|---|---| +| `dfe-transform-elastic` | The service, module by module in the next table | +| `dfe-transforms` | One module per source pipeline (`filebeat::::`), re-exported from the generated crates | +| `dfe-transforms--` | The generated filebeat modules for a contiguous run of packages, compiled from `crates/dfe-transforms/src/filebeat` so no single rustc holds the whole tree | +| `dfe-runtime` | Grok caching, enrichment, `codegen_api`, the `Transform` trait and the prelude | +| `dfe-painless` | Painless matching: the `known_patterns` ladder, `plan`, `params`, `patterns.lock`, and `bespoke/`, one module per source of hand-written runners resolved by script hash ahead of every matcher | +| `dfe-core` | `Event`, error types, date formats, syslog priority and the regex cache | +| `dfe-parse` | Zero-copy parsers that replace grok and regex patterns | +| `pgo-driver` | The PGO workload driver, built apart so the workload's build never rebuilds the transforms | ---- +The generated transforms reach the Painless matchers through `dfe-runtime`, which re-exports the moved modules at their old `painless_*` paths. Code written against these crates names the crate that holds the code. -## Service binary +`grok_cache` sends two whole-pattern IPv4 forms to `dfe-parse` and answers a lone `%{GREEDYDATA}` with a newline scan. Every other pattern runs on its compiled regex ([parsers.md](parsers.md)). The third-party libraries are in [Key dependencies](performance.md#key-dependencies). -`src/` is the service that resolves a configured source name to one compiled transform and -runs it over every batch on the scalo runtime, over the bus (Kafka) or the direct (gRPC push) -transport. +`src/` resolves the configured source name to one compiled transform and runs it over every batch on the scalo runtime, over the bus (Kafka) or the direct (gRPC push) transport. | Module | Responsibility | |---|---| | `main.rs` | Entry point, hands off to `cli.rs` | -| `cli.rs` | Subcommands: `sources` (list registered transforms), `emit-dockerfile`, `emit-compose`, `emit-config`, `emit-catalogue`, plus scalo's `run`, `version`, `config-check`, `config-schema`, `generate-artefacts` and `metrics-manifest` | -| `config.rs` | The config fields: `source.*`, `sink.*`, `geoip`, read once at startup, and `work_state`, which decides whether an instance has work or idles. Also `Transport`, the `bus`/`direct` selector on each side | -| `config/loader.rs` | Reading it, from the scalo cascade or an explicit `--config` file, and `SCALO_SECTIONS` -- the sections scalo reads for itself, which the unread-key warning skips | -| `config/validate.rs` | Refusing a configuration that cannot work. The range checks run ahead of the idle gate, so a value someone set out of range still refuses while an unconfigured instance idles | -| `registry.rs` | Source name to `Transform` lookup, plus each source's `Intake` (which envelopes it accepts), `Framing` and dataset | +| `cli.rs` | Subcommands: `sources`, `emit-dockerfile`, `emit-compose`, `emit-config` and `emit-catalogue`, plus scalo's `run`, `version`, `config-check`, `config-schema`, `generate-artefacts` and `metrics-manifest` | +| `config.rs` | The config fields: `source.*`, `sink.*` and `geoip`, read once at startup, and `work_state`, which decides whether an instance has work or idles. Also `Transport`, the `bus`/`direct` selector on each side | +| `config/loader.rs` | Loading, from the scalo cascade or an explicit `--config` file, and `SCALO_SECTIONS`, the sections scalo reads for itself, which the unread-key warning skips | +| `config/validate.rs` | Rejecting a configuration that cannot work. The range checks run ahead of the idle gate, so an out-of-range value fails even on an instance with no work | +| `registry.rs` | Source name to `Transform`, plus each source's `Intake` (the envelopes it accepts), `Framing` and dataset | | `envelope.rs` | Detects which of the three producer families wrapped an event and unwraps it into the bare event every transform expects | +| `agentless.rs`, `detect/` | Agentless mode: names each event's source from the marker table, runs that source's transform and stamps `_source` | | `pipeline.rs` | Batch processing: NDJSON parse, envelope unwrap, transform, serialise, with per-batch outcome counts | -| `service.rs` | Runs `pipeline.rs` on scalo's `BatchEngine` pipeline between the scalo consumer and producer -- Kafka on the bus, a Push listener and a gRPC client on direct -- and maps what the sink answers onto the delivery semantics below | -| `deployment.rs` | The single deployment contract: Dockerfile, Helm chart, compose fragment and KEDA scaler are all generated from here. Its tests pin the Dockerfile, `config.example.yaml`, `docs/deployment-contract.json` and the `docs/` config artefacts against a fresh regen -- not every artefact, see the README | +| `service.rs` | Runs `pipeline.rs` on scalo's `BatchEngine` between the scalo consumer and producer, Kafka on the bus and a Push listener and gRPC client on direct, and maps what the sink answers onto the delivery rules below | +| `deployment.rs` | The deployment contract the Dockerfile, chart, compose fragment and KEDA scaler come from. Its tests pin some of the generated artefacts, [CONTRIBUTING.md](../CONTRIBUTING.md#generated-artefacts) says which | | `metrics.rs` | Metric definitions registered with scalo's `MetricsManager` | | `error.rs` | The service's top-level error type | -Nothing is hot-reloaded, the two ways in (`--config` against the scalo cascade) -are not equivalent, and only one env spelling reaches a `--config` file: -[configuration.md](configuration.md) carries all three, and `src/config.rs` states them at the -top of the file. +Nothing is hot-reloaded, `--config` and the scalo cascade are not equivalent, and one env spelling reaches a `--config` file. [configuration.md](configuration.md) covers all three. + +## Envelopes: the same transform, a different wrapper -## Envelopes: the same pipeline, a different wrapper +`source.envelope` is `auto` (the default), `beats`, `receiver` or `fetcher`. -`source.envelope` selects `auto` (the default), `beats`, `receiver` or `fetcher`. +- `beats` -- the raw payload as a string in `message`. The transforms are written for this, and Elastic Agent and anything passing a Beats document through unaltered produce it. +- `receiver` -- [dfe-receiver](https://github.com/hyperi-io/dfe-receiver)'s JSON on any of its transports. The syslog arm rebuilds a line into `message`, and the rest pass their payload through with the receiver's field names moved onto the ECS paths they would shadow. +- `fetcher` -- [dfe-fetcher](https://github.com/hyperi-io/dfe-fetcher)'s JSON, the provider's payload at the top level, for sources where Elastic's agent input is a pure transport. -- **beats** -- the raw vendor payload as a string in `message`, which is what the transforms are - written for, and what Elastic Agent and anything passing a Beats document through unaltered - also produce. -- **receiver** -- [dfe-receiver](https://github.com/hyperi-io/dfe-receiver)'s JSON on any of its - transports. The syslog arm rebuilds a line into `message`, and the rest pass their payload - through with their own field names moved onto the ECS paths those names would otherwise shadow. -- **fetcher** -- [dfe-fetcher](https://github.com/hyperi-io/dfe-fetcher)'s JSON, the provider's - own payload at the top level, for the sources where Elastic's agent input is a pure transport. +`auto` resolves the family per event. A scalo `WorkBatch` spans partitions, so one batch can hold two producers' wrappers. The cost is a few top-level key checks, against an unwrap that cost 2,740 ns an event when benchmarked. -`auto` resolves the family PER EVENT rather than per batch: a scalo `WorkBatch` spans -partitions, so one batch can carry two producers' wrappers and reading only its first event -unwrapped every one of them the same way. The cost is a handful of top-level key checks against -an unwrap measured at 2,740 ns an event. +A bare payload with no producer marker is the one quiet failure. Detection falls back to `beats`, there is no `message` to unpack, and the transform runs and emits with almost nothing renamed. Output that looks like the input with an `ecs.version` added is this. -A bare vendor object carrying no producer marker is the one failure that is not loud. Detection -falls back to `beats`, there is no `message` to unpack, and the transform still runs and emits -with almost nothing renamed. Output that looks like the input with an `ecs.version` bolted on is -this. +`registry.rs` records a syslog framing for every source the receiver can deliver: -Two syslog framings, recorded per source in `registry.rs` as the `Framing` enum: +- `Body` sources, such as `panw` and `cisco_meraki`, read `message` as CSV or key-value, so the body goes through with no header. A tagless PAN-OS line arrives with its first CSV field in `appname`, and when that is not a valid tag the body is rebuilt from `appname`, `procid` and `message`. +- `Line` sources, such as `fortinet`, `cisco_ios` and `cisco_nexus`, grok the header out of `message`. The line is the receiver's opt-in `_raw` where present, else an RFC 3164 line rebuilt from the parsed fields. That is enough for `fortinet`, which wants ``, and not for `cisco_ios` or `cisco_nexus`, which want a source IP and a sequence number the receiver does not keep. -- **Body** -- `panw.*` and `cisco_meraki` read `message` as CSV or key-value. The receiver's - body goes through with no header, because a prefixed header would corrupt the first field. - The receiver's parser takes the first token as the tag even when the device wrote none, so a - tagless PAN-OS line arrives with its first CSV field in `appname`. When `appname` is not a - valid tag, the body is rebuilt from `appname`, `procid` and `message`. `_raw`, which is - opt-in, is read only when there is no parsed message. -- **Line** -- `fortinet`, `cisco_ios` and `cisco_nexus` grok the header out of `message`, so a - line is put back: the receiver's `_raw` verbatim if present, otherwise an RFC 3164 line - rebuilt from the parsed fields. Reconstruction is enough for `fortinet` (`` only); it is - not enough for `cisco_ios` (wants a source IP) or `cisco_nexus` (wants a sequence number), - since the receiver keeps neither. +A pinned envelope a source cannot arrive in is rejected at startup against the source's `Intake`. okta is pulled from an API, so it accepts `beats` and `fetcher` and rejects `receiver`. `auto` has no event to check at startup, so the same mismatch is counted per batch on `envelope_unaccepted_total`. -A PINNED envelope a source cannot arrive in is rejected at startup, checked against that -source's `Intake` -- okta is pulled from an API, so it accepts `beats` and `fetcher` and refuses -`receiver`. `auto` cannot be checked that way, since there is no event yet, so the same -mismatch is counted per batch instead. +## Delivery is at least once -## Delivery: at-least-once, held by the pipeline +The loop is scalo's `BatchEngine` (`service.rs::run_loop`). It holds each block's source acknowledgement, the Kafka offset commit on the bus or the answer to a push on direct, until the sink has taken every event built from it. `source.acknowledgements.enabled: false` releases it at receipt, and a crash then loses what was in flight. -The loop is scalo's `BatchEngine` pipeline (`service.rs::run_loop`): process is parse, transform and serialise, and the sink produces the result. The pipeline holds each block's source acknowledgement -- the Kafka offset commit on the bus, the answer to a push on direct -- until the sink has taken every event built from it. `source.acknowledgements.enabled: false` releases it at receipt instead, and a crash then loses what was in flight. +When the sink answers `Backpressured`, the block is held and retried until the sink takes it, and nothing after it is committed. A failure no retry can fix stops the loop with the block unreleased, so the restarted consumer reads it again. The direct listener holds pushes from the moment it binds, so a push that lands before the loop starts waits for delivery rather than being answered on arrival. -A sink that refuses for now is retried for as long as it refuses, with the block held and nothing after it committed, so a broker outage is waited out rather than crashed on. Only a refusal no retry can fix stops the loop, with the block unreleased for the restarted consumer to read again. The direct listener is built armed, so a push that lands before the loop starts is held rather than answered at enqueue. +A push is held at most 18 s (`PUSH_MAX_HOLD`), inside the 20 s the stage in front waits, and the bytes held are leased on the memory guard. Every send a held block waits on gives up at 15 s (`NEXT_HOP_SEND_TIMEOUT_MS`): a direct sink's gRPC deadline, and on the bus the producer's `message.timeout.ms` when a direct source holds its pushes. A send that outlasted the hold would land after the push was answered `unavailable`, and its retry would be a duplicate. -A push is held at most 18 s (`PUSH_MAX_HOLD`), inside the 20 s the stage in front waits, and the bytes held are leased on the memory guard. Every send the held block waits on gives up sooner, at 15 s (`NEXT_HOP_SEND_TIMEOUT_MS`): a direct sink's gRPC deadline always, and on the bus the producer's `message.timeout.ms` while a direct source holds its pushes. A send that outlasted the hold would land after the push was answered `unavailable`, so the retry it provokes is a duplicate. +Duplicates are the accepted cost: a retried block re-sends the records that landed, so every consumer downstream must be idempotent. -All four `SendResult` variants are handled, and three of them are not delivery: +The sink's answer decides what happens to the block: -| Variant | Meaning | Response | +| `SendResult` | Meaning | Response | |---|---|---| -| `Ok` | The broker accepted it | Count as delivered | +| `Ok` | The sink took it | Counted delivered | | `Backpressured` | A full producer queue, or a broker or listener out of reach | Retried until the sink takes it | -| `Fatal` | Recoverable (a timeout): as `Backpressured`. Otherwise authorisation, a missing topic, a closed transport | Stop, block unreleased | -| `FilteredDlq` | The sink transport dead-lettered the send | Dropped and counted on `send_filtered_dlq_total` and `records_filtered_total`, block released as dropped, never counted delivered | - -A record the sink transport would refuse -- over its `message.max.bytes`, or matched by an outbound `dlq` filter -- is screened out by the pipeline before the send. Input that is not JSON is refused in process as a dead letter with the reason `payload is not JSON`: a whole record when no line of it parses, otherwise the bad line alone. Both go to the DLQ `dlq` configures, a file DLQ by default, and the block is released once the DLQ confirms it holds them. With `dlq.enabled: false` they are dropped and counted on `pipeline_dead_letters_dropped_total{reason}` and `dlq_dropped_total` (`service.rs::with_dead_letters` builds that engine for the tests). +| `Fatal` | A timeout is retried like `Backpressured`. Anything else, such as authorisation, a missing topic or a closed transport, is not | Stop, block unreleased | +| `FilteredDlq` | The sink transport dead-lettered the send | Dropped, counted on `send_filtered_dlq_total` and `records_filtered_total`, block released, never counted delivered | -JSON is the only payload format. MessagePack, supported in DFE/XDR 2.0 and 2.1, is deprecated in DFE 2.2 and no longer accepted: the JSON path (SIMD parsing with sonic-rs, zstd on the wire) is fast enough that MessagePack gave no CPU saving. +A record the sink transport would reject, over its `message.max.bytes` or matched by an outbound `dlq` filter, is screened out before the send. Input that is not JSON is a dead letter with the reason `payload is not JSON`: the whole record when no line parses, else the bad line. Both go to the DLQ `dlq` configures, a file DLQ by default, and the block is released once the DLQ holds them. With `dlq.enabled: false` they are dropped and counted on `pipeline_dead_letters_dropped_total{reason}` and `dlq_dropped_total`. -**One record per event.** `pipeline.rs::serialise_events` gives each event its own payload and `service.rs::send_blocks` sends them via `TransportSender::send_batch`, because dfe-loader parses exactly one JSON document per message ([#67](https://github.com/hyperi-io/dfe-transform-elastic/issues/67)). `sink.max_message_bytes` (default 8,000,000 bytes, the DFE record cap) bounds each payload under the 16 MiB ceiling the producer and dfe-loader's consumer share, and the `send_batch` block under gRPC's 16 MiB ceiling. An event over the budget is dropped and counted on `events_oversize_total`: no broker would take it, and retrying blocks the partition. +Each event goes out as a separate record. `pipeline.rs::serialise_events` gives each one a payload, and `service.rs::send_blocks` sends them in blocks through `TransportSender::send_batch`, because dfe-loader parses one JSON document per message ([#67](https://github.com/hyperi-io/dfe-transform-elastic/issues/67)). `sink.max_message_bytes`, 8,000,000 bytes by default, bounds each payload under the 16 MiB producer ceiling and each block under gRPC's 16 MiB. An event over the budget is dropped and counted on `events_oversize_total`, because no broker would take it and retrying would block the partition. -Duplicates are the accepted cost: a retried block re-sends the records that already landed, so every downstream consumer must be idempotent. +## One batch through the service ---- - -## Event lifecycle - -From Kafka message bytes to transformed output: +With `source.acknowledgements` on, the default, the source is acknowledged after the sink takes the batch. ```mermaid sequenceDiagram - participant K as Kafka Consumer - participant E as Event - participant C as TransformChain - participant T1 as Transform 1
(set) - participant T2 as Transform 2
(grok to regex) - participant T3 as Transform 3
(geoip) - participant O as Output - - K->>E: Event::from_bytes(payload) - E->>C: chain.execute(&mut event) - C->>T1: transform(&mut event) - T1->>E: event.set("event.kind", "event") - T1-->>C: Ok(Continue) - C->>T2: transform(&mut event) - T2->>E: event.get_str("message") - T2->>E: event.set("source.ip", parsed.ip) - T2-->>C: Ok(Continue) - C->>T3: transform(&mut event) - T3->>E: event.get_str("source.ip") - T3->>E: event.set("source.geo.*", geoip_result) - T3-->>C: Ok(Continue) - C-->>O: TransformResult::Continue - O->>O: serde_json::to_vec(&event) + accTitle: L4 Detail -- One batch through the service + accDescr: The source hands a block to the pipeline. For each event the resolver unwraps the envelope and the transform runs. The pipeline sends one record per surviving event, and acknowledges the source once the sink takes them. + autonumber + participant source as Source + participant pipeline as Pipeline + participant resolver as Envelope resolver + participant transform as Transform + participant sink as Sink + source->>pipeline: Block of records + loop Each event + pipeline->>resolver: Resolve and unwrap + pipeline->>transform: Transform event + transform-->>pipeline: Continue or Drop + end + pipeline->>sink: One record per event + sink-->>pipeline: SendResult + pipeline->>source: Acknowledge block ``` ---- +*Figure 3. L4 Detail -- One batch through the service. Solid arrows are requests, dashed arrows are responses.* Up: [L3 Components -- dfe-transform-elastic](#four-steps-between-the-producers-and-the-sink). + +JSON is the one payload format. A MessagePack record, which DFE/XDR 2.0 and 2.1 accepted, is a dead letter like any other binary. + +An event that fails to unwrap or transform is counted on `events_errored_total` and left out of the output, and the batch continues. A transform that returns `Continue` has its `event.original` settled as [parity.md](parity.md) describes. ## Event model -The `Event` type wraps `serde_json::Value` with dotted-path navigation for ECS field access. +`Event` in `dfe-core` wraps a `serde_json::Value` with dotted-path access for ECS fields. + +A transform mutates an `Event` in place and returns `Continue`, `Drop` or an error. ```mermaid classDiagram + accTitle: L4 Detail -- Event and Transform types + accDescr: A Transform mutates an Event and returns a TransformResult, or a TransformError on failure. class Event { -inner: serde_json::Value - +new(value: Value) Event +from_json(json: &str) Result~Event~ - +from_bytes(buf: &mut [u8]) Result~Event~ - +as_value(&self) &Value - +into_value(self) Value - +get(path: &str) Option~&Value~ +get_str(path: &str) Option~&str~ - +get_i64(path: &str) Option~i64~ - +get_f64(path: &str) Option~f64~ - +get_bool(path: &str) Option~bool~ - +get_array(path: &str) Option~&Vec~ - +get_object(path: &str) Option~&Map~ - +has(path: &str) bool - +set(path: &str, value: Value) Result + +set(path: &str, value) Result +remove(path: &str) Option~Value~ +rename(from: &str, to: &str) Result - +append(path: &str, value: Value) Result - +merge(other: &Event, deep: bool) Result - } - - class TransformResult { - <> - Continue - Drop } - class Transform { <> +name(&self) &str +transform(&self, event: &mut Event) Result~TransformResult~ } - - class TransformChain { - -transforms: Vec~Box~dyn Transform~~ - +new(transforms: Vec) TransformChain - +execute(&self, event: &mut Event) Result~TransformResult~ + class TransformResult { + <> + Continue + Drop } - class TransformError { <> FieldNotFound TypeMismatch ParseError - EnrichmentError ProcessorError } - - TransformChain --> Transform : contains Transform --> Event : mutates Transform --> TransformResult : returns Transform --> TransformError : may return ``` -### Dotted-path access +*Figure 4. L4 Detail -- Event and Transform types. The enumerations list the main variants, not all of them.* Up: [L3 Components -- dfe-transform-elastic](#four-steps-between-the-producers-and-the-sink). -All field access uses ECS-style dotted paths (e.g., `source.geo.city_name`). The `set()` method -auto-creates intermediate objects: +`set()` creates intermediate objects. A read of a segment that is not a field falls back to a key with the dot inside it, and a numeric segment indexes an array, the way Elastic's templates read `{{_temp.user_parts.0}}`. ```rust -// Navigates into {"source": {"geo": {"city_name": ...}}} +// Reads {"source": {"geo": {"city_name": ...}}} event.get_str("source.geo.city_name") // Creates {"source": {"geo": {}}} if missing, then sets city_name event.set("source.geo.city_name", "Sydney")?; ``` -### Design decisions - -- **`serde_json::Value` over custom types:** Simpler to implement, and the map underneath is an `IndexMap` because `preserve_order` is not optional here -- Elasticsearch's own ingest documents are insertion-ordered and parity rests on it. Typed structs can be layered on later for hot-path fields. -- **`from_bytes` uses `serde_json`, not simd-json:** measured on this workload and it is the faster of the two, because getting a `serde_json::Value` out of simd-json goes tape to serde deserializer to `Value`, which is strictly more work than parsing straight into the same tree. simd-json is a dev-dependency of `dfe-runtime` now, kept only for `crates/dfe-runtime/benches/json_parse.rs`, which is the measurement. Re-run it before reopening this. -- **Dotted-path splitting:** Split on `.` with no escaping. ECS field names never contain dots within a single field segment. +- The map under `serde_json::Value` is an `IndexMap`, because `preserve_order` is on: Elasticsearch's ingest documents keep insertion order and parity rests on it. +- `Event::from_bytes` parses with `serde_json`, not simd-json. Getting a `serde_json::Value` out of simd-json builds two trees for one document, and `crates/dfe-runtime/benches/json_parse.rs` shows it slower. simd-json stays a dev-dependency of `dfe-runtime` for that bench. ---- +## A failed processor ends in one of four places -## Transform pipeline +A generated transform is one function that runs its pipeline's processors in order, each behind its `if` condition. The `Transform` trait is object-safe, and the registry hands out `&'static dyn Transform`. -How transforms compose and handle errors: +A failed processor is skipped, handled, or ends the pipeline, and the event is lost when the pipeline has no handler. ```mermaid -flowchart TD - start([Event enters chain]) --> t1 - - subgraph TransformChain - t1[Transform 1] -->|Continue| t2[Transform 2] - t2 -->|Continue| t3[Transform 3] - t3 -->|Continue| tn[Transform N] - end - - t1 -->|Drop| dropped([Event dropped]) - t2 -->|Drop| dropped - t3 -->|Drop| dropped - - t1 -->|Error| err{on_failure
handler?} - t2 -->|Error| err - t3 -->|Error| err - - err -->|yes| onfail[Run on_failure
transforms] - err -->|no / ignore_failure| next[Continue to
next transform] - onfail --> next - - tn -->|Continue| output([Event emitted]) +flowchart LR + accTitle: L4 Detail -- A failed processor + accDescr: A failed processor with ignore_failure set is skipped. Otherwise the processor's on_failure handlers run if it has any. Failing that, the pipeline's on_failure handlers run and keep the event, and with none the event is dropped and counted. + + ignore{"ignore_failure
set?"} + skip(["Next processor"]) + proc{"Processor's
on_failure?"} + handlers(["Its handlers run"]) + pipe{"Pipeline
on_failure?"} + kept(["Pipeline handlers
run, event kept"]) + dropped(["Dropped and
counted"]) + + ignore -->|yes| skip + ignore -->|no| proc + proc -->|yes| handlers + proc -->|no| pipe + pipe -->|yes| kept + pipe -->|no| dropped + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef decision fill:#FCF1D6,stroke:#9A6700,stroke-width:1px,color:#3D2A00 + classDef error fill:#FBE3D6,stroke:#C2410C,stroke-width:1px,color:#4A1A05 + class ignore,proc,pipe decision + class dropped error ``` -### Error handling strategy - -Mirrors Elastic ingest pipeline semantics: +*Figure 5. L4 Detail -- A failed processor. Orange is the event lost to the output.* Up: [L3 Components -- dfe-transform-elastic](#four-steps-between-the-producers-and-the-sink). -| Elastic Behaviour | Rust Implementation | +| Elastic behaviour | Generated Rust | |---|---| -| `on_failure` block | Nested `TransformChain` executed on error | -| `ignore_failure: true` | Wrap in `.ok()` / `let _ =`, continue | -| No error handling | Propagate error, stop chain | -| `tag` on failure | `event.append("tags", "error_tag")` | - -At the service boundary, `pipeline.rs::transform_batch_with` treats a transform error the same -way as a rejected envelope: the event is counted and left out of the batch output, and -processing continues with the next event. One malformed record cannot stall a partition. - -### Transform trait contract - -```rust -pub trait Transform: Send + Sync { - /// Human-readable processor name (for error context) - fn name(&self) -> &str; - - /// Apply transform to event, returning Continue or Drop - fn transform(&self, event: &mut Event) -> Result; -} -``` - -- **Object-safe:** `dyn Transform` for heterogeneous chains -- **`Send + Sync`:** Safe for concurrent use across threads -- **Infallible transforms** (set, remove, rename) still return `Result` for uniform error propagation +| `if` | The processor runs inside `if _cond { ... }` | +| `ignore_failure: true` | The processor runs in a closure whose error is discarded | +| A processor's `on_failure` | Its handlers run with `_ingest.on_failure_message`, `_ingest.on_failure_processor_type` and `_ingest.on_failure_processor_tag` set, then cleared | +| The pipeline's `on_failure` | Its handlers run, and the event is emitted | +| No handler at all | The error leaves the transform, and `pipeline.rs` drops the event and counts it on `events_errored_total` | +| `drop` | The transform returns `TransformResult::Drop` | + +Most Elastic pipelines declare a pipeline-level `on_failure` that sets `event.kind: pipeline_error`, appends to `error.message` and tags `preserve_original_event`, so the generated transform does the same. diff --git a/docs/compat-policy.md b/docs/compat-policy.md index c8657ef58..ae6cca4f6 100644 --- a/docs/compat-policy.md +++ b/docs/compat-policy.md @@ -1,165 +1,27 @@ # compat policy -- what counts as a difference -The ratchet that holds per-source scores, the rule sections in `tests/compare-policy.yaml` that decide which differences are defects, and the divergences kept on purpose. The corpus test and its baseline run in the private dfe-transform-elastic-dev repository. The compare policy stays here, because the runtime's test support reads it. The how-to that produces the corpus is [compat.md](compat.md), and what parity means in the first place is [parity.md](parity.md). +`tests/compare-policy.yaml` is the one definition of which differences between our output and Elasticsearch's are defects. The comparison harness reads it through `crates/dfe-runtime/src/testutil/policy.rs`, which is why it lives here beside the runtime. How the comparison runs is [compat.md](compat.md), and what parity means is [parity.md](parity.md). -## The ratchet - -The compat baseline is the ratchet the corpus test asserts, and the -test prints the exact line for every source that improved. A fall needs a -stated reason and is never mechanical. - -## A provenance mismatch fails - -The baseline is asserted only when the fixtures' confirmed documents name the -`pipelines_commit` and engine version the baseline names. A score against -different pipelines is a different measurement, so ratcheting one against the -other would fail for the wrong reason, and the test FAILS rather than print. -Two variables lift it: `DFE_COMPAT_ONLY` scores a subset and checks neither -ratchet, and the acknowledgement below. - -`DFE_COMPAT_ALLOW_UNRATCHETED=1` is the acknowledgement, for the one -legitimate case: the confirmed documents have moved and the baseline has not -caught up yet. It lets a run whose provenance the baseline does not name pass, -which leaves those scores unenforced for that run and is why it has to be typed. - -## The second ratchet: patterns that never apply - -`never_ran` in the compat baseline holds the count of scripts that -bound a pattern and never once ran it. A pattern can MATCH a script and decline -every event, so only running the corpus shows whether it applied. It is not a -defect count: a script can legitimately never run because no event carries its -source field. - -## The ratchet does not gate EXTRA fields - -`fields_wrong` counts missing and mismatched fields only. A field we emit that -Elasticsearch does not is counted separately as `fields_extra`, printed in the -run and compared against nothing. - -An event still needs an empty diff to count as matched, so extras are gated at -EVENT granularity -- but on a source already scoring zero matched events, which -is most of the ones carrying real debt, nothing checks them at all. The run -prints the current figure; it is not restated here, for the same reason. - -Extras are the best single signal that a source is emitting its fields RAW rather -than parsed. axonius carried 2,313 of them on 3,705 fields, all of the -form `.event.data.` where Elasticsearch had `.` -- one -unread hoist, and reading it took the source from 11% of events to 94%. - -So **record the extras figure before and after any change to the pattern -ladder**. A pattern that stops claiming a script hands it to whatever sits -below, and if -that writes a field Elastic never emits, the totals hold while only the extras -move -- which the ratchet passes green. That has happened once and the change -was reverted; it was caught by having written the previous number down and by -nothing else. - -## What matters, and what does not - -Byte equality is not the goal. `tests/compare-policy.yaml` is the single -definition of which differences are defects, read by this tool and by the Rust -comparison harness. Every rule carries a reason, and every run reports what it -excluded. Adding an entry HIDES a difference, which is why the reason is -mandatory and why the section a rule lands in is a decision rather than a -filing choice: +Byte equality is not the goal. Every rule has a `why:` that says why the difference is not a defect, because adding a rule hides that difference from every run. The section a rule lands in decides what follows: | Section | What it claims | What follows from it | |---|---|---| -| `nondeterministic` | the value varies run to run and carries no information | removed at capture, never compared at all | -| `not_emitted` | Elastic or Beats plumbing DFE does not produce | never a defect | -| `known_different` | both sides produce it, differently | DEBT. Each entry should eventually go away and the count is expected to FALL | -| `corrected` | Elasticsearch is wrong and we are right | permanent. It never goes away, and matching the bug would have scored better | -| `unordered` | the field's value is a SET | the same members in another sequence is equality, not a difference | +| `nondeterministic` | The value varies run to run and says nothing | Removed before comparison, never compared | +| `not_emitted` | Elastic or Beats plumbing DFE does not produce | Never a defect | +| `known_different` | Both sides produce it, differently | Debt. Each entry should go, and the count should fall | +| `corrected` | Elasticsearch is wrong and we are right | Permanent. Matching the bug would have scored better | +| `unordered` | The field's value is a set | The same members in another order are equal | -`known_different` and `corrected` read alike and mean opposite things, and the -test is whether the difference is ours to close. An array NOT named in -`unordered` is compared in order, deliberately. +`known_different` and `corrected` read alike and mean opposite things. The test is whether the difference is ours to close. An array not listed in `unordered` is compared in order. -## Scope a rule to its source, or it blinds every other one +Scope a rule to its source, or it blinds every other one. A rule with `sources: [name, ...]` holds for those sources alone. Leave `sources` off where the reason is a fact about the field itself: `event.ingested` is write time everywhere. -A rule may carry `sources: [name, ...]` and then holds for those sources alone. -Omit it only where the reason is a fact about the FIELD -- `event.ingested` is -write time everywhere. An unscoped quirk is the expensive mistake: `event.kind` -and `error.message` were once excluded corpus-wide for one source's date -failure, so a transform emitting `pipeline_error` anywhere else did not register -as a difference at all. `unordered` is never scoped, because a set is a fact -about the field. +An unscoped quirk is the expensive mistake. `event.kind` and `error.message` were once excluded everywhere for one source's date failure, so a transform emitting `pipeline_error` for any other source did not register as a difference. `unordered` is never scoped, because a set is a fact about the field. -The two readers differ. `crates/dfe-runtime/src/testutil/policy.rs` honours -`sources` and chains the four rule sections into ONE skip set, keeping no column -breakdown -- a skipped path is not compared, whichever section it came -from. The audit tool that confirms the documents keeps them apart to build its -columns and does not read `sources` yet, so every rule applies globally there, -the wider reading of the two. And a comparison that does not know its source cannot honour scoping, so -every scoped rule applies to it, which is how the committed-fixture tests run. - -## The audit's four columns - -Every run reports what it excluded: - -```text -excluded 156 field differences by 5 rules: - 93x source.geo DB-IP Lite against MaxMind GeoLite2. - 24x tags Beats and test-harness tagging. Not vendor data. -``` - -What it still counts is sorted into four columns, which is where the sections -above land: - -- **real** -- the only one that means something is wrong. Everything the policy - did not explain. -- **order** -- `unordered`, where the members match and the sequence does not. - An undeclared array in a different order is still real. -- **meta** -- `not_emitted`. -- **enrich** -- `known_different` and `corrected` together. `nondeterministic` - reaches no column, because it is dropped before the comparison. - -Each fixture's own `dynamic_fields` and `numeric_keyword_fields` are honoured -on top of the policy. A source reports `CURRENT` when `real` is zero, and the -`clean` count is the events with no real difference. - -**The corpus test runs with GeoIP OFF** (`geoip_global::disable`). Skipping the -geo fields is not enough, because the enrichment has side effects that ARE -compared: gcp/vpcflow renames `source.as.asn` onto `source.as.number`, and an -ASN hit DB-IP Lite has where MaxMind's GeoLite2 does not makes that rename land -on an occupied target -- which fails the document and skips the twenty-nine -removes behind it, costing 262 events. Comparing against output built from a -database we do not have means running without one. The fixture floors still -exercise enrichment, over the Elastic fixtures in dfe-transform-elastic-dev and the samples in this repository's `tests/fixtures/`. +`policy.rs` honours `sources` and chains the four rule sections into one skip set, so a skipped path is not compared whichever section it came from. A comparison that does not know its source cannot honour scoping, so every scoped rule applies to it. ## Correct beats bug-compatible -The corpus is the reference for what a pipeline MEANS, not a specification of -what Elasticsearch does in every case. Where its output is wrong, we emit the -right value and record the divergence -- we do not reproduce the bug. - -Two live examples. Elasticsearch 9.2.2 overflows a 2,826 ms duration to -`event.duration: -1468967296` where the correct value is 2,826,000,000, because -it multiplies into a 32-bit int. And symantec_endpoint_security carries 51 -events with a non-JSON `event.original`, so both engines fail the same `json` -processor and report it in their own words -- the failure agrees and only the -wording is the runtime's own. - -Both are recorded in `tests/compare-policy.yaml` with the reason attached, so -the difference is excluded from scoring and stays visible to anyone reading it. -A divergence taken this way is a decision with a name on it, and it is never -the quiet option: matching the bug would have scored better. - -### Most of Elasticsearch's own failures never reach a policy entry - -`compat_corpus.rs` drops any expected document whose `event.kind` is -`pipeline_error`, or that carries a bare `_compat_error` object, BEFORE -comparing -- counting it as `events_unanswered`, because scoring against a -failed run counts our correct output as a miss. 405 captured events carry an -Elasticsearch-side `error.message` and that gate excludes 336 of them. - -So a policy entry is needed only where the failed document is still SCORED, -which means Elasticsearch failed a processor and its `on_failure` left -`event.kind` as `event`. That is why cisco_nexus needs no entry in this -harness despite 27 unparseable timestamps -- all 27 set `pipeline_error` and -the source scores 45/45. Its entries in `compare-policy.yaml` still do work -for the audit tool, which applies the rules globally. +Elasticsearch's output is the reference for what a pipeline means, not a specification of what it does in every case. Where its output is wrong, we emit the right value and record the divergence in `corrected` rather than reproduce the bug. -**Read the scorer before concluding an Elasticsearch failure costs anything.** -A scan of the confirmed documents cannot see this gate, and sizing a prize off the captures -alone over-counts it by roughly six times. +Elasticsearch 9.2.2 multiplies milliseconds into a 32-bit int, so azure's 2,826 ms duration becomes `event.duration: -1468967296` where the correct value is 2,826,000,000. symantec_endpoint_security has events with a non-JSON `event.original`, so both engines fail the same `json` processor and report it in different words: the failure agrees and the wording differs. Both rules name the bug in their `why:`, so the difference is left out of the score and stays visible to anyone reading the file. diff --git a/docs/compat.md b/docs/compat.md index 134d96717..da6d1a30c 100644 --- a/docs/compat.md +++ b/docs/compat.md @@ -1,37 +1,47 @@ -# compat -- confirmed output from the real Elastic engine +# compat -- measuring against the Elastic engine -compat runs a fixture's sample lines through the source's own ingest pipeline in a throwaway Elasticsearch container and keeps the documents the engine returns. Those documents are ground truth: what the real engine produces, rather than what we believe it produces. The corpus test scores each transform against the confirmed fixtures it can load. +compat runs a source's sample lines through that source's ingest pipeline in a throwaway Elasticsearch container and keeps the documents the engine returns. The corpus test scores each transform against them field by field. The corpus and its test are not part of this repository. -The samples, the confirmed documents, the corpus test and its baseline all live in the private dfe-transform-elastic-dev repository, which builds against this repository's crates. This repository carries none of them. +What a difference means, and which differences are kept on purpose, is [compat-policy.md](compat-policy.md). -What a difference MEANS -- the ratchet, the compare policy, and the divergences kept on purpose -- is [compat-policy.md](compat-policy.md). +Elasticsearch's output is the reference, and the score is per source. ```mermaid flowchart LR - L["sample lines
+ test config"] --> B["message envelope
+ config fields"] - P["the source's
ingest pipeline"] --> I["install into ES"] - B --> S["_simulate"] - I --> S - S --> C["<fixture>-expected.json"] - C --> T["compat_corpus.rs
scores the transform"] + accTitle: L4 Detail -- How a transform is scored + accDescr: Sample lines go through the source's ingest pipeline in Elasticsearch, which returns confirmed documents. The same lines go through the transform, and the two outputs are compared field by field under the compare policy. + + samples[("Sample lines")] + + subgraph engines["Both engines"] + es["Elasticsearch
ingest pipeline"] + transform["The transform"] + end + + compare["Compare
per field"] + score(["Per-source
score"]) + + samples --> engines + engines --> compare + compare --> score + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef core fill:#DCEBF7,stroke:#0072B2,stroke-width:2px,color:#0B2A3F + classDef store fill:#DDF3EC,stroke:#00785A,stroke-width:1px,color:#0B3327 + classDef ext fill:#EEF0F2,stroke:#6B7280,stroke-width:1px,stroke-dasharray:4 3,color:#1F2937 + classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 + class samples store + class es ext + class transform core + class engines group ``` -## What a confirmed fixture holds +*Figure 1. L4 Detail -- How a transform is scored. The dashed box is external.* Up: [L3 Components -- dfe-transform-elastic](architecture.md#four-steps-between-the-producers-and-the-sink). -A confirmed sample in dfe-transform-elastic-dev has `-expected.json` beside it. The samples under this repository's `tests/fixtures/` are inputs only. - -| Key | Holds | -|---|---| -| `meta` | The package and data stream, the engine version, the pipeline commit, the entry pipeline, and the fields the test config marks as nondeterministic | -| `input` | The documents the engine was sent, exactly as the pipeline saw them | -| `expected` | The documents it returned, one per input, or `_compat_error` where the pipeline dropped or failed the event | - -`compat_corpus.rs` reads these files, runs the inputs of every capture it can score through the transform, and compares field by field. It starts no container and needs no network. - -The confirmed documents are produced by the meta project that maintains the transforms, which runs the engine. +The comparison reads `tests/compare-policy.yaml` through `dfe_runtime::testutil::policy`, so the rules live here beside the runtime. ## Known limits -- **o365 and panw are incomplete.** Part of their transformation runs inside Filebeat before Elasticsearch sees the event: o365 has a JavaScript processor chain, and panw applies `decode_csv_fields` to `message`. The simulate API cannot reach either, so for those two the confirmed output covers only the Elasticsearch half. -- **GeoIP is not compared.** The corpus test disables its GeoIP enricher and excludes GeoIP-derived fields from the comparison. An event can still carry geo fields its source supplied. -- **The engine never ships.** The Elasticsearch image is a development dependency of the meta project only and must never become part of the shipped service. +- o365 and panw are incomplete. Part of their transformation runs inside Filebeat before Elasticsearch sees the event: o365 has a JavaScript processor chain, and panw applies `decode_csv_fields` to `message`. The simulate API reaches neither, so for those two the confirmed output covers the Elasticsearch half. +- GeoIP is not compared. The corpus test turns its GeoIP enricher off and excludes GeoIP-derived fields. An event can still have geo fields its source supplied. +- The engine never ships. Elasticsearch is a development dependency of the measurement, never part of the service. diff --git a/docs/configuration.md b/docs/configuration.md index 7d3357f0a..05b764db7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,92 +1,88 @@ # Configuration -Where the values come from, which env spelling reaches a `--config` file, how the transport is selected, and which sections the file cannot carry. The field-by-field reference is generated rather than written here: `config-schema.yaml` beside this page carries every key with its description, and `config.example.yaml` at the repo root is the complete default set, pinned against drift by a test. `dfe-transform-elastic emit-config` reprints it. +Where the values come from, which env spelling reaches a `--config` file, how the transport is selected, and which sections the file cannot hold. The field-by-field reference is generated: `config-schema.yaml` beside this page lists every key with its description, and `config.example.yaml` at the repo root is the deployment contract's configuration, with illustrative source and sink values, pinned against drift by a test. `dfe-transform-elastic emit-config` reprints it. ## The configuration is read once, at startup -`Config::load` reads the configuration once and hands it to the batch loop by reference, so every value needs a restart to change. There is no hot reload: the broker connections, the resolved transform, the GeoIP databases and the metrics labels are all fixed at startup. +`Config::load` reads the configuration once and hands it to the batch loop by reference, so every value needs a restart to change. The broker connections, the resolved transform, the GeoIP databases and the metrics labels are fixed at startup. -The two ways in are not equivalent. With no `--config` the scalo cascade applies: `defaults`, `settings` and `settings.` files, `.yaml` or `.yml`, searched in `./`, `./config/`, `/config/` and `~/.config/dfe-transform-elastic/`, then the `DFE_TRANSFORM_ELASTIC_*` environment. With `--config`, which is what the container passes, `Config::load` reads the named file into `Config` and also hands it to scalo's cascade as its settings layer, so the scalo sections written in it apply. Kafka credentials are unaffected either way: they are read from `KAFKA_*` separately. +The two ways in are not equivalent. With no `--config` the scalo cascade applies: `defaults`, `settings` and `settings.` files, `.yaml` or `.yml`, searched in `./`, `./config/` and `/config/`, then the `DFE_TRANSFORM_ELASTIC_*` environment. `~/.config/dfe-transform-elastic/` is searched when `APP_NAME=dfe-transform-elastic` is set. With `--config`, which is what the container passes, `Config::load` reads the named file into `Config` and hands it to scalo's cascade as its settings layer, so the scalo sections written in it apply. Kafka credentials come from `KAFKA_*` either way. -## Only the flat env spelling reaches a `--config` file - -The FLAT, single-underscore form is applied to the loaded configuration on both branches, so it works against a `--config` deployment: +The flat, single-underscore env form is applied to the loaded configuration on both paths, so it is the form that works against a `--config` deployment: DFE_TRANSFORM_ELASTIC_SOURCE_TOPICS=a,b DFE_TRANSFORM_ELASTIC_SOURCE_BATCH_SIZE=5000 DFE_TRANSFORM_ELASTIC_SINK_TOPIC=out -It covers every `source.*` and `sink.*` field. `geoip` is not among them: it is scalo's own type, so it stays settable from the file and the cascade only. scalo's DOUBLE-underscore form is resolved from the cascade, and a `--config` deployment reads `source.*` and `sink.*` straight from the file, so it never reaches them. +It covers every `source.*` and `sink.*` field. `geoip` is scalo's type, so it takes no flat form and is set from the file or the cascade. scalo's double-underscore form is a cascade key, and a `--config` deployment reads `source.*` and `sink.*` from the file, so that form never reaches them. ## `transport` selects the bus or a direct push, per side -Each side carries a `transport` of `bus` (the default) or `direct`. A config file and the flat env form also take `kafka` for `bus` and `grpc` for `direct`, the names dfe-engine renders, and a config always writes `bus` and `direct` back. A config file refuses any other name, and an unrecognised env value keeps the configured transport rather than moving the deployment. The chart publishes the push port 6000 when `source.transport` is `direct` or `grpc`. Both transports are compiled in by default and both are constructed: on `direct` the service binds a scalo Push listener on `source.listen` (default `0.0.0.0:6000`) and pushes to `sink.endpoint` (default `http://dfe-loader:6000`). The listener answers a push only once its events are delivered, so a refused push is the upstream sender's to retry. +Each side has a `transport` of `bus` (the default) or `direct`. The file and the flat env form take `kafka` for `bus` and `grpc` for `direct` as well, and a config always writes `bus` and `direct` back. The file rejects any other name, and an unrecognised env value keeps the configured transport. The chart publishes push port 6000 when `source.transport` is `direct` or `grpc`. + +On `direct` the service binds a scalo Push listener on `source.listen` (default `0.0.0.0:6000`) and pushes to `sink.endpoint` (default `http://dfe-loader:6000`). The listener answers a push once its events are delivered, so a rejected push is the sender's to retry. DFE_TRANSFORM_ELASTIC_SOURCE_TRANSPORT=direct DFE_TRANSFORM_ELASTIC_SOURCE_LISTEN=0.0.0.0:6000 DFE_TRANSFORM_ELASTIC_SINK_ENDPOINT=http://dfe-loader:6000 -Every `source` and `sink` key may be left out of the file. `source.name`, `source.topics`, `source.group_id`, `source.brokers` and `sink.topic` then default to empty. An instance with no source name idles, as does a bus instance with no topics. Once the configuration names work, startup refuses a bus source with no brokers or group. - -On `direct` the brokers and group may stay empty. Empty topics do not idle there, because the listener is the work. In a DFE deployment the slim and mesh profiles run both sides on `direct`, and single and scale run on `bus`. - -`sink.topic` is required on both transports once the configuration names work. On `direct` it is the routing key every push carries, and dfe-loader routes a record with no `_source` on that key, so a push without one lands under the loader's `default_topic`. +Every `source` and `sink` key may be left out. `source.name`, `source.topics`, `source.group_id`, `source.brokers` and `sink.topic` then default to empty, and an instance with no source name idles, as does a bus instance with no topics. Once the configuration names work, startup rejects a bus source with no brokers or group. On `direct` the brokers and group may stay empty, and empty topics do not idle, because the listener is the work. In a DFE deployment the slim and mesh profiles run both sides on `direct`, and single and scale run on `bus`. -The listener speaks plaintext gRPC with no authentication of its own, and the push to the endpoint travels in the clear. Both belong behind the mesh route dfe-infra provisions for them, never on an interface reachable from outside the cluster. +`sink.topic` is required on both transports once the configuration names work. On `direct` it is the routing key every push has, and dfe-loader routes a record with no `_source` on that key, so a push without one lands under the loader's `default_topic`. The listener speaks plaintext gRPC with no authentication, and the push to the endpoint is unencrypted. Both belong behind the mesh route dfe-infra provisions, never on an interface reachable from outside the cluster. ## Agentless mode serves one syslog stream of many device types -`source.mode: agentless` reads the source off each event instead of from `source.name`, so one pod can consume a dfe-receiver syslog topic that carries firewalls, switches and proxies together. Each event is matched against a table of markers (literals, key sets and CEF vendor and product pairs), the winner's transform runs, and the output is stamped `_source` with that source's dfe-schemas catalogue label, which is what dfe-loader routes on. The markers are catalogue data: each source's `catalogue.detect` block in dfe-schemas, compiled into `src/detect/markers.rs` from the release `scripts/dfe-schemas.toml` pins. `source.name` must stay empty. `source.envelope` must be `auto` or `receiver`. +`source.mode: agentless` reads the source off each event instead of from `source.name`, so one pod can consume a dfe-receiver syslog topic with firewalls, switches and proxies on it together. Each event is matched against a table of markers (literals, key sets and CEF `Device Vendor` and `Device Product` pairs), the winner's transform runs, and the output is stamped `_source` with that source's dfe-schemas catalogue label, which dfe-loader routes on. The markers are each source's `catalogue.detect` block in dfe-schemas, compiled into `src/detect/markers.rs` from the release `scripts/dfe-schemas.toml` pins. `source.name` must stay empty, and `source.envelope` must be `auto` or `receiver`. - `source.mode`: `agent` (default) or `agentless`. - `source.detect.unmatched`: what an event no marker claims gets. `passthrough` (default) unwraps it to the Beats layout, runs no transform and leaves `_source` off, so the loader lands it under `sink.topic`'s default table. `fallback` runs `source.detect.fallback`'s transform and stamps its label. `stop` ends the service with the block unreleased, so a restart reads it again. -- `source.detect.fallback`: a registered source, required when `unmatched` is `fallback` and refused otherwise. -- `source.detect.candidates`: the sources the detector may name. Empty means all of them. A name it does not know is refused at startup. Narrowing it hands shared lines to the source left in: ASA-format lines go to ASA, and to the IOS catch-all once ASA is out. Citrix CEF goes to the ADC source unless it is excluded, and then to WAF. +- `source.detect.fallback`: a registered source, required when `unmatched` is `fallback` and rejected otherwise. +- `source.detect.candidates`: the sources the detector may pick. Empty means all of them, and an unknown name fails startup. Narrowing it hands shared lines to the source left in: ASA-format lines go to ASA, and to the IOS catch-all once ASA is out. Citrix CEF goes to the ADC source unless it is excluded, and then to WAF. DFE_TRANSFORM_ELASTIC_SOURCE_MODE=agentless DFE_TRANSFORM_ELASTIC_SOURCE_DETECT_UNMATCHED=fallback DFE_TRANSFORM_ELASTIC_SOURCE_DETECT_FALLBACK=filebeat.cisco_ios.default DFE_TRANSFORM_ELASTIC_SOURCE_DETECT_CANDIDATES=filebeat.cisco_asa.default,filebeat.cisco_ftd.default -Fetcher events carry the provider's own payload, so they count as unmatched. Three counters report the split: `events_detected_total{source}`, `events_unmatched_total` and `events_fallback_total`. +Fetcher events hold the provider's payload, so they count as unmatched. Three counters report the split: `events_detected_total{source}`, `events_unmatched_total` and `events_fallback_total`. -Detection costs about 2,080 ns an event (481K events a second on one core), so a 20,000-event batch spends about 42 ms on it. That is 20,000 compat-corpus events of 718 bytes on average, measured on one AMD EPYC 7302 core with the bench profile. `cargo bench --bench detect` re-derives it. +Detection cost about 2,080 ns an event (481K events a second on one core) over 20,000 syslog lines averaging 718 bytes, on one AMD EPYC 7302 core with the bench profile, so a 20,000-event batch spends about 42 ms on it. `cargo bench --bench detect` reads the parity corpus, and skips without it. -Run it on the receiver's `_raw` capture where you can. The receiver splits each line with `syslog_loose`, and a marker that straddled the split (`Pleasant Password Server:`, an `amp:` tag) is gone from the parsed fields. On the compat corpus the detector names 8,243 of 8,885 events of detectable sources from the raw line and 8,004 from the parsed fields. +Run it on the receiver's `_raw` capture where you can. The receiver splits each line with `syslog_loose`, and a marker that straddled the split (`Pleasant Password Server:`, an `amp:` tag) is gone from the parsed fields. Over 8,885 test events of detectable sources the detector identified 8,243 from the raw line and 8,004 from the parsed fields. -`scripts/replay_syslog.py` feeds an agentless stack from the committed fixtures under `tests/fixtures/unencumbered/`, standard library only. `--sources` takes catalogue labels from the same table the detector stamps, and `--list` shows the ones with fixtures. A syslog header is added only to a fixture line that lacks one, and the `%` the ASA fixtures dropped is put back. UDP sends one datagram per line and TCP is newline framed. There is no `http://` form, because dfe-receiver's `/ingest` takes JSON and not syslog. +`scripts/replay_syslog.py` feeds an agentless stack from the committed fixtures under `tests/fixtures/unencumbered/`, standard library only. `--sources` takes catalogue labels from the table the detector stamps, and `--list` shows the ones with fixtures. A syslog header is added to a fixture line that lacks one, and the `%` the ASA fixtures dropped is put back. UDP sends one datagram per line and TCP is newline framed. There is no `http://` form, because dfe-receiver's `/ingest` takes JSON and not syslog. scripts/replay_syslog.py --list scripts/replay_syslog.py --target udp://dfe-receiver:514 --sources cisco-asa,fortinet,panw --rate 50 --loop --duration 120 ## `source.acknowledgements` holds the source until delivery -`source.acknowledgements.enabled` (default `true`) holds the source's acknowledgement -- the Kafka offset commit on the bus, the answer to a push on direct -- until every event built from a record is delivered, or dropped and counted. A sink outage is then waited out, and a crash redelivers rather than loses. `false` acknowledges at receipt, before the transform runs, and a crash loses what was in flight. The flat env form is `DFE_TRANSFORM_ELASTIC_SOURCE_ACKNOWLEDGEMENTS_ENABLED`. +`source.acknowledgements.enabled` (default `true`) holds the source's acknowledgement, the Kafka offset commit on the bus or the answer to a push on direct, until every event built from a record is delivered, or dropped and counted. A sink outage is then waited out, and a crash redelivers rather than loses. `false` acknowledges at receipt, before the transform runs, and a crash loses what was in flight. The flat env form is `DFE_TRANSFORM_ELASTIC_SOURCE_ACKNOWLEDGEMENTS_ENABLED`. On a direct source a push is held at most 18 s, so every send it waits on gives up at 15 s: a direct sink's gRPC deadline, and a bus sink's `message.timeout.ms` (librdkafka's default is 300 s). ## `sink.max_message_bytes` caps one outbound record -The default is the DFE record cap, 8,000,000 bytes: dfe-receiver refuses a request body over it, and every broker DFE deploys takes a record that size (Confluent Basic's 8,388,608 limit leaves room for the framing). The producer this service builds allows 16 MiB, so a deployment with larger brokers raises the budget up to that. An event that serialises larger is dropped, logged, counted on `events_oversize_total`, and its source is still acknowledged. +The default is the DFE record cap, 8,000,000 bytes: dfe-receiver rejects a request body over it, and every broker DFE deploys takes a record that size (Confluent Basic's 8,388,608 limit leaves room for the framing). The producer this service builds allows 16 MiB, so a deployment with larger brokers can raise the budget up to that. An event that serialises larger is dropped, logged and counted on `events_oversize_total`, and its source is acknowledged. -Against a broker or topic whose `message.max.bytes` is lower -- a stock Kafka broker takes about 1 MB -- set `sink.max_message_bytes` below that limit. Left at the default, a broker below 8,000,000 bytes refuses the events between the two instead: each is dropped as a dead letter, counted on `send_filtered_dlq_total`, and the partition moves on. Validation refuses a value under 4096 or over 16,777,216. +Against a broker or topic with a lower `message.max.bytes` (a stock Kafka broker takes about 1 MB), set `sink.max_message_bytes` below that limit. Left at the default, the broker rejects the events between the two: each is dead-lettered by the sink transport, counted on `send_filtered_dlq_total`, and the partition moves on. Validation rejects a value under 4096 or over 16,777,216. DFE_TRANSFORM_ELASTIC_SINK_MAX_MESSAGE_BYTES=900000 ## A record that will not parse goes to the DLQ, one that will not transform is counted -A record that is not JSON, and an event the sink transport would refuse, go to the DLQ `dlq` configures: by default NDJSON files under `/var/spool/scalo/dlq`, on the `dlq` volume the chart mounts at 1Gi. The source is acknowledged only once the DLQ holds them, so a full or broken DLQ holds the block rather than losing it. `dlq` is scalo's `DlqConfig`, so `mode`, `file.rotation` and `file.max_age_days` are set there. With `dlq.enabled: false` they are dropped and counted on `pipeline_dead_letters_dropped_total` instead. +A record that is not JSON, and an event the sink transport would reject, go to the DLQ `dlq` configures: by default NDJSON files under `/var/spool/scalo/dlq`, on the `dlq` volume the chart mounts at 1Gi. The source is acknowledged once the DLQ holds them, so a full or broken DLQ holds the block rather than losing it. `dlq` is scalo's `DlqConfig`, so `mode`, `file.rotation` and `file.max_age_days` are set there. With `dlq.enabled: false` they are dropped and counted on `pipeline_dead_letters_dropped_total`. -An event whose transform returns an error, or that will not serialise, is dropped and counted on `events_errored_total`, and its source is acknowledged, so it is not redelivered. +Invalid UTF-8 is decoded with U+FFFD replacements, as Beats does for a file it cannot decode, and counted on `lossy_payloads_total`. Input that is not JSON is counted on `parse_errors_total`. An event that fails to transform, or will not serialise, is dropped and counted on `events_errored_total`, and its source is acknowledged, so it is not redelivered. Each `Backpressured` answer from the sink counts on `send_backpressure_total`. -## scalo's own sections are set in the same file +## scalo's sections are set in the same file -The `--config` file is a layer of scalo's cascade, so `scaling`, `self_regulation`, `logger` and the other sections scalo reads for itself apply when written in it. The DOUBLE-underscore env form overrides them, for example `DFE_TRANSFORM_ELASTIC_SCALING__MEMORY_GATE_THRESHOLD`. The effective scaling values are logged once at startup, with whether they came from the cascade or the default. +The `--config` file is a layer of scalo's cascade, so `scaling`, `self_regulation`, `logger` and the other sections scalo reads for itself apply when written in it. The double-underscore env form overrides them, for example `DFE_TRANSFORM_ELASTIC_SCALING__MEMORY_GATE_THRESHOLD`. The effective scaling values are logged once at startup, with whether they came from the cascade or the default. -`SCALO_SECTIONS` in `src/config/loader.rs` is the full list, and the unread-key warning skips it. One of them does nothing here: `batch_processing` sizes scalo's own batch engine, which the service loop does not run. Its receives take their record cap from `source.batch_size` and their byte cap from `source.max_batch_bytes`. scalo still refuses to start on a `batch_processing` section that does not parse. Every other key no field and no scalo section reads gets one warning at startup. `geoip` is not on the list: the service declares that section itself and hands it to scalo. KEDA replica scaling is separate: it is driven by the chart's `keda.*` values and the `scaling_pressure` gauge. +`SCALO_SECTIONS` in `src/config/loader.rs` is the full list, and the unread-key warning skips it. `batch_processing` does nothing here: it sizes scalo's batch engine, which the service loop does not run. The loop takes its record cap from `source.batch_size` and its byte cap from `source.max_batch_bytes`, though scalo still fails to start on a `batch_processing` section that does not parse. Every other key that no field and no scalo section reads gets one warning at startup. `geoip` is not on the list, because the service declares that section and hands it to scalo. -The contract's ScaledObject scales on CPU alone (80%, 1 to 10 replicas). In a DFE deployment, dfe-infra's values set the CPU target to 70% in every profile. The single, scale and mesh profiles add the scaling-pressure trigger (target 70, read through dfe-keda-shim), and slim runs on CPU alone. The engine's [KEDA scaling page](https://github.com/hyperi-io/dfe-engine/blob/main/docs/deployment/keda-scaling.md) covers the triggers for every app. +KEDA replica scaling is set by the chart's `keda.*` values and the `scaling_pressure` gauge, not this file. The contract's ScaledObject scales on CPU alone (80%, 1 to 10 replicas). In a DFE deployment, dfe-infra's values set the CPU target to 70% in every profile, and the single, scale and mesh profiles add the scaling-pressure trigger (target 70, read through dfe-keda-shim). The engine's [KEDA scaling page](https://github.com/hyperi-io/dfe-engine/blob/main/docs/deployment/keda-scaling.md) covers the triggers for every app. ## The metrics listener is scalo's, not a config field -`/livez`, `/readyz`, `/metrics` and `/metrics/manifest` are served from scalo's `--metrics-addr` (env `METRICS_ADDR`, default `0.0.0.0:9090`), so charts and deployments override that rather than a YAML field. On `direct` the push listener is a second port, 6000 by default, declared in the deployment contract so the generated Dockerfile exposes both. +`/livez`, `/readyz`, `/metrics`, `/metrics/manifest` and `/scaling/pressure` are served from scalo's `--metrics-addr` (env `METRICS_ADDR`, default `0.0.0.0:9090`), so charts and deployments override that rather than a YAML field. `/scaling/pressure` is the figure KEDA reads. `dfe-transform-elastic metrics-manifest` prints every metric without starting the service. On `direct` the push listener is a second port, 6000 by default, declared in the deployment contract so the generated Dockerfile exposes both. diff --git a/docs/enrichment.md b/docs/enrichment.md index bf5305309..57e55b0c4 100644 --- a/docs/enrichment.md +++ b/docs/enrichment.md @@ -1,67 +1,50 @@ # Enrichment -GeoIP, user agent and community ID: the three runtime enrichers loaded once and shared across -every transform. The code map around them is [architecture.md](architecture.md), and the -processors that call them are listed in [parsers.md](parsers.md). +GeoIP, user agent and community ID are the three runtime enrichers in `crates/dfe-runtime/src/enrichment/`, loaded once and shared across every transform. A generated transform calls one where its pipeline has a `geoip`, `user_agent` or `community_id` processor ([parsers.md](parsers.md#processor-taxonomy)). The code map is [architecture.md](architecture.md). -Runtime enrichment modules loaded once, shared across transforms: +Each enricher reads one kind of field and writes one subtree back onto the same event. ```mermaid -flowchart TD - subgraph "Enrichment Runtime (dfe-runtime)" - geoip[GeoIP
MMDB via maxminddb
mmap + bounded cache] - ua[User Agent
Regex-based
UA parsing] - cid[Community ID
v1 hash
algorithm] +flowchart LR + accTitle: L4 Detail -- Enrichers + accDescr: The GeoIP enricher reads an IP field and writes geo fields. The user agent enricher reads a user agent string and writes user_agent fields. The community ID enricher reads the flow fields and writes network.community_id. + + ipf["IP field"] + uaf["User agent
string"] + flow["Flow fields"] + + subgraph enrichers["dfe-runtime enrichment"] + geoip["GeoIP"] + ua["User agent"] + cid["Community ID"] end - subgraph "Data Sources" - mmdb[(DB-IP Lite city mmdb
provisioned by scalo)] - uareg[(UA regex
patterns)] - end - - event[Event] -->|ip field| geoip - event -->|ua field| ua - event -->|network fields| cid - - mmdb --> geoip - uareg --> ua - - geoip -->|geo.*| event - ua -->|user_agent.*| event - cid -->|network.community_id| event -``` + geo["geo.*"] + uaout["user_agent.*"] + cidout["network.
community_id"] ---- + ipf --> geoip + uaf --> ua + flow --> cid + geoip --> geo + ua --> uaout + cid --> cidout -## Enrichment trait - -```rust -pub trait Enrichment: Send + Sync { - fn enrich(&self, event: &mut Event) -> Result<()>; -} + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 + class enrichers group ``` ---- - -## GeoIP design - -- **Provisioning:** the config's `geoip` section is scalo's `GeoIpConfig` verbatim, because provisioning is shared across the fleet while the lookup engine and its cache stay in `dfe-runtime`. It defaults to ENABLED with auto-download from DB-IP Lite, so an operator gets databases without configuring anything, and it is read once at startup. -- **Storage:** mmap'd MMDB via `maxminddb` crate (zero-copy read) -- **Cache:** a bounded cache in front of the MMDB reader (`crates/dfe-runtime/src/enrichment/geoip_cache.rs`), 100,000 entries by default. When full it evicts the oldest-stored quarter, not the least recently used: a hit takes only the read lock and updates no recency. A generated data-stream module can carry several geoip calls, so one event can hit the cache several times. -- **Output fields:** `country_name`, `country_iso_code`, `city_name`, `location.lat`, `location.lon`, `continent_name`, `timezone` -- **Failure:** `ignore_missing` support, errors tagged not fatal - ---- - -## User agent design +*Figure 1. L4 Detail -- Enrichers.* Up: [L3 Components -- dfe-transform-elastic](architecture.md#four-steps-between-the-producers-and-the-sink). -- **Parser:** Regex-based UA string parsing -- **Output fields:** `name`, `version`, `os.name`, `os.version`, `device.name`, `device.type` +GeoIP provisioning, storage, cache, output and failure: ---- +- Provisioning: the config's `geoip` section is scalo's `GeoIpConfig` as-is, because provisioning is shared across the fleet, and the lookup and its cache stay in `dfe-runtime`. It defaults to enabled, with auto-download from DB-IP Lite, and is read once at startup. +- Storage: the MMDB is memory-mapped through the `maxminddb` crate. +- Cache: a bounded cache in front of the reader (`geoip_cache.rs`), 100,000 entries by default. When full it evicts the oldest-stored quarter rather than tracking use, so a hit takes the read lock and updates nothing. A transform with several `geoip` processors hits the cache several times an event, counted on `geoip_cache_hits_total` and `geoip_cache_misses_total`. +- Output fields: `continent_name`, `country_name`, `country_iso_code`, `region_name`, `region_iso_code`, `city_name`, `location` (`lat`, `lon`) and `timezone`, plus `asn`, `organization_name` and `network` from an ASN database. +- Failure: none reaches the event. A generated `geoip` call skips an absent field, and a value that is not an IP, a private address or a lookup miss writes no fields (`geoip_global.rs`). -## Community ID design +The user agent parser matches pre-compiled regexes against the string and writes `original`, `name`, `version`, `os.name`, `os.version`, `os.full`, `device.name` and, where it can tell, `device.type`. Its output differs from Elastic's ua-parser in small ways. -- **Algorithm:** Community ID v1 (SHA-1 based network flow hash) -- **Input fields:** `source.ip`, `destination.ip`, `source.port`, `destination.port`, `network.transport` -- **Output:** `network.community_id` (e.g., `1:LQU9qZlK+B5F3KDmev6m5PMibrg=`) +Community ID is v1, a SHA-1 hash of the flow, read from `source.ip`, `destination.ip`, `source.port`, `destination.port` and `network.transport` or `network.iana_number`, with `icmp.type` and `icmp.code` for ICMP. It writes `network.community_id`, such as `1:LQU9qZlK+B5F3KDmev6m5PMibrg=`. diff --git a/docs/parity.md b/docs/parity.md index bcc7cdd72..9a492b6e3 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -1,83 +1,48 @@ # Parity -What this service promises against Elastic's own output, what it deliberately leaves alone, and -how the claim is measured. The code map it sits beside is [architecture.md](architecture.md), -and the mechanics of the measurement are [compat.md](compat.md) and -[compat-policy.md](compat-policy.md). +The goal is the same ECS output Elastic's ingest pipeline gives for each supported Beats and Elastic Agent integration. Parity is scored per source, and some sources match fully and others do not yet. The code map is [architecture.md](architecture.md), and how the scoring works is [compat.md](compat.md) and [compat-policy.md](compat-policy.md). -**Goal:** 1:1 (or better) parity with Elastic ingest pipeline output for the supported -Beats and Elastic Agent integrations. Parity is measured per source against Elastic's -output on the compat corpus and ratcheted in the compat baseline, so some -sources match fully and others do not yet. +This is an independent transform service, not a port of Elasticsearch. Data sent through DFE should get the field mapping, type coercion and enrichment it would get going straight to Elasticsearch. Where Elasticsearch's output is wrong, the service emits the correct value and records the divergence ([compat-policy.md](compat-policy.md#correct-beats-bug-compatible)). -This is NOT reverse engineering Elasticsearch. We are building an independent transform -service that produces the same normalised ECS output as the Elastic ingest pipeline would. -Customers sending Beats/Agent data through DFE should get identical field mapping, type -coercion, and enrichment as if the data went directly to Elasticsearch. +Parity is checked against documents a running Elasticsearch produced, compared per field ([compat.md](compat.md)). That corpus and its test are not in this repository. [compat-policy.md](compat-policy.md) covers `tests/compare-policy.yaml`, which decides what counts as a difference. -**"Or better" means:** Where Elastic pipelines have known limitations (regex-only parsing, -single-threaded Painless execution, per-document GeoIP lookups), we can exceed their -performance while maintaining output compatibility. - ---- +This repository's floors run on every clone: `integration/`, `untested_sources.rs` and `unencumbered.rs` in `crates/dfe-transforms/tests/`, over the licence-clean samples in `tests/fixtures/unencumbered/` and the synthetic samples in `tests/fixtures/synthetic/`. They assert no panic, no new errors and enrichment, never parity ([testing.md](testing.md)). ## What we replicate -The Elastic ingest pipeline is the primary transform layer. Data arrives from Beats/Agent -as JSON events, and the ingest pipeline applies processors sequentially to normalise, -enrich, and route the data. - -| Elastic Component | dfe-transform-elastic Equivalent | Status | -|-------------------|----------------------------------|--------| -| **Ingest processors** (27 used) | Rust processor implementations, one per Elastic processor type | Done | -| **Painless scripts** | A hand-written runner from `crates/dfe-painless/src/bespoke/` where one is registered for that script, otherwise pattern-matched against `params.rs` and the ladder in `common.rs`; unrecognised scripts are skipped | see [Painless coverage](parsers.md#painless-coverage) -- the figure is a corpus-run output, not a constant, so re-derive it rather than quoting one from here | -| **Foreach processor** | Per-event loop inside the transform function | Okta, O365 | -| **Pipeline chaining** | A nested pipeline is INLINED into the calling module between `// Begin nested pipeline` / `// End nested pipeline` markers, so there is no call and no dispatch on the hot path | Done. `rg -c "Begin nested pipeline" crates/dfe-transforms/src/` counts today's call sites | -| **GeoIP enrichment** | Global MMDB enricher, auto-detected at startup, behind a bounded cache | Done (DB-IP Lite) | -| **User Agent parsing** | Regex-based parser | Done (minor diffs from Elastic UA parser) | -| **Community ID** | Hash-based network flow ID | Done | -| **Conditional evaluation** | Native Rust `if`/`match` expressions per condition pattern | Nearly. A condition the transpiler cannot read emits `if false`, making the processor it guards dead code -- `skipped_processors` in `tests/ratchets.json` is the live count, with a per-source table beside it | -| **On-failure handlers** | Wrapped around EVERY processor by the generator, alongside `if` and `ignore_failure`, so a new processor cannot quietly omit them. A raised error runs the handlers and records `_ingest.on_failure_message` and `_ingest.on_failure_processor_type` | Done | - ---- - -## What we do not replicate - -| Elastic Component | Why Not | Our Approach | -|-------------------|---------|--------------| -| Beat/Agent collection | We receive from Kafka, not from endpoints | Upstream responsibility | -| Transport parsing (syslog, CEF) | Done by Beat/Agent, or by dfe-receiver for the syslog envelope | Upstream responsibility | -| Agent-side processors (`add_host_metadata`, `add_cloud_metadata`) | Run on source machine | Fields arrive pre-populated in event | -| `@custom` pipelines | User-specific, not part of integration | Not applicable | -| `final_pipeline` | Elasticsearch-specific routing | Not applicable | -| `enrich` processor | Requires Elasticsearch enrich index | Alternative enrichment via dfe-loader | -| `inference` processor | ML model execution | Not applicable for DFE | -| `reroute` processor | Elasticsearch index routing | Kafka topic routing instead | - -The Beat or Agent collects raw data, adds host/cloud metadata, applies its own local -processors and JSON-encodes the result before it ever reaches Kafka (see -[System context](architecture.md#system-context) for the full flow). Everything from there is -this service's job: JSON parse, ingest pipeline logic, enrichment, ECS normalisation. - -One deliberate difference in the output: `event.original` is not a second copy of the vendor line. Elastic keeps it, but the default output here is the extracted JSON alone. Where `pipeline::run_batch` emits an event: - -- Tagged `preserve_original_event` (Elastic's own opt-in, and what its pipelines add on failure): `event.original` stays. -- Holding any field beyond the static and shipper fields in `ENVELOPE` (`dfe-runtime` `transform.rs`): `event.original` is removed. -- Holding only those fields, so nothing was parsed: `event.original` moves back to `message`, and `events_unparsed_original_restored_total` counts it. +Beats and Elastic Agent send JSON events, and the ingest pipeline applies its processors in order to normalise, enrich and route them. Each row below is how the generated code does the same. + +| Elastic component | dfe-transform-elastic equivalent | Status | +|---|---|---| +| Ingest processors | Rust per processor type, listed in [parsers.md](parsers.md#processor-taxonomy) | Done for every processor a shipped pipeline uses | +| Painless scripts | A hand-written runner from `crates/dfe-painless/src/bespoke/` where one is registered for the script, else a match against `params.rs` and the ladder in `common.rs`. An unrecognised script is counted and skipped | Partial, see [Painless coverage](parsers.md#painless-coverage) | +| `foreach` | A loop inside the transform function | Done | +| Pipeline chaining | A nested pipeline is inlined into the calling module between `// Begin nested pipeline` and `// End nested pipeline` markers, so there is no call on the hot path. `rg -c "Begin nested pipeline" crates/dfe-transforms/src/` counts the sites | Done | +| GeoIP enrichment | One MMDB enricher, found at startup, behind a bounded cache | Done, on DB-IP Lite | +| User agent parsing | Regex-based parser | Done, with small differences from Elastic's parser | +| Community ID | Hash-based network flow ID | Done | +| Conditions | Native Rust `if` per condition. A condition the transpiler cannot read emits `if false`, so the processor it guards never runs | Partial. `skipped_processors` in `tests/ratchets.json` counts them, with a per-source table | +| `on_failure` handlers | Generated around every processor that declares them, beside `if` and `ignore_failure` ([architecture.md](architecture.md#a-failed-processor-ends-in-one-of-four-places)) | Done | + +The service leaves these out on purpose: + +| Elastic component | Why not | Our approach | +|---|---|---| +| Beat or Agent collection | Events arrive from Kafka or a push, not from endpoints | Upstream | +| Transport parsing (syslog, CEF) | Done by the Beat or Agent, or by dfe-receiver for the syslog envelope | Upstream | +| Agent-side processors (`add_host_metadata`, `add_cloud_metadata`) | Run on the source machine | The fields arrive set | +| `@custom` pipelines | Per user, not part of an integration | Not applicable | +| `final_pipeline` | Elasticsearch index routing | Not applicable | +| `enrich` processor | Needs an Elasticsearch enrich index | dfe-loader's enrichment columns | +| `inference` processor | ML model execution | Not applicable | +| `reroute` processor | Elasticsearch index routing | Kafka topic routing | + +The Beat or Agent collects the data, adds host and cloud metadata, runs its local processors and JSON-encodes the result before it reaches Kafka. From there the job is this service's: JSON parse, ingest pipeline logic, enrichment and ECS normalisation. + +One difference is deliberate: `event.original` is not a second copy of the line. Elastic keeps it, and the default output here is the extracted JSON alone. When `pipeline.rs` emits an event: + +- tagged `preserve_original_event` (Elastic's opt-in, which its pipelines add on failure), `event.original` stays. +- holding any field beyond the static and shipper fields in `ENVELOPE` (`crates/dfe-runtime/src/transform.rs`), `event.original` is removed. +- holding nothing else, so nothing was parsed, `event.original` moves back to `message`, counted on `events_unparsed_original_restored_total`. A transform called directly still sets `event.original`. - ---- - -## How parity is verified - -**Parity is measured ONLY against the compat corpus.** -`compat_corpus.rs` compares per FIELD against output captured from a real Elasticsearch and ratchets the compat baseline, which holds every per-source score. Both run in the private dfe-transform-elastic-dev repository, beside the corpus. -[compat.md](compat.md) and [compat-policy.md](compat-policy.md) are the full account: how the -corpus is generated, what `tests/compare-policy.yaml` excludes and why, and how the ratchet -refuses to be lowered. - -Elastic's integration samples are Elastic-licensed and kept in dfe-transform-elastic-dev, which runs the corpus test and its own floors over them. This repository's floors run on every clone: `integration/`, `untested_sources.rs` and `unencumbered.rs`, over the licence-clean samples in `tests/fixtures/unencumbered/` and the synthetic samples in `tests/fixtures/synthetic/`. They assert no panic, no new errors and enrichment, never parity. - -Per-processor runtime status and code pattern: see -[Processor taxonomy](parsers.md#processor-taxonomy). diff --git a/docs/parsers.md b/docs/parsers.md index 8996eff7c..ac39684ec 100644 --- a/docs/parsers.md +++ b/docs/parsers.md @@ -1,45 +1,48 @@ # Parsers and processors -The three-layer parser strategy that replaces grok and regex, every ingest processor the -transforms use, and how far the Painless matchers reach. The code map around it is -[architecture.md](architecture.md), and the speed numbers behind the layers are -[performance.md](performance.md). +Grok runs on compiled regex at almost every call site. Three whole-pattern forms take a native path instead, and the composite builder and DFA fallback in `dfe-parse` are built but not called by any transform. This page covers that dispatch, every ingest processor the transforms use, and how far the Painless matchers reach. The code map is [architecture.md](architecture.md), and the libraries are [performance.md](performance.md). ---- +`grok_cache::native_form` recognises whole-pattern forms and rejects a near-miss rather than guessing at one: a pattern with literal text around its captures stays on the regex. -## Three layers over grok and regex - -Three-layer strategy for replacing grok/regex with native Rust, implemented in `dfe-parse`. -Layer 1 is WIRED, for the three whole-pattern forms `grok_cache::native_form` recognises. -Everything else still falls back to the compiled regex the grok expands to. Layers 2 and 3 -exist in `dfe-parse`, but nothing outside that crate calls them yet. The dispatch table -below is the current state, and widening it is the gap this architecture exists to close. +A grok call site in a native form skips the regex, and every other site runs its compiled regex. ```mermaid -flowchart TD - input[Grok Pattern String] --> expand["Expand grok aliases
%{IP} -> regex"] - expand --> analyse{Analyse each
capture group} - - analyse -->|All replaceable| l1[Layer 1: Native Parsers
parse_ipv4, parse_int, etc.
10-20x faster] - analyse -->|Some replaceable| l2[Layer 2: Composite
Chain L1 parsers +
literal separators
not called yet] - analyse -->|Irreducibly complex| l3[Layer 3: DFA Fallback
regex-automata pre-compiled
not called yet] +flowchart LR + accTitle: L4 Detail -- Grok dispatch + accDescr: A grok call site checks whether its pattern is one of the native whole-pattern forms. If so it runs a dfe-parse parser or a newline scan, otherwise the compiled regex. Both return the captured fields. + + site(["Grok call site"]) + form{"Native
form?"} + native["dfe-parse or
newline scan"] + regex["Compiled regex"] + fields(["Captured fields"]) + + site --> form + form -->|yes| native + form -->|no| regex + native --> fields + regex --> fields + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef decision fill:#FCF1D6,stroke:#9A6700,stroke-width:1px,color:#3D2A00 + class form decision +``` - l1 --> output[Structured fields] - l2 --> output - l3 --> output +*Figure 1. L4 Detail -- Grok dispatch.* Up: [L3 Components -- dfe-transform-elastic](architecture.md#four-steps-between-the-producers-and-the-sink). - style l1 fill:#2d5,stroke:#1a3,color:#fff - style l2 fill:#28a,stroke:#167,color:#fff - style l3 fill:#a72,stroke:#841,color:#fff -``` +| Pattern | Path | Native against regex | +|---|---|---| +| `^%{IPV4:f}$` | `dfe_parse::ip::parse_ipv4` | 29 ns against 214 ns | +| `^%{IPV4:a}:%{PORT:p}$` | the two chained | 47 ns against 208 ns | +| `%{GREEDYDATA:f}`, anchored or not | `split_once('\n')` | 11 ns against 1,098 ns | ---- +The timings come from `cargo bench -p dfe-runtime --bench grok`. The third form reaches no parser: `line_anchored` makes it `(?m)^.*$` because joni anchors to lines, so it captures the first line and the native form is a newline scan. Count the sites from the generated tree (`cached_grok!` and `cached_grok_mapped!`) rather than from a figure written here. -## Layer 1: common pattern replacements (`ip.rs`, `numeric.rs`, `string.rs`, `timestamp.rs`) +`^%{DATA:f}$` is excluded although it reads the same: `%{DATA}` is the lazy `.*?`, and on a `\r\n` line ending greedy keeps the `\r` in the capture where lazy stops before it. Every dispatched form has an equivalence test running both paths over the same inputs, because the native path is an optimisation over the regex and never a replacement for it. -Zero-regex native Rust parsers for the 15 most common grok patterns: +`dfe-parse` holds more than the dispatch uses. Layer 1 is native parsers for 15 common grok patterns, in `ip.rs`, `network.rs`, `numeric.rs`, `string.rs` and `timestamp.rs`: -| Grok Pattern | Rust Parser | Technique | +| Grok pattern | Rust parser | Technique | |---|---|---| | `%{IPV4}` | `parse_ipv4()` | Octet validation, direct byte scan | | `%{IPV6}` | `parse_ipv6()` | Colon-group parser | @@ -47,64 +50,26 @@ Zero-regex native Rust parsers for the 15 most common grok patterns: | `%{INT}` | `parse_int::()` | Direct digit scan, generic over type | | `%{NUMBER}` | `parse_number()` | Integer or float with optional exponent | | `%{WORD}` | `take_word()` | Byte-level alphanumeric scan | -| `%{GREEDYDATA}` | `take_greedy()` | Return remaining slice (zero cost) | +| `%{GREEDYDATA}` | `take_greedy()` | Returns the remaining slice | | `%{TIMESTAMP_ISO8601}` | `parse_iso8601()` | Fixed-position field extraction | -| `%{SYSLOGTIMESTAMP}` | `parse_syslog_timestamp()` | Month lookup table + digit parse | +| `%{SYSLOGTIMESTAMP}` | `parse_syslog_timestamp()` | Month lookup table and digit parse | | `%{LOGLEVEL}` | `parse_loglevel()` | Case-insensitive keyword match | -| `%{QUOTEDSTRING}` | `take_quoted()` | memchr for quote, handle escapes | +| `%{QUOTEDSTRING}` | `take_quoted()` | memchr for the quote, handles escapes | | `%{HOSTNAME}` | `parse_hostname()` | Label-dot-label (RFC 1123) | | `%{MAC}` | `parse_mac()` | Fixed 6-group hex parse | | `%{URI}` | `parse_uri()` | scheme://authority/path?query#fragment | | `%{UUID}` | `parse_uuid()` | Fixed 8-4-4-4-12 hex pattern | -**What is WIRED is a much smaller set than what exists.** The table above is -`dfe-parse`'s capability, not the dispatch. `grok_cache::native_form` recognises -whole-pattern forms only, and by design refuses a near-miss rather than guessing -at one — a pattern with literal text around its captures stays on the regex, -because the regex engine is good at exactly that. Three forms are dispatched -today: - -| Pattern | Path | Measured | -|---|---|---| -| `^%{IPV4:f}$` | `dfe_parse::ip::parse_ipv4` | 29 ns against 214 ns | -| `^%{IPV4:a}:%{PORT:p}$` | the two chained | 47 ns against 208 ns | -| `%{GREEDYDATA:f}`, anchored or not | `split_once('\n')` | 11 ns against 1,098 ns | - -The third reaches no parser at all: `line_anchored` makes it `(?m)^.*$` because -joni anchors to lines, so it captures the FIRST LINE and the native form is a -newline scan. It covers 202 call sites over 37 distinct patterns, 4.7% of the -4,299 grok sites in the generated tree (3,496 `cached_grok!` plus 803 -`cached_grok_mapped!`). - -Those are DERIVED, not typed: re-count them from the generated tree rather than -editing them. - -`^%{DATA:f}$` is deliberately excluded despite reading the same: `%{DATA}` is -the lazy `.*?`, and on a `\r\n` line ending greedy keeps the `\r` in the capture -where lazy stops before it. - -Every dispatched form carries an equivalence test running both paths over the -same inputs, because the native path is an optimisation over the regex and never -a replacement for it. - -**Parser convention:** All parsers take `&str`, return `ParseResult<'_, T>` where -`T` is the parsed value (often `&str` for zero-copy): +Every parser takes `&str` and returns `ParseResult<'_, T>`, the remaining input and the parsed value, following the winnow and nom convention: ```rust pub type ParseResult<'a, T> = Result<(&'a str, T), ParseError>; ``` -The return tuple contains `(remaining_input, parsed_value)`, following winnow/nom convention. - ---- - -## Layer 2: composite parser builder (`composite.rs`) - -Built in `dfe-parse` and not yet called by the transforms. Chains Layer 1 parsers with literal separators for full grok-equivalent patterns: +Layer 2, `composite.rs`, chains Layer 1 parsers with literal separators. Layer 3, `dfa.rs`, wraps `regex_automata::meta::Regex` and returns named captures. Both are built and tested in `dfe-parse`, and no transform calls either. ```rust // Grok: %{IPORHOST:source_ip}:%{INT:source_port} -> %{IPORHOST:dest_ip}:%{INT:dest_port} -// Becomes: let parser = CompositeParser::builder() .capture("source_ip", parse_ip_or_host) .literal(":") @@ -116,116 +81,48 @@ let parser = CompositeParser::builder() .build(); ``` ---- - -## Layer 3: pre-compiled DFA fallback (`dfa.rs`) - -Built in `dfe-parse` and not yet called by the transforms. For patterns that can't be decomposed into Layer 1/2: - -- Compile regex to DFA at build time via `regex-automata` -- Serialise DFA bytes, load at runtime with zero compilation cost -- Named capture groups mapped to field names - ---- - ## Processor taxonomy -Every processor the transforms use, by implementation weight, runtime status, and which -sources exercise it. - -### Simple (~11) — direct `Event` API calls, no parser dependency - -| Processor | Status | Used By | Code Pattern | -|---|---|---|---| -| `set` | Done | All | `event.set(path, value)?` | -| `append` | Done | All | `event.append(path, value)?` | -| `remove` | Done | All | `event.remove(path)` | -| `rename` | Done | All | `event.rename(from, to)?` | -| `uppercase` | Done | Panw | `event.set(path, s.to_uppercase())?` | -| `lowercase` | Done | All | `event.set(path, s.to_lowercase())?` | -| `trim` | Done | Cisco | `event.set(path, s.trim())?` | -| `convert` | Done | All | Type coercion: `str→i64`, `str→f64`, `i64→str` | -| `drop` | Done | All | `return Ok(TransformResult::Drop)` | -| `split` | Done | O365 | `event.set(path, s.split(sep).collect())?` | -| `join` | Done | SentinelOne | `join_values(value, sep)`, `None` on a non-array | -| `uri_parts` | Done | Panw | Scheme, host, port, path, query components | -| `dot_expander` | Done | GCP | `dot_expand(event, path, field)` | -| `fail` | Done | CrowdStrike | Returns a `TransformError` carrying the message | -| `terminate` | Done | Defender | Stops the pipeline and KEEPS the document | -| `urldecode` | Done | Zscaler | `url_decode`, form-encoded so `+` is a space | -| `sort` | Done | M365 Defender | `sort_values`, errors on anything with no natural ordering | - -### Medium (~8) — read a field value and reshape it - -| Processor | Status | Used By | Code Pattern | -|---|---|---|---| -| `grok` | Done (regex fallback) | All | `grok_to_regex` builds a pattern, `regex::Regex` matches it | -| `dissect` | Done | Cisco, Meraki | Tokenizer-based split parser | -| `json` | Done | All | `codegen_api::parse_json_str`, on `serde_json` | -| `kv` | Done | Okta | Key-value split, configurable delimiters | -| `csv` | Done | Fortinet | Separator/quote config | -| `foreach` | Done | Okta, O365, sysmon | Each element bound to `_ingest._value`, the inner processor run, the list rebuilt | -| `date` | Done | All | `chrono` against the formats a source emits | -| `gsub` | Done | O365 | `regex::Regex::replace_all` | - -### Complex (~8) — enrichment runtime or a Painless pattern match - -| Processor | Status | Used By | Code Pattern | -|---|---|---|---| -| `script` (Painless) | Pattern-match, see below | All | Rust matching a recognised script pattern, or a no-op | -| `geoip` | Done (DB-IP) | All with IPs | `enrichment::geoip::enrich(event, field, prefix)?` | -| `user_agent` | Done | O365, Okta | `enrichment::user_agent::enrich(event, field, prefix)?` | -| `community_id` | Done | Panw, Fortinet | `enrichment::community_id::enrich(event)?` | -| `registered_domain` | Done | Panw | `codegen_api::registered_domain_lookup`, a `psl` lookup against Mozilla's Public Suffix List | -| `network_direction` | Done | Fortinet, Panw | CIDR-based internal/external classification | -| `fingerprint` | Done | O365, M365 | `fingerprint_default`: SHA-1, base64, NUL before each value. `salt` and `method` are refused, so the default is the only path. | -| `pipeline` (nested) | Done | most multi-pipeline packages -- derive with `rg -l "Begin nested pipeline"` | Inlined into the caller, not a call | - -Not implemented (not used by any transform's pipeline): bytes, cef, date_index_name, -enrich, geo_grid, html_strip, inference, redact, reroute, set_security_user. -The generator ERRORS on one rather than skipping it, so a package that starts -using one fails to onboard and says which -- that is how `join`, `sort`, -`urldecode`, `dot_expander`, `fail` and `terminate` got written. - ---- - -## Painless coverage - -A script the runtime cannot execute is skipped rather than failing the event, so the skips have -to be counted or they are indistinguishable from a script that did nothing. `painless_exec` -lives in `crates/dfe-painless/src/plan.rs` and is re-exported through -`crates/dfe-runtime/src/codegen_api.rs` for the generated call sites. It resolves a -hand-written runner from `crates/dfe-painless/src/bespoke/` first, then the params patterns -in `params.rs`, then the `known_patterns` ladder in `common.rs`, and records the outcome -through `crates/dfe-painless/src/stats.rs`. - -A generated call site holds no script. The generator resolves the dispatch and emits -`compiled_painless!` with the script's hash and that plan as JSON, plus the text only where a -bound runner still reads it. The runtime looks bespoke runners up by the hash, and the counters -key each script by it. - -**Two measurements, and only the second is honest about reach.** - -`painless_coverage.rs` drives a few dozen fixture files and holds a floor of 100%, none skipped. That floor is real but its driver is narrow, so it says those fixtures are fully covered, not that the runtime is. +Every processor the transforms use, and the code the generator emits for it. -The corpus test, `compat_corpus.rs`, runs the same counters over every confirmed capture it can load and score, and that is what says it. Both tests run in dfe-transform-elastic-dev, beside the Elastic fixtures they read. Both figures are run outputs rather than constants, so derive them rather than quoting a number from here: the corpus run prints handled, skipped and distinct-script counts, and `never_ran` is ratcheted in the compat baseline. - -The gap between the two is the point. A pattern can MATCH a script statically and then decline at -run time, which reads as covered from everywhere except the corpus run. - ---- +| Processor | Weight | Code pattern | +|---|---|---| +| `set`, `append`, `remove`, `rename` | Simple | `event.set`, `event.append`, `event.remove`, `event.rename` | +| `uppercase`, `lowercase`, `trim` | Simple | Read the string, write it back changed | +| `convert` | Simple | `convert_value` to string, integer, long, float, double, boolean or ip | +| `drop` | Simple | Returns `TransformResult::Drop` | +| `split` | Simple | Splits a string into an array | +| `join` | Simple | `join_values(value, sep)`, `None` on a non-array | +| `uri_parts` | Simple | Scheme, host, port, path and query components | +| `dot_expander` | Simple | `dot_expand(event, path, field)` | +| `fail` | Simple | Returns a `TransformError` with the message | +| `terminate` | Simple | Stops the pipeline and keeps the document | +| `urldecode` | Simple | `url_decode`, form-encoded so `+` is a space | +| `sort` | Simple | `sort_values`, an error on anything with no natural order | +| `html_strip` | Simple | `html_strip` | +| `grok` | Medium | `cached_grok!`, one compiled pattern per call site, on `regex` or `fancy-regex` | +| `dissect` | Medium | Tokenizer-based split | +| `json` | Medium | `codegen_api::parse_json_str`, on `serde_json` | +| `kv` | Medium | Key-value split with configurable delimiters | +| `csv` | Medium | The `csv` crate, with the processor's separator and quote | +| `foreach` | Medium | Each element bound to `_ingest._value`, the inner processor run, the list rebuilt | +| `date` | Medium | `parse_date_out` against the formats the processor names | +| `gsub` | Medium | A cached regex `replace_all` | +| `script` (Painless) | Complex | A matched runner, or counted and skipped. See below | +| `geoip` | Complex | `enrichment::geoip` | +| `user_agent` | Complex | `enrichment::user_agent` | +| `community_id` | Complex | `enrichment::community_id` | +| `registered_domain` | Complex | `codegen_api::registered_domain_lookup`, a `psl` lookup against Mozilla's Public Suffix List | +| `network_direction` | Complex | CIDR-based internal and external classification | +| `fingerprint` | Complex | `fingerprint_default` (SHA-1, base64, NUL before each value), or `fingerprint_with` for a `salt` and SHA-256, SHA-512 or MurmurHash3. Any other method fails | +| `pipeline` (nested) | Complex | Inlined into the caller, not a call. `rg -l "Begin nested pipeline"` lists the modules | + +Not implemented, and used by no shipped pipeline: `bytes`, `cef`, `date_index_name`, `enrich`, `geo_grid`, `inference`, `redact`, `reroute` and `set_security_user`. -## Future: dfe-parsers, a standalone parser crate +## Painless coverage -**Vision:** extract the per-source message-parsing logic in `dfe-transforms` into a standalone -`dfe-parsers` crate that any DFE Rust project can depend on. It would take `&str` / `Value` -input and return structured output with no knowledge of Beats, Elastic Agent, or transport, so -a syslog feed project outside this repo could parse a raw line and get the same structured -output as if it had come through Beats. +A script the runtime cannot execute is skipped rather than failing the event, so the skips are counted, or they would look like a script that did nothing. `painless_exec` lives in `crates/dfe-painless/src/plan.rs` and is re-exported through `crates/dfe-runtime/src/codegen_api.rs`. It resolves a hand-written runner from `crates/dfe-painless/src/bespoke/` first, then the params patterns in `params.rs`, then the `known_patterns` ladder in `common.rs`, and records the outcome through `crates/dfe-painless/src/stats.rs`. The service exports the outcome as `painless_handled_total` and `painless_unhandled_total`. -`dfe-transforms` would become a thin layer over it: Beats/Agent envelope handling, ECS field -naming, and the enrichment calls, with the actual message parsing delegated out. +A generated call site holds no script. The generator resolves the dispatch and emits `compiled_painless!` with the script's hash and that plan as JSON, plus the normalised text where a bound runner still reads it. The runtime looks bespoke runners up by the hash, and the counters key each script by it. -The nearer half of that is already standing: `dfe-parse` is a workspace crate of its own, and -dfe-transform-splack runs a copy of it -- [shared-crates.md](shared-crates.md) is the account of -what the two projects share and what it would take to extract it. +Two measures, and the second alone says how far the matchers reach. `crates/dfe-transforms/tests/painless_binding.rs` reads every plan out of the generated tree and asks what it binds to, which says a script is claimed but not that the runner writes anything. The parity corpus run counts how many scripts ran and how many were skipped, and those figures are run outputs, never constants. A pattern can match a script statically and decline at run time, which reads as covered everywhere except that run. diff --git a/docs/performance.md b/docs/performance.md index a11722392..e504b77d8 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -1,109 +1,54 @@ # Performance -What the hot path is built out of -- SIMD, zero-copy, reused buffers, the release profile and -the libraries underneath -- plus the targets the native parsers are held to and where the -benchmarks live. The code map around it is [architecture.md](architecture.md), and the parsers -themselves are [parsers.md](parsers.md). +What the hot path is built from: SIMD byte search, borrowed values, reused buffers, the release profile and the libraries underneath, plus where the benchmarks live. The code map is [architecture.md](architecture.md), and the grok dispatch and its timings are [parsers.md](parsers.md). ---- +`memchr` scans for delimiters in `crates/dfe-parse/src/string.rs`, and `aho-corasick` finds every agentless marker in one pass over the line. Both dispatch on the CPU features the build targets in `.cargo/config.toml`: -## SIMD acceleration +- `x86_64-unknown-linux-gnu`: `-C target-cpu=x86-64-v3` (AVX2, BMI1 and BMI2, FMA, Haswell and later) +- `aarch64-unknown-linux-gnu`: `-C target-cpu=generic` (the NEON baseline), linked with mold +- A build with no `--target`: `-C target-cpu=native` -| Library | Use | SIMD Features | -|---|---|---| -| `memchr` | Delimiter scanning in parsers (`string.rs`) | AVX2/SSE2 byte search | -| `regex-automata` | Layer 3 DFA fallback (`dfa.rs`) | Compiled DFA (no backtracking) | +JSON parsing is not SIMD. simd-json lost to `serde_json` on this workload ([architecture.md](architecture.md#event-model)). -JSON deserialisation is NOT on this list. simd-json was measured against -`serde_json` on this workload and lost on both paths -- see -[Event model](architecture.md#event-model). +Values are borrowed where the types allow it. A parser returns `ParseResult<'a, &'a str>`, a slice of its input, and `event.get_str()` returns an `Option<&str>` into the event. A helper that may or may not change a string, such as `url_decode`, returns an `Option>`, borrowing unchanged input. Events hold their tree: `Event::from_json` parses each NDJSON line into an owned `serde_json::Value`. -Build targets in `.cargo/config.toml`: +`pipeline.rs` sizes its vectors from the batch with `Vec::with_capacity`, and `serialise_events` writes every event through one reused scratch buffer. -- **x86_64 release:** `-C target-cpu=x86-64-v3` (AVX2, BMI1/2, FMA — Haswell+) -- **aarch64 release:** `-C target-cpu=generic` (NEON baseline) -- **Development:** `-C target-cpu=native` +The release profile in `Cargo.toml` is thin LTO, one codegen unit, `panic = "abort"`, stripped, at `opt-level = 3`. The generated transform crates override it with `opt-level = 1` and 16 codegen units, which leaves the transform benches unchanged, because thin LTO re-optimises that code at the binary's level, and keeps each rustc's memory down. Splitting the modules across crates by package is what keeps each rustc inside a 16 GiB runner. ---- +The benchmarks are Criterion, except `allocations`, which counts allocations with its own allocator and prints a report. Run one with `cargo bench -p --bench `. -## Zero-copy patterns - -1. **Kafka → Event:** `Event::from_bytes` borrows from the message buffer rather than copying it -2. **Parser returns:** `ParseResult<'a, &'a str>` — parsed values reference input slice -3. **Field access:** `event.get_str()` returns `Option<&str>` borrowing from inner Value -4. **Cow for transforms:** `Cow<'_, str>` when field may or may not need modification - ---- - -## Buffer reuse - -- Pre-allocated `Vec` buffers for serialisation output -- `String` buffers cleared and reused across events for format operations -- `Vec::with_capacity()` based on expected batch sizes - ---- - -## Release profile - -```toml -[profile.release] -lto = "thin" -codegen-units = 1 -panic = "abort" -strip = true -opt-level = 3 -``` - ---- - -## Native parser targets - -| Operation | Regex/Grok | Native Rust | Expected Speedup | -|---|---|---|---| -| IP address parse | ~200ns | ~15ns | ~13x | -| Timestamp (ISO8601) | ~500ns | ~40ns | ~12x | -| Integer parse | ~100ns | ~10ns | ~10x | -| Syslog header (5 fields) | ~2us | ~100ns | ~20x | -| Full DNS log line (8 fields) | ~5us | ~300ns | ~15x | - -JSON parsing is deliberately absent: it is not a grok replacement, and the one -candidate that would have belonged here lost its own benchmark -(`crates/dfe-runtime/benches/json_parse.rs`). - ---- - -## Benchmark framework - -- **Per-parser:** Criterion benchmarks comparing each dfe-parse parser against its regex equivalent (`crates/dfe-parse/benches/parsers.rs`) -- **End-to-end:** Full pipeline (JSON deserialise → transform chain → serialise) - ---- +| Crate | Benches | +|---|---| +| root | `detect.rs`, `envelope.rs`, `original.rs` | +| `dfe-core` | `codec.rs`, `date_formats.rs`, `event.rs`, `hashing.rs` | +| `dfe-parse` | `parsers.rs`, each parser against its regex equivalent | +| `dfe-runtime` | `grok.rs`, `geoip.rs`, `json_parse.rs` | +| `dfe-transforms` | `transforms.rs`, `allocations.rs` | ## Key dependencies -The libraries the hot path is built on. `Cargo.toml` carries the version ranges and the reason -for each bound; this is the map of which crate does what. +The libraries the hot path is built on. `Cargo.toml` has the version ranges and the reason for each bound, and this is the map of which crate does what. | Crate | Purpose | |---|---| -| `serde` / `serde_json` | Serialisation framework, and both JSON paths. `preserve_order` is on, so object keys keep document order | -| `simd-json` | DEV-dependency of `dfe-runtime` only, for `crates/dfe-runtime/benches/json_parse.rs` -- the bench that demoted it | -| `scalo` (`yaml`) | YAML config parsing, through `scalo::yaml` on serde-saphyr | +| `serde` / `serde_json` | Serialisation, and both JSON paths. `preserve_order` is on, so object keys keep document order | +| `simd-json` | Dev-dependency of `dfe-runtime` alone, for `crates/dfe-runtime/benches/json_parse.rs`, the bench that demoted it | +| `scalo` | Config, logging, metrics, Kafka and gRPC transport, deployment, memory guard, scaling and DLQ: the service's runtime. Its `yaml` support parses the config on serde-saphyr | | `memchr` | SIMD byte search (`crates/dfe-parse/src/string.rs`) | -| `regex-automata` | Pre-compiled DFA regex (`crates/dfe-parse/src/dfa.rs`) | -| `regex` | Grok-derived pattern matching in `dfe-transforms` | -| `fancy-regex` | Backtracking fallback for the vendor patterns `regex` rejects by design -- lookaround and backreferences. Reached only when `regex` fails to COMPILE, so the hot path is unchanged | +| `aho-corasick` | The agentless marker scan (`src/detect/`) | +| `regex-automata` | The Layer 3 parser in `crates/dfe-parse/src/dfa.rs` | +| `regex` | Grok-derived pattern matching | +| `fancy-regex` | Backtracking fallback for the patterns `regex` rejects by design, lookaround and backreferences. Reached when `regex` fails to compile, so the hot path is unchanged | | `psl` | Mozilla's Public Suffix List, compiled in, for `registered_domain` | | `parking_lot` | The lock in front of the GeoIP cache, taken per event across one transform thread per partition | | `chrono` | Timestamp handling | -| `maxminddb` | GeoIP lookups (mmap) | -| `base64` | Fingerprinting and Community ID encoding | +| `maxminddb` | GeoIP lookups, memory-mapped | +| `base64` | Fingerprint and Community ID encoding | | `sha1`, `sha2` | Fingerprint and Community ID digests, and Painless's `sha1()` and `sha256()` as lower-case hex | -| `csv` | CSV processor | -| `scalo` | Config, logging, metrics, Kafka transport, deployment, memory guard, scaling -- the service binary's runtime | -| `clap` | Service CLI surface | +| `csv` | The `csv` processor | +| `clap` | The service's command line | | `tokio` | Async runtime | -| `thiserror` | Error types in every crate, including the service binary's `src/error.rs` | +| `thiserror` | Error types in `dfe-core`, `dfe-parse` and the service's `src/error.rs` | | `tracing` | Logging | -| `criterion` | Benchmarking framework | -| `testcontainers` | Ephemeral Kafka broker for round-trip tests | +| `criterion` | Benchmarks | diff --git a/docs/schema-generation.md b/docs/schema-generation.md index a937fde4d..02fdf6f15 100644 --- a/docs/schema-generation.md +++ b/docs/schema-generation.md @@ -1,147 +1,34 @@ -# Schema Generation +# Schema generation -This service transforms Beats/Agent JSON into normalised events. It is also the -only place that knows, per data stream, **what fields actually come out** — the -ingest pipelines declare intent, the compat corpus records what Elasticsearch -really emitted, and only here do the two meet. +Status: partly built, and none of it in this repository. The ECS meta schema in dfe-schemas, `meta/elastic/ecs.yaml`, is generated. Per-source package schemas, the evidence each column records and the Sigma field maps below are design, not code. [#16](https://github.com/hyperi-io/dfe-transform-elastic/issues/16) tracks generating schemas for every Beat and Elastic Agent source. -That knowledge is published as ClickHouse schemas to -[dfe-schemas](https://github.com/hyperi-io/dfe-schemas), by pull request. +The Elastic side of DFE needs one place that knows, per data stream, which fields come out. The ingest pipelines declare intent and the parity corpus records what Elasticsearch emitted, so the two meet where the transforms are built. That knowledge goes to [dfe-schemas](https://github.com/hyperi-io/dfe-schemas) as ClickHouse schemas, by pull request. dfe-engine defines how schemas work (`SchemaLoader`, `SchemaBuilderV2`, `MetaSchema`), imports Elastic index templates, and builds the sigma, ecs and cim views in `fieldmap/`. -## Where the pieces live +## The design -| Repo | Role | -|---|---| -| `dfe-schemas` | The schemas themselves. Version-tree YAML, immutable published versions, 13 primitive types, the `@source:`/`@generated:` expression language. | -| `dfe-engine` | Defines how schemas work: `SchemaLoader`, `SchemaBuilderV2`, `MetaSchema`, the Elastic index-template importer, and `fieldmap/` for the sigma/ecs/cim views. | -| `dfe-transform-elastic` (here) | Ships the generated Elastic-side transforms, schemas and field maps. The generator that produces them runs outside this repo. | +The deployable unit is a data stream. A table holding every filebeat field is not something anyone deploys, and a filebeat vocabulary is. So a schema has three layers: -## The deployable unit is a data stream +- the meta schema, the ECS fields every sub schema inherits. +- a sub schema, one package's namespace, grouping its data streams. +- a deployable table, the common header plus meta plus sub, at data-stream grain. -Measured over the compat corpus: +The grouping key exists: every entry in `src/registry.rs` is `("..", transform, intake, ".")`. `dfe-transform-elastic sources` prints the first element. -| | | -|---|---| -| distinct leaf field paths | 50,109 | -| data streams / packages | 939 / 335 | -| median data stream width | 47 fields | -| widest package | 4,573 (amazon_security_lake) | -| distinct top-level namespaces | 1,010 | +Types come from each package's `fields/*.yml`, not from output. Output gives JSON types alone, and cannot separate `keyword` from `text` or `long` from `scaled_float`, which are the distinctions that drive the DDL. `external: ecs` there is the reliable ECS discriminator: inferring it from the top-level namespace is wrong both ways, because `process.*` is ECS and extended by packages alike. A package's `fields.yml` gives the ECS field names it uses but no types, so the ECS layer takes its types from the ECS spec. -A "complete filebeat" **table** is therefore not a thing anyone deploys. A -complete-filebeat **vocabulary** is. So: +dfe-engine's `services/schema/elastic_schema_service.py` walks an index template's `mappings.properties`, and `fields.yml` is a different layout, so its `_walk_mapping` does not transfer. Two functions must stay identical across the Python and any port, or the generated views break against the generated tables: `_map_es_type` and `_column_name_from_field_path`. -- **Meta schema** — the shared definition every sub schema inherits (ECS). -- **Sub schema** — a package's own namespace, grouping its data streams. -- **Deployable table** — common header + meta + sub, at data-stream grain. +Each column would record its evidence: declared and emitted, declared and never emitted (a parity gap, or a field this data never has), or emitted and not declared. Published versions in dfe-schemas are immutable, so a regeneration adds a version, and the same inputs must produce byte-identical output or every run raises a spurious version. -The grouping needs no new taxonomy. `src/registry.rs` already holds one entry -per source, written as -`("..", transform, intake, ".")`, -and the fourth element is the grouping key. `dfe-transform-elastic sources` -lists them, so count them there rather than from a figure written here -- the -table grows with every onboarded source. +Sigma, ECS and CIM mappings are data, not in-stream renaming. dfe-engine builds those views from field-map tables (`fieldmap/FieldMapRegistry`, `ViewGenerator`, surfaced through `SchemaBuilderV2`'s `view_ddls`) in the form `mappings: {: }`, and the generator would write those tables: -## Types come from `fields/*.yml`, not from the corpus - -The corpus gives JSON types only. It cannot separate `keyword` from `text` -(a lowcardinality dimension against a fulltext index), or `long` from -`scaled_float` — and those distinctions are exactly what drive the DDL. - -Types come from each package's `fields/*.yml`, read at the same pipeline commit -every confirmed fixture records as `pipelines_commit`, so a generated schema and -the parity corpus describe the same pipelines. - -`external: ecs` there is the only reliable ECS/vendor discriminator. Inferring -it from the top-level namespace is wrong in both directions: `process.*` is ECS -*and* vendor-extended, and some vendor namespaces collide with ECS names. - -### The two layers have different sources, and only one is complete - -- **Vendor sub-schema** — fully derivable from a data stream's `fields.yml`, - which declares the package namespace as nested `type: group` trees with types. - No further dependency. -- **ECS meta schema** — the integrations clone gives the ECS field *names* a - package uses (22,037 `external: ecs` declarations across 1,556 files) and - sometimes `dimension: true`, but **no types**. Types need the ECS spec - (`elastic/ecs`, `generated/ecs/ecs_flat.yml`), which is not on disk. - -Not every data stream declares its ECS fields either: `aws.cloudtrail` has no -`ecs.yml` and relies on the stack's `ecs@mappings` component template. So the -ECS layer cannot be assembled from the integrations clone alone. - -## Reuse the importer's type mapping, not its walker - -`dfe-engine`'s `services/schema/elastic_schema_service.py` walks an **index -template**'s `mappings.properties`. Our input is `fields.yml`'s -`name`/`type`/`group`/`fields` layout, which is a different format, so -`_walk_mapping` does not transfer. - -Two pieces must stay identical across the Python and the Rust port, or the -generated views break against the generated tables: - -- `_map_es_type` — the 13 primitives -- `_column_name_from_field_path` — dotted path to underscored column - -A conformance test holds those two together, the same way the compat corpus -holds the transforms to Elasticsearch. - -Reading index templates directly would reuse the walker as well, but the compat -Elasticsearch installs ingest pipelines only, never whole packages, so it has no -index templates to read. Getting them means standing up Fleet or -`elastic-package`, which buys fidelity at a cost worth taking only if the -`fields.yml` route proves insufficient. - -## Every column carries its evidence - -A schema built from Elastic's declaration alone creates columns that are always -NULL for fields we never emit. Each generated column records which it is: - -| Evidence | Meaning | -|---|---| -| declared and emitted | In `fields/*.yml` and produced in the corpus. Ship it. | -| declared, never emitted | A parity gap, or a field this data never carries. | -| emitted, not declared | We produce it; the mapping does not declare it. | - -The third row matters beyond schemas: the corpus ratchet counts only missing and -mismatched fields, so the *extra* fields it reports are gated by nothing. The -run prints the current figure and it moves on every matcher change, so read it -there rather than from a number written here. Schema generation is the first -thing that would surface them. - -## Regeneration must be deterministic - -Published versions in dfe-schemas are immutable, so a regeneration **adds** a -version and never rewrites one. The same corpus plus the same `integrations_sha` -must produce byte-identical output — otherwise every run raises a spurious -version bump. That is a correctness property, and it has a test. - -## Sigma, ECS and CIM mapping: data, not in-stream renaming - -`dfe-engine` already builds sigma/ecs/cim ClickHouse VIEWs from field-map tables -(`fieldmap/FieldMapRegistry`, `ViewGenerator`, surfaced through -`SchemaBuilderV2`'s `view_ddls`). The format is -`mappings: {: }`. - -We generate those tables. We do not rename fields in the stream: - -1. The query-time mechanism already exists; in-stream duplicates it. -2. Renaming would destroy ECS, which our output *is* — `ecs.version` is stamped - by very nearly every generated transform module - (`rg -c ecs.version crates/dfe-transforms/src/ --stats` for today's count). - It would bake one Sigma rule generation into stored data that can never be - reinterpreted. -3. SigmaHQ ships rule changes constantly. As data, a change is a pull request; - in-stream, it is a fleet redeploy plus a backfill. +1. The query-time mechanism exists, and renaming in the stream would duplicate it. +2. Renaming would destroy ECS, which the output is: nearly every generated module stamps `ecs.version` (`rg -c ecs.version crates/dfe-transforms/src/` counts them). +3. SigmaHQ changes rules often. As data a change is a pull request, and in the stream it is a fleet redeploy plus a backfill. 4. Per-event renaming costs throughput for a query-time concern. -5. One ECS field serves several Sigma logsources; in-stream forces one choice. - -Where no SigmaHQ logsource correspondence exists, we emit **nothing**. A wrong -mapping is worse than a missing one, because the rule then silently matches the -wrong column. +5. One ECS field serves several Sigma logsources, and renaming in the stream forces one choice. -The ECS map needs no judgement — it is the dotted path with dots as -underscores, matching `_column_name_from_field_path`. +Where no SigmaHQ logsource corresponds, the map has no entry. A wrong mapping is worse than a missing one, because the rule then matches the wrong column. The ECS map needs no judgement: `_column_name_from_field_path` drops a leading `@` from each segment and joins the segments with underscores. diff --git a/docs/shared-crates.md b/docs/shared-crates.md index 3c925fc6c..b879f7e95 100644 --- a/docs/shared-crates.md +++ b/docs/shared-crates.md @@ -1,31 +1,20 @@ # Shared crates: what dfe-transform-splack took a copy of -## Today: independent copies +dfe-transform-splack, the Splunk counterpart, has copies of `dfe-parse` and `dfe-runtime`, and no extraction is scheduled. The two projects evolve their copies apart on purpose: both are changing, and a shared crate would make every breaking change block both at once. The duplication costs less than that coordination would. -`dfe-transform-splack` (the Splunk equivalent) took a copy of `dfe-parse` and -`dfe-runtime`. The two projects evolve their copies independently, and that is -deliberate: both are still changing, and extracting them now would make -every breaking change block both projects at once. +This workspace has since split its library code into five crates, `dfe-core`, `dfe-painless`, `dfe-parse`, `dfe-runtime` and `dfe-transforms`, so the one copy boundary is four. `dfe-runtime` re-exports the moved modules at their old paths, so a copy taken against the old layout still compiles, against a compatibility surface rather than the current one. -The duplication costs less than that coordination would. - -**The workspace has since split to five crates** — `dfe-core`, `dfe-painless`, -`dfe-parse`, `dfe-runtime`, `dfe-transforms` — so what was one copy boundary is -now four. `dfe-runtime` re-exports the moved modules at their old paths, so a -copy taken against the old layout still compiles; it is reading a compatibility -surface, not the current one. - -## What is common, conceptually +The common half, by crate: | Component | dfe-core | dfe-parse | dfe-runtime | -|-----------|----------|-----------|-------------| +|---|---|---|---| | IP parsers (v4, v6, host) | | Yes | | | Timestamp parsers (ISO 8601, syslog, epoch) | | Yes | | | Numeric parsers (int, float) | | Yes | | | String parsers (word, quoted, greedy) | | Yes | | | Network parsers (MAC, hostname, URI) | | Yes | | | Composite parser builder | | Yes | | -| DFA fallback (pre-compiled regex) | | Yes | | +| DFA fallback | | Yes | | | `Event` (dotted-path JSON wrapper) | Yes | | | | Error types | Yes | | | | Date formats and syslog priority | Yes | | | @@ -36,41 +25,15 @@ surface, not the current one. | Prelude re-exports | | | Yes | | Test utilities (harness, diff, flatten) | | | Yes | -## What is not - -| Project | Specific code | -|---------|---------------| -| dfe-transform-elastic | the whole `dfe-painless` crate, and `codegen_api.rs` in `dfe-runtime` — Painless semantics have no Splunk counterpart | -| dfe-transform-splack | Splunk eval expressions and lookup transforms | - -Putting the Painless matchers in their own crate makes that boundary explicit -in the manifest rather than a note in a table. It does not by itself prove -non-use: `dfe-runtime` re-exports `dfe-painless` at the old `painless_*` paths, -and `dfe-transforms` reaches the matchers that way rather than through a direct -edge. So a copy has to be checked for BOTH — a `dfe-painless` dependency, and -the re-export paths through `dfe-runtime`. +What is not common: the whole `dfe-painless` crate and `codegen_api.rs` in `dfe-runtime`, because Painless has no Splunk counterpart, and splack's Splunk eval expressions and lookup transforms. A separate `dfe-painless` makes that boundary visible in the manifest without proving non-use: `dfe-runtime` re-exports `dfe-painless` at the old `painless_*` paths, and the generated transforms reach the matchers that way. A copy has to be checked for both, a `dfe-painless` dependency and the re-export paths. ## Extracting later When both sides settle: -1. Diff the two copies to find what is genuinely common, rather than assuming. -2. Stand up a shared private workspace repo. **The name `dfe-core` is not - available for it**: this workspace ships a crate called `dfe-core` that is a - different thing -- workspace-internal, never published. Anything extracted - needs another name. -3. Move the common code there; project-specific code stays put. +1. Diff the two copies to find what is common, rather than assuming. +2. Stand up a shared private workspace repo. It needs a name other than `dfe-core`, which this workspace uses for an unpublished internal crate. +3. Move the common code there, and leave project-specific code where it is. 4. Both projects depend on it by git reference. -## When to do it - -Not on a date — on these signals: - -- Both parser sets have stopped growing. -- The `Event` API has settled: no breaking changes to get/set/remove. -- The `Transform` trait is identical in both. -- Test utilities are being copy-pasted between the two with no edits. -- A change to shared logic already requires updating both by hand. - -Until then, divergence is expected. If a parser or an `Event` method is missing, add it -locally rather than waiting for the other project. +The signals that it is time: both parser sets have stopped growing, the `Event` API has had no breaking change to get, set or remove, the `Transform` trait is identical in both, test utilities are copied between the two with no edits, and a change to shared logic needs both updated by hand. Until then divergence is expected. A missing parser or `Event` method is added locally rather than waiting for the other project. diff --git a/docs/testing.md b/docs/testing.md index ccd514f0d..b4eb431f5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1,46 +1,46 @@ # Testing -The test tracks -- the parity corpus, the fixture floors, the scanners over the generated tree -- and the match modes and non-English sweeps they run under. The code map around them is [architecture.md](architecture.md), what parity means is [parity.md](parity.md), and how the corpus is produced is [compat.md](compat.md). +Parity is scored by a corpus and a test that are not in this repository ([compat.md](compat.md)). The two tracks here, the floors and the scanners, run on every clone, and neither measures parity. The code map is [architecture.md](architecture.md), and what parity means is [parity.md](parity.md). -Three tracks, and only the first measures parity. It runs in the private dfe-transform-elastic-dev repository, because its corpus is Elastic's integration samples and the documents Elastic's engine produced for them, which this repository does not carry. +The floors read the committed fixtures, and the scanners read the generated tree against the ratchet file. ```mermaid flowchart LR - subgraph "Parity -- in dfe-transform-elastic-dev" - corpus[(Elastic samples and
confirmed documents)] - cc[compat_corpus.rs
compare per FIELD] - policy[compare-policy.yaml
what is not a defect] - end - - subgraph "Floors -- panic / error / enrichment" - fixtures[(tests/fixtures/
unencumbered and synthetic)] - floors[integration.rs
untested_sources.rs
unencumbered.rs] - end - - subgraph "Scanners -- over the generated tree" - ratchets[ratchets.json
dead branches, skipped] - binding[painless_binding.rs
bespoke_registry.rs
grok_pattern_coverage.rs] - end - - corpus --> cc - policy --> cc + accTitle: L4 Detail -- Floors and scanners + accDescr: The floor tests read the committed fixtures. The scanner tests read the generated transforms and compare their counts with the ratchet file. Both run in the CI test job. + + fixtures[("tests/fixtures")] + tree[("Generated
transforms")] + ratchets[("ratchets.json")] + floors["Floor tests"] + scanners["Scanner tests"] + ci(["CI test job"]) + fixtures --> floors + tree --> scanners + ratchets --> scanners + floors --> ci + scanners --> ci + + classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A + classDef store fill:#DDF3EC,stroke:#00785A,stroke-width:1px,color:#0B3327 + class fixtures,tree,ratchets store ``` -`tests/compare-policy.yaml` stays here because `dfe_runtime::testutil::policy` reads it, and dfe-transform-elastic-dev builds against this repository's runtime. +*Figure 1. L4 Detail -- Floors and scanners. Green is committed data.* Up: [L3 Components -- dfe-transform-elastic](architecture.md#four-steps-between-the-producers-and-the-sink). ---- +The floors are `integration.rs`, `untested_sources.rs` and `unencumbered.rs`. The scanners are `painless_binding.rs`, `bespoke_registry.rs`, `grok_pattern_coverage.rs` and `skipped_processors.rs`, and `tests/ratchets.json` holds the counts they ratchet, such as `dead_branch_sites` and `skipped_processors`. `tests/compare-policy.yaml` is read through `dfe_runtime::testutil::policy` by the parity comparison. ## The fixtures `tests/fixtures/` holds two sets, and `tests/fixtures/PROVENANCE.md` says where each comes from: -- `unencumbered/` -- log samples from CC0, MIT and Apache-2.0 projects, under their own licences. +- `unencumbered/` -- log samples from CC0, MIT and Apache-2.0 projects, under their licences. - `synthetic/` -- samples written for this repository, one file per log variant, each following the layout the product's maker publishes, with invented values. -Neither has expected output, so the floors over them assert what holds regardless of it: no panic, no new errors, and every event gains a field. An event the pipeline's failure handler marks with `error.message` counts as an error. The Beats envelopes under `tests/envelopes/beats/` are written the same way as the synthetic set. +Neither has expected output, so the floors assert what holds regardless: no panic, errors at or under each fixture's ceiling, and every event without an error gains a field. `untested_sources.rs` sets its floors per case. An event the pipeline's failure handler marks with `error.message` counts as an error. The Beats envelopes under `tests/envelopes/beats/` are written the same way as the synthetic set. -The fixture tests read this committed data with no variable to set, so they run whenever CI does: +The fixture tests read the committed data with no variable to set, so they run whenever CI does: ```bash cargo test -p dfe-transforms --test integration --test untested_sources \ @@ -49,34 +49,24 @@ cargo test -p dfe-runtime --all-features --lib --test date_repro cargo test --no-default-features --test envelope --test envelope_detection ``` -`crates/dfe-transforms/tests/unencumbered.rs` fails when `tests/fixtures/` holds anything but `unencumbered/`, `synthetic/` and the provenance, or when any file there carries one of three marker strings: the paths of Elastic's two source repositories, or its licence's name. - ---- +`crates/dfe-transforms/tests/unencumbered.rs` fails when `tests/fixtures/` holds anything but `unencumbered/`, `synthetic/` and the provenance, or when any file there contains one of three marker strings: the paths of Elastic's two source repositories, or its licence's name. -## Match modes +The test diff helper's `MatchMode` is defined in `crates/dfe-runtime/src/testutil/diff.rs`: -`crates/dfe-runtime/src/testutil/diff.rs` defines `MatchMode`: - -| Mode | Use Case | Behaviour | +| Mode | Use | Behaviour | |---|---|---| -| **Exact** | Default | Field-for-field JSON match | -| **Semantic** | Non-deterministic fields | Ignore timestamps with "now", generated UUIDs, GeoIP, user agent | -| **Subset** | Fields set by Beats runtime | Expected is subset of actual (extra fields OK) | - ---- - -## Non-English input +| `Exact` | Default | Field-for-field JSON match | +| `Semantic` | Against Elastic's output | Like `Exact`, skipping every path the compare policy excludes | +| `Subset` | Fields the Beats runtime sets | Expected is a subset of actual, extra fields allowed | -`tests/unicode.rs` runs every registered transform against seventeen scripts and a set of -degenerate inputs, so multi-byte and malformed text is a first-class case rather than an edge -case discovered in production. +`tests/unicode.rs` runs every registered transform against seventeen samples: scripts, right-to-left text, combining marks, emoji and degenerate inputs such as a byte-order mark mid-string. -## Core, full and the shared broker +## Core and the shared broker -Core is every test, and it runs on every push that triggers CI: the unit tests, the floors above, `tests/broker.rs` and `tests/kafka_tls.rs` against a real broker, and the memory tests (`tests/ndjson_fanout.rs`, `tests/asa_teardown.rs`) held to the deployment contract. No test here is `#[ignore]`d. +Core is every test, and it runs on every push that triggers CI: the unit tests, the floors and scanners above, `tests/broker.rs` and `tests/kafka_tls.rs` against a broker, and the memory tests (`tests/ndjson_fanout.rs`, `tests/asa_teardown.rs`) held to the deployment contract. No test here is `#[ignore]`d. -Nextest is required (`test.rust.nextest`): the memory tests read the process's peak resident set and `tests/config_cascade.rs` reads scalo's process-wide cascade, so each needs a process of its own. +Nextest is required (`test.rust.nextest` in `.hyperi-ci.yaml`): the memory tests read the process's peak resident set and `tests/config_cascade.rs` reads scalo's process-wide cascade, so each needs a process to itself. -The Kafka tests share one broker per run, on the image scalo pins as `test_support::KAFKA_IMAGE`. The first test process starts it through scalo's `shared_kafka`, the rest join it by name, and it is removed when the run's nextest process exits, locally and in CI alike. The two tests that stop the broker or raise its record ceiling start one alone (`start_alone`), and `tests/kafka_tls.rs` starts scalo's `TlsBroker`. Each is removed when its test ends. A live broker named in `KAFKA_BROKERS` is used instead where one answers. Without Docker the tests skip locally and fail in CI. +The Kafka tests share one broker per run, on the image scalo pins as `test_support::KAFKA_IMAGE`. The first test process starts it through scalo's `shared_kafka`, the rest join it by name, and it is removed when the run's nextest process exits, locally and in CI alike. The two tests that stop the broker or raise its record ceiling start one alone (`start_alone`), and `tests/kafka_tls.rs` starts scalo's `TlsBroker`. Each is removed when its test ends. A broker listed in `KAFKA_BROKERS` is used instead where one answers. Without Docker the tests skip locally and fail in CI. The standard these follow is hyperi-ai's `standards/universal/testing.md`. diff --git a/scripts/dfe-schemas.toml b/scripts/dfe-schemas.toml index e18ee2dd5..6691c4b48 100644 --- a/scripts/dfe-schemas.toml +++ b/scripts/dfe-schemas.toml @@ -1,5 +1,3 @@ -# The dfe-schemas wheel src/detect/labels.rs and src/detect/markers.rs are generated from. -# Bump both values together from https://pypi.org/pypi/dfe-schemas//json, then run -# scripts/gen_detect_tables.py. tests/scripts_suite.rs fails the build when the files are stale. +# Bump both values together: CONTRIBUTING.md, "The catalogue and the detect tables". version = "0.2.28" sha256 = "ae42d7c73df2483ee7e5d05eaeb7ffe82153b1ff27d3118a6529b6dc45f203c3" diff --git a/sources.yaml b/sources.yaml index 20cfc636b..a5936859b 100644 --- a/sources.yaml +++ b/sources.yaml @@ -1,48 +1,9 @@ # SPDX-License-Identifier: BUSL-1.1 # Copyright (c) 2026 HYPERI PTY LIMITED # -# Every source this service transforms, and everything about it that its name -# does not tell you. Four places used to carry their own copy of this and drift -# apart: the compat capture, the pipeline vendoring, the regeneration driver and -# the service registry. They all read this file now. -# -# Nothing here is derivable. Fortinet's package is `fortinet_fortigate` while -# its fixtures sit under `fortigate`; one azure package is four of our modules; -# cisco_meraki's pipelines are at `cisco/meraki` and its fixtures at -# `cisco/meraki/logs`. -# -# Keys, per source: -# -# package Integration package directory. -# data_stream Data stream directory within that package. -# pipelines Directory under the vendored pipelines tree holding the copy. -# fixtures Directory under dfe-transform-elastic-dev's fixture tree. -# beats Beats module and fileset carrying the same source, if any. -# intakes Every way this source's payload can reach the service. The -# transform is the same for all of them; only the wrapper -# differs, and `src/envelope.rs` unwraps it. -# framing Receiver intakes only. `line` if the pipeline groks the -# syslog header out of `message`, `body` if a prefixed header -# would corrupt its first field. -# transforms Generated transforms wired into the registry, as -# `filebeat..`. -# -# The three intakes, and what earns each: -# -# beats Beats or Elastic Agent. Always, since every source here has an -# Elastic integration. -# receiver A device pushes the data, so dfe-receiver can take it on any of -# its transports. The integration says so by shipping tcp/udp -# agent streams. -# fetcher Elastic's agent input is a PURE TRANSPORT -- httpjson, cel, -# aws-s3, azure-eventhub, streaming -- so the ingest pipeline does -# all the parsing and dfe-fetcher can obtain the same bytes. These -# are the sources where the agent adds nothing we would lose. - -# The shape of this file, for readers outside this repo: the DFE engine -# offers it as the source catalogue from a release asset and refuses a shape -# it does not know. Bump it when a per-source key is renamed or removed, -# never for a new source or a new key. +# The source catalogue: CONTRIBUTING.md, "The catalogue and the detect tables". + +# Bump on a renamed or removed per-source key, never for a new source or key. catalogue_version: 1 sources: diff --git a/src/envelope.rs b/src/envelope.rs index 2c5556da1..2d83de26f 100644 --- a/src/envelope.rs +++ b/src/envelope.rs @@ -15,7 +15,7 @@ //! [`Envelope::Receiver`] is what dfe-receiver produces, on any of its //! transports -- syslog, gelf, fluent, splunk-hec, otlp, grpc, prometheus, //! netflow, sflow. The syslog arm puts a syslog-shaped line back in `message`, -//! because that is what the transforms grok; the rest pass their payload +//! because that is what the transforms grok. The rest pass their payload //! through with their own field names moved onto the ECS paths those names //! would otherwise shadow. //! @@ -25,7 +25,7 @@ //! `azure-eventhub` -- because there the ingest pipeline does all the parsing //! and fetching the data ourselves loses nothing. //! -//! Which one it is comes from [`detect`], off the first event of each batch. +//! Which one it is comes from [`detect`], read off each event. //! `source.envelope` in the config pins it instead, for the shapes that carry //! no marker at all. @@ -321,7 +321,7 @@ impl Envelope { /// What one event's wrapper turned out to be. /// -/// The family selects the unwrap; the variant names which of that producer's +/// The family selects the unwrap. The variant names which of that producer's /// shapes it is, and is carried so the logs and metrics can report it. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Detected { @@ -375,7 +375,7 @@ pub fn detect(event: &Event) -> Detected { return owned(Envelope::Receiver, "_source"); } // OTLP's generic mode tags `_signal` and the Vector gRPC source tags - // metrics and traces `_vector_type`; neither writes `_source`. + // metrics and traces `_vector_type`. Neither writes `_source`. if keys.contains_key("_signal") { return Detected { family: Envelope::Receiver, @@ -738,7 +738,7 @@ const VARIANT_MAX: usize = 64; /// The value comes off the wire in `_source` or `_source_fetcher` with no /// length and no character restriction, and it reaches a log line and an output /// field unaltered. Non-graphic bytes are dropped rather than escaped, because -/// a transport name is ASCII; a value left with nothing after that is reported +/// a transport name is ASCII. A value left with nothing after that is reported /// as [`UNNAMED`], the same as a non-string. fn safe_variant(raw: &str) -> Cow<'static, str> { let trimmed: String = raw @@ -838,7 +838,7 @@ fn unwrap_fetcher(event: &mut Event) -> crate::Result<()> { /// /// `variant` is what detection returned, which is `_source` for the six /// transports that write one and the marker key for the ones that do not. -/// syslog is the only transport that rebuilds a line; the rest pass their +/// syslog is the only transport that rebuilds a line. The rest pass their /// payload through with their own names moved out of ECS's way. /// /// `_source` itself is READ here and left on the event -- see [`RECEIVER_KEYS`] @@ -1395,7 +1395,7 @@ fn lift_to_ecs(event: &mut Event) -> crate::Result<()> { /// Move each `from` onto its `to`, skipping the fields this event does not /// carry. /// -/// A source that feeds a LATER destination as well is cloned; the last +/// A source that feeds a LATER destination as well is cloned, and the last /// destination for a source takes the value itself. Of the eight pairs only /// `hostname` has two destinations, so seven events' worth of clone per /// receiver event goes away, and every source here is in [`RECEIVER_KEYS`] and diff --git a/tests/compare-policy.yaml b/tests/compare-policy.yaml index 7e75e5162..ba5a44e8d 100644 --- a/tests/compare-policy.yaml +++ b/tests/compare-policy.yaml @@ -1,26 +1,7 @@ -# What matters when comparing DFE output against Elastic's. -# -# Byte equality is not the goal. DFE deliberately does not emit Elastic's -# plumbing, and resolves some enrichment against different databases. This file -# is the single definition of which differences are defects, read by -# scripts/compat.py and by the Rust comparison harness. -# -# Adding an entry hides a difference, so every entry carries a reason and every -# run reports how many fields each rule excluded. -# -# An entry may carry `sources: [name, ...]`, and it then holds for those sources -# alone. Omit it only where the reason is a fact about the FIELD rather than one -# source's quirk -- `event.ingested` is write time everywhere. A quirk left -# unscoped blinds every other source: `event.kind` and `error.message` were -# excluded corpus-wide for a cisco_nexus date failure, so a transform emitting -# `pipeline_error` anywhere else did not show up as a difference at all. -# -# scripts/compat.py does not read `sources` yet and applies every rule globally, -# so its audit is the wider reading of the two. +# What counts as a difference against Elastic's output: docs/compat-policy.md. version: 1 -# Carry no information and vary run to run. Removed at capture, not compared. nondeterministic: - path: _ingest.timestamp why: Written by the simulate API on every call. @@ -66,8 +47,7 @@ nondeterministic: instant gives 3-1788220800000. Wrong on 4 of 7 events and on no other field of theirs, so the id before the dash is not what differs. -# Elastic and Beats plumbing DFE does not emit. Never a defect. Prefix match on -# the dotted path, list indices removed first. +# Prefix match on the dotted path, list indices removed first. not_emitted: - prefix: tags why: Beats and test-harness tagging, plus _tag_on_failure. Not vendor data. @@ -100,8 +80,6 @@ not_emitted: - prefix: data_stream why: Destination routing, decided downstream of the transform. -# Produced by both sides, differently. Debt, not plumbing: each entry should -# eventually go away, and the count is expected to fall rather than grow. known_different: - prefix: source.geo why: DB-IP Lite against MaxMind GeoLite2. @@ -164,10 +142,6 @@ known_different: sources: [bbot] why: The other half of the same split. -# Permanent divergences, each with its reason named -- a divergence without one -# is indistinguishable from a defect of our own. Most are Elasticsearch being -# wrong where we are right. The rest is text only the vendor's own runtime can -# produce, which no amount of work here will reproduce. corrected: - prefix: event.duration sources: [azure] @@ -215,8 +189,6 @@ corrected: we do; JSON carries no Set to tell apart from a List, so this can never be matched. -# ECS fields whose values are sets. Same members in a different sequence is -# equality, not a difference. unordered: - event.category - event.type @@ -229,19 +201,8 @@ unordered: - threat.tactic.name - threat.technique.name - okta.debug_context.debug_data.risk_behaviors - # o365 collects the forwarding addresses into a Java HashSet and lists it, - # so Elastic's order is the hash-bucket walk and carries no meaning. + # o365 lists a Java HashSet, so Elastic's order means nothing. - email.to.address - # symantec_endpoint_security collects both of these into a Java HashSet the - # same way. Its email direction ids 0 then 1 map to unknown then inbound and - # every capture holds ["inbound","unknown"]; six attribute ids go in in - # numeric order and come back [encrypted, hidden, readonly, archive, - # compressed, directory]. Reproducing the bucket walk would bake a - # meaningless order into shipped data to score a test. + # symantec_endpoint_security lists a Java HashSet the same way. - email.direction - file.attributes - -# Per-fixture declarations Elastic already ships are honoured in addition to the -# above, read from each `*-config.yml`: -# dynamic_fields value need only match a regex -# numeric_keyword_fields compare the number as a string From 6b4de90ccad488e835c05ee72d5a05131a5f5292 Mon Sep 17 00:00:00 2001 From: Derek Date: Mon, 16 Nov 2026 14:19:25 +1100 Subject: [PATCH 2/2] docs(elastic): keep the parity corpus out of the docs The docs describe the compare policy without corpus or scoring details. - compat.md covers compare-policy.yaml and two known limits. The scoring diagram and scoring process description are removed. - Remove corpus mentions and detector benchmark figures from parity.md, testing.md, configuration.md, parsers.md, schema-generation.md and CONTRIBUTING.md. - testing.md no longer calls `Exact` the default match mode. MatchMode has no default. - src/envelope.rs doc comments say auto detection runs per event. docs/config-schema.{json,yaml} and docs/deployment-contract.json are regenerated with `generate-artefacts`. --- CONTRIBUTING.md | 2 +- README.md | 2 +- docs/README.md | 2 +- docs/compat-policy.md | 2 +- docs/compat.md | 47 +++-------------------------------- docs/config-schema.json | 4 +-- docs/config-schema.yaml | 6 ++--- docs/configuration.md | 4 +-- docs/deployment-contract.json | 4 +-- docs/parity.md | 4 +-- docs/parsers.md | 2 +- docs/schema-generation.md | 2 +- docs/testing.md | 6 ++--- src/envelope.rs | 12 ++++----- 14 files changed, 29 insertions(+), 70 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9495f12e1..4fc279751 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -192,7 +192,7 @@ Two traps worth knowing before you burn a run on them: - **Never set `CARGO_BUILD_JOBS` by hand.** A hand-set value beats the project's pinning and any build governor on the host. -- **The parity corpus is not here, and must never be added.** `unencumbered.rs` fails when `tests/fixtures/` holds anything but `unencumbered/`, `synthetic/` and the provenance, or any file naming Elastic's source repositories or licence. +- **`tests/fixtures/` holds `unencumbered/` and `synthetic/` and nothing else.** `unencumbered.rs` fails a tree holding anything more, or any file with one of its marker strings. ## Working in this repo diff --git a/README.md b/README.md index 2f7969a30..b8185f1b9 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ The source catalogue, `sources.yaml`, is compiled into the binary and `dfe-trans - [docs/architecture.md](docs/architecture.md): the code map, the envelopes and delivery. - [docs/parity.md](docs/parity.md): what the service promises against Elastic's output. -- [docs/compat.md](docs/compat.md) and [docs/compat-policy.md](docs/compat-policy.md): how parity is scored, and what counts as a difference. +- [docs/compat.md](docs/compat.md) and [docs/compat-policy.md](docs/compat-policy.md): what counts as a difference from Elastic's output, and the known limits. - [docs/configuration.md](docs/configuration.md): the cascade, the env spellings and the sections a file cannot hold. - [docs/enrichment.md](docs/enrichment.md): GeoIP, user agent and community ID. - [docs/README.md](docs/README.md): every page, including parsers, performance and testing. diff --git a/docs/README.md b/docs/README.md index a35366dbe..a4ac38a84 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,7 +11,7 @@ Everything deeper than the [README](../README.md). Start at [architecture.md](ar | [enrichment.md](enrichment.md) | GeoIP, user agent and community ID, the three runtime enrichers | | [performance.md](performance.md) | SIMD, borrowed values, the release profile, the benchmarks and the key dependencies | | [testing.md](testing.md) | The floors, the scanners and the shared Kafka broker | -| [compat.md](compat.md) | How a transform is scored against the real Elastic engine, and the known limits | +| [compat.md](compat.md) | The compare policy this repository holds, and the known limits | | [compat-policy.md](compat-policy.md) | What counts as a difference: the `compare-policy.yaml` sections and the divergences kept on purpose | | [schema-generation.md](schema-generation.md) | The design for publishing the Elastic-side field knowledge to dfe-schemas | | [shared-crates.md](shared-crates.md) | What dfe-transform-splack took a copy of, and what extracting the common half would take | diff --git a/docs/compat-policy.md b/docs/compat-policy.md index ae6cca4f6..14dd5fa20 100644 --- a/docs/compat-policy.md +++ b/docs/compat-policy.md @@ -1,6 +1,6 @@ # compat policy -- what counts as a difference -`tests/compare-policy.yaml` is the one definition of which differences between our output and Elasticsearch's are defects. The comparison harness reads it through `crates/dfe-runtime/src/testutil/policy.rs`, which is why it lives here beside the runtime. How the comparison runs is [compat.md](compat.md), and what parity means is [parity.md](parity.md). +`tests/compare-policy.yaml` is the one definition of which differences between our output and Elasticsearch's are defects. The comparison harness reads it through `crates/dfe-runtime/src/testutil/policy.rs`, which is why it lives here beside the runtime. The known limits are in [compat.md](compat.md), and what parity means is [parity.md](parity.md). Byte equality is not the goal. Every rule has a `why:` that says why the difference is not a defect, because adding a rule hides that difference from every run. The section a rule lands in decides what follows: diff --git a/docs/compat.md b/docs/compat.md index da6d1a30c..4f00b4f8c 100644 --- a/docs/compat.md +++ b/docs/compat.md @@ -1,47 +1,8 @@ -# compat -- measuring against the Elastic engine +# compat -- comparing output against Elastic's -compat runs a source's sample lines through that source's ingest pipeline in a throwaway Elasticsearch container and keeps the documents the engine returns. The corpus test scores each transform against them field by field. The corpus and its test are not part of this repository. - -What a difference means, and which differences are kept on purpose, is [compat-policy.md](compat-policy.md). - -Elasticsearch's output is the reference, and the score is per source. - -```mermaid -flowchart LR - accTitle: L4 Detail -- How a transform is scored - accDescr: Sample lines go through the source's ingest pipeline in Elasticsearch, which returns confirmed documents. The same lines go through the transform, and the two outputs are compared field by field under the compare policy. - - samples[("Sample lines")] - - subgraph engines["Both engines"] - es["Elasticsearch
ingest pipeline"] - transform["The transform"] - end - - compare["Compare
per field"] - score(["Per-source
score"]) - - samples --> engines - engines --> compare - compare --> score - - classDef default fill:#F8FAFC,stroke:#64748B,stroke-width:1px,color:#0F172A - classDef core fill:#DCEBF7,stroke:#0072B2,stroke-width:2px,color:#0B2A3F - classDef store fill:#DDF3EC,stroke:#00785A,stroke-width:1px,color:#0B3327 - classDef ext fill:#EEF0F2,stroke:#6B7280,stroke-width:1px,stroke-dasharray:4 3,color:#1F2937 - classDef group fill:none,stroke:#64748B,stroke-width:1px,stroke-dasharray:2 4 - class samples store - class es ext - class transform core - class engines group -``` - -*Figure 1. L4 Detail -- How a transform is scored. The dashed box is external.* Up: [L3 Components -- dfe-transform-elastic](architecture.md#four-steps-between-the-producers-and-the-sink). - -The comparison reads `tests/compare-policy.yaml` through `dfe_runtime::testutil::policy`, so the rules live here beside the runtime. +Parity is not scored in this repository. What this repository holds is `tests/compare-policy.yaml`, which `dfe_runtime::testutil::policy` reads to decide which differences between this service's output and Elasticsearch's are defects. [compat-policy.md](compat-policy.md) covers its sections and the divergences kept on purpose. ## Known limits -- o365 and panw are incomplete. Part of their transformation runs inside Filebeat before Elasticsearch sees the event: o365 has a JavaScript processor chain, and panw applies `decode_csv_fields` to `message`. The simulate API reaches neither, so for those two the confirmed output covers the Elasticsearch half. -- GeoIP is not compared. The corpus test turns its GeoIP enricher off and excludes GeoIP-derived fields. An event can still have geo fields its source supplied. -- The engine never ships. Elasticsearch is a development dependency of the measurement, never part of the service. +- o365 and panw depend on processing inside Filebeat, before the ingest pipeline runs: o365 has a JavaScript processor chain, and panw applies `decode_csv_fields` to `message`. The transforms implement the ingest pipeline half. +- GeoIP resolves against DB-IP Lite, not MaxMind GeoLite2, so geo fields can differ from Elastic's, and `compare-policy.yaml` excludes them. diff --git a/docs/config-schema.json b/docs/config-schema.json index 2ddd824ab..6d7e4f356 100644 --- a/docs/config-schema.json +++ b/docs/config-schema.json @@ -242,10 +242,10 @@ ] }, "EnvelopeSetting": { - "description": "What the operator said about the envelope.\n\n`auto` is the default: the family is read off the first event of each batch,\nso a deployment that changes producer needs no config change. Naming one\npins it, which is the way through for a shape that carries no marker.", + "description": "What the operator said about the envelope.\n\n`auto` is the default: the family is read off each event, so a deployment\nthat changes producer needs no config change. Naming one\npins it, which is the way through for a shape that carries no marker.", "oneOf": [ { - "description": "Detect the family from each batch's first event.", + "description": "Detect the family from each event.", "type": "string", "const": "auto" }, diff --git a/docs/config-schema.yaml b/docs/config-schema.yaml index c02a22cc5..6a4e5156d 100644 --- a/docs/config-schema.yaml +++ b/docs/config-schema.yaml @@ -250,11 +250,11 @@ $defs: description: |- What the operator said about the envelope. - `auto` is the default: the family is read off the first event of each batch, - so a deployment that changes producer needs no config change. Naming one + `auto` is the default: the family is read off each event, so a deployment + that changes producer needs no config change. Naming one pins it, which is the way through for a shape that carries no marker. oneOf: - - description: Detect the family from each batch's first event. + - description: Detect the family from each event. type: string const: auto - description: Beats, Elastic Agent, or anything passed through unaltered. diff --git a/docs/configuration.md b/docs/configuration.md index 05b764db7..033fe45d9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -46,9 +46,7 @@ Every `source` and `sink` key may be left out. `source.name`, `source.topics`, ` Fetcher events hold the provider's payload, so they count as unmatched. Three counters report the split: `events_detected_total{source}`, `events_unmatched_total` and `events_fallback_total`. -Detection cost about 2,080 ns an event (481K events a second on one core) over 20,000 syslog lines averaging 718 bytes, on one AMD EPYC 7302 core with the bench profile, so a 20,000-event batch spends about 42 ms on it. `cargo bench --bench detect` reads the parity corpus, and skips without it. - -Run it on the receiver's `_raw` capture where you can. The receiver splits each line with `syslog_loose`, and a marker that straddled the split (`Pleasant Password Server:`, an `amp:` tag) is gone from the parsed fields. Over 8,885 test events of detectable sources the detector identified 8,243 from the raw line and 8,004 from the parsed fields. +Run it on the receiver's `_raw` capture where you can. The receiver splits each line with `syslog_loose`, and a marker that straddled the split (`Pleasant Password Server:`, an `amp:` tag) is gone from the parsed fields. `scripts/replay_syslog.py` feeds an agentless stack from the committed fixtures under `tests/fixtures/unencumbered/`, standard library only. `--sources` takes catalogue labels from the table the detector stamps, and `--list` shows the ones with fixtures. A syslog header is added to a fixture line that lacks one, and the `%` the ASA fixtures dropped is put back. UDP sends one datagram per line and TCP is newline framed. There is no `http://` form, because dfe-receiver's `/ingest` takes JSON and not syslog. diff --git a/docs/deployment-contract.json b/docs/deployment-contract.json index 0f5b7e690..37c33aafe 100644 --- a/docs/deployment-contract.json +++ b/docs/deployment-contract.json @@ -393,10 +393,10 @@ ] }, "EnvelopeSetting": { - "description": "What the operator said about the envelope.\n\n`auto` is the default: the family is read off the first event of each batch,\nso a deployment that changes producer needs no config change. Naming one\npins it, which is the way through for a shape that carries no marker.", + "description": "What the operator said about the envelope.\n\n`auto` is the default: the family is read off each event, so a deployment\nthat changes producer needs no config change. Naming one\npins it, which is the way through for a shape that carries no marker.", "oneOf": [ { - "description": "Detect the family from each batch's first event.", + "description": "Detect the family from each event.", "type": "string", "const": "auto" }, diff --git a/docs/parity.md b/docs/parity.md index 9a492b6e3..69efad950 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -1,10 +1,10 @@ # Parity -The goal is the same ECS output Elastic's ingest pipeline gives for each supported Beats and Elastic Agent integration. Parity is scored per source, and some sources match fully and others do not yet. The code map is [architecture.md](architecture.md), and how the scoring works is [compat.md](compat.md) and [compat-policy.md](compat-policy.md). +The goal is the same ECS output Elastic's ingest pipeline gives for each supported Beats and Elastic Agent integration. Parity is scored per source, and some sources match fully and others do not yet. The code map is [architecture.md](architecture.md), and what counts as a difference is [compat-policy.md](compat-policy.md). This is an independent transform service, not a port of Elasticsearch. Data sent through DFE should get the field mapping, type coercion and enrichment it would get going straight to Elasticsearch. Where Elasticsearch's output is wrong, the service emits the correct value and records the divergence ([compat-policy.md](compat-policy.md#correct-beats-bug-compatible)). -Parity is checked against documents a running Elasticsearch produced, compared per field ([compat.md](compat.md)). That corpus and its test are not in this repository. [compat-policy.md](compat-policy.md) covers `tests/compare-policy.yaml`, which decides what counts as a difference. +[compat-policy.md](compat-policy.md) covers `tests/compare-policy.yaml`, which decides what counts as a difference. This repository's floors run on every clone: `integration/`, `untested_sources.rs` and `unencumbered.rs` in `crates/dfe-transforms/tests/`, over the licence-clean samples in `tests/fixtures/unencumbered/` and the synthetic samples in `tests/fixtures/synthetic/`. They assert no panic, no new errors and enrichment, never parity ([testing.md](testing.md)). diff --git a/docs/parsers.md b/docs/parsers.md index ac39684ec..f83b51745 100644 --- a/docs/parsers.md +++ b/docs/parsers.md @@ -125,4 +125,4 @@ A script the runtime cannot execute is skipped rather than failing the event, so A generated call site holds no script. The generator resolves the dispatch and emits `compiled_painless!` with the script's hash and that plan as JSON, plus the normalised text where a bound runner still reads it. The runtime looks bespoke runners up by the hash, and the counters key each script by it. -Two measures, and the second alone says how far the matchers reach. `crates/dfe-transforms/tests/painless_binding.rs` reads every plan out of the generated tree and asks what it binds to, which says a script is claimed but not that the runner writes anything. The parity corpus run counts how many scripts ran and how many were skipped, and those figures are run outputs, never constants. A pattern can match a script statically and decline at run time, which reads as covered everywhere except that run. +`crates/dfe-transforms/tests/painless_binding.rs` reads every plan out of the generated tree and asks what it binds to, which says a script is claimed but not that the runner writes anything. The two counters say what ran on live traffic. A pattern can match a script statically and decline at run time, which reads as covered everywhere except those counters. diff --git a/docs/schema-generation.md b/docs/schema-generation.md index 02fdf6f15..d0cb0357d 100644 --- a/docs/schema-generation.md +++ b/docs/schema-generation.md @@ -5,7 +5,7 @@ Status: partly built, and none of it in this repository. The ECS meta schema in dfe-schemas, `meta/elastic/ecs.yaml`, is generated. Per-source package schemas, the evidence each column records and the Sigma field maps below are design, not code. [#16](https://github.com/hyperi-io/dfe-transform-elastic/issues/16) tracks generating schemas for every Beat and Elastic Agent source. -The Elastic side of DFE needs one place that knows, per data stream, which fields come out. The ingest pipelines declare intent and the parity corpus records what Elasticsearch emitted, so the two meet where the transforms are built. That knowledge goes to [dfe-schemas](https://github.com/hyperi-io/dfe-schemas) as ClickHouse schemas, by pull request. dfe-engine defines how schemas work (`SchemaLoader`, `SchemaBuilderV2`, `MetaSchema`), imports Elastic index templates, and builds the sigma, ecs and cim views in `fieldmap/`. +The Elastic side of DFE needs one place that knows, per data stream, which fields come out. The ingest pipelines declare intent and the transforms produce the output, so the two meet where the transforms are built. That knowledge goes to [dfe-schemas](https://github.com/hyperi-io/dfe-schemas) as ClickHouse schemas, by pull request. dfe-engine defines how schemas work (`SchemaLoader`, `SchemaBuilderV2`, `MetaSchema`), imports Elastic index templates, and builds the sigma, ecs and cim views in `fieldmap/`. ## The design diff --git a/docs/testing.md b/docs/testing.md index b4eb431f5..99ae3b6ed 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1,6 +1,6 @@ # Testing -Parity is scored by a corpus and a test that are not in this repository ([compat.md](compat.md)). The two tracks here, the floors and the scanners, run on every clone, and neither measures parity. The code map is [architecture.md](architecture.md), and what parity means is [parity.md](parity.md). +The two test tracks, the floors and the scanners, run on every clone, and neither measures parity. The code map is [architecture.md](architecture.md), and what parity means is [parity.md](parity.md). The floors read the committed fixtures, and the scanners read the generated tree against the ratchet file. @@ -49,13 +49,13 @@ cargo test -p dfe-runtime --all-features --lib --test date_repro cargo test --no-default-features --test envelope --test envelope_detection ``` -`crates/dfe-transforms/tests/unencumbered.rs` fails when `tests/fixtures/` holds anything but `unencumbered/`, `synthetic/` and the provenance, or when any file there contains one of three marker strings: the paths of Elastic's two source repositories, or its licence's name. +`crates/dfe-transforms/tests/unencumbered.rs` fails when `tests/fixtures/` holds anything but `unencumbered/`, `synthetic/` and the provenance, or when any file there contains one of its `MARKERS` strings. The test diff helper's `MatchMode` is defined in `crates/dfe-runtime/src/testutil/diff.rs`: | Mode | Use | Behaviour | |---|---|---| -| `Exact` | Default | Field-for-field JSON match | +| `Exact` | Strict comparison | Field-for-field JSON match | | `Semantic` | Against Elastic's output | Like `Exact`, skipping every path the compare policy excludes | | `Subset` | Fields the Beats runtime sets | Expected is a subset of actual, extra fields allowed | diff --git a/src/envelope.rs b/src/envelope.rs index 2d83de26f..c19f7974b 100644 --- a/src/envelope.rs +++ b/src/envelope.rs @@ -208,8 +208,8 @@ const DEFAULT_SEVERITY: u8 = 5; /// What the operator said about the envelope. /// -/// `auto` is the default: the family is read off the first event of each batch, -/// so a deployment that changes producer needs no config change. Naming one +/// `auto` is the default: the family is read off each event, so a deployment +/// that changes producer needs no config change. Naming one /// pins it, which is the way through for a shape that carries no marker. #[derive( Debug, @@ -224,7 +224,7 @@ const DEFAULT_SEVERITY: u8 = 5; )] #[serde(rename_all = "snake_case")] pub enum EnvelopeSetting { - /// Detect the family from each batch's first event. + /// Detect the family from each event. #[default] Auto, /// Beats, Elastic Agent, or anything passed through unaltered. @@ -665,12 +665,12 @@ fn ingest_time(event: &Event) -> Option { )) } -/// Which envelope a batch will be unwrapped with, and what was odd about it. +/// Which envelope an event will be unwrapped with, and what was odd about it. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Resolution { /// How to unwrap, and what to stamp back on. pub delivery: Delivery, - /// What the batch's first event looked like, when there was one. + /// What the event looked like, when there was one. pub detected: Option, /// The operator pinned a family the events do not look like. pub contradicted: bool, @@ -678,7 +678,7 @@ pub struct Resolution { pub unaccepted: bool, } -/// Decide how a batch arrived, from its first event. +/// Decide how an event arrived. /// /// A pinned setting wins over what the events look like, because the whole /// point of pinning is to name a shape detection cannot see. A contradiction is