diff --git a/Cargo.lock b/Cargo.lock index 7e29a2ed4..273392d95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2049,6 +2049,32 @@ dependencies = [ "tokio", ] +[[package]] +name = "lading_proptest" +version = "0.1.0" +dependencies = [ + "anyhow", + "bollard", + "bytes", + "flate2", + "http-body-util", + "hyper", + "hyper-util", + "nix 0.30.1", + "proptest", + "rustc-hash", + "serde", + "serde_json", + "serde_yaml", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tracing", + "tracing-subscriber", + "uuid", + "zstd", +] + [[package]] name = "lazy_static" version = "1.5.0" diff --git a/Cargo.toml b/Cargo.toml index c5a4d94a5..a7577d22f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] resolver = "2" members = [ + "integration/lading_proptest", "integration/sheepdog", "integration/ducks", "lading", diff --git a/integration/lading_proptest/ADAPTIVE_SAMPLING.md b/integration/lading_proptest/ADAPTIVE_SAMPLING.md new file mode 100644 index 000000000..3bdf0879c --- /dev/null +++ b/integration/lading_proptest/ADAPTIVE_SAMPLING.md @@ -0,0 +1,248 @@ +# Adaptive Sampling Testing + +Testing the Datadog Agent's experimental adaptive sampling feature — a per-pattern credit-based rate limiter that groups structurally similar logs and limits throughput per pattern. + +## How Adaptive Sampling Works + +### Overview + +The agent tokenizes each log line into a structural pattern (classifying bytes as digits, punctuation, character runs, timestamps, etc. — but not the actual values). Structurally similar lines share a pattern, and each pattern has its own independent credit pool that controls how many logs of that pattern are allowed through. + +### Pipeline Position + +Adaptive sampling runs **after** multiline aggregation and truncation. The sampler sees the final assembled message content: + +``` +Tailer → Decoder → Multiline Aggregation → Truncation → ADAPTIVE SAMPLING → Sender +``` + +### The Credit Model + +Each pattern tracks credits (a float counter): + +1. **New pattern discovered:** starts with `burst_size` credits (e.g., 10) +2. **Log emitted:** costs 1 credit +3. **Log dropped:** when credits < 1.0; the drop count is tracked +4. **Credit refill:** `elapsed_seconds × rate_limit` credits added since last seen, capped at `burst_size` +5. **Sampled count tag:** when a log is finally emitted after drops, it gets tagged `adaptive_sampler_sampled_count:` where N = number dropped since last emit + +### Pattern Matching + +Two logs share a pattern if their token sequences match at ≥ `match_threshold` (default 90%) similarity. Tokenization converts log content into abstract categories: + +- `"2024-01-15 INFO request id=123"` → `[D4 Dash D2 Dash D2 Space Info Space C7 Space Id Equal D3]` +- `"2024-01-16 INFO request id=456"` → same token sequence → same pattern +- `"2024-01-15 ERROR connection failed"` → different tokens → different pattern + +### Important Log Protection + +When `protect_important_logs=true` (default), lines containing keywords like `ERROR`, `FATAL`, `PANIC`, `WARN`, `CRITICAL`, `EXCEPTION`, `CRASH`, `FAILURE`, `DEADLOCK`, `TIMEOUT` bypass sampling entirely. No pattern entry is created and the log is always emitted. + +## Configuration + +All settings are under `logs_config.experimental_adaptive_sampling.*` in `datadog.yaml` or via environment variables prefixed with `DD_LOGS_CONFIG_EXPERIMENTAL_ADAPTIVE_SAMPLING_`. + +| Setting | Default | Description | +|---|---|---| +| `enabled` | `false` | Master on/off switch | +| `burst_size` | `1000.0` | Initial credits per pattern and credit cap | +| `rate_limit` | `1.0` | Credits refilled per second per pattern | +| `max_patterns` | `1000` | Max distinct patterns tracked; least-frequent evicted when full | +| `match_threshold` | `0.9` | Token similarity required to share a pattern (0-1] | +| `tokenizer_max_input_bytes` | `2048` | Bytes of each log tokenized for pattern matching | +| `protect_important_logs` | `true` | Bypass sampling for critical severity keywords | + +### Per-Source Override + +```yaml +logs: + - type: file + path: /var/log/app.log + experimental_adaptive_sampling: + enabled: true +``` + +### Test-Friendly Values + +For testing we use small values to make behavior observable in short timeframes: + +| Setting | Test Value | Why | +|---|---|---| +| `burst_size` | `10` | Don't need 1000 lines to exhaust the burst | +| `rate_limit` | `2.0` | Refills fast enough to observe in 5-10 seconds | + +## Current Test Coverage + +### Credit Model (implemented) + +A model-based property test that simulates the credit state machine and compares predictions against agent output. The model tracks credits per the algorithm above (start with `burst_size`, consume 1 per emit, refill at `rate_limit` per second, cap at `burst_size`). It is intentionally simpler than the agent — no tokenizer, no pattern table, no hot-path optimization. + +**What it tests:** +- Initial burst passes through (first `burst_size` lines delivered) +- Rate limiting after burst exhaustion (excess lines dropped) +- Credit refill over time (sleep N seconds, ~N×rate_limit more lines delivered) +- Sampled count tagging (`adaptive_sampler_sampled_count:` tag on first post-drop emit) +- Variable-length action sequences (write, sleep, write, sleep, write, ...) + +**Current status:** Passing at tolerance=0 (exact model match) across 5 cases with variable-length action sequences of 3-7 steps. + +**Limitations:** +- Single pattern only (all lines have identical token structure via zero-padded numeric IDs) +- Fixed config (`burst_size=10`, `rate_limit=2.0`) +- Does not test tokenizer grouping decisions + +### Key Discovery: UUID Tokenization + +UUIDs contain varying mixes of hex digits and letters (e.g., `0c7db8bb` vs `3c7c10de`). The agent's tokenizer classifies these as different token sequences, causing lines with different UUIDs to be treated as different patterns. Fix: use zero-padded numeric IDs (`0000000001`, `0000000002`) which always produce identical token sequences. + +## Testing Strategy Discussion + +There are two separable concerns in adaptive sampling: + +### 1. Credit Arithmetic (the model) + +Given a known set of patterns and a sequence of writes/sleeps, does the agent correctly apply the credit rules? This is what the current model tests. It needs to scale to: + +- **Varying `burst_size` and `rate_limit`** — proptest generates different config values per case, model adapts +- **Multiple patterns** — model tracks independent credit pools per pattern +- **Credit cap behavior** — long sleep beyond burst_size/rate_limit shouldn't accumulate unbounded credits + +### 2. Pattern Grouping (the tokenizer) + +Given two log lines and a config, does the agent correctly decide "same pattern" or "different pattern"? This is affected by three config values: + +- **`match_threshold`** — what percentage of tokens must match +- **`tokenizer_max_input_bytes`** — how many bytes are tokenized +- **`protect_important_logs`** — whether severity keywords bypass everything + +#### The Modeling Problem + +We can't build a simplified tokenizer model because the agent's tokenizer has special-case keyword recognition (months, timezones, log levels like INFO/ERROR). A simplified model that doesn't know these keywords would constantly disagree with the agent, producing false positive failures. + +#### Black-Box Pattern Probing + +Instead of modeling the tokenizer, probe it directly. With `burst_size=1`: + +1. Send line A → always arrives (new pattern, 1 credit) +2. Send line B → if it arrives, agent sees A and B as different patterns. If dropped, same pattern. + +This is a pure black-box test of the tokenizer's grouping decisions. No model needed — just send pairs and observe. + +**Properties to assert on probing results:** +- **Symmetry:** if A matches B, then B matches A +- **Consistency:** the same pair always produces the same decision +- **Structural sensitivity:** changing structure (adding fields, changing digit count) should produce different patterns +- **Value insensitivity:** changing just values (different number, different word of same length) should keep the same pattern + +These properties don't need to know the tokenizer's rules — they test that the tokenizer behaves *reasonably* and *consistently*, and discover edge cases mechanically. + +#### Connecting the Two: Probing Drives the Model + +The pattern probing tests can generate knowledge about what the agent considers "same pattern" vs "different pattern." This knowledge can then drive the credit model tests: + +1. Probing discovers: "lines with format X and format Y are considered the same pattern by the agent at match_threshold=0.9" +2. Credit model test generates a mix of X and Y lines, knows they share credits, asserts credit behavior correctly + +This means the credit model doesn't need to assume pattern groupings — it learns them from the probing tests. The two test layers compose: + +``` +Pattern Probing (black-box) + → discovers grouping decisions + → feeds into ↓ + +Credit Model (model-based) + → uses discovered groupings + → asserts credit arithmetic +``` + +This avoids both problems: no simplified tokenizer to maintain, and the credit model doesn't bake in assumptions about what the tokenizer does. + +**Open question:** How to operationalize this composition. Options: +- Run probing first, persist results, credit model reads them (complex, stateful) +- Run probing inline at the start of each credit test to discover groupings on the fly (simpler, but adds container time) +- Define pattern groups empirically based on probing results and hardcode them (simplest, but needs updating if tokenizer changes) + +## Planned Work (by priority — TBD with team) + +### P1: Vary credit config + +Make `burst_size` and `rate_limit` proptest-generated instead of constants. The model already accepts these as parameters. This tests that credit arithmetic works across many config combos without any code changes to the model. + +Lower bound on `burst_size` to avoid degenerate cases (e.g., ≥ 3). + +### P1: Multi-pattern credit testing + +Extend the model to track credits per pattern. Generate lines in two+ structurally distinct groups (e.g., all-digit content vs all-letter content). Assert each group gets independent credits. Requires extending `SamplingAction::Write` with pattern tags. + +### P2: Pattern probing scenario + +Build the `burst_size=1` probing mechanism. Generate diverse line pairs, observe same/different decisions. Assert behavioral properties (symmetry, consistency, structural sensitivity, value insensitivity). + +### P2: Important log bypass + +Exhaust credits, then send a line with ERROR/FATAL. Assert it arrives. Test with `protect_important_logs=true` and `false` to verify the flag works. + +### P3: Pattern eviction + +Fill pattern table to `max_patterns`, introduce a new pattern. Verify least-frequent is evicted. Requires generating many distinct patterns. + +### P3: Probing-driven credit tests + +Use probing results to discover pattern groupings, then feed those groupings into multi-pattern credit tests. The most ambitious goal — composes both test layers. + +### P3: match_threshold boundary testing + +Generate pairs of lines with known structural similarity. Vary `match_threshold` and verify the grouping decision flips at the expected point. + +### P3: tokenizer_max_input_bytes testing + +Generate lines identical in the first N bytes but different after. Vary `tokenizer_max_input_bytes` and verify the agent only considers the configured prefix. + +### P4: Interaction with multiline + +Send multiline entries that share a pattern. Verify sampling operates on the aggregated message, not individual raw lines. + +### P3: Agent downtime / ingest-time sampling + +Model the scenario where a log pattern that is never sampled under normal operation (written slowly enough that credits refill between lines) gets sampled after agent downtime. During downtime, lines accumulate in the file. When the agent restarts and reads the backlog, it sees them as a burst and samples them. This demonstrates that sampling is based on ingest time, not write time. + +Requires: agent stop/restart within a test case (new orchestrator capability), or a simpler simulation where lines that would have been written slowly are instead written all at once (modeling what the agent sees post-restart). Open questions around file offset tracking on restart and whether to demonstrate normal-operation-no-sampling first vs just the post-downtime burst. + +### P4: FUSE-based timing control + +Use lading's `logrotate_fs` to control exactly when lines appear, removing file tailing latency. Would allow tighter credit refill assertions and testing of sub-second timing behavior. + +## Implementation Notes + +### Action Sequences + +The orchestrator supports variable-length action sequences via `run_action_sequence`. Each case gets a proptest-generated sequence of `Write(N)` and `Sleep(N)` steps. The model walks the same sequence and predicts which lines should be delivered. + +### Temp Directory Contents + +Each test case preserves (with `LADING_KEEP_TEMP=1`): + +``` +lading_proptest_XXXXXX/ + config/datadog.yaml + config/conf.d/proptest.d/conf.yaml + logs/proptest.log # raw input + output.json # full intake payloads + output_messages.txt # one message per line with byte count + summary.txt # action sequence, counts, drops +``` + +### Line ID Strategy + +Lines use zero-padded numeric IDs (`0000000001`) instead of UUIDs. This ensures the agent's tokenizer produces identical token sequences for all lines within a pattern group. UUIDs contain varying hex digit/letter mixes that the tokenizer classifies differently. + +## Agent Source References + +| File | Purpose | +|---|---| +| `pkg/logs/internal/decoder/preprocessor/sampler.go` | Core sampling algorithm | +| `pkg/logs/internal/decoder/preprocessor/tokens.go` | Token types and matching | +| `pkg/logs/internal/decoder/preprocessor/tokenizer.go` | Log line tokenization | +| `pkg/logs/internal/decoder/preprocessor/sampler_test.go` | Unit tests (89 tests) | +| `pkg/config/setup/common_settings.go` | Configuration defaults | +| `pkg/logs/internal/decoder/preprocessor/adaptive_sampler.allium` | Formal specification | diff --git a/integration/lading_proptest/Cargo.toml b/integration/lading_proptest/Cargo.toml new file mode 100644 index 000000000..8f0bab09c --- /dev/null +++ b/integration/lading_proptest/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "lading_proptest" +version = "0.1.0" +edition = "2024" +description = "Property-based integration testing for the Datadog Agent logs pipeline" +authors = ["Single Machine Performance Team"] +license = "MIT" +publish = false + +[dependencies] +anyhow = { workspace = true } +bollard = { version = "0.19" } +bytes = { workspace = true } +flate2 = { version = "1.1.2" } +http-body-util = { workspace = true } +hyper = { workspace = true, features = ["server", "http1"] } +hyper-util = { workspace = true, features = ["http1", "server-auto", "tokio"] } +nix = { version = "0.30", default-features = false, features = ["signal"] } +proptest = { workspace = true } +rustc-hash = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +serde_yaml = { workspace = true } +tempfile = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = [ + "fs", + "io-util", + "macros", + "net", + "process", + "rt-multi-thread", + "signal", + "sync", + "time", +] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter"] } +uuid = { workspace = true, features = ["v4"] } +zstd = { version = "0.13.3" } + +[lints] +workspace = true diff --git a/integration/lading_proptest/MIGRATION.md b/integration/lading_proptest/MIGRATION.md new file mode 100644 index 000000000..05c9ee5f3 --- /dev/null +++ b/integration/lading_proptest/MIGRATION.md @@ -0,0 +1,158 @@ +# Migration: Adaptive Sampling Testing → datadog-agent + +This document tracks the migration of e2e property testing for adaptive sampling from the `lading` repo prototype to the `datadog-agent` repo, where it will live alongside the implementation it tests. + +## Background + +We built `lading_proptest` as a prototype in the lading repo to prove that property-based e2e testing of the Datadog Agent's logs pipeline works. It does — we have passing tests for truncation, multiline aggregation (timestamp, JSON, mixed), and adaptive sampling with a model-based property checker at tolerance=0. + +The next phase moves this work to `datadog-agent` where it belongs. + +## What We Built (lading_proptest prototype) + +### Infrastructure +- **Fake intake server** (Rust, hyper) — accepts agent HTTP log payloads, decompresses gzip/zstd, parses JSON, stores entries in memory for assertion +- **Agent orchestrator** — starts/stops Docker containers or local binaries, manages config, temp dirs, timing +- **Action sequence executor** — write logs, sleep, write more — enables timing-dependent tests +- **Output dumping** — `output.json`, `output_messages.txt`, `summary.txt` per case for human inspection + +### Scenarios +- **Truncation** — all 5 log formats, property: agent correctly truncates at `max_message_size_bytes` with `...TRUNCATED...` marker +- **Timestamp multiline** — 3 formats (TimestampPrefixed, Syslog5424, ApacheCommon), property: continuations merged with correct header +- **JSON multiline** — 11 structural variants (flat, nested, deep, arrays, unicode, escapes, etc.), property: compacted JSON preserves all fields. Discovered top-level array limitation mechanically. +- **Mixed multiline** — all formats interleaved, property: format transitions don't break aggregation +- **Adaptive sampling** — model-based credit state machine, variable-length action sequences, tolerance=0 exact matching + +### Key Ideas Worth Preserving +1. **UUID markers for correlation** — embed a unique ID in each log line so we can match inputs to outputs. For adaptive sampling, use zero-padded numeric IDs (UUIDs have variable hex/letter mixes that the tokenizer treats as different patterns). +2. **Model-based property testing** — define a simplified model of expected behavior, simulate both model and real system on same inputs, compare. The model is intentionally simpler than the implementation. +3. **Action sequences** — write/sleep/write enables timing-dependent tests (credit refill, burst exhaustion). +4. **Structural diversity** — proptest generates diverse input structures (JSON variants, format mixes, variable sequence lengths) to mechanically discover edge cases. +5. **`summary.txt`** — human-readable test case summary showing the action sequence, line counts, and results. Critical for debugging and demos. +6. **Temp dir preservation** — on failure (always) and on success (with `LADING_KEEP_TEMP=1`), preserve all artifacts for inspection. + +### Key Discoveries +- Agent's JSON aggregator doesn't support top-level arrays (`[{...}]`) — discovered mechanically by generating diverse JSON structures +- UUIDs have variable hex digit/letter ratios that the tokenizer classifies as different patterns — discovered when adaptive sampling didn't trigger +- Agent appends `...TRUNCATED...` (15 bytes) to truncated head chunks, splits overflow into separate tail entries +- Aggregated multiline messages use literal `\n` (two chars `\` and `n`), not actual newline byte + +## What to Reuse from datadog-agent + +The agent repo already has infrastructure we should use instead of rebuilding: + +### `test/fakeintake/` +Go-based fake intake server that already handles: +- POST to `/api/v2/logs` and `/v1/input` +- Automatic gzip/zstd decompression +- JSON log payload parsing +- Payload retrieval via `/fakeintake/payloads?endpoint=` +- Payload flushing via `/fakeintake/flushPayloads` + +This replaces our Rust intake server. It's more mature and maintained by the agent team. + +### `test/regression/` +Existing regression test infrastructure using lading configs. We may be able to integrate with or extend this. + +### Agent CI infrastructure +Existing Docker-based test workflows we can hook into. + +## What to Build New in datadog-agent + +### E2E Test Harness +A test runner (Go or Rust, TBD) that: +1. Starts fakeintake +2. Configures and starts the agent (container or local binary) +3. Executes action sequences (write log files, sleep) +4. Collects payloads from fakeintake +5. Runs property assertions +6. Outputs human-readable results + +### Property Framework +Composable property checks against fakeintake payloads. The model-based approach for adaptive sampling, plus structural checks for truncation and multiline. + +### Allium Spec Integration (future) +Parse the `.allium` spec to derive: +- State invariants → post-condition assertions after each action +- Behavioral properties → end-to-end property checks +- Operation rules → action sequence generation guidance + +This is the long-term goal — when the spec evolves, tests evolve automatically. + +## Migration Plan + +### Phase 1: Allium Spec Cleanup (current focus) +- Complete the in-progress allium work (pattern isolation, log emission, credit recovery, table ordering) +- Ensure spec is comprehensive and maintained alongside code + +### Phase 2: Unit Test Expansion +- Inspect existing 19 unit tests in `sampler_test.go` +- Consider converting to proptest-style (Go equivalent: `rapid` or `gopter`) for broader input coverage +- Add tests for spec properties not currently covered: + - CreditRecovery + - Isolation invariant + - TableOrdered (stable sort) + - LogEmission contract (tag format/timing) + - Config validation + +### Phase 3: E2E Test Integration +- Build e2e test harness in datadog-agent repo +- Reuse fakeintake, agent container/binary infrastructure +- Port adaptive sampling model-based tests +- Port truncation and multiline tests if valuable +- Wire into CI as a separate slow test suite (nightly or merge-gate) + +### Phase 4: Spec-Driven Testing (aspirational) +- Parse allium spec +- Auto-generate property assertions from invariants +- Auto-generate action sequences from operation rules +- Tests evolve when spec evolves + +## Architecture in datadog-agent + +``` +datadog-agent/ + pkg/logs/internal/decoder/preprocessor/ + adaptive_sampler.allium ← spec (source of truth) + sampler.go ← implementation + sampler_test.go ← unit tests (fast, every commit) + sampler_proptest_test.go ← property tests (fast, every commit) + test/ + e2e/ ← or similar location + adaptive_sampling/ + model.go ← credit state machine model + properties.go ← property assertions + scenarios.go ← action sequence generation + e2e_test.go ← test entry points (slow, nightly) + fakeintake/ ← existing, reused +``` + +## CI Integration + +| Test Type | Speed | When to Run | What it Catches | +|---|---|---|---| +| Unit tests | < 1s | Every commit | Algorithm logic bugs | +| Property tests | < 10s | Every commit | Edge cases in isolated components | +| E2E tests | ~1 min per case | Nightly / merge gate | Pipeline integration, config plumbing, timing, output format | + +## Open Questions + +- **Go or Rust for e2e harness?** The agent is Go, fakeintake is Go, but our prototype is Rust. Go is probably the right choice for agent-repo integration. The property generation could use `gopter` or `rapid` for Go proptest equivalents. +- **How to handle the slow e2e test speed?** Each case is ~50s (30s warmup + 15s drain + overhead). With FUSE-based log delivery the warmup could potentially be reduced. Alternatively, keep the agent running across cases for scenarios where state leakage is acceptable. +- **Allium spec parsing** — is there an existing parser? If allium is TLA+-like, there may be tooling. If it's custom, we'd need to build a parser. +- **Multi-pattern testing** — the black-box pattern probing approach (burst_size=1, send pairs, observe same/different) is the cleanest way to test the tokenizer without reimplementing it. Should this be a unit test, e2e test, or both? + +## References + +### lading_proptest docs (this repo) +- `README.md` — overview, running instructions, env vars +- `PROPERTIES.md` — detailed property documentation +- `ADAPTIVE_SAMPLING.md` — adaptive sampling testing strategy and discussion +- `MIGRATION.md` — this document + +### Agent repo +- `pkg/logs/internal/decoder/preprocessor/adaptive_sampler.allium` — formal spec +- `pkg/logs/internal/decoder/preprocessor/sampler.go` — implementation +- `pkg/logs/internal/decoder/preprocessor/sampler_test.go` — unit tests +- `test/fakeintake/` — fake intake server +- Branch: `blt/add_pattern_isolation_invariant_to_adaptive_sampling_spec` — spec additions in progress diff --git a/integration/lading_proptest/PROPERTIES.md b/integration/lading_proptest/PROPERTIES.md new file mode 100644 index 000000000..9cd69ff7b --- /dev/null +++ b/integration/lading_proptest/PROPERTIES.md @@ -0,0 +1,166 @@ +# Property Reference + +Detailed documentation for each property assertion used in `lading_proptest`. Each property checks a specific relationship between the test input (`LogBatch`) and agent output (`Vec`). + +## Input Correlation Model + +Every log line we write embeds a UUID that lets us correlate inputs with outputs after the agent processes them. The agent treats these markers as opaque content — it doesn't know about them. + +### Markers + +**`[PROPTEST:]`** — Embedded in every header/standalone line. Appears at the start of the line (before any content) so it survives truncation. Used by all non-JSON formats. + +**`"proptest_id":""`** — The JSON equivalent. A field in every JSON object we generate. Survives `json.Compact()` since it's a valid JSON field. + +**`[CONT::]`** — Embedded in timestamp multiline continuation lines. The `header-uuid` ties it to its parent entry. The `seq` (0, 1, 2, ...) tracks ordering. After aggregation, these markers appear inside the parent entry's message joined with literal `\n`. + +### Line IDs + +Each `LogLine` in a `LogBatch` has an `id` field used internally for tracking (not written to the file — only the `content` is written). The ID conventions: + +| Pattern | Meaning | Example | +|---|---|---| +| `` | Header or standalone line | `a1b2c3d4-...` | +| `:cont:` | Timestamp multiline continuation | `a1b2c3d4-...:cont:0` | +| `:json_line:` | JSON multiline fragment | `a1b2c3d4-...:json_line:1` | + +Properties filter out `:cont:` and `:json_line:` IDs when checking delivery — these fragments merge into their parent entry and aren't expected as standalone output entries. + +### Metadata on LogBatch + +- **`expected_continuations: Vec<(String, usize)>`** — For timestamp multiline: maps each header UUID to its expected continuation count. Used by `MultilineAggregated` to verify completeness. +- **`expected_json: Option>`** — For JSON multiline: maps each UUID to the expected parsed JSON value. Used by `JsonIntegrity` to verify field preservation after compaction. + +--- + +## AllLinesDelivered + +**Intent:** No data loss — every log entry we wrote to the file shows up somewhere in the agent's output. + +**How it works:** +1. Collects all input UUIDs, filtering out fragment IDs (`:cont:` and `:json_line:` suffixes — these are internal line fragments that get merged into their parent entry) +2. Scans every output message for ALL `[PROPTEST:uuid]` and `"proptest_id":"uuid"` patterns using `extract_all_ids` (not just the first match per message — important because aggregated messages can contain multiple UUIDs) +3. Set difference: any input UUID not found in any output message is "missing" + +**Used by:** Truncation, Multiline, JSON Multiline, Mixed Multiline + +**Known limitations:** +- The Apache path detection (`GET //`) in `extract_all_ids` only finds the first Apache UUID per message. If two Apache entries got merged (unlikely since Apache lines have timestamps that trigger `startGroup`), the second UUID could be missed. + +--- + +## NoExtraLines + +**Intent:** The agent doesn't fabricate data — no output entry has a UUID we didn't send. + +**How it works:** +1. Same input ID filtering as `AllLinesDelivered` (skips `:cont:` and `:json_line:`) +2. For each output entry, extracts the **first** UUID only (uses `extract_id`, not `extract_all_ids`) +3. If that UUID isn't in the input set, it's "extra" + +**Used by:** Currently not included in any scenario (available for future use). + +**Known limitations:** +- Uses `extract_id` (first UUID only). If an aggregated output message has multiple UUIDs, only the first is checked against the input set. A hypothetical bug where the agent injects a fabricated UUID as a non-first UUID in an aggregated message would not be caught. Very unlikely to matter in practice. + +--- + +## ContentPreserved + +**Intent:** For non-truncated, non-multiline lines, the content passes through the agent unchanged. + +**How it works:** +1. Builds a map of input UUID → input content (skips `:cont:` lines) +2. For each output entry, extracts the first UUID, looks up the expected input content +3. Checks bidirectional containment: either the output contains the input OR the input contains the output +4. Skips entries with `TRUNCATED` in the message (those are handled by `TruncationRespected`) + +**Used by:** Truncation + +**Known limitations:** +- **The bidirectional check is loose.** The reverse check (`expected.contains(output)`) could mask data loss where the agent delivered only a small fragment of the input. The intent is to allow the agent to wrap content with metadata, but a tiny output would incorrectly pass. +- **Doesn't filter `:json_line:` IDs in the input map.** Not a bug — JSON fragment IDs won't match any output UUID so they're harmlessly skipped — but inconsistent with other properties. +- **Doesn't verify byte-exact content.** Checks containment, not equality. If the agent modified characters within the content but preserved the overall string, this wouldn't catch it. + +--- + +## TruncationRespected + +**Intent:** Lines over the configured `max_message_size_bytes` get truncated. Lines under the limit arrive intact. + +**How it works:** +1. Skips output entries starting with `...TRUNCATED...` (tail chunks from split messages — these are expected byproducts) +2. For each output entry with a UUID, looks up the original input line length +3. **If input was over the limit:** + - Head chunk must be ≤ `max_message_bytes + 15` (the `...TRUNCATED...` marker is 15 bytes, appended by the agent) + - Head chunk must end with the literal string `...TRUNCATED...` +4. **If input was under the limit:** + - Output must be at least 50% of input length (catches severe data loss while allowing minor agent overhead) + +**Used by:** Truncation + +**Known limitations:** +- **The 50% threshold for under-limit lines is arbitrary.** An output at 51% of input length passes, which is still significant data loss. A tighter check (e.g., output length ≥ input length - 100 bytes) would catch more issues but risks false positives from agent metadata. +- **Doesn't verify content of under-limit lines.** Checks size only, not that the actual bytes match. `ContentPreserved` is included alongside this property in the truncation scenario to cover content verification. + +--- + +## MultilineAggregated + +**Intent:** Continuation lines are merged with the correct header, all expected continuations are present, and they appear in order. + +**How it works:** +1. If `expected_continuations` is empty, passes trivially (non-multiline scenario) +2. Builds a map of header UUID → expected continuation count from `LogBatch.expected_continuations` +3. For each output entry with a UUID that's in the expected map: + - Extracts all `[CONT:header_id:seq]` markers from the output message + - **No cross-contamination:** every continuation's `header_id` must match the entry's UUID + - **Completeness:** every expected sequence number (0 through count-1) must be present + - **Ordering:** sequence numbers must appear in ascending order + +**Used by:** Multiline, Mixed Multiline + +**Known limitations:** +- **Doesn't verify continuation content.** Checks that markers are present and ordered, but doesn't verify the text between markers. If the agent corrupted continuation content but preserved the `[CONT:...]` markers, this wouldn't catch it. +- **Doesn't check for unexpected extra continuations.** If the output had continuations 0-5 but we only expected 0-2, the extras aren't flagged. In practice this can't happen since we control the input, but it's a gap in the assertion. + +--- + +## ExpectedEntryCount + +**Intent:** The number of logical output entries matches how many we generated. + +**How it works:** +1. Counts output entries where `extract_id` returns `Some` (filters out agent internal messages, `...TRUNCATED...` tail chunks, and any other entries without a proptest UUID) +2. Compares to the expected count passed at construction + +**Used by:** Multiline (timestamp-based) + +**Known limitations:** +- **Counts by first UUID only.** If a plain text line merges into another entry, the count is correct (merged entry counts as 1). If a truncated line produces a head + tail, only the head has a UUID so it counts as 1. This is the intended behavior. +- **Not suitable for truncation or JSON scenarios.** Truncation produces extra tail chunks. JSON `TopLevelArray` entries get split into multiple outputs. These would throw off the count. That's why this property is only used by the multiline timestamp scenario where the entry count is predictable. + +--- + +## JsonIntegrity + +**Intent:** Every JSON entry arrives in the output as valid JSON with all expected fields and values preserved after the agent's compaction. + +**How it works:** +1. If `expected_json` is `None`, passes trivially (non-JSON scenario) +2. For each expected `(uuid, json_value)` pair: + - Finds the output entry containing that UUID by string search + - Parses the output message as JSON + - Verifies the `proptest_id` field matches + - Recursively compares all fields: + - **Objects:** checks every expected key is present with the correct value (key-by-key) + - **Arrays:** checks length matches, then compares element-by-element + - **Primitives:** checks equality +3. Reports specific failures with dotted field paths (e.g., `data.f_0`, `items[2]`) + +**Used by:** JSON Multiline, Mixed Multiline + +**Known limitations:** +- **Finds output by string containment.** `entry.message.contains(expected_id)` could theoretically match a UUID appearing as a substring of another value. UUIDs are 36 chars with a specific dash pattern so false matches are extremely unlikely. +- **One-directional field check (expected ⊆ actual).** If the agent added extra fields not in the expected JSON, this wouldn't catch it. This is intentional — the agent may add metadata fields like tags. +- **TopLevelArray error message is confusing.** When the expected JSON is `[{...}]` but the agent outputs `{...}` (because top-level arrays aren't supported), the root-level comparison reports `field '' expected [...], got {...}` — the empty path `''` is unclear. The failure is correctly detected but the diagnostic could be improved. diff --git a/integration/lading_proptest/README.md b/integration/lading_proptest/README.md new file mode 100644 index 000000000..77b107b8f --- /dev/null +++ b/integration/lading_proptest/README.md @@ -0,0 +1,181 @@ +# lading_proptest + +Property-based integration testing for the Datadog Agent logs pipeline. Generates structurally diverse log data, feeds it to a real agent via file tailing, collects what the agent sends to a fake HTTP intake, and asserts properties on the output. + +## How It Works + +Each test case: + +1. **Proptest generates parameters** — log format, structure, line counts, etc. +2. **Input is generated** — unique log lines with embedded UUID markers for correlation +3. **Orchestrator runs the agent** — starts a Docker container (or local binary), mounts config and log files +4. **Logs are written** — the generated lines are written to a file the agent tails +5. **Agent processes and sends** — the agent tails the file, applies multiline aggregation/truncation/etc., and POSTs JSON payloads to our fake intake +6. **Intake collects** — our HTTP server decompresses and parses the agent's payloads +7. **Properties are checked** — declarative assertions verify data integrity, field preservation, correct aggregation, etc. + +On failure, proptest shrinks the input to find a minimal reproducing case. Each shrink attempt spins up a new agent container. + +## Scenarios + +### Truncation + +Tests that the agent correctly truncates lines exceeding `max_message_size_bytes`. + +- Generates lines of varying sizes: well under limit, near boundary, well over +- Verifies lines under the limit arrive intact +- Verifies lines over the limit are split into a head chunk (with `...TRUNCATED...` appended) and tail chunks +- Tests across all 5 log formats: PlainText, Json, Syslog5424, ApacheCommon, TimestampPrefixed +- Config variants: truncation limit at 1KB, 64KB, 256KB (agent default) + +**Test functions:** `truncation`, `truncation_plain_text`, `truncation_json`, `truncation_syslog`, `truncation_apache`, `truncation_timestamp_prefixed` + +### Timestamp Multiline + +Tests that the agent aggregates continuation lines with timestamp-prefixed headers. + +- Generates header lines with distinct timestamps (so each triggers `startGroup`) +- Continuation lines have no timestamp or format structure +- Verifies all continuations are merged into the correct header's output entry +- Verifies continuations are in order +- Tests across 3 timestamp formats: TimestampPrefixed, Syslog5424, ApacheCommon + +**Test functions:** `multiline_timestamp_prefixed`, `multiline_syslog`, `multiline_apache`, `multiline_all_formats` + +### JSON Multiline + +Tests the agent's JSON aggregation — buffering incomplete JSON objects and compacting them with `json.Compact()`. + +- Generates structurally diverse valid JSON split across multiple lines +- 11 structural variants: Flat, Nested, WithArray, TopLevelArray, DeepNested, MixedNesting, EmptyObject, EmptyArray, EscapedStrings, UnicodeValues, MixedValueTypes +- Verifies compacted output is valid JSON with all fields preserved +- Does not encode knowledge of agent internals — discovers gaps mechanically + +**Known discovery:** TopLevelArray (`[{...}]`) is not supported by the agent's JSON aggregator. The test finds this every time the variant appears. + +**Test function:** `json_multiline` + +### Mixed Multiline + +Tests interactions between the agent's two aggregation systems when different formats are interleaved in the same file. + +- Mixes all 3 timestamp formats, 5+ JSON variants, and plain text lines in random order +- Tests format transitions: syslog → JSON → apache → plain text → JSON +- Tests JSON with embedded timestamp values (JSON detection should take priority) +- Verifies both aggregation systems work correctly when rapidly alternating + +**Test function:** `mixed_multiline` + +## Running + +### Prerequisites + +- Docker with the agent image pulled: + ```bash + docker pull datadog/agent:7.78.2-full + ``` +- If using Colima on macOS, set `DOCKER_HOST`: + ```bash + export DOCKER_HOST=unix:///Users/$USER/.colima/default/docker.sock + ``` + +### Basic Usage + +```bash +# Run a specific test (1 case, no shrinking) +PROPTEST_MAX_SHRINK_ITERS=0 \ + DD_AGENT_IMAGE=datadog/agent:7.78.2-full \ + PROPTEST_CASES=1 \ + RUST_LOG=info \ + cargo test -p lading_proptest --lib truncation_plain_text -- --nocapture --test-threads=1 + +# Run all truncation tests +cargo test -p lading_proptest --lib truncation -- --nocapture --test-threads=1 + +# Run all multiline tests +cargo test -p lading_proptest --lib multiline -- --nocapture --test-threads=1 + +# Run everything +cargo test -p lading_proptest --lib tests:: -- --nocapture --test-threads=1 +``` + +### Unit Tests Only (no agent needed) + +```bash +cargo test -p lading_proptest --lib log_format +cargo test -p lading_proptest --lib log_gen +``` + +## Environment Variables + +| Variable | Default | Description | +|---|---|---| +| `DD_AGENT_IMAGE` | `datadog/agent:latest` | Docker image for the agent container | +| `DD_AGENT_BINARY` | (none) | Path to agent binary (uses binary mode instead of container) | +| `DOCKER_HOST` | `/var/run/docker.sock` | Docker daemon socket (set for Colima on macOS) | +| `PROPTEST_CASES` | `1` | Number of proptest cases per test function | +| `PROPTEST_MAX_SHRINK_ITERS` | auto | Max shrink attempts on failure (set `0` to disable) | +| `RUST_LOG` | (none) | Tracing log level (`info`, `debug`, `trace`) | +| `LADING_KEEP_TEMP` | (unset) | If set, preserves temp directories even on success | + +## Temp Directory + +Each test case creates a temp directory under `~/.lading_proptest_tmp/` containing: + +``` +lading_proptest_XXXXXX/ + config/ + datadog.yaml # Agent main config + conf.d/proptest.d/ + conf.yaml # Log source config + logs/ + proptest.log # Input: the log file the agent tails + output.json # Output: full JSON of all received intake entries + output_messages.txt # Output: one message per line with byte count +``` + +On test success, the directory is cleaned up (unless `LADING_KEEP_TEMP=1`). On failure, it is preserved and the path is printed in diagnostics. + +## Properties + +| Property | What it checks | +|---|---| +| `AllLinesDelivered` | Every input UUID appears in at least one output entry | +| `NoExtraLines` | No output entry has a UUID not in the input | +| `ContentPreserved` | Non-truncated lines match content exactly | +| `TruncationRespected` | Over-limit lines truncated, under-limit lines intact | +| `MultilineAggregated` | All expected continuations present and in order | +| `ExpectedEntryCount` | Output entry count matches expected logical entries | +| `JsonIntegrity` | Output JSON has all expected fields with correct values | + +## Agent Safety + +The agent is configured to not contact real Datadog infrastructure: + +- `DD_LOGS_CONFIG_LOGS_DD_URL` points to our local intake server +- `DD_DD_URL=http://localhost:1` catches any non-logs telemetry +- `DD_API_KEY=fake_key_for_proptest` would be rejected by real intake +- APM, process collection, metadata collection, and inventories are disabled + +Orphaned containers are cleaned up via a `Drop` implementation that force-removes them. + +## Timing + +Each test case takes ~49 seconds: +- 30s pipeline warmup (agent initialization) +- 15s drain wait (agent processing + sending) +- ~4s container start/stop overhead + +With `PROPTEST_CASES=5`, expect ~4 minutes per test function. With `--test-threads=1`, test functions run sequentially. + +## Future Work + +- **JSON multiline: top-level arrays** — the agent currently does not support aggregating top-level JSON arrays (`[{...}]`). The `TopLevelArray` variant is included to detect if/when support is added. +- **Non-atomic write simulation** — model scenarios where multiple applications write to the same log file, causing interleaved/corrupted multi-line entries. When a pretty-printed JSON object is written line-by-line and another process writes between lines, the agent sees invalid JSON mid-buffer. This could test the agent's error recovery and buffer flushing behavior. Relevant when writes exceed `PIPE_BUF` (4KB) or when applications flush per-line rather than per-entry. +- **Explicit multiline patterns** — PlainText with `log_processing_rules` regex patterns for aggregation (currently untested since PlainText has no auto-detection signal). +- **JSON multiline: incomplete objects** — test what happens when a JSON object is never closed (e.g., application crashes mid-write). Does the agent flush on timeout? How long? +- **Intake hard cap (~900KB)** — the Datadog intake enforces a hard size limit around 900KB per message, independent of the agent's `max_message_size_bytes` setting. Test behavior when the agent's configured limit is set above the intake cap, and verify the intake rejection is handled gracefully. +- **Action sequences** — extend scenarios with timed actions (write lines, sleep, rotate file, write more) to test the agent's behavior with file rotation, delayed writes, and aggregation timeouts. +- **TCP/UDP log delivery** — currently all tests use file tailing. Testing network-based log sources exercises different agent code paths. +- **Pipeline warmup optimization** — the 30s fixed warmup is conservative. Polling the agent's health or metrics endpoint could reduce this. +- **Parallel test execution** — running test functions in parallel on different ports. Requires a dedicated machine to avoid resource contention. diff --git a/integration/lading_proptest/proptest-regressions/lib.txt b/integration/lading_proptest/proptest-regressions/lib.txt new file mode 100644 index 000000000..f81f522f8 --- /dev/null +++ b/integration/lading_proptest/proptest-regressions/lib.txt @@ -0,0 +1 @@ +cc cc5b3078f725e938ba698986c08ab9c131ec0baab3a2a553e5aefdf63f49a16f # shrinks to params = JsonMultilineParams { entries: [WithArray { item_count: 4 }, UnicodeValues, UnicodeValues, TopLevelArray, EmptyObject, MixedValueTypes, DeepNested { depth: 3 }, Flat { field_count: 5 }] } diff --git a/integration/lading_proptest/src/agent.rs b/integration/lading_proptest/src/agent.rs new file mode 100644 index 000000000..18d2cfa47 --- /dev/null +++ b/integration/lading_proptest/src/agent.rs @@ -0,0 +1,375 @@ +//! Agent abstraction for running the Datadog Agent as a container or binary. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use tokio::process::Child; +use tracing::{debug, info, warn}; + +/// Errors from agent lifecycle operations. +#[derive(thiserror::Error, Debug)] +pub enum Error { + /// I/O error during agent operations. + #[error("agent I/O error: {0}")] + Io(#[from] std::io::Error), + /// Docker API error. + #[error("docker error: {0}")] + Docker(#[from] bollard::errors::Error), + /// Agent failed to become ready within timeout. + #[error("agent readiness timeout after {0:?}")] + ReadinessTimeout(Duration), + /// Agent exited unexpectedly. + #[error("agent exited unexpectedly with status: {0}")] + UnexpectedExit(String), +} + +/// How to run the agent. +#[derive(Debug, Clone)] +pub enum AgentTarget { + /// Run the agent as a Docker container. + Container(ContainerAgentConfig), + /// Run the agent as a local binary. + Binary(BinaryAgentConfig), +} + +/// Configuration for running the agent as a Docker container. +#[derive(Debug, Clone)] +pub struct ContainerAgentConfig { + /// Docker image, e.g. `"datadog/agent:latest"`. + pub image: String, + /// Additional environment variables. + pub extra_env: Vec<(String, String)>, +} + +/// Configuration for running the agent as a local binary. +#[derive(Debug, Clone)] +pub struct BinaryAgentConfig { + /// Path to the agent binary. + pub binary_path: PathBuf, + /// Additional environment variables. + pub extra_env: Vec<(String, String)>, +} + +/// A running agent instance. +/// +/// If dropped without calling [`RunningAgent::stop`], containers are +/// force-removed in a background thread to prevent orphans on panic or +/// Ctrl+C. +#[derive(Debug)] +pub struct RunningAgent { + inner: Option, +} + +#[derive(Debug)] +enum RunningAgentInner { + Container { + docker: bollard::Docker, + container_id: String, + }, + Binary { + child: Child, + }, +} + +impl Drop for RunningAgent { + fn drop(&mut self) { + if let Some(RunningAgentInner::Container { docker, container_id }) = self.inner.take() { + warn!("RunningAgent dropped without stop() — force-removing container {container_id}"); + // Spawn a blocking thread so this works even outside a tokio context. + let _ = std::thread::spawn(move || { + let rt = tokio::runtime::Runtime::new(); + if let Ok(rt) = rt { + let _ = rt.block_on(async { + let opts = + bollard::query_parameters::RemoveContainerOptionsBuilder::default() + .force(true) + .build(); + docker.remove_container(&container_id, Some(opts)).await + }); + } + }) + .join(); + } + } +} + +impl AgentTarget { + /// Start the agent with the given configuration. + /// + /// # Errors + /// + /// Returns error if the agent fails to start. + pub async fn start( + &self, + config_dir: &Path, + log_dir: &Path, + intake_port: u16, + ) -> Result { + match self { + Self::Container(cfg) => start_container(cfg, config_dir, log_dir, intake_port).await, + Self::Binary(cfg) => start_binary(cfg, config_dir, intake_port), + } + } + + /// Build an `AgentTarget` from environment variables. + /// + /// Checks `DD_AGENT_BINARY` first (binary mode), then falls back to + /// `DD_AGENT_IMAGE` (container mode, default `datadog/agent:latest`). + /// + /// # Panics + /// + /// This function does not panic. + #[must_use] + pub fn from_env() -> Self { + if let Ok(binary) = std::env::var("DD_AGENT_BINARY") { + Self::Binary(BinaryAgentConfig { + binary_path: PathBuf::from(binary), + extra_env: Vec::new(), + }) + } else { + let image = std::env::var("DD_AGENT_IMAGE") + .unwrap_or_else(|_| "datadog/agent:latest".to_string()); + Self::Container(ContainerAgentConfig { + image, + extra_env: Vec::new(), + }) + } + } +} + +impl RunningAgent { + /// Wait for the agent to become ready. + /// + /// # Errors + /// + /// Returns error if readiness is not achieved within the timeout. + /// + /// # Panics + /// + /// Panics if called after [`RunningAgent::stop`]. + pub async fn wait_ready(&self, timeout: Duration) -> Result<(), Error> { + let deadline = tokio::time::Instant::now() + timeout; + let inner = self.inner.as_ref().expect("agent already stopped"); + loop { + if tokio::time::Instant::now() >= deadline { + return Err(Error::ReadinessTimeout(timeout)); + } + match inner { + RunningAgentInner::Container { docker, container_id } => { + if let Ok(info) = docker + .inspect_container( + container_id, + None::, + ) + .await + && let Some(state) = &info.state + && state.running == Some(true) + { + debug!("container {container_id} is running"); + return Ok(()); + } + } + RunningAgentInner::Binary { child } => { + if child.id().is_some() { + debug!("agent binary process is alive"); + return Ok(()); + } + } + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + + /// Stop the agent gracefully. + /// + /// # Errors + /// + /// Returns error if the agent cannot be stopped. + pub async fn stop(mut self) -> Result<(), Error> { + let Some(inner) = self.inner.take() else { + return Ok(()); + }; + match inner { + RunningAgentInner::Container { docker, container_id } => { + info!("stopping container {container_id}"); + let stop_options = + bollard::query_parameters::StopContainerOptionsBuilder::default() + .t(30) + .build(); + docker + .stop_container(&container_id, Some(stop_options)) + .await?; + let remove_options = + bollard::query_parameters::RemoveContainerOptionsBuilder::default() + .force(true) + .build(); + docker + .remove_container(&container_id, Some(remove_options)) + .await?; + Ok(()) + } + RunningAgentInner::Binary { mut child } => { + info!("stopping agent binary"); + #[cfg(unix)] + { + if let Some(pid) = child.id() { + let nix_pid = + nix::unistd::Pid::from_raw(i32::try_from(pid).unwrap_or(0)); + let _ = + nix::sys::signal::kill(nix_pid, nix::sys::signal::Signal::SIGTERM); + } + } + if let Ok(result) = + tokio::time::timeout(Duration::from_secs(30), child.wait()).await + { + let status = result?; + debug!("agent exited with status: {status}"); + } else { + warn!("agent did not exit after SIGTERM, killing"); + child.kill().await?; + } + Ok(()) + } + } + } +} + +async fn start_container( + cfg: &ContainerAgentConfig, + config_dir: &Path, + log_dir: &Path, + intake_port: u16, +) -> Result { + use bollard::models::{ContainerCreateBody, HostConfig, Mount, MountTypeEnum}; + use bollard::query_parameters::{CreateContainerOptionsBuilder, StartContainerOptions}; + + let docker = bollard::Docker::connect_with_local_defaults()?; + + let host_addr = if cfg!(target_os = "macos") { + format!("host.docker.internal:{intake_port}") + } else { + format!("172.17.0.1:{intake_port}") + }; + + let mut env = vec![ + "DD_API_KEY=fake_key_for_proptest".to_string(), + "DD_HOSTNAME=lading-proptest".to_string(), + "DD_LOGS_ENABLED=true".to_string(), + format!("DD_LOGS_CONFIG_LOGS_DD_URL={host_addr}"), + "DD_LOGS_CONFIG_LOGS_NO_SSL=true".to_string(), + "DD_LOGS_CONFIG_FORCE_USE_HTTP=true".to_string(), + "DD_LOGS_CONFIG_BATCH_WAIT=1".to_string(), + // Point all non-logs telemetry at a dead endpoint so the agent + // does not make outbound requests to datadoghq.com. + "DD_DD_URL=http://localhost:1".to_string(), + // Disable non-logs subsystems + "DD_APM_ENABLED=false".to_string(), + "DD_PROCESS_CONFIG_PROCESS_COLLECTION_ENABLED=false".to_string(), + "DD_ENABLE_METADATA_COLLECTION=false".to_string(), + "DD_INVENTORIES_ENABLED=false".to_string(), + ]; + for (k, v) in &cfg.extra_env { + env.push(format!("{k}={v}")); + } + + let config_mount = config_dir + .to_str() + .expect("config_dir must be valid UTF-8"); + let log_mount = log_dir.to_str().expect("log_dir must be valid UTF-8"); + + let container_config = ContainerCreateBody { + image: Some(cfg.image.clone()), + env: Some(env), + host_config: Some(HostConfig { + mounts: Some(vec![ + Mount { + target: Some("/etc/datadog-agent/datadog.yaml".to_string()), + source: Some(format!("{config_mount}/datadog.yaml")), + typ: Some(MountTypeEnum::BIND), + read_only: Some(true), + ..Default::default() + }, + Mount { + target: Some("/etc/datadog-agent/conf.d".to_string()), + source: Some(format!("{config_mount}/conf.d")), + typ: Some(MountTypeEnum::BIND), + read_only: Some(true), + ..Default::default() + }, + Mount { + target: Some("/var/log/proptest".to_string()), + source: Some(log_mount.to_string()), + typ: Some(MountTypeEnum::BIND), + read_only: Some(true), + ..Default::default() + }, + ]), + extra_hosts: if cfg!(target_os = "linux") { + Some(vec!["host.docker.internal:host-gateway".to_string()]) + } else { + None + }, + ..Default::default() + }), + ..Default::default() + }; + + let name = format!("lading-proptest-{}", uuid::Uuid::new_v4()); + let create_options = CreateContainerOptionsBuilder::default() + .name(&name) + .build(); + let container = docker + .create_container(Some(create_options), container_config) + .await?; + + docker + .start_container(&container.id, None::) + .await?; + + info!("started container {} ({})", name, container.id); + + Ok(RunningAgent { + inner: Some(RunningAgentInner::Container { + docker, + container_id: container.id, + }), + }) +} + +fn start_binary( + cfg: &BinaryAgentConfig, + config_dir: &Path, + _intake_port: u16, +) -> Result { + use std::process::Stdio; + + let stdout_path = config_dir + .parent() + .unwrap_or(config_dir) + .join("agent_stdout.log"); + let stderr_path = config_dir + .parent() + .unwrap_or(config_dir) + .join("agent_stderr.log"); + + let stdout_file = std::fs::File::create(&stdout_path)?; + let stderr_file = std::fs::File::create(&stderr_path)?; + + let mut cmd = tokio::process::Command::new(&cfg.binary_path); + cmd.arg("run") + .arg("--cfgpath") + .arg(config_dir) + .stdout(Stdio::from(stdout_file)) + .stderr(Stdio::from(stderr_file)); + + for (k, v) in &cfg.extra_env { + cmd.env(k, v); + } + + let child = cmd.spawn()?; + info!("started agent binary from {:?}", cfg.binary_path); + + Ok(RunningAgent { + inner: Some(RunningAgentInner::Binary { child }), + }) +} diff --git a/integration/lading_proptest/src/config.rs b/integration/lading_proptest/src/config.rs new file mode 100644 index 000000000..2d008dd5e --- /dev/null +++ b/integration/lading_proptest/src/config.rs @@ -0,0 +1,191 @@ +//! Datadog Agent configuration template builders. +//! +//! Generates the YAML configuration files needed to run the agent with +//! the correct settings for each test scenario. + +use std::path::Path; + +/// Errors from configuration generation. +#[derive(thiserror::Error, Debug)] +pub enum Error { + /// I/O error writing config files. + #[error("config I/O error: {0}")] + Io(#[from] std::io::Error), +} + +/// Log source configuration variant. +#[derive(Debug, Clone)] +pub enum LogSourceConfig { + /// Simple file tailing with no special processing. + Simple, + /// Auto multiline detection enabled. + AutoMultiline, + /// Explicit multiline pattern. + ExplicitMultiline { + /// Regex pattern that starts a new logical log entry. + pattern: String, + }, + /// Auto multiline with JSON detection and aggregation enabled. + JsonMultiline, + /// Adaptive sampling enabled with configurable parameters. + AdaptiveSampling { + /// Initial credits per pattern and credit cap. + burst_size: f64, + /// Credits refilled per second per pattern. + rate_limit: f64, + }, +} + +/// Parameters for generating agent configuration. +#[derive(Debug, Clone)] +pub struct AgentConfigParams { + /// Port of the log intake server. + pub intake_port: u16, + /// Path where log files will be written (inside the container or on host). + pub log_file_path: String, + /// Whether to enable compression on the intake connection. + pub use_compression: bool, + /// Batch wait time in milliseconds (lower = faster drain). + pub batch_wait_ms: u64, + /// Log source configuration. + pub log_source_config: LogSourceConfig, + /// Maximum message size in bytes (for truncation testing). + /// If `None`, uses the agent's default (256 KB). + pub max_message_size_bytes: Option, +} + +/// Write agent configuration files to the given directory. +/// +/// Creates: +/// - `datadog.yaml` — main agent config +/// - `conf.d/proptest.d/conf.yaml` — log source config +/// +/// # Errors +/// +/// Returns error if file I/O fails. +#[expect(clippy::too_many_lines)] +pub fn write_agent_config(config_dir: &Path, params: &AgentConfigParams) -> Result<(), Error> { + // Write main datadog.yaml + let datadog_yaml = format!( + r#"# Generated by lading_proptest +api_key: fake_key_for_proptest +hostname: lading-proptest + +# Point all non-logs telemetry at a dead endpoint so the agent +# does not make outbound requests to datadoghq.com. +dd_url: "http://localhost:1" + +logs_enabled: true +enable_metadata_collection: false +inventories_enabled: false + +logs_config: + logs_dd_url: "127.0.0.1:{intake_port}" + logs_no_ssl: true + force_use_http: true + use_compression: {use_compression} + batch_wait: {batch_wait} + batch_max_size: 100 + open_files_limit: 100 + max_message_size_bytes: {max_message_size_bytes} +{adaptive_sampling_config} +apm_config: + enabled: false + +process_config: + process_collection: + enabled: false + +security_agent: + runtime_security_config: + enabled: false + +compliance_config: + enabled: false +"#, + intake_port = params.intake_port, + use_compression = params.use_compression, + batch_wait = params.batch_wait_ms, + max_message_size_bytes = params.max_message_size_bytes.unwrap_or(256 * 1024), + adaptive_sampling_config = match ¶ms.log_source_config { + LogSourceConfig::AdaptiveSampling { burst_size, rate_limit } => { + format!( + " experimental_adaptive_sampling:\n enabled: true\n burst_size: {burst_size}\n rate_limit: {rate_limit}\n protect_important_logs: false\n" + ) + } + _ => String::new(), + }, + ); + + std::fs::write(config_dir.join("datadog.yaml"), datadog_yaml)?; + + // Write log source config + let conf_d = config_dir.join("conf.d").join("proptest.d"); + std::fs::create_dir_all(&conf_d)?; + + let log_source = match ¶ms.log_source_config { + LogSourceConfig::Simple => { + format!( + r#"logs: + - type: file + path: "{log_path}" + service: proptest + source: proptest +"#, + log_path = params.log_file_path, + ) + } + LogSourceConfig::AutoMultiline => { + format!( + r#"logs: + - type: file + path: "{log_path}" + service: proptest + source: proptest + auto_multi_line_detection: true +"#, + log_path = params.log_file_path, + ) + } + LogSourceConfig::ExplicitMultiline { pattern } => { + format!( + r#"logs: + - type: file + path: "{log_path}" + service: proptest + source: proptest + log_processing_rules: + - type: multi_line + name: proptest_multiline + pattern: '{pattern}' +"#, + log_path = params.log_file_path, + ) + } + LogSourceConfig::JsonMultiline => { + format!( + r#"logs: + - type: file + path: "{log_path}" + service: proptest + source: proptest + auto_multi_line_detection: true + auto_multi_line: + enable_json_detection: true + enable_json_aggregation: true +"#, + log_path = params.log_file_path, + ) + } + LogSourceConfig::AdaptiveSampling { .. } => { + format!( + "logs:\n - type: file\n path: \"{log_path}\"\n service: proptest\n source: proptest\n experimental_adaptive_sampling:\n enabled: true\n", + log_path = params.log_file_path, + ) + } + }; + + std::fs::write(conf_d.join("conf.yaml"), log_source)?; + + Ok(()) +} diff --git a/integration/lading_proptest/src/diagnostics.rs b/integration/lading_proptest/src/diagnostics.rs new file mode 100644 index 000000000..95617628e --- /dev/null +++ b/integration/lading_proptest/src/diagnostics.rs @@ -0,0 +1,74 @@ +//! Failure diagnostics formatting. +//! +//! When a property fails, format the failure with enough context for debugging: +//! input lines, output entries, property details, and temp directory path. + +use std::fmt::Write; + +use crate::log_format::LogFormat; +use crate::orchestrator::TestCaseResult; + +/// Format a test case result for display when a property has failed. +#[must_use] +pub fn format_failure(result: &TestCaseResult) -> String { + let mut out = String::new(); + + out.push_str("=== PROPERTY TEST FAILURE ===\n\n"); + + for prop_result in &result.property_results { + if let Err(failure) = prop_result { + let _ = writeln!(out, "FAILED: {failure}"); + } + } + + let _ = write!( + out, + "\n--- Input ({} lines, format: {:?}) ---\n", + result.input.lines.len(), + result.input.format, + ); + let max_show = 20; + for (i, line) in result.input.lines.iter().take(max_show).enumerate() { + if line.content.len() > 120 { + let _ = writeln!(out, " [{i}] {}...", &line.content[..120]); + } else { + let _ = writeln!(out, " [{i}] {}", line.content); + } + } + if result.input.lines.len() > max_show { + let _ = writeln!( + out, + " ... and {} more lines", + result.input.lines.len() - max_show + ); + } + + let _ = write!( + out, + "\n--- Output ({} entries) ---\n", + result.output.len(), + ); + for (i, entry) in result.output.iter().take(max_show).enumerate() { + let id = LogFormat::extract_id(&entry.message).unwrap_or("???"); + if entry.message.len() > 120 { + let _ = writeln!(out, " [{i}] id={id} msg={}...", &entry.message[..120]); + } else { + let _ = writeln!(out, " [{i}] id={id} msg={}", entry.message); + } + } + if result.output.len() > max_show { + let _ = writeln!( + out, + " ... and {} more entries", + result.output.len() - max_show + ); + } + + let _ = write!( + out, + "\n--- Debug ---\nTemp dir (preserved): {}\n", + result.temp_dir().display(), + ); + + out +} diff --git a/integration/lading_proptest/src/intake.rs b/integration/lading_proptest/src/intake.rs new file mode 100644 index 000000000..71ffcf734 --- /dev/null +++ b/integration/lading_proptest/src/intake.rs @@ -0,0 +1,293 @@ +//! Log intake HTTP server that captures Datadog Agent log payloads. +//! +//! The agent sends JSON arrays of log objects to `/api/v2/logs` or `/v1/input`. +//! This server accepts those payloads, decompresses if needed, parses the JSON, +//! and stores the individual log entries for later property assertion. + +use std::borrow::Cow; +use std::io; +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; + +use bytes::Bytes; +use flate2::read::GzDecoder; +use http_body_util::{BodyExt, Full}; +use hyper::body::Incoming; +use hyper::service::service_fn; +use hyper::{Method, Request, Response, header}; +use hyper_util::rt::{TokioExecutor, TokioIo}; +use hyper_util::server::conn::auto; +use serde::Deserialize; +use tokio::net::TcpListener; +use tracing::{debug, trace, warn}; + +/// Errors from the log intake server. +#[derive(thiserror::Error, Debug)] +pub enum Error { + /// I/O error. + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + /// Hyper HTTP error. + #[error("HTTP error: {0}")] + Hyper(#[from] hyper::Error), +} + +/// A single log entry as received from the Datadog Agent. +#[derive(Debug, Clone, serde::Serialize, Deserialize)] +pub struct ReceivedLogEntry { + /// The log message content. + pub message: String, + /// The log status/level (e.g., "info", "error"). + #[serde(default)] + pub status: Option, + /// The timestamp (can be integer or string depending on agent version). + #[serde(default)] + pub timestamp: Option, + /// The hostname. + #[serde(default)] + pub hostname: Option, + /// The service name. + #[serde(default)] + pub service: Option, + /// The source identifier. + #[serde(default)] + pub ddsource: Option, + /// Comma-separated tags. + #[serde(default)] + pub ddtags: Option, +} + +/// Thread-safe storage for received log entries. +#[derive(Debug, Clone, Default)] +pub struct LogStore { + entries: Arc>>, +} + +impl LogStore { + /// Create a new empty log store. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Append entries to the store. + pub fn append(&self, new_entries: Vec) { + if let Ok(mut entries) = self.entries.lock() { + entries.extend(new_entries); + } + } + + /// Take all stored entries, leaving the store empty. + #[must_use] + pub fn take(&self) -> Vec { + self.entries + .lock() + .map(|mut entries| std::mem::take(&mut *entries)) + .unwrap_or_default() + } + + /// Get the current number of stored entries. + #[must_use] + pub fn len(&self) -> usize { + self.entries.lock().map(|e| e.len()).unwrap_or(0) + } + + /// Check if the store is empty. + #[must_use] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// The log intake HTTP server. +#[derive(Debug)] +pub struct LogIntakeServer { + /// The address the server is listening on. + addr: SocketAddr, + /// The log store shared with the request handler. + store: LogStore, + /// Shutdown signal sender. + shutdown_tx: tokio::sync::oneshot::Sender<()>, + /// Server task handle. + handle: tokio::task::JoinHandle<()>, +} + +impl LogIntakeServer { + /// Start the log intake server on an ephemeral port. + /// + /// # Errors + /// + /// Returns error if the server cannot bind to a port. + pub async fn start() -> Result { + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let store = LogStore::new(); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + + let server_store = store.clone(); + let handle = tokio::spawn(async move { + run_server(listener, server_store, shutdown_rx).await; + }); + + debug!("log intake server listening on {addr}"); + + Ok(Self { + addr, + store, + shutdown_tx, + handle, + }) + } + + /// The port the server is listening on. + #[must_use] + pub fn port(&self) -> u16 { + self.addr.port() + } + + /// The full address the server is listening on. + #[must_use] + pub fn addr(&self) -> SocketAddr { + self.addr + } + + /// The log store containing received entries. + #[must_use] + pub fn store(&self) -> &LogStore { + &self.store + } + + /// Stop the server and return all collected log entries. + pub async fn stop(self) -> Vec { + // Signal shutdown + let _ = self.shutdown_tx.send(()); + // Wait for server to finish + let _ = self.handle.await; + self.store.take() + } +} + +async fn run_server( + listener: TcpListener, + store: LogStore, + mut shutdown_rx: tokio::sync::oneshot::Receiver<()>, +) { + loop { + tokio::select! { + accept_result = listener.accept() => { + match accept_result { + Ok((stream, peer)) => { + trace!("accepted connection from {peer}"); + let store = store.clone(); + tokio::spawn(async move { + let io = TokioIo::new(stream); + let svc = service_fn(move |req| { + let store = store.clone(); + async move { handle_request(req, &store).await } + }); + if let Err(e) = auto::Builder::new(TokioExecutor::new()) + .http1() + .serve_connection(io, svc) + .await + { + debug!("connection error: {e}"); + } + }); + } + Err(e) => { + warn!("accept error: {e}"); + } + } + } + _ = &mut shutdown_rx => { + debug!("log intake server shutting down"); + return; + } + } + } +} + +async fn handle_request( + req: Request, + store: &LogStore, +) -> Result>, hyper::Error> { + let method = req.method().clone(); + let path = req.uri().path().to_string(); + + if method == Method::POST + && (path.contains("/api/v2/logs") || path.contains("/v1/input")) + { + let content_encoding = req + .headers() + .get(header::CONTENT_ENCODING) + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_string(); + + let body = req.collect().await?.to_bytes(); + + debug!( + path = %path, + content_encoding = %content_encoding, + body_size = body.len(), + "received log payload", + ); + + match decompress_if_needed(&body, &content_encoding) { + Ok(decompressed) => { + // The agent may send empty objects `{}` as health/keepalive + // pings. Skip payloads that are too small to contain log data. + if decompressed.len() <= 2 { + trace!("ignoring empty payload ({} bytes)", decompressed.len()); + } else { + match serde_json::from_slice::>(&decompressed) { + Ok(entries) => { + debug!("parsed {} log entries", entries.len()); + store.append(entries); + } + Err(e) => { + // Try parsing as a single entry (some agent versions + // send objects rather than arrays). + if let Ok(entry) = + serde_json::from_slice::(&decompressed) + { + store.append(vec![entry]); + } else { + warn!("failed to parse log payload ({} bytes): {e}", decompressed.len()); + } + } + } + } + } + Err(e) => { + warn!("failed to decompress payload: {e}"); + } + } + } else { + trace!("non-log request: {method} {path}"); + } + + // Always return 200 OK — the agent may probe various health endpoints. + Ok(Response::builder() + .status(200) + .header("Content-Type", "application/json") + .body(Full::new(Bytes::from_static(b"{\"status\":\"ok\"}"))) + .expect("response builder should not fail")) +} + +fn decompress_if_needed<'a>(body: &'a [u8], content_encoding: &str) -> Result, io::Error> { + match content_encoding { + "gzip" => { + let mut decoder = GzDecoder::new(body); + let mut decompressed = Vec::new(); + io::Read::read_to_end(&mut decoder, &mut decompressed)?; + Ok(Cow::Owned(decompressed)) + } + "zstd" => { + let decompressed = + zstd::decode_all(body).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + Ok(Cow::Owned(decompressed)) + } + _ => Ok(Cow::Borrowed(body)), + } +} diff --git a/integration/lading_proptest/src/lib.rs b/integration/lading_proptest/src/lib.rs new file mode 100644 index 000000000..eccaf8a74 --- /dev/null +++ b/integration/lading_proptest/src/lib.rs @@ -0,0 +1,339 @@ +//! Property-based integration tests for the Datadog Agent logs pipeline. +//! +//! This crate feeds generated log data to the DD Agent, collects what the +//! agent sends to a fake intake endpoint, and asserts properties on the output. +//! +//! # Running +//! +//! ```bash +//! # Single case, container mode (default) +//! PROPTEST_CASES=1 cargo test -p lading_proptest -- --nocapture --test-threads=1 +//! +//! # Binary mode +//! DD_AGENT_BINARY=/path/to/agent PROPTEST_CASES=1 cargo test -p lading_proptest -- --nocapture --test-threads=1 +//! +//! # Specific scenario +//! PROPTEST_CASES=1 cargo test -p lading_proptest truncation -- --nocapture --test-threads=1 +//! ``` +//! +//! These tests require a DD Agent binary or Docker image. Set the agent target +//! via environment variables: +//! +//! - `DD_AGENT_IMAGE=datadog/agent:latest` for container mode (default) +//! - `DD_AGENT_BINARY=/path/to/agent` for binary mode + +pub mod agent; +pub mod config; +pub mod diagnostics; +pub mod intake; +pub mod log_format; +pub mod log_gen; +pub mod orchestrator; +pub mod property; +pub mod scenario; + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use crate::diagnostics; + use crate::log_format::LogFormat; + use crate::orchestrator::{self, OrchestratorConfig}; + use crate::scenario::Scenario; + use crate::scenario::json_multiline::JsonMultilineScenario; + use crate::scenario::mixed_multiline::MixedMultilineScenario; + use crate::scenario::multiline::MultilineScenario; + use crate::scenario::truncation::TruncationScenario; + + fn init_tracing() { + let _ = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_test_writer() + .try_init(); + } + + fn orchestrator_config() -> OrchestratorConfig { + init_tracing(); + OrchestratorConfig::default() + } + + /// Bridge proptest (sync) to tokio (async). + fn run_async(f: F) -> F::Output { + tokio::runtime::Runtime::new() + .expect("failed to create tokio runtime") + .block_on(f) + } + + proptest! { + // Default 1 case; override via PROPTEST_CASES env var (read by proptest natively). + #![proptest_config(ProptestConfig::with_cases(1))] + + #[test] + fn multiline_timestamp_prefixed( + params in crate::scenario::multiline::strategy_with_format( + LogFormat::TimestampPrefixed, + ) + ) { + let config = orchestrator_config(); + let result = run_async( + orchestrator::run_case::(&config, ¶ms) + ).expect("test infrastructure failure"); + + for prop_result in &result.property_results { + prop_assert!( + prop_result.is_ok(), + "{}", + diagnostics::format_failure(&result), + ); + } + } + + #[test] + fn multiline_syslog( + params in crate::scenario::multiline::strategy_with_format( + LogFormat::Syslog5424, + ) + ) { + let config = orchestrator_config(); + let result = run_async( + orchestrator::run_case::(&config, ¶ms) + ).expect("test infrastructure failure"); + + for prop_result in &result.property_results { + prop_assert!( + prop_result.is_ok(), + "{}", + diagnostics::format_failure(&result), + ); + } + } + + #[test] + fn multiline_apache( + params in crate::scenario::multiline::strategy_with_format( + LogFormat::ApacheCommon, + ) + ) { + let config = orchestrator_config(); + let result = run_async( + orchestrator::run_case::(&config, ¶ms) + ).expect("test infrastructure failure"); + + for prop_result in &result.property_results { + prop_assert!( + prop_result.is_ok(), + "{}", + diagnostics::format_failure(&result), + ); + } + } + + // PlainText multiline deferred — needs explicit regex patterns. + + #[test] + fn json_multiline( + params in JsonMultilineScenario::strategy() + ) { + let config = orchestrator_config(); + let result = run_async( + orchestrator::run_case::(&config, ¶ms) + ).expect("test infrastructure failure"); + + for prop_result in &result.property_results { + prop_assert!( + prop_result.is_ok(), + "{}", + diagnostics::format_failure(&result), + ); + } + } + + #[test] + fn mixed_multiline( + params in MixedMultilineScenario::strategy() + ) { + let config = orchestrator_config(); + let result = run_async( + orchestrator::run_case::(&config, ¶ms) + ).expect("test infrastructure failure"); + + for prop_result in &result.property_results { + prop_assert!( + prop_result.is_ok(), + "{}", + diagnostics::format_failure(&result), + ); + } + } + + #[test] + fn multiline_all_formats( + params in MultilineScenario::strategy() + ) { + let config = orchestrator_config(); + let result = run_async( + orchestrator::run_case::(&config, ¶ms) + ).expect("test infrastructure failure"); + + for prop_result in &result.property_results { + prop_assert!( + prop_result.is_ok(), + "{}", + diagnostics::format_failure(&result), + ); + } + } + + #[test] + fn truncation( + params in TruncationScenario::strategy() + ) { + let config = orchestrator_config(); + let result = run_async( + orchestrator::run_case::(&config, ¶ms) + ).expect("test infrastructure failure"); + + for prop_result in &result.property_results { + prop_assert!( + prop_result.is_ok(), + "{}", + diagnostics::format_failure(&result), + ); + } + } + + #[test] + fn truncation_plain_text( + params in crate::scenario::truncation::strategy_with_format( + LogFormat::PlainText, + ) + ) { + let config = orchestrator_config(); + let result = run_async( + orchestrator::run_case::(&config, ¶ms) + ).expect("test infrastructure failure"); + + for prop_result in &result.property_results { + prop_assert!( + prop_result.is_ok(), + "{}", + diagnostics::format_failure(&result), + ); + } + } + + #[test] + fn truncation_json( + params in crate::scenario::truncation::strategy_with_format( + LogFormat::Json, + ) + ) { + let config = orchestrator_config(); + let result = run_async( + orchestrator::run_case::(&config, ¶ms) + ).expect("test infrastructure failure"); + + for prop_result in &result.property_results { + prop_assert!( + prop_result.is_ok(), + "{}", + diagnostics::format_failure(&result), + ); + } + } + + #[test] + fn truncation_syslog( + params in crate::scenario::truncation::strategy_with_format( + LogFormat::Syslog5424, + ) + ) { + let config = orchestrator_config(); + let result = run_async( + orchestrator::run_case::(&config, ¶ms) + ).expect("test infrastructure failure"); + + for prop_result in &result.property_results { + prop_assert!( + prop_result.is_ok(), + "{}", + diagnostics::format_failure(&result), + ); + } + } + + #[test] + fn truncation_apache( + params in crate::scenario::truncation::strategy_with_format( + LogFormat::ApacheCommon, + ) + ) { + let config = orchestrator_config(); + let result = run_async( + orchestrator::run_case::(&config, ¶ms) + ).expect("test infrastructure failure"); + + for prop_result in &result.property_results { + prop_assert!( + prop_result.is_ok(), + "{}", + diagnostics::format_failure(&result), + ); + } + } + + #[test] + fn truncation_timestamp_prefixed( + params in crate::scenario::truncation::strategy_with_format( + LogFormat::TimestampPrefixed, + ) + ) { + let config = orchestrator_config(); + let result = run_async( + orchestrator::run_case::(&config, ¶ms) + ).expect("test infrastructure failure"); + + for prop_result in &result.property_results { + prop_assert!( + prop_result.is_ok(), + "{}", + diagnostics::format_failure(&result), + ); + } + } + } + + // Adaptive sampling uses run_action_sequence (not run_case), so it + // lives outside the proptest! macro as a regular proptest test. + proptest! { + #![proptest_config(ProptestConfig::with_cases(1))] + + #[test] + fn adaptive_sampling( + params in crate::scenario::adaptive_sampling::strategy() + ) { + let config = orchestrator_config(); + let seq = crate::scenario::adaptive_sampling::build_actions(¶ms); + let properties = crate::scenario::adaptive_sampling::build_properties(&seq); + let log_source = crate::scenario::adaptive_sampling::log_source_config(); + + let result = run_async( + orchestrator::run_action_sequence( + &config, + log_source, + None, + &seq.actions, + properties, + ) + ).expect("test infrastructure failure"); + + for prop_result in &result.property_results { + prop_assert!( + prop_result.is_ok(), + "{}", + diagnostics::format_failure(&result), + ); + } + } + } +} diff --git a/integration/lading_proptest/src/log_format.rs b/integration/lading_proptest/src/log_format.rs new file mode 100644 index 000000000..e5e8563cc --- /dev/null +++ b/integration/lading_proptest/src/log_format.rs @@ -0,0 +1,659 @@ +//! Log format generation with embedded UUID markers. +//! +//! Each [`LogFormat`] variant produces format-valid log lines with a UUID +//! embedded in a position that survives the agent's processing (truncation, +//! multiline aggregation, etc.). +//! +//! These are lightweight implementations that do NOT depend on `lading_payload`. +//! They produce content sufficient to trigger the agent's format-specific +//! behavior (JSON detection, timestamp detection, syslog parsing) without +//! needing full spec compliance. + +use proptest::prelude::*; + +/// Marker prefix used to identify proptest-generated log lines. +pub const PROPTEST_MARKER: &str = "PROPTEST"; + +/// Marker prefix for continuation lines in multiline scenarios. +pub const CONTINUATION_MARKER: &str = "CONT"; + +/// Supported log formats for test generation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LogFormat { + /// Plain text: `[PROPTEST:] ` + PlainText, + /// JSON object: `{"proptest_id":"","message":"","level":"info",...}` + Json, + /// RFC 5424 syslog: `<134>1 host app 1234 - - [PROPTEST:] ` + Syslog5424, + /// Apache Combined Log: `192.168.1.1 - - [timestamp] "GET //path HTTP/1.1" 200 ` + ApacheCommon, + /// Timestamp-prefixed: `2024-01-15T10:30:00.000Z [PROPTEST:] ` + /// + /// Primary format for multiline testing as it triggers the agent's datetime + /// detection. + TimestampPrefixed, +} + +impl LogFormat { + /// Format a single log line with the given UUID and content. + /// + /// Uses a fixed timestamp. For multiline scenarios where each header needs + /// a distinct timestamp, use [`format_multiline_header`] instead. + #[must_use] + pub fn format_line(&self, id: &str, content: &str) -> String { + self.format_line_with_index(id, 0, content) + } + + /// Format a log line with a timestamp that varies by `entry_index`. + /// + /// For formats with timestamps (`TimestampPrefixed`, `Syslog5424`, + /// `ApacheCommon`), the seconds field increments by `entry_index` so the + /// agent's auto multiline detector sees each header as a distinct + /// `startGroup` event. + #[must_use] + pub fn format_line_with_index(&self, id: &str, entry_index: usize, content: &str) -> String { + let secs = entry_index % 60; + match self { + Self::PlainText => { + format!("[{PROPTEST_MARKER}:{id}] {content}") + } + Self::Json => { + // Escape content for JSON safety. + let escaped = content.replace('\\', "\\\\").replace('"', "\\\""); + format!( + r#"{{"proptest_id":"{id}","message":"{escaped}","level":"info","logger":"proptest"}}"# + ) + } + Self::Syslog5424 => { + format!( + "<134>1 2024-01-15T10:30:{secs:02}.000000Z proptest.host proptest-app 1234 ID001 - [{PROPTEST_MARKER}:{id}] {content}" + ) + } + Self::ApacheCommon => { + let len = content.len(); + format!( + "192.168.1.42 - proptest [15/Jan/2024:10:30:{secs:02} +0000] \"GET /{id}/{} HTTP/1.1\" 200 {len}", + content.replace('"', "%22").replace(' ', "%20"), + ) + } + Self::TimestampPrefixed => { + format!("2024-01-15T10:30:{secs:02}.000Z [{PROPTEST_MARKER}:{id}] {content}") + } + } + } + + /// Format a continuation line for multiline scenarios. + /// + /// Continuation lines lack any leading structure (no timestamp, no syslog + /// header, no JSON opening brace) so the agent's auto multiline detection + /// labels them `aggregate` and appends them to the current buffer. + /// + /// No artificial indentation — the agent doesn't use whitespace for + /// detection, and we don't want to make aggregation easier than it would + /// be with real-world input. + #[must_use] + pub fn format_continuation(&self, header_id: &str, seq: usize, content: &str) -> String { + // All formats use the same continuation shape: just the marker + content. + // No indentation, no format-specific structure. + let _ = self; // format doesn't affect continuation lines + format!("[{CONTINUATION_MARKER}:{header_id}:{seq}] {content}") + } + + /// Extract the first proptest UUID from an agent output message. + /// + /// The agent may have modified the line (truncation, aggregation), but the + /// UUID marker should still be present if the line was delivered. + #[must_use] + pub fn extract_id(message: &str) -> Option<&str> { + Self::extract_all_ids(message).into_iter().next() + } + + /// Extract all proptest UUIDs from an agent output message. + /// + /// An aggregated message may contain multiple `[PROPTEST:]` markers + /// (e.g., when a plain text line is merged into a preceding entry's + /// buffer). This returns all of them. + #[must_use] + pub fn extract_all_ids(message: &str) -> Vec<&str> { + let mut ids = Vec::new(); + let marker = format!("[{PROPTEST_MARKER}:"); + + // Find all [PROPTEST:] patterns + let mut search_from = 0; + while let Some(start) = message[search_from..].find(&marker) { + let abs_start = search_from + start + marker.len(); + if let Some(end) = message[abs_start..].find(']') { + ids.push(&message[abs_start..abs_start + end]); + } + search_from = abs_start; + } + + // Find all proptest_id in JSON + let json_marker = "\"proptest_id\":\""; + search_from = 0; + while let Some(start) = message[search_from..].find(json_marker) { + let abs_start = search_from + start + json_marker.len(); + if let Some(end) = message[abs_start..].find('"') { + let id = &message[abs_start..abs_start + end]; + if !ids.contains(&id) { + ids.push(id); + } + } + search_from = abs_start; + } + + // Find UUID in Apache path: GET // + if let Some(start) = message.find("GET /") { + let after_prefix = start + "GET /".len(); + if let Some(end) = message[after_prefix..].find('/') { + let candidate = &message[after_prefix..after_prefix + end]; + if candidate.len() == 36 && candidate.contains('-') && !ids.contains(&candidate) { + ids.push(candidate); + } + } + } + + ids + } + + /// Extract continuation markers from an agent output message. + /// + /// Returns a list of `(header_id, sequence_number)` pairs found. + #[must_use] + pub fn extract_continuations(message: &str) -> Vec<(&str, usize)> { + let mut results = Vec::new(); + let marker = format!("[{CONTINUATION_MARKER}:"); + let mut search_from = 0; + while let Some(start) = message[search_from..].find(&marker) { + let abs_start = search_from + start + marker.len(); + if let Some(end) = message[abs_start..].find(']') { + let inner = &message[abs_start..abs_start + end]; + if let Some(colon) = inner.find(':') { + let header_id = &inner[..colon]; + if let Ok(seq) = inner[colon + 1..].parse::() { + results.push((header_id, seq)); + } + } + } + search_from = abs_start; + } + results + } +} + +// --- JSON structure generation for JSON multiline testing --- + +/// The shape of a JSON entry for multiline testing. +/// +/// Each variant generates structurally different valid JSON to exercise +/// different code paths in the agent's JSON aggregator. +#[derive(Debug, Clone, Copy)] +pub enum JsonStructure { + /// Flat object: `{"proptest_id":"uuid","f_0":"v","f_1":"v",...}` + Flat { + /// Number of additional fields beyond `proptest_id`. + field_count: usize, + }, + /// Nested object: `{"proptest_id":"uuid","data":{"f_0":"v",...}}` + Nested { + /// Number of fields inside the nested `data` object. + field_count: usize, + }, + /// Object containing an array of strings: `{"proptest_id":"uuid","items":["v0","v1",...]}` + WithArray { + /// Number of items in the array. + item_count: usize, + }, + /// Top-level array: `[{"proptest_id":"uuid","f_0":"v"}]` + /// + /// **Known agent limitation**: the agent's JSON aggregator does not support + /// top-level arrays. The incremental validator returns `Invalid` when it + /// sees `[` at `objCount == 0`, causing the buffer to flush unchanged. + /// This variant is included to detect if/when the agent adds array support. + TopLevelArray, + /// Deeply nested objects: `{"proptest_id":"uuid","l0":{"l1":{"l2":"val"}}}` + /// Tests brace depth tracking at 3+ levels. + DeepNested { + /// Nesting depth (2 = two levels of inner objects). + depth: usize, + }, + /// Array of objects inside an object: + /// `{"proptest_id":"uuid","items":[{"k0":"v0"},{"k1":"v1"}]}` + /// Tests mixed `[` and `{` bracket/brace tracking. + MixedNesting { + /// Number of objects in the array. + obj_count: usize, + }, + /// Object with an empty nested object: `{"proptest_id":"uuid","data":{}}` + EmptyObject, + /// Object with an empty array: `{"proptest_id":"uuid","items":[]}` + EmptyArray, + /// Object with escaped characters in string values. + /// Tests JSON escape sequences that could confuse line-level parsing. + EscapedStrings, + /// Object with Unicode values (CJK, emoji, accented chars). + UnicodeValues, + /// Object with non-string value types (numbers, booleans, null). + MixedValueTypes, +} + +impl JsonStructure { + /// Render the complete JSON value as a single string. + /// + /// # Panics + /// + /// Panics if JSON serialization fails (indicates a bug in generation). + #[must_use] + pub fn render_complete(&self, id: &str) -> String { + match self { + Self::Flat { field_count } => { + let extra: Vec = (0..*field_count) + .map(|i| format!(r#""f_{i}":"value_{i}""#)) + .collect(); + if extra.is_empty() { + format!(r#"{{"proptest_id":"{id}"}}"#) + } else { + format!(r#"{{"proptest_id":"{id}",{}}}"#, extra.join(",")) + } + } + Self::Nested { field_count } => { + let inner: Vec = (0..*field_count) + .map(|i| format!(r#""f_{i}":"value_{i}""#)) + .collect(); + format!( + r#"{{"proptest_id":"{id}","data":{{{}}}}}"#, + inner.join(",") + ) + } + Self::WithArray { item_count } => { + let items: Vec = (0..*item_count) + .map(|i| format!(r#""item_{i}""#)) + .collect(); + format!( + r#"{{"proptest_id":"{id}","items":[{}]}}"#, + items.join(",") + ) + } + Self::TopLevelArray => { + format!(r#"[{{"proptest_id":"{id}","f_0":"value_0"}}]"#) + } + Self::DeepNested { depth } => { + // Build from inside out: innermost has a value field + let mut json = r#""leaf":"deep_value""#.to_string(); + for level in (0..*depth).rev() { + json = format!(r#""l_{level}":{{{json}}}"#); + } + format!(r#"{{"proptest_id":"{id}",{json}}}"#) + } + Self::MixedNesting { obj_count } => { + let objects: Vec = (0..*obj_count) + .map(|i| format!(r#"{{"k_{i}":"v_{i}"}}"#)) + .collect(); + format!( + r#"{{"proptest_id":"{id}","items":[{}]}}"#, + objects.join(",") + ) + } + Self::EmptyObject => { + format!(r#"{{"proptest_id":"{id}","data":{{}}}}"#) + } + Self::EmptyArray => { + format!(r#"{{"proptest_id":"{id}","items":[]}}"#) + } + Self::EscapedStrings => { + // Use serde_json to ensure valid escaping + let obj = serde_json::json!({ + "proptest_id": id, + "msg": "line1\nline2\ttab", + "path": "C:\\foo\\bar", + "quote": "he said \"hello\"" + }); + serde_json::to_string(&obj).expect("serde_json serialization cannot fail") + } + Self::UnicodeValues => { + let obj = serde_json::json!({ + "proptest_id": id, + "cjk": "テスト", + "emoji": "🔥", + "accented": "café" + }); + serde_json::to_string(&obj).expect("serde_json serialization cannot fail") + } + Self::MixedValueTypes => { + format!( + r#"{{"proptest_id":"{id}","count":42,"ratio":3.14,"active":true,"data":null}}"# + ) + } + } + } + + /// Split the JSON into multiple lines for multiline file tailing. + /// + /// Splits after commas and opening braces — the same places a + /// pretty-printer would insert line breaks. + #[must_use] + #[expect(clippy::too_many_lines)] + pub fn render_lines(&self, id: &str) -> Vec { + match self { + Self::Flat { field_count } => { + let mut lines = Vec::with_capacity(field_count + 1); + // First line: opening brace + proptest_id field + trailing comma + if *field_count > 0 { + lines.push(format!(r#"{{"proptest_id":"{id}","#)); + for i in 0..*field_count { + if i == field_count - 1 { + // Last field: include closing brace + lines.push(format!(r#""f_{i}":"value_{i}"}}"#)); + } else { + lines.push(format!(r#""f_{i}":"value_{i}","#)); + } + } + } else { + lines.push(format!(r#"{{"proptest_id":"{id}"}}"#)); + } + lines + } + Self::Nested { field_count } => { + let mut lines = Vec::new(); + lines.push(format!(r#"{{"proptest_id":"{id}","#)); + if *field_count > 0 { + lines.push(r#""data":{"#.to_string()); + for i in 0..*field_count { + if i == field_count - 1 { + lines.push(format!(r#""f_{i}":"value_{i}"}}}}"#)); + } else { + lines.push(format!(r#""f_{i}":"value_{i}","#)); + } + } + } else { + lines.push(r#""data":{}}"#.to_string()); + } + lines + } + Self::WithArray { item_count } => { + let mut lines = Vec::new(); + lines.push(format!(r#"{{"proptest_id":"{id}","#)); + let items: Vec = (0..*item_count) + .map(|i| format!(r#""item_{i}""#)) + .collect(); + lines.push(format!(r#""items":[{}]}}"#, items.join(","))); + lines + } + Self::TopLevelArray => { + vec![ + "[".to_string(), + format!(r#"{{"proptest_id":"{id}","f_0":"value_0"}}"#), + "]".to_string(), + ] + } + Self::DeepNested { depth } => { + let mut lines = Vec::new(); + lines.push(format!(r#"{{"proptest_id":"{id}","#)); + // One line per nesting level opening + for level in 0..*depth { + lines.push(format!(r#""l_{level}":{{"#)); + } + // Innermost value + all closing braces + let closing: String = "}".repeat(depth + 1); + lines.push(format!(r#""leaf":"deep_value"{closing}"#)); + lines + } + Self::MixedNesting { obj_count } => { + let mut lines = Vec::new(); + lines.push(format!(r#"{{"proptest_id":"{id}","#)); + lines.push(r#""items":["#.to_string()); + for i in 0..*obj_count { + let comma = if i < obj_count - 1 { "," } else { "" }; + lines.push(format!(r#"{{"k_{i}":"v_{i}"}}{comma}"#)); + } + lines.push("]}".to_string()); + lines + } + Self::EmptyObject => { + vec![ + format!(r#"{{"proptest_id":"{id}","#), + r#""data":{}}"#.to_string(), + ] + } + Self::EmptyArray => { + vec![ + format!(r#"{{"proptest_id":"{id}","#), + r#""items":[]}"#.to_string(), + ] + } + Self::EscapedStrings => { + // Get the complete valid JSON, then split it into lines + // by breaking after known field boundaries. + let complete = self.render_complete(id); + split_json_after_fields(&complete, 1) + } + Self::UnicodeValues => { + let complete = self.render_complete(id); + split_json_after_fields(&complete, 1) + } + Self::MixedValueTypes => { + vec![ + format!(r#"{{"proptest_id":"{id}","#), + r#""count":42,"#.to_string(), + r#""ratio":3.14,"#.to_string(), + r#""active":true,"#.to_string(), + r#""data":null}"#.to_string(), + ] + } + } + } + + /// Build the expected JSON value for property assertion. + /// + /// Returns the parsed JSON that the agent should produce after + /// compaction (for objects) or pass-through (for arrays). + /// + /// # Panics + /// + /// Panics if the generated JSON is not valid (indicates a bug in + /// the generation logic). + #[must_use] + pub fn expected_json(&self, id: &str) -> serde_json::Value { + let complete = self.render_complete(id); + serde_json::from_str(&complete).expect("generated JSON must be valid") + } +} + +/// Split a JSON string into multiple lines by breaking after commas that +/// follow complete string values. Splits after the first `n_first_fields` +/// comma-separated fields on the first line, then puts the rest on a second line. +fn split_json_after_fields(json: &str, n_first_fields: usize) -> Vec { + // Find the Nth comma at the top level (not inside strings) + let mut commas_found = 0; + let mut in_string = false; + let mut escape_next = false; + + for (i, ch) in json.char_indices() { + if escape_next { + escape_next = false; + continue; + } + if ch == '\\' && in_string { + escape_next = true; + continue; + } + if ch == '"' { + in_string = !in_string; + continue; + } + if !in_string && ch == ',' { + commas_found += 1; + if commas_found == n_first_fields { + // Split here: everything up to and including this comma on line 1 + return vec![ + json[..=i].to_string(), + json[i + 1..].to_string(), + ]; + } + } + } + + // Couldn't split — return as single line + vec![json.to_string()] +} + +/// Proptest strategy that generates a [`JsonStructure`] variant. +/// +/// Generates a mix of all 11 structural variants to maximize the chance +/// of finding edge cases in the agent's JSON processing. +pub fn json_structure_strategy() -> impl Strategy { + prop_oneof![ + // Parameterized variants + (2_usize..8).prop_map(|field_count| JsonStructure::Flat { field_count }), + (1_usize..5).prop_map(|field_count| JsonStructure::Nested { field_count }), + (1_usize..6).prop_map(|item_count| JsonStructure::WithArray { item_count }), + (2_usize..5).prop_map(|depth| JsonStructure::DeepNested { depth }), + (1_usize..4).prop_map(|obj_count| JsonStructure::MixedNesting { obj_count }), + // Fixed variants + Just(JsonStructure::TopLevelArray), + Just(JsonStructure::EmptyObject), + Just(JsonStructure::EmptyArray), + Just(JsonStructure::EscapedStrings), + Just(JsonStructure::UnicodeValues), + Just(JsonStructure::MixedValueTypes), + ] +} + +/// Proptest strategy that generates a [`LogFormat`] variant. +pub fn log_format_strategy() -> impl Strategy { + prop_oneof![ + Just(LogFormat::PlainText), + Just(LogFormat::Json), + Just(LogFormat::Syslog5424), + Just(LogFormat::ApacheCommon), + Just(LogFormat::TimestampPrefixed), + ] +} + +/// Proptest strategy for log formats suitable for multiline testing. +/// +/// Only includes formats with timestamps that the agent's auto multiline +/// detector recognizes as `startGroup` signals. `PlainText` (no timestamp) +/// and `Json` (complete objects get `noAggregate`) are excluded. +pub fn multiline_format_strategy() -> impl Strategy { + prop_oneof![ + Just(LogFormat::TimestampPrefixed), + Just(LogFormat::Syslog5424), + Just(LogFormat::ApacheCommon), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plain_text_roundtrip() { + let id = "550e8400-e29b-41d4-a716-446655440000"; + let line = LogFormat::PlainText.format_line(id, "hello world"); + assert_eq!( + line, + "[PROPTEST:550e8400-e29b-41d4-a716-446655440000] hello world" + ); + assert_eq!(LogFormat::extract_id(&line), Some(id)); + } + + #[test] + fn json_roundtrip() { + let id = "550e8400-e29b-41d4-a716-446655440000"; + let line = LogFormat::Json.format_line(id, "hello world"); + assert!(line.starts_with('{')); + assert!(line.contains("proptest_id")); + assert_eq!(LogFormat::extract_id(&line), Some(id)); + } + + #[test] + fn syslog_roundtrip() { + let id = "550e8400-e29b-41d4-a716-446655440000"; + let line = LogFormat::Syslog5424.format_line(id, "test message"); + assert!(line.starts_with("<134>")); + assert_eq!(LogFormat::extract_id(&line), Some(id)); + } + + #[test] + fn apache_roundtrip() { + let id = "550e8400-e29b-41d4-a716-446655440000"; + let line = LogFormat::ApacheCommon.format_line(id, "test"); + assert!(line.starts_with("192.168.1.42")); + assert_eq!(LogFormat::extract_id(&line), Some(id)); + } + + #[test] + fn timestamp_prefixed_roundtrip() { + let id = "550e8400-e29b-41d4-a716-446655440000"; + let line = LogFormat::TimestampPrefixed.format_line(id, "test message"); + assert!(line.starts_with("2024-")); + assert_eq!(LogFormat::extract_id(&line), Some(id)); + } + + #[test] + fn continuation_extraction() { + let header_id = "abc123"; + let line = LogFormat::PlainText.format_continuation(header_id, 2, "continuation text"); + let continuations = LogFormat::extract_continuations(&line); + assert_eq!(continuations.len(), 1); + assert_eq!(continuations[0], ("abc123", 2)); + } + + #[test] + fn extract_continuations_from_joined_output() { + // The agent joins aggregated lines with literal \n (0x5C 0x6E), + // not actual newlines. Verify extraction works with this format. + let header_id = "abc123"; + let joined = format!( + "2024-01-15T10:30:00.000Z [PROPTEST:{header_id}] header content\\n [CONT:{header_id}:0] cont 0\\n [CONT:{header_id}:1] cont 1" + ); + let continuations = LogFormat::extract_continuations(&joined); + assert_eq!(continuations.len(), 2); + assert_eq!(continuations[0], ("abc123", 0)); + assert_eq!(continuations[1], ("abc123", 1)); + } + + #[test] + fn multiline_header_varies_timestamp() { + let id = "test-uuid"; + let line0 = LogFormat::TimestampPrefixed.format_line_with_index(id, 0, "msg"); + let line1 = LogFormat::TimestampPrefixed.format_line_with_index(id, 1, "msg"); + let line2 = LogFormat::TimestampPrefixed.format_line_with_index(id, 2, "msg"); + assert!(line0.contains("10:30:00")); + assert!(line1.contains("10:30:01")); + assert!(line2.contains("10:30:02")); + // All should still have extractable IDs + assert_eq!(LogFormat::extract_id(&line0), Some(id)); + assert_eq!(LogFormat::extract_id(&line1), Some(id)); + } + + #[test] + fn syslog_multiline_header_varies_timestamp() { + let id = "test-uuid"; + let line0 = LogFormat::Syslog5424.format_line_with_index(id, 0, "msg"); + let line3 = LogFormat::Syslog5424.format_line_with_index(id, 3, "msg"); + assert!(line0.contains("10:30:00.000000Z")); + assert!(line3.contains("10:30:03.000000Z")); + } + + #[test] + fn apache_multiline_header_varies_timestamp() { + let id = "test-uuid"; + let line0 = LogFormat::ApacheCommon.format_line_with_index(id, 0, "msg"); + let line5 = LogFormat::ApacheCommon.format_line_with_index(id, 5, "msg"); + assert!(line0.contains("10:30:00 +0000")); + assert!(line5.contains("10:30:05 +0000")); + } + + #[test] + fn extract_id_from_truncated_line() { + // UUID is near the start, so it should survive truncation + let id = "550e8400-e29b-41d4-a716-446655440000"; + let line = LogFormat::PlainText.format_line(id, "very long content"); + // Simulate truncation: take only the first 80 chars + let truncated = &line[..line.len().min(80)]; + assert_eq!(LogFormat::extract_id(truncated), Some(id)); + } +} diff --git a/integration/lading_proptest/src/log_gen.rs b/integration/lading_proptest/src/log_gen.rs new file mode 100644 index 000000000..a94d13c12 --- /dev/null +++ b/integration/lading_proptest/src/log_gen.rs @@ -0,0 +1,237 @@ +//! Proptest strategies for generating unique log lines. +//! +//! Each generated line contains a UUID marker for input-output correlation +//! during property assertion. See [`crate::log_format`] for format details. + +use proptest::prelude::*; + +use crate::log_format::LogFormat; + +/// A single log line with a unique identifier. +#[derive(Debug, Clone)] +pub struct LogLine { + /// Unique identifier for this line (UUID). + pub id: String, + /// The full text content of the line as written to disk. + pub content: String, +} + +/// A batch of log lines representing one test case's input. +#[derive(Debug, Clone)] +pub struct LogBatch { + /// The individual log lines. + pub lines: Vec, + /// The format used to generate these lines. + pub format: LogFormat, + /// For multiline scenarios: maps header UUID → expected continuation count. + /// Empty for non-multiline scenarios. + pub expected_continuations: Vec<(String, usize)>, + /// For JSON multiline scenarios: maps UUID → expected JSON value. + /// `None` for non-JSON scenarios. + pub expected_json: Option>, +} + +/// A multiline log entry consisting of a header and continuation lines. +#[derive(Debug, Clone)] +pub struct MultilineEntry { + /// The header line (starts the logical entry). + pub header: LogLine, + /// Continuation lines that should be aggregated with the header. + pub continuations: Vec, +} + +/// Strategy for generating a batch of simple single-line logs. +/// +/// Each line is unique (distinct UUID) and formatted according to the given +/// [`LogFormat`]. +pub fn simple_log_batch( + format: LogFormat, + line_count: impl Strategy, + line_len_range: std::ops::Range, +) -> impl Strategy { + line_count.prop_flat_map(move |count| { + proptest::collection::vec(line_len_range.clone(), count).prop_map(move |lens| { + let mut lines = Vec::with_capacity(lens.len()); + for len in &lens { + let id = uuid::Uuid::new_v4().to_string(); + // Generate deterministic content based on line index for + // simplicity; the UUID provides uniqueness. + let filler: String = (0..*len) + .map(|i| char::from(b'a' + u8::try_from(i % 26).unwrap_or(0))) + .collect(); + let content = format.format_line(&id, &filler); + lines.push(LogLine { id, content }); + } + LogBatch { lines, format, expected_continuations: Vec::new(), expected_json: None } + }) + }) +} + +/// Strategy for generating multiline log entries. +/// +/// Each entry has a header line (with timestamp/structure per format) and +/// 0-N continuation lines (indented, no leading structure). +pub fn multiline_log_batch( + format: LogFormat, + entry_count: impl Strategy, + max_continuations: usize, +) -> impl Strategy { + entry_count.prop_flat_map(move |count| { + proptest::collection::vec(0_usize..=max_continuations, count).prop_map( + move |continuation_counts| { + let mut lines = Vec::new(); + for cont_count in &continuation_counts { + let header_id = uuid::Uuid::new_v4().to_string(); + let header_content = format.format_line(&header_id, "log entry header"); + lines.push(LogLine { + id: header_id.clone(), + content: header_content, + }); + for seq in 0..*cont_count { + let cont_content = format.format_continuation( + &header_id, + seq, + &format!("continuation line {seq}"), + ); + // Continuation lines share the header's ID conceptually + // but get their own line entry. The property checker uses + // the CONT marker to associate them. + lines.push(LogLine { + id: format!("{header_id}:cont:{seq}"), + content: cont_content, + }); + } + } + LogBatch { lines, format, expected_continuations: Vec::new(), expected_json: None } + }, + ) + }) +} + +/// Strategy for generating lines near truncation boundaries. +/// +/// Produces a mix of: +/// - Lines well under the limit +/// - Lines near the boundary (+/- 100 bytes) +/// - Lines well over the limit +pub fn truncation_log_batch( + format: LogFormat, + line_count: impl Strategy, + max_message_bytes: usize, +) -> impl Strategy { + line_count.prop_flat_map(move |count| { + // Generate a mix of line length categories + proptest::collection::vec( + prop_oneof![ + // Well under limit (100-1000 bytes) + 100_usize..1000, + // Near boundary (limit - 100 to limit + 100) + (max_message_bytes.saturating_sub(100))..=(max_message_bytes + 100), + // Well over limit (limit + 1000 to limit * 2) + (max_message_bytes + 1000)..=(max_message_bytes * 2), + ], + count, + ) + .prop_map(move |lengths| { + let mut lines = Vec::with_capacity(lengths.len()); + for target_len in &lengths { + let id = uuid::Uuid::new_v4().to_string(); + // The format_line call adds overhead (markers, format structure). + // We generate filler content sized so the total line is + // approximately target_len bytes. + let overhead_estimate = format.format_line(&id, "").len(); + let filler_len = target_len.saturating_sub(overhead_estimate); + let filler: String = (0..filler_len) + .map(|i| char::from(b'a' + u8::try_from(i % 26).unwrap_or(0))) + .collect(); + let content = format.format_line(&id, &filler); + lines.push(LogLine { id, content }); + } + LogBatch { lines, format, expected_continuations: Vec::new(), expected_json: None } + }) + }) +} + +/// Strategy that generates a [`LogFormat`] alongside a simple log batch. +/// +/// Useful for tests that should exercise all formats. +pub fn simple_log_batch_any_format() -> impl Strategy { + crate::log_format::log_format_strategy().prop_flat_map(|format| { + simple_log_batch(format, 3_usize..20, 10..100) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use proptest::test_runner::{Config, TestRunner}; + + #[test] + fn simple_batch_has_unique_ids() { + let mut runner = TestRunner::new(Config::with_cases(10)); + runner + .run( + &simple_log_batch(LogFormat::PlainText, 5_usize..20, 10..100), + |batch| { + let ids: rustc_hash::FxHashSet<&str> = + batch.lines.iter().map(|l| l.id.as_str()).collect(); + // All IDs should be unique + prop_assert_eq!(ids.len(), batch.lines.len()); + Ok(()) + }, + ) + .unwrap(); + } + + #[test] + fn multiline_batch_has_headers_and_continuations() { + let mut runner = TestRunner::new(Config::with_cases(10)); + runner + .run( + &multiline_log_batch(LogFormat::TimestampPrefixed, 3_usize..10, 3), + |batch| { + // Should have at least as many lines as entries (headers) + prop_assert!(!batch.lines.is_empty()); + // First line should be a header (not a continuation) + prop_assert!( + !batch.lines[0].content.contains("CONT:"), + "first line should be a header" + ); + Ok(()) + }, + ) + .unwrap(); + } + + #[test] + fn truncation_batch_has_mixed_sizes() { + let limit = 256 * 1024; // 256 KB + let mut runner = TestRunner::new(Config::with_cases(10)); + runner + .run( + &truncation_log_batch(LogFormat::PlainText, Just(9_usize), limit), + |batch| { + // With 9 lines and 3 categories, we should have a mix + let under: Vec<_> = batch + .lines + .iter() + .filter(|l| l.content.len() < limit - 100) + .collect(); + let over: Vec<_> = batch + .lines + .iter() + .filter(|l| l.content.len() > limit + 100) + .collect(); + // At least some should be under and some over + // (probabilistic, but with 9 lines across 3 categories + // it's very likely) + prop_assert!( + !under.is_empty() || !over.is_empty(), + "should have a mix of sizes" + ); + Ok(()) + }, + ) + .unwrap(); + } +} diff --git a/integration/lading_proptest/src/orchestrator.rs b/integration/lading_proptest/src/orchestrator.rs new file mode 100644 index 000000000..27b1158f0 --- /dev/null +++ b/integration/lading_proptest/src/orchestrator.rs @@ -0,0 +1,467 @@ +//! Deterministic test execution engine. +//! +//! The orchestrator manages the full lifecycle of a single proptest case: +//! start the intake server, configure and launch the agent, feed logs, +//! wait for drain, collect output, and check properties. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use tempfile::TempDir; +use tokio::io::AsyncWriteExt; +use tracing::{debug, info}; + +use crate::agent::AgentTarget; +use crate::config::{self, AgentConfigParams, LogSourceConfig}; +use crate::intake::{LogIntakeServer, ReceivedLogEntry}; +use crate::log_gen::{LogBatch, LogLine}; +use crate::property::{Property, PropertyFailure}; +use crate::scenario::Scenario; + +/// Return a temp directory base that is visible inside Docker VMs. +/// +/// Colima/Lima only mount `$HOME` by default. The system temp dir +/// (`/var/folders/` on macOS) is not visible inside the VM, so bind +/// mounts from there fail. We use `$HOME/.lading_proptest_tmp` instead. +fn dirs_or_home() -> PathBuf { + let base = PathBuf::from( + std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()), + ) + .join(".lading_proptest_tmp"); + std::fs::create_dir_all(&base).expect("failed to create temp base dir"); + base +} + +/// Errors from the orchestrator. +#[derive(thiserror::Error, Debug)] +pub enum Error { + /// I/O error. + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + /// Intake server error. + #[error("intake server error: {0}")] + Intake(#[from] crate::intake::Error), + /// Agent error. + #[error("agent error: {0}")] + Agent(#[from] crate::agent::Error), + /// Configuration error. + #[error("config error: {0}")] + Config(#[from] crate::config::Error), +} + +/// Configuration for the orchestrator. +#[derive(Debug, Clone)] +pub struct OrchestratorConfig { + /// How to run the agent. + pub agent_target: AgentTarget, + /// Duration to wait after agent readiness before writing logs, + /// giving the logs pipeline time to initialize and start tailing. + pub pipeline_warmup: Duration, + /// Duration to wait after last log sent before collecting output. + pub drain_timeout: Duration, + /// Maximum time to wait for agent readiness. + pub readiness_timeout: Duration, + /// Whether the agent should use compression. + pub use_compression: bool, + /// Agent batch wait time in ms (lower = faster drain). + pub batch_wait_ms: u64, + /// Always preserve the temp directory, even on success. + /// Set via `LADING_KEEP_TEMP=1`. + pub keep_temp: bool, +} + +impl Default for OrchestratorConfig { + fn default() -> Self { + Self { + agent_target: AgentTarget::from_env(), + pipeline_warmup: Duration::from_secs(30), + drain_timeout: Duration::from_secs(15), + readiness_timeout: Duration::from_secs(60), + use_compression: true, + batch_wait_ms: 1000, + keep_temp: std::env::var("LADING_KEEP_TEMP").is_ok(), + } + } +} + +/// Result of a single test case execution. +#[derive(Debug)] +pub struct TestCaseResult { + /// The input log batch. + pub input: LogBatch, + /// The received output entries. + pub output: Vec, + /// Results of each property check. + pub property_results: Vec>, + /// Path to the temp directory (persisted on failure for debugging). + temp_dir_path: PathBuf, + /// The temp directory handle (dropped on success to clean up). + _temp_dir: Option, +} + +impl TestCaseResult { + /// Path to the temp directory containing configs, logs, and agent output. + #[must_use] + pub fn temp_dir(&self) -> &Path { + &self.temp_dir_path + } + + /// Whether all properties passed. + #[must_use] + pub fn all_passed(&self) -> bool { + self.property_results.iter().all(Result::is_ok) + } +} + +/// Run a single proptest case to completion. +/// +/// # Errors +/// +/// Returns error if any infrastructure step fails. Property failures +/// are returned in the result, not as errors. +/// +/// # Panics +/// +/// Panics if the log directory path is not valid UTF-8. +/// +/// This function: +/// 1. Creates a temp directory for configs and log files +/// 2. Starts the log intake server (ephemeral port) +/// 3. Writes agent configuration +/// 4. Starts the agent +/// 5. Waits for agent readiness +/// 6. Writes log lines to the file +/// 7. Waits for drain (`drain_timeout` after last write) +/// 8. Stops the agent +/// 9. Collects output from the intake server +/// 10. Checks all properties +/// +/// # Errors +/// +/// Returns error if any infrastructure step fails. Property failures +/// are returned in the result, not as errors. +pub async fn run_case( + config: &OrchestratorConfig, + params: &S::Params, +) -> Result { + // 1. Create temp directory under $HOME so it's visible inside Docker VMs + // (Colima/Lima only mount $HOME by default, not /var/folders/). + let temp_base = dirs_or_home(); + let temp_dir = TempDir::with_prefix_in("lading_proptest_", temp_base)?; + let temp_path = temp_dir.path().to_owned(); + let config_dir = temp_path.join("config"); + let log_dir = temp_path.join("logs"); + std::fs::create_dir_all(&config_dir)?; + std::fs::create_dir_all(&log_dir)?; + + info!("test case temp dir: {}", temp_path.display()); + + // 2. Start intake server + let intake = LogIntakeServer::start().await?; + let intake_port = intake.port(); + debug!("intake server on port {intake_port}"); + + // 3. Write agent config + let log_file_path = match &config.agent_target { + AgentTarget::Container(_) => "/var/log/proptest/proptest.log".to_string(), + AgentTarget::Binary(_) => { + log_dir + .join("proptest.log") + .to_str() + .expect("log path must be valid UTF-8") + .to_string() + } + }; + + let agent_config_params = AgentConfigParams { + intake_port, + log_file_path, + use_compression: config.use_compression, + batch_wait_ms: config.batch_wait_ms, + log_source_config: S::log_source_config(params), + max_message_size_bytes: S::max_message_size_bytes(params), + }; + config::write_agent_config(&config_dir, &agent_config_params)?; + + // 4. Start agent + let agent = config.agent_target.start(&config_dir, &log_dir, intake_port).await?; + + // 5. Wait for readiness + agent.wait_ready(config.readiness_timeout).await?; + debug!("agent is ready, waiting for logs pipeline to initialize"); + + // Give the agent time to fully initialize its logs pipeline and start + // tailing files. The container being "running" doesn't mean the logs + // pipeline is ready. + tokio::time::sleep(config.pipeline_warmup).await; + debug!("pipeline warmup complete"); + + // 6. Generate and write log lines + let input = S::generate_input(params); + let log_file = log_dir.join("proptest.log"); + write_log_batch(&log_file, &input).await?; + info!("wrote {} log lines to {}", input.lines.len(), log_file.display()); + + // 7. Wait for drain + debug!("waiting {:?} for drain", config.drain_timeout); + tokio::time::sleep(config.drain_timeout).await; + + // 8. Stop agent + agent.stop().await?; + debug!("agent stopped"); + + // 9. Collect output + let output = intake.stop().await; + info!("collected {} output entries", output.len()); + + // Dump output and summary to temp dir for inspection + dump_output(&temp_path, &output)?; + let proptest_output = output + .iter() + .filter(|e| crate::log_format::LogFormat::extract_id(&e.message).is_some()) + .count(); + dump_summary(&temp_path, &[ + "=== Single Batch ===".to_string(), + format!("Lines written: {}", input.lines.len()), + format!("Total output entries: {}", output.len()), + format!("Proptest output entries: {proptest_output}"), + ])?; + + // 10. Check properties + let properties = S::properties(params); + let property_results: Vec> = properties + .iter() + .map(|prop| prop.check(&input, &output)) + .collect(); + + let all_passed = property_results.iter().all(Result::is_ok); + + Ok(TestCaseResult { + input, + output, + property_results, + temp_dir_path: temp_path, + _temp_dir: if !all_passed || config.keep_temp { + let path = temp_dir.keep(); + info!("preserving temp dir: {}", path.display()); + None + } else { + Some(temp_dir) + }, + }) +} + +/// Write a log batch to a file, one line per line. +async fn write_log_batch(path: &Path, batch: &LogBatch) -> Result<(), std::io::Error> { + let mut file = tokio::fs::File::create(path).await?; + for line in &batch.lines { + file.write_all(line.content.as_bytes()).await?; + file.write_all(b"\n").await?; + } + file.flush().await?; + Ok(()) +} + +// --- Action Sequence Support --- + +/// A step in an action sequence. +#[derive(Debug, Clone)] +pub enum Action { + /// Append lines to the log file. + WriteLines(Vec), + /// Sleep for a duration (e.g., to allow credit refill). + Sleep(Duration), +} + +/// Run an action sequence against the agent. +/// +/// Like [`run_case`] but instead of writing all logs at once, executes a +/// sequence of write and sleep actions. Used for scenarios that depend on +/// timing (e.g., adaptive sampling credit refill). +/// +/// # Errors +/// +/// Returns error if any infrastructure step fails. +/// +/// # Panics +/// +/// Panics if the log directory path is not valid UTF-8. +#[expect(clippy::too_many_lines)] +pub async fn run_action_sequence( + config: &OrchestratorConfig, + log_source_config: LogSourceConfig, + max_message_size_bytes: Option, + actions: &[Action], + properties: Vec>, +) -> Result { + let temp_base = dirs_or_home(); + let temp_dir = TempDir::with_prefix_in("lading_proptest_", temp_base)?; + let temp_path = temp_dir.path().to_owned(); + let config_dir = temp_path.join("config"); + let log_dir = temp_path.join("logs"); + std::fs::create_dir_all(&config_dir)?; + std::fs::create_dir_all(&log_dir)?; + + info!("test case temp dir: {}", temp_path.display()); + + let intake = LogIntakeServer::start().await?; + let intake_port = intake.port(); + debug!("intake server on port {intake_port}"); + + let log_file_path = match &config.agent_target { + AgentTarget::Container(_) => "/var/log/proptest/proptest.log".to_string(), + AgentTarget::Binary(_) => { + log_dir + .join("proptest.log") + .to_str() + .expect("log path must be valid UTF-8") + .to_string() + } + }; + + let agent_config_params = AgentConfigParams { + intake_port, + log_file_path, + use_compression: config.use_compression, + batch_wait_ms: config.batch_wait_ms, + log_source_config, + max_message_size_bytes, + }; + config::write_agent_config(&config_dir, &agent_config_params)?; + + let agent = config.agent_target.start(&config_dir, &log_dir, intake_port).await?; + agent.wait_ready(config.readiness_timeout).await?; + debug!("agent is ready, waiting for logs pipeline to initialize"); + tokio::time::sleep(config.pipeline_warmup).await; + debug!("pipeline warmup complete"); + + // Execute actions, collecting all written lines and a summary log + let mut all_lines = Vec::new(); + let mut summary_lines: Vec = Vec::new(); + let log_file = log_dir.join("proptest.log"); + let mut total_lines_written: usize = 0; + + summary_lines.push("=== Action Sequence ===".to_string()); + + for (step_idx, action) in actions.iter().enumerate() { + match action { + Action::WriteLines(lines) => { + append_lines(&log_file, lines).await?; + let first_id = lines.first().map_or("?", |l| l.id.as_str()); + let last_id = lines.last().map_or("?", |l| l.id.as_str()); + summary_lines.push(format!( + "Step {step_idx}: Write {} lines (IDs {first_id}..{last_id})", + lines.len(), + )); + info!("wrote {} lines to {}", lines.len(), log_file.display()); + total_lines_written += lines.len(); + all_lines.extend_from_slice(lines); + } + Action::Sleep(duration) => { + summary_lines.push(format!("Step {step_idx}: Sleep {duration:?}")); + info!("sleeping {:?}", duration); + tokio::time::sleep(*duration).await; + } + } + } + + summary_lines.push(format!("\nTotal lines written: {total_lines_written}")); + + // Drain + debug!("waiting {:?} for drain", config.drain_timeout); + tokio::time::sleep(config.drain_timeout).await; + + agent.stop().await?; + debug!("agent stopped"); + + let output = intake.stop().await; + info!("collected {} output entries", output.len()); + + summary_lines.push(format!("Total output entries: {}", output.len())); + + // Count proptest entries vs agent-internal entries + let proptest_output = output + .iter() + .filter(|e| crate::log_format::LogFormat::extract_id(&e.message).is_some()) + .count(); + summary_lines.push(format!("Proptest output entries: {proptest_output}")); + summary_lines.push(format!( + "Lines dropped: {}", + total_lines_written.saturating_sub(proptest_output), + )); + + dump_output(&temp_path, &output)?; + dump_summary(&temp_path, &summary_lines)?; + + let input = LogBatch { + lines: all_lines, + format: crate::log_format::LogFormat::PlainText, + expected_continuations: Vec::new(), + expected_json: None, + }; + + let property_results: Vec> = properties + .iter() + .map(|prop| prop.check(&input, &output)) + .collect(); + + let all_passed = property_results.iter().all(Result::is_ok); + + Ok(TestCaseResult { + input, + output, + property_results, + temp_dir_path: temp_path, + _temp_dir: if !all_passed || config.keep_temp { + let path = temp_dir.keep(); + info!("preserving temp dir: {}", path.display()); + None + } else { + Some(temp_dir) + }, + }) +} + +/// Append lines to an existing file (or create it). +async fn append_lines(path: &Path, lines: &[LogLine]) -> Result<(), std::io::Error> { + use tokio::fs::OpenOptions; + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(path) + .await?; + for line in lines { + file.write_all(line.content.as_bytes()).await?; + file.write_all(b"\n").await?; + } + file.flush().await?; + Ok(()) +} + +/// Dump received output entries to the temp dir for manual inspection. +fn dump_output(temp_path: &Path, output: &[ReceivedLogEntry]) -> Result<(), std::io::Error> { + use std::io::Write; + + // Raw JSON array of all received entries + let json_path = temp_path.join("output.json"); + let json = serde_json::to_string_pretty(output).unwrap_or_else(|_| "[]".to_string()); + std::fs::write(&json_path, json)?; + + // One message per line (easier to diff against input) + let messages_path = temp_path.join("output_messages.txt"); + let mut f = std::fs::File::create(&messages_path)?; + for (i, entry) in output.iter().enumerate() { + writeln!(f, "[{i}] ({} bytes) {}", entry.message.len(), entry.message)?; + } + + info!("output dumped to {} and {}", json_path.display(), messages_path.display()); + Ok(()) +} + +/// Dump a summary of the test case execution. +fn dump_summary(temp_path: &Path, lines: &[String]) -> Result<(), std::io::Error> { + let path = temp_path.join("summary.txt"); + std::fs::write(&path, lines.join("\n"))?; + info!("summary dumped to {}", path.display()); + Ok(()) +} diff --git a/integration/lading_proptest/src/property.rs b/integration/lading_proptest/src/property.rs new file mode 100644 index 000000000..89f66f782 --- /dev/null +++ b/integration/lading_proptest/src/property.rs @@ -0,0 +1,755 @@ +//! Declarative property assertion system. +//! +//! Properties are composable checks that verify relationships between +//! test input ([`LogBatch`]) and agent output ([`ReceivedLogEntry`]). + +use std::fmt; + +use rustc_hash::{FxHashMap, FxHashSet}; + +use crate::intake::ReceivedLogEntry; +use crate::log_format::LogFormat; +use crate::log_gen::LogBatch; + +/// A property that can be checked against test input and output. +pub trait Property: fmt::Debug { + /// Human-readable name of this property. + fn name(&self) -> &'static str; + + /// Check the property. + /// + /// Returns `Ok(())` if the property holds, or a [`PropertyFailure`] + /// describing what went wrong. + /// + /// # Errors + /// + /// Returns [`PropertyFailure`] when the property does not hold. + fn check( + &self, + input: &LogBatch, + output: &[ReceivedLogEntry], + ) -> Result<(), PropertyFailure>; +} + +/// Detailed information about a property failure. +#[derive(Debug, Clone)] +pub struct PropertyFailure { + /// The name of the property that failed. + pub property_name: String, + /// A human-readable description of the failure. + pub description: String, + /// Key-value pairs providing additional context. + pub details: Vec<(String, String)>, +} + +impl fmt::Display for PropertyFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + writeln!(f, "Property '{}' failed: {}", self.property_name, self.description)?; + for (key, value) in &self.details { + writeln!(f, " {key}: {value}")?; + } + Ok(()) + } +} + +// --- Built-in Properties --- + +/// Every unique input line UUID appears in at least one output entry. +#[derive(Debug, Copy, Clone)] +pub struct AllLinesDelivered; + +impl Property for AllLinesDelivered { + fn name(&self) -> &'static str { + "all_lines_delivered" + } + + fn check( + &self, + input: &LogBatch, + output: &[ReceivedLogEntry], + ) -> Result<(), PropertyFailure> { + // Collect all header IDs from input. Skip internal line fragment IDs: + // - "uuid:cont:N" — timestamp multiline continuation lines + // - "uuid:json_line:N" — JSON multiline continuation lines + let input_ids: FxHashSet<&str> = input + .lines + .iter() + .filter(|l| !l.id.contains(":cont:") && !l.id.contains(":json_line:")) + .map(|l| l.id.as_str()) + .collect(); + + // Collect all IDs found in output messages. Use extract_all_ids + // because aggregated messages may contain multiple UUIDs (e.g., a + // plain text line merged into a preceding entry's buffer). + let output_ids: FxHashSet<&str> = output + .iter() + .flat_map(|entry| LogFormat::extract_all_ids(&entry.message)) + .collect(); + + let missing: Vec<&str> = input_ids.difference(&output_ids).copied().collect(); + + if missing.is_empty() { + Ok(()) + } else { + Err(PropertyFailure { + property_name: self.name().to_string(), + description: format!( + "{} of {} input lines not found in output", + missing.len(), + input_ids.len() + ), + details: vec![ + ("missing_ids".to_string(), format!("{missing:?}")), + ("input_count".to_string(), input_ids.len().to_string()), + ("output_count".to_string(), output.len().to_string()), + ], + }) + } + } +} + +/// No output entry contains a UUID that was not in the input. +#[derive(Debug, Copy, Clone)] +pub struct NoExtraLines; + +impl Property for NoExtraLines { + fn name(&self) -> &'static str { + "no_extra_lines" + } + + fn check( + &self, + input: &LogBatch, + output: &[ReceivedLogEntry], + ) -> Result<(), PropertyFailure> { + let input_ids: FxHashSet<&str> = input + .lines + .iter() + .filter(|l| !l.id.contains(":cont:") && !l.id.contains(":json_line:")) + .map(|l| l.id.as_str()) + .collect(); + + let extra: Vec = output + .iter() + .filter_map(|entry| { + LogFormat::extract_id(&entry.message).and_then(|id| { + if input_ids.contains(id) { + None + } else { + Some(id.to_string()) + } + }) + }) + .collect(); + + if extra.is_empty() { + Ok(()) + } else { + Err(PropertyFailure { + property_name: self.name().to_string(), + description: format!( + "{} output entries have IDs not in input", + extra.len() + ), + details: vec![("extra_ids".to_string(), format!("{extra:?}"))], + }) + } + } +} + +/// For non-truncated lines, the message content matches the input exactly. +#[derive(Debug, Copy, Clone)] +pub struct ContentPreserved; + +impl Property for ContentPreserved { + fn name(&self) -> &'static str { + "content_preserved" + } + + fn check( + &self, + input: &LogBatch, + output: &[ReceivedLogEntry], + ) -> Result<(), PropertyFailure> { + // Build a map from ID to input content + let input_by_id: FxHashMap<&str, &str> = input + .lines + .iter() + .filter(|l| !l.id.contains(":cont:")) + .map(|l| (l.id.as_str(), l.content.as_str())) + .collect(); + + let mut mismatches = Vec::new(); + + for entry in output { + let Some(id) = LogFormat::extract_id(&entry.message) else { + continue; + }; + let Some(expected) = input_by_id.get(id) else { + continue; + }; + + // Check if the output message contains the expected content. + // The agent may wrap the content (add metadata, etc.) so we check + // containment rather than exact equality. + if !entry.message.contains(expected) && !expected.contains(&entry.message) { + // Could be truncated — skip those (TruncationRespected handles them) + if !entry.message.contains("TRUNCATED") { + mismatches.push((id.to_string(), (*expected).to_string(), entry.message.clone())); + } + } + } + + if mismatches.is_empty() { + Ok(()) + } else { + let detail: Vec = mismatches + .iter() + .take(5) + .map(|(id, expected, actual)| { + format!("id={id}: expected contains '{expected}', got '{actual}'") + }) + .collect(); + Err(PropertyFailure { + property_name: self.name().to_string(), + description: format!("{} lines had content mismatches", mismatches.len()), + details: vec![ + ("mismatches".to_string(), detail.join("\n")), + ("total_mismatches".to_string(), mismatches.len().to_string()), + ], + }) + } + } +} + +/// Lines exceeding the configured max message size are truncated; lines under +/// the limit are delivered intact. +/// +/// The Datadog Agent's truncation behavior: +/// - Lines under the limit pass through intact. +/// - Lines over the limit are split into a **head chunk** (content up to the +/// limit with `...TRUNCATED...` appended) and one or more **tail chunks** +/// (prefixed with `...TRUNCATED...` containing the remaining content). +/// - The head chunk size is `max_message_size_bytes + TRUNCATED_MARKER.len()`. +/// - Tail chunks are standalone output entries with no UUID. +#[derive(Debug, Copy, Clone)] +pub struct TruncationRespected { + /// The agent's `max_message_size_bytes` setting. + pub max_message_bytes: usize, +} + +/// The marker the agent appends/prepends on truncated messages. +const TRUNCATED_MARKER: &str = "...TRUNCATED..."; + +impl Property for TruncationRespected { + fn name(&self) -> &'static str { + "truncation_respected" + } + + fn check( + &self, + input: &LogBatch, + output: &[ReceivedLogEntry], + ) -> Result<(), PropertyFailure> { + let input_by_id: FxHashMap<&str, &str> = input + .lines + .iter() + .filter(|l| !l.id.contains(":cont:")) + .map(|l| (l.id.as_str(), l.content.as_str())) + .collect(); + + // The agent appends "...TRUNCATED..." to the head chunk, so the + // output message can be up to this many bytes over the raw limit. + let max_head_bytes = self.max_message_bytes + TRUNCATED_MARKER.len(); + + let mut violations = Vec::new(); + + for entry in output { + // Skip tail chunks — they are expected byproducts of truncation + if entry.message.starts_with(TRUNCATED_MARKER) { + continue; + } + + let Some(id) = LogFormat::extract_id(&entry.message) else { + continue; + }; + let Some(input_content) = input_by_id.get(id) else { + continue; + }; + + let input_len = input_content.len(); + let output_len = entry.message.len(); + + if input_len > self.max_message_bytes { + // Input was over limit — head chunk should be truncated. + // The agent appends "...TRUNCATED..." so output can be up to + // max_message_bytes + 15 bytes. + if output_len > max_head_bytes { + violations.push(format!( + "id={id}: input {input_len}B over limit, head chunk {output_len}B exceeds max head size {max_head_bytes}B" + )); + } + // The head chunk should also contain the truncation marker + if !entry.message.ends_with(TRUNCATED_MARKER) { + violations.push(format!( + "id={id}: input {input_len}B over limit, but head chunk missing '{TRUNCATED_MARKER}' suffix" + )); + } + } else { + // Input was under limit — output should be delivered intact + if output_len < input_len / 2 { + violations.push(format!( + "id={id}: input {input_len}B under limit, but output only {output_len}B (possible data loss)" + )); + } + } + } + + if violations.is_empty() { + Ok(()) + } else { + Err(PropertyFailure { + property_name: self.name().to_string(), + description: format!("{} truncation violations", violations.len()), + details: vec![ + ("violations".to_string(), violations.join("\n")), + ( + "max_message_bytes".to_string(), + self.max_message_bytes.to_string(), + ), + ], + }) + } + } +} + +/// Continuation lines are correctly aggregated with their header line. +/// +/// Verifies: +/// - All expected continuations for each header are present in its output entry +/// - Continuations are in ascending sequence order +/// - No continuations from a different header appear in an entry +/// +/// Uses `LogBatch::expected_continuations` to know what to expect. If that +/// field is empty (non-multiline scenarios), this property passes trivially. +#[derive(Debug, Copy, Clone)] +pub struct MultilineAggregated; + +impl Property for MultilineAggregated { + fn name(&self) -> &'static str { + "multiline_aggregated" + } + + fn check( + &self, + input: &LogBatch, + output: &[ReceivedLogEntry], + ) -> Result<(), PropertyFailure> { + // If no continuation metadata, pass trivially (non-multiline scenario) + if input.expected_continuations.is_empty() { + return Ok(()); + } + + // Build expected: header_id → continuation count + let expected: FxHashMap<&str, usize> = input + .expected_continuations + .iter() + .map(|(id, count)| (id.as_str(), *count)) + .collect(); + + let mut failures = Vec::new(); + + for entry in output { + let Some(header_id) = LogFormat::extract_id(&entry.message) else { + continue; + }; + + let Some(&expected_count) = expected.get(header_id) else { + continue; + }; + + // Extract continuations found in this output entry + let continuations = LogFormat::extract_continuations(&entry.message); + + // Check no cross-contamination + for (cont_header_id, _seq) in &continuations { + if *cont_header_id != header_id { + failures.push(format!( + "header {header_id} contains continuation from different header {cont_header_id}" + )); + } + } + + // Check all expected continuations are present + let found_seqs: Vec = continuations + .iter() + .filter(|(hid, _)| *hid == header_id) + .map(|(_, seq)| *seq) + .collect(); + + for expected_seq in 0..expected_count { + if !found_seqs.contains(&expected_seq) { + failures.push(format!( + "header {header_id} missing continuation seq {expected_seq} (expected {expected_count} continuations, found {found_seqs:?})" + )); + } + } + + // Check ordering (continuations should appear in ascending order) + for window in found_seqs.windows(2) { + if window[0] >= window[1] { + failures.push(format!( + "header {header_id} has out-of-order continuations: seq {} before seq {}", + window[0], window[1] + )); + } + } + } + + if failures.is_empty() { + Ok(()) + } else { + Err(PropertyFailure { + property_name: self.name().to_string(), + description: format!("{} multiline aggregation failures", failures.len()), + details: vec![("failures".to_string(), failures.join("\n"))], + }) + } + } +} + +/// The number of output entries matches the expected count. +#[derive(Debug, Copy, Clone)] +pub struct ExpectedEntryCount { + /// Expected number of output entries. + pub expected: usize, +} + +impl Property for ExpectedEntryCount { + fn name(&self) -> &'static str { + "expected_entry_count" + } + + fn check( + &self, + _input: &LogBatch, + output: &[ReceivedLogEntry], + ) -> Result<(), PropertyFailure> { + // Count output entries that have a proptest marker (ignore agent + // internal log entries that may also arrive). + let proptest_entries: usize = output + .iter() + .filter(|entry| LogFormat::extract_id(&entry.message).is_some()) + .count(); + + if proptest_entries == self.expected { + Ok(()) + } else { + Err(PropertyFailure { + property_name: self.name().to_string(), + description: format!( + "expected {expected} entries, got {proptest_entries}", + expected = self.expected, + ), + details: vec![ + ("expected".to_string(), self.expected.to_string()), + ("actual".to_string(), proptest_entries.to_string()), + ("total_output".to_string(), output.len().to_string()), + ], + }) + } + } +} + +/// JSON data integrity: every input JSON entry arrives in the output with +/// all fields preserved. +/// +/// Parses output messages as JSON and verifies that the `proptest_id` field +/// matches and all expected fields are present with correct values. This +/// property does not encode knowledge of which JSON structures the agent +/// supports — it discovers gaps mechanically. +#[derive(Debug, Copy, Clone)] +pub struct JsonIntegrity; + +impl Property for JsonIntegrity { + fn name(&self) -> &'static str { + "json_integrity" + } + + fn check( + &self, + input: &LogBatch, + output: &[ReceivedLogEntry], + ) -> Result<(), PropertyFailure> { + let Some(expected_entries) = &input.expected_json else { + // No JSON metadata — pass trivially + return Ok(()); + }; + + let mut failures = Vec::new(); + + for (expected_id, expected_json) in expected_entries { + // Find the output entry containing this UUID + let matching_output = output.iter().find(|entry| { + entry.message.contains(expected_id) + }); + + let Some(output_entry) = matching_output else { + failures.push(format!( + "id={expected_id}: not found in any output entry" + )); + continue; + }; + + // Try to parse the output message as JSON + let parsed: Result = + serde_json::from_str(&output_entry.message); + + match parsed { + Ok(output_json) => { + // Verify the proptest_id field + if let Some(id_val) = output_json.get("proptest_id") + && id_val.as_str() != Some(expected_id.as_str()) + { + failures.push(format!( + "id={expected_id}: proptest_id mismatch, got {id_val}" + )); + } + + // Verify all expected fields are present + check_json_fields( + expected_id, + expected_json, + &output_json, + "", + &mut failures, + ); + } + Err(e) => { + failures.push(format!( + "id={expected_id}: output is not valid JSON: {e} (message: {})", + &output_entry.message[..output_entry.message.len().min(200)] + )); + } + } + } + + if failures.is_empty() { + Ok(()) + } else { + Err(PropertyFailure { + property_name: self.name().to_string(), + description: format!("{} JSON integrity failures", failures.len()), + details: vec![("failures".to_string(), failures.join("\n"))], + }) + } + } +} + +/// Recursively verify that all fields in `expected` are present in `actual`. +fn check_json_fields( + id: &str, + expected: &serde_json::Value, + actual: &serde_json::Value, + path: &str, + failures: &mut Vec, +) { + match (expected, actual) { + (serde_json::Value::Object(exp_map), serde_json::Value::Object(act_map)) => { + for (key, exp_val) in exp_map { + let field_path = if path.is_empty() { + key.clone() + } else { + format!("{path}.{key}") + }; + match act_map.get(key) { + Some(act_val) => { + check_json_fields(id, exp_val, act_val, &field_path, failures); + } + None => { + failures.push(format!( + "id={id}: missing field '{field_path}'" + )); + } + } + } + } + (serde_json::Value::Array(exp_arr), serde_json::Value::Array(act_arr)) => { + if exp_arr.len() != act_arr.len() { + failures.push(format!( + "id={id}: array at '{path}' has {} elements, expected {}", + act_arr.len(), + exp_arr.len() + )); + } + for (i, (exp_item, act_item)) in + exp_arr.iter().zip(act_arr.iter()).enumerate() + { + let item_path = format!("{path}[{i}]"); + check_json_fields(id, exp_item, act_item, &item_path, failures); + } + } + (exp, act) => { + if exp != act { + failures.push(format!( + "id={id}: field '{path}' expected {exp}, got {act}" + )); + } + } + } +} + +// --- Adaptive Sampling Properties --- + +/// Model-based adaptive sampling check. +/// +/// Simulates the agent's credit-based rate limiter as a simple state machine: +/// - Each `WriteLines` action consumes 1 credit per line (if available) +/// - Each `Sleep` action refills credits at `rate_limit` per second +/// - Compares the model's prediction of which IDs should be delivered +/// against what actually appeared in the output +/// +/// Also verifies the `adaptive_sampler_sampled_count` tag appears when +/// the model predicts drops occurred. +/// +/// The model is intentionally simpler than the agent's implementation — +/// no tokenizer, no pattern table, no hot-path optimization. It just +/// tracks credits. This tests that the real system adheres to the model. +#[derive(Debug, Clone)] +pub struct AdaptiveSamplingModel { + /// The action sequence that was executed, with IDs for each write step. + pub actions: Vec, + /// Configured burst size (initial credits and cap). + pub burst_size: f64, + /// Credits refilled per second. + pub rate_limit: f64, + /// Allowed deviation per check point (absorbs timing jitter). + /// Goal is to eventually get this to 0. + pub tolerance: usize, +} + +/// A step in the sampling action sequence, annotated with line IDs. +#[derive(Debug, Clone)] +pub enum SamplingAction { + /// Lines written to the log file, with their IDs. + Write(Vec), + /// Sleep duration in seconds. + Sleep(u64), +} + +impl Property for AdaptiveSamplingModel { + fn name(&self) -> &'static str { + "adaptive_sampling_model" + } + + fn check( + &self, + _input: &LogBatch, + output: &[ReceivedLogEntry], + ) -> Result<(), PropertyFailure> { + // Simulate the credit model + let mut credits = self.burst_size; + let mut expected_delivered: Vec = Vec::new(); + let mut expected_dropped: usize = 0; + + for action in &self.actions { + match action { + SamplingAction::Write(ids) => { + for id in ids { + if credits >= 1.0 { + expected_delivered.push(id.clone()); + credits -= 1.0; + } else { + expected_dropped += 1; + } + } + } + SamplingAction::Sleep(seconds) => { + #[expect(clippy::cast_precision_loss)] + let refill = *seconds as f64 * self.rate_limit; + credits += refill; + if credits > self.burst_size { + credits = self.burst_size; + } + } + } + } + + // Collect all IDs found in output + let output_ids: FxHashSet<&str> = output + .iter() + .flat_map(|entry| LogFormat::extract_all_ids(&entry.message)) + .collect(); + + let mut failures = Vec::new(); + + // Check: model-predicted deliveries vs actual + let model_delivered_count = expected_delivered.len(); + let actual_delivered_count = expected_delivered + .iter() + .filter(|id| output_ids.contains(id.as_str())) + .count(); + + let min_expected = model_delivered_count.saturating_sub(self.tolerance); + let max_expected = model_delivered_count + self.tolerance; + if actual_delivered_count < min_expected || actual_delivered_count > max_expected { + // Report which specific IDs the model expected but didn't find + let missing: Vec<&str> = expected_delivered + .iter() + .filter(|id| !output_ids.contains(id.as_str())) + .map(String::as_str) + .take(10) + .collect(); + failures.push(format!( + "model predicted {model_delivered_count} delivered, got {actual_delivered_count} (±{}). missing: {missing:?}", + self.tolerance, + )); + } + + // Check: total output should be close to model prediction + // (some output entries may be agent-internal without our IDs) + let total_proptest_output: usize = output + .iter() + .filter(|e| LogFormat::extract_id(&e.message).is_some()) + .count(); + if total_proptest_output < min_expected || total_proptest_output > max_expected { + failures.push(format!( + "total proptest output: {total_proptest_output}, model predicted: {model_delivered_count} (±{})", + self.tolerance, + )); + } + + // Check: if the model predicted any drops, the sampled count tag + // should appear on at least one output entry + if expected_dropped > 0 { + let has_sampled_tag = output.iter().any(|entry| { + entry + .ddtags + .as_deref() + .unwrap_or("") + .contains("adaptive_sampler_sampled_count:") + }); + if !has_sampled_tag { + failures.push(format!( + "model predicted {expected_dropped} drops, but no output has adaptive_sampler_sampled_count tag" + )); + } + } + + if failures.is_empty() { + Ok(()) + } else { + Err(PropertyFailure { + property_name: self.name().to_string(), + description: format!("{} model violations", failures.len()), + details: vec![ + ("failures".to_string(), failures.join("\n")), + ("model_delivered".to_string(), model_delivered_count.to_string()), + ("model_dropped".to_string(), expected_dropped.to_string()), + ("actual_output".to_string(), total_proptest_output.to_string()), + ("credits_remaining".to_string(), format!("{credits:.1}")), + ], + }) + } + } +} diff --git a/integration/lading_proptest/src/scenario.rs b/integration/lading_proptest/src/scenario.rs new file mode 100644 index 000000000..77fafb03b --- /dev/null +++ b/integration/lading_proptest/src/scenario.rs @@ -0,0 +1,51 @@ +//! Scenario framework for property-based testing. +//! +//! A scenario defines what kind of log data to generate, how to configure +//! the agent, and what properties to assert on the output. + +pub mod adaptive_sampling; +pub mod json_multiline; +pub mod mixed_multiline; +pub mod multiline; +pub mod truncation; + +use proptest::prelude::*; + +use crate::config::LogSourceConfig; +use crate::log_format::LogFormat; +use crate::log_gen::LogBatch; +use crate::property::Property; + +/// A test scenario defines the full test lifecycle for one category of +/// agent behavior. +/// +/// Each scenario provides: +/// - A proptest strategy that generates parameters +/// - Agent configuration appropriate for the behavior under test +/// - Input log data generated from those parameters +/// - Properties to assert on the agent's output +pub trait Scenario: std::fmt::Debug { + /// The proptest-generated parameters for this scenario. + type Params: std::fmt::Debug + Clone; + + /// The proptest strategy that generates parameters. + fn strategy() -> BoxedStrategy; + + /// Build log source configuration for the agent. + fn log_source_config(params: &Self::Params) -> LogSourceConfig; + + /// The log format used for this test case. + fn log_format(params: &Self::Params) -> LogFormat; + + /// Generate the log batch (input data) from the parameters. + fn generate_input(params: &Self::Params) -> LogBatch; + + /// Return the list of properties to assert on the output. + fn properties(params: &Self::Params) -> Vec>; + + /// Optional: maximum message size override for the agent config. + /// Returns `None` to use the agent's default (256 KB). + fn max_message_size_bytes(_params: &Self::Params) -> Option { + None + } +} diff --git a/integration/lading_proptest/src/scenario/adaptive_sampling.rs b/integration/lading_proptest/src/scenario/adaptive_sampling.rs new file mode 100644 index 000000000..f35c09aa8 --- /dev/null +++ b/integration/lading_proptest/src/scenario/adaptive_sampling.rs @@ -0,0 +1,154 @@ +//! Adaptive sampling scenario. +//! +//! Tests the agent's experimental adaptive sampling feature — a per-pattern +//! credit-based rate limiter. Uses action sequences (write, sleep, write, ...) +//! of variable length to stress-test the credit model. + +use std::time::Duration; + +use proptest::prelude::*; + +use crate::config::LogSourceConfig; +use crate::log_format::PROPTEST_MARKER; +use crate::log_gen::LogLine; +use crate::orchestrator::Action; +use crate::property; + +/// Fixed adaptive sampling parameters for testing. +const BURST_SIZE: usize = 10; +/// Credits per second. +const RATE_LIMIT: f64 = 2.0; +/// Tolerance for count assertions. 0 = exact model matching. +const TOLERANCE: usize = 0; + +/// A single step in a generated action sequence. +#[derive(Debug, Clone, Copy)] +pub enum Step { + /// Write N lines. + Write(usize), + /// Sleep N seconds. + Sleep(u64), +} + +/// Parameters for adaptive sampling testing — a variable-length +/// sequence of write and sleep steps. +#[derive(Debug, Clone)] +pub struct AdaptiveSamplingParams { + /// The steps to execute. + pub steps: Vec, +} + +/// Strategy for a single step. +fn step_strategy() -> impl Strategy { + prop_oneof![ + (5_usize..25).prop_map(Step::Write), + (2_u64..8).prop_map(Step::Sleep), + ] +} + +/// Proptest strategy for adaptive sampling parameters. +/// +/// Generates a variable-length sequence of write and sleep steps. +/// Always starts with a write (to exercise the burst) and ensures +/// at least one sleep exists (to exercise credit refill). +pub fn strategy() -> BoxedStrategy { + // Generate 2-6 additional steps after the mandatory write+sleep+write + ( + 15_usize..25, // initial write count + 2_u64..8, // first sleep seconds + 5_usize..15, // second write count + proptest::collection::vec(step_strategy(), 0..4), // extra steps + ) + .prop_map(|(first_write, first_sleep, second_write, extra)| { + let mut steps = vec![ + Step::Write(first_write), + Step::Sleep(first_sleep), + Step::Write(second_write), + ]; + steps.extend(extra); + AdaptiveSamplingParams { steps } + }) + .boxed() +} + +/// Counter for generating unique numeric IDs across all calls within a case. +static LINE_COUNTER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + +/// Generate N lines that all share the same structural pattern. +/// +/// Uses a zero-padded numeric ID instead of UUIDs so the agent's tokenizer +/// produces identical token sequences for every line. +#[must_use] +pub fn generate_same_pattern_lines(count: usize) -> Vec { + (0..count) + .map(|_| { + let n = LINE_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let id = format!("{n:010}"); + let content = format!("[{PROPTEST_MARKER}:{id}] adaptive sampling test message"); + LogLine { id, content } + }) + .collect() +} + +/// Result of building an action sequence. +#[derive(Debug)] +pub struct ActionSequenceResult { + /// The orchestrator actions to execute. + pub actions: Vec, + /// The model actions for property checking (with IDs). + pub sampling_actions: Vec, +} + +/// Build the action sequence from params. +#[must_use] +pub fn build_actions(params: &AdaptiveSamplingParams) -> ActionSequenceResult { + LINE_COUNTER.store(0, std::sync::atomic::Ordering::Relaxed); + + let mut actions = Vec::new(); + let mut sampling_actions = Vec::new(); + + for step in ¶ms.steps { + match step { + Step::Write(count) => { + let lines = generate_same_pattern_lines(*count); + let ids: Vec = lines.iter().map(|l| l.id.clone()).collect(); + sampling_actions.push(property::SamplingAction::Write(ids)); + actions.push(Action::WriteLines(lines)); + } + Step::Sleep(seconds) => { + sampling_actions.push(property::SamplingAction::Sleep(*seconds)); + actions.push(Action::Sleep(Duration::from_secs(*seconds))); + } + } + } + + ActionSequenceResult { + actions, + sampling_actions, + } +} + +/// Build the properties to assert. +#[must_use] +pub fn build_properties( + result: &ActionSequenceResult, +) -> Vec> { + #[expect(clippy::cast_precision_loss)] + let burst_size = BURST_SIZE as f64; + vec![Box::new(property::AdaptiveSamplingModel { + actions: result.sampling_actions.clone(), + burst_size, + rate_limit: RATE_LIMIT, + tolerance: TOLERANCE, + })] +} + +/// The log source configuration for adaptive sampling. +#[must_use] +pub fn log_source_config() -> LogSourceConfig { + #[expect(clippy::cast_precision_loss)] + LogSourceConfig::AdaptiveSampling { + burst_size: BURST_SIZE as f64, + rate_limit: RATE_LIMIT, + } +} diff --git a/integration/lading_proptest/src/scenario/json_multiline.rs b/integration/lading_proptest/src/scenario/json_multiline.rs new file mode 100644 index 000000000..3ff9b3290 --- /dev/null +++ b/integration/lading_proptest/src/scenario/json_multiline.rs @@ -0,0 +1,87 @@ +//! JSON multiline aggregation scenario. +//! +//! Generates structurally diverse valid JSON (flat objects, nested objects, +//! objects with arrays, top-level arrays) split across multiple lines. +//! Asserts that the agent correctly compacts multi-line JSON objects and +//! preserves all fields. +//! +//! This scenario does not encode knowledge of the agent's internal behavior. +//! It generates valid JSON and asserts data integrity — if the agent fails +//! to handle a particular JSON structure, the test discovers it mechanically. + +use proptest::prelude::*; + +use crate::config::LogSourceConfig; +use crate::log_format::{self, JsonStructure, LogFormat}; +use crate::log_gen::{self, LogBatch}; +use crate::property::{self, Property}; +use crate::scenario::Scenario; + +/// Parameters for JSON multiline testing. +#[derive(Debug, Clone)] +pub struct JsonMultilineParams { + /// The JSON structures to generate, one per logical entry. + pub entries: Vec, +} + +/// JSON multiline scenario. +#[derive(Debug, Copy, Clone)] +pub struct JsonMultilineScenario; + +impl Scenario for JsonMultilineScenario { + type Params = JsonMultilineParams; + + fn strategy() -> BoxedStrategy { + proptest::collection::vec(log_format::json_structure_strategy(), 3..10) + .prop_map(|entries| JsonMultilineParams { entries }) + .boxed() + } + + fn log_source_config(_params: &Self::Params) -> LogSourceConfig { + LogSourceConfig::JsonMultiline + } + + fn log_format(_params: &Self::Params) -> LogFormat { + LogFormat::Json + } + + fn generate_input(params: &Self::Params) -> LogBatch { + let mut lines = Vec::new(); + let mut expected_json: Vec<(String, serde_json::Value)> = Vec::new(); + + for structure in ¶ms.entries { + let id = uuid::Uuid::new_v4().to_string(); + + // Store the expected JSON for property checking + expected_json.push((id.clone(), structure.expected_json(&id))); + + // Split the JSON across multiple lines for the log file + let entry_lines = structure.render_lines(&id); + for (i, line) in entry_lines.into_iter().enumerate() { + let line_id = if i == 0 { + // First line of each entry carries the UUID + id.clone() + } else { + format!("{id}:json_line:{i}") + }; + lines.push(log_gen::LogLine { + id: line_id, + content: line, + }); + } + } + + LogBatch { + lines, + format: LogFormat::Json, + expected_continuations: Vec::new(), + expected_json: Some(expected_json), + } + } + + fn properties(_params: &Self::Params) -> Vec> { + vec![ + Box::new(property::JsonIntegrity), + ] + } +} diff --git a/integration/lading_proptest/src/scenario/mixed_multiline.rs b/integration/lading_proptest/src/scenario/mixed_multiline.rs new file mode 100644 index 000000000..6e5ab8f76 --- /dev/null +++ b/integration/lading_proptest/src/scenario/mixed_multiline.rs @@ -0,0 +1,278 @@ +//! Mixed format multiline scenario. +//! +//! Interleaves all timestamp formats (`TimestampPrefixed`, `Syslog5424`, +//! `ApacheCommon`), all JSON structures, and plain text lines in the same +//! file. Tests interactions between the agent's detection chain (JSON +//! detector → timestamp detector → default aggregate) and aggregation +//! systems when formats change rapidly. + +use proptest::prelude::*; + +use crate::config::LogSourceConfig; +use crate::log_format::{self, LogFormat}; +use crate::log_gen::{self, LogBatch}; +use crate::property::{self, Property}; +use crate::scenario::Scenario; + +/// A single entry in the mixed sequence. +#[derive(Debug, Clone, Copy)] +pub enum MixedEntry { + /// Timestamp header with N continuations, using a specific format. + Timestamp { + /// The timestamp format for the header line. + format: LogFormat, + /// Number of continuation lines. + continuations: usize, + }, + /// Single timestamp line, no continuations. + TimestampOnly { + /// The timestamp format. + format: LogFormat, + }, + /// Plain text line with no format structure. + /// Defaults to `aggregate` — will merge with whatever is currently buffered. + PlainText, + /// Flat JSON object split across lines. + JsonFlat { + /// Number of fields beyond `proptest_id`. + field_count: usize, + }, + /// Nested JSON object split across lines. + JsonNested { + /// Number of fields in the nested object. + field_count: usize, + }, + /// JSON with a timestamp embedded as a string value. + JsonWithTimestampValue, + /// Complete single-line JSON. + JsonSingleLine, + /// JSON with mixed value types (numbers, booleans, null). + JsonMixedTypes, + /// Deeply nested JSON object. + JsonDeepNested { + /// Nesting depth. + depth: usize, + }, +} + +/// Parameters for mixed multiline testing. +#[derive(Debug, Clone)] +pub struct MixedMultilineParams { + /// The sequence of entries to generate. + pub entries: Vec, +} + +/// Mixed multiline scenario. +#[derive(Debug, Copy, Clone)] +pub struct MixedMultilineScenario; + +/// Strategy for a timestamp format (only formats with detectable timestamps). +fn timestamp_format_strategy() -> impl Strategy { + prop_oneof![ + Just(LogFormat::TimestampPrefixed), + Just(LogFormat::Syslog5424), + Just(LogFormat::ApacheCommon), + ] +} + +/// Strategy that generates a single [`MixedEntry`]. +fn mixed_entry_strategy() -> impl Strategy { + prop_oneof![ + // Timestamp entries with varying formats and continuations + (timestamp_format_strategy(), 1_usize..4) + .prop_map(|(format, continuations)| MixedEntry::Timestamp { format, continuations }), + timestamp_format_strategy() + .prop_map(|format| MixedEntry::TimestampOnly { format }), + // Plain text — no structure, tests default aggregate behavior + Just(MixedEntry::PlainText), + // JSON variants + (2_usize..6).prop_map(|field_count| MixedEntry::JsonFlat { field_count }), + (1_usize..4).prop_map(|field_count| MixedEntry::JsonNested { field_count }), + Just(MixedEntry::JsonWithTimestampValue), + Just(MixedEntry::JsonSingleLine), + Just(MixedEntry::JsonMixedTypes), + (2_usize..4).prop_map(|depth| MixedEntry::JsonDeepNested { depth }), + ] +} + +impl Scenario for MixedMultilineScenario { + type Params = MixedMultilineParams; + + fn strategy() -> BoxedStrategy { + proptest::collection::vec(mixed_entry_strategy(), 5..15) + .prop_map(|entries| MixedMultilineParams { entries }) + .boxed() + } + + fn log_source_config(_params: &Self::Params) -> LogSourceConfig { + // JsonMultiline enables auto_multi_line_detection + JSON detection + + // JSON aggregation. Datetime detection defaults to true, so both + // aggregation systems are active. + LogSourceConfig::JsonMultiline + } + + fn log_format(_params: &Self::Params) -> LogFormat { + LogFormat::PlainText + } + + fn generate_input(params: &Self::Params) -> LogBatch { + let mut lines = Vec::new(); + let mut expected_continuations = Vec::new(); + let mut expected_json: Vec<(String, serde_json::Value)> = Vec::new(); + let mut timestamp_index: usize = 0; + + for entry in ¶ms.entries { + let id = uuid::Uuid::new_v4().to_string(); + + match entry { + MixedEntry::Timestamp { format, continuations } => { + let header = format + .format_line_with_index(&id, timestamp_index, "mixed entry header"); + timestamp_index += 1; + lines.push(log_gen::LogLine { + id: id.clone(), + content: header, + }); + + expected_continuations.push((id.clone(), *continuations)); + + for seq in 0..*continuations { + let cont = format + .format_continuation(&id, seq, &format!("continuation {seq}")); + lines.push(log_gen::LogLine { + id: format!("{id}:cont:{seq}"), + content: cont, + }); + } + } + + MixedEntry::TimestampOnly { format } => { + let header = format + .format_line_with_index(&id, timestamp_index, "standalone entry"); + timestamp_index += 1; + lines.push(log_gen::LogLine { + id: id.clone(), + content: header, + }); + expected_continuations.push((id, 0)); + } + + MixedEntry::PlainText => { + // A line with no timestamp or JSON structure. + // The agent labels this `aggregate` and appends it to + // whatever is currently buffered. + let content = format!("[PROPTEST:{id}] plain text with no format structure"); + lines.push(log_gen::LogLine { id, content }); + // We don't add to expected_continuations or expected_json. + // AllLinesDelivered will check this UUID appears somewhere. + } + + MixedEntry::JsonFlat { field_count } => { + let structure = log_format::JsonStructure::Flat { + field_count: *field_count, + }; + push_json_entry(&mut lines, &mut expected_json, &id, &structure); + } + + MixedEntry::JsonNested { field_count } => { + let structure = log_format::JsonStructure::Nested { + field_count: *field_count, + }; + push_json_entry(&mut lines, &mut expected_json, &id, &structure); + } + + MixedEntry::JsonWithTimestampValue => { + let obj = serde_json::json!({ + "proptest_id": id, + "timestamp": "2024-01-15T10:30:00.000Z", + "msg": "json with embedded timestamp" + }); + let complete = serde_json::to_string(&obj) + .expect("serde_json serialization cannot fail"); + expected_json.push((id.clone(), obj)); + push_split_json(&mut lines, &id, &complete); + } + + MixedEntry::JsonSingleLine => { + let obj = serde_json::json!({ + "proptest_id": id, + "msg": "single line json" + }); + let complete = serde_json::to_string(&obj) + .expect("serde_json serialization cannot fail"); + expected_json.push((id.clone(), obj)); + lines.push(log_gen::LogLine { id, content: complete }); + } + + MixedEntry::JsonMixedTypes => { + let structure = log_format::JsonStructure::MixedValueTypes; + push_json_entry(&mut lines, &mut expected_json, &id, &structure); + } + + MixedEntry::JsonDeepNested { depth } => { + let structure = log_format::JsonStructure::DeepNested { depth: *depth }; + push_json_entry(&mut lines, &mut expected_json, &id, &structure); + } + } + } + + LogBatch { + lines, + format: LogFormat::PlainText, + expected_continuations, + expected_json: Some(expected_json), + } + } + + fn properties(_params: &Self::Params) -> Vec> { + vec![ + Box::new(property::AllLinesDelivered), + Box::new(property::MultilineAggregated), + Box::new(property::JsonIntegrity), + ] + } +} + +/// Helper: push a JSON structure's lines and expected value. +fn push_json_entry( + lines: &mut Vec, + expected_json: &mut Vec<(String, serde_json::Value)>, + id: &str, + structure: &log_format::JsonStructure, +) { + expected_json.push((id.to_string(), structure.expected_json(id))); + for (i, line) in structure.render_lines(id).into_iter().enumerate() { + lines.push(log_gen::LogLine { + id: if i == 0 { + id.to_string() + } else { + format!("{id}:json_line:{i}") + }, + content: line, + }); + } +} + +/// Helper: split a JSON string after the first comma and push as lines. +fn push_split_json( + lines: &mut Vec, + id: &str, + complete: &str, +) { + let split_point = complete.find(',').unwrap_or(complete.len()); + if split_point < complete.len() - 1 { + lines.push(log_gen::LogLine { + id: id.to_string(), + content: complete[..=split_point].to_string(), + }); + lines.push(log_gen::LogLine { + id: format!("{id}:json_line:1"), + content: complete[split_point + 1..].to_string(), + }); + } else { + lines.push(log_gen::LogLine { + id: id.to_string(), + content: complete.to_string(), + }); + } +} diff --git a/integration/lading_proptest/src/scenario/multiline.rs b/integration/lading_proptest/src/scenario/multiline.rs new file mode 100644 index 000000000..5bba857a9 --- /dev/null +++ b/integration/lading_proptest/src/scenario/multiline.rs @@ -0,0 +1,121 @@ +//! Multiline aggregation scenario. +//! +//! Tests that the agent correctly aggregates continuation lines with their +//! header line based on format-specific timestamp detection. +//! +//! Only formats with timestamps work for auto multiline detection: +//! `TimestampPrefixed`, `Syslog5424`, `ApacheCommon`. `PlainText` has no +//! `startGroup` signal (all lines merge). `Json` complete objects get +//! `noAggregate` (flushed standalone). JSON multiline (incomplete objects +//! spanning lines) is deferred to a future scenario. + +use proptest::prelude::*; + +use crate::config::LogSourceConfig; +use crate::log_format::{self, LogFormat}; +use crate::log_gen::{self, LogBatch}; +use crate::property::{self, Property}; +use crate::scenario::Scenario; + +/// Parameters for multiline aggregation testing. +#[derive(Debug, Clone, Copy)] +pub struct MultilineParams { + /// Number of logical log entries (each may span multiple lines). + pub entry_count: usize, + /// Maximum continuation lines per entry. + pub max_continuations: usize, + /// The log format to use. + pub format: LogFormat, +} + +/// Multiline aggregation scenario. +#[derive(Debug, Copy, Clone)] +pub struct MultilineScenario; + +impl Scenario for MultilineScenario { + type Params = MultilineParams; + + fn strategy() -> BoxedStrategy { + ( + 3_usize..20, + 1_usize..5, + log_format::multiline_format_strategy(), + ) + .prop_map(|(entry_count, max_continuations, format)| MultilineParams { + entry_count, + max_continuations, + format, + }) + .boxed() + } + + fn log_source_config(_params: &Self::Params) -> LogSourceConfig { + LogSourceConfig::AutoMultiline + } + + fn log_format(params: &Self::Params) -> LogFormat { + params.format + } + + fn generate_input(params: &Self::Params) -> LogBatch { + let mut lines = Vec::new(); + let mut expected_continuations = Vec::new(); + + for entry_idx in 0..params.entry_count { + let header_id = uuid::Uuid::new_v4().to_string(); + + // Header line with a distinct timestamp per entry so the agent + // sees each as a new `startGroup`. + let header_content = params + .format + .format_line_with_index(&header_id, entry_idx, "log entry header"); + lines.push(log_gen::LogLine { + id: header_id.clone(), + content: header_content, + }); + + // Vary continuation count deterministically per entry. + let cont_count = entry_idx % (params.max_continuations + 1); + expected_continuations.push((header_id.clone(), cont_count)); + + for seq in 0..cont_count { + let cont_content = params.format.format_continuation( + &header_id, + seq, + &format!("continuation line {seq}"), + ); + lines.push(log_gen::LogLine { + id: format!("{header_id}:cont:{seq}"), + content: cont_content, + }); + } + } + LogBatch { + lines, + format: params.format, + expected_continuations, + expected_json: None, + } + } + + fn properties(params: &Self::Params) -> Vec> { + vec![ + Box::new(property::AllLinesDelivered), + Box::new(property::MultilineAggregated), + Box::new(property::ExpectedEntryCount { + expected: params.entry_count, + }), + ] + } +} + +/// Strategy for multiline scenario with a pinned format. +pub fn strategy_with_format(format: LogFormat) -> BoxedStrategy { + (3_usize..20, 1_usize..5) + .prop_map(move |(entry_count, max_continuations)| MultilineParams { + entry_count, + max_continuations, + format, + }) + .boxed() +} diff --git a/integration/lading_proptest/src/scenario/truncation.rs b/integration/lading_proptest/src/scenario/truncation.rs new file mode 100644 index 000000000..330a8aed8 --- /dev/null +++ b/integration/lading_proptest/src/scenario/truncation.rs @@ -0,0 +1,132 @@ +//! Truncation boundary scenario. +//! +//! Tests that the agent correctly truncates lines exceeding the configured +//! `max_message_size_bytes` while delivering lines under the limit intact. + +use proptest::prelude::*; + +use crate::config::LogSourceConfig; +use crate::log_format::{self, LogFormat}; +use crate::log_gen::{self, LogBatch}; +use crate::property::{self, Property}; +use crate::scenario::Scenario; + +/// The Datadog Agent's default max message size in bytes. +pub const DEFAULT_MAX_MESSAGE_BYTES: usize = 256 * 1024; + +/// Parameters for truncation testing. +#[derive(Debug, Clone, Copy)] +pub struct TruncationParams { + /// Number of log lines to generate. + pub line_count: usize, + /// The log format to use. + pub format: LogFormat, + /// The max message size the agent is configured with. + pub max_message_bytes: usize, +} + +/// Truncation scenario. +#[derive(Debug, Copy, Clone)] +pub struct TruncationScenario; + +impl Scenario for TruncationScenario { + type Params = TruncationParams; + + fn strategy() -> BoxedStrategy { + ( + 5_usize..15, + log_format::log_format_strategy(), + // Test at different truncation limits + prop_oneof![ + Just(1024_usize), // 1 KB + Just(64 * 1024), // 64 KB + Just(DEFAULT_MAX_MESSAGE_BYTES), // 256 KB (default) + ], + ) + .prop_map(|(line_count, format, max_message_bytes)| TruncationParams { + line_count, + format, + max_message_bytes, + }) + .boxed() + } + + fn log_source_config(_params: &Self::Params) -> LogSourceConfig { + LogSourceConfig::Simple + } + + fn log_format(params: &Self::Params) -> LogFormat { + params.format + } + + fn generate_input(params: &Self::Params) -> LogBatch { + let limit = params.max_message_bytes; + let mut lines = Vec::with_capacity(params.line_count); + + for i in 0..params.line_count { + let id = uuid::Uuid::new_v4().to_string(); + + // Distribute lines across three size categories + let target_len = match i % 3 { + // Well under limit + 0 => 100 + (i * 37) % 900, + // Near boundary (+/- 50 bytes) + 1 => limit.saturating_sub(50) + (i * 13) % 100, + // Well over limit + _ => limit + 1000 + (i * 71) % (limit / 2), + }; + + let overhead_estimate = params.format.format_line(&id, "").len(); + let filler_len = target_len.saturating_sub(overhead_estimate); + let filler: String = (0..filler_len) + .map(|j| char::from(b'a' + u8::try_from(j % 26).unwrap_or(0))) + .collect(); + let content = params.format.format_line(&id, &filler); + lines.push(log_gen::LogLine { id, content }); + } + + LogBatch { + lines, + format: params.format, + expected_continuations: Vec::new(), + expected_json: None, + } + } + + fn properties(params: &Self::Params) -> Vec> { + vec![ + Box::new(property::AllLinesDelivered), + Box::new(property::TruncationRespected { + max_message_bytes: params.max_message_bytes, + }), + Box::new(property::ContentPreserved), + ] + } + + fn max_message_size_bytes(params: &Self::Params) -> Option { + // Only override if not using the default + if params.max_message_bytes == DEFAULT_MAX_MESSAGE_BYTES { + None + } else { + Some(params.max_message_bytes) + } + } +} + +/// Strategy for truncation scenario with a pinned format. +pub fn strategy_with_format(format: LogFormat) -> BoxedStrategy { + ( + 5_usize..15, + prop_oneof![ + Just(1024_usize), + Just(64 * 1024), + Just(DEFAULT_MAX_MESSAGE_BYTES), + ], + ) + .prop_map(move |(line_count, max_message_bytes)| TruncationParams { + line_count, + format, + max_message_bytes, + }) + .boxed() +}