diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7e1656..81d4885 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,28 +11,77 @@ jobs: - uses: actions/setup-go@v5 with: go-version: "1.22" + + # The engine resolves every technique's command from the pinned Atomic Red + # Team tree. Without it the atomic, replay and precision tests cannot run, + # so the fetch is part of the build, not an optional step. + - name: vendor atomic red team (pinned) + run: make atomics + - name: build run: go build ./cmd/... ./internal/... - name: vet run: go vet ./cmd/... ./internal/... + - name: gofmt + run: | + unformatted=$(gofmt -l cmd internal) + if [ -n "$unformatted" ]; then + echo "not gofmt-clean:"; echo "$unformatted"; exit 1 + fi + + # ./cmd/... is included deliberately: the atomic-resolution, attribution, + # replay and diff guards live there, and running only ./internal/... would + # skip every one of them. - name: test - run: go test ./internal/... -count=1 + run: go test ./cmd/... ./internal/... -count=1 + + # Runs the real matcher over captured-format telemetry with no lab. This + # is the check that fixtures alone cannot provide. + - name: replay sample dataset + run: go run ./cmd/purpleloop replay testdata/sample-run > /dev/null + + # Coverage without a false-positive rate is half a metric. Exits non-zero + # if any rule fires on the benign administrator workload. + - name: precision (false-positive gate) + run: go run ./cmd/purpleloop precision + detections: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: sigma lint + - name: sigma lint (repo linter) run: | pip install pyyaml python3 scripts/lint-sigma.py + + # pySigma enforces spec rules the hand-rolled Go parser tolerates. It + # caught a non-UUID rule identifier the Go parser accepted silently. + # + # The two excluded validators are SigmaHQ house conventions for their own + # repository, not parts of the Sigma specification, and both conflict with + # deliberate choices here: rules are named for the technique they detect + # (T1082.yml), and the pipeline canary is a positive control rather than an + # ATT&CK detection, so it carries a purpleloop.* tag. + - name: sigma check (pySigma spec validation) + run: | + pip install sigma-cli + sigma check --fail-on-issues \ + -x filename_length -x namespace_tag \ + detections/linux detections/windows detections/canary + - name: fixture regression run: python3 scripts/regression-test.py + secrets: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 + # gitleaks-action requires a token to scan pull requests since v2. Without + # it the job fails on every PR for a configuration reason rather than a + # finding — which trains everyone to ignore a red secrets check. - name: gitleaks uses: gitleaks/gitleaks-action@v2 - # Phase 3 adds a `detections` job: sigma lint + regression vs fixtures. + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 3901cd8..e9517e4 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ reports/ # os / editor .DS_Store *.swp + +# local go test cache (see review remediation) +.gotmp/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bbdbd0..2403265 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,171 @@ All notable changes to this project follow Keep a Changelog and Semantic Versioning. ## [Unreleased] +### Fixed (2026-08 review — the engine now executes the atomics it claims to) +- **Every technique ran the same hardcoded `id; whoami`.** `atomic_ids` was parsed from plan + YAML, carried through emulation stages and the arbiter feed and written into the proof + chain — but never resolved to a command. A ten-technique campaign ran one command ten times + and labelled the results T1082, T1033, T1518… so every verdict but `T1059.004`'s was about a + command nobody asked for. The README's claim that Execute "runs Atomic Red Team tests" was + not true of the code. New `internal/atomic` package resolves each `atomic_id` against the + pinned ART tree; resolution is strict (`override → vendored ART → error`) and there is no + default command. +- **Five of ten techniques in `plans/discovery.yml` named Windows atomics** while the campaign + targets a Linux victim (`T1082-1`, `T1033-1`, `T1007-1`, `T1016-1`, `T1049-1` are all + `command_prompt` tests). Invisible while everything ran `id; whoami`. Plans and emulation + chains now name the correct Linux tests, and a platform mismatch is a loud `ERROR`. +- **Containment breach in the pinned atomics.** `T1059.004-1` writes a script that pings + `8.8.8.8` — traffic leaving the isolated lab network, against the project's own rule. Its + `host` input is pinned to loopback via an override that keeps the upstream command. +- **`ERROR` was counted as `missed`** in the HTML headline, and `SKIPPED_PREREQ`/`ERROR` were + inside the dashboard's coverage denominator. A harness failure is not a detection gap; + coverage is now measured only over techniques that actually exercised a detection. +- **Atomic Red Team was recorded, not pinned.** `lab-fetch.sh` cloned HEAD then overwrote the + pin file with whatever it got, so a re-clone could silently change every command the engine + runs. `scripts/fetch-atomics.sh` checks out the recorded commit and verifies HEAD matches. +- Dashboard no longer hardcodes `campaign: "discovery"` and `build: "v1.3.0"`; both come from + the run. `gap.why` is populated from the verdict's note instead of always being empty. +- `make help` printed "Makefile" in place of every target name (`grep` needed `-h`). + +### Fixed (2026-08 — the detections had the same defect as the engine) +- **Nine of the ten Linux Sigma rules were the identical rule.** Every one matched + `Image endswith /id or /whoami`, differing only in title, because they were written against + an engine that executed `id; whoami` for every technique. All nine shared one byte-identical + positive fixture, so the "10 rules, 20 fixtures, CI enforced" detection-as-code claim was + testing one rule nine times against one event. Each technique now has a detection for what + its own atomic actually does, with distinct fixtures whose negatives include a *neighbouring* + technique's activity — so a rule that is secretly a copy fails its own negative test. +- **Two false positives found by the new precision gate and fixed**: `T1087.001` fired on + `getent hosts localhost` (a DNS lookup, not account enumeration) and `T1135` fired on a bare + `mount`. Both rules keyed on a tool name rather than what the tool was being asked to do. +- **`win_proc_create.yml` had a non-UUID identifier** (`win-proc-create-001`), a Sigma spec + violation the hand-rolled Go parser accepted silently. Found by adding `sigma check`. +- **The Windows path was unreachable from `run`.** `runTasks` hardcoded a Linux target, rules + directory and executor; `SSHExecutor` and the Windows rule existed only inside the standalone + `canary` subcommand, which also hardcoded the IP `192.168.88.13`. Targets are now declared in + `lab/targets.yml`, techniques are dispatched by the platform their rule-map entry names, and + the canary runs once per platform in use — a healthy Linux canary says nothing about whether + Windows telemetry is flowing. +- **`history.json` stored only a coverage percentage**, so no run could be compared with + another. It now carries per-technique verdicts, which is what `purpleloop diff` needs. + +### Fixed (2026-08 — lab reproducibility) +- **Archive logging did not survive a restart.** The collector reads + `archives.json`, which the manager only writes when `` is `yes`; the image ships + it as `no`. It had been enabled by hand *inside the running container*, which cannot work: + the manager bind-mounts `/wazuh-config-mount/etc/ossec.conf` from the host and its entrypoint + copies that over `/var/ossec/etc/ossec.conf` on every start, so the edit is wiped by the very + restart needed to apply it. The stack then comes up healthy, the agent enrolls, and every + technique reports `NO_TELEMETRY` — which reads as a detection problem and is not one. + `scripts/enable-archives.sh` (`make archives`) patches the host-side source of the config + mount and then *verifies* the setting is live inside the container. + +### Fixed (2026-08 — victim telemetry honesty) +- **The victim entrypoint hid a total telemetry failure.** It ran + `service auditd start || auditd || true`, so every failure was swallowed: the container came + up looking healthy with no process-creation source at all, and the pipeline then reported + `NO_TELEMETRY` for every technique — which reads as a detection problem and is not one. The + entrypoint now probes the kernel audit subsystem, loads `execve` rules when it is usable, and + otherwise says loudly that this victim has no process-creation telemetry and that runs will + be `INCONCLUSIVE` as a result. +- **`--settle-timeout` / `--settle-poll` are configurable.** The 60s default is below the + victim's 360s `` collection interval, so even command-output telemetry could not + land inside a technique's window. Documented on the flag itself. +- Verified and documented in `docs/PITFALLS.md` what actually blocks Linux process-creation + telemetry: auditd is impossible on a host whose kernel exposes no audit sysctls (reproduced + with `--privileged --network host`), and Sysmon for Linux only runs under systemd (its + `-service` mode still shells out to `systemctl`). Closing the gap needs a systemd-based + victim — a deliberate change to the lab's security posture, not a config tweak. + +### Added (2026-08 — Linux-Sysmon readiness) +- Normalizer handles a `data.sysmon` path: Linux-Sysmon `EventID 1` is treated as genuine + process creation, any other Sysmon event ID is low fidelity and can never satisfy a + `process_creation` rule. Events are deliberately not routed through `data.win.*`, which is + Wazuh's Windows eventchannel convention. Tested in `internal/evaluator/sysmon_linux_test.go`, + including that a non-creation event with identical fields does not match. +- Documented in `docs/PITFALLS.md` why the Linux victim is a container without process-creation + telemetry rather than a container running Sysmon: Sysmon for Linux works, but its eBPF is + kernel-global and the event carries no container identity, so scoping it either leaks host + processes (PID reuse) or drops the atomics' own short-lived ones. Both manufacture false + verdicts. The fix is a Linux VM, as the Windows victim already is. + +### Added (2026-08) +- `purpleloop precision` and `internal/precision`: measures the false-positive rate against a + benign administrator workload (`emulation/benign-baseline.yml`, 26 commands across 7 + categories) including adjacent cases — benign uses of the binaries the rules watch. Exits + non-zero on any false positive, so it gates CI. +- `purpleloop replay` and `internal/capture`: every real run writes + `reports/runs//events.jsonl`; replay re-runs the real evaluator over it with no lab. CI + replays `testdata/sample-run` on every push, so rule edits are checked against telemetry + rather than only against fixtures written to match them. The capture writer refuses synthetic + runs, whose "telemetry" is fabricated. +- `purpleloop diff [--fail-on-regression]`: names the technique that regressed + (`T1082 DETECTED -> MISSED`) instead of only moving a percentage. Verdicts that mean "never + exercised" share a rank, so ERROR -> SKIPPED_PREREQ is a change, not a regression. +- `collector.Settler`: polls until an expected rule matches or ingestion settles, replacing the + fixed `time.Sleep(10s)` per technique and its 10-minute query window. +- Attribution labels on every verdict (`window_and_host_scoped`, `window_scoped`, + `window_overlap`, `unscoped`) and `detect_latency_ms`. Overlap is computed campaign-wide once + every window is known. This changes no verdict; it states the quality of the evidence behind + one. +- `mappings/attack_rule_map.json` with `expected_rules[]` — a technique is DETECTED when ANY + expected rule matches. Replaces the technique->rule map hardcoded in `cmd/purpleloop`. +- `untested_rules` accounting: every rule is mapped to a technique or declared untested with a + reason, enforced by a CI lint, and reported per run. +- `collection_gaps` in the dashboard output, naming the technique and rules behind each + `NO_TELEMETRY`, for handoff to the sibling detection-decay repo. +- `lab/targets.yml` + `internal/lab`, `plans/windows-discovery.yml`, `plans/cross-platform.yml`. +- `make archives` / `scripts/enable-archives.sh`. +- `sigma check` (pySigma) in CI, and a `gofmt` gate. CI now runs `./cmd/...` too — the + atomic-resolution, attribution, replay and diff guards live there and were being skipped. +- Regression guards for both halves of the defect: no two techniques in a campaign may resolve + to the same command, and no two rules may share detection logic. + +### Added (2026-08, atomics) +- `internal/atomic`: ART loader with input-argument interpolation, prerequisite parsing, + platform support checks and lab overrides. +- `SKIPPED_PREREQ` verdict — an atomic whose own prerequisites are unmet never ran, and is no + longer reported as a detection miss. +- `mappings/atomic-overrides.yml`: two override kinds — pinned inputs (keeps the upstream + command, used for lab containment) and replaced commands (only where no usable upstream test + exists). Every entry requires a `reason`, which the loader enforces and the report shows. +- Proof chains carry atomic provenance (`name`, `guid`, `source`, `override_reason`) and a + `note` explaining any non-coverage verdict. Campaign results record the ART commit. +- `make atomics` target and `scripts/fetch-atomics.sh`. +- Regression guard: a test fails if any two techniques in the shipped campaign resolve to the + same command — the defect above, made unshippable. + +### Fixed (2026-07 code review — trust contracts enforced in code, not just docs) +- **Canary gating now runs in the pipeline.** Every `run` path executes the positive + control against the same Linux/Docker executor+collector the campaign uses; if it does + not fire, the run is marked `INCONCLUSIVE` and its coverage is not presented or published. + Previously the canary ran only in the standalone `canary` subcommand and no `run` was ever + gated. +- **Dashboard canary status is the real result**, never hardcoded `healthy:true`. +- **Dry/synthetic runs are marked non-evidentiary end to end** (`CampaignResult.Synthetic`): + loud banner in the HTML/Navigator output, and such runs are never appended to `history.json` + or published to `docs/data/coverage.json`. +- **Partial container config is rejected** — a real run requires both `--victim-container` + and `--manager-container` (or `--dry-run`), so a real atomic is never paired with a + synthetic collector. +- **Fidelity gate corrected**: only genuine process-creation events (Sysmon 1 / Security 4688, + or the image+commandLine signature) are treated as process telemetry. Enumeration/network + events (e.g. 4798 `callerProcessName`) no longer credit a process-creation detection. +- **Unsupported Sigma modifiers** (base64, cidr, windash, …) now yield `INCONCLUSIVE`, not a + silent `MISSED`. +- **Sigma matcher correctness**: wildcards compile to anchored regexps (`?` supported, + `*\svchost.exe` no longer matches `…svchost.exe.malware`, metacharacters escaped); + `N of prefix_*` / `all of prefix_*` expand by glob and no longer drop the trailing + `and not filter`; `Field: null` matches absent fields; `not(...)` (no space) parses; `of + them` excludes filter/falsepositive identifiers. +- **Evidence window** pads only the end (ingestion lag), not the start — a query can no + longer reach back and credit an earlier run's events (false `DETECTED`). +- **Collector date pre-filter** widened ±1 day so a non-UTC Wazuh timestamp near a day + boundary is not dropped before the precise instant check. +- Removed the hardcoded `admin:SecretPassword` from `scripts/verify-lab.sh` (now sourced from + env / the gitignored secrets file, matching the manager check). Removed dead + `BaseURL`/`User`/`Pass` collector fields and the unused `canary.Check` dry-run parameter. + ### Added - Evidence fidelity: normalizer tags each event's source; `process_creation` rules now only accept genuine process-creation telemetry (Sysmon/EventChannel eventdata, auditd diff --git a/Makefile b/Makefile index 7f0c93f..c8e6129 100644 --- a/Makefile +++ b/Makefile @@ -6,10 +6,10 @@ export TECHNIQUE ?= T1059.004 LAB_DIR := lab/wazuh-docker/single-node -.PHONY: help host-prep lab-fetch lab-up lab-down verify run build vet reset snapshot restore +.PHONY: help host-prep lab-fetch atomics lab-up lab-down archives verify run build vet reset snapshot restore help: ## list targets - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + @grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ awk 'BEGIN{FS=":.*?## "}{printf " \033[36m%-12s\033[0m %s\n", $$1, $$2}' host-prep: ## one-time host tweak (vm.max_map_count) + tooling check @@ -18,9 +18,15 @@ host-prep: ## one-time host tweak (vm.max_map_count) + tooling check lab-fetch: ## pull pinned Wazuh single-node + Atomic Red Team, install override @bash scripts/lab-fetch.sh +atomics: ## vendor Atomic Red Team at the pinned commit (required before any run) + @bash scripts/fetch-atomics.sh + lab-up: ## bring the merged Wazuh stack + victim up @cd $(LAB_DIR) && docker compose up -d - @echo "give the indexer ~60-90s, then: make verify" + @echo "give the indexer ~60-90s, then: make archives && make verify" + +archives: ## enable Wazuh archive logging (the collector reads archives.json) + @bash scripts/enable-archives.sh lab-down: ## stop the stack (keeps data) @cd $(LAB_DIR) && docker compose down @@ -32,7 +38,7 @@ verify: ## binary health check + canary — must pass before proceeding canary: ## run the pipeline positive control @go run ./cmd/purpleloop canary -build: ## compile everything +build: ## compile everything (scoped: the vendored ART tree is not Go source) @go build ./cmd/... ./internal/... vet: ## static checks diff --git a/README.md b/README.md index 4435c6d..fb99f7b 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,13 @@ rules, and produces an evidence-backed coverage report. No guessing, no presence *[More detailed view → Interactive Architecture Map](https://jayelbotvibe-web.github.io/purple-loop/)* 1. **Feed** loads techniques from a plan, arbiter export, or emulation script -2. **Execute** runs Atomic Red Team tests on lab victims (Docker Linux + VMware Windows) -3. **Collect** queries Wazuh archives for raw telemetry in the execution window -4. **Evaluate** normalizes events and matches them against Sigma rules using a native Go parser -5. **Report** produces JSON, HTML coverage grid, or ATT&CK Navigator layer export +2. **Resolve** turns each `atomic_id` into a real command from the pinned Atomic Red Team + tree — strictly. An unresolvable ID, or one whose atomic does not support the target + platform, is an `ERROR` naming the problem, never a substituted default command +3. **Execute** runs that atomic on the lab victim (Docker Linux + VMware Windows) +4. **Collect** queries Wazuh archives for raw telemetry in the execution window +5. **Evaluate** normalizes events and matches them against Sigma rules using a native Go parser +6. **Report** produces JSON, HTML coverage grid, or ATT&CK Navigator layer export ## Pipeline canary (positive control) @@ -68,6 +71,38 @@ This removes the ambiguity that caused v1.0's false 100% coverage — you can no } ``` +### Precision, not just coverage + +```bash +purpleloop precision +# Precision: 100% (26 benign commands, 10 rules) +``` + +Coverage answers *does it fire when it should*. Precision answers *does it stay quiet when it +should* — a rule matching everything scores perfect coverage and is worthless. Every detection +is measured against a benign administrator workload including **adjacent cases**: benign uses +of the very binaries the rules watch. This found two real false positives on introduction +(`T1087.001` fired on `getent hosts`, a DNS lookup; `T1135` fired on a bare `mount`), and both +rules were narrowed. It exits non-zero, so it gates CI. + +### Replay a captured run — no lab required + +```bash +purpleloop replay testdata/sample-run +``` + +Every real run writes `reports/runs//events.jsonl`. `replay` re-runs the **real** evaluator +over that telemetry with no Docker and no Wazuh, so a rule edit can be checked against every +past run: *would this have broken a detection that already worked?* CI runs it on every push. + +### Compare two runs + +```bash +purpleloop diff reports/runs/ reports/runs/ --fail-on-regression +# REGRESSED (1) +# T1082 DETECTED -> MISSED +``` + Run: `purpleloop serve` for a **local web dashboard** with all past runs at `http://127.0.0.1:8787`. [Live dashboard on GitHub Pages](https://jayelbotvibe-web.github.io/purple-loop/dashboard.html). @@ -76,9 +111,19 @@ Run: `purpleloop serve` for a **local web dashboard** with all past runs at `htt ```bash git clone https://github.com/jayelbotvibe-web/purple-loop.git && cd purple-loop +make atomics # vendor Atomic Red Team at the pinned commit — required bash scripts/startup.sh ``` +Bringing the lab up by hand needs one extra step, because the collector reads the manager's +full event archive rather than only rule-triggered alerts: + +```bash +make lab-up +make archives # enable ; without it EVERY technique reports NO_TELEMETRY +make verify # health gates + pipeline canary +``` + Full guide: [STARTUP.md](STARTUP.md) — covers lab, Windows VM, arbiter connection, campaigns, troubleshooting. ```bash @@ -106,6 +151,33 @@ the lab and validating whether the corresponding behavioral (TTP) detection fire [`threat-intel-arbiter → arbiter-live.json → purple-loop run --arbiter`] +## Atomic resolution and lab overrides + +Every technique's command comes from the Atomic Red Team commit recorded in +`mappings/atomic-red-team.commit`, checked out and **verified** by `make atomics`. Resolution +is strict: `override → vendored ART → error`. There is deliberately no default command — a +fabricated input produces a fabricated verdict, so an atomic that cannot be resolved fails +loudly instead. + +Three verdicts exist for "this never tested a detection", and none of them counts toward +coverage: + +| Verdict | Means | +|---|---| +| `ERROR` | The atomic could not be resolved, or does not support the target platform | +| `SKIPPED_PREREQ` | The atomic's own `prereq_command` failed — it never ran | +| `NO_TELEMETRY` | It ran, but the events the rule needs never arrived | + +`mappings/atomic-overrides.yml` adjusts atomics that cannot run truthfully in this lab. Every +entry requires a `reason`, which appears in the report, so a substitution is never mistaken for +the upstream atomic. Two kinds, narrower preferred: + +- **Pinned inputs** keep the upstream command and override only its declared inputs. Used for + containment — upstream `T1059.004-1` pings `8.8.8.8`, which leaves the isolated lab network, + so `host` is pinned to loopback. +- **Replaced commands** substitute the atomic wholesale, only where no usable upstream test + exists. `T1518` (Software Discovery) has six atomics and not one supports Linux. + ## Design decisions / scope **Behavioral (TTP) validation, by design.** Purple Loop validates ATT&CK technique @@ -126,10 +198,15 @@ orchestrator. Lab runs isolated on `purpleloop-lab` Docker network. ## Detection-as-code -Every Sigma rule has **positive + negative fixtures**. CI enforces: -- All positives must match -- All negatives must reject -- A broken rule fails `go test ./internal/...` and turns CI red +Every Sigma rule has **positive + negative fixtures**, and each rule detects what its own +technique's atomic actually does. CI enforces: +- All positives must match, all negatives must reject +- **No two rules may share detection logic** — a campaign whose detections are one rule under + many names reports coverage it does not have +- Every rule is mapped to a technique or explicitly declared untested **with a reason** +- Every rule stays silent on the benign workload (`purpleloop precision`) +- `sigma check` (pySigma) validates against the Sigma spec, not just this repo's parser +- A broken rule fails `go test ./cmd/... ./internal/...` and turns CI red ```bash go test ./internal/evaluator/ -v -run Regression @@ -155,7 +232,12 @@ process-creation rule — so a log line that merely mentions a binary cannot pro - **Lab-contained only.** Never run against production or targets outside `purpleloop-lab`. - **Linux Sysmon gap.** The Linux victim has command-output telemetry but not Sysmon process-creation events. This is a known gap; Windows Sysmon detection is confirmed working. +- **Collection gaps hand off to [detection-decay](https://github.com/jayelbotvibe-web/detection-decay).** A `NO_TELEMETRY` verdict is the symptom — the rule never saw the events it needs. detection-decay models the cause (source death, field drift). Every run emits `collection_gaps` naming the technique and the rules whose fields went missing. +- **One atomic per technique.** A technique runs the first atomic its plan or the rule map names. Exercising several per technique is not yet supported. - **SSVC mapping.** The arbiter uses pre-SSVC labels (Schedule/Monitor/Track) mapped to SSVC v2.1 equivalents. +- **Coverage denominator.** Coverage is measured over techniques that actually exercised a + detection. `ERROR`, `SKIPPED_PREREQ`, `NO_TELEMETRY` and `INCONCLUSIVE` are reported + separately rather than counted as gaps — a harness failure is not a detection gap. ## License diff --git a/RESULTS.md b/RESULTS.md index 24c982f..bc9285a 100644 --- a/RESULTS.md +++ b/RESULTS.md @@ -1 +1,64 @@ -"chains":[{"technique_id":"T1059.004","verdict":"DETECTED","rule_matched":"detections/windows/win_proc_create.yml","events_collected":73,"executed_at":"2026-07-04T18:09:20Z","evidence":[{"id":"archives-7594","matched":true,"fields":{"Image":"C:\\Windows\\SysWOW64\\net.exe","CommandLine":"net.exe accounts","ParentImage":"C:\\Program Files (x86)\\ossec-agent\\wazuh-agent.exe","User":"NT AUTHORITY\\SYSTEM"}}]}]} \ No newline at end of file +# Results + +## Live run — 2026-08-29, Linux victim + +``` +$ purpleloop run --technique T1082 \ + --victim-container purpleloop-victim \ + --manager-container single-node-wazuh.manager-1 + +INCONCLUSIVE: canary not detected on linux — coverage will not be reported as valid +``` + +| | | +|---|---| +| Atomic resolved | `T1082-3` — *List OS Information* (`uname -a`, `/etc/os-release`, …) | +| Source | Atomic Red Team @ `9415359` | +| Canary | **not detected** on linux | +| Run verdict | `INCONCLUSIVE` — coverage suppressed | +| Technique verdict | `INCONCLUSIVE` | +| Events collected | 0 (`no telemetry arrived within 1m3s of execution`) | + +**This is the correct outcome, and it is the point of the canary.** The Linux victim ships +command-output telemetry (`` command monitoring on a 360-second cycle) but no +process-creation source. The canary rule requires `category: process_creation`, so the positive +control cannot fire, so the pipeline is unproven — and an unproven pipeline may not report +coverage. Every technique in the run is `INCONCLUSIVE` rather than `MISSED`, because a missed +detection is a claim about a rule and nothing here tested a rule. + +The three trust contracts were verified against this run: + +- the per-run artifact was written to `reports/runs//coverage.json` for inspection; +- it was **not** appended to `reports/history.json`; +- it was **not** published to `docs/data/coverage.json`. + +**To close this gap** the Linux victim needs Sysmon-for-Linux (or auditd `execve` forwarding +that carries `Image`/`CommandLine`). Windows Sysmon Event ID 1 is confirmed working and is the +platform where detection has been demonstrated. + +--- + +## What the previous version of this file claimed + +The file previously held a single JSON blob reporting: + +```json +{"technique_id":"T1059.004","verdict":"DETECTED", + "rule_matched":"detections/windows/win_proc_create.yml", + "evidence":[{"fields":{"Image":"C:\\Windows\\SysWOW64\\net.exe", + "ParentImage":"C:\\Program Files (x86)\\ossec-agent\\wazuh-agent.exe"}}]} +``` + +Three things are wrong with it, all consequences of the engine defect fixed in the 2026-08 +changelog entries: + +1. **`T1059.004` is Bash on Linux.** The evidence is a Windows process. +2. **The rule credited is the Windows rule**, not the Linux rule for that technique. +3. **The "attack" is the monitoring agent itself** — `net.exe` spawned by `wazuh-agent.exe`. + Nothing the atomic did produced this event. + +The old engine executed `id; whoami` for every technique, collected whatever appeared in a +ten-minute window, and credited an unrelated match. The evidence-fidelity gate, the corrected +window, the platform check and the atomic resolver each independently prevent this result now. +It is kept here because a project that claims evidence should show what its own bad evidence +looked like. diff --git a/STARTUP.md b/STARTUP.md index fbda297..0d921eb 100644 --- a/STARTUP.md +++ b/STARTUP.md @@ -83,8 +83,17 @@ the Wazuh agent is connected (agent_control -l shows agent 002 Active). ### 3.1 Starting 1. Power on the VM in VMware -2. Confirm bridged IP: run `ipconfig` on the VM → should be `192.168.88.13` -3. Test SSH: `ssh windows-vm@192.168.88.13 hostname` +2. Find its address: run `ipconfig` on the VM. It depends on how the VM is + networked — bridged, NAT (`vmnet8`) and host-only (`vmnet1`) each give a + different subnet — so there is no address to expect here. +3. Export it, and test SSH: + ```bash + export WINDOWS_SSH_HOST= + export WINDOWS_SSH_USER=windows-vm + export WINDOWS_SSH_PASS= + ssh "$WINDOWS_SSH_USER@$WINDOWS_SSH_HOST" hostname + ``` + `lab/targets.yml` reads these; nothing in the repo hardcodes an address. 4. Confirm services: ```powershell Get-Service WazuhSvc, Sysmon64 | Format-Table Name, Status @@ -207,11 +216,12 @@ at boot. To prevent boot auto-start: `sudo systemctl disable docker`. | Symptom | Check | |---------|-------| -| Canary fails | `ssh windows-vm@192.168.88.13 "sc query WazuhSvc Sysmon64"` | +| Canary fails | `ssh "$WINDOWS_SSH_USER@$WINDOWS_SSH_HOST" "sc query WazuhSvc Sysmon64"` | | Agent not Active | `docker logs single-node-wazuh.manager-1 \| grep -i "auth\|enroll" \| tail -5` | -| No Sysmon events | `ssh windows-vm@192.168.88.13 "wevtutil qe Microsoft-Windows-Sysmon/Operational /c:2"` | +| No Sysmon events | `ssh "$WINDOWS_SSH_USER@$WINDOWS_SSH_HOST" "wevtutil qe Microsoft-Windows-Sysmon/Operational /c:2"` | | Dashboard 502 | `docker logs single-node-wazuh.indexer-1 \| tail -5` | | Build fails | `go mod tidy && make build` | +| Canary: "no address for the windows victim" | `WINDOWS_SSH_HOST` is unset. There is no default — see 3.1. | | Port conflicts | `docker ps --format '{{.Names}} {{.Ports}}'` | | vm.max_map_count reset | `sysctl vm.max_map_count` (must be ≥262144) | diff --git a/cmd/purpleloop/diff.go b/cmd/purpleloop/diff.go new file mode 100644 index 0000000..9569c92 --- /dev/null +++ b/cmd/purpleloop/diff.go @@ -0,0 +1,149 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "sort" +) + +// runDiff compares two runs and reports what changed per technique. +// +// A coverage percentage that moves from 80% to 70% tells you something broke +// but not what. This names it: "T1082 regressed DETECTED -> MISSED". That is +// the output detection engineering actually acts on, and it was impossible +// while the history index stored only a percentage. +func runDiff() { + fs := flag.NewFlagSet("diff", flag.ExitOnError) + failOnRegression := fs.Bool("fail-on-regression", false, "exit non-zero if any technique regressed") + if err := fs.Parse(os.Args[2:]); err != nil { + os.Exit(2) + } + if fs.NArg() != 2 { + fmt.Fprintln(os.Stderr, "usage: purpleloop diff [--fail-on-regression] ") + fmt.Fprintln(os.Stderr, " each argument is a run directory or its coverage.json") + fmt.Fprintln(os.Stderr, " flags must precede the run paths") + os.Exit(2) + } + + before, err := readCoverage(fs.Arg(0)) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + after, err := readCoverage(fs.Arg(1)) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + regressions, improvements, other := compareRuns(before, after) + + fmt.Printf("%s -> %s\n\n", fs.Arg(0), fs.Arg(1)) + printChanges("REGRESSED", regressions) + printChanges("IMPROVED", improvements) + printChanges("CHANGED", other) + + if len(regressions)+len(improvements)+len(other) == 0 { + fmt.Println("No per-technique changes.") + } + if *failOnRegression && len(regressions) > 0 { + os.Exit(1) + } +} + +type change struct{ Technique, From, To string } + +// verdictRank orders outcomes by how much detection they demonstrate, so a +// move down the scale is a regression. Verdicts that mean "never exercised" +// deliberately share the bottom: moving between them is a change worth showing +// but is not a detection regression. +var verdictRank = map[string]int{ + "ERROR": 0, + "SKIPPED_PREREQ": 0, + "INCONCLUSIVE": 0, + "NO_TELEMETRY": 1, + "MISSED": 2, + "DETECTED": 3, +} + +func compareRuns(before, after map[string]string) (regressions, improvements, other []change) { + seen := map[string]bool{} + for t := range before { + seen[t] = true + } + for t := range after { + seen[t] = true + } + techniques := make([]string, 0, len(seen)) + for t := range seen { + techniques = append(techniques, t) + } + sort.Strings(techniques) + + for _, t := range techniques { + from, okB := before[t] + to, okA := after[t] + switch { + case !okB: + other = append(other, change{t, "(absent)", to}) + continue + case !okA: + other = append(other, change{t, from, "(absent)"}) + continue + case from == to: + continue + } + c := change{t, from, to} + switch { + case verdictRank[to] < verdictRank[from]: + regressions = append(regressions, c) + case verdictRank[to] > verdictRank[from]: + improvements = append(improvements, c) + default: + other = append(other, c) + } + } + return regressions, improvements, other +} + +func printChanges(label string, cs []change) { + if len(cs) == 0 { + return + } + fmt.Printf("%s (%d)\n", label, len(cs)) + for _, c := range cs { + fmt.Printf(" %-12s %s -> %s\n", c.Technique, c.From, c.To) + } + fmt.Println() +} + +// readCoverage loads a run's per-technique verdicts from its coverage.json. +func readCoverage(path string) (map[string]string, error) { + if fi, err := os.Stat(path); err == nil && fi.IsDir() { + path = filepath.Join(path, "coverage.json") + } + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read run: %w", err) + } + var doc struct { + Techniques []struct { + ID string `json:"id"` + Verdict string `json:"verdict"` + } `json:"techniques"` + } + if err := json.Unmarshal(data, &doc); err != nil { + return nil, fmt.Errorf("parse %s: %w", path, err) + } + if len(doc.Techniques) == 0 { + return nil, fmt.Errorf("%s has no technique results", path) + } + out := make(map[string]string, len(doc.Techniques)) + for _, t := range doc.Techniques { + out[t.ID] = t.Verdict + } + return out, nil +} diff --git a/cmd/purpleloop/main.go b/cmd/purpleloop/main.go index 8a84846..7012730 100644 --- a/cmd/purpleloop/main.go +++ b/cmd/purpleloop/main.go @@ -10,37 +10,41 @@ import ( "net" "net/http" "os" + "path/filepath" "strings" "time" + "github.com/jayelbotvibe-web/purple-loop/internal/atomic" "github.com/jayelbotvibe-web/purple-loop/internal/canary" + "github.com/jayelbotvibe-web/purple-loop/internal/capture" "github.com/jayelbotvibe-web/purple-loop/internal/collector" "github.com/jayelbotvibe-web/purple-loop/internal/evaluator" "github.com/jayelbotvibe-web/purple-loop/internal/executor" "github.com/jayelbotvibe-web/purple-loop/internal/feed" + "github.com/jayelbotvibe-web/purple-loop/internal/lab" + "github.com/jayelbotvibe-web/purple-loop/internal/mapping" "github.com/jayelbotvibe-web/purple-loop/internal/model" "github.com/jayelbotvibe-web/purple-loop/internal/report" "github.com/jayelbotvibe-web/purple-loop/internal/server" ) -// techniqueRuleMap maps technique IDs to their Sigma rule files. -var techniqueRuleMap = map[string]string{ - "T1059.004": "detections/linux/proc_creation_susp_shell.yml", - "T1087.001": "detections/linux/T1087.001.yml", - "T1082": "detections/linux/T1082.yml", - "T1033": "detections/linux/T1033.yml", - "T1007": "detections/linux/T1007.yml", - "T1016": "detections/linux/T1016.yml", - "T1049": "detections/linux/T1049.yml", - "T1069.001": "detections/linux/T1069.001.yml", - "T1069": "detections/linux/T1069.001.yml", - "T1135": "detections/linux/T1135.yml", - "T1518": "detections/linux/T1518.yml", -} +// Default locations for the vendored Atomic Red Team tree and the lab +// overrides that sit in front of it. +const ( + defaultAtomicsRoot = "lab/atomic-red-team" + defaultOverrides = "mappings/atomic-overrides.yml" + defaultRuleMap = "mappings/attack_rule_map.json" + defaultTargets = "lab/targets.yml" + atomicsCommitRecord = "mappings/atomic-red-team.commit" + + // collectWindow is how far past the execution the collector looks. Phase 3 + // replaces this with a poll bounded by the ingestion watermark. + collectWindow = 10 * time.Minute +) func main() { if len(os.Args) < 2 { - fmt.Fprintln(os.Stderr, "usage: purpleloop run|canary [...]") + fmt.Fprintln(os.Stderr, "usage: purpleloop run|replay|precision|diff|canary|serve [...]") os.Exit(1) } @@ -54,6 +58,21 @@ func main() { return } + if os.Args[1] == "replay" { + runReplay() + return + } + + if os.Args[1] == "precision" { + runPrecision() + return + } + + if os.Args[1] == "diff" { + runDiff() + return + } + if os.Args[1] != "run" { fmt.Fprintln(os.Stderr, "usage: purpleloop run [--technique | --plan ] [flags]") os.Exit(2) @@ -67,10 +86,27 @@ func main() { dryRun := fs.Bool("dry-run", false, "run the pipeline without a live lab") victim := fs.String("victim-container", "", "Docker container for execution (e.g. purpleloop-victim)") manager := fs.String("manager-container", "", "Docker container for Wazuh manager") + atomicsRoot := fs.String("atomics", defaultAtomicsRoot, "vendored Atomic Red Team checkout") + ruleMapPath := fs.String("rule-map", defaultRuleMap, "technique -> atomic -> expected rules mapping") + targetsPath := fs.String("targets", defaultTargets, "lab target inventory") + settleTimeout := fs.Duration("settle-timeout", 60*time.Second, + "how long to wait for an execution's telemetry before calling it NO_TELEMETRY. "+ + "Must exceed the victim's collection interval: Wazuh command monitoring "+ + "defaults to 360s, so a 60s budget can never see it") + settlePoll := fs.Duration("settle-poll", 3*time.Second, "gap between telemetry polls") + overridesPath := fs.String("overrides", defaultOverrides, "lab overrides for unusable atomics") _ = fs.Parse(os.Args[2:]) ctx := context.Background() + // A partial container config would fire a REAL atomic on the victim while + // collecting from a dry (synthetic) SIEM — a real attack paired with a + // fabricated event. Reject it: a real run needs both, or pass --dry-run. + if !*dryRun && (*victim == "") != (*manager == "") { + fmt.Fprintln(os.Stderr, "error: a real run needs BOTH --victim-container and --manager-container (or pass --dry-run)") + os.Exit(2) + } + // Warn loudly whenever any pipeline stage is synthetic, so a dry-run report // can never be mistaken for real detection evidence. if *dryRun || *victim == "" || *manager == "" { @@ -80,31 +116,69 @@ func main() { fmt.Fprintln(os.Stderr, "└─────────────────────────────────────────────────────────────┘") } + // No registry, no run. Every technique's command comes from the vendored + // Atomic Red Team tree; without it there is nothing honest to execute. + reg, err := loadRegistry(*atomicsRoot, *overridesPath) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + rules, err := mapping.LoadRuleMap(*ruleMapPath) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + targets, err := lab.Load(*targetsPath) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + opts := runOpts{ + output: *output, + dryRun: *dryRun, + victimContainer: *victim, + managerContainer: *manager, + registry: reg, + rules: rules, + targets: targets, + settleTimeout: *settleTimeout, + settlePoll: *settlePoll, + } + switch { case *arbiterFile != "": - if err := runArbiter(ctx, *arbiterFile, *output, *dryRun, *victim, *manager); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + err = runArbiter(ctx, *arbiterFile, opts) case *emulationFile != "": - if err := runEmulation(ctx, *emulationFile, *output, *dryRun, *victim, *manager); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + err = runEmulation(ctx, *emulationFile, opts) case *planFile != "": - if err := runCampaign(ctx, *planFile, *output, *dryRun, *victim, *manager); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + err = runCampaign(ctx, *planFile, opts) case *technique != "": - if err := runOne(ctx, *technique, *output, *dryRun, *victim, *manager); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + err = runOne(ctx, *technique, opts) default: fmt.Fprintln(os.Stderr, "error: --technique or --plan is required") os.Exit(2) } + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +// loadRegistry opens the vendored Atomic Red Team tree. A run cannot proceed +// without it: with no registry there is no real command to execute, and the +// engine must never fall back to a fabricated one. +func loadRegistry(root, overrides string) (*atomic.Registry, error) { + reg, err := atomic.Load(root, overrides) + if err != nil { + return nil, err + } + if b, err := os.ReadFile(atomicsCommitRecord); err == nil { + reg.SetCommit(string(b)) + } + return reg, nil } func newExec(dryRun bool, victimContainer string) model.Executor { @@ -134,7 +208,297 @@ func newReporter(output string) model.Reporter { return report.JSONReporter{Out: os.Stdout} } -func runCampaign(ctx context.Context, planPath, output string, dryRun bool, victimContainer, managerContainer string) error { +// runOpts carries the shared pipeline configuration for a run command. +type runOpts struct { + output string + dryRun bool + victimContainer string + managerContainer string + campaign string + settleTimeout time.Duration + settlePoll time.Duration + registry *atomic.Registry + rules *mapping.RuleMap + targets *lab.Targets +} + +// with returns a copy of the options labelled with the campaign that is running. +func (o runOpts) with(campaign string) runOpts { + o.campaign = campaign + return o +} + +// synthetic reports whether any pipeline stage is dry/synthetic. Such a run's +// results are NOT real telemetry and must be marked non-evidentiary. +func (o runOpts) synthetic() bool { + return o.dryRun || o.victimContainer == "" || o.managerContainer == "" +} + +// runTasks drives a set of tasks through the pipeline, applying BOTH hard +// contracts in one place so no run command can bypass them: +// - synthetic marking (dry pipeline → not real evidence), and +// - canary gating (a real run whose positive control does not fire is +// INCONCLUSIVE and its coverage must not be presented as valid). +func runTasks(ctx context.Context, tasks []model.TechniqueTask, o runOpts) error { + result := model.CampaignResult{ + StartedAt: time.Now().UTC(), + Synthetic: o.synthetic(), + Campaign: o.campaign, + AtomicsCommit: o.registry.Commit(), + UntestedRules: untestedFor(o.rules, tasks), + } + + // Group work by the platform its rule-map entry declares, so a campaign can + // span the Linux container and the Windows VM in one run. Previously every + // technique went to a hardcoded Linux target and the Windows rule was + // unreachable from `run`. + byPlatform := map[string][]model.TechniqueTask{} + var order []string + for _, task := range tasks { + p := o.platformFor(task.TechniqueID) + if _, seen := byPlatform[p]; !seen { + order = append(order, p) + } + byPlatform[p] = append(byPlatform[p], task) + } + + for _, platform := range order { + plat := o.stackFor(platform) + if plat.err != nil { + for _, task := range byPlatform[platform] { + result.Chains = append(result.Chains, model.ProofChain{ + TechniqueID: task.TechniqueID, + SourceCVE: task.SourceCVE, + ArbiterPriority: task.Priority, + Verdict: model.Errored, + Note: plat.err.Error(), + }) + } + continue + } + + // Canary gate, per platform. A campaign spanning two hosts must prove + // BOTH pipelines: a healthy Linux canary says nothing about whether + // Windows telemetry is flowing. + if !result.Synthetic { + cr := canary.Checker{RulesDir: "detections"}.Run(ctx, canary.NewMarker(), + plat.exec, plat.coll, platform, plat.target) + if !cr.Healthy { + result.Inconclusive = true + result.CanaryDetail = appendDetail(result.CanaryDetail, canaryFailDetail(cr)) + fmt.Fprintf(os.Stderr, "INCONCLUSIVE: %s — coverage will not be reported as valid\n", + canaryFailDetail(cr)) + } else { + result.CanaryHealthy = true + } + } + + for _, task := range byPlatform[platform] { + chain, err := runTechnique(ctx, plat.exec, plat.coll, plat.eval, o.registry, o.rules, task, + plat.target, collector.Settler{Timeout: o.settleTimeout, PollInterval: o.settlePoll}) + if err != nil { + chain = model.ProofChain{TechniqueID: task.TechniqueID, Verdict: model.Errored, Note: err.Error()} + } + chain.SourceCVE = task.SourceCVE + chain.ArbiterPriority = task.Priority + result.Chains = append(result.Chains, chain) + } + } + + // A failed positive control on ANY platform invalidates the run: without a + // working pipeline no per-technique verdict is trustworthy. + if result.Inconclusive { + result.CanaryHealthy = false + for i := range result.Chains { + if result.Chains[i].Verdict != model.Errored { + result.Chains[i].Verdict = model.Inconclusive + } + } + } + + labelAttribution(result.Chains, o.synthetic(), primaryHost(o)) + + // Capture the raw telemetry so this run can be re-evaluated against future + // rule changes with no lab. Synthetic runs are refused by the writer. + if dir := captureDir(o.output, result.StartedAt); dir != "" { + if err := capture.Write(dir, result, primaryHost(o)); err != nil { + fmt.Fprintf(os.Stderr, "warning: could not capture telemetry: %v\n", err) + } + } + + return newReporter(o.output).Write(result) +} + +// platformStack is the executor/collector/evaluator triple for one platform. +type platformStack struct { + exec model.Executor + coll model.Collector + eval model.Evaluator + target model.Target + err error +} + +// platformFor reports which lab platform a technique belongs on. +func (o runOpts) platformFor(technique string) string { + if o.rules != nil { + if e, ok := o.rules.Entry(technique); ok && e.Platform != "" { + return strings.ToLower(e.Platform) + } + } + return "linux" +} + +// stackFor builds the pipeline for one platform from the lab inventory. +func (o runOpts) stackFor(platform string) platformStack { + t, ok := o.targets.ForPlatform(platform) + if !ok { + return platformStack{err: fmt.Errorf( + "no lab target declared for platform %q — add one to lab/targets.yml", platform)} + } + + st := platformStack{ + target: t.Model(), + eval: evaluator.RuleMatcherEvaluator{RulesDir: orDefaultStr(t.RulesDir, "detections/"+platform)}, + coll: newColl(o.dryRun, o.managerContainer), + } + + switch { + case o.synthetic(): + st.exec = executor.DryExecutor{} + case t.Executor == "ssh": + host, user, pass := t.SSHConfig() + if host == "" { + return platformStack{err: fmt.Errorf( + "target %s needs an SSH host (set %s)", t.Name, t.SSHHostEnv)} + } + st.exec = &executor.SSHExecutor{Host: host, User: user, Password: pass} + default: + container := t.Container + if o.victimContainer != "" { + container = o.victimContainer + } + st.exec = executor.DockerExecutor{Container: container} + } + return st +} + +// primaryHost is the host recorded on captured telemetry. A single-platform +// campaign names its target; a mixed one records none, because the dataset +// spans hosts and claiming one would be wrong. +func primaryHost(o runOpts) string { + if t, ok := o.targets.ForPlatform("linux"); ok { + return t.Name + } + return "" +} + +func appendDetail(existing, add string) string { + if existing == "" { + return add + } + return existing + "; " + add +} + +func orDefaultStr(v, def string) string { + if strings.TrimSpace(v) == "" { + return def + } + return v +} + +// untestedFor names every rule this campaign will not exercise, with why. +// Rules declared untested in the map keep their declared reason; a mapped rule +// whose technique is absent from this campaign is reported as not-in-campaign. +// Coverage is only honest next to what it did not cover. +func untestedFor(rules *mapping.RuleMap, tasks []model.TechniqueTask) map[string]string { + if rules == nil { + return nil + } + out := map[string]string{} + for rule, reason := range rules.UntestedRules { + out[rule] = reason + } + + inCampaign := map[string]bool{} + for _, t := range tasks { + for _, r := range rules.ExpectedRules(t.TechniqueID) { + inCampaign[r] = true + } + } + for rule := range rules.MappedRules() { + if !inCampaign[rule] { + out[rule] = "not_in_campaign — mapped to a technique this plan does not run" + } + } + if len(out) == 0 { + return nil + } + return out +} + +// captureDir mirrors DashboardReporter's per-run layout so a run's dataset sits +// beside its coverage.json. +func captureDir(output string, startedAt time.Time) string { + dir := output + if dir == "" || strings.HasSuffix(dir, ".html") || strings.HasSuffix(dir, ".json") { + dir = "reports" + } + runID := fmt.Sprintf("campaign-%s", startedAt.UTC().Format("20060102T150405Z")) + return filepath.Join(dir, "runs", runID) +} + +// labelAttribution records how tightly each verdict is bound to its own +// execution. Overlap is a campaign-level property — two techniques whose +// collection windows intersect can each pick up the other's events — so it can +// only be decided once every window is known. +// +// This does not change any verdict. It states the quality of the evidence +// behind one, which is the difference between a coverage number and a claim. +func labelAttribution(chains []model.ProofChain, synthetic bool, host string) { + for i := range chains { + c := &chains[i] + + // A synthetic pipeline produced no real telemetry, so nothing can be + // attributed to a real execution. + if synthetic { + c.Attribution = model.Unscoped + continue + } + // A technique that never ran has no window to scope. + if c.Window.Start.IsZero() || c.Window.End.IsZero() { + c.Attribution = model.Unscoped + continue + } + + overlapped := false + for j := range chains { + if i == j || chains[j].Window.Start.IsZero() { + continue + } + if c.Window.Overlaps(chains[j].Window) { + overlapped = true + break + } + } + switch { + case overlapped: + c.Attribution = model.WindowOverlap + case host != "": + c.Attribution = model.WindowAndHostScoped + default: + c.Attribution = model.WindowScoped + } + } +} + +func canaryFailDetail(cr canary.Result) string { + if cr.Err != nil { + return fmt.Sprintf("canary not detected on %s: %v", cr.Platform, cr.Err) + } + return fmt.Sprintf("canary not detected on %s", cr.Platform) +} + +func runCampaign(ctx context.Context, planPath string, o runOpts) error { var f feed.StaticFeed if err := f.Load(planPath); err != nil { return fmt.Errorf("load plan: %w", err) @@ -143,28 +507,10 @@ func runCampaign(ctx context.Context, planPath, output string, dryRun bool, vict if err != nil { return fmt.Errorf("feed: %w", err) } - - exec := newExec(dryRun, victimContainer) - coll := newColl(dryRun, managerContainer) - eval := evaluator.RuleMatcherEvaluator{RulesDir: "detections/linux"} - rep := newReporter(output) - target := model.Target{Host: "victim01", Kind: "linux"} - - result := model.CampaignResult{StartedAt: time.Now().UTC()} - for _, task := range tasks { - chain, err := runTechnique(ctx, exec, coll, eval, task, target) - if err != nil { - chain = model.ProofChain{ - TechniqueID: task.TechniqueID, - Verdict: model.Errored, - } - } - result.Chains = append(result.Chains, chain) - } - return rep.Write(result) + return runTasks(ctx, tasks, o.with(f.Name)) } -func runArbiter(ctx context.Context, arbiterPath, output string, dryRun bool, victimContainer, managerContainer string) error { +func runArbiter(ctx context.Context, arbiterPath string, o runOpts) error { var f feed.ArbiterFeed if err := f.Load(arbiterPath); err != nil { return fmt.Errorf("load arbiter export: %w", err) @@ -173,150 +519,263 @@ func runArbiter(ctx context.Context, arbiterPath, output string, dryRun bool, vi if err != nil { return fmt.Errorf("arbiter feed: %w", err) } - - exec := newExec(dryRun, victimContainer) - coll := newColl(dryRun, managerContainer) - eval := evaluator.RuleMatcherEvaluator{RulesDir: "detections/linux"} - rep := newReporter(output) - target := model.Target{Host: "victim01", Kind: "linux"} - - result := model.CampaignResult{StartedAt: time.Now().UTC()} - for _, task := range tasks { - chain, err := runTechnique(ctx, exec, coll, eval, task, target) - if err != nil { - chain = model.ProofChain{ - TechniqueID: task.TechniqueID, - Verdict: model.Errored, - } - } - chain.SourceCVE = task.SourceCVE - chain.ArbiterPriority = task.Priority - result.Chains = append(result.Chains, chain) - } - return rep.Write(result) + return runTasks(ctx, tasks, o.with("arbiter")) } -func runEmulation(ctx context.Context, emuPath, output string, dryRun bool, victimContainer, managerContainer string) error { +func runEmulation(ctx context.Context, emuPath string, o runOpts) error { plan, err := feed.LoadEmulation(emuPath) if err != nil { return fmt.Errorf("load emulation: %w", err) } + var tasks []model.TechniqueTask + for _, stage := range plan.Stages { + tasks = append(tasks, stage.ToTasks()...) + } + return runTasks(ctx, tasks, o.with(plan.Name)) +} + +func runOne(ctx context.Context, technique string, o runOpts) error { + // Deliberately leave AtomicIDs empty so atomicIDFor consults the rule map. + // Synthesizing "-1" here pre-empted it and picked ART's first + // test, which for most techniques is the WINDOWS one — `--technique T1082` + // resolved to a command_prompt atomic and errored on the Linux victim, even + // though the rule map names T1082-3. + task := model.TechniqueTask{TechniqueID: technique} + return runTasks(ctx, []model.TechniqueTask{task}, o.with(technique)) +} - exec := newExec(dryRun, victimContainer) - coll := newColl(dryRun, managerContainer) - eval := evaluator.RuleMatcherEvaluator{RulesDir: "detections/linux"} - rep := newReporter(output) - target := model.Target{Host: "victim01", Kind: "linux"} +// atomicIDFor picks the atomic this task should run: the plan's choice first, +// then the mapping's. Only if neither declares one does it guess the +// technique's first test — and the resolved test's platform is still checked +// before anything executes. +func atomicIDFor(task model.TechniqueTask, rules *mapping.RuleMap) string { + if len(task.AtomicIDs) > 0 && strings.TrimSpace(task.AtomicIDs[0]) != "" { + return strings.TrimSpace(task.AtomicIDs[0]) + } + if rules != nil { + if id := rules.AtomicFor(task.TechniqueID); id != "" { + return id + } + } + return task.TechniqueID + "-1" +} - result := model.CampaignResult{StartedAt: time.Now().UTC()} - for _, stage := range plan.Stages { - for _, task := range stage.ToTasks() { - chain, err := runTechnique(ctx, exec, coll, eval, task, target) - if err != nil { - chain = model.ProofChain{ - TechniqueID: task.TechniqueID, - Verdict: model.Errored, - } +// unmetPrereq runs each dependency's prereq_command on the target and returns a +// description of the first one that fails. A technique whose prerequisites are +// not met never ran, so it is SKIPPED_PREREQ — reporting it as MISSED would +// invent a detection gap out of an unmet dependency. +func unmetPrereq(ctx context.Context, exec model.Executor, test atomic.Test, target model.Target) string { + for _, dep := range test.Dependencies { + if dep.PrereqCommand == "" { + continue + } + probe := model.AtomicTest{ + ID: test.ID + "-prereq", + TechniqueID: test.TechniqueID, + Command: dep.PrereqCommand, + Executor: test.DependencyExecutor, + } + res, err := exec.Run(ctx, probe, target) + if err != nil { + return fmt.Sprintf("prerequisite check failed to run (%s): %v", dep.Description, err) + } + if res.ExitCode != 0 { + desc := dep.Description + if desc == "" { + desc = dep.PrereqCommand } - // per-stage verdict tracking via priority field - chain.ArbiterPriority = task.Priority - result.Chains = append(result.Chains, chain) + return "unmet prerequisite: " + desc } } - return rep.Write(result) + return "" } -func runOne(ctx context.Context, technique, output string, dryRun bool, victimContainer, managerContainer string) error { - exec := newExec(dryRun, victimContainer) - coll := newColl(dryRun, managerContainer) - eval := evaluator.RuleMatcherEvaluator{RulesDir: "detections/linux"} - rep := newReporter(output) - - task := model.TechniqueTask{TechniqueID: technique, AtomicIDs: []string{technique + "-1"}} - target := model.Target{Host: "victim01", Kind: "linux"} - - chain, err := runTechnique(ctx, exec, coll, eval, task, target) +// runTechnique resolves the task's atomic to a REAL command, runs it, collects +// telemetry and evaluates the mapped rule. +// +// Resolution is strict on purpose. Every failure below returns a verdict that +// names what went wrong rather than a coverage number: an unresolvable atomic, +// a platform mismatch and an unmet prerequisite are all reasons a detection was +// never exercised, and none of them is evidence that a detection is missing. +func runTechnique(ctx context.Context, exec model.Executor, coll model.Collector, eval model.Evaluator, + reg *atomic.Registry, rules *mapping.RuleMap, task model.TechniqueTask, target model.Target, + settler collector.Settler) (model.ProofChain, error) { + + atomicID := atomicIDFor(task, rules) + chain := model.ProofChain{TechniqueID: task.TechniqueID} + + test, err := reg.Resolve(atomicID) if err != nil { - return err + chain.Verdict = model.Errored + chain.Note = err.Error() + return chain, nil } - return rep.Write(model.CampaignResult{StartedAt: time.Now().UTC(), Chains: []model.ProofChain{chain}}) -} -func runTechnique(ctx context.Context, exec model.Executor, coll model.Collector, eval model.Evaluator, task model.TechniqueTask, target model.Target) (model.ProofChain, error) { - atomicID := task.TechniqueID + "-1" - if len(task.AtomicIDs) > 0 { - atomicID = task.AtomicIDs[0] + chain.Atomic = model.AtomicTest{ + ID: test.ID, + TechniqueID: test.TechniqueID, + Name: test.Name, + GUID: test.GUID, + Command: test.Command, + CleanupCommand: test.CleanupCommand, + Executor: test.Executor, + Source: test.Source, + OverrideReason: test.OverrideReason, } - atomic := model.AtomicTest{ - ID: atomicID, - TechniqueID: task.TechniqueID, - Command: "id; whoami", - Executor: "sh", + if !test.SupportsPlatform(target.Kind) { + chain.Verdict = model.Errored + chain.Note = fmt.Sprintf("atomic %s supports %v, but target %s is %s — fix the plan's atomic_ids", + test.ID, test.SupportedPlatforms, target.Host, target.Kind) + return chain, nil } - run, err := exec.Run(ctx, atomic, target) + if unmet := unmetPrereq(ctx, exec, test, target); unmet != "" { + chain.Verdict = model.SkippedPrereq + chain.Note = unmet + return chain, nil + } + + run, err := exec.Run(ctx, chain.Atomic, target) if err != nil { - return model.ProofChain{}, fmt.Errorf("execute: %w", err) + return chain, fmt.Errorf("execute: %w", err) } - // Cleanup is best-effort per atomic after run - _ = exec.Cleanup(ctx, atomic, target) + chain.ExecutedAt = run.StartedAt - // Let Wazuh ingest telemetry before querying - time.Sleep(10 * time.Second) + // Cleanup is best-effort per atomic after run. + _ = exec.Cleanup(ctx, chain.Atomic, target) - events, err := coll.Query(ctx, run.Window(10*time.Minute), target.Host) - if err != nil { - return model.ProofChain{}, fmt.Errorf("collect: %w", err) + window := run.Window(collectWindow) + chain.Window = window + + expected := rules.ExpectedRules(task.TechniqueID) + chain.RulesExpected = expected + if len(expected) == 0 { + chain.Verdict = model.Errored + chain.Note = "no Sigma rule mapped for " + task.TechniqueID + " in the rule map" + return chain, nil } - // Resolve rule path from embedded technique mapping - rulePath := "" // empty → evaluator returns MISSED for unmapped techniques - if mapped, ok := techniqueRuleMap[task.TechniqueID]; ok { - rulePath = mapped + // Poll until an expected rule matches or ingestion settles, rather than + // sleeping a fixed interval and querying once. + settled, err := settler.Settle(ctx, coll, window, target.Host, + func(events []model.Event) bool { + v, _, _, err := evaluateExpected(eval, expected, task.TechniqueID, events) + return err == nil && v == model.Detected + }) + if err != nil { + return chain, fmt.Errorf("collect: %w", err) + } + events := settled.Events + chain.EventsCollected = len(events) + chain.Collected = events + if settled.TimedOut && len(events) == 0 { + chain.Note = fmt.Sprintf("no telemetry arrived within %s of execution", settled.Waited.Round(time.Second)) } - rule := model.SigmaRule{Path: rulePath, Title: task.TechniqueID} - verdict, evidence, err := eval.Evaluate(rule, events) + + // A technique is DETECTED when ANY expected rule matches. Evaluating only + // the first would understate coverage whenever a second rule is the one that + // legitimately catches the behaviour. + verdict, evidence, matched, err := evaluateExpected(eval, expected, task.TechniqueID, events) if err != nil { - return model.ProofChain{}, fmt.Errorf("evaluate: %w", err) + return chain, fmt.Errorf("evaluate: %w", err) } - ruleMatched := "" - if verdict == model.Detected { - ruleMatched = rule.Path + chain.Verdict = verdict + chain.Evidence = evidence + chain.RulesMatched = matched + if len(matched) > 0 { + chain.RuleMatched = matched[0] + chain.DetectLatencyMS = detectLatency(run.StartedAt, evidence) } + return chain, nil +} + +// evaluateExpected runs every expected rule and reports the best outcome. The +// verdict precedence is DETECTED > MISSED > INCONCLUSIVE > NO_TELEMETRY: a rule +// that matched is proof of detection regardless of what the others did, while +// NO_TELEMETRY only stands if no rule ever saw usable events. +func evaluateExpected(eval model.Evaluator, expected []string, title string, events []model.Event) ( + model.Verdict, []model.Event, []string, error) { - return model.ProofChain{ - TechniqueID: task.TechniqueID, - SourceCVE: task.SourceCVE, - ArbiterPriority: task.Priority, - Atomic: atomic, - ExecutedAt: run.StartedAt, - EventsCollected: len(events), - RuleMatched: ruleMatched, - Verdict: verdict, - Evidence: evidence, - }, nil + best := model.NoTelemetry + var evidence []model.Event + var matched []string + + rank := map[model.Verdict]int{ + model.NoTelemetry: 0, + model.Inconclusive: 1, + model.Missed: 2, + model.Detected: 3, + } + + for _, path := range expected { + v, ev, err := eval.Evaluate(model.SigmaRule{Path: path, Title: title}, events) + if err != nil { + return model.Errored, nil, nil, err + } + if v == model.Detected { + matched = append(matched, path) + evidence = append(evidence, ev...) + } + if rank[v] > rank[best] { + best = v + } + } + return best, evidence, matched, nil +} + +// detectLatency is the gap between execution start and the earliest matching +// event — how long the pipeline took to make the behaviour visible. +func detectLatency(start time.Time, evidence []model.Event) int64 { + earliest := time.Time{} + for _, e := range evidence { + if e.Timestamp.IsZero() { + continue + } + if earliest.IsZero() || e.Timestamp.Before(earliest) { + earliest = e.Timestamp + } + } + if earliest.IsZero() || earliest.Before(start) { + return 0 + } + return earliest.Sub(start).Milliseconds() } func runCanaryCmd() { marker := canary.NewMarker() ctx := context.Background() - sshHost := os.Getenv("WINDOWS_SSH_HOST") - sshUser := os.Getenv("WINDOWS_SSH_USER") - sshPass := os.Getenv("WINDOWS_SSH_PASS") - if sshHost == "" { - sshHost = "192.168.88.13" + + // The Windows host comes from the lab inventory, not from a literal here. + // It used to default to 192.168.88.13, an address that only existed on one + // machine's bridged network — on any other host the canary silently probed + // something that was not the victim, or nothing at all. + targets, err := lab.Load(defaultTargets) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) } - if sshUser == "" { - sshUser = "windows-vm" + t, ok := targets.ForPlatform("windows") + if !ok { + fmt.Fprintf(os.Stderr, "error: no windows target declared in %s\n", defaultTargets) + os.Exit(1) } + sshHost, sshUser, sshPass := t.SSHConfig() + if sshHost == "" { + fmt.Fprintf(os.Stderr, + "error: no address for the windows victim. Set %s to the IP the VM "+ + "actually has (run `ipconfig` on it) — there is no default, because a\n"+ + "wrong one probes a host that is not your victim.\n", t.SSHHostEnv) + os.Exit(1) + } + exec := &executor.SSHExecutor{Host: sshHost, User: sshUser, Password: sshPass} coll := &collector.WazuhCollector{ManagerContainer: "single-node-wazuh.manager-1"} target := model.Target{Host: "windows-vm", Kind: "windows"} fmt.Printf("Canary marker: %s\n", marker) - result := canary.Check(ctx, marker, exec, coll, "windows", target, false) + result := canary.Check(ctx, marker, exec, coll, "windows", target) if result.Healthy { fmt.Printf("Canary: DETECTED on %s (evidence: %d events)\n", result.Platform, len(result.Evidence)) } else { diff --git a/cmd/purpleloop/main_test.go b/cmd/purpleloop/main_test.go new file mode 100644 index 0000000..5aa134a --- /dev/null +++ b/cmd/purpleloop/main_test.go @@ -0,0 +1,286 @@ +package main + +import ( + "testing" + "time" + + "github.com/jayelbotvibe-web/purple-loop/internal/mapping" + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +func win(startSec, endSec int) model.TimeWindow { + base := time.Date(2026, 8, 29, 10, 0, 0, 0, time.UTC) + return model.TimeWindow{ + Start: base.Add(time.Duration(startSec) * time.Second), + End: base.Add(time.Duration(endSec) * time.Second), + } +} + +func TestLabelAttributionSyntheticIsUnscoped(t *testing.T) { + chains := []model.ProofChain{{Window: win(0, 10)}} + labelAttribution(chains, true, "victim01") + if chains[0].Attribution != model.Unscoped { + t.Errorf("synthetic run = %q, want %q", chains[0].Attribution, model.Unscoped) + } +} + +func TestLabelAttributionDetectsOverlap(t *testing.T) { + chains := []model.ProofChain{ + {TechniqueID: "T1082", Window: win(0, 60)}, + {TechniqueID: "T1033", Window: win(30, 90)}, // overlaps the first + {TechniqueID: "T1007", Window: win(200, 260)}, + } + labelAttribution(chains, false, "victim01") + + if chains[0].Attribution != model.WindowOverlap { + t.Errorf("T1082 = %q, want %q", chains[0].Attribution, model.WindowOverlap) + } + if chains[1].Attribution != model.WindowOverlap { + t.Errorf("T1033 = %q, want %q", chains[1].Attribution, model.WindowOverlap) + } + if chains[2].Attribution != model.WindowAndHostScoped { + t.Errorf("T1007 = %q, want %q", chains[2].Attribution, model.WindowAndHostScoped) + } +} + +func TestLabelAttributionWithoutHost(t *testing.T) { + chains := []model.ProofChain{{Window: win(0, 10)}} + labelAttribution(chains, false, "") + if chains[0].Attribution != model.WindowScoped { + t.Errorf("no host = %q, want %q", chains[0].Attribution, model.WindowScoped) + } +} + +// A technique that never ran has no window, so nothing can be attributed to it. +func TestLabelAttributionNoWindow(t *testing.T) { + chains := []model.ProofChain{{TechniqueID: "T1518", Verdict: model.SkippedPrereq}} + labelAttribution(chains, false, "victim01") + if chains[0].Attribution != model.Unscoped { + t.Errorf("unrun technique = %q, want %q", chains[0].Attribution, model.Unscoped) + } +} + +// stubEval returns a canned verdict per rule path. +type stubEval map[string]model.Verdict + +func (s stubEval) Evaluate(rule model.SigmaRule, events []model.Event) (model.Verdict, []model.Event, error) { + v, ok := s[rule.Path] + if !ok { + v = model.Missed + } + if v == model.Detected { + return v, []model.Event{{ID: "ev-" + rule.Path}}, nil + } + return v, nil, nil +} + +// The point of expected_rules: a second rule catching the behaviour is still a +// detection. Evaluating only the first would report a gap that does not exist. +func TestEvaluateExpectedAnyRuleDetects(t *testing.T) { + eval := stubEval{"a.yml": model.Missed, "b.yml": model.Detected} + v, ev, matched, err := evaluateExpected(eval, []string{"a.yml", "b.yml"}, "T1082", []model.Event{{}}) + if err != nil { + t.Fatal(err) + } + if v != model.Detected { + t.Errorf("verdict = %q, want DETECTED", v) + } + if len(matched) != 1 || matched[0] != "b.yml" { + t.Errorf("matched = %v, want [b.yml]", matched) + } + if len(ev) == 0 { + t.Error("a detection must carry its evidence") + } +} + +// NO_TELEMETRY only stands when no rule saw usable events. A real MISSED from +// any rule outranks it, because something was actually evaluated. +func TestEvaluateExpectedVerdictPrecedence(t *testing.T) { + cases := []struct { + name string + eval stubEval + want model.Verdict + }{ + {"missed beats no_telemetry", stubEval{"a.yml": model.NoTelemetry, "b.yml": model.Missed}, model.Missed}, + {"inconclusive beats no_telemetry", stubEval{"a.yml": model.NoTelemetry, "b.yml": model.Inconclusive}, model.Inconclusive}, + {"detected beats all", stubEval{"a.yml": model.Missed, "b.yml": model.Detected}, model.Detected}, + {"all silent stays no_telemetry", stubEval{"a.yml": model.NoTelemetry, "b.yml": model.NoTelemetry}, model.NoTelemetry}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + v, _, _, err := evaluateExpected(c.eval, []string{"a.yml", "b.yml"}, "T", []model.Event{{}}) + if err != nil { + t.Fatal(err) + } + if v != c.want { + t.Errorf("verdict = %q, want %q", v, c.want) + } + }) + } +} + +func TestDetectLatency(t *testing.T) { + start := time.Date(2026, 8, 29, 10, 0, 0, 0, time.UTC) + ev := []model.Event{ + {Timestamp: start.Add(5 * time.Second)}, + {Timestamp: start.Add(2 * time.Second)}, // earliest wins + } + if got := detectLatency(start, ev); got != 2000 { + t.Errorf("latency = %dms, want 2000", got) + } + // An event stamped before the execution cannot be its evidence. + if got := detectLatency(start, []model.Event{{Timestamp: start.Add(-time.Second)}}); got != 0 { + t.Errorf("pre-execution event gave latency %d, want 0", got) + } + if got := detectLatency(start, nil); got != 0 { + t.Errorf("no evidence gave latency %d, want 0", got) + } +} + +// Replaying the shipped dataset runs the REAL evaluator over real event shapes, +// with no lab, no Docker and no Wazuh. This is the regression test the project +// could not previously have: rule edits are now checked against telemetry +// rather than only against fixtures written to match them. +// +// The expected verdicts are deliberately mixed. A sample where everything +// passes would not catch a rule that matches everything. +func TestReplaySampleDataset(t *testing.T) { + result, err := replayDataset("../../testdata/sample-run", + "../../mappings/attack_rule_map.json", "../../detections/linux") + if err != nil { + t.Fatalf("replay: %v", err) + } + + want := map[string]model.Verdict{ + "T1082": model.Detected, + "T1033": model.Detected, + "T1049": model.Detected, + "T1518": model.Missed, // ran, produced telemetry, rule legitimately did not match + } + if len(result.Chains) != len(want) { + t.Fatalf("chains = %d, want %d", len(result.Chains), len(want)) + } + for _, c := range result.Chains { + w, ok := want[c.TechniqueID] + if !ok { + t.Errorf("unexpected technique %s in replay", c.TechniqueID) + continue + } + if c.Verdict != w { + t.Errorf("%s = %s, want %s (note: %s)", c.TechniqueID, c.Verdict, w, c.Note) + } + if c.EventsCollected == 0 { + t.Errorf("%s replayed with no events", c.TechniqueID) + } + } +} + +// A replay is a real evaluation of real telemetry, so it must never be marked +// synthetic — that flag means "fabricated", and mislabelling would either +// suppress a valid result or launder a fake one. +func TestReplayIsNotSynthetic(t *testing.T) { + result, err := replayDataset("../../testdata/sample-run", + "../../mappings/attack_rule_map.json", "../../detections/linux") + if err != nil { + t.Fatal(err) + } + if result.Synthetic { + t.Error("a replay of captured telemetry is not a synthetic run") + } +} + +func TestCompareRunsClassifiesChanges(t *testing.T) { + before := map[string]string{ + "T1082": "DETECTED", + "T1033": "MISSED", + "T1049": "DETECTED", + "T1007": "NO_TELEMETRY", + "T1016": "DETECTED", + } + after := map[string]string{ + "T1082": "MISSED", // regression — the one that matters + "T1033": "DETECTED", // improvement + "T1049": "DETECTED", // unchanged + "T1007": "ERROR", // regression: it used to at least collect + "T1518": "DETECTED", // newly present + // T1016 absent + } + + reg, imp, other := compareRuns(before, after) + + if len(reg) != 2 { + t.Errorf("regressions = %v, want 2 — T1082 and T1007; T1016 dropping out is a change, not a regression", reg) + } + found := map[string]string{} + for _, c := range reg { + found[c.Technique] = c.From + "->" + c.To + } + if found["T1082"] != "DETECTED->MISSED" { + t.Errorf("T1082 = %q, want DETECTED->MISSED", found["T1082"]) + } + if found["T1007"] != "NO_TELEMETRY->ERROR" { + t.Errorf("T1007 = %q, want NO_TELEMETRY->ERROR", found["T1007"]) + } + + if len(imp) != 1 || imp[0].Technique != "T1033" { + t.Errorf("improvements = %v, want just T1033", imp) + } + + // Techniques appearing or disappearing are changes, but not regressions. + otherSet := map[string]bool{} + for _, c := range other { + otherSet[c.Technique] = true + } + if !otherSet["T1518"] || !otherSet["T1016"] { + t.Errorf("other = %v, want T1518 (new) and T1016 (absent)", other) + } +} + +// Verdicts that all mean "never exercised" must not read as regressions of one +// another — moving from ERROR to SKIPPED_PREREQ is a different explanation of +// the same non-result, not a detection getting worse. +func TestCompareRunsNotExercisedVerdictsAreEqualRank(t *testing.T) { + reg, imp, other := compareRuns( + map[string]string{"T1": "ERROR"}, + map[string]string{"T1": "SKIPPED_PREREQ"}) + if len(reg) != 0 || len(imp) != 0 { + t.Errorf("want no regression/improvement, got reg=%v imp=%v", reg, imp) + } + if len(other) != 1 { + t.Errorf("want the change reported as 'other', got %v", other) + } +} + +func TestCompareRunsIdenticalIsQuiet(t *testing.T) { + m := map[string]string{"T1082": "DETECTED", "T1033": "MISSED"} + reg, imp, other := compareRuns(m, m) + if len(reg)+len(imp)+len(other) != 0 { + t.Errorf("identical runs must produce no changes: %v %v %v", reg, imp, other) + } +} + +// `--technique T1082` must use the atomic the rule map names (T1082-3, Linux), +// not ART's first test (T1082-1, Windows). Guessing "-1" picks the +// Windows atomic for most techniques, which then errors on the Linux victim. +func TestAtomicIDForPrefersRuleMapOverGuess(t *testing.T) { + rules, err := mapping.LoadRuleMap("../../mappings/attack_rule_map.json") + if err != nil { + t.Fatal(err) + } + task := model.TechniqueTask{TechniqueID: "T1082"} // no atomic named + if got := atomicIDFor(task, rules); got != "T1082-3" { + t.Errorf("atomicIDFor = %q, want T1082-3 from the rule map", got) + } + + // An explicit plan choice still wins over the map. + task.AtomicIDs = []string{"T1082-8"} + if got := atomicIDFor(task, rules); got != "T1082-8" { + t.Errorf("atomicIDFor = %q, want the plan's explicit T1082-8", got) + } + + // Only with neither does it fall back to the guess. + unmapped := model.TechniqueTask{TechniqueID: "T9999"} + if got := atomicIDFor(unmapped, rules); got != "T9999-1" { + t.Errorf("atomicIDFor = %q, want the T9999-1 fallback", got) + } +} diff --git a/cmd/purpleloop/precision.go b/cmd/purpleloop/precision.go new file mode 100644 index 0000000..dc6946c --- /dev/null +++ b/cmd/purpleloop/precision.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + + "github.com/jayelbotvibe-web/purple-loop/internal/evaluator" + "github.com/jayelbotvibe-web/purple-loop/internal/precision" +) + +// runPrecision measures the false-positive rate of the shipped detections +// against a benign administrator workload. +// +// Coverage answers "does it fire when it should". This answers "does it stay +// quiet when it should", which coverage alone cannot: a rule matching +// everything scores 100% coverage and is useless. Reported together, they are +// a claim about a detection set; reported alone, coverage is a half-truth. +func runPrecision() { + fs := flag.NewFlagSet("precision", flag.ExitOnError) + baseline := fs.String("baseline", "emulation/benign-baseline.yml", "benign workload definition") + rulesDir := fs.String("rules-dir", "detections/linux", "rules to measure") + asJSON := fs.Bool("json", false, "emit JSON instead of a table") + if err := fs.Parse(os.Args[2:]); err != nil { + os.Exit(2) + } + + b, err := precision.LoadBaseline(*baseline) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + res, err := precision.Measure(evaluator.RuleMatcherEvaluator{RulesDir: *rulesDir}, *rulesDir, b) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + if *asJSON { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(res); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + } else { + fmt.Printf("Precision: %d%% (%d benign commands, %d rules)\n", + res.PrecisionPct, res.CommandsRun, res.RulesChecked) + if len(res.FalsePositives) == 0 { + fmt.Println("No rule fired on benign activity.") + } + for _, fp := range res.FalsePositives { + fmt.Printf(" FALSE POSITIVE %s\n fired on %q (%s)\n", + fp.Rule, fp.Command, fp.Category) + } + } + + // A false positive is a finding, not a note: exit non-zero so this gates CI + // and a release the same way a failing detection would. + if len(res.FalsePositives) > 0 { + os.Exit(1) + } +} diff --git a/cmd/purpleloop/replay.go b/cmd/purpleloop/replay.go new file mode 100644 index 0000000..6cf6ceb --- /dev/null +++ b/cmd/purpleloop/replay.go @@ -0,0 +1,112 @@ +package main + +import ( + "flag" + "fmt" + "os" + "time" + + "github.com/jayelbotvibe-web/purple-loop/internal/capture" + "github.com/jayelbotvibe-web/purple-loop/internal/evaluator" + "github.com/jayelbotvibe-web/purple-loop/internal/mapping" + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +// runReplay re-evaluates a captured run's telemetry with no lab attached. +// +// It runs the SAME evaluator over the SAME events the live run collected, so a +// replay of an unchanged run reproduces its verdicts exactly — that equality is +// the test. Once rules change, the difference is the answer to the question CI +// could not previously ask: would this edit have broken a detection that +// already worked? +func runReplay() { + fs := flag.NewFlagSet("replay", flag.ExitOnError) + output := fs.String("output", "", "output file (.html) or empty for JSON on stdout") + ruleMapPath := fs.String("rule-map", defaultRuleMap, "technique -> expected rules mapping") + rulesDir := fs.String("rules-dir", "detections/linux", "rules root for the evaluator") + if err := fs.Parse(os.Args[2:]); err != nil { + os.Exit(2) + } + if fs.NArg() < 1 { + fmt.Fprintln(os.Stderr, "usage: purpleloop replay [--output report.html]") + os.Exit(2) + } + + result, err := replayDataset(fs.Arg(0), *ruleMapPath, *rulesDir) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + if err := newReporter(*output).Write(result); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +// replayDataset re-evaluates a captured dataset and returns the campaign result. +// Split out from the command so CI can assert that replaying a dataset +// reproduces the verdicts it was captured with. +func replayDataset(path, ruleMapPath, rulesDir string) (model.CampaignResult, error) { + ds, err := capture.Read(path) + if err != nil { + return model.CampaignResult{}, err + } + rules, err := mapping.LoadRuleMap(ruleMapPath) + if err != nil { + return model.CampaignResult{}, err + } + + eval := evaluator.RuleMatcherEvaluator{RulesDir: rulesDir} + order, events, meta := ds.ByTechnique() + + // A replay is a real evaluation of real telemetry, so it is NOT synthetic. + // It is also not a live run: nothing was executed and the canary did not + // fire here, so canary health is inherited from the captured run rather + // than asserted. Coverage from a replay describes the rules, not the lab. + result := model.CampaignResult{ + StartedAt: time.Now().UTC(), + Campaign: "replay:" + path, + CanaryHealthy: true, + } + + for _, tech := range order { + rec := meta[tech] + evs := events[tech] + chain := model.ProofChain{ + TechniqueID: tech, + Atomic: model.AtomicTest{ID: rec.AtomicID, TechniqueID: tech}, + ExecutedAt: rec.ExecutedAt, + Window: rec.Window, + EventsCollected: len(evs), + Attribution: model.WindowAndHostScoped, + } + + expected := rules.ExpectedRules(tech) + chain.RulesExpected = expected + if len(expected) == 0 { + chain.Verdict = model.Errored + chain.Note = "no Sigma rule mapped for " + tech + " in the rule map" + result.Chains = append(result.Chains, chain) + continue + } + + verdict, evidence, matched, err := evaluateExpected(eval, expected, tech, evs) + if err != nil { + chain.Verdict = model.Errored + chain.Note = err.Error() + result.Chains = append(result.Chains, chain) + continue + } + chain.Verdict = verdict + chain.Evidence = evidence + chain.RulesMatched = matched + if len(matched) > 0 { + chain.RuleMatched = matched[0] + chain.DetectLatencyMS = detectLatency(rec.ExecutedAt, evidence) + } + result.Chains = append(result.Chains, chain) + } + + return result, nil +} diff --git a/detections/linux/T1007.yml b/detections/linux/T1007.yml index a7a12ef..2e5c83d 100644 --- a/detections/linux/T1007.yml +++ b/detections/linux/T1007.yml @@ -1,23 +1,32 @@ -title: Discovery - T1007 -id: 792aae53-0000-0000-0000-000000000000 +title: System Service Discovery (Linux) +id: a1c0f2e4-0004-4b10-9c21-5f0a10070010 status: experimental -description: Detects T1007 discovery activity. +description: Detects service enumeration, as emitted by atomic T1007-3 (systemctl --type=service / service -e) and T1007-8 (init script listing). references: - https://attack.mitre.org/techniques/T1007/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + systemctl_list: + Image|endswith: '/systemctl' + CommandLine|contains: + - '--type=service' + - 'list-units' + - 'list-unit-files' + service_enum: Image|endswith: - - '/id' - - '/whoami' - condition: selection + - '/service' + - '/chkconfig' + initscripts: + CommandLine|contains: '/etc/init.d' + condition: systemctl_list or service_enum or initscripts falsepositives: - - Legitimate administrative scripts + - Administrators and provisioning tools inventorying services + - Monitoring agents polling unit state level: low tags: - attack.discovery - - attack.T1007 + - attack.t1007 diff --git a/detections/linux/T1016.yml b/detections/linux/T1016.yml index 95fe6b7..b9ab8ed 100644 --- a/detections/linux/T1016.yml +++ b/detections/linux/T1016.yml @@ -1,23 +1,37 @@ -title: Discovery - T1016 -id: 7d506977-0000-0000-0000-000000000000 +title: System Network Configuration Discovery (Linux) +id: a1c0f2e4-0005-4b10-9c21-5f0a10160010 status: experimental -description: Detects T1016 discovery activity. +description: Detects local network configuration enumeration, as emitted by atomic T1016-3 (arp -a; ifconfig; ip addr; netstat -ant; route). references: - https://attack.mitre.org/techniques/T1016/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + legacy_tools: Image|endswith: - - '/id' - - '/whoami' - condition: selection + - '/ifconfig' + - '/arp' + - '/route' + - '/netstat' + iproute: + Image|endswith: '/ip' + CommandLine|contains: + - ' addr' + - ' route' + - ' link' + - ' neigh' + resolver: + CommandLine|contains: + - '/etc/resolv.conf' + - '/etc/hosts' + condition: legacy_tools or iproute or resolver falsepositives: - - Legitimate administrative scripts + - Network troubleshooting by administrators + - Container entrypoints inspecting their own interfaces level: low tags: - attack.discovery - - attack.T1016 + - attack.t1016 diff --git a/detections/linux/T1033.yml b/detections/linux/T1033.yml index 64cd252..343b837 100644 --- a/detections/linux/T1033.yml +++ b/detections/linux/T1033.yml @@ -1,23 +1,30 @@ -title: Discovery - T1033 -id: 641f50c0-0000-0000-0000-000000000000 +title: System Owner and User Discovery (Linux) +id: a1c0f2e4-0003-4b10-9c21-5f0a10330010 status: experimental -description: Detects T1033 discovery activity. +description: Detects enumeration of the current and logged-on users, as emitted by atomic T1033-2 (users; w; who). references: - https://attack.mitre.org/techniques/T1033/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + session_enum: + Image|endswith: + - '/users' + - '/who' + - '/w' + - '/last' + identity: Image|endswith: - - '/id' - '/whoami' - condition: selection + - '/logname' + condition: session_enum or identity falsepositives: - - Legitimate administrative scripts + - Interactive administrators checking who is logged in + - Monitoring agents sampling session counts level: low tags: - attack.discovery - - attack.T1033 + - attack.t1033 diff --git a/detections/linux/T1049.yml b/detections/linux/T1049.yml index fb4feef..1d0eb80 100644 --- a/detections/linux/T1049.yml +++ b/detections/linux/T1049.yml @@ -1,23 +1,33 @@ -title: Discovery - T1049 -id: e3b2e995-0000-0000-0000-000000000000 +title: System Network Connections Discovery (Linux) +id: a1c0f2e4-0006-4b10-9c21-5f0a10490010 status: experimental -description: Detects T1049 discovery activity. +description: Detects enumeration of active network connections, as emitted by atomic T1049-4 (ss -antp / lsof -i) and T1049-6 (sockstat). references: - https://attack.mitre.org/techniques/T1049/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + socket_stats: Image|endswith: - - '/id' - - '/whoami' - condition: selection + - '/ss' + - '/sockstat' + lsof_network: + Image|endswith: '/lsof' + CommandLine|contains: '-i' + netstat_conns: + Image|endswith: '/netstat' + CommandLine|contains: + - '-a' + - '-t' + - '-n' + condition: socket_stats or lsof_network or netstat_conns falsepositives: - - Legitimate administrative scripts + - Administrators diagnosing connectivity + - Health checks probing listening ports level: low tags: - attack.discovery - - attack.T1049 + - attack.t1049 diff --git a/detections/linux/T1069.001.yml b/detections/linux/T1069.001.yml index 62c3b80..92833f6 100644 --- a/detections/linux/T1069.001.yml +++ b/detections/linux/T1069.001.yml @@ -1,23 +1,31 @@ -title: Discovery - T1069.001 -id: 2d3e4d06-0000-0000-0000-000000000000 +title: Local Permission Group Discovery (Linux) +id: a1c0f2e4-0007-4b10-9c21-5f0a10690010 status: experimental -description: Detects T1069.001 discovery activity. +description: Detects local group enumeration, as emitted by atomic T1069.001-1 (groups; cat /etc/group; getent group). references: - https://attack.mitre.org/techniques/T1069/001/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + group_db: + CommandLine|contains: + - '/etc/group' + - '/etc/sudoers' + group_tools: Image|endswith: - - '/id' - - '/whoami' - condition: selection + - '/groups' + - '/groupmems' + getent_group: + Image|endswith: '/getent' + CommandLine|contains: 'group' + condition: group_db or group_tools or getent_group falsepositives: - - Legitimate administrative scripts + - Provisioning tools reconciling group membership + - Login flows resolving supplementary groups level: low tags: - attack.discovery - - attack.T1069001 + - attack.t1069.001 diff --git a/detections/linux/T1082.yml b/detections/linux/T1082.yml index 59feb80..aa86868 100644 --- a/detections/linux/T1082.yml +++ b/detections/linux/T1082.yml @@ -1,23 +1,33 @@ -title: Discovery - T1082 -id: 422ee334-0000-0000-0000-000000000000 +title: System Information Discovery (Linux) +id: a1c0f2e4-0002-4b10-9c21-5f0a10820010 status: experimental -description: Detects T1082 discovery activity. +description: Detects host and OS fingerprinting, as emitted by atomic T1082-3 (uname -a plus reads of the release files) and by kernel-module listing. references: - https://attack.mitre.org/techniques/T1082/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + uname: + Image|endswith: '/uname' + release_files: + CommandLine|contains: + - '/etc/os-release' + - '/etc/lsb-release' + - '/etc/redhat-release' + - '/etc/issue' + host_fingerprint: Image|endswith: - - '/id' - - '/whoami' - condition: selection + - '/hostnamectl' + - '/lsmod' + - '/dmidecode' + condition: uname or release_files or host_fingerprint falsepositives: - - Legitimate administrative scripts + - Package managers and installers fingerprinting the distribution + - Configuration-management agents collecting host facts level: low tags: - attack.discovery - - attack.T1082 + - attack.t1082 diff --git a/detections/linux/T1087.001.yml b/detections/linux/T1087.001.yml index 1bb51c1..9889793 100644 --- a/detections/linux/T1087.001.yml +++ b/detections/linux/T1087.001.yml @@ -1,23 +1,33 @@ -title: Discovery - T1087.001 -id: 06a5d919-0000-0000-0000-000000000000 +title: Local Account Enumeration (Linux) +id: a1c0f2e4-0001-4b10-9c21-5f0a10870010 status: experimental -description: Detects T1087.001 discovery activity. +description: Detects local account enumeration by reading the account databases, as emitted by atomic T1087.001-1 (cat /etc/passwd) and by getent/lastlog equivalents. references: - https://attack.mitre.org/techniques/T1087/001/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + read_account_db: + CommandLine|contains: + - '/etc/passwd' + - '/etc/shadow' + enum_tools: Image|endswith: - - '/id' - - '/whoami' - condition: selection + - '/lastlog' + - '/lslogins' + getent_account_db: + Image|endswith: '/getent' + CommandLine|contains: + - 'passwd' + - 'shadow' + condition: read_account_db or enum_tools or getent_account_db falsepositives: - - Legitimate administrative scripts + - Backup and configuration-management tooling reading /etc/passwd + - Identity agents resolving accounts through getent passwd level: low tags: - attack.discovery - - attack.T1087001 + - attack.t1087.001 diff --git a/detections/linux/T1135.yml b/detections/linux/T1135.yml index aa1b561..1b7b6f5 100644 --- a/detections/linux/T1135.yml +++ b/detections/linux/T1135.yml @@ -1,23 +1,36 @@ -title: Discovery - T1135 -id: 59fe172d-0000-0000-0000-000000000000 +title: Network Share Discovery (Linux) +id: a1c0f2e4-0008-4b10-9c21-5f0a11350010 status: experimental -description: Detects T1135 discovery activity. +description: Detects network share enumeration, as emitted by the lab's T1135-1 override (reads of /proc/mounts filtered for network filesystems, showmount, smbclient) and by upstream smbstatus. references: - https://attack.mitre.org/techniques/T1135/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + smb_tools: Image|endswith: - - '/id' - - '/whoami' - condition: selection + - '/smbclient' + - '/smbstatus' + - '/showmount' + - '/nmblookup' + mount_table: + CommandLine|contains: + - '/proc/mounts' + - '/etc/fstab' + network_mount: + Image|endswith: '/mount' + CommandLine|contains: + - 'cifs' + - 'nfs' + - 'smb' + condition: smb_tools or mount_table or network_mount falsepositives: - - Legitimate administrative scripts -level: low + - Storage agents reading /proc/mounts or /etc/fstab + - Backup tooling enumerating network filesystems +level: medium tags: - attack.discovery - - attack.T1135 + - attack.t1135 diff --git a/detections/linux/T1518.yml b/detections/linux/T1518.yml index 78df919..731663f 100644 --- a/detections/linux/T1518.yml +++ b/detections/linux/T1518.yml @@ -1,23 +1,38 @@ -title: Discovery - T1518 -id: 4c1736b3-0000-0000-0000-000000000000 +title: Software Discovery (Linux) +id: a1c0f2e4-0009-4b10-9c21-5f0a15180010 status: experimental -description: Detects T1518 discovery activity. +description: Detects installed-software enumeration, as emitted by the lab's T1518-1 override (dpkg -l / rpm -qa / apk info). Upstream Atomic Red Team ships no Linux test for this technique. references: - https://attack.mitre.org/techniques/T1518/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + dpkg_list: + Image|endswith: '/dpkg' + CommandLine|contains: + - '-l' + - '--list' + - '--get-selections' + rpm_query: + Image|endswith: '/rpm' + CommandLine|contains: + - '-qa' + - '--query' + other_pkg: Image|endswith: - - '/id' - - '/whoami' - condition: selection + - '/apk' + - '/dpkg-query' + CommandLine|contains: + - 'info' + - '-l' + condition: dpkg_list or rpm_query or other_pkg falsepositives: - - Legitimate administrative scripts + - Vulnerability scanners and inventory agents + - Package managers during a normal install or upgrade level: low tags: - attack.discovery - - attack.T1518 + - attack.t1518 diff --git a/detections/tests/T1007/negative_events.jsonl b/detections/tests/T1007/negative_events.jsonl index 9c96041..68fd2bd 100644 --- a/detections/tests/T1007/negative_events.jsonl +++ b/detections/tests/T1007/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/bin/systemctl", "ParentImage": "/bin/bash", "CommandLine": "systemctl restart nginx", "User": "root"} +{"Image": "/usr/bin/who", "ParentImage": "/bin/sh", "CommandLine": "who", "User": "root"} diff --git a/detections/tests/T1007/positive_events.jsonl b/detections/tests/T1007/positive_events.jsonl index b3e8f64..d745217 100644 --- a/detections/tests/T1007/positive_events.jsonl +++ b/detections/tests/T1007/positive_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/bin/systemctl", "ParentImage": "/bin/bash", "CommandLine": "systemctl --type=service", "User": "root"} +{"Image": "/usr/sbin/service", "ParentImage": "/bin/bash", "CommandLine": "service --status-all", "User": "root"} diff --git a/detections/tests/T1016/negative_events.jsonl b/detections/tests/T1016/negative_events.jsonl index 9c96041..85770ef 100644 --- a/detections/tests/T1016/negative_events.jsonl +++ b/detections/tests/T1016/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/sbin/ip", "ParentImage": "/bin/bash", "CommandLine": "ip --version", "User": "root"} +{"Image": "/usr/bin/ss", "ParentImage": "/bin/bash", "CommandLine": "ss -antp", "User": "root"} diff --git a/detections/tests/T1016/positive_events.jsonl b/detections/tests/T1016/positive_events.jsonl index b3e8f64..8b66ca6 100644 --- a/detections/tests/T1016/positive_events.jsonl +++ b/detections/tests/T1016/positive_events.jsonl @@ -1 +1,3 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/sbin/arp", "ParentImage": "/bin/sh", "CommandLine": "arp -a", "User": "root"} +{"Image": "/usr/sbin/ifconfig", "ParentImage": "/bin/sh", "CommandLine": "ifconfig", "User": "root"} +{"Image": "/usr/sbin/ip", "ParentImage": "/bin/sh", "CommandLine": "ip addr show", "User": "root"} diff --git a/detections/tests/T1033/negative_events.jsonl b/detections/tests/T1033/negative_events.jsonl index 9c96041..8382d93 100644 --- a/detections/tests/T1033/negative_events.jsonl +++ b/detections/tests/T1033/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/bin/uname", "ParentImage": "/bin/sh", "CommandLine": "uname -a", "User": "root"} +{"Image": "/usr/bin/wget", "ParentImage": "/bin/bash", "CommandLine": "wget http://mirror/pkg.deb", "User": "root"} diff --git a/detections/tests/T1033/positive_events.jsonl b/detections/tests/T1033/positive_events.jsonl index b3e8f64..931eb04 100644 --- a/detections/tests/T1033/positive_events.jsonl +++ b/detections/tests/T1033/positive_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/bin/who", "ParentImage": "/bin/sh", "CommandLine": "who", "User": "root"} +{"Image": "/usr/bin/w", "ParentImage": "/bin/sh", "CommandLine": "w", "User": "root"} diff --git a/detections/tests/T1049/negative_events.jsonl b/detections/tests/T1049/negative_events.jsonl index 9c96041..2124241 100644 --- a/detections/tests/T1049/negative_events.jsonl +++ b/detections/tests/T1049/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/bin/lsof", "ParentImage": "/bin/bash", "CommandLine": "lsof /var/log/syslog", "User": "root"} +{"Image": "/usr/sbin/ifconfig", "ParentImage": "/bin/sh", "CommandLine": "ifconfig", "User": "root"} diff --git a/detections/tests/T1049/positive_events.jsonl b/detections/tests/T1049/positive_events.jsonl index b3e8f64..36d3c0f 100644 --- a/detections/tests/T1049/positive_events.jsonl +++ b/detections/tests/T1049/positive_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/bin/ss", "ParentImage": "/bin/bash", "CommandLine": "ss -antp", "User": "root"} +{"Image": "/usr/bin/lsof", "ParentImage": "/bin/bash", "CommandLine": "lsof -i -nP", "User": "root"} diff --git a/detections/tests/T1069.001/negative_events.jsonl b/detections/tests/T1069.001/negative_events.jsonl index 9c96041..3c933c9 100644 --- a/detections/tests/T1069.001/negative_events.jsonl +++ b/detections/tests/T1069.001/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/bin/cat", "ParentImage": "/bin/sh", "CommandLine": "cat /etc/passwd", "User": "root"} +{"Image": "/usr/bin/id", "ParentImage": "/bin/sh", "CommandLine": "id", "User": "root"} diff --git a/detections/tests/T1069.001/positive_events.jsonl b/detections/tests/T1069.001/positive_events.jsonl index b3e8f64..6dca966 100644 --- a/detections/tests/T1069.001/positive_events.jsonl +++ b/detections/tests/T1069.001/positive_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/bin/groups", "ParentImage": "/bin/sh", "CommandLine": "groups", "User": "root"} +{"Image": "/usr/bin/cat", "ParentImage": "/bin/sh", "CommandLine": "cat /etc/group", "User": "root"} diff --git a/detections/tests/T1082/negative_events.jsonl b/detections/tests/T1082/negative_events.jsonl index 9c96041..7369f26 100644 --- a/detections/tests/T1082/negative_events.jsonl +++ b/detections/tests/T1082/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/bin/cat", "ParentImage": "/bin/sh", "CommandLine": "cat /etc/passwd", "User": "root"} +{"Image": "/usr/bin/apt", "ParentImage": "/bin/bash", "CommandLine": "apt update", "User": "root"} diff --git a/detections/tests/T1082/positive_events.jsonl b/detections/tests/T1082/positive_events.jsonl index b3e8f64..d28916c 100644 --- a/detections/tests/T1082/positive_events.jsonl +++ b/detections/tests/T1082/positive_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/bin/uname", "ParentImage": "/bin/sh", "CommandLine": "uname -a", "User": "root"} +{"Image": "/usr/bin/cat", "ParentImage": "/bin/sh", "CommandLine": "cat /etc/os-release", "User": "root"} diff --git a/detections/tests/T1087.001/negative_events.jsonl b/detections/tests/T1087.001/negative_events.jsonl index 9c96041..bc423eb 100644 --- a/detections/tests/T1087.001/negative_events.jsonl +++ b/detections/tests/T1087.001/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/bin/cat", "ParentImage": "/bin/bash", "CommandLine": "cat /etc/hostname", "User": "root"} +{"Image": "/usr/bin/uname", "ParentImage": "/bin/sh", "CommandLine": "uname -a", "User": "root"} diff --git a/detections/tests/T1087.001/positive_events.jsonl b/detections/tests/T1087.001/positive_events.jsonl index b3e8f64..5166a64 100644 --- a/detections/tests/T1087.001/positive_events.jsonl +++ b/detections/tests/T1087.001/positive_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/bin/cat", "ParentImage": "/bin/sh", "CommandLine": "cat /etc/passwd", "User": "root"} +{"Image": "/usr/bin/getent", "ParentImage": "/bin/bash", "CommandLine": "getent passwd", "User": "root"} diff --git a/detections/tests/T1135/negative_events.jsonl b/detections/tests/T1135/negative_events.jsonl index 9c96041..5ab4fb0 100644 --- a/detections/tests/T1135/negative_events.jsonl +++ b/detections/tests/T1135/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/bin/cat", "ParentImage": "/bin/sh", "CommandLine": "cat /proc/cpuinfo", "User": "root"} +{"Image": "/usr/bin/dpkg", "ParentImage": "/bin/sh", "CommandLine": "dpkg -l", "User": "root"} diff --git a/detections/tests/T1135/positive_events.jsonl b/detections/tests/T1135/positive_events.jsonl index b3e8f64..38682c2 100644 --- a/detections/tests/T1135/positive_events.jsonl +++ b/detections/tests/T1135/positive_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/bin/cat", "ParentImage": "/bin/sh", "CommandLine": "cat /proc/mounts", "User": "root"} +{"Image": "/usr/bin/smbclient", "ParentImage": "/bin/sh", "CommandLine": "smbclient -L localhost -N", "User": "root"} diff --git a/detections/tests/T1518/negative_events.jsonl b/detections/tests/T1518/negative_events.jsonl index 9c96041..3895ffa 100644 --- a/detections/tests/T1518/negative_events.jsonl +++ b/detections/tests/T1518/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/bin/dpkg", "ParentImage": "/bin/bash", "CommandLine": "dpkg --configure -a", "User": "root"} +{"Image": "/usr/bin/uname", "ParentImage": "/bin/sh", "CommandLine": "uname -a", "User": "root"} diff --git a/detections/tests/T1518/positive_events.jsonl b/detections/tests/T1518/positive_events.jsonl index b3e8f64..94a3aef 100644 --- a/detections/tests/T1518/positive_events.jsonl +++ b/detections/tests/T1518/positive_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/bin/dpkg", "ParentImage": "/bin/sh", "CommandLine": "dpkg -l", "User": "root"} +{"Image": "/usr/bin/rpm", "ParentImage": "/bin/sh", "CommandLine": "rpm -qa", "User": "root"} diff --git a/detections/windows/win_proc_create.yml b/detections/windows/win_proc_create.yml index fe9d9b1..d110e76 100644 --- a/detections/windows/win_proc_create.yml +++ b/detections/windows/win_proc_create.yml @@ -1,5 +1,5 @@ title: Windows Process Creation -id: win-proc-create-001 +id: a1c0f2e4-0010-4b10-9c21-5f0a10590030 status: experimental description: Detects Windows process creation via Sysmon Event ID 1. references: diff --git a/docs/PITFALLS.md b/docs/PITFALLS.md index 99d6ec3..8a3e049 100644 --- a/docs/PITFALLS.md +++ b/docs/PITFALLS.md @@ -35,3 +35,113 @@ Check here before debugging. ## Laptop slows mid-campaign - Thermal throttling on the mobile CPU, not a resource fault. Plug in, ensure airflow, continue. Do NOT change host config to "fix" it. + +--- + +## Linux process-creation telemetry: what actually blocks it + +The README's "Linux Sysmon gap" is often read as *not configured yet*. It is more specific than +that, and both halves were verified on the 2026-08-29 lab host rather than assumed. + +### auditd cannot run on a host whose kernel does not expose the audit subsystem + +The victim image installs `auditd` and the compose override grants `AUDIT_CONTROL` / +`AUDIT_READ`. Those capabilities *are* effective inside the container — and it still fails: + +``` +$ docker exec purpleloop-victim auditctl -l +Error sending rule list data request (Operation not permitted) +``` + +This is not a container permissions problem. It fails identically with `--privileged` **and** +`--network host`, so it is neither a capability nor a network-namespace issue: + +``` +$ docker run --rm --privileged --network host --entrypoint sh purpleloop/victim:latest \ + -c "auditctl -s" +Error sending status request (Operation not permitted) +``` + +The host itself has no audit sysctls at all: + +``` +$ cat /proc/sys/kernel/audit_backlog_limit +cat: /proc/sys/kernel/audit_backlog_limit: No such file or directory +$ ls /proc/sys/kernel/ | grep -i audit # no results +``` + +`kauditd` exists as a kernel thread, so `CONFIG_AUDIT` is compiled in, but the control +interface is unavailable. **On such a host, no amount of container configuration produces +auditd telemetry.** + +### Sysmon for Linux requires systemd + +Sysmon for Linux 1.5.2 installs and its eBPF objects are present (the host has +`/sys/kernel/btf/vmlinux`, so eBPF itself is fine). But it only runs as a systemd service: + +``` +$ /opt/sysmon/sysmon -i /opt/sysmon/config.xml -service +sh: 1: systemctl: not found +# process exits +``` + +The `-service` flag in the shipped unit's `ExecStart` still shells out to `systemctl`. The +victim container runs `tail -F` as PID 1, so there is no systemd for it to talk to. + +### What this means for a run + +The pipeline canary requires `category: process_creation`. With no process-creation source the +canary cannot fire, so every Linux run is correctly `INCONCLUSIVE` and coverage is suppressed. +**That is the canary working, not the canary failing.** Do not "fix" it by relaxing the canary +rule or by letting command-output telemetry satisfy a process-creation rule — the +evidence-fidelity gate exists precisely to refuse that, and removing it is how v1.0 produced a +false 100%. + +### Closing it + +Closing the gap needs a victim that can run systemd — a systemd-based image running privileged +with PID 1 as systemd, so Sysmon for Linux can install and start. That is a deliberate change +to the lab's security posture (a privileged container in a lab that otherwise runs +unprivileged), and it is a decision to take explicitly rather than drift into. + +Until then, Windows Sysmon Event ID 1 is the platform where detection is demonstrated. The +`--settle-timeout` default of 60s is also below the victim's 360s `` collection +interval; raise it if you want even low-fidelity command output to land inside a technique's +window. + +## Why the Linux victim is not a container with Sysmon + +Sysmon for Linux was built, run and measured on this lab before being rejected. Recording the +result so it is not re-attempted blind. + +**It works.** With systemd as PID 1 (`privileged: true`, `cgroup: host`, `/sys/kernel/btf` +mounted) the service starts and emits genuine `EventID 1` process-creation events carrying +`Image`, `CommandLine`, `ParentImage` and `User` — exactly the fields the evaluator needs. The +undocumented `-service` flag in its own systemd unit is what runs it; it has no foreground mode. + +**And it cannot be scoped to the victim.** Sysmon's eBPF is kernel-global. Inside a container it +observes every process on the host, and the event carries no container, cgroup or namespace +field — `RuleName`, `LogonId` and `TerminalSessionId` do not distinguish one. Measured results +from attempting to scope it after the fact by resolving the event's host PID through a +read-only `/proc` mount: + +| Approach | Result | +|---|---| +| cgroup lookup on the event's PID | **Host binaries leaked.** `/usr/lib/cargo/bin/coreutils/tr`, `nvidia-smi`, `vmrun` — none of which exist in the Ubuntu image — were accepted. Short-lived host processes exit, the kernel recycles the PID, a container process takes it, and the lookup answers about the wrong process. | +| the same, plus verifying `/proc//comm` matches the event's image | **No leakage, but most events lost.** Only processes living longer than the syslog→converter latency survive; `uname`, `id` and `who` — the atomics themselves — were dropped. | + +Both directions are wrong in ways this project exists to prevent. The first manufactures false +`DETECTED` verdicts from activity typed on the host; the second manufactures false `MISSED` and +`NO_TELEMETRY` from a collector that simply could not keep up. There is no third signal in the +event to break the tie. + +**The fix is a VM, not a container.** The Windows victim already is one, which is precisely why +Windows telemetry is unambiguous: everything the sensor sees belongs to the victim, because the +sensor and the victim share a kernel that nothing else uses. A Linux VM running Sysmon for Linux +would give the Linux side the same property. + +The engine is already ready for it: the normalizer has a `data.sysmon` path with EventID-1 +fidelity gating and tests (`internal/evaluator/sysmon_linux_test.go`), so a Linux-Sysmon source +needs no Go changes — only a Wazuh agent shipping the events with those keys. Note the events +are deliberately *not* mapped through `data.win.*`; that path is Wazuh's Windows eventchannel +convention and using it for Linux would make the telemetry misreport its own origin. diff --git a/emulation/apt29-subset.yml b/emulation/apt29-subset.yml index 26fe721..d4f1726 100644 --- a/emulation/apt29-subset.yml +++ b/emulation/apt29-subset.yml @@ -14,15 +14,15 @@ stages: description: Enumerate users, systems, and network techniques: - technique_id: T1033 - atomic_ids: [T1033-1] + atomic_ids: [T1033-2] - technique_id: T1087.001 atomic_ids: [T1087.001-1] - technique_id: T1082 - atomic_ids: [T1082-1] + atomic_ids: [T1082-3] - technique_id: T1016 - atomic_ids: [T1016-1] + atomic_ids: [T1016-3] - technique_id: T1049 - atomic_ids: [T1049-1] + atomic_ids: [T1049-4] - name: persistence description: Establish persistence via boot/logon autostart and account manipulation techniques: @@ -34,7 +34,7 @@ stages: description: Collect data from local system and network shares techniques: - technique_id: T1007 - atomic_ids: [T1007-1] + atomic_ids: [T1007-3] - technique_id: T1518 atomic_ids: [T1518-1] - technique_id: T1135 diff --git a/emulation/benign-baseline.yml b/emulation/benign-baseline.yml new file mode 100644 index 0000000..c6c88c9 --- /dev/null +++ b/emulation/benign-baseline.yml @@ -0,0 +1,134 @@ +# Benign administrator workload — the precision baseline. +# +# CI's negative fixtures prove a rule rejects an event crafted to be rejected. +# That is a weak claim: the fixture was written by the same person as the rule. +# This is the stronger one — ordinary administration that a detection must stay +# silent about. Any rule that fires here is a measured false positive, reported +# with the command that triggered it. +# +# `image` and `command_line` are the process-creation event the command +# produces, so precision can be measured offline in CI as well as live in the +# lab. Commands are read-only and safe to run on the victim container. +name: benign-admin-baseline +description: Read-only administrator activity that no detection should fire on + +commands: + # --- package management --- + - category: package-management + command: apt-get --version + image: /usr/bin/apt-get + command_line: apt-get --version + - category: package-management + command: dpkg --configure -a --dry-run + image: /usr/bin/dpkg + command_line: dpkg --configure -a --dry-run + - category: package-management + command: apt-cache policy bash + image: /usr/bin/apt-cache + command_line: apt-cache policy bash + + # --- routine file operations --- + - category: file-ops + command: ls -la /var/log + image: /usr/bin/ls + command_line: ls -la /var/log + - category: file-ops + command: du -sh /var/log + image: /usr/bin/du + command_line: du -sh /var/log + - category: file-ops + command: df -h + image: /usr/bin/df + command_line: df -h + - category: file-ops + command: cat /etc/hostname + image: /usr/bin/cat + command_line: cat /etc/hostname + - category: file-ops + command: tail -n 20 /var/log/dpkg.log + image: /usr/bin/tail + command_line: tail -n 20 /var/log/dpkg.log + + # --- process and resource inspection --- + - category: process-inspection + command: ps aux + image: /usr/bin/ps + command_line: ps aux + - category: process-inspection + command: free -m + image: /usr/bin/free + command_line: free -m + - category: process-inspection + command: uptime + image: /usr/bin/uptime + command_line: uptime + + # --- service operation (NOT enumeration) --- + - category: service-ops + command: systemctl is-active cron + image: /usr/bin/systemctl + command_line: systemctl is-active cron + - category: service-ops + command: systemctl status cron + image: /usr/bin/systemctl + command_line: systemctl status cron + + # --- network operation (NOT reconnaissance) --- + - category: network-ops + command: ping -c 1 127.0.0.1 + image: /usr/bin/ping + command_line: ping -c 1 127.0.0.1 + - category: network-ops + command: ip --version + image: /usr/sbin/ip + command_line: ip --version + - category: network-ops + command: curl --version + image: /usr/bin/curl + command_line: curl --version + + # --- ADJACENT cases: benign uses of the very binaries the detections watch. + # These are what make this test worth running. A rule keyed on a tool name + # rather than on what the tool is being asked to do will fire here. + - category: adjacent-benign + command: mount + image: /usr/bin/mount + command_line: mount + - category: adjacent-benign + command: getent hosts localhost + image: /usr/bin/getent + command_line: getent hosts localhost + - category: adjacent-benign + command: lsof /var/log/syslog + image: /usr/bin/lsof + command_line: lsof /var/log/syslog + - category: adjacent-benign + command: systemctl restart cron + image: /usr/bin/systemctl + command_line: systemctl restart cron + - category: adjacent-benign + command: cat /proc/cpuinfo + image: /usr/bin/cat + command_line: cat /proc/cpuinfo + - category: adjacent-benign + command: dpkg --configure -a + image: /usr/bin/dpkg + command_line: dpkg --configure -a + + # --- shell and text utilities --- + - category: shell-utils + command: echo hello + image: /usr/bin/echo + command_line: echo hello + - category: shell-utils + command: date + image: /usr/bin/date + command_line: date + - category: shell-utils + command: env + image: /usr/bin/env + command_line: env + - category: shell-utils + command: grep --version + image: /usr/bin/grep + command_line: grep --version diff --git a/emulation/discovery-chain.yml b/emulation/discovery-chain.yml index 3aa614d..9d9443c 100644 --- a/emulation/discovery-chain.yml +++ b/emulation/discovery-chain.yml @@ -7,24 +7,24 @@ stages: description: Initial reconnaissance — who is on the system techniques: - technique_id: T1033 - atomic_ids: [T1033-1] + atomic_ids: [T1033-2] - technique_id: T1087.001 atomic_ids: [T1087.001-1] - name: system-info description: Gather system information techniques: - technique_id: T1082 - atomic_ids: [T1082-1] + atomic_ids: [T1082-3] - technique_id: T1518 atomic_ids: [T1518-1] - technique_id: T1007 - atomic_ids: [T1007-1] + atomic_ids: [T1007-3] - name: network description: Discover network configuration and connections techniques: - technique_id: T1016 - atomic_ids: [T1016-1] + atomic_ids: [T1016-3] - technique_id: T1049 - atomic_ids: [T1049-1] + atomic_ids: [T1049-4] - technique_id: T1135 atomic_ids: [T1135-1] diff --git a/internal/atomic/atomic.go b/internal/atomic/atomic.go new file mode 100644 index 0000000..b40cf32 --- /dev/null +++ b/internal/atomic/atomic.go @@ -0,0 +1,342 @@ +// Package atomic resolves Atomic Red Team test IDs (e.g. "T1082-3") into the +// concrete command that will run on a victim. +// +// This package exists because the engine previously fabricated its input: every +// technique executed the same hardcoded "id; whoami" regardless of which atomic +// the plan named, so every verdict but one was about a command nobody asked for. +// Resolution here is strict by design — an ID that cannot be resolved is an +// error the caller must surface, never a silent fallback to some default +// command. A fabricated input produces a fabricated verdict. +package atomic + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +// Source records where a resolved test's command came from, so a report can +// show that a lab-specific substitution was made rather than hiding it. +const ( + SourceART = "atomic-red-team" + SourceOverride = "override" +) + +// Dependency is a prerequisite an atomic needs before it can run. +type Dependency struct { + Description string `yaml:"description"` + PrereqCommand string `yaml:"prereq_command"` + GetPrereqCommand string `yaml:"get_prereq_command"` +} + +// InputArg is a declared input with its default value. +type InputArg struct { + Description string `yaml:"description"` + Type string `yaml:"type"` + Default any `yaml:"default"` +} + +// Test is one resolved atomic: the real command, executor and cleanup. +type Test struct { + ID string + Index int + GUID string + Name string + TechniqueID string + Description string + SupportedPlatforms []string + Executor string + Command string + CleanupCommand string + ElevationRequired bool + Dependencies []Dependency + DependencyExecutor string + Source string + OverrideReason string +} + +// SupportsPlatform reports whether the test declares support for a target kind +// ("linux", "windows", "macos"). +func (t Test) SupportsPlatform(kind string) bool { + for _, p := range t.SupportedPlatforms { + if strings.EqualFold(p, kind) { + return true + } + } + return false +} + +// ---- on-disk ART schema ---- + +type artFile struct { + AttackTechnique string `yaml:"attack_technique"` + DisplayName string `yaml:"display_name"` + AtomicTests []artTest `yaml:"atomic_tests"` +} + +type artTest struct { + Name string `yaml:"name"` + GUID string `yaml:"auto_generated_guid"` + Description string `yaml:"description"` + SupportedPlatforms []string `yaml:"supported_platforms"` + InputArguments map[string]InputArg `yaml:"input_arguments"` + DependencyExecutor string `yaml:"dependency_executor_name"` + Dependencies []Dependency `yaml:"dependencies"` + Executor artExecutor `yaml:"executor"` +} + +type artExecutor struct { + Name string `yaml:"name"` + Command string `yaml:"command"` + CleanupCommand string `yaml:"cleanup_command"` + ElevationRequired bool `yaml:"elevation_required"` +} + +// ---- overrides ---- + +// Override adjusts an ART atomic that cannot run truthfully in this lab. Every +// override carries a reason, which the report shows, so the change stays +// visible. +// +// There are two kinds, and the narrower one is preferred: +// +// - InputArguments alone pins an upstream atomic's declared inputs (the +// mechanism ART itself provides) while keeping its command. Use this to +// keep a test inside the lab network. +// - Command replaces the atomic wholesale. Only for a technique with no +// usable upstream test on this platform. +type Override struct { + Reason string `yaml:"reason"` + Command string `yaml:"command"` + Executor string `yaml:"executor"` + CleanupCommand string `yaml:"cleanup_command"` + SupportedPlatforms []string `yaml:"supported_platforms"` + Name string `yaml:"name"` + InputArguments map[string]string `yaml:"input_arguments"` +} + +type overrideFile struct { + Overrides map[string]Override `yaml:"overrides"` +} + +// Registry resolves atomic IDs against a vendored ART tree plus lab overrides. +type Registry struct { + root string + commit string + overrides map[string]Override +} + +// Load opens a vendored Atomic Red Team checkout. root is the repository root +// (the directory containing "atomics"). overridePath may be empty. +func Load(root, overridePath string) (*Registry, error) { + atomicsDir := filepath.Join(root, "atomics") + if fi, err := os.Stat(atomicsDir); err != nil || !fi.IsDir() { + return nil, fmt.Errorf("atomic red team not vendored at %s — run 'make atomics'", root) + } + r := &Registry{root: root, overrides: map[string]Override{}} + + // Best effort: the recorded commit makes a report reproducible. + if b, err := os.ReadFile(filepath.Join(root, "COMMIT")); err == nil { + r.commit = strings.TrimSpace(string(b)) + } + + if overridePath != "" { + if err := r.loadOverrides(overridePath); err != nil { + return nil, err + } + } + return r, nil +} + +func (r *Registry) loadOverrides(path string) error { + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return nil // overrides are optional + } + if err != nil { + return fmt.Errorf("read overrides: %w", err) + } + var f overrideFile + if err := yaml.Unmarshal(data, &f); err != nil { + return fmt.Errorf("parse overrides %s: %w", path, err) + } + for id, o := range f.Overrides { + if strings.TrimSpace(o.Reason) == "" { + return fmt.Errorf("override %s has no reason — every substitution must be justified", id) + } + if strings.TrimSpace(o.Command) == "" && len(o.InputArguments) == 0 { + return fmt.Errorf("override %s must set either command or input_arguments", id) + } + r.overrides[id] = o + } + return nil +} + +// SetCommit records the vendored ART revision for reporting. +func (r *Registry) SetCommit(c string) { r.commit = strings.TrimSpace(c) } + +// Commit returns the vendored ART revision, or "" if unknown. +func (r *Registry) Commit() string { return r.commit } + +var idRe = regexp.MustCompile(`^(T\d{4}(?:\.\d{3})?)-(\d+)$`) + +// ParseID splits an atomic ID such as "T1069.001-1" into its technique and +// 1-based test index. +func ParseID(atomicID string) (technique string, index int, err error) { + m := idRe.FindStringSubmatch(strings.TrimSpace(atomicID)) + if m == nil { + return "", 0, fmt.Errorf("malformed atomic id %q (want e.g. T1082-3)", atomicID) + } + i, _ := strconv.Atoi(m[2]) + if i < 1 { + return "", 0, fmt.Errorf("atomic index in %q must be 1-based", atomicID) + } + return m[1], i, nil +} + +// Resolve returns the concrete test for an atomic ID. Overrides win over the +// vendored tree; an unresolvable ID is an error, never a default command. +func (r *Registry) Resolve(atomicID string) (Test, error) { + technique, index, err := ParseID(atomicID) + if err != nil { + return Test{}, err + } + + o, overridden := r.overrides[atomicID] + if overridden && strings.TrimSpace(o.Command) != "" { + platforms := o.SupportedPlatforms + if len(platforms) == 0 { + platforms = []string{"linux"} + } + name := o.Name + if name == "" { + name = "lab override for " + atomicID + } + return Test{ + ID: atomicID, + Index: index, + Name: name, + TechniqueID: technique, + SupportedPlatforms: platforms, + Executor: orDefault(o.Executor, "sh"), + Command: strings.TrimSpace(o.Command), + CleanupCommand: strings.TrimSpace(o.CleanupCommand), + Source: SourceOverride, + OverrideReason: o.Reason, + }, nil + } + + path := filepath.Join(r.root, "atomics", technique, technique+".yaml") + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return Test{}, fmt.Errorf("technique %s has no atomics in the vendored tree (%s)", technique, path) + } + if err != nil { + return Test{}, fmt.Errorf("read %s: %w", path, err) + } + + var f artFile + if err := yaml.Unmarshal(data, &f); err != nil { + return Test{}, fmt.Errorf("parse %s: %w", path, err) + } + if index > len(f.AtomicTests) { + return Test{}, fmt.Errorf("%s: technique %s has %d atomic tests, index %d requested", + atomicID, technique, len(f.AtomicTests), index) + } + + at := f.AtomicTests[index-1] + subst := defaults(at.InputArguments) + + // An input-only override pins declared inputs (e.g. a ping target) without + // touching the upstream command, so the test stays the upstream test. + pinned := "" + if overridden { + keys := make([]string, 0, len(o.InputArguments)) + for k, v := range o.InputArguments { + subst[k] = v + keys = append(keys, k) + } + sort.Strings(keys) + if len(keys) > 0 { + pinned = fmt.Sprintf("inputs pinned (%s): %s", strings.Join(keys, ", "), o.Reason) + } + } + + deps := make([]Dependency, 0, len(at.Dependencies)) + for _, d := range at.Dependencies { + deps = append(deps, Dependency{ + Description: strings.TrimSpace(interpolate(d.Description, subst)), + PrereqCommand: strings.TrimSpace(interpolate(d.PrereqCommand, subst)), + GetPrereqCommand: strings.TrimSpace(interpolate(d.GetPrereqCommand, subst)), + }) + } + + return Test{ + ID: atomicID, + Index: index, + GUID: at.GUID, + Name: at.Name, + TechniqueID: technique, + Description: strings.TrimSpace(at.Description), + SupportedPlatforms: at.SupportedPlatforms, + Executor: orDefault(at.Executor.Name, "sh"), + Command: strings.TrimSpace(interpolate(at.Executor.Command, subst)), + CleanupCommand: strings.TrimSpace(interpolate(at.Executor.CleanupCommand, subst)), + ElevationRequired: at.Executor.ElevationRequired, + Dependencies: deps, + DependencyExecutor: orDefault(at.DependencyExecutor, orDefault(at.Executor.Name, "sh")), + Source: SourceART, + OverrideReason: pinned, + }, nil +} + +// defaults flattens declared input arguments to their default values. +func defaults(args map[string]InputArg) map[string]string { + out := make(map[string]string, len(args)) + for k, v := range args { + if v.Default == nil { + out[k] = "" + continue + } + out[k] = strings.TrimSpace(fmt.Sprintf("%v", v.Default)) + } + return out +} + +var placeholderRe = regexp.MustCompile(`#\{([a-zA-Z0-9_]+)\}`) + +// interpolate substitutes #{name} placeholders with their defaults. ART allows +// a default to itself contain a placeholder, so it resolves iteratively; the +// bound stops a self-referential default from looping forever. +func interpolate(s string, subst map[string]string) string { + if s == "" || len(subst) == 0 { + return s + } + for i := 0; i < 5; i++ { + replaced := placeholderRe.ReplaceAllStringFunc(s, func(m string) string { + key := placeholderRe.FindStringSubmatch(m)[1] + if v, ok := subst[key]; ok { + return v + } + return m // unknown placeholder stays visible rather than silently emptied + }) + if replaced == s { + break + } + s = replaced + } + return s +} + +func orDefault(v, def string) string { + if strings.TrimSpace(v) == "" { + return def + } + return v +} diff --git a/internal/atomic/atomic_test.go b/internal/atomic/atomic_test.go new file mode 100644 index 0000000..9802c80 --- /dev/null +++ b/internal/atomic/atomic_test.go @@ -0,0 +1,183 @@ +package atomic + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// artRoot is the vendored tree. Tests that need it skip when it is absent so a +// clean checkout without `make atomics` still runs the rest of the suite. +const artRoot = "../../lab/atomic-red-team" + +func requireART(t *testing.T) *Registry { + t.Helper() + if _, err := os.Stat(filepath.Join(artRoot, "atomics")); err != nil { + t.Skip("atomic-red-team not vendored; run 'make atomics'") + } + reg, err := Load(artRoot, "../../mappings/atomic-overrides.yml") + if err != nil { + t.Fatalf("load registry: %v", err) + } + return reg +} + +func TestParseID(t *testing.T) { + cases := []struct { + in string + tech string + index int + bad bool + }{ + {in: "T1082-3", tech: "T1082", index: 3}, + {in: "T1069.001-1", tech: "T1069.001", index: 1}, + {in: "T1059.004-17", tech: "T1059.004", index: 17}, + {in: "T1082", bad: true}, + {in: "T1082-0", bad: true}, + {in: "1082-1", bad: true}, + {in: "", bad: true}, + } + for _, c := range cases { + tech, idx, err := ParseID(c.in) + if c.bad { + if err == nil { + t.Errorf("ParseID(%q) = (%s,%d), want error", c.in, tech, idx) + } + continue + } + if err != nil { + t.Errorf("ParseID(%q): %v", c.in, err) + continue + } + if tech != c.tech || idx != c.index { + t.Errorf("ParseID(%q) = (%s,%d), want (%s,%d)", c.in, tech, idx, c.tech, c.index) + } + } +} + +func TestInterpolateDefaults(t *testing.T) { + subst := map[string]string{"output_file": "/tmp/x.txt", "host": "127.0.0.1"} + got := interpolate("uname -a >> #{output_file}; ping #{host}", subst) + want := "uname -a >> /tmp/x.txt; ping 127.0.0.1" + if got != want { + t.Errorf("interpolate = %q, want %q", got, want) + } +} + +// An unknown placeholder must stay visible. Silently emptying it would produce +// a subtly different command than the atomic describes. +func TestInterpolateLeavesUnknownPlaceholder(t *testing.T) { + got := interpolate("run #{nope}", map[string]string{"other": "x"}) + if got != "run #{nope}" { + t.Errorf("interpolate = %q, want the placeholder preserved", got) + } +} + +func TestResolveRealAtomic(t *testing.T) { + reg := requireART(t) + test, err := reg.Resolve("T1082-3") + if err != nil { + t.Fatalf("resolve: %v", err) + } + if !strings.Contains(test.Command, "uname -a") { + t.Errorf("T1082-3 command = %q, want it to contain 'uname -a'", test.Command) + } + if strings.Contains(test.Command, "#{") { + t.Errorf("T1082-3 command still has an uninterpolated placeholder: %q", test.Command) + } + if !test.SupportsPlatform("linux") || test.SupportsPlatform("windows") { + t.Errorf("T1082-3 platforms = %v, want linux and not windows", test.SupportedPlatforms) + } + if test.Source != SourceART { + t.Errorf("source = %q, want %q", test.Source, SourceART) + } +} + +// The engine must never invent a command. An unresolvable ID is an error the +// caller has to surface, which is the whole point of this package. +func TestResolveRefusesUnknown(t *testing.T) { + reg := requireART(t) + for _, id := range []string{"T9999-1", "T1082-999", "garbage"} { + if got, err := reg.Resolve(id); err == nil { + t.Errorf("Resolve(%q) = %+v, want error", id, got) + } + } +} + +func TestOverrideReplacesCommand(t *testing.T) { + reg := requireART(t) + test, err := reg.Resolve("T1518-1") + if err != nil { + t.Fatalf("resolve: %v", err) + } + if test.Source != SourceOverride { + t.Fatalf("T1518-1 source = %q, want %q", test.Source, SourceOverride) + } + if test.OverrideReason == "" { + t.Error("an override must carry a visible reason") + } + if !test.SupportsPlatform("linux") { + t.Error("the T1518 override must declare linux support") + } +} + +// An input-only override keeps the upstream command but pins its inputs. +func TestInputPinningKeepsUpstreamCommand(t *testing.T) { + reg := requireART(t) + test, err := reg.Resolve("T1059.004-1") + if err != nil { + t.Fatalf("resolve: %v", err) + } + if test.Source != SourceART { + t.Errorf("source = %q, want the upstream atomic to be preserved", test.Source) + } + if strings.Contains(test.Command, "8.8.8.8") { + t.Error("containment: the atomic still reaches 8.8.8.8, outside purpleloop-lab") + } + if !strings.Contains(test.Command, "127.0.0.1") { + t.Errorf("pinned host not applied: %q", test.Command) + } + if test.OverrideReason == "" { + t.Error("a pinned input must be reported") + } +} + +func TestOverrideRequiresReason(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "ov.yml") + if err := os.WriteFile(path, []byte("overrides:\n T1082-1:\n command: id\n"), 0644); err != nil { + t.Fatal(err) + } + if _, err := Load(artRoot, path); err == nil { + t.Error("an override without a reason must be rejected") + } +} + +// The regression guard for the defect this package exists to fix: every +// technique in the shipped campaign must resolve to its OWN command. If two +// techniques share one, the campaign is testing one behaviour under many names. +func TestCampaignAtomicsAreDistinct(t *testing.T) { + reg := requireART(t) + ids := []string{ + "T1059.004-1", "T1087.001-1", "T1082-3", "T1033-2", "T1007-3", + "T1016-3", "T1049-4", "T1069.001-1", "T1135-1", "T1518-1", + } + seen := map[string]string{} + for _, id := range ids { + test, err := reg.Resolve(id) + if err != nil { + t.Errorf("resolve %s: %v", id, err) + continue + } + if !test.SupportsPlatform("linux") { + t.Errorf("%s does not support linux (platforms %v) — the Linux campaign cannot run it", + id, test.SupportedPlatforms) + } + if prev, dup := seen[test.Command]; dup { + t.Errorf("%s and %s resolve to the SAME command — a campaign must not test one behaviour under many names:\n%s", + prev, id, test.Command) + } + seen[test.Command] = id + } +} diff --git a/internal/canary/canary.go b/internal/canary/canary.go index e760764..e94cd54 100644 --- a/internal/canary/canary.go +++ b/internal/canary/canary.go @@ -115,7 +115,7 @@ func (ck Checker) Run(ctx context.Context, marker string, exec model.Executor, // Check executes the canary on one platform and verifies the marker was // detected, using production defaults. func Check(ctx context.Context, marker string, exec model.Executor, - coll model.Collector, platform string, target model.Target, dryRun bool) Result { + coll model.Collector, platform string, target model.Target) Result { return Checker{}.Run(ctx, marker, exec, coll, platform, target) } diff --git a/internal/capture/capture.go b/internal/capture/capture.go new file mode 100644 index 0000000..7b2ab1f --- /dev/null +++ b/internal/capture/capture.go @@ -0,0 +1,141 @@ +// Package capture persists the raw telemetry a campaign collected, so a run can +// be re-evaluated later without a lab. +// +// This is what makes rule changes testable against reality. CI has fixtures — +// events hand-written to prove a rule matches what its author intended — but a +// fixture is a claim about the world, not a sample of it. A captured run lets +// the same matcher be re-run over telemetry the lab actually produced, and lets +// a rule edit be checked against every past run: "would this have broken a +// DETECTED we already had?" +package capture + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +// Filename is the dataset written beside a run's coverage.json. +const Filename = "events.jsonl" + +// Record is one collected event with the execution it belongs to. The atomic +// and window travel with the event so a replay can reproduce the exact +// evaluation, not merely re-match a pile of logs. +type Record struct { + TechniqueID string `json:"technique_id"` + AtomicID string `json:"atomic_id"` + Host string `json:"host"` + ExecutedAt time.Time `json:"executed_at"` + Window model.TimeWindow `json:"window"` + Event model.Event `json:"event"` +} + +// Dataset is a replayable run, grouped by technique. +type Dataset struct { + Records []Record +} + +// Write persists every collected event for a campaign to dir/events.jsonl. +// A synthetic run is refused: its "telemetry" is fabricated, and a fabricated +// dataset replayed later would be indistinguishable from a real one. +func Write(dir string, run model.CampaignResult, host string) error { + if run.Synthetic { + return nil + } + + // No events means no dataset. Writing an empty file would leave something + // that looks like a capture but that Read rejects — the run's own report + // already records that nothing was collected. + total := 0 + for _, c := range run.Chains { + total += len(c.Collected) + } + if total == 0 { + return nil + } + + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + f, err := os.Create(filepath.Join(dir, Filename)) + if err != nil { + return err + } + defer f.Close() + + w := bufio.NewWriter(f) + enc := json.NewEncoder(w) + for _, c := range run.Chains { + for _, e := range c.Collected { + rec := Record{ + TechniqueID: c.TechniqueID, + AtomicID: c.Atomic.ID, + Host: host, + ExecutedAt: c.ExecutedAt, + Window: c.Window, + Event: e, + } + if err := enc.Encode(rec); err != nil { + return err + } + } + } + return w.Flush() +} + +// Read loads a captured dataset. dir may be the run directory or the file. +func Read(path string) (*Dataset, error) { + if fi, err := os.Stat(path); err == nil && fi.IsDir() { + path = filepath.Join(path, Filename) + } + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open dataset: %w", err) + } + defer f.Close() + + ds := &Dataset{} + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) // archive events can be large + line := 0 + for sc.Scan() { + line++ + raw := sc.Bytes() + if len(raw) == 0 { + continue + } + var rec Record + if err := json.Unmarshal(raw, &rec); err != nil { + return nil, fmt.Errorf("dataset line %d: %w", line, err) + } + ds.Records = append(ds.Records, rec) + } + if err := sc.Err(); err != nil { + return nil, fmt.Errorf("read dataset: %w", err) + } + if len(ds.Records) == 0 { + return nil, fmt.Errorf("dataset %s is empty", path) + } + return ds, nil +} + +// ByTechnique groups the dataset's events, preserving first-seen order so a +// replay reproduces the original campaign's sequence. +func (d *Dataset) ByTechnique() ([]string, map[string][]model.Event, map[string]Record) { + var order []string + events := map[string][]model.Event{} + meta := map[string]Record{} + for _, r := range d.Records { + if _, seen := events[r.TechniqueID]; !seen { + order = append(order, r.TechniqueID) + meta[r.TechniqueID] = r + } + events[r.TechniqueID] = append(events[r.TechniqueID], r.Event) + } + return order, events, meta +} diff --git a/internal/capture/capture_test.go b/internal/capture/capture_test.go new file mode 100644 index 0000000..d906b1d --- /dev/null +++ b/internal/capture/capture_test.go @@ -0,0 +1,122 @@ +package capture + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +func sampleRun(synthetic bool) model.CampaignResult { + start := time.Date(2026, 8, 29, 10, 0, 0, 0, time.UTC) + mk := func(tech, atomicID string, n int) model.ProofChain { + evs := make([]model.Event, n) + for i := range evs { + evs[i] = model.Event{ + ID: tech + "-ev", + Timestamp: start.Add(time.Duration(i) * time.Second), + Raw: json.RawMessage(`{"Image":"/usr/bin/uname","CommandLine":"uname -a"}`), + } + } + return model.ProofChain{ + TechniqueID: tech, + Atomic: model.AtomicTest{ID: atomicID, TechniqueID: tech}, + ExecutedAt: start, + Window: model.TimeWindow{Start: start, End: start.Add(time.Minute)}, + Collected: evs, + } + } + return model.CampaignResult{ + StartedAt: start, + Synthetic: synthetic, + Chains: []model.ProofChain{mk("T1082", "T1082-3", 2), mk("T1033", "T1033-2", 1)}, + } +} + +func TestWriteReadRoundTrip(t *testing.T) { + dir := t.TempDir() + if err := Write(dir, sampleRun(false), "victim01"); err != nil { + t.Fatalf("write: %v", err) + } + ds, err := Read(dir) + if err != nil { + t.Fatalf("read: %v", err) + } + if len(ds.Records) != 3 { + t.Fatalf("records = %d, want 3", len(ds.Records)) + } + + order, events, meta := ds.ByTechnique() + if len(order) != 2 || order[0] != "T1082" { + t.Errorf("order = %v, want T1082 first (campaign sequence preserved)", order) + } + if len(events["T1082"]) != 2 || len(events["T1033"]) != 1 { + t.Errorf("grouping wrong: %d/%d", len(events["T1082"]), len(events["T1033"])) + } + if meta["T1082"].AtomicID != "T1082-3" { + t.Errorf("atomic = %q, want T1082-3 — replay needs the atomic that produced the events", + meta["T1082"].AtomicID) + } + if meta["T1082"].Host != "victim01" { + t.Errorf("host = %q, want victim01", meta["T1082"].Host) + } + if meta["T1082"].Window.Start.IsZero() { + t.Error("the collection window must survive capture") + } +} + +// A synthetic run's telemetry is fabricated. Capturing it would produce a +// dataset indistinguishable from a real one on replay, so it is refused. +func TestWriteRefusesSyntheticRun(t *testing.T) { + dir := t.TempDir() + if err := Write(dir, sampleRun(true), "victim01"); err != nil { + t.Fatalf("write: %v", err) + } + if _, err := Read(dir); err == nil { + t.Error("a synthetic run must not produce a replayable dataset") + } +} + +func TestReadRejectsEmptyAndMissing(t *testing.T) { + if _, err := Read(filepath.Join(t.TempDir(), "nope")); err == nil { + t.Error("a missing dataset must error") + } +} + +// The raw event must survive byte-for-byte: replay re-runs the real matcher +// over it, so any normalisation here would change the verdict. +func TestRawEventPreservedExactly(t *testing.T) { + dir := t.TempDir() + run := sampleRun(false) + want := string(run.Chains[0].Collected[0].Raw) + + if err := Write(dir, run, "victim01"); err != nil { + t.Fatal(err) + } + ds, err := Read(dir) + if err != nil { + t.Fatal(err) + } + if got := string(ds.Records[0].Event.Raw); got != want { + t.Errorf("raw event changed through capture:\n got %s\nwant %s", got, want) + } +} + +// A run that collected nothing must not leave an empty file that looks like a +// dataset but that Read rejects. +func TestWriteSkipsRunWithNoEvents(t *testing.T) { + dir := t.TempDir() + run := model.CampaignResult{ + StartedAt: time.Now(), + Chains: []model.ProofChain{{TechniqueID: "T1082", Verdict: model.NoTelemetry}}, + } + if err := Write(dir, run, "victim01"); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dir, Filename)); !os.IsNotExist(err) { + t.Error("a run with no collected events must not write a dataset file") + } +} diff --git a/internal/collector/prefilter_test.go b/internal/collector/prefilter_test.go index c5a4fee..fc7bbb8 100644 --- a/internal/collector/prefilter_test.go +++ b/internal/collector/prefilter_test.go @@ -19,8 +19,11 @@ func TestDateAlternation(t *testing.T) { w model.TimeWindow want string }{ - {"same day", mk("2026-07-04T09:00:00Z", "2026-07-04T09:05:00Z"), "2026-07-04"}, - {"spans midnight", mk("2026-07-04T23:59:00Z", "2026-07-05T00:02:00Z"), "2026-07-04|2026-07-05"}, + // The alternation is widened by one day on each side so a non-UTC + // Wazuh timestamp whose literal date lands on an adjacent day is not + // dropped by the coarse pre-filter (the precise instant check still bounds it). + {"same day", mk("2026-07-04T09:00:00Z", "2026-07-04T09:05:00Z"), "2026-07-03|2026-07-04|2026-07-05"}, + {"spans midnight", mk("2026-07-04T23:59:00Z", "2026-07-05T00:02:00Z"), "2026-07-03|2026-07-04|2026-07-05|2026-07-06"}, {"zero window", model.TimeWindow{}, ""}, {"reversed", mk("2026-07-05T00:00:00Z", "2026-07-04T00:00:00Z"), ""}, } diff --git a/internal/collector/settle.go b/internal/collector/settle.go new file mode 100644 index 0000000..1b9445c --- /dev/null +++ b/internal/collector/settle.go @@ -0,0 +1,119 @@ +package collector + +import ( + "context" + "time" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +// Settler polls a collector until the telemetry for an execution has arrived, +// instead of sleeping a fixed amount and hoping. +// +// The pipeline previously slept 10s per technique and then queried once. That +// is wrong in both directions: too long when the SIEM has already indexed the +// events (ten techniques spend 100s asleep), and too short when it has not, +// which produces a NO_TELEMETRY that is really an impatience artifact. +// +// Settle returns as soon as one of three things is true: +// +// - Satisfied reports a match — nothing later can improve on a detection; +// - the event count stops growing for StableFor consecutive polls, meaning +// ingestion has settled and no further events are coming; +// - Timeout elapses, which is a genuine "the events never arrived". +type Settler struct { + PollInterval time.Duration // gap between polls (default 3s) + Timeout time.Duration // total budget (default 60s) + StableFor int // polls with no new events before settling (default 2) + MinWait time.Duration // floor before the first poll (default 2s) +} + +func (s Settler) withDefaults() Settler { + if s.PollInterval <= 0 { + s.PollInterval = 3 * time.Second + } + if s.Timeout <= 0 { + s.Timeout = 60 * time.Second + } + if s.StableFor <= 0 { + s.StableFor = 2 + } + if s.MinWait < 0 { + s.MinWait = 0 + } + if s.MinWait == 0 { + s.MinWait = 2 * time.Second + } + return s +} + +// SettleResult reports what the poll loop observed. +type SettleResult struct { + Events []model.Event + Satisfied bool // the caller's predicate matched + Waited time.Duration // how long collection actually took + TimedOut bool +} + +// Settle polls coll for the window until satisfied, stable, or timed out. +// satisfied may be nil, in which case Settle waits for ingestion to stabilise. +func (s Settler) Settle(ctx context.Context, coll model.Collector, window model.TimeWindow, + host string, satisfied func([]model.Event) bool) (SettleResult, error) { + + s = s.withDefaults() + started := time.Now() + deadline := started.Add(s.Timeout) + + // A brief floor before the first query: an immediate poll almost always + // races the SIEM's indexer and wastes a round trip. + if err := sleepCtx(ctx, s.MinWait); err != nil { + return SettleResult{}, err + } + + var last []model.Event + stable := 0 + + for { + events, err := coll.Query(ctx, window, host) + if err != nil { + return SettleResult{Events: last, Waited: time.Since(started)}, err + } + + if satisfied != nil && satisfied(events) { + return SettleResult{Events: events, Satisfied: true, Waited: time.Since(started)}, nil + } + + if len(events) == len(last) { + stable++ + } else { + stable = 0 + } + last = events + + // Ingestion has settled: the count held steady across consecutive polls, + // so waiting longer cannot change the verdict. Requiring at least one + // event avoids settling instantly on an empty archive that is merely slow. + if stable >= s.StableFor && len(events) > 0 { + return SettleResult{Events: events, Waited: time.Since(started)}, nil + } + + if time.Now().After(deadline) { + return SettleResult{Events: events, Waited: time.Since(started), TimedOut: true}, nil + } + if err := sleepCtx(ctx, s.PollInterval); err != nil { + return SettleResult{Events: events, Waited: time.Since(started)}, err + } + } +} + +func sleepCtx(ctx context.Context, d time.Duration) error { + if d <= 0 { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(d): + return nil + } +} diff --git a/internal/collector/settle_test.go b/internal/collector/settle_test.go new file mode 100644 index 0000000..754ac35 --- /dev/null +++ b/internal/collector/settle_test.go @@ -0,0 +1,95 @@ +package collector + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +// scriptedCollector returns a different event slice on each successive call. +type scriptedCollector struct { + rounds [][]model.Event + calls int +} + +func (c *scriptedCollector) Query(ctx context.Context, w model.TimeWindow, host string) ([]model.Event, error) { + i := c.calls + c.calls++ + if i >= len(c.rounds) { + i = len(c.rounds) - 1 + } + return c.rounds[i], nil +} + +func ev(n int) []model.Event { + out := make([]model.Event, n) + for i := range out { + out[i] = model.Event{ID: string(rune('a' + i)), Raw: json.RawMessage(`{}`)} + } + return out +} + +func fast() Settler { + return Settler{PollInterval: time.Millisecond, Timeout: 200 * time.Millisecond, StableFor: 2, MinWait: time.Millisecond} +} + +// A match ends collection immediately — nothing later can improve on it. +func TestSettleStopsOnSatisfied(t *testing.T) { + c := &scriptedCollector{rounds: [][]model.Event{ev(1), ev(2), ev(3)}} + res, err := fast().Settle(context.Background(), c, model.TimeWindow{}, "h", + func(e []model.Event) bool { return len(e) >= 2 }) + if err != nil { + t.Fatal(err) + } + if !res.Satisfied { + t.Error("want Satisfied") + } + if c.calls != 2 { + t.Errorf("polled %d times, want 2 — it should stop as soon as the predicate matches", c.calls) + } +} + +// Once the count holds steady, ingestion has settled and waiting longer cannot +// change the verdict. +func TestSettleStopsWhenStable(t *testing.T) { + c := &scriptedCollector{rounds: [][]model.Event{ev(1), ev(2), ev(2), ev(2)}} + res, err := fast().Settle(context.Background(), c, model.TimeWindow{}, "h", func([]model.Event) bool { return false }) + if err != nil { + t.Fatal(err) + } + if res.Satisfied || res.TimedOut { + t.Errorf("want a settled result, got satisfied=%v timedout=%v", res.Satisfied, res.TimedOut) + } + if len(res.Events) != 2 { + t.Errorf("events = %d, want 2", len(res.Events)) + } +} + +// An archive that never produces an event must time out rather than settle at +// zero: "nothing arrived yet" and "nothing will arrive" are different claims, +// and only the timeout can distinguish them. +func TestSettleDoesNotSettleOnEmpty(t *testing.T) { + c := &scriptedCollector{rounds: [][]model.Event{{}}} + res, err := fast().Settle(context.Background(), c, model.TimeWindow{}, "h", func([]model.Event) bool { return false }) + if err != nil { + t.Fatal(err) + } + if !res.TimedOut { + t.Error("an always-empty archive must time out, not settle") + } + if len(res.Events) != 0 { + t.Errorf("events = %d, want 0", len(res.Events)) + } +} + +func TestSettleRespectsContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + c := &scriptedCollector{rounds: [][]model.Event{{}}} + if _, err := fast().Settle(ctx, c, model.TimeWindow{}, "h", nil); err == nil { + t.Error("a cancelled context must abort collection") + } +} diff --git a/internal/collector/wazuh.go b/internal/collector/wazuh.go index f7264cd..9a489b7 100644 --- a/internal/collector/wazuh.go +++ b/internal/collector/wazuh.go @@ -38,11 +38,10 @@ func parseTimestamp(s string) (time.Time, error) { return time.Time{}, fmt.Errorf("cannot parse timestamp %q", s) } -// WazuhCollector reads agent alerts from the manager's alerts.json log. +// WazuhCollector reads agent alerts from the manager's archives.json via +// `docker exec`. With no ManagerContainer (and no test alertsPath) it runs in +// dry mode and returns a single synthetic event. type WazuhCollector struct { - BaseURL string // e.g. https://localhost:55000 — empty => dry mode - User string - Pass string ManagerContainer string // docker container name, e.g. single-node-wazuh.manager-1 // alertsPath overrides the source for testing; empty uses docker exec. @@ -156,9 +155,13 @@ func dateAlternation(w model.TimeWindow) string { if w.Start.IsZero() || w.End.IsZero() || w.End.Before(w.Start) { return "" } - const maxDays = 31 - start := w.Start.UTC().Truncate(24 * time.Hour) - end := w.End.UTC().Truncate(24 * time.Hour) + const maxDays = 34 + // Widen by a day on each side: Wazuh may stamp events with a non-UTC offset, + // so a line's literal YYYY-MM-DD can be the local date while the window is + // computed in UTC. Including the adjacent days keeps a day-boundary event in + // the coarse pre-filter; the precise instant comparison below still bounds it. + start := w.Start.UTC().Truncate(24 * time.Hour).Add(-24 * time.Hour) + end := w.End.UTC().Truncate(24 * time.Hour).Add(24 * time.Hour) var stamps []string for d := start; !d.After(end); d = d.Add(24 * time.Hour) { stamps = append(stamps, d.Format("2006-01-02")) diff --git a/internal/evaluator/distinct_test.go b/internal/evaluator/distinct_test.go new file mode 100644 index 0000000..7affa8d --- /dev/null +++ b/internal/evaluator/distinct_test.go @@ -0,0 +1,145 @@ +package evaluator + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// Nine of the ten Linux rules were once byte-identical in logic — every one +// matched `Image endswith /id or /whoami`, differing only in title — because +// they were written against an engine that executed `id; whoami` for every +// technique. The CI regression suite passed the whole time, since all nine +// shared one positive fixture too. +// +// A campaign whose detections are one rule under many names reports coverage it +// does not have. This test makes that unshippable. +func TestRulesHaveDistinctLogic(t *testing.T) { + dirs := []string{"../../detections/linux", "../../detections/windows"} + byLogic := map[string][]string{} + + for _, dir := range dirs { + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read %s: %v", dir, err) + } + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".yml") { + continue + } + path := filepath.Join(dir, e.Name()) + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + var doc struct { + Detection map[string]any `yaml:"detection"` + } + if err := yaml.Unmarshal(data, &doc); err != nil { + t.Fatalf("parse %s: %v", path, err) + } + if len(doc.Detection) == 0 { + t.Errorf("%s has no detection block", path) + continue + } + // Canonicalise so formatting differences do not mask identical logic. + canon, err := json.Marshal(normalize(doc.Detection)) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(canon) + key := hex.EncodeToString(sum[:8]) + byLogic[key] = append(byLogic[key], filepath.Join(filepath.Base(dir), e.Name())) + } + } + + for _, rules := range byLogic { + if len(rules) > 1 { + t.Errorf("these rules have IDENTICAL detection logic: %s\n"+ + "each technique needs a detection for what its own atomic does — "+ + "duplicate logic reports coverage the campaign does not have", + strings.Join(rules, ", ")) + } + } +} + +// normalize converts a YAML tree into a form with deterministic ordering, so +// two logically identical detections hash the same. +func normalize(v any) any { + switch x := v.(type) { + case map[string]any: + out := make(map[string]any, len(x)) + for k, val := range x { + out[k] = normalize(val) + } + return out + case map[any]any: + out := make(map[string]any, len(x)) + for k, val := range x { + out[strings.TrimSpace(toString(k))] = normalize(val) + } + return out + case []any: + out := make([]any, len(x)) + for i, val := range x { + out[i] = normalize(val) + } + return out + default: + return v + } +} + +func toString(v any) string { + if s, ok := v.(string); ok { + return s + } + b, _ := json.Marshal(v) + return string(b) +} + +// Every rule mapped to a technique must actually reference that technique in +// its tags, so a mapping cannot quietly point at an unrelated detection. +func TestRuleTagsMatchTechnique(t *testing.T) { + entries, err := os.ReadDir("../../detections/linux") + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, ".yml") { + continue + } + tech := strings.TrimSuffix(name, ".yml") + if !strings.HasPrefix(tech, "T1") { + continue // rules named by behaviour rather than technique + } + data, err := os.ReadFile(filepath.Join("../../detections/linux", name)) + if err != nil { + t.Fatal(err) + } + var doc struct { + Tags []string `yaml:"tags"` + } + if err := yaml.Unmarshal(data, &doc); err != nil { + t.Fatal(err) + } + want := "attack." + strings.ToLower(tech) + found := false + for _, tag := range doc.Tags { + if strings.EqualFold(strings.TrimSpace(tag), want) { + found = true + break + } + } + if !found { + t.Errorf("%s does not tag %s (tags: %v)", name, want, doc.Tags) + } + } +} diff --git a/internal/evaluator/fidelity_test.go b/internal/evaluator/fidelity_test.go index d963dc5..ab14412 100644 --- a/internal/evaluator/fidelity_test.go +++ b/internal/evaluator/fidelity_test.go @@ -55,7 +55,9 @@ func TestNormalizer_FidelityTagging(t *testing.T) { raw string want string }{ - {"windows eventdata", `{"data":{"win":{"eventdata":{"image":"C:\\a.exe"}}}}`, FidelityProcess}, + {"windows process creation", `{"data":{"win":{"eventdata":{"image":"C:\\a.exe","commandLine":"a.exe"}}}}`, FidelityProcess}, + {"windows sysmon1 by eventID", `{"data":{"win":{"system":{"eventID":"1"},"eventdata":{"image":"C:\\a.exe"}}}}`, FidelityProcess}, + {"windows non-creation enumeration (4798)", `{"data":{"win":{"eventdata":{"callerProcessName":"C:\\net1.exe","subjectUserName":"SYSTEM"}}}}`, FidelityLog}, {"auditd execve", `{"data":{"audit":{"execve":{"a0":"id"}}}}`, FidelityProcess}, {"top-level synthetic", `{"Image":"/usr/bin/id","CommandLine":"id"}`, FidelityProcess}, {"full_log scrape", `{"full_log":"ossec: output: 'id': uid=0"}`, FidelityLog}, diff --git a/internal/evaluator/fixes_test.go b/internal/evaluator/fixes_test.go new file mode 100644 index 0000000..a78f79c --- /dev/null +++ b/internal/evaluator/fixes_test.go @@ -0,0 +1,102 @@ +package evaluator + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +// TestGlobMatch_Anchored guards the wildcard fixes: patterns are anchored to the +// whole value (no false DETECTED from trailing/leading slop) and `?` matches +// exactly one character. +func TestGlobMatch_Anchored(t *testing.T) { + cases := []struct { + pattern, value string + want bool + }{ + {`*\svchost.exe`, `c:\windows\svchost.exe`, true}, + {`*\svchost.exe`, `c:\x\svchost.exe.malware`, false}, // trailing anchored + {`cmd*`, `cmd.exe`, true}, + {`cmd*`, `xcmd.exe`, false}, // leading anchored + {`C:\Users\?\evil.exe`, `C:\Users\a\evil.exe`, true}, + {`C:\Users\?\evil.exe`, `C:\Users\ab\evil.exe`, false}, // ? = one char + } + for _, c := range cases { + if got := globMatch(c.pattern, c.value); got != c.want { + t.Errorf("globMatch(%q, %q) = %v, want %v", c.pattern, c.value, got, c.want) + } + } +} + +// TestCondition_AggregateGlobHonorsTrailingFilter guards the P0 fix: `N of +// prefix_*` expands by glob AND the trailing `and not filter` is no longer +// dropped by the parser (which would lose filter suppression → false DETECTED). +func TestCondition_AggregateGlobHonorsTrailingFilter(t *testing.T) { + expr, err := parseCondition("1 of selection_* and not filter") + if err != nil { + t.Fatalf("parse: %v", err) + } + detections := map[string]FieldMap{ + "selection_a": {"Image": {Values: []string{"/bin/evil"}}}, + "filter": {"User": {Values: []string{"root"}}}, + } + // Matches selection_a AND filter → "1 of selection_*" true, "not filter" false. + if evalExpr(expr, detections, map[string]string{"Image": "/bin/evil", "User": "root"}) { + t.Error("filter should suppress via the trailing 'and not filter'") + } + // Matches selection_a, not filter → true. + if !evalExpr(expr, detections, map[string]string{"Image": "/bin/evil", "User": "alice"}) { + t.Error("selection matches and filter does not → should match") + } +} + +// TestMatchNull guards Sigma `Field: null` (match when absent/empty). +func TestMatchNull(t *testing.T) { + expr, _ := parseCondition("sel") + detections := map[string]FieldMap{"sel": {"CommandLine": {MatchNull: true}}} + if !evalExpr(expr, detections, map[string]string{"Image": "x"}) { + t.Error("Field: null should match when the field is absent") + } + if evalExpr(expr, detections, map[string]string{"CommandLine": "present"}) { + t.Error("Field: null should NOT match when the field is present") + } +} + +// TestNot_NoSpaceBeforeParen guards `not(...)` (no space) parsing. +func TestNot_NoSpaceBeforeParen(t *testing.T) { + expr, err := parseCondition("not(sel)") + if err != nil { + t.Fatalf("parse: %v", err) + } + detections := map[string]FieldMap{"sel": {"Image": {Values: []string{"/bin/evil"}}}} + if evalExpr(expr, detections, map[string]string{"Image": "/bin/evil"}) { + t.Error("not(sel) should be false when sel matches") + } + if !evalExpr(expr, detections, map[string]string{"Image": "/bin/ok"}) { + t.Error("not(sel) should be true when sel does not match") + } +} + +// TestEvaluate_UnsupportedModifierInconclusive guards that a rule using a +// modifier we can't evaluate is reported INCONCLUSIVE, never a silent MISS. +func TestEvaluate_UnsupportedModifierInconclusive(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "r.yml") + rule := "title: t\nlogsource:\n category: process_creation\ndetection:\n" + + " sel:\n CommandLine|base64|contains: whoami\n condition: sel\n" + if err := os.WriteFile(path, []byte(rule), 0644); err != nil { + t.Fatal(err) + } + eval := RuleMatcherEvaluator{RulesDir: dir} + ev := []model.Event{{Raw: json.RawMessage(`{"Image":"/bin/x","CommandLine":"whoami"}`)}} + v, _, err := eval.Evaluate(model.SigmaRule{Path: path}, ev) + if err != nil { + t.Fatalf("evaluate: %v", err) + } + if v != model.Inconclusive { + t.Errorf("unsupported-modifier rule → %s, want INCONCLUSIVE", v) + } +} diff --git a/internal/evaluator/matcher.go b/internal/evaluator/matcher.go index 22955d2..3acb918 100644 --- a/internal/evaluator/matcher.go +++ b/internal/evaluator/matcher.go @@ -52,6 +52,13 @@ func evalIdent(name string, detections map[string]FieldMap, event map[string]str // All field entries in the identifier must match (AND) for field, entry := range fm { val, exists := event[field] + if entry.MatchNull { + // Sigma `Field: null` matches when the field is absent or empty. + if exists && val != "" { + return false + } + continue + } if !exists { return false } @@ -127,23 +134,22 @@ func matchField(eventValue string, entry FieldEntry) bool { return matchNumeric(eventValue, candidate, numOp) } - v := strings.ToLower(eventValue) - c := strings.ToLower(candidate) - - // If candidate contains * wildcards, use glob-like matching - if strings.Contains(c, "*") { - return matchWildcard(v, c) - } - + // Express the modifier (or an explicit-wildcard value) as a Sigma glob, + // then match as an anchored, case-insensitive regexp. Anchoring pins the + // literal segments to the ends of the value (so `*\svchost.exe` does not + // match `…svchost.exe.malware`), `?` matches one character, and regex + // metacharacters in the value can't leak. switch { - case hasEndsWith: - return strings.HasSuffix(v, c) - case hasStartsWith: - return strings.HasPrefix(v, c) case hasContains: - return strings.Contains(v, c) + return globMatch("*"+candidate+"*", eventValue) + case hasStartsWith: + return globMatch(candidate+"*", eventValue) + case hasEndsWith: + return globMatch("*"+candidate, eventValue) + case strings.ContainsAny(candidate, "*?"): + return globMatch(candidate, eventValue) default: - return v == c + return strings.EqualFold(eventValue, candidate) } } @@ -185,38 +191,40 @@ func matchNumeric(eventValue, candidate, op string) bool { return false } -// matchWildcard performs case-insensitive glob matching where * matches any -// sequence of characters. It splits the pattern on * and matches each literal -// segment in order within the value string. -func matchWildcard(value, pattern string) bool { - // A single "*" matches everything - if pattern == "*" { - return true - } +// globMatch reports whether value matches a Sigma glob pattern, case-insensitively +// and anchored to the whole value. +func globMatch(pattern, value string) bool { + re := globToRegexp(pattern) + return re != nil && re.MatchString(value) +} - segments := strings.Split(pattern, "*") - pos := 0 - for _, seg := range segments { - if seg == "" { - continue // leading *, trailing *, or consecutive ** - } - idx := strings.Index(value[pos:], seg) - if idx < 0 { - return false +// globToRegexp compiles a Sigma wildcard pattern into an anchored, +// case-insensitive regexp: `*` → any run, `?` → one character; every other +// character (including `\`, a literal path separator in this dialect) is +// escaped. Returns nil if compilation fails (treated as no match by globMatch). +func globToRegexp(glob string) *regexp.Regexp { + var b strings.Builder + b.WriteString("(?is)^") + for i := 0; i < len(glob); i++ { + switch glob[i] { + case '*': + b.WriteString(".*") + case '?': + b.WriteString(".") + default: + b.WriteString(regexp.QuoteMeta(string(glob[i]))) } - pos += idx + len(seg) } - return true + b.WriteString("$") + re, err := regexp.Compile(b.String()) + if err != nil { + return nil + } + return re } func evalOneOf(e OneOfExpr, detections map[string]FieldMap, event map[string]string) bool { - names := e.Names - if len(names) == 0 { - // "1 of them" — expand to all search-identifiers - for k := range detections { - names = append(names, k) - } - } + names := expandIdentNames(e.Names, detections) matched := 0 for _, name := range names { if evalIdent(name, detections, event) { @@ -227,13 +235,7 @@ func evalOneOf(e OneOfExpr, detections map[string]FieldMap, event map[string]str } func evalAllOf(e AllOfExpr, detections map[string]FieldMap, event map[string]string) bool { - names := e.Names - if len(names) == 0 { - // "all of them" — expand to all search-identifiers - for k := range detections { - names = append(names, k) - } - } + names := expandIdentNames(e.Names, detections) for _, name := range names { if !evalIdent(name, detections, event) { return false @@ -241,3 +243,37 @@ func evalAllOf(e AllOfExpr, detections map[string]FieldMap, event map[string]str } return len(names) > 0 // vacuous truth over empty set → false } + +// expandIdentNames resolves aggregate operands to concrete identifier names. +// Empty input means "of them" → every identifier EXCEPT filter/falsepositive +// selections (which are meant to be referenced explicitly, not folded into an +// OR/AND). A name containing */? is a glob expanded against the identifier keys. +func expandIdentNames(names []string, detections map[string]FieldMap) []string { + if len(names) == 0 { + var out []string + for k := range detections { + if strings.HasPrefix(k, "filter") || strings.HasPrefix(k, "falsepositive") { + continue + } + out = append(out, k) + } + return out + } + var out []string + for _, n := range names { + if strings.ContainsAny(n, "*?") { + re := globToRegexp(n) + if re == nil { + continue + } + for k := range detections { + if re.MatchString(k) { + out = append(out, k) + } + } + continue + } + out = append(out, n) + } + return out +} diff --git a/internal/evaluator/normalizer.go b/internal/evaluator/normalizer.go index 0916a22..433993e 100644 --- a/internal/evaluator/normalizer.go +++ b/internal/evaluator/normalizer.go @@ -52,21 +52,77 @@ func (Normalizer) Normalize(raw json.RawMessage) map[string]string { highFidelity = true } - // Try Windows Sysmon / EventChannel paths + // Try Windows Sysmon / EventChannel paths. + // + // CRITICAL: only genuine PROCESS-CREATION events (Sysmon EventID 1, Security + // 4688) may be tagged high fidelity. Other Windows events (network + // connection=3, account enumeration=4798, handle=4656, …) also carry a + // process name (callerProcessName/processName) but are NOT proof a process + // was created. Tagging them as process telemetry credited a process-creation + // detection off an enumeration/network event — a false DETECTED. if data, ok := event["data"].(map[string]any); ok { if win, ok := data["win"].(map[string]any); ok { if ed, ok := win["eventdata"].(map[string]any); ok { - // Sysmon process creation (sysmon 1) - getString(ed, "image", &out, "Image") - getString(ed, "parentImage", &out, "ParentImage") - getString(ed, "commandLine", &out, "CommandLine") - getString(ed, "user", &out, "User") - getString(ed, "parentUser", &out, "User") - // Security Audit fallbacks - getString(ed, "callerProcessName", &out, "Image") - getString(ed, "subjectUserName", &out, "User") - getString(ed, "processName", &out, "Image") + eventID := winEventID(win) + if eventID != "" { + out["EventID"] = eventID + } + if isProcessCreation(ed, eventID) { + // Real process creation — its fields are evidence. + getString(ed, "image", &out, "Image") + getString(ed, "parentImage", &out, "ParentImage") + getString(ed, "commandLine", &out, "CommandLine") + getString(ed, "user", &out, "User") + getString(ed, "parentUser", &out, "User") + getString(ed, "newProcessName", &out, "Image") // Security 4688 + getString(ed, "parentProcessName", &out, "ParentImage") + getString(ed, "subjectUserName", &out, "User") + highFidelity = true + } else { + // Non-creation event: extract for context only, LOW fidelity, + // so process_creation rules can never match it. + getString(ed, "image", &out, "Image") + getString(ed, "callerProcessName", &out, "Image") + getString(ed, "processName", &out, "Image") + getString(ed, "commandLine", &out, "CommandLine") + getString(ed, "user", &out, "User") + getString(ed, "subjectUserName", &out, "User") + if out["Image"] != "" || out["CommandLine"] != "" { + lowFidelity = true + } + } + } + } + } + + // Try Linux-Sysmon (eBPF) paths. + // + // Sysmon for Linux emits the same process-creation semantics as its Windows + // counterpart — EventID 1 with Image, CommandLine, ParentImage, User — but it + // is NOT an eventchannel event, so it gets its own path rather than being + // smuggled through data.win. The same gate applies: only EventID 1 is + // process creation, and only process creation is high fidelity. + if data, ok := event["data"].(map[string]any); ok { + if sm, ok := data["sysmon"].(map[string]any); ok { + eventID := anyToString(sm["eventID"]) + if eventID != "" { + out["EventID"] = eventID + } + if eventID == "1" { + getString(sm, "image", &out, "Image") + getString(sm, "parentImage", &out, "ParentImage") + getString(sm, "commandLine", &out, "CommandLine") + getString(sm, "user", &out, "User") highFidelity = true + } else { + // Any other Sysmon event (network, file, …) carries a process + // name without being proof a process was created. + getString(sm, "image", &out, "Image") + getString(sm, "commandLine", &out, "CommandLine") + getString(sm, "user", &out, "User") + if out["Image"] != "" || out["CommandLine"] != "" { + lowFidelity = true + } } } } @@ -135,6 +191,50 @@ func (Normalizer) Normalize(raw json.RawMessage) map[string]string { return out } +// isProcessCreation reports whether a Windows eventdata block is genuine +// process-creation telemetry: Sysmon EventID 1 / Security 4688, or — when the +// structured EventID is absent (as in some captured events) — the +// process-creation field signature (a command line plus an image/new-process +// name). Enumeration/network events (callerProcessName only, no command line) +// are correctly excluded. +func isProcessCreation(ed map[string]any, eventID string) bool { + if eventID == "1" || eventID == "4688" { + return true + } + return nonEmptyStr(ed, "commandLine") && (nonEmptyStr(ed, "image") || nonEmptyStr(ed, "newProcessName")) +} + +func nonEmptyStr(m map[string]any, key string) bool { + s, ok := m[key].(string) + return ok && s != "" +} + +// anyToString renders a JSON value that may be a string or a number. +func anyToString(v any) string { + switch t := v.(type) { + case string: + return t + case float64: + return fmt.Sprintf("%d", int(t)) + } + return "" +} + +// winEventID extracts data.win.system.eventID (string or numeric) or "". +func winEventID(win map[string]any) string { + sys, ok := win["system"].(map[string]any) + if !ok { + return "" + } + switch v := sys["eventID"].(type) { + case string: + return v + case float64: + return fmt.Sprintf("%d", int(v)) + } + return "" +} + func getString(m map[string]any, key string, out *map[string]string, target string) { if v, ok := m[key].(string); ok && (*out)[target] == "" { (*out)[target] = v diff --git a/internal/evaluator/rule.go b/internal/evaluator/rule.go index 0d35b4b..78c75af 100644 --- a/internal/evaluator/rule.go +++ b/internal/evaluator/rule.go @@ -12,11 +12,12 @@ import ( // Rule represents a parsed Sigma rule's detection block. type Rule struct { - Path string - Title string - Category string // logsource.category, e.g. "process_creation" - Detections map[string]FieldMap // search-identifier → field conditions - Condition Expr // parsed condition tree + Path string + Title string + Category string // logsource.category, e.g. "process_creation" + Detections map[string]FieldMap // search-identifier → field conditions + Condition Expr // parsed condition tree + Unsupported string // non-empty if the rule uses a feature we can't evaluate } // FieldMap is a search-identifier's field→value mapping with modifiers. @@ -26,6 +27,16 @@ type FieldMap map[string]FieldEntry type FieldEntry struct { Values []string // list of values to match against Modifiers []string // e.g. "contains", "startswith", "endswith", "all" + MatchNull bool // Sigma `Field: null` — matches when the field is absent/empty +} + +// supportedModifiers is the set of value modifiers this engine evaluates +// correctly. Any other modifier (base64, cidr, windash, fieldref, …) means the +// rule cannot be faithfully evaluated; such a rule is reported INCONCLUSIVE +// rather than silently scored as a clean MISS. +var supportedModifiers = map[string]bool{ + "all": true, "endswith": true, "startswith": true, "contains": true, + "re": true, "lt": true, "lte": true, "gt": true, "gte": true, } // Expr is a node in the condition expression tree. @@ -97,6 +108,18 @@ func (RuleParser) Parse(path string) (*Rule, error) { rule.Detections[name] = fm } + // Flag any unsupported modifier so the evaluator can report INCONCLUSIVE + // instead of a misleading MISS. + for _, fm := range rule.Detections { + for _, entry := range fm { + for _, m := range entry.Modifiers { + if !supportedModifiers[m] { + rule.Unsupported = "unsupported Sigma modifier: " + m + } + } + } + } + // Parse condition expression cond, err := parseCondition(raw.Detection.Condition) if err != nil { @@ -156,6 +179,8 @@ func parseFieldMap(val any) (FieldMap, error) { // Parse value(s). Scalars may be strings, numbers, or booleans // (e.g. `EventID: 4688`, `Count|gt: 5`); coerce each to its string form. switch vv := v.(type) { + case nil: + entry.MatchNull = true // Sigma `Field: null` — match when absent/empty case string: entry.Values = []string{vv} case []any: @@ -259,8 +284,10 @@ func (p *condParser) parseAnd() (Expr, error) { func (p *condParser) parseNot() (Expr, error) { p.skip() - if strings.HasPrefix(p.s[p.pos:], "not ") { - p.pos += 4 + rest := p.s[p.pos:] + // Accept "not " and also "not(" (no space before the paren). + if strings.HasPrefix(rest, "not ") || strings.HasPrefix(rest, "not\t") || strings.HasPrefix(rest, "not(") { + p.pos += 3 // consume "not"; the recursive skip() handles the space or "(" child, err := p.parseNot() if err != nil { return nil, err @@ -333,43 +360,33 @@ func (p *condParser) parseOneOf() (Expr, error) { } // Read identifiers var names []string - for { - p.skip() - if strings.HasPrefix(p.s[p.pos:], "them") { - p.pos += 4 - // "1 of them" — we don't know the list, pass through - break - } - if strings.HasPrefix(p.s[p.pos:], "(") { - // "1 of (a, b, c, ...)" - p.pos++ // skip ( - for { - id, err := p.parseIdent() - if err != nil { - return nil, err - } - if id != nil { - names = append(names, id.(IdentExpr).Name) - } - p.skip() - if p.pos < len(p.s) && p.s[p.pos] == ',' { - p.pos++ - continue - } - if p.pos < len(p.s) && p.s[p.pos] == ')' { - p.pos++ - break - } - break + p.skip() + switch { + case strings.HasPrefix(p.s[p.pos:], "them"): + p.pos += 4 // "1 of them" — Names stays empty; the matcher expands it + case p.pos < len(p.s) && p.s[p.pos] == '(': + p.pos++ // skip ( + for { + if name := p.readAggIdent(); name != "" { + names = append(names, name) + } + p.skip() + if p.pos < len(p.s) && p.s[p.pos] == ',' { + p.pos++ + continue + } + if p.pos < len(p.s) && p.s[p.pos] == ')' { + p.pos++ } break } - // "1 of them" without parens — just "them" - if strings.HasPrefix(p.s[p.pos:], "them") { - p.pos += 4 - break + default: + // Bare identifier or glob pattern, e.g. "1 of selection_*". Consuming it + // here (rather than stopping at '*') is what lets the following + // "and not filter" parse instead of being silently dropped. + if name := p.readAggIdent(); name != "" { + names = append(names, name) } - break } return OneOfExpr{N: n, Names: names}, nil } @@ -383,12 +400,8 @@ func (p *condParser) parseAllOf() (Expr, error) { return AllOfExpr{Names: names}, nil } for { - id, err := p.parseIdent() - if err != nil { - return nil, err - } - if id != nil { - names = append(names, id.(IdentExpr).Name) + if name := p.readAggIdent(); name != "" { + names = append(names, name) } p.skip() if p.pos < len(p.s) && p.s[p.pos] == ',' { @@ -403,3 +416,14 @@ func (p *condParser) parseAllOf() (Expr, error) { func isIdent(c byte) bool { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' } + +// readAggIdent reads an aggregate operand identifier, allowing the `*`/`?` +// glob characters so patterns like "selection_*" are captured whole. +func (p *condParser) readAggIdent() string { + p.skip() + start := p.pos + for p.pos < len(p.s) && (isIdent(p.s[p.pos]) || p.s[p.pos] == '*' || p.s[p.pos] == '?') { + p.pos++ + } + return p.s[start:p.pos] +} diff --git a/internal/evaluator/sigma.go b/internal/evaluator/sigma.go index 8f63284..b8e11b4 100644 --- a/internal/evaluator/sigma.go +++ b/internal/evaluator/sigma.go @@ -48,6 +48,12 @@ func (e RuleMatcherEvaluator) Evaluate(rule model.SigmaRule, events []model.Even return model.Errored, nil, fmt.Errorf("parse rule %s: %w", rule.Path, err) } + // A rule using a feature we cannot faithfully evaluate must NOT be scored as + // a clean MISS — that silently under-reports coverage. Report INCONCLUSIVE. + if parsedRule.Unsupported != "" { + return model.Inconclusive, nil, nil + } + // For process-creation rules, only genuine process-creation telemetry can // justify a verdict. Command-output/metadata scrapes (full_log, decoder // name) are tagged low fidelity by the normalizer and are excluded here, so diff --git a/internal/evaluator/sysmon_linux_test.go b/internal/evaluator/sysmon_linux_test.go new file mode 100644 index 0000000..bc485e5 --- /dev/null +++ b/internal/evaluator/sysmon_linux_test.go @@ -0,0 +1,91 @@ +package evaluator + +import ( + "encoding/json" + "testing" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +func normalizeRaw(t *testing.T, raw string) map[string]string { + t.Helper() + return Normalizer{}.Normalize(json.RawMessage(raw)) +} + +// Sysmon for Linux EventID 1 is genuine process creation and must satisfy a +// process_creation rule. +func TestLinuxSysmonProcessCreateIsHighFidelity(t *testing.T) { + got := normalizeRaw(t, `{"data":{"sysmon":{ + "eventID":"1","image":"/usr/bin/uname","commandLine":"uname -a", + "parentImage":"/bin/bash","user":"root"}}}`) + + if got["Image"] != "/usr/bin/uname" { + t.Errorf("Image = %q", got["Image"]) + } + if got["CommandLine"] != "uname -a" { + t.Errorf("CommandLine = %q", got["CommandLine"]) + } + if got["ParentImage"] != "/bin/bash" { + t.Errorf("ParentImage = %q", got["ParentImage"]) + } + if got[FidelityKey] != FidelityProcess { + t.Errorf("fidelity = %q, want %q", got[FidelityKey], FidelityProcess) + } +} + +// The eventID may arrive as a JSON number rather than a string. +func TestLinuxSysmonNumericEventID(t *testing.T) { + got := normalizeRaw(t, `{"data":{"sysmon":{ + "eventID":1,"image":"/usr/bin/id","commandLine":"id"}}}`) + if got[FidelityKey] != FidelityProcess { + t.Errorf("numeric eventID not recognised: fidelity = %q", got[FidelityKey]) + } + if got["EventID"] != "1" { + t.Errorf("EventID = %q, want \"1\"", got["EventID"]) + } +} + +// A non-creation Sysmon event carries a process name without being proof a +// process was created — the same trap that produced a false DETECTED off a +// Windows 4798 enumeration event. +func TestLinuxSysmonNonCreationIsLowFidelity(t *testing.T) { + got := normalizeRaw(t, `{"data":{"sysmon":{ + "eventID":"3","image":"/usr/bin/curl","commandLine":"curl http://x"}}}`) + + if got[FidelityKey] == FidelityProcess { + t.Error("a Sysmon network event must not be tagged as process creation") + } + if got[FidelityKey] != FidelityLog { + t.Errorf("fidelity = %q, want %q", got[FidelityKey], FidelityLog) + } +} + +// End to end: a Linux-Sysmon event must be able to satisfy a real shipped rule, +// and a non-creation event with the same fields must not. +func TestLinuxSysmonSatisfiesShippedRule(t *testing.T) { + e := RuleMatcherEvaluator{RulesDir: "../../detections/linux"} + rule := model.SigmaRule{Path: "../../detections/linux/T1082.yml", Title: "T1082"} + + creation := []model.Event{{Raw: json.RawMessage(`{"data":{"sysmon":{ + "eventID":"1","image":"/usr/bin/uname","commandLine":"uname -a","user":"root"}}}`)}} + verdict, evidence, err := e.Evaluate(rule, creation) + if err != nil { + t.Fatal(err) + } + if verdict != model.Detected { + t.Errorf("verdict = %s, want DETECTED", verdict) + } + if len(evidence) == 0 { + t.Error("a detection must carry evidence") + } + + nonCreation := []model.Event{{Raw: json.RawMessage(`{"data":{"sysmon":{ + "eventID":"3","image":"/usr/bin/uname","commandLine":"uname -a","user":"root"}}}`)}} + verdict, _, err = e.Evaluate(rule, nonCreation) + if err != nil { + t.Fatal(err) + } + if verdict == model.Detected { + t.Error("a non-creation event must never satisfy a process_creation rule") + } +} diff --git a/internal/executor/ssh.go b/internal/executor/ssh.go index 2b35a6c..9970b25 100644 --- a/internal/executor/ssh.go +++ b/internal/executor/ssh.go @@ -12,7 +12,7 @@ import ( // SSHExecutor runs atomics via SSH on remote targets (Windows VM, etc). // Uses sshpass for password auth in lab environments. type SSHExecutor struct { - Host string // e.g. 192.168.88.13 + Host string // victim address; from WINDOWS_SSH_HOST via lab/targets.yml User string // e.g. windows-vm Password string } diff --git a/internal/feed/static.go b/internal/feed/static.go index 02cd454..a4badfd 100644 --- a/internal/feed/static.go +++ b/internal/feed/static.go @@ -27,6 +27,7 @@ type PlanTechnique struct { // StaticFeed yields tasks from a YAML plan file. type StaticFeed struct { + Name string Tasks []model.TechniqueTask } @@ -40,6 +41,7 @@ func (f *StaticFeed) Load(path string) error { if err := yaml.Unmarshal(data, &plan); err != nil { return fmt.Errorf("parse plan: %w", err) } + f.Name = plan.Name f.Tasks = make([]model.TechniqueTask, 0, len(plan.Techniques)) for i, t := range plan.Techniques { f.Tasks = append(f.Tasks, model.TechniqueTask{ diff --git a/internal/lab/targets.go b/internal/lab/targets.go new file mode 100644 index 0000000..9109b8d --- /dev/null +++ b/internal/lab/targets.go @@ -0,0 +1,107 @@ +// Package lab describes the hosts a campaign can dispatch techniques to. +// +// The engine previously hardcoded one Linux target in the run path, so a +// Windows rule could never be exercised by `run` even though the executor and +// the rule both existed. Targets are configuration now, and credentials are +// read from the environment rather than compiled into the binary. +package lab + +import ( + "fmt" + "os" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +// Target is one lab host and how to reach it. +type Target struct { + Name string `yaml:"name"` + Kind string `yaml:"kind"` + Executor string `yaml:"executor"` // "docker" | "ssh" + Container string `yaml:"container"` + RulesDir string `yaml:"rules_dir"` + + SSHHostEnv string `yaml:"ssh_host_env"` + SSHUserEnv string `yaml:"ssh_user_env"` + SSHPassEnv string `yaml:"ssh_pass_env"` + DefaultHost string `yaml:"default_host"` + DefaultUser string `yaml:"default_user"` +} + +// Targets is the lab inventory. +type Targets struct { + Targets []Target `yaml:"targets"` +} + +// Load reads the lab target inventory. +func Load(path string) (*Targets, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read targets: %w", err) + } + var t Targets + if err := yaml.Unmarshal(data, &t); err != nil { + return nil, fmt.Errorf("parse targets %s: %w", path, err) + } + if len(t.Targets) == 0 { + return nil, fmt.Errorf("targets %s declares no hosts", path) + } + for i, tt := range t.Targets { + if tt.Name == "" || tt.Kind == "" || tt.Executor == "" { + return nil, fmt.Errorf("target %d needs name, kind and executor", i) + } + } + return &t, nil +} + +// ForPlatform returns the target that runs a given platform's techniques. +func (t *Targets) ForPlatform(kind string) (Target, bool) { + for _, tt := range t.Targets { + if strings.EqualFold(tt.Kind, kind) { + return tt, true + } + } + return Target{}, false +} + +// Kinds lists the distinct platforms the inventory can reach. +func (t *Targets) Kinds() []string { + seen := map[string]bool{} + var out []string + for _, tt := range t.Targets { + k := strings.ToLower(tt.Kind) + if !seen[k] { + seen[k] = true + out = append(out, k) + } + } + return out +} + +// Model converts to the engine's target type. +func (t Target) Model() model.Target { + return model.Target{Host: t.Name, Kind: t.Kind} +} + +// SSHConfig resolves connection details from the environment, falling back to +// the declared defaults. A password is never defaulted. +func (t Target) SSHConfig() (host, user, pass string) { + host = envOr(t.SSHHostEnv, t.DefaultHost) + user = envOr(t.SSHUserEnv, t.DefaultUser) + if t.SSHPassEnv != "" { + pass = os.Getenv(t.SSHPassEnv) + } + return host, user, pass +} + +func envOr(key, def string) string { + if key != "" { + if v := os.Getenv(key); v != "" { + return v + } + } + return def +} diff --git a/internal/lab/targets_test.go b/internal/lab/targets_test.go new file mode 100644 index 0000000..0f066f8 --- /dev/null +++ b/internal/lab/targets_test.go @@ -0,0 +1,78 @@ +package lab + +import ( + "testing" +) + +const shipped = "../../lab/targets.yml" + +func TestShippedInventoryCoversBothPlatforms(t *testing.T) { + ts, err := Load(shipped) + if err != nil { + t.Fatalf("load: %v", err) + } + for _, kind := range []string{"linux", "windows"} { + tgt, ok := ts.ForPlatform(kind) + if !ok { + t.Errorf("no %s target declared — the cross-platform claim needs one", kind) + continue + } + if tgt.RulesDir == "" { + t.Errorf("%s target declares no rules_dir", kind) + } + } +} + +// Credentials must come from the environment. The Windows host used to be a +// literal in cmd/purpleloop/main.go. +func TestSSHConfigPrefersEnvironment(t *testing.T) { + ts, err := Load(shipped) + if err != nil { + t.Fatal(err) + } + win, ok := ts.ForPlatform("windows") + if !ok { + t.Fatal("no windows target") + } + if win.SSHHostEnv == "" || win.SSHPassEnv == "" { + t.Fatal("the windows target must read host and password from the environment") + } + + t.Setenv(win.SSHHostEnv, "10.0.0.5") + t.Setenv(win.SSHUserEnv, "lab-user") + t.Setenv(win.SSHPassEnv, "from-env") + + host, user, pass := win.SSHConfig() + if host != "10.0.0.5" || user != "lab-user" || pass != "from-env" { + t.Errorf("SSHConfig = (%s,%s,%s), want the environment values", host, user, pass) + } +} + +// A password must never come from a default — only from the environment. +func TestSSHPasswordIsNeverDefaulted(t *testing.T) { + ts, err := Load(shipped) + if err != nil { + t.Fatal(err) + } + win, _ := ts.ForPlatform("windows") + t.Setenv(win.SSHPassEnv, "") + if _, _, pass := win.SSHConfig(); pass != "" { + t.Errorf("password = %q with an empty env var; it must not fall back to a literal", pass) + } +} + +func TestForPlatformUnknownKind(t *testing.T) { + ts, err := Load(shipped) + if err != nil { + t.Fatal(err) + } + if _, ok := ts.ForPlatform("macos"); ok { + t.Error("an undeclared platform must not resolve to a target") + } +} + +func TestLoadRejectsMissingFile(t *testing.T) { + if _, err := Load("nope.yml"); err == nil { + t.Error("a missing inventory must error") + } +} diff --git a/internal/mapping/mapping.go b/internal/mapping/mapping.go index f782a4a..ab331a7 100644 --- a/internal/mapping/mapping.go +++ b/internal/mapping/mapping.go @@ -38,3 +38,74 @@ func (m CVEMap) Resolve(cve string) (techniques []string, techniqueAtomics map[s } return entry.Techniques, entry.Atomics, nil } + +// ---- technique -> atomic -> expected detections ---- + +// TechniqueEntry declares which atomic exercises a technique and which Sigma +// rules are expected to fire for it. expected_rules is a list, not one path: +// a technique may legitimately be caught by any of several rules, and asserting +// a single one understates coverage. +type TechniqueEntry struct { + Atomics []string `json:"atomics"` + Platform string `json:"platform"` + ExpectedRules []string `json:"expected_rules"` +} + +// RuleMap is the single source of truth for what a campaign should run and what +// should catch it. It replaces the technique->rule map that was hardcoded in +// cmd/purpleloop, so the mapping is data the repository can lint rather than +// code nobody can inspect. +type RuleMap struct { + Techniques map[string]TechniqueEntry `json:"techniques"` + UntestedRules map[string]string `json:"untested_rules"` +} + +// LoadRuleMap reads the technique/rule mapping file. +func LoadRuleMap(path string) (*RuleMap, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read rule map: %w", err) + } + var m RuleMap + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("parse rule map %s: %w", path, err) + } + if len(m.Techniques) == 0 { + return nil, fmt.Errorf("rule map %s declares no techniques", path) + } + return &m, nil +} + +// Entry returns the mapping for a technique. +func (m *RuleMap) Entry(technique string) (TechniqueEntry, bool) { + e, ok := m.Techniques[technique] + return e, ok +} + +// ExpectedRules returns the rules that should fire for a technique. +func (m *RuleMap) ExpectedRules(technique string) []string { + if e, ok := m.Techniques[technique]; ok { + return e.ExpectedRules + } + return nil +} + +// AtomicFor returns the mapped atomic ID for a technique, if one is declared. +func (m *RuleMap) AtomicFor(technique string) string { + if e, ok := m.Techniques[technique]; ok && len(e.Atomics) > 0 { + return e.Atomics[0] + } + return "" +} + +// MappedRules returns every rule path referenced by any technique, so a lint +// can tell which rules on disk are never exercised. +func (m *RuleMap) MappedRules() map[string]bool { + out := map[string]bool{} + for _, e := range m.Techniques { + for _, r := range e.ExpectedRules { + out[r] = true + } + } + return out +} diff --git a/internal/mapping/rulemap_test.go b/internal/mapping/rulemap_test.go new file mode 100644 index 0000000..aac9ab7 --- /dev/null +++ b/internal/mapping/rulemap_test.go @@ -0,0 +1,105 @@ +package mapping + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +const ( + ruleMapPath = "../../mappings/attack_rule_map.json" + detectionsDir = "../../detections" +) + +func loadShipped(t *testing.T) *RuleMap { + t.Helper() + m, err := LoadRuleMap(ruleMapPath) + if err != nil { + t.Fatalf("load rule map: %v", err) + } + return m +} + +func TestRuleMapEntriesAreComplete(t *testing.T) { + m := loadShipped(t) + for id, e := range m.Techniques { + if len(e.Atomics) == 0 { + t.Errorf("%s declares no atomic", id) + } + if len(e.ExpectedRules) == 0 { + t.Errorf("%s declares no expected rules", id) + } + if e.Platform == "" { + t.Errorf("%s declares no platform", id) + } + } +} + +// Every rule a technique expects must exist. A mapping pointing at a deleted +// rule would silently report MISSED forever. +func TestExpectedRulesExistOnDisk(t *testing.T) { + m := loadShipped(t) + for id, e := range m.Techniques { + for _, r := range e.ExpectedRules { + path := filepath.Join("../..", r) + if _, err := os.Stat(path); err != nil { + t.Errorf("%s expects %s, which does not exist", id, r) + } + } + } +} + +// The untested-rule lint: every Sigma rule in the repository is either mapped +// to a technique or explicitly declared untested WITH a reason. Without this a +// rule can be added, never exercised, and still be counted as coverage the +// project does not actually have. +func TestEveryRuleIsMappedOrDeclaredUntested(t *testing.T) { + m := loadShipped(t) + mapped := m.MappedRules() + + err := filepath.Walk(detectionsDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || !strings.HasSuffix(path, ".yml") { + return nil + } + if strings.Contains(path, string(filepath.Separator)+"tests"+string(filepath.Separator)) { + return nil // fixtures, not rules + } + rel := strings.TrimPrefix(filepath.ToSlash(path), "../../") + + if mapped[rel] { + return nil + } + reason, declared := m.UntestedRules[rel] + if !declared { + t.Errorf("rule %s is neither mapped to a technique nor declared in untested_rules — "+ + "an unexercised rule must not be silently counted as coverage", rel) + return nil + } + if strings.TrimSpace(reason) == "" { + t.Errorf("rule %s is declared untested with no reason", rel) + } + return nil + }) + if err != nil { + t.Fatalf("walk detections: %v", err) + } +} + +// The reverse guard: untested_rules must not name a rule that is also mapped, +// or a rule that no longer exists. +func TestUntestedRulesAreConsistent(t *testing.T) { + m := loadShipped(t) + mapped := m.MappedRules() + for rule := range m.UntestedRules { + if mapped[rule] { + t.Errorf("%s is listed as untested but is also mapped to a technique", rule) + } + if _, err := os.Stat(filepath.Join("../..", rule)); err != nil { + t.Errorf("%s is listed as untested but does not exist", rule) + } + } +} diff --git a/internal/model/model.go b/internal/model/model.go index 5553fbc..e1fbc28 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -18,12 +18,42 @@ const ( NoTelemetry Verdict = "NO_TELEMETRY" Inconclusive Verdict = "INCONCLUSIVE" Errored Verdict = "ERROR" + + // SkippedPrereq means the atomic's own prerequisites were not met, so it + // never ran. That is not a detection gap and must never be counted as one — + // the same class of error as crediting a low-fidelity event as evidence. + SkippedPrereq Verdict = "SKIPPED_PREREQ" +) + +// Attribution states how tightly a verdict is bound to the execution that +// claims it. A DETECTED whose window overlapped another technique's run is not +// the same quality of evidence as one that did not, and a report that does not +// say so overstates its own rigour. +type Attribution string + +const ( + // WindowAndHostScoped is the strongest: the telemetry query was bounded to + // this execution's window AND filtered to this target host. + WindowAndHostScoped Attribution = "window_and_host_scoped" + // WindowScoped means time-bounded, but the host was not resolved. + WindowScoped Attribution = "window_scoped" + // WindowOverlap means another technique in this campaign ran inside this + // one's collection window, so a matching event may belong to either. + WindowOverlap Attribution = "window_overlap" + // Unscoped means the verdict cannot be attributed — a synthetic pipeline. + Unscoped Attribution = "unscoped" ) // TimeWindow bounds a telemetry query to an execution's real time span. type TimeWindow struct { - Start time.Time - End time.Time + Start time.Time `json:"start"` + End time.Time `json:"end"` +} + +// Overlaps reports whether two collection windows intersect. Two techniques +// whose windows overlap can each collect the other's events. +func (w TimeWindow) Overlaps(o TimeWindow) bool { + return w.Start.Before(o.End) && o.Start.Before(w.End) } // Target is a host inside the isolated lab network. Attacks NEVER target @@ -34,12 +64,21 @@ type Target struct { } // AtomicTest is a single Atomic Red Team test mapped to an ATT&CK technique. +// +// Name/GUID/Source exist so a report can prove WHICH upstream atomic produced a +// command. Source is "atomic-red-team" for a vendored test and "override" for a +// lab substitution, in which case OverrideReason says why — a substituted +// command must never read as the upstream atomic. type AtomicTest struct { ID string `json:"id"` TechniqueID string `json:"technique_id"` + Name string `json:"name,omitempty"` + GUID string `json:"guid,omitempty"` Command string `json:"command"` CleanupCommand string `json:"cleanup_command,omitempty"` Executor string `json:"executor"` + Source string `json:"source,omitempty"` + OverrideReason string `json:"override_reason,omitempty"` } // RunResult is what the Executor records about one atomic execution. @@ -52,10 +91,13 @@ type RunResult struct { Stderr string `json:"-"` } -// Window returns the execution's time span, padded so the SIEM has room to -// have ingested the resulting events. +// Window returns the execution's time span, padded only on the END so the SIEM +// has room to ingest the resulting events. The start is NOT padded backward: +// widening it into the past can capture an earlier run's matching events and +// credit them to this execution (false DETECTED). A small epsilon absorbs +// minor clock skew without reaching a prior test. func (r RunResult) Window(pad time.Duration) TimeWindow { - return TimeWindow{Start: r.StartedAt.Add(-pad), End: r.FinishedAt.Add(pad)} + return TimeWindow{Start: r.StartedAt.Add(-2 * time.Second), End: r.FinishedAt.Add(pad)} } // Event is one normalised telemetry record pulled from the SIEM. @@ -91,12 +133,62 @@ type ProofChain struct { RuleMatched string `json:"rule_matched"` Verdict Verdict `json:"verdict"` Evidence []Event `json:"evidence"` + + // Note carries the reason behind a non-coverage verdict — why a technique + // ERRORed, which prerequisite was unmet, which platform mismatched. An + // unexplained gap is not actionable. + Note string `json:"note,omitempty"` + + // RulesExpected lists every rule that should have fired; RulesMatched lists + // those that did. RuleMatched above stays as the first match, preserving the + // documented proof-chain shape. + RulesExpected []string `json:"rules_expected,omitempty"` + RulesMatched []string `json:"rules_matched,omitempty"` + + // Attribution states how tightly this verdict is bound to its execution. + Attribution Attribution `json:"attribution,omitempty"` + + // Window is the telemetry query span, kept so overlapping executions can be + // detected across the campaign. + Window TimeWindow `json:"window"` + + // DetectLatencyMS is the gap between execution start and the earliest + // matching event. Zero when nothing matched. + DetectLatencyMS int64 `json:"detect_latency_ms,omitempty"` + + // Collected is every event the query returned, not just the matching ones. + // It is excluded from the proof-chain JSON (that contract carries evidence, + // which is the matching subset) and exists so a run can be captured and + // replayed against future rule changes. + Collected []Event `json:"-"` } // CampaignResult aggregates every technique validated in one run. +// +// The three trust fields enforce the hard contracts in code (not just docs): +// - Synthetic: any pipeline stage was dry/synthetic, so the chains are NOT +// real telemetry and must never be presented or published as evidence. +// - CanaryHealthy / Inconclusive: the pipeline positive control. On a real +// run, if the canary did not fire the run is INCONCLUSIVE and its coverage +// must not be treated as valid. type CampaignResult struct { - StartedAt time.Time `json:"started_at"` - Chains []ProofChain `json:"chains"` + StartedAt time.Time `json:"started_at"` + Chains []ProofChain `json:"chains"` + Synthetic bool `json:"synthetic"` // dry/synthetic pipeline — not real evidence + CanaryHealthy bool `json:"canary_healthy"` // positive control fired (real runs) + Inconclusive bool `json:"inconclusive"` // canary failed on a real run → coverage invalid + CanaryDetail string `json:"canary_detail,omitempty"` // why the canary failed + + // Campaign names the plan that ran; AtomicsCommit records the vendored + // Atomic Red Team revision, so a report states exactly which upstream + // commands produced it. + Campaign string `json:"campaign,omitempty"` + AtomicsCommit string `json:"atomics_commit,omitempty"` + + // UntestedRules names detections this campaign did not exercise, with the + // reason. An honest denominator needs the rules that were never tried, not + // only the ones that were. + UntestedRules map[string]string `json:"untested_rules,omitempty"` } // ---- interfaces: the swappable seams of the system ---- diff --git a/internal/precision/precision.go b/internal/precision/precision.go new file mode 100644 index 0000000..7f783ec --- /dev/null +++ b/internal/precision/precision.go @@ -0,0 +1,143 @@ +// Package precision measures false positives: whether detections stay silent +// during ordinary administration. +// +// Coverage without a false-positive rate is half a metric. A rule that matches +// everything scores perfect coverage and is worthless. The repository's +// negative fixtures prove a rule rejects an event crafted to be rejected, which +// is a claim about the author's intent; this measures the rule against a +// workload written independently of it. +package precision + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +// Command is one benign action and the process-creation event it produces. +type Command struct { + Category string `yaml:"category"` + Command string `yaml:"command"` + Image string `yaml:"image"` + CommandLine string `yaml:"command_line"` +} + +// Baseline is the benign workload. +type Baseline struct { + Name string `yaml:"name"` + Description string `yaml:"description"` + Commands []Command `yaml:"commands"` +} + +// LoadBaseline reads the benign workload definition. +func LoadBaseline(path string) (*Baseline, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read baseline: %w", err) + } + var b Baseline + if err := yaml.Unmarshal(data, &b); err != nil { + return nil, fmt.Errorf("parse baseline %s: %w", path, err) + } + if len(b.Commands) == 0 { + return nil, fmt.Errorf("baseline %s declares no commands", path) + } + for i, c := range b.Commands { + if c.Image == "" || c.CommandLine == "" { + return nil, fmt.Errorf("baseline entry %d (%q) needs both image and command_line", i, c.Command) + } + } + return &b, nil +} + +// Event renders a benign command as the process-creation event it would emit. +func (c Command) Event() model.Event { + raw, _ := json.Marshal(map[string]string{ + "Image": c.Image, + "ParentImage": "/bin/bash", + "CommandLine": c.CommandLine, + "User": "root", + }) + return model.Event{ID: "benign-" + c.Image, Raw: raw} +} + +// FalsePositive is one rule firing on one benign command. +type FalsePositive struct { + Rule string `json:"rule"` + Command string `json:"command"` + Category string `json:"category"` +} + +// Result is the outcome of a precision run. +type Result struct { + Baseline string `json:"baseline"` + CommandsRun int `json:"commands_run"` + RulesChecked int `json:"rules_checked"` + FalsePositives []FalsePositive `json:"false_positives"` + // PrecisionPct is the share of benign commands that fired no rule at all. + PrecisionPct int `json:"precision_pct"` +} + +// Measure evaluates every rule against every benign command. A rule that fires +// is a false positive named with the command that triggered it, so the finding +// is actionable rather than a bare number. +func Measure(eval model.Evaluator, rulesDir string, b *Baseline) (*Result, error) { + rules, err := listRules(rulesDir) + if err != nil { + return nil, err + } + + res := &Result{Baseline: b.Name, CommandsRun: len(b.Commands), RulesChecked: len(rules)} + noisy := map[string]bool{} + + for _, c := range b.Commands { + events := []model.Event{c.Event()} + for _, rule := range rules { + verdict, _, err := eval.Evaluate(model.SigmaRule{Path: rule, Title: rule}, events) + if err != nil { + return nil, fmt.Errorf("evaluate %s: %w", rule, err) + } + if verdict == model.Detected { + res.FalsePositives = append(res.FalsePositives, FalsePositive{ + Rule: rule, Command: c.Command, Category: c.Category, + }) + noisy[c.Command] = true + } + } + } + + clean := len(b.Commands) - len(noisy) + if len(b.Commands) > 0 { + res.PrecisionPct = clean * 100 / len(b.Commands) + } + sort.Slice(res.FalsePositives, func(i, j int) bool { + if res.FalsePositives[i].Rule != res.FalsePositives[j].Rule { + return res.FalsePositives[i].Rule < res.FalsePositives[j].Rule + } + return res.FalsePositives[i].Command < res.FalsePositives[j].Command + }) + return res, nil +} + +func listRules(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("read rules dir: %w", err) + } + var out []string + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".yml") { + continue + } + out = append(out, filepath.Join(dir, e.Name())) + } + sort.Strings(out) + return out, nil +} diff --git a/internal/precision/precision_test.go b/internal/precision/precision_test.go new file mode 100644 index 0000000..fe7e71e --- /dev/null +++ b/internal/precision/precision_test.go @@ -0,0 +1,66 @@ +package precision + +import ( + "testing" + + "github.com/jayelbotvibe-web/purple-loop/internal/evaluator" +) + +const ( + baselinePath = "../../emulation/benign-baseline.yml" + rulesDir = "../../detections/linux" +) + +// The precision gate. Every shipped rule must stay silent during ordinary +// administration, including the adjacent cases — benign uses of the very +// binaries the detections watch. A rule keyed on a tool name rather than on +// what the tool is being asked to do fires here and fails the build. +// +// This found two real false positives when it was introduced: T1087.001 fired +// on `getent hosts` (a DNS lookup, not account enumeration) and T1135 fired on +// a bare `mount`. Both rules were narrowed. +func TestRulesAreSilentOnBenignWorkload(t *testing.T) { + b, err := LoadBaseline(baselinePath) + if err != nil { + t.Fatalf("load baseline: %v", err) + } + res, err := Measure(evaluator.RuleMatcherEvaluator{RulesDir: rulesDir}, rulesDir, b) + if err != nil { + t.Fatalf("measure: %v", err) + } + for _, fp := range res.FalsePositives { + t.Errorf("false positive: %s fired on benign %q (%s)", fp.Rule, fp.Command, fp.Category) + } + if res.PrecisionPct != 100 { + t.Errorf("precision = %d%%, want 100%% (%d commands, %d rules)", + res.PrecisionPct, res.CommandsRun, res.RulesChecked) + } +} + +// A baseline with no adjacent cases proves nothing — it would pass against a +// rule set that matches only exact strings. Guard the guard. +func TestBaselineIncludesAdjacentCases(t *testing.T) { + b, err := LoadBaseline(baselinePath) + if err != nil { + t.Fatal(err) + } + adjacent := 0 + for _, c := range b.Commands { + if c.Category == "adjacent-benign" { + adjacent++ + } + } + if adjacent < 5 { + t.Errorf("baseline has %d adjacent-benign commands, want at least 5 — "+ + "without benign uses of the watched binaries this test is vacuous", adjacent) + } + if len(b.Commands) < 20 { + t.Errorf("baseline has %d commands, want at least 20", len(b.Commands)) + } +} + +func TestLoadBaselineRejectsIncomplete(t *testing.T) { + if _, err := LoadBaseline("does-not-exist.yml"); err == nil { + t.Error("a missing baseline must error") + } +} diff --git a/internal/report/dashboard.go b/internal/report/dashboard.go index 33a0f5c..fa5fbad 100644 --- a/internal/report/dashboard.go +++ b/internal/report/dashboard.go @@ -6,6 +6,7 @@ import ( "math" "os" "path/filepath" + "runtime/debug" "github.com/jayelbotvibe-web/purple-loop/internal/model" ) @@ -37,15 +38,30 @@ func (r DashboardReporter) Write(result model.CampaignResult) error { return err } + // Synthetic or inconclusive runs are NOT valid coverage: keep the per-run + // artifact above (marked, for the operator) but never append them to the + // history index or publish them to the public Pages snapshot. + if result.Synthetic || result.Inconclusive { + return nil + } + // History index s := d["summary"].(map[string]any) c := d["canary"].(map[string]any) + // Per-technique verdicts go into the index so a later run can be diffed + // against this one. A history of coverage percentages cannot answer the + // question that matters — WHICH technique regressed. + verdicts := map[string]string{} + for _, ch := range result.Chains { + verdicts[ch.TechniqueID] = string(ch.Verdict) + } entry := map[string]any{ "id": runID, "campaign": d["campaign"], "generated_at": d["generated_at"], "coverage_pct": s["coverage_pct"], "canary_healthy": c["healthy"], + "techniques": verdicts, } if err := appendHistory(dir, entry); err != nil { return err @@ -79,18 +95,28 @@ func appendHistory(dir string, entry map[string]any) error { } func buildCoverage(result model.CampaignResult) map[string]any { + campaign := result.Campaign + if campaign == "" { + campaign = "unnamed" + } d := map[string]any{ - "campaign": "discovery", - "generated_at": result.StartedAt.UTC().Format("2006-01-02T15:04:05Z"), - "build": "v1.3.0", + "campaign": campaign, + "generated_at": result.StartedAt.UTC().Format("2006-01-02T15:04:05Z"), + "build": buildVersion(), + "atomics_commit": result.AtomicsCommit, } - // Canary (per-platform from result if available, else default healthy) + // Canary — the REAL positive-control result for this run (never hardcoded). + // A synthetic or inconclusive run is not healthy: the dashboard keys its + // "inconclusive" banner off this flag. + canaryHealthy := result.CanaryHealthy && !result.Synthetic && !result.Inconclusive d["canary"] = map[string]any{ - "healthy": true, + "healthy": canaryHealthy, + "synthetic": result.Synthetic, + "inconclusive": result.Inconclusive, + "detail": result.CanaryDetail, "platforms": []map[string]any{ - {"name": "linux", "healthy": true}, - {"name": "windows", "healthy": true}, + {"name": "linux", "healthy": canaryHealthy}, }, } @@ -105,19 +131,28 @@ func buildCoverage(result model.CampaignResult) map[string]any { noTel := counts[model.NoTelemetry] inconclusive := counts[model.Inconclusive] - denom := total - inconclusive - noTel + // Coverage is measured only over techniques that actually exercised a + // detection. Anything that never ran — INCONCLUSIVE, NO_TELEMETRY, + // SKIPPED_PREREQ, ERROR — is excluded from the denominator rather than + // silently counted as a gap. + skipped := counts[model.SkippedPrereq] + errored := counts[model.Errored] + denom := total - inconclusive - noTel - skipped - errored pct := 0 if denom > 0 { pct = int(math.Round(float64(detected) / float64(denom) * 100)) } d["summary"] = map[string]any{ - "total": total, - "detected": detected, - "missed": missed, - "no_telemetry": noTel, - "inconclusive": inconclusive, - "coverage_pct": pct, + "total": total, + "detected": detected, + "missed": missed, + "no_telemetry": noTel, + "inconclusive": inconclusive, + "skipped_prereq": skipped, + "errored": errored, + "exercised": denom, + "coverage_pct": pct, } // Readiness (stub — arbiter campaign not wired here yet) @@ -133,8 +168,31 @@ func buildCoverage(result model.CampaignResult) map[string]any { "Defense Evasion", "Credential Access", "Discovery", "Command & Control"} d["tactics"] = tactics - // Untested (empty for now) - d["untested"] = map[string]int{} + // Collection gaps, for handoff to detection-decay. NO_TELEMETRY says a rule + // never saw the events it needs; it does not say why. detection-decay models + // the cause (source death, field drift), so name the technique and the rule + // whose fields went missing rather than leaving the operator to guess. + var gaps []map[string]any + for _, c := range result.Chains { + if c.Verdict != model.NoTelemetry { + continue + } + gaps = append(gaps, map[string]any{ + "technique_id": c.TechniqueID, + "rules_expected": c.RulesExpected, + "atomic": c.Atomic.ID, + "window": map[string]any{"start": c.Window.Start, "end": c.Window.End}, + "note": c.Note, + }) + } + d["collection_gaps"] = gaps + + // Untested rules, with the reason each was not exercised. + untested := result.UntestedRules + if untested == nil { + untested = map[string]string{} + } + d["untested"] = untested // Techniques var techs []map[string]any @@ -167,9 +225,19 @@ func buildCoverage(result model.CampaignResult) map[string]any { } else { t["evidence"] = "" } - // Gap for MISSED - if c.Verdict == model.Missed { - t["gap"] = map[string]string{"why": "", "next": ""} + // Provenance: which upstream atomic produced this command, and whether + // it was substituted for the lab. + t["attribution"] = string(c.Attribution) + t["detect_latency_ms"] = c.DetectLatencyMS + t["rules_expected"] = c.RulesExpected + t["rules_matched"] = c.RulesMatched + t["atomic_name"] = c.Atomic.Name + t["atomic_source"] = c.Atomic.Source + t["override_reason"] = c.Atomic.OverrideReason + + // Gap: why this technique did not produce coverage. + if c.Verdict != model.Detected { + t["gap"] = map[string]string{"why": c.Note, "next": ""} } else { t["gap"] = nil } @@ -197,4 +265,11 @@ var techniqueMeta = map[string]struct{ name, tactic string }{ // Ensure interface compliance at compile time. var _ model.Reporter = DashboardReporter{} -func init() { _ = fmt.Sprintf("") } // suppress unused import +// buildVersion reports the compiled version rather than a hardcoded string, so +// a report always names the binary that produced it. +func buildVersion() string { + if bi, ok := debug.ReadBuildInfo(); ok && bi.Main.Version != "" && bi.Main.Version != "(devel)" { + return bi.Main.Version + } + return "devel" +} diff --git a/internal/report/report.go b/internal/report/report.go index f918174..88d27e9 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -58,38 +58,83 @@ tr:hover{background:#1a1a2e} .summary span{font-size:1.2em}

Purple Loop — Coverage Report

-

` + html.EscapeString(run.StartedAt.Format("2006-01-02 15:04:05 UTC")) + `

-
`) +

` + html.EscapeString(run.StartedAt.Format("2006-01-02 15:04:05 UTC")) + `

`) - for _, v := range []model.Verdict{model.Detected, model.Missed, model.NoTelemetry, model.Inconclusive, model.Errored} { + // Trust banners — an operator must never mistake a synthetic or + // canary-failed report for verified coverage. + if run.Synthetic { + f.WriteString(`
⚠ SYNTHETIC / DRY-RUN PIPELINE — these results are NOT real telemetry and must not be used as evidence.
`) + } + if run.Inconclusive { + f.WriteString(`
⚠ INCONCLUSIVE — the pipeline positive control (canary) did not fire; coverage is not valid.
` + html.EscapeString(run.CanaryDetail) + `
`) + } + + f.WriteString(`
`) + + for _, v := range []model.Verdict{model.Detected, model.Missed, model.NoTelemetry, model.Inconclusive, model.SkippedPrereq, model.Errored} { if n, ok := counts[v]; ok && n > 0 { f.WriteString(fmt.Sprintf(`%s: %d`, v, v, n)) } } f.WriteString(`
` + narrativeHeadline(counts) + `
-`) +
PriorityCVETechniqueVerdictEventsRule
`) for _, c := range run.Chains { evCount := fmt.Sprintf("%d", c.EventsCollected) - rule := c.RuleMatched - if rule == "" { - rule = "—" - } prio := fmt.Sprintf("%.2f", c.ArbiterPriority) - cve := c.SourceCVE - if cve == "" { - cve = "—" + cve := orDash(c.SourceCVE) + + // The last column carries the rule that fired, or — when nothing did — + // the reason, so a gap is never left unexplained. + detail := c.RuleMatched + if detail == "" { + detail = orDash(c.Note) + } + + // Provenance: a substituted command must not read as the upstream atomic. + atomicCell := html.EscapeString(orDash(c.Atomic.ID)) + if c.Atomic.OverrideReason != "" { + atomicCell += ` ⚑` + } + + latency := "—" + if c.DetectLatencyMS > 0 { + latency = fmt.Sprintf("%.1fs", float64(c.DetectLatencyMS)/1000) + } + + attr := string(c.Attribution) + attrStyle := "" + switch c.Attribution { + case model.WindowOverlap: + attrStyle = ` style="color:#f59e0b" title="another technique ran inside this collection window — a matching event may belong to either"` + case model.Unscoped: + attrStyle = ` style="color:#9ca3af"` } - f.WriteString(fmt.Sprintf(``, - prio, html.EscapeString(cve), html.EscapeString(c.TechniqueID), - c.Verdict, c.Verdict, evCount, html.EscapeString(rule))) + + f.WriteString(fmt.Sprintf( + `%s`, + prio, html.EscapeString(cve), html.EscapeString(c.TechniqueID), atomicCell, + c.Verdict, c.Verdict, attrStyle, html.EscapeString(orDash(attr)), + latency, evCount, html.EscapeString(detail))) } f.WriteString("
PriorityCVETechniqueAtomicVerdictAttributionLatencyEventsRule / why not
%s%s%s%s%s%s
%s%s%s%s%s%s%s%s
") return nil } +// narrativeHeadline reports coverage over the techniques that were actually +// exercised. A technique that ERRORed (unresolvable atomic, platform mismatch), +// was SKIPPED_PREREQ, or produced no telemetry never tested a detection, so +// counting any of them as "missed" would invent detection gaps out of +// harness failures — the same error class as presence-based coverage. +func orDash(s string) string { + if s == "" { + return "—" + } + return s +} + func narrativeHeadline(counts map[model.Verdict]int) string { total := 0 for _, n := range counts { @@ -100,9 +145,13 @@ func narrativeHeadline(counts map[model.Verdict]int) string { } det := counts[model.Detected] miss := counts[model.Missed] - errs := counts[model.Errored] - return fmt.Sprintf("Top %d exploited-in-the-wild: %d detected / %d missed", - total, det, miss+errs) + notRun := counts[model.Errored] + counts[model.SkippedPrereq] + counts[model.NoTelemetry] + counts[model.Inconclusive] + headline := fmt.Sprintf("%d of %d techniques exercised: %d detected / %d missed", + det+miss, total, det, miss) + if notRun > 0 { + headline += fmt.Sprintf(" — %d not exercised (see per-technique notes)", notRun) + } + return headline } // NavigatorLayerReporter writes an ATT&CK Navigator layer JSON file. @@ -134,14 +183,21 @@ func (r NavigatorLayerReporter) Write(run model.CampaignResult) error { color string score int }{ - model.Detected: {"#4caf50", 100}, - model.Missed: {"#f44336", 0}, - model.Errored: {"#e91e63", 0}, + model.Detected: {"#4caf50", 100}, + model.Missed: {"#f44336", 0}, + model.Errored: {"#e91e63", 0}, + model.SkippedPrereq: {"#9e9e9e", 0}, } + desc := fmt.Sprintf("Detection coverage from campaign run at %s", run.StartedAt.Format(time.RFC3339)) + if run.Synthetic { + desc = "SYNTHETIC / DRY-RUN — NOT real telemetry. " + desc + } else if run.Inconclusive { + desc = "INCONCLUSIVE — canary did not fire, coverage not valid. " + desc + } layer := navLayer{ Name: "Purple Loop Coverage", - Description: fmt.Sprintf("Detection coverage from campaign run at %s", run.StartedAt.Format(time.RFC3339)), + Description: desc, Domain: "mitre-enterprise", Versions: map[string]string{"layer": "4.5", "attack": "16"}, } diff --git a/internal/report/synthetic_test.go b/internal/report/synthetic_test.go new file mode 100644 index 0000000..f755718 --- /dev/null +++ b/internal/report/synthetic_test.go @@ -0,0 +1,72 @@ +package report + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +// TestDashboard_SyntheticNotPublished guards that a synthetic (dry) run writes +// a marked per-run artifact but is NEVER appended to history or published to the +// public docs/data snapshot. +func TestDashboard_SyntheticNotPublished(t *testing.T) { + tmp := t.TempDir() + old, _ := os.Getwd() + defer os.Chdir(old) + if err := os.Chdir(tmp); err != nil { + t.Fatal(err) + } + + r := DashboardReporter{Dir: "reports"} + result := model.CampaignResult{ + StartedAt: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), + Synthetic: true, + Chains: []model.ProofChain{{TechniqueID: "T1", Verdict: model.Detected}}, + } + if err := r.Write(result); err != nil { + t.Fatalf("write: %v", err) + } + + if _, err := os.Stat("docs/data/coverage.json"); !os.IsNotExist(err) { + t.Error("synthetic run must NOT publish docs/data/coverage.json") + } + if _, err := os.Stat(filepath.Join("reports", "history.json")); !os.IsNotExist(err) { + t.Error("synthetic run must NOT append to history.json") + } + if runs, _ := filepath.Glob("reports/runs/*/coverage.json"); len(runs) != 1 { + t.Errorf("expected 1 marked per-run artifact, got %d", len(runs)) + } +} + +// TestBuildCoverage_CanaryReflectsResult guards that the dashboard canary flag +// is the REAL result, never hardcoded true. +func TestBuildCoverage_CanaryReflectsResult(t *testing.T) { + inc := buildCoverage(model.CampaignResult{Inconclusive: true}) + if inc["canary"].(map[string]any)["healthy"].(bool) { + t.Error("inconclusive run must report canary healthy=false") + } + syn := buildCoverage(model.CampaignResult{CanaryHealthy: true, Synthetic: true}) + if syn["canary"].(map[string]any)["healthy"].(bool) { + t.Error("synthetic run must report canary healthy=false") + } + real := buildCoverage(model.CampaignResult{CanaryHealthy: true}) + if !real["canary"].(map[string]any)["healthy"].(bool) { + t.Error("healthy real run must report canary healthy=true") + } +} + +// TestHTML_SyntheticBanner guards the loud non-evidentiary banner in the HTML report. +func TestHTML_SyntheticBanner(t *testing.T) { + path := filepath.Join(t.TempDir(), "r.html") + if err := (HTMLReporter{Path: path}).Write(model.CampaignResult{Synthetic: true}); err != nil { + t.Fatal(err) + } + b, _ := os.ReadFile(path) + if !strings.Contains(string(b), "SYNTHETIC") { + t.Error("HTML report of a synthetic run must show a SYNTHETIC banner") + } +} diff --git a/lab/docker-compose.override.yml b/lab/docker-compose.override.yml index 0e09799..0acccb5 100644 --- a/lab/docker-compose.override.yml +++ b/lab/docker-compose.override.yml @@ -13,6 +13,12 @@ services: container_name: purpleloop-victim hostname: victim01 restart: unless-stopped + # Deliberately unprivileged. Running Sysmon for Linux here was tried and + # measured (docs/PITFALLS.md): it needs privileged + systemd, and its eBPF + # then observes the whole host with no container identity in the event, so + # the telemetry cannot be attributed to this victim safely. A privileged + # container in a security lab is not worth paying for a source that would + # produce false attributions. cap_add: - AUDIT_CONTROL - AUDIT_READ diff --git a/lab/targets.yml b/lab/targets.yml new file mode 100644 index 0000000..a599c7d --- /dev/null +++ b/lab/targets.yml @@ -0,0 +1,34 @@ +# Lab targets. +# +# Before this file, `run` hardcoded a single Linux target (victim01 / Docker / +# detections/linux) and the Windows executor was reachable ONLY from the +# standalone `canary` subcommand — which also hardcoded an IP. The v1.0 +# "cross-platform coverage" claim could not be exercised through `run` at all. +# +# A technique is dispatched to the target whose `kind` matches the `platform` +# its rule-map entry declares. The canary runs per platform actually in use, so +# a campaign spanning both hosts proves both pipelines before trusting either. + +targets: + - name: victim01 + kind: linux + executor: docker + container: purpleloop-victim + rules_dir: detections/linux + + - name: windows-vm + kind: windows + executor: ssh + rules_dir: detections/windows + # Host and credentials come from the environment, never from source: + # WINDOWS_SSH_HOST, WINDOWS_SSH_USER, WINDOWS_SSH_PASS + # + # There is deliberately NO default host. The address depends on how your + # hypervisor is networked -- bridged, NAT (vmnet8) and host-only (vmnet1) + # all give different subnets -- so a committed default is right on exactly + # one machine and quietly wrong everywhere else. Find it with `ipconfig` on + # the VM and export WINDOWS_SSH_HOST. + ssh_host_env: WINDOWS_SSH_HOST + ssh_user_env: WINDOWS_SSH_USER + ssh_pass_env: WINDOWS_SSH_PASS + default_user: windows-vm diff --git a/lab/victim/Dockerfile b/lab/victim/Dockerfile index 3ac1bc7..94ec34d 100644 --- a/lab/victim/Dockerfile +++ b/lab/victim/Dockerfile @@ -1,5 +1,16 @@ -# Ubuntu victim: runs the Wazuh agent + auditd so attacks generate telemetry -# that ships to the manager. Kept intentionally small. +# Ubuntu victim: the Wazuh agent plus whatever process-creation telemetry this +# host can actually provide. +# +# NOT privileged, and NOT running Sysmon for Linux. That was tried and measured +# — see docs/PITFALLS.md, "Why the Linux victim is not a container with Sysmon". +# Sysmon works, but its eBPF is kernel-global: inside a container it observes +# every process on the host, and the event carries no container identity. Scoping +# it after the fact by resolving host PIDs is defeated by PID reuse, which let +# host binaries through; verifying process identity closes that leak but then +# drops the short-lived processes the atomics actually spawn. Either way the +# telemetry is wrong in a direction this project refuses to ship. +# +# The Linux equivalent of the Windows victim is a VM, not a container. FROM ubuntu:22.04 ARG WAZUH_VERSION=4.9.2 diff --git a/lab/victim/entrypoint.sh b/lab/victim/entrypoint.sh index d4bedae..f232827 100755 --- a/lab/victim/entrypoint.sh +++ b/lab/victim/entrypoint.sh @@ -1,16 +1,51 @@ #!/usr/bin/env bash -# Start auditd and the Wazuh agent (auto-enrolling against $WAZUH_MANAGER), -# then hold the container open. -set -e +# Start the telemetry sources and the Wazuh agent, then hold the container open. +# +# Telemetry status is reported LOUDLY at start-up. This used to run +# service auditd start || auditd || true +# which swallowed every failure, so the container came up looking healthy with +# no process-creation source at all. The pipeline then reported NO_TELEMETRY for +# every technique, which reads as a detection problem and is not one — it is a +# victim that never had the telemetry to begin with. +set -u : "${WAZUH_MANAGER:=wazuh.manager}" : "${WAZUH_AGENT_NAME:=victim01}" -# point the agent at the manager -sed -i "s|
.*
|
${WAZUH_MANAGER}
|g" /var/ossec/etc/ossec.conf || true +banner() { echo "[victim] $*"; } -service auditd start || auditd || true -/var/ossec/bin/wazuh-control start || true +# Point the agent at the manager. +sed -i "s|
.*
|
${WAZUH_MANAGER}
|g" \ + /var/ossec/etc/ossec.conf || banner "WARNING: could not set manager address" -# tail agent log so `docker logs` is useful; keep PID 1 alive +# --- process-creation telemetry --------------------------------------------- +# auditd needs the kernel audit subsystem. It is unavailable in many +# environments (kernel built without it, audit locked, or the netlink interface +# refused) and there is no point pretending otherwise: say so once, clearly, +# rather than failing silently. +PROCESS_TELEMETRY="none" +if auditctl -s >/dev/null 2>&1; then + if service auditd start >/dev/null 2>&1 || auditd >/dev/null 2>&1; then + # Capture execve so events carry Image and CommandLine, which is what the + # evaluator's process_creation rules require. + auditctl -a always,exit -F arch=b64 -S execve -k purpleloop_exec >/dev/null 2>&1 + auditctl -a always,exit -F arch=b32 -S execve -k purpleloop_exec >/dev/null 2>&1 + PROCESS_TELEMETRY="auditd" + banner "process-creation telemetry: auditd (execve rules loaded)" + else + banner "WARNING: kernel audit is available but auditd failed to start" + fi +else + banner "WARNING: kernel audit subsystem unavailable — auditctl cannot talk to it." + banner "WARNING: this victim has NO process-creation telemetry." + banner "WARNING: the pipeline canary cannot fire, so runs will be INCONCLUSIVE." + banner "WARNING: this is a lab capability gap, NOT a detection gap." + banner "WARNING: see docs/PITFALLS.md for what was measured and why a VM is the fix." +fi + +/var/ossec/bin/wazuh-control start || banner "WARNING: wazuh-agent failed to start" + +banner "agent=${WAZUH_AGENT_NAME} manager=${WAZUH_MANAGER} process_telemetry=${PROCESS_TELEMETRY}" + +# Tail the agent log so `docker logs` is useful; keep PID 1 alive. touch /var/ossec/logs/ossec.log exec tail -F /var/ossec/logs/ossec.log diff --git a/mappings/atomic-overrides.yml b/mappings/atomic-overrides.yml new file mode 100644 index 0000000..078cc2f --- /dev/null +++ b/mappings/atomic-overrides.yml @@ -0,0 +1,59 @@ +# Lab overrides for Atomic Red Team tests. +# +# Resolution order is: override -> vendored ART -> error. An override is a +# deliberate, visible substitution for an atomic that cannot run truthfully in +# THIS lab (no Linux test exists, needs elevation the victim container lacks, or +# installs packages on an offline victim). Every entry MUST carry a reason; the +# loader rejects one that does not, and the reason is printed in the report so a +# substituted command is never mistaken for the upstream atomic. +# +# An override is NOT a way to make a technique pass. If no honest command exists +# for a technique on this platform, leave it out and let the run report the gap. + +overrides: + # CONTAINMENT. Upstream T1059.004-1 writes a script that pings #{host}, + # whose default is 8.8.8.8 — traffic that leaves the isolated lab network and + # breaks the project's own containment rule ("never target anything outside + # purpleloop-lab"). Pinning the input keeps the upstream command and its + # detection surface intact while holding the traffic inside the lab. + T1059.004-1: + reason: >- + ART default pings 8.8.8.8, which leaves the purpleloop-lab network. + Pinned to loopback; the command and its telemetry are otherwise upstream. + input_arguments: + host: 127.0.0.1 + + # T1518 (Software Discovery) has SIX atomic tests upstream and not one of them + # supports Linux — they are Windows browser/dotnet checks plus one macOS + # Safari check. The campaign validates T1518 against a Linux victim, so there + # is nothing upstream to run. This is the equivalent enumeration a Linux + # adversary performs, and it is flagged as a substitution in every report. + T1518-1: + reason: >- + No Linux atomic exists upstream (all 6 tests are windows/macos). This is + the Linux-equivalent installed-software enumeration. + name: Software Discovery (Linux equivalent) + executor: sh + supported_platforms: [linux] + command: | + if [ -x "$(command -v dpkg)" ]; then dpkg -l | head -50; fi + if [ -x "$(command -v rpm)" ]; then rpm -qa | head -50; fi + if [ -x "$(command -v apk)" ]; then apk info | head -50; fi + ls -1 /usr/bin | head -50 + + # T1135-2 is the only Linux network-share atomic. It requires elevation and a + # samba package the offline victim container does not carry, so upstream it + # would always report SKIPPED_PREREQ and never exercise a detection. This + # enumerates shares with what the container actually has. + T1135-1: + reason: >- + Upstream T1135-2 needs samba + elevation; the victim container has + neither, so it can only ever skip. This uses the tools present. + name: Network Share Discovery (unprivileged Linux) + executor: sh + supported_platforms: [linux] + command: | + cat /proc/mounts | grep -Ei 'cifs|nfs|smb' || echo "no network mounts" + if [ -x "$(command -v showmount)" ]; then showmount -e localhost; fi + if [ -x "$(command -v smbclient)" ]; then smbclient -L localhost -N; fi + mount | head -30 diff --git a/mappings/attack_rule_map.json b/mappings/attack_rule_map.json new file mode 100644 index 0000000..fd9d97a --- /dev/null +++ b/mappings/attack_rule_map.json @@ -0,0 +1,116 @@ +{ + "_note": "Single source of truth for technique -> atomic -> expected detections. Replaces the technique->rule map that was hardcoded in cmd/purpleloop/main.go. 'expected_rules' lists every rule that SHOULD fire for this technique; a technique is DETECTED when any of them matches. 'platform' is the lab target kind the atomic runs on.", + "techniques": { + "T1059.004": { + "atomics": [ + "T1059.004-1" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/proc_creation_susp_shell.yml" + ] + }, + "T1087.001": { + "atomics": [ + "T1087.001-1" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1087.001.yml" + ] + }, + "T1082": { + "atomics": [ + "T1082-3" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1082.yml" + ] + }, + "T1033": { + "atomics": [ + "T1033-2" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1033.yml" + ] + }, + "T1007": { + "atomics": [ + "T1007-3" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1007.yml" + ] + }, + "T1016": { + "atomics": [ + "T1016-3" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1016.yml" + ] + }, + "T1049": { + "atomics": [ + "T1049-4" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1049.yml" + ] + }, + "T1069.001": { + "atomics": [ + "T1069.001-1" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1069.001.yml" + ] + }, + "T1069": { + "atomics": [ + "T1069.001-1" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1069.001.yml" + ] + }, + "T1135": { + "atomics": [ + "T1135-1" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1135.yml" + ] + }, + "T1518": { + "atomics": [ + "T1518-1" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1518.yml" + ] + }, + "T1059.003": { + "atomics": [ + "T1059.003-2" + ], + "platform": "windows", + "expected_rules": [ + "detections/windows/win_proc_create.yml" + ] + } + }, + "untested_rules": { + "detections/canary/pipeline_canary.yml": "not_a_detection \u2014 pipeline positive control, exercised by the canary gate on every run" + } +} diff --git a/mappings/attack_rule_map.seed.json b/mappings/attack_rule_map.seed.json deleted file mode 100644 index bacbaa6..0000000 --- a/mappings/attack_rule_map.seed.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "_note": "Seed mapping technique -> atomic test ids -> detection rules. Phase 4 adds CVE->technique resolution.", - "T1059.004": { - "atomics": ["T1059.004-1"], - "rules": ["detections/linux/proc_creation_susp_shell.yml"] - }, - "T1087.001": { - "atomics": ["T1087.001-1"], - "rules": ["detections/linux/T1087.001.yml"] - }, - "T1082": { - "atomics": ["T1082-1"], - "rules": ["detections/linux/T1082.yml"] - }, - "T1033": { - "atomics": ["T1033-1"], - "rules": ["detections/linux/T1033.yml"] - }, - "T1007": { - "atomics": ["T1007-1"], - "rules": ["detections/linux/T1007.yml"] - }, - "T1016": { - "atomics": ["T1016-1"], - "rules": ["detections/linux/T1016.yml"] - }, - "T1049": { - "atomics": ["T1049-1"], - "rules": ["detections/linux/T1049.yml"] - }, - "T1069.001": { - "atomics": ["T1069.001-1"], - "rules": ["detections/linux/T1069.001.yml"] - }, - "T1135": { - "atomics": ["T1135-1"], - "rules": ["detections/linux/T1135.yml"] - }, - "T1518": { - "atomics": ["T1518-1"], - "rules": ["detections/linux/T1518.yml"] - } -} diff --git a/plans/cross-platform.yml b/plans/cross-platform.yml new file mode 100644 index 0000000..fbb6a04 --- /dev/null +++ b/plans/cross-platform.yml @@ -0,0 +1,13 @@ +# A campaign spanning both lab hosts in one run. Each technique is dispatched to +# the target matching the platform its rule-map entry declares, and the pipeline +# canary runs once per platform in use — a healthy Linux canary says nothing +# about whether Windows telemetry is flowing. +name: cross-platform +description: Linux discovery plus Windows command-shell execution in a single run +techniques: + - technique_id: T1082 + atomic_ids: [T1082-3] + - technique_id: T1033 + atomic_ids: [T1033-2] + - technique_id: T1059.003 + atomic_ids: [T1059.003-2] diff --git a/plans/discovery.yml b/plans/discovery.yml index 380d9fa..cf75c40 100644 --- a/plans/discovery.yml +++ b/plans/discovery.yml @@ -8,15 +8,15 @@ techniques: - technique_id: T1087.001 atomic_ids: [T1087.001-1] - technique_id: T1082 - atomic_ids: [T1082-1] + atomic_ids: [T1082-3] - technique_id: T1033 - atomic_ids: [T1033-1] + atomic_ids: [T1033-2] - technique_id: T1007 - atomic_ids: [T1007-1] + atomic_ids: [T1007-3] - technique_id: T1016 - atomic_ids: [T1016-1] + atomic_ids: [T1016-3] - technique_id: T1049 - atomic_ids: [T1049-1] + atomic_ids: [T1049-4] - technique_id: T1069 atomic_ids: [T1069.001-1] - technique_id: T1135 diff --git a/plans/windows-discovery.yml b/plans/windows-discovery.yml new file mode 100644 index 0000000..67a630c --- /dev/null +++ b/plans/windows-discovery.yml @@ -0,0 +1,12 @@ +# Windows campaign. Exercises the Windows executor, the Windows target from +# lab/targets.yml and detections/windows — the cross-platform path that was +# previously unreachable from `run` (SSHExecutor existed only inside the +# standalone `canary` subcommand). +# +# Requires a Windows victim reachable over SSH: +# export WINDOWS_SSH_HOST=... WINDOWS_SSH_USER=... WINDOWS_SSH_PASS=... +name: windows-discovery +description: Windows command-shell execution against the Sysmon-instrumented victim +techniques: + - technique_id: T1059.003 + atomic_ids: [T1059.003-2] diff --git a/scripts/enable-archives.sh b/scripts/enable-archives.sh new file mode 100755 index 0000000..0f2bace --- /dev/null +++ b/scripts/enable-archives.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Enable Wazuh archive logging, persistently. +# +# The collector reads /var/ossec/logs/archives/archives.json — every event, not +# just rule-triggered alerts. That file is only written when is +# yes, and the Wazuh manager image ships it as no. +# +# Patching the RUNNING container does not work. The manager bind-mounts +# /wazuh-config-mount/etc/ossec.conf from the host and its entrypoint copies +# that file over /var/ossec/etc/ossec.conf on every start, so an in-container +# edit is wiped by the very restart needed to apply it. This was previously done +# by hand inside the container, which is why archive logging silently vanished +# whenever the stack was restarted: the lab came up healthy, the agent enrolled, +# and every technique reported NO_TELEMETRY because the archive the collector +# reads was never written. +# +# So: patch the host-side source of the config mount, then restart. +set -euo pipefail + +MANAGER="${1:-single-node-wazuh.manager-1}" + +if ! docker ps -a --format '{{.Names}}' | grep -qx "$MANAGER"; then + echo "manager container '$MANAGER' not found" >&2 + exit 1 +fi + +# Resolve the host path backing the config mount, rather than assuming a layout. +SRC=$(docker inspect "$MANAGER" \ + --format '{{range .Mounts}}{{if eq .Destination "/wazuh-config-mount/etc/ossec.conf"}}{{.Source}}{{end}}{{end}}') + +if [ -z "$SRC" ]; then + echo "no /wazuh-config-mount/etc/ossec.conf mount on $MANAGER" >&2 + echo "this lab layout is unexpected — enable in the manager config by hand" >&2 + exit 1 +fi +if [ ! -w "$SRC" ]; then + echo "config source is not writable: $SRC" >&2 + exit 1 +fi + +if grep -q 'yes' "$SRC"; then + echo "archive logging already enabled in $SRC" +else + if ! grep -q '' "$SRC"; then + echo "no element in $SRC — cannot patch safely" >&2 + exit 1 + fi + cp "$SRC" "$SRC.bak.$(date +%Y%m%d%H%M%S)" + sed -i 's|no|yes|' "$SRC" + echo "enabled in $SRC" +fi + +docker restart "$MANAGER" >/dev/null +echo "manager restarted" + +# Verify it actually took effect inside the container — the point of this script +# is that the obvious approach silently does not. +for _ in $(seq 1 20); do + if docker exec "$MANAGER" sh -c "grep -q 'yes' /var/ossec/etc/ossec.conf" 2>/dev/null; then + echo "verified: archive logging active in the manager" + exit 0 + fi + sleep 3 +done +echo "FAILED: is not yes inside the container after restart" >&2 +exit 1 diff --git a/scripts/fetch-atomics.sh b/scripts/fetch-atomics.sh new file mode 100755 index 0000000..b5c6d2c --- /dev/null +++ b/scripts/fetch-atomics.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Vendor Atomic Red Team at the commit recorded in mappings/atomic-red-team.commit. +# +# This PINS: it checks out the recorded commit and verifies HEAD matches, so two +# people running it get byte-identical atomics. (The older path in lab-fetch.sh +# cloned HEAD and then recorded whatever it happened to get, which is a record, +# not a pin — a re-clone months later would silently change every command the +# engine runs.) +set -euo pipefail +cd "$(dirname "$0")/.." + +art=lab/atomic-red-team +pin_file=mappings/atomic-red-team.commit + +if [ ! -f "$pin_file" ]; then + echo "no pin recorded at $pin_file" >&2 + exit 1 +fi +pin=$(tr -d '[:space:]' < "$pin_file") + +if [ ! -d "$art/.git" ]; then + echo "cloning atomic-red-team…" + rm -rf "$art" + git clone --filter=blob:none --no-checkout \ + https://github.com/redcanaryco/atomic-red-team.git "$art" +fi + +echo "checking out pinned commit ${pin}" +git -C "$art" fetch --depth 1 origin "$pin" 2>/dev/null || git -C "$art" fetch origin +git -C "$art" checkout --quiet --detach "$pin" + +actual=$(git -C "$art" rev-parse HEAD) +if [ "$actual" != "$pin" ]; then + echo "PIN MISMATCH: wanted $pin, got $actual" >&2 + exit 1 +fi + +count=$(find "$art/atomics" -maxdepth 1 -mindepth 1 -type d | wc -l) +echo "atomic-red-team pinned at $actual (${count} techniques)" diff --git a/scripts/lab-fetch.sh b/scripts/lab-fetch.sh index c084a51..615354f 100755 --- a/scripts/lab-fetch.sh +++ b/scripts/lab-fetch.sh @@ -19,13 +19,12 @@ cp lab/docker-compose.override.yml "$dest/single-node/docker-compose.override.ym cp lab/victim/entrypoint.sh lab/victim/entrypoint.sh 2>/dev/null || true echo "override installed into $dest/single-node/" -# pin Atomic Red Team and record the commit for reproducibility -art=lab/atomic-red-team -if [ ! -d "$art" ]; then - git clone --depth 1 https://github.com/redcanaryco/atomic-red-team.git "$art" -fi -( cd "$art" && git rev-parse HEAD ) > mappings/atomic-red-team.commit -echo "atomic-red-team pinned at $(cat mappings/atomic-red-team.commit)" +# Vendor Atomic Red Team at the RECORDED commit. This used to clone HEAD and +# then overwrite the pin file with whatever it got, so the "pin" tracked +# upstream instead of holding it — every re-clone could silently change the +# commands the engine executes. scripts/fetch-atomics.sh checks out the +# recorded commit and verifies it. +bash scripts/fetch-atomics.sh # generate certs (official helper) if not already present if [ ! -f "$dest/single-node/config/wazuh_indexer_ssl_certs/root-ca.pem" ]; then diff --git a/scripts/startup.sh b/scripts/startup.sh index 0590ee8..b0df8ea 100755 --- a/scripts/startup.sh +++ b/scripts/startup.sh @@ -32,7 +32,9 @@ echo "[5/5] Startup complete." echo "" echo "Next steps:" echo " 1. Power on Windows VM in VMware" -echo " 2. ssh windows-vm@192.168.88.13 hostname # verify" +echo " 2. Find the VM address (ipconfig on the VM), then:" +echo " export WINDOWS_SSH_HOST=
WINDOWS_SSH_USER=windows-vm" +echo " ssh \"\$WINDOWS_SSH_USER@\$WINDOWS_SSH_HOST\" hostname # verify" echo " 3. Generate arbiter export:" echo " hermes -z 'export top 10 alerts...' --profile threatlib" echo " 4. Run campaign:" diff --git a/scripts/verify-lab.sh b/scripts/verify-lab.sh index d5e85bf..7a50dec 100755 --- a/scripts/verify-lab.sh +++ b/scripts/verify-lab.sh @@ -18,17 +18,21 @@ MANAGER=$(docker compose -f lab/wazuh-docker/single-node/docker-compose.yml ps - echo "== verify-lab ==" +# Lab credentials come from the environment or the gitignored secrets file, +# never hardcoded in this tracked script. +if [ -f lab/secrets/wazuh-api.env ]; then + # shellcheck disable=SC1091 + source lab/secrets/wazuh-api.env +fi + # 1) indexer cluster health green +indexer_auth="${WAZUH_INDEXER_USER:-admin}:${WAZUH_INDEXER_PASSWORD:?WAZUH_INDEXER_PASSWORD not set}" health=$(docker exec "$INDEXER" \ - curl -s -k -u admin:SecretPassword https://localhost:9200/_cluster/health 2>/dev/null || true) + curl -s -k -u "$indexer_auth" https://localhost:9200/_cluster/health 2>/dev/null || true) echo "$health" | grep -q '"status":"green"' && ok "indexer cluster health green" \ || no "indexer cluster health not green (got: ${health:-none})" -# 2) manager API authenticates -WAZUH_PASS="${WAZUH_API_PASSWORD:-}" -if [ -z "$WAZUH_PASS" ] && [ -f lab/secrets/wazuh-api.env ]; then - source lab/secrets/wazuh-api.env -fi +# 2) manager API authenticates (credentials sourced above) tok=$(docker exec "$MANAGER" \ curl -s -k -u "${WAZUH_API_USER:-wazuh-wui}:${WAZUH_API_PASSWORD:?WAZUH_API_PASSWORD not set}" -X POST \ "https://localhost:55000/security/user/authenticate?raw=true" 2>/dev/null || true) diff --git a/testdata/sample-run/README.md b/testdata/sample-run/README.md new file mode 100644 index 0000000..08dc0ff --- /dev/null +++ b/testdata/sample-run/README.md @@ -0,0 +1,18 @@ +# Sample replay dataset + +`events.jsonl` is a dataset in the captured-run format, so `purpleloop replay` can be +exercised without a lab: + +```bash +go run ./cmd/purpleloop replay testdata/sample-run +``` + +**This is not lab telemetry.** The events are the same ones the CI fixtures assert, shaped +into the capture format. It demonstrates the replay path and gives a newcomer a real verdict +from the real matcher — it is not evidence about any real detection pipeline. A genuine +dataset is produced by a real run: `reports/runs//events.jsonl`, which the capture +writer refuses to create for a synthetic run. + +`T1518` is included deliberately as a **MISSED**: its atomic ran and produced telemetry, but +the events do not satisfy the rule. A sample where everything passes would teach the wrong +thing. diff --git a/testdata/sample-run/events.jsonl b/testdata/sample-run/events.jsonl new file mode 100644 index 0000000..cadb59c --- /dev/null +++ b/testdata/sample-run/events.jsonl @@ -0,0 +1,5 @@ +{"technique_id": "T1082", "atomic_id": "T1082-3", "host": "victim01", "executed_at": "2026-08-29T10:00:00Z", "window": {"start": "2026-08-29T10:00:00Z", "end": "2026-08-29T10:01:00Z"}, "event": {"event_id": "T1082-0", "timestamp": "2026-08-29T10:00:03Z", "raw": {"Image": "/usr/bin/uname", "ParentImage": "/bin/sh", "CommandLine": "uname -a", "User": "root"}}} +{"technique_id": "T1082", "atomic_id": "T1082-3", "host": "victim01", "executed_at": "2026-08-29T10:00:00Z", "window": {"start": "2026-08-29T10:00:00Z", "end": "2026-08-29T10:01:00Z"}, "event": {"event_id": "T1082-1", "timestamp": "2026-08-29T10:00:04Z", "raw": {"Image": "/usr/bin/cat", "ParentImage": "/bin/sh", "CommandLine": "cat /etc/os-release", "User": "root"}}} +{"technique_id": "T1033", "atomic_id": "T1033-2", "host": "victim01", "executed_at": "2026-08-29T10:02:00Z", "window": {"start": "2026-08-29T10:02:00Z", "end": "2026-08-29T10:03:00Z"}, "event": {"event_id": "T1033-0", "timestamp": "2026-08-29T10:02:03Z", "raw": {"Image": "/usr/bin/who", "ParentImage": "/bin/sh", "CommandLine": "who", "User": "root"}}} +{"technique_id": "T1049", "atomic_id": "T1049-4", "host": "victim01", "executed_at": "2026-08-29T10:04:00Z", "window": {"start": "2026-08-29T10:04:00Z", "end": "2026-08-29T10:05:00Z"}, "event": {"event_id": "T1049-0", "timestamp": "2026-08-29T10:04:03Z", "raw": {"Image": "/usr/bin/ss", "ParentImage": "/bin/bash", "CommandLine": "ss -antp", "User": "root"}}} +{"technique_id": "T1518", "atomic_id": "T1518-1", "host": "victim01", "executed_at": "2026-08-29T10:06:00Z", "window": {"start": "2026-08-29T10:06:00Z", "end": "2026-08-29T10:07:00Z"}, "event": {"event_id": "T1518-0", "timestamp": "2026-08-29T10:06:03Z", "raw": {"Image": "/usr/bin/ls", "ParentImage": "/bin/sh", "CommandLine": "ls -1 /usr/bin", "User": "root"}}}