From 5b3dbc5ea29cfaafe98a728ee08c77030bb29c2e Mon Sep 17 00:00:00 2001 From: jayelbotvibe-web Date: Sat, 25 Jul 2026 13:02:17 +0800 Subject: [PATCH 01/13] fix(integrity): enforce canary/dry-run trust contracts in code The pipeline's trust guarantees were documented but not enforced, so the tool's output could be mistaken for real detection evidence. - Canary gating now runs in every `run` path (runTasks) against the same Linux/Docker executor+collector the campaign uses. A run whose positive control does not fire is marked INCONCLUSIVE and its coverage is not presented as valid. Previously the canary ran only in the `canary` subcommand and no run was ever gated; model.Inconclusive was never assigned. - Dashboard canary status is the real result, not a hardcoded healthy:true. - Dry/synthetic runs are marked non-evidentiary end to end (CampaignResult.Synthetic): loud HTML/Navigator banner, and never appended to history.json or published to docs/data/coverage.json. - Reject partial container config so a real atomic is never paired with a synthetic collector. - Evidence window pads only the end, not the start (false DETECTED from an earlier run's events). - Collector: widen date pre-filter +/-1 day (non-UTC boundary); remove dead BaseURL/User/Pass fields and the unused canary dryRun param. - verify-lab.sh: source the indexer credential from env/secrets. - Refactor the four run* commands onto one runTasks helper. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 3 + CHANGELOG.md | 31 ++++++ cmd/purpleloop/main.go | 152 ++++++++++++++------------- internal/canary/canary.go | 2 +- internal/collector/prefilter_test.go | 7 +- internal/collector/wazuh.go | 17 +-- internal/model/model.go | 24 ++++- internal/report/dashboard.go | 20 +++- internal/report/report.go | 22 +++- internal/report/synthetic_test.go | 72 +++++++++++++ scripts/verify-lab.sh | 16 +-- 11 files changed, 265 insertions(+), 101 deletions(-) create mode 100644 internal/report/synthetic_test.go diff --git a/.gitignore b/.gitignore index 3901cd8..e9517e4 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ reports/ # os / editor .DS_Store *.swp + +# local go test cache (see review remediation) +.gotmp/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bbdbd0..7da22bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,37 @@ All notable changes to this project follow Keep a Changelog and Semantic Versioning. ## [Unreleased] +### Fixed (2026-07 code review — trust contracts enforced in code, not just docs) +- **Canary gating now runs in the pipeline.** Every `run` path executes the positive + control against the same Linux/Docker executor+collector the campaign uses; if it does + not fire, the run is marked `INCONCLUSIVE` and its coverage is not presented or published. + Previously the canary ran only in the standalone `canary` subcommand and no `run` was ever + gated. +- **Dashboard canary status is the real result**, never hardcoded `healthy:true`. +- **Dry/synthetic runs are marked non-evidentiary end to end** (`CampaignResult.Synthetic`): + loud banner in the HTML/Navigator output, and such runs are never appended to `history.json` + or published to `docs/data/coverage.json`. +- **Partial container config is rejected** — a real run requires both `--victim-container` + and `--manager-container` (or `--dry-run`), so a real atomic is never paired with a + synthetic collector. +- **Fidelity gate corrected**: only genuine process-creation events (Sysmon 1 / Security 4688, + or the image+commandLine signature) are treated as process telemetry. Enumeration/network + events (e.g. 4798 `callerProcessName`) no longer credit a process-creation detection. +- **Unsupported Sigma modifiers** (base64, cidr, windash, …) now yield `INCONCLUSIVE`, not a + silent `MISSED`. +- **Sigma matcher correctness**: wildcards compile to anchored regexps (`?` supported, + `*\svchost.exe` no longer matches `…svchost.exe.malware`, metacharacters escaped); + `N of prefix_*` / `all of prefix_*` expand by glob and no longer drop the trailing + `and not filter`; `Field: null` matches absent fields; `not(...)` (no space) parses; `of + them` excludes filter/falsepositive identifiers. +- **Evidence window** pads only the end (ingestion lag), not the start — a query can no + longer reach back and credit an earlier run's events (false `DETECTED`). +- **Collector date pre-filter** widened ±1 day so a non-UTC Wazuh timestamp near a day + boundary is not dropped before the precise instant check. +- Removed the hardcoded `admin:SecretPassword` from `scripts/verify-lab.sh` (now sourced from + env / the gitignored secrets file, matching the manager check). Removed dead + `BaseURL`/`User`/`Pass` collector fields and the unused `canary.Check` dry-run parameter. + ### Added - Evidence fidelity: normalizer tags each event's source; `process_creation` rules now only accept genuine process-creation telemetry (Sysmon/EventChannel eventdata, auditd diff --git a/cmd/purpleloop/main.go b/cmd/purpleloop/main.go index 8a84846..ae0917c 100644 --- a/cmd/purpleloop/main.go +++ b/cmd/purpleloop/main.go @@ -71,6 +71,14 @@ func main() { ctx := context.Background() + // A partial container config would fire a REAL atomic on the victim while + // collecting from a dry (synthetic) SIEM — a real attack paired with a + // fabricated event. Reject it: a real run needs both, or pass --dry-run. + if !*dryRun && (*victim == "") != (*manager == "") { + fmt.Fprintln(os.Stderr, "error: a real run needs BOTH --victim-container and --manager-container (or pass --dry-run)") + os.Exit(2) + } + // Warn loudly whenever any pipeline stage is synthetic, so a dry-run report // can never be mistaken for real detection evidence. if *dryRun || *victim == "" || *manager == "" { @@ -134,34 +142,81 @@ func newReporter(output string) model.Reporter { return report.JSONReporter{Out: os.Stdout} } -func runCampaign(ctx context.Context, planPath, output string, dryRun bool, victimContainer, managerContainer string) error { - var f feed.StaticFeed - if err := f.Load(planPath); err != nil { - return fmt.Errorf("load plan: %w", err) - } - tasks, err := f.Next(ctx) - if err != nil { - return fmt.Errorf("feed: %w", err) - } +// runOpts carries the shared pipeline configuration for a run command. +type runOpts struct { + output string + dryRun bool + victimContainer string + managerContainer string +} + +// synthetic reports whether any pipeline stage is dry/synthetic. Such a run's +// results are NOT real telemetry and must be marked non-evidentiary. +func (o runOpts) synthetic() bool { + return o.dryRun || o.victimContainer == "" || o.managerContainer == "" +} - exec := newExec(dryRun, victimContainer) - coll := newColl(dryRun, managerContainer) +// runTasks drives a set of tasks through the pipeline, applying BOTH hard +// contracts in one place so no run command can bypass them: +// - synthetic marking (dry pipeline → not real evidence), and +// - canary gating (a real run whose positive control does not fire is +// INCONCLUSIVE and its coverage must not be presented as valid). +func runTasks(ctx context.Context, tasks []model.TechniqueTask, o runOpts) error { + exec := newExec(o.dryRun, o.victimContainer) + coll := newColl(o.dryRun, o.managerContainer) eval := evaluator.RuleMatcherEvaluator{RulesDir: "detections/linux"} - rep := newReporter(output) target := model.Target{Host: "victim01", Kind: "linux"} - result := model.CampaignResult{StartedAt: time.Now().UTC()} + result := model.CampaignResult{StartedAt: time.Now().UTC(), Synthetic: o.synthetic()} + + // Canary gate — the pipeline positive control, run against the SAME + // executor/collector/target the campaign uses (the Linux/Docker path). + // Skipped for synthetic runs (already marked non-evidentiary above). + if !result.Synthetic { + cr := canary.Check(ctx, canary.NewMarker(), exec, coll, target.Kind, target) + result.CanaryHealthy = cr.Healthy + if !cr.Healthy { + result.Inconclusive = true + result.CanaryDetail = canaryFailDetail(cr) + fmt.Fprintf(os.Stderr, "INCONCLUSIVE: %s — coverage will not be reported as valid\n", result.CanaryDetail) + } + } + for _, task := range tasks { chain, err := runTechnique(ctx, exec, coll, eval, task, target) if err != nil { - chain = model.ProofChain{ - TechniqueID: task.TechniqueID, - Verdict: model.Errored, - } + chain = model.ProofChain{TechniqueID: task.TechniqueID, Verdict: model.Errored} + } + chain.SourceCVE = task.SourceCVE + chain.ArbiterPriority = task.Priority + if result.Inconclusive { + // Without a working positive control, no per-technique verdict is + // trustworthy — do not present DETECTED/MISSED as coverage. + chain.Verdict = model.Inconclusive } result.Chains = append(result.Chains, chain) } - return rep.Write(result) + + return newReporter(o.output).Write(result) +} + +func canaryFailDetail(cr canary.Result) string { + if cr.Err != nil { + return fmt.Sprintf("canary not detected on %s: %v", cr.Platform, cr.Err) + } + return fmt.Sprintf("canary not detected on %s", cr.Platform) +} + +func runCampaign(ctx context.Context, planPath, output string, dryRun bool, victimContainer, managerContainer string) error { + var f feed.StaticFeed + if err := f.Load(planPath); err != nil { + return fmt.Errorf("load plan: %w", err) + } + tasks, err := f.Next(ctx) + if err != nil { + return fmt.Errorf("feed: %w", err) + } + return runTasks(ctx, tasks, runOpts{output, dryRun, victimContainer, managerContainer}) } func runArbiter(ctx context.Context, arbiterPath, output string, dryRun bool, victimContainer, managerContainer string) error { @@ -173,27 +228,7 @@ func runArbiter(ctx context.Context, arbiterPath, output string, dryRun bool, vi if err != nil { return fmt.Errorf("arbiter feed: %w", err) } - - exec := newExec(dryRun, victimContainer) - coll := newColl(dryRun, managerContainer) - eval := evaluator.RuleMatcherEvaluator{RulesDir: "detections/linux"} - rep := newReporter(output) - target := model.Target{Host: "victim01", Kind: "linux"} - - result := model.CampaignResult{StartedAt: time.Now().UTC()} - for _, task := range tasks { - chain, err := runTechnique(ctx, exec, coll, eval, task, target) - if err != nil { - chain = model.ProofChain{ - TechniqueID: task.TechniqueID, - Verdict: model.Errored, - } - } - chain.SourceCVE = task.SourceCVE - chain.ArbiterPriority = task.Priority - result.Chains = append(result.Chains, chain) - } - return rep.Write(result) + return runTasks(ctx, tasks, runOpts{output, dryRun, victimContainer, managerContainer}) } func runEmulation(ctx context.Context, emuPath, output string, dryRun bool, victimContainer, managerContainer string) error { @@ -201,45 +236,16 @@ func runEmulation(ctx context.Context, emuPath, output string, dryRun bool, vict if err != nil { return fmt.Errorf("load emulation: %w", err) } - - exec := newExec(dryRun, victimContainer) - coll := newColl(dryRun, managerContainer) - eval := evaluator.RuleMatcherEvaluator{RulesDir: "detections/linux"} - rep := newReporter(output) - target := model.Target{Host: "victim01", Kind: "linux"} - - result := model.CampaignResult{StartedAt: time.Now().UTC()} + var tasks []model.TechniqueTask for _, stage := range plan.Stages { - for _, task := range stage.ToTasks() { - chain, err := runTechnique(ctx, exec, coll, eval, task, target) - if err != nil { - chain = model.ProofChain{ - TechniqueID: task.TechniqueID, - Verdict: model.Errored, - } - } - // per-stage verdict tracking via priority field - chain.ArbiterPriority = task.Priority - result.Chains = append(result.Chains, chain) - } + tasks = append(tasks, stage.ToTasks()...) } - return rep.Write(result) + return runTasks(ctx, tasks, runOpts{output, dryRun, victimContainer, managerContainer}) } func runOne(ctx context.Context, technique, output string, dryRun bool, victimContainer, managerContainer string) error { - exec := newExec(dryRun, victimContainer) - coll := newColl(dryRun, managerContainer) - eval := evaluator.RuleMatcherEvaluator{RulesDir: "detections/linux"} - rep := newReporter(output) - task := model.TechniqueTask{TechniqueID: technique, AtomicIDs: []string{technique + "-1"}} - target := model.Target{Host: "victim01", Kind: "linux"} - - chain, err := runTechnique(ctx, exec, coll, eval, task, target) - if err != nil { - return err - } - return rep.Write(model.CampaignResult{StartedAt: time.Now().UTC(), Chains: []model.ProofChain{chain}}) + return runTasks(ctx, []model.TechniqueTask{task}, runOpts{output, dryRun, victimContainer, managerContainer}) } func runTechnique(ctx context.Context, exec model.Executor, coll model.Collector, eval model.Evaluator, task model.TechniqueTask, target model.Target) (model.ProofChain, error) { @@ -316,7 +322,7 @@ func runCanaryCmd() { target := model.Target{Host: "windows-vm", Kind: "windows"} fmt.Printf("Canary marker: %s\n", marker) - result := canary.Check(ctx, marker, exec, coll, "windows", target, false) + result := canary.Check(ctx, marker, exec, coll, "windows", target) if result.Healthy { fmt.Printf("Canary: DETECTED on %s (evidence: %d events)\n", result.Platform, len(result.Evidence)) } else { diff --git a/internal/canary/canary.go b/internal/canary/canary.go index e760764..e94cd54 100644 --- a/internal/canary/canary.go +++ b/internal/canary/canary.go @@ -115,7 +115,7 @@ func (ck Checker) Run(ctx context.Context, marker string, exec model.Executor, // Check executes the canary on one platform and verifies the marker was // detected, using production defaults. func Check(ctx context.Context, marker string, exec model.Executor, - coll model.Collector, platform string, target model.Target, dryRun bool) Result { + coll model.Collector, platform string, target model.Target) Result { return Checker{}.Run(ctx, marker, exec, coll, platform, target) } diff --git a/internal/collector/prefilter_test.go b/internal/collector/prefilter_test.go index c5a4fee..fc7bbb8 100644 --- a/internal/collector/prefilter_test.go +++ b/internal/collector/prefilter_test.go @@ -19,8 +19,11 @@ func TestDateAlternation(t *testing.T) { w model.TimeWindow want string }{ - {"same day", mk("2026-07-04T09:00:00Z", "2026-07-04T09:05:00Z"), "2026-07-04"}, - {"spans midnight", mk("2026-07-04T23:59:00Z", "2026-07-05T00:02:00Z"), "2026-07-04|2026-07-05"}, + // The alternation is widened by one day on each side so a non-UTC + // Wazuh timestamp whose literal date lands on an adjacent day is not + // dropped by the coarse pre-filter (the precise instant check still bounds it). + {"same day", mk("2026-07-04T09:00:00Z", "2026-07-04T09:05:00Z"), "2026-07-03|2026-07-04|2026-07-05"}, + {"spans midnight", mk("2026-07-04T23:59:00Z", "2026-07-05T00:02:00Z"), "2026-07-03|2026-07-04|2026-07-05|2026-07-06"}, {"zero window", model.TimeWindow{}, ""}, {"reversed", mk("2026-07-05T00:00:00Z", "2026-07-04T00:00:00Z"), ""}, } diff --git a/internal/collector/wazuh.go b/internal/collector/wazuh.go index f7264cd..9a489b7 100644 --- a/internal/collector/wazuh.go +++ b/internal/collector/wazuh.go @@ -38,11 +38,10 @@ func parseTimestamp(s string) (time.Time, error) { return time.Time{}, fmt.Errorf("cannot parse timestamp %q", s) } -// WazuhCollector reads agent alerts from the manager's alerts.json log. +// WazuhCollector reads agent alerts from the manager's archives.json via +// `docker exec`. With no ManagerContainer (and no test alertsPath) it runs in +// dry mode and returns a single synthetic event. type WazuhCollector struct { - BaseURL string // e.g. https://localhost:55000 — empty => dry mode - User string - Pass string ManagerContainer string // docker container name, e.g. single-node-wazuh.manager-1 // alertsPath overrides the source for testing; empty uses docker exec. @@ -156,9 +155,13 @@ func dateAlternation(w model.TimeWindow) string { if w.Start.IsZero() || w.End.IsZero() || w.End.Before(w.Start) { return "" } - const maxDays = 31 - start := w.Start.UTC().Truncate(24 * time.Hour) - end := w.End.UTC().Truncate(24 * time.Hour) + const maxDays = 34 + // Widen by a day on each side: Wazuh may stamp events with a non-UTC offset, + // so a line's literal YYYY-MM-DD can be the local date while the window is + // computed in UTC. Including the adjacent days keeps a day-boundary event in + // the coarse pre-filter; the precise instant comparison below still bounds it. + start := w.Start.UTC().Truncate(24 * time.Hour).Add(-24 * time.Hour) + end := w.End.UTC().Truncate(24 * time.Hour).Add(24 * time.Hour) var stamps []string for d := start; !d.After(end); d = d.Add(24 * time.Hour) { stamps = append(stamps, d.Format("2006-01-02")) diff --git a/internal/model/model.go b/internal/model/model.go index 5553fbc..6ccaebf 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -52,10 +52,13 @@ type RunResult struct { Stderr string `json:"-"` } -// Window returns the execution's time span, padded so the SIEM has room to -// have ingested the resulting events. +// Window returns the execution's time span, padded only on the END so the SIEM +// has room to ingest the resulting events. The start is NOT padded backward: +// widening it into the past can capture an earlier run's matching events and +// credit them to this execution (false DETECTED). A small epsilon absorbs +// minor clock skew without reaching a prior test. func (r RunResult) Window(pad time.Duration) TimeWindow { - return TimeWindow{Start: r.StartedAt.Add(-pad), End: r.FinishedAt.Add(pad)} + return TimeWindow{Start: r.StartedAt.Add(-2 * time.Second), End: r.FinishedAt.Add(pad)} } // Event is one normalised telemetry record pulled from the SIEM. @@ -94,9 +97,20 @@ type ProofChain struct { } // CampaignResult aggregates every technique validated in one run. +// +// The three trust fields enforce the hard contracts in code (not just docs): +// - Synthetic: any pipeline stage was dry/synthetic, so the chains are NOT +// real telemetry and must never be presented or published as evidence. +// - CanaryHealthy / Inconclusive: the pipeline positive control. On a real +// run, if the canary did not fire the run is INCONCLUSIVE and its coverage +// must not be treated as valid. type CampaignResult struct { - StartedAt time.Time `json:"started_at"` - Chains []ProofChain `json:"chains"` + StartedAt time.Time `json:"started_at"` + Chains []ProofChain `json:"chains"` + Synthetic bool `json:"synthetic"` // dry/synthetic pipeline — not real evidence + CanaryHealthy bool `json:"canary_healthy"` // positive control fired (real runs) + Inconclusive bool `json:"inconclusive"` // canary failed on a real run → coverage invalid + CanaryDetail string `json:"canary_detail,omitempty"` // why the canary failed } // ---- interfaces: the swappable seams of the system ---- diff --git a/internal/report/dashboard.go b/internal/report/dashboard.go index 33a0f5c..22e6abe 100644 --- a/internal/report/dashboard.go +++ b/internal/report/dashboard.go @@ -37,6 +37,13 @@ func (r DashboardReporter) Write(result model.CampaignResult) error { return err } + // Synthetic or inconclusive runs are NOT valid coverage: keep the per-run + // artifact above (marked, for the operator) but never append them to the + // history index or publish them to the public Pages snapshot. + if result.Synthetic || result.Inconclusive { + return nil + } + // History index s := d["summary"].(map[string]any) c := d["canary"].(map[string]any) @@ -85,12 +92,17 @@ func buildCoverage(result model.CampaignResult) map[string]any { "build": "v1.3.0", } - // Canary (per-platform from result if available, else default healthy) + // Canary — the REAL positive-control result for this run (never hardcoded). + // A synthetic or inconclusive run is not healthy: the dashboard keys its + // "inconclusive" banner off this flag. + canaryHealthy := result.CanaryHealthy && !result.Synthetic && !result.Inconclusive d["canary"] = map[string]any{ - "healthy": true, + "healthy": canaryHealthy, + "synthetic": result.Synthetic, + "inconclusive": result.Inconclusive, + "detail": result.CanaryDetail, "platforms": []map[string]any{ - {"name": "linux", "healthy": true}, - {"name": "windows", "healthy": true}, + {"name": "linux", "healthy": canaryHealthy}, }, } diff --git a/internal/report/report.go b/internal/report/report.go index f918174..bbcdf78 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -58,8 +58,18 @@ tr:hover{background:#1a1a2e} .summary span{font-size:1.2em}

Purple Loop — Coverage Report

-

` + html.EscapeString(run.StartedAt.Format("2006-01-02 15:04:05 UTC")) + `

-
`) +

` + html.EscapeString(run.StartedAt.Format("2006-01-02 15:04:05 UTC")) + `

`) + + // Trust banners — an operator must never mistake a synthetic or + // canary-failed report for verified coverage. + if run.Synthetic { + f.WriteString(`
⚠ SYNTHETIC / DRY-RUN PIPELINE — these results are NOT real telemetry and must not be used as evidence.
`) + } + if run.Inconclusive { + f.WriteString(`
⚠ INCONCLUSIVE — the pipeline positive control (canary) did not fire; coverage is not valid.
` + html.EscapeString(run.CanaryDetail) + `
`) + } + + f.WriteString(`
`) for _, v := range []model.Verdict{model.Detected, model.Missed, model.NoTelemetry, model.Inconclusive, model.Errored} { if n, ok := counts[v]; ok && n > 0 { @@ -139,9 +149,15 @@ func (r NavigatorLayerReporter) Write(run model.CampaignResult) error { model.Errored: {"#e91e63", 0}, } + desc := fmt.Sprintf("Detection coverage from campaign run at %s", run.StartedAt.Format(time.RFC3339)) + if run.Synthetic { + desc = "SYNTHETIC / DRY-RUN — NOT real telemetry. " + desc + } else if run.Inconclusive { + desc = "INCONCLUSIVE — canary did not fire, coverage not valid. " + desc + } layer := navLayer{ Name: "Purple Loop Coverage", - Description: fmt.Sprintf("Detection coverage from campaign run at %s", run.StartedAt.Format(time.RFC3339)), + Description: desc, Domain: "mitre-enterprise", Versions: map[string]string{"layer": "4.5", "attack": "16"}, } diff --git a/internal/report/synthetic_test.go b/internal/report/synthetic_test.go new file mode 100644 index 0000000..f755718 --- /dev/null +++ b/internal/report/synthetic_test.go @@ -0,0 +1,72 @@ +package report + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +// TestDashboard_SyntheticNotPublished guards that a synthetic (dry) run writes +// a marked per-run artifact but is NEVER appended to history or published to the +// public docs/data snapshot. +func TestDashboard_SyntheticNotPublished(t *testing.T) { + tmp := t.TempDir() + old, _ := os.Getwd() + defer os.Chdir(old) + if err := os.Chdir(tmp); err != nil { + t.Fatal(err) + } + + r := DashboardReporter{Dir: "reports"} + result := model.CampaignResult{ + StartedAt: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), + Synthetic: true, + Chains: []model.ProofChain{{TechniqueID: "T1", Verdict: model.Detected}}, + } + if err := r.Write(result); err != nil { + t.Fatalf("write: %v", err) + } + + if _, err := os.Stat("docs/data/coverage.json"); !os.IsNotExist(err) { + t.Error("synthetic run must NOT publish docs/data/coverage.json") + } + if _, err := os.Stat(filepath.Join("reports", "history.json")); !os.IsNotExist(err) { + t.Error("synthetic run must NOT append to history.json") + } + if runs, _ := filepath.Glob("reports/runs/*/coverage.json"); len(runs) != 1 { + t.Errorf("expected 1 marked per-run artifact, got %d", len(runs)) + } +} + +// TestBuildCoverage_CanaryReflectsResult guards that the dashboard canary flag +// is the REAL result, never hardcoded true. +func TestBuildCoverage_CanaryReflectsResult(t *testing.T) { + inc := buildCoverage(model.CampaignResult{Inconclusive: true}) + if inc["canary"].(map[string]any)["healthy"].(bool) { + t.Error("inconclusive run must report canary healthy=false") + } + syn := buildCoverage(model.CampaignResult{CanaryHealthy: true, Synthetic: true}) + if syn["canary"].(map[string]any)["healthy"].(bool) { + t.Error("synthetic run must report canary healthy=false") + } + real := buildCoverage(model.CampaignResult{CanaryHealthy: true}) + if !real["canary"].(map[string]any)["healthy"].(bool) { + t.Error("healthy real run must report canary healthy=true") + } +} + +// TestHTML_SyntheticBanner guards the loud non-evidentiary banner in the HTML report. +func TestHTML_SyntheticBanner(t *testing.T) { + path := filepath.Join(t.TempDir(), "r.html") + if err := (HTMLReporter{Path: path}).Write(model.CampaignResult{Synthetic: true}); err != nil { + t.Fatal(err) + } + b, _ := os.ReadFile(path) + if !strings.Contains(string(b), "SYNTHETIC") { + t.Error("HTML report of a synthetic run must show a SYNTHETIC banner") + } +} diff --git a/scripts/verify-lab.sh b/scripts/verify-lab.sh index d5e85bf..7a50dec 100755 --- a/scripts/verify-lab.sh +++ b/scripts/verify-lab.sh @@ -18,17 +18,21 @@ MANAGER=$(docker compose -f lab/wazuh-docker/single-node/docker-compose.yml ps - echo "== verify-lab ==" +# Lab credentials come from the environment or the gitignored secrets file, +# never hardcoded in this tracked script. +if [ -f lab/secrets/wazuh-api.env ]; then + # shellcheck disable=SC1091 + source lab/secrets/wazuh-api.env +fi + # 1) indexer cluster health green +indexer_auth="${WAZUH_INDEXER_USER:-admin}:${WAZUH_INDEXER_PASSWORD:?WAZUH_INDEXER_PASSWORD not set}" health=$(docker exec "$INDEXER" \ - curl -s -k -u admin:SecretPassword https://localhost:9200/_cluster/health 2>/dev/null || true) + curl -s -k -u "$indexer_auth" https://localhost:9200/_cluster/health 2>/dev/null || true) echo "$health" | grep -q '"status":"green"' && ok "indexer cluster health green" \ || no "indexer cluster health not green (got: ${health:-none})" -# 2) manager API authenticates -WAZUH_PASS="${WAZUH_API_PASSWORD:-}" -if [ -z "$WAZUH_PASS" ] && [ -f lab/secrets/wazuh-api.env ]; then - source lab/secrets/wazuh-api.env -fi +# 2) manager API authenticates (credentials sourced above) tok=$(docker exec "$MANAGER" \ curl -s -k -u "${WAZUH_API_USER:-wazuh-wui}:${WAZUH_API_PASSWORD:?WAZUH_API_PASSWORD not set}" -X POST \ "https://localhost:55000/security/user/authenticate?raw=true" 2>/dev/null || true) From c6f61ac241557e6f643c5b771ee3737c8985015e Mon Sep 17 00:00:00 2001 From: jayelbotvibe-web Date: Sat, 25 Jul 2026 13:02:33 +0800 Subject: [PATCH 02/13] fix(evaluator): correct fidelity gate and Sigma matcher semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fidelity: only genuine process-creation events (Sysmon 1 / Security 4688, or the image+commandLine signature) are tagged process telemetry. Enumeration/ network events (e.g. 4798 callerProcessName) no longer credit a process-creation detection (false DETECTED on real telemetry). EventID is now carried into the normalized event. - Unsupported modifiers (base64/cidr/windash/...) → INCONCLUSIVE, not a silent MISSED. - Wildcards compile to anchored, case-insensitive regexps: `?` supported, `*\svchost.exe` no longer matches `...svchost.exe.malware`, metacharacters escaped; modifiers (contains/startswith/endswith) anchor correctly. - `N of prefix_*` / `all of prefix_*` expand by glob AND the parser no longer drops the trailing `and not filter` (was losing filter suppression). - `Field: null` matches absent/empty fields; `not(...)` (no space) parses; `of them` excludes filter/falsepositive identifiers. Co-Authored-By: Claude Opus 4.8 --- internal/evaluator/fidelity_test.go | 4 +- internal/evaluator/fixes_test.go | 102 ++++++++++++++++++++++ internal/evaluator/matcher.go | 130 ++++++++++++++++++---------- internal/evaluator/normalizer.go | 81 ++++++++++++++--- internal/evaluator/rule.go | 116 +++++++++++++++---------- internal/evaluator/sigma.go | 6 ++ 6 files changed, 333 insertions(+), 106 deletions(-) create mode 100644 internal/evaluator/fixes_test.go diff --git a/internal/evaluator/fidelity_test.go b/internal/evaluator/fidelity_test.go index d963dc5..ab14412 100644 --- a/internal/evaluator/fidelity_test.go +++ b/internal/evaluator/fidelity_test.go @@ -55,7 +55,9 @@ func TestNormalizer_FidelityTagging(t *testing.T) { raw string want string }{ - {"windows eventdata", `{"data":{"win":{"eventdata":{"image":"C:\\a.exe"}}}}`, FidelityProcess}, + {"windows process creation", `{"data":{"win":{"eventdata":{"image":"C:\\a.exe","commandLine":"a.exe"}}}}`, FidelityProcess}, + {"windows sysmon1 by eventID", `{"data":{"win":{"system":{"eventID":"1"},"eventdata":{"image":"C:\\a.exe"}}}}`, FidelityProcess}, + {"windows non-creation enumeration (4798)", `{"data":{"win":{"eventdata":{"callerProcessName":"C:\\net1.exe","subjectUserName":"SYSTEM"}}}}`, FidelityLog}, {"auditd execve", `{"data":{"audit":{"execve":{"a0":"id"}}}}`, FidelityProcess}, {"top-level synthetic", `{"Image":"/usr/bin/id","CommandLine":"id"}`, FidelityProcess}, {"full_log scrape", `{"full_log":"ossec: output: 'id': uid=0"}`, FidelityLog}, diff --git a/internal/evaluator/fixes_test.go b/internal/evaluator/fixes_test.go new file mode 100644 index 0000000..a78f79c --- /dev/null +++ b/internal/evaluator/fixes_test.go @@ -0,0 +1,102 @@ +package evaluator + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +// TestGlobMatch_Anchored guards the wildcard fixes: patterns are anchored to the +// whole value (no false DETECTED from trailing/leading slop) and `?` matches +// exactly one character. +func TestGlobMatch_Anchored(t *testing.T) { + cases := []struct { + pattern, value string + want bool + }{ + {`*\svchost.exe`, `c:\windows\svchost.exe`, true}, + {`*\svchost.exe`, `c:\x\svchost.exe.malware`, false}, // trailing anchored + {`cmd*`, `cmd.exe`, true}, + {`cmd*`, `xcmd.exe`, false}, // leading anchored + {`C:\Users\?\evil.exe`, `C:\Users\a\evil.exe`, true}, + {`C:\Users\?\evil.exe`, `C:\Users\ab\evil.exe`, false}, // ? = one char + } + for _, c := range cases { + if got := globMatch(c.pattern, c.value); got != c.want { + t.Errorf("globMatch(%q, %q) = %v, want %v", c.pattern, c.value, got, c.want) + } + } +} + +// TestCondition_AggregateGlobHonorsTrailingFilter guards the P0 fix: `N of +// prefix_*` expands by glob AND the trailing `and not filter` is no longer +// dropped by the parser (which would lose filter suppression → false DETECTED). +func TestCondition_AggregateGlobHonorsTrailingFilter(t *testing.T) { + expr, err := parseCondition("1 of selection_* and not filter") + if err != nil { + t.Fatalf("parse: %v", err) + } + detections := map[string]FieldMap{ + "selection_a": {"Image": {Values: []string{"/bin/evil"}}}, + "filter": {"User": {Values: []string{"root"}}}, + } + // Matches selection_a AND filter → "1 of selection_*" true, "not filter" false. + if evalExpr(expr, detections, map[string]string{"Image": "/bin/evil", "User": "root"}) { + t.Error("filter should suppress via the trailing 'and not filter'") + } + // Matches selection_a, not filter → true. + if !evalExpr(expr, detections, map[string]string{"Image": "/bin/evil", "User": "alice"}) { + t.Error("selection matches and filter does not → should match") + } +} + +// TestMatchNull guards Sigma `Field: null` (match when absent/empty). +func TestMatchNull(t *testing.T) { + expr, _ := parseCondition("sel") + detections := map[string]FieldMap{"sel": {"CommandLine": {MatchNull: true}}} + if !evalExpr(expr, detections, map[string]string{"Image": "x"}) { + t.Error("Field: null should match when the field is absent") + } + if evalExpr(expr, detections, map[string]string{"CommandLine": "present"}) { + t.Error("Field: null should NOT match when the field is present") + } +} + +// TestNot_NoSpaceBeforeParen guards `not(...)` (no space) parsing. +func TestNot_NoSpaceBeforeParen(t *testing.T) { + expr, err := parseCondition("not(sel)") + if err != nil { + t.Fatalf("parse: %v", err) + } + detections := map[string]FieldMap{"sel": {"Image": {Values: []string{"/bin/evil"}}}} + if evalExpr(expr, detections, map[string]string{"Image": "/bin/evil"}) { + t.Error("not(sel) should be false when sel matches") + } + if !evalExpr(expr, detections, map[string]string{"Image": "/bin/ok"}) { + t.Error("not(sel) should be true when sel does not match") + } +} + +// TestEvaluate_UnsupportedModifierInconclusive guards that a rule using a +// modifier we can't evaluate is reported INCONCLUSIVE, never a silent MISS. +func TestEvaluate_UnsupportedModifierInconclusive(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "r.yml") + rule := "title: t\nlogsource:\n category: process_creation\ndetection:\n" + + " sel:\n CommandLine|base64|contains: whoami\n condition: sel\n" + if err := os.WriteFile(path, []byte(rule), 0644); err != nil { + t.Fatal(err) + } + eval := RuleMatcherEvaluator{RulesDir: dir} + ev := []model.Event{{Raw: json.RawMessage(`{"Image":"/bin/x","CommandLine":"whoami"}`)}} + v, _, err := eval.Evaluate(model.SigmaRule{Path: path}, ev) + if err != nil { + t.Fatalf("evaluate: %v", err) + } + if v != model.Inconclusive { + t.Errorf("unsupported-modifier rule → %s, want INCONCLUSIVE", v) + } +} diff --git a/internal/evaluator/matcher.go b/internal/evaluator/matcher.go index 22955d2..3acb918 100644 --- a/internal/evaluator/matcher.go +++ b/internal/evaluator/matcher.go @@ -52,6 +52,13 @@ func evalIdent(name string, detections map[string]FieldMap, event map[string]str // All field entries in the identifier must match (AND) for field, entry := range fm { val, exists := event[field] + if entry.MatchNull { + // Sigma `Field: null` matches when the field is absent or empty. + if exists && val != "" { + return false + } + continue + } if !exists { return false } @@ -127,23 +134,22 @@ func matchField(eventValue string, entry FieldEntry) bool { return matchNumeric(eventValue, candidate, numOp) } - v := strings.ToLower(eventValue) - c := strings.ToLower(candidate) - - // If candidate contains * wildcards, use glob-like matching - if strings.Contains(c, "*") { - return matchWildcard(v, c) - } - + // Express the modifier (or an explicit-wildcard value) as a Sigma glob, + // then match as an anchored, case-insensitive regexp. Anchoring pins the + // literal segments to the ends of the value (so `*\svchost.exe` does not + // match `…svchost.exe.malware`), `?` matches one character, and regex + // metacharacters in the value can't leak. switch { - case hasEndsWith: - return strings.HasSuffix(v, c) - case hasStartsWith: - return strings.HasPrefix(v, c) case hasContains: - return strings.Contains(v, c) + return globMatch("*"+candidate+"*", eventValue) + case hasStartsWith: + return globMatch(candidate+"*", eventValue) + case hasEndsWith: + return globMatch("*"+candidate, eventValue) + case strings.ContainsAny(candidate, "*?"): + return globMatch(candidate, eventValue) default: - return v == c + return strings.EqualFold(eventValue, candidate) } } @@ -185,38 +191,40 @@ func matchNumeric(eventValue, candidate, op string) bool { return false } -// matchWildcard performs case-insensitive glob matching where * matches any -// sequence of characters. It splits the pattern on * and matches each literal -// segment in order within the value string. -func matchWildcard(value, pattern string) bool { - // A single "*" matches everything - if pattern == "*" { - return true - } +// globMatch reports whether value matches a Sigma glob pattern, case-insensitively +// and anchored to the whole value. +func globMatch(pattern, value string) bool { + re := globToRegexp(pattern) + return re != nil && re.MatchString(value) +} - segments := strings.Split(pattern, "*") - pos := 0 - for _, seg := range segments { - if seg == "" { - continue // leading *, trailing *, or consecutive ** - } - idx := strings.Index(value[pos:], seg) - if idx < 0 { - return false +// globToRegexp compiles a Sigma wildcard pattern into an anchored, +// case-insensitive regexp: `*` → any run, `?` → one character; every other +// character (including `\`, a literal path separator in this dialect) is +// escaped. Returns nil if compilation fails (treated as no match by globMatch). +func globToRegexp(glob string) *regexp.Regexp { + var b strings.Builder + b.WriteString("(?is)^") + for i := 0; i < len(glob); i++ { + switch glob[i] { + case '*': + b.WriteString(".*") + case '?': + b.WriteString(".") + default: + b.WriteString(regexp.QuoteMeta(string(glob[i]))) } - pos += idx + len(seg) } - return true + b.WriteString("$") + re, err := regexp.Compile(b.String()) + if err != nil { + return nil + } + return re } func evalOneOf(e OneOfExpr, detections map[string]FieldMap, event map[string]string) bool { - names := e.Names - if len(names) == 0 { - // "1 of them" — expand to all search-identifiers - for k := range detections { - names = append(names, k) - } - } + names := expandIdentNames(e.Names, detections) matched := 0 for _, name := range names { if evalIdent(name, detections, event) { @@ -227,13 +235,7 @@ func evalOneOf(e OneOfExpr, detections map[string]FieldMap, event map[string]str } func evalAllOf(e AllOfExpr, detections map[string]FieldMap, event map[string]string) bool { - names := e.Names - if len(names) == 0 { - // "all of them" — expand to all search-identifiers - for k := range detections { - names = append(names, k) - } - } + names := expandIdentNames(e.Names, detections) for _, name := range names { if !evalIdent(name, detections, event) { return false @@ -241,3 +243,37 @@ func evalAllOf(e AllOfExpr, detections map[string]FieldMap, event map[string]str } return len(names) > 0 // vacuous truth over empty set → false } + +// expandIdentNames resolves aggregate operands to concrete identifier names. +// Empty input means "of them" → every identifier EXCEPT filter/falsepositive +// selections (which are meant to be referenced explicitly, not folded into an +// OR/AND). A name containing */? is a glob expanded against the identifier keys. +func expandIdentNames(names []string, detections map[string]FieldMap) []string { + if len(names) == 0 { + var out []string + for k := range detections { + if strings.HasPrefix(k, "filter") || strings.HasPrefix(k, "falsepositive") { + continue + } + out = append(out, k) + } + return out + } + var out []string + for _, n := range names { + if strings.ContainsAny(n, "*?") { + re := globToRegexp(n) + if re == nil { + continue + } + for k := range detections { + if re.MatchString(k) { + out = append(out, k) + } + } + continue + } + out = append(out, n) + } + return out +} diff --git a/internal/evaluator/normalizer.go b/internal/evaluator/normalizer.go index 0916a22..d477498 100644 --- a/internal/evaluator/normalizer.go +++ b/internal/evaluator/normalizer.go @@ -52,21 +52,45 @@ func (Normalizer) Normalize(raw json.RawMessage) map[string]string { highFidelity = true } - // Try Windows Sysmon / EventChannel paths + // Try Windows Sysmon / EventChannel paths. + // + // CRITICAL: only genuine PROCESS-CREATION events (Sysmon EventID 1, Security + // 4688) may be tagged high fidelity. Other Windows events (network + // connection=3, account enumeration=4798, handle=4656, …) also carry a + // process name (callerProcessName/processName) but are NOT proof a process + // was created. Tagging them as process telemetry credited a process-creation + // detection off an enumeration/network event — a false DETECTED. if data, ok := event["data"].(map[string]any); ok { if win, ok := data["win"].(map[string]any); ok { if ed, ok := win["eventdata"].(map[string]any); ok { - // Sysmon process creation (sysmon 1) - getString(ed, "image", &out, "Image") - getString(ed, "parentImage", &out, "ParentImage") - getString(ed, "commandLine", &out, "CommandLine") - getString(ed, "user", &out, "User") - getString(ed, "parentUser", &out, "User") - // Security Audit fallbacks - getString(ed, "callerProcessName", &out, "Image") - getString(ed, "subjectUserName", &out, "User") - getString(ed, "processName", &out, "Image") - highFidelity = true + eventID := winEventID(win) + if eventID != "" { + out["EventID"] = eventID + } + if isProcessCreation(ed, eventID) { + // Real process creation — its fields are evidence. + getString(ed, "image", &out, "Image") + getString(ed, "parentImage", &out, "ParentImage") + getString(ed, "commandLine", &out, "CommandLine") + getString(ed, "user", &out, "User") + getString(ed, "parentUser", &out, "User") + getString(ed, "newProcessName", &out, "Image") // Security 4688 + getString(ed, "parentProcessName", &out, "ParentImage") + getString(ed, "subjectUserName", &out, "User") + highFidelity = true + } else { + // Non-creation event: extract for context only, LOW fidelity, + // so process_creation rules can never match it. + getString(ed, "image", &out, "Image") + getString(ed, "callerProcessName", &out, "Image") + getString(ed, "processName", &out, "Image") + getString(ed, "commandLine", &out, "CommandLine") + getString(ed, "user", &out, "User") + getString(ed, "subjectUserName", &out, "User") + if out["Image"] != "" || out["CommandLine"] != "" { + lowFidelity = true + } + } } } } @@ -135,6 +159,39 @@ func (Normalizer) Normalize(raw json.RawMessage) map[string]string { return out } +// isProcessCreation reports whether a Windows eventdata block is genuine +// process-creation telemetry: Sysmon EventID 1 / Security 4688, or — when the +// structured EventID is absent (as in some captured events) — the +// process-creation field signature (a command line plus an image/new-process +// name). Enumeration/network events (callerProcessName only, no command line) +// are correctly excluded. +func isProcessCreation(ed map[string]any, eventID string) bool { + if eventID == "1" || eventID == "4688" { + return true + } + return nonEmptyStr(ed, "commandLine") && (nonEmptyStr(ed, "image") || nonEmptyStr(ed, "newProcessName")) +} + +func nonEmptyStr(m map[string]any, key string) bool { + s, ok := m[key].(string) + return ok && s != "" +} + +// winEventID extracts data.win.system.eventID (string or numeric) or "". +func winEventID(win map[string]any) string { + sys, ok := win["system"].(map[string]any) + if !ok { + return "" + } + switch v := sys["eventID"].(type) { + case string: + return v + case float64: + return fmt.Sprintf("%d", int(v)) + } + return "" +} + func getString(m map[string]any, key string, out *map[string]string, target string) { if v, ok := m[key].(string); ok && (*out)[target] == "" { (*out)[target] = v diff --git a/internal/evaluator/rule.go b/internal/evaluator/rule.go index 0d35b4b..78c75af 100644 --- a/internal/evaluator/rule.go +++ b/internal/evaluator/rule.go @@ -12,11 +12,12 @@ import ( // Rule represents a parsed Sigma rule's detection block. type Rule struct { - Path string - Title string - Category string // logsource.category, e.g. "process_creation" - Detections map[string]FieldMap // search-identifier → field conditions - Condition Expr // parsed condition tree + Path string + Title string + Category string // logsource.category, e.g. "process_creation" + Detections map[string]FieldMap // search-identifier → field conditions + Condition Expr // parsed condition tree + Unsupported string // non-empty if the rule uses a feature we can't evaluate } // FieldMap is a search-identifier's field→value mapping with modifiers. @@ -26,6 +27,16 @@ type FieldMap map[string]FieldEntry type FieldEntry struct { Values []string // list of values to match against Modifiers []string // e.g. "contains", "startswith", "endswith", "all" + MatchNull bool // Sigma `Field: null` — matches when the field is absent/empty +} + +// supportedModifiers is the set of value modifiers this engine evaluates +// correctly. Any other modifier (base64, cidr, windash, fieldref, …) means the +// rule cannot be faithfully evaluated; such a rule is reported INCONCLUSIVE +// rather than silently scored as a clean MISS. +var supportedModifiers = map[string]bool{ + "all": true, "endswith": true, "startswith": true, "contains": true, + "re": true, "lt": true, "lte": true, "gt": true, "gte": true, } // Expr is a node in the condition expression tree. @@ -97,6 +108,18 @@ func (RuleParser) Parse(path string) (*Rule, error) { rule.Detections[name] = fm } + // Flag any unsupported modifier so the evaluator can report INCONCLUSIVE + // instead of a misleading MISS. + for _, fm := range rule.Detections { + for _, entry := range fm { + for _, m := range entry.Modifiers { + if !supportedModifiers[m] { + rule.Unsupported = "unsupported Sigma modifier: " + m + } + } + } + } + // Parse condition expression cond, err := parseCondition(raw.Detection.Condition) if err != nil { @@ -156,6 +179,8 @@ func parseFieldMap(val any) (FieldMap, error) { // Parse value(s). Scalars may be strings, numbers, or booleans // (e.g. `EventID: 4688`, `Count|gt: 5`); coerce each to its string form. switch vv := v.(type) { + case nil: + entry.MatchNull = true // Sigma `Field: null` — match when absent/empty case string: entry.Values = []string{vv} case []any: @@ -259,8 +284,10 @@ func (p *condParser) parseAnd() (Expr, error) { func (p *condParser) parseNot() (Expr, error) { p.skip() - if strings.HasPrefix(p.s[p.pos:], "not ") { - p.pos += 4 + rest := p.s[p.pos:] + // Accept "not " and also "not(" (no space before the paren). + if strings.HasPrefix(rest, "not ") || strings.HasPrefix(rest, "not\t") || strings.HasPrefix(rest, "not(") { + p.pos += 3 // consume "not"; the recursive skip() handles the space or "(" child, err := p.parseNot() if err != nil { return nil, err @@ -333,43 +360,33 @@ func (p *condParser) parseOneOf() (Expr, error) { } // Read identifiers var names []string - for { - p.skip() - if strings.HasPrefix(p.s[p.pos:], "them") { - p.pos += 4 - // "1 of them" — we don't know the list, pass through - break - } - if strings.HasPrefix(p.s[p.pos:], "(") { - // "1 of (a, b, c, ...)" - p.pos++ // skip ( - for { - id, err := p.parseIdent() - if err != nil { - return nil, err - } - if id != nil { - names = append(names, id.(IdentExpr).Name) - } - p.skip() - if p.pos < len(p.s) && p.s[p.pos] == ',' { - p.pos++ - continue - } - if p.pos < len(p.s) && p.s[p.pos] == ')' { - p.pos++ - break - } - break + p.skip() + switch { + case strings.HasPrefix(p.s[p.pos:], "them"): + p.pos += 4 // "1 of them" — Names stays empty; the matcher expands it + case p.pos < len(p.s) && p.s[p.pos] == '(': + p.pos++ // skip ( + for { + if name := p.readAggIdent(); name != "" { + names = append(names, name) + } + p.skip() + if p.pos < len(p.s) && p.s[p.pos] == ',' { + p.pos++ + continue + } + if p.pos < len(p.s) && p.s[p.pos] == ')' { + p.pos++ } break } - // "1 of them" without parens — just "them" - if strings.HasPrefix(p.s[p.pos:], "them") { - p.pos += 4 - break + default: + // Bare identifier or glob pattern, e.g. "1 of selection_*". Consuming it + // here (rather than stopping at '*') is what lets the following + // "and not filter" parse instead of being silently dropped. + if name := p.readAggIdent(); name != "" { + names = append(names, name) } - break } return OneOfExpr{N: n, Names: names}, nil } @@ -383,12 +400,8 @@ func (p *condParser) parseAllOf() (Expr, error) { return AllOfExpr{Names: names}, nil } for { - id, err := p.parseIdent() - if err != nil { - return nil, err - } - if id != nil { - names = append(names, id.(IdentExpr).Name) + if name := p.readAggIdent(); name != "" { + names = append(names, name) } p.skip() if p.pos < len(p.s) && p.s[p.pos] == ',' { @@ -403,3 +416,14 @@ func (p *condParser) parseAllOf() (Expr, error) { func isIdent(c byte) bool { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' } + +// readAggIdent reads an aggregate operand identifier, allowing the `*`/`?` +// glob characters so patterns like "selection_*" are captured whole. +func (p *condParser) readAggIdent() string { + p.skip() + start := p.pos + for p.pos < len(p.s) && (isIdent(p.s[p.pos]) || p.s[p.pos] == '*' || p.s[p.pos] == '?') { + p.pos++ + } + return p.s[start:p.pos] +} diff --git a/internal/evaluator/sigma.go b/internal/evaluator/sigma.go index 8f63284..b8e11b4 100644 --- a/internal/evaluator/sigma.go +++ b/internal/evaluator/sigma.go @@ -48,6 +48,12 @@ func (e RuleMatcherEvaluator) Evaluate(rule model.SigmaRule, events []model.Even return model.Errored, nil, fmt.Errorf("parse rule %s: %w", rule.Path, err) } + // A rule using a feature we cannot faithfully evaluate must NOT be scored as + // a clean MISS — that silently under-reports coverage. Report INCONCLUSIVE. + if parsedRule.Unsupported != "" { + return model.Inconclusive, nil, nil + } + // For process-creation rules, only genuine process-creation telemetry can // justify a verdict. Command-output/metadata scrapes (full_log, decoder // name) are tagged low fidelity by the normalizer and are excluded here, so From 2d5a3464b3393638a96b83f5a4dee012863a5b72 Mon Sep 17 00:00:00 2001 From: jayelbotvibe-web Date: Sat, 29 Aug 2026 15:08:32 +0800 Subject: [PATCH 03/13] feat(atomic): resolve real Atomic Red Team commands instead of a hardcoded one Every technique executed the same hardcoded "id; whoami". atomic_ids was parsed from plan YAML, carried through emulation stages and the arbiter feed, and written into the proof chain -- but never resolved to a command. A ten-technique campaign ran one command ten times and labelled the results T1082, T1033, T1518..., so every verdict but T1059.004's was about a command nobody asked for. The README's claim that Execute "runs Atomic Red Team tests" was not true of the code. New internal/atomic resolves each atomic_id against the pinned ART tree. Resolution is strict -- override, then vendored ART, then error -- with no default command, because a fabricated input produces a fabricated verdict. Fixing it immediately exposed two more defects: - Five of ten techniques in plans/discovery.yml named WINDOWS atomics while the campaign targets a Linux victim (T1082-1, T1033-1, T1007-1, T1016-1, T1049-1 are all command_prompt tests). Invisible while everything ran id; whoami. Plans now name the correct Linux tests and a platform mismatch is a loud ERROR. - T1059.004-1 writes a script that pings 8.8.8.8, traffic leaving the isolated lab network against the project's own containment rule. Its host input is pinned to loopback via an override that keeps the upstream command. Also: ART was recorded, not pinned. lab-fetch.sh cloned HEAD then overwrote the pin file with whatever it got, so a re-clone could silently change every command the engine runs. scripts/fetch-atomics.sh checks out the recorded commit and verifies HEAD matches. Adds SKIPPED_PREREQ: an atomic whose own prerequisites are unmet never ran, and is no longer reported as a detection miss. A test fails if any two techniques in the shipped campaign resolve to the same command, making the original defect unshippable. Co-Authored-By: Claude Opus 5 --- Makefile | 14 +- cmd/purpleloop/main.go | 650 ++++++++++++++++++++++++----- emulation/apt29-subset.yml | 10 +- emulation/discovery-chain.yml | 10 +- internal/atomic/atomic.go | 342 +++++++++++++++ internal/atomic/atomic_test.go | 183 ++++++++ internal/feed/static.go | 2 + internal/model/model.go | 82 +++- mappings/atomic-overrides.yml | 59 +++ mappings/attack_rule_map.seed.json | 43 -- plans/discovery.yml | 10 +- scripts/fetch-atomics.sh | 39 ++ scripts/lab-fetch.sh | 13 +- 13 files changed, 1280 insertions(+), 177 deletions(-) create mode 100644 internal/atomic/atomic.go create mode 100644 internal/atomic/atomic_test.go create mode 100644 mappings/atomic-overrides.yml delete mode 100644 mappings/attack_rule_map.seed.json create mode 100755 scripts/fetch-atomics.sh diff --git a/Makefile b/Makefile index 7f0c93f..c8e6129 100644 --- a/Makefile +++ b/Makefile @@ -6,10 +6,10 @@ export TECHNIQUE ?= T1059.004 LAB_DIR := lab/wazuh-docker/single-node -.PHONY: help host-prep lab-fetch lab-up lab-down verify run build vet reset snapshot restore +.PHONY: help host-prep lab-fetch atomics lab-up lab-down archives verify run build vet reset snapshot restore help: ## list targets - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ + @grep -hE '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \ awk 'BEGIN{FS=":.*?## "}{printf " \033[36m%-12s\033[0m %s\n", $$1, $$2}' host-prep: ## one-time host tweak (vm.max_map_count) + tooling check @@ -18,9 +18,15 @@ host-prep: ## one-time host tweak (vm.max_map_count) + tooling check lab-fetch: ## pull pinned Wazuh single-node + Atomic Red Team, install override @bash scripts/lab-fetch.sh +atomics: ## vendor Atomic Red Team at the pinned commit (required before any run) + @bash scripts/fetch-atomics.sh + lab-up: ## bring the merged Wazuh stack + victim up @cd $(LAB_DIR) && docker compose up -d - @echo "give the indexer ~60-90s, then: make verify" + @echo "give the indexer ~60-90s, then: make archives && make verify" + +archives: ## enable Wazuh archive logging (the collector reads archives.json) + @bash scripts/enable-archives.sh lab-down: ## stop the stack (keeps data) @cd $(LAB_DIR) && docker compose down @@ -32,7 +38,7 @@ verify: ## binary health check + canary — must pass before proceeding canary: ## run the pipeline positive control @go run ./cmd/purpleloop canary -build: ## compile everything +build: ## compile everything (scoped: the vendored ART tree is not Go source) @go build ./cmd/... ./internal/... vet: ## static checks diff --git a/cmd/purpleloop/main.go b/cmd/purpleloop/main.go index ae0917c..be41df9 100644 --- a/cmd/purpleloop/main.go +++ b/cmd/purpleloop/main.go @@ -10,37 +10,41 @@ import ( "net" "net/http" "os" + "path/filepath" "strings" "time" + "github.com/jayelbotvibe-web/purple-loop/internal/atomic" "github.com/jayelbotvibe-web/purple-loop/internal/canary" + "github.com/jayelbotvibe-web/purple-loop/internal/capture" "github.com/jayelbotvibe-web/purple-loop/internal/collector" "github.com/jayelbotvibe-web/purple-loop/internal/evaluator" "github.com/jayelbotvibe-web/purple-loop/internal/executor" "github.com/jayelbotvibe-web/purple-loop/internal/feed" + "github.com/jayelbotvibe-web/purple-loop/internal/lab" + "github.com/jayelbotvibe-web/purple-loop/internal/mapping" "github.com/jayelbotvibe-web/purple-loop/internal/model" "github.com/jayelbotvibe-web/purple-loop/internal/report" "github.com/jayelbotvibe-web/purple-loop/internal/server" ) -// techniqueRuleMap maps technique IDs to their Sigma rule files. -var techniqueRuleMap = map[string]string{ - "T1059.004": "detections/linux/proc_creation_susp_shell.yml", - "T1087.001": "detections/linux/T1087.001.yml", - "T1082": "detections/linux/T1082.yml", - "T1033": "detections/linux/T1033.yml", - "T1007": "detections/linux/T1007.yml", - "T1016": "detections/linux/T1016.yml", - "T1049": "detections/linux/T1049.yml", - "T1069.001": "detections/linux/T1069.001.yml", - "T1069": "detections/linux/T1069.001.yml", - "T1135": "detections/linux/T1135.yml", - "T1518": "detections/linux/T1518.yml", -} +// Default locations for the vendored Atomic Red Team tree and the lab +// overrides that sit in front of it. +const ( + defaultAtomicsRoot = "lab/atomic-red-team" + defaultOverrides = "mappings/atomic-overrides.yml" + defaultRuleMap = "mappings/attack_rule_map.json" + defaultTargets = "lab/targets.yml" + atomicsCommitRecord = "mappings/atomic-red-team.commit" + + // collectWindow is how far past the execution the collector looks. Phase 3 + // replaces this with a poll bounded by the ingestion watermark. + collectWindow = 10 * time.Minute +) func main() { if len(os.Args) < 2 { - fmt.Fprintln(os.Stderr, "usage: purpleloop run|canary [...]") + fmt.Fprintln(os.Stderr, "usage: purpleloop run|replay|precision|diff|canary|serve [...]") os.Exit(1) } @@ -54,6 +58,21 @@ func main() { return } + if os.Args[1] == "replay" { + runReplay() + return + } + + if os.Args[1] == "precision" { + runPrecision() + return + } + + if os.Args[1] == "diff" { + runDiff() + return + } + if os.Args[1] != "run" { fmt.Fprintln(os.Stderr, "usage: purpleloop run [--technique | --plan ] [flags]") os.Exit(2) @@ -67,6 +86,15 @@ func main() { dryRun := fs.Bool("dry-run", false, "run the pipeline without a live lab") victim := fs.String("victim-container", "", "Docker container for execution (e.g. purpleloop-victim)") manager := fs.String("manager-container", "", "Docker container for Wazuh manager") + atomicsRoot := fs.String("atomics", defaultAtomicsRoot, "vendored Atomic Red Team checkout") + ruleMapPath := fs.String("rule-map", defaultRuleMap, "technique -> atomic -> expected rules mapping") + targetsPath := fs.String("targets", defaultTargets, "lab target inventory") + settleTimeout := fs.Duration("settle-timeout", 60*time.Second, + "how long to wait for an execution's telemetry before calling it NO_TELEMETRY. "+ + "Must exceed the victim's collection interval: Wazuh command monitoring "+ + "defaults to 360s, so a 60s budget can never see it") + settlePoll := fs.Duration("settle-poll", 3*time.Second, "gap between telemetry polls") + overridesPath := fs.String("overrides", defaultOverrides, "lab overrides for unusable atomics") _ = fs.Parse(os.Args[2:]) ctx := context.Background() @@ -88,31 +116,69 @@ func main() { fmt.Fprintln(os.Stderr, "└─────────────────────────────────────────────────────────────┘") } + // No registry, no run. Every technique's command comes from the vendored + // Atomic Red Team tree; without it there is nothing honest to execute. + reg, err := loadRegistry(*atomicsRoot, *overridesPath) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + rules, err := mapping.LoadRuleMap(*ruleMapPath) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + targets, err := lab.Load(*targetsPath) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + opts := runOpts{ + output: *output, + dryRun: *dryRun, + victimContainer: *victim, + managerContainer: *manager, + registry: reg, + rules: rules, + targets: targets, + settleTimeout: *settleTimeout, + settlePoll: *settlePoll, + } + switch { case *arbiterFile != "": - if err := runArbiter(ctx, *arbiterFile, *output, *dryRun, *victim, *manager); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + err = runArbiter(ctx, *arbiterFile, opts) case *emulationFile != "": - if err := runEmulation(ctx, *emulationFile, *output, *dryRun, *victim, *manager); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + err = runEmulation(ctx, *emulationFile, opts) case *planFile != "": - if err := runCampaign(ctx, *planFile, *output, *dryRun, *victim, *manager); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + err = runCampaign(ctx, *planFile, opts) case *technique != "": - if err := runOne(ctx, *technique, *output, *dryRun, *victim, *manager); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) - } + err = runOne(ctx, *technique, opts) default: fmt.Fprintln(os.Stderr, "error: --technique or --plan is required") os.Exit(2) } + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +// loadRegistry opens the vendored Atomic Red Team tree. A run cannot proceed +// without it: with no registry there is no real command to execute, and the +// engine must never fall back to a fabricated one. +func loadRegistry(root, overrides string) (*atomic.Registry, error) { + reg, err := atomic.Load(root, overrides) + if err != nil { + return nil, err + } + if b, err := os.ReadFile(atomicsCommitRecord); err == nil { + reg.SetCommit(string(b)) + } + return reg, nil } func newExec(dryRun bool, victimContainer string) model.Executor { @@ -148,6 +214,18 @@ type runOpts struct { dryRun bool victimContainer string managerContainer string + campaign string + settleTimeout time.Duration + settlePoll time.Duration + registry *atomic.Registry + rules *mapping.RuleMap + targets *lab.Targets +} + +// with returns a copy of the options labelled with the campaign that is running. +func (o runOpts) with(campaign string) runOpts { + o.campaign = campaign + return o } // synthetic reports whether any pipeline stage is dry/synthetic. Such a run's @@ -162,44 +240,257 @@ func (o runOpts) synthetic() bool { // - canary gating (a real run whose positive control does not fire is // INCONCLUSIVE and its coverage must not be presented as valid). func runTasks(ctx context.Context, tasks []model.TechniqueTask, o runOpts) error { - exec := newExec(o.dryRun, o.victimContainer) - coll := newColl(o.dryRun, o.managerContainer) - eval := evaluator.RuleMatcherEvaluator{RulesDir: "detections/linux"} - target := model.Target{Host: "victim01", Kind: "linux"} - - result := model.CampaignResult{StartedAt: time.Now().UTC(), Synthetic: o.synthetic()} - - // Canary gate — the pipeline positive control, run against the SAME - // executor/collector/target the campaign uses (the Linux/Docker path). - // Skipped for synthetic runs (already marked non-evidentiary above). - if !result.Synthetic { - cr := canary.Check(ctx, canary.NewMarker(), exec, coll, target.Kind, target) - result.CanaryHealthy = cr.Healthy - if !cr.Healthy { - result.Inconclusive = true - result.CanaryDetail = canaryFailDetail(cr) - fmt.Fprintf(os.Stderr, "INCONCLUSIVE: %s — coverage will not be reported as valid\n", result.CanaryDetail) + result := model.CampaignResult{ + StartedAt: time.Now().UTC(), + Synthetic: o.synthetic(), + Campaign: o.campaign, + AtomicsCommit: o.registry.Commit(), + UntestedRules: untestedFor(o.rules, tasks), + } + + // Group work by the platform its rule-map entry declares, so a campaign can + // span the Linux container and the Windows VM in one run. Previously every + // technique went to a hardcoded Linux target and the Windows rule was + // unreachable from `run`. + byPlatform := map[string][]model.TechniqueTask{} + var order []string + for _, task := range tasks { + p := o.platformFor(task.TechniqueID) + if _, seen := byPlatform[p]; !seen { + order = append(order, p) + } + byPlatform[p] = append(byPlatform[p], task) + } + + for _, platform := range order { + plat := o.stackFor(platform) + if plat.err != nil { + for _, task := range byPlatform[platform] { + result.Chains = append(result.Chains, model.ProofChain{ + TechniqueID: task.TechniqueID, + SourceCVE: task.SourceCVE, + ArbiterPriority: task.Priority, + Verdict: model.Errored, + Note: plat.err.Error(), + }) + } + continue + } + + // Canary gate, per platform. A campaign spanning two hosts must prove + // BOTH pipelines: a healthy Linux canary says nothing about whether + // Windows telemetry is flowing. + if !result.Synthetic { + cr := canary.Checker{RulesDir: "detections"}.Run(ctx, canary.NewMarker(), + plat.exec, plat.coll, platform, plat.target) + if !cr.Healthy { + result.Inconclusive = true + result.CanaryDetail = appendDetail(result.CanaryDetail, canaryFailDetail(cr)) + fmt.Fprintf(os.Stderr, "INCONCLUSIVE: %s — coverage will not be reported as valid\n", + canaryFailDetail(cr)) + } else { + result.CanaryHealthy = true + } + } + + for _, task := range byPlatform[platform] { + chain, err := runTechnique(ctx, plat.exec, plat.coll, plat.eval, o.registry, o.rules, task, + plat.target, collector.Settler{Timeout: o.settleTimeout, PollInterval: o.settlePoll}) + if err != nil { + chain = model.ProofChain{TechniqueID: task.TechniqueID, Verdict: model.Errored, Note: err.Error()} + } + chain.SourceCVE = task.SourceCVE + chain.ArbiterPriority = task.Priority + result.Chains = append(result.Chains, chain) } } - for _, task := range tasks { - chain, err := runTechnique(ctx, exec, coll, eval, task, target) - if err != nil { - chain = model.ProofChain{TechniqueID: task.TechniqueID, Verdict: model.Errored} + // A failed positive control on ANY platform invalidates the run: without a + // working pipeline no per-technique verdict is trustworthy. + if result.Inconclusive { + result.CanaryHealthy = false + for i := range result.Chains { + if result.Chains[i].Verdict != model.Errored { + result.Chains[i].Verdict = model.Inconclusive + } } - chain.SourceCVE = task.SourceCVE - chain.ArbiterPriority = task.Priority - if result.Inconclusive { - // Without a working positive control, no per-technique verdict is - // trustworthy — do not present DETECTED/MISSED as coverage. - chain.Verdict = model.Inconclusive + } + + labelAttribution(result.Chains, o.synthetic(), primaryHost(o)) + + // Capture the raw telemetry so this run can be re-evaluated against future + // rule changes with no lab. Synthetic runs are refused by the writer. + if dir := captureDir(o.output, result.StartedAt); dir != "" { + if err := capture.Write(dir, result, primaryHost(o)); err != nil { + fmt.Fprintf(os.Stderr, "warning: could not capture telemetry: %v\n", err) } - result.Chains = append(result.Chains, chain) } return newReporter(o.output).Write(result) } +// platformStack is the executor/collector/evaluator triple for one platform. +type platformStack struct { + exec model.Executor + coll model.Collector + eval model.Evaluator + target model.Target + err error +} + +// platformFor reports which lab platform a technique belongs on. +func (o runOpts) platformFor(technique string) string { + if o.rules != nil { + if e, ok := o.rules.Entry(technique); ok && e.Platform != "" { + return strings.ToLower(e.Platform) + } + } + return "linux" +} + +// stackFor builds the pipeline for one platform from the lab inventory. +func (o runOpts) stackFor(platform string) platformStack { + t, ok := o.targets.ForPlatform(platform) + if !ok { + return platformStack{err: fmt.Errorf( + "no lab target declared for platform %q — add one to lab/targets.yml", platform)} + } + + st := platformStack{ + target: t.Model(), + eval: evaluator.RuleMatcherEvaluator{RulesDir: orDefaultStr(t.RulesDir, "detections/"+platform)}, + coll: newColl(o.dryRun, o.managerContainer), + } + + switch { + case o.synthetic(): + st.exec = executor.DryExecutor{} + case t.Executor == "ssh": + host, user, pass := t.SSHConfig() + if host == "" { + return platformStack{err: fmt.Errorf( + "target %s needs an SSH host (set %s)", t.Name, t.SSHHostEnv)} + } + st.exec = &executor.SSHExecutor{Host: host, User: user, Password: pass} + default: + container := t.Container + if o.victimContainer != "" { + container = o.victimContainer + } + st.exec = executor.DockerExecutor{Container: container} + } + return st +} + +// primaryHost is the host recorded on captured telemetry. A single-platform +// campaign names its target; a mixed one records none, because the dataset +// spans hosts and claiming one would be wrong. +func primaryHost(o runOpts) string { + if t, ok := o.targets.ForPlatform("linux"); ok { + return t.Name + } + return "" +} + +func appendDetail(existing, add string) string { + if existing == "" { + return add + } + return existing + "; " + add +} + +func orDefaultStr(v, def string) string { + if strings.TrimSpace(v) == "" { + return def + } + return v +} + +// untestedFor names every rule this campaign will not exercise, with why. +// Rules declared untested in the map keep their declared reason; a mapped rule +// whose technique is absent from this campaign is reported as not-in-campaign. +// Coverage is only honest next to what it did not cover. +func untestedFor(rules *mapping.RuleMap, tasks []model.TechniqueTask) map[string]string { + if rules == nil { + return nil + } + out := map[string]string{} + for rule, reason := range rules.UntestedRules { + out[rule] = reason + } + + inCampaign := map[string]bool{} + for _, t := range tasks { + for _, r := range rules.ExpectedRules(t.TechniqueID) { + inCampaign[r] = true + } + } + for rule := range rules.MappedRules() { + if !inCampaign[rule] { + out[rule] = "not_in_campaign — mapped to a technique this plan does not run" + } + } + if len(out) == 0 { + return nil + } + return out +} + +// captureDir mirrors DashboardReporter's per-run layout so a run's dataset sits +// beside its coverage.json. +func captureDir(output string, startedAt time.Time) string { + dir := output + if dir == "" || strings.HasSuffix(dir, ".html") || strings.HasSuffix(dir, ".json") { + dir = "reports" + } + runID := fmt.Sprintf("campaign-%s", startedAt.UTC().Format("20060102T150405Z")) + return filepath.Join(dir, "runs", runID) +} + +// labelAttribution records how tightly each verdict is bound to its own +// execution. Overlap is a campaign-level property — two techniques whose +// collection windows intersect can each pick up the other's events — so it can +// only be decided once every window is known. +// +// This does not change any verdict. It states the quality of the evidence +// behind one, which is the difference between a coverage number and a claim. +func labelAttribution(chains []model.ProofChain, synthetic bool, host string) { + for i := range chains { + c := &chains[i] + + // A synthetic pipeline produced no real telemetry, so nothing can be + // attributed to a real execution. + if synthetic { + c.Attribution = model.Unscoped + continue + } + // A technique that never ran has no window to scope. + if c.Window.Start.IsZero() || c.Window.End.IsZero() { + c.Attribution = model.Unscoped + continue + } + + overlapped := false + for j := range chains { + if i == j || chains[j].Window.Start.IsZero() { + continue + } + if c.Window.Overlaps(chains[j].Window) { + overlapped = true + break + } + } + switch { + case overlapped: + c.Attribution = model.WindowOverlap + case host != "": + c.Attribution = model.WindowAndHostScoped + default: + c.Attribution = model.WindowScoped + } + } +} + func canaryFailDetail(cr canary.Result) string { if cr.Err != nil { return fmt.Sprintf("canary not detected on %s: %v", cr.Platform, cr.Err) @@ -207,7 +498,7 @@ func canaryFailDetail(cr canary.Result) string { return fmt.Sprintf("canary not detected on %s", cr.Platform) } -func runCampaign(ctx context.Context, planPath, output string, dryRun bool, victimContainer, managerContainer string) error { +func runCampaign(ctx context.Context, planPath string, o runOpts) error { var f feed.StaticFeed if err := f.Load(planPath); err != nil { return fmt.Errorf("load plan: %w", err) @@ -216,10 +507,10 @@ func runCampaign(ctx context.Context, planPath, output string, dryRun bool, vict if err != nil { return fmt.Errorf("feed: %w", err) } - return runTasks(ctx, tasks, runOpts{output, dryRun, victimContainer, managerContainer}) + return runTasks(ctx, tasks, o.with(f.Name)) } -func runArbiter(ctx context.Context, arbiterPath, output string, dryRun bool, victimContainer, managerContainer string) error { +func runArbiter(ctx context.Context, arbiterPath string, o runOpts) error { var f feed.ArbiterFeed if err := f.Load(arbiterPath); err != nil { return fmt.Errorf("load arbiter export: %w", err) @@ -228,10 +519,10 @@ func runArbiter(ctx context.Context, arbiterPath, output string, dryRun bool, vi if err != nil { return fmt.Errorf("arbiter feed: %w", err) } - return runTasks(ctx, tasks, runOpts{output, dryRun, victimContainer, managerContainer}) + return runTasks(ctx, tasks, o.with("arbiter")) } -func runEmulation(ctx context.Context, emuPath, output string, dryRun bool, victimContainer, managerContainer string) error { +func runEmulation(ctx context.Context, emuPath string, o runOpts) error { plan, err := feed.LoadEmulation(emuPath) if err != nil { return fmt.Errorf("load emulation: %w", err) @@ -240,69 +531,216 @@ func runEmulation(ctx context.Context, emuPath, output string, dryRun bool, vict for _, stage := range plan.Stages { tasks = append(tasks, stage.ToTasks()...) } - return runTasks(ctx, tasks, runOpts{output, dryRun, victimContainer, managerContainer}) + return runTasks(ctx, tasks, o.with(plan.Name)) } -func runOne(ctx context.Context, technique, output string, dryRun bool, victimContainer, managerContainer string) error { - task := model.TechniqueTask{TechniqueID: technique, AtomicIDs: []string{technique + "-1"}} - return runTasks(ctx, []model.TechniqueTask{task}, runOpts{output, dryRun, victimContainer, managerContainer}) +func runOne(ctx context.Context, technique string, o runOpts) error { + // Deliberately leave AtomicIDs empty so atomicIDFor consults the rule map. + // Synthesizing "-1" here pre-empted it and picked ART's first + // test, which for most techniques is the WINDOWS one — `--technique T1082` + // resolved to a command_prompt atomic and errored on the Linux victim, even + // though the rule map names T1082-3. + task := model.TechniqueTask{TechniqueID: technique} + return runTasks(ctx, []model.TechniqueTask{task}, o.with(technique)) } -func runTechnique(ctx context.Context, exec model.Executor, coll model.Collector, eval model.Evaluator, task model.TechniqueTask, target model.Target) (model.ProofChain, error) { - atomicID := task.TechniqueID + "-1" - if len(task.AtomicIDs) > 0 { - atomicID = task.AtomicIDs[0] +// atomicIDFor picks the atomic this task should run: the plan's choice first, +// then the mapping's. Only if neither declares one does it guess the +// technique's first test — and the resolved test's platform is still checked +// before anything executes. +func atomicIDFor(task model.TechniqueTask, rules *mapping.RuleMap) string { + if len(task.AtomicIDs) > 0 && strings.TrimSpace(task.AtomicIDs[0]) != "" { + return strings.TrimSpace(task.AtomicIDs[0]) + } + if rules != nil { + if id := rules.AtomicFor(task.TechniqueID); id != "" { + return id + } } + return task.TechniqueID + "-1" +} - atomic := model.AtomicTest{ - ID: atomicID, - TechniqueID: task.TechniqueID, - Command: "id; whoami", - Executor: "sh", +// unmetPrereq runs each dependency's prereq_command on the target and returns a +// description of the first one that fails. A technique whose prerequisites are +// not met never ran, so it is SKIPPED_PREREQ — reporting it as MISSED would +// invent a detection gap out of an unmet dependency. +func unmetPrereq(ctx context.Context, exec model.Executor, test atomic.Test, target model.Target) string { + for _, dep := range test.Dependencies { + if dep.PrereqCommand == "" { + continue + } + probe := model.AtomicTest{ + ID: test.ID + "-prereq", + TechniqueID: test.TechniqueID, + Command: dep.PrereqCommand, + Executor: test.DependencyExecutor, + } + res, err := exec.Run(ctx, probe, target) + if err != nil { + return fmt.Sprintf("prerequisite check failed to run (%s): %v", dep.Description, err) + } + if res.ExitCode != 0 { + desc := dep.Description + if desc == "" { + desc = dep.PrereqCommand + } + return "unmet prerequisite: " + desc + } } + return "" +} - run, err := exec.Run(ctx, atomic, target) +// runTechnique resolves the task's atomic to a REAL command, runs it, collects +// telemetry and evaluates the mapped rule. +// +// Resolution is strict on purpose. Every failure below returns a verdict that +// names what went wrong rather than a coverage number: an unresolvable atomic, +// a platform mismatch and an unmet prerequisite are all reasons a detection was +// never exercised, and none of them is evidence that a detection is missing. +func runTechnique(ctx context.Context, exec model.Executor, coll model.Collector, eval model.Evaluator, + reg *atomic.Registry, rules *mapping.RuleMap, task model.TechniqueTask, target model.Target, + settler collector.Settler) (model.ProofChain, error) { + + atomicID := atomicIDFor(task, rules) + chain := model.ProofChain{TechniqueID: task.TechniqueID} + + test, err := reg.Resolve(atomicID) if err != nil { - return model.ProofChain{}, fmt.Errorf("execute: %w", err) + chain.Verdict = model.Errored + chain.Note = err.Error() + return chain, nil } - // Cleanup is best-effort per atomic after run - _ = exec.Cleanup(ctx, atomic, target) - // Let Wazuh ingest telemetry before querying - time.Sleep(10 * time.Second) + chain.Atomic = model.AtomicTest{ + ID: test.ID, + TechniqueID: test.TechniqueID, + Name: test.Name, + GUID: test.GUID, + Command: test.Command, + CleanupCommand: test.CleanupCommand, + Executor: test.Executor, + Source: test.Source, + OverrideReason: test.OverrideReason, + } + + if !test.SupportsPlatform(target.Kind) { + chain.Verdict = model.Errored + chain.Note = fmt.Sprintf("atomic %s supports %v, but target %s is %s — fix the plan's atomic_ids", + test.ID, test.SupportedPlatforms, target.Host, target.Kind) + return chain, nil + } - events, err := coll.Query(ctx, run.Window(10*time.Minute), target.Host) + if unmet := unmetPrereq(ctx, exec, test, target); unmet != "" { + chain.Verdict = model.SkippedPrereq + chain.Note = unmet + return chain, nil + } + + run, err := exec.Run(ctx, chain.Atomic, target) if err != nil { - return model.ProofChain{}, fmt.Errorf("collect: %w", err) + return chain, fmt.Errorf("execute: %w", err) } + chain.ExecutedAt = run.StartedAt + + // Cleanup is best-effort per atomic after run. + _ = exec.Cleanup(ctx, chain.Atomic, target) - // Resolve rule path from embedded technique mapping - rulePath := "" // empty → evaluator returns MISSED for unmapped techniques - if mapped, ok := techniqueRuleMap[task.TechniqueID]; ok { - rulePath = mapped + window := run.Window(collectWindow) + chain.Window = window + + expected := rules.ExpectedRules(task.TechniqueID) + chain.RulesExpected = expected + if len(expected) == 0 { + chain.Verdict = model.Errored + chain.Note = "no Sigma rule mapped for " + task.TechniqueID + " in the rule map" + return chain, nil } - rule := model.SigmaRule{Path: rulePath, Title: task.TechniqueID} - verdict, evidence, err := eval.Evaluate(rule, events) + + // Poll until an expected rule matches or ingestion settles, rather than + // sleeping a fixed interval and querying once. + settled, err := settler.Settle(ctx, coll, window, target.Host, + func(events []model.Event) bool { + v, _, _, err := evaluateExpected(eval, expected, task.TechniqueID, events) + return err == nil && v == model.Detected + }) if err != nil { - return model.ProofChain{}, fmt.Errorf("evaluate: %w", err) + return chain, fmt.Errorf("collect: %w", err) + } + events := settled.Events + chain.EventsCollected = len(events) + chain.Collected = events + if settled.TimedOut && len(events) == 0 { + chain.Note = fmt.Sprintf("no telemetry arrived within %s of execution", settled.Waited.Round(time.Second)) } - ruleMatched := "" - if verdict == model.Detected { - ruleMatched = rule.Path + // A technique is DETECTED when ANY expected rule matches. Evaluating only + // the first would understate coverage whenever a second rule is the one that + // legitimately catches the behaviour. + verdict, evidence, matched, err := evaluateExpected(eval, expected, task.TechniqueID, events) + if err != nil { + return chain, fmt.Errorf("evaluate: %w", err) } - return model.ProofChain{ - TechniqueID: task.TechniqueID, - SourceCVE: task.SourceCVE, - ArbiterPriority: task.Priority, - Atomic: atomic, - ExecutedAt: run.StartedAt, - EventsCollected: len(events), - RuleMatched: ruleMatched, - Verdict: verdict, - Evidence: evidence, - }, nil + chain.Verdict = verdict + chain.Evidence = evidence + chain.RulesMatched = matched + if len(matched) > 0 { + chain.RuleMatched = matched[0] + chain.DetectLatencyMS = detectLatency(run.StartedAt, evidence) + } + return chain, nil +} + +// evaluateExpected runs every expected rule and reports the best outcome. The +// verdict precedence is DETECTED > MISSED > INCONCLUSIVE > NO_TELEMETRY: a rule +// that matched is proof of detection regardless of what the others did, while +// NO_TELEMETRY only stands if no rule ever saw usable events. +func evaluateExpected(eval model.Evaluator, expected []string, title string, events []model.Event) ( + model.Verdict, []model.Event, []string, error) { + + best := model.NoTelemetry + var evidence []model.Event + var matched []string + + rank := map[model.Verdict]int{ + model.NoTelemetry: 0, + model.Inconclusive: 1, + model.Missed: 2, + model.Detected: 3, + } + + for _, path := range expected { + v, ev, err := eval.Evaluate(model.SigmaRule{Path: path, Title: title}, events) + if err != nil { + return model.Errored, nil, nil, err + } + if v == model.Detected { + matched = append(matched, path) + evidence = append(evidence, ev...) + } + if rank[v] > rank[best] { + best = v + } + } + return best, evidence, matched, nil +} + +// detectLatency is the gap between execution start and the earliest matching +// event — how long the pipeline took to make the behaviour visible. +func detectLatency(start time.Time, evidence []model.Event) int64 { + earliest := time.Time{} + for _, e := range evidence { + if e.Timestamp.IsZero() { + continue + } + if earliest.IsZero() || e.Timestamp.Before(earliest) { + earliest = e.Timestamp + } + } + if earliest.IsZero() || earliest.Before(start) { + return 0 + } + return earliest.Sub(start).Milliseconds() } func runCanaryCmd() { diff --git a/emulation/apt29-subset.yml b/emulation/apt29-subset.yml index 26fe721..d4f1726 100644 --- a/emulation/apt29-subset.yml +++ b/emulation/apt29-subset.yml @@ -14,15 +14,15 @@ stages: description: Enumerate users, systems, and network techniques: - technique_id: T1033 - atomic_ids: [T1033-1] + atomic_ids: [T1033-2] - technique_id: T1087.001 atomic_ids: [T1087.001-1] - technique_id: T1082 - atomic_ids: [T1082-1] + atomic_ids: [T1082-3] - technique_id: T1016 - atomic_ids: [T1016-1] + atomic_ids: [T1016-3] - technique_id: T1049 - atomic_ids: [T1049-1] + atomic_ids: [T1049-4] - name: persistence description: Establish persistence via boot/logon autostart and account manipulation techniques: @@ -34,7 +34,7 @@ stages: description: Collect data from local system and network shares techniques: - technique_id: T1007 - atomic_ids: [T1007-1] + atomic_ids: [T1007-3] - technique_id: T1518 atomic_ids: [T1518-1] - technique_id: T1135 diff --git a/emulation/discovery-chain.yml b/emulation/discovery-chain.yml index 3aa614d..9d9443c 100644 --- a/emulation/discovery-chain.yml +++ b/emulation/discovery-chain.yml @@ -7,24 +7,24 @@ stages: description: Initial reconnaissance — who is on the system techniques: - technique_id: T1033 - atomic_ids: [T1033-1] + atomic_ids: [T1033-2] - technique_id: T1087.001 atomic_ids: [T1087.001-1] - name: system-info description: Gather system information techniques: - technique_id: T1082 - atomic_ids: [T1082-1] + atomic_ids: [T1082-3] - technique_id: T1518 atomic_ids: [T1518-1] - technique_id: T1007 - atomic_ids: [T1007-1] + atomic_ids: [T1007-3] - name: network description: Discover network configuration and connections techniques: - technique_id: T1016 - atomic_ids: [T1016-1] + atomic_ids: [T1016-3] - technique_id: T1049 - atomic_ids: [T1049-1] + atomic_ids: [T1049-4] - technique_id: T1135 atomic_ids: [T1135-1] diff --git a/internal/atomic/atomic.go b/internal/atomic/atomic.go new file mode 100644 index 0000000..b40cf32 --- /dev/null +++ b/internal/atomic/atomic.go @@ -0,0 +1,342 @@ +// Package atomic resolves Atomic Red Team test IDs (e.g. "T1082-3") into the +// concrete command that will run on a victim. +// +// This package exists because the engine previously fabricated its input: every +// technique executed the same hardcoded "id; whoami" regardless of which atomic +// the plan named, so every verdict but one was about a command nobody asked for. +// Resolution here is strict by design — an ID that cannot be resolved is an +// error the caller must surface, never a silent fallback to some default +// command. A fabricated input produces a fabricated verdict. +package atomic + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + + "gopkg.in/yaml.v3" +) + +// Source records where a resolved test's command came from, so a report can +// show that a lab-specific substitution was made rather than hiding it. +const ( + SourceART = "atomic-red-team" + SourceOverride = "override" +) + +// Dependency is a prerequisite an atomic needs before it can run. +type Dependency struct { + Description string `yaml:"description"` + PrereqCommand string `yaml:"prereq_command"` + GetPrereqCommand string `yaml:"get_prereq_command"` +} + +// InputArg is a declared input with its default value. +type InputArg struct { + Description string `yaml:"description"` + Type string `yaml:"type"` + Default any `yaml:"default"` +} + +// Test is one resolved atomic: the real command, executor and cleanup. +type Test struct { + ID string + Index int + GUID string + Name string + TechniqueID string + Description string + SupportedPlatforms []string + Executor string + Command string + CleanupCommand string + ElevationRequired bool + Dependencies []Dependency + DependencyExecutor string + Source string + OverrideReason string +} + +// SupportsPlatform reports whether the test declares support for a target kind +// ("linux", "windows", "macos"). +func (t Test) SupportsPlatform(kind string) bool { + for _, p := range t.SupportedPlatforms { + if strings.EqualFold(p, kind) { + return true + } + } + return false +} + +// ---- on-disk ART schema ---- + +type artFile struct { + AttackTechnique string `yaml:"attack_technique"` + DisplayName string `yaml:"display_name"` + AtomicTests []artTest `yaml:"atomic_tests"` +} + +type artTest struct { + Name string `yaml:"name"` + GUID string `yaml:"auto_generated_guid"` + Description string `yaml:"description"` + SupportedPlatforms []string `yaml:"supported_platforms"` + InputArguments map[string]InputArg `yaml:"input_arguments"` + DependencyExecutor string `yaml:"dependency_executor_name"` + Dependencies []Dependency `yaml:"dependencies"` + Executor artExecutor `yaml:"executor"` +} + +type artExecutor struct { + Name string `yaml:"name"` + Command string `yaml:"command"` + CleanupCommand string `yaml:"cleanup_command"` + ElevationRequired bool `yaml:"elevation_required"` +} + +// ---- overrides ---- + +// Override adjusts an ART atomic that cannot run truthfully in this lab. Every +// override carries a reason, which the report shows, so the change stays +// visible. +// +// There are two kinds, and the narrower one is preferred: +// +// - InputArguments alone pins an upstream atomic's declared inputs (the +// mechanism ART itself provides) while keeping its command. Use this to +// keep a test inside the lab network. +// - Command replaces the atomic wholesale. Only for a technique with no +// usable upstream test on this platform. +type Override struct { + Reason string `yaml:"reason"` + Command string `yaml:"command"` + Executor string `yaml:"executor"` + CleanupCommand string `yaml:"cleanup_command"` + SupportedPlatforms []string `yaml:"supported_platforms"` + Name string `yaml:"name"` + InputArguments map[string]string `yaml:"input_arguments"` +} + +type overrideFile struct { + Overrides map[string]Override `yaml:"overrides"` +} + +// Registry resolves atomic IDs against a vendored ART tree plus lab overrides. +type Registry struct { + root string + commit string + overrides map[string]Override +} + +// Load opens a vendored Atomic Red Team checkout. root is the repository root +// (the directory containing "atomics"). overridePath may be empty. +func Load(root, overridePath string) (*Registry, error) { + atomicsDir := filepath.Join(root, "atomics") + if fi, err := os.Stat(atomicsDir); err != nil || !fi.IsDir() { + return nil, fmt.Errorf("atomic red team not vendored at %s — run 'make atomics'", root) + } + r := &Registry{root: root, overrides: map[string]Override{}} + + // Best effort: the recorded commit makes a report reproducible. + if b, err := os.ReadFile(filepath.Join(root, "COMMIT")); err == nil { + r.commit = strings.TrimSpace(string(b)) + } + + if overridePath != "" { + if err := r.loadOverrides(overridePath); err != nil { + return nil, err + } + } + return r, nil +} + +func (r *Registry) loadOverrides(path string) error { + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return nil // overrides are optional + } + if err != nil { + return fmt.Errorf("read overrides: %w", err) + } + var f overrideFile + if err := yaml.Unmarshal(data, &f); err != nil { + return fmt.Errorf("parse overrides %s: %w", path, err) + } + for id, o := range f.Overrides { + if strings.TrimSpace(o.Reason) == "" { + return fmt.Errorf("override %s has no reason — every substitution must be justified", id) + } + if strings.TrimSpace(o.Command) == "" && len(o.InputArguments) == 0 { + return fmt.Errorf("override %s must set either command or input_arguments", id) + } + r.overrides[id] = o + } + return nil +} + +// SetCommit records the vendored ART revision for reporting. +func (r *Registry) SetCommit(c string) { r.commit = strings.TrimSpace(c) } + +// Commit returns the vendored ART revision, or "" if unknown. +func (r *Registry) Commit() string { return r.commit } + +var idRe = regexp.MustCompile(`^(T\d{4}(?:\.\d{3})?)-(\d+)$`) + +// ParseID splits an atomic ID such as "T1069.001-1" into its technique and +// 1-based test index. +func ParseID(atomicID string) (technique string, index int, err error) { + m := idRe.FindStringSubmatch(strings.TrimSpace(atomicID)) + if m == nil { + return "", 0, fmt.Errorf("malformed atomic id %q (want e.g. T1082-3)", atomicID) + } + i, _ := strconv.Atoi(m[2]) + if i < 1 { + return "", 0, fmt.Errorf("atomic index in %q must be 1-based", atomicID) + } + return m[1], i, nil +} + +// Resolve returns the concrete test for an atomic ID. Overrides win over the +// vendored tree; an unresolvable ID is an error, never a default command. +func (r *Registry) Resolve(atomicID string) (Test, error) { + technique, index, err := ParseID(atomicID) + if err != nil { + return Test{}, err + } + + o, overridden := r.overrides[atomicID] + if overridden && strings.TrimSpace(o.Command) != "" { + platforms := o.SupportedPlatforms + if len(platforms) == 0 { + platforms = []string{"linux"} + } + name := o.Name + if name == "" { + name = "lab override for " + atomicID + } + return Test{ + ID: atomicID, + Index: index, + Name: name, + TechniqueID: technique, + SupportedPlatforms: platforms, + Executor: orDefault(o.Executor, "sh"), + Command: strings.TrimSpace(o.Command), + CleanupCommand: strings.TrimSpace(o.CleanupCommand), + Source: SourceOverride, + OverrideReason: o.Reason, + }, nil + } + + path := filepath.Join(r.root, "atomics", technique, technique+".yaml") + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return Test{}, fmt.Errorf("technique %s has no atomics in the vendored tree (%s)", technique, path) + } + if err != nil { + return Test{}, fmt.Errorf("read %s: %w", path, err) + } + + var f artFile + if err := yaml.Unmarshal(data, &f); err != nil { + return Test{}, fmt.Errorf("parse %s: %w", path, err) + } + if index > len(f.AtomicTests) { + return Test{}, fmt.Errorf("%s: technique %s has %d atomic tests, index %d requested", + atomicID, technique, len(f.AtomicTests), index) + } + + at := f.AtomicTests[index-1] + subst := defaults(at.InputArguments) + + // An input-only override pins declared inputs (e.g. a ping target) without + // touching the upstream command, so the test stays the upstream test. + pinned := "" + if overridden { + keys := make([]string, 0, len(o.InputArguments)) + for k, v := range o.InputArguments { + subst[k] = v + keys = append(keys, k) + } + sort.Strings(keys) + if len(keys) > 0 { + pinned = fmt.Sprintf("inputs pinned (%s): %s", strings.Join(keys, ", "), o.Reason) + } + } + + deps := make([]Dependency, 0, len(at.Dependencies)) + for _, d := range at.Dependencies { + deps = append(deps, Dependency{ + Description: strings.TrimSpace(interpolate(d.Description, subst)), + PrereqCommand: strings.TrimSpace(interpolate(d.PrereqCommand, subst)), + GetPrereqCommand: strings.TrimSpace(interpolate(d.GetPrereqCommand, subst)), + }) + } + + return Test{ + ID: atomicID, + Index: index, + GUID: at.GUID, + Name: at.Name, + TechniqueID: technique, + Description: strings.TrimSpace(at.Description), + SupportedPlatforms: at.SupportedPlatforms, + Executor: orDefault(at.Executor.Name, "sh"), + Command: strings.TrimSpace(interpolate(at.Executor.Command, subst)), + CleanupCommand: strings.TrimSpace(interpolate(at.Executor.CleanupCommand, subst)), + ElevationRequired: at.Executor.ElevationRequired, + Dependencies: deps, + DependencyExecutor: orDefault(at.DependencyExecutor, orDefault(at.Executor.Name, "sh")), + Source: SourceART, + OverrideReason: pinned, + }, nil +} + +// defaults flattens declared input arguments to their default values. +func defaults(args map[string]InputArg) map[string]string { + out := make(map[string]string, len(args)) + for k, v := range args { + if v.Default == nil { + out[k] = "" + continue + } + out[k] = strings.TrimSpace(fmt.Sprintf("%v", v.Default)) + } + return out +} + +var placeholderRe = regexp.MustCompile(`#\{([a-zA-Z0-9_]+)\}`) + +// interpolate substitutes #{name} placeholders with their defaults. ART allows +// a default to itself contain a placeholder, so it resolves iteratively; the +// bound stops a self-referential default from looping forever. +func interpolate(s string, subst map[string]string) string { + if s == "" || len(subst) == 0 { + return s + } + for i := 0; i < 5; i++ { + replaced := placeholderRe.ReplaceAllStringFunc(s, func(m string) string { + key := placeholderRe.FindStringSubmatch(m)[1] + if v, ok := subst[key]; ok { + return v + } + return m // unknown placeholder stays visible rather than silently emptied + }) + if replaced == s { + break + } + s = replaced + } + return s +} + +func orDefault(v, def string) string { + if strings.TrimSpace(v) == "" { + return def + } + return v +} diff --git a/internal/atomic/atomic_test.go b/internal/atomic/atomic_test.go new file mode 100644 index 0000000..9802c80 --- /dev/null +++ b/internal/atomic/atomic_test.go @@ -0,0 +1,183 @@ +package atomic + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// artRoot is the vendored tree. Tests that need it skip when it is absent so a +// clean checkout without `make atomics` still runs the rest of the suite. +const artRoot = "../../lab/atomic-red-team" + +func requireART(t *testing.T) *Registry { + t.Helper() + if _, err := os.Stat(filepath.Join(artRoot, "atomics")); err != nil { + t.Skip("atomic-red-team not vendored; run 'make atomics'") + } + reg, err := Load(artRoot, "../../mappings/atomic-overrides.yml") + if err != nil { + t.Fatalf("load registry: %v", err) + } + return reg +} + +func TestParseID(t *testing.T) { + cases := []struct { + in string + tech string + index int + bad bool + }{ + {in: "T1082-3", tech: "T1082", index: 3}, + {in: "T1069.001-1", tech: "T1069.001", index: 1}, + {in: "T1059.004-17", tech: "T1059.004", index: 17}, + {in: "T1082", bad: true}, + {in: "T1082-0", bad: true}, + {in: "1082-1", bad: true}, + {in: "", bad: true}, + } + for _, c := range cases { + tech, idx, err := ParseID(c.in) + if c.bad { + if err == nil { + t.Errorf("ParseID(%q) = (%s,%d), want error", c.in, tech, idx) + } + continue + } + if err != nil { + t.Errorf("ParseID(%q): %v", c.in, err) + continue + } + if tech != c.tech || idx != c.index { + t.Errorf("ParseID(%q) = (%s,%d), want (%s,%d)", c.in, tech, idx, c.tech, c.index) + } + } +} + +func TestInterpolateDefaults(t *testing.T) { + subst := map[string]string{"output_file": "/tmp/x.txt", "host": "127.0.0.1"} + got := interpolate("uname -a >> #{output_file}; ping #{host}", subst) + want := "uname -a >> /tmp/x.txt; ping 127.0.0.1" + if got != want { + t.Errorf("interpolate = %q, want %q", got, want) + } +} + +// An unknown placeholder must stay visible. Silently emptying it would produce +// a subtly different command than the atomic describes. +func TestInterpolateLeavesUnknownPlaceholder(t *testing.T) { + got := interpolate("run #{nope}", map[string]string{"other": "x"}) + if got != "run #{nope}" { + t.Errorf("interpolate = %q, want the placeholder preserved", got) + } +} + +func TestResolveRealAtomic(t *testing.T) { + reg := requireART(t) + test, err := reg.Resolve("T1082-3") + if err != nil { + t.Fatalf("resolve: %v", err) + } + if !strings.Contains(test.Command, "uname -a") { + t.Errorf("T1082-3 command = %q, want it to contain 'uname -a'", test.Command) + } + if strings.Contains(test.Command, "#{") { + t.Errorf("T1082-3 command still has an uninterpolated placeholder: %q", test.Command) + } + if !test.SupportsPlatform("linux") || test.SupportsPlatform("windows") { + t.Errorf("T1082-3 platforms = %v, want linux and not windows", test.SupportedPlatforms) + } + if test.Source != SourceART { + t.Errorf("source = %q, want %q", test.Source, SourceART) + } +} + +// The engine must never invent a command. An unresolvable ID is an error the +// caller has to surface, which is the whole point of this package. +func TestResolveRefusesUnknown(t *testing.T) { + reg := requireART(t) + for _, id := range []string{"T9999-1", "T1082-999", "garbage"} { + if got, err := reg.Resolve(id); err == nil { + t.Errorf("Resolve(%q) = %+v, want error", id, got) + } + } +} + +func TestOverrideReplacesCommand(t *testing.T) { + reg := requireART(t) + test, err := reg.Resolve("T1518-1") + if err != nil { + t.Fatalf("resolve: %v", err) + } + if test.Source != SourceOverride { + t.Fatalf("T1518-1 source = %q, want %q", test.Source, SourceOverride) + } + if test.OverrideReason == "" { + t.Error("an override must carry a visible reason") + } + if !test.SupportsPlatform("linux") { + t.Error("the T1518 override must declare linux support") + } +} + +// An input-only override keeps the upstream command but pins its inputs. +func TestInputPinningKeepsUpstreamCommand(t *testing.T) { + reg := requireART(t) + test, err := reg.Resolve("T1059.004-1") + if err != nil { + t.Fatalf("resolve: %v", err) + } + if test.Source != SourceART { + t.Errorf("source = %q, want the upstream atomic to be preserved", test.Source) + } + if strings.Contains(test.Command, "8.8.8.8") { + t.Error("containment: the atomic still reaches 8.8.8.8, outside purpleloop-lab") + } + if !strings.Contains(test.Command, "127.0.0.1") { + t.Errorf("pinned host not applied: %q", test.Command) + } + if test.OverrideReason == "" { + t.Error("a pinned input must be reported") + } +} + +func TestOverrideRequiresReason(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "ov.yml") + if err := os.WriteFile(path, []byte("overrides:\n T1082-1:\n command: id\n"), 0644); err != nil { + t.Fatal(err) + } + if _, err := Load(artRoot, path); err == nil { + t.Error("an override without a reason must be rejected") + } +} + +// The regression guard for the defect this package exists to fix: every +// technique in the shipped campaign must resolve to its OWN command. If two +// techniques share one, the campaign is testing one behaviour under many names. +func TestCampaignAtomicsAreDistinct(t *testing.T) { + reg := requireART(t) + ids := []string{ + "T1059.004-1", "T1087.001-1", "T1082-3", "T1033-2", "T1007-3", + "T1016-3", "T1049-4", "T1069.001-1", "T1135-1", "T1518-1", + } + seen := map[string]string{} + for _, id := range ids { + test, err := reg.Resolve(id) + if err != nil { + t.Errorf("resolve %s: %v", id, err) + continue + } + if !test.SupportsPlatform("linux") { + t.Errorf("%s does not support linux (platforms %v) — the Linux campaign cannot run it", + id, test.SupportedPlatforms) + } + if prev, dup := seen[test.Command]; dup { + t.Errorf("%s and %s resolve to the SAME command — a campaign must not test one behaviour under many names:\n%s", + prev, id, test.Command) + } + seen[test.Command] = id + } +} diff --git a/internal/feed/static.go b/internal/feed/static.go index 02cd454..a4badfd 100644 --- a/internal/feed/static.go +++ b/internal/feed/static.go @@ -27,6 +27,7 @@ type PlanTechnique struct { // StaticFeed yields tasks from a YAML plan file. type StaticFeed struct { + Name string Tasks []model.TechniqueTask } @@ -40,6 +41,7 @@ func (f *StaticFeed) Load(path string) error { if err := yaml.Unmarshal(data, &plan); err != nil { return fmt.Errorf("parse plan: %w", err) } + f.Name = plan.Name f.Tasks = make([]model.TechniqueTask, 0, len(plan.Techniques)) for i, t := range plan.Techniques { f.Tasks = append(f.Tasks, model.TechniqueTask{ diff --git a/internal/model/model.go b/internal/model/model.go index 6ccaebf..e1fbc28 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -18,12 +18,42 @@ const ( NoTelemetry Verdict = "NO_TELEMETRY" Inconclusive Verdict = "INCONCLUSIVE" Errored Verdict = "ERROR" + + // SkippedPrereq means the atomic's own prerequisites were not met, so it + // never ran. That is not a detection gap and must never be counted as one — + // the same class of error as crediting a low-fidelity event as evidence. + SkippedPrereq Verdict = "SKIPPED_PREREQ" +) + +// Attribution states how tightly a verdict is bound to the execution that +// claims it. A DETECTED whose window overlapped another technique's run is not +// the same quality of evidence as one that did not, and a report that does not +// say so overstates its own rigour. +type Attribution string + +const ( + // WindowAndHostScoped is the strongest: the telemetry query was bounded to + // this execution's window AND filtered to this target host. + WindowAndHostScoped Attribution = "window_and_host_scoped" + // WindowScoped means time-bounded, but the host was not resolved. + WindowScoped Attribution = "window_scoped" + // WindowOverlap means another technique in this campaign ran inside this + // one's collection window, so a matching event may belong to either. + WindowOverlap Attribution = "window_overlap" + // Unscoped means the verdict cannot be attributed — a synthetic pipeline. + Unscoped Attribution = "unscoped" ) // TimeWindow bounds a telemetry query to an execution's real time span. type TimeWindow struct { - Start time.Time - End time.Time + Start time.Time `json:"start"` + End time.Time `json:"end"` +} + +// Overlaps reports whether two collection windows intersect. Two techniques +// whose windows overlap can each collect the other's events. +func (w TimeWindow) Overlaps(o TimeWindow) bool { + return w.Start.Before(o.End) && o.Start.Before(w.End) } // Target is a host inside the isolated lab network. Attacks NEVER target @@ -34,12 +64,21 @@ type Target struct { } // AtomicTest is a single Atomic Red Team test mapped to an ATT&CK technique. +// +// Name/GUID/Source exist so a report can prove WHICH upstream atomic produced a +// command. Source is "atomic-red-team" for a vendored test and "override" for a +// lab substitution, in which case OverrideReason says why — a substituted +// command must never read as the upstream atomic. type AtomicTest struct { ID string `json:"id"` TechniqueID string `json:"technique_id"` + Name string `json:"name,omitempty"` + GUID string `json:"guid,omitempty"` Command string `json:"command"` CleanupCommand string `json:"cleanup_command,omitempty"` Executor string `json:"executor"` + Source string `json:"source,omitempty"` + OverrideReason string `json:"override_reason,omitempty"` } // RunResult is what the Executor records about one atomic execution. @@ -94,6 +133,34 @@ type ProofChain struct { RuleMatched string `json:"rule_matched"` Verdict Verdict `json:"verdict"` Evidence []Event `json:"evidence"` + + // Note carries the reason behind a non-coverage verdict — why a technique + // ERRORed, which prerequisite was unmet, which platform mismatched. An + // unexplained gap is not actionable. + Note string `json:"note,omitempty"` + + // RulesExpected lists every rule that should have fired; RulesMatched lists + // those that did. RuleMatched above stays as the first match, preserving the + // documented proof-chain shape. + RulesExpected []string `json:"rules_expected,omitempty"` + RulesMatched []string `json:"rules_matched,omitempty"` + + // Attribution states how tightly this verdict is bound to its execution. + Attribution Attribution `json:"attribution,omitempty"` + + // Window is the telemetry query span, kept so overlapping executions can be + // detected across the campaign. + Window TimeWindow `json:"window"` + + // DetectLatencyMS is the gap between execution start and the earliest + // matching event. Zero when nothing matched. + DetectLatencyMS int64 `json:"detect_latency_ms,omitempty"` + + // Collected is every event the query returned, not just the matching ones. + // It is excluded from the proof-chain JSON (that contract carries evidence, + // which is the matching subset) and exists so a run can be captured and + // replayed against future rule changes. + Collected []Event `json:"-"` } // CampaignResult aggregates every technique validated in one run. @@ -111,6 +178,17 @@ type CampaignResult struct { CanaryHealthy bool `json:"canary_healthy"` // positive control fired (real runs) Inconclusive bool `json:"inconclusive"` // canary failed on a real run → coverage invalid CanaryDetail string `json:"canary_detail,omitempty"` // why the canary failed + + // Campaign names the plan that ran; AtomicsCommit records the vendored + // Atomic Red Team revision, so a report states exactly which upstream + // commands produced it. + Campaign string `json:"campaign,omitempty"` + AtomicsCommit string `json:"atomics_commit,omitempty"` + + // UntestedRules names detections this campaign did not exercise, with the + // reason. An honest denominator needs the rules that were never tried, not + // only the ones that were. + UntestedRules map[string]string `json:"untested_rules,omitempty"` } // ---- interfaces: the swappable seams of the system ---- diff --git a/mappings/atomic-overrides.yml b/mappings/atomic-overrides.yml new file mode 100644 index 0000000..078cc2f --- /dev/null +++ b/mappings/atomic-overrides.yml @@ -0,0 +1,59 @@ +# Lab overrides for Atomic Red Team tests. +# +# Resolution order is: override -> vendored ART -> error. An override is a +# deliberate, visible substitution for an atomic that cannot run truthfully in +# THIS lab (no Linux test exists, needs elevation the victim container lacks, or +# installs packages on an offline victim). Every entry MUST carry a reason; the +# loader rejects one that does not, and the reason is printed in the report so a +# substituted command is never mistaken for the upstream atomic. +# +# An override is NOT a way to make a technique pass. If no honest command exists +# for a technique on this platform, leave it out and let the run report the gap. + +overrides: + # CONTAINMENT. Upstream T1059.004-1 writes a script that pings #{host}, + # whose default is 8.8.8.8 — traffic that leaves the isolated lab network and + # breaks the project's own containment rule ("never target anything outside + # purpleloop-lab"). Pinning the input keeps the upstream command and its + # detection surface intact while holding the traffic inside the lab. + T1059.004-1: + reason: >- + ART default pings 8.8.8.8, which leaves the purpleloop-lab network. + Pinned to loopback; the command and its telemetry are otherwise upstream. + input_arguments: + host: 127.0.0.1 + + # T1518 (Software Discovery) has SIX atomic tests upstream and not one of them + # supports Linux — they are Windows browser/dotnet checks plus one macOS + # Safari check. The campaign validates T1518 against a Linux victim, so there + # is nothing upstream to run. This is the equivalent enumeration a Linux + # adversary performs, and it is flagged as a substitution in every report. + T1518-1: + reason: >- + No Linux atomic exists upstream (all 6 tests are windows/macos). This is + the Linux-equivalent installed-software enumeration. + name: Software Discovery (Linux equivalent) + executor: sh + supported_platforms: [linux] + command: | + if [ -x "$(command -v dpkg)" ]; then dpkg -l | head -50; fi + if [ -x "$(command -v rpm)" ]; then rpm -qa | head -50; fi + if [ -x "$(command -v apk)" ]; then apk info | head -50; fi + ls -1 /usr/bin | head -50 + + # T1135-2 is the only Linux network-share atomic. It requires elevation and a + # samba package the offline victim container does not carry, so upstream it + # would always report SKIPPED_PREREQ and never exercise a detection. This + # enumerates shares with what the container actually has. + T1135-1: + reason: >- + Upstream T1135-2 needs samba + elevation; the victim container has + neither, so it can only ever skip. This uses the tools present. + name: Network Share Discovery (unprivileged Linux) + executor: sh + supported_platforms: [linux] + command: | + cat /proc/mounts | grep -Ei 'cifs|nfs|smb' || echo "no network mounts" + if [ -x "$(command -v showmount)" ]; then showmount -e localhost; fi + if [ -x "$(command -v smbclient)" ]; then smbclient -L localhost -N; fi + mount | head -30 diff --git a/mappings/attack_rule_map.seed.json b/mappings/attack_rule_map.seed.json deleted file mode 100644 index bacbaa6..0000000 --- a/mappings/attack_rule_map.seed.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "_note": "Seed mapping technique -> atomic test ids -> detection rules. Phase 4 adds CVE->technique resolution.", - "T1059.004": { - "atomics": ["T1059.004-1"], - "rules": ["detections/linux/proc_creation_susp_shell.yml"] - }, - "T1087.001": { - "atomics": ["T1087.001-1"], - "rules": ["detections/linux/T1087.001.yml"] - }, - "T1082": { - "atomics": ["T1082-1"], - "rules": ["detections/linux/T1082.yml"] - }, - "T1033": { - "atomics": ["T1033-1"], - "rules": ["detections/linux/T1033.yml"] - }, - "T1007": { - "atomics": ["T1007-1"], - "rules": ["detections/linux/T1007.yml"] - }, - "T1016": { - "atomics": ["T1016-1"], - "rules": ["detections/linux/T1016.yml"] - }, - "T1049": { - "atomics": ["T1049-1"], - "rules": ["detections/linux/T1049.yml"] - }, - "T1069.001": { - "atomics": ["T1069.001-1"], - "rules": ["detections/linux/T1069.001.yml"] - }, - "T1135": { - "atomics": ["T1135-1"], - "rules": ["detections/linux/T1135.yml"] - }, - "T1518": { - "atomics": ["T1518-1"], - "rules": ["detections/linux/T1518.yml"] - } -} diff --git a/plans/discovery.yml b/plans/discovery.yml index 380d9fa..cf75c40 100644 --- a/plans/discovery.yml +++ b/plans/discovery.yml @@ -8,15 +8,15 @@ techniques: - technique_id: T1087.001 atomic_ids: [T1087.001-1] - technique_id: T1082 - atomic_ids: [T1082-1] + atomic_ids: [T1082-3] - technique_id: T1033 - atomic_ids: [T1033-1] + atomic_ids: [T1033-2] - technique_id: T1007 - atomic_ids: [T1007-1] + atomic_ids: [T1007-3] - technique_id: T1016 - atomic_ids: [T1016-1] + atomic_ids: [T1016-3] - technique_id: T1049 - atomic_ids: [T1049-1] + atomic_ids: [T1049-4] - technique_id: T1069 atomic_ids: [T1069.001-1] - technique_id: T1135 diff --git a/scripts/fetch-atomics.sh b/scripts/fetch-atomics.sh new file mode 100755 index 0000000..b5c6d2c --- /dev/null +++ b/scripts/fetch-atomics.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Vendor Atomic Red Team at the commit recorded in mappings/atomic-red-team.commit. +# +# This PINS: it checks out the recorded commit and verifies HEAD matches, so two +# people running it get byte-identical atomics. (The older path in lab-fetch.sh +# cloned HEAD and then recorded whatever it happened to get, which is a record, +# not a pin — a re-clone months later would silently change every command the +# engine runs.) +set -euo pipefail +cd "$(dirname "$0")/.." + +art=lab/atomic-red-team +pin_file=mappings/atomic-red-team.commit + +if [ ! -f "$pin_file" ]; then + echo "no pin recorded at $pin_file" >&2 + exit 1 +fi +pin=$(tr -d '[:space:]' < "$pin_file") + +if [ ! -d "$art/.git" ]; then + echo "cloning atomic-red-team…" + rm -rf "$art" + git clone --filter=blob:none --no-checkout \ + https://github.com/redcanaryco/atomic-red-team.git "$art" +fi + +echo "checking out pinned commit ${pin}" +git -C "$art" fetch --depth 1 origin "$pin" 2>/dev/null || git -C "$art" fetch origin +git -C "$art" checkout --quiet --detach "$pin" + +actual=$(git -C "$art" rev-parse HEAD) +if [ "$actual" != "$pin" ]; then + echo "PIN MISMATCH: wanted $pin, got $actual" >&2 + exit 1 +fi + +count=$(find "$art/atomics" -maxdepth 1 -mindepth 1 -type d | wc -l) +echo "atomic-red-team pinned at $actual (${count} techniques)" diff --git a/scripts/lab-fetch.sh b/scripts/lab-fetch.sh index c084a51..615354f 100755 --- a/scripts/lab-fetch.sh +++ b/scripts/lab-fetch.sh @@ -19,13 +19,12 @@ cp lab/docker-compose.override.yml "$dest/single-node/docker-compose.override.ym cp lab/victim/entrypoint.sh lab/victim/entrypoint.sh 2>/dev/null || true echo "override installed into $dest/single-node/" -# pin Atomic Red Team and record the commit for reproducibility -art=lab/atomic-red-team -if [ ! -d "$art" ]; then - git clone --depth 1 https://github.com/redcanaryco/atomic-red-team.git "$art" -fi -( cd "$art" && git rev-parse HEAD ) > mappings/atomic-red-team.commit -echo "atomic-red-team pinned at $(cat mappings/atomic-red-team.commit)" +# Vendor Atomic Red Team at the RECORDED commit. This used to clone HEAD and +# then overwrite the pin file with whatever it got, so the "pin" tracked +# upstream instead of holding it — every re-clone could silently change the +# commands the engine executes. scripts/fetch-atomics.sh checks out the +# recorded commit and verifies it. +bash scripts/fetch-atomics.sh # generate certs (official helper) if not already present if [ ! -f "$dest/single-node/config/wazuh_indexer_ssl_certs/root-ca.pem" ]; then From 0829208bf05841aab9d43dd5c602939624e7e66a Mon Sep 17 00:00:00 2001 From: jayelbotvibe-web Date: Sat, 29 Aug 2026 15:08:50 +0800 Subject: [PATCH 04/13] fix(detections): give each technique its own rule instead of nine copies Nine of the ten Linux Sigma rules were the identical rule. Every one matched "Image endswith /id or /whoami", differing only in title, because they were written against an engine that executed id; whoami for every technique. All nine shared one byte-identical positive fixture, so the "10 rules, 20 fixtures, CI enforced" detection-as-code claim was testing one rule nine times against one event. Each technique now has a detection for what its own atomic actually does, with distinct fixtures whose NEGATIVES include a neighbouring technique's activity -- so a rule that is secretly a copy fails its own negative test. win_proc_create.yml carried a non-UUID identifier (win-proc-create-001), a Sigma spec violation the hand-rolled Go parser accepted silently. Found by adding sigma check. Adds a data.sysmon normalizer path so a Linux-Sysmon source needs no Go changes: EventID 1 is process creation, any other Sysmon event is low fidelity and can never satisfy a process_creation rule. Deliberately not routed through data.win.*, which is Wazuh's Windows eventchannel convention -- using it for Linux would make the telemetry misreport its own origin. A test fails if any two rules share detection logic. Co-Authored-By: Claude Opus 5 --- detections/linux/T1007.yml | 29 ++-- detections/linux/T1016.yml | 34 ++-- detections/linux/T1033.yml | 25 +-- detections/linux/T1049.yml | 30 ++-- detections/linux/T1069.001.yml | 28 ++-- detections/linux/T1082.yml | 30 ++-- detections/linux/T1087.001.yml | 30 ++-- detections/linux/T1135.yml | 35 +++-- detections/linux/T1518.yml | 35 +++-- detections/tests/T1007/negative_events.jsonl | 3 +- detections/tests/T1007/positive_events.jsonl | 3 +- detections/tests/T1016/negative_events.jsonl | 3 +- detections/tests/T1016/positive_events.jsonl | 4 +- detections/tests/T1033/negative_events.jsonl | 3 +- detections/tests/T1033/positive_events.jsonl | 3 +- detections/tests/T1049/negative_events.jsonl | 3 +- detections/tests/T1049/positive_events.jsonl | 3 +- .../tests/T1069.001/negative_events.jsonl | 3 +- .../tests/T1069.001/positive_events.jsonl | 3 +- detections/tests/T1082/negative_events.jsonl | 3 +- detections/tests/T1082/positive_events.jsonl | 3 +- .../tests/T1087.001/negative_events.jsonl | 3 +- .../tests/T1087.001/positive_events.jsonl | 3 +- detections/tests/T1135/negative_events.jsonl | 3 +- detections/tests/T1135/positive_events.jsonl | 3 +- detections/tests/T1518/negative_events.jsonl | 3 +- detections/tests/T1518/positive_events.jsonl | 3 +- detections/windows/win_proc_create.yml | 2 +- internal/evaluator/distinct_test.go | 145 ++++++++++++++++++ internal/evaluator/normalizer.go | 45 +++++- 30 files changed, 413 insertions(+), 110 deletions(-) create mode 100644 internal/evaluator/distinct_test.go diff --git a/detections/linux/T1007.yml b/detections/linux/T1007.yml index a7a12ef..2e5c83d 100644 --- a/detections/linux/T1007.yml +++ b/detections/linux/T1007.yml @@ -1,23 +1,32 @@ -title: Discovery - T1007 -id: 792aae53-0000-0000-0000-000000000000 +title: System Service Discovery (Linux) +id: a1c0f2e4-0004-4b10-9c21-5f0a10070010 status: experimental -description: Detects T1007 discovery activity. +description: Detects service enumeration, as emitted by atomic T1007-3 (systemctl --type=service / service -e) and T1007-8 (init script listing). references: - https://attack.mitre.org/techniques/T1007/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + systemctl_list: + Image|endswith: '/systemctl' + CommandLine|contains: + - '--type=service' + - 'list-units' + - 'list-unit-files' + service_enum: Image|endswith: - - '/id' - - '/whoami' - condition: selection + - '/service' + - '/chkconfig' + initscripts: + CommandLine|contains: '/etc/init.d' + condition: systemctl_list or service_enum or initscripts falsepositives: - - Legitimate administrative scripts + - Administrators and provisioning tools inventorying services + - Monitoring agents polling unit state level: low tags: - attack.discovery - - attack.T1007 + - attack.t1007 diff --git a/detections/linux/T1016.yml b/detections/linux/T1016.yml index 95fe6b7..b9ab8ed 100644 --- a/detections/linux/T1016.yml +++ b/detections/linux/T1016.yml @@ -1,23 +1,37 @@ -title: Discovery - T1016 -id: 7d506977-0000-0000-0000-000000000000 +title: System Network Configuration Discovery (Linux) +id: a1c0f2e4-0005-4b10-9c21-5f0a10160010 status: experimental -description: Detects T1016 discovery activity. +description: Detects local network configuration enumeration, as emitted by atomic T1016-3 (arp -a; ifconfig; ip addr; netstat -ant; route). references: - https://attack.mitre.org/techniques/T1016/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + legacy_tools: Image|endswith: - - '/id' - - '/whoami' - condition: selection + - '/ifconfig' + - '/arp' + - '/route' + - '/netstat' + iproute: + Image|endswith: '/ip' + CommandLine|contains: + - ' addr' + - ' route' + - ' link' + - ' neigh' + resolver: + CommandLine|contains: + - '/etc/resolv.conf' + - '/etc/hosts' + condition: legacy_tools or iproute or resolver falsepositives: - - Legitimate administrative scripts + - Network troubleshooting by administrators + - Container entrypoints inspecting their own interfaces level: low tags: - attack.discovery - - attack.T1016 + - attack.t1016 diff --git a/detections/linux/T1033.yml b/detections/linux/T1033.yml index 64cd252..343b837 100644 --- a/detections/linux/T1033.yml +++ b/detections/linux/T1033.yml @@ -1,23 +1,30 @@ -title: Discovery - T1033 -id: 641f50c0-0000-0000-0000-000000000000 +title: System Owner and User Discovery (Linux) +id: a1c0f2e4-0003-4b10-9c21-5f0a10330010 status: experimental -description: Detects T1033 discovery activity. +description: Detects enumeration of the current and logged-on users, as emitted by atomic T1033-2 (users; w; who). references: - https://attack.mitre.org/techniques/T1033/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + session_enum: + Image|endswith: + - '/users' + - '/who' + - '/w' + - '/last' + identity: Image|endswith: - - '/id' - '/whoami' - condition: selection + - '/logname' + condition: session_enum or identity falsepositives: - - Legitimate administrative scripts + - Interactive administrators checking who is logged in + - Monitoring agents sampling session counts level: low tags: - attack.discovery - - attack.T1033 + - attack.t1033 diff --git a/detections/linux/T1049.yml b/detections/linux/T1049.yml index fb4feef..1d0eb80 100644 --- a/detections/linux/T1049.yml +++ b/detections/linux/T1049.yml @@ -1,23 +1,33 @@ -title: Discovery - T1049 -id: e3b2e995-0000-0000-0000-000000000000 +title: System Network Connections Discovery (Linux) +id: a1c0f2e4-0006-4b10-9c21-5f0a10490010 status: experimental -description: Detects T1049 discovery activity. +description: Detects enumeration of active network connections, as emitted by atomic T1049-4 (ss -antp / lsof -i) and T1049-6 (sockstat). references: - https://attack.mitre.org/techniques/T1049/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + socket_stats: Image|endswith: - - '/id' - - '/whoami' - condition: selection + - '/ss' + - '/sockstat' + lsof_network: + Image|endswith: '/lsof' + CommandLine|contains: '-i' + netstat_conns: + Image|endswith: '/netstat' + CommandLine|contains: + - '-a' + - '-t' + - '-n' + condition: socket_stats or lsof_network or netstat_conns falsepositives: - - Legitimate administrative scripts + - Administrators diagnosing connectivity + - Health checks probing listening ports level: low tags: - attack.discovery - - attack.T1049 + - attack.t1049 diff --git a/detections/linux/T1069.001.yml b/detections/linux/T1069.001.yml index 62c3b80..92833f6 100644 --- a/detections/linux/T1069.001.yml +++ b/detections/linux/T1069.001.yml @@ -1,23 +1,31 @@ -title: Discovery - T1069.001 -id: 2d3e4d06-0000-0000-0000-000000000000 +title: Local Permission Group Discovery (Linux) +id: a1c0f2e4-0007-4b10-9c21-5f0a10690010 status: experimental -description: Detects T1069.001 discovery activity. +description: Detects local group enumeration, as emitted by atomic T1069.001-1 (groups; cat /etc/group; getent group). references: - https://attack.mitre.org/techniques/T1069/001/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + group_db: + CommandLine|contains: + - '/etc/group' + - '/etc/sudoers' + group_tools: Image|endswith: - - '/id' - - '/whoami' - condition: selection + - '/groups' + - '/groupmems' + getent_group: + Image|endswith: '/getent' + CommandLine|contains: 'group' + condition: group_db or group_tools or getent_group falsepositives: - - Legitimate administrative scripts + - Provisioning tools reconciling group membership + - Login flows resolving supplementary groups level: low tags: - attack.discovery - - attack.T1069001 + - attack.t1069.001 diff --git a/detections/linux/T1082.yml b/detections/linux/T1082.yml index 59feb80..aa86868 100644 --- a/detections/linux/T1082.yml +++ b/detections/linux/T1082.yml @@ -1,23 +1,33 @@ -title: Discovery - T1082 -id: 422ee334-0000-0000-0000-000000000000 +title: System Information Discovery (Linux) +id: a1c0f2e4-0002-4b10-9c21-5f0a10820010 status: experimental -description: Detects T1082 discovery activity. +description: Detects host and OS fingerprinting, as emitted by atomic T1082-3 (uname -a plus reads of the release files) and by kernel-module listing. references: - https://attack.mitre.org/techniques/T1082/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + uname: + Image|endswith: '/uname' + release_files: + CommandLine|contains: + - '/etc/os-release' + - '/etc/lsb-release' + - '/etc/redhat-release' + - '/etc/issue' + host_fingerprint: Image|endswith: - - '/id' - - '/whoami' - condition: selection + - '/hostnamectl' + - '/lsmod' + - '/dmidecode' + condition: uname or release_files or host_fingerprint falsepositives: - - Legitimate administrative scripts + - Package managers and installers fingerprinting the distribution + - Configuration-management agents collecting host facts level: low tags: - attack.discovery - - attack.T1082 + - attack.t1082 diff --git a/detections/linux/T1087.001.yml b/detections/linux/T1087.001.yml index 1bb51c1..9889793 100644 --- a/detections/linux/T1087.001.yml +++ b/detections/linux/T1087.001.yml @@ -1,23 +1,33 @@ -title: Discovery - T1087.001 -id: 06a5d919-0000-0000-0000-000000000000 +title: Local Account Enumeration (Linux) +id: a1c0f2e4-0001-4b10-9c21-5f0a10870010 status: experimental -description: Detects T1087.001 discovery activity. +description: Detects local account enumeration by reading the account databases, as emitted by atomic T1087.001-1 (cat /etc/passwd) and by getent/lastlog equivalents. references: - https://attack.mitre.org/techniques/T1087/001/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + read_account_db: + CommandLine|contains: + - '/etc/passwd' + - '/etc/shadow' + enum_tools: Image|endswith: - - '/id' - - '/whoami' - condition: selection + - '/lastlog' + - '/lslogins' + getent_account_db: + Image|endswith: '/getent' + CommandLine|contains: + - 'passwd' + - 'shadow' + condition: read_account_db or enum_tools or getent_account_db falsepositives: - - Legitimate administrative scripts + - Backup and configuration-management tooling reading /etc/passwd + - Identity agents resolving accounts through getent passwd level: low tags: - attack.discovery - - attack.T1087001 + - attack.t1087.001 diff --git a/detections/linux/T1135.yml b/detections/linux/T1135.yml index aa1b561..1b7b6f5 100644 --- a/detections/linux/T1135.yml +++ b/detections/linux/T1135.yml @@ -1,23 +1,36 @@ -title: Discovery - T1135 -id: 59fe172d-0000-0000-0000-000000000000 +title: Network Share Discovery (Linux) +id: a1c0f2e4-0008-4b10-9c21-5f0a11350010 status: experimental -description: Detects T1135 discovery activity. +description: Detects network share enumeration, as emitted by the lab's T1135-1 override (reads of /proc/mounts filtered for network filesystems, showmount, smbclient) and by upstream smbstatus. references: - https://attack.mitre.org/techniques/T1135/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + smb_tools: Image|endswith: - - '/id' - - '/whoami' - condition: selection + - '/smbclient' + - '/smbstatus' + - '/showmount' + - '/nmblookup' + mount_table: + CommandLine|contains: + - '/proc/mounts' + - '/etc/fstab' + network_mount: + Image|endswith: '/mount' + CommandLine|contains: + - 'cifs' + - 'nfs' + - 'smb' + condition: smb_tools or mount_table or network_mount falsepositives: - - Legitimate administrative scripts -level: low + - Storage agents reading /proc/mounts or /etc/fstab + - Backup tooling enumerating network filesystems +level: medium tags: - attack.discovery - - attack.T1135 + - attack.t1135 diff --git a/detections/linux/T1518.yml b/detections/linux/T1518.yml index 78df919..731663f 100644 --- a/detections/linux/T1518.yml +++ b/detections/linux/T1518.yml @@ -1,23 +1,38 @@ -title: Discovery - T1518 -id: 4c1736b3-0000-0000-0000-000000000000 +title: Software Discovery (Linux) +id: a1c0f2e4-0009-4b10-9c21-5f0a15180010 status: experimental -description: Detects T1518 discovery activity. +description: Detects installed-software enumeration, as emitted by the lab's T1518-1 override (dpkg -l / rpm -qa / apk info). Upstream Atomic Red Team ships no Linux test for this technique. references: - https://attack.mitre.org/techniques/T1518/ author: purple-loop -date: 2026/07/04 +date: 2026/08/29 logsource: product: linux category: process_creation detection: - selection: + dpkg_list: + Image|endswith: '/dpkg' + CommandLine|contains: + - '-l' + - '--list' + - '--get-selections' + rpm_query: + Image|endswith: '/rpm' + CommandLine|contains: + - '-qa' + - '--query' + other_pkg: Image|endswith: - - '/id' - - '/whoami' - condition: selection + - '/apk' + - '/dpkg-query' + CommandLine|contains: + - 'info' + - '-l' + condition: dpkg_list or rpm_query or other_pkg falsepositives: - - Legitimate administrative scripts + - Vulnerability scanners and inventory agents + - Package managers during a normal install or upgrade level: low tags: - attack.discovery - - attack.T1518 + - attack.t1518 diff --git a/detections/tests/T1007/negative_events.jsonl b/detections/tests/T1007/negative_events.jsonl index 9c96041..68fd2bd 100644 --- a/detections/tests/T1007/negative_events.jsonl +++ b/detections/tests/T1007/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/bin/systemctl", "ParentImage": "/bin/bash", "CommandLine": "systemctl restart nginx", "User": "root"} +{"Image": "/usr/bin/who", "ParentImage": "/bin/sh", "CommandLine": "who", "User": "root"} diff --git a/detections/tests/T1007/positive_events.jsonl b/detections/tests/T1007/positive_events.jsonl index b3e8f64..d745217 100644 --- a/detections/tests/T1007/positive_events.jsonl +++ b/detections/tests/T1007/positive_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/bin/systemctl", "ParentImage": "/bin/bash", "CommandLine": "systemctl --type=service", "User": "root"} +{"Image": "/usr/sbin/service", "ParentImage": "/bin/bash", "CommandLine": "service --status-all", "User": "root"} diff --git a/detections/tests/T1016/negative_events.jsonl b/detections/tests/T1016/negative_events.jsonl index 9c96041..85770ef 100644 --- a/detections/tests/T1016/negative_events.jsonl +++ b/detections/tests/T1016/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/sbin/ip", "ParentImage": "/bin/bash", "CommandLine": "ip --version", "User": "root"} +{"Image": "/usr/bin/ss", "ParentImage": "/bin/bash", "CommandLine": "ss -antp", "User": "root"} diff --git a/detections/tests/T1016/positive_events.jsonl b/detections/tests/T1016/positive_events.jsonl index b3e8f64..8b66ca6 100644 --- a/detections/tests/T1016/positive_events.jsonl +++ b/detections/tests/T1016/positive_events.jsonl @@ -1 +1,3 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/sbin/arp", "ParentImage": "/bin/sh", "CommandLine": "arp -a", "User": "root"} +{"Image": "/usr/sbin/ifconfig", "ParentImage": "/bin/sh", "CommandLine": "ifconfig", "User": "root"} +{"Image": "/usr/sbin/ip", "ParentImage": "/bin/sh", "CommandLine": "ip addr show", "User": "root"} diff --git a/detections/tests/T1033/negative_events.jsonl b/detections/tests/T1033/negative_events.jsonl index 9c96041..8382d93 100644 --- a/detections/tests/T1033/negative_events.jsonl +++ b/detections/tests/T1033/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/bin/uname", "ParentImage": "/bin/sh", "CommandLine": "uname -a", "User": "root"} +{"Image": "/usr/bin/wget", "ParentImage": "/bin/bash", "CommandLine": "wget http://mirror/pkg.deb", "User": "root"} diff --git a/detections/tests/T1033/positive_events.jsonl b/detections/tests/T1033/positive_events.jsonl index b3e8f64..931eb04 100644 --- a/detections/tests/T1033/positive_events.jsonl +++ b/detections/tests/T1033/positive_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/bin/who", "ParentImage": "/bin/sh", "CommandLine": "who", "User": "root"} +{"Image": "/usr/bin/w", "ParentImage": "/bin/sh", "CommandLine": "w", "User": "root"} diff --git a/detections/tests/T1049/negative_events.jsonl b/detections/tests/T1049/negative_events.jsonl index 9c96041..2124241 100644 --- a/detections/tests/T1049/negative_events.jsonl +++ b/detections/tests/T1049/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/bin/lsof", "ParentImage": "/bin/bash", "CommandLine": "lsof /var/log/syslog", "User": "root"} +{"Image": "/usr/sbin/ifconfig", "ParentImage": "/bin/sh", "CommandLine": "ifconfig", "User": "root"} diff --git a/detections/tests/T1049/positive_events.jsonl b/detections/tests/T1049/positive_events.jsonl index b3e8f64..36d3c0f 100644 --- a/detections/tests/T1049/positive_events.jsonl +++ b/detections/tests/T1049/positive_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/bin/ss", "ParentImage": "/bin/bash", "CommandLine": "ss -antp", "User": "root"} +{"Image": "/usr/bin/lsof", "ParentImage": "/bin/bash", "CommandLine": "lsof -i -nP", "User": "root"} diff --git a/detections/tests/T1069.001/negative_events.jsonl b/detections/tests/T1069.001/negative_events.jsonl index 9c96041..3c933c9 100644 --- a/detections/tests/T1069.001/negative_events.jsonl +++ b/detections/tests/T1069.001/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/bin/cat", "ParentImage": "/bin/sh", "CommandLine": "cat /etc/passwd", "User": "root"} +{"Image": "/usr/bin/id", "ParentImage": "/bin/sh", "CommandLine": "id", "User": "root"} diff --git a/detections/tests/T1069.001/positive_events.jsonl b/detections/tests/T1069.001/positive_events.jsonl index b3e8f64..6dca966 100644 --- a/detections/tests/T1069.001/positive_events.jsonl +++ b/detections/tests/T1069.001/positive_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/bin/groups", "ParentImage": "/bin/sh", "CommandLine": "groups", "User": "root"} +{"Image": "/usr/bin/cat", "ParentImage": "/bin/sh", "CommandLine": "cat /etc/group", "User": "root"} diff --git a/detections/tests/T1082/negative_events.jsonl b/detections/tests/T1082/negative_events.jsonl index 9c96041..7369f26 100644 --- a/detections/tests/T1082/negative_events.jsonl +++ b/detections/tests/T1082/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/bin/cat", "ParentImage": "/bin/sh", "CommandLine": "cat /etc/passwd", "User": "root"} +{"Image": "/usr/bin/apt", "ParentImage": "/bin/bash", "CommandLine": "apt update", "User": "root"} diff --git a/detections/tests/T1082/positive_events.jsonl b/detections/tests/T1082/positive_events.jsonl index b3e8f64..d28916c 100644 --- a/detections/tests/T1082/positive_events.jsonl +++ b/detections/tests/T1082/positive_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/bin/uname", "ParentImage": "/bin/sh", "CommandLine": "uname -a", "User": "root"} +{"Image": "/usr/bin/cat", "ParentImage": "/bin/sh", "CommandLine": "cat /etc/os-release", "User": "root"} diff --git a/detections/tests/T1087.001/negative_events.jsonl b/detections/tests/T1087.001/negative_events.jsonl index 9c96041..bc423eb 100644 --- a/detections/tests/T1087.001/negative_events.jsonl +++ b/detections/tests/T1087.001/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/bin/cat", "ParentImage": "/bin/bash", "CommandLine": "cat /etc/hostname", "User": "root"} +{"Image": "/usr/bin/uname", "ParentImage": "/bin/sh", "CommandLine": "uname -a", "User": "root"} diff --git a/detections/tests/T1087.001/positive_events.jsonl b/detections/tests/T1087.001/positive_events.jsonl index b3e8f64..5166a64 100644 --- a/detections/tests/T1087.001/positive_events.jsonl +++ b/detections/tests/T1087.001/positive_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/bin/cat", "ParentImage": "/bin/sh", "CommandLine": "cat /etc/passwd", "User": "root"} +{"Image": "/usr/bin/getent", "ParentImage": "/bin/bash", "CommandLine": "getent passwd", "User": "root"} diff --git a/detections/tests/T1135/negative_events.jsonl b/detections/tests/T1135/negative_events.jsonl index 9c96041..5ab4fb0 100644 --- a/detections/tests/T1135/negative_events.jsonl +++ b/detections/tests/T1135/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/bin/cat", "ParentImage": "/bin/sh", "CommandLine": "cat /proc/cpuinfo", "User": "root"} +{"Image": "/usr/bin/dpkg", "ParentImage": "/bin/sh", "CommandLine": "dpkg -l", "User": "root"} diff --git a/detections/tests/T1135/positive_events.jsonl b/detections/tests/T1135/positive_events.jsonl index b3e8f64..38682c2 100644 --- a/detections/tests/T1135/positive_events.jsonl +++ b/detections/tests/T1135/positive_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/bin/cat", "ParentImage": "/bin/sh", "CommandLine": "cat /proc/mounts", "User": "root"} +{"Image": "/usr/bin/smbclient", "ParentImage": "/bin/sh", "CommandLine": "smbclient -L localhost -N", "User": "root"} diff --git a/detections/tests/T1518/negative_events.jsonl b/detections/tests/T1518/negative_events.jsonl index 9c96041..3895ffa 100644 --- a/detections/tests/T1518/negative_events.jsonl +++ b/detections/tests/T1518/negative_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/apt","ParentImage":"/bin/bash","CommandLine":"apt update"} +{"Image": "/usr/bin/dpkg", "ParentImage": "/bin/bash", "CommandLine": "dpkg --configure -a", "User": "root"} +{"Image": "/usr/bin/uname", "ParentImage": "/bin/sh", "CommandLine": "uname -a", "User": "root"} diff --git a/detections/tests/T1518/positive_events.jsonl b/detections/tests/T1518/positive_events.jsonl index b3e8f64..94a3aef 100644 --- a/detections/tests/T1518/positive_events.jsonl +++ b/detections/tests/T1518/positive_events.jsonl @@ -1 +1,2 @@ -{"Image":"/usr/bin/id","ParentImage":"/bin/sh","CommandLine":"id"} +{"Image": "/usr/bin/dpkg", "ParentImage": "/bin/sh", "CommandLine": "dpkg -l", "User": "root"} +{"Image": "/usr/bin/rpm", "ParentImage": "/bin/sh", "CommandLine": "rpm -qa", "User": "root"} diff --git a/detections/windows/win_proc_create.yml b/detections/windows/win_proc_create.yml index fe9d9b1..d110e76 100644 --- a/detections/windows/win_proc_create.yml +++ b/detections/windows/win_proc_create.yml @@ -1,5 +1,5 @@ title: Windows Process Creation -id: win-proc-create-001 +id: a1c0f2e4-0010-4b10-9c21-5f0a10590030 status: experimental description: Detects Windows process creation via Sysmon Event ID 1. references: diff --git a/internal/evaluator/distinct_test.go b/internal/evaluator/distinct_test.go new file mode 100644 index 0000000..7affa8d --- /dev/null +++ b/internal/evaluator/distinct_test.go @@ -0,0 +1,145 @@ +package evaluator + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// Nine of the ten Linux rules were once byte-identical in logic — every one +// matched `Image endswith /id or /whoami`, differing only in title — because +// they were written against an engine that executed `id; whoami` for every +// technique. The CI regression suite passed the whole time, since all nine +// shared one positive fixture too. +// +// A campaign whose detections are one rule under many names reports coverage it +// does not have. This test makes that unshippable. +func TestRulesHaveDistinctLogic(t *testing.T) { + dirs := []string{"../../detections/linux", "../../detections/windows"} + byLogic := map[string][]string{} + + for _, dir := range dirs { + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read %s: %v", dir, err) + } + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".yml") { + continue + } + path := filepath.Join(dir, e.Name()) + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + var doc struct { + Detection map[string]any `yaml:"detection"` + } + if err := yaml.Unmarshal(data, &doc); err != nil { + t.Fatalf("parse %s: %v", path, err) + } + if len(doc.Detection) == 0 { + t.Errorf("%s has no detection block", path) + continue + } + // Canonicalise so formatting differences do not mask identical logic. + canon, err := json.Marshal(normalize(doc.Detection)) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(canon) + key := hex.EncodeToString(sum[:8]) + byLogic[key] = append(byLogic[key], filepath.Join(filepath.Base(dir), e.Name())) + } + } + + for _, rules := range byLogic { + if len(rules) > 1 { + t.Errorf("these rules have IDENTICAL detection logic: %s\n"+ + "each technique needs a detection for what its own atomic does — "+ + "duplicate logic reports coverage the campaign does not have", + strings.Join(rules, ", ")) + } + } +} + +// normalize converts a YAML tree into a form with deterministic ordering, so +// two logically identical detections hash the same. +func normalize(v any) any { + switch x := v.(type) { + case map[string]any: + out := make(map[string]any, len(x)) + for k, val := range x { + out[k] = normalize(val) + } + return out + case map[any]any: + out := make(map[string]any, len(x)) + for k, val := range x { + out[strings.TrimSpace(toString(k))] = normalize(val) + } + return out + case []any: + out := make([]any, len(x)) + for i, val := range x { + out[i] = normalize(val) + } + return out + default: + return v + } +} + +func toString(v any) string { + if s, ok := v.(string); ok { + return s + } + b, _ := json.Marshal(v) + return string(b) +} + +// Every rule mapped to a technique must actually reference that technique in +// its tags, so a mapping cannot quietly point at an unrelated detection. +func TestRuleTagsMatchTechnique(t *testing.T) { + entries, err := os.ReadDir("../../detections/linux") + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, ".yml") { + continue + } + tech := strings.TrimSuffix(name, ".yml") + if !strings.HasPrefix(tech, "T1") { + continue // rules named by behaviour rather than technique + } + data, err := os.ReadFile(filepath.Join("../../detections/linux", name)) + if err != nil { + t.Fatal(err) + } + var doc struct { + Tags []string `yaml:"tags"` + } + if err := yaml.Unmarshal(data, &doc); err != nil { + t.Fatal(err) + } + want := "attack." + strings.ToLower(tech) + found := false + for _, tag := range doc.Tags { + if strings.EqualFold(strings.TrimSpace(tag), want) { + found = true + break + } + } + if !found { + t.Errorf("%s does not tag %s (tags: %v)", name, want, doc.Tags) + } + } +} diff --git a/internal/evaluator/normalizer.go b/internal/evaluator/normalizer.go index d477498..433993e 100644 --- a/internal/evaluator/normalizer.go +++ b/internal/evaluator/normalizer.go @@ -74,7 +74,7 @@ func (Normalizer) Normalize(raw json.RawMessage) map[string]string { getString(ed, "commandLine", &out, "CommandLine") getString(ed, "user", &out, "User") getString(ed, "parentUser", &out, "User") - getString(ed, "newProcessName", &out, "Image") // Security 4688 + getString(ed, "newProcessName", &out, "Image") // Security 4688 getString(ed, "parentProcessName", &out, "ParentImage") getString(ed, "subjectUserName", &out, "User") highFidelity = true @@ -95,6 +95,38 @@ func (Normalizer) Normalize(raw json.RawMessage) map[string]string { } } + // Try Linux-Sysmon (eBPF) paths. + // + // Sysmon for Linux emits the same process-creation semantics as its Windows + // counterpart — EventID 1 with Image, CommandLine, ParentImage, User — but it + // is NOT an eventchannel event, so it gets its own path rather than being + // smuggled through data.win. The same gate applies: only EventID 1 is + // process creation, and only process creation is high fidelity. + if data, ok := event["data"].(map[string]any); ok { + if sm, ok := data["sysmon"].(map[string]any); ok { + eventID := anyToString(sm["eventID"]) + if eventID != "" { + out["EventID"] = eventID + } + if eventID == "1" { + getString(sm, "image", &out, "Image") + getString(sm, "parentImage", &out, "ParentImage") + getString(sm, "commandLine", &out, "CommandLine") + getString(sm, "user", &out, "User") + highFidelity = true + } else { + // Any other Sysmon event (network, file, …) carries a process + // name without being proof a process was created. + getString(sm, "image", &out, "Image") + getString(sm, "commandLine", &out, "CommandLine") + getString(sm, "user", &out, "User") + if out["Image"] != "" || out["CommandLine"] != "" { + lowFidelity = true + } + } + } + } + // Try Linux auditd paths if data, ok := event["data"].(map[string]any); ok { if audit, ok := data["audit"].(map[string]any); ok { @@ -177,6 +209,17 @@ func nonEmptyStr(m map[string]any, key string) bool { return ok && s != "" } +// anyToString renders a JSON value that may be a string or a number. +func anyToString(v any) string { + switch t := v.(type) { + case string: + return t + case float64: + return fmt.Sprintf("%d", int(t)) + } + return "" +} + // winEventID extracts data.win.system.eventID (string or numeric) or "". func winEventID(win map[string]any) string { sys, ok := win["system"].(map[string]any) From 8638b80fa8a49616ffe25e53990ffa6ac735dcb3 Mon Sep 17 00:00:00 2001 From: jayelbotvibe-web Date: Sat, 29 Aug 2026 15:08:50 +0800 Subject: [PATCH 05/13] feat(mapping): expected_rules, attribution labels and detection latency Retires the technique->rule map hardcoded in cmd/purpleloop, which duplicated mappings/attack_rule_map.seed.json -- a file no code loaded. The mapping is now data the repository can lint, extended with expected_rules[] so a technique is DETECTED when ANY expected rule matches; asserting a single rule understated coverage whenever a second rule was the one that legitimately caught the behaviour. Adds attribution labels to every verdict (window_and_host_scoped, window_scoped, window_overlap, unscoped). The hard part already existed -- RunResult.Window pads only the end, with a documented rationale -- what was missing was saying how well-scoped each verdict is. Overlap is a campaign-level property, so it is computed once every window is known. This changes no verdict; it states the quality of the evidence behind one. Adds detect_latency_ms from data already collected. Report corrections: - ERROR was counted as "missed" in the HTML headline, and SKIPPED_PREREQ/ERROR sat inside the dashboard's coverage denominator. A harness failure is not a detection gap; coverage is measured only over techniques that exercised one. - campaign and build were hardcoded ("discovery", "v1.3.0") regardless of what ran; both now come from the run. - untested_rules is populated with a reason per rule, enforced by a CI lint, so a rule can no longer be added, never exercised, and still count as coverage. - gap.why is filled from the verdict's note instead of always being empty. Co-Authored-By: Claude Opus 5 --- cmd/purpleloop/main_test.go | 286 +++++++++++++++++++++++++++++++ internal/mapping/mapping.go | 71 ++++++++ internal/mapping/rulemap_test.go | 105 ++++++++++++ internal/report/dashboard.go | 95 ++++++++-- internal/report/report.go | 76 ++++++-- mappings/attack_rule_map.json | 116 +++++++++++++ 6 files changed, 715 insertions(+), 34 deletions(-) create mode 100644 cmd/purpleloop/main_test.go create mode 100644 internal/mapping/rulemap_test.go create mode 100644 mappings/attack_rule_map.json diff --git a/cmd/purpleloop/main_test.go b/cmd/purpleloop/main_test.go new file mode 100644 index 0000000..5aa134a --- /dev/null +++ b/cmd/purpleloop/main_test.go @@ -0,0 +1,286 @@ +package main + +import ( + "testing" + "time" + + "github.com/jayelbotvibe-web/purple-loop/internal/mapping" + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +func win(startSec, endSec int) model.TimeWindow { + base := time.Date(2026, 8, 29, 10, 0, 0, 0, time.UTC) + return model.TimeWindow{ + Start: base.Add(time.Duration(startSec) * time.Second), + End: base.Add(time.Duration(endSec) * time.Second), + } +} + +func TestLabelAttributionSyntheticIsUnscoped(t *testing.T) { + chains := []model.ProofChain{{Window: win(0, 10)}} + labelAttribution(chains, true, "victim01") + if chains[0].Attribution != model.Unscoped { + t.Errorf("synthetic run = %q, want %q", chains[0].Attribution, model.Unscoped) + } +} + +func TestLabelAttributionDetectsOverlap(t *testing.T) { + chains := []model.ProofChain{ + {TechniqueID: "T1082", Window: win(0, 60)}, + {TechniqueID: "T1033", Window: win(30, 90)}, // overlaps the first + {TechniqueID: "T1007", Window: win(200, 260)}, + } + labelAttribution(chains, false, "victim01") + + if chains[0].Attribution != model.WindowOverlap { + t.Errorf("T1082 = %q, want %q", chains[0].Attribution, model.WindowOverlap) + } + if chains[1].Attribution != model.WindowOverlap { + t.Errorf("T1033 = %q, want %q", chains[1].Attribution, model.WindowOverlap) + } + if chains[2].Attribution != model.WindowAndHostScoped { + t.Errorf("T1007 = %q, want %q", chains[2].Attribution, model.WindowAndHostScoped) + } +} + +func TestLabelAttributionWithoutHost(t *testing.T) { + chains := []model.ProofChain{{Window: win(0, 10)}} + labelAttribution(chains, false, "") + if chains[0].Attribution != model.WindowScoped { + t.Errorf("no host = %q, want %q", chains[0].Attribution, model.WindowScoped) + } +} + +// A technique that never ran has no window, so nothing can be attributed to it. +func TestLabelAttributionNoWindow(t *testing.T) { + chains := []model.ProofChain{{TechniqueID: "T1518", Verdict: model.SkippedPrereq}} + labelAttribution(chains, false, "victim01") + if chains[0].Attribution != model.Unscoped { + t.Errorf("unrun technique = %q, want %q", chains[0].Attribution, model.Unscoped) + } +} + +// stubEval returns a canned verdict per rule path. +type stubEval map[string]model.Verdict + +func (s stubEval) Evaluate(rule model.SigmaRule, events []model.Event) (model.Verdict, []model.Event, error) { + v, ok := s[rule.Path] + if !ok { + v = model.Missed + } + if v == model.Detected { + return v, []model.Event{{ID: "ev-" + rule.Path}}, nil + } + return v, nil, nil +} + +// The point of expected_rules: a second rule catching the behaviour is still a +// detection. Evaluating only the first would report a gap that does not exist. +func TestEvaluateExpectedAnyRuleDetects(t *testing.T) { + eval := stubEval{"a.yml": model.Missed, "b.yml": model.Detected} + v, ev, matched, err := evaluateExpected(eval, []string{"a.yml", "b.yml"}, "T1082", []model.Event{{}}) + if err != nil { + t.Fatal(err) + } + if v != model.Detected { + t.Errorf("verdict = %q, want DETECTED", v) + } + if len(matched) != 1 || matched[0] != "b.yml" { + t.Errorf("matched = %v, want [b.yml]", matched) + } + if len(ev) == 0 { + t.Error("a detection must carry its evidence") + } +} + +// NO_TELEMETRY only stands when no rule saw usable events. A real MISSED from +// any rule outranks it, because something was actually evaluated. +func TestEvaluateExpectedVerdictPrecedence(t *testing.T) { + cases := []struct { + name string + eval stubEval + want model.Verdict + }{ + {"missed beats no_telemetry", stubEval{"a.yml": model.NoTelemetry, "b.yml": model.Missed}, model.Missed}, + {"inconclusive beats no_telemetry", stubEval{"a.yml": model.NoTelemetry, "b.yml": model.Inconclusive}, model.Inconclusive}, + {"detected beats all", stubEval{"a.yml": model.Missed, "b.yml": model.Detected}, model.Detected}, + {"all silent stays no_telemetry", stubEval{"a.yml": model.NoTelemetry, "b.yml": model.NoTelemetry}, model.NoTelemetry}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + v, _, _, err := evaluateExpected(c.eval, []string{"a.yml", "b.yml"}, "T", []model.Event{{}}) + if err != nil { + t.Fatal(err) + } + if v != c.want { + t.Errorf("verdict = %q, want %q", v, c.want) + } + }) + } +} + +func TestDetectLatency(t *testing.T) { + start := time.Date(2026, 8, 29, 10, 0, 0, 0, time.UTC) + ev := []model.Event{ + {Timestamp: start.Add(5 * time.Second)}, + {Timestamp: start.Add(2 * time.Second)}, // earliest wins + } + if got := detectLatency(start, ev); got != 2000 { + t.Errorf("latency = %dms, want 2000", got) + } + // An event stamped before the execution cannot be its evidence. + if got := detectLatency(start, []model.Event{{Timestamp: start.Add(-time.Second)}}); got != 0 { + t.Errorf("pre-execution event gave latency %d, want 0", got) + } + if got := detectLatency(start, nil); got != 0 { + t.Errorf("no evidence gave latency %d, want 0", got) + } +} + +// Replaying the shipped dataset runs the REAL evaluator over real event shapes, +// with no lab, no Docker and no Wazuh. This is the regression test the project +// could not previously have: rule edits are now checked against telemetry +// rather than only against fixtures written to match them. +// +// The expected verdicts are deliberately mixed. A sample where everything +// passes would not catch a rule that matches everything. +func TestReplaySampleDataset(t *testing.T) { + result, err := replayDataset("../../testdata/sample-run", + "../../mappings/attack_rule_map.json", "../../detections/linux") + if err != nil { + t.Fatalf("replay: %v", err) + } + + want := map[string]model.Verdict{ + "T1082": model.Detected, + "T1033": model.Detected, + "T1049": model.Detected, + "T1518": model.Missed, // ran, produced telemetry, rule legitimately did not match + } + if len(result.Chains) != len(want) { + t.Fatalf("chains = %d, want %d", len(result.Chains), len(want)) + } + for _, c := range result.Chains { + w, ok := want[c.TechniqueID] + if !ok { + t.Errorf("unexpected technique %s in replay", c.TechniqueID) + continue + } + if c.Verdict != w { + t.Errorf("%s = %s, want %s (note: %s)", c.TechniqueID, c.Verdict, w, c.Note) + } + if c.EventsCollected == 0 { + t.Errorf("%s replayed with no events", c.TechniqueID) + } + } +} + +// A replay is a real evaluation of real telemetry, so it must never be marked +// synthetic — that flag means "fabricated", and mislabelling would either +// suppress a valid result or launder a fake one. +func TestReplayIsNotSynthetic(t *testing.T) { + result, err := replayDataset("../../testdata/sample-run", + "../../mappings/attack_rule_map.json", "../../detections/linux") + if err != nil { + t.Fatal(err) + } + if result.Synthetic { + t.Error("a replay of captured telemetry is not a synthetic run") + } +} + +func TestCompareRunsClassifiesChanges(t *testing.T) { + before := map[string]string{ + "T1082": "DETECTED", + "T1033": "MISSED", + "T1049": "DETECTED", + "T1007": "NO_TELEMETRY", + "T1016": "DETECTED", + } + after := map[string]string{ + "T1082": "MISSED", // regression — the one that matters + "T1033": "DETECTED", // improvement + "T1049": "DETECTED", // unchanged + "T1007": "ERROR", // regression: it used to at least collect + "T1518": "DETECTED", // newly present + // T1016 absent + } + + reg, imp, other := compareRuns(before, after) + + if len(reg) != 2 { + t.Errorf("regressions = %v, want 2 — T1082 and T1007; T1016 dropping out is a change, not a regression", reg) + } + found := map[string]string{} + for _, c := range reg { + found[c.Technique] = c.From + "->" + c.To + } + if found["T1082"] != "DETECTED->MISSED" { + t.Errorf("T1082 = %q, want DETECTED->MISSED", found["T1082"]) + } + if found["T1007"] != "NO_TELEMETRY->ERROR" { + t.Errorf("T1007 = %q, want NO_TELEMETRY->ERROR", found["T1007"]) + } + + if len(imp) != 1 || imp[0].Technique != "T1033" { + t.Errorf("improvements = %v, want just T1033", imp) + } + + // Techniques appearing or disappearing are changes, but not regressions. + otherSet := map[string]bool{} + for _, c := range other { + otherSet[c.Technique] = true + } + if !otherSet["T1518"] || !otherSet["T1016"] { + t.Errorf("other = %v, want T1518 (new) and T1016 (absent)", other) + } +} + +// Verdicts that all mean "never exercised" must not read as regressions of one +// another — moving from ERROR to SKIPPED_PREREQ is a different explanation of +// the same non-result, not a detection getting worse. +func TestCompareRunsNotExercisedVerdictsAreEqualRank(t *testing.T) { + reg, imp, other := compareRuns( + map[string]string{"T1": "ERROR"}, + map[string]string{"T1": "SKIPPED_PREREQ"}) + if len(reg) != 0 || len(imp) != 0 { + t.Errorf("want no regression/improvement, got reg=%v imp=%v", reg, imp) + } + if len(other) != 1 { + t.Errorf("want the change reported as 'other', got %v", other) + } +} + +func TestCompareRunsIdenticalIsQuiet(t *testing.T) { + m := map[string]string{"T1082": "DETECTED", "T1033": "MISSED"} + reg, imp, other := compareRuns(m, m) + if len(reg)+len(imp)+len(other) != 0 { + t.Errorf("identical runs must produce no changes: %v %v %v", reg, imp, other) + } +} + +// `--technique T1082` must use the atomic the rule map names (T1082-3, Linux), +// not ART's first test (T1082-1, Windows). Guessing "-1" picks the +// Windows atomic for most techniques, which then errors on the Linux victim. +func TestAtomicIDForPrefersRuleMapOverGuess(t *testing.T) { + rules, err := mapping.LoadRuleMap("../../mappings/attack_rule_map.json") + if err != nil { + t.Fatal(err) + } + task := model.TechniqueTask{TechniqueID: "T1082"} // no atomic named + if got := atomicIDFor(task, rules); got != "T1082-3" { + t.Errorf("atomicIDFor = %q, want T1082-3 from the rule map", got) + } + + // An explicit plan choice still wins over the map. + task.AtomicIDs = []string{"T1082-8"} + if got := atomicIDFor(task, rules); got != "T1082-8" { + t.Errorf("atomicIDFor = %q, want the plan's explicit T1082-8", got) + } + + // Only with neither does it fall back to the guess. + unmapped := model.TechniqueTask{TechniqueID: "T9999"} + if got := atomicIDFor(unmapped, rules); got != "T9999-1" { + t.Errorf("atomicIDFor = %q, want the T9999-1 fallback", got) + } +} diff --git a/internal/mapping/mapping.go b/internal/mapping/mapping.go index f782a4a..ab331a7 100644 --- a/internal/mapping/mapping.go +++ b/internal/mapping/mapping.go @@ -38,3 +38,74 @@ func (m CVEMap) Resolve(cve string) (techniques []string, techniqueAtomics map[s } return entry.Techniques, entry.Atomics, nil } + +// ---- technique -> atomic -> expected detections ---- + +// TechniqueEntry declares which atomic exercises a technique and which Sigma +// rules are expected to fire for it. expected_rules is a list, not one path: +// a technique may legitimately be caught by any of several rules, and asserting +// a single one understates coverage. +type TechniqueEntry struct { + Atomics []string `json:"atomics"` + Platform string `json:"platform"` + ExpectedRules []string `json:"expected_rules"` +} + +// RuleMap is the single source of truth for what a campaign should run and what +// should catch it. It replaces the technique->rule map that was hardcoded in +// cmd/purpleloop, so the mapping is data the repository can lint rather than +// code nobody can inspect. +type RuleMap struct { + Techniques map[string]TechniqueEntry `json:"techniques"` + UntestedRules map[string]string `json:"untested_rules"` +} + +// LoadRuleMap reads the technique/rule mapping file. +func LoadRuleMap(path string) (*RuleMap, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read rule map: %w", err) + } + var m RuleMap + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("parse rule map %s: %w", path, err) + } + if len(m.Techniques) == 0 { + return nil, fmt.Errorf("rule map %s declares no techniques", path) + } + return &m, nil +} + +// Entry returns the mapping for a technique. +func (m *RuleMap) Entry(technique string) (TechniqueEntry, bool) { + e, ok := m.Techniques[technique] + return e, ok +} + +// ExpectedRules returns the rules that should fire for a technique. +func (m *RuleMap) ExpectedRules(technique string) []string { + if e, ok := m.Techniques[technique]; ok { + return e.ExpectedRules + } + return nil +} + +// AtomicFor returns the mapped atomic ID for a technique, if one is declared. +func (m *RuleMap) AtomicFor(technique string) string { + if e, ok := m.Techniques[technique]; ok && len(e.Atomics) > 0 { + return e.Atomics[0] + } + return "" +} + +// MappedRules returns every rule path referenced by any technique, so a lint +// can tell which rules on disk are never exercised. +func (m *RuleMap) MappedRules() map[string]bool { + out := map[string]bool{} + for _, e := range m.Techniques { + for _, r := range e.ExpectedRules { + out[r] = true + } + } + return out +} diff --git a/internal/mapping/rulemap_test.go b/internal/mapping/rulemap_test.go new file mode 100644 index 0000000..aac9ab7 --- /dev/null +++ b/internal/mapping/rulemap_test.go @@ -0,0 +1,105 @@ +package mapping + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +const ( + ruleMapPath = "../../mappings/attack_rule_map.json" + detectionsDir = "../../detections" +) + +func loadShipped(t *testing.T) *RuleMap { + t.Helper() + m, err := LoadRuleMap(ruleMapPath) + if err != nil { + t.Fatalf("load rule map: %v", err) + } + return m +} + +func TestRuleMapEntriesAreComplete(t *testing.T) { + m := loadShipped(t) + for id, e := range m.Techniques { + if len(e.Atomics) == 0 { + t.Errorf("%s declares no atomic", id) + } + if len(e.ExpectedRules) == 0 { + t.Errorf("%s declares no expected rules", id) + } + if e.Platform == "" { + t.Errorf("%s declares no platform", id) + } + } +} + +// Every rule a technique expects must exist. A mapping pointing at a deleted +// rule would silently report MISSED forever. +func TestExpectedRulesExistOnDisk(t *testing.T) { + m := loadShipped(t) + for id, e := range m.Techniques { + for _, r := range e.ExpectedRules { + path := filepath.Join("../..", r) + if _, err := os.Stat(path); err != nil { + t.Errorf("%s expects %s, which does not exist", id, r) + } + } + } +} + +// The untested-rule lint: every Sigma rule in the repository is either mapped +// to a technique or explicitly declared untested WITH a reason. Without this a +// rule can be added, never exercised, and still be counted as coverage the +// project does not actually have. +func TestEveryRuleIsMappedOrDeclaredUntested(t *testing.T) { + m := loadShipped(t) + mapped := m.MappedRules() + + err := filepath.Walk(detectionsDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || !strings.HasSuffix(path, ".yml") { + return nil + } + if strings.Contains(path, string(filepath.Separator)+"tests"+string(filepath.Separator)) { + return nil // fixtures, not rules + } + rel := strings.TrimPrefix(filepath.ToSlash(path), "../../") + + if mapped[rel] { + return nil + } + reason, declared := m.UntestedRules[rel] + if !declared { + t.Errorf("rule %s is neither mapped to a technique nor declared in untested_rules — "+ + "an unexercised rule must not be silently counted as coverage", rel) + return nil + } + if strings.TrimSpace(reason) == "" { + t.Errorf("rule %s is declared untested with no reason", rel) + } + return nil + }) + if err != nil { + t.Fatalf("walk detections: %v", err) + } +} + +// The reverse guard: untested_rules must not name a rule that is also mapped, +// or a rule that no longer exists. +func TestUntestedRulesAreConsistent(t *testing.T) { + m := loadShipped(t) + mapped := m.MappedRules() + for rule := range m.UntestedRules { + if mapped[rule] { + t.Errorf("%s is listed as untested but is also mapped to a technique", rule) + } + if _, err := os.Stat(filepath.Join("../..", rule)); err != nil { + t.Errorf("%s is listed as untested but does not exist", rule) + } + } +} diff --git a/internal/report/dashboard.go b/internal/report/dashboard.go index 22e6abe..fa5fbad 100644 --- a/internal/report/dashboard.go +++ b/internal/report/dashboard.go @@ -6,6 +6,7 @@ import ( "math" "os" "path/filepath" + "runtime/debug" "github.com/jayelbotvibe-web/purple-loop/internal/model" ) @@ -47,12 +48,20 @@ func (r DashboardReporter) Write(result model.CampaignResult) error { // History index s := d["summary"].(map[string]any) c := d["canary"].(map[string]any) + // Per-technique verdicts go into the index so a later run can be diffed + // against this one. A history of coverage percentages cannot answer the + // question that matters — WHICH technique regressed. + verdicts := map[string]string{} + for _, ch := range result.Chains { + verdicts[ch.TechniqueID] = string(ch.Verdict) + } entry := map[string]any{ "id": runID, "campaign": d["campaign"], "generated_at": d["generated_at"], "coverage_pct": s["coverage_pct"], "canary_healthy": c["healthy"], + "techniques": verdicts, } if err := appendHistory(dir, entry); err != nil { return err @@ -86,10 +95,15 @@ func appendHistory(dir string, entry map[string]any) error { } func buildCoverage(result model.CampaignResult) map[string]any { + campaign := result.Campaign + if campaign == "" { + campaign = "unnamed" + } d := map[string]any{ - "campaign": "discovery", - "generated_at": result.StartedAt.UTC().Format("2006-01-02T15:04:05Z"), - "build": "v1.3.0", + "campaign": campaign, + "generated_at": result.StartedAt.UTC().Format("2006-01-02T15:04:05Z"), + "build": buildVersion(), + "atomics_commit": result.AtomicsCommit, } // Canary — the REAL positive-control result for this run (never hardcoded). @@ -117,19 +131,28 @@ func buildCoverage(result model.CampaignResult) map[string]any { noTel := counts[model.NoTelemetry] inconclusive := counts[model.Inconclusive] - denom := total - inconclusive - noTel + // Coverage is measured only over techniques that actually exercised a + // detection. Anything that never ran — INCONCLUSIVE, NO_TELEMETRY, + // SKIPPED_PREREQ, ERROR — is excluded from the denominator rather than + // silently counted as a gap. + skipped := counts[model.SkippedPrereq] + errored := counts[model.Errored] + denom := total - inconclusive - noTel - skipped - errored pct := 0 if denom > 0 { pct = int(math.Round(float64(detected) / float64(denom) * 100)) } d["summary"] = map[string]any{ - "total": total, - "detected": detected, - "missed": missed, - "no_telemetry": noTel, - "inconclusive": inconclusive, - "coverage_pct": pct, + "total": total, + "detected": detected, + "missed": missed, + "no_telemetry": noTel, + "inconclusive": inconclusive, + "skipped_prereq": skipped, + "errored": errored, + "exercised": denom, + "coverage_pct": pct, } // Readiness (stub — arbiter campaign not wired here yet) @@ -145,8 +168,31 @@ func buildCoverage(result model.CampaignResult) map[string]any { "Defense Evasion", "Credential Access", "Discovery", "Command & Control"} d["tactics"] = tactics - // Untested (empty for now) - d["untested"] = map[string]int{} + // Collection gaps, for handoff to detection-decay. NO_TELEMETRY says a rule + // never saw the events it needs; it does not say why. detection-decay models + // the cause (source death, field drift), so name the technique and the rule + // whose fields went missing rather than leaving the operator to guess. + var gaps []map[string]any + for _, c := range result.Chains { + if c.Verdict != model.NoTelemetry { + continue + } + gaps = append(gaps, map[string]any{ + "technique_id": c.TechniqueID, + "rules_expected": c.RulesExpected, + "atomic": c.Atomic.ID, + "window": map[string]any{"start": c.Window.Start, "end": c.Window.End}, + "note": c.Note, + }) + } + d["collection_gaps"] = gaps + + // Untested rules, with the reason each was not exercised. + untested := result.UntestedRules + if untested == nil { + untested = map[string]string{} + } + d["untested"] = untested // Techniques var techs []map[string]any @@ -179,9 +225,19 @@ func buildCoverage(result model.CampaignResult) map[string]any { } else { t["evidence"] = "" } - // Gap for MISSED - if c.Verdict == model.Missed { - t["gap"] = map[string]string{"why": "", "next": ""} + // Provenance: which upstream atomic produced this command, and whether + // it was substituted for the lab. + t["attribution"] = string(c.Attribution) + t["detect_latency_ms"] = c.DetectLatencyMS + t["rules_expected"] = c.RulesExpected + t["rules_matched"] = c.RulesMatched + t["atomic_name"] = c.Atomic.Name + t["atomic_source"] = c.Atomic.Source + t["override_reason"] = c.Atomic.OverrideReason + + // Gap: why this technique did not produce coverage. + if c.Verdict != model.Detected { + t["gap"] = map[string]string{"why": c.Note, "next": ""} } else { t["gap"] = nil } @@ -209,4 +265,11 @@ var techniqueMeta = map[string]struct{ name, tactic string }{ // Ensure interface compliance at compile time. var _ model.Reporter = DashboardReporter{} -func init() { _ = fmt.Sprintf("") } // suppress unused import +// buildVersion reports the compiled version rather than a hardcoded string, so +// a report always names the binary that produced it. +func buildVersion() string { + if bi, ok := debug.ReadBuildInfo(); ok && bi.Main.Version != "" && bi.Main.Version != "(devel)" { + return bi.Main.Version + } + return "devel" +} diff --git a/internal/report/report.go b/internal/report/report.go index bbcdf78..88d27e9 100644 --- a/internal/report/report.go +++ b/internal/report/report.go @@ -71,35 +71,70 @@ tr:hover{background:#1a1a2e} f.WriteString(`
`) - for _, v := range []model.Verdict{model.Detected, model.Missed, model.NoTelemetry, model.Inconclusive, model.Errored} { + for _, v := range []model.Verdict{model.Detected, model.Missed, model.NoTelemetry, model.Inconclusive, model.SkippedPrereq, model.Errored} { if n, ok := counts[v]; ok && n > 0 { f.WriteString(fmt.Sprintf(`%s: %d`, v, v, n)) } } f.WriteString(`
` + narrativeHeadline(counts) + `
-`) +
PriorityCVETechniqueVerdictEventsRule
`) for _, c := range run.Chains { evCount := fmt.Sprintf("%d", c.EventsCollected) - rule := c.RuleMatched - if rule == "" { - rule = "—" - } prio := fmt.Sprintf("%.2f", c.ArbiterPriority) - cve := c.SourceCVE - if cve == "" { - cve = "—" + cve := orDash(c.SourceCVE) + + // The last column carries the rule that fired, or — when nothing did — + // the reason, so a gap is never left unexplained. + detail := c.RuleMatched + if detail == "" { + detail = orDash(c.Note) + } + + // Provenance: a substituted command must not read as the upstream atomic. + atomicCell := html.EscapeString(orDash(c.Atomic.ID)) + if c.Atomic.OverrideReason != "" { + atomicCell += ` ⚑` } - f.WriteString(fmt.Sprintf(``, - prio, html.EscapeString(cve), html.EscapeString(c.TechniqueID), - c.Verdict, c.Verdict, evCount, html.EscapeString(rule))) + + latency := "—" + if c.DetectLatencyMS > 0 { + latency = fmt.Sprintf("%.1fs", float64(c.DetectLatencyMS)/1000) + } + + attr := string(c.Attribution) + attrStyle := "" + switch c.Attribution { + case model.WindowOverlap: + attrStyle = ` style="color:#f59e0b" title="another technique ran inside this collection window — a matching event may belong to either"` + case model.Unscoped: + attrStyle = ` style="color:#9ca3af"` + } + + f.WriteString(fmt.Sprintf( + `%s`, + prio, html.EscapeString(cve), html.EscapeString(c.TechniqueID), atomicCell, + c.Verdict, c.Verdict, attrStyle, html.EscapeString(orDash(attr)), + latency, evCount, html.EscapeString(detail))) } f.WriteString("
PriorityCVETechniqueAtomicVerdictAttributionLatencyEventsRule / why not
%s%s%s%s%s%s
%s%s%s%s%s%s%s%s
") return nil } +// narrativeHeadline reports coverage over the techniques that were actually +// exercised. A technique that ERRORed (unresolvable atomic, platform mismatch), +// was SKIPPED_PREREQ, or produced no telemetry never tested a detection, so +// counting any of them as "missed" would invent detection gaps out of +// harness failures — the same error class as presence-based coverage. +func orDash(s string) string { + if s == "" { + return "—" + } + return s +} + func narrativeHeadline(counts map[model.Verdict]int) string { total := 0 for _, n := range counts { @@ -110,9 +145,13 @@ func narrativeHeadline(counts map[model.Verdict]int) string { } det := counts[model.Detected] miss := counts[model.Missed] - errs := counts[model.Errored] - return fmt.Sprintf("Top %d exploited-in-the-wild: %d detected / %d missed", - total, det, miss+errs) + notRun := counts[model.Errored] + counts[model.SkippedPrereq] + counts[model.NoTelemetry] + counts[model.Inconclusive] + headline := fmt.Sprintf("%d of %d techniques exercised: %d detected / %d missed", + det+miss, total, det, miss) + if notRun > 0 { + headline += fmt.Sprintf(" — %d not exercised (see per-technique notes)", notRun) + } + return headline } // NavigatorLayerReporter writes an ATT&CK Navigator layer JSON file. @@ -144,9 +183,10 @@ func (r NavigatorLayerReporter) Write(run model.CampaignResult) error { color string score int }{ - model.Detected: {"#4caf50", 100}, - model.Missed: {"#f44336", 0}, - model.Errored: {"#e91e63", 0}, + model.Detected: {"#4caf50", 100}, + model.Missed: {"#f44336", 0}, + model.Errored: {"#e91e63", 0}, + model.SkippedPrereq: {"#9e9e9e", 0}, } desc := fmt.Sprintf("Detection coverage from campaign run at %s", run.StartedAt.Format(time.RFC3339)) diff --git a/mappings/attack_rule_map.json b/mappings/attack_rule_map.json new file mode 100644 index 0000000..fd9d97a --- /dev/null +++ b/mappings/attack_rule_map.json @@ -0,0 +1,116 @@ +{ + "_note": "Single source of truth for technique -> atomic -> expected detections. Replaces the technique->rule map that was hardcoded in cmd/purpleloop/main.go. 'expected_rules' lists every rule that SHOULD fire for this technique; a technique is DETECTED when any of them matches. 'platform' is the lab target kind the atomic runs on.", + "techniques": { + "T1059.004": { + "atomics": [ + "T1059.004-1" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/proc_creation_susp_shell.yml" + ] + }, + "T1087.001": { + "atomics": [ + "T1087.001-1" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1087.001.yml" + ] + }, + "T1082": { + "atomics": [ + "T1082-3" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1082.yml" + ] + }, + "T1033": { + "atomics": [ + "T1033-2" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1033.yml" + ] + }, + "T1007": { + "atomics": [ + "T1007-3" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1007.yml" + ] + }, + "T1016": { + "atomics": [ + "T1016-3" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1016.yml" + ] + }, + "T1049": { + "atomics": [ + "T1049-4" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1049.yml" + ] + }, + "T1069.001": { + "atomics": [ + "T1069.001-1" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1069.001.yml" + ] + }, + "T1069": { + "atomics": [ + "T1069.001-1" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1069.001.yml" + ] + }, + "T1135": { + "atomics": [ + "T1135-1" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1135.yml" + ] + }, + "T1518": { + "atomics": [ + "T1518-1" + ], + "platform": "linux", + "expected_rules": [ + "detections/linux/T1518.yml" + ] + }, + "T1059.003": { + "atomics": [ + "T1059.003-2" + ], + "platform": "windows", + "expected_rules": [ + "detections/windows/win_proc_create.yml" + ] + } + }, + "untested_rules": { + "detections/canary/pipeline_canary.yml": "not_a_detection \u2014 pipeline positive control, exercised by the canary gate on every run" + } +} From 5551f4ef57012d977545e3860ed32ea3bc3ec057 Mon Sep 17 00:00:00 2001 From: jayelbotvibe-web Date: Sat, 29 Aug 2026 15:09:06 +0800 Subject: [PATCH 06/13] feat(replay): capture run telemetry and re-evaluate it without a lab Replaces the fixed time.Sleep(10s) per technique and its 10-minute query window with collector.Settler, which polls until an expected rule matches or ingestion settles. The old wait was wrong in both directions: too long when the SIEM had already indexed the events (ten techniques spent 100s asleep), too short when it had not, producing a NO_TELEMETRY that was really an impatience artifact. Every real run now writes reports/runs//events.jsonl, and `purpleloop replay ` re-runs the REAL evaluator over it with no Docker and no Wazuh. This is the regression test the project could not previously have: CI has fixtures, but a fixture is a claim about the world written by the same person as the rule, not a sample of it. A rule edit can now be checked against every past run -- would this have broken a detection that already worked? The capture writer refuses synthetic runs, whose telemetry is fabricated and would be indistinguishable from a real dataset on replay, and skips a run that collected nothing rather than leaving an empty file that looks like a capture. testdata/sample-run ships a dataset in the capture format so replay is exercisable from a clean clone. Its README states plainly that it is not lab telemetry, and it deliberately includes a MISSED -- a sample where everything passes would teach the wrong thing. Co-Authored-By: Claude Opus 5 --- cmd/purpleloop/replay.go | 112 ++++++++++++++++++++++++ internal/capture/capture.go | 141 ++++++++++++++++++++++++++++++ internal/capture/capture_test.go | 122 ++++++++++++++++++++++++++ internal/collector/settle.go | 119 +++++++++++++++++++++++++ internal/collector/settle_test.go | 95 ++++++++++++++++++++ testdata/sample-run/README.md | 18 ++++ testdata/sample-run/events.jsonl | 5 ++ 7 files changed, 612 insertions(+) create mode 100644 cmd/purpleloop/replay.go create mode 100644 internal/capture/capture.go create mode 100644 internal/capture/capture_test.go create mode 100644 internal/collector/settle.go create mode 100644 internal/collector/settle_test.go create mode 100644 testdata/sample-run/README.md create mode 100644 testdata/sample-run/events.jsonl diff --git a/cmd/purpleloop/replay.go b/cmd/purpleloop/replay.go new file mode 100644 index 0000000..6cf6ceb --- /dev/null +++ b/cmd/purpleloop/replay.go @@ -0,0 +1,112 @@ +package main + +import ( + "flag" + "fmt" + "os" + "time" + + "github.com/jayelbotvibe-web/purple-loop/internal/capture" + "github.com/jayelbotvibe-web/purple-loop/internal/evaluator" + "github.com/jayelbotvibe-web/purple-loop/internal/mapping" + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +// runReplay re-evaluates a captured run's telemetry with no lab attached. +// +// It runs the SAME evaluator over the SAME events the live run collected, so a +// replay of an unchanged run reproduces its verdicts exactly — that equality is +// the test. Once rules change, the difference is the answer to the question CI +// could not previously ask: would this edit have broken a detection that +// already worked? +func runReplay() { + fs := flag.NewFlagSet("replay", flag.ExitOnError) + output := fs.String("output", "", "output file (.html) or empty for JSON on stdout") + ruleMapPath := fs.String("rule-map", defaultRuleMap, "technique -> expected rules mapping") + rulesDir := fs.String("rules-dir", "detections/linux", "rules root for the evaluator") + if err := fs.Parse(os.Args[2:]); err != nil { + os.Exit(2) + } + if fs.NArg() < 1 { + fmt.Fprintln(os.Stderr, "usage: purpleloop replay [--output report.html]") + os.Exit(2) + } + + result, err := replayDataset(fs.Arg(0), *ruleMapPath, *rulesDir) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + if err := newReporter(*output).Write(result); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +// replayDataset re-evaluates a captured dataset and returns the campaign result. +// Split out from the command so CI can assert that replaying a dataset +// reproduces the verdicts it was captured with. +func replayDataset(path, ruleMapPath, rulesDir string) (model.CampaignResult, error) { + ds, err := capture.Read(path) + if err != nil { + return model.CampaignResult{}, err + } + rules, err := mapping.LoadRuleMap(ruleMapPath) + if err != nil { + return model.CampaignResult{}, err + } + + eval := evaluator.RuleMatcherEvaluator{RulesDir: rulesDir} + order, events, meta := ds.ByTechnique() + + // A replay is a real evaluation of real telemetry, so it is NOT synthetic. + // It is also not a live run: nothing was executed and the canary did not + // fire here, so canary health is inherited from the captured run rather + // than asserted. Coverage from a replay describes the rules, not the lab. + result := model.CampaignResult{ + StartedAt: time.Now().UTC(), + Campaign: "replay:" + path, + CanaryHealthy: true, + } + + for _, tech := range order { + rec := meta[tech] + evs := events[tech] + chain := model.ProofChain{ + TechniqueID: tech, + Atomic: model.AtomicTest{ID: rec.AtomicID, TechniqueID: tech}, + ExecutedAt: rec.ExecutedAt, + Window: rec.Window, + EventsCollected: len(evs), + Attribution: model.WindowAndHostScoped, + } + + expected := rules.ExpectedRules(tech) + chain.RulesExpected = expected + if len(expected) == 0 { + chain.Verdict = model.Errored + chain.Note = "no Sigma rule mapped for " + tech + " in the rule map" + result.Chains = append(result.Chains, chain) + continue + } + + verdict, evidence, matched, err := evaluateExpected(eval, expected, tech, evs) + if err != nil { + chain.Verdict = model.Errored + chain.Note = err.Error() + result.Chains = append(result.Chains, chain) + continue + } + chain.Verdict = verdict + chain.Evidence = evidence + chain.RulesMatched = matched + if len(matched) > 0 { + chain.RuleMatched = matched[0] + chain.DetectLatencyMS = detectLatency(rec.ExecutedAt, evidence) + } + result.Chains = append(result.Chains, chain) + } + + return result, nil +} diff --git a/internal/capture/capture.go b/internal/capture/capture.go new file mode 100644 index 0000000..7b2ab1f --- /dev/null +++ b/internal/capture/capture.go @@ -0,0 +1,141 @@ +// Package capture persists the raw telemetry a campaign collected, so a run can +// be re-evaluated later without a lab. +// +// This is what makes rule changes testable against reality. CI has fixtures — +// events hand-written to prove a rule matches what its author intended — but a +// fixture is a claim about the world, not a sample of it. A captured run lets +// the same matcher be re-run over telemetry the lab actually produced, and lets +// a rule edit be checked against every past run: "would this have broken a +// DETECTED we already had?" +package capture + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +// Filename is the dataset written beside a run's coverage.json. +const Filename = "events.jsonl" + +// Record is one collected event with the execution it belongs to. The atomic +// and window travel with the event so a replay can reproduce the exact +// evaluation, not merely re-match a pile of logs. +type Record struct { + TechniqueID string `json:"technique_id"` + AtomicID string `json:"atomic_id"` + Host string `json:"host"` + ExecutedAt time.Time `json:"executed_at"` + Window model.TimeWindow `json:"window"` + Event model.Event `json:"event"` +} + +// Dataset is a replayable run, grouped by technique. +type Dataset struct { + Records []Record +} + +// Write persists every collected event for a campaign to dir/events.jsonl. +// A synthetic run is refused: its "telemetry" is fabricated, and a fabricated +// dataset replayed later would be indistinguishable from a real one. +func Write(dir string, run model.CampaignResult, host string) error { + if run.Synthetic { + return nil + } + + // No events means no dataset. Writing an empty file would leave something + // that looks like a capture but that Read rejects — the run's own report + // already records that nothing was collected. + total := 0 + for _, c := range run.Chains { + total += len(c.Collected) + } + if total == 0 { + return nil + } + + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + f, err := os.Create(filepath.Join(dir, Filename)) + if err != nil { + return err + } + defer f.Close() + + w := bufio.NewWriter(f) + enc := json.NewEncoder(w) + for _, c := range run.Chains { + for _, e := range c.Collected { + rec := Record{ + TechniqueID: c.TechniqueID, + AtomicID: c.Atomic.ID, + Host: host, + ExecutedAt: c.ExecutedAt, + Window: c.Window, + Event: e, + } + if err := enc.Encode(rec); err != nil { + return err + } + } + } + return w.Flush() +} + +// Read loads a captured dataset. dir may be the run directory or the file. +func Read(path string) (*Dataset, error) { + if fi, err := os.Stat(path); err == nil && fi.IsDir() { + path = filepath.Join(path, Filename) + } + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open dataset: %w", err) + } + defer f.Close() + + ds := &Dataset{} + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) // archive events can be large + line := 0 + for sc.Scan() { + line++ + raw := sc.Bytes() + if len(raw) == 0 { + continue + } + var rec Record + if err := json.Unmarshal(raw, &rec); err != nil { + return nil, fmt.Errorf("dataset line %d: %w", line, err) + } + ds.Records = append(ds.Records, rec) + } + if err := sc.Err(); err != nil { + return nil, fmt.Errorf("read dataset: %w", err) + } + if len(ds.Records) == 0 { + return nil, fmt.Errorf("dataset %s is empty", path) + } + return ds, nil +} + +// ByTechnique groups the dataset's events, preserving first-seen order so a +// replay reproduces the original campaign's sequence. +func (d *Dataset) ByTechnique() ([]string, map[string][]model.Event, map[string]Record) { + var order []string + events := map[string][]model.Event{} + meta := map[string]Record{} + for _, r := range d.Records { + if _, seen := events[r.TechniqueID]; !seen { + order = append(order, r.TechniqueID) + meta[r.TechniqueID] = r + } + events[r.TechniqueID] = append(events[r.TechniqueID], r.Event) + } + return order, events, meta +} diff --git a/internal/capture/capture_test.go b/internal/capture/capture_test.go new file mode 100644 index 0000000..d906b1d --- /dev/null +++ b/internal/capture/capture_test.go @@ -0,0 +1,122 @@ +package capture + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +func sampleRun(synthetic bool) model.CampaignResult { + start := time.Date(2026, 8, 29, 10, 0, 0, 0, time.UTC) + mk := func(tech, atomicID string, n int) model.ProofChain { + evs := make([]model.Event, n) + for i := range evs { + evs[i] = model.Event{ + ID: tech + "-ev", + Timestamp: start.Add(time.Duration(i) * time.Second), + Raw: json.RawMessage(`{"Image":"/usr/bin/uname","CommandLine":"uname -a"}`), + } + } + return model.ProofChain{ + TechniqueID: tech, + Atomic: model.AtomicTest{ID: atomicID, TechniqueID: tech}, + ExecutedAt: start, + Window: model.TimeWindow{Start: start, End: start.Add(time.Minute)}, + Collected: evs, + } + } + return model.CampaignResult{ + StartedAt: start, + Synthetic: synthetic, + Chains: []model.ProofChain{mk("T1082", "T1082-3", 2), mk("T1033", "T1033-2", 1)}, + } +} + +func TestWriteReadRoundTrip(t *testing.T) { + dir := t.TempDir() + if err := Write(dir, sampleRun(false), "victim01"); err != nil { + t.Fatalf("write: %v", err) + } + ds, err := Read(dir) + if err != nil { + t.Fatalf("read: %v", err) + } + if len(ds.Records) != 3 { + t.Fatalf("records = %d, want 3", len(ds.Records)) + } + + order, events, meta := ds.ByTechnique() + if len(order) != 2 || order[0] != "T1082" { + t.Errorf("order = %v, want T1082 first (campaign sequence preserved)", order) + } + if len(events["T1082"]) != 2 || len(events["T1033"]) != 1 { + t.Errorf("grouping wrong: %d/%d", len(events["T1082"]), len(events["T1033"])) + } + if meta["T1082"].AtomicID != "T1082-3" { + t.Errorf("atomic = %q, want T1082-3 — replay needs the atomic that produced the events", + meta["T1082"].AtomicID) + } + if meta["T1082"].Host != "victim01" { + t.Errorf("host = %q, want victim01", meta["T1082"].Host) + } + if meta["T1082"].Window.Start.IsZero() { + t.Error("the collection window must survive capture") + } +} + +// A synthetic run's telemetry is fabricated. Capturing it would produce a +// dataset indistinguishable from a real one on replay, so it is refused. +func TestWriteRefusesSyntheticRun(t *testing.T) { + dir := t.TempDir() + if err := Write(dir, sampleRun(true), "victim01"); err != nil { + t.Fatalf("write: %v", err) + } + if _, err := Read(dir); err == nil { + t.Error("a synthetic run must not produce a replayable dataset") + } +} + +func TestReadRejectsEmptyAndMissing(t *testing.T) { + if _, err := Read(filepath.Join(t.TempDir(), "nope")); err == nil { + t.Error("a missing dataset must error") + } +} + +// The raw event must survive byte-for-byte: replay re-runs the real matcher +// over it, so any normalisation here would change the verdict. +func TestRawEventPreservedExactly(t *testing.T) { + dir := t.TempDir() + run := sampleRun(false) + want := string(run.Chains[0].Collected[0].Raw) + + if err := Write(dir, run, "victim01"); err != nil { + t.Fatal(err) + } + ds, err := Read(dir) + if err != nil { + t.Fatal(err) + } + if got := string(ds.Records[0].Event.Raw); got != want { + t.Errorf("raw event changed through capture:\n got %s\nwant %s", got, want) + } +} + +// A run that collected nothing must not leave an empty file that looks like a +// dataset but that Read rejects. +func TestWriteSkipsRunWithNoEvents(t *testing.T) { + dir := t.TempDir() + run := model.CampaignResult{ + StartedAt: time.Now(), + Chains: []model.ProofChain{{TechniqueID: "T1082", Verdict: model.NoTelemetry}}, + } + if err := Write(dir, run, "victim01"); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dir, Filename)); !os.IsNotExist(err) { + t.Error("a run with no collected events must not write a dataset file") + } +} diff --git a/internal/collector/settle.go b/internal/collector/settle.go new file mode 100644 index 0000000..1b9445c --- /dev/null +++ b/internal/collector/settle.go @@ -0,0 +1,119 @@ +package collector + +import ( + "context" + "time" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +// Settler polls a collector until the telemetry for an execution has arrived, +// instead of sleeping a fixed amount and hoping. +// +// The pipeline previously slept 10s per technique and then queried once. That +// is wrong in both directions: too long when the SIEM has already indexed the +// events (ten techniques spend 100s asleep), and too short when it has not, +// which produces a NO_TELEMETRY that is really an impatience artifact. +// +// Settle returns as soon as one of three things is true: +// +// - Satisfied reports a match — nothing later can improve on a detection; +// - the event count stops growing for StableFor consecutive polls, meaning +// ingestion has settled and no further events are coming; +// - Timeout elapses, which is a genuine "the events never arrived". +type Settler struct { + PollInterval time.Duration // gap between polls (default 3s) + Timeout time.Duration // total budget (default 60s) + StableFor int // polls with no new events before settling (default 2) + MinWait time.Duration // floor before the first poll (default 2s) +} + +func (s Settler) withDefaults() Settler { + if s.PollInterval <= 0 { + s.PollInterval = 3 * time.Second + } + if s.Timeout <= 0 { + s.Timeout = 60 * time.Second + } + if s.StableFor <= 0 { + s.StableFor = 2 + } + if s.MinWait < 0 { + s.MinWait = 0 + } + if s.MinWait == 0 { + s.MinWait = 2 * time.Second + } + return s +} + +// SettleResult reports what the poll loop observed. +type SettleResult struct { + Events []model.Event + Satisfied bool // the caller's predicate matched + Waited time.Duration // how long collection actually took + TimedOut bool +} + +// Settle polls coll for the window until satisfied, stable, or timed out. +// satisfied may be nil, in which case Settle waits for ingestion to stabilise. +func (s Settler) Settle(ctx context.Context, coll model.Collector, window model.TimeWindow, + host string, satisfied func([]model.Event) bool) (SettleResult, error) { + + s = s.withDefaults() + started := time.Now() + deadline := started.Add(s.Timeout) + + // A brief floor before the first query: an immediate poll almost always + // races the SIEM's indexer and wastes a round trip. + if err := sleepCtx(ctx, s.MinWait); err != nil { + return SettleResult{}, err + } + + var last []model.Event + stable := 0 + + for { + events, err := coll.Query(ctx, window, host) + if err != nil { + return SettleResult{Events: last, Waited: time.Since(started)}, err + } + + if satisfied != nil && satisfied(events) { + return SettleResult{Events: events, Satisfied: true, Waited: time.Since(started)}, nil + } + + if len(events) == len(last) { + stable++ + } else { + stable = 0 + } + last = events + + // Ingestion has settled: the count held steady across consecutive polls, + // so waiting longer cannot change the verdict. Requiring at least one + // event avoids settling instantly on an empty archive that is merely slow. + if stable >= s.StableFor && len(events) > 0 { + return SettleResult{Events: events, Waited: time.Since(started)}, nil + } + + if time.Now().After(deadline) { + return SettleResult{Events: events, Waited: time.Since(started), TimedOut: true}, nil + } + if err := sleepCtx(ctx, s.PollInterval); err != nil { + return SettleResult{Events: events, Waited: time.Since(started)}, err + } + } +} + +func sleepCtx(ctx context.Context, d time.Duration) error { + if d <= 0 { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(d): + return nil + } +} diff --git a/internal/collector/settle_test.go b/internal/collector/settle_test.go new file mode 100644 index 0000000..754ac35 --- /dev/null +++ b/internal/collector/settle_test.go @@ -0,0 +1,95 @@ +package collector + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +// scriptedCollector returns a different event slice on each successive call. +type scriptedCollector struct { + rounds [][]model.Event + calls int +} + +func (c *scriptedCollector) Query(ctx context.Context, w model.TimeWindow, host string) ([]model.Event, error) { + i := c.calls + c.calls++ + if i >= len(c.rounds) { + i = len(c.rounds) - 1 + } + return c.rounds[i], nil +} + +func ev(n int) []model.Event { + out := make([]model.Event, n) + for i := range out { + out[i] = model.Event{ID: string(rune('a' + i)), Raw: json.RawMessage(`{}`)} + } + return out +} + +func fast() Settler { + return Settler{PollInterval: time.Millisecond, Timeout: 200 * time.Millisecond, StableFor: 2, MinWait: time.Millisecond} +} + +// A match ends collection immediately — nothing later can improve on it. +func TestSettleStopsOnSatisfied(t *testing.T) { + c := &scriptedCollector{rounds: [][]model.Event{ev(1), ev(2), ev(3)}} + res, err := fast().Settle(context.Background(), c, model.TimeWindow{}, "h", + func(e []model.Event) bool { return len(e) >= 2 }) + if err != nil { + t.Fatal(err) + } + if !res.Satisfied { + t.Error("want Satisfied") + } + if c.calls != 2 { + t.Errorf("polled %d times, want 2 — it should stop as soon as the predicate matches", c.calls) + } +} + +// Once the count holds steady, ingestion has settled and waiting longer cannot +// change the verdict. +func TestSettleStopsWhenStable(t *testing.T) { + c := &scriptedCollector{rounds: [][]model.Event{ev(1), ev(2), ev(2), ev(2)}} + res, err := fast().Settle(context.Background(), c, model.TimeWindow{}, "h", func([]model.Event) bool { return false }) + if err != nil { + t.Fatal(err) + } + if res.Satisfied || res.TimedOut { + t.Errorf("want a settled result, got satisfied=%v timedout=%v", res.Satisfied, res.TimedOut) + } + if len(res.Events) != 2 { + t.Errorf("events = %d, want 2", len(res.Events)) + } +} + +// An archive that never produces an event must time out rather than settle at +// zero: "nothing arrived yet" and "nothing will arrive" are different claims, +// and only the timeout can distinguish them. +func TestSettleDoesNotSettleOnEmpty(t *testing.T) { + c := &scriptedCollector{rounds: [][]model.Event{{}}} + res, err := fast().Settle(context.Background(), c, model.TimeWindow{}, "h", func([]model.Event) bool { return false }) + if err != nil { + t.Fatal(err) + } + if !res.TimedOut { + t.Error("an always-empty archive must time out, not settle") + } + if len(res.Events) != 0 { + t.Errorf("events = %d, want 0", len(res.Events)) + } +} + +func TestSettleRespectsContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + c := &scriptedCollector{rounds: [][]model.Event{{}}} + if _, err := fast().Settle(ctx, c, model.TimeWindow{}, "h", nil); err == nil { + t.Error("a cancelled context must abort collection") + } +} diff --git a/testdata/sample-run/README.md b/testdata/sample-run/README.md new file mode 100644 index 0000000..08dc0ff --- /dev/null +++ b/testdata/sample-run/README.md @@ -0,0 +1,18 @@ +# Sample replay dataset + +`events.jsonl` is a dataset in the captured-run format, so `purpleloop replay` can be +exercised without a lab: + +```bash +go run ./cmd/purpleloop replay testdata/sample-run +``` + +**This is not lab telemetry.** The events are the same ones the CI fixtures assert, shaped +into the capture format. It demonstrates the replay path and gives a newcomer a real verdict +from the real matcher — it is not evidence about any real detection pipeline. A genuine +dataset is produced by a real run: `reports/runs//events.jsonl`, which the capture +writer refuses to create for a synthetic run. + +`T1518` is included deliberately as a **MISSED**: its atomic ran and produced telemetry, but +the events do not satisfy the rule. A sample where everything passes would teach the wrong +thing. diff --git a/testdata/sample-run/events.jsonl b/testdata/sample-run/events.jsonl new file mode 100644 index 0000000..cadb59c --- /dev/null +++ b/testdata/sample-run/events.jsonl @@ -0,0 +1,5 @@ +{"technique_id": "T1082", "atomic_id": "T1082-3", "host": "victim01", "executed_at": "2026-08-29T10:00:00Z", "window": {"start": "2026-08-29T10:00:00Z", "end": "2026-08-29T10:01:00Z"}, "event": {"event_id": "T1082-0", "timestamp": "2026-08-29T10:00:03Z", "raw": {"Image": "/usr/bin/uname", "ParentImage": "/bin/sh", "CommandLine": "uname -a", "User": "root"}}} +{"technique_id": "T1082", "atomic_id": "T1082-3", "host": "victim01", "executed_at": "2026-08-29T10:00:00Z", "window": {"start": "2026-08-29T10:00:00Z", "end": "2026-08-29T10:01:00Z"}, "event": {"event_id": "T1082-1", "timestamp": "2026-08-29T10:00:04Z", "raw": {"Image": "/usr/bin/cat", "ParentImage": "/bin/sh", "CommandLine": "cat /etc/os-release", "User": "root"}}} +{"technique_id": "T1033", "atomic_id": "T1033-2", "host": "victim01", "executed_at": "2026-08-29T10:02:00Z", "window": {"start": "2026-08-29T10:02:00Z", "end": "2026-08-29T10:03:00Z"}, "event": {"event_id": "T1033-0", "timestamp": "2026-08-29T10:02:03Z", "raw": {"Image": "/usr/bin/who", "ParentImage": "/bin/sh", "CommandLine": "who", "User": "root"}}} +{"technique_id": "T1049", "atomic_id": "T1049-4", "host": "victim01", "executed_at": "2026-08-29T10:04:00Z", "window": {"start": "2026-08-29T10:04:00Z", "end": "2026-08-29T10:05:00Z"}, "event": {"event_id": "T1049-0", "timestamp": "2026-08-29T10:04:03Z", "raw": {"Image": "/usr/bin/ss", "ParentImage": "/bin/bash", "CommandLine": "ss -antp", "User": "root"}}} +{"technique_id": "T1518", "atomic_id": "T1518-1", "host": "victim01", "executed_at": "2026-08-29T10:06:00Z", "window": {"start": "2026-08-29T10:06:00Z", "end": "2026-08-29T10:07:00Z"}, "event": {"event_id": "T1518-0", "timestamp": "2026-08-29T10:06:03Z", "raw": {"Image": "/usr/bin/ls", "ParentImage": "/bin/sh", "CommandLine": "ls -1 /usr/bin", "User": "root"}}} From 7a6ae6c6ce7f842be67814ef6175c04b9e31175d Mon Sep 17 00:00:00 2001 From: jayelbotvibe-web Date: Sat, 29 Aug 2026 15:09:06 +0800 Subject: [PATCH 07/13] feat(precision): measure false positives against a benign workload Coverage without a false-positive rate is half a metric: a rule that matches everything scores perfect coverage and is worthless. The repository's negative fixtures prove a rule rejects an event crafted to be rejected, which is a claim about the author's intent. This measures every rule against a workload written independently of it. emulation/benign-baseline.yml is 26 read-only administrator commands across seven categories, including adjacent cases -- benign uses of the very binaries the detections watch. Those are what make the test worth running: a rule keyed on a tool name rather than on what the tool is being asked to do fires there. It found two real false positives on introduction, both now fixed: - T1087.001 fired on `getent hosts localhost`, a DNS lookup, not account enumeration. It now requires an account database. - T1135 fired on a bare `mount`, routine administration. It now requires a network filesystem. `purpleloop precision` exits non-zero on any false positive, so it gates CI. Co-Authored-By: Claude Opus 5 --- cmd/purpleloop/precision.go | 64 ++++++++++++ emulation/benign-baseline.yml | 134 +++++++++++++++++++++++++ internal/precision/precision.go | 143 +++++++++++++++++++++++++++ internal/precision/precision_test.go | 66 +++++++++++++ 4 files changed, 407 insertions(+) create mode 100644 cmd/purpleloop/precision.go create mode 100644 emulation/benign-baseline.yml create mode 100644 internal/precision/precision.go create mode 100644 internal/precision/precision_test.go diff --git a/cmd/purpleloop/precision.go b/cmd/purpleloop/precision.go new file mode 100644 index 0000000..dc6946c --- /dev/null +++ b/cmd/purpleloop/precision.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + + "github.com/jayelbotvibe-web/purple-loop/internal/evaluator" + "github.com/jayelbotvibe-web/purple-loop/internal/precision" +) + +// runPrecision measures the false-positive rate of the shipped detections +// against a benign administrator workload. +// +// Coverage answers "does it fire when it should". This answers "does it stay +// quiet when it should", which coverage alone cannot: a rule matching +// everything scores 100% coverage and is useless. Reported together, they are +// a claim about a detection set; reported alone, coverage is a half-truth. +func runPrecision() { + fs := flag.NewFlagSet("precision", flag.ExitOnError) + baseline := fs.String("baseline", "emulation/benign-baseline.yml", "benign workload definition") + rulesDir := fs.String("rules-dir", "detections/linux", "rules to measure") + asJSON := fs.Bool("json", false, "emit JSON instead of a table") + if err := fs.Parse(os.Args[2:]); err != nil { + os.Exit(2) + } + + b, err := precision.LoadBaseline(*baseline) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + res, err := precision.Measure(evaluator.RuleMatcherEvaluator{RulesDir: *rulesDir}, *rulesDir, b) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + if *asJSON { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(res); err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + } else { + fmt.Printf("Precision: %d%% (%d benign commands, %d rules)\n", + res.PrecisionPct, res.CommandsRun, res.RulesChecked) + if len(res.FalsePositives) == 0 { + fmt.Println("No rule fired on benign activity.") + } + for _, fp := range res.FalsePositives { + fmt.Printf(" FALSE POSITIVE %s\n fired on %q (%s)\n", + fp.Rule, fp.Command, fp.Category) + } + } + + // A false positive is a finding, not a note: exit non-zero so this gates CI + // and a release the same way a failing detection would. + if len(res.FalsePositives) > 0 { + os.Exit(1) + } +} diff --git a/emulation/benign-baseline.yml b/emulation/benign-baseline.yml new file mode 100644 index 0000000..c6c88c9 --- /dev/null +++ b/emulation/benign-baseline.yml @@ -0,0 +1,134 @@ +# Benign administrator workload — the precision baseline. +# +# CI's negative fixtures prove a rule rejects an event crafted to be rejected. +# That is a weak claim: the fixture was written by the same person as the rule. +# This is the stronger one — ordinary administration that a detection must stay +# silent about. Any rule that fires here is a measured false positive, reported +# with the command that triggered it. +# +# `image` and `command_line` are the process-creation event the command +# produces, so precision can be measured offline in CI as well as live in the +# lab. Commands are read-only and safe to run on the victim container. +name: benign-admin-baseline +description: Read-only administrator activity that no detection should fire on + +commands: + # --- package management --- + - category: package-management + command: apt-get --version + image: /usr/bin/apt-get + command_line: apt-get --version + - category: package-management + command: dpkg --configure -a --dry-run + image: /usr/bin/dpkg + command_line: dpkg --configure -a --dry-run + - category: package-management + command: apt-cache policy bash + image: /usr/bin/apt-cache + command_line: apt-cache policy bash + + # --- routine file operations --- + - category: file-ops + command: ls -la /var/log + image: /usr/bin/ls + command_line: ls -la /var/log + - category: file-ops + command: du -sh /var/log + image: /usr/bin/du + command_line: du -sh /var/log + - category: file-ops + command: df -h + image: /usr/bin/df + command_line: df -h + - category: file-ops + command: cat /etc/hostname + image: /usr/bin/cat + command_line: cat /etc/hostname + - category: file-ops + command: tail -n 20 /var/log/dpkg.log + image: /usr/bin/tail + command_line: tail -n 20 /var/log/dpkg.log + + # --- process and resource inspection --- + - category: process-inspection + command: ps aux + image: /usr/bin/ps + command_line: ps aux + - category: process-inspection + command: free -m + image: /usr/bin/free + command_line: free -m + - category: process-inspection + command: uptime + image: /usr/bin/uptime + command_line: uptime + + # --- service operation (NOT enumeration) --- + - category: service-ops + command: systemctl is-active cron + image: /usr/bin/systemctl + command_line: systemctl is-active cron + - category: service-ops + command: systemctl status cron + image: /usr/bin/systemctl + command_line: systemctl status cron + + # --- network operation (NOT reconnaissance) --- + - category: network-ops + command: ping -c 1 127.0.0.1 + image: /usr/bin/ping + command_line: ping -c 1 127.0.0.1 + - category: network-ops + command: ip --version + image: /usr/sbin/ip + command_line: ip --version + - category: network-ops + command: curl --version + image: /usr/bin/curl + command_line: curl --version + + # --- ADJACENT cases: benign uses of the very binaries the detections watch. + # These are what make this test worth running. A rule keyed on a tool name + # rather than on what the tool is being asked to do will fire here. + - category: adjacent-benign + command: mount + image: /usr/bin/mount + command_line: mount + - category: adjacent-benign + command: getent hosts localhost + image: /usr/bin/getent + command_line: getent hosts localhost + - category: adjacent-benign + command: lsof /var/log/syslog + image: /usr/bin/lsof + command_line: lsof /var/log/syslog + - category: adjacent-benign + command: systemctl restart cron + image: /usr/bin/systemctl + command_line: systemctl restart cron + - category: adjacent-benign + command: cat /proc/cpuinfo + image: /usr/bin/cat + command_line: cat /proc/cpuinfo + - category: adjacent-benign + command: dpkg --configure -a + image: /usr/bin/dpkg + command_line: dpkg --configure -a + + # --- shell and text utilities --- + - category: shell-utils + command: echo hello + image: /usr/bin/echo + command_line: echo hello + - category: shell-utils + command: date + image: /usr/bin/date + command_line: date + - category: shell-utils + command: env + image: /usr/bin/env + command_line: env + - category: shell-utils + command: grep --version + image: /usr/bin/grep + command_line: grep --version diff --git a/internal/precision/precision.go b/internal/precision/precision.go new file mode 100644 index 0000000..7f783ec --- /dev/null +++ b/internal/precision/precision.go @@ -0,0 +1,143 @@ +// Package precision measures false positives: whether detections stay silent +// during ordinary administration. +// +// Coverage without a false-positive rate is half a metric. A rule that matches +// everything scores perfect coverage and is worthless. The repository's +// negative fixtures prove a rule rejects an event crafted to be rejected, which +// is a claim about the author's intent; this measures the rule against a +// workload written independently of it. +package precision + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +// Command is one benign action and the process-creation event it produces. +type Command struct { + Category string `yaml:"category"` + Command string `yaml:"command"` + Image string `yaml:"image"` + CommandLine string `yaml:"command_line"` +} + +// Baseline is the benign workload. +type Baseline struct { + Name string `yaml:"name"` + Description string `yaml:"description"` + Commands []Command `yaml:"commands"` +} + +// LoadBaseline reads the benign workload definition. +func LoadBaseline(path string) (*Baseline, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read baseline: %w", err) + } + var b Baseline + if err := yaml.Unmarshal(data, &b); err != nil { + return nil, fmt.Errorf("parse baseline %s: %w", path, err) + } + if len(b.Commands) == 0 { + return nil, fmt.Errorf("baseline %s declares no commands", path) + } + for i, c := range b.Commands { + if c.Image == "" || c.CommandLine == "" { + return nil, fmt.Errorf("baseline entry %d (%q) needs both image and command_line", i, c.Command) + } + } + return &b, nil +} + +// Event renders a benign command as the process-creation event it would emit. +func (c Command) Event() model.Event { + raw, _ := json.Marshal(map[string]string{ + "Image": c.Image, + "ParentImage": "/bin/bash", + "CommandLine": c.CommandLine, + "User": "root", + }) + return model.Event{ID: "benign-" + c.Image, Raw: raw} +} + +// FalsePositive is one rule firing on one benign command. +type FalsePositive struct { + Rule string `json:"rule"` + Command string `json:"command"` + Category string `json:"category"` +} + +// Result is the outcome of a precision run. +type Result struct { + Baseline string `json:"baseline"` + CommandsRun int `json:"commands_run"` + RulesChecked int `json:"rules_checked"` + FalsePositives []FalsePositive `json:"false_positives"` + // PrecisionPct is the share of benign commands that fired no rule at all. + PrecisionPct int `json:"precision_pct"` +} + +// Measure evaluates every rule against every benign command. A rule that fires +// is a false positive named with the command that triggered it, so the finding +// is actionable rather than a bare number. +func Measure(eval model.Evaluator, rulesDir string, b *Baseline) (*Result, error) { + rules, err := listRules(rulesDir) + if err != nil { + return nil, err + } + + res := &Result{Baseline: b.Name, CommandsRun: len(b.Commands), RulesChecked: len(rules)} + noisy := map[string]bool{} + + for _, c := range b.Commands { + events := []model.Event{c.Event()} + for _, rule := range rules { + verdict, _, err := eval.Evaluate(model.SigmaRule{Path: rule, Title: rule}, events) + if err != nil { + return nil, fmt.Errorf("evaluate %s: %w", rule, err) + } + if verdict == model.Detected { + res.FalsePositives = append(res.FalsePositives, FalsePositive{ + Rule: rule, Command: c.Command, Category: c.Category, + }) + noisy[c.Command] = true + } + } + } + + clean := len(b.Commands) - len(noisy) + if len(b.Commands) > 0 { + res.PrecisionPct = clean * 100 / len(b.Commands) + } + sort.Slice(res.FalsePositives, func(i, j int) bool { + if res.FalsePositives[i].Rule != res.FalsePositives[j].Rule { + return res.FalsePositives[i].Rule < res.FalsePositives[j].Rule + } + return res.FalsePositives[i].Command < res.FalsePositives[j].Command + }) + return res, nil +} + +func listRules(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("read rules dir: %w", err) + } + var out []string + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".yml") { + continue + } + out = append(out, filepath.Join(dir, e.Name())) + } + sort.Strings(out) + return out, nil +} diff --git a/internal/precision/precision_test.go b/internal/precision/precision_test.go new file mode 100644 index 0000000..fe7e71e --- /dev/null +++ b/internal/precision/precision_test.go @@ -0,0 +1,66 @@ +package precision + +import ( + "testing" + + "github.com/jayelbotvibe-web/purple-loop/internal/evaluator" +) + +const ( + baselinePath = "../../emulation/benign-baseline.yml" + rulesDir = "../../detections/linux" +) + +// The precision gate. Every shipped rule must stay silent during ordinary +// administration, including the adjacent cases — benign uses of the very +// binaries the detections watch. A rule keyed on a tool name rather than on +// what the tool is being asked to do fires here and fails the build. +// +// This found two real false positives when it was introduced: T1087.001 fired +// on `getent hosts` (a DNS lookup, not account enumeration) and T1135 fired on +// a bare `mount`. Both rules were narrowed. +func TestRulesAreSilentOnBenignWorkload(t *testing.T) { + b, err := LoadBaseline(baselinePath) + if err != nil { + t.Fatalf("load baseline: %v", err) + } + res, err := Measure(evaluator.RuleMatcherEvaluator{RulesDir: rulesDir}, rulesDir, b) + if err != nil { + t.Fatalf("measure: %v", err) + } + for _, fp := range res.FalsePositives { + t.Errorf("false positive: %s fired on benign %q (%s)", fp.Rule, fp.Command, fp.Category) + } + if res.PrecisionPct != 100 { + t.Errorf("precision = %d%%, want 100%% (%d commands, %d rules)", + res.PrecisionPct, res.CommandsRun, res.RulesChecked) + } +} + +// A baseline with no adjacent cases proves nothing — it would pass against a +// rule set that matches only exact strings. Guard the guard. +func TestBaselineIncludesAdjacentCases(t *testing.T) { + b, err := LoadBaseline(baselinePath) + if err != nil { + t.Fatal(err) + } + adjacent := 0 + for _, c := range b.Commands { + if c.Category == "adjacent-benign" { + adjacent++ + } + } + if adjacent < 5 { + t.Errorf("baseline has %d adjacent-benign commands, want at least 5 — "+ + "without benign uses of the watched binaries this test is vacuous", adjacent) + } + if len(b.Commands) < 20 { + t.Errorf("baseline has %d commands, want at least 20", len(b.Commands)) + } +} + +func TestLoadBaselineRejectsIncomplete(t *testing.T) { + if _, err := LoadBaseline("does-not-exist.yml"); err == nil { + t.Error("a missing baseline must error") + } +} From db0c62390c27b659df31f39ce2259d0dbcd6b71e Mon Sep 17 00:00:00 2001 From: jayelbotvibe-web Date: Sat, 29 Aug 2026 15:09:22 +0800 Subject: [PATCH 08/13] feat(lab): reach Windows from `run`, and diff two runs per technique runTasks hardcoded a Linux target, rules directory and executor. SSHExecutor and the Windows rule existed only inside the standalone `canary` subcommand, which also hardcoded the IP 192.168.88.13 -- so the v1.0 cross-platform coverage claim could not be exercised through `run` at all. Targets are declared in lab/targets.yml, techniques are dispatched to the target matching the platform their rule-map entry names, and the canary runs once per platform in use: a healthy Linux canary says nothing about whether Windows telemetry is flowing. Credentials come from the environment, never from source. `purpleloop diff [--fail-on-regression]` names the technique that regressed -- "T1082 DETECTED -> MISSED" -- instead of only moving a percentage. That was impossible while history.json stored a coverage number and nothing else; it now carries per-technique verdicts. Verdicts meaning "never exercised" share a rank, so ERROR -> SKIPPED_PREREQ is a change, not a regression. Co-Authored-By: Claude Opus 5 --- cmd/purpleloop/diff.go | 149 +++++++++++++++++++++++++++++++++++ internal/lab/targets.go | 107 +++++++++++++++++++++++++ internal/lab/targets_test.go | 78 ++++++++++++++++++ lab/targets.yml | 30 +++++++ plans/cross-platform.yml | 13 +++ plans/windows-discovery.yml | 12 +++ 6 files changed, 389 insertions(+) create mode 100644 cmd/purpleloop/diff.go create mode 100644 internal/lab/targets.go create mode 100644 internal/lab/targets_test.go create mode 100644 lab/targets.yml create mode 100644 plans/cross-platform.yml create mode 100644 plans/windows-discovery.yml diff --git a/cmd/purpleloop/diff.go b/cmd/purpleloop/diff.go new file mode 100644 index 0000000..9569c92 --- /dev/null +++ b/cmd/purpleloop/diff.go @@ -0,0 +1,149 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "path/filepath" + "sort" +) + +// runDiff compares two runs and reports what changed per technique. +// +// A coverage percentage that moves from 80% to 70% tells you something broke +// but not what. This names it: "T1082 regressed DETECTED -> MISSED". That is +// the output detection engineering actually acts on, and it was impossible +// while the history index stored only a percentage. +func runDiff() { + fs := flag.NewFlagSet("diff", flag.ExitOnError) + failOnRegression := fs.Bool("fail-on-regression", false, "exit non-zero if any technique regressed") + if err := fs.Parse(os.Args[2:]); err != nil { + os.Exit(2) + } + if fs.NArg() != 2 { + fmt.Fprintln(os.Stderr, "usage: purpleloop diff [--fail-on-regression] ") + fmt.Fprintln(os.Stderr, " each argument is a run directory or its coverage.json") + fmt.Fprintln(os.Stderr, " flags must precede the run paths") + os.Exit(2) + } + + before, err := readCoverage(fs.Arg(0)) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + after, err := readCoverage(fs.Arg(1)) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + + regressions, improvements, other := compareRuns(before, after) + + fmt.Printf("%s -> %s\n\n", fs.Arg(0), fs.Arg(1)) + printChanges("REGRESSED", regressions) + printChanges("IMPROVED", improvements) + printChanges("CHANGED", other) + + if len(regressions)+len(improvements)+len(other) == 0 { + fmt.Println("No per-technique changes.") + } + if *failOnRegression && len(regressions) > 0 { + os.Exit(1) + } +} + +type change struct{ Technique, From, To string } + +// verdictRank orders outcomes by how much detection they demonstrate, so a +// move down the scale is a regression. Verdicts that mean "never exercised" +// deliberately share the bottom: moving between them is a change worth showing +// but is not a detection regression. +var verdictRank = map[string]int{ + "ERROR": 0, + "SKIPPED_PREREQ": 0, + "INCONCLUSIVE": 0, + "NO_TELEMETRY": 1, + "MISSED": 2, + "DETECTED": 3, +} + +func compareRuns(before, after map[string]string) (regressions, improvements, other []change) { + seen := map[string]bool{} + for t := range before { + seen[t] = true + } + for t := range after { + seen[t] = true + } + techniques := make([]string, 0, len(seen)) + for t := range seen { + techniques = append(techniques, t) + } + sort.Strings(techniques) + + for _, t := range techniques { + from, okB := before[t] + to, okA := after[t] + switch { + case !okB: + other = append(other, change{t, "(absent)", to}) + continue + case !okA: + other = append(other, change{t, from, "(absent)"}) + continue + case from == to: + continue + } + c := change{t, from, to} + switch { + case verdictRank[to] < verdictRank[from]: + regressions = append(regressions, c) + case verdictRank[to] > verdictRank[from]: + improvements = append(improvements, c) + default: + other = append(other, c) + } + } + return regressions, improvements, other +} + +func printChanges(label string, cs []change) { + if len(cs) == 0 { + return + } + fmt.Printf("%s (%d)\n", label, len(cs)) + for _, c := range cs { + fmt.Printf(" %-12s %s -> %s\n", c.Technique, c.From, c.To) + } + fmt.Println() +} + +// readCoverage loads a run's per-technique verdicts from its coverage.json. +func readCoverage(path string) (map[string]string, error) { + if fi, err := os.Stat(path); err == nil && fi.IsDir() { + path = filepath.Join(path, "coverage.json") + } + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read run: %w", err) + } + var doc struct { + Techniques []struct { + ID string `json:"id"` + Verdict string `json:"verdict"` + } `json:"techniques"` + } + if err := json.Unmarshal(data, &doc); err != nil { + return nil, fmt.Errorf("parse %s: %w", path, err) + } + if len(doc.Techniques) == 0 { + return nil, fmt.Errorf("%s has no technique results", path) + } + out := make(map[string]string, len(doc.Techniques)) + for _, t := range doc.Techniques { + out[t.ID] = t.Verdict + } + return out, nil +} diff --git a/internal/lab/targets.go b/internal/lab/targets.go new file mode 100644 index 0000000..9109b8d --- /dev/null +++ b/internal/lab/targets.go @@ -0,0 +1,107 @@ +// Package lab describes the hosts a campaign can dispatch techniques to. +// +// The engine previously hardcoded one Linux target in the run path, so a +// Windows rule could never be exercised by `run` even though the executor and +// the rule both existed. Targets are configuration now, and credentials are +// read from the environment rather than compiled into the binary. +package lab + +import ( + "fmt" + "os" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +// Target is one lab host and how to reach it. +type Target struct { + Name string `yaml:"name"` + Kind string `yaml:"kind"` + Executor string `yaml:"executor"` // "docker" | "ssh" + Container string `yaml:"container"` + RulesDir string `yaml:"rules_dir"` + + SSHHostEnv string `yaml:"ssh_host_env"` + SSHUserEnv string `yaml:"ssh_user_env"` + SSHPassEnv string `yaml:"ssh_pass_env"` + DefaultHost string `yaml:"default_host"` + DefaultUser string `yaml:"default_user"` +} + +// Targets is the lab inventory. +type Targets struct { + Targets []Target `yaml:"targets"` +} + +// Load reads the lab target inventory. +func Load(path string) (*Targets, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read targets: %w", err) + } + var t Targets + if err := yaml.Unmarshal(data, &t); err != nil { + return nil, fmt.Errorf("parse targets %s: %w", path, err) + } + if len(t.Targets) == 0 { + return nil, fmt.Errorf("targets %s declares no hosts", path) + } + for i, tt := range t.Targets { + if tt.Name == "" || tt.Kind == "" || tt.Executor == "" { + return nil, fmt.Errorf("target %d needs name, kind and executor", i) + } + } + return &t, nil +} + +// ForPlatform returns the target that runs a given platform's techniques. +func (t *Targets) ForPlatform(kind string) (Target, bool) { + for _, tt := range t.Targets { + if strings.EqualFold(tt.Kind, kind) { + return tt, true + } + } + return Target{}, false +} + +// Kinds lists the distinct platforms the inventory can reach. +func (t *Targets) Kinds() []string { + seen := map[string]bool{} + var out []string + for _, tt := range t.Targets { + k := strings.ToLower(tt.Kind) + if !seen[k] { + seen[k] = true + out = append(out, k) + } + } + return out +} + +// Model converts to the engine's target type. +func (t Target) Model() model.Target { + return model.Target{Host: t.Name, Kind: t.Kind} +} + +// SSHConfig resolves connection details from the environment, falling back to +// the declared defaults. A password is never defaulted. +func (t Target) SSHConfig() (host, user, pass string) { + host = envOr(t.SSHHostEnv, t.DefaultHost) + user = envOr(t.SSHUserEnv, t.DefaultUser) + if t.SSHPassEnv != "" { + pass = os.Getenv(t.SSHPassEnv) + } + return host, user, pass +} + +func envOr(key, def string) string { + if key != "" { + if v := os.Getenv(key); v != "" { + return v + } + } + return def +} diff --git a/internal/lab/targets_test.go b/internal/lab/targets_test.go new file mode 100644 index 0000000..0f066f8 --- /dev/null +++ b/internal/lab/targets_test.go @@ -0,0 +1,78 @@ +package lab + +import ( + "testing" +) + +const shipped = "../../lab/targets.yml" + +func TestShippedInventoryCoversBothPlatforms(t *testing.T) { + ts, err := Load(shipped) + if err != nil { + t.Fatalf("load: %v", err) + } + for _, kind := range []string{"linux", "windows"} { + tgt, ok := ts.ForPlatform(kind) + if !ok { + t.Errorf("no %s target declared — the cross-platform claim needs one", kind) + continue + } + if tgt.RulesDir == "" { + t.Errorf("%s target declares no rules_dir", kind) + } + } +} + +// Credentials must come from the environment. The Windows host used to be a +// literal in cmd/purpleloop/main.go. +func TestSSHConfigPrefersEnvironment(t *testing.T) { + ts, err := Load(shipped) + if err != nil { + t.Fatal(err) + } + win, ok := ts.ForPlatform("windows") + if !ok { + t.Fatal("no windows target") + } + if win.SSHHostEnv == "" || win.SSHPassEnv == "" { + t.Fatal("the windows target must read host and password from the environment") + } + + t.Setenv(win.SSHHostEnv, "10.0.0.5") + t.Setenv(win.SSHUserEnv, "lab-user") + t.Setenv(win.SSHPassEnv, "from-env") + + host, user, pass := win.SSHConfig() + if host != "10.0.0.5" || user != "lab-user" || pass != "from-env" { + t.Errorf("SSHConfig = (%s,%s,%s), want the environment values", host, user, pass) + } +} + +// A password must never come from a default — only from the environment. +func TestSSHPasswordIsNeverDefaulted(t *testing.T) { + ts, err := Load(shipped) + if err != nil { + t.Fatal(err) + } + win, _ := ts.ForPlatform("windows") + t.Setenv(win.SSHPassEnv, "") + if _, _, pass := win.SSHConfig(); pass != "" { + t.Errorf("password = %q with an empty env var; it must not fall back to a literal", pass) + } +} + +func TestForPlatformUnknownKind(t *testing.T) { + ts, err := Load(shipped) + if err != nil { + t.Fatal(err) + } + if _, ok := ts.ForPlatform("macos"); ok { + t.Error("an undeclared platform must not resolve to a target") + } +} + +func TestLoadRejectsMissingFile(t *testing.T) { + if _, err := Load("nope.yml"); err == nil { + t.Error("a missing inventory must error") + } +} diff --git a/lab/targets.yml b/lab/targets.yml new file mode 100644 index 0000000..3788bb0 --- /dev/null +++ b/lab/targets.yml @@ -0,0 +1,30 @@ +# Lab targets. +# +# Before this file, `run` hardcoded a single Linux target (victim01 / Docker / +# detections/linux) and the Windows executor was reachable ONLY from the +# standalone `canary` subcommand — which also hardcoded an IP. The v1.0 +# "cross-platform coverage" claim could not be exercised through `run` at all. +# +# A technique is dispatched to the target whose `kind` matches the `platform` +# its rule-map entry declares. The canary runs per platform actually in use, so +# a campaign spanning both hosts proves both pipelines before trusting either. + +targets: + - name: victim01 + kind: linux + executor: docker + container: purpleloop-victim + rules_dir: detections/linux + + - name: windows-vm + kind: windows + executor: ssh + rules_dir: detections/windows + # Host and credentials come from the environment, never from source: + # WINDOWS_SSH_HOST, WINDOWS_SSH_USER, WINDOWS_SSH_PASS + # (The IP was previously hardcoded at cmd/purpleloop/main.go:314.) + ssh_host_env: WINDOWS_SSH_HOST + ssh_user_env: WINDOWS_SSH_USER + ssh_pass_env: WINDOWS_SSH_PASS + default_host: 192.168.88.13 + default_user: windows-vm diff --git a/plans/cross-platform.yml b/plans/cross-platform.yml new file mode 100644 index 0000000..fbb6a04 --- /dev/null +++ b/plans/cross-platform.yml @@ -0,0 +1,13 @@ +# A campaign spanning both lab hosts in one run. Each technique is dispatched to +# the target matching the platform its rule-map entry declares, and the pipeline +# canary runs once per platform in use — a healthy Linux canary says nothing +# about whether Windows telemetry is flowing. +name: cross-platform +description: Linux discovery plus Windows command-shell execution in a single run +techniques: + - technique_id: T1082 + atomic_ids: [T1082-3] + - technique_id: T1033 + atomic_ids: [T1033-2] + - technique_id: T1059.003 + atomic_ids: [T1059.003-2] diff --git a/plans/windows-discovery.yml b/plans/windows-discovery.yml new file mode 100644 index 0000000..67a630c --- /dev/null +++ b/plans/windows-discovery.yml @@ -0,0 +1,12 @@ +# Windows campaign. Exercises the Windows executor, the Windows target from +# lab/targets.yml and detections/windows — the cross-platform path that was +# previously unreachable from `run` (SSHExecutor existed only inside the +# standalone `canary` subcommand). +# +# Requires a Windows victim reachable over SSH: +# export WINDOWS_SSH_HOST=... WINDOWS_SSH_USER=... WINDOWS_SSH_PASS=... +name: windows-discovery +description: Windows command-shell execution against the Sysmon-instrumented victim +techniques: + - technique_id: T1059.003 + atomic_ids: [T1059.003-2] From e65d33cdcd2cebcdcf5ed806e1957a674620c285 Mon Sep 17 00:00:00 2001 From: jayelbotvibe-web Date: Sat, 29 Aug 2026 15:09:22 +0800 Subject: [PATCH 09/13] fix(lab): make archive logging persistent and telemetry failure visible Archive logging did not survive a restart. The collector reads archives.json, which the manager only writes when is yes; the image ships it as no. It had been enabled by hand INSIDE the running container, which cannot work: the manager bind-mounts /wazuh-config-mount/etc/ossec.conf from the host and its entrypoint copies that over /var/ossec/etc/ossec.conf on every start, so the edit is wiped by the very restart needed to apply it. The stack then comes up healthy, the agent enrols, and every technique reports NO_TELEMETRY -- which reads as a detection problem and is not one. scripts/enable-archives.sh (make archives) patches the host-side source of the config mount and then verifies the setting is live inside the container. The victim entrypoint hid a total telemetry failure: it ran service auditd start || auditd || true so every failure was swallowed. It now probes the kernel audit subsystem, loads execve rules when usable, and otherwise says loudly that this victim has no process-creation telemetry and that runs will be INCONCLUSIVE as a result. Co-Authored-By: Claude Opus 5 --- lab/docker-compose.override.yml | 6 +++ lab/victim/Dockerfile | 15 +++++++- lab/victim/entrypoint.sh | 51 +++++++++++++++++++++---- scripts/enable-archives.sh | 66 +++++++++++++++++++++++++++++++++ 4 files changed, 128 insertions(+), 10 deletions(-) create mode 100755 scripts/enable-archives.sh diff --git a/lab/docker-compose.override.yml b/lab/docker-compose.override.yml index 0e09799..0acccb5 100644 --- a/lab/docker-compose.override.yml +++ b/lab/docker-compose.override.yml @@ -13,6 +13,12 @@ services: container_name: purpleloop-victim hostname: victim01 restart: unless-stopped + # Deliberately unprivileged. Running Sysmon for Linux here was tried and + # measured (docs/PITFALLS.md): it needs privileged + systemd, and its eBPF + # then observes the whole host with no container identity in the event, so + # the telemetry cannot be attributed to this victim safely. A privileged + # container in a security lab is not worth paying for a source that would + # produce false attributions. cap_add: - AUDIT_CONTROL - AUDIT_READ diff --git a/lab/victim/Dockerfile b/lab/victim/Dockerfile index 3ac1bc7..94ec34d 100644 --- a/lab/victim/Dockerfile +++ b/lab/victim/Dockerfile @@ -1,5 +1,16 @@ -# Ubuntu victim: runs the Wazuh agent + auditd so attacks generate telemetry -# that ships to the manager. Kept intentionally small. +# Ubuntu victim: the Wazuh agent plus whatever process-creation telemetry this +# host can actually provide. +# +# NOT privileged, and NOT running Sysmon for Linux. That was tried and measured +# — see docs/PITFALLS.md, "Why the Linux victim is not a container with Sysmon". +# Sysmon works, but its eBPF is kernel-global: inside a container it observes +# every process on the host, and the event carries no container identity. Scoping +# it after the fact by resolving host PIDs is defeated by PID reuse, which let +# host binaries through; verifying process identity closes that leak but then +# drops the short-lived processes the atomics actually spawn. Either way the +# telemetry is wrong in a direction this project refuses to ship. +# +# The Linux equivalent of the Windows victim is a VM, not a container. FROM ubuntu:22.04 ARG WAZUH_VERSION=4.9.2 diff --git a/lab/victim/entrypoint.sh b/lab/victim/entrypoint.sh index d4bedae..f232827 100755 --- a/lab/victim/entrypoint.sh +++ b/lab/victim/entrypoint.sh @@ -1,16 +1,51 @@ #!/usr/bin/env bash -# Start auditd and the Wazuh agent (auto-enrolling against $WAZUH_MANAGER), -# then hold the container open. -set -e +# Start the telemetry sources and the Wazuh agent, then hold the container open. +# +# Telemetry status is reported LOUDLY at start-up. This used to run +# service auditd start || auditd || true +# which swallowed every failure, so the container came up looking healthy with +# no process-creation source at all. The pipeline then reported NO_TELEMETRY for +# every technique, which reads as a detection problem and is not one — it is a +# victim that never had the telemetry to begin with. +set -u : "${WAZUH_MANAGER:=wazuh.manager}" : "${WAZUH_AGENT_NAME:=victim01}" -# point the agent at the manager -sed -i "s|
.*
|
${WAZUH_MANAGER}
|g" /var/ossec/etc/ossec.conf || true +banner() { echo "[victim] $*"; } -service auditd start || auditd || true -/var/ossec/bin/wazuh-control start || true +# Point the agent at the manager. +sed -i "s|
.*
|
${WAZUH_MANAGER}
|g" \ + /var/ossec/etc/ossec.conf || banner "WARNING: could not set manager address" -# tail agent log so `docker logs` is useful; keep PID 1 alive +# --- process-creation telemetry --------------------------------------------- +# auditd needs the kernel audit subsystem. It is unavailable in many +# environments (kernel built without it, audit locked, or the netlink interface +# refused) and there is no point pretending otherwise: say so once, clearly, +# rather than failing silently. +PROCESS_TELEMETRY="none" +if auditctl -s >/dev/null 2>&1; then + if service auditd start >/dev/null 2>&1 || auditd >/dev/null 2>&1; then + # Capture execve so events carry Image and CommandLine, which is what the + # evaluator's process_creation rules require. + auditctl -a always,exit -F arch=b64 -S execve -k purpleloop_exec >/dev/null 2>&1 + auditctl -a always,exit -F arch=b32 -S execve -k purpleloop_exec >/dev/null 2>&1 + PROCESS_TELEMETRY="auditd" + banner "process-creation telemetry: auditd (execve rules loaded)" + else + banner "WARNING: kernel audit is available but auditd failed to start" + fi +else + banner "WARNING: kernel audit subsystem unavailable — auditctl cannot talk to it." + banner "WARNING: this victim has NO process-creation telemetry." + banner "WARNING: the pipeline canary cannot fire, so runs will be INCONCLUSIVE." + banner "WARNING: this is a lab capability gap, NOT a detection gap." + banner "WARNING: see docs/PITFALLS.md for what was measured and why a VM is the fix." +fi + +/var/ossec/bin/wazuh-control start || banner "WARNING: wazuh-agent failed to start" + +banner "agent=${WAZUH_AGENT_NAME} manager=${WAZUH_MANAGER} process_telemetry=${PROCESS_TELEMETRY}" + +# Tail the agent log so `docker logs` is useful; keep PID 1 alive. touch /var/ossec/logs/ossec.log exec tail -F /var/ossec/logs/ossec.log diff --git a/scripts/enable-archives.sh b/scripts/enable-archives.sh new file mode 100755 index 0000000..0f2bace --- /dev/null +++ b/scripts/enable-archives.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Enable Wazuh archive logging, persistently. +# +# The collector reads /var/ossec/logs/archives/archives.json — every event, not +# just rule-triggered alerts. That file is only written when is +# yes, and the Wazuh manager image ships it as no. +# +# Patching the RUNNING container does not work. The manager bind-mounts +# /wazuh-config-mount/etc/ossec.conf from the host and its entrypoint copies +# that file over /var/ossec/etc/ossec.conf on every start, so an in-container +# edit is wiped by the very restart needed to apply it. This was previously done +# by hand inside the container, which is why archive logging silently vanished +# whenever the stack was restarted: the lab came up healthy, the agent enrolled, +# and every technique reported NO_TELEMETRY because the archive the collector +# reads was never written. +# +# So: patch the host-side source of the config mount, then restart. +set -euo pipefail + +MANAGER="${1:-single-node-wazuh.manager-1}" + +if ! docker ps -a --format '{{.Names}}' | grep -qx "$MANAGER"; then + echo "manager container '$MANAGER' not found" >&2 + exit 1 +fi + +# Resolve the host path backing the config mount, rather than assuming a layout. +SRC=$(docker inspect "$MANAGER" \ + --format '{{range .Mounts}}{{if eq .Destination "/wazuh-config-mount/etc/ossec.conf"}}{{.Source}}{{end}}{{end}}') + +if [ -z "$SRC" ]; then + echo "no /wazuh-config-mount/etc/ossec.conf mount on $MANAGER" >&2 + echo "this lab layout is unexpected — enable in the manager config by hand" >&2 + exit 1 +fi +if [ ! -w "$SRC" ]; then + echo "config source is not writable: $SRC" >&2 + exit 1 +fi + +if grep -q 'yes' "$SRC"; then + echo "archive logging already enabled in $SRC" +else + if ! grep -q '' "$SRC"; then + echo "no element in $SRC — cannot patch safely" >&2 + exit 1 + fi + cp "$SRC" "$SRC.bak.$(date +%Y%m%d%H%M%S)" + sed -i 's|no|yes|' "$SRC" + echo "enabled in $SRC" +fi + +docker restart "$MANAGER" >/dev/null +echo "manager restarted" + +# Verify it actually took effect inside the container — the point of this script +# is that the obvious approach silently does not. +for _ in $(seq 1 20); do + if docker exec "$MANAGER" sh -c "grep -q 'yes' /var/ossec/etc/ossec.conf" 2>/dev/null; then + echo "verified: archive logging active in the manager" + exit 0 + fi + sleep 3 +done +echo "FAILED: is not yes inside the container after restart" >&2 +exit 1 From 3cdf9802c268f32ae8301a6549e9ea1da2883bd2 Mon Sep 17 00:00:00 2001 From: jayelbotvibe-web Date: Sat, 29 Aug 2026 15:09:40 +0800 Subject: [PATCH 10/13] ci: run cmd tests, sigma check, gofmt, replay and the precision gate CI ran only ./internal/... , skipping every guard in ./cmd/... -- atomic resolution, attribution, replay and diff. It now runs both. Adds four gates: - gofmt, which go vet does not cover - sigma check (pySigma), which enforces spec rules the hand-rolled Go parser tolerates; it caught a non-UUID rule identifier. The two excluded validators are SigmaHQ house conventions for their own repository, not the specification. - replay of testdata/sample-run: the real matcher over real event shapes - purpleloop precision: the false-positive gate `make atomics` is part of the build, not an optional step: without the pinned ART tree the engine has no command to run. Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 50 +++++++++++++- internal/evaluator/sysmon_linux_test.go | 91 +++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 internal/evaluator/sysmon_linux_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7e1656..1956ab6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,22 +11,67 @@ jobs: - uses: actions/setup-go@v5 with: go-version: "1.22" + + # The engine resolves every technique's command from the pinned Atomic Red + # Team tree. Without it the atomic, replay and precision tests cannot run, + # so the fetch is part of the build, not an optional step. + - name: vendor atomic red team (pinned) + run: make atomics + - name: build run: go build ./cmd/... ./internal/... - name: vet run: go vet ./cmd/... ./internal/... + - name: gofmt + run: | + unformatted=$(gofmt -l cmd internal) + if [ -n "$unformatted" ]; then + echo "not gofmt-clean:"; echo "$unformatted"; exit 1 + fi + + # ./cmd/... is included deliberately: the atomic-resolution, attribution, + # replay and diff guards live there, and running only ./internal/... would + # skip every one of them. - name: test - run: go test ./internal/... -count=1 + run: go test ./cmd/... ./internal/... -count=1 + + # Runs the real matcher over captured-format telemetry with no lab. This + # is the check that fixtures alone cannot provide. + - name: replay sample dataset + run: go run ./cmd/purpleloop replay testdata/sample-run > /dev/null + + # Coverage without a false-positive rate is half a metric. Exits non-zero + # if any rule fires on the benign administrator workload. + - name: precision (false-positive gate) + run: go run ./cmd/purpleloop precision + detections: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: sigma lint + - name: sigma lint (repo linter) run: | pip install pyyaml python3 scripts/lint-sigma.py + + # pySigma enforces spec rules the hand-rolled Go parser tolerates. It + # caught a non-UUID rule identifier the Go parser accepted silently. + # + # The two excluded validators are SigmaHQ house conventions for their own + # repository, not parts of the Sigma specification, and both conflict with + # deliberate choices here: rules are named for the technique they detect + # (T1082.yml), and the pipeline canary is a positive control rather than an + # ATT&CK detection, so it carries a purpleloop.* tag. + - name: sigma check (pySigma spec validation) + run: | + pip install sigma-cli + sigma check --fail-on-issues \ + -x filename_length -x namespace_tag \ + detections/linux detections/windows detections/canary + - name: fixture regression run: python3 scripts/regression-test.py + secrets: runs-on: ubuntu-latest steps: @@ -35,4 +80,3 @@ jobs: fetch-depth: 0 - name: gitleaks uses: gitleaks/gitleaks-action@v2 - # Phase 3 adds a `detections` job: sigma lint + regression vs fixtures. diff --git a/internal/evaluator/sysmon_linux_test.go b/internal/evaluator/sysmon_linux_test.go new file mode 100644 index 0000000..bc485e5 --- /dev/null +++ b/internal/evaluator/sysmon_linux_test.go @@ -0,0 +1,91 @@ +package evaluator + +import ( + "encoding/json" + "testing" + + "github.com/jayelbotvibe-web/purple-loop/internal/model" +) + +func normalizeRaw(t *testing.T, raw string) map[string]string { + t.Helper() + return Normalizer{}.Normalize(json.RawMessage(raw)) +} + +// Sysmon for Linux EventID 1 is genuine process creation and must satisfy a +// process_creation rule. +func TestLinuxSysmonProcessCreateIsHighFidelity(t *testing.T) { + got := normalizeRaw(t, `{"data":{"sysmon":{ + "eventID":"1","image":"/usr/bin/uname","commandLine":"uname -a", + "parentImage":"/bin/bash","user":"root"}}}`) + + if got["Image"] != "/usr/bin/uname" { + t.Errorf("Image = %q", got["Image"]) + } + if got["CommandLine"] != "uname -a" { + t.Errorf("CommandLine = %q", got["CommandLine"]) + } + if got["ParentImage"] != "/bin/bash" { + t.Errorf("ParentImage = %q", got["ParentImage"]) + } + if got[FidelityKey] != FidelityProcess { + t.Errorf("fidelity = %q, want %q", got[FidelityKey], FidelityProcess) + } +} + +// The eventID may arrive as a JSON number rather than a string. +func TestLinuxSysmonNumericEventID(t *testing.T) { + got := normalizeRaw(t, `{"data":{"sysmon":{ + "eventID":1,"image":"/usr/bin/id","commandLine":"id"}}}`) + if got[FidelityKey] != FidelityProcess { + t.Errorf("numeric eventID not recognised: fidelity = %q", got[FidelityKey]) + } + if got["EventID"] != "1" { + t.Errorf("EventID = %q, want \"1\"", got["EventID"]) + } +} + +// A non-creation Sysmon event carries a process name without being proof a +// process was created — the same trap that produced a false DETECTED off a +// Windows 4798 enumeration event. +func TestLinuxSysmonNonCreationIsLowFidelity(t *testing.T) { + got := normalizeRaw(t, `{"data":{"sysmon":{ + "eventID":"3","image":"/usr/bin/curl","commandLine":"curl http://x"}}}`) + + if got[FidelityKey] == FidelityProcess { + t.Error("a Sysmon network event must not be tagged as process creation") + } + if got[FidelityKey] != FidelityLog { + t.Errorf("fidelity = %q, want %q", got[FidelityKey], FidelityLog) + } +} + +// End to end: a Linux-Sysmon event must be able to satisfy a real shipped rule, +// and a non-creation event with the same fields must not. +func TestLinuxSysmonSatisfiesShippedRule(t *testing.T) { + e := RuleMatcherEvaluator{RulesDir: "../../detections/linux"} + rule := model.SigmaRule{Path: "../../detections/linux/T1082.yml", Title: "T1082"} + + creation := []model.Event{{Raw: json.RawMessage(`{"data":{"sysmon":{ + "eventID":"1","image":"/usr/bin/uname","commandLine":"uname -a","user":"root"}}}`)}} + verdict, evidence, err := e.Evaluate(rule, creation) + if err != nil { + t.Fatal(err) + } + if verdict != model.Detected { + t.Errorf("verdict = %s, want DETECTED", verdict) + } + if len(evidence) == 0 { + t.Error("a detection must carry evidence") + } + + nonCreation := []model.Event{{Raw: json.RawMessage(`{"data":{"sysmon":{ + "eventID":"3","image":"/usr/bin/uname","commandLine":"uname -a","user":"root"}}}`)}} + verdict, _, err = e.Evaluate(rule, nonCreation) + if err != nil { + t.Fatal(err) + } + if verdict == model.Detected { + t.Error("a non-creation event must never satisfy a process_creation rule") + } +} From 14f267effad9a2f04325393e189886f8fb829043 Mon Sep 17 00:00:00 2001 From: jayelbotvibe-web Date: Sat, 29 Aug 2026 15:09:40 +0800 Subject: [PATCH 11/13] docs: describe what the engine actually does, and what the lab cannot README documented behaviour the code did not have. It now describes strict atomic resolution, the three verdicts that mean "this never tested a detection" and are excluded from coverage, the override mechanism, and the new precision, replay and diff commands. RESULTS.md held a single JSON blob claiming T1059.004 -- a Linux Bash technique -- was DETECTED, with evidence being net.exe on Windows spawned by the Wazuh agent itself. Three things wrong with it, all consequences of the hardcoded id; whoami: wrong platform, wrong rule credited, and the "attack" was the monitoring agent's own activity. Replaced with a real run, and the old blob kept with an explanation, because a project that claims evidence should show what its own bad evidence looked like. PITFALLS records two measured environmental limits so they are not re-attempted blind: auditd cannot work on a host whose kernel exposes no audit sysctls (reproduced under --privileged --network host), and Sysmon for Linux runs only under systemd. Running Sysmon in a privileged container was built and rejected: its eBPF is kernel-global and the event carries no container identity, so scoping it either leaks host processes through PID reuse or drops the atomics' own short-lived ones. Both manufacture false verdicts. The fix is a Linux VM, as the Windows victim already is. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 134 +++++++++++++++++++++++++++++++++++++++++++++++ README.md | 98 +++++++++++++++++++++++++++++++--- RESULTS.md | 65 ++++++++++++++++++++++- docs/PITFALLS.md | 110 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 398 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7da22bb..2403265 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,140 @@ All notable changes to this project follow Keep a Changelog and Semantic Versioning. ## [Unreleased] +### Fixed (2026-08 review — the engine now executes the atomics it claims to) +- **Every technique ran the same hardcoded `id; whoami`.** `atomic_ids` was parsed from plan + YAML, carried through emulation stages and the arbiter feed and written into the proof + chain — but never resolved to a command. A ten-technique campaign ran one command ten times + and labelled the results T1082, T1033, T1518… so every verdict but `T1059.004`'s was about a + command nobody asked for. The README's claim that Execute "runs Atomic Red Team tests" was + not true of the code. New `internal/atomic` package resolves each `atomic_id` against the + pinned ART tree; resolution is strict (`override → vendored ART → error`) and there is no + default command. +- **Five of ten techniques in `plans/discovery.yml` named Windows atomics** while the campaign + targets a Linux victim (`T1082-1`, `T1033-1`, `T1007-1`, `T1016-1`, `T1049-1` are all + `command_prompt` tests). Invisible while everything ran `id; whoami`. Plans and emulation + chains now name the correct Linux tests, and a platform mismatch is a loud `ERROR`. +- **Containment breach in the pinned atomics.** `T1059.004-1` writes a script that pings + `8.8.8.8` — traffic leaving the isolated lab network, against the project's own rule. Its + `host` input is pinned to loopback via an override that keeps the upstream command. +- **`ERROR` was counted as `missed`** in the HTML headline, and `SKIPPED_PREREQ`/`ERROR` were + inside the dashboard's coverage denominator. A harness failure is not a detection gap; + coverage is now measured only over techniques that actually exercised a detection. +- **Atomic Red Team was recorded, not pinned.** `lab-fetch.sh` cloned HEAD then overwrote the + pin file with whatever it got, so a re-clone could silently change every command the engine + runs. `scripts/fetch-atomics.sh` checks out the recorded commit and verifies HEAD matches. +- Dashboard no longer hardcodes `campaign: "discovery"` and `build: "v1.3.0"`; both come from + the run. `gap.why` is populated from the verdict's note instead of always being empty. +- `make help` printed "Makefile" in place of every target name (`grep` needed `-h`). + +### Fixed (2026-08 — the detections had the same defect as the engine) +- **Nine of the ten Linux Sigma rules were the identical rule.** Every one matched + `Image endswith /id or /whoami`, differing only in title, because they were written against + an engine that executed `id; whoami` for every technique. All nine shared one byte-identical + positive fixture, so the "10 rules, 20 fixtures, CI enforced" detection-as-code claim was + testing one rule nine times against one event. Each technique now has a detection for what + its own atomic actually does, with distinct fixtures whose negatives include a *neighbouring* + technique's activity — so a rule that is secretly a copy fails its own negative test. +- **Two false positives found by the new precision gate and fixed**: `T1087.001` fired on + `getent hosts localhost` (a DNS lookup, not account enumeration) and `T1135` fired on a bare + `mount`. Both rules keyed on a tool name rather than what the tool was being asked to do. +- **`win_proc_create.yml` had a non-UUID identifier** (`win-proc-create-001`), a Sigma spec + violation the hand-rolled Go parser accepted silently. Found by adding `sigma check`. +- **The Windows path was unreachable from `run`.** `runTasks` hardcoded a Linux target, rules + directory and executor; `SSHExecutor` and the Windows rule existed only inside the standalone + `canary` subcommand, which also hardcoded the IP `192.168.88.13`. Targets are now declared in + `lab/targets.yml`, techniques are dispatched by the platform their rule-map entry names, and + the canary runs once per platform in use — a healthy Linux canary says nothing about whether + Windows telemetry is flowing. +- **`history.json` stored only a coverage percentage**, so no run could be compared with + another. It now carries per-technique verdicts, which is what `purpleloop diff` needs. + +### Fixed (2026-08 — lab reproducibility) +- **Archive logging did not survive a restart.** The collector reads + `archives.json`, which the manager only writes when `` is `yes`; the image ships + it as `no`. It had been enabled by hand *inside the running container*, which cannot work: + the manager bind-mounts `/wazuh-config-mount/etc/ossec.conf` from the host and its entrypoint + copies that over `/var/ossec/etc/ossec.conf` on every start, so the edit is wiped by the very + restart needed to apply it. The stack then comes up healthy, the agent enrolls, and every + technique reports `NO_TELEMETRY` — which reads as a detection problem and is not one. + `scripts/enable-archives.sh` (`make archives`) patches the host-side source of the config + mount and then *verifies* the setting is live inside the container. + +### Fixed (2026-08 — victim telemetry honesty) +- **The victim entrypoint hid a total telemetry failure.** It ran + `service auditd start || auditd || true`, so every failure was swallowed: the container came + up looking healthy with no process-creation source at all, and the pipeline then reported + `NO_TELEMETRY` for every technique — which reads as a detection problem and is not one. The + entrypoint now probes the kernel audit subsystem, loads `execve` rules when it is usable, and + otherwise says loudly that this victim has no process-creation telemetry and that runs will + be `INCONCLUSIVE` as a result. +- **`--settle-timeout` / `--settle-poll` are configurable.** The 60s default is below the + victim's 360s `` collection interval, so even command-output telemetry could not + land inside a technique's window. Documented on the flag itself. +- Verified and documented in `docs/PITFALLS.md` what actually blocks Linux process-creation + telemetry: auditd is impossible on a host whose kernel exposes no audit sysctls (reproduced + with `--privileged --network host`), and Sysmon for Linux only runs under systemd (its + `-service` mode still shells out to `systemctl`). Closing the gap needs a systemd-based + victim — a deliberate change to the lab's security posture, not a config tweak. + +### Added (2026-08 — Linux-Sysmon readiness) +- Normalizer handles a `data.sysmon` path: Linux-Sysmon `EventID 1` is treated as genuine + process creation, any other Sysmon event ID is low fidelity and can never satisfy a + `process_creation` rule. Events are deliberately not routed through `data.win.*`, which is + Wazuh's Windows eventchannel convention. Tested in `internal/evaluator/sysmon_linux_test.go`, + including that a non-creation event with identical fields does not match. +- Documented in `docs/PITFALLS.md` why the Linux victim is a container without process-creation + telemetry rather than a container running Sysmon: Sysmon for Linux works, but its eBPF is + kernel-global and the event carries no container identity, so scoping it either leaks host + processes (PID reuse) or drops the atomics' own short-lived ones. Both manufacture false + verdicts. The fix is a Linux VM, as the Windows victim already is. + +### Added (2026-08) +- `purpleloop precision` and `internal/precision`: measures the false-positive rate against a + benign administrator workload (`emulation/benign-baseline.yml`, 26 commands across 7 + categories) including adjacent cases — benign uses of the binaries the rules watch. Exits + non-zero on any false positive, so it gates CI. +- `purpleloop replay` and `internal/capture`: every real run writes + `reports/runs//events.jsonl`; replay re-runs the real evaluator over it with no lab. CI + replays `testdata/sample-run` on every push, so rule edits are checked against telemetry + rather than only against fixtures written to match them. The capture writer refuses synthetic + runs, whose "telemetry" is fabricated. +- `purpleloop diff
[--fail-on-regression]`: names the technique that regressed + (`T1082 DETECTED -> MISSED`) instead of only moving a percentage. Verdicts that mean "never + exercised" share a rank, so ERROR -> SKIPPED_PREREQ is a change, not a regression. +- `collector.Settler`: polls until an expected rule matches or ingestion settles, replacing the + fixed `time.Sleep(10s)` per technique and its 10-minute query window. +- Attribution labels on every verdict (`window_and_host_scoped`, `window_scoped`, + `window_overlap`, `unscoped`) and `detect_latency_ms`. Overlap is computed campaign-wide once + every window is known. This changes no verdict; it states the quality of the evidence behind + one. +- `mappings/attack_rule_map.json` with `expected_rules[]` — a technique is DETECTED when ANY + expected rule matches. Replaces the technique->rule map hardcoded in `cmd/purpleloop`. +- `untested_rules` accounting: every rule is mapped to a technique or declared untested with a + reason, enforced by a CI lint, and reported per run. +- `collection_gaps` in the dashboard output, naming the technique and rules behind each + `NO_TELEMETRY`, for handoff to the sibling detection-decay repo. +- `lab/targets.yml` + `internal/lab`, `plans/windows-discovery.yml`, `plans/cross-platform.yml`. +- `make archives` / `scripts/enable-archives.sh`. +- `sigma check` (pySigma) in CI, and a `gofmt` gate. CI now runs `./cmd/...` too — the + atomic-resolution, attribution, replay and diff guards live there and were being skipped. +- Regression guards for both halves of the defect: no two techniques in a campaign may resolve + to the same command, and no two rules may share detection logic. + +### Added (2026-08, atomics) +- `internal/atomic`: ART loader with input-argument interpolation, prerequisite parsing, + platform support checks and lab overrides. +- `SKIPPED_PREREQ` verdict — an atomic whose own prerequisites are unmet never ran, and is no + longer reported as a detection miss. +- `mappings/atomic-overrides.yml`: two override kinds — pinned inputs (keeps the upstream + command, used for lab containment) and replaced commands (only where no usable upstream test + exists). Every entry requires a `reason`, which the loader enforces and the report shows. +- Proof chains carry atomic provenance (`name`, `guid`, `source`, `override_reason`) and a + `note` explaining any non-coverage verdict. Campaign results record the ART commit. +- `make atomics` target and `scripts/fetch-atomics.sh`. +- Regression guard: a test fails if any two techniques in the shipped campaign resolve to the + same command — the defect above, made unshippable. + ### Fixed (2026-07 code review — trust contracts enforced in code, not just docs) - **Canary gating now runs in the pipeline.** Every `run` path executes the positive control against the same Linux/Docker executor+collector the campaign uses; if it does diff --git a/README.md b/README.md index 4435c6d..fb99f7b 100644 --- a/README.md +++ b/README.md @@ -27,10 +27,13 @@ rules, and produces an evidence-backed coverage report. No guessing, no presence *[More detailed view → Interactive Architecture Map](https://jayelbotvibe-web.github.io/purple-loop/)* 1. **Feed** loads techniques from a plan, arbiter export, or emulation script -2. **Execute** runs Atomic Red Team tests on lab victims (Docker Linux + VMware Windows) -3. **Collect** queries Wazuh archives for raw telemetry in the execution window -4. **Evaluate** normalizes events and matches them against Sigma rules using a native Go parser -5. **Report** produces JSON, HTML coverage grid, or ATT&CK Navigator layer export +2. **Resolve** turns each `atomic_id` into a real command from the pinned Atomic Red Team + tree — strictly. An unresolvable ID, or one whose atomic does not support the target + platform, is an `ERROR` naming the problem, never a substituted default command +3. **Execute** runs that atomic on the lab victim (Docker Linux + VMware Windows) +4. **Collect** queries Wazuh archives for raw telemetry in the execution window +5. **Evaluate** normalizes events and matches them against Sigma rules using a native Go parser +6. **Report** produces JSON, HTML coverage grid, or ATT&CK Navigator layer export ## Pipeline canary (positive control) @@ -68,6 +71,38 @@ This removes the ambiguity that caused v1.0's false 100% coverage — you can no } ``` +### Precision, not just coverage + +```bash +purpleloop precision +# Precision: 100% (26 benign commands, 10 rules) +``` + +Coverage answers *does it fire when it should*. Precision answers *does it stay quiet when it +should* — a rule matching everything scores perfect coverage and is worthless. Every detection +is measured against a benign administrator workload including **adjacent cases**: benign uses +of the very binaries the rules watch. This found two real false positives on introduction +(`T1087.001` fired on `getent hosts`, a DNS lookup; `T1135` fired on a bare `mount`), and both +rules were narrowed. It exits non-zero, so it gates CI. + +### Replay a captured run — no lab required + +```bash +purpleloop replay testdata/sample-run +``` + +Every real run writes `reports/runs//events.jsonl`. `replay` re-runs the **real** evaluator +over that telemetry with no Docker and no Wazuh, so a rule edit can be checked against every +past run: *would this have broken a detection that already worked?* CI runs it on every push. + +### Compare two runs + +```bash +purpleloop diff reports/runs/ reports/runs/ --fail-on-regression +# REGRESSED (1) +# T1082 DETECTED -> MISSED +``` + Run: `purpleloop serve` for a **local web dashboard** with all past runs at `http://127.0.0.1:8787`. [Live dashboard on GitHub Pages](https://jayelbotvibe-web.github.io/purple-loop/dashboard.html). @@ -76,9 +111,19 @@ Run: `purpleloop serve` for a **local web dashboard** with all past runs at `htt ```bash git clone https://github.com/jayelbotvibe-web/purple-loop.git && cd purple-loop +make atomics # vendor Atomic Red Team at the pinned commit — required bash scripts/startup.sh ``` +Bringing the lab up by hand needs one extra step, because the collector reads the manager's +full event archive rather than only rule-triggered alerts: + +```bash +make lab-up +make archives # enable ; without it EVERY technique reports NO_TELEMETRY +make verify # health gates + pipeline canary +``` + Full guide: [STARTUP.md](STARTUP.md) — covers lab, Windows VM, arbiter connection, campaigns, troubleshooting. ```bash @@ -106,6 +151,33 @@ the lab and validating whether the corresponding behavioral (TTP) detection fire [`threat-intel-arbiter → arbiter-live.json → purple-loop run --arbiter`] +## Atomic resolution and lab overrides + +Every technique's command comes from the Atomic Red Team commit recorded in +`mappings/atomic-red-team.commit`, checked out and **verified** by `make atomics`. Resolution +is strict: `override → vendored ART → error`. There is deliberately no default command — a +fabricated input produces a fabricated verdict, so an atomic that cannot be resolved fails +loudly instead. + +Three verdicts exist for "this never tested a detection", and none of them counts toward +coverage: + +| Verdict | Means | +|---|---| +| `ERROR` | The atomic could not be resolved, or does not support the target platform | +| `SKIPPED_PREREQ` | The atomic's own `prereq_command` failed — it never ran | +| `NO_TELEMETRY` | It ran, but the events the rule needs never arrived | + +`mappings/atomic-overrides.yml` adjusts atomics that cannot run truthfully in this lab. Every +entry requires a `reason`, which appears in the report, so a substitution is never mistaken for +the upstream atomic. Two kinds, narrower preferred: + +- **Pinned inputs** keep the upstream command and override only its declared inputs. Used for + containment — upstream `T1059.004-1` pings `8.8.8.8`, which leaves the isolated lab network, + so `host` is pinned to loopback. +- **Replaced commands** substitute the atomic wholesale, only where no usable upstream test + exists. `T1518` (Software Discovery) has six atomics and not one supports Linux. + ## Design decisions / scope **Behavioral (TTP) validation, by design.** Purple Loop validates ATT&CK technique @@ -126,10 +198,15 @@ orchestrator. Lab runs isolated on `purpleloop-lab` Docker network. ## Detection-as-code -Every Sigma rule has **positive + negative fixtures**. CI enforces: -- All positives must match -- All negatives must reject -- A broken rule fails `go test ./internal/...` and turns CI red +Every Sigma rule has **positive + negative fixtures**, and each rule detects what its own +technique's atomic actually does. CI enforces: +- All positives must match, all negatives must reject +- **No two rules may share detection logic** — a campaign whose detections are one rule under + many names reports coverage it does not have +- Every rule is mapped to a technique or explicitly declared untested **with a reason** +- Every rule stays silent on the benign workload (`purpleloop precision`) +- `sigma check` (pySigma) validates against the Sigma spec, not just this repo's parser +- A broken rule fails `go test ./cmd/... ./internal/...` and turns CI red ```bash go test ./internal/evaluator/ -v -run Regression @@ -155,7 +232,12 @@ process-creation rule — so a log line that merely mentions a binary cannot pro - **Lab-contained only.** Never run against production or targets outside `purpleloop-lab`. - **Linux Sysmon gap.** The Linux victim has command-output telemetry but not Sysmon process-creation events. This is a known gap; Windows Sysmon detection is confirmed working. +- **Collection gaps hand off to [detection-decay](https://github.com/jayelbotvibe-web/detection-decay).** A `NO_TELEMETRY` verdict is the symptom — the rule never saw the events it needs. detection-decay models the cause (source death, field drift). Every run emits `collection_gaps` naming the technique and the rules whose fields went missing. +- **One atomic per technique.** A technique runs the first atomic its plan or the rule map names. Exercising several per technique is not yet supported. - **SSVC mapping.** The arbiter uses pre-SSVC labels (Schedule/Monitor/Track) mapped to SSVC v2.1 equivalents. +- **Coverage denominator.** Coverage is measured over techniques that actually exercised a + detection. `ERROR`, `SKIPPED_PREREQ`, `NO_TELEMETRY` and `INCONCLUSIVE` are reported + separately rather than counted as gaps — a harness failure is not a detection gap. ## License diff --git a/RESULTS.md b/RESULTS.md index 24c982f..bc9285a 100644 --- a/RESULTS.md +++ b/RESULTS.md @@ -1 +1,64 @@ -"chains":[{"technique_id":"T1059.004","verdict":"DETECTED","rule_matched":"detections/windows/win_proc_create.yml","events_collected":73,"executed_at":"2026-07-04T18:09:20Z","evidence":[{"id":"archives-7594","matched":true,"fields":{"Image":"C:\\Windows\\SysWOW64\\net.exe","CommandLine":"net.exe accounts","ParentImage":"C:\\Program Files (x86)\\ossec-agent\\wazuh-agent.exe","User":"NT AUTHORITY\\SYSTEM"}}]}]} \ No newline at end of file +# Results + +## Live run — 2026-08-29, Linux victim + +``` +$ purpleloop run --technique T1082 \ + --victim-container purpleloop-victim \ + --manager-container single-node-wazuh.manager-1 + +INCONCLUSIVE: canary not detected on linux — coverage will not be reported as valid +``` + +| | | +|---|---| +| Atomic resolved | `T1082-3` — *List OS Information* (`uname -a`, `/etc/os-release`, …) | +| Source | Atomic Red Team @ `9415359` | +| Canary | **not detected** on linux | +| Run verdict | `INCONCLUSIVE` — coverage suppressed | +| Technique verdict | `INCONCLUSIVE` | +| Events collected | 0 (`no telemetry arrived within 1m3s of execution`) | + +**This is the correct outcome, and it is the point of the canary.** The Linux victim ships +command-output telemetry (`` command monitoring on a 360-second cycle) but no +process-creation source. The canary rule requires `category: process_creation`, so the positive +control cannot fire, so the pipeline is unproven — and an unproven pipeline may not report +coverage. Every technique in the run is `INCONCLUSIVE` rather than `MISSED`, because a missed +detection is a claim about a rule and nothing here tested a rule. + +The three trust contracts were verified against this run: + +- the per-run artifact was written to `reports/runs//coverage.json` for inspection; +- it was **not** appended to `reports/history.json`; +- it was **not** published to `docs/data/coverage.json`. + +**To close this gap** the Linux victim needs Sysmon-for-Linux (or auditd `execve` forwarding +that carries `Image`/`CommandLine`). Windows Sysmon Event ID 1 is confirmed working and is the +platform where detection has been demonstrated. + +--- + +## What the previous version of this file claimed + +The file previously held a single JSON blob reporting: + +```json +{"technique_id":"T1059.004","verdict":"DETECTED", + "rule_matched":"detections/windows/win_proc_create.yml", + "evidence":[{"fields":{"Image":"C:\\Windows\\SysWOW64\\net.exe", + "ParentImage":"C:\\Program Files (x86)\\ossec-agent\\wazuh-agent.exe"}}]} +``` + +Three things are wrong with it, all consequences of the engine defect fixed in the 2026-08 +changelog entries: + +1. **`T1059.004` is Bash on Linux.** The evidence is a Windows process. +2. **The rule credited is the Windows rule**, not the Linux rule for that technique. +3. **The "attack" is the monitoring agent itself** — `net.exe` spawned by `wazuh-agent.exe`. + Nothing the atomic did produced this event. + +The old engine executed `id; whoami` for every technique, collected whatever appeared in a +ten-minute window, and credited an unrelated match. The evidence-fidelity gate, the corrected +window, the platform check and the atomic resolver each independently prevent this result now. +It is kept here because a project that claims evidence should show what its own bad evidence +looked like. diff --git a/docs/PITFALLS.md b/docs/PITFALLS.md index 99d6ec3..8a3e049 100644 --- a/docs/PITFALLS.md +++ b/docs/PITFALLS.md @@ -35,3 +35,113 @@ Check here before debugging. ## Laptop slows mid-campaign - Thermal throttling on the mobile CPU, not a resource fault. Plug in, ensure airflow, continue. Do NOT change host config to "fix" it. + +--- + +## Linux process-creation telemetry: what actually blocks it + +The README's "Linux Sysmon gap" is often read as *not configured yet*. It is more specific than +that, and both halves were verified on the 2026-08-29 lab host rather than assumed. + +### auditd cannot run on a host whose kernel does not expose the audit subsystem + +The victim image installs `auditd` and the compose override grants `AUDIT_CONTROL` / +`AUDIT_READ`. Those capabilities *are* effective inside the container — and it still fails: + +``` +$ docker exec purpleloop-victim auditctl -l +Error sending rule list data request (Operation not permitted) +``` + +This is not a container permissions problem. It fails identically with `--privileged` **and** +`--network host`, so it is neither a capability nor a network-namespace issue: + +``` +$ docker run --rm --privileged --network host --entrypoint sh purpleloop/victim:latest \ + -c "auditctl -s" +Error sending status request (Operation not permitted) +``` + +The host itself has no audit sysctls at all: + +``` +$ cat /proc/sys/kernel/audit_backlog_limit +cat: /proc/sys/kernel/audit_backlog_limit: No such file or directory +$ ls /proc/sys/kernel/ | grep -i audit # no results +``` + +`kauditd` exists as a kernel thread, so `CONFIG_AUDIT` is compiled in, but the control +interface is unavailable. **On such a host, no amount of container configuration produces +auditd telemetry.** + +### Sysmon for Linux requires systemd + +Sysmon for Linux 1.5.2 installs and its eBPF objects are present (the host has +`/sys/kernel/btf/vmlinux`, so eBPF itself is fine). But it only runs as a systemd service: + +``` +$ /opt/sysmon/sysmon -i /opt/sysmon/config.xml -service +sh: 1: systemctl: not found +# process exits +``` + +The `-service` flag in the shipped unit's `ExecStart` still shells out to `systemctl`. The +victim container runs `tail -F` as PID 1, so there is no systemd for it to talk to. + +### What this means for a run + +The pipeline canary requires `category: process_creation`. With no process-creation source the +canary cannot fire, so every Linux run is correctly `INCONCLUSIVE` and coverage is suppressed. +**That is the canary working, not the canary failing.** Do not "fix" it by relaxing the canary +rule or by letting command-output telemetry satisfy a process-creation rule — the +evidence-fidelity gate exists precisely to refuse that, and removing it is how v1.0 produced a +false 100%. + +### Closing it + +Closing the gap needs a victim that can run systemd — a systemd-based image running privileged +with PID 1 as systemd, so Sysmon for Linux can install and start. That is a deliberate change +to the lab's security posture (a privileged container in a lab that otherwise runs +unprivileged), and it is a decision to take explicitly rather than drift into. + +Until then, Windows Sysmon Event ID 1 is the platform where detection is demonstrated. The +`--settle-timeout` default of 60s is also below the victim's 360s `` collection +interval; raise it if you want even low-fidelity command output to land inside a technique's +window. + +## Why the Linux victim is not a container with Sysmon + +Sysmon for Linux was built, run and measured on this lab before being rejected. Recording the +result so it is not re-attempted blind. + +**It works.** With systemd as PID 1 (`privileged: true`, `cgroup: host`, `/sys/kernel/btf` +mounted) the service starts and emits genuine `EventID 1` process-creation events carrying +`Image`, `CommandLine`, `ParentImage` and `User` — exactly the fields the evaluator needs. The +undocumented `-service` flag in its own systemd unit is what runs it; it has no foreground mode. + +**And it cannot be scoped to the victim.** Sysmon's eBPF is kernel-global. Inside a container it +observes every process on the host, and the event carries no container, cgroup or namespace +field — `RuleName`, `LogonId` and `TerminalSessionId` do not distinguish one. Measured results +from attempting to scope it after the fact by resolving the event's host PID through a +read-only `/proc` mount: + +| Approach | Result | +|---|---| +| cgroup lookup on the event's PID | **Host binaries leaked.** `/usr/lib/cargo/bin/coreutils/tr`, `nvidia-smi`, `vmrun` — none of which exist in the Ubuntu image — were accepted. Short-lived host processes exit, the kernel recycles the PID, a container process takes it, and the lookup answers about the wrong process. | +| the same, plus verifying `/proc//comm` matches the event's image | **No leakage, but most events lost.** Only processes living longer than the syslog→converter latency survive; `uname`, `id` and `who` — the atomics themselves — were dropped. | + +Both directions are wrong in ways this project exists to prevent. The first manufactures false +`DETECTED` verdicts from activity typed on the host; the second manufactures false `MISSED` and +`NO_TELEMETRY` from a collector that simply could not keep up. There is no third signal in the +event to break the tie. + +**The fix is a VM, not a container.** The Windows victim already is one, which is precisely why +Windows telemetry is unambiguous: everything the sensor sees belongs to the victim, because the +sensor and the victim share a kernel that nothing else uses. A Linux VM running Sysmon for Linux +would give the Linux side the same property. + +The engine is already ready for it: the normalizer has a `data.sysmon` path with EventID-1 +fidelity gating and tests (`internal/evaluator/sysmon_linux_test.go`), so a Linux-Sysmon source +needs no Go changes — only a Wazuh agent shipping the events with those keys. Note the events +are deliberately *not* mapped through `data.win.*`; that path is Wazuh's Windows eventchannel +convention and using it for Linux would make the telemetry misreport its own origin. From 74f5569afa324f10687ba839813c2da146397403 Mon Sep 17 00:00:00 2001 From: jayelbotvibe-web Date: Sat, 29 Aug 2026 15:12:59 +0800 Subject: [PATCH 12/13] ci: give gitleaks the token it needs to scan pull requests The secrets job failed on every pull request with "GITHUB_TOKEN is now required to scan pull requests" -- a breaking change in gitleaks-action v2, not a finding. A secrets check that is permanently red teaches everyone to ignore it, which is worse than not running one. Co-Authored-By: Claude Opus 5 --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1956ab6..81d4885 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,5 +78,10 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + # gitleaks-action requires a token to scan pull requests since v2. Without + # it the job fails on every PR for a configuration reason rather than a + # finding — which trains everyone to ignore a red secrets check. - name: gitleaks uses: gitleaks/gitleaks-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 9e6222818de2dada9e4b5f429cce0f79503a78df Mon Sep 17 00:00:00 2001 From: jayelbotvibe-web Date: Sat, 29 Aug 2026 15:49:01 +0800 Subject: [PATCH 13/13] fix: remove the hardcoded Windows victim address 192.168.88.13 was asserted as fact in seven places -- the canary's fallback, lab/targets.yml's default_host, an ssh.go comment, four spots in STARTUP.md and one in startup.sh. It only ever existed on one machine's bridged network. On any other host, bridged, NAT (vmnet8) and host-only (vmnet1) give different subnets, so the documented address is not merely absent but points somewhere else: the canary would probe a host that is not the victim, and the setup guide told you to expect an address you will never see. `canary` now reads the Windows host from lab/targets.yml like `run` does, and there is deliberately no default. With WINDOWS_SSH_HOST unset it says so and exits, naming the variable, rather than quietly probing a wrong address. STARTUP.md now tells you to find the address with ipconfig and export it, which is the only instruction that is true on every host. Co-Authored-By: Claude Opus 5 --- STARTUP.md | 18 ++++++++++++++---- cmd/purpleloop/main.go | 29 ++++++++++++++++++++++------- internal/executor/ssh.go | 2 +- lab/targets.yml | 8 ++++++-- scripts/startup.sh | 4 +++- 5 files changed, 46 insertions(+), 15 deletions(-) diff --git a/STARTUP.md b/STARTUP.md index fbda297..0d921eb 100644 --- a/STARTUP.md +++ b/STARTUP.md @@ -83,8 +83,17 @@ the Wazuh agent is connected (agent_control -l shows agent 002 Active). ### 3.1 Starting 1. Power on the VM in VMware -2. Confirm bridged IP: run `ipconfig` on the VM → should be `192.168.88.13` -3. Test SSH: `ssh windows-vm@192.168.88.13 hostname` +2. Find its address: run `ipconfig` on the VM. It depends on how the VM is + networked — bridged, NAT (`vmnet8`) and host-only (`vmnet1`) each give a + different subnet — so there is no address to expect here. +3. Export it, and test SSH: + ```bash + export WINDOWS_SSH_HOST= + export WINDOWS_SSH_USER=windows-vm + export WINDOWS_SSH_PASS= + ssh "$WINDOWS_SSH_USER@$WINDOWS_SSH_HOST" hostname + ``` + `lab/targets.yml` reads these; nothing in the repo hardcodes an address. 4. Confirm services: ```powershell Get-Service WazuhSvc, Sysmon64 | Format-Table Name, Status @@ -207,11 +216,12 @@ at boot. To prevent boot auto-start: `sudo systemctl disable docker`. | Symptom | Check | |---------|-------| -| Canary fails | `ssh windows-vm@192.168.88.13 "sc query WazuhSvc Sysmon64"` | +| Canary fails | `ssh "$WINDOWS_SSH_USER@$WINDOWS_SSH_HOST" "sc query WazuhSvc Sysmon64"` | | Agent not Active | `docker logs single-node-wazuh.manager-1 \| grep -i "auth\|enroll" \| tail -5` | -| No Sysmon events | `ssh windows-vm@192.168.88.13 "wevtutil qe Microsoft-Windows-Sysmon/Operational /c:2"` | +| No Sysmon events | `ssh "$WINDOWS_SSH_USER@$WINDOWS_SSH_HOST" "wevtutil qe Microsoft-Windows-Sysmon/Operational /c:2"` | | Dashboard 502 | `docker logs single-node-wazuh.indexer-1 \| tail -5` | | Build fails | `go mod tidy && make build` | +| Canary: "no address for the windows victim" | `WINDOWS_SSH_HOST` is unset. There is no default — see 3.1. | | Port conflicts | `docker ps --format '{{.Names}} {{.Ports}}'` | | vm.max_map_count reset | `sysctl vm.max_map_count` (must be ≥262144) | diff --git a/cmd/purpleloop/main.go b/cmd/purpleloop/main.go index be41df9..7012730 100644 --- a/cmd/purpleloop/main.go +++ b/cmd/purpleloop/main.go @@ -746,15 +746,30 @@ func detectLatency(start time.Time, evidence []model.Event) int64 { func runCanaryCmd() { marker := canary.NewMarker() ctx := context.Background() - sshHost := os.Getenv("WINDOWS_SSH_HOST") - sshUser := os.Getenv("WINDOWS_SSH_USER") - sshPass := os.Getenv("WINDOWS_SSH_PASS") - if sshHost == "" { - sshHost = "192.168.88.13" + + // The Windows host comes from the lab inventory, not from a literal here. + // It used to default to 192.168.88.13, an address that only existed on one + // machine's bridged network — on any other host the canary silently probed + // something that was not the victim, or nothing at all. + targets, err := lab.Load(defaultTargets) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) } - if sshUser == "" { - sshUser = "windows-vm" + t, ok := targets.ForPlatform("windows") + if !ok { + fmt.Fprintf(os.Stderr, "error: no windows target declared in %s\n", defaultTargets) + os.Exit(1) } + sshHost, sshUser, sshPass := t.SSHConfig() + if sshHost == "" { + fmt.Fprintf(os.Stderr, + "error: no address for the windows victim. Set %s to the IP the VM "+ + "actually has (run `ipconfig` on it) — there is no default, because a\n"+ + "wrong one probes a host that is not your victim.\n", t.SSHHostEnv) + os.Exit(1) + } + exec := &executor.SSHExecutor{Host: sshHost, User: sshUser, Password: sshPass} coll := &collector.WazuhCollector{ManagerContainer: "single-node-wazuh.manager-1"} target := model.Target{Host: "windows-vm", Kind: "windows"} diff --git a/internal/executor/ssh.go b/internal/executor/ssh.go index 2b35a6c..9970b25 100644 --- a/internal/executor/ssh.go +++ b/internal/executor/ssh.go @@ -12,7 +12,7 @@ import ( // SSHExecutor runs atomics via SSH on remote targets (Windows VM, etc). // Uses sshpass for password auth in lab environments. type SSHExecutor struct { - Host string // e.g. 192.168.88.13 + Host string // victim address; from WINDOWS_SSH_HOST via lab/targets.yml User string // e.g. windows-vm Password string } diff --git a/lab/targets.yml b/lab/targets.yml index 3788bb0..a599c7d 100644 --- a/lab/targets.yml +++ b/lab/targets.yml @@ -22,9 +22,13 @@ targets: rules_dir: detections/windows # Host and credentials come from the environment, never from source: # WINDOWS_SSH_HOST, WINDOWS_SSH_USER, WINDOWS_SSH_PASS - # (The IP was previously hardcoded at cmd/purpleloop/main.go:314.) + # + # There is deliberately NO default host. The address depends on how your + # hypervisor is networked -- bridged, NAT (vmnet8) and host-only (vmnet1) + # all give different subnets -- so a committed default is right on exactly + # one machine and quietly wrong everywhere else. Find it with `ipconfig` on + # the VM and export WINDOWS_SSH_HOST. ssh_host_env: WINDOWS_SSH_HOST ssh_user_env: WINDOWS_SSH_USER ssh_pass_env: WINDOWS_SSH_PASS - default_host: 192.168.88.13 default_user: windows-vm diff --git a/scripts/startup.sh b/scripts/startup.sh index 0590ee8..b0df8ea 100755 --- a/scripts/startup.sh +++ b/scripts/startup.sh @@ -32,7 +32,9 @@ echo "[5/5] Startup complete." echo "" echo "Next steps:" echo " 1. Power on Windows VM in VMware" -echo " 2. ssh windows-vm@192.168.88.13 hostname # verify" +echo " 2. Find the VM address (ipconfig on the VM), then:" +echo " export WINDOWS_SSH_HOST=
WINDOWS_SSH_USER=windows-vm" +echo " ssh \"\$WINDOWS_SSH_USER@\$WINDOWS_SSH_HOST\" hostname # verify" echo " 3. Generate arbiter export:" echo " hermes -z 'export top 10 alerts...' --profile threatlib" echo " 4. Run campaign:"