From 8cb3f4f99136caa57c946490e3235f4ac0fb83de Mon Sep 17 00:00:00 2001 From: Ron Heichman Date: Mon, 7 Sep 2026 14:28:44 +0000 Subject: [PATCH] feat(rules): add `rules test --json` machine-readable result contract Add an optional --json mode to `numbat rules test` that emits a versioned NDJSON result stream on stdout: one event_result object per successfully evaluated fixture line plus one terminal summary object. The stream distinguishes findings (with enforcement eligibility and rule version), malformed input, evaluator failure, and shell-parse coverage as separate result classes, so a downstream consumer never has to parse human CLI text or infer a clean no-match from an empty stdout. The change extends the existing rules test seam and its evalFixture sibling; no parallel evaluator is introduced. rule.Engine gains an EvalDetailed companion to Eval that returns per-rule errors and shell-analysis diagnostics without changing Eval's error text or the existing tab-separated output. The legacy stdout format is byte-identical when --json is absent. The contract is documented in docs/schema/rules-test-result.v1.md and the CLI reference. Behavioral checks in rules_test_json_test.go exercise all five result classes end-to-end through runCLI. Built with Codex --- cmd/numbat/rules.go | 14 +- cmd/numbat/rules_json.go | 437 +++++++++++++ cmd/numbat/rules_test_json_test.go | 904 +++++++++++++++++++++++++++ docs/cli.md | 11 + docs/schema/rules-test-result.v1.md | 191 ++++++ internal/archguard/archguard_test.go | 2 +- internal/rule/engine.go | 78 +++ 7 files changed, 1633 insertions(+), 4 deletions(-) create mode 100644 cmd/numbat/rules_json.go create mode 100644 cmd/numbat/rules_test_json_test.go create mode 100644 docs/schema/rules-test-result.v1.md diff --git a/cmd/numbat/rules.go b/cmd/numbat/rules.go index 2b40a61..f0745a0 100644 --- a/cmd/numbat/rules.go +++ b/cmd/numbat/rules.go @@ -306,7 +306,10 @@ func runRulesList(args []string, stdout, stderr io.Writer) int { // runRulesTest evaluates the compiled rules against a fixture of NDJSON // events (one model.Event per line) and prints each match as "rule_id\tevent_id". -// It is the deterministic, offline check that rules fire as intended. +// It is the deterministic, offline check that rules fire as intended. With +// --json the same evaluation emits a machine-readable NDJSON result contract +// on stdout instead: one event_result per successfully evaluated fixture +// line plus a terminal summary. See docs/schema/rules-test-result.v1.md. func runRulesTest(args []string, stdout, stderr io.Writer) int { fs := flag.NewFlagSet("rules test", flag.ContinueOnError) fs.SetOutput(stderr) @@ -315,11 +318,12 @@ func runRulesTest(args []string, stdout, stderr io.Writer) int { expectNone := fs.Bool("expect-none", false, "exit non-zero if any rule matches (for negative fixtures)") var expect multiFlag fs.Var(&expect, "expect", "rule ID expected to match at least once (repeatable)") + jsonOut := fs.Bool("json", false, "emit a machine-readable NDJSON result stream (schema rules-test-result.v1) instead of tab-separated matches") var rf ruleFlags rf.register(fs) fs.Usage = func() { - fmt.Fprintln(stderr, "usage: numbat rules test --fixture FILE [--require-match] [--expect RULE_ID ...] [--expect-none] [--rules-dir DIR ...] [--no-builtin-rules]") - fmt.Fprintln(stderr, "\nEvaluate fixture events and print rule_idevent_id for each match.") + fmt.Fprintln(stderr, "usage: numbat rules test --fixture FILE [--json] [--require-match] [--expect RULE_ID ...] [--expect-none] [--rules-dir DIR ...] [--no-builtin-rules]") + fmt.Fprintln(stderr, "\nEvaluate fixture events and print rule_idevent_id for each match, or a JSON result stream with --json.") fs.PrintDefaults() } if err := fs.Parse(args); err != nil { @@ -356,6 +360,10 @@ func runRulesTest(args []string, stdout, stderr io.Writer) int { return 1 } + if *jsonOut { + return runRulesTestJSON(eng, f, stdout, stderr, *requireMatch, *expectNone, expect) + } + matched, matchedRules, evalErr := evalFixture(eng, f, stdout) if evalErr != nil { fmt.Fprintln(stderr, evalErr.Error()) diff --git a/cmd/numbat/rules_json.go b/cmd/numbat/rules_json.go new file mode 100644 index 0000000..1c2ded7 --- /dev/null +++ b/cmd/numbat/rules_json.go @@ -0,0 +1,437 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "io" + + "github.com/perplexityai/numbat/internal/model" + "github.com/perplexityai/numbat/internal/rule" + "github.com/perplexityai/numbat/internal/sequence" + "github.com/perplexityai/numbat/internal/version" +) + +// rulesTestResultSchemaVersion identifies the machine-readable result stream +// contract emitted by `rules test --json`. It is intentionally separate from +// the wire record schema (model.SchemaVersion) because this envelope is a +// CLI-adjacent surface, not a record shape emitted by the pipeline. Any +// breaking change to the envelope requires a new major version and updated +// consumers. +const rulesTestResultSchemaVersion = "rules-test-result.v1" + +// rulesTestEventResult is one line of the NDJSON stream: the direct-evaluator +// result for a single fixture line. Status distinguishes the result classes +// the contract must expose: +// +// - "completed": the event was evaluated and any matching rule is +// reported in Findings. This is the only status that +// legitimately reports zero findings as "no match". +// - "malformed_input": the fixture line could not be decoded, failed +// model.Event validation, or the input stream itself +// failed to scan; Error.Kind is "decode", "validate", +// or "scan". Fixture processing stops at this event. +// - "evaluation_failure": at least one compiled rule's CEL program errored +// at runtime, or the sequence tracker returned an +// error; EvaluatorErrors names every failing rule +// (a per-rule CEL failure and a sequence-tracker +// failure may both appear for the same event). +// Error.Kind is "sequence" when the tracker failed, +// otherwise "evaluation". Fixture processing stops. +type rulesTestEventResult struct { + Type string `json:"type"` + SchemaVersion string `json:"schema_version"` + FixtureLine int `json:"fixture_line"` + EventID string `json:"event_id,omitempty"` + Status string `json:"status"` + Findings []rulesTestFinding `json:"findings,omitempty"` + EvaluatorErrors []rulesTestEvaluatorError `json:"evaluator_errors,omitempty"` + Coverage *rulesTestCoverage `json:"coverage,omitempty"` + Error *rulesTestEventErrorDetail `json:"error,omitempty"` +} + +// rulesTestFinding is one matched rule for the event. RuleVersion is copied +// from the compiled rule so downstream systems can reason about drift. +// EnforcementEligible is the same flag that would gate the live enforce +// path (already accounting for shell-enforcement-safety); Via reports whether +// the finding came from a single-event evaluation or a completed sequence +// chain so a consumer can attribute chain findings to the terminating event. +type rulesTestFinding struct { + RuleID string `json:"rule_id"` + RuleVersion string `json:"rule_version"` + Severity string `json:"severity,omitempty"` + EnforcementEligible bool `json:"enforcement_eligible"` + Via string `json:"via"` +} + +// rulesTestEvaluatorError names one rule whose CEL program failed at runtime. +// Emitted alongside any matches that other rules produced for the same event; +// per-rule failure never suppresses another rule's match. +type rulesTestEvaluatorError struct { + RuleID string `json:"rule_id,omitempty"` + Message string `json:"message"` +} + +// rulesTestCoverage carries the shared shell-analysis health signal for the +// event. shell_parse is one of "ok" (no shell analysis run or the parse was +// clean), "degraded" (analysis produced errors but at least one command was +// still usable), and "unusable" (parse produced no usable commands and rules +// that depend on shell_commands were skipped). sequence_tracker_active is +// true when the compiled catalog contains at least one sequence rule (i.e. +// a window tracker exists for this run); it does not imply this specific +// event was folded into a window (the tracker only observes events that +// carry a session_id). +type rulesTestCoverage struct { + ShellParse string `json:"shell_parse"` + SequenceTrackerActive bool `json:"sequence_tracker_active"` +} + +// rulesTestEventErrorDetail describes a fixture-stopping failure for one +// event. Kind is one of: "decode" (invalid JSON), "validate" +// (schema/contract violation from model.Event.Validate), "scan" (input +// stream failed to scan, e.g. line too long or IO error; Message names the +// next unread fixture line), "evaluation" (per-rule CEL runtime error, +// aggregated), or "sequence" (window-tracker failure). +type rulesTestEventErrorDetail struct { + Kind string `json:"kind"` + Message string `json:"message"` +} + +// rulesTestSummary is the terminal object in the stream. Status is +// "completed" when every fixture line was evaluated (including a clean +// no-match run) and "partial" when processing stopped at a malformed or +// failing event. AssertionOutcome captures the --require-match / --expect / +// --expect-none result independently of Status: a fully evaluated fixture +// whose assertions failed is Status="completed", AssertionOutcome="failed". +type rulesTestSummary struct { + Type string `json:"type"` + SchemaVersion string `json:"schema_version"` + Status string `json:"status"` + EventsEvaluated int `json:"events_evaluated"` + Matches int `json:"matches"` + RulesLoaded int `json:"rules_loaded"` + EnforceEligible int `json:"enforce_eligible_rules"` + AssertionOutcome string `json:"assertion_outcome"` + AssertionMissing []string `json:"assertion_missing,omitempty"` + StoppedAt *rulesTestSummaryStopping `json:"stopped_at,omitempty"` + NumbatVersion string `json:"numbat_version"` + RecordSchema string `json:"record_schema"` +} + +// rulesTestSummaryStopping identifies the fixture line and reason a partial +// run stopped. Reason mirrors the last event_result's status ("malformed_input" +// or "evaluation_failure") so a consumer can dispatch on the summary alone. +type rulesTestSummaryStopping struct { + FixtureLine int `json:"fixture_line"` + Reason string `json:"reason"` +} + +// rulesTestJSONDeliveryExitCode is returned when a stdout write fails at any +// point in the JSON result stream. It is deliberately distinct from the +// partial-run exit (1) and the completed-success exit (0) so a downstream +// consumer that only sees the process exit can distinguish "the evaluator +// ran but the machine-readable result could not be delivered" from "the +// evaluator observed a failure and reported it in a well-formed stream". +const rulesTestJSONDeliveryExitCode = 2 + +// runRulesTestJSON is the --json branch of `numbat rules test`. It shares +// no state with the legacy path, but reuses the compiled Engine and the +// same sequence.Tracker construction as evalFixture so its match set is +// byte-equivalent to the legacy stream. The JSON encoder is bound to +// stdout; every event_result and the terminal summary are one line each. +// Every write goes through the emit closure so a stdout failure never +// masquerades as a clean run. +func runRulesTestJSON(eng *rule.Engine, r io.Reader, stdout, stderr io.Writer, requireMatch, expectNone bool, expect multiFlag) int { + enc := json.NewEncoder(stdout) + enc.SetEscapeHTML(false) + + // deliveryFailed records that at least one enc.Encode returned an error. + // Once set, no further JSON is written to stdout (a downstream consumer + // that treats a missing or invalid terminal stream as failure will react + // correctly), and the function returns rulesTestJSONDeliveryExitCode. + // This exists because evaluating successfully is not the same as + // delivering the result: silently swallowing a stdout write failure would + // let a Guardian-side JSON parser see zero events and infer a clean + // no-match, which is the exact confusion this contract was added to + // prevent. + deliveryFailed := false + emit := func(v interface{}) bool { + if deliveryFailed { + return false + } + if err := enc.Encode(v); err != nil { + deliveryFailed = true + // Best-effort diagnostic; a broken stderr is deliberately ignored + // because the nonzero exit code already signals delivery failure to + // the caller, and a stderr write error here would only mask the + // original stdout failure. + _, _ = fmt.Fprintln(stderr, "write rules-test-result:", err.Error()) + return false + } + return true + } + + enforceEligible := eng.CountEnforceEligibleRules() + + var tracker *sequence.Tracker + seqRules := eng.SequenceRules() + if len(seqRules) > 0 { + tracker = sequence.NewTracker(seqRules, sequence.DefaultConfig()) + } + + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024) + + matched := 0 + matchedRules := map[string]int{} + eventsEvaluated := 0 + line := 0 + var stopped *rulesTestSummaryStopping + streamStatus := "completed" + + // summary is emitted at the end regardless of the outcome; capture stops in + // stopped so a downstream consumer sees the exact fixture line that failed. +loop: + for sc.Scan() { + line++ + raw := sc.Bytes() + if len(raw) == 0 { + continue + } + var ev model.Event + if err := json.Unmarshal(raw, &ev); err != nil { + emit(rulesTestEventResult{ + Type: "event_result", + SchemaVersion: rulesTestResultSchemaVersion, + FixtureLine: line, + Status: "malformed_input", + Error: &rulesTestEventErrorDetail{ + Kind: "decode", + Message: fmt.Sprintf("fixture line %d: %s", line, err.Error()), + }, + }) + stopped = &rulesTestSummaryStopping{FixtureLine: line, Reason: "malformed_input"} + streamStatus = "partial" + break loop + } + ev = ev.NormalizePaths() + if err := ev.Validate(); err != nil { + emit(rulesTestEventResult{ + Type: "event_result", + SchemaVersion: rulesTestResultSchemaVersion, + FixtureLine: line, + EventID: ev.EventID, + Status: "malformed_input", + Error: &rulesTestEventErrorDetail{ + Kind: "validate", + Message: fmt.Sprintf("fixture line %d: %s", line, err.Error()), + }, + }) + stopped = &rulesTestSummaryStopping{FixtureLine: line, Reason: "malformed_input"} + streamStatus = "partial" + break loop + } + + singleMatches, evalErrs, diag := eng.EvalDetailed(ev) + coverage := &rulesTestCoverage{ + ShellParse: classifyShellParse(diag), + SequenceTrackerActive: tracker != nil, + } + + findings := make([]rulesTestFinding, 0, len(singleMatches)) + for _, m := range singleMatches { + matched++ + matchedRules[m.Rule.ID]++ + findings = append(findings, rulesTestFinding{ + RuleID: m.Rule.ID, + RuleVersion: m.Rule.Version, + Severity: m.Rule.Severity, + EnforcementEligible: m.EnforcementMatch, + Via: "engine", + }) + } + + // Convert per-rule single-event evaluator errors into the wire form up + // front so the sequence-error branch can preserve them alongside a + // tracker failure. Per-rule CEL errors and a sequence-tracker error can + // legitimately co-occur for one event; neither may suppress the other. + evaluatorErrors := make([]rulesTestEvaluatorError, 0, len(evalErrs)+1) + for _, e := range evalErrs { + evaluatorErrors = append(evaluatorErrors, rulesTestEvaluatorError{ + RuleID: e.RuleID, + Message: e.Message, + }) + } + + sequenceErrorMessage := "" + if tracker != nil { + observation, err := tracker.Observe(ev) + if err != nil { + // A sequence-tracker error is a fixture-stopping evaluator failure: + // the window state cannot be trusted for the remainder of the run. + // Preserve any per-rule CEL errors above; the two failures are + // independent and both belong in evaluator_errors. + sequenceErrorMessage = err.Error() + evaluatorErrors = append(evaluatorErrors, rulesTestEvaluatorError{ + Message: err.Error(), + }) + } else { + enforceIndex := map[string]bool{} + for _, r := range observation.EnforcementRules { + enforceIndex[r.ID] = true + } + for _, c := range observation.Findings { + matched++ + matchedRules[c.Rule.ID]++ + findings = append(findings, rulesTestFinding{ + RuleID: c.Rule.ID, + RuleVersion: c.Rule.Version, + Severity: c.Rule.Severity, + EnforcementEligible: enforceIndex[c.Rule.ID], + Via: "sequence", + }) + } + } + } + + status := "completed" + if len(evalErrs) > 0 || sequenceErrorMessage != "" { + // Any evaluator failure stops the fixture. Matches from other rules on + // this same event still surface in Findings; failures are never + // silently suppressed. + status = "evaluation_failure" + } + + result := rulesTestEventResult{ + Type: "event_result", + SchemaVersion: rulesTestResultSchemaVersion, + FixtureLine: line, + EventID: ev.EventID, + Status: status, + Findings: findings, + EvaluatorErrors: evaluatorErrors, + Coverage: coverage, + } + if status == "evaluation_failure" { + // error.kind is "sequence" when the tracker itself failed (so a + // consumer can distinguish a window-state failure from a CEL runtime + // error), otherwise "evaluation" for one or more per-rule CEL errors. + switch { + case sequenceErrorMessage != "": + result.Error = &rulesTestEventErrorDetail{ + Kind: "sequence", + Message: fmt.Sprintf("fixture line %d: %s", line, sequenceErrorMessage), + } + default: + result.Error = &rulesTestEventErrorDetail{ + Kind: "evaluation", + Message: fmt.Sprintf("fixture line %d: %d rule(s) failed evaluation", line, len(evalErrs)), + } + } + } + emit(result) + + // Every fixture line that reached direct evaluation is counted, even one + // whose evaluation ended in failure. events_evaluated therefore denotes + // "events that reached direct evaluation", not "events with matches"; + // matches is the total findings across all event_results and may exceed + // events_evaluated when several rules match one event. + eventsEvaluated++ + if status == "evaluation_failure" { + stopped = &rulesTestSummaryStopping{FixtureLine: line, Reason: "evaluation_failure"} + streamStatus = "partial" + break loop + } + } + if err := sc.Err(); err != nil { + // A scanner failure (over-long line, IO error) must still terminate the + // stream with a well-formed summary; a downstream consumer relying on + // the terminal object to distinguish partial from truncated pipe would + // otherwise hit bare EOF. Report the scan failure at the next fixture + // line so its position is unambiguous, then fall through to the summary + // emission below. + emit(rulesTestEventResult{ + Type: "event_result", + SchemaVersion: rulesTestResultSchemaVersion, + FixtureLine: line + 1, + Status: "malformed_input", + Error: &rulesTestEventErrorDetail{ + Kind: "scan", + Message: fmt.Sprintf("scan fixture: %s", err.Error()), + }, + }) + stopped = &rulesTestSummaryStopping{FixtureLine: line + 1, Reason: "malformed_input"} + streamStatus = "partial" + fmt.Fprintln(stderr, "scan fixture:", err.Error()) + } + + // AssertionOutcome is only meaningful for a completed run; a partial run + // intentionally reports "unchecked" so a consumer never conflates a + // fixture-processing failure with an assertion failure. + assertionOutcome := "unchecked" + var assertionMissing []string + if streamStatus == "completed" { + assertionOutcome = "passed" + if requireMatch && matched == 0 { + assertionOutcome = "failed" + } + if expectNone && matched > 0 { + assertionOutcome = "failed" + } + if missing := missingExpectedRules(expect, matchedRules); len(missing) > 0 { + assertionOutcome = "failed" + assertionMissing = missing + } + if len(expect) == 0 && !requireMatch && !expectNone { + assertionOutcome = "unchecked" + } + } + + summary := rulesTestSummary{ + Type: "summary", + SchemaVersion: rulesTestResultSchemaVersion, + Status: streamStatus, + EventsEvaluated: eventsEvaluated, + Matches: matched, + RulesLoaded: eng.Len(), + EnforceEligible: enforceEligible, + AssertionOutcome: assertionOutcome, + AssertionMissing: assertionMissing, + StoppedAt: stopped, + NumbatVersion: version.String(), + RecordSchema: model.SchemaVersion, + } + emit(summary) + + // A delivery failure at any point in the stream (event_result OR summary) + // dominates the exit code: the caller must not observe "evaluation ran + // cleanly, exit 0" when part or all of the machine-readable stream never + // reached the descriptor. The legacy tab-separated path signals the same + // class of failure with exit 1 and a stderr message; the JSON path uses a + // distinct exit (2) so a consumer that also parses exit codes can tell + // "partial run reported correctly" from "result stream truncated". + if deliveryFailed { + return rulesTestJSONDeliveryExitCode + } + if streamStatus == "partial" { + return 1 + } + if assertionOutcome == "failed" { + return 1 + } + return 0 +} + +// classifyShellParse maps rule.EvalDiagnostics to the coverage.shell_parse +// enum defined in rules-test-result.v1. It preserves the invariant that a +// consumer can distinguish a bounded/unusable shell parse from a clean +// no-match without reading log lines. +func classifyShellParse(diag rule.EvalDiagnostics) string { + if diag.ShellParseError == nil { + return "ok" + } + if diag.ShellUsable { + return "degraded" + } + return "unusable" +} diff --git a/cmd/numbat/rules_test_json_test.go b/cmd/numbat/rules_test_json_test.go new file mode 100644 index 0000000..f4f2edb --- /dev/null +++ b/cmd/numbat/rules_test_json_test.go @@ -0,0 +1,904 @@ +package main + +// Behavioral checks for the machine-readable `rules test --json` result +// contract. The contract binds each observed input line to one of five +// distinct classes so a downstream consumer (e.g. Guardian) never has to +// parse human CLI text. These tests were authored before the feature +// implementation as RED-first checks: they must fail on the current main +// (no --json flag) and pass after the smallest supported implementation +// lands. Every class is exercised end-to-end through runCLI. + +import ( + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// jsonRulesTestEvent mirrors the per-event object the CLI emits when --json +// is set. Reading it back through this struct is the machine-readable +// contract downstream consumers rely on. +type jsonRulesTestEvent struct { + Type string `json:"type"` + SchemaVersion string `json:"schema_version"` + FixtureLine int `json:"fixture_line"` + EventID string `json:"event_id,omitempty"` + Status string `json:"status"` + Findings []jsonRulesTestFinding `json:"findings,omitempty"` + EvaluatorErrors []jsonRulesTestEvalError `json:"evaluator_errors,omitempty"` + Coverage *jsonRulesTestCoverage `json:"coverage,omitempty"` + Error *jsonRulesTestErrorDetail `json:"error,omitempty"` +} + +type jsonRulesTestFinding struct { + RuleID string `json:"rule_id"` + RuleVersion string `json:"rule_version"` + EnforcementEligible bool `json:"enforcement_eligible"` + Via string `json:"via"` +} + +type jsonRulesTestEvalError struct { + RuleID string `json:"rule_id,omitempty"` + Message string `json:"message"` +} + +type jsonRulesTestCoverage struct { + ShellParse string `json:"shell_parse"` + SequenceTrackerActive bool `json:"sequence_tracker_active"` +} + +type jsonRulesTestErrorDetail struct { + Kind string `json:"kind"` + Message string `json:"message"` +} + +type jsonRulesTestSummary struct { + Type string `json:"type"` + SchemaVersion string `json:"schema_version"` + Status string `json:"status"` + EventsEvaluated int `json:"events_evaluated"` + Matches int `json:"matches"` + RulesLoaded int `json:"rules_loaded"` + EnforceEligible int `json:"enforce_eligible_rules"` + AssertionOutcome string `json:"assertion_outcome"` + AssertionMissing []string `json:"assertion_missing,omitempty"` + StoppedAt *jsonRulesTestStoppedAtBlock `json:"stopped_at,omitempty"` + NumbatVersion string `json:"numbat_version"` + RecordSchema string `json:"record_schema"` +} + +type jsonRulesTestStoppedAtBlock struct { + FixtureLine int `json:"fixture_line"` + Reason string `json:"reason"` +} + +// parseJSONStream splits stdout into per-line JSON objects. Every line must +// decode; the CLI's contract is one JSON object per line and one terminal +// summary object. +func parseJSONStream(t *testing.T, out string) ([]jsonRulesTestEvent, jsonRulesTestSummary) { + t.Helper() + var events []jsonRulesTestEvent + var summary jsonRulesTestSummary + var sawSummary bool + for i, line := range strings.Split(strings.TrimRight(out, "\n"), "\n") { + if line == "" { + continue + } + var probe struct { + Type string `json:"type"` + } + if err := json.Unmarshal([]byte(line), &probe); err != nil { + t.Fatalf("line %d not JSON: %v; content=%q", i+1, err, line) + } + switch probe.Type { + case "event_result": + var ev jsonRulesTestEvent + if err := json.Unmarshal([]byte(line), &ev); err != nil { + t.Fatalf("line %d event_result decode: %v", i+1, err) + } + events = append(events, ev) + case "summary": + if err := json.Unmarshal([]byte(line), &summary); err != nil { + t.Fatalf("line %d summary decode: %v", i+1, err) + } + sawSummary = true + default: + t.Fatalf("line %d unexpected type %q; content=%q", i+1, probe.Type, line) + } + } + if !sawSummary { + t.Fatalf("stream missing terminal summary; stdout=%q", out) + } + return events, summary +} + +// writeTempFile writes body to a temp file under t.TempDir and returns its +// absolute path. Keeps fixture bodies inline in each test for readability +// rather than adding one-off testdata files. +func writeTempFile(t *testing.T, name, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + return path +} + +// TestRulesTestJSONFindingsClass: a completed evaluation with matches emits +// one event_result per input, each match is a finding object with rule_id, +// rule_version, enforcement_eligible, and via; the summary status is +// "completed" and matches count sums the findings. +func TestRulesTestJSONFindingsClass(t *testing.T) { + // Two positive events (secrets.agent_read_env, secrets.read_private_key) + // plus one benign one. Reusing the shipped secrets_fixture keeps the test + // bound to real embedded rules rather than a synthetic engine. + out, errb, code := runCLI("rules", "test", "--json", "--fixture", "testdata/secrets_fixture.ndjson") + if code != 0 { + t.Fatalf("exit = %d, stderr=%q, stdout=%q", code, errb, out) + } + events, summary := parseJSONStream(t, out) + if len(events) < 3 { + t.Fatalf("want at least 3 event_result lines, got %d", len(events)) + } + if summary.Status != "completed" { + t.Fatalf("summary status = %q, want %q", summary.Status, "completed") + } + if summary.SchemaVersion == "" || summary.RecordSchema == "" { + t.Fatalf("summary missing schema/record identifiers: %+v", summary) + } + if summary.AssertionOutcome != "unchecked" { + t.Fatalf("assertion_outcome = %q, want %q", summary.AssertionOutcome, "unchecked") + } + // Find e1 (cat .env) — must have a finding for secrets.agent_read_env with + // rule_version and via="engine". Its enforcement_eligible reflects the + // engine's compiled decision; the test asserts the field is present and + // well-typed rather than pinning its value across rule updates. + var e1 *jsonRulesTestEvent + for i := range events { + if events[i].EventID == "e1" { + e1 = &events[i] + break + } + } + if e1 == nil { + t.Fatalf("missing event_result for e1: %+v", events) + } + if e1.Status != "completed" { + t.Fatalf("e1 status = %q, want completed", e1.Status) + } + var seen bool + for _, f := range e1.Findings { + if f.RuleID == "secrets.agent_read_env" { + if f.RuleVersion == "" { + t.Fatalf("e1 finding missing rule_version: %+v", f) + } + if f.Via != "engine" { + t.Fatalf("e1 finding via = %q, want engine", f.Via) + } + seen = true + } + } + if !seen { + t.Fatalf("e1 missing expected finding for secrets.agent_read_env: %+v", e1) + } + // e3 (.env.example) must complete with no findings — the no-match case. + var e3 *jsonRulesTestEvent + for i := range events { + if events[i].EventID == "e3" { + e3 = &events[i] + break + } + } + if e3 == nil || e3.Status != "completed" { + t.Fatalf("missing completed event_result for e3: %+v", e3) + } + if len(e3.Findings) != 0 { + t.Fatalf("e3 should have no findings: %+v", e3.Findings) + } + // Rules loaded + enforce-eligible counts are populated so the consumer can + // distinguish disabled/empty catalogs from active ones. + if summary.RulesLoaded < 1 { + t.Fatalf("rules_loaded = %d, want >=1", summary.RulesLoaded) + } +} + +// TestRulesTestJSONMalformedInputClass: a fixture line that fails JSON decode +// or event validation is reported as status="malformed_input" with the +// fixture line number, and the summary status is "partial" (fixture +// processing stopped before the end). Exit code is 1. +func TestRulesTestJSONMalformedInputClass(t *testing.T) { + // Line 1 is a valid benign event; line 2 is not valid JSON. Line 3 would + // be valid but must never be reported: the stream stops at the malformed + // input. + body := `{"schema_version":"0.3.0","event_id":"ok1","source_agent":"claude-code","source_type":"artifact","event_type":"file.read","file_path":"/app/main.go","confidence":"high","evidence":{"artifact_type":"claude_jsonl","local_path":"/x","line":1}} +{not valid json +{"schema_version":"0.3.0","event_id":"ok2","source_agent":"claude-code","source_type":"artifact","event_type":"file.read","file_path":"/app/other.go","confidence":"high","evidence":{"artifact_type":"claude_jsonl","local_path":"/x","line":3}} +` + fixture := writeTempFile(t, "malformed.ndjson", body) + out, _, code := runCLI("rules", "test", "--json", "--fixture", fixture) + if code != 1 { + t.Fatalf("exit = %d, want 1; stdout=%q", code, out) + } + events, summary := parseJSONStream(t, out) + if summary.Status != "partial" { + t.Fatalf("summary.status = %q, want partial", summary.Status) + } + if summary.StoppedAt == nil || summary.StoppedAt.FixtureLine != 2 || summary.StoppedAt.Reason != "malformed_input" { + t.Fatalf("stopped_at = %+v, want fixture_line=2 reason=malformed_input", summary.StoppedAt) + } + // The last event_result must be the malformed one. + last := events[len(events)-1] + if last.Status != "malformed_input" { + t.Fatalf("last status = %q, want malformed_input; events=%+v", last.Status, events) + } + if last.FixtureLine != 2 { + t.Fatalf("last fixture_line = %d, want 2", last.FixtureLine) + } + if last.Error == nil || last.Error.Kind != "decode" || last.Error.Message == "" { + t.Fatalf("last error block = %+v, want kind=decode with message", last.Error) + } + // Fixture line 3 must never appear (stream stops at first failure). + for _, e := range events { + if e.EventID == "ok2" || e.FixtureLine == 3 { + t.Fatalf("saw event past malformed line: %+v", e) + } + } +} + +// TestRulesTestJSONEvaluatorFailureClass: a runtime CEL error surfaces as +// status="evaluation_failure" with per-rule evaluator_errors and does not +// silently drop the failing rule. The summary reports partial and the exit +// code is 1. +func TestRulesTestJSONEvaluatorFailureClass(t *testing.T) { + // A rule directory whose expression indexes an empty string out of range + // triggers a runtime CEL evaluation error, mirroring the pattern used in + // TestEvalFixtureSurfacesRuntimeEvalError. + ruleDir := t.TempDir() + ruleYAML := "id: test.eval_boom\nversion: \"1.0\"\ntitle: eval boom\nseverity: low\nexpr: 'event.event_type == \"command.exec\" && event.command[10] == \"x\"'\n" + if err := os.WriteFile(filepath.Join(ruleDir, "boom.yaml"), []byte(ruleYAML), 0o600); err != nil { + t.Fatal(err) + } + body := `{"schema_version":"0.3.0","event_id":"boom1","source_agent":"claude-code","source_type":"artifact","event_type":"command.exec","command":"hi","confidence":"high","evidence":{"artifact_type":"claude_jsonl","local_path":"/x","line":1}} +` + fixture := writeTempFile(t, "eval_fail.ndjson", body) + out, errb, code := runCLI("rules", "test", "--json", "--no-builtin-rules", "--rules-dir", ruleDir, "--fixture", fixture) + if code != 1 { + t.Fatalf("exit = %d, want 1; stderr=%q, stdout=%q", code, errb, out) + } + events, summary := parseJSONStream(t, out) + if summary.Status != "partial" { + t.Fatalf("summary.status = %q, want partial", summary.Status) + } + if summary.StoppedAt == nil || summary.StoppedAt.Reason != "evaluation_failure" { + t.Fatalf("stopped_at = %+v, want reason=evaluation_failure", summary.StoppedAt) + } + if len(events) == 0 { + t.Fatalf("no events emitted") + } + last := events[len(events)-1] + if last.Status != "evaluation_failure" { + t.Fatalf("last status = %q, want evaluation_failure", last.Status) + } + if last.EventID != "boom1" { + t.Fatalf("last event_id = %q, want boom1", last.EventID) + } + if len(last.EvaluatorErrors) == 0 { + t.Fatalf("expected at least one evaluator_error: %+v", last) + } + var sawBoom bool + for _, e := range last.EvaluatorErrors { + if e.RuleID == "test.eval_boom" && e.Message != "" { + sawBoom = true + } + } + if !sawBoom { + t.Fatalf("expected evaluator_error naming test.eval_boom: %+v", last.EvaluatorErrors) + } +} + +// TestRulesTestJSONCoverageHealthClass: an event whose command exceeds the +// shell parser's bounded coverage (>64 statements) reports coverage.shell_parse +// = "unusable" and completes without a match, distinct from a clean no-match. +// This is the coverage/evaluation-health signal Guardian needs to avoid +// inferring "clean" from a bounded-analysis skip. +func TestRulesTestJSONCoverageHealthClass(t *testing.T) { + // Build a command with 100 chained statements to exceed maxShellCommands=64. + var parts []string + for i := 0; i < 100; i++ { + parts = append(parts, "true") + } + command := strings.Join(parts, "; ") + body := `{"schema_version":"0.3.0","event_id":"cov1","source_agent":"claude-code","source_type":"artifact","event_type":"command.exec","command":` + mustJSONString(command) + `,"confidence":"high","evidence":{"artifact_type":"claude_jsonl","local_path":"/x","line":1}} +` + fixture := writeTempFile(t, "coverage.ndjson", body) + out, errb, code := runCLI("rules", "test", "--json", "--fixture", fixture) + // The event either completes with shell_parse=unusable (bounded analysis) + // or is surfaced as an evaluation_failure. Both are valid direct-evaluator + // results; the test asserts that the machine-readable output distinguishes + // them from a clean no-match — no bare empty stdout with exit 0. + switch code { + case 0: + events, summary := parseJSONStream(t, out) + if summary.Status != "completed" { + t.Fatalf("summary.status = %q, want completed", summary.Status) + } + if len(events) != 1 { + t.Fatalf("events = %d, want 1", len(events)) + } + cov := events[0].Coverage + if cov == nil || cov.ShellParse != "unusable" { + t.Fatalf("coverage.shell_parse = %+v, want unusable", cov) + } + case 1: + events, summary := parseJSONStream(t, out) + if summary.Status != "partial" { + t.Fatalf("summary.status = %q, want partial", summary.Status) + } + if len(events) == 0 { + t.Fatalf("expected at least one event_result; stderr=%q", errb) + } + last := events[len(events)-1] + if last.Status != "evaluation_failure" { + t.Fatalf("last status = %q, want evaluation_failure; got=%+v", last.Status, last) + } + default: + t.Fatalf("unexpected exit = %d; stdout=%q stderr=%q", code, out, errb) + } +} + +// TestRulesTestJSONEnforcementEligibleFlag: a rule declared enforce:true +// against a matching event reports enforcement_eligible=true, distinguishing +// an enforce-eligible finding from an advisory one at the CLI seam. +func TestRulesTestJSONEnforcementEligibleFlag(t *testing.T) { + ruleDir := t.TempDir() + ruleYAML := "id: test.enforce_eligible\nversion: \"1.0\"\ntitle: enforce-eligible test rule\nseverity: low\nenforce: true\nexpr: 'event.event_type == \"file.read\" && event.file_path == \"/tmp/target\"'\n" + if err := os.WriteFile(filepath.Join(ruleDir, "enf.yaml"), []byte(ruleYAML), 0o600); err != nil { + t.Fatal(err) + } + body := `{"schema_version":"0.3.0","event_id":"enf1","source_agent":"claude-code","source_type":"artifact","event_type":"file.read","file_path":"/tmp/target","confidence":"high","evidence":{"artifact_type":"claude_jsonl","local_path":"/x","line":1}} +` + fixture := writeTempFile(t, "enf.ndjson", body) + out, errb, code := runCLI("rules", "test", "--json", "--no-builtin-rules", "--rules-dir", ruleDir, "--fixture", fixture) + if code != 0 { + t.Fatalf("exit = %d, want 0; stderr=%q, stdout=%q", code, errb, out) + } + events, summary := parseJSONStream(t, out) + if len(events) != 1 || events[0].EventID != "enf1" { + t.Fatalf("events = %+v", events) + } + if len(events[0].Findings) != 1 { + t.Fatalf("findings = %+v", events[0].Findings) + } + if !events[0].Findings[0].EnforcementEligible { + t.Fatalf("finding not enforcement-eligible: %+v", events[0].Findings[0]) + } + if summary.EnforceEligible < 1 { + t.Fatalf("summary.enforce_eligible_rules = %d, want >=1", summary.EnforceEligible) + } +} + +// TestRulesTestJSONAssertionOutcome: --expect-none against a positive fixture +// with --json completes evaluation (summary.status=completed) but reports +// assertion_outcome=failed and returns exit 1. This is the "completed +// evaluation with failed assertion" case S02a called out — distinct from +// fixture-processing failure. +func TestRulesTestJSONAssertionOutcome(t *testing.T) { + out, errb, code := runCLI("rules", "test", "--json", "--fixture", "testdata/secrets_fixture.ndjson", "--expect-none") + if code != 1 { + t.Fatalf("exit = %d, want 1; stderr=%q, stdout=%q", code, errb, out) + } + _, summary := parseJSONStream(t, out) + if summary.Status != "completed" { + t.Fatalf("summary.status = %q, want completed (assertion failed after complete evaluation)", summary.Status) + } + if summary.AssertionOutcome != "failed" { + t.Fatalf("assertion_outcome = %q, want failed", summary.AssertionOutcome) + } +} + +// TestRulesTestJSONBackwardCompatible: without --json, existing stdout format +// (rule_idevent_id) is byte-identical to the pre-change behavior. Bound +// to the shipped secrets fixture so a formatting drift shows up immediately. +func TestRulesTestJSONBackwardCompatible(t *testing.T) { + out, _, code := runCLI("rules", "test", "--fixture", "testdata/secrets_fixture.ndjson") + if code != 0 { + t.Fatalf("exit = %d", code) + } + // Legacy tab-separated form: no JSON must leak into stdout when --json is + // absent. + if strings.Contains(out, "\"type\":") || strings.Contains(out, "\"schema_version\":") { + t.Fatalf("legacy stdout contains JSON envelope: %q", out) + } + if !strings.Contains(out, "secrets.agent_read_env\te1") { + t.Fatalf("legacy stdout missing tab-separated match: %q", out) + } +} + +// TestRulesTestJSONRejectsConflictingFlags: --json is documented as +// mutually exclusive with the assertion flag combinations that are already +// rejected. The stray combination check must extend cleanly. +func TestRulesTestJSONHelpMentionsFlag(t *testing.T) { + out, errb, code := runCLI("rules", "test", "--help") + if code != 0 { + t.Fatalf("--help exit = %d", code) + } + helpText := out + errb + if !strings.Contains(helpText, "--json") && !strings.Contains(helpText, "-json") { + t.Fatalf("--help missing --json flag documentation: stdout=%q stderr=%q", out, errb) + } +} + +// mustJSONString marshals s into a JSON string literal, for embedding inside +// hand-written NDJSON fixtures. Escapes quotes, semicolons, and control +// characters so a fixture body remains valid JSON. +func mustJSONString(s string) string { + b, err := json.Marshal(s) + if err != nil { + panic(err) + } + return string(b) +} + +// TestRulesTestJSONMalformedInputValidateKind: a syntactically valid JSON +// line whose event fails model.Event.Validate() (unknown event_type) must +// emit status=malformed_input with error.kind="validate", distinct from +// kind="decode". The distinction lets a downstream consumer route validation +// regressions separately from parse errors. +func TestRulesTestJSONMalformedInputValidateKind(t *testing.T) { + tmp := t.TempDir() + fx := filepath.Join(tmp, "validate.ndjson") + // Structurally valid JSON, but event_type is unknown so Validate() rejects. + body := "{\"schema_version\":\"0.3.0\",\"event_id\":\"v1\",\"source_agent\":\"claude-code\",\"source_type\":\"artifact\",\"event_type\":\"WRONG_TYPE\",\"file_path\":\"/x\",\"confidence\":\"high\",\"evidence\":{\"artifact_type\":\"claude_jsonl\",\"local_path\":\"/x\",\"line\":1}}\n" + if err := os.WriteFile(fx, []byte(body), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + out, errb, code := runCLI("rules", "test", "--json", "--fixture", fx) + if code != 1 { + t.Fatalf("exit = %d, want 1; stderr=%q, stdout=%q", code, errb, out) + } + events, summary := parseJSONStream(t, out) + if len(events) != 1 { + t.Fatalf("want 1 event_result, got %d: %+v", len(events), events) + } + if events[0].Status != "malformed_input" { + t.Fatalf("status = %q, want malformed_input", events[0].Status) + } + if events[0].Error == nil || events[0].Error.Kind != "validate" { + t.Fatalf("error.kind = %+v, want validate", events[0].Error) + } + if summary.Status != "partial" || summary.StoppedAt == nil || summary.StoppedAt.Reason != "malformed_input" { + t.Fatalf("summary = %+v, want partial/malformed_input", summary) + } +} + +// TestRulesTestJSONScanErrorEmitsSummary: when the input scanner fails +// mid-stream (a fixture line exceeds bufio.Scanner's buffer, or an IO error +// occurs), the contract still guarantees exactly one terminal summary object. +// A downstream consumer must be able to distinguish scan failure from a +// truncated pipe or a binary that never wrote anything to stdout. +func TestRulesTestJSONScanErrorEmitsSummary(t *testing.T) { + tmp := t.TempDir() + fx := filepath.Join(tmp, "huge.ndjson") + // A single line larger than bufio.Scanner's 64 KiB default token buffer + // forces a scan error. Nine MiB is comfortably above every reasonable + // bump the implementation might apply. Any well-formed but oversized + // event exercises the same failure path. + huge := strings.Repeat("A", 9*1024*1024) + body := "{\"schema_version\":\"0.3.0\",\"event_id\":\"huge\",\"source_agent\":\"claude-code\",\"source_type\":\"artifact\",\"event_type\":\"file.read\",\"file_path\":\"/x\",\"confidence\":\"high\",\"tags\":[" + mustJSONString(huge) + "],\"evidence\":{\"artifact_type\":\"claude_jsonl\",\"local_path\":\"/x\",\"line\":1}}\n" + if err := os.WriteFile(fx, []byte(body), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + out, errb, code := runCLI("rules", "test", "--json", "--fixture", fx) + if code != 1 { + t.Fatalf("exit = %d, want 1; stdout=%q stderr=%q", code, out, errb) + } + if strings.TrimSpace(out) == "" { + t.Fatalf("stdout is empty; contract requires terminal summary on scan error") + } + events, summary := parseJSONStream(t, out) + if summary.Type != "summary" { + t.Fatalf("missing terminal summary; got %+v %+v", events, summary) + } + if summary.Status != "partial" || summary.StoppedAt == nil { + t.Fatalf("summary = %+v, want partial with stopped_at populated", summary) + } + // At least one event_result should carry error.kind="scan"; the exact + // count depends on whether the first line reached decode before the + // scanner rejected it, but a scan-classified event_result must exist. + sawScan := false + for _, e := range events { + if e.Error != nil && e.Error.Kind == "scan" { + sawScan = true + if e.Status != "malformed_input" { + t.Fatalf("scan event status = %q, want malformed_input", e.Status) + } + } + } + if !sawScan { + t.Fatalf("no event_result with error.kind=scan found in %+v", events) + } +} + +// TestRulesTestJSONSequenceAndCELErrorCoexist asserts that a per-rule CEL +// error and a real sequence-tracker error on the SAME event both surface in +// evaluator_errors. The scenario is deterministic: the sequence rule's +// first-step expression indexes tags[11] (out of bounds), and a direct rule +// indexes tags[10] (also out of bounds). session_id is set so the tracker +// observes the event. The compiled CLI must emit both errors, classify the +// stop as `error.kind: "sequence"` (tracker failure dominates classification), +// mark the summary partial, and exit 1. +func TestRulesTestJSONSequenceAndCELErrorCoexist(t *testing.T) { + tmp := t.TempDir() + rulesDir := filepath.Join(tmp, "rules") + if err := os.MkdirAll(rulesDir, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + brokenRule := `id: both.broken +version: "1.0" +title: broken rule +severity: low +expr: 'event.event_type == "command.exec" && event.tags[10] == "x"' +` + if err := os.WriteFile(filepath.Join(rulesDir, "broken.yaml"), []byte(brokenRule), 0o600); err != nil { + t.Fatalf("write broken.yaml: %v", err) + } + // The sequence tracker evaluates step 1 against every observed event that + // carries a session_id. Indexing tags[11] on an event whose tags slice is + // shorter than 12 elements deterministically errors the tracker. + seqRule := `id: chain.demo +version: "1.0" +title: demo sequence +severity: medium +sequence: + within_events: 8 + steps: + - expr: 'event.event_type == "command.exec" && event.tags[11] == "x"' + - expr: 'event.event_type == "command.exec"' +` + if err := os.WriteFile(filepath.Join(rulesDir, "seq.yaml"), []byte(seqRule), 0o600); err != nil { + t.Fatalf("write seq.yaml: %v", err) + } + fx := filepath.Join(tmp, "both.ndjson") + body := "{\"schema_version\":\"0.3.0\",\"event_id\":\"b1\",\"source_agent\":\"claude-code\",\"source_type\":\"artifact\",\"event_type\":\"command.exec\",\"command\":\"echo hi\",\"session_id\":\"s1\",\"confidence\":\"high\",\"tags\":[],\"evidence\":{\"artifact_type\":\"claude_jsonl\",\"local_path\":\"/x\",\"line\":1}}\n" + if err := os.WriteFile(fx, []byte(body), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + out, errb, code := runCLI("rules", "test", "--json", "--no-builtin-rules", "--rules-dir", rulesDir, "--fixture", fx) + if code != 1 { + t.Fatalf("exit = %d, want 1; stderr=%q, stdout=%q", code, errb, out) + } + events, summary := parseJSONStream(t, out) + if len(events) != 1 { + t.Fatalf("want 1 event_result, got %d", len(events)) + } + ev := events[0] + if ev.Status != "evaluation_failure" { + t.Fatalf("status = %q, want evaluation_failure", ev.Status) + } + if ev.Error == nil || ev.Error.Kind != "sequence" { + t.Fatalf("error.kind = %+v, want \"sequence\" (tracker failure classifies the stop)", ev.Error) + } + sawBroken := false + sawSequence := false + for _, e := range ev.EvaluatorErrors { + switch e.RuleID { + case "both.broken": + sawBroken = true + case "": + sawSequence = true + } + } + if !sawBroken || !sawSequence { + t.Fatalf("want BOTH per-rule (rule_id=both.broken) and sequence (rule_id=\"\") errors; got %+v", ev.EvaluatorErrors) + } + if summary.Status != "partial" { + t.Fatalf("summary.status = %q, want partial", summary.Status) + } +} + +// TestRulesTestJSONCountIdentities pins the precise semantics of the two +// summary counters. events_evaluated is the number of fixture lines that +// reached direct evaluation (including a final line that ended in an +// evaluation failure). matches is the total findings emitted across all +// event_results. matches may exceed events_evaluated when several direct +// rules match one event, and each sequence finding also increments matches +// without adding to events_evaluated. There is no ordering invariant between +// the two counters. +func TestRulesTestJSONCountIdentities(t *testing.T) { + t.Run("two direct rules match one event: matches>events_evaluated", func(t *testing.T) { + tmp := t.TempDir() + rulesDir := filepath.Join(tmp, "rules") + if err := os.MkdirAll(rulesDir, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + for i, name := range []string{"a", "b"} { + r := "id: pair.match_" + name + "\nversion: \"1.0\"\ntitle: pair " + name + "\nseverity: low\nexpr: 'event.event_type == \"command.exec\"'\n" + if err := os.WriteFile(filepath.Join(rulesDir, name+".yaml"), []byte(r), 0o600); err != nil { + t.Fatalf("write %d: %v", i, err) + } + } + fx := filepath.Join(tmp, "one.ndjson") + body := "{\"schema_version\":\"0.3.0\",\"event_id\":\"e1\",\"source_agent\":\"claude-code\",\"source_type\":\"artifact\",\"event_type\":\"command.exec\",\"command\":\"ls\",\"confidence\":\"high\",\"tags\":[],\"evidence\":{\"artifact_type\":\"claude_jsonl\",\"local_path\":\"/x\",\"line\":1}}\n" + if err := os.WriteFile(fx, []byte(body), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + out, errb, code := runCLI("rules", "test", "--json", "--no-builtin-rules", "--rules-dir", rulesDir, "--fixture", fx) + if code != 0 { + t.Fatalf("exit = %d, want 0; stderr=%q", code, errb) + } + events, summary := parseJSONStream(t, out) + if summary.EventsEvaluated != 1 { + t.Fatalf("events_evaluated = %d, want 1", summary.EventsEvaluated) + } + if summary.Matches != 2 { + t.Fatalf("matches = %d, want 2 (two rules match one event)", summary.Matches) + } + if len(events) != 1 || len(events[0].Findings) != 2 { + t.Fatalf("want 1 event_result with 2 findings; got %+v", events) + } + }) + + t.Run("partial final event still counts in events_evaluated", func(t *testing.T) { + tmp := t.TempDir() + rulesDir := filepath.Join(tmp, "rules") + if err := os.MkdirAll(rulesDir, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + matchRule := `id: pair.match +version: "1.0" +title: match rule +severity: low +expr: 'event.event_type == "file.read"' +` + brokenRule := `id: pair.broken +version: "1.0" +title: broken rule +severity: low +expr: 'event.event_type == "file.read" && event.tags[10] == "x"' +` + if err := os.WriteFile(filepath.Join(rulesDir, "m.yaml"), []byte(matchRule), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + if err := os.WriteFile(filepath.Join(rulesDir, "b.yaml"), []byte(brokenRule), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + fx := filepath.Join(tmp, "pair.ndjson") + body := "{\"schema_version\":\"0.3.0\",\"event_id\":\"p1\",\"source_agent\":\"claude-code\",\"source_type\":\"artifact\",\"event_type\":\"file.read\",\"file_path\":\"/x\",\"confidence\":\"high\",\"tags\":[],\"evidence\":{\"artifact_type\":\"claude_jsonl\",\"local_path\":\"/x\",\"line\":1}}\n" + if err := os.WriteFile(fx, []byte(body), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + out, errb, code := runCLI("rules", "test", "--json", "--no-builtin-rules", "--rules-dir", rulesDir, "--fixture", fx) + if code != 1 { + t.Fatalf("exit = %d, want 1; stderr=%q", code, errb) + } + _, summary := parseJSONStream(t, out) + if summary.EventsEvaluated != 1 { + t.Fatalf("events_evaluated = %d, want 1 (line reached evaluation)", summary.EventsEvaluated) + } + if summary.Matches != 1 { + t.Fatalf("matches = %d, want 1 (pair.match still fires)", summary.Matches) + } + if summary.Status != "partial" { + t.Fatalf("status = %q, want partial", summary.Status) + } + }) + + t.Run("sequence findings add to matches without adding events_evaluated", func(t *testing.T) { + tmp := t.TempDir() + rulesDir := filepath.Join(tmp, "rules") + if err := os.MkdirAll(rulesDir, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + // Two-step sequence that fires on the second event in the same session. + seqRule := `id: chain.pair +version: "1.0" +title: two-step chain +severity: medium +sequence: + within_events: 8 + steps: + - expr: 'event.event_type == "command.exec"' + - expr: 'event.event_type == "command.exec"' +` + if err := os.WriteFile(filepath.Join(rulesDir, "seq.yaml"), []byte(seqRule), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + fx := filepath.Join(tmp, "chain.ndjson") + body := "{\"schema_version\":\"0.3.0\",\"event_id\":\"e1\",\"source_agent\":\"claude-code\",\"source_type\":\"artifact\",\"event_type\":\"command.exec\",\"command\":\"ls\",\"session_id\":\"s1\",\"confidence\":\"high\",\"tags\":[],\"evidence\":{\"artifact_type\":\"claude_jsonl\",\"local_path\":\"/x\",\"line\":1}}\n" + + "{\"schema_version\":\"0.3.0\",\"event_id\":\"e2\",\"source_agent\":\"claude-code\",\"source_type\":\"artifact\",\"event_type\":\"command.exec\",\"command\":\"pwd\",\"session_id\":\"s1\",\"confidence\":\"high\",\"tags\":[],\"evidence\":{\"artifact_type\":\"claude_jsonl\",\"local_path\":\"/x\",\"line\":2}}\n" + if err := os.WriteFile(fx, []byte(body), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + out, errb, code := runCLI("rules", "test", "--json", "--no-builtin-rules", "--rules-dir", rulesDir, "--fixture", fx) + if code != 0 { + t.Fatalf("exit = %d, want 0; stderr=%q", code, errb) + } + _, summary := parseJSONStream(t, out) + if summary.EventsEvaluated != 2 { + t.Fatalf("events_evaluated = %d, want 2", summary.EventsEvaluated) + } + if summary.Matches < 1 { + t.Fatalf("matches = %d, want >= 1 (sequence should fire on e2)", summary.Matches) + } + }) +} + +// failingWriter returns errFailingWriter after the first `okBytes` bytes; used +// to synthesize the OS-level partial-write / EPIPE / EBADF class without +// depending on /dev/full being addressable inside the sandbox. +type failingWriter struct { + okBytes int + written int +} + +var errFailingWriter = errors.New("failingWriter: synthetic stdout failure") + +func (f *failingWriter) Write(p []byte) (int, error) { + remaining := f.okBytes - f.written + if remaining <= 0 { + return 0, errFailingWriter + } + if len(p) <= remaining { + f.written += len(p) + return len(p), nil + } + f.written += remaining + return remaining, errFailingWriter +} + +// runRulesTestJSONForTest builds a compiled catalog identical to the CLI +// (--no-builtin-rules with a directory of operator rules) and drives +// runRulesTestJSON directly against a custom stdout writer. Used for the +// delivery-failure checks that need to observe encoder behavior on a broken +// writer without spawning a subprocess. +func runRulesTestJSONForTest(t *testing.T, rulesDir, fixturePath string, stdout, stderr io.Writer) int { + t.Helper() + eng, err := buildEngine([]string{rulesDir}, true) + if err != nil { + t.Fatalf("load rules: %v", err) + } + f, err := os.Open(fixturePath) + if err != nil { + t.Fatalf("open fixture: %v", err) + } + defer f.Close() + return runRulesTestJSON(eng, f, stdout, stderr, false, false, nil) +} + +// TestRulesTestJSONDeliveryFailureOnEventResult asserts that a stdout write +// failure while emitting the FIRST event_result returns a non-zero exit code +// (rulesTestJSONDeliveryExitCode), reports the error on stderr, and does not +// silently succeed. This closes the class where a Guardian-side reader sees +// zero events and mis-infers a clean no-match. +func TestRulesTestJSONDeliveryFailureOnEventResult(t *testing.T) { + tmp := t.TempDir() + rulesDir := filepath.Join(tmp, "rules") + if err := os.MkdirAll(rulesDir, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + r := `id: deliver.match +version: "1.0" +title: deliver match +severity: low +expr: 'event.event_type == "command.exec"' +` + if err := os.WriteFile(filepath.Join(rulesDir, "r.yaml"), []byte(r), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + fx := filepath.Join(tmp, "one.ndjson") + body := "{\"schema_version\":\"0.3.0\",\"event_id\":\"e1\",\"source_agent\":\"claude-code\",\"source_type\":\"artifact\",\"event_type\":\"command.exec\",\"command\":\"ls\",\"confidence\":\"high\",\"tags\":[],\"evidence\":{\"artifact_type\":\"claude_jsonl\",\"local_path\":\"/x\",\"line\":1}}\n" + if err := os.WriteFile(fx, []byte(body), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + stdout := &failingWriter{okBytes: 0} + var stderr strings.Builder + code := runRulesTestJSONForTest(t, rulesDir, fx, stdout, &stderr) + if code != rulesTestJSONDeliveryExitCode { + t.Fatalf("exit = %d, want %d (delivery failure); stderr=%q", code, rulesTestJSONDeliveryExitCode, stderr.String()) + } + if !strings.Contains(stderr.String(), "write rules-test-result") { + t.Fatalf("stderr missing write-failure notice: %q", stderr.String()) + } +} + +// TestRulesTestJSONDeliveryFailureOnSummaryOnly asserts that when the +// event_result writes succeed but the terminal summary write fails, the exit +// code still reflects delivery failure. A consumer must not treat a run whose +// summary was truncated as a successful run. +func TestRulesTestJSONDeliveryFailureOnSummaryOnly(t *testing.T) { + tmp := t.TempDir() + rulesDir := filepath.Join(tmp, "rules") + if err := os.MkdirAll(rulesDir, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + r := `id: summary.match +version: "1.0" +title: summary match +severity: low +expr: 'event.event_type == "command.exec"' +` + if err := os.WriteFile(filepath.Join(rulesDir, "r.yaml"), []byte(r), 0o600); err != nil { + t.Fatalf("write: %v", err) + } + fx := filepath.Join(tmp, "one.ndjson") + body := "{\"schema_version\":\"0.3.0\",\"event_id\":\"e1\",\"source_agent\":\"claude-code\",\"source_type\":\"artifact\",\"event_type\":\"command.exec\",\"command\":\"ls\",\"confidence\":\"high\",\"tags\":[],\"evidence\":{\"artifact_type\":\"claude_jsonl\",\"local_path\":\"/x\",\"line\":1}}\n" + if err := os.WriteFile(fx, []byte(body), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + // Determine how many bytes the first event_result will occupy, then let + // the writer accept exactly those bytes so only the summary write fails. + var probe strings.Builder + probeCode := runRulesTestJSONForTest(t, rulesDir, fx, &probe, io.Discard) + if probeCode != 0 { + t.Fatalf("probe exit = %d, want 0; stdout=%q", probeCode, probe.String()) + } + nl := strings.IndexByte(probe.String(), '\n') + if nl <= 0 { + t.Fatalf("no newline in probe output: %q", probe.String()) + } + firstLineBytes := nl + 1 + stdout := &failingWriter{okBytes: firstLineBytes} + var stderr strings.Builder + code := runRulesTestJSONForTest(t, rulesDir, fx, stdout, &stderr) + if code != rulesTestJSONDeliveryExitCode { + t.Fatalf("exit = %d, want %d (summary delivery failure); stderr=%q", code, rulesTestJSONDeliveryExitCode, stderr.String()) + } + if !strings.Contains(stderr.String(), "write rules-test-result") { + t.Fatalf("stderr missing write-failure notice: %q", stderr.String()) + } +} + +// TestRulesTestJSONCompiledCLIDevFull spawns the compiled numbat binary with +// stdout redirected to /dev/full so every OS-level write returns ENOSPC, and +// asserts the JSON mode reports a nonzero exit code, matching the legacy +// tab-separated mode's behavior on the same failure. Skipped when /dev/full is +// not present (non-Linux hosts or minimal sandboxes without the char device). +func TestRulesTestJSONCompiledCLIDevFull(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("compiled-CLI OS write-failure check requires linux /dev/full") + } + if _, err := os.Stat("/dev/full"); err != nil { + t.Skipf("/dev/full unavailable: %v", err) + } + // Build the binary into a temp path so the test does not depend on a + // preinstalled binary or PATH shape. + bin := filepath.Join(t.TempDir(), "numbat") + buildCmd := exec.Command("go", "build", "-o", bin, "./") + buildCmd.Dir = "." + if out, err := buildCmd.CombinedOutput(); err != nil { + t.Fatalf("build numbat: %v\n%s", err, out) + } + fx := filepath.Join(t.TempDir(), "fx.ndjson") + body := "{\"schema_version\":\"0.3.0\",\"event_id\":\"e1\",\"source_agent\":\"claude-code\",\"source_type\":\"artifact\",\"event_type\":\"command.exec\",\"command\":\"ls\",\"confidence\":\"high\",\"tags\":[],\"evidence\":{\"artifact_type\":\"claude_jsonl\",\"local_path\":\"/x\",\"line\":1}}\n" + if err := os.WriteFile(fx, []byte(body), 0o600); err != nil { + t.Fatalf("write fixture: %v", err) + } + devFull, err := os.OpenFile("/dev/full", os.O_WRONLY, 0) + if err != nil { + t.Skipf("open /dev/full: %v", err) + } + defer devFull.Close() + cmd := exec.Command(bin, "rules", "test", "--json", "--fixture", fx) + cmd.Stdout = devFull + var stderr strings.Builder + cmd.Stderr = &stderr + err = cmd.Run() + if err == nil { + t.Fatalf("compiled --json rules test exited 0 with stdout=/dev/full; want nonzero exit. stderr=%q", stderr.String()) + } + exitErr, ok := err.(*exec.ExitError) + if !ok { + t.Fatalf("unexpected error type %T: %v", err, err) + } + if exitErr.ExitCode() == 0 { + t.Fatalf("exit code = 0, want nonzero; stderr=%q", stderr.String()) + } + if !strings.Contains(stderr.String(), "write") { + t.Fatalf("stderr missing write-failure notice: %q", stderr.String()) + } +} diff --git a/docs/cli.md b/docs/cli.md index f2f68cd..a35c1e0 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -759,6 +759,10 @@ violations, and rule-evaluation errors report the fixture line number. `check`, ``` --fixture FILE NDJSON events file to evaluate (required) +--json emit a machine-readable NDJSON result stream + (schema rules-test-result.v1) instead of + tab-separated matches; see + docs/schema/rules-test-result.v1.md --require-match exit non-zero if no rule matches (for positive fixtures) --expect RULE_ID exit non-zero if this rule id does not match at least once (repeatable) @@ -774,8 +778,15 @@ numbat rules list numbat rules test --fixture events.ndjson --require-match numbat rules test --fixture positive.ndjson --expect secrets.agent_read_env numbat rules test --fixture negative.ndjson --expect-none +numbat rules test --json --fixture events.ndjson ``` +The `--json` mode emits one `event_result` object per fixture line and one +terminal `summary` object, distinguishing findings, enforcement eligibility, +shell-parse coverage, malformed input, and evaluator failure. It is intended +for downstream consumers that need a stable, versioned direct-evaluation +result (see [`docs/schema/rules-test-result.v1.md`](schema/rules-test-result.v1.md)). + ## case bundles `case build` curates captured record streams into a portable `case.numbat` diff --git a/docs/schema/rules-test-result.v1.md b/docs/schema/rules-test-result.v1.md new file mode 100644 index 0000000..4979f0b --- /dev/null +++ b/docs/schema/rules-test-result.v1.md @@ -0,0 +1,191 @@ +# rules-test-result.v1 + +`numbat rules test --json` emits an NDJSON result stream on stdout. This +document is the authoritative machine-readable contract; every consumer must +treat unknown fields as reserved. The schema versions independently of the +record wire schema (`model.SchemaVersion`) because it describes a CLI-adjacent +direct-evaluator surface, not a record shape emitted by the pipeline. + +- Every line is one JSON object. +- One `event_result` object per fixture line that reached evaluation. +- Exactly one terminal `summary` object. +- Stdout carries only these objects; logs, human-facing messages, and errors + go to stderr. + +Exit codes: + +- `0`: `summary.status == "completed"` with a passed or unchecked assertion, + delivered as a well-formed NDJSON stream terminated by exactly one + `summary` object. +- `1`: `summary.status == "partial"` OR a failed assertion, delivered as a + well-formed NDJSON stream terminated by exactly one `summary` object. + This is a successfully delivered result, not a delivery failure. +- `2`: one of two cases that share this exit code and are distinguished + by looking at stdout and stderr: + - Handled JSON delivery failure. A stdout write for an `event_result` + or the terminal `summary` returned an OS write error (ENOSPC, + read-only descriptor); the process caught it, wrote a + `write rules-test-result:` diagnostic to stderr, and short-circuited + the remainder of the stream. Any partial stream on stdout is not + terminated by a `summary`. + - Usage or setup failure raised before any stream is produced. For + example, `rules test --json` without `--fixture` prints a usage + diagnostic to stderr and exits 2 with empty stdout. + +Exit code alone is not sufficient to identify delivery failure. A +usage/setup failure exits 2 before any stream, and abnormal process +termination (SIGPIPE on a real broken pipe, SIGKILL, panic, host death) +may end the process before the handled exit-2 path runs and before a +terminal `summary` is written - the observed exit status in that case is +signal-encoded (for example a Python subprocess returncode of `-13` for +SIGPIPE), not `2`. + +Consumer rule: only a well-formed NDJSON stream terminated by exactly one +`summary` object AND an exit status compatible with that summary +(`0`/`1`/`2` as described above) proves the documented outcome. A missing +terminal `summary`, a malformed or truncated final JSON line, or an +abnormal/signal-encoded termination cannot prove a clean no-match and +must be treated as an indeterminate delivery outcome - not silently +reclassified as delivery failure, and not silently accepted as success. +Consumers should read the summary object rather than infer completeness +from the exit code alone. + +## event_result + +``` +{ + "type": "event_result", + "schema_version": "rules-test-result.v1", + "fixture_line": 1, + "event_id": "e1", + "status": "completed", + "findings": [ + { + "rule_id": "secrets.agent_read_env", + "rule_version": "1.0", + "severity": "high", + "enforcement_eligible": true, + "via": "engine" + } + ], + "coverage": {"shell_parse": "ok", "sequence_tracker_active": false} +} +``` + +A `status == "completed"` event carries no `evaluator_errors` and no +`error`; both fields are omitted from the wire when empty. Any per-rule CEL +failure or sequence-tracker error promotes the event to +`status == "evaluation_failure"`, at which point those errors appear in +`evaluator_errors` and `error` is populated with the failing `kind`. + +- `fixture_line` (int, required): 1-based line number in the fixture. Blank + lines are skipped and do not receive an `event_result`. +- `event_id` (string, optional): copied from the decoded event; absent when + decoding failed before an id was known. +- `status` (string enum, required): one of + - `completed`: the event was evaluated and any matches appear in + `findings`. `findings: []` (or the field omitted) is the only legitimate + representation of a clean no-match. A downstream consumer must not + infer "no match" from a missing `event_result`. + - `malformed_input`: the fixture line failed JSON decode or event + validation, or the input stream itself could not be scanned (line too + long, IO error). `error.kind` is `decode`, `validate`, or `scan`. + Fixture processing stops at this event; the summary reports + `status: partial`. A `scan` failure attributes the failing + `fixture_line` to the next unread line and carries no `event_id`. + - `evaluation_failure`: at least one rule's CEL program errored at + runtime, or the sequence tracker returned an error. Matches from other + rules on the same event still appear in `findings`; every failing rule + still appears in `evaluator_errors`, and a per-rule CEL failure and a + sequence-tracker failure may both surface for the same event. + `error.kind` names the failure that stopped the fixture (`sequence` + when the tracker failed, otherwise `evaluation`). Fixture processing + stops. +- `findings` (array, optional): one entry per matching rule. + - `rule_id` (string, required) + - `rule_version` (string, required) + - `severity` (string, optional): the compiled rule's declared severity. + - `enforcement_eligible` (bool, required): mirrors the flag that gates + the live enforce path (accounting for shell-enforcement-safety). A + finding with `enforcement_eligible: false` is advisory even though the + rule declares `enforce: true`. + - `via` (string enum, required): `engine` for a single-event evaluation, + `sequence` for a completed sequence chain. Sequence findings cite the + event that terminated the chain. +- `evaluator_errors` (array, optional): named rule failures for the event. + - `rule_id` (string, optional): missing when the failure was not + attributable to one rule (for example, a sequence tracker error). + - `message` (string, required): raw error text; treat as diagnostic only. +- `coverage` (object, optional): + - `shell_parse` (string enum, required): `ok`, `degraded`, or `unusable`. + `unusable` means rules that read `shell_commands` were skipped for the + event; a consumer must not infer a clean no-match in that case. + - `sequence_tracker_active` (bool, required): true when the compiled + catalog contains at least one sequence rule (i.e. a window tracker + exists for this run). The tracker is only asked to observe events that + carry a `session_id`; a `true` value does not by itself imply this + event was folded into a window. + +- `error` (object, optional): populated when `status` is not `completed`. + - `kind` (string enum, required): `decode`, `validate`, `scan`, + `evaluation`, or `sequence`. + - `message` (string, required): includes the fixture line for context. + +## summary + +``` +{ + "type": "summary", + "schema_version": "rules-test-result.v1", + "status": "completed", + "events_evaluated": 3, + "matches": 2, + "rules_loaded": 51, + "enforce_eligible_rules": 24, + "assertion_outcome": "unchecked", + "assertion_missing": null, + "stopped_at": null, + "numbat_version": "dev+abc123", + "record_schema": "0.3.0" +} +``` + +- `status` (string enum, required): `completed` if every fixture line was + processed, `partial` if the stream stopped early. A completed run with a + failed assertion is still `completed`; see `assertion_outcome`. +- `events_evaluated` (int, required): number of fixture lines that + reached direct evaluation, including any final line that ended in + `evaluation_failure`. Lines that failed decode/validate/scan before + evaluation are excluded. +- `matches` (int, required): total findings emitted across all + `event_results` (single-event and sequence combined). `matches` may + exceed `events_evaluated`: several direct rules can match one event, and + each sequence finding adds to `matches` without adding to + `events_evaluated`. Consumers must not assert `matches <= + events_evaluated`. +- `rules_loaded` (int, required): count of compiled rules in the effective + catalog. +- `enforce_eligible_rules` (int, required): count of compiled rules that + may block in live enforce mode. Independent of whether any matched. +- `assertion_outcome` (string enum, required): `passed`, `failed`, or + `unchecked`. A `partial` run always reports `unchecked` so that + fixture-processing failures are not conflated with assertion failures. +- `assertion_missing` (array, optional): rule ids passed via `--expect` that + produced no matches when `assertion_outcome == "failed"`. +- `stopped_at` (object, optional): populated when `status == "partial"` + and only then; every `partial` run carries it. + - `fixture_line` (int, required) + - `reason` (string enum, required): `malformed_input` or + `evaluation_failure`; mirrors the last event_result's status. +- `numbat_version` (string, required): the running binary's version string + (as `numbat version` would report). +- `record_schema` (string, required): the current event/finding wire schema + version (`model.SchemaVersion`). Independent of `schema_version` above. + +## Compatibility + +A future `rules-test-result.v2` will change `schema_version`. Additive +fields inside a version (new optional keys, new enum values in a documented +enum extension) do not bump the major version; consumers must ignore unknown +keys and treat unknown enum values as an unrecognized class rather than +`completed`. diff --git a/internal/archguard/archguard_test.go b/internal/archguard/archguard_test.go index 164ae5c..3bb758d 100644 --- a/internal/archguard/archguard_test.go +++ b/internal/archguard/archguard_test.go @@ -208,7 +208,7 @@ func classifyCmdFile(name string) (forbidden []string, ok bool) { name == "ship.go" || strings.HasPrefix(name, "ship_"): return forbidForensics, true case name == "content.go" || name == "main.go" || name == "rules.go" || name == "rules_companion.go" || - name == "run_id.go" || name == "sink.go" || name == "version.go": + name == "rules_json.go" || name == "run_id.go" || name == "sink.go" || name == "version.go": return nil, true case name == "agents.go": return nil, true diff --git a/internal/rule/engine.go b/internal/rule/engine.go index a3b08a7..d95a86d 100644 --- a/internal/rule/engine.go +++ b/internal/rule/engine.go @@ -625,6 +625,20 @@ func (e *Engine) HasEnforceEligibleRules() bool { return false } +// CountEnforceEligibleRules returns the number of enabled compiled rules +// that may block in live hook enforce mode. It is the count companion of +// HasEnforceEligibleRules and is used by machine-readable surfaces that +// need to report catalog shape (for example, `rules test --json`). +func (e *Engine) CountEnforceEligibleRules() int { + n := 0 + for _, c := range e.rules { + if c.rule.IsEnforceEligible() { + n++ + } + } + return n +} + // SequenceRules returns the compiled sequence rules in load order, for a // window tracker to evaluate. Empty when the load contains none. func (e *Engine) SequenceRules() []*SequenceRule { @@ -646,6 +660,70 @@ func (e *Engine) RuleIDs() []string { return ids } +// EvalError names an individual rule whose CEL program failed at runtime for +// one event. It is emitted by EvalDetailed so consumers can attribute a +// failure to a specific rule without parsing a joined error string. +type EvalError struct { + RuleID string + Message string +} + +// EvalDiagnostics carries the non-per-rule signals produced while evaluating +// one event: the shell-analysis error (if any) and its usability flags. It +// lets callers distinguish a bounded, unusable shell parse from a clean +// no-match without inspecting internal state. +type EvalDiagnostics struct { + ShellParseError error + ShellUsable bool + ShellEnforcementSafe bool +} + +// EvalDetailed is the machine-readable companion of Eval. It returns the +// matches, the per-rule evaluator errors, and the shared shell-analysis +// diagnostics for one event. A per-rule failure does not suppress matches +// from other rules. Callers that only want the joined error text should +// keep using Eval; EvalDetailed is intended for surfaces that expose the +// distinction between "clean no-match", "evaluator failure", and +// "bounded/unusable coverage" as separate result classes (for example, the +// `rules test --json` result contract). +func (e *Engine) EvalDetailed(ev model.Event) ([]Match, []EvalError, EvalDiagnostics) { + activations := prepareActivations(e.env.CELTypeAdapter(), ev, e.usesShellCommands) + diag := EvalDiagnostics{ + ShellParseError: activations.err, + ShellUsable: activations.shellUsable, + ShellEnforcementSafe: activations.shellEnforcementSafe, + } + var ( + matches []Match + errs []EvalError + ) + for _, c := range e.rules { + if c.seq != nil { + continue + } + if activations.err != nil && c.program.usesShellCommands && !activations.shellUsable { + continue + } + out, _, err := c.program.program.Eval(activations.detection) + if err != nil { + errs = append(errs, EvalError{RuleID: c.rule.ID, Message: err.Error()}) + continue + } + if asBool(out) { + enforcementMatch := c.rule.IsEnforceEligible() + if enforcementMatch && c.program.usesShellCommands && !activations.shellEnforcementSafe { + enforcementMatch = false + } + matches = append(matches, Match{ + Rule: cloneRule(c.rule), + Event: ev, + EnforcementMatch: enforcementMatch, + }) + } + } + return matches, errs, diag +} + // Eval runs every compiled single-event rule against one event and returns // the matches in rule load order. Sequence rules are skipped — they match // chains, not events, and are evaluated by the window tracker that holds