diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5d53338..5eb7858f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -303,7 +303,7 @@ jobs: name: Published compose (${{ matrix.pack }}) needs: demo-packs runs-on: ubuntu-26.04 - timeout-minutes: 50 + timeout-minutes: 65 strategy: fail-fast: false matrix: @@ -350,35 +350,6 @@ jobs: docker compose -f "$FILE" up -d --wait --wait-timeout 1200 curl -fsSL --retry 5 --retry-all-errors -o /dev/null -w 'console answered %{http_code} at %{url_effective}\n' http://localhost:3000/ - # Every source with events the pack replays, which need no egress, so a pack whose demo.json lists no sources checks none. Fetcher tables wait on their upstream and are not checked - - name: Land the pack's data - timeout-minutes: 10 - working-directory: ${{ env.COMPOSE_DIR }} - run: | - tables=() - for source in $(jq -r '.sources[]' "$GITHUB_WORKSPACE/$PACK_DIR/demo.json"); do - if [ -d "$GITHUB_WORKSPACE/$PACK_DIR/events/$source" ]; then - tables+=("$source") - fi - done - if [ "${#tables[@]}" -eq 0 ]; then - echo "$PACK_DIR replays no source, so no table is checked" - fi - for table in "${tables[@]}"; do - for attempt in $(seq 60); do - rows="$(docker compose -f "$FILE" exec -T clickhouse sh -c ". /run/dfe-secrets/env && clickhouse-client --password \"\$CLICKHOUSE_PASSWORD\" --query 'SELECT count() FROM dfe.\`$table\`'" || echo 0)" - if [ "${rows:-0}" -gt 0 ]; then - echo "dfe.$table holds $rows rows" - break - fi - if [ "$attempt" -eq 60 ]; then - echo "::error::no rows reached dfe.$table" - exit 1 - fi - sleep 5 - done - done - # The secrets are this runner's own: no service holds one in its environment, ClickHouse refuses its empty default password, and the login the file prints is held for a change - name: Run on secrets minted here working-directory: ${{ env.COMPOSE_DIR }} @@ -413,6 +384,14 @@ jobs: with urllib.request.urlopen(request, timeout=30) as answer: assert json.load(answer).get("password_change_required") is True, "the first login is not held for a change"' + # Runs after the secrets step because the console login check replaces the login's issued password, which that step expects held. One checker proves the pack worked: its replayed sources land parsed rows, the intake and default tables stay empty, every source is deployed, each hunt fires under its ceiling, each dashboard and saved search returns rows, the console login can search, replayed fetcher polls land and the dead-letter queue stays empty. What a pack expects is data in resources//verify.json + - name: Verify the pack worked + timeout-minutes: 35 + working-directory: ${{ env.COMPOSE_DIR }} + env: + PACK: ${{ matrix.pack }} + run: python3 "$GITHUB_WORKSPACE/scripts/demo_verify.py" --pack "$PACK" --file "$FILE" + - name: Show what did not come up if: failure() working-directory: ${{ env.COMPOSE_DIR }} diff --git a/docs/evaluating.md b/docs/evaluating.md index 8d5e069b..2ed43b26 100644 --- a/docs/evaluating.md +++ b/docs/evaluating.md @@ -173,23 +173,23 @@ The demo is one set of pieces, listed in [`resources/demo/demo.json`](../resourc | Piece | Where | What it is | |-------|-------|------------| | Sources | [`resources/demo/sources/`](../resources/demo/sources/) | One `POST /sources` body per file: the `demo-syslog` intake, the `cisco-asa` and `fortinet` tables, and two fetchers | -| Fetchers | `wikipedia-recentchanges`, `component-releases` | Pull beside the receiver's push. `wikipedia-recentchanges` polls the MediaWiki recent changes API every 5 seconds, asking each window once, with the first poll bringing in the last hour. `component-releases` takes the latest GitHub release of each public DFE component every 15 minutes, to stay inside GitHub's 60 unauthenticated requests an hour | +| Fetchers | `feodo-c2`, `component-releases` | Pull beside the receiver's push. `feodo-c2` takes the abuse.ch Feodo Tracker botnet command-and-control blocklist every 5 minutes, as one snapshot a poll, and until the fetcher and engine land a dump unit in its own table its rows land in `dfe.main`. `component-releases` takes the latest GitHub release of each public DFE component every 15 minutes, to stay inside GitHub's 60 unauthenticated requests an hour | | Archive | `demo-syslog` has `archive: true` | On a Kafka profile dfe-archiver writes the raw syslog lines to the `archiver-data` volume (`file:///var/lib/dfe-archiver/archive`). dfe-archiver has no retention of its own, so the pack runs a pruner that keeps the archive to 15 minutes and 200 MB. A gRPC profile runs no archiver, so there the Source is created without archiving | | Console | [`resources/demo/console/`](../resources/demo/console/) | A dashboard per source and one for the stack's health, and saved searches with a live tail for each streamed source. Every name starts `demo-` | -| Tables | `dfe.cisco-asa`, `dfe.fortinet`, `dfe.demo-syslog`, `dfe.component-releases`, `dfe.wikipedia-recentchanges` | `make demo-stop` leaves them, with their rows, because only the engine issues ClickHouse DDL and it never drops a table. The next `make demo` goes on in them, and `make clean` removes them with the volumes | +| Tables | `dfe.cisco-asa`, `dfe.fortinet`, `dfe.demo-syslog`, `dfe.component-releases`, `dfe.feodo-c2` | `make demo-stop` leaves them, with their rows, because only the engine issues ClickHouse DDL and it never drops a table. The next `make demo` goes on in them, and `make clean` removes them with the volumes | Each device loops over the sample lines in [`resources/demo/events//`](../resources/demo/events/), which `python3 resources/demo/events/generate.py` regenerates. The run turns on dfe-receiver's syslog listener (UDP and TCP on port 514) and recreates what that changes. It returns once every device table has gained rows and the archive holds a file, and leaves the stream running: ```console demo - console demo-cisco-asa: http://localhost:8090/dashboards/ -demo - console demo-wikipedia-live: http://localhost:8090/search/?isLive=true +demo - console demo-feodo-c2-all: http://localhost:8090/search/ demo - dfe.cisco-asa: +N rows demo - dfe.fortinet: +N rows demo - dfe.demo-syslog: +0 rows demo - dfe.main: +0 rows demo - dfe.component-releases: 0 rows -demo - dfe.wikipedia-recentchanges: N rows +demo - dfe.feodo-c2: 0 rows demo - dfe-archiver archive of demo-syslog: N files demo - demo-syslog_land/default/2026/11/03/01-0001-.jsonl.zst 77645 bytes demo - first line of the newest: {"message":"%ASA-6-302014: Teardown TCP connection ... diff --git a/resources/demo-cyber/demo.json b/resources/demo-cyber/demo.json index 6473eb3e..6039e975 100644 --- a/resources/demo-cyber/demo.json +++ b/resources/demo-cyber/demo.json @@ -26,7 +26,9 @@ ] }, "hunts": [ - "demo-cyber-drop-sign-in" + "demo-cyber-anonymous-link", + "demo-cyber-drop-sign-in", + "demo-cyber-ransomware" ], "sources": [ "cisco-asa", diff --git a/resources/demo-cyber/hunts/demo-cyber-anonymous-link.yaml b/resources/demo-cyber/hunts/demo-cyber-anonymous-link.yaml new file mode 100644 index 00000000..67dacdc7 --- /dev/null +++ b/resources/demo-cyber/hunts/demo-cyber-anonymous-link.yaml @@ -0,0 +1,19 @@ +# Project: dfe-docker +# File: resources/demo-cyber/hunts/demo-cyber-anonymous-link.yaml +# Purpose: The cyber demo's hunt, running its anonymous-link rule every five minutes +# Language: YAML +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +name: demo-cyber-anonymous-link +display_name: Anonymous links used on Microsoft 365 files +cron: "*/5 * * * *" +# A required label: the runner takes each detection's org from the matched row. +customers: + - default +rules: + - demo-cyber-anonymous-link +global_source_table_name: dfe.o365 +global_target_table_name: dfe.detection +checkpoint_timestamp_field: _timestamp_load diff --git a/resources/demo-cyber/hunts/demo-cyber-ransomware.yaml b/resources/demo-cyber/hunts/demo-cyber-ransomware.yaml new file mode 100644 index 00000000..107c5ddc --- /dev/null +++ b/resources/demo-cyber/hunts/demo-cyber-ransomware.yaml @@ -0,0 +1,19 @@ +# Project: dfe-docker +# File: resources/demo-cyber/hunts/demo-cyber-ransomware.yaml +# Purpose: The cyber demo's hunt, running its ransomware rule every five minutes +# Language: YAML +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +name: demo-cyber-ransomware +display_name: Ransomware behaviour on endpoints +cron: "*/5 * * * *" +# A required label: the runner takes each detection's org from the matched row. +customers: + - default +rules: + - demo-cyber-ransomware +global_source_table_name: dfe.crowdstrike +global_target_table_name: dfe.detection +checkpoint_timestamp_field: _timestamp_load diff --git a/resources/demo-cyber/rules/demo-cyber-anonymous-link.yaml b/resources/demo-cyber/rules/demo-cyber-anonymous-link.yaml new file mode 100644 index 00000000..49ac814f --- /dev/null +++ b/resources/demo-cyber/rules/demo-cyber-anonymous-link.yaml @@ -0,0 +1,14 @@ +# Project: dfe-docker +# File: resources/demo-cyber/rules/demo-cyber-anonymous-link.yaml +# Purpose: The cyber demo's rule: an anonymous sharing link used to open a Microsoft 365 file +# Language: YAML +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +name: demo-cyber-anonymous-link +display_name: Anonymous link used on a Microsoft 365 file +severity: medium +source: o365 +source_type: raw +user_sql: "SELECT * FROM dfe.`o365` WHERE event_action = 'AnonymousLinkUsed'" diff --git a/resources/demo-cyber/rules/demo-cyber-ransomware.yaml b/resources/demo-cyber/rules/demo-cyber-ransomware.yaml new file mode 100644 index 00000000..0d221991 --- /dev/null +++ b/resources/demo-cyber/rules/demo-cyber-ransomware.yaml @@ -0,0 +1,14 @@ +# Project: dfe-docker +# File: resources/demo-cyber/rules/demo-cyber-ransomware.yaml +# Purpose: The cyber demo's rule: a high-severity CrowdStrike Falcon detection of ransomware behaviour +# Language: YAML +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +name: demo-cyber-ransomware +display_name: Ransomware behaviour on an endpoint +severity: critical +source: crowdstrike +source_type: raw +user_sql: "SELECT * FROM dfe.`crowdstrike` WHERE rule_name = 'Ransomware Behaviour' AND event_severity >= 70" diff --git a/resources/demo-cyber/verify.json b/resources/demo-cyber/verify.json new file mode 100644 index 00000000..a5547a67 --- /dev/null +++ b/resources/demo-cyber/verify.json @@ -0,0 +1,63 @@ +{ + "hunts": { + "demo-cyber-anonymous-link": { + "max_alerts": 126 + }, + "demo-cyber-drop-sign-in": { + "max_alerts": 126 + }, + "demo-cyber-ransomware": { + "max_alerts": 126 + } + }, + "parsed": { + "cisco-asa": { + "fields": [ + "event_action", + "source_ip", + "destination_ip", + "observer_vendor" + ], + "min_ratio": 0.9 + }, + "crowdstrike": { + "fields": [ + "event_action", + "user_name", + "source_ip" + ], + "min_ratio": 0.99 + }, + "fortinet": { + "fields": [ + "event_action", + "event_outcome", + "source_ip", + "destination_ip", + "source_port", + "network_transport", + "observer_vendor" + ], + "min_ratio": 0.99 + }, + "o365": { + "fields": [ + "event_action", + "event_outcome", + "user_name", + "client_ip" + ], + "min_ratio": 0.99 + }, + "okta": { + "fields": [ + "event_action", + "event_outcome", + "client_ip", + "user_name" + ], + "min_ratio": 0.99 + } + }, + "zero_ok": [] +} diff --git a/resources/demo-telco/console/dashboards/demo-telco-bgp-peers.json b/resources/demo-telco/console/dashboards/demo-telco-bgp-peers.json index 6b1e2311..51e4a2fc 100644 --- a/resources/demo-telco/console/dashboards/demo-telco-bgp-peers.json +++ b/resources/demo-telco/console/dashboards/demo-telco-bgp-peers.json @@ -51,12 +51,12 @@ "config": { "configType": "sql", "displayType": "table", - "name": "Peers over 20 withdrawals a minute, from hunt demo-telco-peer-withdrawals", + "name": "Peers over 20 withdrawals a minute", "source": "bgp-updates", - "sqlTemplate": "SELECT formatDateTime(toStartOfMinute(_timestamp), '%H:%i') AS \"RIS minute (UTC)\",\n concat(splitByChar('-', toString(_json.attrs.source_id))[2], ' (rrc', splitByChar('-', toString(_json.attrs.source_id))[1], ')') AS \"Peer\",\n count() AS \"Withdrawals\"\nFROM dfe.detection\nWHERE hunt_name = 'demo-telco-peer-withdrawals' AND $__timeFilter(_timestamp + INTERVAL 12 HOUR)\nGROUP BY toStartOfMinute(_timestamp), 1, 2\nHAVING \"Withdrawals\" > 20\nORDER BY toStartOfMinute(_timestamp) DESC, 3 DESC\nLIMIT 20" + "sqlTemplate": "SELECT formatDateTime(toStartOfMinute(_timestamp), '%H:%i') AS \"RIS minute (UTC)\",\n concat(splitByChar('-', toString(_json.attrs.source_id))[2], ' (rrc', splitByChar('-', toString(_json.attrs.source_id))[1], ')') AS \"Peer\",\n count() AS \"Withdrawals\"\nFROM dfe.`bgp-updates`\nWHERE toString(_json.type) = 'W' AND $__timeFilter(_timestamp + INTERVAL 12 HOUR)\nGROUP BY toStartOfMinute(_timestamp), 1, 2\nHAVING \"Withdrawals\" > 20\nORDER BY toStartOfMinute(_timestamp) DESC, 3 DESC\nLIMIT 20" }, "h": 6, - "id": "hunt", + "id": "bursts", "w": 12, "x": 0, "y": 15 diff --git a/resources/demo-telco/console/searches/demo-telco-prefix-history.json b/resources/demo-telco/console/searches/demo-telco-prefix-history.json index 13e05988..6dd0fdba 100644 --- a/resources/demo-telco/console/searches/demo-telco-prefix-history.json +++ b/resources/demo-telco/console/searches/demo-telco-prefix-history.json @@ -6,6 +6,6 @@ "tags": [ "demo" ], - "where": "toString(_json.attrs.target_prefix) = '172.69.12.0/24'", + "where": "toString(_json.attrs.target_prefix) = '173.82.227.0/24'", "whereLanguage": "sql" } diff --git a/resources/demo-telco/demo.json b/resources/demo-telco/demo.json index 4287a4db..d75521e4 100644 --- a/resources/demo-telco/demo.json +++ b/resources/demo-telco/demo.json @@ -20,7 +20,9 @@ ] }, "hunts": [ - "demo-telco-peer-withdrawals" + "demo-telco-hss-too-busy", + "demo-telco-peer-withdrawals", + "demo-telco-weak-signal" ], "sources": [ "bgp-updates", diff --git a/resources/demo-telco/hunts/demo-telco-hss-too-busy.yaml b/resources/demo-telco/hunts/demo-telco-hss-too-busy.yaml new file mode 100644 index 00000000..8d8389cc --- /dev/null +++ b/resources/demo-telco/hunts/demo-telco-hss-too-busy.yaml @@ -0,0 +1,19 @@ +# Project: dfe-docker +# File: resources/demo-telco/hunts/demo-telco-hss-too-busy.yaml +# Purpose: The telco demo's hunt, running its HSS overload rule every five minutes +# Language: YAML +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +name: demo-telco-hss-too-busy +display_name: HSS overload on S6a +cron: "*/5 * * * *" +# A required label: the runner takes each detection's org from the matched row. +customers: + - default +rules: + - demo-telco-hss-too-busy +global_source_table_name: dfe.core-probes +global_target_table_name: dfe.detection +checkpoint_timestamp_field: _timestamp_load diff --git a/resources/demo-telco/hunts/demo-telco-weak-signal.yaml b/resources/demo-telco/hunts/demo-telco-weak-signal.yaml new file mode 100644 index 00000000..23eca4b3 --- /dev/null +++ b/resources/demo-telco/hunts/demo-telco-weak-signal.yaml @@ -0,0 +1,19 @@ +# Project: dfe-docker +# File: resources/demo-telco/hunts/demo-telco-weak-signal.yaml +# Purpose: The telco demo's hunt, running its weak-signal rule every five minutes +# Language: YAML +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +name: demo-telco-weak-signal +display_name: Cells with a very weak signal +cron: "*/5 * * * *" +# A required label: the runner takes each detection's org from the matched row. +customers: + - default +rules: + - demo-telco-weak-signal +global_source_table_name: dfe.ran-kpis +global_target_table_name: dfe.detection +checkpoint_timestamp_field: _timestamp_load diff --git a/resources/demo-telco/rules/demo-telco-hss-too-busy.yaml b/resources/demo-telco/rules/demo-telco-hss-too-busy.yaml new file mode 100644 index 00000000..e1922671 --- /dev/null +++ b/resources/demo-telco/rules/demo-telco-hss-too-busy.yaml @@ -0,0 +1,14 @@ +# Project: dfe-docker +# File: resources/demo-telco/rules/demo-telco-hss-too-busy.yaml +# Purpose: The telco demo's rule: an S6a request the HSS refused with DIAMETER_TOO_BUSY +# Language: YAML +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +name: demo-telco-hss-too-busy +display_name: HSS refused an S6a request as too busy +severity: high +source: core-probes +source_type: raw +user_sql: "SELECT * FROM dfe.`core-probes` WHERE toString(_json.cause) = 'DIAMETER_TOO_BUSY'" diff --git a/resources/demo-telco/rules/demo-telco-peer-withdrawals.yaml b/resources/demo-telco/rules/demo-telco-peer-withdrawals.yaml index a43e64b4..184e4f6a 100644 --- a/resources/demo-telco/rules/demo-telco-peer-withdrawals.yaml +++ b/resources/demo-telco/rules/demo-telco-peer-withdrawals.yaml @@ -11,5 +11,5 @@ display_name: BGP withdrawal from a RIS peer severity: medium source: bgp-updates source_type: raw -# A rule matches single rows and may not read another table, so the burst, more than 20 withdrawals in a minute from one peer, is counted where the demo-telco-bgp-peers dashboard reads the detections. +# A rule matches single rows and may not read another table, so the burst, more than 20 withdrawals in a minute from one peer, is counted where the demo-telco-bgp-peers dashboard reads the updates, and each run records at most the engine's per-run detection cap. user_sql: "SELECT * FROM dfe.`bgp-updates` WHERE toString(_json.type) = 'W'" diff --git a/resources/demo-telco/rules/demo-telco-weak-signal.yaml b/resources/demo-telco/rules/demo-telco-weak-signal.yaml new file mode 100644 index 00000000..5c613dcf --- /dev/null +++ b/resources/demo-telco/rules/demo-telco-weak-signal.yaml @@ -0,0 +1,14 @@ +# Project: dfe-docker +# File: resources/demo-telco/rules/demo-telco-weak-signal.yaml +# Purpose: The telco demo's rule: a cell reporting an average RSRP below -102 dBm +# Language: YAML +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +name: demo-telco-weak-signal +display_name: Cell reporting a very weak signal +severity: medium +source: ran-kpis +source_type: raw +user_sql: "SELECT * FROM dfe.`ran-kpis` WHERE toFloat64OrZero(toString(_json.rsrp_dbm)) < -102" diff --git a/resources/demo-telco/verify.json b/resources/demo-telco/verify.json new file mode 100644 index 00000000..792a14cb --- /dev/null +++ b/resources/demo-telco/verify.json @@ -0,0 +1,38 @@ +{ + "hunts": { + "demo-telco-hss-too-busy": { + "max_alerts": 126 + }, + "demo-telco-peer-withdrawals": { + "max_alerts": 126 + }, + "demo-telco-weak-signal": { + "max_alerts": 126 + } + }, + "parsed": { + "core-probes": { + "fields": [ + "_json.protocol", + "_json.procedure", + "_json.result", + "_json.latency_ms", + "_json.source_node" + ], + "min_ratio": 0.99 + }, + "ran-kpis": { + "fields": [ + "_json.cell", + "_json.site", + "_json.rsrp_dbm", + "_json.prb_util_dl_pct", + "_json.dl_throughput_mbps" + ], + "min_ratio": 0.99 + } + }, + "zero_ok": [ + "demo-telco-bgp-peers/bursts" + ] +} diff --git a/resources/demo-theatre/console/searches/demo-theatre-vessel-track.json b/resources/demo-theatre/console/searches/demo-theatre-vessel-track.json index 9d3c7c2e..6adb4b0e 100644 --- a/resources/demo-theatre/console/searches/demo-theatre-vessel-track.json +++ b/resources/demo-theatre/console/searches/demo-theatre-vessel-track.json @@ -6,6 +6,6 @@ "tags": [ "demo" ], - "where": "toString(_json.mmsi) = '209297000'", + "where": "toString(_json.mmsi) = (SELECT toString(_json.mmsi) FROM dfe.`ais-tracks` GROUP BY 1 ORDER BY count() DESC, 1 LIMIT 1)", "whereLanguage": "sql" } diff --git a/resources/demo-theatre/demo.json b/resources/demo-theatre/demo.json index 6efff6c3..ba63c570 100644 --- a/resources/demo-theatre/demo.json +++ b/resources/demo-theatre/demo.json @@ -23,6 +23,7 @@ ] }, "hunts": [ + "demo-theatre-aircraft-low", "demo-theatre-vessel-silent" ], "sources": [ diff --git a/resources/demo-theatre/hunts/demo-theatre-aircraft-low.yaml b/resources/demo-theatre/hunts/demo-theatre-aircraft-low.yaml new file mode 100644 index 00000000..bf226a11 --- /dev/null +++ b/resources/demo-theatre/hunts/demo-theatre-aircraft-low.yaml @@ -0,0 +1,19 @@ +# Project: dfe-docker +# File: resources/demo-theatre/hunts/demo-theatre-aircraft-low.yaml +# Purpose: The theatre demo's hunt, running its low-altitude rule every five minutes +# Language: YAML +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +name: demo-theatre-aircraft-low +display_name: Aircraft below 1,500 feet +cron: "*/5 * * * *" +# A required label: the runner takes each detection's org from the matched row. +customers: + - default +rules: + - demo-theatre-aircraft-low +global_source_table_name: dfe.adsb-tracks +global_target_table_name: dfe.detection +checkpoint_timestamp_field: _timestamp_load diff --git a/resources/demo-theatre/rules/demo-theatre-aircraft-low.yaml b/resources/demo-theatre/rules/demo-theatre-aircraft-low.yaml new file mode 100644 index 00000000..af785cfe --- /dev/null +++ b/resources/demo-theatre/rules/demo-theatre-aircraft-low.yaml @@ -0,0 +1,15 @@ +# Project: dfe-docker +# File: resources/demo-theatre/rules/demo-theatre-aircraft-low.yaml +# Purpose: The theatre demo's rule: an aircraft reporting a barometric altitude under 1,500 feet +# Language: YAML +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +name: demo-theatre-aircraft-low +display_name: Aircraft below 1,500 feet +severity: low +source: adsb-tracks +source_type: raw +# Aircraft in the circuit or on approach and departure round Sydney Kingsford Smith, which the air picture holds on every poll. A grounded aircraft reports "ground", which reads as 0. +user_sql: "SELECT * FROM dfe.`adsb-tracks` WHERE toFloat64OrZero(toString(_json.alt_baro)) BETWEEN 1 AND 1499" diff --git a/resources/demo-theatre/rules/demo-theatre-vessel-silent.yaml b/resources/demo-theatre/rules/demo-theatre-vessel-silent.yaml index baa5b662..577c7ed2 100644 --- a/resources/demo-theatre/rules/demo-theatre-vessel-silent.yaml +++ b/resources/demo-theatre/rules/demo-theatre-vessel-silent.yaml @@ -11,6 +11,6 @@ display_name: Vessel silent for five minutes severity: medium source: ais-tracks source_type: raw -# Each poll lands every vessel's latest report, so a report between 300 and 310 seconds old matches on one poll per silence. +# Each poll lands every vessel's latest report, so a report between 300 and 330 seconds old matches on up to three polls per silence, enough that one lands within minutes of the feed's first poll. # A rule reads only its own row, so the track coming back is the demo-theatre-churn dashboard's returning tracks. -user_sql: "SELECT * FROM dfe.`ais-tracks` WHERE dateDiff('second', fromUnixTimestamp64Milli(toInt64OrZero(toString(`ais-tracks`._json.properties.timestampExternal))), `ais-tracks`._timestamp) BETWEEN 300 AND 309" +user_sql: "SELECT * FROM dfe.`ais-tracks` WHERE dateDiff('second', fromUnixTimestamp64Milli(toInt64OrZero(toString(`ais-tracks`._json.properties.timestampExternal))), `ais-tracks`._timestamp) BETWEEN 300 AND 329" diff --git a/resources/demo-theatre/verify.json b/resources/demo-theatre/verify.json new file mode 100644 index 00000000..95c35dc4 --- /dev/null +++ b/resources/demo-theatre/verify.json @@ -0,0 +1,34 @@ +{ + "hunts": { + "demo-theatre-aircraft-low": { + "max_alerts": 126 + }, + "demo-theatre-vessel-silent": { + "max_alerts": 126 + } + }, + "parsed": { + "adsb-tracks": { + "fields": [ + "_json.hex", + "_json.lat", + "_json.lon", + "_json.alt_baro", + "_json.seen_pos" + ], + "min_ratio": 0.9 + }, + "ais-tracks": { + "fields": [ + "_json.mmsi", + "_json.properties.sog", + "_json.properties.cog", + "_json.properties.timestampExternal" + ], + "min_ratio": 0.99 + } + }, + "zero_ok": [ + "demo-theatre-aircraft-late" + ] +} diff --git a/resources/demo/console/dashboards/demo-feodo-c2.json b/resources/demo/console/dashboards/demo-feodo-c2.json new file mode 100644 index 00000000..3bd751d6 --- /dev/null +++ b/resources/demo/console/dashboards/demo-feodo-c2.json @@ -0,0 +1,64 @@ +{ + "name": "demo-feodo-c2", + "tags": [ + "demo" + ], + "tiles": [ + { + "config": { + "configType": "sql", + "displayType": "number", + "name": "Command-and-control servers on the blocklist", + "source": "feodo-c2", + "sqlTemplate": "SELECT count() AS \"servers\"\nFROM (SELECT * FROM dfe.`feodo-c2` WHERE ip_address IS NOT NULL ORDER BY _timestamp_load DESC LIMIT 1 BY ip_address, port)" + }, + "h": 5, + "id": "total", + "w": 8, + "x": 0, + "y": 0 + }, + { + "config": { + "configType": "sql", + "displayType": "table", + "name": "Servers by malware family", + "source": "feodo-c2", + "sqlTemplate": "SELECT malware AS \"Malware\",\n count() AS \"Servers\",\n countIf(status = 'online') AS \"Online\",\n max(last_online) AS \"Last online\"\nFROM (SELECT * FROM dfe.`feodo-c2` WHERE ip_address IS NOT NULL ORDER BY _timestamp_load DESC LIMIT 1 BY ip_address, port)\nGROUP BY 1\nORDER BY 2 DESC" + }, + "h": 5, + "id": "families", + "w": 16, + "x": 8, + "y": 0 + }, + { + "config": { + "configType": "sql", + "displayType": "table", + "name": "Servers and where they are hosted", + "source": "feodo-c2", + "sqlTemplate": "SELECT replaceOne(IPv6NumToString(assumeNotNull(ip_address)), '::ffff:', '') AS \"Address\",\n port AS \"Port\",\n malware AS \"Malware\",\n status AS \"Status\",\n country AS \"Country\",\n as_name AS \"Network\",\n first_seen AS \"First seen\",\n last_online AS \"Last online\"\nFROM (SELECT * FROM dfe.`feodo-c2` WHERE ip_address IS NOT NULL ORDER BY _timestamp_load DESC LIMIT 1 BY ip_address, port)\nORDER BY last_online DESC" + }, + "h": 8, + "id": "servers", + "w": 24, + "x": 0, + "y": 5 + }, + { + "config": { + "configType": "sql", + "displayType": "line", + "name": "Blocklist rows landed per poll", + "source": "feodo-c2", + "sqlTemplate": "SELECT $__timeInterval(_timestamp_load) AS ts,\n count() AS \"rows landed\"\nFROM dfe.`feodo-c2`\nWHERE $__timeFilter(_timestamp_load)\nGROUP BY ts\nORDER BY ts" + }, + "h": 5, + "id": "polls", + "w": 24, + "x": 0, + "y": 13 + } + ] +} diff --git a/resources/demo/console/dashboards/demo-stack-health.json b/resources/demo/console/dashboards/demo-stack-health.json index 5196735c..50fe092f 100644 --- a/resources/demo/console/dashboards/demo-stack-health.json +++ b/resources/demo/console/dashboards/demo-stack-health.json @@ -10,7 +10,7 @@ "displayType": "stacked_bar", "name": "Rows per table per minute", "source": "all_sources", - "sqlTemplate": "SELECT $__timeInterval(_timestamp_load) AS ts,\n toString(_source) AS \"table\",\n count() * 60 / $__interval_s AS \"rows/min\"\nFROM dfe.all_sources\nWHERE $__timeFilter(_timestamp_load) AND _table = toString(_source) AND toString(_source) IN ('cisco-asa', 'fortinet', 'demo-syslog', 'component-releases', 'wikipedia-recentchanges')\nGROUP BY ts, \"table\"\nORDER BY ts" + "sqlTemplate": "SELECT $__timeInterval(_timestamp_load) AS ts,\n toString(_source) AS \"table\",\n count() * 60 / $__interval_s AS \"rows/min\"\nFROM dfe.all_sources\nWHERE $__timeFilter(_timestamp_load) AND _table = toString(_source) AND toString(_source) IN ('cisco-asa', 'fortinet', 'demo-syslog', 'component-releases', 'feodo-c2')\nGROUP BY ts, \"table\"\nORDER BY ts" }, "h": 7, "id": "rows", @@ -24,7 +24,7 @@ "displayType": "table", "name": "Rows landed", "source": "all_sources", - "sqlTemplate": "SELECT toString(_source) AS \"Table\",\n count() AS \"Rows\"\nFROM dfe.all_sources\nWHERE $__timeFilter(_timestamp_load) AND _table = toString(_source) AND toString(_source) IN ('cisco-asa', 'fortinet', 'demo-syslog', 'component-releases', 'wikipedia-recentchanges')\nGROUP BY 1\nORDER BY 2 DESC" + "sqlTemplate": "SELECT toString(_source) AS \"Table\",\n count() AS \"Rows\"\nFROM dfe.all_sources\nWHERE $__timeFilter(_timestamp_load) AND _table = toString(_source) AND toString(_source) IN ('cisco-asa', 'fortinet', 'demo-syslog', 'component-releases', 'feodo-c2')\nGROUP BY 1\nORDER BY 2 DESC" }, "h": 7, "id": "total", diff --git a/resources/demo/console/dashboards/demo-wikipedia-recentchanges.json b/resources/demo/console/dashboards/demo-wikipedia-recentchanges.json deleted file mode 100644 index 6eed883d..00000000 --- a/resources/demo/console/dashboards/demo-wikipedia-recentchanges.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "name": "demo-wikipedia-recentchanges", - "tags": [ - "demo" - ], - "tiles": [ - { - "config": { - "configType": "sql", - "displayType": "line", - "name": "Edits per minute", - "source": "wikipedia-recentchanges", - "sqlTemplate": "SELECT $__timeInterval(_timestamp) AS ts,\n count() * 60 / $__interval_s AS \"edits/min\"\nFROM dfe.`wikipedia-recentchanges`\nWHERE $__timeFilter(_timestamp)\nGROUP BY ts\nORDER BY ts" - }, - "h": 6, - "id": "epm", - "w": 12, - "x": 0, - "y": 0 - }, - { - "config": { - "configType": "sql", - "displayType": "stacked_bar", - "name": "Bot and human changes", - "source": "wikipedia-recentchanges", - "sqlTemplate": "SELECT $__timeInterval(_timestamp) AS ts,\n if(dynamicType(_json.bot) = 'None', 'human', 'bot') AS who,\n count() AS changes\nFROM dfe.`wikipedia-recentchanges`\nWHERE $__timeFilter(_timestamp)\nGROUP BY ts, who\nORDER BY ts" - }, - "h": 6, - "id": "bots", - "w": 12, - "x": 12, - "y": 0 - }, - { - "config": { - "configType": "sql", - "displayType": "table", - "name": "Top pages (edits and new pages)", - "source": "wikipedia-recentchanges", - "sqlTemplate": "SELECT toString(_json.title) AS \"Page\",\n count() AS \"Changes\",\n uniqExact(toString(_json.user)) AS \"Editors\"\nFROM dfe.`wikipedia-recentchanges`\nWHERE $__timeFilter(_timestamp) AND toString(_json.type) IN ('edit', 'new')\nGROUP BY 1\nORDER BY 2 DESC\nLIMIT 12" - }, - "h": 12, - "id": "pages", - "w": 12, - "x": 0, - "y": 6 - }, - { - "config": { - "configType": "sql", - "displayType": "table", - "name": "Wikis and change types", - "source": "wikipedia-recentchanges", - "sqlTemplate": "SELECT toString(_json._dfe_fetcher_wiki) AS \"Wiki\",\n toString(_json.type) AS \"Type\",\n count() AS \"Changes\"\nFROM dfe.`wikipedia-recentchanges`\nWHERE $__timeFilter(_timestamp)\nGROUP BY 1, 2\nORDER BY 3 DESC" - }, - "h": 5, - "id": "wikis", - "w": 12, - "x": 12, - "y": 6 - }, - { - "config": { - "configType": "sql", - "displayType": "table", - "name": "Size change distribution (bytes)", - "source": "wikipedia-recentchanges", - "sqlTemplate": "SELECT multiIf(d < -2000, 'shrank over 2000', d < -200, 'shrank 200 to 2000', d < 0, 'shrank under 200', d = 0, 'no change', d <= 200, 'grew under 200', d <= 2000, 'grew 200 to 2000', 'grew over 2000') AS \"Change\",\n count() AS \"Edits\"\nFROM (SELECT toInt64OrZero(toString(_json.newlen)) - toInt64OrZero(toString(_json.oldlen)) AS d FROM dfe.`wikipedia-recentchanges` WHERE $__timeFilter(_timestamp) AND toString(_json.type) = 'edit')\nGROUP BY 1\nORDER BY min(d)" - }, - "h": 7, - "id": "sizes", - "w": 12, - "x": 12, - "y": 11 - } - ] -} diff --git a/resources/demo/console/searches/demo-feodo-c2-all.json b/resources/demo/console/searches/demo-feodo-c2-all.json new file mode 100644 index 00000000..3a1b28dd --- /dev/null +++ b/resources/demo/console/searches/demo-feodo-c2-all.json @@ -0,0 +1,11 @@ +{ + "name": "demo-feodo-c2-all", + "orderBy": "_timestamp_load DESC", + "select": "_timestamp_load, replaceOne(IPv6NumToString(assumeNotNull(ip_address)), '::ffff:', '') AS address, port, malware, status, country, as_name, first_seen, last_online", + "source": "feodo-c2", + "tags": [ + "demo" + ], + "where": "ip_address IS NOT NULL", + "whereLanguage": "sql" +} diff --git a/resources/demo/console/searches/demo-feodo-c2-online.json b/resources/demo/console/searches/demo-feodo-c2-online.json new file mode 100644 index 00000000..aeca1ba4 --- /dev/null +++ b/resources/demo/console/searches/demo-feodo-c2-online.json @@ -0,0 +1,11 @@ +{ + "name": "demo-feodo-c2-online", + "orderBy": "_timestamp_load DESC", + "select": "_timestamp_load, replaceOne(IPv6NumToString(assumeNotNull(ip_address)), '::ffff:', '') AS address, port, malware, status, country, as_name, first_seen, last_online", + "source": "feodo-c2", + "tags": [ + "demo" + ], + "where": "ip_address IS NOT NULL AND status = 'online'", + "whereLanguage": "sql" +} diff --git a/resources/demo/console/searches/demo-syslog-live.json b/resources/demo/console/searches/demo-syslog-live.json deleted file mode 100644 index f404e75f..00000000 --- a/resources/demo/console/searches/demo-syslog-live.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "demo-syslog-live", - "orderBy": "_timestamp_load DESC", - "select": "_timestamp_load, message, hostname, app_name", - "source": "demo-syslog", - "tags": [ - "demo" - ], - "where": "", - "whereLanguage": "sql" -} diff --git a/resources/demo/console/searches/demo-wikipedia-bots.json b/resources/demo/console/searches/demo-wikipedia-bots.json deleted file mode 100644 index 746c4ce9..00000000 --- a/resources/demo/console/searches/demo-wikipedia-bots.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "demo-wikipedia-bots", - "orderBy": "_timestamp DESC", - "select": "_timestamp, toString(_json.type) AS type, toString(_json.title) AS page, toString(_json.user) AS editor, toInt64OrZero(toString(_json.newlen)) - toInt64OrZero(toString(_json.oldlen)) AS bytes, toString(_json.comment) AS comment", - "source": "wikipedia-recentchanges", - "tags": [ - "demo" - ], - "where": "dynamicType(_json.bot) != 'None'", - "whereLanguage": "sql" -} diff --git a/resources/demo/console/searches/demo-wikipedia-large-edits.json b/resources/demo/console/searches/demo-wikipedia-large-edits.json deleted file mode 100644 index 5c43a84d..00000000 --- a/resources/demo/console/searches/demo-wikipedia-large-edits.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "demo-wikipedia-large-edits", - "orderBy": "_timestamp DESC", - "select": "_timestamp, toString(_json.type) AS type, toString(_json.title) AS page, toString(_json.user) AS editor, toInt64OrZero(toString(_json.newlen)) - toInt64OrZero(toString(_json.oldlen)) AS bytes, toString(_json.comment) AS comment", - "source": "wikipedia-recentchanges", - "tags": [ - "demo" - ], - "where": "toString(_json.type) = 'edit' AND abs(toInt64OrZero(toString(_json.newlen)) - toInt64OrZero(toString(_json.oldlen))) > 2000", - "whereLanguage": "sql" -} diff --git a/resources/demo/console/searches/demo-wikipedia-live.json b/resources/demo/console/searches/demo-wikipedia-live.json deleted file mode 100644 index 7f2102ae..00000000 --- a/resources/demo/console/searches/demo-wikipedia-live.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "demo-wikipedia-live", - "orderBy": "_timestamp DESC", - "select": "_timestamp, toString(_json.type) AS type, toString(_json.title) AS page, toString(_json.user) AS editor, toInt64OrZero(toString(_json.newlen)) - toInt64OrZero(toString(_json.oldlen)) AS bytes, toString(_json.comment) AS comment", - "source": "wikipedia-recentchanges", - "tags": [ - "demo" - ], - "where": "", - "whereLanguage": "sql" -} diff --git a/resources/demo/demo.json b/resources/demo/demo.json index ed851b17..d5cc194f 100644 --- a/resources/demo/demo.json +++ b/resources/demo/demo.json @@ -11,29 +11,31 @@ "dashboards": [ "demo-cisco-asa", "demo-component-releases", + "demo-feodo-c2", "demo-fortinet", "demo-stack-health", - "demo-syslog", - "demo-wikipedia-recentchanges" + "demo-syslog" ], "searches": [ "demo-cisco-asa-denied", "demo-cisco-asa-live", "demo-component-releases-all", + "demo-feodo-c2-all", + "demo-feodo-c2-online", "demo-fortinet-denied", "demo-fortinet-live", - "demo-syslog-live", - "demo-syslog-unclaimed", - "demo-wikipedia-bots", - "demo-wikipedia-large-edits", - "demo-wikipedia-live" + "demo-syslog-unclaimed" ] }, "sources": [ "cisco-asa", "component-releases", "demo-syslog", - "fortinet", - "wikipedia-recentchanges" + "feodo-c2", + "fortinet" + ], + "hunts": [ + "demo-asa-logon-failed", + "demo-fortinet-ssh-denied" ] } diff --git a/resources/demo/hunts/demo-asa-logon-failed.yaml b/resources/demo/hunts/demo-asa-logon-failed.yaml new file mode 100644 index 00000000..1daf2b37 --- /dev/null +++ b/resources/demo/hunts/demo-asa-logon-failed.yaml @@ -0,0 +1,19 @@ +# Project: dfe-docker +# File: resources/demo/hunts/demo-asa-logon-failed.yaml +# Purpose: The demo's hunt, running its failed-logon rule every five minutes +# Language: YAML +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +name: demo-asa-logon-failed +display_name: Failed logons on Cisco ASAs +cron: "*/5 * * * *" +# A required label: the runner takes each detection's org from the matched row. +customers: + - default +rules: + - demo-asa-logon-failed +global_source_table_name: dfe.cisco-asa +global_target_table_name: dfe.detection +checkpoint_timestamp_field: _timestamp_load diff --git a/resources/demo/hunts/demo-fortinet-ssh-denied.yaml b/resources/demo/hunts/demo-fortinet-ssh-denied.yaml new file mode 100644 index 00000000..25889b3b --- /dev/null +++ b/resources/demo/hunts/demo-fortinet-ssh-denied.yaml @@ -0,0 +1,19 @@ +# Project: dfe-docker +# File: resources/demo/hunts/demo-fortinet-ssh-denied.yaml +# Purpose: The demo's hunt, running its denied-SSH rule every five minutes +# Language: YAML +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +name: demo-fortinet-ssh-denied +display_name: SSH connections denied by FortiGates +cron: "*/5 * * * *" +# A required label: the runner takes each detection's org from the matched row. +customers: + - default +rules: + - demo-fortinet-ssh-denied +global_source_table_name: dfe.fortinet +global_target_table_name: dfe.detection +checkpoint_timestamp_field: _timestamp_load diff --git a/resources/demo/rules/demo-asa-logon-failed.yaml b/resources/demo/rules/demo-asa-logon-failed.yaml new file mode 100644 index 00000000..92bb82d8 --- /dev/null +++ b/resources/demo/rules/demo-asa-logon-failed.yaml @@ -0,0 +1,14 @@ +# Project: dfe-docker +# File: resources/demo/rules/demo-asa-logon-failed.yaml +# Purpose: The demo's rule: a failed administrator logon on a Cisco ASA +# Language: YAML +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +name: demo-asa-logon-failed +display_name: Failed logon on a Cisco ASA +severity: medium +source: cisco-asa +source_type: raw +user_sql: "SELECT * FROM dfe.`cisco-asa` WHERE event_action = 'logon-failed'" diff --git a/resources/demo/rules/demo-fortinet-ssh-denied.yaml b/resources/demo/rules/demo-fortinet-ssh-denied.yaml new file mode 100644 index 00000000..51cdc788 --- /dev/null +++ b/resources/demo/rules/demo-fortinet-ssh-denied.yaml @@ -0,0 +1,14 @@ +# Project: dfe-docker +# File: resources/demo/rules/demo-fortinet-ssh-denied.yaml +# Purpose: The demo's rule: a FortiGate deny on the SSH port +# Language: YAML +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +name: demo-fortinet-ssh-denied +display_name: SSH connection denied by a FortiGate +severity: low +source: fortinet +source_type: raw +user_sql: "SELECT * FROM dfe.`fortinet` WHERE event_action = 'deny' AND destination_port = 22" diff --git a/resources/demo/sources/feodo-c2.yaml b/resources/demo/sources/feodo-c2.yaml new file mode 100644 index 00000000..b753da3b --- /dev/null +++ b/resources/demo/sources/feodo-c2.yaml @@ -0,0 +1,42 @@ +# Project: dfe-docker +# File: resources/demo/sources/feodo-c2.yaml +# Purpose: A demo fetcher that pulls the abuse.ch Feodo Tracker botnet command-and-control blocklist +# Language: YAML +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +description: Botnet command-and-control servers (Emotet, QakBot and similar) on the abuse.ch Feodo Tracker IP blocklist, one row per server address and port. The tracker rebuilds the list every five minutes and publishes it under CC0 with no key (https://feodotracker.abuse.ch/blocklist/) +display_name: Feodo Tracker C2 servers +fetcher: + config: + auth: + mode: none + # The tracker asks clients to fetch at most every five minutes + interval_secs: 300 + profile: + auth: + accepts: [none] + base_url: "{{ vars.api_url }}" + endpoints: + - path: /downloads/ipblocklist.json + rows: + decoder: json_array + unit: blocklist + headers: + Accept: application/json + User-Agent: DFE-demo (https://github.com/hyperi-io/dfe-docker) + retry: + never_retry: [401, 403] + shape: dump + vars: + api_url: https://feodotracker.abuse.ch + source_type: rest + topic: own +header: + type: common-header/timeseries + version: 1.0.1 +schema: + meta_schema: meta/abusech/feodo_c2 + meta_schema_version: 1.0.0 +source: feodo-c2 diff --git a/resources/demo/sources/wikipedia-recentchanges.yaml b/resources/demo/sources/wikipedia-recentchanges.yaml deleted file mode 100644 index aa917413..00000000 --- a/resources/demo/sources/wikipedia-recentchanges.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# Project: dfe-docker -# File: resources/demo/sources/wikipedia-recentchanges.yaml -# Purpose: A demo fetcher that pulls live English Wikipedia recent changes -# Language: YAML -# -# License: BUSL-1.1 -# Copyright: (c) 2026 HYPERI PTY LIMITED - -description: Live English Wikipedia recent changes, every edit and log entry as it happens -display_name: Wikipedia recent changes -fetcher: - config: - auth: - mode: none - interval_secs: 5 - profile: - auth: - accepts: [none] - base_url: "{{ vars.api_url }}" - endpoints: - - add_fields: - _dfe_fetcher_wiki: "{{ vars.wiki }}" - paginate: - from: "body:/continue/rccontinue" - into: "query:rccontinue" - strategy: cursor - path: /w/api.php - query: - action: query - format: json - list: recentchanges - rcdir: newer - rcend: "{{ int(window.end) - 2 }}" - rclimit: "100" - rcprop: title|ids|sizes|flags|user|timestamp|comment - rcstart: "{{ int(window.start) - 1 }}" - rows: - at: /query/recentchanges - decoder: json_at - unit: recentchanges - headers: - Accept: application/json - User-Agent: DFE-demo/{{ vars.demo_version }} (https://github.com/hyperi-io/dfe-docker) - probe: - path: /w/api.php - query: - action: query - format: json - meta: siteinfo - retry: - never_retry: [401, 403] - shape: incremental - window: - format: epoch_secs - lookback: 30s - # At about four edits a second a longer window passes the fetcher's page ceiling - step: 5m - vars: - api_url: https://en.wikipedia.org - demo_version: "2.2" - wiki: en.wikipedia.org - source_type: rest - topic: own -header: - type: common-header/timeseries - version: 1.0.1 -schema: - meta_schema: meta/syslog - meta_schema_version: 1.0.0 -source: wikipedia-recentchanges diff --git a/resources/demo/verify.json b/resources/demo/verify.json new file mode 100644 index 00000000..5f2f5016 --- /dev/null +++ b/resources/demo/verify.json @@ -0,0 +1,53 @@ +{ + "hunts": { + "demo-asa-logon-failed": { + "max_alerts": 126 + }, + "demo-fortinet-ssh-denied": { + "max_alerts": 126 + } + }, + "parsed": { + "cisco-asa": { + "fields": [ + "event_action", + "source_ip", + "destination_ip", + "observer_vendor" + ], + "min_ratio": 0.9 + }, + "fortinet": { + "fields": [ + "event_action", + "event_outcome", + "source_ip", + "destination_ip", + "source_port", + "network_transport", + "observer_vendor" + ], + "min_ratio": 0.99 + } + }, + "zero_ok": [ + "demo-stack-health/archerr", + "demo-syslog/unclaimed", + "demo-syslog-unclaimed", + "demo-component-releases/latest", + "demo-component-releases/polls", + "demo-component-releases-all", + "demo-feodo-c2/total", + "demo-feodo-c2/families", + "demo-feodo-c2/servers", + "demo-feodo-c2/polls", + "demo-feodo-c2-all", + "demo-feodo-c2-online" + ], + "waiting": { + "reason": "a fetcher and engine that land a dump unit in its own table, where the 2.2.2 ones land it in dfe.main", + "sources": [ + "feodo-c2" + ] + } +} diff --git a/scripts/demo_verify.py b/scripts/demo_verify.py new file mode 100644 index 00000000..dc2850bf --- /dev/null +++ b/scripts/demo_verify.py @@ -0,0 +1,1203 @@ +#!/usr/bin/env python3 +# Project: dfe-docker +# File: scripts/demo_verify.py +# Purpose: Prove a published demo pack worked end to end, not only that its stack came up and rows arrived +# Language: Python +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +"""Check a demo pack's published stack did its job, from the host that started it. + +CI runs it in each "Published compose" leg once the stack is healthy: + + demo_verify.py --pack demo-cyber --file dfe-demo-cyber.yaml + +Every call into the stack goes through one `docker compose exec` into dfe-engine, which logs in from the engine's own secrets file and makes the requests there, so no credential reaches this process or the runner's log. SQL runs through the engine's query API, the one dfe-ui's search calls. The checks: + + landed every source whose events the pack replays holds rows + fetchers each fetcher source whose polls the pack replays holds rows, and the others say why they are skipped + parsed no row carries `error.message` or a `pipeline_error` tag, and each field the pack's verify.json names is set on at least its `min_ratio` of rows + unclaimed the intake's own table and the default table hold nothing + sources every pack source is deployed and in the state its file declares + hunts each rule of each hunt the pack ships wrote detections, and the hunt wrote no more than its `max_alerts` + console every dashboard and saved search the pack names is installed and its queries return rows + search a saved search's query answers through the engine's query API as the console login + dlq dfe.dlq and each dead-letter topic exist and hold nothing + +What a pack expects is data in resources//verify.json: + + parsed {: {"fields": [, ...], "min_ratio": 0.95}} + hunts {: {"max_alerts": }}, which is six five-minute runs of the stack's cap (publish_compose.DEMO_MAX_DETECTIONS) plus the summary row each run writes past it + waiting {"reason": , "sources": [, ...]} sources that cannot land in their own table until a fix ships, which the landing, fetcher, parsed and unclaimed checks leave out and say so + zero_ok ["/" or "", ...] queries that return no rows or read 0 on purpose + +Exit codes: + 0 every check held + 1 a check failed, naming the table, the count and the query + 2 usage error +""" + +import argparse +import json +import re +import subprocess +import sys +import time +import typing +from collections.abc import Callable, Sequence +from dataclasses import dataclass, field +from pathlib import Path + +# Bytecode for the repo modules imported below goes to .tmp/pycache, as it does under make +sys.pycache_prefix = str(Path(__file__).resolve().parent.parent / ".tmp" / "pycache") + +from _common import _print, demo_pack_dir # noqa: E402 + +DATABASE = "dfe" +DEFAULT_TABLE = "main" +DETECTION_TABLE = "detection" +DLQ_TABLE = "dlq" +ENGINE_SERVICE = "dfe-engine" +KAFKA_SERVICE = "kafka-redpanda" +ENGINE_API = "http://localhost:8000/api/v1" +HYPERDX_API = "http://dfe-hyperdx:8000" +DATASOURCE = "clickhouse:default" +BREAKGLASS_USER = "breakglass" +BREAKGLASS_PASSWORD_KEY = "DFE_ENGINE_AUTH_BREAKGLASS_PASSWORD" +SECRETS_FILE = "/run/dfe-secrets/env" +DLQ_TOPICS = ( + "dfe_archiver_dlq", + "dfe_fetcher_dlq", + "dfe_loader_dlq", + "dfe_receiver_dlq", + "dfe_transform_dlq", +) +VERIFY_FILE = "verify.json" +DEFAULT_MIN_RATIO = 0.95 +CONSOLE_RANGE_SECONDS = 3600 +CONSOLE_INTERVAL_SECONDS = 60 +LAND_TIMEOUT_SECONDS = 300.0 +FETCH_TIMEOUT_SECONDS = 600.0 +HUNT_TIMEOUT_SECONDS = 900.0 +CONSOLE_TIMEOUT_SECONDS = 300.0 +POLL_INTERVAL_SECONDS = 5.0 +EXEC_TIMEOUT_SECONDS = 240 +CHECKS = ( + "landed", + "fetchers", + "parsed", + "unclaimed", + "sources", + "hunts", + "console", + "search", + "dlq", +) +# A Source file's own first-level `state:` and the intake's agentless mode +STATE_LINE = re.compile(r"^state:\s*(\w+)\s*$", re.MULTILINE) +AGENTLESS_LINE = re.compile(r"^\s+mode:\s*agentless\s*$", re.MULTILINE) +FETCHER_LINE = re.compile(r"^fetcher:", re.MULTILINE) +RULE_LINE = re.compile(r"^\s+-\s+(\S+)\s*$") +FIELD_PATH = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$") +TABLE_NAME = re.compile(r"^[A-Za-z0-9_-]+$") +# What a column holds when it is empty: an unset field, an empty array or object +EMPTY_VALUES = ("", "[]", "{}", "null") +ERROR_ROW = ( + "(JSONHas(toJSONString(_json), 'error', 'message') " + "OR position(toJSONString(_tags), 'pipeline_error') > 0)" +) + +# Runs in dfe-engine: logs in as each identity a request names, once, replacing an issued password first as a person's first use of the console login does, makes each request and prints the answers as one JSON line +ENGINE_SCRIPT = ( + "import json, os, secrets, sys, urllib.error, urllib.request\n" + "base = sys.argv[1]\n" + "def call(method, url, payload, token):\n" + " body = None if payload is None else json.dumps(payload).encode()\n" + " request = urllib.request.Request(url, data=body, method=method)\n" + " request.add_header('Content-Type', 'application/json')\n" + " if token:\n" + " request.add_header('Authorization', 'Bearer ' + token)\n" + " try:\n" + " with urllib.request.urlopen(request, timeout=120) as answer:\n" + " raw, status = answer.read().decode(), answer.status\n" + " except urllib.error.HTTPError as error:\n" + " raw, status = error.read().decode(), error.code\n" + " try:\n" + " return [status, json.loads(raw)]\n" + " except ValueError:\n" + " return [status, raw]\n" + "def login(username, password):\n" + " status, body = call('POST', base + '/auth/login', {'username': username, 'password': password}, '')\n" + " token = body.get('access_token', '') if isinstance(body, dict) else ''\n" + " if token and body.get('password_change_required') is True:\n" + " fresh = secrets.token_hex(16)\n" + " status, body = call('POST', base + '/auth/accounts/reset-password', {'current_password': password, 'new_password': fresh}, token)\n" + " if status != 200:\n" + " return ''\n" + " status, body = call('POST', base + '/auth/login', {'username': username, 'password': fresh}, '')\n" + " token = body.get('access_token', '') if isinstance(body, dict) else ''\n" + " return token\n" + "spec = json.load(sys.stdin)\n" + "tokens = {}\n" + "def token_for(identity):\n" + " if identity not in tokens:\n" + " if identity == 'console' and spec.get('console_login'):\n" + " tokens[identity] = login(spec['console_login']['username'], spec['console_login']['password'])\n" + " elif identity == 'breakglass':\n" + " tokens[identity] = login(spec['breakglass_user'], os.environ[spec['breakglass_key']])\n" + " else:\n" + " tokens[identity] = ''\n" + " return tokens[identity]\n" + "print(json.dumps([call(item['method'], item['url'], item.get('payload'), token_for(item['as'])) for item in spec['requests']]))\n" +) + + +class VerifyError(Exception): + """A pack or stack this checker cannot read, with what to change.""" + + +@dataclass(frozen=True, slots=True) +class Request: + """One call made inside the engine's container, as `breakglass` or the minted `console` login.""" + + method: str + url: str + payload: dict | None = None + identity: str = "breakglass" + + +@dataclass(frozen=True, slots=True) +class Reply: + """What a request answered: its HTTP status and decoded body.""" + + status: int + body: typing.Any + + +@dataclass(frozen=True, slots=True) +class Failure: + """One failed check, saying what failed where.""" + + check: str + detail: str + # False where waiting cannot change the answer, such as an alert count already over its ceiling + retry: bool = True + + +@dataclass(frozen=True, slots=True) +class Result: + """One check's outcome: what held, what failed or why it was skipped.""" + + check: str + detail: str + failures: tuple[Failure, ...] = () + skipped: bool = False + + +@dataclass(slots=True) +class Pack: + """A demo pack's files, read without YAML so a bare python3 can run this. + + Attributes: + name: The pack's directory name, `demo` or `demo-`. + sources: The sources demo.json lists. + states: Each source's `state:` in its file, active where it declares none. + replayed: Sources whose events the pack sends at the receiver. + polled: Fetcher sources whose captured polls the pack serves. + fetchers: Every source that is a fetcher. + intake: The agentless intake's source name, None for a pack with no syslog. + dashboards: Names of the dashboards demo.json lists. + searches: Names of the saved searches demo.json lists. + hunts: Each hunt demo.json lists, with the rules its file runs. + parsed: verify.json's expected fields per source. + hunt_limits: verify.json's alert ceiling per hunt. + zero_ok: verify.json's tiles and searches that return no rows or read 0 on purpose. + waiting_sources: verify.json's sources whose rows cannot land in their own table yet, left out of the landing checks. + waiting_reason: Why they cannot, which the skipped checks print. + """ + + name: str + sources: list[str] + states: dict[str, str] + replayed: list[str] + polled: list[str] + fetchers: list[str] + intake: str | None + dashboards: list[str] + searches: list[str] + hunts: dict[str, list[str]] + parsed: dict[str, dict] + hunt_limits: dict[str, dict] + zero_ok: set[str] = field(default_factory=set) + waiting_sources: list[str] = field(default_factory=list) + waiting_reason: str = "" + + +class Stack(typing.Protocol): + """A running stack: a way into dfe-engine, the broker's dead-letter topics and the compose file's services.""" + + def calls(self, requests: Sequence[Request]) -> list[Reply]: ... + + def services(self) -> list[str]: ... + + def topic_marks(self, topics: Sequence[str]) -> dict[str, int | None]: ... + + +@dataclass(frozen=True, slots=True) +class Compose: + """A stack reached through `docker compose -f `, with the console login it holds for the search check.""" + + file: Path + context: str = "" + console_login: tuple[str, str] | None = None + login_fault: str = "" + + def _compose(self, *args: str) -> list[str]: + context = ["--context", self.context] if self.context else [] + return ["docker", *context, "compose", "-f", str(self.file), *args] + + def _run(self, command: list[str], *, stdin: str | None = None): + return subprocess.run( + command, + capture_output=True, + check=False, + encoding="utf-8", + errors="replace", + input=stdin, + timeout=EXEC_TIMEOUT_SECONDS, + ) + + def calls(self, requests: Sequence[Request]) -> list[Reply]: + """Send every request in one exec into the engine and return the answers in order.""" + login = self.console_login + spec = { + "breakglass_key": BREAKGLASS_PASSWORD_KEY, + "breakglass_user": BREAKGLASS_USER, + "console_login": ( + {"username": login[0], "password": login[1]} if login else None + ), + "requests": [ + { + "as": item.identity, + "method": item.method, + "payload": item.payload, + "url": item.url, + } + for item in requests + ], + } + result = self._run( + self._compose( + "exec", + "-T", + ENGINE_SERVICE, + "sh", + "-c", + f'set -a && . {SECRETS_FILE} && python -c "$0" {ENGINE_API}', + ENGINE_SCRIPT, + ), + stdin=json.dumps(spec), + ) + if result.returncode != 0: + raise VerifyError( + f"{ENGINE_SERVICE} exec failed with {result.returncode}: " + f"{result.stderr.strip()[-400:]}" + ) + try: + answers = json.loads(result.stdout.strip().splitlines()[-1]) + except (IndexError, ValueError) as error: + raise VerifyError( + f"{ENGINE_SERVICE} answered no JSON: {result.stdout.strip()[-400:]!r}" + ) from error + return [Reply(status=status, body=body) for status, body in answers] + + def services(self) -> list[str]: + """The compose file's service names.""" + result = self._run(self._compose("config", "--services")) + return result.stdout.split() if result.returncode == 0 else [] + + def topic_marks(self, topics: Sequence[str]) -> dict[str, int | None]: + """Each topic's high watermark summed over its partitions, None where the broker cannot say.""" + marks: dict[str, int | None] = {} + for topic in topics: + result = self._run( + self._compose( + "exec", "-T", KAFKA_SERVICE, "rpk", "topic", "describe", topic, "-p" + ) + ) + marks[topic] = ( + watermark_total(text=result.stdout) if result.returncode == 0 else None + ) + return marks + + def with_minted_login(self) -> "Compose": + """This stack with the console login secrets-init mints, held in memory and never printed.""" + result = self._run(self._compose("run", "--rm", "secrets-init", "login")) + login = parse_login(text=result.stdout) if result.returncode == 0 else None + fault = "" + if login is None: + # A line holding the password is the only one that can match, so none that did not is printed + fault = ( + f"secrets-init login exited {result.returncode} and printed no 'Console login: / ' line. " + f"Its stderr ends: {result.stderr.strip()[-300:]!r}" + ) + return Compose( + file=self.file, + context=self.context, + console_login=login, + login_fault=fault, + ) + + +def parse_login(*, text: str) -> tuple[str, str] | None: + """The name and password in secrets-init's `Console login: / ` line.""" + found = re.search(r"^Console login: (.+?) / (\S+)$", text, re.MULTILINE) + return (found.group(1), found.group(2)) if found else None + + +def watermark_total(*, text: str) -> int | None: + """The sum of the HIGH-WATERMARK column of `rpk topic describe -p`, None when it has no such column.""" + lines = [line.split() for line in text.strip().splitlines()] + if len(lines) < 2 or "HIGH-WATERMARK" not in lines[0]: + return None + column = lines[0].index("HIGH-WATERMARK") + try: + return sum(int(row[column]) for row in lines[1:]) + except (IndexError, ValueError): + return None + + +def table_ref(*, table: str) -> str: + """The `dfe`.`` reference, refusing a name that could change the query.""" + if not TABLE_NAME.fullmatch(table): + raise VerifyError(f"{table!r} is not a table name") + return f"{DATABASE}.`{table}`" + + +def present(*, name: str) -> str: + """A ClickHouse condition that is true where the column or `_json.` path holds a value.""" + if not FIELD_PATH.fullmatch(name): + raise VerifyError(f"{name!r} is not a column or a `_json.` path") + values = ", ".join(f"'{empty}'" for empty in EMPTY_VALUES) + return f"ifNull(toString({name}) NOT IN ({values}), 0)" + + +def coverage_query(*, fields: Sequence[str], table: str) -> str: + """One row: the table's rows, those carrying an error, then the rows holding each field in order.""" + cells = [f"countIf({present(name=name)})" for name in fields] + columns = ["count()", f"countIf({ERROR_ROW})", *cells] + return f"SELECT {', '.join(columns)} FROM {table_ref(table=table)}" + + +def expand_macros(*, sql: str) -> str: + """A dashboard tile's SQL with the console's time macros filled in for the last CONSOLE_RANGE_SECONDS.""" + window = f"INTERVAL {CONSOLE_RANGE_SECONDS} SECOND" + sql = re.sub( + r"\$__timeFilter\(([^)]*)\)", + lambda found: ( + f"({found.group(1)} >= now() - {window} AND {found.group(1)} <= now())" + ), + sql, + ) + sql = re.sub( + r"\$__timeInterval\(([^)]*)\)", + lambda found: ( + f"toStartOfInterval({found.group(1)}, INTERVAL {CONSOLE_INTERVAL_SECONDS} SECOND)" + ), + sql, + ) + return sql.replace("$__interval_s", str(CONSOLE_INTERVAL_SECONDS)) + + +def search_query(*, search: dict, table: str) -> str: + """The count a saved search's select and where read from its table, which errors where either is wrong.""" + where = str(search.get("where") or "").strip() or "1" + select = str(search.get("select") or "").strip() or "*" + return ( + f"SELECT count() FROM (SELECT {select} FROM {table_ref(table=table)} " + f"WHERE {where} LIMIT 100)" + ) + + +def first_number(*, rows: Sequence[dict]) -> float | None: + """The first cell of the first row as a number, None when there is none or it is not one.""" + if not rows: + return None + try: + return float(next(iter(rows[0].values()))) + except (StopIteration, TypeError, ValueError): + return None + + +def rows_of(*, reply: Reply) -> list[dict] | None: + """The rows a query answered, None when it did not answer 200 with a `rows` list.""" + if reply.status != 200 or not isinstance(reply.body, dict): + return None + rows = reply.body.get("rows") + return rows if isinstance(rows, list) else None + + +def fault_of(*, reply: Reply) -> str: + """Why a query did not answer, from the engine's problem body.""" + body = reply.body + detail = body.get("detail", body) if isinstance(body, dict) else body + return f"HTTP {reply.status}: {str(detail)[:300]}" + + +def sql_request(*, sql: str, identity: str = "breakglass") -> Request: + """The engine query API call that runs `sql` against the data database.""" + return Request( + identity=identity, + method="POST", + payload={"datasource": DATASOURCE, "query": sql}, + url=f"{ENGINE_API}/queries/raw", + ) + + +def run_sql( + *, queries: Sequence[str], stack: Stack, identity: str = "breakglass" +) -> list[Reply]: + """Run every query in one exec and return each answer.""" + return stack.calls([sql_request(sql=sql, identity=identity) for sql in queries]) + + +def count_tables(*, stack: Stack, tables: Sequence[str]) -> dict[str, int | None]: + """The rows in each table, None where the count could not be read.""" + queries = [f"SELECT count() FROM {table_ref(table=table)}" for table in tables] + counts: dict[str, int | None] = {} + for table, reply in zip(tables, run_sql(queries=queries, stack=stack), strict=True): + rows = rows_of(reply=reply) + number = first_number(rows=rows) if rows else None + counts[table] = None if number is None else int(number) + return counts + + +def wait_for( + *, + probe: Callable[[], Result], + timeout: float, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, +) -> Result: + """Run `probe` again every POLL_INTERVAL_SECONDS until it holds or `timeout` has passed, and return the last result.""" + deadline = clock() + timeout + while True: + result = probe() + waitable = any(failure.retry for failure in result.failures) + if not waitable or clock() >= deadline: + return result + sleep(POLL_INTERVAL_SECONDS) + + +def check_landed(*, pack: Pack, stack: Stack) -> Result: + """Every source whose events the pack replays holds rows.""" + if not pack.replayed: + return Result("landed", f"{pack.name} replays no source's events", skipped=True) + counts = count_tables(stack=stack, tables=pack.replayed) + failures = tuple( + Failure( + "landed", + f"{DATABASE}.{table} holds {'an unreadable count' if rows is None else f'{rows} rows'}. " + f"Query: SELECT count() FROM {table_ref(table=table)}", + ) + for table, rows in counts.items() + if not rows + ) + return Result( + "landed", + ", ".join(f"{table} {rows}" for table, rows in counts.items()), + failures, + ) + + +def check_parsed(*, pack: Pack, stack: Stack) -> Result: + """No replayed or polled row carries an error and each expected field is set on at least its ratio of rows.""" + tables = [*pack.replayed, *pack.polled] + if not tables: + return Result("parsed", f"{pack.name} replays no source's events", skipped=True) + plans = [] + for table in tables: + expected = pack.parsed.get(table, {}) + fields = list(expected.get("fields", [])) + plans.append( + (table, fields, float(expected.get("min_ratio", DEFAULT_MIN_RATIO))) + ) + queries = [coverage_query(fields=fields, table=table) for table, fields, _ in plans] + failures: list[Failure] = [] + held = [] + for (table, fields, ratio), sql, reply in zip( + plans, queries, run_sql(queries=queries, stack=stack), strict=True + ): + rows = rows_of(reply=reply) + if not rows: + failures.append( + Failure( + "parsed", + f"{table_ref(table=table)} could not be read ({fault_of(reply=reply)}). Query: {sql}", + ) + ) + continue + cells = [int(cell) for cell in rows[0].values()] + total, errored, set_counts = cells[0], cells[1], cells[2:] + if total == 0: + # A fetcher table with no rows is the fetchers check's failure to name + if table not in pack.polled: + failures.append( + Failure( + "parsed", f"{table_ref(table=table)} holds 0 rows. Query: {sql}" + ) + ) + continue + if errored: + failures.append( + Failure( + "parsed", + f"{table_ref(table=table)}: {errored} of {total} rows carry error.message " + f"or a pipeline_error tag. Query: SELECT substring(toJSONString(_json), 1, 300) " + f"FROM {table_ref(table=table)} WHERE {ERROR_ROW} LIMIT 3", + ) + ) + for name, number in zip(fields, set_counts, strict=True): + if number < ratio * total: + failures.append( + Failure( + "parsed", + f"{table_ref(table=table)}: {name} is set on {number} of {total} rows " + f"({number / total:.0%}), under the {ratio:.0%} verify.json expects. " + f"Query: SELECT countIf({present(name=name)}) FROM {table_ref(table=table)}", + ) + ) + held.append(f"{table} {total}") + return Result("parsed", ", ".join(held), tuple(failures)) + + +def check_unclaimed(*, pack: Pack, stack: Stack) -> Result: + """The intake's own table and the default table hold nothing, bar rows of a source waiting on a fix, and no waiting source has its own rows yet.""" + # A waiting source lands in the default table until the fix ships, so only its rows are left out of the count + leave_out = "" + for source in pack.waiting_sources: + table_ref(table=source) + leave_out += f" AND NOT startsWith(toString(_source), '{source}')" + tables = [*([pack.intake] if pack.intake else []), DEFAULT_TABLE] + queries = [ + f"SELECT count() FROM {table_ref(table=table)}" + + (f" WHERE 1{leave_out}" if table == DEFAULT_TABLE else "") + for table in tables + ] + waiting = [ + f"SELECT count() FROM {table_ref(table=s)}" for s in pack.waiting_sources + ] + replies = run_sql(queries=[*queries, *waiting], stack=stack) + counts: dict[str, int | None] = {} + for table, reply in zip(tables, replies, strict=False): + rows = rows_of(reply=reply) + number = first_number(rows=rows) if rows else None + counts[table] = None if number is None else int(number) + failures = [] + for (table, rows), sql in zip(counts.items(), queries, strict=True): + if rows == 0: + continue + why = ( + "its count could not be read" + if rows is None + else f"holds {rows} rows no source claimed" + ) + failures.append( + Failure( + "unclaimed", + f"{table_ref(table=table)} {why}. Query: {sql}", + ) + ) + for source, reply in zip( + pack.waiting_sources, replies[len(queries) :], strict=True + ): + rows = rows_of(reply=reply) + number = first_number(rows=rows) if rows else None + if number: + failures.append( + Failure( + "unclaimed", + f"{table_ref(table=source)} holds {int(number)} rows, so {source} no longer waits on {pack.waiting_reason}. " + f"Remove it from {VERIFY_FILE} waiting", + retry=False, + ) + ) + held = ", ".join(f"{table} {rows}" for table, rows in counts.items()) + if pack.waiting_sources: + held += f"; {', '.join(pack.waiting_sources)} rows left out of {DEFAULT_TABLE}, waiting on {pack.waiting_reason}" + return Result("unclaimed", held, tuple(failures)) + + +def check_sources(*, pack: Pack, stack: Stack) -> Result: + """Every pack source is deployed and in the state its file declares.""" + (reply,) = stack.calls([Request("GET", f"{ENGINE_API}/sources?page_size=500")]) + items = reply.body.get("items") if isinstance(reply.body, dict) else None + if reply.status != 200 or not isinstance(items, list): + return Result( + "sources", + "", + ( + Failure( + "sources", f"GET /api/v1/sources answered {fault_of(reply=reply)}" + ), + ), + ) + held = {item.get("name"): item for item in items if isinstance(item, dict)} + failures = [] + for source in pack.sources: + item = held.get(source) + want = pack.states[source] + if item is None: + failures.append( + Failure("sources", f"{source} is not in GET /api/v1/sources") + ) + continue + if item.get("state") != want: + failures.append( + Failure( + "sources", + f"{source} is {item.get('state')!r}, its file declares {want!r}", + ) + ) + if not item.get("deployed_version") or item.get("deployed_version") != item.get( + "current" + ): + failures.append( + Failure( + "sources", + f"{source} is deployed at {item.get('deployed_version')!r} where its current version is {item.get('current')!r}", + ) + ) + return Result("sources", f"{len(pack.sources)} sources deployed", tuple(failures)) + + +def check_hunts(*, pack: Pack, stack: Stack) -> Result: + """Each rule of each hunt wrote detections, and each hunt stayed under its ceiling.""" + if not pack.hunts: + return Result("hunts", f"{pack.name} ships no hunt", skipped=True) + names = ", ".join(f"'{name}'" for name in pack.hunts) + sql = ( + f"SELECT hunt_name, rule_id, count() FROM {table_ref(table=DETECTION_TABLE)} " + f"WHERE hunt_name IN ({names}) GROUP BY hunt_name, rule_id" + ) + (reply,) = run_sql(queries=[sql], stack=stack) + rows = rows_of(reply=reply) + if rows is None: + return Result( + "hunts", + "", + ( + Failure( + "hunts", + f"{table_ref(table=DETECTION_TABLE)} could not be read ({fault_of(reply=reply)}). Query: {sql}", + ), + ), + ) + seen: dict[tuple[str, str], int] = {} + for row in rows: + hunt, rule, number = list(row.values())[:3] + seen[(str(hunt), str(rule))] = int(number) + failures = [] + held = [] + for hunt, rules in pack.hunts.items(): + total = sum(number for (name, _), number in seen.items() if name == hunt) + for rule in rules: + if not seen.get((hunt, rule)): + failures.append( + Failure( + "hunts", + f"hunt {hunt}: rule {rule} wrote no detection yet. Query: {sql}", + ) + ) + ceiling = pack.hunt_limits.get(hunt, {}).get("max_alerts") + if not isinstance(ceiling, int): + failures.append( + Failure( + "hunts", + f"{VERIFY_FILE} names no max_alerts for hunt {hunt}", + retry=False, + ) + ) + elif total > ceiling: + failures.append( + Failure( + "hunts", + f"hunt {hunt} wrote {total} detections, over its ceiling of {ceiling}. Query: {sql}", + retry=False, + ) + ) + held.append(f"{hunt} {total}") + return Result("hunts", ", ".join(held), tuple(failures)) + + +def console_requests() -> list[Request]: + """The reads that list what the console holds: its sources, dashboards and saved searches.""" + return [ + Request("GET", f"{HYPERDX_API}/sources"), + Request("GET", f"{HYPERDX_API}/dashboards"), + Request("GET", f"{HYPERDX_API}/saved-search"), + ] + + +def read_console( + *, stack: Stack +) -> tuple[dict[str, dict], dict[str, dict], dict[str, str]] | str: + """The installed dashboards and searches by name and each source's table by id, or why they cannot be read.""" + replies = stack.calls(console_requests()) + for reply, what in zip( + replies, ("sources", "dashboards", "saved-search"), strict=True + ): + if reply.status != 200 or not isinstance(reply.body, list): + return f"GET {HYPERDX_API}/{what} answered {fault_of(reply=reply)}" + sources = { + str(item.get("id")): str((item.get("from") or {}).get("tableName")) + for item in replies[0].body + if isinstance(item, dict) + } + dashboards = { + str(item.get("name")): item + for item in replies[1].body + if isinstance(item, dict) + } + searches = { + str(item.get("name")): item + for item in replies[2].body + if isinstance(item, dict) + } + return dashboards, searches, sources + + +def check_console(*, pack: Pack, stack: Stack) -> Result: + """Every dashboard and saved search the pack names is installed and its queries return rows.""" + if not pack.dashboards and not pack.searches: + return Result( + "console", f"{pack.name} installs no dashboard or search", skipped=True + ) + read = read_console(stack=stack) + if isinstance(read, str): + return Result("console", "", (Failure("console", read),)) + dashboards, searches, tables = read + failures: list[Failure] = [] + plan: list[tuple[str, str, str, str]] = [] + for name in pack.dashboards: + found = dashboards.get(name) + if found is None: + failures.append( + Failure("console", f"dashboard {name} is not installed in dfe-hyperdx") + ) + continue + for tile in found.get("tiles") or []: + config = tile.get("config") or {} + sql = config.get("sqlTemplate") + if sql: + plan.append( + ( + f"dashboard {name}, tile {tile.get('id')}", + f"{name}/{tile.get('id')}", + str(config.get("displayType")), + expand_macros(sql=sql), + ) + ) + for name in pack.searches: + found = searches.get(name) + if found is None: + failures.append( + Failure( + "console", f"saved search {name} is not installed in dfe-hyperdx" + ) + ) + continue + table = tables.get(str(found.get("source"))) + if not table: + failures.append( + Failure( + "console", + f"saved search {name} reads source {found.get('source')!r}, which dfe-hyperdx does not hold", + ) + ) + continue + plan.append( + ( + f"saved search {name}", + name, + "search", + search_query(search=found, table=table), + ) + ) + replies = run_sql(queries=[sql for *_, sql in plan], stack=stack) + for (label, key, kind, sql), reply in zip(plan, replies, strict=True): + rows = rows_of(reply=reply) + if rows is None: + failures.append( + Failure( + "console", + f"{label} did not run ({fault_of(reply=reply)}). Query: {sql}", + ) + ) + elif key in pack.zero_ok: + continue + elif not rows or (kind in ("search", "number") and not first_number(rows=rows)): + failures.append( + Failure( + "console", + f"{label} returns no rows or reads 0, and {key} is not in {VERIFY_FILE} zero_ok. Query: {sql}", + ) + ) + return Result( + "console", + f"{len(pack.dashboards)} dashboards and {len(pack.searches)} searches, {len(plan)} queries", + tuple(failures), + ) + + +def check_search(*, pack: Pack, stack: Stack) -> Result: + """A saved search's query returns rows through the engine's query API as the console login.""" + names = [name for name in pack.searches if name not in pack.zero_ok] + if not names: + return Result( + "search", + f"{pack.name} installs no saved search that must return rows", + skipped=True, + ) + fault = getattr(stack, "login_fault", "") + if fault: + return Result("search", "", (Failure("search", fault),)) + read = read_console(stack=stack) + if isinstance(read, str): + return Result("search", "", (Failure("search", read),)) + _, searches, tables = read + failures: list[Failure] = [] + plan: list[tuple[str, str]] = [] + for name in names: + found = searches.get(name) + table = tables.get(str(found.get("source"))) if found else None + if not found or not table: + failures.append( + Failure( + "search", + f"saved search {name} is not installed or reads no table", + ) + ) + continue + plan.append((name, search_query(search=found, table=table))) + replies = run_sql(queries=[sql for _, sql in plan], stack=stack, identity="console") + for (name, sql), reply in zip(plan, replies, strict=True): + rows = rows_of(reply=reply) + if rows is None: + why = f"got {fault_of(reply=reply)}" + elif not first_number(rows=rows): + why = "returned no rows" + else: + continue + failures.append( + Failure("search", f"{name} as the console login {why}. Query: {sql}") + ) + return Result( + "search", + f"{len(plan)} saved searches returned rows as the console login", + tuple(failures), + ) + + +def check_fetchers(*, pack: Pack, stack: Stack) -> Result: + """Each fetcher source whose polls the pack replays holds rows, and the others are skipped with their reason.""" + if not pack.fetchers: + return Result("fetchers", f"{pack.name} has no fetcher source", skipped=True) + waiting = [source for source in pack.fetchers if source in pack.waiting_sources] + unreplayed = [ + source + for source in pack.fetchers + if source not in pack.polled and source not in waiting + ] + notes = [] + if unreplayed: + notes.append( + f"skipped {', '.join(unreplayed)}: no captured poll in events//demo.log, so they wait on their upstream" + ) + if waiting: + notes.append(f"skipped {', '.join(waiting)}: waiting on {pack.waiting_reason}") + note = "; ".join(notes) + if not pack.polled: + return Result("fetchers", note, skipped=True) + counts = count_tables(stack=stack, tables=pack.polled) + failures = tuple( + Failure( + "fetchers", + f"{table_ref(table=table)} holds {'an unreadable count' if rows is None else f'{rows} rows'} after its replayed polls. Query: SELECT count() FROM {table_ref(table=table)}", + ) + for table, rows in counts.items() + if not rows + ) + held = ", ".join(f"{table} {rows}" for table, rows in counts.items()) + return Result("fetchers", f"{held}{'; ' + note if note else ''}", failures) + + +def check_dlq(*, pack: Pack, stack: Stack) -> Result: + """dfe.dlq and each dead-letter topic exist and hold nothing, and a record names its source and reason.""" + sql = ( + f"SELECT service, ifNull(_source, ''), destination, reason, count() " + f"FROM {table_ref(table=DLQ_TABLE)} GROUP BY service, _source, destination, reason" + ) + (reply,) = run_sql(queries=[sql], stack=stack) + rows = rows_of(reply=reply) + failures: list[Failure] = [] + if rows is None: + failures.append( + Failure( + "dlq", + f"{table_ref(table=DLQ_TABLE)} could not be read ({fault_of(reply=reply)}). Query: {sql}", + ) + ) + for row in rows or []: + service, source, destination, reason, number = list(row.values())[:5] + failures.append( + Failure( + "dlq", + f"{number} records dead-lettered by {service} for source {source!r} heading to {destination!r}: {str(reason)[:300]}. Query: {sql}", + ) + ) + marks: dict[str, int | None] = {} + if KAFKA_SERVICE in stack.services(): + marks = stack.topic_marks(DLQ_TOPICS) + for topic, mark in marks.items(): + if mark is None: + failures.append( + Failure( + "dlq", + f"topic {topic} is missing or unreadable, and a record sent to it is held and back-pressures its source. The engine creates every DLQ topic at startup: rpk topic describe {topic} -p", + ) + ) + elif mark: + failures.append( + Failure( + "dlq", + f"topic {topic} holds {mark} records: rpk topic consume {topic} -n 3", + ) + ) + seen = "table dfe.dlq" + ( + f" and {len(marks)} topics" if marks else ", no broker in this stack" + ) + return Result("dlq", f"{seen} hold nothing", tuple(failures)) + + +def verify( + *, + only: Sequence[str] = CHECKS, + pack: Pack, + stack: Stack, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, + wait_scale: float = 1.0, +) -> list[Result]: + """Run each wanted check in order, the slow ones until they hold or their wait runs out, and return every result. + + `wait_scale` multiplies every wait, 0 for a single try. + """ + waits = { + "landed": LAND_TIMEOUT_SECONDS, + "hunts": HUNT_TIMEOUT_SECONDS, + "console": CONSOLE_TIMEOUT_SECONDS, + "fetchers": FETCH_TIMEOUT_SECONDS, + } + functions = { + "landed": check_landed, + "parsed": check_parsed, + "unclaimed": check_unclaimed, + "sources": check_sources, + "hunts": check_hunts, + "console": check_console, + "search": check_search, + "fetchers": check_fetchers, + "dlq": check_dlq, + } + results = [] + for name in CHECKS: + if name not in only: + continue + probe = lambda check=functions[name]: check(pack=pack, stack=stack) # noqa: E731 + if name in waits: + results.append( + wait_for( + probe=probe, + timeout=waits[name] * wait_scale, + clock=clock, + sleep=sleep, + ) + ) + else: + results.append(probe()) + return results + + +def load_pack(*, name: str, root: Path | None = None) -> Pack: + """Read the pack's files, refusing a pack that is missing one or holds a file this cannot read.""" + short = name.removeprefix("demo-") if name != "demo" else "" + root = root or demo_pack_dir(pack=short) + manifest = _json_file(path=root / "demo.json") + verify_data = _json_file(path=root / VERIFY_FILE) + sources = _names(value=manifest.get("sources"), what="demo.json sources") + console = manifest.get("console") or {} + texts = {source: _source_text(root=root, source=source) for source in sources} + intake = next( + (source for source, text in texts.items() if AGENTLESS_LINE.search(text)), None + ) + fetchers = [source for source, text in texts.items() if FETCHER_LINE.search(text)] + waiting = verify_data.get("waiting") or {} + if not isinstance(waiting, dict): + raise VerifyError(f"{VERIFY_FILE} waiting needs an object") + waiting_sources = _names( + value=waiting.get("sources", []), what=f"{VERIFY_FILE} waiting sources" + ) + captured = [ + source + for source in sources + if (root / "events" / source / "demo.log").is_file() + and source not in waiting_sources + ] + hunts = { + hunt: _hunt_rules(path=root / "hunts" / f"{hunt}.yaml") + for hunt in _names(value=manifest.get("hunts") or [], what="demo.json hunts") + } + return Pack( + name=name, + sources=sources, + states={ + source: (found.group(1) if (found := STATE_LINE.search(text)) else "active") + for source, text in texts.items() + }, + replayed=[ + source for source in captured if source not in fetchers and source != intake + ], + polled=[source for source in captured if source in fetchers], + fetchers=fetchers, + intake=intake, + dashboards=_names(value=console.get("dashboards"), what="demo.json dashboards"), + searches=_names(value=console.get("searches"), what="demo.json searches"), + hunts=hunts, + parsed=_objects(value=verify_data.get("parsed"), what=f"{VERIFY_FILE} parsed"), + hunt_limits=_objects( + value=verify_data.get("hunts"), what=f"{VERIFY_FILE} hunts" + ), + waiting_sources=waiting_sources, + waiting_reason=str(waiting.get("reason", "")), + zero_ok=set( + _names( + value=verify_data.get("zero_ok") or [], what=f"{VERIFY_FILE} zero_ok" + ) + ), + ) + + +def _json_file(*, path: Path) -> dict: + try: + loaded = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise VerifyError(f"{str(path)!r} is not readable JSON: {error}") from error + if not isinstance(loaded, dict): + raise VerifyError(f"{str(path)!r} needs a JSON object") + return loaded + + +def _names(*, value: object, what: str) -> list[str]: + if not isinstance(value, list) or not all( + isinstance(item, str) and item for item in value + ): + raise VerifyError(f"{what} needs a list of names") + return list(value) + + +def _objects(*, value: object, what: str) -> dict[str, dict]: + if not isinstance(value, dict) or not all( + isinstance(item, dict) for item in value.values() + ): + raise VerifyError(f"{what} needs an object of objects") + return dict(value) + + +def _source_text(*, root: Path, source: str) -> str: + path = root / "sources" / f"{source}.yaml" + try: + return path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as error: + raise VerifyError(f"{str(path)!r} is not readable: {error}") from error + + +def _hunt_rules(*, path: Path) -> list[str]: + """The rule names under a hunt file's `rules:` list.""" + try: + lines = path.read_text(encoding="utf-8").splitlines() + except (OSError, UnicodeDecodeError) as error: + raise VerifyError(f"{str(path)!r} is not readable: {error}") from error + rules: list[str] = [] + inside = False + for line in lines: + if line.startswith("rules:"): + inside = True + elif inside and (found := RULE_LINE.match(line)): + rules.append(found.group(1)) + elif inside and line.strip() and not line.startswith("#"): + break + if not rules: + raise VerifyError(f"{str(path)!r} lists no rules") + return rules + + +def _report(*, results: Sequence[Result]) -> int: + failed = 0 + for result in results: + if result.skipped: + _print(msg=f"SKIP {result.check}: {result.detail}") + elif result.failures: + failed += 1 + for failure in result.failures: + _print(msg=f"FAIL {failure.check}: {failure.detail}") + else: + _print(msg=f"PASS {result.check}: {result.detail}") + return 1 if failed else 0 + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--pack", required=True, help="the pack: demo, or demo-") + parser.add_argument( + "--file", + required=True, + type=Path, + help="the pack's published compose file, which is up", + ) + parser.add_argument( + "--context", default="", help="the docker context the stack runs in" + ) + parser.add_argument( + "--wait-scale", + default=1.0, + type=float, + help="multiply every wait, 0 to try each check once", + ) + parser.add_argument( + "--only", + default=",".join(CHECKS), + help=f"comma-separated checks to run, from {', '.join(CHECKS)}", + ) + args = parser.parse_args() + only = [name for name in args.only.split(",") if name] + unknown = sorted(set(only) - set(CHECKS)) + if unknown: + parser.error(f"no check {unknown}, the checks are {', '.join(CHECKS)}") + try: + pack = load_pack(name=args.pack) + stack = Compose(file=args.file.resolve(), context=args.context) + if "search" in only: + stack = stack.with_minted_login() + results = verify(only=only, pack=pack, stack=stack, wait_scale=args.wait_scale) + except (VerifyError, OSError, subprocess.TimeoutExpired) as error: + _print(msg=f"FAIL {error}") + return 1 + return _report(results=results) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/publish_compose.py b/scripts/publish_compose.py index fad7c3be..1d8ecfcf 100644 --- a/scripts/publish_compose.py +++ b/scripts/publish_compose.py @@ -98,6 +98,10 @@ ("from its /proc at /hostfs", "from the collector's own /proc"), ) } +# Detection rows one rule writes in one hunt run on a demo, past which the run writes a summary row carrying the true match count. A replay loops the same events, so an uncapped rule that matches one writes its whole window every five minutes +HUNT_RUNNER_SERVICE = "dfe-hunt-runner" +MAX_DETECTIONS_KEY = "DFE_ENGINE_HUNTS_MAX_DETECTIONS_PER_RUN" +DEMO_MAX_DETECTIONS = 20 # The service whose deploy limits and DNS settings the demo's helpers take SIDECAR_SERVICE = "dlq-init" TRANSFORM_ELASTIC_CONTRACT = "contract-dfe-transform-elastic" @@ -639,6 +643,13 @@ def demo_overlay( engine_env[HOSTED_INSTANCES_KEY] = demo_hosted(roles=roles) engine_env[APPLY_COMMAND_KEY] = apply_command(tag=demo_pack_tag(pack=pack)) engine_env[CREDENTIAL_COMMAND_KEY] = login_command(tag=demo_pack_tag(pack=pack)) + if HUNT_RUNNER_SERVICE not in services: + raise PublishError( + f"the demo model has no {HUNT_RUNNER_SERVICE!r} to cap the pack's hunts on" + ) + services[HUNT_RUNNER_SERVICE].setdefault("environment", {})[MAX_DETECTIONS_KEY] = ( + str(DEMO_MAX_DETECTIONS) + ) sidecar = services[SIDECAR_SERVICE] helper = { "deploy": sidecar["deploy"], diff --git a/scripts/tests/test_demo.py b/scripts/tests/test_demo.py index cccb84c0..245cc759 100644 --- a/scripts/tests/test_demo.py +++ b/scripts/tests/test_demo.py @@ -1219,31 +1219,32 @@ def test_the_committed_fetchers_land_in_their_own_tables_inside_their_rate_limit fetchers = {body["source"]: body["fetcher"] for body in loaded.fetchers} releases = fetchers["component-releases"]["config"] - assert sorted(fetchers) == ["component-releases", "wikipedia-recentchanges"] + assert sorted(fetchers) == ["component-releases", "feodo-c2"] assert all(fetcher["topic"] == "own" for fetcher in fetchers.values()) # Unauthenticated GitHub allows 60 requests an hour per IP and each poll spends one per repo assert releases["interval_secs"] >= 60 * len(releases["vars"]["repos"]) for name in [loaded.intake["source"], *loaded.events, *fetchers]: assert SOURCE_NAME.fullmatch(string=name), name - def test_the_wikipedia_fetcher_asks_for_each_window_once_and_names_itself( + def test_the_feodo_fetcher_polls_the_blocklist_no_faster_than_the_tracker_asks( self, ) -> None: - wiki = {b["source"]: b for b in _committed().fetchers}[ - "wikipedia-recentchanges" - ] - config = wiki["fetcher"]["config"] + feodo = {b["source"]: b for b in _committed().fetchers}["feodo-c2"] + config = feodo["fetcher"]["config"] profile = config["profile"] - query = profile["endpoints"][0]["query"] - - assert config["interval_secs"] == 5 - assert profile["window"]["format"] == "epoch_secs" - # Whole seconds, both ends inclusive, so the next window must start one second past this one's end - assert "int(window.start) - 1" in query["rcstart"] - assert "int(window.end) - 2" in query["rcend"] - assert query["rcdir"] == "newer" + units = {endpoint["unit"]: endpoint for endpoint in profile["endpoints"]} + + # The tracker rebuilds the list every five minutes and asks clients to fetch no faster + assert config["interval_secs"] >= 300 + assert profile["shape"] == "dump" + # make demo has no demo-replay service, so the source carries no replay unit + assert sorted(units) == ["blocklist"] + assert units["blocklist"]["path"] == "/downloads/ipblocklist.json" + # A streaming decoder is bounded per read, so the fetcher refuses a request timeout on it + assert "timeout_secs" not in units["blocklist"] + assert feodo["schema"]["meta_schema"] == "meta/abusech/feodo_c2" agent = profile["headers"]["User-Agent"] - assert agent.startswith("DFE-demo/") + assert agent.startswith("DFE-demo") assert "https://github.com/hyperi-io/dfe-docker" in agent def test_the_committed_sources_keep_their_keys_alphabetical(self) -> None: diff --git a/scripts/tests/test_demo_archive.py b/scripts/tests/test_demo_archive.py index 198e6e7b..99a426fd 100644 --- a/scripts/tests/test_demo_archive.py +++ b/scripts/tests/test_demo_archive.py @@ -179,12 +179,18 @@ def test_refuses_a_manifest_that_disagrees_with_the_sources( with pytest.raises(demo.DemoError, match=message): demo.check_manifest(demo=loaded, manifest=manifest) - def test_the_cyber_pack_carries_each_rule_its_hunt_runs(self) -> None: - hunts = demo.pack_hunts(root=demo_pack_dir(pack="cyber")) + @pytest.mark.parametrize("pack", ["", *demo_pack_names()]) + def test_each_pack_carries_each_rule_its_hunts_run(self, pack: str) -> None: + root = demo_pack_dir(pack=pack) + manifest = json.loads((root / "demo.json").read_text(encoding="utf-8")) + + hunts = demo.pack_hunts(root=root) - assert [name for name, _ in hunts.hunts] == ["demo-cyber-drop-sign-in"] - assert [name for name, _ in hunts.rules] == ["demo-cyber-drop-sign-in"] - assert hunts.rules[0][1]["cel_filter"] + assert sorted(name for name, _ in hunts.hunts) == sorted(manifest["hunts"]) + assert manifest["hunts"], f"{root.name} ships no hunt" + ran = {rule for _, body in hunts.hunts for rule in body["rules"]} + assert ran == {name for name, _ in hunts.rules} + assert all(body["user_sql"] for _, body in hunts.rules) @pytest.mark.parametrize( ("gets", "posts", "expected", "post_calls"), @@ -395,8 +401,9 @@ def test_every_committed_tile_is_sql_over_the_demo_tables(self) -> None: assert len(ids) == len(set(ids)), body["name"] for tile in body["tiles"]: assert tile["config"]["configType"] == "sql", tile["id"] - assert "$__timeFilter" in tile["config"]["sqlTemplate"] or ( - "component-releases" in tile["config"]["sqlTemplate"] + sql = tile["config"]["sqlTemplate"] + assert "$__timeFilter" in sql or ( + "component-releases" in sql or "feodo-c2" in sql ), tile["id"] @@ -406,7 +413,7 @@ def test_deletes_only_what_this_checkouts_demo_created( ) -> None: marker = tmp_path / "installed.json" monkeypatch.setattr(demo, "INSTALLED_FILE", marker) - manifest = {"sources": ["cisco-asa", "fortinet", "wikipedia-recentchanges"]} + manifest = {"sources": ["cisco-asa", "fortinet", "feodo-c2"]} assert demo._installed_sources(manifest=manifest) == [] diff --git a/scripts/tests/test_demo_verify.py b/scripts/tests/test_demo_verify.py new file mode 100644 index 00000000..d0accd4b --- /dev/null +++ b/scripts/tests/test_demo_verify.py @@ -0,0 +1,772 @@ +# Project: dfe-docker +# File: tests/test_demo_verify.py +# Purpose: Assert each demo_verify check passes a pack that worked and fails one that did not, naming the table, the count and the query +# Language: Python +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED + +"""The pack checker over a stack that answers from a table, plus every real pack's own files. + +The stack here is a function from a request to a reply, so each check runs its real queries against canned answers: nothing in demo_verify is replaced, and a check that built the wrong SQL finds no answer for it. +""" + +import re +from collections.abc import Callable, Sequence + +import pytest + +import demo_verify as dv +import publish_compose +from _common import demo_pack_names + + +class Answers: + """A stack whose replies come from `answer`, keyed on the request, and whose broker holds `marks`.""" + + def __init__( + self, + answer: Callable[[dv.Request], dv.Reply], + *, + marks: dict[str, int | None] | None = None, + services: Sequence[str] = ("dfe-engine", "kafka-redpanda"), + ) -> None: + self.answer = answer + self.marks = marks if marks is not None else dict.fromkeys(dv.DLQ_TOPICS, 0) + self.listed = list(services) + self.sent: list[dv.Request] = [] + + def calls(self, requests: Sequence[dv.Request]) -> list[dv.Reply]: + self.sent.extend(requests) + return [self.answer(request) for request in requests] + + def services(self) -> list[str]: + return self.listed + + def topic_marks(self, topics: Sequence[str]) -> dict[str, int | None]: + return {topic: self.marks.get(topic) for topic in topics} + + +def rows(*cells: object) -> dv.Reply: + return dv.Reply(200, {"rows": [{f"c{i}": cell for i, cell in enumerate(cells)}]}) + + +def table(*lines: Sequence[object]) -> dv.Reply: + return dv.Reply( + 200, + {"rows": [{f"c{i}": cell for i, cell in enumerate(line)} for line in lines]}, + ) + + +NONE = dv.Reply(200, {"rows": []}) +REFUSED = dv.Reply(500, {"detail": "The query failed"}) + + +def pack(**changes: object) -> dv.Pack: + base: dict = { + "name": "demo-test", + "sources": ["okta", "wiki", "demo-syslog"], + "states": {"okta": "active", "wiki": "active", "demo-syslog": "active"}, + "replayed": ["okta"], + "polled": [], + "fetchers": [], + "intake": "demo-syslog", + "dashboards": ["demo-t"], + "searches": ["demo-t-live"], + "hunts": {"demo-hunt": ["demo-rule"]}, + "parsed": {"okta": {"fields": ["event_action", "_json.a.b"], "min_ratio": 0.9}}, + "hunt_limits": {"demo-hunt": {"max_alerts": 10}}, + "zero_ok": set(), + } + base.update(changes) + return dv.Pack(**base) + + +def only(check: str, **kwargs: object) -> dv.Result: + results = dv.verify(only=[check], **kwargs, sleep=lambda _: None) + assert [result.check for result in results] == [check] + return results[0] + + +def clock_of(step: float) -> Callable[[], float]: + now = [0.0] + + def tick() -> float: + now[0] += step + return now[0] + + return tick + + +def landing(counts: dict[str, int]) -> Callable[[dv.Request], dv.Reply]: + def answer(request: dv.Request) -> dv.Reply: + query = request.payload["query"] + for name, count in counts.items(): + if f"`{name}`" in query: + return rows(count) + return rows(0) + + return answer + + +@pytest.mark.parametrize( + ("name", "fragment"), + [ + ("event_action", "toString(event_action) NOT IN"), + ("_json.a.b", "toString(_json.a.b) NOT IN"), + ], +) +def test_present_reads_a_column_or_a_json_path(name: str, fragment: str) -> None: + assert fragment in dv.present(name=name) + + +@pytest.mark.parametrize( + "name", ["a; SELECT 1", "a b", "", "1a", "a..b", "a)", "`a`", "a-b"] +) +def test_a_field_that_could_change_the_query_is_refused(name: str) -> None: + with pytest.raises(dv.VerifyError): + dv.present(name=name) + + +@pytest.mark.parametrize("table_name", ["a`b", "a b", "a;b", "", "a.b"]) +def test_a_table_name_that_could_change_the_query_is_refused(table_name: str) -> None: + with pytest.raises(dv.VerifyError): + dv.table_ref(table=table_name) + + +def test_coverage_counts_rows_errors_then_each_field_in_order() -> None: + sql = dv.coverage_query(fields=["a", "b"], table="cisco-asa") + assert sql.startswith("SELECT count(), countIf((JSONHas(") + assert sql.index("toString(a)") < sql.index("toString(b)") + assert sql.endswith("FROM dfe.`cisco-asa`") + + +def test_time_macros_become_a_window_an_interval_and_a_step() -> None: + sql = dv.expand_macros( + sql="SELECT $__timeInterval(t), count() * 60 / $__interval_s FROM x WHERE $__timeFilter(t)" + ) + assert "$__" not in sql + assert "toStartOfInterval(t, INTERVAL 60 SECOND)" in sql + assert "t >= now() - INTERVAL 3600 SECOND" in sql + assert "/ 60 " in sql + + +@pytest.mark.parametrize( + ("text", "want"), + [ + ( + "region\nPARTITION LEADER EPOCH REPLICAS LOG-START-OFFSET HIGH-WATERMARK\n0 0 1 [0] 0 7\n1 0 1 [0] 0 5\n", + 12, + ), + ( + "PARTITION LEADER EPOCH REPLICAS LOG-START-OFFSET HIGH-WATERMARK\n0 0 1 [0] 0 0\n", + 0, + ), + ("PARTITION LEADER\n0 0\n", None), + ("", None), + ( + "PARTITION LEADER EPOCH REPLICAS LOG-START-OFFSET HIGH-WATERMARK\n0 0 1 [0] 0 x\n", + None, + ), + ], +) +def test_a_topics_watermark_is_summed_or_unknown(text: str, want: int | None) -> None: + assert ( + dv.watermark_total( + text=text.split("\n", 1)[1] if text.startswith("region") else text + ) + == want + ) + + +@pytest.mark.parametrize( + ("text", "want"), + [ + ("Console login: admin / s3cret\n", ("admin", "s3cret")), + ("noise\nConsole login: a.b@c.io / pw\nmore", ("a.b@c.io", "pw")), + ("Console login: only\n", None), + ("", None), + ], +) +def test_the_minted_login_is_read_from_its_line(text: str, want: tuple | None) -> None: + assert dv.parse_login(text=text) == want + + +def test_landed_passes_a_pack_whose_sources_hold_rows() -> None: + result = only("landed", pack=pack(), stack=Answers(landing({"okta": 5}))) + assert not result.failures + + +def test_landed_fails_a_source_with_no_rows_and_names_its_query() -> None: + result = only( + "landed", + pack=pack(), + stack=Answers(landing({"okta": 0})), + clock=clock_of(1000), + ) + assert [failure.check for failure in result.failures] == ["landed"] + assert "dfe.okta holds 0 rows" in result.failures[0].detail + assert "SELECT count() FROM dfe.`okta`" in result.failures[0].detail + + +def coverage( + total: int, errored: int, *fields: int +) -> Callable[[dv.Request], dv.Reply]: + return lambda request: rows(total, errored, *fields) + + +@pytest.mark.parametrize( + ("answer", "needle"), + [ + (coverage(100, 0, 100, 100), None), + (coverage(100, 0, 90, 95), None), + ( + coverage(100, 3, 100, 100), + "3 of 100 rows carry error.message or a pipeline_error tag", + ), + (coverage(100, 0, 100, 89), "_json.a.b is set on 89 of 100 rows (89%)"), + (coverage(100, 0, 0, 0), "event_action is set on 0 of 100 rows"), + (coverage(0, 0, 0, 0), "holds 0 rows"), + (lambda request: REFUSED, "could not be read"), + ], +) +def test_parsed_judges_errors_and_each_expected_field( + answer: Callable[[dv.Request], dv.Reply], needle: str | None +) -> None: + result = only("parsed", pack=pack(), stack=Answers(answer)) + if needle is None: + assert not result.failures + else: + assert any(needle in failure.detail for failure in result.failures), result + + +def test_parsed_names_the_query_that_finds_the_errored_rows() -> None: + result = only("parsed", pack=pack(), stack=Answers(coverage(10, 2, 10, 10))) + assert ( + "WHERE (JSONHas(toJSONString(_json), 'error', 'message')" + in result.failures[0].detail + ) + + +@pytest.mark.parametrize( + ("counts", "failed"), + [ + ({"demo-syslog": 0, "main": 0}, []), + ({"demo-syslog": 4, "main": 0}, ["demo-syslog"]), + ({"demo-syslog": 0, "main": 2}, ["main"]), + ({"demo-syslog": 1, "main": 1}, ["demo-syslog", "main"]), + ], +) +def test_unclaimed_fails_a_table_no_source_should_have_filled( + counts: dict[str, int], failed: list[str] +) -> None: + result = only("unclaimed", pack=pack(), stack=Answers(landing(counts))) + assert [f.detail.split("`")[1] for f in result.failures] == failed + + +def test_unclaimed_reads_only_the_default_table_for_a_pack_with_no_intake() -> None: + stack = Answers(landing({"main": 0})) + assert not only("unclaimed", pack=pack(intake=None), stack=stack).failures + assert len(stack.sent) == 1 + + +def listing(*items: dict) -> Callable[[dv.Request], dv.Reply]: + return lambda request: dv.Reply(200, {"items": list(items)}) + + +def source(name: str, *, state: str = "active", deployed: str | None = "1.0.0") -> dict: + return { + "name": name, + "state": state, + "current": "1.0.0", + "deployed_version": deployed, + } + + +@pytest.mark.parametrize( + ("items", "needle"), + [ + ([source("okta"), source("wiki"), source("demo-syslog")], None), + ([source("okta"), source("demo-syslog")], "wiki is not in GET /api/v1/sources"), + ( + [source("okta"), source("wiki", state="dormant"), source("demo-syslog")], + "wiki is 'dormant', its file declares 'active'", + ), + ( + [source("okta"), source("wiki", deployed=None), source("demo-syslog")], + "wiki is deployed at None", + ), + ( + [source("okta"), source("wiki", deployed="0.9.0"), source("demo-syslog")], + "deployed at '0.9.0'", + ), + ], +) +def test_sources_must_be_deployed_in_the_state_their_file_declares( + items: list[dict], needle: str | None +) -> None: + result = only("sources", pack=pack(), stack=Answers(listing(*items))) + if needle is None: + assert not result.failures + else: + assert any(needle in failure.detail for failure in result.failures) + + +def test_a_dormant_source_passes_where_its_file_declares_dormant() -> None: + items = [source("okta"), source("wiki"), source("demo-syslog", state="dormant")] + states = {"okta": "active", "wiki": "active", "demo-syslog": "dormant"} + assert not only( + "sources", pack=pack(states=states), stack=Answers(listing(*items)) + ).failures + + +def test_sources_fails_when_the_engine_does_not_answer() -> None: + result = only("sources", pack=pack(), stack=Answers(lambda request: REFUSED)) + assert "GET /api/v1/sources answered HTTP 500" in result.failures[0].detail + + +@pytest.mark.parametrize( + ("detections", "limits", "needle", "retry"), + [ + ( + [("demo-hunt", "demo-rule", 3)], + {"demo-hunt": {"max_alerts": 10}}, + None, + None, + ), + ( + [("demo-hunt", "demo-rule", 10)], + {"demo-hunt": {"max_alerts": 10}}, + None, + None, + ), + ( + [], + {"demo-hunt": {"max_alerts": 10}}, + "rule demo-rule wrote no detection yet", + True, + ), + ( + [("demo-hunt", "demo-rule", 11)], + {"demo-hunt": {"max_alerts": 10}}, + "wrote 11 detections, over its ceiling of 10", + False, + ), + ( + [("demo-hunt", "demo-rule", 3)], + {}, + "names no max_alerts for hunt demo-hunt", + False, + ), + ( + [("demo-hunt", "other-rule", 3)], + {"demo-hunt": {"max_alerts": 10}}, + "rule demo-rule wrote no detection yet", + True, + ), + ], +) +def test_hunts_must_fire_every_rule_and_stay_under_the_ceiling( + detections: list[tuple], limits: dict, needle: str | None, retry: bool | None +) -> None: + stack = Answers(lambda request: table(*detections) if detections else NONE) + result = only( + "hunts", pack=pack(hunt_limits=limits), stack=stack, clock=clock_of(1000) + ) + if needle is None: + assert not result.failures + else: + assert any(needle in failure.detail for failure in result.failures) + assert all(failure.retry is retry for failure in result.failures) + + +def test_a_hunt_over_its_ceiling_is_not_waited_on() -> None: + sleeps: list[float] = [] + stack = Answers(lambda request: table(("demo-hunt", "demo-rule", 99))) + dv.verify(only=["hunts"], pack=pack(), stack=stack, sleep=sleeps.append) + assert sleeps == [] + + +def console( + *, dashboards: list[dict], searches: list[dict] +) -> Callable[[dv.Request, Callable], dv.Reply]: + def answer(request: dv.Request, run: Callable[[str], dv.Reply]) -> dv.Reply: + if request.url.endswith("/sources"): + return dv.Reply(200, [{"id": "s1", "from": {"tableName": "okta"}}]) + if request.url.endswith("/dashboards"): + return dv.Reply(200, dashboards) + if request.url.endswith("/saved-search"): + return dv.Reply(200, searches) + return run(request.payload["query"]) + + return answer + + +TILE = { + "id": "t1", + "config": { + "displayType": "table", + "sqlTemplate": "SELECT 1 FROM dfe.okta WHERE $__timeFilter(t)", + }, +} +NUMBER = { + "id": "n1", + "config": {"displayType": "number", "sqlTemplate": "SELECT count() FROM dfe.okta"}, +} +SEARCH = {"name": "demo-t-live", "source": "s1", "select": "a", "where": "b = 1"} + + +def run_console( + dashboards: list[dict], + searches: list[dict], + run: Callable[[str], dv.Reply], + **changes: object, +) -> dv.Result: + answer = console(dashboards=dashboards, searches=searches) + stack = Answers(lambda request: answer(request, run)) + return only("console", pack=pack(**changes), stack=stack, clock=clock_of(1000)) + + +@pytest.mark.parametrize( + ("dashboards", "searches", "run", "changes", "needle"), + [ + ( + [{"name": "demo-t", "tiles": [TILE]}], + [SEARCH], + lambda sql: rows(4), + {}, + None, + ), + ([], [SEARCH], lambda sql: rows(4), {}, "dashboard demo-t is not installed"), + ( + [{"name": "demo-t", "tiles": [TILE]}], + [], + lambda sql: rows(4), + {}, + "saved search demo-t-live is not installed", + ), + ( + [{"name": "demo-t", "tiles": [TILE]}], + [SEARCH], + lambda sql: NONE, + {}, + "dashboard demo-t, tile t1 returns no rows or reads 0", + ), + ( + [{"name": "demo-t", "tiles": [TILE]}], + [SEARCH], + lambda sql: rows(0) if "LIMIT 100" in sql else rows(1), + {}, + "saved search demo-t-live returns no rows or reads 0", + ), + ( + [{"name": "demo-t", "tiles": [TILE]}], + [SEARCH], + lambda sql: REFUSED, + {}, + "did not run (HTTP 500", + ), + ( + [{"name": "demo-t", "tiles": [NUMBER]}], + [SEARCH], + lambda sql: rows(0) if "count()" in sql and "LIMIT" not in sql else rows(1), + {}, + "demo-t/n1 is not in verify.json zero_ok", + ), + ( + [{"name": "demo-t", "tiles": [NUMBER]}], + [SEARCH], + lambda sql: rows(0) if "count()" in sql and "LIMIT" not in sql else rows(1), + {"zero_ok": {"demo-t/n1"}}, + None, + ), + ( + [{"name": "demo-t", "tiles": [TILE]}], + [SEARCH], + lambda sql: NONE if "LIMIT" not in sql else rows(0), + {"zero_ok": {"demo-t/t1", "demo-t-live"}}, + None, + ), + ], +) +def test_console_runs_every_tile_and_search( + dashboards, searches, run, changes, needle +) -> None: + result = run_console(dashboards, searches, run, **changes) + if needle is None: + assert not result.failures, result + else: + assert any(needle in failure.detail for failure in result.failures), result + + +def test_console_refuses_a_search_that_reads_a_source_the_console_lacks() -> None: + result = run_console( + [{"name": "demo-t", "tiles": []}], + [{**SEARCH, "source": "gone"}], + lambda sql: rows(1), + ) + assert ( + "reads source 'gone', which dfe-hyperdx does not hold" + in result.failures[0].detail + ) + + +def test_search_runs_the_first_saved_search_as_the_console_login() -> None: + answer = console(dashboards=[], searches=[SEARCH]) + stack = Answers(lambda request: answer(request, lambda sql: rows(3))) + assert not only("search", pack=pack(), stack=stack).failures + assert stack.sent[-1].identity == "console" + + +@pytest.mark.parametrize( + ("run", "needle"), + [ + ( + lambda sql: dv.Reply(403, {"detail": "password change required"}), + "as the console login got HTTP 403", + ), + (lambda sql: rows(0), "as the console login returned no rows"), + ], +) +def test_search_fails_where_the_console_login_cannot_read_rows(run, needle) -> None: + answer = console(dashboards=[], searches=[SEARCH]) + result = only( + "search", pack=pack(), stack=Answers(lambda request: answer(request, run)) + ) + assert needle in result.failures[0].detail + + +def test_fetchers_skip_a_pack_with_none_and_say_why_for_an_unreplayed_one() -> None: + assert only("fetchers", pack=pack(), stack=Answers(landing({}))).skipped + unreplayed = only( + "fetchers", pack=pack(fetchers=["wiki"]), stack=Answers(landing({})) + ) + assert unreplayed.skipped + assert "skipped wiki: no captured poll" in unreplayed.detail + + +@pytest.mark.parametrize( + ("count", "failed"), + [(7, False), (0, True)], +) +def test_fetchers_need_rows_where_the_polls_are_replayed( + count: int, failed: bool +) -> None: + result = only( + "fetchers", + pack=pack(fetchers=["wiki"], polled=["wiki"]), + stack=Answers(landing({"wiki": count})), + clock=clock_of(1000), + ) + assert bool(result.failures) is failed + if failed: + assert ( + "dfe.`wiki` holds 0 rows after its replayed polls" + in result.failures[0].detail + ) + + +@pytest.mark.parametrize( + ("dead", "marks", "services", "needles"), + [ + (NONE, None, None, []), + ( + table(("dfe-loader", "okta", "dfe.okta", "type mismatch", 3)), + None, + None, + [ + "3 records dead-lettered by dfe-loader for source 'okta'", + "type mismatch", + ], + ), + (NONE, {"dfe_loader_dlq": 2}, None, ["topic dfe_loader_dlq holds 2 records"]), + ( + NONE, + {"dfe_fetcher_dlq": None}, + None, + ["topic dfe_fetcher_dlq is missing or unreadable"], + ), + (NONE, {"dfe_loader_dlq": 9}, ["dfe-engine"], []), + (REFUSED, None, None, ["could not be read"]), + ], +) +def test_dlq_fails_on_any_record_in_the_table_or_a_topic( + dead, marks, services, needles +) -> None: + stack = Answers( + lambda request: dead, + marks={**dict.fromkeys(dv.DLQ_TOPICS, 0), **(marks or {})}, + services=services or ["dfe-engine", "kafka-redpanda"], + ) + result = only("dlq", pack=pack(), stack=stack) + for needle in needles: + assert any(needle in failure.detail for failure in result.failures), result + assert bool(result.failures) is bool(needles) + + +def test_verify_runs_the_checks_in_order_and_only_the_wanted_ones() -> None: + stack = Answers( + lambda request: ( + rows(1) + if "count()" in request.payload["query"] + and "GROUP BY" not in request.payload["query"] + else NONE + ) + ) + results = dv.verify(only=["dlq", "landed"], pack=pack(), stack=stack) + assert [result.check for result in results] == ["landed", "dlq"] + + +def test_a_check_that_never_holds_is_asked_again_until_its_wait_ends() -> None: + sleeps: list[float] = [] + result = dv.wait_for( + probe=lambda: dv.Result("x", "", (dv.Failure("x", "not yet"),)), + timeout=20, + clock=clock_of(5), + sleep=sleeps.append, + ) + assert result.failures + assert len(sleeps) == 3 + + +@pytest.mark.parametrize("name", demo_pack_names()) +def test_a_real_pack_reads_and_declares_what_it_expects(name: str) -> None: + loaded = dv.load_pack(name=f"demo-{name}") + assert loaded.sources + assert set(loaded.hunts) == set(loaded.hunt_limits) + ceiling = 6 * (publish_compose.DEMO_MAX_DETECTIONS + 1) + assert all( + spec.get("max_alerts") == ceiling for spec in loaded.hunt_limits.values() + ) + assert set(loaded.parsed) <= set(loaded.sources) + for source in [*loaded.replayed, *loaded.polled]: + assert loaded.parsed.get(source, {}).get("fields"), ( + f"{source} names no expected field" + ) + for spec in loaded.parsed.values(): + for field in spec["fields"]: + assert re.fullmatch(dv.FIELD_PATH.pattern, field) + + +def test_the_base_demo_reads_and_declares_what_it_expects() -> None: + loaded = dv.load_pack(name="demo") + assert loaded.intake == "demo-syslog" + assert set(loaded.hunts) == set(loaded.hunt_limits) + for source in loaded.replayed: + assert loaded.parsed.get(source, {}).get("fields") + + +def test_a_pack_with_no_verify_file_is_refused(tmp_path) -> None: + (tmp_path / "demo.json").write_text( + '{"sources": [], "console": {"dashboards": [], "searches": []}}' + ) + with pytest.raises(dv.VerifyError, match="verify.json"): + dv.load_pack(name="demo-x", root=tmp_path) + + +def test_a_wait_scale_of_zero_tries_each_check_once() -> None: + sleeps: list[float] = [] + result = dv.verify( + only=["landed"], + pack=pack(), + stack=Answers(landing({"okta": 0})), + sleep=sleeps.append, + wait_scale=0, + )[0] + + assert result.failures + assert sleeps == [] + + +def test_search_skips_a_search_that_may_return_nothing() -> None: + answer = console(dashboards=[], searches=[SEARCH, {**SEARCH, "name": "demo-t-b"}]) + stack = Answers(lambda request: answer(request, lambda sql: rows(3))) + result = only( + "search", + pack=pack(searches=["demo-t-live", "demo-t-b"], zero_ok={"demo-t-live"}), + stack=stack, + ) + + assert result.detail.startswith("1 saved searches returned rows") + + +def test_a_waiting_source_is_named_by_fetchers_and_only_its_rows_leave_the_default_table_count() -> ( + None +): + waiting = pack( + fetchers=["wiki"], + waiting_sources=["wiki"], + waiting_reason="a loader fix", + ) + + def answer(request: dv.Request) -> dv.Reply: + query = request.payload["query"] + if "dfe.`main`" in query: + return rows( + 0 if "NOT startsWith(toString(_source), 'wiki')" in query else 5 + ) + return rows(0) + + stack = Answers(answer) + + fetchers = only("fetchers", pack=waiting, stack=stack) + unclaimed = only("unclaimed", pack=waiting, stack=stack) + + assert fetchers.skipped + assert "skipped wiki: waiting on a loader fix" in fetchers.detail + assert not unclaimed.failures + assert "wiki rows left out of main" in unclaimed.detail + + +def test_another_source_s_row_in_the_default_table_still_fails_while_one_waits() -> ( + None +): + waiting = pack(waiting_sources=["wiki"], waiting_reason="a loader fix") + stack = Answers( + lambda request: rows(2) if "dfe.`main`" in request.payload["query"] else rows(0) + ) + + unclaimed = only("unclaimed", pack=waiting, stack=stack) + + assert "dfe.`main` holds 2 rows no source claimed" in unclaimed.failures[0].detail + + +def test_a_waiting_entry_fails_once_the_source_holds_rows_of_its_own() -> None: + waiting = pack(waiting_sources=["wiki"], waiting_reason="a loader fix") + stack = Answers( + lambda request: rows(3) if "dfe.`wiki`" in request.payload["query"] else rows(0) + ) + + unclaimed = only("unclaimed", pack=waiting, stack=stack) + + assert "no longer waits on a loader fix" in unclaimed.failures[0].detail + assert "Remove it from verify.json waiting" in unclaimed.failures[0].detail + + +def test_search_names_why_the_console_login_could_not_be_read() -> None: + stack = Answers(lambda request: rows(3)) + stack.login_fault = "secrets-init login exited 1 and printed no line" + + result = only("search", pack=pack(), stack=stack) + + assert ( + result.failures[0].detail == "secrets-init login exited 1 and printed no line" + ) + + +def test_search_runs_every_saved_search_and_names_the_one_that_returns_nothing() -> ( + None +): + second = {**SEARCH, "name": "demo-t-b", "where": "empty = 1"} + answer = console(dashboards=[], searches=[SEARCH, second]) + stack = Answers( + lambda request: answer( + request, lambda sql: rows(0) if "empty = 1" in sql else rows(3) + ) + ) + + result = only( + "search", pack=pack(searches=["demo-t-live", "demo-t-b"]), stack=stack + ) + + assert [failure.detail.split(" ")[0] for failure in result.failures] == ["demo-t-b"] diff --git a/scripts/tests/test_publish_compose.py b/scripts/tests/test_publish_compose.py index e44ca3f1..fc26f6c9 100644 --- a/scripts/tests/test_publish_compose.py +++ b/scripts/tests/test_publish_compose.py @@ -662,6 +662,7 @@ def _stack(services: list[str]) -> dict: DEMO_STACK = [ "dfe-engine", + "dfe-hunt-runner", "dfe-receiver", "dfe-hyperdx", "dlq-init", @@ -726,6 +727,32 @@ def test_holds_the_data_path_until_the_sources_exist(self) -> None: "--force-recreate {service}" ) + def test_caps_the_detections_a_rule_writes_each_run(self) -> None: + model = publish_compose.demo_overlay( + archive=ARCHIVE, + image="ghcr.io/hyperi-io/dfe-docker@sha256:" + "b" * 64, + model=_stack(DEMO_STACK), + roles=ROLES, + stream=["scripts/replay_syslog.py", "tcp", "20"], + ) + + runner = model["services"]["dfe-hunt-runner"]["environment"] + assert runner[publish_compose.MAX_DETECTIONS_KEY] == str( + publish_compose.DEMO_MAX_DETECTIONS + ) + + def test_refuses_a_stack_with_no_hunt_runner(self) -> None: + with pytest.raises(publish_compose.PublishError, match="dfe-hunt-runner"): + publish_compose.demo_overlay( + archive=ARCHIVE, + image="ghcr.io/hyperi-io/dfe-docker@sha256:" + "b" * 64, + model=_stack( + [name for name in DEMO_STACK if name != "dfe-hunt-runner"] + ), + roles=ROLES, + stream=["scripts/replay_syslog.py", "tcp", "20"], + ) + def test_a_pack_with_no_sources_seeds_itself_and_replays_nothing(self) -> None: stack = _stack( [name for name in DEMO_STACK if name != "dfe-transform-elastic-demo-syslog"]