diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb3decd..2a24aca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,6 +67,8 @@ jobs: uses: hyperi-io/hyperi-ci/.github/workflows/rust-ci.yml@main with: publish-target: both + # The filebeat test reads the pipeline and cases from this repo, beside the checkout. + sibling-checkouts: hyperi-io/dfe-transform-vrl tag: ${{ inputs.tag || '' }} from-head: ${{ inputs.from-head || '' }} bump: ${{ inputs.bump || 'auto' }} diff --git a/.gitleaks.toml b/.gitleaks.toml index 634b37f..d9b4a40 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -9,6 +9,6 @@ description = "Project-specific false positive allowlist" paths = [ '''(^|/)ci/''', '''(^|/)ai/''', - # Commits before the history rewrite still hold the deleted file, and gitleaks scans them. + # The Meraki event subtype '8021x_eap_failure' matches a key rule. '''pipelines/filebeat/filebeat\.yaml''', ] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bb9ee88..e6321a1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -151,6 +151,7 @@ A Rust supervisor that runs Vector as a child process so a Vector pipeline behav | `src/health.rs`, `src/metrics.rs`, `src/metrics/scrape.rs` | Readiness publishing, and the scrape that merges `vector_*` into scalo's registry | | `src/deployment.rs`, `src/vector-layer.dockerfile` | `contract()`, the source the Dockerfile and the released chart are generated from, and the two-binary image override spliced onto scalo's generated Dockerfile | | `templates/bus.yaml`, `templates/direct.yaml` | Whole runnable Vector topologies, one per transport, commented per field | +| `pipelines/filebeat/` | The bundled filebeat pipeline. The image does not carry it | | `docs/` | `architecture.md` for why the service is built this way and the invariants, `internals.md` for field-by-field depth, `migration.md`, `librdkafka.md`, the generated `config-schema.*` and `capability-catalog.*` | | `tests/` | `integration`, `e2e`, `smoke`, and `TESTING.md` for how the broker and Vector binary are resolved | | `scripts/fetch-vector.sh`, `scripts/pgo-workload.sh` | Downloads the pinned Vector for tests, and drives the PGO/BOLT workload | @@ -170,7 +171,7 @@ What green does NOT mean: - The default run leaves out the `#[ignore]` cases in `tests/integration/vector_validate.rs` and `tests/e2e/kafka.rs`. The `vector_validate` ones run Vector's own validator over each buffer, SASL and TLS variant of an assembled config, so until you pass `--run-ignored all` those variants are unchecked. - Locally, a missing broker or Vector binary SKIPS rather than fails. In CI that absence is an assertion failure for the cases that run by default, so CI is stricter than a green laptop run. -- `e2e::app_kafka` runs by default: the shipped binary over a shared broker with an authored transform file. Without Docker it skips locally and fails under CI. +- `e2e::filebeat_kafka` runs by default and needs a dfe-transform-vrl checkout, for its cases and the bundled pipeline: beside this repo, or named by `DFE_TRANSFORM_VRL_DIR`. Without one it skips locally and fails under CI, which checks the repo out beside this one. - A docs-only push runs no CI at all. `paths-ignore` for `docs/**` and `**.md` is on the `push` trigger but not on `pull_request`, so the PR is the only place a docs change is checked. A silent push is not a pass. ### What tends to bite @@ -208,3 +209,4 @@ Outbound -- what depends on this repo: Two more relationships that are real but are NOT declared edges in the suite graph, so no gate enforces them: - dfe-infra `apps.yaml` is the SSoT for how this app is deployed -- multiplicity, scaling, transports, endpoints and which compiler derives its routing. dfe-engine reads that manifest, so adding or changing the app is a manifest edit, never an engine release. +- dfe-transform-vrl holds the filebeat cases (`tests/fixtures/filebeat/cases.yaml`) and the bundled pipeline that `e2e::filebeat_kafka` grades against. The dependency is a test-time checkout only, which CI makes and a laptop may not. diff --git a/pipelines/filebeat/README.md b/pipelines/filebeat/README.md new file mode 100644 index 0000000..db221c2 --- /dev/null +++ b/pipelines/filebeat/README.md @@ -0,0 +1,36 @@ + + +# Bundled pipeline: filebeat-compat (INTERIM) + +A pre-canned transform file that keeps DFE 2.1 filebeat feeds working on dfe-transform-vector: the same VRL dfe-transform-vrl runs, wrapped in a Vector `remap` and carrying the enrichment table it needs. Point `transforms.dir` at this directory, mount the table, and the pipeline runs -- there is nothing to author. The modules, the routing rules, the departures from DFE 2.1 and the known limitations are documented once, in dfe-transform-vrl `pipelines/filebeat/README.md`, and apply here unchanged. + +> **INTERIM.** Elastic compatibility is being replaced by +> `dfe-transform-elastic` (Rust-native). New integrations +> should NOT build on this pipeline. + +## Using it + +```yaml +# config.yaml +dfe_source: "filebeat" # derives filebeat_land -> filebeat_load +transforms: + dir: "/etc/dfe-transform-vector/transforms" +``` + +Mount `filebeat.yaml` into that directory and `timezones.csv` at the path the pipeline's `enrichment_tables` entry names (`/etc/dfe-transform-vector/data/timezones.csv`), or edit that one path to wherever the table lives inside `transforms.data_dir`. Both files are here. + +The wrapper writes each transform file into Vector's config dir with its wired `inputs` added, and a transform file may declare a top-level `enrichment_tables` key, so Vector merges the declaration and the VRL's timezone lookups resolve. `vector validate` runs before Vector starts, so a pipeline that cannot compile stops the app rather than passing nothing on. + +Events carry the DFE 2.1 Kafka fields (`{message, tags, timestamp}`), and the optional `._conf.tz_offset` / `._conf.tz_map` fields drive the ios and meraki timezone handling. + +## Provenance + +`filebeat.yaml` embeds dfe-transform-vrl `pipelines/filebeat/filebeat.vrl` @ `e0b7087`, the same program bar two lines of its header comment, and `timezones.csv` is that repo's table copied. Refresh by copying both again rather than editing here -- the two transform apps grade against the same cases (`tests/e2e/filebeat_kafka.rs` in each), and that only means something while the program is the same one. diff --git a/pipelines/filebeat/filebeat.yaml b/pipelines/filebeat/filebeat.yaml new file mode 100644 index 0000000..789108d --- /dev/null +++ b/pipelines/filebeat/filebeat.yaml @@ -0,0 +1,5583 @@ +# Project: dfe-transform-vector +# File: pipelines/filebeat/filebeat.yaml +# Purpose: Bundled filebeat pipeline - the DFE 2.1 filebeat VRL as a Vector remap +# Language: YAML +# +# License: BUSL-1.1 +# Copyright: (c) 2026 HYPERI PTY LIMITED +# +# A pre-canned transform file for `transforms.dir`. The wrapper parses it and +# writes it into Vector's config dir with its wired `inputs`, so the top-level +# `enrichment_tables` key reaches Vector and the VRL's timezone lookups resolve. +# `inputs: [dfe_source]` binds it to whatever Kafka topic `dfe_source` derives, +# and the wrapper wires the transform's output to `dfe_sink`. +# +# The `path` below is where the dfe-docker compose stack mounts the table. The +# image does not ship timezones.csv, so any other deployment mounts it at that +# path or points `path` at its own copy; startup validation names the path when +# the file is not there. +# +# The VRL body is dfe-transform-vrl pipelines/filebeat/filebeat.vrl @ e0b7087, +# bar two lines of its header comment, and that repo maintains it. Refresh by copying +# that file again rather than editing the copy here, so the two transform apps +# keep passing the same cases. + +enrichment_tables: + timezones: + type: file + file: + path: /etc/dfe-transform-vector/data/timezones.csv + encoding: + type: csv + schema: + abbreviation: string + +transforms: + filebeat: + type: remap + inputs: + - dfe_source + source: | + ############################################################################## + # dfe-transform-vrl bundled pipeline: filebeat-compat -- INTERIM + # + # Single-file port of the DFE 2.1 Vector filebeat transform templates + # (dfe-vector-templates src/core_templates 14x family): + # 140-transform-filebeat-cisco-meraki-logs + # 141-transform-filebeat-cisco-ios-logs + # 143..149-transform-filebeat-cisco-umbrella-*-logs (7 files, one body) + # + # The DFE 2.1 Vector deployment split this across per-module files to work + # around Vector's large-VRL-file limits, and its route transform fanned + # every {message,tags,timestamp} event to EVERY module transform, relying + # on drop_on_abort to discard the wrong parses. Embedded VRL has no + # file-size problem and runs one program per event, so this port detects + # the module first and runs only that module's transform. The seven + # umbrella templates differed only in the hardcoded .log.file.path seed; + # they collapse to one body driven by the detected log type. + # + # INTERIM: this pipeline keeps DFE 2.1 filebeat feeds working on + # dfe-transform-vrl. Elastic compatibility is being replaced by + # dfe-transform-elastic (Rust-native), and new integrations should NOT + # build on this file. + # + # Deliberate departures from the 2.1 templates: + # - geoip enrichment is NOT ported: dfe-loader owns geoip enrichment in + # the current DFE pipeline layout. + # - unmatched events are tagged "filebeat_unmatched" and passed through + # instead of being silently dropped by the route fan-out. + # + # Requirements (matching the 2.1 deployment contract): + # - enrichment table "timezones" (bundled timezones.csv, key column + # "abbreviation") for cisco-ios and meraki timestamp resolution. + # - events MAY carry ._conf.tz_offset (abbreviation like "UTC" or offset + # like "-0600"); without it the ios/meraki @timestamp mapping is + # skipped, exactly as in DFE 2.1. + ############################################################################## + + _ingest = {} + _ingest.timestamp = now() + + # --- module detection ------------------------------------------------------ + _module = "unknown" + _umbrella_type = "" + _msg = string(.message) ?? string(.event.original) ?? "" + + if match(_msg, r'^<134>1 \d{9,10}\.\d+ ') { + # Meraki syslog: <134>1 ... + # (DFE 2.1 routed on the <134> prefix alone; the epoch check keeps a + # local0.info Cisco IOS device out of the meraki branch.) + _module = "cisco_meraki_logs" + } else if match(_msg, r'^<\d{1,3}>') { + # Any other syslog-priority-prefixed line is Cisco IOS (the only + # syslog module in the 14x family besides Meraki). + _module = "cisco_ios_logs" + } else if match(_msg, r'%[A-Z][A-Z0-9_-]*-\d-[A-Z0-9_]+') && !match(_msg, r'^"?(\d{4}-\d{2}-\d{2} |\d+",")') { + # Cisco IOS %FACILITY-SEVERITY-MNEMONIC marker without a syslog + # header. The facility itself may carry dashes + # (e.g. %SNMP-SW1-3-RESPONSE_DELAYED). Umbrella CSV rows open with an + # ISO date (audit: a numeric id column), so a marker-lookalike inside + # a CSV field (a proxy URL, a dlp filename) must not steal the line + # from the CSV branch. + _module = "cisco_ios_logs" + } else { + # Cisco Umbrella exports are CSV; DFE 2.1 carried the log type in the + # S3 object path. Honour an upstream-provided .log.file.path first, + # then fall back to column-shape detection (counts verified against + # the elastic/integrations pipeline test corpus). + _path = string(.log.file.path) ?? "" + if contains(_path, "auditlogs") { + _umbrella_type = "auditlogs" + } else if contains(_path, "firewalllogs") { + _umbrella_type = "cloudfirewalllogs" + } else if contains(_path, "dlplogs") { + _umbrella_type = "dlplogs" + } else if contains(_path, "dnslogs") { + _umbrella_type = "dnslogs" + } else if contains(_path, "intrusionlogs") { + _umbrella_type = "intrusionlogs" + } else if contains(_path, "iplogs") { + _umbrella_type = "iplogs" + } else if contains(_path, "proxylogs") { + _umbrella_type = "proxylogs" + } + + if _umbrella_type == "" && _msg != "" { + # Same `", "` normalisation the umbrella body applies before + # parse_csv (the export quotes fields inconsistently). + _csv, _csv_err = parse_csv(replace(_msg, r'", "', s'","')) + if _csv_err == null { + _n = length(_csv) + _c0 = string(_csv[0]) ?? "" + _c1 = string(_csv[1]) ?? "" + _ts_re = r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}' + if _n == 7 && match(_c0, _ts_re) { + _umbrella_type = "iplogs" + } else if _n == 9 && match(_c1, _ts_re) { + # v5 audit format: numeric event id, then the timestamp. + _umbrella_type = "auditlogs" + } else if _n == 13 && match(_c0, _ts_re) { + _umbrella_type = "dnslogs" + } else if _n >= 10 && _n <= 12 && match(_c0, _ts_re) && match(string(_csv[6]) ?? "", r'^\d+ \(') { + # Older dns export without the trailing policy columns; + # column 6 is the " (TYPE)" dns query type. + _umbrella_type = "dnslogs" + } else if (_n == 14 || _n == 16) && match(_c0, _ts_re) { + # cloud firewall: 14 columns, or 16 with fqdns + + # destination list ids appended. + _umbrella_type = "cloudfirewalllogs" + } else if _n == 17 && match(_c0, _ts_re) { + # dlp and intrusion both carry 17 columns; intrusion has + # numeric gid/sid at columns 3 and 4. + if match(string(_csv[3]) ?? "", r'^\d+$') && match(string(_csv[4]) ?? "", r'^\d+$') { + _umbrella_type = "intrusionlogs" + } else { + _umbrella_type = "dlplogs" + } + } else if _n >= 20 && match(_c0, _ts_re) { + _umbrella_type = "proxylogs" + } + } + } + if _umbrella_type != "" { + _module = "cisco_umbrella" + } + } + + if _module == "cisco_meraki_logs" { + _ingest.pipeline = "cisco_meraki_logs" + _ingest.on_failure_pipeline = "cisco_meraki_logs" + .tags = ["preserve_original_event", "forwarded"] + .ecs.version = "8.11.0" + if .event.original == null { + if .message != null { + if .event.original == null { + .event.original = del(.message) + } else { + assert!(false, ".event.original must be null or override: true") + } + } + } + if .event.original != null { + tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} %{DATA:_temp.ts_nano} %{DATA:observer.hostname} %{DATA:cisco_meraki.event_type} %{DATA}$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "failed to dissect .event.original: " + to_string(err)) + } + } else { + err = ".event.original must not be nullish" + assert!(false, err) + } + if ._conf.tz_offset != null && ._conf.tz_offset != "local" { + date = null + err = "no datetime formats provided" + if err != null { + datetime = to_string!(._temp.ts_nano) + external_timezone = ._conf.tz_offset + if false { + external_timezone = null + } + records = if match!(external_timezone, r'[\+-]?[0-9]{1,2}:?[0-9]{2}') { + [{ "offset": external_timezone }] + } else { + find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": external_timezone }) + } + if length(records) == 1 { + offset = records[0].offset + date, err = parse_timestamp(value: join!(value: [datetime, offset], separator: " "), format: join!(value: ["%s%.f", "%:z"], separator: " ")) + } else { + if length(records) > 1 { + err = "ambiguous timezone abbreviation" + } else { + if length(records) == 0 { + err = "no timezone abbreviation found" + } + } + } + } + if err == null { + .@timestamp = date + } else { + _ingest.on_failure_processor_type = "date" + _ingest.on_failure_processor_tag = "" + _ingest.on_failure_message = to_string(err) + if !is_array(.error.message) { + if .error.message == null { + .error.message = [] + } else { + .error.message = [.error.message] + } + } + .error.message = append!({ + expr = .error.message + if false { + expr = null + } + expr + }, [join!(["failed to parse time field (", to_string!(._temp.ts_nano), "): ", _ingest.on_failure_message], "")]) + } + } + if ._conf.tz_offset == null || ._conf.tz_offset == "local" { + date = null + err = "no datetime formats provided" + if err != null { + datetime = to_string!(._temp.ts_nano) + date, err = parse_timestamp(value: join!(value: [datetime], separator: " "), format: join!(value: ["%s%.f"], separator: " ")) + } + if err == null { + .@timestamp = date + } else { + _ingest.on_failure_processor_type = "date" + _ingest.on_failure_processor_tag = "" + _ingest.on_failure_message = to_string(err) + if !is_array(.error.message) { + if .error.message == null { + .error.message = [] + } else { + .error.message = [.error.message] + } + } + .error.message = append!({ + expr = .error.message + if false { + expr = null + } + expr + }, [join!(["failed to parse time field (", to_string!(._temp.ts_nano), "): ", _ingest.on_failure_message], "")]) + } + } + if { + expr = ["flows", "firewall", "vpn_firewall", "cellular_firewall", "bridge_anyconnect_client_vpn_firewall"] + value = .cisco_meraki.event_type + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + _ingest.timestamp = now() + if _ingest.pipeline == null { + _ingest.pipeline = "" + _ingest.on_failure_pipeline = "" + } + if !is_nullish(.event.original) { + value, err = parse_groks(value: .event.original, patterns: [s'.*?%{TYPE}( %{NOTSPACE:cisco_meraki.flows.op})? src=%{IP:source.ip} dst=%{IP:destination.ip}( mac=%{MAC:source.mac})? protocol=%{NOTSPACE:network.protocol}( type=%{NOTSPACE})?( sport=%{NONNEGINT:source.port})?( dport=%{NONNEGINT:destination.port})?( pattern: %{GREEDYDATA:cisco_meraki.firewall.pattern})?.*'], aliases: { "TYPE": s'(flows|firewall|vpn_firewall|cellular_firewall|bridge_anyconnect_client_vpn_firewall)' }) + if err == null { + if value.destination.ip != null { + value.destination.ip = to_string!(value.destination.ip) + } + if value.destination.port != null { + value.destination.port = parse_int!(value.destination.port) + } + if value.source.ip != null { + value.source.ip = to_string!(value.source.ip) + } + if value.source.port != null { + value.source.port = parse_int!(value.source.port) + } + . = merge(., value, deep: true) + } else { + assert!(false, "failed to grok .event.original:" + to_string(err)) + } + } else { + assert!(false, ".event.original must not be nullish") + } + if .cisco_meraki.firewall.pattern != null && (starts_with!({ + expr = .cisco_meraki.firewall.pattern + if false { + expr = null + } + expr + }, "allow") || starts_with!({ + expr = .cisco_meraki.firewall.pattern + if false { + expr = null + } + expr + }, "deny")) { + if !is_nullish(.cisco_meraki.firewall.pattern) { + value, err = parse_groks(value: .cisco_meraki.firewall.pattern, patterns: [s'.*?%{NOTSPACE:cisco_meraki.firewall.action} %{GREEDYDATA:cisco_meraki.firewall.rule}.*']) + if err == null { + . = merge(., value, deep: true) + } + } + } + if .cisco_meraki.firewall.rule != null { + if !exists(.cisco_meraki.firewall.pattern) { + assert!(false, "tried to remove field '.cisco_meraki.firewall.pattern' that doesn't exist") + } + del(.cisco_meraki.firewall.pattern) + } + if !is_nullish(.source.mac) { + if is_array(.source.mac) { + .source.mac = map_values(array!(.source.mac)) -> |value| { + replace!(value, r'[:.]', "-") + } + } else { + if false { + .source.mac = null + } + target, err = replace(.source.mac, r'[:.]', "-") + if err == null { + .source.mac = target + } else { + assert!(false, to_string(err)) + } + } + } + if .cisco_meraki.flows.op == null { + .cisco_meraki.event_subtype = "ip_session_initiated" + } + if .cisco_meraki.flows.op == "allow" { + .cisco_meraki.event_subtype = "flow_allowed" + } + if .cisco_meraki.flows.op == "deny" { + .cisco_meraki.event_subtype = "flow_denied" + } + } + if .cisco_meraki.event_type == "ip_flow_start" || .cisco_meraki.event_type == "ip_flow_end" { + _ingest.timestamp = now() + if _ingest.pipeline == null { + _ingest.pipeline = "" + _ingest.on_failure_pipeline = "" + } + if .event.original != null { + tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} %{DATA} %{DATA} %{DATA:_temp.event_type} %{DATA:_temp.event}$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "failed to dissect .event.original: " + to_string(err)) + } + } else { + err = ".event.original must not be nullish" + assert!(false, err) + } + if ._temp.event != null { + if !is_nullish(._temp.event) { + kv, err = parse_key_value(._temp.event, key_value_delimiter: "=", field_delimiter: " ", accept_standalone_key: true, whitespace: "strict") + if err == null { + standalone_key = false + for_each(kv) -> |_, value| { + if value == true { + standalone_key = true + } + } + if standalone_key { + err = "field ._temp.event does not contain value_split [=]" + } + } + if err == null { + if . == null { + . = { } + } + . = merge(object(.), kv, deep: true) + } else { + assert!(false, to_string(err)) + } + } else { + err = "._temp.event must not be nullish" + assert!(false, to_string(err)) + } + } + if .translated_src_ip != null { + if !is_nullish(.translated_src_ip) { + if is_ipv4(string!(.translated_src_ip)) || is_ipv6(string!(.translated_src_ip)) { + .source.ip = .translated_src_ip + } else { + err = "Failed converting .translated_src_ip to an IP address" + assert!(false, err) + } + } else { + err = ".translated_src_ip must be present" + assert!(false, err) + } + } + if .translated_src_ip == null && .src != null { + if !is_nullish(.src) { + if is_ipv4(string!(.src)) || is_ipv6(string!(.src)) { + .source.ip = .src + } else { + err = "Failed converting .src to an IP address" + assert!(false, err) + } + } else { + err = ".src must be present" + assert!(false, err) + } + } + if .translated_src_ip != null && .translated_port != null { + if !is_nullish(.translated_port) { + if is_string(.translated_port) { + temp = .translated_port + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .source.port = temp + } else { + err_msg = join!(["Failed converting .translated_port to an integer. Found (", to_string!(.translated_port), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.translated_port) { + .source.port = .translated_port + } else { + err = "Failed converting .translated_port to an integer" + assert!(false, err) + } + } + } else { + err = ".translated_port must be present" + assert!(false, err) + } + } + if .translated_src_ip == null && .sport != null { + if !is_nullish(.sport) { + if is_string(.sport) { + temp = .sport + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .source.port = temp + } else { + err_msg = join!(["Failed converting .sport to an integer. Found (", to_string!(.sport), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.sport) { + .source.port = .sport + } else { + err = "Failed converting .sport to an integer" + assert!(false, err) + } + } + } else { + err = ".sport must be present" + assert!(false, err) + } + } + if .translated_dst_ip != null { + if !is_nullish(.translated_dst_ip) { + if is_ipv4(string!(.translated_dst_ip)) || is_ipv6(string!(.translated_dst_ip)) { + .destination.ip = .translated_dst_ip + } else { + err = "Failed converting .translated_dst_ip to an IP address" + assert!(false, err) + } + } else { + err = ".translated_dst_ip must be present" + assert!(false, err) + } + } + if .translated_dst_ip == null && .dst != null { + if !is_nullish(.dst) { + if is_ipv4(string!(.dst)) || is_ipv6(string!(.dst)) { + .destination.ip = .dst + } else { + err = "Failed converting .dst to an IP address" + assert!(false, err) + } + } else { + err = ".dst must be present" + assert!(false, err) + } + } + if .translated_dst_ip != null && .translated_port != null { + if !is_nullish(.translated_port) { + if is_string(.translated_port) { + temp = .translated_port + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .destination.port = temp + } else { + err_msg = join!(["Failed converting .translated_port to an integer. Found (", to_string!(.translated_port), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.translated_port) { + .destination.port = .translated_port + } else { + err = "Failed converting .translated_port to an integer" + assert!(false, err) + } + } + } else { + err = ".translated_port must be present" + assert!(false, err) + } + } + if .translated_dst_ip == null && .dport != null { + if !is_nullish(.dport) { + if is_string(.dport) { + temp = .dport + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .destination.port = temp + } else { + err_msg = join!(["Failed converting .dport to an integer. Found (", to_string!(.dport), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.dport) { + .destination.port = .dport + } else { + err = "Failed converting .dport to an integer" + assert!(false, err) + } + } + } else { + err = ".dport must be present" + assert!(false, err) + } + } + if .protocol != null { + if .network.protocol == null { + .network.protocol = del(.protocol) + } else { + assert!(false, ".network.protocol must be null or override: true") + } + } else { + assert!(false, ".protocol must not be null") + } + } + if .cisco_meraki.event_type == "airmarshal_events" { + _ingest.timestamp = now() + if _ingest.pipeline == null { + _ingest.pipeline = "" + _ingest.on_failure_pipeline = "" + } + if .event.original != null { + tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} airmarshal_events type=%{DATA:type} %{DATA}$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "failed to dissect .event.original: " + to_string(err)) + } + } else { + err = ".event.original must not be nullish" + assert!(false, err) + } + if .type != null { + if .cisco_meraki.event_subtype == null { + .cisco_meraki.event_subtype = del(.type) + } else { + assert!(false, ".cisco_meraki.event_subtype must be null or override: true") + } + } else { + assert!(false, ".type must not be null") + } + if !is_nullish(.event.original) { + value, err = parse_groks(value: .event.original, patterns: [s'.*?%{GREEDYDATA} ssid=%{QS:_temp.ssid}%{SPACE}%{GREEDYDATA:_temp.kvline}.*']) + if err == null { + . = merge(., value, deep: true) + } else { + assert!(false, "failed to grok .event.original:" + to_string(err)) + } + } else { + assert!(false, ".event.original must not be nullish") + } + if ._temp.ssid != null { + tmp, err = parse_groks(value: ._temp.ssid, patterns: [s'^\'%{DATA:_temp.kv.ssid}\'.*?$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "failed to dissect ._temp.ssid: " + to_string(err)) + } + } else { + err = "._temp.ssid must not be nullish" + assert!(false, err) + } + if !is_nullish(._temp.kvline) { + kv, err = parse_key_value(._temp.kvline, key_value_delimiter: "=", field_delimiter: " ", accept_standalone_key: true, whitespace: "strict") + if err == null { + standalone_key = false + for_each(kv) -> |_, value| { + if value == true { + standalone_key = true + } + } + if standalone_key { + err = "field ._temp.kvline does not contain value_split [=]" + } + } + if err == null { + if ._temp.kv == null { + ._temp.kv = { } + } + ._temp.kv = merge(object!(._temp.kv), kv, deep: true) + } else { + assert!(false, to_string(err)) + } + } else { + err = "._temp.kvline must not be nullish" + assert!(false, to_string(err)) + } + if ._temp.kv.ssid != null { + if ._temp.kv.ssid != null { + if .network.name == null { + .network.name = del(._temp.kv.ssid) + } else { + assert!(false, ".network.name must be null or override: true") + } + } else { + assert!(false, "._temp.kv.ssid must not be null") + } + } + if ._temp.kv.bssid != null { + if .cisco_meraki.bssid == null { + .cisco_meraki.bssid = del(._temp.kv.bssid) + } else { + assert!(false, ".cisco_meraki.bssid must be null or override: true") + } + } else { + assert!(false, "._temp.kv.bssid must not be null") + } + if .cisco_meraki.event_subtype == "ssid_spoofing_detected" { + if ._temp.kv.vap != null { + if .cisco_meraki.vap == null { + .cisco_meraki.vap = del(._temp.kv.vap) + } else { + assert!(false, ".cisco_meraki.vap must be null or override: true") + } + } else { + assert!(false, "._temp.kv.vap must not be null") + } + } + if !is_nullish(._temp.kv.src) { + if is_array(._temp.kv.src) { + .source.mac = map_values(array!(._temp.kv.src)) -> |value| { + replace!(value, r'[-:.]', "-") + } + } else { + if false { + ._temp.kv.src = null + } + target, err = replace(._temp.kv.src, r'[-:.]', "-") + if err == null { + .source.mac = target + } else { + assert!(false, to_string(err)) + } + } + } else { + assert!(false, "._temp.kv.src must not be nullish") + } + if !is_nullish(._temp.kv.dst) { + if is_array(._temp.kv.dst) { + .destination.mac = map_values(array!(._temp.kv.dst)) -> |value| { + replace!(value, r'[-:.]', "-") + } + } else { + if false { + ._temp.kv.dst = null + } + target, err = replace(._temp.kv.dst, r'[-:.]', "-") + if err == null { + .destination.mac = target + } else { + assert!(false, to_string(err)) + } + } + } else { + assert!(false, "._temp.kv.dst must not be nullish") + } + if .cisco_meraki.event_subtype == "rogue_ssid_detected" { + if !is_nullish(._temp.kv.wired_mac) { + if is_array(._temp.kv.wired_mac) { + ._temp.observer.mac = map_values(array!(._temp.kv.wired_mac)) -> |value| { + replace!(value, r'[-:.]', "-") + } + } else { + if false { + ._temp.kv.wired_mac = null + } + target, err = replace(._temp.kv.wired_mac, r'[-:.]', "-") + if err == null { + ._temp.observer.mac = target + } else { + assert!(false, to_string(err)) + } + } + } else { + assert!(false, "._temp.kv.wired_mac must not be nullish") + } + } + if ._temp.observer.mac != null { + if !is_array(.observer.mac) { + if .observer.mac == null { + .observer.mac = [] + } else { + .observer.mac = [.observer.mac] + } + } + .observer.mac = append!({ + expr = .observer.mac + if false { + expr = null + } + expr + }, [._temp.observer.mac]) + } + if .cisco_meraki.event_subtype == "rogue_ssid_detected" { + if ._temp.kv.vlan_id != null { + if .network.vlan.id == null { + .network.vlan.id = del(._temp.kv.vlan_id) + } else { + assert!(false, ".network.vlan.id must be null or override: true") + } + } else { + assert!(false, "._temp.kv.vlan_id must not be null") + } + } + if ._temp.kv.channel != null { + if .cisco_meraki.channel == null { + .cisco_meraki.channel = del(._temp.kv.channel) + } else { + assert!(false, ".cisco_meraki.channel must be null or override: true") + } + } else { + assert!(false, "._temp.kv.channel must not be null") + } + if ._temp.kv.fc_type != null { + if .cisco_meraki.fc_type == null { + .cisco_meraki.fc_type = del(._temp.kv.fc_type) + } else { + assert!(false, ".cisco_meraki.fc_type must be null or override: true") + } + } else { + assert!(false, "._temp.kv.fc_type must not be null") + } + if ._temp.kv.fc_subtype != null { + if .cisco_meraki.fc_subtype == null { + .cisco_meraki.fc_subtype = del(._temp.kv.fc_subtype) + } else { + assert!(false, ".cisco_meraki.fc_subtype must be null or override: true") + } + } else { + assert!(false, "._temp.kv.fc_subtype must not be null") + } + } + if .cisco_meraki.event_type == "security_event" { + _ingest.timestamp = now() + if _ingest.pipeline == null { + _ingest.pipeline = "" + _ingest.on_failure_pipeline = "" + } + if .event.original != null { + tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} security_event %{DATA:type} %{DATA}$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "failed to dissect .event.original: " + to_string(err)) + } + } else { + err = ".event.original must not be nullish" + assert!(false, err) + } + if .type != null { + if .cisco_meraki.event_subtype == null { + .cisco_meraki.event_subtype = del(.type) + } else { + assert!(false, ".cisco_meraki.event_subtype must be null or override: true") + } + } else { + assert!(false, ".type must not be null") + } + if !is_nullish(.event.original) { + value, err = parse_groks(value: .event.original, patterns: [s'.*?^%{DATA} (security_event|ids-alerts) (%{WORD}\s)?%{DATA:_temp.kvs}(\smessage:\s?%{DATA:message})?$.*']) + if err == null { + . = merge(., value, deep: true) + } else { + assert!(false, "failed to grok .event.original:" + to_string(err)) + } + } else { + assert!(false, ".event.original must not be nullish") + } + if !is_nullish(._temp.kvs) { + kv, err = parse_key_value(._temp.kvs, key_value_delimiter: "=", field_delimiter: " ", accept_standalone_key: true, whitespace: "strict") + if err == null { + standalone_key = false + for_each(kv) -> |_, value| { + if value == true { + standalone_key = true + } + } + if standalone_key { + err = "field ._temp.kvs does not contain value_split [=]" + } + } + if err == null { + if . == null { + . = { } + } + . = merge(object(.), kv, deep: true) + } else { + assert!(false, to_string(err)) + } + } else { + err = "._temp.kvs must not be nullish" + assert!(false, to_string(err)) + } + if .priority != null { + if .cisco_meraki.security.priority == null { + .cisco_meraki.security.priority = del(.priority) + } else { + assert!(false, ".cisco_meraki.security.priority must be null or override: true") + } + } + if .signature != null { + if .cisco_meraki.security.signature == null { + .cisco_meraki.security.signature = del(.signature) + } else { + assert!(false, ".cisco_meraki.security.signature must be null or override: true") + } + } + if !is_nullish(.dhost) { + if is_array(.dhost) { + .cisco_meraki.security.dhost = map_values(array!(.dhost)) -> |value| { + replace!(value, r'[-:.]', "-") + } + } else { + if false { + .dhost = null + } + target, err = replace(.dhost, r'[-:.]', "-") + if err == null { + .cisco_meraki.security.dhost = target + } else { + assert!(false, to_string(err)) + } + } + } + if .direction != null { + if .network.direction == null { + .network.direction = del(.direction) + } else { + assert!(false, ".network.direction must be null or override: true") + } + } + if !is_nullish(.protocol) { + if is_array(.protocol) { + .protocol = map_values(array!(.protocol)) -> |value| { + downcase!(value) + } + } else { + tmp, err = downcase(.protocol) + if err == null { + .network.protocol = tmp + } else { + assert!(false, to_string(err)) + } + } + } + if .decision != null { + if .cisco_meraki.security.decision == null { + .cisco_meraki.security.decision = del(.decision) + } else { + assert!(false, ".cisco_meraki.security.decision must be null or override: true") + } + } + if .url != null { + uri, err = parse_url(.url, default_known_ports: true) + if err == null { + if uri.username == "" { + del(uri.username) + } + if uri.password == "" { + del(uri.password) + } + uri.query = encode_key_value!(uri.query, field_delimiter: "&", key_value_delimiter: "=") + if uri.query == "" { + del(uri.query) + } + if uri.path != null { + uri.path = decode_percent!(uri.path) + } + uri.original = .url + uri.extension = parse_regex(uri.path, r'.*\.(?P[a-zA-Z0-9_\-]+)$').extension ?? null + uri.domain = del(uri.host) + if uri.username != null && uri.password != null { + uri.user_info = join!([uri.username, uri.password], separator: ":") + } + .url = compact(uri) + } + } + if !is_nullish(.mac) { + if is_array(.mac) { + .cisco_meraki.security.mac = map_values(array!(.mac)) -> |value| { + replace!(value, r'[-:.]', "-") + } + } else { + if false { + .mac = null + } + target, err = replace(.mac, r'[-:.]', "-") + if err == null { + .cisco_meraki.security.mac = target + } else { + assert!(false, to_string(err)) + } + } + } + if .name != null { + if .file.name == null { + .file.name = del(.name) + } else { + assert!(false, ".file.name must be null or override: true") + } + } + if .sha256 != null { + if .file.hash.sha256 == null { + .file.hash.sha256 = del(.sha256) + } else { + assert!(false, ".file.hash.sha256 must be null or override: true") + } + } + if .disposition != null { + if .cisco_meraki.disposition == null { + .cisco_meraki.disposition = del(.disposition) + } else { + assert!(false, ".cisco_meraki.disposition must be null or override: true") + } + } + if .action != null { + if .cisco_meraki.security.action == null { + .cisco_meraki.security.action = del(.action) + } else { + assert!(false, ".cisco_meraki.security.action must be null or override: true") + } + } + if .cisco_meraki.event_subtype != "security_filtering_disposition_change" && .src != null { + if !is_nullish(.src) { + value, err = parse_groks(value: .src, patterns: [s'.*?^%{IPV4:_temp.src_ip}:%{PORT:sport}$.*', s'.*?^\[%{IPV6:_temp.src_ip}\]:%{PORT:sport}$.*', s'.*?^%{IPV6NOCOMPRESS:_temp.src_ip}:%{PORT:sport}$.*', s'.*?^%{IPV6:_temp.src_ip}%{IPV6PORTSEP}%{PORT:sport}$.*'], aliases: { "IPV6NOCOMPRESS": s'([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}', "IPV6PORTSEP": s'(?: port |[p#.])', "PORT": s'[0-9]+' }) + if err == null { + . = merge(., value, deep: true) + } else { + assert!(false, "failed to grok .src:" + to_string(err)) + } + } else { + assert!(false, ".src must not be nullish") + } + } + if !is_nullish(._temp.src_ip) { + if is_ipv4(string!(._temp.src_ip)) || is_ipv6(string!(._temp.src_ip)) { + .source.ip = ._temp.src_ip + } else { + err = "Failed converting ._temp.src_ip to an IP address" + assert!(false, err) + } + } + if .sport != "0" { + if !is_nullish(.sport) { + if is_string(.sport) { + temp = .sport + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .source.port = temp + } + } else { + if is_integer(.sport) { + .source.port = .sport + } + } + } + } + if .cisco_meraki.event_subtype != "security_filtering_disposition_change" && .dst != null { + if !is_nullish(.dst) { + value, err = parse_groks(value: .dst, patterns: [s'.*?^%{IPV4:_temp.dst_ip}:%{PORT:dport}$.*', s'.*?^\[%{IPV6:_temp.dst_ip}\]:%{PORT:dport}$.*', s'.*?^%{IPV6NOCOMPRESS:_temp.dst_ip}:%{PORT:dport}$.*', s'.*?^%{IPV6:_temp.dst_ip}%{IPV6PORTSEP}%{PORT:dport}$.*'], aliases: { "IPV6NOCOMPRESS": s'([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}', "IPV6PORTSEP": s'(?: port |[p#.])', "PORT": s'[0-9]+' }) + if err == null { + . = merge(., value, deep: true) + } else { + assert!(false, "failed to grok .dst:" + to_string(err)) + } + } else { + assert!(false, ".dst must not be nullish") + } + } + if !is_nullish(._temp.dst_ip) { + if is_ipv4(string!(._temp.dst_ip)) || is_ipv6(string!(._temp.dst_ip)) { + .destination.ip = ._temp.dst_ip + } + } + if .dport != "0" && .cisco_meraki.event_subtype != "security_filtering_disposition_change" { + if !is_nullish(.dport) { + if is_string(.dport) { + temp = .dport + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .destination.port = temp + } + } else { + if is_integer(.dport) { + .destination.port = .dport + } + } + } + } + } + if .cisco_meraki.event_type == "ids-alerts" { + _ingest.timestamp = now() + if _ingest.pipeline == null { + _ingest.pipeline = "" + _ingest.on_failure_pipeline = "" + } + if .event.original != null { + tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} ids-alerts signature=%{DATA:signature} priority=%{DATA:priority} timestamp=%{DATA:timestamp} direction=%{DATA:direction} protocol=%{DATA:protocol} src=%{DATA:src}$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "failed to dissect .event.original: " + to_string(err)) + } + } else { + err = ".event.original must not be nullish" + assert!(false, err) + } + .cisco_meraki.event_subtype = "ids_alerted" + if .priority != null { + if .cisco_meraki.security.priority == null { + .cisco_meraki.security.priority = del(.priority) + } else { + assert!(false, ".cisco_meraki.security.priority must be null or override: true") + } + } else { + assert!(false, ".priority must not be null") + } + if .signature != null { + if .cisco_meraki.security.signature == null { + .cisco_meraki.security.signature = del(.signature) + } else { + assert!(false, ".cisco_meraki.security.signature must be null or override: true") + } + } else { + assert!(false, ".signature must not be null") + } + if .direction != null { + if .network.direction == null { + .network.direction = del(.direction) + } else { + assert!(false, ".network.direction must be null or override: true") + } + } else { + assert!(false, ".direction must not be null") + } + if !is_nullish(.protocol) { + if is_array(.protocol) { + .protocol = map_values(array!(.protocol)) -> |value| { + downcase!(value) + } + } else { + tmp, err = downcase(.protocol) + if err == null { + .network.protocol = tmp + } else { + assert!(false, to_string(err)) + } + } + } else { + assert!(false, ".protocol must not be nullish") + } + if .src != null { + if !is_nullish(.src) { + value, err = parse_groks(value: .src, patterns: [s'.*?^%{IPV4:_temp.src_ip}:%{PORT:sport}$.*', s'.*?^\[%{IPV6:_temp.src_ip}\]:%{PORT:sport}$.*', s'.*?^%{IPV6NOCOMPRESS:_temp.src_ip}:%{PORT:sport}$.*', s'.*?^%{IPV6:_temp.src_ip}%{IPV6PORTSEP}%{PORT:sport}$.*'], aliases: { "IPV6NOCOMPRESS": s'([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}', "IPV6PORTSEP": s'(?: port |[p#.])', "PORT": s'[0-9]+' }) + if err == null { + . = merge(., value, deep: true) + } else { + assert!(false, "failed to grok .src:" + to_string(err)) + } + } else { + assert!(false, ".src must not be nullish") + } + } + if !is_nullish(._temp.src_ip) { + if is_ipv4(string!(._temp.src_ip)) || is_ipv6(string!(._temp.src_ip)) { + .source.ip = ._temp.src_ip + } + } + if !is_nullish(.sport) { + if is_string(.sport) { + temp = .sport + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .source.port = temp + } + } else { + if is_integer(.sport) { + .source.port = .sport + } + } + } + } + if .cisco_meraki.event_type == "events" { + _ingest.timestamp = now() + if _ingest.pipeline == null { + _ingest.pipeline = "" + _ingest.on_failure_pipeline = "" + } + if .event.original != null { + tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} events %{DATA:msgtype} %{DATA}$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "failed to dissect .event.original: " + to_string(err)) + } + } else { + err = ".event.original must not be nullish" + assert!(false, err) + } + if downcase(string!(.msgtype)) == "site-to-site" { + .cisco_meraki.event_subtype = "Site-to-Site VPN" + } + if downcase(string!(.msgtype)) == "client_vpn_connect" { + .cisco_meraki.event_subtype = "client_vpn_connect" + } + if downcase(string!(.msgtype)) == "blocked" { + .cisco_meraki.event_subtype = "blocked" + } + if downcase(string!(.msgtype)) == "auth" { + .cisco_meraki.event_subtype = "auth" + } + if downcase(string!(.msgtype)) == "port" { + .cisco_meraki.event_subtype = "port" + } + if downcase(string!(.msgtype)) == "carrier_change" { + .cisco_meraki.event_subtype = "carrier_change" + } + if starts_with!({ + expr = .msgtype + if false { + expr = null + } + expr + }, "type=") { + if .event.original != null { + tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} events type=%{DATA:type} %{DATA}$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "failed to dissect .event.original: " + to_string(err)) + } + } else { + err = ".event.original must not be nullish" + assert!(false, err) + } + } + if .type != null { + if .type != null { + if .cisco_meraki.event_subtype == null { + .cisco_meraki.event_subtype = del(.type) + } else { + assert!(false, ".cisco_meraki.event_subtype must be null or override: true") + } + } else { + assert!(false, ".type must not be null") + } + } + if downcase(string!(.msgtype)) == "dhcp" { + if .event.original != null { + tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} events dhcp %{DATA:_temp.dhcp_op} %{DATA:_temp.dhcp_op2} %{DATA}$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "failed to dissect .event.original: " + to_string(err)) + } + } else { + err = ".event.original must not be nullish" + assert!(false, err) + } + } + if downcase(string!(.msgtype)) == "dhcp" { + .network.protocol = "dhcp" + } + if downcase(string!(.msgtype)) == "dhcp" && downcase(string!(._temp.dhcp_op)) == "lease" { + if .event.original != null { + tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} events dhcp lease of ip %{DATA:_temp.client_ip} from %{DATA} mac %{DATA:server.mac} for client mac %{DATA:client.mac} %{DATA}$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "failed to dissect .event.original: " + to_string(err)) + } + } else { + err = ".event.original must not be nullish" + assert!(false, err) + } + } + if downcase(string!(.msgtype)) == "dhcp" && downcase(string!(._temp.dhcp_op)) == "no" && downcase(string!(._temp.dhcp_op2)) == "offers" { + if .event.original != null { + tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} events dhcp no offers for mac %{DATA:client.mac} %{DATA}$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "failed to dissect .event.original: " + to_string(err)) + } + } else { + err = ".event.original must not be nullish" + assert!(false, err) + } + } + if downcase(string!(.msgtype)) == "dhcp" && ._temp.dhcp_op == "lease" { + .cisco_meraki.event_subtype = "dhcp_offer" + } + if downcase(string!(.msgtype)) == "dhcp" && downcase(string!(._temp.dhcp_op)) == "no" && downcase(string!(._temp.dhcp_op2)) == "offers" { + .cisco_meraki.event_subtype = "dhcp_no_offer" + } + if downcase(string!(.msgtype)) == "dhcp" { + if !is_nullish(.event.original) { + value, err = parse_groks(value: .event.original, patterns: [s'.*?events dhcp %{GREEDYDATA:message}$.*']) + if err == null { + . = merge(., value, deep: true) + } else { + assert!(false, "failed to grok .event.original:" + to_string(err)) + } + } else { + assert!(false, ".event.original must not be nullish") + } + } + if starts_with!({ + expr = .event.original + if false { + expr = null + } + expr + }, "<") && .cisco_meraki.event_subtype == "Site-to-Site VPN" { + if !is_nullish(.event.original) { + value, err = parse_groks(value: .event.original, patterns: [s'.*?%{SYSLOGHDR}%{SPACE}%{NUMBER}%{SPACE}%{WORDORHOST}%{SPACE}events%{SPACE}(?i)Site-to-Site VPN:%{GREEDYDATA:cisco_meraki.site_to_site_vpn.raw}.*'], aliases: { "SYSLOGHDR": s'%{SYSLOGPRI}%{SYSLOGVER}', "SYSLOGPRI": s'<%{NONNEGINT:log.syslog.priority}>', "SYSLOGVER": s'\b(?:\d{1,2})\b', "WORDORHOST": s'(?:%{WORD}|%{HOSTNAME})' }) + if err == null { + if value.log.syslog.priority != null { + value.log.syslog.priority = parse_int!(value.log.syslog.priority) + } + . = merge(., value, deep: true) + } else { + assert!(false, "failed to grok .event.original:" + to_string(err)) + } + } else { + assert!(false, ".event.original must not be nullish") + } + } + if starts_with!({ + expr = .event.original + if false { + expr = null + } + expr + }, "<") && .cisco_meraki.event_subtype == "blocked" { + if !is_nullish(.event.original) { + value, err = parse_groks(value: .event.original, patterns: [s'.*?^%{SYSLOGHDR}%{SPACE}%{NUMBER}%{SPACE}%{WORDORHOST}%{SPACE}events%{SPACE}(?%{BLOCKEDARP:_temp.blocked_arp} from %{MAC:source.mac} with IP %{IP:source.ip} on %{NOTSPACE} %{GREEDYDATA:observer.ingress.vlan.id})$.*'], aliases: { "BLOCKEDARP": s'Blocked ARP Packet', "SYSLOGHDR": s'%{SYSLOGPRI}%{SYSLOGVER}', "SYSLOGPRI": s'<%{NONNEGINT:log.syslog.priority}>', "SYSLOGVER": s'\b(?:\d{1,2})\b', "WORDORHOST": s'(?:%{WORD}|%{HOSTNAME})' }) + if err == null { + if value.log.syslog.priority != null { + value.log.syslog.priority = parse_int!(value.log.syslog.priority) + } + . = merge(., value, deep: true) + } else { + assert!(false, "failed to grok .event.original:" + to_string(err)) + } + } else { + assert!(false, ".event.original must not be nullish") + } + } + if !is_nullish(.source.mac) { + if is_array(.source.mac) { + .source.mac = map_values(array!(.source.mac)) -> |value| { + replace!(value, r'[:.]', "-") + } + } else { + if false { + .source.mac = null + } + target, err = replace(.source.mac, r'[:.]', "-") + if err == null { + .source.mac = target + } else { + assert!(false, to_string(err)) + } + } + } + if !is_nullish(.source.mac) { + .source.mac = upcase!(.source.mac) + } + if ._temp.blocked_arp != null { + .cisco_meraki.event_subtype = "arp_blocked" + } + if starts_with!({ + expr = .event.original + if false { + expr = null + } + expr + }, "<") && .cisco_meraki.event_subtype == "port" { + if !is_nullish(.event.original) { + value, err = parse_groks(value: .event.original, patterns: [s'.*?^(?i)%{SYSLOGHDR}%{SPACE}%{NUMBER}%{SPACE}%{WORDORHOST}%{SPACE}events%{SPACE}(?port %{NOTSPACE:cisco_meraki.port} %{PORTACTION:_temp.port_action}(?: from %{NOTSPACE:cisco_meraki.old_port_status} to %{NOTSPACE:cisco_meraki.new_port_status}|.*))$.*'], aliases: { "PORTACTION": s'(?:changed stp role|status changed)', "SYSLOGHDR": s'%{SYSLOGPRI}%{SYSLOGVER}', "SYSLOGPRI": s'<%{NONNEGINT:log.syslog.priority}>', "SYSLOGVER": s'\b(?:\d{1,2})\b', "WORDORHOST": s'(?:%{WORD}|%{HOSTNAME})' }) + if err == null { + if value.log.syslog.priority != null { + value.log.syslog.priority = parse_int!(value.log.syslog.priority) + } + . = merge(., value, deep: true) + } else { + assert!(false, "failed to grok .event.original:" + to_string(err)) + } + } else { + assert!(false, ".event.original must not be nullish") + } + } + if !is_nullish(._temp.port_action) { + if is_array(._temp.port_action) { + ._temp.port_action = map_values(array!(._temp.port_action)) -> |value| { + replace!(value, r' ', "_") + } + } else { + if false { + ._temp.port_action = null + } + target, err = replace(._temp.port_action, r' ', "_") + if err == null { + ._temp.port_action = target + } else { + assert!(false, to_string(err)) + } + } + } + if !is_nullish(._temp.port_action) { + if is_array(._temp.port_action) { + ._temp.port_action = map_values(array!(._temp.port_action)) -> |value| { + downcase!(value) + } + } else { + tmp, err = downcase(._temp.port_action) + if err == null { + ._temp.port_action = tmp + } else { + assert!(false, to_string(err)) + } + } + } + if ._temp.port_action != null { + .cisco_meraki.event_subtype = join!(["port_", to_string!(._temp.port_action)], "") + } + if starts_with!({ + expr = .event.original + if false { + expr = null + } + expr + }, "<") && .cisco_meraki.event_subtype == "carrier_change" { + if !is_nullish(.event.original) { + value, err = parse_groks(value: .event.original, patterns: [s'.*?^(?i)%{SYSLOGHDR}%{SPACE}%{NUMBER}%{SPACE}%{WORDORHOST}%{SPACE}events carrier_change device%{SPACE}%{NOTSPACE:cisco_meraki.mxport} up %{NOTSPACE:_temp.up}.*$.*'], aliases: { "SYSLOGHDR": s'%{SYSLOGPRI}%{SYSLOGVER}', "SYSLOGPRI": s'<%{NONNEGINT:log.syslog.priority}>', "SYSLOGVER": s'\b(?:\d{1,2})\b', "WORDORHOST": s'(?:%{WORD}|%{HOSTNAME})' }) + if err == null { + if value.log.syslog.priority != null { + value.log.syslog.priority = parse_int!(value.log.syslog.priority) + } + . = merge(., value, deep: true) + } else { + assert!(false, "failed to grok .event.original:" + to_string(err)) + } + } else { + assert!(false, ".event.original must not be nullish") + } + } + if ._temp.up == "true" { + .cisco_meraki.new_port_status = "up" + } + if ._temp.up == "false" { + .cisco_meraki.new_port_status = "down" + } + if starts_with!({ + expr = .event.original + if false { + expr = null + } + expr + }, "<") && { + expr = ["dfs_event", "association", "disassociation", "aps_association_reject", "multiple_dhcp_servers_detected", "wpa_deauth", "wpa_auth", "vpn_connectivity_change", "8021x_eap_failure", "8021x_auth", "8021x_deauth", "8021x_eap_success", "splash_auth", "device_packet_flood"] + value = .cisco_meraki.event_subtype + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + if !is_nullish(.event.original) { + value, err = parse_groks(value: .event.original, patterns: [s'.*?%{SYSLOGHDR}%{SPACE}%{NUMBER}%{SPACE}%{WORDORHOST}%{SPACE}events%{SPACE}%{GREEDYDATA:_temp.rest}.*'], aliases: { "SYSLOGHDR": s'%{SYSLOGPRI}%{SYSLOGVER}', "SYSLOGPRI": s'<%{NONNEGINT:log.syslog.priority}>', "SYSLOGVER": s'\b(?:\d{1,2})\b', "WORDORHOST": s'(?:%{WORD}|%{HOSTNAME})' }) + if err == null { + if value.log.syslog.priority != null { + value.log.syslog.priority = parse_int!(value.log.syslog.priority) + } + . = merge(., value, deep: true) + } else { + assert!(false, "failed to grok .event.original:" + to_string(err)) + } + } else { + assert!(false, ".event.original must not be nullish") + } + } + if ._temp.rest != null && { + expr = ["dfs_event", "association", "disassociation", "aps_association_reject", "multiple_dhcp_servers_detected", "wpa_deauth", "wpa_auth", "8021x_eap_failure", "8021x_auth", "8021x_deauth", "8021x_eap_success", "splash_auth", "device_packet_flood"] + value = .cisco_meraki.event_subtype + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + if !is_nullish(._temp.rest) { + kv, err = parse_key_value(._temp.rest, key_value_delimiter: "=", field_delimiter: " ", accept_standalone_key: true, whitespace: "strict") + if err == null { + standalone_key = false + for_each(kv) -> |_, value| { + if value == true { + standalone_key = true + } + } + if standalone_key { + err = "field ._temp.rest does not contain value_split [=]" + } + } + if err == null { + target = get(., ["cisco_meraki", .cisco_meraki.event_subtype]) ?? { } + if target == null { + target = { } + } + . = set!(., ["cisco_meraki", .cisco_meraki.event_subtype], merge!(target, kv, deep: true)) + } else { + assert!(false, to_string(err)) + } + } else { + err = "._temp.rest must not be nullish" + assert!(false, to_string(err)) + } + } + if ._temp.rest != null && .cisco_meraki.event_subtype == "vpn_connectivity_change" { + if !is_nullish(._temp.rest) { + kv, err = parse_key_value(._temp.rest, key_value_delimiter: "=", field_delimiter: " ", accept_standalone_key: true, whitespace: "strict") + if err == null { + standalone_key = false + for_each(kv) -> |_, value| { + if value == true { + standalone_key = true + } + } + if standalone_key { + err = "field ._temp.rest does not contain value_split [=]" + } + } + if err == null { + if .cisco_meraki.site_to_site_vpn.connectivity_change == null { + .cisco_meraki.site_to_site_vpn.connectivity_change = { } + } + .cisco_meraki.site_to_site_vpn.connectivity_change = merge(object!(.cisco_meraki.site_to_site_vpn.connectivity_change), kv, deep: true) + } else { + assert!(false, to_string(err)) + } + } else { + err = "._temp.rest must not be nullish" + assert!(false, to_string(err)) + } + } + if .cisco_meraki.event_subtype == "multiple_dhcp_servers_detected" { + .network.protocol = "dhcp" + } + if .cisco_meraki.event_subtype == "multiple_dhcp_servers_detected" { + if .cisco_meraki.multiple_dhcp_servers_detected.original_server_mac != null { + if .server.mac == null { + .server.mac = del(.cisco_meraki.multiple_dhcp_servers_detected.original_server_mac) + } else { + assert!(false, ".server.mac must be null or override: true") + } + } else { + assert!(false, ".cisco_meraki.multiple_dhcp_servers_detected.original_server_mac must not be null") + } + } + if .cisco_meraki.event_subtype == "multiple_dhcp_servers_detected" { + if !is_nullish(.cisco_meraki.multiple_dhcp_servers_detected.original_server_ip) { + value, err = parse_groks(value: .cisco_meraki.multiple_dhcp_servers_detected.original_server_ip, patterns: [s'.*?^%{IPV4:cisco_meraki.multiple_dhcp_servers_detected.original_server_ip}$.*', s'.*?^%{IPV6:cisco_meraki.multiple_dhcp_servers_detected.original_server_ip}$.*']) + if err == null { + . = merge(., value, deep: true) + } + } + } + if .cisco_meraki.event_subtype == "multiple_dhcp_servers_detected" { + if !is_nullish(.cisco_meraki.multiple_dhcp_servers_detected.original_server_ip) { + if is_ipv4(string!(.cisco_meraki.multiple_dhcp_servers_detected.original_server_ip)) || is_ipv6(string!(.cisco_meraki.multiple_dhcp_servers_detected.original_server_ip)) { + .server.ip = .cisco_meraki.multiple_dhcp_servers_detected.original_server_ip + } + } + } + if .server.ip != null { + if !exists(.cisco_meraki.multiple_dhcp_servers_detected.original_server_ip) { + assert!(false, "tried to remove field '.cisco_meraki.multiple_dhcp_servers_detected.original_server_ip' that doesn't exist") + } + del(.cisco_meraki.multiple_dhcp_servers_detected.original_server_ip) + } + if .cisco_meraki.event_subtype == "multiple_dhcp_servers_detected" { + if !is_array(.related.ip) { + if .related.ip == null { + .related.ip = [] + } else { + .related.ip = [.related.ip] + } + } + .related.ip = append!({ + expr = .related.ip + if false { + expr = null + } + expr + }, [.server.ip]) + } + if .cisco_meraki.event_subtype == "multiple_dhcp_servers_detected" { + if !is_nullish(.cisco_meraki.multiple_dhcp_servers_detected.server_ip) { + value, err = parse_groks(value: .cisco_meraki.multiple_dhcp_servers_detected.server_ip, patterns: [s'.*?^%{IPV4:cisco_meraki.multiple_dhcp_servers_detected.server_ip}$.*', s'.*?^%{IPV6:cisco_meraki.multiple_dhcp_servers_detected.server_ip}$.*']) + if err == null { + . = merge(., value, deep: true) + } else { + assert!(false, "failed to grok .cisco_meraki.multiple_dhcp_servers_detected.server_ip:" + to_string(err)) + } + } else { + assert!(false, ".cisco_meraki.multiple_dhcp_servers_detected.server_ip must not be nullish") + } + } + if .cisco_meraki.event_subtype == "multiple_dhcp_servers_detected" { + if !is_nullish(.cisco_meraki.multiple_dhcp_servers_detected.server_ip) { + if is_ipv4(string!(.cisco_meraki.multiple_dhcp_servers_detected.server_ip)) || is_ipv6(string!(.cisco_meraki.multiple_dhcp_servers_detected.server_ip)) { + .cisco_meraki.multiple_dhcp_servers_detected.server_ip = .cisco_meraki.multiple_dhcp_servers_detected.server_ip + } else { + err = "Failed converting .cisco_meraki.multiple_dhcp_servers_detected.server_ip to an IP address" + assert!(false, err) + } + } else { + err = ".cisco_meraki.multiple_dhcp_servers_detected.server_ip must be present" + assert!(false, err) + } + } + if .cisco_meraki.event_subtype == "multiple_dhcp_servers_detected" { + if !is_array(.related.ip) { + if .related.ip == null { + .related.ip = [] + } else { + .related.ip = [.related.ip] + } + } + .related.ip = append!({ + expr = .related.ip + if false { + expr = null + } + expr + }, [.cisco_meraki.multiple_dhcp_servers_detected.server_ip]) + } + if .cisco_meraki.event_subtype == "wpa_deauth" { + if .cisco_meraki.wpa_deauth.client_mac != null { + if .client.mac == null { + .client.mac = del(.cisco_meraki.wpa_deauth.client_mac) + } else { + assert!(false, ".client.mac must be null or override: true") + } + } else { + assert!(false, ".cisco_meraki.wpa_deauth.client_mac must not be null") + } + } + if .cisco_meraki.event_subtype == "client_vpn_connect" { + if .event.original != null { + tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} events client_vpn_connect user id \'%{DATA:user.name}\' local ip %{DATA:network.forwarded_ip} connected from %{DATA:_temp.client_ip}$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "failed to dissect .event.original: " + to_string(err)) + } + } else { + err = ".event.original must not be nullish" + assert!(false, err) + } + } + if .cisco_meraki.event_subtype == "client_vpn_connect" { + if !is_nullish(.event.original) { + value, err = parse_groks(value: .event.original, patterns: [s'.*?events client_vpn_connect %{GREEDYDATA:message}$.*']) + if err == null { + . = merge(., value, deep: true) + } else { + assert!(false, "failed to grok .event.original:" + to_string(err)) + } + } else { + assert!(false, ".event.original must not be nullish") + } + } + if ._temp.client_ip != null { + if !is_nullish(._temp.client_ip) { + value, err = parse_groks(value: ._temp.client_ip, patterns: [s'.*?^%{IPV4:_temp.client_ip}$.*', s'.*?^%{IPV6:_temp.client_ip}$.*']) + if err == null { + . = merge(., value, deep: true) + } + } + } + if ._temp.client_ip != null { + if !is_nullish(._temp.client_ip) { + if is_ipv4(string!(._temp.client_ip)) || is_ipv6(string!(._temp.client_ip)) { + .client.ip = ._temp.client_ip + } + } + } + if !is_nullish(.client.mac) { + if is_array(.client.mac) { + .client.mac = map_values(array!(.client.mac)) -> |value| { + replace!(value, r'[:.]', "-") + } + } else { + if false { + .client.mac = null + } + target, err = replace(.client.mac, r'[:.]', "-") + if err == null { + .client.mac = target + } else { + assert!(false, to_string(err)) + } + } + } + if !is_nullish(.client.mac) { + .client.mac = upcase!(.client.mac) + } + if !is_nullish(.server.mac) { + if is_array(.server.mac) { + .server.mac = map_values(array!(.server.mac)) -> |value| { + replace!(value, r'[:.]', "-") + } + } else { + if false { + .server.mac = null + } + target, err = replace(.server.mac, r'[:.]', "-") + if err == null { + .server.mac = target + } else { + assert!(false, to_string(err)) + } + } + } + if !is_nullish(.server.mac) { + .server.mac = upcase!(.server.mac) + } + } + if .cisco_meraki.event_type == "urls" { + _ingest.timestamp = now() + if _ingest.pipeline == null { + _ingest.pipeline = "" + _ingest.on_failure_pipeline = "" + } + if .event.original != null { + tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} urls src=%{DATA:src} dst=%{DATA:dst} mac=%{DATA:mac} request: %{DATA:http.request.method} %{DATA:url.original}$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "failed to dissect .event.original: " + to_string(err)) + } + } else { + err = ".event.original must not be nullish" + assert!(false, err) + } + if .mac != null { + tmp, err = parse_groks(value: .mac, patterns: [s'^%{DATA:mac} agent=%{DATA:user_agent.original}$']) + if err == null { + . = merge(., tmp, deep: true) + } + } + if !is_nullish(.src) { + value, err = parse_groks(value: .src, patterns: [s'.*?^%{IPV4:_temp.src_ip}:%{PORT:sport}$.*', s'.*?^\[%{IPV6:_temp.src_ip}\]:%{PORT:sport}$.*', s'.*?^%{IPV6NOCOMPRESS:_temp.src_ip}:%{PORT:sport}$.*', s'.*?^%{IPV6:_temp.src_ip}%{IPV6PORTSEP}%{PORT:sport}$.*'], aliases: { "IPV6NOCOMPRESS": s'([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}', "IPV6PORTSEP": s'(?: port |[p#.])', "PORT": s'[0-9]+' }) + if err == null { + . = merge(., value, deep: true) + } else { + assert!(false, "failed to grok .src:" + to_string(err)) + } + } else { + assert!(false, ".src must not be nullish") + } + if !is_nullish(._temp.src_ip) { + if is_ipv4(string!(._temp.src_ip)) || is_ipv6(string!(._temp.src_ip)) { + .source.ip = ._temp.src_ip + } else { + err = "Failed converting ._temp.src_ip to an IP address" + assert!(false, err) + } + } else { + err = "._temp.src_ip must be present" + assert!(false, err) + } + if !is_nullish(.sport) { + if is_string(.sport) { + temp = .sport + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .source.port = temp + } + } else { + if is_integer(.sport) { + .source.port = .sport + } + } + } + if !is_nullish(.dst) { + value, err = parse_groks(value: .dst, patterns: [s'.*?^%{IPV4:_temp.dst_ip}:%{PORT:dport}$.*', s'.*?^\[%{IPV6:_temp.dst_ip}\]:%{PORT:dport}$.*', s'.*?^%{IPV6NOCOMPRESS:_temp.dst_ip}:%{PORT:dport}$.*', s'.*?^%{IPV6:_temp.dst_ip}%{IPV6PORTSEP}%{PORT:dport}$.*'], aliases: { "IPV6NOCOMPRESS": s'([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}', "IPV6PORTSEP": s'(?: port |[p#.])', "PORT": s'[0-9]+' }) + if err == null { + . = merge(., value, deep: true) + } else { + assert!(false, "failed to grok .dst:" + to_string(err)) + } + } else { + assert!(false, ".dst must not be nullish") + } + if !is_nullish(._temp.dst_ip) { + if is_ipv4(string!(._temp.dst_ip)) || is_ipv6(string!(._temp.dst_ip)) { + .destination.ip = ._temp.dst_ip + } + } + if .dport != "0" && .cisco_meraki.event_subtype != "security_filtering_disposition_change" { + if !is_nullish(.dport) { + if is_string(.dport) { + temp = .dport + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .destination.port = temp + } + } else { + if is_integer(.dport) { + .destination.port = .dport + } + } + } + } + if !is_nullish(.mac) { + if is_array(.mac) { + .cisco_meraki.urls.mac = map_values(array!(.mac)) -> |value| { + replace!(value, r'[-:.]', "-") + } + } else { + if false { + .mac = null + } + target, err = replace(.mac, r'[-:.]', "-") + if err == null { + .cisco_meraki.urls.mac = target + } else { + assert!(false, to_string(err)) + } + } + } else { + assert!(false, ".mac must not be nullish") + } + if downcase(string!(.http.request.method)) != "unknown" { + .cisco_meraki.event_subtype = "http_access" + } + if downcase(string!(.http.request.method)) == "unknown" { + .cisco_meraki.event_subtype = "http_access_error" + } + if is_string(.user_agent.original) { + user_agent = parse_user_agent!(.user_agent.original) + .user_agent.version = user_agent.browser.version + .user_agent.os.name = user_agent.os.family + .user_agent.os.version = user_agent.os.version + .user_agent.os.full, _ = join([user_agent.os.family, user_agent.os.version], separator: " ") + .user_agent.name = user_agent.browser.family + if is_nullish(.user_agent.name) { + .user_agent.name = "Other" + } + .user_agent.device.name = user_agent.device.category + if is_nullish(.user_agent.device.name) { + .user_agent.device.name = "Other" + } + .user_agent = compact(.user_agent, recursive: true) + if .user_agent.original == null { + .user_agent.original = "" + } else { + .user_agent.original = .user_agent.original + } + } + if .url.original != null && .url.original != "" { + uri, err = parse_url(.url.original, default_known_ports: true) + if err == null { + if uri.username == "" { + del(uri.username) + } + if uri.password == "" { + del(uri.password) + } + uri.query = encode_key_value!(uri.query, field_delimiter: "&", key_value_delimiter: "=") + if uri.query == "" { + del(uri.query) + } + if uri.path != null { + uri.path = decode_percent!(uri.path) + } + uri.original = .url.original + uri.extension = parse_regex(uri.path, r'.*\.(?P[a-zA-Z0-9_\-]+)$').extension ?? null + uri.domain = del(uri.host) + if uri.username != null && uri.password != null { + uri.user_info = join!([uri.username, uri.password], separator: ":") + } + .url = compact(uri) + } else { + assert!(false, to_string(err)) + } + } + if .url.domain != null { + tmp, err = parse_etld(.url.domain, plus_parts: 1) + if err == null { + .url.registered_domain = tmp.etld_plus + .url.top_level_domain = tmp.etld + .url.domain = .url.domain + .url.subdomain = { + subdomain = replace!(.url.domain, tmp.etld_plus, "") + if subdomain != "" { + subdomain = slice!(subdomain, 0, -1) + } + subdomain + } + .url = compact(.url) + } else { + assert!(false, to_string(err)) + } + } + } + if !is_array(.event.category) { + if .event.category == null { + .event.category = [] + } else { + .event.category = [.event.category] + } + } + .event.category = append!({ + expr = .event.category + if false { + expr = null + } + expr + }, ["network"]) + if !is_array(.event.type) { + if .event.type == null { + .event.type = [] + } else { + .event.type = [.event.type] + } + } + .event.type = append!({ + expr = .event.type + if false { + expr = null + } + expr + }, ["info"]) + if .cisco_meraki.event_subtype != null { + params = { "eventmap": { "8021x_deauth": { "action": s'wifi-8021x-failed-auth-or-deauth', "category": ["authentication"], "type": ["end", "denied"] }, "8021x_eap_failure": { "action": s'wifi-8021x-failed-authentication-attempt', "category": ["authentication"], "type": ["end", "denied"] }, "8021x_eap_success": { "action": s'wifi-8021x-auth', "category": ["authentication"], "type": ["start"] }, "Site-to-Site VPN": { "action": s'site-to-site-vpn', "type": ["access"] }, "aps_association_reject": { "action": s'association-rejected-for-load-balancing' }, "association": { "action": s'wifi-association-request', "type": ["access", "connection"] }, "client_vpn_connect": { "action": s'site-to-site-vpn', "category": ["session"], "type": ["access", "allowed", "start"] }, "device_packet_flood": { "action": s'wireless-packet-flood-detected' }, "dfs_event": { "action": s'dynamic-frequency-selection-detected' }, "dhcp_no_offer": { "action": s'dhcp-no-offer', "type": ["access", "denied"] }, "dhcp_offer": { "action": s'dhcp-offer', "type": ["access", "allowed"] }, "disassociation": { "action": s'wifi-disassociation-request', "category": ["session"], "type": ["access", "end"] }, "flow_allowed": { "action": s'layer3-firewall-allowed-flow', "type": ["connection", "start"] }, "flow_denied": { "action": s'layer3-firewall-denied-flow', "type": ["access", "denied"] }, "http_access": { "action": s'http-access', "category": ["web"], "type": ["access"] }, "http_access_error": { "action": s'http-access-error', "category": ["web"], "type": ["error"] }, "ids_alerted": { "action": s'ids-signature-matched', "category": ["intrusion_detection"] }, "ip_session_initiated": { "action": s'ip-session-initiated', "type": ["access", "start"] }, "multiple_dhcp_servers_detected": { "action": ["multiple_dhcp_servers_detected"], "type": ["protocol"] }, "rogue_ssid_detected": { "action": s'rogue-ssid-detected' }, "security_filtering_disposition_change": { "action": s'issued-retrospective-malicious-disposition', "category": ["file", "malware"] }, "security_filtering_file_scanned": { "action": s'malicious-file-actioned', "category": ["file", "malware"] }, "splash_auth": { "action": s'splash-authentication', "category": ["authentication"], "type": ["start"] }, "ssid_spoofing_detected": { "action": s'ssid-spoofing-detected' }, "vpn_connectivity_change": { "action": s'vpn-connectivity-change', "category": ["session"], "type": ["connection"] }, "wpa_auth": { "action": s'wifi-wpa-authentication', "category": ["authentication"], "type": ["start", "access"] }, "wpa_deauth": { "action": s'wifi-wpa-failed-auth-or-deauth', "category": ["authentication"], "type": ["end", "denied"] } } } + eventMap = get!(params, ["eventmap"]) + eventData = get!(eventMap, [.cisco_meraki.event_subtype]) + shouldSkip_1 = false + return_1 = if eventData == null { + .event.action = .cisco_meraki.event_subtype + shouldSkip_1 = true + null + } + if !(shouldSkip_1 || false) { + eventCategory = get!(eventData, ["category"]) + eventType = get!(eventData, ["type"]) + eventAction = get!(eventData, ["action"]) + if eventType != null { + for_each({ + arr = eventType + if false { + arr = null + } + array!(arr) + }) -> |_index, t| { + if false { + .event.type = null + } + .event.type = push!(.event.type, t) + } + } + if eventCategory != null { + for_each({ + arr = eventCategory + if false { + arr = null + } + array!(arr) + }) -> |_index, c| { + if false { + .event.category = null + } + .event.category = push!(.event.category, c) + } + } + if eventAction != null { + .event.action = eventAction + } + } else { + return_1 + } + } + if .source.as.asn != null { + if .source.as.number == null { + .source.as.number = del(.source.as.asn) + } else { + assert!(false, ".source.as.number must be null or override: true") + } + } + if .source.as.organization_name != null { + if .source.as.organization.name == null { + .source.as.organization.name = del(.source.as.organization_name) + } else { + assert!(false, ".source.as.organization.name must be null or override: true") + } + } + if .destination.as.asn != null { + if .destination.as.number == null { + .destination.as.number = del(.destination.as.asn) + } else { + assert!(false, ".destination.as.number must be null or override: true") + } + } + if .destination.as.organization_name != null { + if .destination.as.organization.name == null { + .destination.as.organization.name = del(.destination.as.organization_name) + } else { + assert!(false, ".destination.as.organization.name must be null or override: true") + } + } + if .client.as.asn != null { + if .client.as.number == null { + .client.as.number = del(.client.as.asn) + } else { + assert!(false, ".client.as.number must be null or override: true") + } + } + if .client.as.organization_name != null { + if .client.as.organization.name == null { + .client.as.organization.name = del(.client.as.organization_name) + } else { + assert!(false, ".client.as.organization.name must be null or override: true") + } + } + del(._temp) + del(._conf) + del(.sport) + del(.dport) + del(.mac) + del(.src) + del(.dst) + del(.translated_src_ip) + del(.translated_dst_ip) + del(.translated_port) + del(.wired_mac) + del(.rssi) + del(.protocol) + del(.dhost) + del(.client_mac) + del(.radio) + del(.sts) + del(.msgtype) + del(.timestamp) + . = compact(.) + if .tags == null || !({ + expr = .tags + value = "preserve_original_event" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + }) { + del(.event.original) + } + } else if _module == "cisco_ios_logs" { + _ingest.pipeline = "cisco_ios_logs" + _ingest.on_failure_pipeline = "cisco_ios_logs" + .tags = ["preserve_original_event"] + .ecs.version = "8.11.0" + .event.category = ["network"] + .event.provider = "firewall" + .observer.vendor = "Cisco" + .observer.product = "IOS" + .observer.type = "firewall" + .event.type = ["info"] + if is_nullish(.event.original) { + .event.original = .message + } + del(.message) + if !is_nullish(.event.original) { + value, err = parse_groks(value: .event.original, patterns: [s'.*?^(:?%{CISCO_PRIORITY_MSGCOUNT})?%{SYSLOGTIMESTAMP} %{IP} %{CISCO_HOSTNAME:log.syslog.hostname}: (?:%{NUMBER:cisco.ios.sequence}: )?(?:%{CISCO_UPTIME:cisco.ios.uptime}|%{CISCO_TIMESTAMP}): %{GREEDYDATA:_temp_.message}$.*', s'.*?^(:?%{CISCO_PRIORITY_MSGCOUNT})?%{SYSLOGTIMESTAMP} (?:%{IP}|%{CISCO_HOSTNAME:log.syslog.hostname}) %{NUMBER:cisco.ios.sequence}: (?:%{CISCO_UPTIME:cisco.ios.uptime}|%{CISCO_TIMESTAMP}): %{GREEDYDATA:_temp_.message}$.*', s'.*?^(:?%{CISCO_PRIORITY_MSGCOUNT})?(?:(?:%{CISCO_HOSTNAME:log.syslog.hostname}|%{IP})[:]? )?(?:%{NUMBER:cisco.ios.sequence}: )?(?:%{CISCO_UPTIME:cisco.ios.uptime}|%{CISCO_TIMESTAMP}): %{GREEDYDATA:_temp_.message}$.*'], aliases: { "CISCO_HOSTNAME": s'[a-zA-Z][0-9a-zA-Z_-]{0,61}[0-9a-zA-Z]?', "CISCO_PRIORITY_MSGCOUNT": s'<%{NONNEGINT:log.syslog.priority}>(?:%{NONNEGINT:cisco.ios.message_count})?(?:: )?', "CISCO_TIMESTAMP": s'[*]?%{CISCOTIMESTAMP:_temp_.cisco_timestamp}(?: %{CISCO_TZ:_temp_.tz})?', "CISCO_TZ": s'[a-zA-Z]{1,4}', "CISCO_UPTIME": s'[0-9a-zA-Z]+' }) + if err == null { + if value.log.syslog.priority != null { + value.log.syslog.priority = parse_int!(value.log.syslog.priority) + } + . = merge(., value, deep: true) + } else { + assert!(false, "grok_header: failed to grok .event.original:" + to_string(err)) + } + } else { + assert!(false, ".event.original must not be nullish") + } + if !is_nullish(._temp_.message) { + value, err = parse_groks(value: ._temp_.message, patterns: [s'.*?^%%{GREEDYDATA:message}$.*', s'.*?^%{GREEDYDATA:_temp_.generic_message}$.*']) + if err == null { + . = merge(., value, deep: true) + } else { + assert!(false, "grok_message: failed to grok ._temp_.message:" + to_string(err)) + } + } else { + assert!(false, "._temp_.message must not be nullish") + } + if .cisco.ios.sequence != null { + .event.sequence = .cisco.ios.sequence + } + if .cisco.ios.message_count != null { + if !is_nullish(.cisco.ios.message_count) { + if is_string(.cisco.ios.message_count) { + temp = .cisco.ios.message_count + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .cisco.ios.message_count = temp + } else { + err_msg = join!(["Failed converting .cisco.ios.message_count to an integer. Found (", to_string!(.cisco.ios.message_count), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.cisco.ios.message_count) { + .cisco.ios.message_count = .cisco.ios.message_count + } else { + err = "Failed converting .cisco.ios.message_count to an integer" + assert!(false, err) + } + } + } else { + err = ".cisco.ios.message_count must be present" + assert!(false, err) + } + } + if .cisco.ios.message_count != null && .event.sequence == null { + .event.sequence = .cisco.ios.message_count + } + if !is_nullish(._temp_.cisco_timestamp) { + if is_array(._temp_.cisco_timestamp) { + ._temp_.cisco_timestamp = map_values(array!(._temp_.cisco_timestamp)) -> |value| { + replace!(value, r' {2,}', " ") + } + } else { + if false { + ._temp_.cisco_timestamp = null + } + target, err = replace(._temp_.cisco_timestamp, r' {2,}', " ") + if err == null { + ._temp_.cisco_timestamp = target + } else { + assert!(false, to_string(err)) + } + } + } + shouldSkip_1 = false + return_1 = if ._temp_.tz != null && ._conf.tz_map != null { + for_each({ + arr = ._conf.tz_map + if false { + arr = null + } + array!(arr) + }) -> |_index, item| { + if item.tz_short == ._temp_.tz { + .event.timezone = item.tz_long + shouldSkip_1 = true + null + } + } + } + if !(shouldSkip_1 || false) { + if ._conf.tz_offset != null { + .event.timezone = ._conf.tz_offset + } + if .event.timezone == null { + .event.timezone = "UTC" + } + } else { + return_1 + } + if ._temp_.cisco_timestamp != null { + date = null + err = "no datetime formats provided" + if err != null { + datetime = to_string!(._temp_.cisco_timestamp) + data, err = parse_regex(value: datetime, pattern: r'(?[A-Z]{3,4}+)$') + records = find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": data.abbreviation }) + if length(records) == 1 { + offset = records[0].offset + datetime = replace!(datetime, data.abbreviation, offset) + date, err = parse_timestamp(value: join!(value: [datetime], separator: " "), format: join!(value: ["%b %d %Y %H:%M:%S.%3f ", "%:z"], separator: " ")) + } else { + if length(records) > 1 { + err = "ambiguous timezone abbreviation" + } else { + if length(records) == 0 { + err = "no timezone abbreviation found" + } + } + } + } + if err != null { + datetime = to_string!(._temp_.cisco_timestamp) + external_timezone = .event.timezone + if false { + external_timezone = null + } + records = if match!(external_timezone, r'[\+-]?[0-9]{1,2}:?[0-9]{2}') { + [{ "offset": external_timezone }] + } else if contains(to_string(external_timezone) ?? "", "/") { + [{ "tz_name": external_timezone }] + } else { + find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": external_timezone }) + } + if length(records) == 1 { + if records[0].tz_name != null { + date, err = parse_timestamp(value: join!(value: [datetime], separator: " "), format: join!(value: ["%b %d %Y %H:%M:%S.%3f"], separator: " "), timezone: string!(records[0].tz_name)) + } else { + offset = records[0].offset + date, err = parse_timestamp(value: join!(value: [datetime, offset], separator: " "), format: join!(value: ["%b %d %Y %H:%M:%S.%3f", "%:z"], separator: " ")) + } + } else { + if length(records) > 1 { + err = "ambiguous timezone abbreviation" + } else { + if length(records) == 0 { + err = "no timezone abbreviation found" + } + } + } + } + if err != null { + datetime = to_string!(._temp_.cisco_timestamp) + data, err = parse_regex(value: datetime, pattern: r'(?[A-Z]{3,4}+)$') + records = find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": data.abbreviation }) + if length(records) == 1 { + offset = records[0].offset + datetime = replace!(datetime, data.abbreviation, offset) + date, err = parse_timestamp(value: join!(value: [datetime], separator: " "), format: join!(value: ["%b %d %Y %H:%M:%S ", "%:z"], separator: " ")) + } else { + if length(records) > 1 { + err = "ambiguous timezone abbreviation" + } else { + if length(records) == 0 { + err = "no timezone abbreviation found" + } + } + } + } + if err != null { + datetime = to_string!(._temp_.cisco_timestamp) + external_timezone = .event.timezone + if false { + external_timezone = null + } + records = if match!(external_timezone, r'[\+-]?[0-9]{1,2}:?[0-9]{2}') { + [{ "offset": external_timezone }] + } else if contains(to_string(external_timezone) ?? "", "/") { + [{ "tz_name": external_timezone }] + } else { + find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": external_timezone }) + } + if length(records) == 1 { + if records[0].tz_name != null { + date, err = parse_timestamp(value: join!(value: [datetime], separator: " "), format: join!(value: ["%b %d %Y %H:%M:%S"], separator: " "), timezone: string!(records[0].tz_name)) + } else { + offset = records[0].offset + date, err = parse_timestamp(value: join!(value: [datetime, offset], separator: " "), format: join!(value: ["%b %d %Y %H:%M:%S", "%:z"], separator: " ")) + } + } else { + if length(records) > 1 { + err = "ambiguous timezone abbreviation" + } else { + if length(records) == 0 { + err = "no timezone abbreviation found" + } + } + } + } + if err != null { + datetime = to_string!(._temp_.cisco_timestamp) + data, err = parse_regex(value: datetime, pattern: r'(?[A-Z]{3,4}+)$') + records = find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": data.abbreviation }) + if length(records) == 1 { + offset = records[0].offset + datetime = replace!(datetime, data.abbreviation, offset) + date, err = parse_timestamp(value: join!(value: [datetime, format_timestamp!(now(), format: "%Y")], separator: " "), format: join!(value: ["%b %d %H:%M:%S.%3f ", "%:z", "%Y"], separator: " ")) + } else { + if length(records) > 1 { + err = "ambiguous timezone abbreviation" + } else { + if length(records) == 0 { + err = "no timezone abbreviation found" + } + } + } + } + if err != null { + datetime = to_string!(._temp_.cisco_timestamp) + external_timezone = .event.timezone + if false { + external_timezone = null + } + records = if match!(external_timezone, r'[\+-]?[0-9]{1,2}:?[0-9]{2}') { + [{ "offset": external_timezone }] + } else if contains(to_string(external_timezone) ?? "", "/") { + [{ "tz_name": external_timezone }] + } else { + find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": external_timezone }) + } + if length(records) == 1 { + if records[0].tz_name != null { + date, err = parse_timestamp(value: join!(value: [datetime, format_timestamp!(now(), format: "%Y")], separator: " "), format: join!(value: ["%b %d %H:%M:%S.%3f", "%Y"], separator: " "), timezone: string!(records[0].tz_name)) + } else { + offset = records[0].offset + date, err = parse_timestamp(value: join!(value: [datetime, offset, format_timestamp!(now(), format: "%Y")], separator: " "), format: join!(value: ["%b %d %H:%M:%S.%3f", "%:z", "%Y"], separator: " ")) + } + } else { + if length(records) > 1 { + err = "ambiguous timezone abbreviation" + } else { + if length(records) == 0 { + err = "no timezone abbreviation found" + } + } + } + } + if err != null { + datetime = to_string!(._temp_.cisco_timestamp) + data, err = parse_regex(value: datetime, pattern: r'(?[A-Z]{3,4}+)$') + records = find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": data.abbreviation }) + if length(records) == 1 { + offset = records[0].offset + datetime = replace!(datetime, data.abbreviation, offset) + date, err = parse_timestamp(value: join!(value: [datetime, format_timestamp!(now(), format: "%Y")], separator: " "), format: join!(value: ["%b %d %H:%M:%S ", "%:z", "%Y"], separator: " ")) + } else { + if length(records) > 1 { + err = "ambiguous timezone abbreviation" + } else { + if length(records) == 0 { + err = "no timezone abbreviation found" + } + } + } + } + if err != null { + datetime = to_string!(._temp_.cisco_timestamp) + external_timezone = .event.timezone + if false { + external_timezone = null + } + records = if match!(external_timezone, r'[\+-]?[0-9]{1,2}:?[0-9]{2}') { + [{ "offset": external_timezone }] + } else if contains(to_string(external_timezone) ?? "", "/") { + [{ "tz_name": external_timezone }] + } else { + find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": external_timezone }) + } + if length(records) == 1 { + if records[0].tz_name != null { + date, err = parse_timestamp(value: join!(value: [datetime, format_timestamp!(now(), format: "%Y")], separator: " "), format: join!(value: ["%b %d %H:%M:%S", "%Y"], separator: " "), timezone: string!(records[0].tz_name)) + } else { + offset = records[0].offset + date, err = parse_timestamp(value: join!(value: [datetime, offset, format_timestamp!(now(), format: "%Y")], separator: " "), format: join!(value: ["%b %d %H:%M:%S", "%:z", "%Y"], separator: " ")) + } + } else { + if length(records) > 1 { + err = "ambiguous timezone abbreviation" + } else { + if length(records) == 0 { + err = "no timezone abbreviation found" + } + } + } + } + if err == null { + .@timestamp = date + } else { + datetime = to_string!(._temp_.cisco_timestamp) + assert!(false, "failed to parse timestamp ._temp_.cisco_timestamp: ({{ datetime }}): " + to_string(err)) + } + } + if !is_nullish(.message) { + value, err = parse_groks(value: .message, patterns: [s'.*?%{DATA:cisco.ios.facility}-%{POSINT:event.severity}-%{DATA:event.code}:\s+(\w+\d+(/\d+)?\:\s+)?([a-zA-Z0-9_]+\:\s+)?%{GREEDYDATA:message}.*']) + if err == null { + . = merge(., value, deep: true) + } else { + assert!(false, "grok_message: failed to grok .message:" + to_string(err)) + } + } + if .cisco.ios.facility == "IOSXE" && .event.code == "PLATFORM" { + if !is_nullish(.message) { + value, err = parse_groks(value: .message, patterns: [s'.*?%%{DATA:cisco.ios.facility}-%{POSINT:event.severity}-%{DATA:event.code}:\s+(\w+\d+(/\d+)?\:\s+)?([a-zA-Z0-9_]+\:\s+)?%{GREEDYDATA:message}.*']) + if err == null { + . = merge(., value, deep: true) + } else { + assert!(false, "grok_child_message: failed to grok .message:" + to_string(err)) + } + } + } + if .cisco.ios.facility == "FW" && .event.code == "SESS_AUDIT_TRAIL" { + if !is_nullish(.message) { + value, err = parse_groks(value: .message, patterns: [s'.*?initiator \(%{IP:source.ip}:%{NUMBER:source.port}\) sent %{NUMBER:source.bytes} bytes -- responder \(%{IP:destination.ip}:%{NUMBER:destination.port}\) sent %{NUMBER:destination.bytes} bytes, from %{NOTSPACE:cisco.ios.interface.name}.*']) + if err == null { + if value.destination.bytes != null { + value.destination.bytes = parse_int!(value.destination.bytes) + } + if value.destination.port != null { + value.destination.port = parse_int!(value.destination.port) + } + if value.source.bytes != null { + value.source.bytes = parse_int!(value.source.bytes) + } + if value.source.port != null { + value.source.port = parse_int!(value.source.port) + } + . = merge(., value, deep: true) + } else { + assert!(false, "grok_audit_details: failed to grok .message:" + to_string(err)) + } + } + } + if .cisco.ios.facility == "FW" && .event.code == "DROP_PKT" { + if !is_nullish(.message) { + value, err = parse_groks(value: .message, patterns: [s'.*?^Dropping %{WORD} %{WORD} from %{NOTSPACE:cisco.ios.interface.name} %{IP:source.ip}:%{NUMBER:source.port} ?=> ?%{IP:destination.ip}:%{NUMBER:destination.port}.*']) + if err == null { + if value.destination.port != null { + value.destination.port = parse_int!(value.destination.port) + } + if value.source.port != null { + value.source.port = parse_int!(value.source.port) + } + . = merge(., value, deep: true) + } else { + assert!(false, "grok_drop_details: failed to grok .message:" + to_string(err)) + } + } + } + if ._temp_.generic_message != null { + if ._temp_.generic_message != null { + if .message == null { + .message = del(._temp_.generic_message) + } else { + assert!(false, ".message must be null or override: true") + } + } else { + assert!(false, "._temp_.generic_message must not be null") + } + } + if !is_nullish(.event.severity) { + if is_string(.event.severity) { + temp = .event.severity + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .event.severity = temp + } else { + err_msg = join!(["Failed converting .event.severity to an integer. Found (", to_string!(.event.severity), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.event.severity) { + .event.severity = .event.severity + } else { + err = "Failed converting .event.severity to an integer" + assert!(false, err) + } + } + } + if !is_nullish(.event.sequence) { + if is_string(.event.sequence) { + temp = .event.sequence + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .event.sequence = temp + } else { + err_msg = join!(["Failed converting .event.sequence to an integer. Found (", to_string!(.event.sequence), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.event.sequence) { + .event.sequence = .event.sequence + } else { + err = "Failed converting .event.sequence to an integer" + assert!(false, err) + } + } + } + if { + expr = ["IPACCESSLOGP", "ACCESSLOGP"] + value = .event.code + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + if .message != null { + tmp, err = parse_groks(value: .message, patterns: [s'^list %{DATA:cisco.ios.access_list} %{DATA:_temp_.event.action} %{DATA:network.transport} %{DATA:source.address}\(%{DATA:source.port}\) %{DATA} %{DATA:destination.address}\(%{DATA:destination.port}\), %{DATA:source.packets} packet.*?$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "dissect_gp: failed to dissect .message: " + to_string(err)) + } + } else { + err = ".message must not be nullish" + assert!(false, err) + } + } + if { + expr = ["IPACCESSLOGDP", "ACCESSLOGDP"] + value = .event.code + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + if .message != null { + tmp, err = parse_groks(value: .message, patterns: [s'^list %{DATA:cisco.ios.access_list} %{DATA:_temp_.event.action} %{DATA:network.transport} %{DATA:source.address} %{DATA} %{DATA:destination.address} \(%{DATA:icmp.type}/%{DATA:icmp.code}\), %{DATA:source.packets} packet.*?$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "dissect_gdp: failed to dissect .message: " + to_string(err)) + } + } else { + err = ".message must not be nullish" + assert!(false, err) + } + } + if .event.code == "IPACCESSLOGRP" { + if .message != null { + tmp, err = parse_groks(value: .message, patterns: [s'^list %{DATA:cisco.ios.access_list} %{DATA:_temp_.event.action} %{DATA:network.transport} %{DATA:source.address} %{DATA} %{DATA:destination.address}, %{DATA:source.packets} packet.*?$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "dissect_grp: failed to dissect .message: " + to_string(err)) + } + } else { + err = ".message must not be nullish" + assert!(false, err) + } + } + if .event.code == "IPACCESSLOGSP" { + if .message != null { + tmp, err = parse_groks(value: .message, patterns: [s'^list %{DATA:cisco.ios.access_list} %{DATA:_temp_.event.action} %{DATA:network.transport} %{DATA:source.address} %{DATA} %{DATA:destination.address} \(%{DATA:igmp.type}\), %{DATA:source.packets} packet.*?$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "dissect_gsp: failed to dissect .message: " + to_string(err)) + } + } else { + err = ".message must not be nullish" + assert!(false, err) + } + } + if .event.code == "ACCESSLOGSP" { + if .message != null { + tmp, err = parse_groks(value: .message, patterns: [s'^list %{DATA:cisco.ios.access_list} %{DATA:_temp_.event.action} %{DATA:network.type} %{DATA:source.address} %{DATA} %{DATA:destination.address}, %{DATA:source.packets} packet.*?$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "dissect_sp: failed to dissect .message: " + to_string(err)) + } + } else { + err = ".message must not be nullish" + assert!(false, err) + } + } + if { + expr = ["IPACCESSLOGNP", "ACCESSLOGNP"] + value = .event.code + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + if .message != null { + tmp, err = parse_groks(value: .message, patterns: [s'^list %{DATA:cisco.ios.access_list} %{DATA:_temp_.event.action} %{DATA:network.iana_number} %{DATA:source.address} %{DATA} %{DATA:destination.address}, %{DATA:source.packets} packet.*?$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "dissect_gnp: failed to dissect .message: " + to_string(err)) + } + } else { + err = ".message must not be nullish" + assert!(false, err) + } + } + if .event.code == "LOGIN_SUCCESS" { + if .message != null { + tmp, err = parse_groks(value: .message, patterns: [s'^%{DATA:cisco.ios.action} %{DATA:_temp_.event.action} \[user: %{DATA:source.user.name}\] \[Source: %{DATA:source.address}\] \[localport: %{DATA:destination.port}\] at %{DATA}$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "dissect_login: failed to dissect .message: " + to_string(err)) + } + } else { + err = ".message must not be nullish" + assert!(false, err) + } + } + if .event.code == "LOGOUT" { + if .message != null { + tmp, err = parse_groks(value: .message, patterns: [s'^User %{DATA:source.user.name} has %{DATA:cisco.ios.action} %{DATA:cisco.ios.session.type} session %{DATA:cisco.ios.session.number}\(%{DATA:source.address}\).*?$']) + if err == null { + . = merge(., tmp, deep: true) + } else { + assert!(false, "dissect_logout: failed to dissect .message: " + to_string(err)) + } + } else { + err = ".message must not be nullish" + assert!(false, err) + } + } + if .event.code == "BADAUTH" { + if !is_nullish(.message) { + value, err = parse_groks(value: .message, patterns: [s'.*?^(?:No|Invalid) MD5 digest from %{DATA:source.address}(\(%{INT:source.port}\)|\:%{INT:source.port}) to %{DATA:destination.address}(\(%{INT:destination.port}\)|\:%{INT:destination.port})(?:(?: \(RST\))? (?:tableid - %{DATA:cisco.ios.tableid}|%{GREEDYDATA:_temp_.rst}))?$.*']) + if err == null { + . = merge(., value, deep: true) + } else { + assert!(false, "grok_badauth: failed to grok .message:" + to_string(err)) + } + } + } + if .event.code == "INVALID_RP_JOIN" { + if !is_nullish(.message) { + value, err = parse_groks(value: .message, patterns: [s'.*?Received \(%{PIM_SOURCE}, %{DATA:cisco.ios.pim.group.ip}\) %{WORD:cisco.ios.action} from %{IP:source.address} for %{DATA:cisco.ios.outcome} %{IP:destination.address}.*'], aliases: { "PIM_SOURCE": s'(%{IP:cisco.ios.pim.source.ip}|%{DATA})' }) + if err == null { + . = merge(., value, deep: true) + } else { + assert!(false, "grok_message_rp_join: failed to grok .message:" + to_string(err)) + } + } else { + assert!(false, ".message must not be nullish") + } + } + if .event.code == "INVALID_RP_JOIN" { + .event.action = "multicast-join" + } + if .event.code == "INVALID_RP_JOIN" { + .event.outcome = "failure" + } + if .event.code == "INVALID_RP_JOIN" { + .event.reason = "Invalid RP" + } + if !is_nullish(.destination.address) { + if is_ipv4(string!(.destination.address)) || is_ipv6(string!(.destination.address)) { + .destination.ip = .destination.address + } else { + err = "Failed converting .destination.address to an IP address" + _ingest.on_failure_processor_type = "on_failure" + _ingest.on_failure_processor_tag = "convert_destination_ip" + _ingest.on_failure_message = to_string(err) + .destination.domain = .destination.address + } + } + if !is_nullish(.source.address) { + if is_ipv4(string!(.source.address)) || is_ipv6(string!(.source.address)) { + .source.ip = .source.address + } else { + err = "Failed converting .source.address to an IP address" + _ingest.on_failure_processor_type = "on_failure" + _ingest.on_failure_processor_tag = "convert_source_ip" + _ingest.on_failure_message = to_string(err) + .source.domain = .source.address + } + } + if !is_nullish(.cisco.ios.pim.source.ip) { + if is_ipv4(string!(.cisco.ios.pim.source.ip)) || is_ipv6(string!(.cisco.ios.pim.source.ip)) { + .cisco.ios.pim.source.ip = .cisco.ios.pim.source.ip + } else { + err = "Failed converting .cisco.ios.pim.source.ip to an IP address" + assert!(false, err) + } + } + if !is_nullish(.source.port) { + if is_string(.source.port) { + temp = .source.port + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .source.port = temp + } else { + err_msg = join!(["Failed converting .source.port to an integer. Found (", to_string!(.source.port), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.source.port) { + .source.port = .source.port + } else { + err = "Failed converting .source.port to an integer" + assert!(false, err) + } + } + } + if !is_nullish(.source.packets) { + if is_string(.source.packets) { + temp = .source.packets + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .source.packets = temp + } else { + err_msg = join!(["Failed converting .source.packets to an integer. Found (", to_string!(.source.packets), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.source.packets) { + .source.packets = .source.packets + } else { + err = "Failed converting .source.packets to an integer" + assert!(false, err) + } + } + } + if !is_nullish(.destination.port) { + if is_string(.destination.port) { + temp = .destination.port + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .destination.port = temp + } else { + err_msg = join!(["Failed converting .destination.port to an integer. Found (", to_string!(.destination.port), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.destination.port) { + .destination.port = .destination.port + } else { + err = "Failed converting .destination.port to an integer" + assert!(false, err) + } + } + } + if .source.bytes != null || .destination.bytes != null { + n = 0 + if .source.bytes != null { + n = int!({ + tmp = n + if false { + tmp = null + } + tmp + }) + int!({ + tmp = .source.bytes + if false { + tmp = null + } + tmp + }) + } + if .destination.bytes != null { + n = int!({ + tmp = n + if false { + tmp = null + } + tmp + }) + int!({ + tmp = .destination.bytes + if false { + tmp = null + } + tmp + }) + } + if .network == null { + .network = { } + } + .network.bytes = n + } + if .source.packets != null { + .network.packets = .source.packets + } + if .source.ip != null && { + expr = .source.ip + value = "." + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + .network.type = "ipv4" + } + if .source.ip != null && .network.type == null { + .network.type = "ipv6" + } + if ._temp_.event.action == "denied" { + .event.action = "deny" + } + if .event.action == "deny" { + if !is_array(.event.type) { + if .event.type == null { + .event.type = [] + } else { + .event.type = [.event.type] + } + } + .event.type = append!({ + expr = .event.type + if false { + expr = null + } + expr + }, ["denied"]) + } + if ._temp_.event.action == "permitted" { + .event.action = "allow" + } + if .event.action == "allow" { + if !is_array(.event.type) { + if .event.type == null { + .event.type = [] + } else { + .event.type = [.event.type] + } + } + .event.type = append!({ + expr = .event.type + if false { + expr = null + } + expr + }, ["allowed"]) + } + if .event.severity == 0 { + .log.level = "emergencies" + } + if .event.severity == 1 { + .log.level = "alert" + } + if .event.severity == 2 { + .log.level = "critical" + } + if .event.severity == 3 { + .log.level = "error" + } + if .event.severity == 4 { + .log.level = "warning" + } + if .event.severity == 5 { + .log.level = "notification" + } + if .event.severity == 6 { + .log.level = "informational" + } + if .event.severity == 7 { + .log.level = "debug" + } + if .source.as.asn != null { + if .source.as.number == null { + .source.as.number = del(.source.as.asn) + } else { + assert!(false, ".source.as.number must be null or override: true") + } + } + if .source.as.organization_name != null { + if .source.as.organization.name == null { + .source.as.organization.name = del(.source.as.organization_name) + } else { + assert!(false, ".source.as.organization.name must be null or override: true") + } + } + if .destination.as.asn != null { + if .destination.as.number == null { + .destination.as.number = del(.destination.as.asn) + } else { + assert!(false, ".destination.as.number must be null or override: true") + } + } + if .destination.as.organization_name != null { + if .destination.as.organization.name == null { + .destination.as.organization.name = del(.destination.as.organization_name) + } else { + assert!(false, ".destination.as.organization.name must be null or override: true") + } + } + if .source.ip != null { + if .related.ip == null { + .related.ip = [.source.ip] + } else { + if is_array(.related.ip) { + if !includes!(.related.ip, .source.ip) { + .related.ip = push!(.related.ip, .source.ip) + } + } else { + if .related.ip != .source.ip { + .related.ip = [.related.ip, .source.ip] + } + } + } + } + if .destination.ip != null { + if .related.ip == null { + .related.ip = [.destination.ip] + } else { + if is_array(.related.ip) { + if !includes!(.related.ip, .destination.ip) { + .related.ip = push!(.related.ip, .destination.ip) + } + } else { + if .related.ip != .destination.ip { + .related.ip = [.related.ip, .destination.ip] + } + } + } + } + if .source.domain != null { + if .related.hosts == null { + .related.hosts = [.source.domain] + } else { + if is_array(.related.hosts) { + if !includes!(.related.hosts, .source.domain) { + .related.hosts = push!(.related.hosts, .source.domain) + } + } else { + if .related.hosts != .source.domain { + .related.hosts = [.related.hosts, .source.domain] + } + } + } + } + if .destination.domain != null { + if .related.hosts == null { + .related.hosts = [.destination.domain] + } else { + if is_array(.related.hosts) { + if !includes!(.related.hosts, .destination.domain) { + .related.hosts = push!(.related.hosts, .destination.domain) + } + } else { + if .related.hosts != .destination.domain { + .related.hosts = [.related.hosts, .destination.domain] + } + } + } + } + if .source.user.name != null { + if .related.user == null { + .related.user = [.source.user.name] + } else { + if is_array(.related.user) { + if !includes!(.related.user, .source.user.name) { + .related.user = push!(.related.user, .source.user.name) + } + } else { + if .related.user != .source.user.name { + .related.user = [.related.user, .source.user.name] + } + } + } + } + iana_numbers = { "icmp": 1, "igmp": 2, "tcp": 6, "udp": 17, "gre": 47, "icmp ipv6": 58, "eigrp": 88, "ospf": 89, "pim": 103, "sctp": 132 } + if !(is_nullish(.source.ip) || is_nullish(.destination.ip) || (is_nullish(.network.iana_number) && is_nullish(.network.transport)) || (.source.ip == "0.0.0.0" && .destination.ip == "0.0.0.0" && .source.port == 0 && .destination.port == 0)) { + iana_number = .network.iana_number + if is_nullish(iana_number) && !is_nullish(.network.transport) { + temp_iana_number, err = get(iana_numbers, [downcase!(.network.transport)]) + if err == null { + iana_number = temp_iana_number + } + } + if iana_number == null { + tmp_iana_number, err = parse_int(.network.transport) + if err == null { + iana_number = tmp_iana_number + } + } + source_port = .source.port + destination_port = .destination.port + if (iana_number == 6 || iana_number == 17) && source_port == null && destination_port == null { + source_port = 0 + destination_port = 0 + } + if (iana_number == 1) { + source_port = .icmp.type + destination_port = .icmp.code + } + if .icmp.type != null && .icmp.code != null { + source_port = to_int!(.icmp.type) + destination_port = to_int!(.icmp.code) + } + community_id = err = null + if source_port != null && destination_port != null { + community_id, err = community_id(source_ip: .source.ip, source_port: source_port, destination_ip: .destination.ip, destination_port: destination_port, protocol: iana_number, seed: 0) + } else { + community_id, err = community_id(source_ip: .source.ip, destination_ip: .destination.ip, protocol: iana_number, seed: 0) + } + if err == null { + .network.community_id = community_id + } + } + del(._temp_) + del(._conf) + if .tags == null || !({ + expr = .tags + value = "preserve_original_event" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + }) { + del(.event.original) + } + } else if _module == "cisco_umbrella" { + _ingest.pipeline = "cisco_umbrella_" + _umbrella_type + _ingest.on_failure_pipeline = _ingest.pipeline + .log.file.path = "/filebeat/cisco_umbrella/" + _umbrella_type + .tags = ["preserve_original_event"] + .ecs.version = "8.11.0" + .observer.vendor = "Cisco" + .observer.product = "Umbrella" + if .event.original == null { + if .message != null { + if .event.original == null { + .event.original = del(.message) + } else { + assert!(false, ".event.original must be null or override: true") + } + } + } + if .log.file.path != null && { + expr = .log.file.path + value = "dnslogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + csv = parse_csv!(replace!(.event.original, r'", "', s'","')) + value = csv[0] + if !is_nullish(value) { + .cisco.umbrella._tmp.time = value + } + value = csv[1] + if !is_nullish(value) { + .cisco.umbrella.identity = value + } + value = csv[2] + if !is_nullish(value) { + .cisco.umbrella.identities = value + } + value = csv[3] + if !is_nullish(value) { + .source.address = value + } + value = csv[4] + if !is_nullish(value) { + .source.nat.ip = value + } + value = csv[5] + if !is_nullish(value) { + .cisco.umbrella.action = value + } + value = csv[6] + if !is_nullish(value) { + .dns.question.type = value + } + value = csv[7] + if !is_nullish(value) { + .dns.response_code = value + } + value = csv[8] + if !is_nullish(value) { + .dns.question.name = value + } + value = csv[9] + if !is_nullish(value) { + .cisco.umbrella.categories = value + } + value = csv[10] + if !is_nullish(value) { + .cisco.umbrella.policy_identity_type = value + } + value = csv[11] + if !is_nullish(value) { + .cisco.umbrella.identity_types = value + } + value = csv[12] + if !is_nullish(value) { + .cisco.umbrella.blocked_categories = value + } + } + if !is_nullish(.dns.question.name) { + if is_array(.dns.question.name) { + .dns.question.name = map_values(array!(.dns.question.name)) -> |value| { + replace!(value, r'\.$', "") + } + } else { + if false { + .dns.question.name = null + } + target, err = replace(.dns.question.name, r'\.$', "") + if err == null { + .dns.question.name = target + } else { + assert!(false, to_string(err)) + } + } + } + if .log.file.path != null && { + expr = .log.file.path + value = "dnslogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + .observer.type = "dns" + } + if .log.file.path != null && { + expr = .log.file.path + value = "iplogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + csv = parse_csv!(replace!(.event.original, r'", "', s'","')) + value = csv[0] + if !is_nullish(value) { + .cisco.umbrella._tmp.time = value + } + value = csv[1] + if !is_nullish(value) { + .cisco.umbrella.identity = value + } + value = csv[2] + if !is_nullish(value) { + .source.address = value + } + value = csv[3] + if !is_nullish(value) { + .source.port = value + } + value = csv[4] + if !is_nullish(value) { + .destination.address = value + } + value = csv[5] + if !is_nullish(value) { + .destination.port = value + } + value = csv[6] + if !is_nullish(value) { + .cisco.umbrella.categories = value + } + } + if .log.file.path != null && { + expr = .log.file.path + value = "iplogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + .observer.type = "firewall" + } + if .log.file.path != null && { + expr = .log.file.path + value = "proxylogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + csv = parse_csv!(replace!(.event.original, r'", "', s'","')) + value = csv[0] + if !is_nullish(value) { + .cisco.umbrella._tmp.time = value + } + value = csv[1] + if !is_nullish(value) { + .cisco.umbrella.identity = value + } + value = csv[2] + if !is_nullish(value) { + .source.address = value + } + value = csv[3] + if !is_nullish(value) { + .source.nat.ip = value + } + value = csv[4] + if !is_nullish(value) { + .destination.address = value + } + value = csv[5] + if !is_nullish(value) { + .http.request.mime_type = value + } + value = csv[6] + if !is_nullish(value) { + .cisco.umbrella.action = value + } + value = csv[7] + if !is_nullish(value) { + .url.original = value + } + value = csv[8] + if !is_nullish(value) { + .http.request.referrer = value + } + value = csv[9] + if !is_nullish(value) { + .user_agent.original = value + } + value = csv[10] + if !is_nullish(value) { + .http.response.status_code = value + } + value = csv[11] + if !is_nullish(value) { + .http.request.bytes = value + } + value = csv[12] + if !is_nullish(value) { + .http.response.bytes = value + } + value = csv[13] + if !is_nullish(value) { + .http.response.body.bytes = value + } + value = csv[14] + if !is_nullish(value) { + .cisco.umbrella.sha_sha256 = value + } + value = csv[15] + if !is_nullish(value) { + .cisco.umbrella.categories = value + } + value = csv[16] + if !is_nullish(value) { + .cisco.umbrella.av_detections = value + } + value = csv[17] + if !is_nullish(value) { + .cisco.umbrella.puas = value + } + value = csv[18] + if !is_nullish(value) { + .cisco.umbrella.amp_disposition = value + } + value = csv[19] + if !is_nullish(value) { + .cisco.umbrella.amp_malware_name = value + } + value = csv[20] + if !is_nullish(value) { + .cisco.umbrella.amp_score = value + } + value = csv[21] + if !is_nullish(value) { + .cisco.umbrella.policy_identity_type = value + } + value = csv[22] + if !is_nullish(value) { + .cisco.umbrella.blocked_categories = value + } + value = csv[23] + if !is_nullish(value) { + .cisco.umbrella.identities = value + } + value = csv[24] + if !is_nullish(value) { + .cisco.umbrella.identity_types = value + } + value = csv[25] + if !is_nullish(value) { + .http.request.method = value + } + value = csv[26] + if !is_nullish(value) { + .cisco.umbrella.dlp_status = value + } + value = csv[27] + if !is_nullish(value) { + .cisco.umbrella.certificate_errors = value + } + value = csv[28] + if !is_nullish(value) { + .file.name = value + } + value = csv[29] + if !is_nullish(value) { + .cisco.umbrella.ruleset_id = value + } + value = csv[30] + if !is_nullish(value) { + .rule.id = value + } + value = csv[31] + if !is_nullish(value) { + .cisco.umbrella.destination_lists_id = value + } + value = csv[32] + if !is_nullish(value) { + .cisco.umbrella.isolate_action = value + } + value = csv[33] + if !is_nullish(value) { + .cisco.umbrella.file_action = value + } + value = csv[34] + if !is_nullish(value) { + .cisco.umbrella.warn_status = value + } + } + if .log.file.path != null && { + expr = .log.file.path + value = "proxylogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + .observer.type = "proxy" + } + if .log.file.path != null && ({ + expr = .log.file.path + value = "cloudfirewalllogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } || { + expr = .log.file.path + value = "firewalllogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + }) { + csv = parse_csv!(replace!(.event.original, r'", "', s'","')) + value = csv[0] + if !is_nullish(value) { + .cisco.umbrella._tmp.time = value + } + value = csv[1] + if !is_nullish(value) { + .cisco.umbrella.origin_id = value + } + value = csv[2] + if !is_nullish(value) { + .cisco.umbrella.identities = value + } + value = csv[3] + if !is_nullish(value) { + .cisco.umbrella.identity_types = value + } + value = csv[4] + if !is_nullish(value) { + .cisco.umbrella.direction = value + } + value = csv[5] + if !is_nullish(value) { + .network.transport = value + } + value = csv[6] + if !is_nullish(value) { + .source.bytes = value + } + value = csv[7] + if !is_nullish(value) { + .source.address = value + } + value = csv[8] + if !is_nullish(value) { + .source.port = value + } + value = csv[9] + if !is_nullish(value) { + .destination.address = value + } + value = csv[10] + if !is_nullish(value) { + .destination.port = value + } + value = csv[11] + if !is_nullish(value) { + .cisco.umbrella.datacenter = value + } + value = csv[12] + if !is_nullish(value) { + .rule.id = value + } + value = csv[13] + if !is_nullish(value) { + .cisco.umbrella.action = value + } + value = csv[14] + if !is_nullish(value) { + .cisco.umbrella.fqdns = value + } + value = csv[15] + if !is_nullish(value) { + .cisco.umbrella.destination_lists_id = value + } + } + if .log.file.path != null && ({ + expr = .log.file.path + value = "cloudfirewalllogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } || { + expr = .log.file.path + value = "firewalllogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + }) { + .observer.type = "firewall" + } + if .log.file.path != null && { + expr = .log.file.path + value = "intrusionlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + csv = parse_csv!(replace!(.event.original, r'", "', s'","')) + value = csv[0] + if !is_nullish(value) { + .cisco.umbrella._tmp.time = value + } + value = csv[1] + if !is_nullish(value) { + .cisco.umbrella.identities = value + } + value = csv[2] + if !is_nullish(value) { + .cisco.umbrella.identity_types = value + } + value = csv[3] + if !is_nullish(value) { + .cisco.umbrella.gid = value + } + value = csv[4] + if !is_nullish(value) { + .cisco.umbrella.sid = value + } + value = csv[5] + if !is_nullish(value) { + .cisco.umbrella.message = value + } + value = csv[6] + if !is_nullish(value) { + .cisco.umbrella.signature_list_id = value + } + value = csv[7] + if !is_nullish(value) { + .cisco.umbrella.severity = value + } + value = csv[8] + if !is_nullish(value) { + .cisco.umbrella.classification = value + } + value = csv[9] + if !is_nullish(value) { + .cisco.umbrella.cves = value + } + value = csv[10] + if !is_nullish(value) { + .network.protocol = value + } + value = csv[11] + if !is_nullish(value) { + .event.id = value + } + value = csv[12] + if !is_nullish(value) { + .source.address = value + } + value = csv[13] + if !is_nullish(value) { + .source.port = value + } + value = csv[14] + if !is_nullish(value) { + .destination.address = value + } + value = csv[15] + if !is_nullish(value) { + .destination.port = value + } + value = csv[16] + if !is_nullish(value) { + .cisco.umbrella.action = value + } + } + if .log.file.path != null && { + expr = .log.file.path + value = "intrusionlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + .observer.type = "idps" + } + if .log.file.path != null && { + expr = .log.file.path + value = "dlplogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + csv = parse_csv!(replace!(.event.original, r'", "', s'","')) + value = csv[0] + if !is_nullish(value) { + .cisco.umbrella._tmp.time = value + } + value = csv[1] + if !is_nullish(value) { + .event.provider = value + } + value = csv[2] + if !is_nullish(value) { + .event.id = value + } + value = csv[3] + if !is_nullish(value) { + .cisco.umbrella.severity = value + } + value = csv[4] + if !is_nullish(value) { + .network.name = value + } + value = csv[5] + if !is_nullish(value) { + .file.owner = value + } + value = csv[6] + if !is_nullish(value) { + .file.name = value + } + value = csv[7] + if !is_nullish(value) { + .network.application = value + } + value = csv[8] + if !is_nullish(value) { + .url.original = value + } + value = csv[9] + if !is_nullish(value) { + .cisco.umbrella.action = value + } + value = csv[10] + if !is_nullish(value) { + .rule.name = value + } + value = csv[11] + if !is_nullish(value) { + .cisco.umbrella.data_classification = value + } + value = csv[12] + if !is_nullish(value) { + .cisco.umbrella.data_identifier = value + } + value = csv[13] + if !is_nullish(value) { + .file.mime_type = value + } + value = csv[14] + if !is_nullish(value) { + .file.size = value + } + value = csv[15] + if !is_nullish(value) { + .file.hash.sha256 = value + } + value = csv[16] + if !is_nullish(value) { + .cisco.umbrella.file_label = value + } + } + if .log.file.path != null && { + expr = .log.file.path + value = "dlplogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + .observer.type = "dlp" + } + if .log.file.path != null && { + expr = .log.file.path + value = "auditlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + csv = parse_csv!(replace!(.event.original, r'", "', s'","')) + value = csv[0] + if !is_nullish(value) { + .event.id = value + } + value = csv[1] + if !is_nullish(value) { + .cisco.umbrella._tmp.time = value + } + value = csv[2] + if !is_nullish(value) { + .user.email = value + } + value = csv[3] + if !is_nullish(value) { + .user.name = value + } + value = csv[4] + if !is_nullish(value) { + .cisco.umbrella.audit.type = value + } + value = csv[5] + if !is_nullish(value) { + .event.action = value + } + value = csv[6] + if !is_nullish(value) { + .source.address = value + } + value = csv[7] + if !is_nullish(value) { + .cisco.umbrella.audit.before = value + } + value = csv[8] + if !is_nullish(value) { + .cisco.umbrella.audit.after = value + } + } + if .cisco.umbrella.audit.type != "global_settings" { + if !is_nullish(.cisco.umbrella.audit.before) { + kv, err = parse_key_value(.cisco.umbrella.audit.before, key_value_delimiter: ":", field_delimiter: "\n", accept_standalone_key: true, whitespace: "strict") + if err == null { + standalone_key = false + for_each(kv) -> |_, value| { + if value == true { + standalone_key = true + } + } + if standalone_key { + err = "field .cisco.umbrella.audit.before does not contain value_split [:]" + } + } + if err == null { + if .cisco.umbrella.audit.before_values == null { + .cisco.umbrella.audit.before_values = { } + } + .cisco.umbrella.audit.before_values = merge(object!(.cisco.umbrella.audit.before_values), kv, deep: true) + } else { + _ingest.on_failure_processor_type = "kv" + _ingest.on_failure_processor_tag = "kv_audit_before" + _ingest.on_failure_message = to_string(err) + if !is_array(.error.message) { + if .error.message == null { + .error.message = [] + } else { + .error.message = [.error.message] + } + } + .error.message = append!({ + expr = .error.message + if false { + expr = null + } + expr + }, [join!(["Processor ", _ingest.on_failure_processor_type, " with tag ", _ingest.on_failure_processor_tag, " in pipeline ", _ingest.on_failure_pipeline, " failed with message: ", _ingest.on_failure_message], "")]) + } + } + } + if .cisco.umbrella.audit.type != "global_settings" { + if !is_nullish(.cisco.umbrella.audit.before) { + .cisco.umbrella.audit.before = split(string!(.cisco.umbrella.audit.before), r'\n') + } + } + if .cisco.umbrella.audit.type != "global_settings" { + if !is_nullish(.cisco.umbrella.audit.after) { + kv, err = parse_key_value(.cisco.umbrella.audit.after, key_value_delimiter: ":", field_delimiter: "\n", accept_standalone_key: true, whitespace: "strict") + if err == null { + standalone_key = false + for_each(kv) -> |_, value| { + if value == true { + standalone_key = true + } + } + if standalone_key { + err = "field .cisco.umbrella.audit.after does not contain value_split [:]" + } + } + if err == null { + if .cisco.umbrella.audit.after_values == null { + .cisco.umbrella.audit.after_values = { } + } + .cisco.umbrella.audit.after_values = merge(object!(.cisco.umbrella.audit.after_values), kv, deep: true) + } else { + _ingest.on_failure_processor_type = "kv" + _ingest.on_failure_processor_tag = "kv_audit_after" + _ingest.on_failure_message = to_string(err) + if !is_array(.error.message) { + if .error.message == null { + .error.message = [] + } else { + .error.message = [.error.message] + } + } + .error.message = append!({ + expr = .error.message + if false { + expr = null + } + expr + }, [join!(["Processor ", _ingest.on_failure_processor_type, " with tag ", _ingest.on_failure_processor_tag, " in pipeline ", _ingest.on_failure_pipeline, " failed with message: ", _ingest.on_failure_message], "")]) + } + } + } + if .cisco.umbrella.audit.type != "global_settings" { + if !is_nullish(.cisco.umbrella.audit.after) { + .cisco.umbrella.audit.after = split(string!(.cisco.umbrella.audit.after), r'\n') + } + } + if .url.original != null { + uri, err = parse_url(.url.original, default_known_ports: true) + if err == null { + if uri.username == "" { + del(uri.username) + } + if uri.password == "" { + del(uri.password) + } + uri.query = encode_key_value!(uri.query, field_delimiter: "&", key_value_delimiter: "=") + if uri.query == "" { + del(uri.query) + } + if uri.path != null { + uri.path = decode_percent!(uri.path) + } + uri.original = .url.original + uri.extension = parse_regex(uri.path, r'.*\.(?P[a-zA-Z0-9_\-]+)$').extension ?? null + uri.domain = del(uri.host) + if uri.username != null && uri.password != null { + uri.user_info = join!([uri.username, uri.password], separator: ":") + } + .url = compact(uri) + } + } + if (is_string(.cisco.umbrella.identity)) && { + expr = .cisco.umbrella.identity + value = "," + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } && (is_string(.cisco.umbrella.identities)) && starts_with!({ + expr = .cisco.umbrella.identities + if false { + expr = null + } + expr + }, .cisco.umbrella.identity) { + identities_tail = slice!(.cisco.umbrella.identities, length!({ + expr = .cisco.umbrella.identity + if false { + expr = null + } + expr + })) + if starts_with!({ + expr = identities_tail + if false { + expr = null + } + expr + }, ",") { + identities_tail = slice!(identities_tail, 1) + } + if .cisco.umbrella._tmp == null { + .cisco.umbrella._tmp = { } + } + .cisco.umbrella._tmp.identities_tail = identities_tail + } + if .cisco.umbrella._tmp.identities_tail != null { + .cisco.umbrella._tmp.identities_tail = split(string!(.cisco.umbrella._tmp.identities_tail), r',\s*') + } + if .cisco.umbrella._tmp.identities_tail != null { + identities = [] + if false { + identities = null + } + identities = push!(identities, .cisco.umbrella.identity) + for_each(array!({ + expr = .cisco.umbrella._tmp.identities_tail + if false { + expr = null + } + expr + })) -> |_index, identity| { + { + if false { + identities = null + } + identities = push!(identities, identity) + } + } + .cisco.umbrella._tmp.identities = identities + } + if .cisco.umbrella._tmp.identities != null { + .cisco.umbrella.identities = .cisco.umbrella._tmp.identities + } + if .cisco.umbrella.identities != null && .cisco.umbrella._tmp.identities == null { + .cisco.umbrella.identities = split(string!(.cisco.umbrella.identities), r',\s*') + } + if .log.file.path != null && { + expr = .log.file.path + value = "dnslogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } && .cisco.umbrella.categories != null { + .cisco.umbrella.categories = split(string!(.cisco.umbrella.categories), r',\s*') + } + if .log.file.path != null && { + expr = .log.file.path + value = "dnslogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } && .cisco.umbrella.blocked_categories != null { + .cisco.umbrella.blocked_categories = split(string!(.cisco.umbrella.blocked_categories), r',\s*') + } + if .cisco.umbrella.identity_types != null { + .cisco.umbrella.identity_types = split(string!(.cisco.umbrella.identity_types), r',\s*') + } + if .cisco.umbrella.fqdns != null { + .cisco.umbrella.fqdns = split(string!(.cisco.umbrella.fqdns), r',\s*') + } + if .cisco.umbrella.identities != null && .cisco.umbrella.identity_types != null && { + temp = .cisco.umbrella.identities + if false { + temp = null + } + length!(temp) + } == { + temp = .cisco.umbrella.identity_types + if false { + temp = null + } + length!(temp) + } { + i = 0 + for_each(array!({ + expr = .cisco.umbrella.identity_types + if false { + expr = null + } + expr + })) -> |_index, cisco_identity_type| { + { + if { + expr = ["AD Users"] + value = cisco_identity_type + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + { + x = get!(.cisco.umbrella.identities, [i]) + { + if .user == null { + .user = { } + } + if .user.name == null { + .user.name = x + } + } + } + } + if { + expr = ["AD Computers", "Roaming Computers", "Anyconnect Roaming Client", "Mobile Devices"] + value = cisco_identity_type + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + { + x = get!(.cisco.umbrella.identities, [i]) + { + if .host == null { + .host = { } + } + if .host.name == null { + .host.name = x + } + } + } + } + if { + expr = ["Sites", "Internal Networks", "Networks", "Network Devices", "Network Tunnels", "CDFW Tunnel Device"] + value = cisco_identity_type + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + { + x = get!(.cisco.umbrella.identities, [i]) + { + if .network == null { + .network = { } + } + if .network.name == null { + al = [] + .network = set!(.network, ["name"], al) + } + if !{ + expr = .network.name + value = x + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + if false { + .network.name = null + } + .network.name = push!(.network.name, x) + } + } + } + } + { + tmp = i + i = i + 1 + tmp + } + } + } + } + if .host.name != null { + if !is_nullish(.host.name) { + if is_array(.host.name) { + .host.name = map_values(array!(.host.name)) -> |value| { + downcase!(value) + } + } else { + tmp, err = downcase(.host.name) + if err == null { + .host.name = tmp + } else { + assert!(false, to_string(err)) + } + } + } else { + assert!(false, ".host.name must not be nullish") + } + } + if .host.name != null && { + expr = .host.name + value = "." + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + if !is_nullish(.host.name) { + value, err = parse_groks(value: .host.name, patterns: [s'.*?^%{DATA:host.hostname}\.%{GREEDYDATA:host.domain}$.*']) + if err == null { + . = merge(., value, deep: true) + } + } + } + if .user.name != null && { + expr = .user.name + value = "@" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } == true { + if !is_nullish(.user.name) { + value, err = parse_groks(value: .user.name, patterns: [s'.*?^%{GREEDYDATA:user.full_name} \(%{DATA:user.name}@%{DATA:user.domain}\)?$.*']) + if err == null { + . = merge(., value, deep: true) + } + } + } + if .user.email == null && .user.name != null && .user.domain != null { + .user.email = join!([to_string!(.user.name), "@", to_string!(.user.domain)], "") + } + if .user.email != null { + .user.id = .user.email + } + if .user.email == "null" { + if !exists(.user.email) { + assert!(false, "tried to remove field '.user.email' that doesn't exist") + } + del(.user.email) + } + if .user.id == "null" { + if !exists(.user.id) { + assert!(false, "tried to remove field '.user.id' that doesn't exist") + } + del(.user.id) + } + if .user.name == "null" { + if !exists(.user.name) { + assert!(false, "tried to remove field '.user.name' that doesn't exist") + } + del(.user.name) + } + if is_object(.user) && length!(.user) == 0 { + del(.user) + } + if .cisco.umbrella._tmp.time != null { + date = null + err = "no datetime formats provided" + if err != null { + datetime = to_string!(.cisco.umbrella._tmp.time) + date, err = parse_timestamp(value: join!(value: [datetime], separator: " "), format: join!(value: ["%Y-%m-%d %H:%M:%S"], separator: " ")) + } + if err != null { + datetime = to_string!(.cisco.umbrella._tmp.time) + date, err = parse_timestamp(value: join!(value: [datetime], separator: " "), format: join!(value: ["%+"], separator: " ")) + } + if err == null { + .@timestamp = date + } else { + datetime = to_string!(.cisco.umbrella._tmp.time) + assert!(false, "failed to parse timestamp .cisco.umbrella._tmp.time: ({{ datetime }}): " + to_string(err)) + } + } + if .dns.question.name != null { + .dns.type = "query" + } + if .dns.question.name != null { + tmp, err = parse_etld(.dns.question.name, plus_parts: 1) + if err == null { + .dns.question.registered_domain = tmp.etld_plus + .dns.question.top_level_domain = tmp.etld + .dns.question.domain = .dns.question.name + .dns.question.subdomain = { + subdomain = replace!(.dns.question.name, tmp.etld_plus, "") + if subdomain != "" { + subdomain = slice!(subdomain, 0, -1) + } + subdomain + } + .dns.question = compact(.dns.question) + } else { + assert!(false, to_string(err)) + } + } + del(.dns.question.domain) + if is_string(.user_agent.original) { + user_agent = parse_user_agent!(.user_agent.original) + .user_agent.version = user_agent.browser.version + .user_agent.os.name = user_agent.os.family + .user_agent.os.version = user_agent.os.version + .user_agent.os.full, _ = join([user_agent.os.family, user_agent.os.version], separator: " ") + .user_agent.name = user_agent.browser.family + if is_nullish(.user_agent.name) { + .user_agent.name = "Other" + } + .user_agent.device.name = user_agent.device.category + if is_nullish(.user_agent.device.name) { + .user_agent.device.name = "Other" + } + .user_agent = compact(.user_agent, recursive: true) + if .user_agent.original == null { + .user_agent.original = "" + } else { + .user_agent.original = .user_agent.original + } + } + if .cisco.umbrella.direction != null { + if !is_nullish(.cisco.umbrella.direction) { + if is_array(.cisco.umbrella.direction) { + .cisco.umbrella.direction = map_values(array!(.cisco.umbrella.direction)) -> |value| { + downcase!(value) + } + } else { + tmp, err = downcase(.cisco.umbrella.direction) + if err == null { + .network.direction = tmp + } else { + assert!(false, to_string(err)) + } + } + } else { + assert!(false, ".cisco.umbrella.direction must not be nullish") + } + } + if .source.bytes != null { + if !is_nullish(.source.bytes) { + if is_string(.source.bytes) { + temp = .source.bytes + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .source.bytes = temp + } else { + err_msg = join!(["Failed converting .source.bytes to an integer. Found (", to_string!(.source.bytes), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.source.bytes) { + .source.bytes = .source.bytes + } else { + err = "Failed converting .source.bytes to an integer" + assert!(false, err) + } + } + } else { + err = ".source.bytes must be present" + assert!(false, err) + } + } + if .source.port != null { + if !is_nullish(.source.port) { + if is_string(.source.port) { + temp = .source.port + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .source.port = temp + } else { + err_msg = join!(["Failed converting .source.port to an integer. Found (", to_string!(.source.port), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.source.port) { + .source.port = .source.port + } else { + err = "Failed converting .source.port to an integer" + assert!(false, err) + } + } + } else { + err = ".source.port must be present" + assert!(false, err) + } + } + if .destination.port != null { + if !is_nullish(.destination.port) { + if is_string(.destination.port) { + temp = .destination.port + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .destination.port = temp + } else { + err_msg = join!(["Failed converting .destination.port to an integer. Found (", to_string!(.destination.port), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.destination.port) { + .destination.port = .destination.port + } else { + err = "Failed converting .destination.port to an integer" + assert!(false, err) + } + } + } else { + err = ".destination.port must be present" + assert!(false, err) + } + } + if .http.request.bytes != null { + if !is_nullish(.http.request.bytes) { + if is_string(.http.request.bytes) { + temp = .http.request.bytes + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .http.request.bytes = temp + } else { + err_msg = join!(["Failed converting .http.request.bytes to an integer. Found (", to_string!(.http.request.bytes), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.http.request.bytes) { + .http.request.bytes = .http.request.bytes + } else { + err = "Failed converting .http.request.bytes to an integer" + assert!(false, err) + } + } + } else { + err = ".http.request.bytes must be present" + assert!(false, err) + } + } + if .http.response.bytes != null { + if !is_nullish(.http.response.bytes) { + if is_string(.http.response.bytes) { + temp = .http.response.bytes + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .http.response.bytes = temp + } else { + err_msg = join!(["Failed converting .http.response.bytes to an integer. Found (", to_string!(.http.response.bytes), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.http.response.bytes) { + .http.response.bytes = .http.response.bytes + } else { + err = "Failed converting .http.response.bytes to an integer" + assert!(false, err) + } + } + } else { + err = ".http.response.bytes must be present" + assert!(false, err) + } + } + if .http.response.body.bytes != null { + if !is_nullish(.http.response.body.bytes) { + if is_string(.http.response.body.bytes) { + temp = .http.response.body.bytes + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .http.response.body.bytes = temp + } else { + err_msg = join!(["Failed converting .http.response.body.bytes to an integer. Found (", to_string!(.http.response.body.bytes), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.http.response.body.bytes) { + .http.response.body.bytes = .http.response.body.bytes + } else { + err = "Failed converting .http.response.body.bytes to an integer" + assert!(false, err) + } + } + } else { + err = ".http.response.body.bytes must be present" + assert!(false, err) + } + } + if .http.response.status_code != null { + if !is_nullish(.http.response.status_code) { + if is_string(.http.response.status_code) { + temp = .http.response.status_code + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .http.response.status_code = temp + } else { + err_msg = join!(["Failed converting .http.response.status_code to an integer. Found (", to_string!(.http.response.status_code), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.http.response.status_code) { + .http.response.status_code = .http.response.status_code + } else { + err = "Failed converting .http.response.status_code to an integer" + assert!(false, err) + } + } + } else { + err = ".http.response.status_code must be present" + assert!(false, err) + } + } + if .cisco.umbrella.action != null { + if !is_nullish(.cisco.umbrella.action) { + if is_array(.cisco.umbrella.action) { + .cisco.umbrella.action = map_values(array!(.cisco.umbrella.action)) -> |value| { + replace!(value, r'\s', "-") + } + } else { + if false { + .cisco.umbrella.action = null + } + target, err = replace(.cisco.umbrella.action, r'\s', "-") + if err == null { + .cisco.umbrella.action = target + } else { + assert!(false, to_string(err)) + } + } + } else { + assert!(false, ".cisco.umbrella.action must not be nullish") + } + } + if .cisco.umbrella.action != null && .log.file.path != null && { + expr = .log.file.path + value = "dnslogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + .event.action = join!(["dns-request-", to_string!(.cisco.umbrella.action)], "") + } + if .http.request.method != null && .log.file.path != null && { + expr = .log.file.path + value = "proxylogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + .event.action = join!(["proxy-request-", to_string!(.http.request.method)], "") + } + if .cisco.umbrella.action != null && .log.file.path != null && ({ + expr = .log.file.path + value = "firewalllogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } || { + expr = .log.file.path + value = "cloudfirewalllogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + }) { + .event.action = join!(["fw-connection-", to_string!(.cisco.umbrella.action)], "") + } + if .cisco.umbrella.action != null && .log.file.path != null && { + expr = .log.file.path + value = "intrusionlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + .event.action = join!(["ips-", to_string!(.cisco.umbrella.action)], "") + } + if .cisco.umbrella.action != null && .log.file.path != null && { + expr = .log.file.path + value = "dlplogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + .event.action = join!(["dlp-", to_string!(.cisco.umbrella.action)], "") + } + if .log.file.path != null && !{ + expr = .log.file.path + value = "auditlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + .event.category = "network" + } + if .cisco.umbrella.action != null && { + expr = ["Allowed", "ALLOWED", "ALLOW", "Would-Block"] + value = .cisco.umbrella.action + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + if !is_array(.event.type) { + if .event.type == null { + .event.type = [] + } else { + .event.type = [.event.type] + } + } + .event.type = append!({ + expr = .event.type + if false { + expr = null + } + expr + }, ["allowed"]) + } + if .cisco.umbrella.action != null && { + expr = ["Blocked", "BLOCKED", "BLOCK"] + value = .cisco.umbrella.action + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + if !is_array(.event.type) { + if .event.type == null { + .event.type = [] + } else { + .event.type = [.event.type] + } + } + .event.type = append!({ + expr = .event.type + if false { + expr = null + } + expr + }, ["denied"]) + } + if .log.file.path != null && !{ + expr = .log.file.path + value = "auditlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + if !is_array(.event.type) { + if .event.type == null { + .event.type = [] + } else { + .event.type = [.event.type] + } + } + .event.type = append!({ + expr = .event.type + if false { + expr = null + } + expr + }, ["connection"]) + } + if .log.file.path != null && { + expr = .log.file.path + value = "auditlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + .event.category = "configuration" + } + if .log.file.path != null && { + expr = .log.file.path + value = "dlplogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + if !is_array(.event.category) { + if .event.category == null { + .event.category = [] + } else { + .event.category = [.event.category] + } + } + .event.category = append!({ + expr = .event.category + if false { + expr = null + } + expr + }, ["file"]) + } + if .log.file.path != null && { + expr = .log.file.path + value = "intrusionlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + if !is_array(.event.category) { + if .event.category == null { + .event.category = [] + } else { + .event.category = [.event.category] + } + } + .event.category = append!({ + expr = .event.category + if false { + expr = null + } + expr + }, ["intrusion_detection"]) + } + if .log.file.path != null && { + expr = .log.file.path + value = "intrusionlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + .event.kind = "alert" + } + if .log.file.path != null && !{ + expr = .log.file.path + value = "intrusionlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + .event.kind = "event" + } + if .log.file.path != null && { + expr = .log.file.path + value = "intrusionlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + if !is_array(.event.type) { + if .event.type == null { + .event.type = [] + } else { + .event.type = [.event.type] + } + } + .event.type = append!({ + expr = .event.type + if false { + expr = null + } + expr + }, ["info"]) + } + if .log.file.path != null && { + expr = .log.file.path + value = "dlplogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } { + if !is_array(.event.type) { + if .event.type == null { + .event.type = [] + } else { + .event.type = [.event.type] + } + } + .event.type = append!({ + expr = .event.type + if false { + expr = null + } + expr + }, ["info"]) + } + if .log.file.path != null && { + expr = .log.file.path + value = "auditlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } && downcase(string!(.event.action)) == "create" { + if !is_array(.event.type) { + if .event.type == null { + .event.type = [] + } else { + .event.type = [.event.type] + } + } + .event.type = append!({ + expr = .event.type + if false { + expr = null + } + expr + }, ["creation"]) + } + if .log.file.path != null && { + expr = .log.file.path + value = "auditlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } && downcase(string!(.event.action)) == "update" { + if !is_array(.event.type) { + if .event.type == null { + .event.type = [] + } else { + .event.type = [.event.type] + } + } + .event.type = append!({ + expr = .event.type + if false { + expr = null + } + expr + }, ["change"]) + } + if .log.file.path != null && { + expr = .log.file.path + value = "auditlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } && downcase(string!(.event.action)) == "delete" { + if !is_array(.event.type) { + if .event.type == null { + .event.type = [] + } else { + .event.type = [.event.type] + } + } + .event.type = append!({ + expr = .event.type + if false { + expr = null + } + expr + }, ["deletion"]) + } + if !is_nullish(.source.address) { + if is_ipv4(string!(.source.address)) || is_ipv6(string!(.source.address)) { + .source.ip = .source.address + } else { + err = "Failed converting .source.address to an IP address" + _ingest.on_failure_processor_type = "on_failure" + _ingest.on_failure_processor_tag = "" + _ingest.on_failure_message = to_string(err) + .source.domain = .source.address + } + } + if !is_nullish(.destination.address) { + if is_ipv4(string!(.destination.address)) || is_ipv6(string!(.destination.address)) { + .destination.ip = .destination.address + } else { + err = "Failed converting .destination.address to an IP address" + _ingest.on_failure_processor_type = "on_failure" + _ingest.on_failure_processor_tag = "" + _ingest.on_failure_message = to_string(err) + .destination.domain = .destination.address + } + } + if !is_nullish(.source.nat.ip) { + if is_ipv4(string!(.source.nat.ip)) || is_ipv6(string!(.source.nat.ip)) { + .source.nat.ip = .source.nat.ip + } else { + err = "Failed converting .source.nat.ip to an IP address" + _ingest.on_failure_processor_type = "on_failure" + _ingest.on_failure_processor_tag = "" + _ingest.on_failure_message = to_string(err) + if !exists(.source.nat.ip) { + assert!(false, "tried to remove field '.source.nat.ip' that doesn't exist") + } + del(.source.nat.ip) + } + } + iana_numbers = { "icmp": 1, "igmp": 2, "tcp": 6, "udp": 17, "gre": 47, "icmp ipv6": 58, "eigrp": 88, "ospf": 89, "pim": 103, "sctp": 132 } + if !(is_nullish(.source.ip) || is_nullish(.destination.ip) || (is_nullish(.network.iana_number) && is_nullish(.network.transport)) || (.source.ip == "0.0.0.0" && .destination.ip == "0.0.0.0" && .source.port == 0 && .destination.port == 0)) { + iana_number = .network.iana_number + if is_nullish(iana_number) && !is_nullish(.network.transport) { + temp_iana_number, err = get(iana_numbers, [downcase!(.network.transport)]) + if err == null { + iana_number = temp_iana_number + } else { + assert!(false, to_string(err)) + } + } + if iana_number == null { + tmp_iana_number, err = parse_int(.network.transport) + if err == null { + iana_number = tmp_iana_number + } + } + source_port = .source.port + destination_port = .destination.port + if (iana_number == 6 || iana_number == 17) && source_port == null && destination_port == null { + source_port = 0 + destination_port = 0 + } + if (iana_number == 1) { + source_port = .icmp.type + destination_port = .icmp.code + } + if .icmp.type != null && .icmp.code != null { + source_port = to_int!(.icmp.type) + destination_port = to_int!(.icmp.code) + } + community_id = err = null + if source_port != null && destination_port != null { + community_id, err = community_id(source_ip: .source.ip, source_port: source_port, destination_ip: .destination.ip, destination_port: destination_port, protocol: iana_number, seed: 0) + } else { + community_id, err = community_id(source_ip: .source.ip, destination_ip: .destination.ip, protocol: iana_number, seed: 0) + } + if err == null { + .network.community_id = community_id + } else { + assert!(false, to_string(err)) + } + } + if .source.as.asn != null { + if .source.as.number == null { + .source.as.number = del(.source.as.asn) + } else { + assert!(false, ".source.as.number must be null or override: true") + } + } + if .source.as.organization_name != null { + if .source.as.organization.name == null { + .source.as.organization.name = del(.source.as.organization_name) + } else { + assert!(false, ".source.as.organization.name must be null or override: true") + } + } + if .destination.as.asn != null { + if .destination.as.number == null { + .destination.as.number = del(.destination.as.asn) + } else { + assert!(false, ".destination.as.number must be null or override: true") + } + } + if .destination.as.organization_name != null { + if .destination.as.organization.name == null { + .destination.as.organization.name = del(.destination.as.organization_name) + } else { + assert!(false, ".destination.as.organization.name must be null or override: true") + } + } + if .cisco.umbrella.severity == "LOW" && .log.file.path != null && ({ + expr = .log.file.path + value = "intrusionlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } || { + expr = .log.file.path + value = "dlplogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + }) { + .event.severity = 1 + } + if .cisco.umbrella.severity == "MEDIUM" && .log.file.path != null && ({ + expr = .log.file.path + value = "intrusionlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } || { + expr = .log.file.path + value = "dlplogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + }) { + .event.severity = 2 + } + if .cisco.umbrella.severity == "HIGH" && .log.file.path != null && ({ + expr = .log.file.path + value = "intrusionlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } || { + expr = .log.file.path + value = "dlplogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + }) { + .event.severity = 3 + } + if .cisco.umbrella.severity == "CRITICAL" && .log.file.path != null && ({ + expr = .log.file.path + value = "intrusionlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } || { + expr = .log.file.path + value = "dlplogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + }) { + .event.severity = 4 + } + if .event.severity == null && .log.file.path != null && ({ + expr = .log.file.path + value = "intrusionlogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + } || { + expr = .log.file.path + value = "dlplogs" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + }) { + .event.severity = 0 + } + if .file.size != null { + if !is_nullish(.file.size) { + if is_string(.file.size) { + temp = .file.size + if temp != "0" { + temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") + } + temp, err = parse_int(temp) + if err == null { + .file.size = temp + } else { + err_msg = join!(["Failed converting .file.size to an integer. Found (", to_string!(.file.size), "), ", to_string(err)], "") + assert!(false, err_msg) + } + } else { + if is_integer(.file.size) { + .file.size = .file.size + } else { + err = "Failed converting .file.size to an integer" + assert!(false, err) + } + } + } else { + err = ".file.size must be present" + assert!(false, err) + } + } + if .user.name != null { + if .related.user == null { + .related.user = [.user.name] + } else { + if is_array(.related.user) { + if !includes!(.related.user, .user.name) { + .related.user = push!(.related.user, .user.name) + } + } else { + if .related.user != .user.name { + .related.user = [.related.user, .user.name] + } + } + } + } + if .source.ip != null { + if .related.ip == null { + .related.ip = [.source.ip] + } else { + if is_array(.related.ip) { + if !includes!(.related.ip, .source.ip) { + .related.ip = push!(.related.ip, .source.ip) + } + } else { + if .related.ip != .source.ip { + .related.ip = [.related.ip, .source.ip] + } + } + } + } + if .source.nat.ip != null { + if .related.ip == null { + .related.ip = [.source.nat.ip] + } else { + if is_array(.related.ip) { + if !includes!(.related.ip, .source.nat.ip) { + .related.ip = push!(.related.ip, .source.nat.ip) + } + } else { + if .related.ip != .source.nat.ip { + .related.ip = [.related.ip, .source.nat.ip] + } + } + } + } + if .destination.ip != null { + if .related.ip == null { + .related.ip = [.destination.ip] + } else { + if is_array(.related.ip) { + if !includes!(.related.ip, .destination.ip) { + .related.ip = push!(.related.ip, .destination.ip) + } + } else { + if .related.ip != .destination.ip { + .related.ip = [.related.ip, .destination.ip] + } + } + } + } + if .host.name != null { + if .related.hosts == null { + .related.hosts = [.host.name] + } else { + if is_array(.related.hosts) { + if !includes!(.related.hosts, .host.name) { + .related.hosts = push!(.related.hosts, .host.name) + } + } else { + if .related.hosts != .host.name { + .related.hosts = [.related.hosts, .host.name] + } + } + } + } + if .source.domain != null { + if .related.hosts == null { + .related.hosts = [.source.domain] + } else { + if is_array(.related.hosts) { + if !includes!(.related.hosts, .source.domain) { + .related.hosts = push!(.related.hosts, .source.domain) + } + } else { + if .related.hosts != .source.domain { + .related.hosts = [.related.hosts, .source.domain] + } + } + } + } + if .dns.question.name != null { + if .related.hosts == null { + .related.hosts = [.dns.question.name] + } else { + if is_array(.related.hosts) { + if !includes!(.related.hosts, .dns.question.name) { + .related.hosts = push!(.related.hosts, .dns.question.name) + } + } else { + if .related.hosts != .dns.question.name { + .related.hosts = [.related.hosts, .dns.question.name] + } + } + } + } + if .cisco.umbrella.fqdns != null { + if !is_nullish(.cisco.umbrella.fqdns) { + .cisco.umbrella.fqdns = map_values(array!(.cisco.umbrella.fqdns)) -> |value| { + _ingest._value = value + if .related.hosts == null { + .related.hosts = [_ingest._value] + } else { + if is_array(.related.hosts) { + if !includes!(.related.hosts, _ingest._value) { + .related.hosts = push!(.related.hosts, _ingest._value) + } + } else { + if .related.hosts != _ingest._value { + .related.hosts = [.related.hosts, _ingest._value] + } + } + } + _ingest._value + } + } else { + err = ".cisco.umbrella.fqdns must not be nullish" + assert!(false, err) + } + } + if .cisco.umbrella.sha_sha256 != null { + if .related.hash == null { + .related.hash = [.cisco.umbrella.sha_sha256] + } else { + if is_array(.related.hash) { + if !includes!(.related.hash, .cisco.umbrella.sha_sha256) { + .related.hash = push!(.related.hash, .cisco.umbrella.sha_sha256) + } + } else { + if .related.hash != .cisco.umbrella.sha_sha256 { + .related.hash = [.related.hash, .cisco.umbrella.sha_sha256] + } + } + } + } + if .file.hash.sha256 != null { + if .related.hash == null { + .related.hash = [.file.hash.sha256] + } else { + if is_array(.related.hash) { + if !includes!(.related.hash, .file.hash.sha256) { + .related.hash = push!(.related.hash, .file.hash.sha256) + } + } else { + if .related.hash != .file.hash.sha256 { + .related.hash = [.related.hash, .file.hash.sha256] + } + } + } + } + del(.cisco.umbrella._tmp) + del(.cisco.umbrella.direction) + del(.cisco.umbrella.action) + del(.log.flags) + if .tags == null || !({ + expr = .tags + value = "preserve_original_event" + if false { + expr = null + value = null + } + if is_array(expr) { + includes(array!(expr), value) + } else { + if is_string(expr) { + contains(string!(expr), string!(value)) + } else { + assert!(false, "contains only works on strings and array") + } + } + }) { + del(.event.original) + } + } else { + # Unknown shape: tag + pass through, never silently drop. + .tags = push(array(.tags) ?? [], "filebeat_unmatched") + } diff --git a/pipelines/filebeat/timezones.csv b/pipelines/filebeat/timezones.csv new file mode 100644 index 0000000..259a4fe --- /dev/null +++ b/pipelines/filebeat/timezones.csv @@ -0,0 +1,213 @@ +abbreviation,name,offset +ACDT, Australian Central Daylight Time, +10:30 +ACST, Australian Central Standard Time, +09:30 +ACT, Acre Time, -05:00 +ACWST, Australian Central Western Standard Time, +8:45 +ADT, Arabia Daylight Time, +04:00 +ADT, Atlantic Daylight Time, -03:00 +AEDT, Australian Eastern Daylight Time, +11:00 +AEST, Australian Eastern Standard Time, +10:00 +AFT, Afghanistan Time, +4:30 +AKDT, Alaska Daylight Time, -08:00 +AKST, Alaska Standard Time, -09:00 +ALMT, Alma-Ata Time, +06:00 +AMST, Amazon Summer Time, -03:00 +AMST, Armenia Summer Time, +05:00 +AMT, Amazon Time, -04:00 +AMT, Armenia Time, +04:00 +ANAST, Anadyr Summer Time, +12:00 +ANAT, Anadyr Time, +12:00 +AQTT, Aqtobe Time, +05:00 +ART, Argentina Time, -03:00 +AST, Arabia Standard Time, +03:00 +AST, Atlantic Standard Time, -04:00 +AWDT, Australian Western Daylight Time, +09:00 +AWST, Australian Western Standard Time, +08:00 +AZOST, Azores Summer Time, +00:00 +AZOT, Azores Time, -01:00 +AZST, Azerbaijan Summer Time, +05:00 +AZT, Azerbaijan Time, +04:00 +AoE, Anywhere on Earth, -12:00 +BNT, Brunei Darussalam Time, +08:00 +BOT, Bolivia Time, -04:00 +BRST, Brasília Summer Time, -02:00 +BRT, Brasília Time, -03:00 +BST, Bangladesh Standard Time, +06:00 +BST, Bougainville Standard Time, +11:00 +BST, British Summer Time, +01:00 +BTT, Bhutan Time, +06:00 +CAST, Casey Time, +08:00 +CAT, Central Africa Time, +02:00 +CCT, Cocos Islands Time, +6:30 +CDT, Central Daylight Time, -05:00 +CDT, Cuba Daylight Time, -04:00 +CEST, Central European Summer Time, +02:00 +CET, Central European Time, +01:00 +CHADT, Chatham Island Daylight Time, +13:45 +CHAST, Chatham Island Standard Time, +12:45 +CHOST, Choibalsan Summer Time, +09:00 +CHOT, Choibalsan Time, +08:00 +CHUT, Chuuk Time, +10:00 +CIDST, Cayman Islands Daylight Saving Time, -04:00 +CIST, Cayman Islands Standard Time, -05:00 +CKT, Cook Island Time, -10:00 +CLST, Chile Summer Time, -03:00 +CLT, Chile Standard Time, -04:00 +COT, Colombia Time, -05:00 +CST, Central Standard Time, -06:00 +CST, China Standard Time, +08:00 +CST, Cuba Standard Time, -05:00 +CVT, Cape Verde Time, -01:00 +CXT, Christmas Island Time, +07:00 +ChST, Chamorro Standard Time, +10:00 +DAVT, Davis Time, +07:00 +DDUT, Dumont-d'Urville Time, +10:00 +EASST, Easter Island Summer Time, -05:00 +EAST, Easter Island Standard Time, -06:00 +EAT, Eastern Africa Time, +03:00 +ECT, Ecuador Time, -05:00 +EDT, Eastern Daylight Time, -04:00 +EEST, Eastern European Summer Time, +03:00 +EET, Eastern European Time, +02:00 +EGST, Eastern Greenland Summer Time, +00:00 +EGT, East Greenland Time, -01:00 +EST, Eastern Standard Time, -05:00 +FET, Further-Eastern European Time, +03:00 +FJST, Fiji Summer Time, +13:00 +FJT, Fiji Time, +12:00 +FKST, Falkland Islands Summer Time, -03:00 +FKT, Falkland Island Time, -04:00 +FNT, Fernando de Noronha Time, -02:00 +GALT, Galapagos Time, -06:00 +GAMT, Gambier Time, -09:00 +GET, Georgia Standard Time, +04:00 +GFT, French Guiana Time, -03:00 +GILT, Gilbert Island Time, +12:00 +GMT, Greenwich Mean Time, +00:00 +GST, Gulf Standard Time, +04:00 +GST, South Georgia Time, -02:00 +GYT, Guyana Time, -04:00 +HDT, Hawaii-Aleutian Daylight Time, -09:00 +HKT, Hong Kong Time, +08:00 +HOVST, Hovd Summer Time, +08:00 +HOVT, Hovd Time, +07:00 +HST, Hawaii Standard Time, -10:00 +ICT, Indochina Time, +07:00 +IDT, Israel Daylight Time, +03:00 +IOT, Indian Chagos Time, +06:00 +IRDT, Iran Daylight Time, +4:30 +IRKST, Irkutsk Summer Time, +09:00 +IRKT, Irkutsk Time, +08:00 +IRST, Iran Standard Time, +3:30 +IST, India Standard Time, +5:30 +IST, Irish Standard Time, +01:00 +IST, Israel Standard Time, +02:00 +JST, Japan Standard Time, +09:00 +KGT, Kyrgyzstan Time, +06:00 +KOST, Kosrae Time, +11:00 +KRAST, Krasnoyarsk Summer Time, +08:00 +KRAT, Krasnoyarsk Time, +07:00 +KST, Korea Standard Time, +09:00 +KUYT, Kuybyshev Time, +04:00 +LHDT, Lord Howe Daylight Time, +11:00 +LHST, Lord Howe Standard Time, +10:30 +LINT, Line Islands Time, +14:00 +MAGST, Magadan Summer Time, +12:00 +MAGT, Magadan Time, +11:00 +MART, Marquesas Time, -9:30 +MAWT, Mawson Time, +05:00 +MDT, Mountain Daylight Time, -06:00 +MHT, Marshall Islands Time, +12:00 +MMT, Myanmar Time, +6:30 +MSD, Moscow Daylight Time, +04:00 +MSK, Moscow Standard Time, +03:00 +MST, Mountain Standard Time, -07:00 +MT, Mountain Time, -7:00 / -6:00 +MUT, Mauritius Time, +04:00 +MVT, Maldives Time, +05:00 +MYT, Malaysia Time, +08:00 +NCT, New Caledonia Time, +11:00 +NDT, Newfoundland Daylight Time, -2:30 +NFT, Norfolk Time, +11:00 +NOVST, Novosibirsk Summer Time, +07:00 +NOVT, Novosibirsk Time, +06:00 +NPT, Nepal Time , +5:45 +NRT, Nauru Time, +12:00 +NST, Newfoundland Standard Time, -3:30 +NUT, Niue Time, -11:00 +NZDT, New Zealand Daylight Time, +13:00 +NZST, New Zealand Standard Time, +12:00 +OMSST, Omsk Summer Time, +07:00 +OMST, Omsk Standard Time, +06:00 +ORAT, Oral Time, +05:00 +PDT, Pacific Daylight Time, -07:00 +PET, Peru Time, -05:00 +PETST, Kamchatka Summer Time, +12:00 +PETT, Kamchatka Time, +12:00 +PGT, Papua New Guinea Time, +10:00 +PHOT, Phoenix Island Time, +13:00 +PHT, Philippine Time, +08:00 +PKT, Pakistan Standard Time, +05:00 +PMDT, Pierre & Miquelon Daylight Time, -02:00 +PMST, Pierre & Miquelon Standard Time, -03:00 +PONT, Pohnpei Standard Time, +11:00 +PST, Pacific Standard Time, -08:00 +PST, Pitcairn Standard Time, -08:00 +PWT, Palau Time, +09:00 +PYST, Paraguay Summer Time, -03:00 +PYT, Paraguay Time, -04:00 +PYT, Pyongyang Time, +8:30 +QYZT, Qyzylorda Time, +06:00 +RET, Reunion Time, +04:00 +ROTT, Rothera Time, -03:00 +SAKT, Sakhalin Time, +11:00 +SAMT, Samara Time, +04:00 +SAST, South Africa Standard Time, +02:00 +SBT, Solomon Islands Time, +11:00 +SCT, Seychelles Time, +04:00 +SGT, Singapore Time, +08:00 +SRET, Srednekolymsk Time, +11:00 +SRT, Suriname Time, -03:00 +SST, Samoa Standard Time, -11:00 +SYOT, Syowa Time, +03:00 +TAHT, Tahiti Time, -10:00 +TFT, French Southern and Antarctic Time, +05:00 +TJT, Tajikistan Time , +05:00 +TKT, Tokelau Time, +13:00 +TLT, East Timor Time, +09:00 +TMT, Turkmenistan Time, +05:00 +TOST, Tonga Summer Time, +14:00 +TOT, Tonga Time, +13:00 +TRT, Turkey Time, +03:00 +TVT, Tuvalu Time, +12:00 +ULAST, Ulaanbaatar Summer Time, +09:00 +ULAT, Ulaanbaatar Time, +08:00 +UTC, Coordinated Universal Time, +00:00 +UYST, Uruguay Summer Time, -02:00 +UYT, Uruguay Time, -03:00 +UZT, Uzbekistan Time, +05:00 +VET, Venezuelan Standard Time, -04:00 +VLAST, Vladivostok Summer Time, +11:00 +VLAT, Vladivostok Time, +10:00 +VOST, Vostok Time, +06:00 +VUT, Vanuatu Time, +11:00 +WAKT, Wake Time, +12:00 +WARST, Western Argentine Summer Time, -03:00 +WAST, West Africa Summer Time, +02:00 +WAT, West Africa Time, +01:00 +WEST, Western European Summer Time, +01:00 +WET, Western European Time, +00:00 +WFT, Wallis and Futuna Time, +12:00 +WGST, Western Greenland Summer Time, -02:00 +WGT, West Greenland Time, -03:00 +WIB, Western Indonesian Time, +07:00 +WIT, Eastern Indonesian Time, +09:00 +WITA, Central Indonesian Time, +08:00 +WST, West Samoa Time, +13:00 +WST, Western Sahara Summer Time, +01:00 +WT, Western Sahara Standard Time, +00:00 +YAKST, Yakutsk Summer Time, +10:00 +YAKT, Yakutsk Time, +09:00 +YAPT, Yap Time, +10:00 +YEKST, Yekaterinburg Summer Time, +06:00 +YEKT, Yekaterinburg Time, +05:00 diff --git a/src/config/loader.rs b/src/config/loader.rs index 774b49d..d8f81e0 100644 --- a/src/config/loader.rs +++ b/src/config/loader.rs @@ -551,7 +551,8 @@ impl Default for BufferConfig { } } -/// Where the engine's file set puts the enrichment tables. +/// Where the bundled filebeat pipeline and the engine's file set put the +/// enrichment tables. pub const DEFAULT_TRANSFORMS_DATA_DIR: &str = "/etc/dfe-transform-vector/data"; /// Transform file loading configuration. diff --git a/src/config/validate.rs b/src/config/validate.rs index f41f7bd..8093471 100644 --- a/src/config/validate.rs +++ b/src/config/validate.rs @@ -664,7 +664,8 @@ mod tests { ))); } - /// A transform file declaring a `file` table at `path`. + /// A transform file declaring a `file` table at `path`, the shape the + /// bundled filebeat pipeline ships. fn file_table(path: &Path) -> String { format!( "enrichment_tables:\n timezones:\n type: file\n file:\n path: {}\n \ @@ -677,7 +678,7 @@ mod tests { fn a_missing_table_file_is_named_with_its_path() { let tmp = tempfile::tempdir().expect("tempdir"); let missing = tmp.path().join("data/timezones.csv"); - write_yaml_file(tmp.path(), "50_000_tables.yaml", &file_table(&missing)); + write_yaml_file(tmp.path(), "50_000_filebeat.yaml", &file_table(&missing)); let unreadable = unreadable_enrichment_tables(tmp.path()); assert_eq!(unreadable.len(), 1, "{unreadable:?}"); @@ -697,7 +698,7 @@ mod tests { let tmp = tempfile::tempdir().expect("tempdir"); let table = tmp.path().join("timezones.csv"); std::fs::write(&table, "abbreviation\nUTC\n").expect("write table"); - write_yaml_file(tmp.path(), "50_000_tables.yaml", &file_table(&table)); + write_yaml_file(tmp.path(), "50_000_filebeat.yaml", &file_table(&table)); assert_eq!(unreadable_enrichment_tables(tmp.path()), Vec::new()); } @@ -707,7 +708,7 @@ mod tests { let tmp = tempfile::tempdir().expect("tempdir"); let dir = tmp.path().join("timezones.csv"); std::fs::create_dir(&dir).expect("create dir"); - write_yaml_file(tmp.path(), "50_000_tables.yaml", &file_table(&dir)); + write_yaml_file(tmp.path(), "50_000_filebeat.yaml", &file_table(&dir)); let unreadable = unreadable_enrichment_tables(tmp.path()); assert_eq!(unreadable.len(), 1, "{unreadable:?}"); @@ -764,7 +765,7 @@ mod tests { let config_dir = tmp.path().join("config"); std::fs::create_dir(&config_dir).expect("config dir"); let missing = tmp.path().join("timezones.csv"); - write_yaml_file(&config_dir, "50_000_tables.yaml", &file_table(&missing)); + write_yaml_file(&config_dir, "50_000_filebeat.yaml", &file_table(&missing)); let vector_config = VectorConfig { binary: always_valid_binary(tmp.path()) diff --git a/tests/TESTING.md b/tests/TESTING.md index 76eba41..41119f6 100644 --- a/tests/TESTING.md +++ b/tests/TESTING.md @@ -17,7 +17,7 @@ There is no mode switch. `KafkaFixture::acquire` resolves a broker on its own: `KafkaFixture::hermetic` skips step 1 outright and takes the shared broker. Anything using the real DFE topic names takes that path, because -`e2e_syslog_land` on a shared live broker is somebody else's data. +`filebeat_land` on a shared live broker is somebody else's data. `KafkaFixture::scram` starts a SCRAM-only broker the test holds alone. scalo names every container under the suite prefix, labels it with the repo, @@ -51,8 +51,12 @@ cargo nextest run --run-ignored all Opt-in (`#[ignore]`) cases are the full tier: `vector_validate` runs the real validator over each buffer, SASL and TLS variant of an assembled config, and two `e2e::kafka` pipelines drive a bare Vector process against a broker. -## The app over Kafka +## The WS21 acceptance case -`e2e::app_kafka` runs in core, on the shared broker, which no live cluster's data shares. -It seeds synthetic syslog-style lines onto `e2e_syslog_land`, runs the shipped -binary with an authored transform file, and grades `e2e_syslog_load` against the fields each case names. +`e2e::filebeat_kafka` runs in core, on the shared broker, which no live cluster's data shares. +It seeds the synthetic filebeat cases onto `filebeat_land`, runs the shipped +binary over them, and grades `filebeat_load` against the fields each case names. + +The cases (`tests/fixtures/filebeat/cases.yaml`) and the bundled pipeline belong +to dfe-transform-vrl, so the case needs that repo checked out beside this one, +or `DFE_TRANSFORM_VRL_DIR` pointing at it. Without one it prints a skip. diff --git a/tests/common/filebeat.rs b/tests/common/filebeat.rs new file mode 100644 index 0000000..157683b --- /dev/null +++ b/tests/common/filebeat.rs @@ -0,0 +1,203 @@ +// A missing or malformed case file is unrecoverable and has no caller to +// report it to, so the loader panics rather than threading a Result through +// every test. +#![allow(dead_code, clippy::panic)] +// Project: dfe-transform-vector +// File: tests/common/filebeat.rs +// Purpose: The filebeat test cases and their comparison, read from dfe-transform-vrl +// Language: Rust +// +// License: BUSL-1.1 +// Copyright: (c) 2026 HYPERI PTY LIMITED + +//! The synthetic filebeat cases in dfe-transform-vrl's +//! `tests/fixtures/filebeat/cases.yaml`, and what counts as agreeing with one. +//! +//! Both transform apps run the SAME bundled pipeline over the SAME cases, and a +//! parity test only means something if both grade on the same curve. +//! dfe-transform-vrl owns the cases and the bundled VRL, and this is the reader +//! for them. It mirrors `dfe-transform-vrl/tests/common/filebeat.rs`. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +use serde::Deserialize; + +/// The dfe-transform-vrl checkout that owns the cases and the pipeline. +/// +/// `DFE_TRANSFORM_VRL_DIR` first, then a `dfe-transform-vrl` sitting beside +/// any ancestor of this checkout -- which finds it from a plain clone and from +/// a `.worktrees/` worktree alike, without naming anybody's workspace +/// layout. `None` means the acceptance case cannot run here, and the test says +/// so rather than asserting on nothing. +pub fn vrl_repo() -> Option<&'static Path> { + static DIR: OnceLock> = OnceLock::new(); + DIR.get_or_init(|| { + let has_pipeline = |dir: &Path| dir.join("pipelines/filebeat/filebeat.vrl").is_file(); + if let Some(explicit) = std::env::var_os("DFE_TRANSFORM_VRL_DIR") { + let dir = PathBuf::from(explicit); + return has_pipeline(&dir).then_some(dir); + } + Path::new(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .map(|a| a.join("dfe-transform-vrl")) + .find(|dir| has_pipeline(dir)) + }) + .as_deref() +} + +/// The bundled filebeat pipeline, or `None` without a dfe-transform-vrl checkout. +pub fn pipeline_vrl() -> Option { + vrl_repo().map(|dir| dir.join("pipelines/filebeat/filebeat.vrl")) +} + +/// The timezones lookup the ios and meraki branches need. +pub fn timezones_csv() -> Option { + vrl_repo().map(|dir| dir.join("pipelines/filebeat/timezones.csv")) +} + +// ========================================================================= +// The cases +// ========================================================================= + +/// The filebeat module a case's line belongs to. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Module { + Meraki, + Ios, + Umbrella, +} + +/// One input line and what the pipeline must make of it. +#[derive(Debug, Clone, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Case { + /// Unique across the file. The e2e suite carries it through the broker to + /// pair each output with its case. + pub name: String, + pub module: Module, + /// The `_conf` object the DFE 2.1 deployment injected (`tz_offset`). + #[serde(default)] + pub conf: Option, + /// The raw log line, as the `message` field. + pub line: String, + /// A subset of the transformed event: every leaf named here must be + /// present and equal in the output. Fields the case does not name are not + /// checked. + pub expect: serde_json::Value, +} + +/// Every case in the file, read once. Callers check [`vrl_repo`] first. +pub fn cases() -> &'static [Case] { + static CASES: OnceLock> = OnceLock::new(); + CASES.get_or_init(|| { + let path = vrl_repo() + .expect("dfe-transform-vrl checkout must be present") + .join("tests/fixtures/filebeat/cases.yaml"); + let text = std::fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("read {}: {e}", path.display())); + let cases: Vec = + scalo::yaml::from_str(&text, "cases.yaml", scalo::yaml::Profile::Data) + .unwrap_or_else(|e| panic!("parse cases.yaml: {e}")); + let mut seen = HashSet::new(); + for case in &cases { + assert!( + seen.insert(case.name.as_str()), + "duplicate case name {}", + case.name + ); + } + cases + }) +} + +/// A DFE 2.1-shaped input event: `{message, tags, timestamp}` plus the +/// `_conf` object the 2.1 deployment injected (`tz_offset`, `tz_map`). +pub fn input_event(message: &str, conf: Option<&serde_json::Value>) -> serde_json::Value { + let mut obj = serde_json::json!({ + "message": message, + "tags": [], + "timestamp": "2026-01-01T00:00:00Z", + }); + if let Some(conf) = conf { + obj["_conf"] = conf.clone(); + } + obj +} + +/// The input event for one case. +pub fn case_event(case: &Case) -> serde_json::Value { + input_event(&case.line, case.conf.as_ref()) +} + +// ========================================================================= +// What counts as agreeing with a case +// ========================================================================= + +/// Compare one transformed event against a case's `expect`: every leaf in +/// `expected` must be present and equal in `ours`. Extra fields on our side +/// (DFE 2.1 event carriage like `_conf`, `timestamp`, and the fields Vector's +/// kafka source adds) are fine. The root `tags` array compares as a set, +/// because the 2.1 template seeds tags in a fixed but arbitrary order. +pub fn diff_expected( + path: &str, + expected: &serde_json::Value, + ours: &serde_json::Value, + problems: &mut Vec, +) { + match expected { + serde_json::Value::Object(map) => { + for (k, v) in map { + let child = if path.is_empty() { + k.clone() + } else { + format!("{path}.{k}") + }; + match ours.get(k) { + Some(o) => diff_expected(&child, v, o, problems), + None => problems.push(format!("{child}: missing (expected {v})")), + } + } + } + serde_json::Value::Array(items) if path == "tags" => { + let have = ours.as_array().map(Vec::as_slice).unwrap_or_default(); + for item in items { + if !have.contains(item) { + problems.push(format!("tags: missing {item}")); + } + } + } + serde_json::Value::Array(items) => match ours.as_array() { + Some(have) if have.len() == items.len() => { + for (i, (e, o)) in items.iter().zip(have).enumerate() { + diff_expected(&format!("{path}[{i}]"), e, o, problems); + } + } + _ => problems.push(format!("{path}: expected array {expected}, got {ours}")), + }, + leaf => { + if ours != leaf { + problems.push(format!("{path}: expected {leaf}, got {ours}")); + } + } + } +} + +/// Everything wrong with `ours` as the transform of `case`. Empty means it +/// agrees. +pub fn problems_for(case: &Case, ours: &serde_json::Value) -> Vec { + let mut problems = Vec::new(); + diff_expected("", &case.expect, ours, &mut problems); + problems +} + +/// Whether the pipeline routed this event to the shape-unmatched catch-all. +/// Every case belongs to a known module, so this must never fire on one. +pub fn is_unmatched(event: &serde_json::Value) -> bool { + event + .get("tags") + .and_then(|t| t.as_array()) + .is_some_and(|t| t.contains(&serde_json::json!("filebeat_unmatched"))) +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 9986737..511e997 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -117,7 +117,7 @@ impl KafkaFixture { /// Acquire the run's shared broker container, never the live cluster. /// /// `acquire` prefers the live cluster, which is wrong for a test that uses - /// the real DFE topic names: `e2e_syslog_land` on a shared live broker is + /// the real DFE topic names: `filebeat_land` on a shared live broker is /// somebody else's data. Returns `None` when no container can be started. pub async fn hermetic(_test: &str) -> Option { match shared_broker().await { diff --git a/tests/e2e.rs b/tests/e2e.rs index 3a1cbb8..19eebdf 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -11,5 +11,9 @@ mod common; +/// The filebeat cases and their comparison, owned by dfe-transform-vrl. +#[path = "common/filebeat.rs"] +mod filebeat_cases; + #[path = "e2e/mod.rs"] mod e2e; diff --git a/tests/e2e/app_kafka.rs b/tests/e2e/filebeat_kafka.rs similarity index 56% rename from tests/e2e/app_kafka.rs rename to tests/e2e/filebeat_kafka.rs index 5bf67e4..fed5918 100644 --- a/tests/e2e/app_kafka.rs +++ b/tests/e2e/filebeat_kafka.rs @@ -1,21 +1,40 @@ // Project: dfe-transform-vector -// File: tests/e2e/app_kafka.rs -// Purpose: The app binary over a real broker with an authored transform file +// File: tests/e2e/filebeat_kafka.rs +// Purpose: WS21 parity: the filebeat cases through the app, Vector and a broker // Language: Rust // // License: BUSL-1.1 // Copyright: (c) 2026 HYPERI PTY LIMITED -//! The shipped binary reading `e2e_syslog_land`, transforming with Vector and -//! writing `e2e_syslog_load`. +//! The transform hop of the WS21 chain, with Vector doing the transforming. //! -//! The instance is bound to its source by NAME: `dfe_source` is the only -//! topic setting in the config, and both topics and the consumer group derive -//! from it. The app also runs `vector validate` before it starts Vector, so a -//! transform file Vector cannot compile fails here. +//! ```text +//! filebeat cases -> filebeat_land -> dfe-transform-vector -> filebeat_load +//! ``` //! -//! The receiver hop that picks the source and the loader hop that lands rows -//! in `ClickHouse` need a cluster and are not covered. +//! This is the parity half of the acceptance case: the same cases, the same +//! bundled filebeat VRL, and the same assertions as +//! `dfe-transform-vrl/tests/e2e/filebeat_kafka.rs`. Only the transform app +//! differs, which is what makes WS21 a test of the transform LAYER rather +//! than of one app. +//! +//! What it exercises that a config test cannot: +//! +//! - The instance is bound to its source by NAME. `dfe_source: filebeat` is +//! the only topic setting in the config; `filebeat_land`, `filebeat_load` +//! and the consumer group are all derived from it. +//! - The VRL arrives as an authored transform file, wrapped in a `remap`, +//! with its lookup table declared alongside it -- so a top-level +//! `enrichment_tables:` reaching Vector through the assembler is proved, +//! not assumed. +//! - The app runs `vector validate` before starting, so a pipeline Vector's +//! VRL cannot compile fails here rather than silently passing nothing on. +//! +//! The cases and the pipeline belong to dfe-transform-vrl. Without that +//! checkout there is nothing to run against and the test says so. +//! +//! NOT covered here, because it needs a cluster: the receiver hop that picks +//! the source, and the loader hop that lands rows in `ClickHouse`. use std::time::{Duration, Instant}; @@ -23,78 +42,41 @@ use scalo::transport::kafka::{KafkaAdmin, KafkaConfig, KafkaTransport}; use scalo::transport::{TransportBase, TransportReceiver, TransportSender}; use crate::common::{self, KafkaFixture}; +use crate::filebeat_cases as fb; -const SOURCE: &str = "e2e_syslog"; -const LAND_TOPIC: &str = "e2e_syslog_land"; -const LOAD_TOPIC: &str = "e2e_syslog_load"; +/// The source this instance is bound to, and the topics that follow from it. +const SOURCE: &str = "filebeat"; +const LAND_TOPIC: &str = "filebeat_land"; +const LOAD_TOPIC: &str = "filebeat_load"; -/// A synthetic line and the fields the transform must set, or `None` where it -/// must tag the event `unparsed` and pass it on. -struct Case { - name: &'static str, - line: &'static str, - expect: Option<(&'static str, &'static str)>, -} +/// A line no filebeat module claims, carried through to prove the +/// shape-unmatched catch-all still passes an event on rather than dropping it. +const UNMATCHED_PROBE: &str = "ws21 probe: a line no filebeat module claims"; -const CASES: &[Case] = &[ - Case { - name: "denied", - line: "denied src=203.0.113.7", - expect: Some(("203.0.113.7", "denied")), - }, - Case { - name: "allowed", - line: "allowed src=198.51.100.23", - expect: Some(("198.51.100.23", "allowed")), - }, - Case { - name: "reset", - line: "conn_reset src=192.0.2.1", - expect: Some(("192.0.2.1", "conn_reset")), - }, - Case { - name: "no_source", - line: "a line with no key value pairs", - expect: None, - }, - Case { - name: "bad_ip", - line: "denied src=not-an-address", - expect: None, - }, -]; - -/// How long the app gets to assemble the config, validate and start Vector. -const READY_TIMEOUT: Duration = Duration::from_secs(120); +/// How long the app gets to assemble the config, run `vector validate` over +/// the 5.5K-line pipeline, and start Vector. +const READY_TIMEOUT: Duration = Duration::from_secs(240); /// How long the cases get to arrive on the sink topic once the app is ready. -const DRAIN_TIMEOUT: Duration = Duration::from_secs(120); - -const TRANSFORM: &str = r#"transforms: - parse_line: - type: remap - inputs: - - dfe_source - source: | - parsed, err = parse_regex(.message, r'^(?P[a-z_]+) src=(?P[0-9.]+)$') - if err == null { - .source.ip = parsed.ip - .event.action = parsed.action - } else { - .tags = ["unparsed"] - } -"#; +const DRAIN_TIMEOUT: Duration = Duration::from_secs(180); +/// Every case as Vector will read it off `filebeat_land`, plus the +/// shape-unmatched probe. +/// +/// Each case event carries its name in a root `_case` field. The pipeline keeps +/// root fields on every branch, so the name comes out the far side and pairs +/// an output with its case whatever order the broker returns them in. fn workload() -> Vec { - CASES + let mut inputs: Vec = fb::cases() .iter() - .map(|c| serde_json::json!({ "message": c.line, "_case": c.name })) - .collect() -} - -fn write_transform(dir: &std::path::Path) { - std::fs::create_dir_all(dir).expect("create the transforms dir"); - std::fs::write(dir.join("01_parse_line.yaml"), TRANSFORM).expect("write the transform file"); + .map(|case| { + let mut event = fb::case_event(case); + event["_case"] = serde_json::json!(case.name); + event + }) + .collect(); + inputs.push(fb::input_event(UNMATCHED_PROBE, None)); + inputs } fn client_config(base: &KafkaConfig, topics: &[&str], group: &str) -> KafkaConfig { @@ -117,6 +99,50 @@ fn free_port() -> u16 { port } +/// The bundled filebeat VRL as a Vector `remap` transform, with the lookup +/// table it needs declared in the same file. +/// +/// The top-level `enrichment_tables` key is the point: the assembler writes +/// this file into Vector's config dir verbatim, so Vector merges the +/// declaration and the VRL's `get_enrichment_table_record` calls resolve. +fn write_transform(dir: &std::path::Path) { + let vrl = std::fs::read_to_string(fb::pipeline_vrl().expect("the bundled pipeline")) + .expect("read the bundled filebeat.vrl"); + let timezones = fb::timezones_csv().expect("the bundled timezones table"); + + let doc = serde_json::json!({ + "enrichment_tables": { + "timezones": { + "type": "file", + "file": { + "path": timezones.to_string_lossy(), + "encoding": { "type": "csv" }, + }, + "schema": { "abbreviation": "string" }, + }, + }, + "transforms": { + "filebeat": { + "type": "remap", + "inputs": ["dfe_source"], + "source": vrl, + }, + }, + }); + + std::fs::create_dir_all(dir).expect("create the transforms dir"); + std::fs::write( + dir.join("01_filebeat.yaml"), + scalo::yaml::to_string(&doc).expect("transform serialises"), + ) + .expect("write the transform file"); +} + +/// The config file the app runs from. +/// +/// `dfe_source` is the ONLY topic setting: leaving `source.topics`, +/// `sink.topic` and `source.group_id` at their defaults is what lets the app +/// derive them, which is the binding under test. fn write_config( dir: &std::path::Path, brokers: &[String], @@ -139,7 +165,13 @@ fn write_config( "encoding": "json", "compression": "none", }, - "transforms": { "dir": transforms_dir.to_string_lossy() }, + "transforms": { + "dir": transforms_dir.to_string_lossy(), + "data_dir": fb::timezones_csv() + .and_then(|table| table.parent().map(std::path::Path::to_path_buf)) + .expect("the bundled timezones table") + .to_string_lossy(), + }, "vector": { "binary": vector_binary.to_string_lossy(), "data_dir": data_dir.to_string_lossy(), @@ -248,14 +280,49 @@ async fn drain( out } +/// Grade every case against the output event that carries its name. +/// Returns one line per case that has no output or whose output disagrees. +fn grade_cases(outputs: &[serde_json::Value]) -> Vec { + let mut failures = Vec::new(); + for case in fb::cases() { + let found = outputs + .iter() + .find(|e| e.get("_case").and_then(serde_json::Value::as_str) == Some(&case.name)); + match found { + None => failures.push(format!("{}: no output event carried its name", case.name)), + Some(event) if fb::is_unmatched(event) => { + failures.push(format!("{}: routed to filebeat_unmatched", case.name)); + } + Some(event) => { + let problems = fb::problems_for(case, event); + if !problems.is_empty() { + failures.push(format!("{}: {}", case.name, problems.join("; "))); + } + } + } + } + failures +} + #[tokio::test] -async fn syslog_lines_round_trip_through_vector() { +async fn filebeat_cases_round_trip_through_vector() { + if fb::vrl_repo().is_none() { + common::require_service_in_ci( + "dfe-transform-vrl checkout", + "the cases and the bundled pipeline grade this test, and CI clones the repo beside this one", + ); + eprintln!( + "SKIP: the cases and the bundled pipeline live in dfe-transform-vrl. \ + Point DFE_TRANSFORM_VRL_DIR at a checkout to run this." + ); + return; + } let Some(vector_binary) = common::vector_binary_path() else { common::require_service_in_ci("Vector binary", "scripts/fetch-vector.sh found nothing"); eprintln!("SKIP: no Vector binary (run scripts/fetch-vector.sh)"); return; }; - let Some(fixture) = KafkaFixture::hermetic("syslog-round-trip").await else { + let Some(fixture) = KafkaFixture::hermetic("filebeat-cases-round-trip").await else { common::require_service_in_ci("Kafka", "no Docker to start a broker container"); eprintln!("SKIP: no Docker, so this test cannot own a broker."); return; @@ -264,18 +331,18 @@ async fn syslog_lines_round_trip_through_vector() { let inputs = workload(); eprintln!( - "replaying {} events through {LAND_TOPIC} -> {LOAD_TOPIC}", + "WS21: replaying {} events through {LAND_TOPIC} -> {LOAD_TOPIC}", inputs.len() ); let admin = - KafkaAdmin::new(&client_config(&base, &[LAND_TOPIC], "admin")).expect("admin client"); + KafkaAdmin::new(&client_config(&base, &[LAND_TOPIC], "ws21-admin")).expect("admin client"); admin .create_topics(&[(LAND_TOPIC, 1, 1), (LOAD_TOPIC, 1, 1)]) .await .expect("create the source-bound topics"); - let producer = KafkaTransport::new(&client_config(&base, &[LAND_TOPIC], "seed")) + let producer = KafkaTransport::new(&client_config(&base, &[LAND_TOPIC], "ws21-seed")) .await .expect("seed producer"); // `send`'s first argument is the DESTINATION, which for Kafka is the topic @@ -298,7 +365,7 @@ async fn syslog_lines_round_trip_through_vector() { let seeded = drain( &base, LAND_TOPIC, - "seed-check", + "ws21-seed-check", inputs.len(), Duration::from_secs(60), ) @@ -362,7 +429,14 @@ async fn syslog_lines_round_trip_through_vector() { "dfe-transform-vector did not report ready on {probes} within {READY_TIMEOUT:?}" ); - let outputs = drain(&base, LOAD_TOPIC, "verify", inputs.len(), DRAIN_TIMEOUT).await; + let outputs = drain( + &base, + LOAD_TOPIC, + "ws21-verify", + inputs.len(), + DRAIN_TIMEOUT, + ) + .await; stop(&mut app).await; // Assertion 1 -- everything that went in came out. @@ -374,44 +448,45 @@ async fn syslog_lines_round_trip_through_vector() { outputs.len() ); - // Assertion 2 -- each line carries the fields its case names. - let mut failures = Vec::new(); - for case in CASES { - let Some(event) = outputs - .iter() - .find(|e| e.get("_case").and_then(serde_json::Value::as_str) == Some(case.name)) - else { - failures.push(format!("{}: no output event carried its name", case.name)); - continue; - }; - let ip = event - .pointer("/source/ip") - .and_then(serde_json::Value::as_str); - let action = event - .pointer("/event/action") - .and_then(serde_json::Value::as_str); - let unparsed = event - .pointer("/tags") - .and_then(serde_json::Value::as_array) - .is_some_and(|t| t.iter().any(|v| v == "unparsed")); - match case.expect { - Some((want_ip, want_action)) => { - if ip != Some(want_ip) || action != Some(want_action) || unparsed { - failures.push(format!("{}: got {event}", case.name)); - } - } - None => { - if !unparsed || ip.is_some() || action.is_some() { - failures.push(format!("{}: expected unparsed, got {event}", case.name)); - } - } - } - } + // Assertion 2 -- the shape-unmatched catch-all passes an event through + // tagged, rather than dropping it. + let probe_at = outputs + .iter() + .position(|e| e.get("message").and_then(serde_json::Value::as_str) == Some(UNMATCHED_PROBE)) + .expect("the unmatched probe must arrive on the sink topic"); + assert!( + fb::is_unmatched(&outputs[probe_at]), + "an unclaimed line must carry filebeat_unmatched: {}", + outputs[probe_at] + ); + + // Assertion 3 -- every case agrees with the fields it names. + let failures = grade_cases(&outputs); assert!( failures.is_empty(), "{} of {} cases did not match:\n{}", failures.len(), - CASES.len(), + fb::cases().len(), failures.join("\n") ); + + // Assertion 4 -- all three module branches actually ran. A pipeline that + // detected nothing would still satisfy the count above by passing + // everything through unmatched. + let branch_ran = |module: &str, probe: &dyn Fn(&serde_json::Value) -> bool| { + assert!( + outputs.iter().any(probe), + "no output event came from the {module} branch" + ); + }; + let product_is = |event: &serde_json::Value, want: &str| { + event + .get("observer") + .and_then(|o| o.get("product")) + .and_then(serde_json::Value::as_str) + == Some(want) + }; + branch_ran("umbrella", &|e| product_is(e, "Umbrella")); + branch_ran("ios", &|e| product_is(e, "IOS")); + branch_ran("meraki", &|e| e.get("cisco_meraki").is_some()); } diff --git a/tests/e2e/mod.rs b/tests/e2e/mod.rs index d58be23..bd701b2 100644 --- a/tests/e2e/mod.rs +++ b/tests/e2e/mod.rs @@ -6,7 +6,7 @@ // License: BUSL-1.1 // Copyright: (c) 2026 HYPERI PTY LIMITED -mod app_kafka; +mod filebeat_kafka; mod kafka; // The Vector metrics merge is asserted inside `kafka` against a real Vector, // and against a fake exporter in `tests/integration/metrics.rs`. diff --git a/tests/integration/vector_validate.rs b/tests/integration/vector_validate.rs index c3d5b46..888c502 100644 --- a/tests/integration/vector_validate.rs +++ b/tests/integration/vector_validate.rs @@ -416,11 +416,11 @@ async fn a_table_file_vector_validate_passes_but_cannot_load_fails_validation() fs::create_dir_all(&transforms).expect("transforms dir"); let table = work_dir.path().join("data/timezones.csv"); fs::write( - transforms.join("01_tables.yaml"), + transforms.join("01_filebeat.yaml"), format!( "enrichment_tables:\n timezones:\n type: file\n file:\n path: {}\n \ encoding:\n type: csv\n schema:\n abbreviation: string\n\ - transforms:\n tables:\n type: remap\n inputs: [dfe_source]\n \ + transforms:\n filebeat:\n type: remap\n inputs: [dfe_source]\n \ source: |\n .seen = true\n", table.display() ),