diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a24aca..eb3decd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,8 +67,6 @@ jobs: uses: hyperi-io/hyperi-ci/.github/workflows/rust-ci.yml@main with: publish-target: both - # The filebeat test reads the pipeline and cases from this repo, beside the checkout. - sibling-checkouts: hyperi-io/dfe-transform-vrl tag: ${{ inputs.tag || '' }} from-head: ${{ inputs.from-head || '' }} bump: ${{ inputs.bump || 'auto' }} diff --git a/.gitleaks.toml b/.gitleaks.toml index d9b4a40..634b37f 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -9,6 +9,6 @@ description = "Project-specific false positive allowlist" paths = [ '''(^|/)ci/''', '''(^|/)ai/''', - # The Meraki event subtype '8021x_eap_failure' matches a key rule. + # Commits before the history rewrite still hold the deleted file, and gitleaks scans them. '''pipelines/filebeat/filebeat\.yaml''', ] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e6321a1..bb9ee88 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -151,7 +151,6 @@ A Rust supervisor that runs Vector as a child process so a Vector pipeline behav | `src/health.rs`, `src/metrics.rs`, `src/metrics/scrape.rs` | Readiness publishing, and the scrape that merges `vector_*` into scalo's registry | | `src/deployment.rs`, `src/vector-layer.dockerfile` | `contract()`, the source the Dockerfile and the released chart are generated from, and the two-binary image override spliced onto scalo's generated Dockerfile | | `templates/bus.yaml`, `templates/direct.yaml` | Whole runnable Vector topologies, one per transport, commented per field | -| `pipelines/filebeat/` | The bundled filebeat pipeline. The image does not carry it | | `docs/` | `architecture.md` for why the service is built this way and the invariants, `internals.md` for field-by-field depth, `migration.md`, `librdkafka.md`, the generated `config-schema.*` and `capability-catalog.*` | | `tests/` | `integration`, `e2e`, `smoke`, and `TESTING.md` for how the broker and Vector binary are resolved | | `scripts/fetch-vector.sh`, `scripts/pgo-workload.sh` | Downloads the pinned Vector for tests, and drives the PGO/BOLT workload | @@ -171,7 +170,7 @@ What green does NOT mean: - The default run leaves out the `#[ignore]` cases in `tests/integration/vector_validate.rs` and `tests/e2e/kafka.rs`. The `vector_validate` ones run Vector's own validator over each buffer, SASL and TLS variant of an assembled config, so until you pass `--run-ignored all` those variants are unchecked. - Locally, a missing broker or Vector binary SKIPS rather than fails. In CI that absence is an assertion failure for the cases that run by default, so CI is stricter than a green laptop run. -- `e2e::filebeat_kafka` runs by default and needs a dfe-transform-vrl checkout, for its cases and the bundled pipeline: beside this repo, or named by `DFE_TRANSFORM_VRL_DIR`. Without one it skips locally and fails under CI, which checks the repo out beside this one. +- `e2e::app_kafka` runs by default: the shipped binary over a shared broker with an authored transform file. Without Docker it skips locally and fails under CI. - A docs-only push runs no CI at all. `paths-ignore` for `docs/**` and `**.md` is on the `push` trigger but not on `pull_request`, so the PR is the only place a docs change is checked. A silent push is not a pass. ### What tends to bite @@ -209,4 +208,3 @@ Outbound -- what depends on this repo: Two more relationships that are real but are NOT declared edges in the suite graph, so no gate enforces them: - dfe-infra `apps.yaml` is the SSoT for how this app is deployed -- multiplicity, scaling, transports, endpoints and which compiler derives its routing. dfe-engine reads that manifest, so adding or changing the app is a manifest edit, never an engine release. -- dfe-transform-vrl holds the filebeat cases (`tests/fixtures/filebeat/cases.yaml`) and the bundled pipeline that `e2e::filebeat_kafka` grades against. The dependency is a test-time checkout only, which CI makes and a laptop may not. diff --git a/pipelines/filebeat/README.md b/pipelines/filebeat/README.md deleted file mode 100644 index db221c2..0000000 --- a/pipelines/filebeat/README.md +++ /dev/null @@ -1,36 +0,0 @@ - - -# Bundled pipeline: filebeat-compat (INTERIM) - -A pre-canned transform file that keeps DFE 2.1 filebeat feeds working on dfe-transform-vector: the same VRL dfe-transform-vrl runs, wrapped in a Vector `remap` and carrying the enrichment table it needs. Point `transforms.dir` at this directory, mount the table, and the pipeline runs -- there is nothing to author. The modules, the routing rules, the departures from DFE 2.1 and the known limitations are documented once, in dfe-transform-vrl `pipelines/filebeat/README.md`, and apply here unchanged. - -> **INTERIM.** Elastic compatibility is being replaced by -> `dfe-transform-elastic` (Rust-native). New integrations -> should NOT build on this pipeline. - -## Using it - -```yaml -# config.yaml -dfe_source: "filebeat" # derives filebeat_land -> filebeat_load -transforms: - dir: "/etc/dfe-transform-vector/transforms" -``` - -Mount `filebeat.yaml` into that directory and `timezones.csv` at the path the pipeline's `enrichment_tables` entry names (`/etc/dfe-transform-vector/data/timezones.csv`), or edit that one path to wherever the table lives inside `transforms.data_dir`. Both files are here. - -The wrapper writes each transform file into Vector's config dir with its wired `inputs` added, and a transform file may declare a top-level `enrichment_tables` key, so Vector merges the declaration and the VRL's timezone lookups resolve. `vector validate` runs before Vector starts, so a pipeline that cannot compile stops the app rather than passing nothing on. - -Events carry the DFE 2.1 Kafka fields (`{message, tags, timestamp}`), and the optional `._conf.tz_offset` / `._conf.tz_map` fields drive the ios and meraki timezone handling. - -## Provenance - -`filebeat.yaml` embeds dfe-transform-vrl `pipelines/filebeat/filebeat.vrl` @ `e0b7087`, the same program bar two lines of its header comment, and `timezones.csv` is that repo's table copied. Refresh by copying both again rather than editing here -- the two transform apps grade against the same cases (`tests/e2e/filebeat_kafka.rs` in each), and that only means something while the program is the same one. diff --git a/pipelines/filebeat/filebeat.yaml b/pipelines/filebeat/filebeat.yaml deleted file mode 100644 index 789108d..0000000 --- a/pipelines/filebeat/filebeat.yaml +++ /dev/null @@ -1,5583 +0,0 @@ -# Project: dfe-transform-vector -# File: pipelines/filebeat/filebeat.yaml -# Purpose: Bundled filebeat pipeline - the DFE 2.1 filebeat VRL as a Vector remap -# Language: YAML -# -# License: BUSL-1.1 -# Copyright: (c) 2026 HYPERI PTY LIMITED -# -# A pre-canned transform file for `transforms.dir`. The wrapper parses it and -# writes it into Vector's config dir with its wired `inputs`, so the top-level -# `enrichment_tables` key reaches Vector and the VRL's timezone lookups resolve. -# `inputs: [dfe_source]` binds it to whatever Kafka topic `dfe_source` derives, -# and the wrapper wires the transform's output to `dfe_sink`. -# -# The `path` below is where the dfe-docker compose stack mounts the table. The -# image does not ship timezones.csv, so any other deployment mounts it at that -# path or points `path` at its own copy; startup validation names the path when -# the file is not there. -# -# The VRL body is dfe-transform-vrl pipelines/filebeat/filebeat.vrl @ e0b7087, -# bar two lines of its header comment, and that repo maintains it. Refresh by copying -# that file again rather than editing the copy here, so the two transform apps -# keep passing the same cases. - -enrichment_tables: - timezones: - type: file - file: - path: /etc/dfe-transform-vector/data/timezones.csv - encoding: - type: csv - schema: - abbreviation: string - -transforms: - filebeat: - type: remap - inputs: - - dfe_source - source: | - ############################################################################## - # dfe-transform-vrl bundled pipeline: filebeat-compat -- INTERIM - # - # Single-file port of the DFE 2.1 Vector filebeat transform templates - # (dfe-vector-templates src/core_templates 14x family): - # 140-transform-filebeat-cisco-meraki-logs - # 141-transform-filebeat-cisco-ios-logs - # 143..149-transform-filebeat-cisco-umbrella-*-logs (7 files, one body) - # - # The DFE 2.1 Vector deployment split this across per-module files to work - # around Vector's large-VRL-file limits, and its route transform fanned - # every {message,tags,timestamp} event to EVERY module transform, relying - # on drop_on_abort to discard the wrong parses. Embedded VRL has no - # file-size problem and runs one program per event, so this port detects - # the module first and runs only that module's transform. The seven - # umbrella templates differed only in the hardcoded .log.file.path seed; - # they collapse to one body driven by the detected log type. - # - # INTERIM: this pipeline keeps DFE 2.1 filebeat feeds working on - # dfe-transform-vrl. Elastic compatibility is being replaced by - # dfe-transform-elastic (Rust-native), and new integrations should NOT - # build on this file. - # - # Deliberate departures from the 2.1 templates: - # - geoip enrichment is NOT ported: dfe-loader owns geoip enrichment in - # the current DFE pipeline layout. - # - unmatched events are tagged "filebeat_unmatched" and passed through - # instead of being silently dropped by the route fan-out. - # - # Requirements (matching the 2.1 deployment contract): - # - enrichment table "timezones" (bundled timezones.csv, key column - # "abbreviation") for cisco-ios and meraki timestamp resolution. - # - events MAY carry ._conf.tz_offset (abbreviation like "UTC" or offset - # like "-0600"); without it the ios/meraki @timestamp mapping is - # skipped, exactly as in DFE 2.1. - ############################################################################## - - _ingest = {} - _ingest.timestamp = now() - - # --- module detection ------------------------------------------------------ - _module = "unknown" - _umbrella_type = "" - _msg = string(.message) ?? string(.event.original) ?? "" - - if match(_msg, r'^<134>1 \d{9,10}\.\d+ ') { - # Meraki syslog: <134>1 ... - # (DFE 2.1 routed on the <134> prefix alone; the epoch check keeps a - # local0.info Cisco IOS device out of the meraki branch.) - _module = "cisco_meraki_logs" - } else if match(_msg, r'^<\d{1,3}>') { - # Any other syslog-priority-prefixed line is Cisco IOS (the only - # syslog module in the 14x family besides Meraki). - _module = "cisco_ios_logs" - } else if match(_msg, r'%[A-Z][A-Z0-9_-]*-\d-[A-Z0-9_]+') && !match(_msg, r'^"?(\d{4}-\d{2}-\d{2} |\d+",")') { - # Cisco IOS %FACILITY-SEVERITY-MNEMONIC marker without a syslog - # header. The facility itself may carry dashes - # (e.g. %SNMP-SW1-3-RESPONSE_DELAYED). Umbrella CSV rows open with an - # ISO date (audit: a numeric id column), so a marker-lookalike inside - # a CSV field (a proxy URL, a dlp filename) must not steal the line - # from the CSV branch. - _module = "cisco_ios_logs" - } else { - # Cisco Umbrella exports are CSV; DFE 2.1 carried the log type in the - # S3 object path. Honour an upstream-provided .log.file.path first, - # then fall back to column-shape detection (counts verified against - # the elastic/integrations pipeline test corpus). - _path = string(.log.file.path) ?? "" - if contains(_path, "auditlogs") { - _umbrella_type = "auditlogs" - } else if contains(_path, "firewalllogs") { - _umbrella_type = "cloudfirewalllogs" - } else if contains(_path, "dlplogs") { - _umbrella_type = "dlplogs" - } else if contains(_path, "dnslogs") { - _umbrella_type = "dnslogs" - } else if contains(_path, "intrusionlogs") { - _umbrella_type = "intrusionlogs" - } else if contains(_path, "iplogs") { - _umbrella_type = "iplogs" - } else if contains(_path, "proxylogs") { - _umbrella_type = "proxylogs" - } - - if _umbrella_type == "" && _msg != "" { - # Same `", "` normalisation the umbrella body applies before - # parse_csv (the export quotes fields inconsistently). - _csv, _csv_err = parse_csv(replace(_msg, r'", "', s'","')) - if _csv_err == null { - _n = length(_csv) - _c0 = string(_csv[0]) ?? "" - _c1 = string(_csv[1]) ?? "" - _ts_re = r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}' - if _n == 7 && match(_c0, _ts_re) { - _umbrella_type = "iplogs" - } else if _n == 9 && match(_c1, _ts_re) { - # v5 audit format: numeric event id, then the timestamp. - _umbrella_type = "auditlogs" - } else if _n == 13 && match(_c0, _ts_re) { - _umbrella_type = "dnslogs" - } else if _n >= 10 && _n <= 12 && match(_c0, _ts_re) && match(string(_csv[6]) ?? "", r'^\d+ \(') { - # Older dns export without the trailing policy columns; - # column 6 is the " (TYPE)" dns query type. - _umbrella_type = "dnslogs" - } else if (_n == 14 || _n == 16) && match(_c0, _ts_re) { - # cloud firewall: 14 columns, or 16 with fqdns + - # destination list ids appended. - _umbrella_type = "cloudfirewalllogs" - } else if _n == 17 && match(_c0, _ts_re) { - # dlp and intrusion both carry 17 columns; intrusion has - # numeric gid/sid at columns 3 and 4. - if match(string(_csv[3]) ?? "", r'^\d+$') && match(string(_csv[4]) ?? "", r'^\d+$') { - _umbrella_type = "intrusionlogs" - } else { - _umbrella_type = "dlplogs" - } - } else if _n >= 20 && match(_c0, _ts_re) { - _umbrella_type = "proxylogs" - } - } - } - if _umbrella_type != "" { - _module = "cisco_umbrella" - } - } - - if _module == "cisco_meraki_logs" { - _ingest.pipeline = "cisco_meraki_logs" - _ingest.on_failure_pipeline = "cisco_meraki_logs" - .tags = ["preserve_original_event", "forwarded"] - .ecs.version = "8.11.0" - if .event.original == null { - if .message != null { - if .event.original == null { - .event.original = del(.message) - } else { - assert!(false, ".event.original must be null or override: true") - } - } - } - if .event.original != null { - tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} %{DATA:_temp.ts_nano} %{DATA:observer.hostname} %{DATA:cisco_meraki.event_type} %{DATA}$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "failed to dissect .event.original: " + to_string(err)) - } - } else { - err = ".event.original must not be nullish" - assert!(false, err) - } - if ._conf.tz_offset != null && ._conf.tz_offset != "local" { - date = null - err = "no datetime formats provided" - if err != null { - datetime = to_string!(._temp.ts_nano) - external_timezone = ._conf.tz_offset - if false { - external_timezone = null - } - records = if match!(external_timezone, r'[\+-]?[0-9]{1,2}:?[0-9]{2}') { - [{ "offset": external_timezone }] - } else { - find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": external_timezone }) - } - if length(records) == 1 { - offset = records[0].offset - date, err = parse_timestamp(value: join!(value: [datetime, offset], separator: " "), format: join!(value: ["%s%.f", "%:z"], separator: " ")) - } else { - if length(records) > 1 { - err = "ambiguous timezone abbreviation" - } else { - if length(records) == 0 { - err = "no timezone abbreviation found" - } - } - } - } - if err == null { - .@timestamp = date - } else { - _ingest.on_failure_processor_type = "date" - _ingest.on_failure_processor_tag = "" - _ingest.on_failure_message = to_string(err) - if !is_array(.error.message) { - if .error.message == null { - .error.message = [] - } else { - .error.message = [.error.message] - } - } - .error.message = append!({ - expr = .error.message - if false { - expr = null - } - expr - }, [join!(["failed to parse time field (", to_string!(._temp.ts_nano), "): ", _ingest.on_failure_message], "")]) - } - } - if ._conf.tz_offset == null || ._conf.tz_offset == "local" { - date = null - err = "no datetime formats provided" - if err != null { - datetime = to_string!(._temp.ts_nano) - date, err = parse_timestamp(value: join!(value: [datetime], separator: " "), format: join!(value: ["%s%.f"], separator: " ")) - } - if err == null { - .@timestamp = date - } else { - _ingest.on_failure_processor_type = "date" - _ingest.on_failure_processor_tag = "" - _ingest.on_failure_message = to_string(err) - if !is_array(.error.message) { - if .error.message == null { - .error.message = [] - } else { - .error.message = [.error.message] - } - } - .error.message = append!({ - expr = .error.message - if false { - expr = null - } - expr - }, [join!(["failed to parse time field (", to_string!(._temp.ts_nano), "): ", _ingest.on_failure_message], "")]) - } - } - if { - expr = ["flows", "firewall", "vpn_firewall", "cellular_firewall", "bridge_anyconnect_client_vpn_firewall"] - value = .cisco_meraki.event_type - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - _ingest.timestamp = now() - if _ingest.pipeline == null { - _ingest.pipeline = "" - _ingest.on_failure_pipeline = "" - } - if !is_nullish(.event.original) { - value, err = parse_groks(value: .event.original, patterns: [s'.*?%{TYPE}( %{NOTSPACE:cisco_meraki.flows.op})? src=%{IP:source.ip} dst=%{IP:destination.ip}( mac=%{MAC:source.mac})? protocol=%{NOTSPACE:network.protocol}( type=%{NOTSPACE})?( sport=%{NONNEGINT:source.port})?( dport=%{NONNEGINT:destination.port})?( pattern: %{GREEDYDATA:cisco_meraki.firewall.pattern})?.*'], aliases: { "TYPE": s'(flows|firewall|vpn_firewall|cellular_firewall|bridge_anyconnect_client_vpn_firewall)' }) - if err == null { - if value.destination.ip != null { - value.destination.ip = to_string!(value.destination.ip) - } - if value.destination.port != null { - value.destination.port = parse_int!(value.destination.port) - } - if value.source.ip != null { - value.source.ip = to_string!(value.source.ip) - } - if value.source.port != null { - value.source.port = parse_int!(value.source.port) - } - . = merge(., value, deep: true) - } else { - assert!(false, "failed to grok .event.original:" + to_string(err)) - } - } else { - assert!(false, ".event.original must not be nullish") - } - if .cisco_meraki.firewall.pattern != null && (starts_with!({ - expr = .cisco_meraki.firewall.pattern - if false { - expr = null - } - expr - }, "allow") || starts_with!({ - expr = .cisco_meraki.firewall.pattern - if false { - expr = null - } - expr - }, "deny")) { - if !is_nullish(.cisco_meraki.firewall.pattern) { - value, err = parse_groks(value: .cisco_meraki.firewall.pattern, patterns: [s'.*?%{NOTSPACE:cisco_meraki.firewall.action} %{GREEDYDATA:cisco_meraki.firewall.rule}.*']) - if err == null { - . = merge(., value, deep: true) - } - } - } - if .cisco_meraki.firewall.rule != null { - if !exists(.cisco_meraki.firewall.pattern) { - assert!(false, "tried to remove field '.cisco_meraki.firewall.pattern' that doesn't exist") - } - del(.cisco_meraki.firewall.pattern) - } - if !is_nullish(.source.mac) { - if is_array(.source.mac) { - .source.mac = map_values(array!(.source.mac)) -> |value| { - replace!(value, r'[:.]', "-") - } - } else { - if false { - .source.mac = null - } - target, err = replace(.source.mac, r'[:.]', "-") - if err == null { - .source.mac = target - } else { - assert!(false, to_string(err)) - } - } - } - if .cisco_meraki.flows.op == null { - .cisco_meraki.event_subtype = "ip_session_initiated" - } - if .cisco_meraki.flows.op == "allow" { - .cisco_meraki.event_subtype = "flow_allowed" - } - if .cisco_meraki.flows.op == "deny" { - .cisco_meraki.event_subtype = "flow_denied" - } - } - if .cisco_meraki.event_type == "ip_flow_start" || .cisco_meraki.event_type == "ip_flow_end" { - _ingest.timestamp = now() - if _ingest.pipeline == null { - _ingest.pipeline = "" - _ingest.on_failure_pipeline = "" - } - if .event.original != null { - tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} %{DATA} %{DATA} %{DATA:_temp.event_type} %{DATA:_temp.event}$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "failed to dissect .event.original: " + to_string(err)) - } - } else { - err = ".event.original must not be nullish" - assert!(false, err) - } - if ._temp.event != null { - if !is_nullish(._temp.event) { - kv, err = parse_key_value(._temp.event, key_value_delimiter: "=", field_delimiter: " ", accept_standalone_key: true, whitespace: "strict") - if err == null { - standalone_key = false - for_each(kv) -> |_, value| { - if value == true { - standalone_key = true - } - } - if standalone_key { - err = "field ._temp.event does not contain value_split [=]" - } - } - if err == null { - if . == null { - . = { } - } - . = merge(object(.), kv, deep: true) - } else { - assert!(false, to_string(err)) - } - } else { - err = "._temp.event must not be nullish" - assert!(false, to_string(err)) - } - } - if .translated_src_ip != null { - if !is_nullish(.translated_src_ip) { - if is_ipv4(string!(.translated_src_ip)) || is_ipv6(string!(.translated_src_ip)) { - .source.ip = .translated_src_ip - } else { - err = "Failed converting .translated_src_ip to an IP address" - assert!(false, err) - } - } else { - err = ".translated_src_ip must be present" - assert!(false, err) - } - } - if .translated_src_ip == null && .src != null { - if !is_nullish(.src) { - if is_ipv4(string!(.src)) || is_ipv6(string!(.src)) { - .source.ip = .src - } else { - err = "Failed converting .src to an IP address" - assert!(false, err) - } - } else { - err = ".src must be present" - assert!(false, err) - } - } - if .translated_src_ip != null && .translated_port != null { - if !is_nullish(.translated_port) { - if is_string(.translated_port) { - temp = .translated_port - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .source.port = temp - } else { - err_msg = join!(["Failed converting .translated_port to an integer. Found (", to_string!(.translated_port), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.translated_port) { - .source.port = .translated_port - } else { - err = "Failed converting .translated_port to an integer" - assert!(false, err) - } - } - } else { - err = ".translated_port must be present" - assert!(false, err) - } - } - if .translated_src_ip == null && .sport != null { - if !is_nullish(.sport) { - if is_string(.sport) { - temp = .sport - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .source.port = temp - } else { - err_msg = join!(["Failed converting .sport to an integer. Found (", to_string!(.sport), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.sport) { - .source.port = .sport - } else { - err = "Failed converting .sport to an integer" - assert!(false, err) - } - } - } else { - err = ".sport must be present" - assert!(false, err) - } - } - if .translated_dst_ip != null { - if !is_nullish(.translated_dst_ip) { - if is_ipv4(string!(.translated_dst_ip)) || is_ipv6(string!(.translated_dst_ip)) { - .destination.ip = .translated_dst_ip - } else { - err = "Failed converting .translated_dst_ip to an IP address" - assert!(false, err) - } - } else { - err = ".translated_dst_ip must be present" - assert!(false, err) - } - } - if .translated_dst_ip == null && .dst != null { - if !is_nullish(.dst) { - if is_ipv4(string!(.dst)) || is_ipv6(string!(.dst)) { - .destination.ip = .dst - } else { - err = "Failed converting .dst to an IP address" - assert!(false, err) - } - } else { - err = ".dst must be present" - assert!(false, err) - } - } - if .translated_dst_ip != null && .translated_port != null { - if !is_nullish(.translated_port) { - if is_string(.translated_port) { - temp = .translated_port - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .destination.port = temp - } else { - err_msg = join!(["Failed converting .translated_port to an integer. Found (", to_string!(.translated_port), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.translated_port) { - .destination.port = .translated_port - } else { - err = "Failed converting .translated_port to an integer" - assert!(false, err) - } - } - } else { - err = ".translated_port must be present" - assert!(false, err) - } - } - if .translated_dst_ip == null && .dport != null { - if !is_nullish(.dport) { - if is_string(.dport) { - temp = .dport - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .destination.port = temp - } else { - err_msg = join!(["Failed converting .dport to an integer. Found (", to_string!(.dport), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.dport) { - .destination.port = .dport - } else { - err = "Failed converting .dport to an integer" - assert!(false, err) - } - } - } else { - err = ".dport must be present" - assert!(false, err) - } - } - if .protocol != null { - if .network.protocol == null { - .network.protocol = del(.protocol) - } else { - assert!(false, ".network.protocol must be null or override: true") - } - } else { - assert!(false, ".protocol must not be null") - } - } - if .cisco_meraki.event_type == "airmarshal_events" { - _ingest.timestamp = now() - if _ingest.pipeline == null { - _ingest.pipeline = "" - _ingest.on_failure_pipeline = "" - } - if .event.original != null { - tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} airmarshal_events type=%{DATA:type} %{DATA}$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "failed to dissect .event.original: " + to_string(err)) - } - } else { - err = ".event.original must not be nullish" - assert!(false, err) - } - if .type != null { - if .cisco_meraki.event_subtype == null { - .cisco_meraki.event_subtype = del(.type) - } else { - assert!(false, ".cisco_meraki.event_subtype must be null or override: true") - } - } else { - assert!(false, ".type must not be null") - } - if !is_nullish(.event.original) { - value, err = parse_groks(value: .event.original, patterns: [s'.*?%{GREEDYDATA} ssid=%{QS:_temp.ssid}%{SPACE}%{GREEDYDATA:_temp.kvline}.*']) - if err == null { - . = merge(., value, deep: true) - } else { - assert!(false, "failed to grok .event.original:" + to_string(err)) - } - } else { - assert!(false, ".event.original must not be nullish") - } - if ._temp.ssid != null { - tmp, err = parse_groks(value: ._temp.ssid, patterns: [s'^\'%{DATA:_temp.kv.ssid}\'.*?$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "failed to dissect ._temp.ssid: " + to_string(err)) - } - } else { - err = "._temp.ssid must not be nullish" - assert!(false, err) - } - if !is_nullish(._temp.kvline) { - kv, err = parse_key_value(._temp.kvline, key_value_delimiter: "=", field_delimiter: " ", accept_standalone_key: true, whitespace: "strict") - if err == null { - standalone_key = false - for_each(kv) -> |_, value| { - if value == true { - standalone_key = true - } - } - if standalone_key { - err = "field ._temp.kvline does not contain value_split [=]" - } - } - if err == null { - if ._temp.kv == null { - ._temp.kv = { } - } - ._temp.kv = merge(object!(._temp.kv), kv, deep: true) - } else { - assert!(false, to_string(err)) - } - } else { - err = "._temp.kvline must not be nullish" - assert!(false, to_string(err)) - } - if ._temp.kv.ssid != null { - if ._temp.kv.ssid != null { - if .network.name == null { - .network.name = del(._temp.kv.ssid) - } else { - assert!(false, ".network.name must be null or override: true") - } - } else { - assert!(false, "._temp.kv.ssid must not be null") - } - } - if ._temp.kv.bssid != null { - if .cisco_meraki.bssid == null { - .cisco_meraki.bssid = del(._temp.kv.bssid) - } else { - assert!(false, ".cisco_meraki.bssid must be null or override: true") - } - } else { - assert!(false, "._temp.kv.bssid must not be null") - } - if .cisco_meraki.event_subtype == "ssid_spoofing_detected" { - if ._temp.kv.vap != null { - if .cisco_meraki.vap == null { - .cisco_meraki.vap = del(._temp.kv.vap) - } else { - assert!(false, ".cisco_meraki.vap must be null or override: true") - } - } else { - assert!(false, "._temp.kv.vap must not be null") - } - } - if !is_nullish(._temp.kv.src) { - if is_array(._temp.kv.src) { - .source.mac = map_values(array!(._temp.kv.src)) -> |value| { - replace!(value, r'[-:.]', "-") - } - } else { - if false { - ._temp.kv.src = null - } - target, err = replace(._temp.kv.src, r'[-:.]', "-") - if err == null { - .source.mac = target - } else { - assert!(false, to_string(err)) - } - } - } else { - assert!(false, "._temp.kv.src must not be nullish") - } - if !is_nullish(._temp.kv.dst) { - if is_array(._temp.kv.dst) { - .destination.mac = map_values(array!(._temp.kv.dst)) -> |value| { - replace!(value, r'[-:.]', "-") - } - } else { - if false { - ._temp.kv.dst = null - } - target, err = replace(._temp.kv.dst, r'[-:.]', "-") - if err == null { - .destination.mac = target - } else { - assert!(false, to_string(err)) - } - } - } else { - assert!(false, "._temp.kv.dst must not be nullish") - } - if .cisco_meraki.event_subtype == "rogue_ssid_detected" { - if !is_nullish(._temp.kv.wired_mac) { - if is_array(._temp.kv.wired_mac) { - ._temp.observer.mac = map_values(array!(._temp.kv.wired_mac)) -> |value| { - replace!(value, r'[-:.]', "-") - } - } else { - if false { - ._temp.kv.wired_mac = null - } - target, err = replace(._temp.kv.wired_mac, r'[-:.]', "-") - if err == null { - ._temp.observer.mac = target - } else { - assert!(false, to_string(err)) - } - } - } else { - assert!(false, "._temp.kv.wired_mac must not be nullish") - } - } - if ._temp.observer.mac != null { - if !is_array(.observer.mac) { - if .observer.mac == null { - .observer.mac = [] - } else { - .observer.mac = [.observer.mac] - } - } - .observer.mac = append!({ - expr = .observer.mac - if false { - expr = null - } - expr - }, [._temp.observer.mac]) - } - if .cisco_meraki.event_subtype == "rogue_ssid_detected" { - if ._temp.kv.vlan_id != null { - if .network.vlan.id == null { - .network.vlan.id = del(._temp.kv.vlan_id) - } else { - assert!(false, ".network.vlan.id must be null or override: true") - } - } else { - assert!(false, "._temp.kv.vlan_id must not be null") - } - } - if ._temp.kv.channel != null { - if .cisco_meraki.channel == null { - .cisco_meraki.channel = del(._temp.kv.channel) - } else { - assert!(false, ".cisco_meraki.channel must be null or override: true") - } - } else { - assert!(false, "._temp.kv.channel must not be null") - } - if ._temp.kv.fc_type != null { - if .cisco_meraki.fc_type == null { - .cisco_meraki.fc_type = del(._temp.kv.fc_type) - } else { - assert!(false, ".cisco_meraki.fc_type must be null or override: true") - } - } else { - assert!(false, "._temp.kv.fc_type must not be null") - } - if ._temp.kv.fc_subtype != null { - if .cisco_meraki.fc_subtype == null { - .cisco_meraki.fc_subtype = del(._temp.kv.fc_subtype) - } else { - assert!(false, ".cisco_meraki.fc_subtype must be null or override: true") - } - } else { - assert!(false, "._temp.kv.fc_subtype must not be null") - } - } - if .cisco_meraki.event_type == "security_event" { - _ingest.timestamp = now() - if _ingest.pipeline == null { - _ingest.pipeline = "" - _ingest.on_failure_pipeline = "" - } - if .event.original != null { - tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} security_event %{DATA:type} %{DATA}$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "failed to dissect .event.original: " + to_string(err)) - } - } else { - err = ".event.original must not be nullish" - assert!(false, err) - } - if .type != null { - if .cisco_meraki.event_subtype == null { - .cisco_meraki.event_subtype = del(.type) - } else { - assert!(false, ".cisco_meraki.event_subtype must be null or override: true") - } - } else { - assert!(false, ".type must not be null") - } - if !is_nullish(.event.original) { - value, err = parse_groks(value: .event.original, patterns: [s'.*?^%{DATA} (security_event|ids-alerts) (%{WORD}\s)?%{DATA:_temp.kvs}(\smessage:\s?%{DATA:message})?$.*']) - if err == null { - . = merge(., value, deep: true) - } else { - assert!(false, "failed to grok .event.original:" + to_string(err)) - } - } else { - assert!(false, ".event.original must not be nullish") - } - if !is_nullish(._temp.kvs) { - kv, err = parse_key_value(._temp.kvs, key_value_delimiter: "=", field_delimiter: " ", accept_standalone_key: true, whitespace: "strict") - if err == null { - standalone_key = false - for_each(kv) -> |_, value| { - if value == true { - standalone_key = true - } - } - if standalone_key { - err = "field ._temp.kvs does not contain value_split [=]" - } - } - if err == null { - if . == null { - . = { } - } - . = merge(object(.), kv, deep: true) - } else { - assert!(false, to_string(err)) - } - } else { - err = "._temp.kvs must not be nullish" - assert!(false, to_string(err)) - } - if .priority != null { - if .cisco_meraki.security.priority == null { - .cisco_meraki.security.priority = del(.priority) - } else { - assert!(false, ".cisco_meraki.security.priority must be null or override: true") - } - } - if .signature != null { - if .cisco_meraki.security.signature == null { - .cisco_meraki.security.signature = del(.signature) - } else { - assert!(false, ".cisco_meraki.security.signature must be null or override: true") - } - } - if !is_nullish(.dhost) { - if is_array(.dhost) { - .cisco_meraki.security.dhost = map_values(array!(.dhost)) -> |value| { - replace!(value, r'[-:.]', "-") - } - } else { - if false { - .dhost = null - } - target, err = replace(.dhost, r'[-:.]', "-") - if err == null { - .cisco_meraki.security.dhost = target - } else { - assert!(false, to_string(err)) - } - } - } - if .direction != null { - if .network.direction == null { - .network.direction = del(.direction) - } else { - assert!(false, ".network.direction must be null or override: true") - } - } - if !is_nullish(.protocol) { - if is_array(.protocol) { - .protocol = map_values(array!(.protocol)) -> |value| { - downcase!(value) - } - } else { - tmp, err = downcase(.protocol) - if err == null { - .network.protocol = tmp - } else { - assert!(false, to_string(err)) - } - } - } - if .decision != null { - if .cisco_meraki.security.decision == null { - .cisco_meraki.security.decision = del(.decision) - } else { - assert!(false, ".cisco_meraki.security.decision must be null or override: true") - } - } - if .url != null { - uri, err = parse_url(.url, default_known_ports: true) - if err == null { - if uri.username == "" { - del(uri.username) - } - if uri.password == "" { - del(uri.password) - } - uri.query = encode_key_value!(uri.query, field_delimiter: "&", key_value_delimiter: "=") - if uri.query == "" { - del(uri.query) - } - if uri.path != null { - uri.path = decode_percent!(uri.path) - } - uri.original = .url - uri.extension = parse_regex(uri.path, r'.*\.(?P[a-zA-Z0-9_\-]+)$').extension ?? null - uri.domain = del(uri.host) - if uri.username != null && uri.password != null { - uri.user_info = join!([uri.username, uri.password], separator: ":") - } - .url = compact(uri) - } - } - if !is_nullish(.mac) { - if is_array(.mac) { - .cisco_meraki.security.mac = map_values(array!(.mac)) -> |value| { - replace!(value, r'[-:.]', "-") - } - } else { - if false { - .mac = null - } - target, err = replace(.mac, r'[-:.]', "-") - if err == null { - .cisco_meraki.security.mac = target - } else { - assert!(false, to_string(err)) - } - } - } - if .name != null { - if .file.name == null { - .file.name = del(.name) - } else { - assert!(false, ".file.name must be null or override: true") - } - } - if .sha256 != null { - if .file.hash.sha256 == null { - .file.hash.sha256 = del(.sha256) - } else { - assert!(false, ".file.hash.sha256 must be null or override: true") - } - } - if .disposition != null { - if .cisco_meraki.disposition == null { - .cisco_meraki.disposition = del(.disposition) - } else { - assert!(false, ".cisco_meraki.disposition must be null or override: true") - } - } - if .action != null { - if .cisco_meraki.security.action == null { - .cisco_meraki.security.action = del(.action) - } else { - assert!(false, ".cisco_meraki.security.action must be null or override: true") - } - } - if .cisco_meraki.event_subtype != "security_filtering_disposition_change" && .src != null { - if !is_nullish(.src) { - value, err = parse_groks(value: .src, patterns: [s'.*?^%{IPV4:_temp.src_ip}:%{PORT:sport}$.*', s'.*?^\[%{IPV6:_temp.src_ip}\]:%{PORT:sport}$.*', s'.*?^%{IPV6NOCOMPRESS:_temp.src_ip}:%{PORT:sport}$.*', s'.*?^%{IPV6:_temp.src_ip}%{IPV6PORTSEP}%{PORT:sport}$.*'], aliases: { "IPV6NOCOMPRESS": s'([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}', "IPV6PORTSEP": s'(?: port |[p#.])', "PORT": s'[0-9]+' }) - if err == null { - . = merge(., value, deep: true) - } else { - assert!(false, "failed to grok .src:" + to_string(err)) - } - } else { - assert!(false, ".src must not be nullish") - } - } - if !is_nullish(._temp.src_ip) { - if is_ipv4(string!(._temp.src_ip)) || is_ipv6(string!(._temp.src_ip)) { - .source.ip = ._temp.src_ip - } else { - err = "Failed converting ._temp.src_ip to an IP address" - assert!(false, err) - } - } - if .sport != "0" { - if !is_nullish(.sport) { - if is_string(.sport) { - temp = .sport - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .source.port = temp - } - } else { - if is_integer(.sport) { - .source.port = .sport - } - } - } - } - if .cisco_meraki.event_subtype != "security_filtering_disposition_change" && .dst != null { - if !is_nullish(.dst) { - value, err = parse_groks(value: .dst, patterns: [s'.*?^%{IPV4:_temp.dst_ip}:%{PORT:dport}$.*', s'.*?^\[%{IPV6:_temp.dst_ip}\]:%{PORT:dport}$.*', s'.*?^%{IPV6NOCOMPRESS:_temp.dst_ip}:%{PORT:dport}$.*', s'.*?^%{IPV6:_temp.dst_ip}%{IPV6PORTSEP}%{PORT:dport}$.*'], aliases: { "IPV6NOCOMPRESS": s'([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}', "IPV6PORTSEP": s'(?: port |[p#.])', "PORT": s'[0-9]+' }) - if err == null { - . = merge(., value, deep: true) - } else { - assert!(false, "failed to grok .dst:" + to_string(err)) - } - } else { - assert!(false, ".dst must not be nullish") - } - } - if !is_nullish(._temp.dst_ip) { - if is_ipv4(string!(._temp.dst_ip)) || is_ipv6(string!(._temp.dst_ip)) { - .destination.ip = ._temp.dst_ip - } - } - if .dport != "0" && .cisco_meraki.event_subtype != "security_filtering_disposition_change" { - if !is_nullish(.dport) { - if is_string(.dport) { - temp = .dport - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .destination.port = temp - } - } else { - if is_integer(.dport) { - .destination.port = .dport - } - } - } - } - } - if .cisco_meraki.event_type == "ids-alerts" { - _ingest.timestamp = now() - if _ingest.pipeline == null { - _ingest.pipeline = "" - _ingest.on_failure_pipeline = "" - } - if .event.original != null { - tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} ids-alerts signature=%{DATA:signature} priority=%{DATA:priority} timestamp=%{DATA:timestamp} direction=%{DATA:direction} protocol=%{DATA:protocol} src=%{DATA:src}$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "failed to dissect .event.original: " + to_string(err)) - } - } else { - err = ".event.original must not be nullish" - assert!(false, err) - } - .cisco_meraki.event_subtype = "ids_alerted" - if .priority != null { - if .cisco_meraki.security.priority == null { - .cisco_meraki.security.priority = del(.priority) - } else { - assert!(false, ".cisco_meraki.security.priority must be null or override: true") - } - } else { - assert!(false, ".priority must not be null") - } - if .signature != null { - if .cisco_meraki.security.signature == null { - .cisco_meraki.security.signature = del(.signature) - } else { - assert!(false, ".cisco_meraki.security.signature must be null or override: true") - } - } else { - assert!(false, ".signature must not be null") - } - if .direction != null { - if .network.direction == null { - .network.direction = del(.direction) - } else { - assert!(false, ".network.direction must be null or override: true") - } - } else { - assert!(false, ".direction must not be null") - } - if !is_nullish(.protocol) { - if is_array(.protocol) { - .protocol = map_values(array!(.protocol)) -> |value| { - downcase!(value) - } - } else { - tmp, err = downcase(.protocol) - if err == null { - .network.protocol = tmp - } else { - assert!(false, to_string(err)) - } - } - } else { - assert!(false, ".protocol must not be nullish") - } - if .src != null { - if !is_nullish(.src) { - value, err = parse_groks(value: .src, patterns: [s'.*?^%{IPV4:_temp.src_ip}:%{PORT:sport}$.*', s'.*?^\[%{IPV6:_temp.src_ip}\]:%{PORT:sport}$.*', s'.*?^%{IPV6NOCOMPRESS:_temp.src_ip}:%{PORT:sport}$.*', s'.*?^%{IPV6:_temp.src_ip}%{IPV6PORTSEP}%{PORT:sport}$.*'], aliases: { "IPV6NOCOMPRESS": s'([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}', "IPV6PORTSEP": s'(?: port |[p#.])', "PORT": s'[0-9]+' }) - if err == null { - . = merge(., value, deep: true) - } else { - assert!(false, "failed to grok .src:" + to_string(err)) - } - } else { - assert!(false, ".src must not be nullish") - } - } - if !is_nullish(._temp.src_ip) { - if is_ipv4(string!(._temp.src_ip)) || is_ipv6(string!(._temp.src_ip)) { - .source.ip = ._temp.src_ip - } - } - if !is_nullish(.sport) { - if is_string(.sport) { - temp = .sport - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .source.port = temp - } - } else { - if is_integer(.sport) { - .source.port = .sport - } - } - } - } - if .cisco_meraki.event_type == "events" { - _ingest.timestamp = now() - if _ingest.pipeline == null { - _ingest.pipeline = "" - _ingest.on_failure_pipeline = "" - } - if .event.original != null { - tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} events %{DATA:msgtype} %{DATA}$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "failed to dissect .event.original: " + to_string(err)) - } - } else { - err = ".event.original must not be nullish" - assert!(false, err) - } - if downcase(string!(.msgtype)) == "site-to-site" { - .cisco_meraki.event_subtype = "Site-to-Site VPN" - } - if downcase(string!(.msgtype)) == "client_vpn_connect" { - .cisco_meraki.event_subtype = "client_vpn_connect" - } - if downcase(string!(.msgtype)) == "blocked" { - .cisco_meraki.event_subtype = "blocked" - } - if downcase(string!(.msgtype)) == "auth" { - .cisco_meraki.event_subtype = "auth" - } - if downcase(string!(.msgtype)) == "port" { - .cisco_meraki.event_subtype = "port" - } - if downcase(string!(.msgtype)) == "carrier_change" { - .cisco_meraki.event_subtype = "carrier_change" - } - if starts_with!({ - expr = .msgtype - if false { - expr = null - } - expr - }, "type=") { - if .event.original != null { - tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} events type=%{DATA:type} %{DATA}$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "failed to dissect .event.original: " + to_string(err)) - } - } else { - err = ".event.original must not be nullish" - assert!(false, err) - } - } - if .type != null { - if .type != null { - if .cisco_meraki.event_subtype == null { - .cisco_meraki.event_subtype = del(.type) - } else { - assert!(false, ".cisco_meraki.event_subtype must be null or override: true") - } - } else { - assert!(false, ".type must not be null") - } - } - if downcase(string!(.msgtype)) == "dhcp" { - if .event.original != null { - tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} events dhcp %{DATA:_temp.dhcp_op} %{DATA:_temp.dhcp_op2} %{DATA}$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "failed to dissect .event.original: " + to_string(err)) - } - } else { - err = ".event.original must not be nullish" - assert!(false, err) - } - } - if downcase(string!(.msgtype)) == "dhcp" { - .network.protocol = "dhcp" - } - if downcase(string!(.msgtype)) == "dhcp" && downcase(string!(._temp.dhcp_op)) == "lease" { - if .event.original != null { - tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} events dhcp lease of ip %{DATA:_temp.client_ip} from %{DATA} mac %{DATA:server.mac} for client mac %{DATA:client.mac} %{DATA}$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "failed to dissect .event.original: " + to_string(err)) - } - } else { - err = ".event.original must not be nullish" - assert!(false, err) - } - } - if downcase(string!(.msgtype)) == "dhcp" && downcase(string!(._temp.dhcp_op)) == "no" && downcase(string!(._temp.dhcp_op2)) == "offers" { - if .event.original != null { - tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} events dhcp no offers for mac %{DATA:client.mac} %{DATA}$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "failed to dissect .event.original: " + to_string(err)) - } - } else { - err = ".event.original must not be nullish" - assert!(false, err) - } - } - if downcase(string!(.msgtype)) == "dhcp" && ._temp.dhcp_op == "lease" { - .cisco_meraki.event_subtype = "dhcp_offer" - } - if downcase(string!(.msgtype)) == "dhcp" && downcase(string!(._temp.dhcp_op)) == "no" && downcase(string!(._temp.dhcp_op2)) == "offers" { - .cisco_meraki.event_subtype = "dhcp_no_offer" - } - if downcase(string!(.msgtype)) == "dhcp" { - if !is_nullish(.event.original) { - value, err = parse_groks(value: .event.original, patterns: [s'.*?events dhcp %{GREEDYDATA:message}$.*']) - if err == null { - . = merge(., value, deep: true) - } else { - assert!(false, "failed to grok .event.original:" + to_string(err)) - } - } else { - assert!(false, ".event.original must not be nullish") - } - } - if starts_with!({ - expr = .event.original - if false { - expr = null - } - expr - }, "<") && .cisco_meraki.event_subtype == "Site-to-Site VPN" { - if !is_nullish(.event.original) { - value, err = parse_groks(value: .event.original, patterns: [s'.*?%{SYSLOGHDR}%{SPACE}%{NUMBER}%{SPACE}%{WORDORHOST}%{SPACE}events%{SPACE}(?i)Site-to-Site VPN:%{GREEDYDATA:cisco_meraki.site_to_site_vpn.raw}.*'], aliases: { "SYSLOGHDR": s'%{SYSLOGPRI}%{SYSLOGVER}', "SYSLOGPRI": s'<%{NONNEGINT:log.syslog.priority}>', "SYSLOGVER": s'\b(?:\d{1,2})\b', "WORDORHOST": s'(?:%{WORD}|%{HOSTNAME})' }) - if err == null { - if value.log.syslog.priority != null { - value.log.syslog.priority = parse_int!(value.log.syslog.priority) - } - . = merge(., value, deep: true) - } else { - assert!(false, "failed to grok .event.original:" + to_string(err)) - } - } else { - assert!(false, ".event.original must not be nullish") - } - } - if starts_with!({ - expr = .event.original - if false { - expr = null - } - expr - }, "<") && .cisco_meraki.event_subtype == "blocked" { - if !is_nullish(.event.original) { - value, err = parse_groks(value: .event.original, patterns: [s'.*?^%{SYSLOGHDR}%{SPACE}%{NUMBER}%{SPACE}%{WORDORHOST}%{SPACE}events%{SPACE}(?%{BLOCKEDARP:_temp.blocked_arp} from %{MAC:source.mac} with IP %{IP:source.ip} on %{NOTSPACE} %{GREEDYDATA:observer.ingress.vlan.id})$.*'], aliases: { "BLOCKEDARP": s'Blocked ARP Packet', "SYSLOGHDR": s'%{SYSLOGPRI}%{SYSLOGVER}', "SYSLOGPRI": s'<%{NONNEGINT:log.syslog.priority}>', "SYSLOGVER": s'\b(?:\d{1,2})\b', "WORDORHOST": s'(?:%{WORD}|%{HOSTNAME})' }) - if err == null { - if value.log.syslog.priority != null { - value.log.syslog.priority = parse_int!(value.log.syslog.priority) - } - . = merge(., value, deep: true) - } else { - assert!(false, "failed to grok .event.original:" + to_string(err)) - } - } else { - assert!(false, ".event.original must not be nullish") - } - } - if !is_nullish(.source.mac) { - if is_array(.source.mac) { - .source.mac = map_values(array!(.source.mac)) -> |value| { - replace!(value, r'[:.]', "-") - } - } else { - if false { - .source.mac = null - } - target, err = replace(.source.mac, r'[:.]', "-") - if err == null { - .source.mac = target - } else { - assert!(false, to_string(err)) - } - } - } - if !is_nullish(.source.mac) { - .source.mac = upcase!(.source.mac) - } - if ._temp.blocked_arp != null { - .cisco_meraki.event_subtype = "arp_blocked" - } - if starts_with!({ - expr = .event.original - if false { - expr = null - } - expr - }, "<") && .cisco_meraki.event_subtype == "port" { - if !is_nullish(.event.original) { - value, err = parse_groks(value: .event.original, patterns: [s'.*?^(?i)%{SYSLOGHDR}%{SPACE}%{NUMBER}%{SPACE}%{WORDORHOST}%{SPACE}events%{SPACE}(?port %{NOTSPACE:cisco_meraki.port} %{PORTACTION:_temp.port_action}(?: from %{NOTSPACE:cisco_meraki.old_port_status} to %{NOTSPACE:cisco_meraki.new_port_status}|.*))$.*'], aliases: { "PORTACTION": s'(?:changed stp role|status changed)', "SYSLOGHDR": s'%{SYSLOGPRI}%{SYSLOGVER}', "SYSLOGPRI": s'<%{NONNEGINT:log.syslog.priority}>', "SYSLOGVER": s'\b(?:\d{1,2})\b', "WORDORHOST": s'(?:%{WORD}|%{HOSTNAME})' }) - if err == null { - if value.log.syslog.priority != null { - value.log.syslog.priority = parse_int!(value.log.syslog.priority) - } - . = merge(., value, deep: true) - } else { - assert!(false, "failed to grok .event.original:" + to_string(err)) - } - } else { - assert!(false, ".event.original must not be nullish") - } - } - if !is_nullish(._temp.port_action) { - if is_array(._temp.port_action) { - ._temp.port_action = map_values(array!(._temp.port_action)) -> |value| { - replace!(value, r' ', "_") - } - } else { - if false { - ._temp.port_action = null - } - target, err = replace(._temp.port_action, r' ', "_") - if err == null { - ._temp.port_action = target - } else { - assert!(false, to_string(err)) - } - } - } - if !is_nullish(._temp.port_action) { - if is_array(._temp.port_action) { - ._temp.port_action = map_values(array!(._temp.port_action)) -> |value| { - downcase!(value) - } - } else { - tmp, err = downcase(._temp.port_action) - if err == null { - ._temp.port_action = tmp - } else { - assert!(false, to_string(err)) - } - } - } - if ._temp.port_action != null { - .cisco_meraki.event_subtype = join!(["port_", to_string!(._temp.port_action)], "") - } - if starts_with!({ - expr = .event.original - if false { - expr = null - } - expr - }, "<") && .cisco_meraki.event_subtype == "carrier_change" { - if !is_nullish(.event.original) { - value, err = parse_groks(value: .event.original, patterns: [s'.*?^(?i)%{SYSLOGHDR}%{SPACE}%{NUMBER}%{SPACE}%{WORDORHOST}%{SPACE}events carrier_change device%{SPACE}%{NOTSPACE:cisco_meraki.mxport} up %{NOTSPACE:_temp.up}.*$.*'], aliases: { "SYSLOGHDR": s'%{SYSLOGPRI}%{SYSLOGVER}', "SYSLOGPRI": s'<%{NONNEGINT:log.syslog.priority}>', "SYSLOGVER": s'\b(?:\d{1,2})\b', "WORDORHOST": s'(?:%{WORD}|%{HOSTNAME})' }) - if err == null { - if value.log.syslog.priority != null { - value.log.syslog.priority = parse_int!(value.log.syslog.priority) - } - . = merge(., value, deep: true) - } else { - assert!(false, "failed to grok .event.original:" + to_string(err)) - } - } else { - assert!(false, ".event.original must not be nullish") - } - } - if ._temp.up == "true" { - .cisco_meraki.new_port_status = "up" - } - if ._temp.up == "false" { - .cisco_meraki.new_port_status = "down" - } - if starts_with!({ - expr = .event.original - if false { - expr = null - } - expr - }, "<") && { - expr = ["dfs_event", "association", "disassociation", "aps_association_reject", "multiple_dhcp_servers_detected", "wpa_deauth", "wpa_auth", "vpn_connectivity_change", "8021x_eap_failure", "8021x_auth", "8021x_deauth", "8021x_eap_success", "splash_auth", "device_packet_flood"] - value = .cisco_meraki.event_subtype - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - if !is_nullish(.event.original) { - value, err = parse_groks(value: .event.original, patterns: [s'.*?%{SYSLOGHDR}%{SPACE}%{NUMBER}%{SPACE}%{WORDORHOST}%{SPACE}events%{SPACE}%{GREEDYDATA:_temp.rest}.*'], aliases: { "SYSLOGHDR": s'%{SYSLOGPRI}%{SYSLOGVER}', "SYSLOGPRI": s'<%{NONNEGINT:log.syslog.priority}>', "SYSLOGVER": s'\b(?:\d{1,2})\b', "WORDORHOST": s'(?:%{WORD}|%{HOSTNAME})' }) - if err == null { - if value.log.syslog.priority != null { - value.log.syslog.priority = parse_int!(value.log.syslog.priority) - } - . = merge(., value, deep: true) - } else { - assert!(false, "failed to grok .event.original:" + to_string(err)) - } - } else { - assert!(false, ".event.original must not be nullish") - } - } - if ._temp.rest != null && { - expr = ["dfs_event", "association", "disassociation", "aps_association_reject", "multiple_dhcp_servers_detected", "wpa_deauth", "wpa_auth", "8021x_eap_failure", "8021x_auth", "8021x_deauth", "8021x_eap_success", "splash_auth", "device_packet_flood"] - value = .cisco_meraki.event_subtype - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - if !is_nullish(._temp.rest) { - kv, err = parse_key_value(._temp.rest, key_value_delimiter: "=", field_delimiter: " ", accept_standalone_key: true, whitespace: "strict") - if err == null { - standalone_key = false - for_each(kv) -> |_, value| { - if value == true { - standalone_key = true - } - } - if standalone_key { - err = "field ._temp.rest does not contain value_split [=]" - } - } - if err == null { - target = get(., ["cisco_meraki", .cisco_meraki.event_subtype]) ?? { } - if target == null { - target = { } - } - . = set!(., ["cisco_meraki", .cisco_meraki.event_subtype], merge!(target, kv, deep: true)) - } else { - assert!(false, to_string(err)) - } - } else { - err = "._temp.rest must not be nullish" - assert!(false, to_string(err)) - } - } - if ._temp.rest != null && .cisco_meraki.event_subtype == "vpn_connectivity_change" { - if !is_nullish(._temp.rest) { - kv, err = parse_key_value(._temp.rest, key_value_delimiter: "=", field_delimiter: " ", accept_standalone_key: true, whitespace: "strict") - if err == null { - standalone_key = false - for_each(kv) -> |_, value| { - if value == true { - standalone_key = true - } - } - if standalone_key { - err = "field ._temp.rest does not contain value_split [=]" - } - } - if err == null { - if .cisco_meraki.site_to_site_vpn.connectivity_change == null { - .cisco_meraki.site_to_site_vpn.connectivity_change = { } - } - .cisco_meraki.site_to_site_vpn.connectivity_change = merge(object!(.cisco_meraki.site_to_site_vpn.connectivity_change), kv, deep: true) - } else { - assert!(false, to_string(err)) - } - } else { - err = "._temp.rest must not be nullish" - assert!(false, to_string(err)) - } - } - if .cisco_meraki.event_subtype == "multiple_dhcp_servers_detected" { - .network.protocol = "dhcp" - } - if .cisco_meraki.event_subtype == "multiple_dhcp_servers_detected" { - if .cisco_meraki.multiple_dhcp_servers_detected.original_server_mac != null { - if .server.mac == null { - .server.mac = del(.cisco_meraki.multiple_dhcp_servers_detected.original_server_mac) - } else { - assert!(false, ".server.mac must be null or override: true") - } - } else { - assert!(false, ".cisco_meraki.multiple_dhcp_servers_detected.original_server_mac must not be null") - } - } - if .cisco_meraki.event_subtype == "multiple_dhcp_servers_detected" { - if !is_nullish(.cisco_meraki.multiple_dhcp_servers_detected.original_server_ip) { - value, err = parse_groks(value: .cisco_meraki.multiple_dhcp_servers_detected.original_server_ip, patterns: [s'.*?^%{IPV4:cisco_meraki.multiple_dhcp_servers_detected.original_server_ip}$.*', s'.*?^%{IPV6:cisco_meraki.multiple_dhcp_servers_detected.original_server_ip}$.*']) - if err == null { - . = merge(., value, deep: true) - } - } - } - if .cisco_meraki.event_subtype == "multiple_dhcp_servers_detected" { - if !is_nullish(.cisco_meraki.multiple_dhcp_servers_detected.original_server_ip) { - if is_ipv4(string!(.cisco_meraki.multiple_dhcp_servers_detected.original_server_ip)) || is_ipv6(string!(.cisco_meraki.multiple_dhcp_servers_detected.original_server_ip)) { - .server.ip = .cisco_meraki.multiple_dhcp_servers_detected.original_server_ip - } - } - } - if .server.ip != null { - if !exists(.cisco_meraki.multiple_dhcp_servers_detected.original_server_ip) { - assert!(false, "tried to remove field '.cisco_meraki.multiple_dhcp_servers_detected.original_server_ip' that doesn't exist") - } - del(.cisco_meraki.multiple_dhcp_servers_detected.original_server_ip) - } - if .cisco_meraki.event_subtype == "multiple_dhcp_servers_detected" { - if !is_array(.related.ip) { - if .related.ip == null { - .related.ip = [] - } else { - .related.ip = [.related.ip] - } - } - .related.ip = append!({ - expr = .related.ip - if false { - expr = null - } - expr - }, [.server.ip]) - } - if .cisco_meraki.event_subtype == "multiple_dhcp_servers_detected" { - if !is_nullish(.cisco_meraki.multiple_dhcp_servers_detected.server_ip) { - value, err = parse_groks(value: .cisco_meraki.multiple_dhcp_servers_detected.server_ip, patterns: [s'.*?^%{IPV4:cisco_meraki.multiple_dhcp_servers_detected.server_ip}$.*', s'.*?^%{IPV6:cisco_meraki.multiple_dhcp_servers_detected.server_ip}$.*']) - if err == null { - . = merge(., value, deep: true) - } else { - assert!(false, "failed to grok .cisco_meraki.multiple_dhcp_servers_detected.server_ip:" + to_string(err)) - } - } else { - assert!(false, ".cisco_meraki.multiple_dhcp_servers_detected.server_ip must not be nullish") - } - } - if .cisco_meraki.event_subtype == "multiple_dhcp_servers_detected" { - if !is_nullish(.cisco_meraki.multiple_dhcp_servers_detected.server_ip) { - if is_ipv4(string!(.cisco_meraki.multiple_dhcp_servers_detected.server_ip)) || is_ipv6(string!(.cisco_meraki.multiple_dhcp_servers_detected.server_ip)) { - .cisco_meraki.multiple_dhcp_servers_detected.server_ip = .cisco_meraki.multiple_dhcp_servers_detected.server_ip - } else { - err = "Failed converting .cisco_meraki.multiple_dhcp_servers_detected.server_ip to an IP address" - assert!(false, err) - } - } else { - err = ".cisco_meraki.multiple_dhcp_servers_detected.server_ip must be present" - assert!(false, err) - } - } - if .cisco_meraki.event_subtype == "multiple_dhcp_servers_detected" { - if !is_array(.related.ip) { - if .related.ip == null { - .related.ip = [] - } else { - .related.ip = [.related.ip] - } - } - .related.ip = append!({ - expr = .related.ip - if false { - expr = null - } - expr - }, [.cisco_meraki.multiple_dhcp_servers_detected.server_ip]) - } - if .cisco_meraki.event_subtype == "wpa_deauth" { - if .cisco_meraki.wpa_deauth.client_mac != null { - if .client.mac == null { - .client.mac = del(.cisco_meraki.wpa_deauth.client_mac) - } else { - assert!(false, ".client.mac must be null or override: true") - } - } else { - assert!(false, ".cisco_meraki.wpa_deauth.client_mac must not be null") - } - } - if .cisco_meraki.event_subtype == "client_vpn_connect" { - if .event.original != null { - tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} events client_vpn_connect user id \'%{DATA:user.name}\' local ip %{DATA:network.forwarded_ip} connected from %{DATA:_temp.client_ip}$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "failed to dissect .event.original: " + to_string(err)) - } - } else { - err = ".event.original must not be nullish" - assert!(false, err) - } - } - if .cisco_meraki.event_subtype == "client_vpn_connect" { - if !is_nullish(.event.original) { - value, err = parse_groks(value: .event.original, patterns: [s'.*?events client_vpn_connect %{GREEDYDATA:message}$.*']) - if err == null { - . = merge(., value, deep: true) - } else { - assert!(false, "failed to grok .event.original:" + to_string(err)) - } - } else { - assert!(false, ".event.original must not be nullish") - } - } - if ._temp.client_ip != null { - if !is_nullish(._temp.client_ip) { - value, err = parse_groks(value: ._temp.client_ip, patterns: [s'.*?^%{IPV4:_temp.client_ip}$.*', s'.*?^%{IPV6:_temp.client_ip}$.*']) - if err == null { - . = merge(., value, deep: true) - } - } - } - if ._temp.client_ip != null { - if !is_nullish(._temp.client_ip) { - if is_ipv4(string!(._temp.client_ip)) || is_ipv6(string!(._temp.client_ip)) { - .client.ip = ._temp.client_ip - } - } - } - if !is_nullish(.client.mac) { - if is_array(.client.mac) { - .client.mac = map_values(array!(.client.mac)) -> |value| { - replace!(value, r'[:.]', "-") - } - } else { - if false { - .client.mac = null - } - target, err = replace(.client.mac, r'[:.]', "-") - if err == null { - .client.mac = target - } else { - assert!(false, to_string(err)) - } - } - } - if !is_nullish(.client.mac) { - .client.mac = upcase!(.client.mac) - } - if !is_nullish(.server.mac) { - if is_array(.server.mac) { - .server.mac = map_values(array!(.server.mac)) -> |value| { - replace!(value, r'[:.]', "-") - } - } else { - if false { - .server.mac = null - } - target, err = replace(.server.mac, r'[:.]', "-") - if err == null { - .server.mac = target - } else { - assert!(false, to_string(err)) - } - } - } - if !is_nullish(.server.mac) { - .server.mac = upcase!(.server.mac) - } - } - if .cisco_meraki.event_type == "urls" { - _ingest.timestamp = now() - if _ingest.pipeline == null { - _ingest.pipeline = "" - _ingest.on_failure_pipeline = "" - } - if .event.original != null { - tmp, err = parse_groks(value: .event.original, patterns: [s'^%{DATA} urls src=%{DATA:src} dst=%{DATA:dst} mac=%{DATA:mac} request: %{DATA:http.request.method} %{DATA:url.original}$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "failed to dissect .event.original: " + to_string(err)) - } - } else { - err = ".event.original must not be nullish" - assert!(false, err) - } - if .mac != null { - tmp, err = parse_groks(value: .mac, patterns: [s'^%{DATA:mac} agent=%{DATA:user_agent.original}$']) - if err == null { - . = merge(., tmp, deep: true) - } - } - if !is_nullish(.src) { - value, err = parse_groks(value: .src, patterns: [s'.*?^%{IPV4:_temp.src_ip}:%{PORT:sport}$.*', s'.*?^\[%{IPV6:_temp.src_ip}\]:%{PORT:sport}$.*', s'.*?^%{IPV6NOCOMPRESS:_temp.src_ip}:%{PORT:sport}$.*', s'.*?^%{IPV6:_temp.src_ip}%{IPV6PORTSEP}%{PORT:sport}$.*'], aliases: { "IPV6NOCOMPRESS": s'([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}', "IPV6PORTSEP": s'(?: port |[p#.])', "PORT": s'[0-9]+' }) - if err == null { - . = merge(., value, deep: true) - } else { - assert!(false, "failed to grok .src:" + to_string(err)) - } - } else { - assert!(false, ".src must not be nullish") - } - if !is_nullish(._temp.src_ip) { - if is_ipv4(string!(._temp.src_ip)) || is_ipv6(string!(._temp.src_ip)) { - .source.ip = ._temp.src_ip - } else { - err = "Failed converting ._temp.src_ip to an IP address" - assert!(false, err) - } - } else { - err = "._temp.src_ip must be present" - assert!(false, err) - } - if !is_nullish(.sport) { - if is_string(.sport) { - temp = .sport - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .source.port = temp - } - } else { - if is_integer(.sport) { - .source.port = .sport - } - } - } - if !is_nullish(.dst) { - value, err = parse_groks(value: .dst, patterns: [s'.*?^%{IPV4:_temp.dst_ip}:%{PORT:dport}$.*', s'.*?^\[%{IPV6:_temp.dst_ip}\]:%{PORT:dport}$.*', s'.*?^%{IPV6NOCOMPRESS:_temp.dst_ip}:%{PORT:dport}$.*', s'.*?^%{IPV6:_temp.dst_ip}%{IPV6PORTSEP}%{PORT:dport}$.*'], aliases: { "IPV6NOCOMPRESS": s'([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4}', "IPV6PORTSEP": s'(?: port |[p#.])', "PORT": s'[0-9]+' }) - if err == null { - . = merge(., value, deep: true) - } else { - assert!(false, "failed to grok .dst:" + to_string(err)) - } - } else { - assert!(false, ".dst must not be nullish") - } - if !is_nullish(._temp.dst_ip) { - if is_ipv4(string!(._temp.dst_ip)) || is_ipv6(string!(._temp.dst_ip)) { - .destination.ip = ._temp.dst_ip - } - } - if .dport != "0" && .cisco_meraki.event_subtype != "security_filtering_disposition_change" { - if !is_nullish(.dport) { - if is_string(.dport) { - temp = .dport - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .destination.port = temp - } - } else { - if is_integer(.dport) { - .destination.port = .dport - } - } - } - } - if !is_nullish(.mac) { - if is_array(.mac) { - .cisco_meraki.urls.mac = map_values(array!(.mac)) -> |value| { - replace!(value, r'[-:.]', "-") - } - } else { - if false { - .mac = null - } - target, err = replace(.mac, r'[-:.]', "-") - if err == null { - .cisco_meraki.urls.mac = target - } else { - assert!(false, to_string(err)) - } - } - } else { - assert!(false, ".mac must not be nullish") - } - if downcase(string!(.http.request.method)) != "unknown" { - .cisco_meraki.event_subtype = "http_access" - } - if downcase(string!(.http.request.method)) == "unknown" { - .cisco_meraki.event_subtype = "http_access_error" - } - if is_string(.user_agent.original) { - user_agent = parse_user_agent!(.user_agent.original) - .user_agent.version = user_agent.browser.version - .user_agent.os.name = user_agent.os.family - .user_agent.os.version = user_agent.os.version - .user_agent.os.full, _ = join([user_agent.os.family, user_agent.os.version], separator: " ") - .user_agent.name = user_agent.browser.family - if is_nullish(.user_agent.name) { - .user_agent.name = "Other" - } - .user_agent.device.name = user_agent.device.category - if is_nullish(.user_agent.device.name) { - .user_agent.device.name = "Other" - } - .user_agent = compact(.user_agent, recursive: true) - if .user_agent.original == null { - .user_agent.original = "" - } else { - .user_agent.original = .user_agent.original - } - } - if .url.original != null && .url.original != "" { - uri, err = parse_url(.url.original, default_known_ports: true) - if err == null { - if uri.username == "" { - del(uri.username) - } - if uri.password == "" { - del(uri.password) - } - uri.query = encode_key_value!(uri.query, field_delimiter: "&", key_value_delimiter: "=") - if uri.query == "" { - del(uri.query) - } - if uri.path != null { - uri.path = decode_percent!(uri.path) - } - uri.original = .url.original - uri.extension = parse_regex(uri.path, r'.*\.(?P[a-zA-Z0-9_\-]+)$').extension ?? null - uri.domain = del(uri.host) - if uri.username != null && uri.password != null { - uri.user_info = join!([uri.username, uri.password], separator: ":") - } - .url = compact(uri) - } else { - assert!(false, to_string(err)) - } - } - if .url.domain != null { - tmp, err = parse_etld(.url.domain, plus_parts: 1) - if err == null { - .url.registered_domain = tmp.etld_plus - .url.top_level_domain = tmp.etld - .url.domain = .url.domain - .url.subdomain = { - subdomain = replace!(.url.domain, tmp.etld_plus, "") - if subdomain != "" { - subdomain = slice!(subdomain, 0, -1) - } - subdomain - } - .url = compact(.url) - } else { - assert!(false, to_string(err)) - } - } - } - if !is_array(.event.category) { - if .event.category == null { - .event.category = [] - } else { - .event.category = [.event.category] - } - } - .event.category = append!({ - expr = .event.category - if false { - expr = null - } - expr - }, ["network"]) - if !is_array(.event.type) { - if .event.type == null { - .event.type = [] - } else { - .event.type = [.event.type] - } - } - .event.type = append!({ - expr = .event.type - if false { - expr = null - } - expr - }, ["info"]) - if .cisco_meraki.event_subtype != null { - params = { "eventmap": { "8021x_deauth": { "action": s'wifi-8021x-failed-auth-or-deauth', "category": ["authentication"], "type": ["end", "denied"] }, "8021x_eap_failure": { "action": s'wifi-8021x-failed-authentication-attempt', "category": ["authentication"], "type": ["end", "denied"] }, "8021x_eap_success": { "action": s'wifi-8021x-auth', "category": ["authentication"], "type": ["start"] }, "Site-to-Site VPN": { "action": s'site-to-site-vpn', "type": ["access"] }, "aps_association_reject": { "action": s'association-rejected-for-load-balancing' }, "association": { "action": s'wifi-association-request', "type": ["access", "connection"] }, "client_vpn_connect": { "action": s'site-to-site-vpn', "category": ["session"], "type": ["access", "allowed", "start"] }, "device_packet_flood": { "action": s'wireless-packet-flood-detected' }, "dfs_event": { "action": s'dynamic-frequency-selection-detected' }, "dhcp_no_offer": { "action": s'dhcp-no-offer', "type": ["access", "denied"] }, "dhcp_offer": { "action": s'dhcp-offer', "type": ["access", "allowed"] }, "disassociation": { "action": s'wifi-disassociation-request', "category": ["session"], "type": ["access", "end"] }, "flow_allowed": { "action": s'layer3-firewall-allowed-flow', "type": ["connection", "start"] }, "flow_denied": { "action": s'layer3-firewall-denied-flow', "type": ["access", "denied"] }, "http_access": { "action": s'http-access', "category": ["web"], "type": ["access"] }, "http_access_error": { "action": s'http-access-error', "category": ["web"], "type": ["error"] }, "ids_alerted": { "action": s'ids-signature-matched', "category": ["intrusion_detection"] }, "ip_session_initiated": { "action": s'ip-session-initiated', "type": ["access", "start"] }, "multiple_dhcp_servers_detected": { "action": ["multiple_dhcp_servers_detected"], "type": ["protocol"] }, "rogue_ssid_detected": { "action": s'rogue-ssid-detected' }, "security_filtering_disposition_change": { "action": s'issued-retrospective-malicious-disposition', "category": ["file", "malware"] }, "security_filtering_file_scanned": { "action": s'malicious-file-actioned', "category": ["file", "malware"] }, "splash_auth": { "action": s'splash-authentication', "category": ["authentication"], "type": ["start"] }, "ssid_spoofing_detected": { "action": s'ssid-spoofing-detected' }, "vpn_connectivity_change": { "action": s'vpn-connectivity-change', "category": ["session"], "type": ["connection"] }, "wpa_auth": { "action": s'wifi-wpa-authentication', "category": ["authentication"], "type": ["start", "access"] }, "wpa_deauth": { "action": s'wifi-wpa-failed-auth-or-deauth', "category": ["authentication"], "type": ["end", "denied"] } } } - eventMap = get!(params, ["eventmap"]) - eventData = get!(eventMap, [.cisco_meraki.event_subtype]) - shouldSkip_1 = false - return_1 = if eventData == null { - .event.action = .cisco_meraki.event_subtype - shouldSkip_1 = true - null - } - if !(shouldSkip_1 || false) { - eventCategory = get!(eventData, ["category"]) - eventType = get!(eventData, ["type"]) - eventAction = get!(eventData, ["action"]) - if eventType != null { - for_each({ - arr = eventType - if false { - arr = null - } - array!(arr) - }) -> |_index, t| { - if false { - .event.type = null - } - .event.type = push!(.event.type, t) - } - } - if eventCategory != null { - for_each({ - arr = eventCategory - if false { - arr = null - } - array!(arr) - }) -> |_index, c| { - if false { - .event.category = null - } - .event.category = push!(.event.category, c) - } - } - if eventAction != null { - .event.action = eventAction - } - } else { - return_1 - } - } - if .source.as.asn != null { - if .source.as.number == null { - .source.as.number = del(.source.as.asn) - } else { - assert!(false, ".source.as.number must be null or override: true") - } - } - if .source.as.organization_name != null { - if .source.as.organization.name == null { - .source.as.organization.name = del(.source.as.organization_name) - } else { - assert!(false, ".source.as.organization.name must be null or override: true") - } - } - if .destination.as.asn != null { - if .destination.as.number == null { - .destination.as.number = del(.destination.as.asn) - } else { - assert!(false, ".destination.as.number must be null or override: true") - } - } - if .destination.as.organization_name != null { - if .destination.as.organization.name == null { - .destination.as.organization.name = del(.destination.as.organization_name) - } else { - assert!(false, ".destination.as.organization.name must be null or override: true") - } - } - if .client.as.asn != null { - if .client.as.number == null { - .client.as.number = del(.client.as.asn) - } else { - assert!(false, ".client.as.number must be null or override: true") - } - } - if .client.as.organization_name != null { - if .client.as.organization.name == null { - .client.as.organization.name = del(.client.as.organization_name) - } else { - assert!(false, ".client.as.organization.name must be null or override: true") - } - } - del(._temp) - del(._conf) - del(.sport) - del(.dport) - del(.mac) - del(.src) - del(.dst) - del(.translated_src_ip) - del(.translated_dst_ip) - del(.translated_port) - del(.wired_mac) - del(.rssi) - del(.protocol) - del(.dhost) - del(.client_mac) - del(.radio) - del(.sts) - del(.msgtype) - del(.timestamp) - . = compact(.) - if .tags == null || !({ - expr = .tags - value = "preserve_original_event" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - }) { - del(.event.original) - } - } else if _module == "cisco_ios_logs" { - _ingest.pipeline = "cisco_ios_logs" - _ingest.on_failure_pipeline = "cisco_ios_logs" - .tags = ["preserve_original_event"] - .ecs.version = "8.11.0" - .event.category = ["network"] - .event.provider = "firewall" - .observer.vendor = "Cisco" - .observer.product = "IOS" - .observer.type = "firewall" - .event.type = ["info"] - if is_nullish(.event.original) { - .event.original = .message - } - del(.message) - if !is_nullish(.event.original) { - value, err = parse_groks(value: .event.original, patterns: [s'.*?^(:?%{CISCO_PRIORITY_MSGCOUNT})?%{SYSLOGTIMESTAMP} %{IP} %{CISCO_HOSTNAME:log.syslog.hostname}: (?:%{NUMBER:cisco.ios.sequence}: )?(?:%{CISCO_UPTIME:cisco.ios.uptime}|%{CISCO_TIMESTAMP}): %{GREEDYDATA:_temp_.message}$.*', s'.*?^(:?%{CISCO_PRIORITY_MSGCOUNT})?%{SYSLOGTIMESTAMP} (?:%{IP}|%{CISCO_HOSTNAME:log.syslog.hostname}) %{NUMBER:cisco.ios.sequence}: (?:%{CISCO_UPTIME:cisco.ios.uptime}|%{CISCO_TIMESTAMP}): %{GREEDYDATA:_temp_.message}$.*', s'.*?^(:?%{CISCO_PRIORITY_MSGCOUNT})?(?:(?:%{CISCO_HOSTNAME:log.syslog.hostname}|%{IP})[:]? )?(?:%{NUMBER:cisco.ios.sequence}: )?(?:%{CISCO_UPTIME:cisco.ios.uptime}|%{CISCO_TIMESTAMP}): %{GREEDYDATA:_temp_.message}$.*'], aliases: { "CISCO_HOSTNAME": s'[a-zA-Z][0-9a-zA-Z_-]{0,61}[0-9a-zA-Z]?', "CISCO_PRIORITY_MSGCOUNT": s'<%{NONNEGINT:log.syslog.priority}>(?:%{NONNEGINT:cisco.ios.message_count})?(?:: )?', "CISCO_TIMESTAMP": s'[*]?%{CISCOTIMESTAMP:_temp_.cisco_timestamp}(?: %{CISCO_TZ:_temp_.tz})?', "CISCO_TZ": s'[a-zA-Z]{1,4}', "CISCO_UPTIME": s'[0-9a-zA-Z]+' }) - if err == null { - if value.log.syslog.priority != null { - value.log.syslog.priority = parse_int!(value.log.syslog.priority) - } - . = merge(., value, deep: true) - } else { - assert!(false, "grok_header: failed to grok .event.original:" + to_string(err)) - } - } else { - assert!(false, ".event.original must not be nullish") - } - if !is_nullish(._temp_.message) { - value, err = parse_groks(value: ._temp_.message, patterns: [s'.*?^%%{GREEDYDATA:message}$.*', s'.*?^%{GREEDYDATA:_temp_.generic_message}$.*']) - if err == null { - . = merge(., value, deep: true) - } else { - assert!(false, "grok_message: failed to grok ._temp_.message:" + to_string(err)) - } - } else { - assert!(false, "._temp_.message must not be nullish") - } - if .cisco.ios.sequence != null { - .event.sequence = .cisco.ios.sequence - } - if .cisco.ios.message_count != null { - if !is_nullish(.cisco.ios.message_count) { - if is_string(.cisco.ios.message_count) { - temp = .cisco.ios.message_count - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .cisco.ios.message_count = temp - } else { - err_msg = join!(["Failed converting .cisco.ios.message_count to an integer. Found (", to_string!(.cisco.ios.message_count), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.cisco.ios.message_count) { - .cisco.ios.message_count = .cisco.ios.message_count - } else { - err = "Failed converting .cisco.ios.message_count to an integer" - assert!(false, err) - } - } - } else { - err = ".cisco.ios.message_count must be present" - assert!(false, err) - } - } - if .cisco.ios.message_count != null && .event.sequence == null { - .event.sequence = .cisco.ios.message_count - } - if !is_nullish(._temp_.cisco_timestamp) { - if is_array(._temp_.cisco_timestamp) { - ._temp_.cisco_timestamp = map_values(array!(._temp_.cisco_timestamp)) -> |value| { - replace!(value, r' {2,}', " ") - } - } else { - if false { - ._temp_.cisco_timestamp = null - } - target, err = replace(._temp_.cisco_timestamp, r' {2,}', " ") - if err == null { - ._temp_.cisco_timestamp = target - } else { - assert!(false, to_string(err)) - } - } - } - shouldSkip_1 = false - return_1 = if ._temp_.tz != null && ._conf.tz_map != null { - for_each({ - arr = ._conf.tz_map - if false { - arr = null - } - array!(arr) - }) -> |_index, item| { - if item.tz_short == ._temp_.tz { - .event.timezone = item.tz_long - shouldSkip_1 = true - null - } - } - } - if !(shouldSkip_1 || false) { - if ._conf.tz_offset != null { - .event.timezone = ._conf.tz_offset - } - if .event.timezone == null { - .event.timezone = "UTC" - } - } else { - return_1 - } - if ._temp_.cisco_timestamp != null { - date = null - err = "no datetime formats provided" - if err != null { - datetime = to_string!(._temp_.cisco_timestamp) - data, err = parse_regex(value: datetime, pattern: r'(?[A-Z]{3,4}+)$') - records = find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": data.abbreviation }) - if length(records) == 1 { - offset = records[0].offset - datetime = replace!(datetime, data.abbreviation, offset) - date, err = parse_timestamp(value: join!(value: [datetime], separator: " "), format: join!(value: ["%b %d %Y %H:%M:%S.%3f ", "%:z"], separator: " ")) - } else { - if length(records) > 1 { - err = "ambiguous timezone abbreviation" - } else { - if length(records) == 0 { - err = "no timezone abbreviation found" - } - } - } - } - if err != null { - datetime = to_string!(._temp_.cisco_timestamp) - external_timezone = .event.timezone - if false { - external_timezone = null - } - records = if match!(external_timezone, r'[\+-]?[0-9]{1,2}:?[0-9]{2}') { - [{ "offset": external_timezone }] - } else if contains(to_string(external_timezone) ?? "", "/") { - [{ "tz_name": external_timezone }] - } else { - find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": external_timezone }) - } - if length(records) == 1 { - if records[0].tz_name != null { - date, err = parse_timestamp(value: join!(value: [datetime], separator: " "), format: join!(value: ["%b %d %Y %H:%M:%S.%3f"], separator: " "), timezone: string!(records[0].tz_name)) - } else { - offset = records[0].offset - date, err = parse_timestamp(value: join!(value: [datetime, offset], separator: " "), format: join!(value: ["%b %d %Y %H:%M:%S.%3f", "%:z"], separator: " ")) - } - } else { - if length(records) > 1 { - err = "ambiguous timezone abbreviation" - } else { - if length(records) == 0 { - err = "no timezone abbreviation found" - } - } - } - } - if err != null { - datetime = to_string!(._temp_.cisco_timestamp) - data, err = parse_regex(value: datetime, pattern: r'(?[A-Z]{3,4}+)$') - records = find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": data.abbreviation }) - if length(records) == 1 { - offset = records[0].offset - datetime = replace!(datetime, data.abbreviation, offset) - date, err = parse_timestamp(value: join!(value: [datetime], separator: " "), format: join!(value: ["%b %d %Y %H:%M:%S ", "%:z"], separator: " ")) - } else { - if length(records) > 1 { - err = "ambiguous timezone abbreviation" - } else { - if length(records) == 0 { - err = "no timezone abbreviation found" - } - } - } - } - if err != null { - datetime = to_string!(._temp_.cisco_timestamp) - external_timezone = .event.timezone - if false { - external_timezone = null - } - records = if match!(external_timezone, r'[\+-]?[0-9]{1,2}:?[0-9]{2}') { - [{ "offset": external_timezone }] - } else if contains(to_string(external_timezone) ?? "", "/") { - [{ "tz_name": external_timezone }] - } else { - find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": external_timezone }) - } - if length(records) == 1 { - if records[0].tz_name != null { - date, err = parse_timestamp(value: join!(value: [datetime], separator: " "), format: join!(value: ["%b %d %Y %H:%M:%S"], separator: " "), timezone: string!(records[0].tz_name)) - } else { - offset = records[0].offset - date, err = parse_timestamp(value: join!(value: [datetime, offset], separator: " "), format: join!(value: ["%b %d %Y %H:%M:%S", "%:z"], separator: " ")) - } - } else { - if length(records) > 1 { - err = "ambiguous timezone abbreviation" - } else { - if length(records) == 0 { - err = "no timezone abbreviation found" - } - } - } - } - if err != null { - datetime = to_string!(._temp_.cisco_timestamp) - data, err = parse_regex(value: datetime, pattern: r'(?[A-Z]{3,4}+)$') - records = find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": data.abbreviation }) - if length(records) == 1 { - offset = records[0].offset - datetime = replace!(datetime, data.abbreviation, offset) - date, err = parse_timestamp(value: join!(value: [datetime, format_timestamp!(now(), format: "%Y")], separator: " "), format: join!(value: ["%b %d %H:%M:%S.%3f ", "%:z", "%Y"], separator: " ")) - } else { - if length(records) > 1 { - err = "ambiguous timezone abbreviation" - } else { - if length(records) == 0 { - err = "no timezone abbreviation found" - } - } - } - } - if err != null { - datetime = to_string!(._temp_.cisco_timestamp) - external_timezone = .event.timezone - if false { - external_timezone = null - } - records = if match!(external_timezone, r'[\+-]?[0-9]{1,2}:?[0-9]{2}') { - [{ "offset": external_timezone }] - } else if contains(to_string(external_timezone) ?? "", "/") { - [{ "tz_name": external_timezone }] - } else { - find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": external_timezone }) - } - if length(records) == 1 { - if records[0].tz_name != null { - date, err = parse_timestamp(value: join!(value: [datetime, format_timestamp!(now(), format: "%Y")], separator: " "), format: join!(value: ["%b %d %H:%M:%S.%3f", "%Y"], separator: " "), timezone: string!(records[0].tz_name)) - } else { - offset = records[0].offset - date, err = parse_timestamp(value: join!(value: [datetime, offset, format_timestamp!(now(), format: "%Y")], separator: " "), format: join!(value: ["%b %d %H:%M:%S.%3f", "%:z", "%Y"], separator: " ")) - } - } else { - if length(records) > 1 { - err = "ambiguous timezone abbreviation" - } else { - if length(records) == 0 { - err = "no timezone abbreviation found" - } - } - } - } - if err != null { - datetime = to_string!(._temp_.cisco_timestamp) - data, err = parse_regex(value: datetime, pattern: r'(?[A-Z]{3,4}+)$') - records = find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": data.abbreviation }) - if length(records) == 1 { - offset = records[0].offset - datetime = replace!(datetime, data.abbreviation, offset) - date, err = parse_timestamp(value: join!(value: [datetime, format_timestamp!(now(), format: "%Y")], separator: " "), format: join!(value: ["%b %d %H:%M:%S ", "%:z", "%Y"], separator: " ")) - } else { - if length(records) > 1 { - err = "ambiguous timezone abbreviation" - } else { - if length(records) == 0 { - err = "no timezone abbreviation found" - } - } - } - } - if err != null { - datetime = to_string!(._temp_.cisco_timestamp) - external_timezone = .event.timezone - if false { - external_timezone = null - } - records = if match!(external_timezone, r'[\+-]?[0-9]{1,2}:?[0-9]{2}') { - [{ "offset": external_timezone }] - } else if contains(to_string(external_timezone) ?? "", "/") { - [{ "tz_name": external_timezone }] - } else { - find_enrichment_table_records!(table: "timezones", condition: { "abbreviation": external_timezone }) - } - if length(records) == 1 { - if records[0].tz_name != null { - date, err = parse_timestamp(value: join!(value: [datetime, format_timestamp!(now(), format: "%Y")], separator: " "), format: join!(value: ["%b %d %H:%M:%S", "%Y"], separator: " "), timezone: string!(records[0].tz_name)) - } else { - offset = records[0].offset - date, err = parse_timestamp(value: join!(value: [datetime, offset, format_timestamp!(now(), format: "%Y")], separator: " "), format: join!(value: ["%b %d %H:%M:%S", "%:z", "%Y"], separator: " ")) - } - } else { - if length(records) > 1 { - err = "ambiguous timezone abbreviation" - } else { - if length(records) == 0 { - err = "no timezone abbreviation found" - } - } - } - } - if err == null { - .@timestamp = date - } else { - datetime = to_string!(._temp_.cisco_timestamp) - assert!(false, "failed to parse timestamp ._temp_.cisco_timestamp: ({{ datetime }}): " + to_string(err)) - } - } - if !is_nullish(.message) { - value, err = parse_groks(value: .message, patterns: [s'.*?%{DATA:cisco.ios.facility}-%{POSINT:event.severity}-%{DATA:event.code}:\s+(\w+\d+(/\d+)?\:\s+)?([a-zA-Z0-9_]+\:\s+)?%{GREEDYDATA:message}.*']) - if err == null { - . = merge(., value, deep: true) - } else { - assert!(false, "grok_message: failed to grok .message:" + to_string(err)) - } - } - if .cisco.ios.facility == "IOSXE" && .event.code == "PLATFORM" { - if !is_nullish(.message) { - value, err = parse_groks(value: .message, patterns: [s'.*?%%{DATA:cisco.ios.facility}-%{POSINT:event.severity}-%{DATA:event.code}:\s+(\w+\d+(/\d+)?\:\s+)?([a-zA-Z0-9_]+\:\s+)?%{GREEDYDATA:message}.*']) - if err == null { - . = merge(., value, deep: true) - } else { - assert!(false, "grok_child_message: failed to grok .message:" + to_string(err)) - } - } - } - if .cisco.ios.facility == "FW" && .event.code == "SESS_AUDIT_TRAIL" { - if !is_nullish(.message) { - value, err = parse_groks(value: .message, patterns: [s'.*?initiator \(%{IP:source.ip}:%{NUMBER:source.port}\) sent %{NUMBER:source.bytes} bytes -- responder \(%{IP:destination.ip}:%{NUMBER:destination.port}\) sent %{NUMBER:destination.bytes} bytes, from %{NOTSPACE:cisco.ios.interface.name}.*']) - if err == null { - if value.destination.bytes != null { - value.destination.bytes = parse_int!(value.destination.bytes) - } - if value.destination.port != null { - value.destination.port = parse_int!(value.destination.port) - } - if value.source.bytes != null { - value.source.bytes = parse_int!(value.source.bytes) - } - if value.source.port != null { - value.source.port = parse_int!(value.source.port) - } - . = merge(., value, deep: true) - } else { - assert!(false, "grok_audit_details: failed to grok .message:" + to_string(err)) - } - } - } - if .cisco.ios.facility == "FW" && .event.code == "DROP_PKT" { - if !is_nullish(.message) { - value, err = parse_groks(value: .message, patterns: [s'.*?^Dropping %{WORD} %{WORD} from %{NOTSPACE:cisco.ios.interface.name} %{IP:source.ip}:%{NUMBER:source.port} ?=> ?%{IP:destination.ip}:%{NUMBER:destination.port}.*']) - if err == null { - if value.destination.port != null { - value.destination.port = parse_int!(value.destination.port) - } - if value.source.port != null { - value.source.port = parse_int!(value.source.port) - } - . = merge(., value, deep: true) - } else { - assert!(false, "grok_drop_details: failed to grok .message:" + to_string(err)) - } - } - } - if ._temp_.generic_message != null { - if ._temp_.generic_message != null { - if .message == null { - .message = del(._temp_.generic_message) - } else { - assert!(false, ".message must be null or override: true") - } - } else { - assert!(false, "._temp_.generic_message must not be null") - } - } - if !is_nullish(.event.severity) { - if is_string(.event.severity) { - temp = .event.severity - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .event.severity = temp - } else { - err_msg = join!(["Failed converting .event.severity to an integer. Found (", to_string!(.event.severity), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.event.severity) { - .event.severity = .event.severity - } else { - err = "Failed converting .event.severity to an integer" - assert!(false, err) - } - } - } - if !is_nullish(.event.sequence) { - if is_string(.event.sequence) { - temp = .event.sequence - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .event.sequence = temp - } else { - err_msg = join!(["Failed converting .event.sequence to an integer. Found (", to_string!(.event.sequence), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.event.sequence) { - .event.sequence = .event.sequence - } else { - err = "Failed converting .event.sequence to an integer" - assert!(false, err) - } - } - } - if { - expr = ["IPACCESSLOGP", "ACCESSLOGP"] - value = .event.code - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - if .message != null { - tmp, err = parse_groks(value: .message, patterns: [s'^list %{DATA:cisco.ios.access_list} %{DATA:_temp_.event.action} %{DATA:network.transport} %{DATA:source.address}\(%{DATA:source.port}\) %{DATA} %{DATA:destination.address}\(%{DATA:destination.port}\), %{DATA:source.packets} packet.*?$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "dissect_gp: failed to dissect .message: " + to_string(err)) - } - } else { - err = ".message must not be nullish" - assert!(false, err) - } - } - if { - expr = ["IPACCESSLOGDP", "ACCESSLOGDP"] - value = .event.code - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - if .message != null { - tmp, err = parse_groks(value: .message, patterns: [s'^list %{DATA:cisco.ios.access_list} %{DATA:_temp_.event.action} %{DATA:network.transport} %{DATA:source.address} %{DATA} %{DATA:destination.address} \(%{DATA:icmp.type}/%{DATA:icmp.code}\), %{DATA:source.packets} packet.*?$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "dissect_gdp: failed to dissect .message: " + to_string(err)) - } - } else { - err = ".message must not be nullish" - assert!(false, err) - } - } - if .event.code == "IPACCESSLOGRP" { - if .message != null { - tmp, err = parse_groks(value: .message, patterns: [s'^list %{DATA:cisco.ios.access_list} %{DATA:_temp_.event.action} %{DATA:network.transport} %{DATA:source.address} %{DATA} %{DATA:destination.address}, %{DATA:source.packets} packet.*?$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "dissect_grp: failed to dissect .message: " + to_string(err)) - } - } else { - err = ".message must not be nullish" - assert!(false, err) - } - } - if .event.code == "IPACCESSLOGSP" { - if .message != null { - tmp, err = parse_groks(value: .message, patterns: [s'^list %{DATA:cisco.ios.access_list} %{DATA:_temp_.event.action} %{DATA:network.transport} %{DATA:source.address} %{DATA} %{DATA:destination.address} \(%{DATA:igmp.type}\), %{DATA:source.packets} packet.*?$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "dissect_gsp: failed to dissect .message: " + to_string(err)) - } - } else { - err = ".message must not be nullish" - assert!(false, err) - } - } - if .event.code == "ACCESSLOGSP" { - if .message != null { - tmp, err = parse_groks(value: .message, patterns: [s'^list %{DATA:cisco.ios.access_list} %{DATA:_temp_.event.action} %{DATA:network.type} %{DATA:source.address} %{DATA} %{DATA:destination.address}, %{DATA:source.packets} packet.*?$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "dissect_sp: failed to dissect .message: " + to_string(err)) - } - } else { - err = ".message must not be nullish" - assert!(false, err) - } - } - if { - expr = ["IPACCESSLOGNP", "ACCESSLOGNP"] - value = .event.code - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - if .message != null { - tmp, err = parse_groks(value: .message, patterns: [s'^list %{DATA:cisco.ios.access_list} %{DATA:_temp_.event.action} %{DATA:network.iana_number} %{DATA:source.address} %{DATA} %{DATA:destination.address}, %{DATA:source.packets} packet.*?$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "dissect_gnp: failed to dissect .message: " + to_string(err)) - } - } else { - err = ".message must not be nullish" - assert!(false, err) - } - } - if .event.code == "LOGIN_SUCCESS" { - if .message != null { - tmp, err = parse_groks(value: .message, patterns: [s'^%{DATA:cisco.ios.action} %{DATA:_temp_.event.action} \[user: %{DATA:source.user.name}\] \[Source: %{DATA:source.address}\] \[localport: %{DATA:destination.port}\] at %{DATA}$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "dissect_login: failed to dissect .message: " + to_string(err)) - } - } else { - err = ".message must not be nullish" - assert!(false, err) - } - } - if .event.code == "LOGOUT" { - if .message != null { - tmp, err = parse_groks(value: .message, patterns: [s'^User %{DATA:source.user.name} has %{DATA:cisco.ios.action} %{DATA:cisco.ios.session.type} session %{DATA:cisco.ios.session.number}\(%{DATA:source.address}\).*?$']) - if err == null { - . = merge(., tmp, deep: true) - } else { - assert!(false, "dissect_logout: failed to dissect .message: " + to_string(err)) - } - } else { - err = ".message must not be nullish" - assert!(false, err) - } - } - if .event.code == "BADAUTH" { - if !is_nullish(.message) { - value, err = parse_groks(value: .message, patterns: [s'.*?^(?:No|Invalid) MD5 digest from %{DATA:source.address}(\(%{INT:source.port}\)|\:%{INT:source.port}) to %{DATA:destination.address}(\(%{INT:destination.port}\)|\:%{INT:destination.port})(?:(?: \(RST\))? (?:tableid - %{DATA:cisco.ios.tableid}|%{GREEDYDATA:_temp_.rst}))?$.*']) - if err == null { - . = merge(., value, deep: true) - } else { - assert!(false, "grok_badauth: failed to grok .message:" + to_string(err)) - } - } - } - if .event.code == "INVALID_RP_JOIN" { - if !is_nullish(.message) { - value, err = parse_groks(value: .message, patterns: [s'.*?Received \(%{PIM_SOURCE}, %{DATA:cisco.ios.pim.group.ip}\) %{WORD:cisco.ios.action} from %{IP:source.address} for %{DATA:cisco.ios.outcome} %{IP:destination.address}.*'], aliases: { "PIM_SOURCE": s'(%{IP:cisco.ios.pim.source.ip}|%{DATA})' }) - if err == null { - . = merge(., value, deep: true) - } else { - assert!(false, "grok_message_rp_join: failed to grok .message:" + to_string(err)) - } - } else { - assert!(false, ".message must not be nullish") - } - } - if .event.code == "INVALID_RP_JOIN" { - .event.action = "multicast-join" - } - if .event.code == "INVALID_RP_JOIN" { - .event.outcome = "failure" - } - if .event.code == "INVALID_RP_JOIN" { - .event.reason = "Invalid RP" - } - if !is_nullish(.destination.address) { - if is_ipv4(string!(.destination.address)) || is_ipv6(string!(.destination.address)) { - .destination.ip = .destination.address - } else { - err = "Failed converting .destination.address to an IP address" - _ingest.on_failure_processor_type = "on_failure" - _ingest.on_failure_processor_tag = "convert_destination_ip" - _ingest.on_failure_message = to_string(err) - .destination.domain = .destination.address - } - } - if !is_nullish(.source.address) { - if is_ipv4(string!(.source.address)) || is_ipv6(string!(.source.address)) { - .source.ip = .source.address - } else { - err = "Failed converting .source.address to an IP address" - _ingest.on_failure_processor_type = "on_failure" - _ingest.on_failure_processor_tag = "convert_source_ip" - _ingest.on_failure_message = to_string(err) - .source.domain = .source.address - } - } - if !is_nullish(.cisco.ios.pim.source.ip) { - if is_ipv4(string!(.cisco.ios.pim.source.ip)) || is_ipv6(string!(.cisco.ios.pim.source.ip)) { - .cisco.ios.pim.source.ip = .cisco.ios.pim.source.ip - } else { - err = "Failed converting .cisco.ios.pim.source.ip to an IP address" - assert!(false, err) - } - } - if !is_nullish(.source.port) { - if is_string(.source.port) { - temp = .source.port - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .source.port = temp - } else { - err_msg = join!(["Failed converting .source.port to an integer. Found (", to_string!(.source.port), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.source.port) { - .source.port = .source.port - } else { - err = "Failed converting .source.port to an integer" - assert!(false, err) - } - } - } - if !is_nullish(.source.packets) { - if is_string(.source.packets) { - temp = .source.packets - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .source.packets = temp - } else { - err_msg = join!(["Failed converting .source.packets to an integer. Found (", to_string!(.source.packets), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.source.packets) { - .source.packets = .source.packets - } else { - err = "Failed converting .source.packets to an integer" - assert!(false, err) - } - } - } - if !is_nullish(.destination.port) { - if is_string(.destination.port) { - temp = .destination.port - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .destination.port = temp - } else { - err_msg = join!(["Failed converting .destination.port to an integer. Found (", to_string!(.destination.port), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.destination.port) { - .destination.port = .destination.port - } else { - err = "Failed converting .destination.port to an integer" - assert!(false, err) - } - } - } - if .source.bytes != null || .destination.bytes != null { - n = 0 - if .source.bytes != null { - n = int!({ - tmp = n - if false { - tmp = null - } - tmp - }) + int!({ - tmp = .source.bytes - if false { - tmp = null - } - tmp - }) - } - if .destination.bytes != null { - n = int!({ - tmp = n - if false { - tmp = null - } - tmp - }) + int!({ - tmp = .destination.bytes - if false { - tmp = null - } - tmp - }) - } - if .network == null { - .network = { } - } - .network.bytes = n - } - if .source.packets != null { - .network.packets = .source.packets - } - if .source.ip != null && { - expr = .source.ip - value = "." - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - .network.type = "ipv4" - } - if .source.ip != null && .network.type == null { - .network.type = "ipv6" - } - if ._temp_.event.action == "denied" { - .event.action = "deny" - } - if .event.action == "deny" { - if !is_array(.event.type) { - if .event.type == null { - .event.type = [] - } else { - .event.type = [.event.type] - } - } - .event.type = append!({ - expr = .event.type - if false { - expr = null - } - expr - }, ["denied"]) - } - if ._temp_.event.action == "permitted" { - .event.action = "allow" - } - if .event.action == "allow" { - if !is_array(.event.type) { - if .event.type == null { - .event.type = [] - } else { - .event.type = [.event.type] - } - } - .event.type = append!({ - expr = .event.type - if false { - expr = null - } - expr - }, ["allowed"]) - } - if .event.severity == 0 { - .log.level = "emergencies" - } - if .event.severity == 1 { - .log.level = "alert" - } - if .event.severity == 2 { - .log.level = "critical" - } - if .event.severity == 3 { - .log.level = "error" - } - if .event.severity == 4 { - .log.level = "warning" - } - if .event.severity == 5 { - .log.level = "notification" - } - if .event.severity == 6 { - .log.level = "informational" - } - if .event.severity == 7 { - .log.level = "debug" - } - if .source.as.asn != null { - if .source.as.number == null { - .source.as.number = del(.source.as.asn) - } else { - assert!(false, ".source.as.number must be null or override: true") - } - } - if .source.as.organization_name != null { - if .source.as.organization.name == null { - .source.as.organization.name = del(.source.as.organization_name) - } else { - assert!(false, ".source.as.organization.name must be null or override: true") - } - } - if .destination.as.asn != null { - if .destination.as.number == null { - .destination.as.number = del(.destination.as.asn) - } else { - assert!(false, ".destination.as.number must be null or override: true") - } - } - if .destination.as.organization_name != null { - if .destination.as.organization.name == null { - .destination.as.organization.name = del(.destination.as.organization_name) - } else { - assert!(false, ".destination.as.organization.name must be null or override: true") - } - } - if .source.ip != null { - if .related.ip == null { - .related.ip = [.source.ip] - } else { - if is_array(.related.ip) { - if !includes!(.related.ip, .source.ip) { - .related.ip = push!(.related.ip, .source.ip) - } - } else { - if .related.ip != .source.ip { - .related.ip = [.related.ip, .source.ip] - } - } - } - } - if .destination.ip != null { - if .related.ip == null { - .related.ip = [.destination.ip] - } else { - if is_array(.related.ip) { - if !includes!(.related.ip, .destination.ip) { - .related.ip = push!(.related.ip, .destination.ip) - } - } else { - if .related.ip != .destination.ip { - .related.ip = [.related.ip, .destination.ip] - } - } - } - } - if .source.domain != null { - if .related.hosts == null { - .related.hosts = [.source.domain] - } else { - if is_array(.related.hosts) { - if !includes!(.related.hosts, .source.domain) { - .related.hosts = push!(.related.hosts, .source.domain) - } - } else { - if .related.hosts != .source.domain { - .related.hosts = [.related.hosts, .source.domain] - } - } - } - } - if .destination.domain != null { - if .related.hosts == null { - .related.hosts = [.destination.domain] - } else { - if is_array(.related.hosts) { - if !includes!(.related.hosts, .destination.domain) { - .related.hosts = push!(.related.hosts, .destination.domain) - } - } else { - if .related.hosts != .destination.domain { - .related.hosts = [.related.hosts, .destination.domain] - } - } - } - } - if .source.user.name != null { - if .related.user == null { - .related.user = [.source.user.name] - } else { - if is_array(.related.user) { - if !includes!(.related.user, .source.user.name) { - .related.user = push!(.related.user, .source.user.name) - } - } else { - if .related.user != .source.user.name { - .related.user = [.related.user, .source.user.name] - } - } - } - } - iana_numbers = { "icmp": 1, "igmp": 2, "tcp": 6, "udp": 17, "gre": 47, "icmp ipv6": 58, "eigrp": 88, "ospf": 89, "pim": 103, "sctp": 132 } - if !(is_nullish(.source.ip) || is_nullish(.destination.ip) || (is_nullish(.network.iana_number) && is_nullish(.network.transport)) || (.source.ip == "0.0.0.0" && .destination.ip == "0.0.0.0" && .source.port == 0 && .destination.port == 0)) { - iana_number = .network.iana_number - if is_nullish(iana_number) && !is_nullish(.network.transport) { - temp_iana_number, err = get(iana_numbers, [downcase!(.network.transport)]) - if err == null { - iana_number = temp_iana_number - } - } - if iana_number == null { - tmp_iana_number, err = parse_int(.network.transport) - if err == null { - iana_number = tmp_iana_number - } - } - source_port = .source.port - destination_port = .destination.port - if (iana_number == 6 || iana_number == 17) && source_port == null && destination_port == null { - source_port = 0 - destination_port = 0 - } - if (iana_number == 1) { - source_port = .icmp.type - destination_port = .icmp.code - } - if .icmp.type != null && .icmp.code != null { - source_port = to_int!(.icmp.type) - destination_port = to_int!(.icmp.code) - } - community_id = err = null - if source_port != null && destination_port != null { - community_id, err = community_id(source_ip: .source.ip, source_port: source_port, destination_ip: .destination.ip, destination_port: destination_port, protocol: iana_number, seed: 0) - } else { - community_id, err = community_id(source_ip: .source.ip, destination_ip: .destination.ip, protocol: iana_number, seed: 0) - } - if err == null { - .network.community_id = community_id - } - } - del(._temp_) - del(._conf) - if .tags == null || !({ - expr = .tags - value = "preserve_original_event" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - }) { - del(.event.original) - } - } else if _module == "cisco_umbrella" { - _ingest.pipeline = "cisco_umbrella_" + _umbrella_type - _ingest.on_failure_pipeline = _ingest.pipeline - .log.file.path = "/filebeat/cisco_umbrella/" + _umbrella_type - .tags = ["preserve_original_event"] - .ecs.version = "8.11.0" - .observer.vendor = "Cisco" - .observer.product = "Umbrella" - if .event.original == null { - if .message != null { - if .event.original == null { - .event.original = del(.message) - } else { - assert!(false, ".event.original must be null or override: true") - } - } - } - if .log.file.path != null && { - expr = .log.file.path - value = "dnslogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - csv = parse_csv!(replace!(.event.original, r'", "', s'","')) - value = csv[0] - if !is_nullish(value) { - .cisco.umbrella._tmp.time = value - } - value = csv[1] - if !is_nullish(value) { - .cisco.umbrella.identity = value - } - value = csv[2] - if !is_nullish(value) { - .cisco.umbrella.identities = value - } - value = csv[3] - if !is_nullish(value) { - .source.address = value - } - value = csv[4] - if !is_nullish(value) { - .source.nat.ip = value - } - value = csv[5] - if !is_nullish(value) { - .cisco.umbrella.action = value - } - value = csv[6] - if !is_nullish(value) { - .dns.question.type = value - } - value = csv[7] - if !is_nullish(value) { - .dns.response_code = value - } - value = csv[8] - if !is_nullish(value) { - .dns.question.name = value - } - value = csv[9] - if !is_nullish(value) { - .cisco.umbrella.categories = value - } - value = csv[10] - if !is_nullish(value) { - .cisco.umbrella.policy_identity_type = value - } - value = csv[11] - if !is_nullish(value) { - .cisco.umbrella.identity_types = value - } - value = csv[12] - if !is_nullish(value) { - .cisco.umbrella.blocked_categories = value - } - } - if !is_nullish(.dns.question.name) { - if is_array(.dns.question.name) { - .dns.question.name = map_values(array!(.dns.question.name)) -> |value| { - replace!(value, r'\.$', "") - } - } else { - if false { - .dns.question.name = null - } - target, err = replace(.dns.question.name, r'\.$', "") - if err == null { - .dns.question.name = target - } else { - assert!(false, to_string(err)) - } - } - } - if .log.file.path != null && { - expr = .log.file.path - value = "dnslogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - .observer.type = "dns" - } - if .log.file.path != null && { - expr = .log.file.path - value = "iplogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - csv = parse_csv!(replace!(.event.original, r'", "', s'","')) - value = csv[0] - if !is_nullish(value) { - .cisco.umbrella._tmp.time = value - } - value = csv[1] - if !is_nullish(value) { - .cisco.umbrella.identity = value - } - value = csv[2] - if !is_nullish(value) { - .source.address = value - } - value = csv[3] - if !is_nullish(value) { - .source.port = value - } - value = csv[4] - if !is_nullish(value) { - .destination.address = value - } - value = csv[5] - if !is_nullish(value) { - .destination.port = value - } - value = csv[6] - if !is_nullish(value) { - .cisco.umbrella.categories = value - } - } - if .log.file.path != null && { - expr = .log.file.path - value = "iplogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - .observer.type = "firewall" - } - if .log.file.path != null && { - expr = .log.file.path - value = "proxylogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - csv = parse_csv!(replace!(.event.original, r'", "', s'","')) - value = csv[0] - if !is_nullish(value) { - .cisco.umbrella._tmp.time = value - } - value = csv[1] - if !is_nullish(value) { - .cisco.umbrella.identity = value - } - value = csv[2] - if !is_nullish(value) { - .source.address = value - } - value = csv[3] - if !is_nullish(value) { - .source.nat.ip = value - } - value = csv[4] - if !is_nullish(value) { - .destination.address = value - } - value = csv[5] - if !is_nullish(value) { - .http.request.mime_type = value - } - value = csv[6] - if !is_nullish(value) { - .cisco.umbrella.action = value - } - value = csv[7] - if !is_nullish(value) { - .url.original = value - } - value = csv[8] - if !is_nullish(value) { - .http.request.referrer = value - } - value = csv[9] - if !is_nullish(value) { - .user_agent.original = value - } - value = csv[10] - if !is_nullish(value) { - .http.response.status_code = value - } - value = csv[11] - if !is_nullish(value) { - .http.request.bytes = value - } - value = csv[12] - if !is_nullish(value) { - .http.response.bytes = value - } - value = csv[13] - if !is_nullish(value) { - .http.response.body.bytes = value - } - value = csv[14] - if !is_nullish(value) { - .cisco.umbrella.sha_sha256 = value - } - value = csv[15] - if !is_nullish(value) { - .cisco.umbrella.categories = value - } - value = csv[16] - if !is_nullish(value) { - .cisco.umbrella.av_detections = value - } - value = csv[17] - if !is_nullish(value) { - .cisco.umbrella.puas = value - } - value = csv[18] - if !is_nullish(value) { - .cisco.umbrella.amp_disposition = value - } - value = csv[19] - if !is_nullish(value) { - .cisco.umbrella.amp_malware_name = value - } - value = csv[20] - if !is_nullish(value) { - .cisco.umbrella.amp_score = value - } - value = csv[21] - if !is_nullish(value) { - .cisco.umbrella.policy_identity_type = value - } - value = csv[22] - if !is_nullish(value) { - .cisco.umbrella.blocked_categories = value - } - value = csv[23] - if !is_nullish(value) { - .cisco.umbrella.identities = value - } - value = csv[24] - if !is_nullish(value) { - .cisco.umbrella.identity_types = value - } - value = csv[25] - if !is_nullish(value) { - .http.request.method = value - } - value = csv[26] - if !is_nullish(value) { - .cisco.umbrella.dlp_status = value - } - value = csv[27] - if !is_nullish(value) { - .cisco.umbrella.certificate_errors = value - } - value = csv[28] - if !is_nullish(value) { - .file.name = value - } - value = csv[29] - if !is_nullish(value) { - .cisco.umbrella.ruleset_id = value - } - value = csv[30] - if !is_nullish(value) { - .rule.id = value - } - value = csv[31] - if !is_nullish(value) { - .cisco.umbrella.destination_lists_id = value - } - value = csv[32] - if !is_nullish(value) { - .cisco.umbrella.isolate_action = value - } - value = csv[33] - if !is_nullish(value) { - .cisco.umbrella.file_action = value - } - value = csv[34] - if !is_nullish(value) { - .cisco.umbrella.warn_status = value - } - } - if .log.file.path != null && { - expr = .log.file.path - value = "proxylogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - .observer.type = "proxy" - } - if .log.file.path != null && ({ - expr = .log.file.path - value = "cloudfirewalllogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } || { - expr = .log.file.path - value = "firewalllogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - }) { - csv = parse_csv!(replace!(.event.original, r'", "', s'","')) - value = csv[0] - if !is_nullish(value) { - .cisco.umbrella._tmp.time = value - } - value = csv[1] - if !is_nullish(value) { - .cisco.umbrella.origin_id = value - } - value = csv[2] - if !is_nullish(value) { - .cisco.umbrella.identities = value - } - value = csv[3] - if !is_nullish(value) { - .cisco.umbrella.identity_types = value - } - value = csv[4] - if !is_nullish(value) { - .cisco.umbrella.direction = value - } - value = csv[5] - if !is_nullish(value) { - .network.transport = value - } - value = csv[6] - if !is_nullish(value) { - .source.bytes = value - } - value = csv[7] - if !is_nullish(value) { - .source.address = value - } - value = csv[8] - if !is_nullish(value) { - .source.port = value - } - value = csv[9] - if !is_nullish(value) { - .destination.address = value - } - value = csv[10] - if !is_nullish(value) { - .destination.port = value - } - value = csv[11] - if !is_nullish(value) { - .cisco.umbrella.datacenter = value - } - value = csv[12] - if !is_nullish(value) { - .rule.id = value - } - value = csv[13] - if !is_nullish(value) { - .cisco.umbrella.action = value - } - value = csv[14] - if !is_nullish(value) { - .cisco.umbrella.fqdns = value - } - value = csv[15] - if !is_nullish(value) { - .cisco.umbrella.destination_lists_id = value - } - } - if .log.file.path != null && ({ - expr = .log.file.path - value = "cloudfirewalllogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } || { - expr = .log.file.path - value = "firewalllogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - }) { - .observer.type = "firewall" - } - if .log.file.path != null && { - expr = .log.file.path - value = "intrusionlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - csv = parse_csv!(replace!(.event.original, r'", "', s'","')) - value = csv[0] - if !is_nullish(value) { - .cisco.umbrella._tmp.time = value - } - value = csv[1] - if !is_nullish(value) { - .cisco.umbrella.identities = value - } - value = csv[2] - if !is_nullish(value) { - .cisco.umbrella.identity_types = value - } - value = csv[3] - if !is_nullish(value) { - .cisco.umbrella.gid = value - } - value = csv[4] - if !is_nullish(value) { - .cisco.umbrella.sid = value - } - value = csv[5] - if !is_nullish(value) { - .cisco.umbrella.message = value - } - value = csv[6] - if !is_nullish(value) { - .cisco.umbrella.signature_list_id = value - } - value = csv[7] - if !is_nullish(value) { - .cisco.umbrella.severity = value - } - value = csv[8] - if !is_nullish(value) { - .cisco.umbrella.classification = value - } - value = csv[9] - if !is_nullish(value) { - .cisco.umbrella.cves = value - } - value = csv[10] - if !is_nullish(value) { - .network.protocol = value - } - value = csv[11] - if !is_nullish(value) { - .event.id = value - } - value = csv[12] - if !is_nullish(value) { - .source.address = value - } - value = csv[13] - if !is_nullish(value) { - .source.port = value - } - value = csv[14] - if !is_nullish(value) { - .destination.address = value - } - value = csv[15] - if !is_nullish(value) { - .destination.port = value - } - value = csv[16] - if !is_nullish(value) { - .cisco.umbrella.action = value - } - } - if .log.file.path != null && { - expr = .log.file.path - value = "intrusionlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - .observer.type = "idps" - } - if .log.file.path != null && { - expr = .log.file.path - value = "dlplogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - csv = parse_csv!(replace!(.event.original, r'", "', s'","')) - value = csv[0] - if !is_nullish(value) { - .cisco.umbrella._tmp.time = value - } - value = csv[1] - if !is_nullish(value) { - .event.provider = value - } - value = csv[2] - if !is_nullish(value) { - .event.id = value - } - value = csv[3] - if !is_nullish(value) { - .cisco.umbrella.severity = value - } - value = csv[4] - if !is_nullish(value) { - .network.name = value - } - value = csv[5] - if !is_nullish(value) { - .file.owner = value - } - value = csv[6] - if !is_nullish(value) { - .file.name = value - } - value = csv[7] - if !is_nullish(value) { - .network.application = value - } - value = csv[8] - if !is_nullish(value) { - .url.original = value - } - value = csv[9] - if !is_nullish(value) { - .cisco.umbrella.action = value - } - value = csv[10] - if !is_nullish(value) { - .rule.name = value - } - value = csv[11] - if !is_nullish(value) { - .cisco.umbrella.data_classification = value - } - value = csv[12] - if !is_nullish(value) { - .cisco.umbrella.data_identifier = value - } - value = csv[13] - if !is_nullish(value) { - .file.mime_type = value - } - value = csv[14] - if !is_nullish(value) { - .file.size = value - } - value = csv[15] - if !is_nullish(value) { - .file.hash.sha256 = value - } - value = csv[16] - if !is_nullish(value) { - .cisco.umbrella.file_label = value - } - } - if .log.file.path != null && { - expr = .log.file.path - value = "dlplogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - .observer.type = "dlp" - } - if .log.file.path != null && { - expr = .log.file.path - value = "auditlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - csv = parse_csv!(replace!(.event.original, r'", "', s'","')) - value = csv[0] - if !is_nullish(value) { - .event.id = value - } - value = csv[1] - if !is_nullish(value) { - .cisco.umbrella._tmp.time = value - } - value = csv[2] - if !is_nullish(value) { - .user.email = value - } - value = csv[3] - if !is_nullish(value) { - .user.name = value - } - value = csv[4] - if !is_nullish(value) { - .cisco.umbrella.audit.type = value - } - value = csv[5] - if !is_nullish(value) { - .event.action = value - } - value = csv[6] - if !is_nullish(value) { - .source.address = value - } - value = csv[7] - if !is_nullish(value) { - .cisco.umbrella.audit.before = value - } - value = csv[8] - if !is_nullish(value) { - .cisco.umbrella.audit.after = value - } - } - if .cisco.umbrella.audit.type != "global_settings" { - if !is_nullish(.cisco.umbrella.audit.before) { - kv, err = parse_key_value(.cisco.umbrella.audit.before, key_value_delimiter: ":", field_delimiter: "\n", accept_standalone_key: true, whitespace: "strict") - if err == null { - standalone_key = false - for_each(kv) -> |_, value| { - if value == true { - standalone_key = true - } - } - if standalone_key { - err = "field .cisco.umbrella.audit.before does not contain value_split [:]" - } - } - if err == null { - if .cisco.umbrella.audit.before_values == null { - .cisco.umbrella.audit.before_values = { } - } - .cisco.umbrella.audit.before_values = merge(object!(.cisco.umbrella.audit.before_values), kv, deep: true) - } else { - _ingest.on_failure_processor_type = "kv" - _ingest.on_failure_processor_tag = "kv_audit_before" - _ingest.on_failure_message = to_string(err) - if !is_array(.error.message) { - if .error.message == null { - .error.message = [] - } else { - .error.message = [.error.message] - } - } - .error.message = append!({ - expr = .error.message - if false { - expr = null - } - expr - }, [join!(["Processor ", _ingest.on_failure_processor_type, " with tag ", _ingest.on_failure_processor_tag, " in pipeline ", _ingest.on_failure_pipeline, " failed with message: ", _ingest.on_failure_message], "")]) - } - } - } - if .cisco.umbrella.audit.type != "global_settings" { - if !is_nullish(.cisco.umbrella.audit.before) { - .cisco.umbrella.audit.before = split(string!(.cisco.umbrella.audit.before), r'\n') - } - } - if .cisco.umbrella.audit.type != "global_settings" { - if !is_nullish(.cisco.umbrella.audit.after) { - kv, err = parse_key_value(.cisco.umbrella.audit.after, key_value_delimiter: ":", field_delimiter: "\n", accept_standalone_key: true, whitespace: "strict") - if err == null { - standalone_key = false - for_each(kv) -> |_, value| { - if value == true { - standalone_key = true - } - } - if standalone_key { - err = "field .cisco.umbrella.audit.after does not contain value_split [:]" - } - } - if err == null { - if .cisco.umbrella.audit.after_values == null { - .cisco.umbrella.audit.after_values = { } - } - .cisco.umbrella.audit.after_values = merge(object!(.cisco.umbrella.audit.after_values), kv, deep: true) - } else { - _ingest.on_failure_processor_type = "kv" - _ingest.on_failure_processor_tag = "kv_audit_after" - _ingest.on_failure_message = to_string(err) - if !is_array(.error.message) { - if .error.message == null { - .error.message = [] - } else { - .error.message = [.error.message] - } - } - .error.message = append!({ - expr = .error.message - if false { - expr = null - } - expr - }, [join!(["Processor ", _ingest.on_failure_processor_type, " with tag ", _ingest.on_failure_processor_tag, " in pipeline ", _ingest.on_failure_pipeline, " failed with message: ", _ingest.on_failure_message], "")]) - } - } - } - if .cisco.umbrella.audit.type != "global_settings" { - if !is_nullish(.cisco.umbrella.audit.after) { - .cisco.umbrella.audit.after = split(string!(.cisco.umbrella.audit.after), r'\n') - } - } - if .url.original != null { - uri, err = parse_url(.url.original, default_known_ports: true) - if err == null { - if uri.username == "" { - del(uri.username) - } - if uri.password == "" { - del(uri.password) - } - uri.query = encode_key_value!(uri.query, field_delimiter: "&", key_value_delimiter: "=") - if uri.query == "" { - del(uri.query) - } - if uri.path != null { - uri.path = decode_percent!(uri.path) - } - uri.original = .url.original - uri.extension = parse_regex(uri.path, r'.*\.(?P[a-zA-Z0-9_\-]+)$').extension ?? null - uri.domain = del(uri.host) - if uri.username != null && uri.password != null { - uri.user_info = join!([uri.username, uri.password], separator: ":") - } - .url = compact(uri) - } - } - if (is_string(.cisco.umbrella.identity)) && { - expr = .cisco.umbrella.identity - value = "," - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } && (is_string(.cisco.umbrella.identities)) && starts_with!({ - expr = .cisco.umbrella.identities - if false { - expr = null - } - expr - }, .cisco.umbrella.identity) { - identities_tail = slice!(.cisco.umbrella.identities, length!({ - expr = .cisco.umbrella.identity - if false { - expr = null - } - expr - })) - if starts_with!({ - expr = identities_tail - if false { - expr = null - } - expr - }, ",") { - identities_tail = slice!(identities_tail, 1) - } - if .cisco.umbrella._tmp == null { - .cisco.umbrella._tmp = { } - } - .cisco.umbrella._tmp.identities_tail = identities_tail - } - if .cisco.umbrella._tmp.identities_tail != null { - .cisco.umbrella._tmp.identities_tail = split(string!(.cisco.umbrella._tmp.identities_tail), r',\s*') - } - if .cisco.umbrella._tmp.identities_tail != null { - identities = [] - if false { - identities = null - } - identities = push!(identities, .cisco.umbrella.identity) - for_each(array!({ - expr = .cisco.umbrella._tmp.identities_tail - if false { - expr = null - } - expr - })) -> |_index, identity| { - { - if false { - identities = null - } - identities = push!(identities, identity) - } - } - .cisco.umbrella._tmp.identities = identities - } - if .cisco.umbrella._tmp.identities != null { - .cisco.umbrella.identities = .cisco.umbrella._tmp.identities - } - if .cisco.umbrella.identities != null && .cisco.umbrella._tmp.identities == null { - .cisco.umbrella.identities = split(string!(.cisco.umbrella.identities), r',\s*') - } - if .log.file.path != null && { - expr = .log.file.path - value = "dnslogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } && .cisco.umbrella.categories != null { - .cisco.umbrella.categories = split(string!(.cisco.umbrella.categories), r',\s*') - } - if .log.file.path != null && { - expr = .log.file.path - value = "dnslogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } && .cisco.umbrella.blocked_categories != null { - .cisco.umbrella.blocked_categories = split(string!(.cisco.umbrella.blocked_categories), r',\s*') - } - if .cisco.umbrella.identity_types != null { - .cisco.umbrella.identity_types = split(string!(.cisco.umbrella.identity_types), r',\s*') - } - if .cisco.umbrella.fqdns != null { - .cisco.umbrella.fqdns = split(string!(.cisco.umbrella.fqdns), r',\s*') - } - if .cisco.umbrella.identities != null && .cisco.umbrella.identity_types != null && { - temp = .cisco.umbrella.identities - if false { - temp = null - } - length!(temp) - } == { - temp = .cisco.umbrella.identity_types - if false { - temp = null - } - length!(temp) - } { - i = 0 - for_each(array!({ - expr = .cisco.umbrella.identity_types - if false { - expr = null - } - expr - })) -> |_index, cisco_identity_type| { - { - if { - expr = ["AD Users"] - value = cisco_identity_type - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - { - x = get!(.cisco.umbrella.identities, [i]) - { - if .user == null { - .user = { } - } - if .user.name == null { - .user.name = x - } - } - } - } - if { - expr = ["AD Computers", "Roaming Computers", "Anyconnect Roaming Client", "Mobile Devices"] - value = cisco_identity_type - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - { - x = get!(.cisco.umbrella.identities, [i]) - { - if .host == null { - .host = { } - } - if .host.name == null { - .host.name = x - } - } - } - } - if { - expr = ["Sites", "Internal Networks", "Networks", "Network Devices", "Network Tunnels", "CDFW Tunnel Device"] - value = cisco_identity_type - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - { - x = get!(.cisco.umbrella.identities, [i]) - { - if .network == null { - .network = { } - } - if .network.name == null { - al = [] - .network = set!(.network, ["name"], al) - } - if !{ - expr = .network.name - value = x - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - if false { - .network.name = null - } - .network.name = push!(.network.name, x) - } - } - } - } - { - tmp = i - i = i + 1 - tmp - } - } - } - } - if .host.name != null { - if !is_nullish(.host.name) { - if is_array(.host.name) { - .host.name = map_values(array!(.host.name)) -> |value| { - downcase!(value) - } - } else { - tmp, err = downcase(.host.name) - if err == null { - .host.name = tmp - } else { - assert!(false, to_string(err)) - } - } - } else { - assert!(false, ".host.name must not be nullish") - } - } - if .host.name != null && { - expr = .host.name - value = "." - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - if !is_nullish(.host.name) { - value, err = parse_groks(value: .host.name, patterns: [s'.*?^%{DATA:host.hostname}\.%{GREEDYDATA:host.domain}$.*']) - if err == null { - . = merge(., value, deep: true) - } - } - } - if .user.name != null && { - expr = .user.name - value = "@" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } == true { - if !is_nullish(.user.name) { - value, err = parse_groks(value: .user.name, patterns: [s'.*?^%{GREEDYDATA:user.full_name} \(%{DATA:user.name}@%{DATA:user.domain}\)?$.*']) - if err == null { - . = merge(., value, deep: true) - } - } - } - if .user.email == null && .user.name != null && .user.domain != null { - .user.email = join!([to_string!(.user.name), "@", to_string!(.user.domain)], "") - } - if .user.email != null { - .user.id = .user.email - } - if .user.email == "null" { - if !exists(.user.email) { - assert!(false, "tried to remove field '.user.email' that doesn't exist") - } - del(.user.email) - } - if .user.id == "null" { - if !exists(.user.id) { - assert!(false, "tried to remove field '.user.id' that doesn't exist") - } - del(.user.id) - } - if .user.name == "null" { - if !exists(.user.name) { - assert!(false, "tried to remove field '.user.name' that doesn't exist") - } - del(.user.name) - } - if is_object(.user) && length!(.user) == 0 { - del(.user) - } - if .cisco.umbrella._tmp.time != null { - date = null - err = "no datetime formats provided" - if err != null { - datetime = to_string!(.cisco.umbrella._tmp.time) - date, err = parse_timestamp(value: join!(value: [datetime], separator: " "), format: join!(value: ["%Y-%m-%d %H:%M:%S"], separator: " ")) - } - if err != null { - datetime = to_string!(.cisco.umbrella._tmp.time) - date, err = parse_timestamp(value: join!(value: [datetime], separator: " "), format: join!(value: ["%+"], separator: " ")) - } - if err == null { - .@timestamp = date - } else { - datetime = to_string!(.cisco.umbrella._tmp.time) - assert!(false, "failed to parse timestamp .cisco.umbrella._tmp.time: ({{ datetime }}): " + to_string(err)) - } - } - if .dns.question.name != null { - .dns.type = "query" - } - if .dns.question.name != null { - tmp, err = parse_etld(.dns.question.name, plus_parts: 1) - if err == null { - .dns.question.registered_domain = tmp.etld_plus - .dns.question.top_level_domain = tmp.etld - .dns.question.domain = .dns.question.name - .dns.question.subdomain = { - subdomain = replace!(.dns.question.name, tmp.etld_plus, "") - if subdomain != "" { - subdomain = slice!(subdomain, 0, -1) - } - subdomain - } - .dns.question = compact(.dns.question) - } else { - assert!(false, to_string(err)) - } - } - del(.dns.question.domain) - if is_string(.user_agent.original) { - user_agent = parse_user_agent!(.user_agent.original) - .user_agent.version = user_agent.browser.version - .user_agent.os.name = user_agent.os.family - .user_agent.os.version = user_agent.os.version - .user_agent.os.full, _ = join([user_agent.os.family, user_agent.os.version], separator: " ") - .user_agent.name = user_agent.browser.family - if is_nullish(.user_agent.name) { - .user_agent.name = "Other" - } - .user_agent.device.name = user_agent.device.category - if is_nullish(.user_agent.device.name) { - .user_agent.device.name = "Other" - } - .user_agent = compact(.user_agent, recursive: true) - if .user_agent.original == null { - .user_agent.original = "" - } else { - .user_agent.original = .user_agent.original - } - } - if .cisco.umbrella.direction != null { - if !is_nullish(.cisco.umbrella.direction) { - if is_array(.cisco.umbrella.direction) { - .cisco.umbrella.direction = map_values(array!(.cisco.umbrella.direction)) -> |value| { - downcase!(value) - } - } else { - tmp, err = downcase(.cisco.umbrella.direction) - if err == null { - .network.direction = tmp - } else { - assert!(false, to_string(err)) - } - } - } else { - assert!(false, ".cisco.umbrella.direction must not be nullish") - } - } - if .source.bytes != null { - if !is_nullish(.source.bytes) { - if is_string(.source.bytes) { - temp = .source.bytes - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .source.bytes = temp - } else { - err_msg = join!(["Failed converting .source.bytes to an integer. Found (", to_string!(.source.bytes), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.source.bytes) { - .source.bytes = .source.bytes - } else { - err = "Failed converting .source.bytes to an integer" - assert!(false, err) - } - } - } else { - err = ".source.bytes must be present" - assert!(false, err) - } - } - if .source.port != null { - if !is_nullish(.source.port) { - if is_string(.source.port) { - temp = .source.port - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .source.port = temp - } else { - err_msg = join!(["Failed converting .source.port to an integer. Found (", to_string!(.source.port), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.source.port) { - .source.port = .source.port - } else { - err = "Failed converting .source.port to an integer" - assert!(false, err) - } - } - } else { - err = ".source.port must be present" - assert!(false, err) - } - } - if .destination.port != null { - if !is_nullish(.destination.port) { - if is_string(.destination.port) { - temp = .destination.port - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .destination.port = temp - } else { - err_msg = join!(["Failed converting .destination.port to an integer. Found (", to_string!(.destination.port), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.destination.port) { - .destination.port = .destination.port - } else { - err = "Failed converting .destination.port to an integer" - assert!(false, err) - } - } - } else { - err = ".destination.port must be present" - assert!(false, err) - } - } - if .http.request.bytes != null { - if !is_nullish(.http.request.bytes) { - if is_string(.http.request.bytes) { - temp = .http.request.bytes - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .http.request.bytes = temp - } else { - err_msg = join!(["Failed converting .http.request.bytes to an integer. Found (", to_string!(.http.request.bytes), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.http.request.bytes) { - .http.request.bytes = .http.request.bytes - } else { - err = "Failed converting .http.request.bytes to an integer" - assert!(false, err) - } - } - } else { - err = ".http.request.bytes must be present" - assert!(false, err) - } - } - if .http.response.bytes != null { - if !is_nullish(.http.response.bytes) { - if is_string(.http.response.bytes) { - temp = .http.response.bytes - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .http.response.bytes = temp - } else { - err_msg = join!(["Failed converting .http.response.bytes to an integer. Found (", to_string!(.http.response.bytes), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.http.response.bytes) { - .http.response.bytes = .http.response.bytes - } else { - err = "Failed converting .http.response.bytes to an integer" - assert!(false, err) - } - } - } else { - err = ".http.response.bytes must be present" - assert!(false, err) - } - } - if .http.response.body.bytes != null { - if !is_nullish(.http.response.body.bytes) { - if is_string(.http.response.body.bytes) { - temp = .http.response.body.bytes - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .http.response.body.bytes = temp - } else { - err_msg = join!(["Failed converting .http.response.body.bytes to an integer. Found (", to_string!(.http.response.body.bytes), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.http.response.body.bytes) { - .http.response.body.bytes = .http.response.body.bytes - } else { - err = "Failed converting .http.response.body.bytes to an integer" - assert!(false, err) - } - } - } else { - err = ".http.response.body.bytes must be present" - assert!(false, err) - } - } - if .http.response.status_code != null { - if !is_nullish(.http.response.status_code) { - if is_string(.http.response.status_code) { - temp = .http.response.status_code - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .http.response.status_code = temp - } else { - err_msg = join!(["Failed converting .http.response.status_code to an integer. Found (", to_string!(.http.response.status_code), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.http.response.status_code) { - .http.response.status_code = .http.response.status_code - } else { - err = "Failed converting .http.response.status_code to an integer" - assert!(false, err) - } - } - } else { - err = ".http.response.status_code must be present" - assert!(false, err) - } - } - if .cisco.umbrella.action != null { - if !is_nullish(.cisco.umbrella.action) { - if is_array(.cisco.umbrella.action) { - .cisco.umbrella.action = map_values(array!(.cisco.umbrella.action)) -> |value| { - replace!(value, r'\s', "-") - } - } else { - if false { - .cisco.umbrella.action = null - } - target, err = replace(.cisco.umbrella.action, r'\s', "-") - if err == null { - .cisco.umbrella.action = target - } else { - assert!(false, to_string(err)) - } - } - } else { - assert!(false, ".cisco.umbrella.action must not be nullish") - } - } - if .cisco.umbrella.action != null && .log.file.path != null && { - expr = .log.file.path - value = "dnslogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - .event.action = join!(["dns-request-", to_string!(.cisco.umbrella.action)], "") - } - if .http.request.method != null && .log.file.path != null && { - expr = .log.file.path - value = "proxylogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - .event.action = join!(["proxy-request-", to_string!(.http.request.method)], "") - } - if .cisco.umbrella.action != null && .log.file.path != null && ({ - expr = .log.file.path - value = "firewalllogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } || { - expr = .log.file.path - value = "cloudfirewalllogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - }) { - .event.action = join!(["fw-connection-", to_string!(.cisco.umbrella.action)], "") - } - if .cisco.umbrella.action != null && .log.file.path != null && { - expr = .log.file.path - value = "intrusionlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - .event.action = join!(["ips-", to_string!(.cisco.umbrella.action)], "") - } - if .cisco.umbrella.action != null && .log.file.path != null && { - expr = .log.file.path - value = "dlplogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - .event.action = join!(["dlp-", to_string!(.cisco.umbrella.action)], "") - } - if .log.file.path != null && !{ - expr = .log.file.path - value = "auditlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - .event.category = "network" - } - if .cisco.umbrella.action != null && { - expr = ["Allowed", "ALLOWED", "ALLOW", "Would-Block"] - value = .cisco.umbrella.action - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - if !is_array(.event.type) { - if .event.type == null { - .event.type = [] - } else { - .event.type = [.event.type] - } - } - .event.type = append!({ - expr = .event.type - if false { - expr = null - } - expr - }, ["allowed"]) - } - if .cisco.umbrella.action != null && { - expr = ["Blocked", "BLOCKED", "BLOCK"] - value = .cisco.umbrella.action - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - if !is_array(.event.type) { - if .event.type == null { - .event.type = [] - } else { - .event.type = [.event.type] - } - } - .event.type = append!({ - expr = .event.type - if false { - expr = null - } - expr - }, ["denied"]) - } - if .log.file.path != null && !{ - expr = .log.file.path - value = "auditlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - if !is_array(.event.type) { - if .event.type == null { - .event.type = [] - } else { - .event.type = [.event.type] - } - } - .event.type = append!({ - expr = .event.type - if false { - expr = null - } - expr - }, ["connection"]) - } - if .log.file.path != null && { - expr = .log.file.path - value = "auditlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - .event.category = "configuration" - } - if .log.file.path != null && { - expr = .log.file.path - value = "dlplogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - if !is_array(.event.category) { - if .event.category == null { - .event.category = [] - } else { - .event.category = [.event.category] - } - } - .event.category = append!({ - expr = .event.category - if false { - expr = null - } - expr - }, ["file"]) - } - if .log.file.path != null && { - expr = .log.file.path - value = "intrusionlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - if !is_array(.event.category) { - if .event.category == null { - .event.category = [] - } else { - .event.category = [.event.category] - } - } - .event.category = append!({ - expr = .event.category - if false { - expr = null - } - expr - }, ["intrusion_detection"]) - } - if .log.file.path != null && { - expr = .log.file.path - value = "intrusionlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - .event.kind = "alert" - } - if .log.file.path != null && !{ - expr = .log.file.path - value = "intrusionlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - .event.kind = "event" - } - if .log.file.path != null && { - expr = .log.file.path - value = "intrusionlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - if !is_array(.event.type) { - if .event.type == null { - .event.type = [] - } else { - .event.type = [.event.type] - } - } - .event.type = append!({ - expr = .event.type - if false { - expr = null - } - expr - }, ["info"]) - } - if .log.file.path != null && { - expr = .log.file.path - value = "dlplogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } { - if !is_array(.event.type) { - if .event.type == null { - .event.type = [] - } else { - .event.type = [.event.type] - } - } - .event.type = append!({ - expr = .event.type - if false { - expr = null - } - expr - }, ["info"]) - } - if .log.file.path != null && { - expr = .log.file.path - value = "auditlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } && downcase(string!(.event.action)) == "create" { - if !is_array(.event.type) { - if .event.type == null { - .event.type = [] - } else { - .event.type = [.event.type] - } - } - .event.type = append!({ - expr = .event.type - if false { - expr = null - } - expr - }, ["creation"]) - } - if .log.file.path != null && { - expr = .log.file.path - value = "auditlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } && downcase(string!(.event.action)) == "update" { - if !is_array(.event.type) { - if .event.type == null { - .event.type = [] - } else { - .event.type = [.event.type] - } - } - .event.type = append!({ - expr = .event.type - if false { - expr = null - } - expr - }, ["change"]) - } - if .log.file.path != null && { - expr = .log.file.path - value = "auditlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } && downcase(string!(.event.action)) == "delete" { - if !is_array(.event.type) { - if .event.type == null { - .event.type = [] - } else { - .event.type = [.event.type] - } - } - .event.type = append!({ - expr = .event.type - if false { - expr = null - } - expr - }, ["deletion"]) - } - if !is_nullish(.source.address) { - if is_ipv4(string!(.source.address)) || is_ipv6(string!(.source.address)) { - .source.ip = .source.address - } else { - err = "Failed converting .source.address to an IP address" - _ingest.on_failure_processor_type = "on_failure" - _ingest.on_failure_processor_tag = "" - _ingest.on_failure_message = to_string(err) - .source.domain = .source.address - } - } - if !is_nullish(.destination.address) { - if is_ipv4(string!(.destination.address)) || is_ipv6(string!(.destination.address)) { - .destination.ip = .destination.address - } else { - err = "Failed converting .destination.address to an IP address" - _ingest.on_failure_processor_type = "on_failure" - _ingest.on_failure_processor_tag = "" - _ingest.on_failure_message = to_string(err) - .destination.domain = .destination.address - } - } - if !is_nullish(.source.nat.ip) { - if is_ipv4(string!(.source.nat.ip)) || is_ipv6(string!(.source.nat.ip)) { - .source.nat.ip = .source.nat.ip - } else { - err = "Failed converting .source.nat.ip to an IP address" - _ingest.on_failure_processor_type = "on_failure" - _ingest.on_failure_processor_tag = "" - _ingest.on_failure_message = to_string(err) - if !exists(.source.nat.ip) { - assert!(false, "tried to remove field '.source.nat.ip' that doesn't exist") - } - del(.source.nat.ip) - } - } - iana_numbers = { "icmp": 1, "igmp": 2, "tcp": 6, "udp": 17, "gre": 47, "icmp ipv6": 58, "eigrp": 88, "ospf": 89, "pim": 103, "sctp": 132 } - if !(is_nullish(.source.ip) || is_nullish(.destination.ip) || (is_nullish(.network.iana_number) && is_nullish(.network.transport)) || (.source.ip == "0.0.0.0" && .destination.ip == "0.0.0.0" && .source.port == 0 && .destination.port == 0)) { - iana_number = .network.iana_number - if is_nullish(iana_number) && !is_nullish(.network.transport) { - temp_iana_number, err = get(iana_numbers, [downcase!(.network.transport)]) - if err == null { - iana_number = temp_iana_number - } else { - assert!(false, to_string(err)) - } - } - if iana_number == null { - tmp_iana_number, err = parse_int(.network.transport) - if err == null { - iana_number = tmp_iana_number - } - } - source_port = .source.port - destination_port = .destination.port - if (iana_number == 6 || iana_number == 17) && source_port == null && destination_port == null { - source_port = 0 - destination_port = 0 - } - if (iana_number == 1) { - source_port = .icmp.type - destination_port = .icmp.code - } - if .icmp.type != null && .icmp.code != null { - source_port = to_int!(.icmp.type) - destination_port = to_int!(.icmp.code) - } - community_id = err = null - if source_port != null && destination_port != null { - community_id, err = community_id(source_ip: .source.ip, source_port: source_port, destination_ip: .destination.ip, destination_port: destination_port, protocol: iana_number, seed: 0) - } else { - community_id, err = community_id(source_ip: .source.ip, destination_ip: .destination.ip, protocol: iana_number, seed: 0) - } - if err == null { - .network.community_id = community_id - } else { - assert!(false, to_string(err)) - } - } - if .source.as.asn != null { - if .source.as.number == null { - .source.as.number = del(.source.as.asn) - } else { - assert!(false, ".source.as.number must be null or override: true") - } - } - if .source.as.organization_name != null { - if .source.as.organization.name == null { - .source.as.organization.name = del(.source.as.organization_name) - } else { - assert!(false, ".source.as.organization.name must be null or override: true") - } - } - if .destination.as.asn != null { - if .destination.as.number == null { - .destination.as.number = del(.destination.as.asn) - } else { - assert!(false, ".destination.as.number must be null or override: true") - } - } - if .destination.as.organization_name != null { - if .destination.as.organization.name == null { - .destination.as.organization.name = del(.destination.as.organization_name) - } else { - assert!(false, ".destination.as.organization.name must be null or override: true") - } - } - if .cisco.umbrella.severity == "LOW" && .log.file.path != null && ({ - expr = .log.file.path - value = "intrusionlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } || { - expr = .log.file.path - value = "dlplogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - }) { - .event.severity = 1 - } - if .cisco.umbrella.severity == "MEDIUM" && .log.file.path != null && ({ - expr = .log.file.path - value = "intrusionlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } || { - expr = .log.file.path - value = "dlplogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - }) { - .event.severity = 2 - } - if .cisco.umbrella.severity == "HIGH" && .log.file.path != null && ({ - expr = .log.file.path - value = "intrusionlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } || { - expr = .log.file.path - value = "dlplogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - }) { - .event.severity = 3 - } - if .cisco.umbrella.severity == "CRITICAL" && .log.file.path != null && ({ - expr = .log.file.path - value = "intrusionlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } || { - expr = .log.file.path - value = "dlplogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - }) { - .event.severity = 4 - } - if .event.severity == null && .log.file.path != null && ({ - expr = .log.file.path - value = "intrusionlogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - } || { - expr = .log.file.path - value = "dlplogs" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - }) { - .event.severity = 0 - } - if .file.size != null { - if !is_nullish(.file.size) { - if is_string(.file.size) { - temp = .file.size - if temp != "0" { - temp = replace!(temp, r'^0*(?P[0-9]+)', "$suffix") - } - temp, err = parse_int(temp) - if err == null { - .file.size = temp - } else { - err_msg = join!(["Failed converting .file.size to an integer. Found (", to_string!(.file.size), "), ", to_string(err)], "") - assert!(false, err_msg) - } - } else { - if is_integer(.file.size) { - .file.size = .file.size - } else { - err = "Failed converting .file.size to an integer" - assert!(false, err) - } - } - } else { - err = ".file.size must be present" - assert!(false, err) - } - } - if .user.name != null { - if .related.user == null { - .related.user = [.user.name] - } else { - if is_array(.related.user) { - if !includes!(.related.user, .user.name) { - .related.user = push!(.related.user, .user.name) - } - } else { - if .related.user != .user.name { - .related.user = [.related.user, .user.name] - } - } - } - } - if .source.ip != null { - if .related.ip == null { - .related.ip = [.source.ip] - } else { - if is_array(.related.ip) { - if !includes!(.related.ip, .source.ip) { - .related.ip = push!(.related.ip, .source.ip) - } - } else { - if .related.ip != .source.ip { - .related.ip = [.related.ip, .source.ip] - } - } - } - } - if .source.nat.ip != null { - if .related.ip == null { - .related.ip = [.source.nat.ip] - } else { - if is_array(.related.ip) { - if !includes!(.related.ip, .source.nat.ip) { - .related.ip = push!(.related.ip, .source.nat.ip) - } - } else { - if .related.ip != .source.nat.ip { - .related.ip = [.related.ip, .source.nat.ip] - } - } - } - } - if .destination.ip != null { - if .related.ip == null { - .related.ip = [.destination.ip] - } else { - if is_array(.related.ip) { - if !includes!(.related.ip, .destination.ip) { - .related.ip = push!(.related.ip, .destination.ip) - } - } else { - if .related.ip != .destination.ip { - .related.ip = [.related.ip, .destination.ip] - } - } - } - } - if .host.name != null { - if .related.hosts == null { - .related.hosts = [.host.name] - } else { - if is_array(.related.hosts) { - if !includes!(.related.hosts, .host.name) { - .related.hosts = push!(.related.hosts, .host.name) - } - } else { - if .related.hosts != .host.name { - .related.hosts = [.related.hosts, .host.name] - } - } - } - } - if .source.domain != null { - if .related.hosts == null { - .related.hosts = [.source.domain] - } else { - if is_array(.related.hosts) { - if !includes!(.related.hosts, .source.domain) { - .related.hosts = push!(.related.hosts, .source.domain) - } - } else { - if .related.hosts != .source.domain { - .related.hosts = [.related.hosts, .source.domain] - } - } - } - } - if .dns.question.name != null { - if .related.hosts == null { - .related.hosts = [.dns.question.name] - } else { - if is_array(.related.hosts) { - if !includes!(.related.hosts, .dns.question.name) { - .related.hosts = push!(.related.hosts, .dns.question.name) - } - } else { - if .related.hosts != .dns.question.name { - .related.hosts = [.related.hosts, .dns.question.name] - } - } - } - } - if .cisco.umbrella.fqdns != null { - if !is_nullish(.cisco.umbrella.fqdns) { - .cisco.umbrella.fqdns = map_values(array!(.cisco.umbrella.fqdns)) -> |value| { - _ingest._value = value - if .related.hosts == null { - .related.hosts = [_ingest._value] - } else { - if is_array(.related.hosts) { - if !includes!(.related.hosts, _ingest._value) { - .related.hosts = push!(.related.hosts, _ingest._value) - } - } else { - if .related.hosts != _ingest._value { - .related.hosts = [.related.hosts, _ingest._value] - } - } - } - _ingest._value - } - } else { - err = ".cisco.umbrella.fqdns must not be nullish" - assert!(false, err) - } - } - if .cisco.umbrella.sha_sha256 != null { - if .related.hash == null { - .related.hash = [.cisco.umbrella.sha_sha256] - } else { - if is_array(.related.hash) { - if !includes!(.related.hash, .cisco.umbrella.sha_sha256) { - .related.hash = push!(.related.hash, .cisco.umbrella.sha_sha256) - } - } else { - if .related.hash != .cisco.umbrella.sha_sha256 { - .related.hash = [.related.hash, .cisco.umbrella.sha_sha256] - } - } - } - } - if .file.hash.sha256 != null { - if .related.hash == null { - .related.hash = [.file.hash.sha256] - } else { - if is_array(.related.hash) { - if !includes!(.related.hash, .file.hash.sha256) { - .related.hash = push!(.related.hash, .file.hash.sha256) - } - } else { - if .related.hash != .file.hash.sha256 { - .related.hash = [.related.hash, .file.hash.sha256] - } - } - } - } - del(.cisco.umbrella._tmp) - del(.cisco.umbrella.direction) - del(.cisco.umbrella.action) - del(.log.flags) - if .tags == null || !({ - expr = .tags - value = "preserve_original_event" - if false { - expr = null - value = null - } - if is_array(expr) { - includes(array!(expr), value) - } else { - if is_string(expr) { - contains(string!(expr), string!(value)) - } else { - assert!(false, "contains only works on strings and array") - } - } - }) { - del(.event.original) - } - } else { - # Unknown shape: tag + pass through, never silently drop. - .tags = push(array(.tags) ?? [], "filebeat_unmatched") - } diff --git a/pipelines/filebeat/timezones.csv b/pipelines/filebeat/timezones.csv deleted file mode 100644 index 259a4fe..0000000 --- a/pipelines/filebeat/timezones.csv +++ /dev/null @@ -1,213 +0,0 @@ -abbreviation,name,offset -ACDT, Australian Central Daylight Time, +10:30 -ACST, Australian Central Standard Time, +09:30 -ACT, Acre Time, -05:00 -ACWST, Australian Central Western Standard Time, +8:45 -ADT, Arabia Daylight Time, +04:00 -ADT, Atlantic Daylight Time, -03:00 -AEDT, Australian Eastern Daylight Time, +11:00 -AEST, Australian Eastern Standard Time, +10:00 -AFT, Afghanistan Time, +4:30 -AKDT, Alaska Daylight Time, -08:00 -AKST, Alaska Standard Time, -09:00 -ALMT, Alma-Ata Time, +06:00 -AMST, Amazon Summer Time, -03:00 -AMST, Armenia Summer Time, +05:00 -AMT, Amazon Time, -04:00 -AMT, Armenia Time, +04:00 -ANAST, Anadyr Summer Time, +12:00 -ANAT, Anadyr Time, +12:00 -AQTT, Aqtobe Time, +05:00 -ART, Argentina Time, -03:00 -AST, Arabia Standard Time, +03:00 -AST, Atlantic Standard Time, -04:00 -AWDT, Australian Western Daylight Time, +09:00 -AWST, Australian Western Standard Time, +08:00 -AZOST, Azores Summer Time, +00:00 -AZOT, Azores Time, -01:00 -AZST, Azerbaijan Summer Time, +05:00 -AZT, Azerbaijan Time, +04:00 -AoE, Anywhere on Earth, -12:00 -BNT, Brunei Darussalam Time, +08:00 -BOT, Bolivia Time, -04:00 -BRST, Brasília Summer Time, -02:00 -BRT, Brasília Time, -03:00 -BST, Bangladesh Standard Time, +06:00 -BST, Bougainville Standard Time, +11:00 -BST, British Summer Time, +01:00 -BTT, Bhutan Time, +06:00 -CAST, Casey Time, +08:00 -CAT, Central Africa Time, +02:00 -CCT, Cocos Islands Time, +6:30 -CDT, Central Daylight Time, -05:00 -CDT, Cuba Daylight Time, -04:00 -CEST, Central European Summer Time, +02:00 -CET, Central European Time, +01:00 -CHADT, Chatham Island Daylight Time, +13:45 -CHAST, Chatham Island Standard Time, +12:45 -CHOST, Choibalsan Summer Time, +09:00 -CHOT, Choibalsan Time, +08:00 -CHUT, Chuuk Time, +10:00 -CIDST, Cayman Islands Daylight Saving Time, -04:00 -CIST, Cayman Islands Standard Time, -05:00 -CKT, Cook Island Time, -10:00 -CLST, Chile Summer Time, -03:00 -CLT, Chile Standard Time, -04:00 -COT, Colombia Time, -05:00 -CST, Central Standard Time, -06:00 -CST, China Standard Time, +08:00 -CST, Cuba Standard Time, -05:00 -CVT, Cape Verde Time, -01:00 -CXT, Christmas Island Time, +07:00 -ChST, Chamorro Standard Time, +10:00 -DAVT, Davis Time, +07:00 -DDUT, Dumont-d'Urville Time, +10:00 -EASST, Easter Island Summer Time, -05:00 -EAST, Easter Island Standard Time, -06:00 -EAT, Eastern Africa Time, +03:00 -ECT, Ecuador Time, -05:00 -EDT, Eastern Daylight Time, -04:00 -EEST, Eastern European Summer Time, +03:00 -EET, Eastern European Time, +02:00 -EGST, Eastern Greenland Summer Time, +00:00 -EGT, East Greenland Time, -01:00 -EST, Eastern Standard Time, -05:00 -FET, Further-Eastern European Time, +03:00 -FJST, Fiji Summer Time, +13:00 -FJT, Fiji Time, +12:00 -FKST, Falkland Islands Summer Time, -03:00 -FKT, Falkland Island Time, -04:00 -FNT, Fernando de Noronha Time, -02:00 -GALT, Galapagos Time, -06:00 -GAMT, Gambier Time, -09:00 -GET, Georgia Standard Time, +04:00 -GFT, French Guiana Time, -03:00 -GILT, Gilbert Island Time, +12:00 -GMT, Greenwich Mean Time, +00:00 -GST, Gulf Standard Time, +04:00 -GST, South Georgia Time, -02:00 -GYT, Guyana Time, -04:00 -HDT, Hawaii-Aleutian Daylight Time, -09:00 -HKT, Hong Kong Time, +08:00 -HOVST, Hovd Summer Time, +08:00 -HOVT, Hovd Time, +07:00 -HST, Hawaii Standard Time, -10:00 -ICT, Indochina Time, +07:00 -IDT, Israel Daylight Time, +03:00 -IOT, Indian Chagos Time, +06:00 -IRDT, Iran Daylight Time, +4:30 -IRKST, Irkutsk Summer Time, +09:00 -IRKT, Irkutsk Time, +08:00 -IRST, Iran Standard Time, +3:30 -IST, India Standard Time, +5:30 -IST, Irish Standard Time, +01:00 -IST, Israel Standard Time, +02:00 -JST, Japan Standard Time, +09:00 -KGT, Kyrgyzstan Time, +06:00 -KOST, Kosrae Time, +11:00 -KRAST, Krasnoyarsk Summer Time, +08:00 -KRAT, Krasnoyarsk Time, +07:00 -KST, Korea Standard Time, +09:00 -KUYT, Kuybyshev Time, +04:00 -LHDT, Lord Howe Daylight Time, +11:00 -LHST, Lord Howe Standard Time, +10:30 -LINT, Line Islands Time, +14:00 -MAGST, Magadan Summer Time, +12:00 -MAGT, Magadan Time, +11:00 -MART, Marquesas Time, -9:30 -MAWT, Mawson Time, +05:00 -MDT, Mountain Daylight Time, -06:00 -MHT, Marshall Islands Time, +12:00 -MMT, Myanmar Time, +6:30 -MSD, Moscow Daylight Time, +04:00 -MSK, Moscow Standard Time, +03:00 -MST, Mountain Standard Time, -07:00 -MT, Mountain Time, -7:00 / -6:00 -MUT, Mauritius Time, +04:00 -MVT, Maldives Time, +05:00 -MYT, Malaysia Time, +08:00 -NCT, New Caledonia Time, +11:00 -NDT, Newfoundland Daylight Time, -2:30 -NFT, Norfolk Time, +11:00 -NOVST, Novosibirsk Summer Time, +07:00 -NOVT, Novosibirsk Time, +06:00 -NPT, Nepal Time , +5:45 -NRT, Nauru Time, +12:00 -NST, Newfoundland Standard Time, -3:30 -NUT, Niue Time, -11:00 -NZDT, New Zealand Daylight Time, +13:00 -NZST, New Zealand Standard Time, +12:00 -OMSST, Omsk Summer Time, +07:00 -OMST, Omsk Standard Time, +06:00 -ORAT, Oral Time, +05:00 -PDT, Pacific Daylight Time, -07:00 -PET, Peru Time, -05:00 -PETST, Kamchatka Summer Time, +12:00 -PETT, Kamchatka Time, +12:00 -PGT, Papua New Guinea Time, +10:00 -PHOT, Phoenix Island Time, +13:00 -PHT, Philippine Time, +08:00 -PKT, Pakistan Standard Time, +05:00 -PMDT, Pierre & Miquelon Daylight Time, -02:00 -PMST, Pierre & Miquelon Standard Time, -03:00 -PONT, Pohnpei Standard Time, +11:00 -PST, Pacific Standard Time, -08:00 -PST, Pitcairn Standard Time, -08:00 -PWT, Palau Time, +09:00 -PYST, Paraguay Summer Time, -03:00 -PYT, Paraguay Time, -04:00 -PYT, Pyongyang Time, +8:30 -QYZT, Qyzylorda Time, +06:00 -RET, Reunion Time, +04:00 -ROTT, Rothera Time, -03:00 -SAKT, Sakhalin Time, +11:00 -SAMT, Samara Time, +04:00 -SAST, South Africa Standard Time, +02:00 -SBT, Solomon Islands Time, +11:00 -SCT, Seychelles Time, +04:00 -SGT, Singapore Time, +08:00 -SRET, Srednekolymsk Time, +11:00 -SRT, Suriname Time, -03:00 -SST, Samoa Standard Time, -11:00 -SYOT, Syowa Time, +03:00 -TAHT, Tahiti Time, -10:00 -TFT, French Southern and Antarctic Time, +05:00 -TJT, Tajikistan Time , +05:00 -TKT, Tokelau Time, +13:00 -TLT, East Timor Time, +09:00 -TMT, Turkmenistan Time, +05:00 -TOST, Tonga Summer Time, +14:00 -TOT, Tonga Time, +13:00 -TRT, Turkey Time, +03:00 -TVT, Tuvalu Time, +12:00 -ULAST, Ulaanbaatar Summer Time, +09:00 -ULAT, Ulaanbaatar Time, +08:00 -UTC, Coordinated Universal Time, +00:00 -UYST, Uruguay Summer Time, -02:00 -UYT, Uruguay Time, -03:00 -UZT, Uzbekistan Time, +05:00 -VET, Venezuelan Standard Time, -04:00 -VLAST, Vladivostok Summer Time, +11:00 -VLAT, Vladivostok Time, +10:00 -VOST, Vostok Time, +06:00 -VUT, Vanuatu Time, +11:00 -WAKT, Wake Time, +12:00 -WARST, Western Argentine Summer Time, -03:00 -WAST, West Africa Summer Time, +02:00 -WAT, West Africa Time, +01:00 -WEST, Western European Summer Time, +01:00 -WET, Western European Time, +00:00 -WFT, Wallis and Futuna Time, +12:00 -WGST, Western Greenland Summer Time, -02:00 -WGT, West Greenland Time, -03:00 -WIB, Western Indonesian Time, +07:00 -WIT, Eastern Indonesian Time, +09:00 -WITA, Central Indonesian Time, +08:00 -WST, West Samoa Time, +13:00 -WST, Western Sahara Summer Time, +01:00 -WT, Western Sahara Standard Time, +00:00 -YAKST, Yakutsk Summer Time, +10:00 -YAKT, Yakutsk Time, +09:00 -YAPT, Yap Time, +10:00 -YEKST, Yekaterinburg Summer Time, +06:00 -YEKT, Yekaterinburg Time, +05:00 diff --git a/src/config/loader.rs b/src/config/loader.rs index d8f81e0..774b49d 100644 --- a/src/config/loader.rs +++ b/src/config/loader.rs @@ -551,8 +551,7 @@ impl Default for BufferConfig { } } -/// Where the bundled filebeat pipeline and the engine's file set put the -/// enrichment tables. +/// Where the engine's file set puts the enrichment tables. pub const DEFAULT_TRANSFORMS_DATA_DIR: &str = "/etc/dfe-transform-vector/data"; /// Transform file loading configuration. diff --git a/src/config/validate.rs b/src/config/validate.rs index 8093471..f41f7bd 100644 --- a/src/config/validate.rs +++ b/src/config/validate.rs @@ -664,8 +664,7 @@ mod tests { ))); } - /// A transform file declaring a `file` table at `path`, the shape the - /// bundled filebeat pipeline ships. + /// A transform file declaring a `file` table at `path`. fn file_table(path: &Path) -> String { format!( "enrichment_tables:\n timezones:\n type: file\n file:\n path: {}\n \ @@ -678,7 +677,7 @@ mod tests { fn a_missing_table_file_is_named_with_its_path() { let tmp = tempfile::tempdir().expect("tempdir"); let missing = tmp.path().join("data/timezones.csv"); - write_yaml_file(tmp.path(), "50_000_filebeat.yaml", &file_table(&missing)); + write_yaml_file(tmp.path(), "50_000_tables.yaml", &file_table(&missing)); let unreadable = unreadable_enrichment_tables(tmp.path()); assert_eq!(unreadable.len(), 1, "{unreadable:?}"); @@ -698,7 +697,7 @@ mod tests { let tmp = tempfile::tempdir().expect("tempdir"); let table = tmp.path().join("timezones.csv"); std::fs::write(&table, "abbreviation\nUTC\n").expect("write table"); - write_yaml_file(tmp.path(), "50_000_filebeat.yaml", &file_table(&table)); + write_yaml_file(tmp.path(), "50_000_tables.yaml", &file_table(&table)); assert_eq!(unreadable_enrichment_tables(tmp.path()), Vec::new()); } @@ -708,7 +707,7 @@ mod tests { let tmp = tempfile::tempdir().expect("tempdir"); let dir = tmp.path().join("timezones.csv"); std::fs::create_dir(&dir).expect("create dir"); - write_yaml_file(tmp.path(), "50_000_filebeat.yaml", &file_table(&dir)); + write_yaml_file(tmp.path(), "50_000_tables.yaml", &file_table(&dir)); let unreadable = unreadable_enrichment_tables(tmp.path()); assert_eq!(unreadable.len(), 1, "{unreadable:?}"); @@ -765,7 +764,7 @@ mod tests { let config_dir = tmp.path().join("config"); std::fs::create_dir(&config_dir).expect("config dir"); let missing = tmp.path().join("timezones.csv"); - write_yaml_file(&config_dir, "50_000_filebeat.yaml", &file_table(&missing)); + write_yaml_file(&config_dir, "50_000_tables.yaml", &file_table(&missing)); let vector_config = VectorConfig { binary: always_valid_binary(tmp.path()) diff --git a/tests/TESTING.md b/tests/TESTING.md index 41119f6..76eba41 100644 --- a/tests/TESTING.md +++ b/tests/TESTING.md @@ -17,7 +17,7 @@ There is no mode switch. `KafkaFixture::acquire` resolves a broker on its own: `KafkaFixture::hermetic` skips step 1 outright and takes the shared broker. Anything using the real DFE topic names takes that path, because -`filebeat_land` on a shared live broker is somebody else's data. +`e2e_syslog_land` on a shared live broker is somebody else's data. `KafkaFixture::scram` starts a SCRAM-only broker the test holds alone. scalo names every container under the suite prefix, labels it with the repo, @@ -51,12 +51,8 @@ cargo nextest run --run-ignored all Opt-in (`#[ignore]`) cases are the full tier: `vector_validate` runs the real validator over each buffer, SASL and TLS variant of an assembled config, and two `e2e::kafka` pipelines drive a bare Vector process against a broker. -## The WS21 acceptance case +## The app over Kafka -`e2e::filebeat_kafka` runs in core, on the shared broker, which no live cluster's data shares. -It seeds the synthetic filebeat cases onto `filebeat_land`, runs the shipped -binary over them, and grades `filebeat_load` against the fields each case names. - -The cases (`tests/fixtures/filebeat/cases.yaml`) and the bundled pipeline belong -to dfe-transform-vrl, so the case needs that repo checked out beside this one, -or `DFE_TRANSFORM_VRL_DIR` pointing at it. Without one it prints a skip. +`e2e::app_kafka` runs in core, on the shared broker, which no live cluster's data shares. +It seeds synthetic syslog-style lines onto `e2e_syslog_land`, runs the shipped +binary with an authored transform file, and grades `e2e_syslog_load` against the fields each case names. diff --git a/tests/common/filebeat.rs b/tests/common/filebeat.rs deleted file mode 100644 index 157683b..0000000 --- a/tests/common/filebeat.rs +++ /dev/null @@ -1,203 +0,0 @@ -// A missing or malformed case file is unrecoverable and has no caller to -// report it to, so the loader panics rather than threading a Result through -// every test. -#![allow(dead_code, clippy::panic)] -// Project: dfe-transform-vector -// File: tests/common/filebeat.rs -// Purpose: The filebeat test cases and their comparison, read from dfe-transform-vrl -// Language: Rust -// -// License: BUSL-1.1 -// Copyright: (c) 2026 HYPERI PTY LIMITED - -//! The synthetic filebeat cases in dfe-transform-vrl's -//! `tests/fixtures/filebeat/cases.yaml`, and what counts as agreeing with one. -//! -//! Both transform apps run the SAME bundled pipeline over the SAME cases, and a -//! parity test only means something if both grade on the same curve. -//! dfe-transform-vrl owns the cases and the bundled VRL, and this is the reader -//! for them. It mirrors `dfe-transform-vrl/tests/common/filebeat.rs`. - -use std::collections::HashSet; -use std::path::{Path, PathBuf}; -use std::sync::OnceLock; - -use serde::Deserialize; - -/// The dfe-transform-vrl checkout that owns the cases and the pipeline. -/// -/// `DFE_TRANSFORM_VRL_DIR` first, then a `dfe-transform-vrl` sitting beside -/// any ancestor of this checkout -- which finds it from a plain clone and from -/// a `.worktrees/` worktree alike, without naming anybody's workspace -/// layout. `None` means the acceptance case cannot run here, and the test says -/// so rather than asserting on nothing. -pub fn vrl_repo() -> Option<&'static Path> { - static DIR: OnceLock> = OnceLock::new(); - DIR.get_or_init(|| { - let has_pipeline = |dir: &Path| dir.join("pipelines/filebeat/filebeat.vrl").is_file(); - if let Some(explicit) = std::env::var_os("DFE_TRANSFORM_VRL_DIR") { - let dir = PathBuf::from(explicit); - return has_pipeline(&dir).then_some(dir); - } - Path::new(env!("CARGO_MANIFEST_DIR")) - .ancestors() - .map(|a| a.join("dfe-transform-vrl")) - .find(|dir| has_pipeline(dir)) - }) - .as_deref() -} - -/// The bundled filebeat pipeline, or `None` without a dfe-transform-vrl checkout. -pub fn pipeline_vrl() -> Option { - vrl_repo().map(|dir| dir.join("pipelines/filebeat/filebeat.vrl")) -} - -/// The timezones lookup the ios and meraki branches need. -pub fn timezones_csv() -> Option { - vrl_repo().map(|dir| dir.join("pipelines/filebeat/timezones.csv")) -} - -// ========================================================================= -// The cases -// ========================================================================= - -/// The filebeat module a case's line belongs to. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum Module { - Meraki, - Ios, - Umbrella, -} - -/// One input line and what the pipeline must make of it. -#[derive(Debug, Clone, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct Case { - /// Unique across the file. The e2e suite carries it through the broker to - /// pair each output with its case. - pub name: String, - pub module: Module, - /// The `_conf` object the DFE 2.1 deployment injected (`tz_offset`). - #[serde(default)] - pub conf: Option, - /// The raw log line, as the `message` field. - pub line: String, - /// A subset of the transformed event: every leaf named here must be - /// present and equal in the output. Fields the case does not name are not - /// checked. - pub expect: serde_json::Value, -} - -/// Every case in the file, read once. Callers check [`vrl_repo`] first. -pub fn cases() -> &'static [Case] { - static CASES: OnceLock> = OnceLock::new(); - CASES.get_or_init(|| { - let path = vrl_repo() - .expect("dfe-transform-vrl checkout must be present") - .join("tests/fixtures/filebeat/cases.yaml"); - let text = std::fs::read_to_string(&path) - .unwrap_or_else(|e| panic!("read {}: {e}", path.display())); - let cases: Vec = - scalo::yaml::from_str(&text, "cases.yaml", scalo::yaml::Profile::Data) - .unwrap_or_else(|e| panic!("parse cases.yaml: {e}")); - let mut seen = HashSet::new(); - for case in &cases { - assert!( - seen.insert(case.name.as_str()), - "duplicate case name {}", - case.name - ); - } - cases - }) -} - -/// A DFE 2.1-shaped input event: `{message, tags, timestamp}` plus the -/// `_conf` object the 2.1 deployment injected (`tz_offset`, `tz_map`). -pub fn input_event(message: &str, conf: Option<&serde_json::Value>) -> serde_json::Value { - let mut obj = serde_json::json!({ - "message": message, - "tags": [], - "timestamp": "2026-01-01T00:00:00Z", - }); - if let Some(conf) = conf { - obj["_conf"] = conf.clone(); - } - obj -} - -/// The input event for one case. -pub fn case_event(case: &Case) -> serde_json::Value { - input_event(&case.line, case.conf.as_ref()) -} - -// ========================================================================= -// What counts as agreeing with a case -// ========================================================================= - -/// Compare one transformed event against a case's `expect`: every leaf in -/// `expected` must be present and equal in `ours`. Extra fields on our side -/// (DFE 2.1 event carriage like `_conf`, `timestamp`, and the fields Vector's -/// kafka source adds) are fine. The root `tags` array compares as a set, -/// because the 2.1 template seeds tags in a fixed but arbitrary order. -pub fn diff_expected( - path: &str, - expected: &serde_json::Value, - ours: &serde_json::Value, - problems: &mut Vec, -) { - match expected { - serde_json::Value::Object(map) => { - for (k, v) in map { - let child = if path.is_empty() { - k.clone() - } else { - format!("{path}.{k}") - }; - match ours.get(k) { - Some(o) => diff_expected(&child, v, o, problems), - None => problems.push(format!("{child}: missing (expected {v})")), - } - } - } - serde_json::Value::Array(items) if path == "tags" => { - let have = ours.as_array().map(Vec::as_slice).unwrap_or_default(); - for item in items { - if !have.contains(item) { - problems.push(format!("tags: missing {item}")); - } - } - } - serde_json::Value::Array(items) => match ours.as_array() { - Some(have) if have.len() == items.len() => { - for (i, (e, o)) in items.iter().zip(have).enumerate() { - diff_expected(&format!("{path}[{i}]"), e, o, problems); - } - } - _ => problems.push(format!("{path}: expected array {expected}, got {ours}")), - }, - leaf => { - if ours != leaf { - problems.push(format!("{path}: expected {leaf}, got {ours}")); - } - } - } -} - -/// Everything wrong with `ours` as the transform of `case`. Empty means it -/// agrees. -pub fn problems_for(case: &Case, ours: &serde_json::Value) -> Vec { - let mut problems = Vec::new(); - diff_expected("", &case.expect, ours, &mut problems); - problems -} - -/// Whether the pipeline routed this event to the shape-unmatched catch-all. -/// Every case belongs to a known module, so this must never fire on one. -pub fn is_unmatched(event: &serde_json::Value) -> bool { - event - .get("tags") - .and_then(|t| t.as_array()) - .is_some_and(|t| t.contains(&serde_json::json!("filebeat_unmatched"))) -} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 511e997..9986737 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -117,7 +117,7 @@ impl KafkaFixture { /// Acquire the run's shared broker container, never the live cluster. /// /// `acquire` prefers the live cluster, which is wrong for a test that uses - /// the real DFE topic names: `filebeat_land` on a shared live broker is + /// the real DFE topic names: `e2e_syslog_land` on a shared live broker is /// somebody else's data. Returns `None` when no container can be started. pub async fn hermetic(_test: &str) -> Option { match shared_broker().await { diff --git a/tests/e2e.rs b/tests/e2e.rs index 19eebdf..3a1cbb8 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -11,9 +11,5 @@ mod common; -/// The filebeat cases and their comparison, owned by dfe-transform-vrl. -#[path = "common/filebeat.rs"] -mod filebeat_cases; - #[path = "e2e/mod.rs"] mod e2e; diff --git a/tests/e2e/filebeat_kafka.rs b/tests/e2e/app_kafka.rs similarity index 56% rename from tests/e2e/filebeat_kafka.rs rename to tests/e2e/app_kafka.rs index fed5918..5bf67e4 100644 --- a/tests/e2e/filebeat_kafka.rs +++ b/tests/e2e/app_kafka.rs @@ -1,40 +1,21 @@ // Project: dfe-transform-vector -// File: tests/e2e/filebeat_kafka.rs -// Purpose: WS21 parity: the filebeat cases through the app, Vector and a broker +// File: tests/e2e/app_kafka.rs +// Purpose: The app binary over a real broker with an authored transform file // Language: Rust // // License: BUSL-1.1 // Copyright: (c) 2026 HYPERI PTY LIMITED -//! The transform hop of the WS21 chain, with Vector doing the transforming. +//! The shipped binary reading `e2e_syslog_land`, transforming with Vector and +//! writing `e2e_syslog_load`. //! -//! ```text -//! filebeat cases -> filebeat_land -> dfe-transform-vector -> filebeat_load -//! ``` +//! The instance is bound to its source by NAME: `dfe_source` is the only +//! topic setting in the config, and both topics and the consumer group derive +//! from it. The app also runs `vector validate` before it starts Vector, so a +//! transform file Vector cannot compile fails here. //! -//! This is the parity half of the acceptance case: the same cases, the same -//! bundled filebeat VRL, and the same assertions as -//! `dfe-transform-vrl/tests/e2e/filebeat_kafka.rs`. Only the transform app -//! differs, which is what makes WS21 a test of the transform LAYER rather -//! than of one app. -//! -//! What it exercises that a config test cannot: -//! -//! - The instance is bound to its source by NAME. `dfe_source: filebeat` is -//! the only topic setting in the config; `filebeat_land`, `filebeat_load` -//! and the consumer group are all derived from it. -//! - The VRL arrives as an authored transform file, wrapped in a `remap`, -//! with its lookup table declared alongside it -- so a top-level -//! `enrichment_tables:` reaching Vector through the assembler is proved, -//! not assumed. -//! - The app runs `vector validate` before starting, so a pipeline Vector's -//! VRL cannot compile fails here rather than silently passing nothing on. -//! -//! The cases and the pipeline belong to dfe-transform-vrl. Without that -//! checkout there is nothing to run against and the test says so. -//! -//! NOT covered here, because it needs a cluster: the receiver hop that picks -//! the source, and the loader hop that lands rows in `ClickHouse`. +//! The receiver hop that picks the source and the loader hop that lands rows +//! in `ClickHouse` need a cluster and are not covered. use std::time::{Duration, Instant}; @@ -42,41 +23,78 @@ use scalo::transport::kafka::{KafkaAdmin, KafkaConfig, KafkaTransport}; use scalo::transport::{TransportBase, TransportReceiver, TransportSender}; use crate::common::{self, KafkaFixture}; -use crate::filebeat_cases as fb; -/// The source this instance is bound to, and the topics that follow from it. -const SOURCE: &str = "filebeat"; -const LAND_TOPIC: &str = "filebeat_land"; -const LOAD_TOPIC: &str = "filebeat_load"; +const SOURCE: &str = "e2e_syslog"; +const LAND_TOPIC: &str = "e2e_syslog_land"; +const LOAD_TOPIC: &str = "e2e_syslog_load"; -/// A line no filebeat module claims, carried through to prove the -/// shape-unmatched catch-all still passes an event on rather than dropping it. -const UNMATCHED_PROBE: &str = "ws21 probe: a line no filebeat module claims"; +/// A synthetic line and the fields the transform must set, or `None` where it +/// must tag the event `unparsed` and pass it on. +struct Case { + name: &'static str, + line: &'static str, + expect: Option<(&'static str, &'static str)>, +} -/// How long the app gets to assemble the config, run `vector validate` over -/// the 5.5K-line pipeline, and start Vector. -const READY_TIMEOUT: Duration = Duration::from_secs(240); +const CASES: &[Case] = &[ + Case { + name: "denied", + line: "denied src=203.0.113.7", + expect: Some(("203.0.113.7", "denied")), + }, + Case { + name: "allowed", + line: "allowed src=198.51.100.23", + expect: Some(("198.51.100.23", "allowed")), + }, + Case { + name: "reset", + line: "conn_reset src=192.0.2.1", + expect: Some(("192.0.2.1", "conn_reset")), + }, + Case { + name: "no_source", + line: "a line with no key value pairs", + expect: None, + }, + Case { + name: "bad_ip", + line: "denied src=not-an-address", + expect: None, + }, +]; + +/// How long the app gets to assemble the config, validate and start Vector. +const READY_TIMEOUT: Duration = Duration::from_secs(120); /// How long the cases get to arrive on the sink topic once the app is ready. -const DRAIN_TIMEOUT: Duration = Duration::from_secs(180); +const DRAIN_TIMEOUT: Duration = Duration::from_secs(120); + +const TRANSFORM: &str = r#"transforms: + parse_line: + type: remap + inputs: + - dfe_source + source: | + parsed, err = parse_regex(.message, r'^(?P[a-z_]+) src=(?P[0-9.]+)$') + if err == null { + .source.ip = parsed.ip + .event.action = parsed.action + } else { + .tags = ["unparsed"] + } +"#; -/// Every case as Vector will read it off `filebeat_land`, plus the -/// shape-unmatched probe. -/// -/// Each case event carries its name in a root `_case` field. The pipeline keeps -/// root fields on every branch, so the name comes out the far side and pairs -/// an output with its case whatever order the broker returns them in. fn workload() -> Vec { - let mut inputs: Vec = fb::cases() + CASES .iter() - .map(|case| { - let mut event = fb::case_event(case); - event["_case"] = serde_json::json!(case.name); - event - }) - .collect(); - inputs.push(fb::input_event(UNMATCHED_PROBE, None)); - inputs + .map(|c| serde_json::json!({ "message": c.line, "_case": c.name })) + .collect() +} + +fn write_transform(dir: &std::path::Path) { + std::fs::create_dir_all(dir).expect("create the transforms dir"); + std::fs::write(dir.join("01_parse_line.yaml"), TRANSFORM).expect("write the transform file"); } fn client_config(base: &KafkaConfig, topics: &[&str], group: &str) -> KafkaConfig { @@ -99,50 +117,6 @@ fn free_port() -> u16 { port } -/// The bundled filebeat VRL as a Vector `remap` transform, with the lookup -/// table it needs declared in the same file. -/// -/// The top-level `enrichment_tables` key is the point: the assembler writes -/// this file into Vector's config dir verbatim, so Vector merges the -/// declaration and the VRL's `get_enrichment_table_record` calls resolve. -fn write_transform(dir: &std::path::Path) { - let vrl = std::fs::read_to_string(fb::pipeline_vrl().expect("the bundled pipeline")) - .expect("read the bundled filebeat.vrl"); - let timezones = fb::timezones_csv().expect("the bundled timezones table"); - - let doc = serde_json::json!({ - "enrichment_tables": { - "timezones": { - "type": "file", - "file": { - "path": timezones.to_string_lossy(), - "encoding": { "type": "csv" }, - }, - "schema": { "abbreviation": "string" }, - }, - }, - "transforms": { - "filebeat": { - "type": "remap", - "inputs": ["dfe_source"], - "source": vrl, - }, - }, - }); - - std::fs::create_dir_all(dir).expect("create the transforms dir"); - std::fs::write( - dir.join("01_filebeat.yaml"), - scalo::yaml::to_string(&doc).expect("transform serialises"), - ) - .expect("write the transform file"); -} - -/// The config file the app runs from. -/// -/// `dfe_source` is the ONLY topic setting: leaving `source.topics`, -/// `sink.topic` and `source.group_id` at their defaults is what lets the app -/// derive them, which is the binding under test. fn write_config( dir: &std::path::Path, brokers: &[String], @@ -165,13 +139,7 @@ fn write_config( "encoding": "json", "compression": "none", }, - "transforms": { - "dir": transforms_dir.to_string_lossy(), - "data_dir": fb::timezones_csv() - .and_then(|table| table.parent().map(std::path::Path::to_path_buf)) - .expect("the bundled timezones table") - .to_string_lossy(), - }, + "transforms": { "dir": transforms_dir.to_string_lossy() }, "vector": { "binary": vector_binary.to_string_lossy(), "data_dir": data_dir.to_string_lossy(), @@ -280,49 +248,14 @@ async fn drain( out } -/// Grade every case against the output event that carries its name. -/// Returns one line per case that has no output or whose output disagrees. -fn grade_cases(outputs: &[serde_json::Value]) -> Vec { - let mut failures = Vec::new(); - for case in fb::cases() { - let found = outputs - .iter() - .find(|e| e.get("_case").and_then(serde_json::Value::as_str) == Some(&case.name)); - match found { - None => failures.push(format!("{}: no output event carried its name", case.name)), - Some(event) if fb::is_unmatched(event) => { - failures.push(format!("{}: routed to filebeat_unmatched", case.name)); - } - Some(event) => { - let problems = fb::problems_for(case, event); - if !problems.is_empty() { - failures.push(format!("{}: {}", case.name, problems.join("; "))); - } - } - } - } - failures -} - #[tokio::test] -async fn filebeat_cases_round_trip_through_vector() { - if fb::vrl_repo().is_none() { - common::require_service_in_ci( - "dfe-transform-vrl checkout", - "the cases and the bundled pipeline grade this test, and CI clones the repo beside this one", - ); - eprintln!( - "SKIP: the cases and the bundled pipeline live in dfe-transform-vrl. \ - Point DFE_TRANSFORM_VRL_DIR at a checkout to run this." - ); - return; - } +async fn syslog_lines_round_trip_through_vector() { let Some(vector_binary) = common::vector_binary_path() else { common::require_service_in_ci("Vector binary", "scripts/fetch-vector.sh found nothing"); eprintln!("SKIP: no Vector binary (run scripts/fetch-vector.sh)"); return; }; - let Some(fixture) = KafkaFixture::hermetic("filebeat-cases-round-trip").await else { + let Some(fixture) = KafkaFixture::hermetic("syslog-round-trip").await else { common::require_service_in_ci("Kafka", "no Docker to start a broker container"); eprintln!("SKIP: no Docker, so this test cannot own a broker."); return; @@ -331,18 +264,18 @@ async fn filebeat_cases_round_trip_through_vector() { let inputs = workload(); eprintln!( - "WS21: replaying {} events through {LAND_TOPIC} -> {LOAD_TOPIC}", + "replaying {} events through {LAND_TOPIC} -> {LOAD_TOPIC}", inputs.len() ); let admin = - KafkaAdmin::new(&client_config(&base, &[LAND_TOPIC], "ws21-admin")).expect("admin client"); + KafkaAdmin::new(&client_config(&base, &[LAND_TOPIC], "admin")).expect("admin client"); admin .create_topics(&[(LAND_TOPIC, 1, 1), (LOAD_TOPIC, 1, 1)]) .await .expect("create the source-bound topics"); - let producer = KafkaTransport::new(&client_config(&base, &[LAND_TOPIC], "ws21-seed")) + let producer = KafkaTransport::new(&client_config(&base, &[LAND_TOPIC], "seed")) .await .expect("seed producer"); // `send`'s first argument is the DESTINATION, which for Kafka is the topic @@ -365,7 +298,7 @@ async fn filebeat_cases_round_trip_through_vector() { let seeded = drain( &base, LAND_TOPIC, - "ws21-seed-check", + "seed-check", inputs.len(), Duration::from_secs(60), ) @@ -429,14 +362,7 @@ async fn filebeat_cases_round_trip_through_vector() { "dfe-transform-vector did not report ready on {probes} within {READY_TIMEOUT:?}" ); - let outputs = drain( - &base, - LOAD_TOPIC, - "ws21-verify", - inputs.len(), - DRAIN_TIMEOUT, - ) - .await; + let outputs = drain(&base, LOAD_TOPIC, "verify", inputs.len(), DRAIN_TIMEOUT).await; stop(&mut app).await; // Assertion 1 -- everything that went in came out. @@ -448,45 +374,44 @@ async fn filebeat_cases_round_trip_through_vector() { outputs.len() ); - // Assertion 2 -- the shape-unmatched catch-all passes an event through - // tagged, rather than dropping it. - let probe_at = outputs - .iter() - .position(|e| e.get("message").and_then(serde_json::Value::as_str) == Some(UNMATCHED_PROBE)) - .expect("the unmatched probe must arrive on the sink topic"); - assert!( - fb::is_unmatched(&outputs[probe_at]), - "an unclaimed line must carry filebeat_unmatched: {}", - outputs[probe_at] - ); - - // Assertion 3 -- every case agrees with the fields it names. - let failures = grade_cases(&outputs); + // Assertion 2 -- each line carries the fields its case names. + let mut failures = Vec::new(); + for case in CASES { + let Some(event) = outputs + .iter() + .find(|e| e.get("_case").and_then(serde_json::Value::as_str) == Some(case.name)) + else { + failures.push(format!("{}: no output event carried its name", case.name)); + continue; + }; + let ip = event + .pointer("/source/ip") + .and_then(serde_json::Value::as_str); + let action = event + .pointer("/event/action") + .and_then(serde_json::Value::as_str); + let unparsed = event + .pointer("/tags") + .and_then(serde_json::Value::as_array) + .is_some_and(|t| t.iter().any(|v| v == "unparsed")); + match case.expect { + Some((want_ip, want_action)) => { + if ip != Some(want_ip) || action != Some(want_action) || unparsed { + failures.push(format!("{}: got {event}", case.name)); + } + } + None => { + if !unparsed || ip.is_some() || action.is_some() { + failures.push(format!("{}: expected unparsed, got {event}", case.name)); + } + } + } + } assert!( failures.is_empty(), "{} of {} cases did not match:\n{}", failures.len(), - fb::cases().len(), + CASES.len(), failures.join("\n") ); - - // Assertion 4 -- all three module branches actually ran. A pipeline that - // detected nothing would still satisfy the count above by passing - // everything through unmatched. - let branch_ran = |module: &str, probe: &dyn Fn(&serde_json::Value) -> bool| { - assert!( - outputs.iter().any(probe), - "no output event came from the {module} branch" - ); - }; - let product_is = |event: &serde_json::Value, want: &str| { - event - .get("observer") - .and_then(|o| o.get("product")) - .and_then(serde_json::Value::as_str) - == Some(want) - }; - branch_ran("umbrella", &|e| product_is(e, "Umbrella")); - branch_ran("ios", &|e| product_is(e, "IOS")); - branch_ran("meraki", &|e| e.get("cisco_meraki").is_some()); } diff --git a/tests/e2e/mod.rs b/tests/e2e/mod.rs index bd701b2..d58be23 100644 --- a/tests/e2e/mod.rs +++ b/tests/e2e/mod.rs @@ -6,7 +6,7 @@ // License: BUSL-1.1 // Copyright: (c) 2026 HYPERI PTY LIMITED -mod filebeat_kafka; +mod app_kafka; mod kafka; // The Vector metrics merge is asserted inside `kafka` against a real Vector, // and against a fake exporter in `tests/integration/metrics.rs`. diff --git a/tests/integration/vector_validate.rs b/tests/integration/vector_validate.rs index 888c502..c3d5b46 100644 --- a/tests/integration/vector_validate.rs +++ b/tests/integration/vector_validate.rs @@ -416,11 +416,11 @@ async fn a_table_file_vector_validate_passes_but_cannot_load_fails_validation() fs::create_dir_all(&transforms).expect("transforms dir"); let table = work_dir.path().join("data/timezones.csv"); fs::write( - transforms.join("01_filebeat.yaml"), + transforms.join("01_tables.yaml"), format!( "enrichment_tables:\n timezones:\n type: file\n file:\n path: {}\n \ encoding:\n type: csv\n schema:\n abbreviation: string\n\ - transforms:\n filebeat:\n type: remap\n inputs: [dfe_source]\n \ + transforms:\n tables:\n type: remap\n inputs: [dfe_source]\n \ source: |\n .seen = true\n", table.display() ),