diff --git a/apps/web/drizzle/0005_maad_measures.sql b/apps/web/drizzle/0005_maad_measures.sql new file mode 100644 index 00000000..06680500 --- /dev/null +++ b/apps/web/drizzle/0005_maad_measures.sql @@ -0,0 +1,29 @@ +PRAGMA foreign_keys=OFF;--> statement-breakpoint +CREATE TABLE `__new_address_structure_stats` ( + `source_id` text NOT NULL, + `granularity` text NOT NULL, + `bucket_start` integer NOT NULL, + `bucket_end` integer NOT NULL, + `ip_version` integer NOT NULL, + `src_locality` text NOT NULL, + `dst_locality` text NOT NULL, + `address_side` text NOT NULL, + `measure` text NOT NULL, + `structure_kind` text NOT NULL, + `values_json` text NOT NULL, + `metadata_json` text NOT NULL, + `processed_at` text DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(`source_id`, `granularity`, `bucket_start`, `ip_version`, `src_locality`, `dst_locality`, `address_side`, `measure`, `structure_kind`), + CONSTRAINT "address_structure_stats_ip_version_check" CHECK("__new_address_structure_stats"."ip_version" IN (4, 6)), + CONSTRAINT "address_structure_stats_src_locality_check" CHECK("__new_address_structure_stats"."src_locality" IN ('all', 'internal', 'external')), + CONSTRAINT "address_structure_stats_dst_locality_check" CHECK("__new_address_structure_stats"."dst_locality" IN ('all', 'internal', 'external')), + CONSTRAINT "address_structure_stats_measure_check" CHECK("__new_address_structure_stats"."measure" IN ('addresses', 'packets', 'bytes')), + CONSTRAINT "address_structure_stats_measure_spectrum_check" CHECK("__new_address_structure_stats"."measure" = 'addresses' OR "__new_address_structure_stats"."structure_kind" <> 'spectrum') +); +--> statement-breakpoint +INSERT INTO `__new_address_structure_stats`("source_id", "granularity", "bucket_start", "bucket_end", "ip_version", "src_locality", "dst_locality", "address_side", "measure", "structure_kind", "values_json", "metadata_json", "processed_at") SELECT "source_id", "granularity", "bucket_start", "bucket_end", "ip_version", "src_locality", "dst_locality", "address_side", 'addresses', "structure_kind", "values_json", "metadata_json", "processed_at" FROM `address_structure_stats`;--> statement-breakpoint +DROP TABLE `address_structure_stats`;--> statement-breakpoint +ALTER TABLE `__new_address_structure_stats` RENAME TO `address_structure_stats`;--> statement-breakpoint +PRAGMA foreign_keys=ON;--> statement-breakpoint +CREATE INDEX `idx_address_structure_stats_query` ON `address_structure_stats` (`granularity`,`bucket_start`,`source_id`,`ip_version`,`src_locality`,`dst_locality`,`address_side`,`measure`,`structure_kind`);--> statement-breakpoint +CREATE INDEX `idx_address_structure_stats_timeseries` ON `address_structure_stats` (`source_id`,`granularity`,`src_locality`,`dst_locality`,`ip_version`,`measure`,`structure_kind`,`bucket_start`); \ No newline at end of file diff --git a/apps/web/drizzle/meta/0005_snapshot.json b/apps/web/drizzle/meta/0005_snapshot.json new file mode 100644 index 00000000..7858b439 --- /dev/null +++ b/apps/web/drizzle/meta/0005_snapshot.json @@ -0,0 +1,1106 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "346de28f-0cfb-4986-8daf-28235ade1fdf", + "prevId": "4a3c1a73-bfe4-4e21-95c9-7ea8a091979f", + "tables": { + "address_count_stats": { + "name": "address_count_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "address_side": { + "name": "address_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_address_count": { + "name": "unique_address_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_address_count_stats_query": { + "name": "idx_address_count_stats_query", + "columns": [ + "granularity", + "bucket_start", + "source_id", + "ip_version", + "src_locality", + "dst_locality", + "address_side" + ], + "isUnique": false + }, + "idx_address_count_stats_timeseries": { + "name": "idx_address_count_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "address_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality", + "address_side" + ], + "name": "address_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "address_count_stats_ip_version_check": { + "name": "address_count_stats_ip_version_check", + "value": "\"address_count_stats\".\"ip_version\" IN (4, 6)" + }, + "address_count_stats_src_locality_check": { + "name": "address_count_stats_src_locality_check", + "value": "\"address_count_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "address_count_stats_dst_locality_check": { + "name": "address_count_stats_dst_locality_check", + "value": "\"address_count_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "address_structure_stats": { + "name": "address_structure_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "address_side": { + "name": "address_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "measure": { + "name": "measure", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "structure_kind": { + "name": "structure_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "values_json": { + "name": "values_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata_json": { + "name": "metadata_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_address_structure_stats_query": { + "name": "idx_address_structure_stats_query", + "columns": [ + "granularity", + "bucket_start", + "source_id", + "ip_version", + "src_locality", + "dst_locality", + "address_side", + "measure", + "structure_kind" + ], + "isUnique": false + }, + "idx_address_structure_stats_timeseries": { + "name": "idx_address_structure_stats_timeseries", + "columns": [ + "source_id", + "granularity", + "src_locality", + "dst_locality", + "ip_version", + "measure", + "structure_kind", + "bucket_start" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "address_structure_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_measure_structure_kind_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality", + "address_side", + "measure", + "structure_kind" + ], + "name": "address_structure_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_measure_structure_kind_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "address_structure_stats_ip_version_check": { + "name": "address_structure_stats_ip_version_check", + "value": "\"address_structure_stats\".\"ip_version\" IN (4, 6)" + }, + "address_structure_stats_src_locality_check": { + "name": "address_structure_stats_src_locality_check", + "value": "\"address_structure_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "address_structure_stats_dst_locality_check": { + "name": "address_structure_stats_dst_locality_check", + "value": "\"address_structure_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + }, + "address_structure_stats_measure_check": { + "name": "address_structure_stats_measure_check", + "value": "\"address_structure_stats\".\"measure\" IN ('addresses', 'packets', 'bytes')" + }, + "address_structure_stats_measure_spectrum_check": { + "name": "address_structure_stats_measure_spectrum_check", + "value": "\"address_structure_stats\".\"measure\" = 'addresses' OR \"address_structure_stats\".\"structure_kind\" <> 'spectrum'" + } + } + }, + "bucket_coverage": { + "name": "bucket_coverage", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "coverage_state": { + "name": "coverage_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "observed_units": { + "name": "observed_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expected_units": { + "name": "expected_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rejected_units": { + "name": "rejected_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_bucket_coverage_query": { + "name": "idx_bucket_coverage_query", + "columns": ["granularity", "bucket_start", "source_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "bucket_coverage_source_id_granularity_bucket_start_pk": { + "columns": ["source_id", "granularity", "bucket_start"], + "name": "bucket_coverage_source_id_granularity_bucket_start_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "bucket_coverage_interval_check": { + "name": "bucket_coverage_interval_check", + "value": "\"bucket_coverage\".\"bucket_end\" > \"bucket_coverage\".\"bucket_start\"" + }, + "bucket_coverage_expected_check": { + "name": "bucket_coverage_expected_check", + "value": "\"bucket_coverage\".\"expected_units\" > 0" + }, + "bucket_coverage_observed_check": { + "name": "bucket_coverage_observed_check", + "value": "\"bucket_coverage\".\"observed_units\" >= 0 AND \"bucket_coverage\".\"observed_units\" <= \"bucket_coverage\".\"expected_units\"" + }, + "bucket_coverage_rejected_check": { + "name": "bucket_coverage_rejected_check", + "value": "\"bucket_coverage\".\"rejected_units\" >= 0 AND \"bucket_coverage\".\"rejected_units\" <= \"bucket_coverage\".\"expected_units\"" + }, + "bucket_coverage_state_check": { + "name": "bucket_coverage_state_check", + "value": "(\"bucket_coverage\".\"coverage_state\" = 'complete' AND \"bucket_coverage\".\"observed_units\" = \"bucket_coverage\".\"expected_units\" AND \"bucket_coverage\".\"rejected_units\" = 0) OR (\"bucket_coverage\".\"coverage_state\" = 'unknown' AND \"bucket_coverage\".\"observed_units\" = 0 AND \"bucket_coverage\".\"rejected_units\" = 0) OR (\"bucket_coverage\".\"coverage_state\" = 'partial' AND NOT (\"bucket_coverage\".\"observed_units\" = \"bucket_coverage\".\"expected_units\" AND \"bucket_coverage\".\"rejected_units\" = 0) AND NOT (\"bucket_coverage\".\"observed_units\" = 0 AND \"bucket_coverage\".\"rejected_units\" = 0))" + } + } + }, + "datasets": { + "name": "datasets", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_start_date": { + "name": "default_start_date", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_mode": { + "name": "source_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'static'" + }, + "discovery_mode": { + "name": "discovery_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'static'" + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "has_locality": { + "name": "has_locality", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "port_count_stats": { + "name": "port_count_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "port_side": { + "name": "port_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "port_range": { + "name": "port_range", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_port_count": { + "name": "unique_port_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_port_count_stats_timeseries": { + "name": "idx_port_count_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "port_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_port_side_port_range_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality", + "port_side", + "port_range" + ], + "name": "port_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_port_side_port_range_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "port_count_stats_ip_version_check": { + "name": "port_count_stats_ip_version_check", + "value": "\"port_count_stats\".\"ip_version\" IN (4, 6)" + }, + "port_count_stats_src_locality_check": { + "name": "port_count_stats_src_locality_check", + "value": "\"port_count_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "port_count_stats_dst_locality_check": { + "name": "port_count_stats_dst_locality_check", + "value": "\"port_count_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "processed_inputs": { + "name": "processed_inputs", + "columns": { + "input_kind": { + "name": "input_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_locator": { + "name": "input_locator", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "discovered_at": { + "name": "discovered_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_processed_inputs_source_bucket": { + "name": "idx_processed_inputs_source_bucket", + "columns": ["source_id", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "processed_inputs_input_kind_input_locator_source_id_bucket_start_pk": { + "columns": ["input_kind", "input_locator", "source_id", "bucket_start"], + "name": "processed_inputs_input_kind_input_locator_source_id_bucket_start_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "protocol_stats": { + "name": "protocol_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_protocols_count": { + "name": "unique_protocols_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "protocols_list": { + "name": "protocols_list", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_protocol_stats_timeseries": { + "name": "idx_protocol_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "protocol_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality" + ], + "name": "protocol_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "protocol_stats_ip_version_check": { + "name": "protocol_stats_ip_version_check", + "value": "\"protocol_stats\".\"ip_version\" IN (4, 6)" + }, + "protocol_stats_src_locality_check": { + "name": "protocol_stats_src_locality_check", + "value": "\"protocol_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "protocol_stats_dst_locality_check": { + "name": "protocol_stats_dst_locality_check", + "value": "\"protocol_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "source_members": { + "name": "source_members", + "columns": { + "dataset_id": { + "name": "dataset_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "member_id": { + "name": "member_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "source_members_dataset_id_source_id_member_id_pk": { + "columns": ["dataset_id", "source_id", "member_id"], + "name": "source_members_dataset_id_source_id_member_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "traffic_stats": { + "name": "traffic_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows": { + "name": "flows", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_tcp": { + "name": "flows_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_udp": { + "name": "flows_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_icmp": { + "name": "flows_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_other": { + "name": "flows_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets": { + "name": "packets", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_tcp": { + "name": "packets_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_udp": { + "name": "packets_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_icmp": { + "name": "packets_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_other": { + "name": "packets_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_tcp": { + "name": "bytes_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_udp": { + "name": "bytes_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_icmp": { + "name": "bytes_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_other": { + "name": "bytes_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_sum_ms": { + "name": "duration_sum_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_count": { + "name": "duration_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_duration_ms": { + "name": "average_duration_ms", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "min_ttl_sum": { + "name": "min_ttl_sum", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "min_ttl_count": { + "name": "min_ttl_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_min_ttl": { + "name": "average_min_ttl", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_ttl_sum": { + "name": "max_ttl_sum", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "max_ttl_count": { + "name": "max_ttl_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_max_ttl": { + "name": "average_max_ttl", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_traffic_stats_query": { + "name": "idx_traffic_stats_query", + "columns": [ + "granularity", + "bucket_start", + "source_id", + "ip_version", + "src_locality", + "dst_locality" + ], + "isUnique": false + }, + "idx_traffic_stats_timeseries": { + "name": "idx_traffic_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "traffic_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality" + ], + "name": "traffic_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "traffic_stats_ip_version_check": { + "name": "traffic_stats_ip_version_check", + "value": "\"traffic_stats\".\"ip_version\" IN (4, 6)" + }, + "traffic_stats_src_locality_check": { + "name": "traffic_stats_src_locality_check", + "value": "\"traffic_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "traffic_stats_dst_locality_check": { + "name": "traffic_stats_dst_locality_check", + "value": "\"traffic_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/apps/web/drizzle/meta/_journal.json b/apps/web/drizzle/meta/_journal.json index ba24b415..0d4ecbd5 100644 --- a/apps/web/drizzle/meta/_journal.json +++ b/apps/web/drizzle/meta/_journal.json @@ -36,6 +36,13 @@ "when": 1790313292395, "tag": "0004_dataset_locality", "breakpoints": true + }, + { + "idx": 5, + "version": "6", + "when": 1790328400092, + "tag": "0005_maad_measures", + "breakpoints": true } ] } diff --git a/apps/web/src/lib/components/filters/MaadMeasureFilter.svelte b/apps/web/src/lib/components/filters/MaadMeasureFilter.svelte new file mode 100644 index 00000000..67c5aee3 --- /dev/null +++ b/apps/web/src/lib/components/filters/MaadMeasureFilter.svelte @@ -0,0 +1,31 @@ + + + diff --git a/apps/web/src/lib/components/netflow/NetflowFileDetailPage.svelte b/apps/web/src/lib/components/netflow/NetflowFileDetailPage.svelte index eebddb0f..9f0f5eb3 100644 --- a/apps/web/src/lib/components/netflow/NetflowFileDetailPage.svelte +++ b/apps/web/src/lib/components/netflow/NetflowFileDetailPage.svelte @@ -6,11 +6,15 @@ import NetflowFileMessageCard from '$lib/components/netflow/NetflowFileMessageCard.svelte'; import NetflowFileRouterCard from '$lib/components/netflow/NetflowFileRouterCard.svelte'; import SegmentedControl from '$lib/components/common/SegmentedControl.svelte'; + import MaadMeasureFilter from '$lib/components/filters/MaadMeasureFilter.svelte'; import { navigateToNetflowFile } from '$lib/utils/netflow-file-navigation'; import { + DEFAULT_MAAD_MEASURE, MAAD_IP_VERSION_OPTIONS, + maadMeasureHasSpectrum, type FlowDirection, - type MaadIpVersion + type MaadIpVersion, + type MaadMeasure } from '$lib/types/types'; import { createDateFromPSTComponents, @@ -36,6 +40,8 @@ let { data }: { data: NetflowFileDetailData } = $props(); let loader = $state.raw | null>(null); + let maadMeasure = $state(DEFAULT_MAAD_MEASURE); + const showSpectrum = $derived(maadMeasureHasSpectrum(maadMeasure)); const formatCount = (value: number | null | undefined) => typeof value === 'number' && Number.isFinite(value) ? value.toLocaleString() : 'N/A'; @@ -59,7 +65,13 @@ const nextSlug = $derived(getNextSlug(data.slug)); function syncLoader() { - loader = getNetflowFileDetailLoader(data.dataset, data.slug, data.direction, data.ipVersion); + loader = getNetflowFileDetailLoader( + data.dataset, + data.slug, + data.direction, + data.ipVersion, + maadMeasure + ); loader.refresh(); } @@ -78,6 +90,14 @@ }); } + function handleMaadMeasureChange({ measure }: { measure: MaadMeasure }) { + if (measure === maadMeasure) { + return; + } + maadMeasure = measure; + syncLoader(); + } + onMount(() => { syncLoader(); }); @@ -106,18 +126,24 @@ : 'N/A'} /> -
- MAAD address family: - ({ - value: String(option.value), - label: option.label - }))} - value={String(data.ipVersion)} - onValueChange={(value) => handleMaadIpVersionChange(Number(value) as MaadIpVersion)} - ariaLabel="Select MAAD IP address family" - buttonClass="sm:min-w-32" - /> +
+
+ MAAD address family: + ({ + value: String(option.value), + label: option.label + }))} + value={String(data.ipVersion)} + onValueChange={(value) => handleMaadIpVersionChange(Number(value) as MaadIpVersion)} + ariaLabel="Select MAAD IP address family" + buttonClass="sm:min-w-32" + /> +
+
+ MAAD measure: + +
{#if loader?.error && !loader.hasRows} @@ -144,7 +170,7 @@ /> {/if} {#each loader.rows as row (row.key)} - + {/each}
{/if} diff --git a/apps/web/src/lib/components/netflow/NetflowFileRouterCard.svelte b/apps/web/src/lib/components/netflow/NetflowFileRouterCard.svelte index 8967bd95..8a897547 100644 --- a/apps/web/src/lib/components/netflow/NetflowFileRouterCard.svelte +++ b/apps/web/src/lib/components/netflow/NetflowFileRouterCard.svelte @@ -6,10 +6,12 @@ let { row, + showSpectrum = true, formatCount, formatTimestampAsPST }: { row: NetflowFileRouterRow; + showSpectrum?: boolean; formatCount: (value: number | null | undefined) => string; formatTimestampAsPST: (timestamp: number) => string; } = $props(); @@ -35,12 +37,18 @@ source={row.source.structure} destination={row.destination.structure} /> - + {#if showSpectrum} + + {:else} +

+ The spectrum is only computed for the addresses measure. +

+ {/if} diff --git a/apps/web/src/lib/components/netflow/file-detail-loader.svelte.ts b/apps/web/src/lib/components/netflow/file-detail-loader.svelte.ts index 29ecf452..e88164ab 100644 --- a/apps/web/src/lib/components/netflow/file-detail-loader.svelte.ts +++ b/apps/web/src/lib/components/netflow/file-detail-loader.svelte.ts @@ -1,8 +1,10 @@ import { DEFAULT_MAAD_IP_VERSION, + DEFAULT_MAAD_MEASURE, type FileIpCounts, type FlowDirection, type MaadIpVersion, + type MaadMeasure, type NetflowFileDetailsResponse, type NetflowFileSummaryRecord, type SpectrumData, @@ -72,14 +74,16 @@ function buildLoaderKey( dataset: string, slug: string, direction: FlowDirection, - ipVersion: MaadIpVersion + ipVersion: MaadIpVersion, + measure: MaadMeasure ): string { - return `${dataset}:${slug}:${direction}:${ipVersion}`; + return `${dataset}:${slug}:${direction}:${ipVersion}:${measure}`; } function buildScopeSearchParams( direction: FlowDirection, - ipVersion: MaadIpVersion + ipVersion: MaadIpVersion, + measure: MaadMeasure ): Record { const params: Record = {}; @@ -91,6 +95,10 @@ function buildScopeSearchParams( params.ipVersion = String(ipVersion); } + if (measure !== DEFAULT_MAAD_MEASURE) { + params.measure = measure; + } + return params; } @@ -112,6 +120,7 @@ class NetflowFileDetailLoader { readonly slug: string; readonly direction: FlowDirection; readonly ipVersion: MaadIpVersion; + readonly measure: MaadMeasure; readonly key: string; private _loading = $state(false); @@ -124,12 +133,19 @@ class NetflowFileDetailLoader { private _lastAccessedAt = Date.now(); private summarySkeletonDelayId: ReturnType | null = null; - constructor(dataset: string, slug: string, direction: FlowDirection, ipVersion: MaadIpVersion) { + constructor( + dataset: string, + slug: string, + direction: FlowDirection, + ipVersion: MaadIpVersion, + measure: MaadMeasure + ) { this.dataset = dataset; this.slug = slug; this.direction = direction; this.ipVersion = ipVersion; - this.key = buildLoaderKey(dataset, slug, direction, ipVersion); + this.measure = measure; + this.key = buildLoaderKey(dataset, slug, direction, ipVersion, measure); } get loading() { @@ -299,7 +315,7 @@ class NetflowFileDetailLoader { const response = await fetch( `/api/netflow/files/${this.slug}/details?${new URLSearchParams({ dataset: this.dataset, - ...buildScopeSearchParams(this.direction, this.ipVersion) + ...buildScopeSearchParams(this.direction, this.ipVersion, this.measure) }).toString()}` ); @@ -379,7 +395,7 @@ class NetflowFileDetailLoader { return `/api/netflow/files/${this.slug}/${kind === 'ipCounts' ? 'ip-counts' : kind}?${new URLSearchParams( { dataset: this.dataset, - ...buildScopeSearchParams(this.direction, this.ipVersion), + ...buildScopeSearchParams(this.direction, this.ipVersion, this.measure), router, source: String(side === 'source') } @@ -412,13 +428,14 @@ export function getNetflowFileDetailLoader( dataset: string, slug: string, direction: FlowDirection, - ipVersion: MaadIpVersion = DEFAULT_MAAD_IP_VERSION + ipVersion: MaadIpVersion = DEFAULT_MAAD_IP_VERSION, + measure: MaadMeasure = DEFAULT_MAAD_MEASURE ) { - const key = buildLoaderKey(dataset, slug, direction, ipVersion); + const key = buildLoaderKey(dataset, slug, direction, ipVersion, measure); let loader = fileDetailLoaders.get(key); if (!loader) { - loader = new NetflowFileDetailLoader(dataset, slug, direction, ipVersion); + loader = new NetflowFileDetailLoader(dataset, slug, direction, ipVersion, measure); fileDetailLoaders.set(key, loader); } diff --git a/apps/web/src/lib/server/db/local-schema.ts b/apps/web/src/lib/server/db/local-schema.ts index 5eaf4f27..d55a5117 100644 --- a/apps/web/src/lib/server/db/local-schema.ts +++ b/apps/web/src/lib/server/db/local-schema.ts @@ -121,13 +121,15 @@ export const localSchemaSql = ` src_locality TEXT NOT NULL CHECK(src_locality IN ('all', 'internal', 'external')), dst_locality TEXT NOT NULL CHECK(dst_locality IN ('all', 'internal', 'external')), address_side TEXT NOT NULL CHECK(address_side IN ('source', 'destination')), + measure TEXT NOT NULL CHECK(measure IN ('addresses', 'packets', 'bytes')), structure_kind TEXT NOT NULL CHECK(structure_kind IN ('structure', 'spectrum', 'dimension')), values_json TEXT NOT NULL, metadata_json TEXT NOT NULL, processed_at TEXT DEFAULT CURRENT_TIMESTAMP, + CHECK(measure = 'addresses' OR structure_kind <> 'spectrum'), PRIMARY KEY( source_id, granularity, bucket_start, ip_version, - src_locality, dst_locality, address_side, structure_kind + src_locality, dst_locality, address_side, measure, structure_kind ) ); @@ -181,12 +183,12 @@ export const localSchemaSql = ` CREATE INDEX IF NOT EXISTS idx_address_structure_stats_query ON address_structure_stats ( granularity, bucket_start, source_id, ip_version, - src_locality, dst_locality, address_side, structure_kind + src_locality, dst_locality, address_side, measure, structure_kind ); CREATE INDEX IF NOT EXISTS idx_address_structure_stats_timeseries ON address_structure_stats ( source_id, granularity, src_locality, dst_locality, - ip_version, structure_kind, bucket_start + ip_version, measure, structure_kind, bucket_start ); CREATE INDEX IF NOT EXISTS idx_port_count_stats_timeseries ON port_count_stats ( diff --git a/apps/web/src/lib/server/db/schema.ts b/apps/web/src/lib/server/db/schema.ts index 0d5e1bb5..f2b060c6 100644 --- a/apps/web/src/lib/server/db/schema.ts +++ b/apps/web/src/lib/server/db/schema.ts @@ -331,6 +331,7 @@ export const addressStructureStats = sqliteTable( srcLocality: text('src_locality', { enum: ['all', 'internal', 'external'] }).notNull(), dstLocality: text('dst_locality', { enum: ['all', 'internal', 'external'] }).notNull(), addressSide: text('address_side', { enum: ['source', 'destination'] }).notNull(), + measure: text('measure', { enum: ['addresses', 'packets', 'bytes'] }).notNull(), structureKind: text('structure_kind', { enum: ['structure', 'spectrum', 'dimension'] }).notNull(), @@ -348,6 +349,7 @@ export const addressStructureStats = sqliteTable( table.srcLocality, table.dstLocality, table.addressSide, + table.measure, table.structureKind ] }), @@ -359,6 +361,7 @@ export const addressStructureStats = sqliteTable( table.srcLocality, table.dstLocality, table.addressSide, + table.measure, table.structureKind ), index('idx_address_structure_stats_timeseries').on( @@ -367,6 +370,7 @@ export const addressStructureStats = sqliteTable( table.srcLocality, table.dstLocality, table.ipVersion, + table.measure, table.structureKind, table.bucketStart ), @@ -378,6 +382,14 @@ export const addressStructureStats = sqliteTable( check( 'address_structure_stats_dst_locality_check', sql`${table.dstLocality} IN ('all', 'internal', 'external')` + ), + check( + 'address_structure_stats_measure_check', + sql`${table.measure} IN ('addresses', 'packets', 'bytes')` + ), + check( + 'address_structure_stats_measure_spectrum_check', + sql`${table.measure} = 'addresses' OR ${table.structureKind} <> 'spectrum'` ) ] ); diff --git a/apps/web/src/lib/server/netflow-v3.ts b/apps/web/src/lib/server/netflow-v3.ts index 7ea1d653..1613a4c8 100644 --- a/apps/web/src/lib/server/netflow-v3.ts +++ b/apps/web/src/lib/server/netflow-v3.ts @@ -1,14 +1,17 @@ import { DEFAULT_MAAD_IP_VERSION, + DEFAULT_MAAD_MEASURE, FLOW_DIRECTIONS, flowDirectionLocalities, IP_GRANULARITIES, MAAD_IP_VERSIONS, + MAAD_MEASURES, type FlowDirection, type FlowLocality, type FlowLocalityPair, type IpGranularity, - type MaadIpVersion + type MaadIpVersion, + type MaadMeasure } from '$lib/types/types'; import type { SourceDefinition } from '$lib/server/datasets'; import type { StructureFunctionPoint } from '$lib/types/types'; @@ -30,6 +33,13 @@ export interface AggregateStatsParams { dstLocality: FlowLocality; } +export interface MaadParams { + ipVersion: MaadIpVersion; + measure: MaadMeasure; +} + +export type MaadStatsParams = AggregateStatsParams & MaadParams; + export interface RequestValidationError { error: string; status: 400; @@ -41,6 +51,7 @@ export type NetflowSchemaVersion = 'v3'; const VALID_IP_GRANULARITIES = new Set(IP_GRANULARITIES); const VALID_FLOW_DIRECTIONS = new Set(FLOW_DIRECTIONS); +const VALID_MAAD_MEASURES = new Set(MAAD_MEASURES); export function assertNetflowV3Database(): void { return; @@ -117,6 +128,56 @@ export function parseMaadIpVersion(url: URL): MaadIpVersion | RequestValidationE return version; } +export function parseMaadMeasure(url: URL): MaadMeasure | RequestValidationError { + const param = url.searchParams.get('measure'); + if (param === null) { + return DEFAULT_MAAD_MEASURE; + } + + if (!VALID_MAAD_MEASURES.has(param)) { + return { + error: `Invalid measure. Expected one of: ${MAAD_MEASURES.join(', ')}`, + status: 400 + }; + } + + return param as MaadMeasure; +} + +export function parseMaadParams(url: URL): MaadParams | RequestValidationError { + const ipVersion = parseMaadIpVersion(url); + if (typeof ipVersion !== 'number') { + return ipVersion; + } + + const measure = parseMaadMeasure(url); + if (typeof measure !== 'string') { + return measure; + } + + return { ipVersion, measure }; +} + +export function parseMaadStatsParams(url: URL): MaadStatsParams | RequestValidationError { + const aggregate = parseAggregateStatsParams(url); + if ('error' in aggregate) { + return aggregate; + } + + const maad = parseMaadParams(url); + if ('error' in maad) { + return maad; + } + + return { ...aggregate, ...maad }; +} + +export function spectrumMeasureError(measure: MaadMeasure): RequestValidationError | null { + return measure === 'addresses' + ? null + : { error: `Spectrum is only computed for the addresses measure, not ${measure}`, status: 400 }; +} + export function parseFlowDirectionParams( url: URL ): ({ direction: FlowDirection } & FlowLocalityPair) | RequestValidationError { diff --git a/apps/web/src/lib/types/types.ts b/apps/web/src/lib/types/types.ts index c6bb5f99..0db4006e 100644 --- a/apps/web/src/lib/types/types.ts +++ b/apps/web/src/lib/types/types.ts @@ -379,6 +379,22 @@ export const MAAD_IP_VERSION_OPTIONS: MaadIpVersionOption[] = [ { value: 6, label: 'IPv6 (/23–/64)' } ]; +export const MAAD_MEASURES = ['addresses', 'packets', 'bytes'] as const; + +export type MaadMeasure = (typeof MAAD_MEASURES)[number]; + +export const DEFAULT_MAAD_MEASURE: MaadMeasure = 'addresses'; + +export const MAAD_MEASURE_OPTIONS: { value: MaadMeasure; label: string }[] = [ + { value: 'addresses', label: 'Addresses' }, + { value: 'packets', label: 'Packets' }, + { value: 'bytes', label: 'Bytes' } +]; + +export function maadMeasureHasSpectrum(measure: MaadMeasure): boolean { + return measure === 'addresses'; +} + export interface SpectrumPoint { alpha: number; f: number; diff --git a/apps/web/src/routes/api/netflow/files/[slug]/details/+server.ts b/apps/web/src/routes/api/netflow/files/[slug]/details/+server.ts index 2c213b25..233b2f90 100644 --- a/apps/web/src/routes/api/netflow/files/[slug]/details/+server.ts +++ b/apps/web/src/routes/api/netflow/files/[slug]/details/+server.ts @@ -14,7 +14,7 @@ import { getDatasetFromRequest, slugToBucketStart, withDb } from '../utils'; import { normalizeStructurePoints, parseFlowDirectionParams, - parseMaadIpVersion + parseMaadParams } from '$lib/server/netflow-v3'; const FIVE_MINUTES = '5m'; @@ -133,9 +133,9 @@ export const GET: RequestHandler = async ({ params, url, platform }) => { return json({ error: flowDirection.error }, { status: flowDirection.status }); } - const ipVersion = parseMaadIpVersion(url); - if (typeof ipVersion !== 'number') { - return json({ error: ipVersion.error }, { status: ipVersion.status }); + const maad = parseMaadParams(url); + if ('error' in maad) { + return json({ error: maad.error }, { status: maad.status }); } if (!slug || slug.length !== 12 || !/^\d{12}$/.test(slug)) { @@ -210,6 +210,7 @@ export const GET: RequestHandler = async ({ params, url, platform }) => { AND ip_version = ? AND src_locality = ? AND dst_locality = ? + AND measure = ? AND structure_kind = 'structure' GROUP BY source_id, bucket_start ), @@ -225,6 +226,7 @@ export const GET: RequestHandler = async ({ params, url, platform }) => { AND ip_version = ? AND src_locality = ? AND dst_locality = ? + AND measure = ? AND structure_kind = 'spectrum' GROUP BY source_id, bucket_start ) @@ -281,14 +283,16 @@ export const GET: RequestHandler = async ({ params, url, platform }) => { flowDirection.dstLocality, FIVE_MINUTES, bucketStart, - ipVersion, + maad.ipVersion, flowDirection.srcLocality, flowDirection.dstLocality, + maad.measure, FIVE_MINUTES, bucketStart, - ipVersion, + maad.ipVersion, flowDirection.srcLocality, flowDirection.dstLocality, + maad.measure, bucketStart ] ); diff --git a/apps/web/src/routes/api/netflow/files/[slug]/spectrum/+server.ts b/apps/web/src/routes/api/netflow/files/[slug]/spectrum/+server.ts index d08133b4..8d0376fe 100644 --- a/apps/web/src/routes/api/netflow/files/[slug]/spectrum/+server.ts +++ b/apps/web/src/routes/api/netflow/files/[slug]/spectrum/+server.ts @@ -1,7 +1,11 @@ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; import type { SpectrumData, SpectrumPoint } from '$lib/types/types'; -import { parseFlowDirectionParams, parseMaadIpVersion } from '$lib/server/netflow-v3'; +import { + parseFlowDirectionParams, + parseMaadParams, + spectrumMeasureError +} from '$lib/server/netflow-v3'; import { getDatasetFromRequest, slugToBucketStart, withDb } from '../utils'; const FIVE_MINUTES = '5m'; @@ -21,9 +25,14 @@ export const GET: RequestHandler = async ({ params, url, platform }) => { return json({ error: flowDirection.error }, { status: flowDirection.status }); } - const ipVersion = parseMaadIpVersion(url); - if (typeof ipVersion !== 'number') { - return json({ error: ipVersion.error }, { status: ipVersion.status }); + const maad = parseMaadParams(url); + if ('error' in maad) { + return json({ error: maad.error }, { status: maad.status }); + } + + const measureError = spectrumMeasureError(maad.measure); + if (measureError) { + return json({ error: measureError.error }, { status: measureError.status }); } if (!slug || slug.length !== 12 || !/^\d{12}$/.test(slug)) { @@ -61,16 +70,18 @@ export const GET: RequestHandler = async ({ params, url, platform }) => { AND src_locality = ? AND dst_locality = ? AND address_side = ? + AND measure = ? AND structure_kind = 'spectrum' LIMIT 1`, [ router, FIVE_MINUTES, bucketStart, - ipVersion, + maad.ipVersion, flowDirection.srcLocality, flowDirection.dstLocality, - isSource ? 'source' : 'destination' + isSource ? 'source' : 'destination', + maad.measure ] ); diff --git a/apps/web/src/routes/api/netflow/files/[slug]/structure/+server.ts b/apps/web/src/routes/api/netflow/files/[slug]/structure/+server.ts index b7c2fcb8..c9ed349f 100644 --- a/apps/web/src/routes/api/netflow/files/[slug]/structure/+server.ts +++ b/apps/web/src/routes/api/netflow/files/[slug]/structure/+server.ts @@ -5,7 +5,7 @@ import { getDatasetFromRequest, slugToBucketStart, withDb } from '../utils'; import { normalizeStructurePoints, parseFlowDirectionParams, - parseMaadIpVersion + parseMaadParams } from '$lib/server/netflow-v3'; const FIVE_MINUTES = '5m'; @@ -25,9 +25,9 @@ export const GET: RequestHandler = async ({ params, url, platform }) => { return json({ error: flowDirection.error }, { status: flowDirection.status }); } - const ipVersion = parseMaadIpVersion(url); - if (typeof ipVersion !== 'number') { - return json({ error: ipVersion.error }, { status: ipVersion.status }); + const maad = parseMaadParams(url); + if ('error' in maad) { + return json({ error: maad.error }, { status: maad.status }); } if (!slug || slug.length !== 12 || !/^\d{12}$/.test(slug)) { @@ -65,16 +65,18 @@ export const GET: RequestHandler = async ({ params, url, platform }) => { AND src_locality = ? AND dst_locality = ? AND address_side = ? + AND measure = ? AND structure_kind = 'structure' LIMIT 1`, [ router, FIVE_MINUTES, bucketStart, - ipVersion, + maad.ipVersion, flowDirection.srcLocality, flowDirection.dstLocality, - isSource ? 'source' : 'destination' + isSource ? 'source' : 'destination', + maad.measure ] ); diff --git a/apps/web/src/routes/api/netflow/spectrum-stats/+server.ts b/apps/web/src/routes/api/netflow/spectrum-stats/+server.ts index c9229dc0..cb46a8e4 100644 --- a/apps/web/src/routes/api/netflow/spectrum-stats/+server.ts +++ b/apps/web/src/routes/api/netflow/spectrum-stats/+server.ts @@ -4,11 +4,7 @@ import type { SpectrumPoint } from '$lib/types/types'; import type { SpectrumStatsPayload, SpectrumStatsResponse } from '$lib/types/spectrum-stats'; import { buildCoverageTimelines } from '$lib/server/db/coverage'; import { getRequestedDataset, withDatasetDb } from '$lib/server/datasets'; -import { - parseAggregateStatsParams, - parseMaadIpVersion, - placeholders -} from '$lib/server/netflow-v3'; +import { parseMaadStatsParams, placeholders, spectrumMeasureError } from '$lib/server/netflow-v3'; type SpectrumStatsRow = SpectrumStatsPayload & { router: string; @@ -53,15 +49,15 @@ function parseSpectrumPoints( } export const GET: RequestHandler = async ({ url, platform }) => { - const params = parseAggregateStatsParams(url); + const params = parseMaadStatsParams(url); if ('error' in params) { return json({ error: params.error }, { status: params.status }); } - const ipVersion = parseMaadIpVersion(url); - if (typeof ipVersion !== 'number') { - return json({ error: ipVersion.error }, { status: ipVersion.status }); + const measureError = spectrumMeasureError(params.measure); + if (measureError) { + return json({ error: measureError.error }, { status: measureError.status }); } - const { routers, granularity, start, end, srcLocality, dstLocality } = params; + const { routers, granularity, start, end, srcLocality, dstLocality, ipVersion, measure } = params; try { const dataset = await getRequestedDataset(url, platform); @@ -75,7 +71,8 @@ export const GET: RequestHandler = async ({ url, platform }) => { dstLocality, start, end, - ipVersion + ipVersion, + measure ]; const query = ` @@ -93,6 +90,7 @@ export const GET: RequestHandler = async ({ url, platform }) => { AND bucket_start >= ? AND bucket_start < ? AND ip_version = ? + AND measure = ? AND structure_kind = 'spectrum' GROUP BY ${sourceColumn}, bucket_start `; diff --git a/apps/web/src/routes/api/netflow/structure-stats/+server.ts b/apps/web/src/routes/api/netflow/structure-stats/+server.ts index b606e12f..a1b29996 100644 --- a/apps/web/src/routes/api/netflow/structure-stats/+server.ts +++ b/apps/web/src/routes/api/netflow/structure-stats/+server.ts @@ -6,8 +6,7 @@ import { buildCoverageTimelines } from '$lib/server/db/coverage'; import { getRequestedDataset, withDatasetDb } from '$lib/server/datasets'; import { normalizeStructurePoints, - parseAggregateStatsParams, - parseMaadIpVersion, + parseMaadStatsParams, placeholders } from '$lib/server/netflow-v3'; @@ -64,15 +63,11 @@ function parseStructurePoints( } export const GET: RequestHandler = async ({ url, platform }) => { - const params = parseAggregateStatsParams(url); + const params = parseMaadStatsParams(url); if ('error' in params) { return json({ error: params.error }, { status: params.status }); } - const ipVersion = parseMaadIpVersion(url); - if (typeof ipVersion !== 'number') { - return json({ error: ipVersion.error }, { status: ipVersion.status }); - } - const { routers, granularity, start, end, srcLocality, dstLocality } = params; + const { routers, granularity, start, end, srcLocality, dstLocality, ipVersion, measure } = params; try { const dataset = await getRequestedDataset(url, platform); @@ -86,7 +81,8 @@ export const GET: RequestHandler = async ({ url, platform }) => { dstLocality, start, end, - ipVersion + ipVersion, + measure ]; const query = ` @@ -104,6 +100,7 @@ export const GET: RequestHandler = async ({ url, platform }) => { AND bucket_start >= ? AND bucket_start < ? AND ip_version = ? + AND measure = ? AND structure_kind = 'structure' GROUP BY ${sourceColumn}, bucket_start `; diff --git a/apps/web/tests/lib/server/migrations.test.ts b/apps/web/tests/lib/server/migrations.test.ts index 2f74327f..194b9d48 100644 --- a/apps/web/tests/lib/server/migrations.test.ts +++ b/apps/web/tests/lib/server/migrations.test.ts @@ -162,6 +162,7 @@ describe('D1 migrations', () => { 'src_locality', 'dst_locality', 'ip_version', + 'measure', 'structure_kind', 'bucket_start' ] @@ -342,4 +343,49 @@ describe('D1 migrations', () => { database.close(); } }); + + it('keys MAAD rows by measure and keeps existing rows as the addresses measure', () => { + const database = new Database(':memory:'); + const migrations = migrationFiles(); + const measureMigration = migrations.findIndex((fileName) => + fs + .readFileSync(path.join(migrationsDirectory, fileName), 'utf8') + .includes('address_structure_stats_measure_check') + ); + try { + for (const migration of migrations.slice(0, measureMigration)) { + applyMigration(database, migration); + } + database + .prepare( + `INSERT INTO address_structure_stats ( + source_id, granularity, bucket_start, bucket_end, ip_version, + src_locality, dst_locality, address_side, structure_kind, + values_json, metadata_json + ) VALUES ('edge', '5m', 0, 300, 4, 'all', 'all', 'source', 'spectrum', '[]', '{}')` + ) + .run(); + + for (const migration of migrations.slice(measureMigration)) { + applyMigration(database, migration); + } + + expect( + database.prepare('SELECT measure, structure_kind FROM address_structure_stats').all() + ).toEqual([{ measure: 'addresses', structure_kind: 'spectrum' }]); + const insert = database.prepare( + `INSERT INTO address_structure_stats ( + source_id, granularity, bucket_start, bucket_end, ip_version, + src_locality, dst_locality, address_side, measure, structure_kind, + values_json, metadata_json + ) VALUES ('edge', '5m', 0, 300, 4, 'all', 'all', 'source', ?, ?, '[]', '{}')` + ); + insert.run('packets', 'structure'); + insert.run('bytes', 'structure'); + expect(() => insert.run('packets', 'spectrum')).toThrow(/CHECK constraint failed/); + expect(() => insert.run('flows', 'structure')).toThrow(/CHECK constraint failed/); + } finally { + database.close(); + } + }); }); diff --git a/apps/web/tests/lib/server/netflow-v3.test.ts b/apps/web/tests/lib/server/netflow-v3.test.ts index 74b25884..bee0781f 100644 --- a/apps/web/tests/lib/server/netflow-v3.test.ts +++ b/apps/web/tests/lib/server/netflow-v3.test.ts @@ -9,6 +9,8 @@ import { parseFlowDirection, parseFlowDirectionParams, parseMaadIpVersion, + parseMaadMeasure, + parseMaadStatsParams, parseSourceIds, parseTimestamp, resolveSourceIds @@ -169,6 +171,50 @@ describe('netflow v3 helpers', () => { } }); + it('parses the MAAD measure with an addresses default', () => { + expect(parseMaadMeasure(new URL('http://localhost/api/test'))).toBe('addresses'); + for (const measure of ['addresses', 'packets', 'bytes']) { + expect(parseMaadMeasure(new URL(`http://localhost/api/test?measure=${measure}`))).toBe( + measure + ); + } + expect(parseMaadMeasure(new URL('http://localhost/api/test?measure=Packets'))).toEqual({ + error: 'Invalid measure. Expected one of: addresses, packets, bytes', + status: 400 + }); + }); + + it('extends aggregate stats params with the MAAD ip version and measure', () => { + expect( + parseMaadStatsParams( + new URL( + 'http://localhost/api/test?routers=r1&startDate=100&endDate=200&direction=egress&ipVersion=6&measure=bytes' + ) + ) + ).toEqual({ + routers: ['r1'], + granularity: '1h', + start: 100, + end: 200, + direction: 'egress', + srcLocality: 'internal', + dstLocality: 'external', + ipVersion: 6, + measure: 'bytes' + }); + expect( + parseMaadStatsParams( + new URL('http://localhost/api/test?routers=r1&startDate=100&endDate=200&measure=flows') + ) + ).toEqual({ + error: 'Invalid measure. Expected one of: addresses, packets, bytes', + status: 400 + }); + expect( + parseMaadStatsParams(new URL('http://localhost/api/test?startDate=100&endDate=200')) + ).toEqual({ error: 'No routers selected', status: 400 }); + }); + it('normalizes structure points from MAAD variants', () => { expect(normalizeStructurePoints([{ q: 1, tauTilde: 2, s: 3 }])).toEqual([ { q: 1, tau: 2, sd: 3 } diff --git a/apps/web/tests/performance/seed-large-db.ts b/apps/web/tests/performance/seed-large-db.ts index 74bd165d..b8b410cb 100644 --- a/apps/web/tests/performance/seed-large-db.ts +++ b/apps/web/tests/performance/seed-large-db.ts @@ -195,11 +195,11 @@ db.exec(` INSERT INTO address_structure_stats ( source_id, granularity, bucket_start, bucket_end, ip_version, - src_locality, dst_locality, address_side, structure_kind, + src_locality, dst_locality, address_side, measure, structure_kind, values_json, metadata_json ) SELECT source_id, '1h', bucket_start, bucket_start + 3600, 4, - src_locality, dst_locality, address_side, structure_kind, + src_locality, dst_locality, address_side, 'addresses', structure_kind, CASE structure_kind WHEN 'spectrum' THEN '[{"alpha":0.1,"f":0.2},{"alpha":0.2,"f":0.3},{"alpha":0.3,"f":0.4}]' ELSE '[{"q":1,"tau":0.2,"sd":0.01},{"q":2,"tau":0.4,"sd":0.02}]' diff --git a/apps/web/tests/routes/api-aggregate-routes.test.ts b/apps/web/tests/routes/api-aggregate-routes.test.ts index 84ac33c6..4a65b1be 100644 --- a/apps/web/tests/routes/api-aggregate-routes.test.ts +++ b/apps/web/tests/routes/api-aggregate-routes.test.ts @@ -456,7 +456,8 @@ describe('aggregate API routes', () => { 'all', 100, 200, - 4 + 4, + 'addresses' ]); await expect(response.json()).resolves.toEqual({ timelines: [ @@ -516,7 +517,8 @@ describe('aggregate API routes', () => { 'all', 100, 200, - 6 + 6, + 'addresses' ]); const spectrumResponse = await getSpectrumStats({ @@ -532,7 +534,8 @@ describe('aggregate API routes', () => { 'all', 100, 200, - 6 + 6, + 'addresses' ]); }); @@ -559,4 +562,57 @@ describe('aggregate API routes', () => { error: 'Invalid ipVersion. Expected one of: 4, 6' }); }); + + it('filters structure stats by the requested MAAD measure', async () => { + const all = vi.fn().mockResolvedValue([]); + vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); + mockDatasetSession({ all }); + + for (const measure of ['addresses', 'packets', 'bytes']) { + all.mockClear(); + const response = await getStructureStats({ + url: new URL( + `http://localhost/api/netflow/structure-stats?routers=r1&startDate=100&endDate=200&measure=${measure}` + ) + } as never); + expect(response.status).toBe(200); + expect(all).toHaveBeenNthCalledWith(1, expect.stringContaining('AND measure = ?'), [ + '1h', + 'r1', + 'all', + 'all', + 100, + 200, + 4, + measure + ]); + } + }); + + it('rejects unknown measures and weighted spectrum requests', async () => { + const all = vi.fn().mockResolvedValue([]); + vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); + mockDatasetSession({ all }); + + const invalid = await getStructureStats({ + url: new URL( + 'http://localhost/api/netflow/structure-stats?routers=r1&startDate=100&endDate=200&measure=flows' + ) + } as never); + expect(invalid.status).toBe(400); + await expect(invalid.json()).resolves.toEqual({ + error: 'Invalid measure. Expected one of: addresses, packets, bytes' + }); + + const weightedSpectrum = await getSpectrumStats({ + url: new URL( + 'http://localhost/api/netflow/spectrum-stats?routers=r1&startDate=100&endDate=200&measure=packets' + ) + } as never); + expect(weightedSpectrum.status).toBe(400); + await expect(weightedSpectrum.json()).resolves.toEqual({ + error: 'Spectrum is only computed for the addresses measure, not packets' + }); + expect(all).not.toHaveBeenCalled(); + }); }); diff --git a/apps/web/tests/routes/netflow-file-routes.test.ts b/apps/web/tests/routes/netflow-file-routes.test.ts index 15c444ff..ed63fe2c 100644 --- a/apps/web/tests/routes/netflow-file-routes.test.ts +++ b/apps/web/tests/routes/netflow-file-routes.test.ts @@ -106,7 +106,8 @@ describe('netflow file helpers and routes', () => { 4, 'internal', 'external', - 'destination' + 'destination', + 'addresses' ]); expect(get).toHaveBeenNthCalledWith(3, expect.any(String), [ 'r1', @@ -115,7 +116,8 @@ describe('netflow file helpers and routes', () => { 4, 'internal', 'external', - 'source' + 'source', + 'addresses' ]); await expect(ipResponse.json()).resolves.toEqual({ ipv4Count: 1, ipv6Count: 3 }); await expect(spectrumResponse.json()).resolves.toEqual({ @@ -213,11 +215,13 @@ describe('netflow file helpers and routes', () => { 4, 'internal', 'external', + 'addresses', '5m', bucketStart, 4, 'internal', 'external', + 'addresses', bucketStart ]); await expect(response.json()).resolves.toEqual({ @@ -335,7 +339,8 @@ describe('netflow file helpers and routes', () => { 6, 'all', 'all', - 'destination' + 'destination', + 'addresses' ]); expect(get).toHaveBeenNthCalledWith(2, expect.any(String), [ 'r1', @@ -344,7 +349,8 @@ describe('netflow file helpers and routes', () => { 6, 'all', 'all', - 'source' + 'source', + 'addresses' ]); }); @@ -376,6 +382,50 @@ describe('netflow file helpers and routes', () => { } }); + it('binds the requested MAAD measure into the per-file and details MAAD queries', async () => { + const get = vi.fn().mockResolvedValue({ valuesJson: '[{"q":1,"tauTilde":2,"sd":0.5}]' }); + const all = vi.fn().mockResolvedValue([]); + vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); + mockDatasetSession({ get, all }); + + const bucketStart = slugToBucketStart('202503010005'); + + const structureResponse = await getStructure({ + params: { slug: '202503010005' }, + url: new URL( + 'http://localhost/api/netflow/files/x/structure?router=r1&source=true&measure=packets' + ) + } as never); + expect(structureResponse.status).toBe(200); + expect(get).toHaveBeenCalledWith(expect.stringContaining('AND measure = ?'), [ + 'r1', + '5m', + bucketStart, + 4, + 'all', + 'all', + 'source', + 'packets' + ]); + + const spectrumResponse = await getSpectrum({ + params: { slug: '202503010005' }, + url: new URL( + 'http://localhost/api/netflow/files/x/spectrum?router=r1&source=true&measure=bytes' + ) + } as never); + expect(spectrumResponse.status).toBe(400); + expect(get).toHaveBeenCalledTimes(1); + + await getDetails({ + params: { slug: '202503010005' }, + url: new URL('http://localhost/api/netflow/files/x/details?dataset=alpha&measure=bytes') + } as never); + const [detailsQuery, detailsParams] = all.mock.calls[0] as [string, unknown[]]; + expect(detailsQuery.match(/AND measure = \?/g)).toHaveLength(2); + expect(detailsParams.filter((value) => value === 'bytes')).toHaveLength(2); + }); + it('binds the requested MAAD ip version into the file details MAAD query', async () => { const all = vi.fn().mockResolvedValue([]); vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); @@ -403,11 +453,13 @@ describe('netflow file helpers and routes', () => { 6, 'all', 'all', + 'addresses', '5m', bucketStart, 6, 'all', 'all', + 'addresses', bucketStart ]); }); diff --git a/docs/code/pipeline-contract.md b/docs/code/pipeline-contract.md index 35374123..cd2dc5ea 100644 --- a/docs/code/pipeline-contract.md +++ b/docs/code/pipeline-contract.md @@ -11,7 +11,7 @@ The pipeline binds each database to one product identity. The identity contains - The canonical endpoint-locality rules - The pipeline timezone - The native decoder contract -- The MAAD enabled state +- The MAAD enabled state, contract version, configuration, and measure set The pipeline rejects a database when its identity differs. Build a new database for a different product identity. @@ -76,6 +76,25 @@ rules or splitting a prefix does not change the identity. Any change to the matc does, and address-file contents are read on every run. The digest keeps private address lists out of database metadata and export manifests. +## MAAD measures + +Each scope and address side keeps one entry per unique address with its summed packets and bytes. +Rollups merge children by summing those counters per address, so a rollup's per-address totals +equal the sums over its five-minute children. The per-address map adds 16 bytes to each address +entry. + +Every MAAD computation runs three measures on the same entries, for both IP versions and every +granularity. `addresses` weighs each address as 1 and stores structure, spectrum, and dimensions. +`packets` and `bytes` weight moments and D1 entropy by the summed counter and store structure and +dimensions only. Prefix validity always uses distinct-address counts. A weighted measure excludes +addresses whose counter sums to 0 and records the excluded count as `zeroWeightAddrs` in the row +metadata. The measure set is part of the result configuration identity. + +Native and CSV ingestion drop flows whose reported packet count is 0 before any statistic +accumulates, so they add no traffic, protocols, ports, or addresses. The bucket keeps its observed +coverage, and publishing logs the dropped flow count per bucket. A CSV row without a packet count +has an unknown count rather than a reported zero, so it still contributes. + ## Coordinated subset runs Repeat `--dataset` for two or more registry entries to build coordinated subset products. Each entry diff --git a/docs/user/querying.md b/docs/user/querying.md index 8f383678..c104c492 100644 --- a/docs/user/querying.md +++ b/docs/user/querying.md @@ -22,16 +22,16 @@ Show a table schema: ## Main tables -| Table | Content | -| ------------------------- | ---------------------------------------------------- | -| `datasets` | Public dataset metadata | -| `source_members` | Logical-source membership | -| `traffic_stats` | Flow, packet, byte, duration, and TTL metrics | -| `protocol_stats` | Unique protocol counts and protocol lists | -| `address_count_stats` | Unique source-address and destination-address counts | -| `port_count_stats` | Unique low-port and high-port counts | -| `address_structure_stats` | MAAD structure, spectrum, and dimension values | -| `processed_inputs` | Input processing state and provenance | +| Table | Content | +| ------------------------- | ---------------------------------------------------------- | +| `datasets` | Public dataset metadata | +| `source_members` | Logical-source membership | +| `traffic_stats` | Flow, packet, byte, duration, and TTL metrics | +| `protocol_stats` | Unique protocol counts and protocol lists | +| `address_count_stats` | Unique source-address and destination-address counts | +| `port_count_stats` | Unique low-port and high-port counts | +| `address_structure_stats` | MAAD structure, spectrum, and dimension values per measure | +| `processed_inputs` | Input processing state and provenance | The `granularity` value is `5m`, `10m`, `30m`, `1h`, or `1d`. @@ -117,6 +117,39 @@ WHERE granularity = '1h' ORDER BY source_id, bucket_start, ip_version, port_side, port_range; ``` +## Query MAAD results + +`address_structure_stats` stores one MAAD analysis per bucket, IP version, locality pair, address +side, and `measure`: + +| `measure` | Weights each address by | Stored `structure_kind` rows | +| ----------- | ---------------------------------------- | ------------------------------------ | +| `addresses` | 1 (distinct addresses) | `structure`, `spectrum`, `dimension` | +| `packets` | Packets summed over the bucket and scope | `structure`, `dimension` | +| `bytes` | Bytes summed over the bucket and scope | `structure`, `dimension` | + +Weighted measures keep the address-count prefix tests and weight only the moments and the entropy. +The pipeline drops flows that report 0 packets before aggregation. Addresses with zero summed +packets or bytes, such as CSV rows without a packet count, are left out of that measure. Their count is stored in +`metadata_json` as `zeroWeightAddrs`. Always filter on `measure`. Otherwise results for different +measures mix: + +```sql +SELECT + source_id, + bucket_start, + address_side, + values_json +FROM address_structure_stats +WHERE granularity = '1h' + AND ip_version = 4 + AND src_locality = 'all' + AND dst_locality = 'all' + AND measure = 'packets' + AND structure_kind = 'dimension' +ORDER BY source_id, bucket_start, address_side; +``` + ## Query observation averages The database stores sums and counts for safe rollup calculations. It also stores calculated averages for direct queries. diff --git a/docs/user/setup-pipeline.md b/docs/user/setup-pipeline.md index 725bb8a1..00dbaa7f 100644 --- a/docs/user/setup-pipeline.md +++ b/docs/user/setup-pipeline.md @@ -213,6 +213,7 @@ On the native path, nfcapd input needs the fork path: set the top-level `"nfdump | `--nfdump` | Names the nfdump executable. | | `--force` | Rewrites selected nfcapd buckets. | | `--no-maad` | Skips the MAAD statistics. | +| `--maad-workers` | Sets MAAD threads (default: CPUs, max 8). | Time limits must align with local-day boundaries. diff --git a/tools/netflow-db/src/compare.rs b/tools/netflow-db/src/compare.rs index 9ce7f516..f3e1e8b7 100644 --- a/tools/netflow-db/src/compare.rs +++ b/tools/netflow-db/src/compare.rs @@ -94,6 +94,7 @@ const TABLES: &[TableSpec] = &[ "src_locality", "dst_locality", "address_side", + "measure", "structure_kind", "bucket_end", ], diff --git a/tools/netflow-db/src/domain.rs b/tools/netflow-db/src/domain.rs index a0ba2069..813e988c 100644 --- a/tools/netflow-db/src/domain.rs +++ b/tools/netflow-db/src/domain.rs @@ -1,7 +1,7 @@ //! Canonical, adapter-independent NetFlow observations and statistical buckets. use std::{ - collections::{BTreeMap, BTreeSet, HashSet}, + collections::{BTreeMap, BTreeSet, HashMap, HashSet, hash_map::Entry}, net::IpAddr, sync::Arc, }; @@ -200,6 +200,7 @@ pub struct FlowObservation { pub protocol: u8, pub packets: i64, pub bytes: i64, + pub packets_reported: bool, pub src_tos: u8, pub time_received_ms: Option, pub time_end_ms: Option, @@ -234,6 +235,7 @@ impl FlowObservation { protocol, packets, bytes, + packets_reported: true, src_tos, time_received_ms: None, time_end_ms: None, @@ -254,6 +256,12 @@ impl FlowObservation { IpVersion::of(self.src_ip) } + #[must_use] + pub const fn with_unreported_packets(mut self) -> Self { + self.packets_reported = false; + self + } + #[must_use] pub const fn with_locality( mut self, @@ -683,11 +691,151 @@ impl FromIterator for AddressSet { } } +/// Packets and bytes attributed to one address within a scope. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct AddressTraffic { + pub packets: u64, + pub bytes: u64, +} + +impl AddressTraffic { + #[must_use] + pub const fn new(packets: u64, bytes: u64) -> Self { + Self { packets, bytes } + } + + fn add(&mut self, other: Self) { + self.packets = self.packets.saturating_add(other.packets); + self.bytes = self.bytes.saturating_add(other.bytes); + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(C, packed)] +struct PackedAddressTraffic { + packets: u64, + bytes: u64, +} + +impl From for AddressTraffic { + fn from(value: PackedAddressTraffic) -> Self { + Self::new(value.packets, value.bytes) + } +} + +impl From for PackedAddressTraffic { + fn from(value: AddressTraffic) -> Self { + Self { + packets: value.packets, + bytes: value.bytes, + } + } +} + +/// Unique IP addresses with their summed traffic and no iteration-order contract. +/// +/// Counters are stored unaligned so each entry costs the address plus 16 bytes. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AddressTotals(HashMap); + +impl AddressTotals { + pub fn add(&mut self, address: IpAddr, traffic: AddressTraffic) { + match self.0.entry(address) { + Entry::Occupied(mut entry) => { + let mut total = AddressTraffic::from(*entry.get()); + total.add(traffic); + entry.insert(total.into()); + } + Entry::Vacant(entry) => { + entry.insert(traffic.into()); + } + } + } + + /// Sum another scope's totals into this one, keeping rollups additive. + pub fn merge(&mut self, other: &Self) { + for (address, traffic) in other.iter() { + self.add(address, traffic); + } + } + + pub fn iter(&self) -> impl Iterator + '_ { + self.0 + .iter() + .map(|(address, traffic)| (*address, AddressTraffic::from(*traffic))) + } + + #[must_use] + pub fn get(&self, address: &IpAddr) -> Option { + self.0.get(address).copied().map(AddressTraffic::from) + } + + #[must_use] + pub fn len(&self) -> usize { + self.0.len() + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl Serialize for AddressTotals { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq( + self.iter() + .map(|(address, traffic)| (address, traffic.packets, traffic.bytes)), + ) + } +} + +impl<'de> Deserialize<'de> for AddressTotals { + fn deserialize>(deserializer: D) -> Result { + let entries = Vec::<(IpAddr, u64, u64)>::deserialize(deserializer)?; + Ok(entries + .into_iter() + .map(|(address, packets, bytes)| (address, AddressTraffic::new(packets, bytes))) + .collect()) + } +} + +impl FromIterator<(IpAddr, AddressTraffic)> for AddressTotals { + fn from_iter>(entries: T) -> Self { + let mut totals = Self::default(); + for (address, traffic) in entries { + totals.add(address, traffic); + } + totals + } +} + +/// The per-address measure a MAAD analysis weights prefixes by. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum MaadMeasure { + Addresses, + Packets, + Bytes, +} + +impl MaadMeasure { + pub const ALL: [Self; 3] = [Self::Addresses, Self::Packets, Self::Bytes]; + + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Addresses => "addresses", + Self::Packets => "packets", + Self::Bytes => "bytes", + } + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct ScopedAddressesFact { pub scope: Scope, pub address_side: AddressSide, - pub addresses: AddressSet, + pub addresses: AddressTotals, } impl ScopedAddressesFact { @@ -695,7 +843,7 @@ impl ScopedAddressesFact { pub fn new( scope: Scope, address_side: AddressSide, - addresses: impl IntoIterator, + addresses: impl IntoIterator, ) -> Self { Self { scope, @@ -911,7 +1059,7 @@ pub struct ScopedProtocols { pub struct ScopedAddresses { pub scope: Scope, pub address_side: AddressSide, - pub addresses: AddressSet, + pub addresses: AddressTotals, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -934,6 +1082,8 @@ pub struct CanonicalBucket { pub addresses: Vec, pub ports: Vec, pub five_minute_starts: BTreeSet, + #[serde(default)] + pub dropped_zero_packet_flows: u64, } fn serialize_fixed_bitset(ports: &FixedBitSet, serializer: S) -> Result @@ -1078,7 +1228,7 @@ pub struct AddressSetRow<'a> { pub key: BucketKey, pub scope: Scope, pub address_side: AddressSide, - pub addresses: &'a AddressSet, + pub addresses: &'a AddressTotals, } #[derive(Debug, Clone, PartialEq)] @@ -1111,9 +1261,10 @@ pub struct StatisticalBucket { coverage: BucketCoverage, traffic: BTreeMap, protocols: BTreeMap>, - addresses: BTreeMap<(Scope, AddressSide), AddressSet>, + addresses: BTreeMap<(Scope, AddressSide), AddressTotals>, ports: BTreeMap<(Scope, PortSide), FixedBitSet>, five_minute_starts: BTreeSet, + dropped_zero_packet_flows: u64, } impl StatisticalBucket { @@ -1134,6 +1285,7 @@ impl StatisticalBucket { addresses: BTreeMap::new(), ports: BTreeMap::new(), five_minute_starts, + dropped_zero_packet_flows: 0, } } @@ -1148,7 +1300,7 @@ impl StatisticalBucket { for side in [AddressSide::Destination, AddressSide::Source] { bucket .addresses - .insert((scope, side), AddressSet::default()); + .insert((scope, side), AddressTotals::default()); bucket.ports.insert((scope, side), empty_ports()); } } @@ -1194,7 +1346,7 @@ impl StatisticalBucket { self.addresses .entry((entry.scope, entry.address_side)) .or_default() - .extend(entry.addresses.iter().copied()); + .merge(&entry.addresses); } for entry in &child.ports { self.ports @@ -1204,6 +1356,9 @@ impl StatisticalBucket { } self.five_minute_starts .extend(child.five_minute_starts.iter().copied()); + self.dropped_zero_packet_flows = self + .dropped_zero_packet_flows + .saturating_add(child.dropped_zero_packet_flows); self.coverage.include(child.coverage)?; Ok(()) } @@ -1255,6 +1410,7 @@ impl StatisticalBucket { }) .collect(), five_minute_starts: self.five_minute_starts.clone(), + dropped_zero_packet_flows: self.dropped_zero_packet_flows, } } @@ -1270,6 +1426,7 @@ impl StatisticalBucket { addresses, ports, five_minute_starts, + dropped_zero_packet_flows, } = self; CanonicalBucket { @@ -1303,6 +1460,7 @@ impl StatisticalBucket { }) .collect(), five_minute_starts, + dropped_zero_packet_flows, } } @@ -1319,7 +1477,17 @@ impl StatisticalBucket { let (source, destination) = observation .locality .ok_or(DomainError::UnclassifiedObservation)?; + if observation.packets_reported && observation.packets == 0 { + self.dropped_zero_packet_flows = self + .dropped_zero_packet_flows + .saturating_add(observation.flow_count.unsigned_abs()); + return Ok(()); + } let scopes = scopes_for(observation.ip_version(), source, destination); + let traffic = AddressTraffic::new( + observation.packets.unsigned_abs(), + observation.bytes.unsigned_abs(), + ); let mut updates = Vec::with_capacity(scopes.len()); for scope in scopes { let mut metrics = self.traffic.get(&scope).cloned().unwrap_or_default(); @@ -1335,11 +1503,11 @@ impl StatisticalBucket { self.addresses .entry((scope, AddressSide::Source)) .or_default() - .insert(observation.src_ip); + .add(observation.src_ip, traffic); self.addresses .entry((scope, AddressSide::Destination)) .or_default() - .insert(observation.dst_ip); + .add(observation.dst_ip, traffic); if let Some(port) = observation.src_port { insert_port( self.ports @@ -1382,7 +1550,7 @@ impl StatisticalBucket { self.addresses .entry((fact.scope, fact.address_side)) .or_default() - .extend(fact.addresses.iter().copied()); + .merge(&fact.addresses); } } @@ -1438,9 +1606,9 @@ mod tests { use serde_json::json; use super::{ - AddressSet, AddressSide, BucketKey, DomainError, EndpointLocality, FlowObservation, - FlowSelection, Granularity, GroupedTrafficFact, IpVersion, Locality, PortRange, Scope, - ScopedAddressesFact, StatisticalBucket, + AddressSet, AddressSide, AddressTotals, AddressTraffic, BucketKey, DomainError, + EndpointLocality, FlowObservation, FlowSelection, Granularity, GroupedTrafficFact, + IpVersion, Locality, PortRange, Scope, ScopedAddressesFact, StatisticalBucket, }; use crate::coverage::{BucketCoverage, CoverageState}; use crate::locality::{LocalityRuleConfig, LocalityRules}; @@ -1488,6 +1656,35 @@ mod tests { assert_eq!(forward.len(), 2); } + #[test] + fn address_totals_sum_traffic_per_address_without_order_semantics() { + let first = "192.0.2.1".parse().unwrap(); + let second = "2001:db8::1".parse().unwrap(); + + let forward: AddressTotals = [ + (first, AddressTraffic::new(1, 10)), + (second, AddressTraffic::new(0, 0)), + (first, AddressTraffic::new(2, 20)), + ] + .into_iter() + .collect(); + let mut reverse: AddressTotals = + [(second, AddressTraffic::new(0, 0))].into_iter().collect(); + reverse.merge(&[(first, AddressTraffic::new(3, 30))].into_iter().collect()); + + assert_eq!(forward, reverse); + assert_eq!(forward.len(), 2); + assert_eq!(forward.get(&first), Some(AddressTraffic::new(3, 30))); + assert_eq!(forward.get(&second), Some(AddressTraffic::new(0, 0))); + let round_trip: AddressTotals = + serde_json::from_value(serde_json::to_value(&forward).unwrap()).unwrap(); + assert_eq!(round_trip, forward); + assert_eq!( + std::mem::size_of::<(IpAddr, super::PackedAddressTraffic)>(), + std::mem::size_of::() + 16 + ); + } + #[test] fn unclassified_observations_are_rejected_by_buckets() { let mut bucket = StatisticalBucket::new(key(Granularity::FiveMinutes, 0, 300)); @@ -1823,7 +2020,10 @@ mod tests { .add(ScopedAddressesFact::new( Scope::new(IpVersion::V4, Locality::External, Locality::Internal), AddressSide::Destination, - [address([203, 0, 113, 1]), address([203, 0, 113, 2])], + [ + (address([203, 0, 113, 1]), AddressTraffic::new(1, 10)), + (address([203, 0, 113, 2]), AddressTraffic::new(0, 0)), + ], )) .unwrap(); let sparse_expected = sparse.finish(); @@ -1947,9 +2147,12 @@ mod tests { }) .unwrap() .addresses, - [address([192, 0, 2, 1]), address([192, 0, 2, 2])] - .into_iter() - .collect::() + [ + (address([192, 0, 2, 1]), AddressTraffic::new(1, 10)), + (address([192, 0, 2, 2]), AddressTraffic::new(2, 20)), + ] + .into_iter() + .collect::() ); let rows = rolled_up.rows(); let counts = rows @@ -1993,7 +2196,10 @@ mod tests { .add(ScopedAddressesFact::new( scope, AddressSide::Destination, - [address([198, 51, 100, 3]), address([198, 51, 100, 2])], + [ + (address([198, 51, 100, 3]), AddressTraffic::new(5, 50)), + (address([198, 51, 100, 2]), AddressTraffic::new(5, 50)), + ], )) .unwrap(); let bucket = bucket.finish(); @@ -2017,12 +2223,12 @@ mod tests { .unwrap() .addresses, &[ - address([198, 51, 100, 1]), - address([198, 51, 100, 2]), - address([198, 51, 100, 3]), + (address([198, 51, 100, 1]), AddressTraffic::new(1, 10)), + (address([198, 51, 100, 2]), AddressTraffic::new(6, 60)), + (address([198, 51, 100, 3]), AddressTraffic::new(5, 50)), ] .into_iter() - .collect::() + .collect::() ); } } diff --git a/tools/netflow-db/src/export.rs b/tools/netflow-db/src/export.rs index 1c7b6e7e..555872ce 100644 --- a/tools/netflow-db/src/export.rs +++ b/tools/netflow-db/src/export.rs @@ -381,6 +381,7 @@ fn required_columns(table: &str) -> Vec<&'static str> { "port_count_stats" => vec!["port_side", "port_range", "unique_port_count"], "address_structure_stats" => vec![ "address_side", + "measure", "structure_kind", "values_json", "metadata_json", diff --git a/tools/netflow-db/src/feed.rs b/tools/netflow-db/src/feed.rs index 45ebb4c5..6416d1a0 100644 --- a/tools/netflow-db/src/feed.rs +++ b/tools/netflow-db/src/feed.rs @@ -414,9 +414,9 @@ fn collect_total_ipv4_addresses(bucket: CanonicalBucket, addresses: &mut BTreeSe { continue; } - for address in scoped.addresses.iter() { + for (address, _) in scoped.addresses.iter() { if let IpAddr::V4(address) = address { - addresses.insert(*address); + addresses.insert(address); } } } diff --git a/tools/netflow-db/src/ingest.rs b/tools/netflow-db/src/ingest.rs index 9f529e88..6b79adbf 100644 --- a/tools/netflow-db/src/ingest.rs +++ b/tools/netflow-db/src/ingest.rs @@ -713,6 +713,7 @@ CREATE TABLE accepted_rows ( protocol INTEGER NOT NULL, packets INTEGER NOT NULL, bytes INTEGER NOT NULL, + packets_reported INTEGER NOT NULL CHECK (packets_reported IN (0, 1)), src_tos INTEGER NOT NULL, time_received_ms INTEGER, time_end_ms INTEGER, @@ -777,7 +778,7 @@ impl UnsortedCsvScanState { "SELECT src_ip, dst_ip, protocol, packets, bytes, src_tos, \ time_received_ms, time_end_ms, time_start_ms, src_port, dst_port, \ dst_tos, duration_ms, min_ttl, max_ttl, flow_count, src_internal, \ - dst_internal \ + dst_internal, packets_reported \ FROM accepted_rows \ WHERE source_id = ?1 AND bucket_start = ?2 \ ORDER BY id", @@ -916,10 +917,11 @@ impl CsvScanAccumulator for UnsortedCsvScanState { "INSERT INTO accepted_rows (\ source_id, bucket_start, src_ip, dst_ip, protocol, packets, bytes, src_tos, \ time_received_ms, time_end_ms, time_start_ms, src_port, dst_port, dst_tos, \ - duration_ms, min_ttl, max_ttl, flow_count, src_internal, dst_internal\ + duration_ms, min_ttl, max_ttl, flow_count, src_internal, dst_internal, \ + packets_reported\ ) VALUES (\ ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, \ - ?19, ?20\ + ?19, ?20, ?21\ )", params![ source_id, @@ -942,6 +944,7 @@ impl CsvScanAccumulator for UnsortedCsvScanState { observation.flow_count, source == EndpointLocality::Internal, destination == EndpointLocality::Internal, + observation.packets_reported, ], )?; Ok(()) @@ -994,6 +997,7 @@ fn staged_observation(row: &rusqlite::Row<'_>) -> Result std::io::Result { #[cfg(test)] mod tests { - use std::{collections::BTreeMap, fs, io::Write, sync::Arc}; + use std::{collections::BTreeMap, fs, io::Write, net::Ipv4Addr, sync::Arc}; use flate2::{Compression, write::GzEncoder}; use serde_json::json; @@ -1591,7 +1595,9 @@ mod tests { use super::*; use crate::config::{CsvSourceConfig, InputOrder}; use crate::coverage::CoverageState; - use crate::domain::{IpVersion, Locality, Scope}; + use crate::domain::{ + AddressSide, AddressTraffic, CanonicalBucket, IpVersion, Locality, Scope, TrafficMetrics, + }; use crate::locality::{LocalityRuleConfig, LocalityRules}; fn tos_rules() -> Arc { @@ -1825,6 +1831,118 @@ mod tests { } } + #[test] + fn csv_scans_drop_zero_packet_flows_but_keep_observed_coverage() { + let directory = tempdir().unwrap(); + let input = directory.path().join("zero-packets.csv"); + fs::write( + &input, + concat!( + "received,src,dst,packets,bytes,protocol\n", + "0,192.0.2.1,198.51.100.1,2,128,TCP\n", + "10,192.0.2.1,198.51.100.9,0,0,TCP\n", + "300,192.0.2.2,198.51.100.3,0,40,UDP\n", + ), + ) + .unwrap(); + let selection = FlowSelection::from_payload(Some(&json!({ + "version": 1, + "kind": "flows", + "ip_prefix": "192.0.2.0/24", + }))) + .unwrap(); + let scope = Scope::new(IpVersion::V4, Locality::All, Locality::All); + let source = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)); + let addresses = |bucket: &CanonicalBucket, side: AddressSide| { + bucket + .addresses + .iter() + .find(|entry| entry.scope == scope && entry.address_side == side) + .unwrap() + .addresses + .clone() + }; + + for input_order in [InputOrder::TimestampAscending, InputOrder::Unsorted] { + let mut config = config(); + config.input_order = input_order; + let mut buckets = Vec::new(); + scan_csv(&input, &config, &selection, |ready| { + buckets.push(ready.bucket); + Ok::<_, ()>(()) + }) + .unwrap(); + + assert_eq!(buckets.len(), 2); + for bucket in &buckets { + assert_eq!(bucket.dropped_zero_packet_flows, 1); + assert_eq!(bucket.coverage.state(), CoverageState::Complete); + } + let metrics = |bucket: &CanonicalBucket| { + bucket + .traffic + .iter() + .find(|entry| entry.scope == scope) + .unwrap() + .metrics + .clone() + }; + assert_eq!(metrics(&buckets[0]).flows, 1); + assert_eq!(metrics(&buckets[0]).packets, 2); + assert_eq!(metrics(&buckets[0]).bytes, 128); + let sources = addresses(&buckets[0], AddressSide::Source); + assert_eq!(sources.len(), 1); + assert_eq!(sources.get(&source), Some(AddressTraffic::new(2, 128))); + let destinations = addresses(&buckets[0], AddressSide::Destination); + assert_eq!(destinations.len(), 1); + assert!( + destinations + .get(&IpAddr::V4(Ipv4Addr::new(198, 51, 100, 1))) + .is_some() + ); + assert_eq!(metrics(&buckets[1]), TrafficMetrics::default()); + assert!(addresses(&buckets[1], AddressSide::Source).is_empty()); + assert!(addresses(&buckets[1], AddressSide::Destination).is_empty()); + } + + let unreported = directory.path().join("no-packets.csv"); + fs::write( + &unreported, + concat!( + "received,src,dst,bytes,protocol\n", + "0,192.0.2.1,198.51.100.1,128,TCP\n", + "10,192.0.2.1,198.51.100.9,,TCP\n", + ), + ) + .unwrap(); + for input_order in [InputOrder::TimestampAscending, InputOrder::Unsorted] { + let mut config = config(); + config.columns.remove("packets"); + config.input_order = input_order; + let mut buckets = Vec::new(); + scan_csv(&unreported, &config, &selection, |ready| { + buckets.push(ready.bucket); + Ok::<_, ()>(()) + }) + .unwrap(); + + assert_eq!(buckets.len(), 1); + assert_eq!(buckets[0].dropped_zero_packet_flows, 0); + assert_eq!( + buckets[0] + .traffic + .iter() + .find(|entry| entry.scope == scope) + .unwrap() + .metrics + .flows, + 2, + "a missing packet count is unknown, not a reported zero" + ); + assert_eq!(addresses(&buckets[0], AddressSide::Destination).len(), 2); + } + } + #[test] fn scan_reads_selected_members_from_compressed_tar() { let directory = tempdir().unwrap(); diff --git a/tools/netflow-db/src/lib.rs b/tools/netflow-db/src/lib.rs index c3e93f1b..dde34b06 100644 --- a/tools/netflow-db/src/lib.rs +++ b/tools/netflow-db/src/lib.rs @@ -3,7 +3,7 @@ #![forbid(unsafe_code)] /// The persisted pipeline contract version produced by this implementation. -pub const PIPELINE_CONTRACT_VERSION: u32 = 6; +pub const PIPELINE_CONTRACT_VERSION: u32 = 7; pub mod compare; pub mod config; diff --git a/tools/netflow-db/src/main.rs b/tools/netflow-db/src/main.rs index 030e9b26..50e8bfa7 100644 --- a/tools/netflow-db/src/main.rs +++ b/tools/netflow-db/src/main.rs @@ -3,6 +3,7 @@ use std::{ fs::File, io::{self, BufRead, BufReader, Write}, net::{Ipv4Addr, Ipv6Addr}, + num::NonZeroUsize, path::PathBuf, }; @@ -95,6 +96,9 @@ struct PipelineArgs { force: bool, #[arg(long)] no_maad: bool, + /// MAAD worker threads (defaults to available parallelism, capped at 8). + #[arg(long)] + maad_workers: Option, /// Leave partial results in place but fail if requested five-minute coverage is incomplete. #[arg(long)] require_complete: bool, @@ -342,6 +346,9 @@ fn main() -> Result<()> { } fn run_pipeline(args: PipelineArgs) -> Result<()> { + if let Some(workers) = args.maad_workers { + netflow_db::publish::set_maad_workers(workers)?; + } let mut selection = serde_json::Map::new(); if args.daily_active_sources { selection.insert("kind".into(), serde_json::json!("daily_active_sources")); @@ -712,3 +719,31 @@ fn parse_boundary(raw: &str, timezone: &str) -> Result { .timestamp() .as_second()) } + +#[cfg(test)] +mod tests { + use super::*; + + fn pipeline_args(extra: &[&str]) -> Result { + let mut argv = vec!["netflow-db", "pipeline", "--config", "pipeline.json"]; + argv.extend_from_slice(extra); + Cli::try_parse_from(argv).map(|cli| match cli.command { + Command::Pipeline(args) => args, + command => panic!("unexpected command {command:?}"), + }) + } + + #[test] + fn maad_workers_must_be_a_positive_count() { + assert_eq!(pipeline_args(&[]).unwrap().maad_workers, None); + assert_eq!( + pipeline_args(&["--maad-workers", "3"]) + .unwrap() + .maad_workers, + NonZeroUsize::new(3) + ); + for invalid in ["0", "-1", "many"] { + assert!(pipeline_args(&["--maad-workers", invalid]).is_err()); + } + } +} diff --git a/tools/netflow-db/src/nfdump.rs b/tools/netflow-db/src/nfdump.rs index 5971ed12..184457d4 100644 --- a/tools/netflow-db/src/nfdump.rs +++ b/tools/netflow-db/src/nfdump.rs @@ -11,10 +11,10 @@ use fixedbitset::FixedBitSet; use crate::{ coverage::BucketCoverage, domain::{ - AddressSet, AddressSide, BucketKey, CanonicalBucket, DAILY_ACTIVE_MIN_BYTES, - DAILY_ACTIVE_MIN_FLOWS, DAILY_ACTIVE_MIN_PACKETS, EndpointLocality, FlowSelection, - Granularity, IpVersion, Scope, ScopedAddresses, ScopedPorts, ScopedProtocols, - ScopedTraffic, TrafficMetrics, ZERO_FILL_LOCALITY_PAIRS, + AddressSet, AddressSide, AddressTotals, AddressTraffic, BucketKey, CanonicalBucket, + DAILY_ACTIVE_MIN_BYTES, DAILY_ACTIVE_MIN_FLOWS, DAILY_ACTIVE_MIN_PACKETS, EndpointLocality, + FlowSelection, Granularity, IpVersion, Scope, ScopedAddresses, ScopedPorts, + ScopedProtocols, ScopedTraffic, TrafficMetrics, ZERO_FILL_LOCALITY_PAIRS, }, }; @@ -276,8 +276,8 @@ impl std::error::Error for NfdumpError { struct ScopeAccumulator { metrics: [i64; METRIC_COUNT], protocols: [u64; 4], - source_addresses: AddressSet, - destination_addresses: AddressSet, + source_addresses: AddressTotals, + destination_addresses: AddressTotals, source_ports: FixedBitSet, destination_ports: FixedBitSet, } @@ -331,8 +331,10 @@ impl ScopeAccumulator { } self.protocols[usize::from(flow.protocol) / 64] |= 1_u64 << (usize::from(flow.protocol) % 64); - self.source_addresses.insert(flow.source_address); - self.destination_addresses.insert(flow.destination_address); + let traffic = AddressTraffic::new(flow.packets.unsigned_abs(), flow.bytes.unsigned_abs()); + self.source_addresses.add(flow.source_address, traffic); + self.destination_addresses + .add(flow.destination_address, traffic); insert_port(&mut self.source_ports, flow.source_port); insert_port(&mut self.destination_ports, flow.destination_port); } @@ -394,7 +396,7 @@ pub(crate) fn reduce_to_bucket( selection: &FlowSelection, ) -> Result { match reduce_stream(&mut input, selection, None) { - Ok(scopes) => Ok(finish_bucket(scopes, key)), + Ok(reduced) => Ok(finish_bucket(reduced, key)), Err(error) => { drain_to_eof(&mut input); Err(error) @@ -409,7 +411,7 @@ pub(crate) fn reduce_to_bucket_with_active_sources( active_sources: &AddressSet, ) -> Result { match reduce_stream(&mut input, selection, Some(active_sources)) { - Ok(scopes) => Ok(finish_bucket(scopes, key)), + Ok(reduced) => Ok(finish_bucket(reduced, key)), Err(error) => { drain_to_eof(&mut input); Err(error) @@ -423,9 +425,9 @@ pub(crate) fn reduce_to_buckets_with_active_sources( selections_and_active_sources: &[(FlowSelection, Arc)], ) -> Result, NfdumpError> { match reduce_stream_for_active_sources(&mut input, selections_and_active_sources) { - Ok(scopes) => Ok(scopes + Ok(reduced) => Ok(reduced .into_iter() - .map(|scopes| finish_bucket(scopes, key.clone())) + .map(|reduced| finish_bucket(reduced, key.clone())) .collect()), Err(error) => { drain_to_eof(&mut input); @@ -443,6 +445,9 @@ pub(crate) fn reduce_to_daily_source_activities( .map(|_| HashMap::::new()) .collect::>(); let result = visit_stream(&mut input, |flow, _, _| { + if flow.packets == 0 { + return Ok(()); + } for (selection, activity) in selections.iter().zip(&mut activities) { if classify_selected(flow, selection).is_none() { continue; @@ -468,14 +473,14 @@ pub(crate) fn reduce_to_daily_source_activities( fn reduce_stream_for_active_sources( input: &mut impl Read, selections_and_active_sources: &[(FlowSelection, Arc)], -) -> Result, NfdumpError> { +) -> Result, NfdumpError> { validate_daily_active_selection_pairs(selections_and_active_sources)?; - let mut scopes = (0..selections_and_active_sources.len()) - .map(|_| std::array::from_fn(|_| ScopeAccumulator::default())) + let mut reduced = (0..selections_and_active_sources.len()) + .map(|_| ReducedScopes::default()) .collect::>(); visit_stream(input, |flow, block_index, record_ordinal| { - for ((selection, active_sources), scopes) in - selections_and_active_sources.iter().zip(&mut scopes) + for ((selection, active_sources), reduced) in + selections_and_active_sources.iter().zip(&mut reduced) { let Some(locality) = classify_selected(flow, selection) else { continue; @@ -483,11 +488,11 @@ fn reduce_stream_for_active_sources( if !active_sources.contains(&flow.source_address) { continue; } - add_flow_to_scopes(scopes, flow, locality, block_index, record_ordinal)?; + reduced.add(flow, locality, block_index, record_ordinal)?; } Ok(()) })?; - Ok(scopes) + Ok(reduced) } fn validate_daily_active_selections(selections: &[FlowSelection]) -> Result<(), NfdumpError> { @@ -538,7 +543,7 @@ fn reduce_stream( input: &mut R, selection: &FlowSelection, active_sources: Option<&AddressSet>, -) -> Result<[ScopeAccumulator; 10], NfdumpError> { +) -> Result { if selection.selects_daily_active_sources() && active_sources.is_none() { return Err(NfdumpError::new( Phase::Aggregate, @@ -546,7 +551,7 @@ fn reduce_stream( ErrorReason::MissingDailyActiveSources, )); } - let mut scopes = std::array::from_fn(|_| ScopeAccumulator::default()); + let mut reduced = ReducedScopes::default(); visit_stream(input, |flow, block_index, record_ordinal| { let Some(locality) = classify_selected(flow, selection) else { return Ok(()); @@ -554,9 +559,47 @@ fn reduce_stream( if active_sources.is_some_and(|sources| !sources.contains(&flow.source_address)) { return Ok(()); } - add_flow_to_scopes(&mut scopes, flow, locality, block_index, record_ordinal) + reduced.add(flow, locality, block_index, record_ordinal) })?; - Ok(scopes) + Ok(reduced) +} + +struct ReducedScopes { + scopes: [ScopeAccumulator; 10], + dropped_zero_packet_flows: u64, +} + +impl Default for ReducedScopes { + fn default() -> Self { + Self { + scopes: std::array::from_fn(|_| ScopeAccumulator::default()), + dropped_zero_packet_flows: 0, + } + } +} + +impl ReducedScopes { + fn add( + &mut self, + flow: &Flow, + locality: (EndpointLocality, EndpointLocality), + block_index: u64, + record_ordinal: u64, + ) -> Result<(), NfdumpError> { + if flow.packets == 0 { + self.dropped_zero_packet_flows = self + .dropped_zero_packet_flows + .saturating_add(flow.flow_count.unsigned_abs()); + return Ok(()); + } + add_flow_to_scopes( + &mut self.scopes, + flow, + locality, + block_index, + record_ordinal, + ) + } } fn add_flow_to_scopes( @@ -1035,12 +1078,12 @@ const fn record_field_at(offset: usize) -> Field { } } -fn finish_bucket(scopes: [ScopeAccumulator; 10], key: BucketKey) -> CanonicalBucket { +fn finish_bucket(reduced: ReducedScopes, key: BucketKey) -> CanonicalBucket { let mut traffic = Vec::with_capacity(10); let mut protocols = Vec::with_capacity(10); let mut addresses = Vec::with_capacity(20); let mut ports = Vec::with_capacity(20); - for (index, accumulator) in scopes.into_iter().enumerate() { + for (index, accumulator) in reduced.scopes.into_iter().enumerate() { let scope = Scope::new( if index < 5 { IpVersion::V4 @@ -1069,6 +1112,7 @@ fn finish_bucket(scopes: [ScopeAccumulator; 10], key: BucketKey) -> CanonicalBuc addresses, ports, five_minute_starts, + dropped_zero_packet_flows: reduced.dropped_zero_packet_flows, } } @@ -1131,6 +1175,10 @@ mod tests { use crate::domain::{Granularity, Locality, Scope}; use crate::locality::{LocalityRuleConfig, LocalityRules}; + fn address_keys(totals: &AddressTotals) -> AddressSet { + totals.iter().map(|(address, _)| address).collect() + } + fn rules(configs: &[LocalityRuleConfig]) -> Arc { Arc::new(LocalityRules::from_config(configs, std::path::Path::new("/")).unwrap()) } @@ -1231,13 +1279,19 @@ mod tests { assert_eq!(bucket.protocols[index].protocols, ["6"]); } assert_eq!( - bucket.addresses[0].addresses, + address_keys(&bucket.addresses[0].addresses), [IpAddr::V4(Ipv4Addr::new(198, 51, 100, 2))] .into_iter() .collect::() ); assert_eq!( - bucket.addresses[1].addresses, + bucket.addresses[0] + .addresses + .get(&IpAddr::V4(Ipv4Addr::new(198, 51, 100, 2))), + Some(AddressTraffic::new(2, 128)) + ); + assert_eq!( + address_keys(&bucket.addresses[1].addresses), [IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1))] .into_iter() .collect::() @@ -1246,6 +1300,63 @@ mod tests { assert!(bucket.ports[1].ports.contains(443)); } + #[test] + fn zero_packet_flows_are_dropped_and_counted_while_the_bucket_stays_observed() { + let mut zero_packets = base_record(); + zero_packets[16..20].copy_from_slice(&[198, 51, 100, 9]); + zero_packets[32..40].copy_from_slice(&0_u64.to_le_bytes()); + zero_packets[48..56].copy_from_slice(&3_u64.to_le_bytes()); + + let zero_only = reduce_to_bucket( + Cursor::new(stream(&[zero_packets])), + key(), + &FlowSelection::default(), + ) + .unwrap(); + assert_eq!(zero_only.dropped_zero_packet_flows, 3); + assert_eq!(zero_only.coverage, BucketCoverage::complete_unit()); + assert_eq!(zero_only.five_minute_starts, [1_700_000_000].into()); + assert_eq!(zero_only.traffic.len(), 10); + assert!( + zero_only + .traffic + .iter() + .all(|traffic| traffic.metrics == TrafficMetrics::default()) + ); + assert!( + zero_only + .addresses + .iter() + .all(|addresses| addresses.addresses.is_empty()) + ); + + let mixed = reduce_to_bucket( + Cursor::new(stream(&[base_record(), zero_packets])), + key(), + &FlowSelection::default(), + ) + .unwrap(); + assert_eq!(mixed.dropped_zero_packet_flows, 3); + assert_eq!(mixed.traffic[0].metrics.flows, 1); + assert_eq!(mixed.traffic[0].metrics.packets, 2); + assert_eq!(mixed.traffic[0].metrics.bytes, 128); + let source = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)); + assert_eq!( + address_keys(&mixed.addresses[1].addresses), + [source].into_iter().collect::() + ); + assert_eq!( + mixed.addresses[1].addresses.get(&source), + Some(AddressTraffic::new(2, 128)) + ); + assert_eq!( + address_keys(&mixed.addresses[0].addresses), + [IpAddr::V4(Ipv4Addr::new(198, 51, 100, 2))] + .into_iter() + .collect::() + ); + } + #[test] fn icmp_requires_and_retains_a_zero_destination_port() { let mut invalid = ONE_V4_TEST_STREAM.to_vec(); @@ -1303,7 +1414,7 @@ mod tests { assert_eq!(bucket.protocols[index].protocols, ["58"]); } assert_eq!( - bucket.addresses[10].addresses, + address_keys(&bucket.addresses[10].addresses), [IpAddr::V6(Ipv6Addr::new(0x2001, 0x0db8, 0, 0, 0, 0, 0, 2))] .into_iter() .collect::() @@ -1487,11 +1598,11 @@ mod tests { }; assert_eq!(buckets.iter().map(all_v4).collect::>(), [3, 0, 3, 4]); assert_eq!( - buckets[0].addresses[1].addresses, + address_keys(&buckets[0].addresses[1].addresses), [source_a].into_iter().collect::() ); assert_eq!( - buckets[3].addresses[1].addresses, + address_keys(&buckets[3].addresses[1].addresses), [source_b].into_iter().collect::() ); } @@ -1794,7 +1905,7 @@ mod tests { assert_eq!(bucket.traffic[3].metrics.flows, 2); assert_eq!(bucket.protocols[0].protocols, ["17", "6"]); assert_eq!( - bucket.addresses[1].addresses, + address_keys(&bucket.addresses[1].addresses), [ IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)), IpAddr::V4(Ipv4Addr::new(192, 0, 2, 2)), diff --git a/tools/netflow-db/src/normalize.rs b/tools/netflow-db/src/normalize.rs index c5de9c5d..f9cf6f9f 100644 --- a/tools/netflow-db/src/normalize.rs +++ b/tools/netflow-db/src/normalize.rs @@ -102,7 +102,8 @@ pub fn normalize_csv_values( let src_ip = parse_ip(required(value(src_column)?, src_column)?)?; let dst_ip = parse_ip(required(value(dst_column)?, dst_column)?)?; let protocol = extract_protocol(optional("protocol")?, config)?; - let packets = extract_nonnegative(optional("packets")?, "packets", 0)?; + let raw_packets = optional("packets")?; + let packets = extract_nonnegative(raw_packets, "packets", 0)?; let bytes = extract_nonnegative(optional("bytes")?, "bytes", 0)?; let src_tos = extract_u8(optional("src_tos")?, "src_tos", 0)?; let dst_tos = extract_u8(optional("dst_tos")?, "dst_tos", 0)?; @@ -123,6 +124,9 @@ pub fn normalize_csv_values( .with_ports(src_port, dst_port) .with_measurements(duration_ms, min_ttl, max_ttl)? .with_flow_count(flow_count)?; + if raw_packets.is_none() { + observation = observation.with_unreported_packets(); + } observation.time_received_ms = timestamps.get("time_received").copied(); observation.time_end_ms = timestamps.get("time_end").copied(); observation.time_start_ms = timestamps.get("time_start").copied(); diff --git a/tools/netflow-db/src/operations.rs b/tools/netflow-db/src/operations.rs index a30339a1..a203719a 100644 --- a/tools/netflow-db/src/operations.rs +++ b/tools/netflow-db/src/operations.rs @@ -120,12 +120,12 @@ pub fn select_web_verification_window( ON st.source_id = ts.source_id AND st.granularity = '5m' AND st.bucket_start = ts.bucket_start AND st.ip_version = 4 AND st.src_locality = 'all' AND st.dst_locality = 'all' - AND st.structure_kind = 'structure' + AND st.measure = 'addresses' AND st.structure_kind = 'structure' JOIN address_structure_stats sp ON sp.source_id = ts.source_id AND sp.granularity = '5m' AND sp.bucket_start = ts.bucket_start AND sp.ip_version = 4 AND sp.src_locality = 'all' AND sp.dst_locality = 'all' - AND sp.structure_kind = 'spectrum' + AND sp.measure = 'addresses' AND sp.structure_kind = 'spectrum' WHERE ts.source_id = ?1 AND ts.granularity = '5m' AND ts.src_locality = 'all' AND ts.dst_locality = 'all' ORDER BY ts.bucket_start LIMIT 1 diff --git a/tools/netflow-db/src/pipeline.rs b/tools/netflow-db/src/pipeline.rs index e22b0ea8..523bc97a 100644 --- a/tools/netflow-db/src/pipeline.rs +++ b/tools/netflow-db/src/pipeline.rs @@ -1514,7 +1514,7 @@ fn bind_identity( }) }); let result_config = json!({ - "version": 5, + "version": 6, "timezone": pipeline.timezone, "locality": pipeline .selection @@ -1532,8 +1532,9 @@ fn bind_identity( "maad": { "enabled": pipeline.run_maad, "backend": "in-process", - "contract_version": 4, - "config": maad_config + "contract_version": 5, + "config": maad_config, + "measures": crate::domain::MaadMeasure::ALL.map(crate::domain::MaadMeasure::as_str), } }); let identity = ProductIdentity::create( @@ -2575,9 +2576,7 @@ fn process_nfcapd_tree_day( .collect::>() }); let decode_pool = build_nfcapd_decode_pool()?; - let mut next = start; - - while next < end { + let plan_batch = |next: i64| -> Result { let batch_starts = nfcapd_batch_starts( next, end, @@ -2587,7 +2586,7 @@ fn process_nfcapd_tree_day( &tree.member_bounds, tree.extend_gaps_to_window, )?; - next = batch_starts + let following = batch_starts .last() .copied() .map(|last| next_local_five_minute_start(last, &timezone)) @@ -2609,8 +2608,14 @@ fn process_nfcapd_tree_day( .collect::>() .into_iter() .collect::>(); - verify_nfdump_revision(first_pipeline)?; - let decoded = decode_pool.install(|| { + Ok(NfcapdTreeBatch { + following, + prepared, + requests, + }) + }; + let decode_batch = |requests: &[NfcapdTreeRequest]| { + decode_pool.install(|| { requests .par_iter() .map(|((member, bucket_start), path)| { @@ -2633,49 +2638,132 @@ fn process_nfcapd_tree_day( Ok::<_, PipelineError>(((member.clone(), *bucket_start), buckets)) }) .collect::, _>>() - })?; - verify_nfdump_revision(first_pipeline)?; - - for (selection_index, sink_index) in pending.iter().copied().enumerate() { - let aggregate = aggregates[sink_index] - .as_mut() - .expect("pending output has aggregate state"); - for timestamp in &prepared { - for job in ×tamp.jobs { - let member_buckets = job - .present - .iter() - .map(|(member, _)| { - &decoded[&(member.clone(), timestamp.bucket_start)][selection_index] - }) - .collect::>(); - let logical = logical_source_bucket( - &job.source_id, - timestamp.bucket_start, - job.expected_units, - &member_buckets, - )?; - aggregate.reject_persisted_siblings( - sinks[sink_index].connection, - &logical, - &timezone, - )?; - publish_nfcapd_bucket( - sinks[sink_index].connection, - &logical, - &job.owners, - &job.absences, - &job.evidence, - true, - sinks[sink_index].pipeline.run_maad, - )?; - aggregate.include(&logical, &timezone)?; - sinks[sink_index].report.rollup_buckets += aggregate.flush_complete( - sinks[sink_index].connection, - sinks[sink_index].pipeline.run_maad, - )?; - sinks[sink_index].report.five_minute_buckets += 1; - } + }) + }; + + if start >= end { + return Ok(()); + } + let first = plan_batch(start)?; + verify_nfdump_revision(first_pipeline)?; + publish_overlapped( + first, + |batch| { + (batch.following < end).then(|| { + verify_nfdump_revision(first_pipeline)?; + plan_batch(batch.following) + }) + }, + |batch| { + let decoded = decode_batch(&batch.requests)?; + verify_nfdump_revision(first_pipeline)?; + Ok(decoded) + }, + |batch, decoded| { + publish_nfcapd_tree_batch( + &batch.prepared, + decoded, + sinks, + pending, + aggregates, + &timezone, + ) + }, + ) +} + +/// Publish batches in order while the following batch decodes on a scoped thread. +/// +/// Planning, revision checks and decoding for batch N+1 report their errors only after +/// batch N is published, and at most two decoded batches are held at once. +fn publish_overlapped( + first: B, + next: impl Fn(&B) -> Option>, + decode: impl Fn(&B) -> Result + Sync, + mut publish: impl FnMut(&B, &D) -> Result<(), PipelineError>, +) -> Result<(), PipelineError> { + let mut batch = first; + let mut decoded = decode(&batch)?; + loop { + let upcoming = next(&batch); + let (published, upcoming_decoded) = std::thread::scope(|scope| { + let decoding = match &upcoming { + Some(Ok(upcoming)) => Some(scope.spawn(|| decode(upcoming))), + _ => None, + }; + let published = publish(&batch, &decoded); + let upcoming_decoded = decoding.map(|handle| { + handle + .join() + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) + }); + (published, upcoming_decoded) + }); + published?; + let Some(upcoming) = upcoming else { + return Ok(()); + }; + let upcoming = upcoming?; + decoded = upcoming_decoded.expect("a planned batch is decoded")?; + batch = upcoming; + } +} + +type NfcapdTreeRequest = ((String, i64), PathBuf); + +struct NfcapdTreeBatch { + following: i64, + prepared: Vec, + requests: Vec, +} + +fn publish_nfcapd_tree_batch( + prepared: &[PreparedTreeTimestamp], + decoded: &BTreeMap<(String, i64), Vec>, + sinks: &mut [ProductSink<'_>], + pending: &[usize], + aggregates: &mut [Option], + timezone: &str, +) -> Result<(), PipelineError> { + for (selection_index, sink_index) in pending.iter().copied().enumerate() { + let aggregate = aggregates[sink_index] + .as_mut() + .expect("pending output has aggregate state"); + for timestamp in prepared { + for job in ×tamp.jobs { + let member_buckets = job + .present + .iter() + .map(|(member, _)| { + &decoded[&(member.clone(), timestamp.bucket_start)][selection_index] + }) + .collect::>(); + let logical = logical_source_bucket( + &job.source_id, + timestamp.bucket_start, + job.expected_units, + &member_buckets, + )?; + aggregate.reject_persisted_siblings( + sinks[sink_index].connection, + &logical, + timezone, + )?; + publish_nfcapd_bucket( + sinks[sink_index].connection, + &logical, + &job.owners, + &job.absences, + &job.evidence, + true, + sinks[sink_index].pipeline.run_maad, + )?; + aggregate.include(&logical, timezone)?; + sinks[sink_index].report.rollup_buckets += aggregate.flush_complete( + sinks[sink_index].connection, + sinks[sink_index].pipeline.run_maad, + )?; + sinks[sink_index].report.five_minute_buckets += 1; } } } @@ -3592,6 +3680,72 @@ mod tests { fs::set_permissions(executable, fs::Permissions::from_mode(0o755)).unwrap(); } + #[test] + fn overlapped_publishing_finishes_a_batch_before_a_later_batch_fails() { + let requests_per_batch = NFCAPD_DECODE_BATCH_SIZE + 1; + let batches = 3; + let published = std::sync::Mutex::new(Vec::new()); + let decoded = std::sync::Mutex::new(Vec::new()); + + let result = publish_overlapped( + 0_usize, + |&batch| (batch + 1 < batches).then_some(Ok(batch + 1)), + |&batch| { + let requests = (0..requests_per_batch) + .map(|request| batch * requests_per_batch + request) + .collect::>(); + decoded.lock().unwrap().push(batch); + if batch == 1 { + return Err(PipelineError::InvalidConfig(format!( + "corrupt capture in batch {batch}" + ))); + } + Ok(requests) + }, + |&batch, requests| { + assert_eq!(requests.len(), requests_per_batch); + published.lock().unwrap().push(batch); + Ok(()) + }, + ); + + assert!( + result + .unwrap_err() + .to_string() + .contains("corrupt capture in batch 1") + ); + assert_eq!(*published.lock().unwrap(), [0]); + assert_eq!(*decoded.lock().unwrap(), [0, 1]); + } + + #[test] + fn overlapped_publishing_reports_a_planning_error_after_the_current_batch() { + let published = std::sync::Mutex::new(Vec::new()); + + let result = publish_overlapped( + 0_usize, + |_| { + Some(Err(PipelineError::InvalidConfig( + "unplannable batch".into(), + ))) + }, + |&batch| Ok(batch), + |&batch, _| { + published.lock().unwrap().push(batch); + Ok(()) + }, + ); + + assert!( + result + .unwrap_err() + .to_string() + .contains("unplannable batch") + ); + assert_eq!(*published.lock().unwrap(), [0]); + } + fn write_nfcapd_day(root: &Path, member: &str, date: &str) { let mut bucket_start = parse_date_start(date, DEFAULT_TIMEZONE).unwrap(); let end = next_date_start(date, DEFAULT_TIMEZONE).unwrap(); diff --git a/tools/netflow-db/src/publish.rs b/tools/netflow-db/src/publish.rs index 36aa866f..8d1369f0 100644 --- a/tools/netflow-db/src/publish.rs +++ b/tools/netflow-db/src/publish.rs @@ -2,16 +2,21 @@ use std::{ net::IpAddr, + num::NonZeroUsize, sync::OnceLock, time::{Duration, Instant}, }; use rayon::prelude::*; use rusqlite::Connection; +use serde::Serialize; use thiserror::Error; use crate::{ - domain::{AddressSetRow, BucketKey, CanonicalBucket, CanonicalRows, DomainError, IpVersion}, + domain::{ + AddressSetRow, AddressTraffic, BucketKey, CanonicalBucket, CanonicalRows, DomainError, + Granularity, IpVersion, MaadMeasure, + }, maad, storage::{ AddressCountStatsRow, AddressStructureStatsRow, BucketCoverageRow, PortCountStatsRow, @@ -32,11 +37,13 @@ pub enum PublishError { Json(#[from] serde_json::Error), #[error("unable to build MAAD worker pool: {0}")] MaadPool(String), + #[error(transparent)] + Maad(#[from] maad::MaadError), #[error("aggregate bucket lacks complete five-minute coverage: {0:?}")] IncompleteCoverage(BucketKey), } -const MAAD_WORKERS: usize = 2; +const MAX_MAAD_WORKERS: usize = 8; static MAAD_POOL: OnceLock> = OnceLock::new(); /// Aggregate timings and work counts for one or more `write_buckets` calls. @@ -65,6 +72,8 @@ pub struct WriteBucketsProfile { pub(crate) port_count_rows: u64, pub(crate) maad_address_sets: u64, pub(crate) maad_addresses: u64, + pub(crate) maad_zero_weight_addresses: u64, + pub(crate) dropped_zero_packet_flows: u64, pub(crate) address_structure_rows: u64, pub(crate) address_structure_json_bytes: u64, } @@ -144,6 +153,18 @@ pub(crate) fn write_buckets_profiled( .collect::>(); insert_bucket_coverage_rows(connection, &coverage_rows)?; for bucket in buckets { + if bucket.key.granularity == Granularity::FiveMinutes + && bucket.dropped_zero_packet_flows > 0 + { + profile.dropped_zero_packet_flows += bucket.dropped_zero_packet_flows; + tracing::info!( + source_id = %bucket.key.source_id, + granularity = bucket.key.granularity.as_str(), + bucket_start = bucket.key.bucket_start, + dropped_zero_packet_flows = bucket.dropped_zero_packet_flows, + "dropped zero-packet flows before aggregation" + ); + } let canonical_rows_started = Instant::now(); let rows = bucket.rows(); profile.canonical_rows_elapsed += canonical_rows_started.elapsed(); @@ -166,8 +187,18 @@ pub(crate) fn write_buckets_profiled( .map(|addresses| count(addresses.addresses.len())) .sum::(); let maad_started = Instant::now(); - let address_structure = maad_rows(&rows.address_sets)?; + let (address_structure, zero_weight_addresses) = maad_rows(&rows.address_sets)?; profile.maad_elapsed += maad_started.elapsed(); + profile.maad_zero_weight_addresses += zero_weight_addresses; + if zero_weight_addresses > 0 { + tracing::info!( + source_id = %bucket.key.source_id, + granularity = bucket.key.granularity.as_str(), + bucket_start = bucket.key.bucket_start, + zero_weight_addresses, + "excluded zero-weight addresses from weighted MAAD measures" + ); + } profile.address_structure_rows += count(address_structure.len()); profile.address_structure_json_bytes += address_structure .iter() @@ -272,80 +303,201 @@ fn insert_rows( }) } +/// Compute every MAAD measure for each scoped address set. +/// +/// Returns the rows and the number of addresses excluded from weighted measures +/// because their summed weight was zero. fn maad_rows( address_sets: &[AddressSetRow<'_>], -) -> Result, PublishError> { +) -> Result<(Vec, u64), PublishError> { if address_sets.is_empty() { - return Ok(Vec::new()); + return Ok((Vec::new(), 0)); } let pool = maad_pool()?; - let rows = pool.install(|| { + let measured = pool.install(|| { address_sets .par_iter() - .map(|addresses| { - let result = maad_result(addresses); - let metadata_json = serde_json::to_string(&result.metadata)?; - let dimensions = dimensions(&addresses.key, addresses.scope); - Ok::<_, serde_json::Error>([ - AddressStructureStatsRow { - dimensions: dimensions.clone(), - address_side: addresses.address_side.as_str().to_owned(), - structure_kind: "structure".into(), - values_json: serde_json::to_string(&result.structure)?, - metadata_json: metadata_json.clone(), - }, - AddressStructureStatsRow { - dimensions: dimensions.clone(), - address_side: addresses.address_side.as_str().to_owned(), - structure_kind: "spectrum".into(), - values_json: serde_json::to_string(&result.spectrum)?, - metadata_json: metadata_json.clone(), - }, - AddressStructureStatsRow { - dimensions, - address_side: addresses.address_side.as_str().to_owned(), - structure_kind: "dimension".into(), - values_json: serde_json::to_string(&result.dimensions)?, - metadata_json, - }, - ]) - }) + .map(scope_rows) .collect::, _>>() })?; - Ok(rows.into_iter().flatten().collect()) + let zero_weight_addresses = measured.iter().map(|(_, excluded)| count(*excluded)).sum(); + Ok(( + measured.into_iter().flat_map(|(rows, _)| rows).collect(), + zero_weight_addresses, + )) } -fn maad_result(addresses: &AddressSetRow<'_>) -> maad::MaadResult { - match addresses.scope.ip_version { - IpVersion::V4 => maad::compute(addresses.addresses.iter().filter_map( - |address| match address { - IpAddr::V4(address) => Some(*address), +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct WeightedMetadata<'a> { + #[serde(flatten)] + metadata: &'a maad::MaadMetadata, + zero_weight_addrs: usize, +} + +/// One scope's MAAD results: distinct addresses, then packets and bytes with +/// the number of zero-weight addresses each weighted measure excluded. +struct ScopeResults { + addresses: maad::MaadResult, + weighted: [(maad::MaadResult, usize); 2], +} + +fn scope_rows( + addresses: &AddressSetRow<'_>, +) -> Result<(Vec, usize), PublishError> { + let results = match addresses.scope.ip_version { + IpVersion::V4 => scope_results(addresses.addresses.iter().filter_map( + |(address, traffic)| match address { + IpAddr::V4(address) => Some((address, traffic)), IpAddr::V6(_) => None, }, - )), - IpVersion::V6 => maad::compute(addresses.addresses.iter().filter_map( - |address| match address { - IpAddr::V6(address) => Some(*address), + ))?, + IpVersion::V6 => scope_results(addresses.addresses.iter().filter_map( + |(address, traffic)| match address { + IpAddr::V6(address) => Some((address, traffic)), IpAddr::V4(_) => None, }, - )), + ))?, + }; + let dimensions = dimensions(&addresses.key, addresses.scope); + let row = |measure: MaadMeasure, + structure_kind: &str, + values_json: String, + metadata_json: &str| AddressStructureStatsRow { + dimensions: dimensions.clone(), + address_side: addresses.address_side.as_str().to_owned(), + measure: measure.as_str().to_owned(), + structure_kind: structure_kind.to_owned(), + values_json, + metadata_json: metadata_json.to_owned(), + }; + let address_metadata = serde_json::to_string(&results.addresses.metadata)?; + let mut rows = vec![ + row( + MaadMeasure::Addresses, + "structure", + serde_json::to_string(&results.addresses.structure)?, + &address_metadata, + ), + row( + MaadMeasure::Addresses, + "spectrum", + serde_json::to_string(&results.addresses.spectrum)?, + &address_metadata, + ), + row( + MaadMeasure::Addresses, + "dimension", + serde_json::to_string(&results.addresses.dimensions)?, + &address_metadata, + ), + ]; + let mut zero_weight_addresses = 0; + for (measure, (result, zero_weight_addrs)) in [MaadMeasure::Packets, MaadMeasure::Bytes] + .into_iter() + .zip(&results.weighted) + { + let metadata_json = serde_json::to_string(&WeightedMetadata { + metadata: &result.metadata, + zero_weight_addrs: *zero_weight_addrs, + })?; + rows.push(row( + measure, + "structure", + serde_json::to_string(&result.structure)?, + &metadata_json, + )); + rows.push(row( + measure, + "dimension", + serde_json::to_string(&result.dimensions)?, + &metadata_json, + )); + zero_weight_addresses += zero_weight_addrs; + } + Ok((rows, zero_weight_addresses)) +} + +/// Compute all measures over one shared prefix walk. Weighted measures leave out +/// zero-weight addresses, so a scope with any falls back to separate walks. +fn scope_results( + entries: impl Iterator, +) -> Result { + let entries = entries.collect::>(); + let zero_packets = entries + .iter() + .filter(|(_, traffic)| traffic.packets == 0) + .count(); + let zero_bytes = entries + .iter() + .filter(|(_, traffic)| traffic.bytes == 0) + .count(); + if zero_packets == 0 && zero_bytes == 0 { + let (addresses, [packets, bytes]) = + maad::compute_measures(entries.iter().map(|&(address, traffic)| { + (address, [traffic.packets as f64, traffic.bytes as f64]) + }))?; + return Ok(ScopeResults { + addresses, + weighted: [(packets, 0), (bytes, 0)], + }); + } + let weighted = |weight: fn(AddressTraffic) -> u64| { + maad::compute_weighted(entries.iter().filter_map(|&(address, traffic)| { + match weight(traffic) { + 0 => None, + value => Some((address, value as f64)), + } + })) + }; + Ok(ScopeResults { + addresses: maad::compute(entries.iter().map(|&(address, _)| address)), + weighted: [ + (weighted(|traffic| traffic.packets)?, zero_packets), + (weighted(|traffic| traffic.bytes)?, zero_bytes), + ], + }) +} + +/// Size the MAAD worker pool before its first use; later calls fail. +pub fn set_maad_workers(workers: NonZeroUsize) -> Result<(), PublishError> { + let mut initialized = false; + let pool = MAAD_POOL.get_or_init(|| { + initialized = true; + build_maad_pool(workers.get()) + }); + if !initialized { + return Err(PublishError::MaadPool( + "the worker pool was already initialized".into(), + )); } + pool.as_ref() + .map(|_| ()) + .map_err(|error| PublishError::MaadPool(error.clone())) } fn maad_pool() -> Result<&'static rayon::ThreadPool, PublishError> { match MAAD_POOL.get_or_init(|| { - rayon::ThreadPoolBuilder::new() - .num_threads(MAAD_WORKERS) - .thread_name(|index| format!("maad-{index}")) - .build() - .map_err(|error| error.to_string()) + build_maad_pool( + std::thread::available_parallelism() + .map_or(1, NonZeroUsize::get) + .min(MAX_MAAD_WORKERS), + ) }) { Ok(pool) => Ok(pool), Err(error) => Err(PublishError::MaadPool(error.clone())), } } +fn build_maad_pool(workers: usize) -> Result { + rayon::ThreadPoolBuilder::new() + .num_threads(workers) + .thread_name(|index| format!("maad-{index}")) + .build() + .map_err(|error| error.to_string()) +} + fn dimensions(key: &BucketKey, scope: crate::domain::Scope) -> StatsDimensions { StatsDimensions { source_id: key.source_id.clone(), @@ -371,7 +523,7 @@ mod tests { use super::*; use crate::{ domain::{ - AddressSet, AddressSide, EndpointLocality, FlowObservation, Granularity, IpVersion, + AddressSide, AddressTotals, EndpointLocality, FlowObservation, Granularity, IpVersion, Locality, Scope, ScopedAddressesFact, StatisticalBucket, }, storage::init_stats_tables, @@ -397,6 +549,20 @@ mod tests { .with_locality(EndpointLocality::Internal, EndpointLocality::External), ) .unwrap(); + builder + .add( + FlowObservation::new( + IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)), + IpAddr::V4(Ipv4Addr::new(198, 51, 100, 9)), + 6, + 0, + 0, + 0, + ) + .unwrap() + .with_locality(EndpointLocality::Internal, EndpointLocality::External), + ) + .unwrap(); let profile = write_buckets_profiled(&connection, &[builder.finish()], true).unwrap(); @@ -416,7 +582,7 @@ mod tests { row.get::<_, i64>(0) }) .unwrap(), - 60 + 140 ); assert_eq!(profile.bucket_keys, 1); assert_eq!(profile.write_calls, 1); @@ -425,14 +591,23 @@ mod tests { assert_eq!(profile.address_count_rows, 20); assert_eq!(profile.port_count_rows, 40); assert_eq!(profile.maad_address_sets, 20); - assert_eq!(profile.address_structure_rows, 60); + assert_eq!(profile.address_structure_rows, 140); + assert_eq!(profile.maad_zero_weight_addresses, 0); + assert_eq!(profile.dropped_zero_packet_flows, 1); assert!(profile.address_structure_json_bytes > 0); assert!(profile.total_elapsed >= profile.other_elapsed()); } + #[test] + fn maad_workers_cannot_resize_an_initialized_pool() { + assert!(maad_pool().unwrap().current_num_threads() >= 1); + let error = set_maad_workers(NonZeroUsize::new(2).unwrap()).unwrap_err(); + assert!(error.to_string().contains("already initialized")); + } + #[test] fn address_insertion_order_does_not_change_persisted_products() { - fn persist(addresses: impl IntoIterator) -> Connection { + fn persist(addresses: impl IntoIterator) -> Connection { let connection = Connection::open_in_memory().unwrap(); init_stats_tables(&connection).unwrap(); let mut builder = @@ -466,7 +641,7 @@ mod tests { ), query( connection, - "SELECT printf('%s|%s|%s|%s|%s|%s|%s', ip_version, src_locality, dst_locality, address_side, structure_kind, values_json, metadata_json) FROM address_structure_stats ORDER BY ip_version, src_locality, dst_locality, address_side, structure_kind", + "SELECT printf('%s|%s|%s|%s|%s|%s|%s|%s', ip_version, src_locality, dst_locality, address_side, measure, structure_kind, values_json, metadata_json) FROM address_structure_stats ORDER BY ip_version, src_locality, dst_locality, address_side, measure, structure_kind", ), ) } @@ -474,8 +649,19 @@ mod tests { let first = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)); let second = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 2)); let third = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 3)); - let forward = persist([first, second, first, third]); - let reverse = persist([third, first, second, second]); + let traffic = AddressTraffic::new; + let forward = persist([ + (first, traffic(1, 10)), + (second, traffic(2, 200)), + (first, traffic(3, 30)), + (third, traffic(4, 4_000)), + ]); + let reverse = persist([ + (third, traffic(4, 4_000)), + (first, traffic(4, 40)), + (second, traffic(1, 100)), + (second, traffic(1, 100)), + ]); assert_eq!(product_rows(&forward), product_rows(&reverse)); } @@ -483,18 +669,22 @@ mod tests { #[test] fn maad_rows_preserve_scope_order_and_bytes() { let key = BucketKey::new("r1", Granularity::FiveMinutes, 0, 300); - let first_addresses = AddressSet::from_iter([ - IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)), - IpAddr::V4(Ipv4Addr::new(192, 0, 2, 2)), - IpAddr::V4(Ipv4Addr::new(192, 0, 2, 3)), - IpAddr::V4(Ipv4Addr::new(192, 0, 2, 4)), - ]); - let second_addresses = AddressSet::from_iter([ - IpAddr::V4(Ipv4Addr::new(198, 51, 100, 1)), - IpAddr::V4(Ipv4Addr::new(198, 51, 100, 2)), - IpAddr::V4(Ipv4Addr::new(198, 51, 100, 3)), - IpAddr::V4(Ipv4Addr::new(198, 51, 100, 4)), - ]); + let first_addresses = (0..64_u8) + .map(|index| { + ( + IpAddr::V4(Ipv4Addr::new(192, index % 4, index, 1)), + AddressTraffic::new(u64::from(index % 5 + 1).pow(3), 100), + ) + }) + .collect::(); + let second_addresses = (0..64_u8) + .map(|index| { + ( + IpAddr::V4(Ipv4Addr::new(198, index % 4, index, 1)), + AddressTraffic::new(5, 5 * u64::from(index) + 5), + ) + }) + .collect::(); let rows = [ AddressSetRow { key: key.clone(), @@ -510,39 +700,88 @@ mod tests { }, ]; - let first = maad_rows(&rows).unwrap(); - let second = maad_rows(&rows).unwrap(); + let (first, first_excluded) = maad_rows(&rows).unwrap(); + let (second, second_excluded) = maad_rows(&rows).unwrap(); assert_eq!(first, second); - assert_eq!(first.len(), 6); + assert_eq!((first_excluded, second_excluded), (0, 0)); + let kinds = |src_locality: &str, side: &str| { + [ + ("addresses", "structure"), + ("addresses", "spectrum"), + ("addresses", "dimension"), + ("packets", "structure"), + ("packets", "dimension"), + ("bytes", "structure"), + ("bytes", "dimension"), + ] + .map(|(measure, kind)| { + ( + src_locality.to_owned(), + side.to_owned(), + measure.to_owned(), + kind.to_owned(), + ) + }) + }; assert_eq!( first - .chunks_exact(3) - .map(|rows| ( - rows[0].dimensions.src_locality.as_str(), - rows[0].address_side.as_str(), - rows.iter() - .map(|row| row.structure_kind.as_str()) - .collect::>(), + .iter() + .map(|row| ( + row.dimensions.src_locality.clone(), + row.address_side.clone(), + row.measure.clone(), + row.structure_kind.clone(), )) .collect::>(), - vec![ - ("all", "source", vec!["structure", "spectrum", "dimension"]), - ( - "external", - "destination", - vec!["structure", "spectrum", "dimension"] - ), - ] + [kinds("all", "source"), kinds("external", "destination")].concat() ); + let packets_structure = |scope_rows: &[AddressStructureStatsRow]| { + scope_rows + .iter() + .find(|row| row.measure == "packets" && row.structure_kind == "structure") + .unwrap() + .values_json + .clone() + }; + let addresses_structure = |scope_rows: &[AddressStructureStatsRow]| { + scope_rows + .iter() + .find(|row| row.measure == "addresses" && row.structure_kind == "structure") + .unwrap() + .values_json + .clone() + }; + assert_ne!( + packets_structure(&first[..7]), + addresses_structure(&first[..7]) + ); + let tau = |json: String| -> Vec { + serde_json::from_str::>(&json) + .unwrap() + .iter() + .map(|row| row["tauTilde"].as_f64().unwrap()) + .collect() + }; + let uniform_packets = tau(packets_structure(&first[7..])); + let addresses = tau(addresses_structure(&first[7..])); + assert_eq!(uniform_packets.len(), addresses.len()); + for (packets, addresses) in uniform_packets.iter().zip(&addresses) { + assert!((packets - addresses).abs() < 1e-12); + } } #[test] fn maad_rows_use_the_ipv6_prefix_range_for_ipv6_scopes() { let base = u128::from(Ipv6Addr::new(0x2001, 0x0db8, 0, 0, 0, 0, 0, 0)); - let addresses = AddressSet::from_iter( - (0..=255_u128).map(|subnet| IpAddr::V6(Ipv6Addr::from(base | (subnet << 64)))), - ); + let addresses = (0..=255_u128) + .map(|subnet| { + ( + IpAddr::V6(Ipv6Addr::from(base | (subnet << 64))), + AddressTraffic::new(1 + (subnet % 7) as u64, 1_500), + ) + }) + .collect::(); let rows = [AddressSetRow { key: BucketKey::new("r1", Granularity::FiveMinutes, 0, 300), scope: Scope::new(IpVersion::V6, Locality::All, Locality::All), @@ -550,13 +789,74 @@ mod tests { addresses: &addresses, }]; - let rows = maad_rows(&rows).unwrap(); + let (rows, _) = maad_rows(&rows).unwrap(); - assert_eq!(rows.len(), 3); + assert_eq!(rows.len(), 7); assert!(rows.iter().all(|row| row.dimensions.ip_version == 6)); - let metadata: serde_json::Value = serde_json::from_str(&rows[0].metadata_json).unwrap(); - assert_eq!(metadata["totalAddrs"], 256); - assert_eq!(metadata["minPrefixLength"], 23); - assert_eq!(metadata["maxPrefixLength"], 63); + for row in &rows { + let metadata: serde_json::Value = serde_json::from_str(&row.metadata_json).unwrap(); + assert_eq!(metadata["totalAddrs"], 256, "{}", row.measure); + assert_eq!(metadata["minPrefixLength"], 23, "{}", row.measure); + assert_eq!(metadata["maxPrefixLength"], 63, "{}", row.measure); + } + } + + #[test] + fn weighted_measures_exclude_zero_weight_addresses_and_count_them() { + let addresses = (0..=255_u8) + .map(|last| { + let packets = u64::from(last % 4 != 0); + let bytes = if last < 10 { 0 } else { 40 * u64::from(last) }; + ( + IpAddr::V4(Ipv4Addr::new(10, last % 16, last, 1)), + AddressTraffic::new(packets, bytes), + ) + }) + .chain([( + IpAddr::V6(Ipv6Addr::new(0x2001, 0x0db8, 0, 0, 0, 0, 0, 1)), + AddressTraffic::new(0, 0), + )]) + .collect::(); + let rows = [AddressSetRow { + key: BucketKey::new("r1", Granularity::FiveMinutes, 0, 300), + scope: Scope::new(IpVersion::V4, Locality::All, Locality::All), + address_side: AddressSide::Destination, + addresses: &addresses, + }]; + + let (rows, excluded) = maad_rows(&rows).unwrap(); + + let metadata = |measure: &str| -> serde_json::Value { + serde_json::from_str( + &rows + .iter() + .find(|row| row.measure == measure && row.structure_kind == "structure") + .unwrap() + .metadata_json, + ) + .unwrap() + }; + assert_eq!(excluded, 64 + 10); + assert_eq!(metadata("addresses")["totalAddrs"], 256); + assert!(metadata("addresses").get("zeroWeightAddrs").is_none()); + assert_eq!(metadata("packets")["totalAddrs"], 192); + assert_eq!(metadata("packets")["zeroWeightAddrs"], 64); + assert_eq!(metadata("bytes")["totalAddrs"], 246); + assert_eq!(metadata("bytes")["zeroWeightAddrs"], 10); + let packets = addresses + .iter() + .filter_map(|(address, traffic)| match address { + IpAddr::V4(address) if traffic.packets > 0 => { + Some((address, traffic.packets as f64)) + } + _ => None, + }); + assert_eq!( + rows.iter() + .find(|row| row.measure == "packets" && row.structure_kind == "structure") + .unwrap() + .values_json, + serde_json::to_string(&maad::compute_weighted(packets).unwrap().structure).unwrap() + ); } } diff --git a/tools/netflow-db/src/storage.rs b/tools/netflow-db/src/storage.rs index 7f793947..03e9713e 100644 --- a/tools/netflow-db/src/storage.rs +++ b/tools/netflow-db/src/storage.rs @@ -37,13 +37,13 @@ pub const STATS_TABLE_NAMES: [&str; 6] = [ /// Versioned product schema bound into every pipeline database identity. pub fn product_schema() -> serde_json::Value { serde_json::json!({ - "version": 5, + "version": 6, "tables": [ {"name": "traffic_stats", "version": 4}, {"name": "protocol_stats", "version": 3}, {"name": "address_count_stats", "version": 3}, {"name": "port_count_stats", "version": 3}, - {"name": "address_structure_stats", "version": 3}, + {"name": "address_structure_stats", "version": 4}, {"name": "bucket_coverage", "version": 2} ] }) @@ -1511,18 +1511,20 @@ pub fn init_stats_tables(connection: &Connection) -> Result<(), StorageError> { src_locality TEXT NOT NULL CHECK (src_locality IN ('all', 'internal', 'external')), dst_locality TEXT NOT NULL CHECK (dst_locality IN ('all', 'internal', 'external')), address_side TEXT NOT NULL CHECK (address_side IN ('source', 'destination')), + measure TEXT NOT NULL CHECK (measure IN ('addresses', 'packets', 'bytes')), structure_kind TEXT NOT NULL CHECK (structure_kind IN ('structure', 'spectrum', 'dimension')), values_json TEXT NOT NULL, metadata_json TEXT NOT NULL, processed_at DATETIME DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (source_id, granularity, bucket_start, ip_version, src_locality, dst_locality, address_side, structure_kind) + CHECK (measure = 'addresses' OR structure_kind <> 'spectrum'), + PRIMARY KEY (source_id, granularity, bucket_start, ip_version, src_locality, dst_locality, address_side, measure, structure_kind) ) WITHOUT ROWID; CREATE INDEX IF NOT EXISTS idx_address_structure_stats_query - ON address_structure_stats (granularity, bucket_start, source_id, ip_version, src_locality, dst_locality, address_side, structure_kind); + ON address_structure_stats (granularity, bucket_start, source_id, ip_version, src_locality, dst_locality, address_side, measure, structure_kind); CREATE INDEX IF NOT EXISTS idx_address_structure_stats_timeseries ON address_structure_stats ( source_id, granularity, src_locality, dst_locality, - ip_version, structure_kind, bucket_start + ip_version, measure, structure_kind, bucket_start ); DROP INDEX IF EXISTS idx_protocol_stats_query; @@ -1920,6 +1922,7 @@ pub struct PortCountStatsRow { pub struct AddressStructureStatsRow { pub dimensions: StatsDimensions, pub address_side: String, + pub measure: String, pub structure_kind: String, pub values_json: String, pub metadata_json: String, @@ -2078,9 +2081,9 @@ pub fn insert_address_structure_stats_rows( " INSERT OR REPLACE INTO address_structure_stats ( source_id, granularity, bucket_start, bucket_end, ip_version, - src_locality, dst_locality, address_side, structure_kind, + src_locality, dst_locality, address_side, measure, structure_kind, values_json, metadata_json - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12) ", )?; for row in rows { @@ -2094,6 +2097,7 @@ pub fn insert_address_structure_stats_rows( d.src_locality, d.dst_locality, row.address_side, + row.measure, row.structure_kind, row.values_json, row.metadata_json @@ -2336,6 +2340,7 @@ impl AddressStructureStatsRow { Self { dimensions: StatsDimensions::example(), address_side: "source".into(), + measure: "addresses".into(), structure_kind: "structure".into(), values_json: "[]".into(), metadata_json: "{}".into(), @@ -3298,6 +3303,7 @@ mod tests { "src_locality", "dst_locality", "ip_version", + "measure", "structure_kind", "bucket_start", ] diff --git a/tools/netflow-db/src/verify.rs b/tools/netflow-db/src/verify.rs index c59ada54..a12abcfd 100644 --- a/tools/netflow-db/src/verify.rs +++ b/tools/netflow-db/src/verify.rs @@ -94,10 +94,8 @@ pub fn verify_database( )); } } - if options.require_maad_data && row_counts["address_structure_stats"] == 0 { - return Err(VerifyError::Incompatible( - "address_structure_stats has no rows".into(), - )); + if options.require_maad_data { + assert_maad_measures_present(&connection)?; } if options.require_processed { assert_processed_inputs_complete(&connection)?; @@ -337,6 +335,7 @@ const REQUIRED_COLUMNS: &[(&str, &[&str])] = &[ "src_locality", "dst_locality", "address_side", + "measure", "structure_kind", "values_json", "metadata_json", @@ -511,6 +510,62 @@ fn ipv4_literal(value: &str) -> Option<&str> { }) } +/// Every scope must carry a structure row for each MAAD measure. +fn assert_maad_measures_present(connection: &Connection) -> Result<(), VerifyError> { + let has_rows: bool = connection.query_row( + "SELECT EXISTS ( + SELECT 1 FROM address_structure_stats WHERE structure_kind = 'structure' + )", + [], + |row| row.get(0), + )?; + if !has_rows { + return Err(VerifyError::Incompatible( + "address_structure_stats has no structure rows".into(), + )); + } + let incomplete = connection + .query_row( + "SELECT source_id, granularity, bucket_start, ip_version, src_locality, + dst_locality, address_side, + SUM(measure = 'addresses'), SUM(measure = 'packets'), SUM(measure = 'bytes') + FROM address_structure_stats + WHERE structure_kind = 'structure' + GROUP BY granularity, bucket_start, source_id, ip_version, src_locality, + dst_locality, address_side + HAVING SUM(measure = 'addresses') = 0 OR SUM(measure = 'packets') = 0 + OR SUM(measure = 'bytes') = 0 + LIMIT 1", + [], + |row| { + let present = [row.get::<_, i64>(7)?, row.get(8)?, row.get(9)?]; + let missing = ["addresses", "packets", "bytes"] + .into_iter() + .zip(present) + .filter(|(_, count)| *count == 0) + .map(|(measure, _)| measure) + .collect::>() + .join(", "); + Ok(format!( + "address_structure_stats is missing {missing} structure rows for \ + source {} {} bucket {} IPv{} {}->{} {}", + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + row.get::<_, String>(6)?, + )) + }, + ) + .optional()?; + match incomplete { + Some(message) => Err(VerifyError::Incompatible(message)), + None => Ok(()), + } +} + fn assert_processed_inputs_complete(connection: &Connection) -> Result<(), VerifyError> { let pending: i64 = connection.query_row( "SELECT COUNT(*) FROM processed_inputs WHERE status != 'processed'", @@ -614,7 +669,7 @@ fn assert_ipv6_maad_rows( SELECT 1 FROM address_structure_stats WHERE source_id = ?1 AND granularity = '1h' AND ip_version = 6 AND src_locality = 'all' AND dst_locality = 'all' - AND structure_kind = 'structure' + AND measure = 'addresses' AND structure_kind = 'structure' AND bucket_start >= ?2 AND bucket_start < ?3 )", params![source_id, bucket_start, bucket_end], @@ -704,7 +759,7 @@ const STRUCTURE_QUERY: &str = " WHERE granularity = '1h' AND source_id IN (?) AND bucket_start >= ? AND bucket_start < ? AND ip_version = 4 AND src_locality = 'all' AND dst_locality = 'all' - AND structure_kind = 'structure' + AND measure = 'addresses' AND structure_kind = 'structure' GROUP BY source_id, bucket_start ORDER BY source_id, bucket_start LIMIT 1"; const SPECTRUM_QUERY: &str = " @@ -715,7 +770,7 @@ const SPECTRUM_QUERY: &str = " WHERE granularity = '1h' AND source_id IN (?) AND bucket_start >= ? AND bucket_start < ? AND ip_version = 4 AND src_locality = 'all' AND dst_locality = 'all' - AND structure_kind = 'spectrum' + AND measure = 'addresses' AND structure_kind = 'spectrum' GROUP BY source_id, bucket_start ORDER BY source_id, bucket_start LIMIT 1"; const FILE_DETAILS_QUERY: &str = " @@ -820,16 +875,88 @@ mod tests { .execute( "INSERT INTO address_structure_stats ( source_id, granularity, bucket_start, bucket_end, ip_version, - src_locality, dst_locality, address_side, structure_kind, + src_locality, dst_locality, address_side, measure, structure_kind, values_json, metadata_json - ) VALUES ('r1', '1h', 0, 3600, 6, 'all', 'all', 'source', 'structure', - '[]', '{}')", + ) VALUES ('r1', '1h', 0, 3600, 6, 'all', 'all', 'source', 'addresses', + 'structure', '[]', '{}')", [], ) .unwrap(); assert!(assert_ipv6_maad_rows(&connection, "r1", 0, 3_600).is_ok()); } + #[test] + fn maad_data_requires_a_structure_row_for_every_measure() { + let connection = Connection::open_in_memory().unwrap(); + init_schema(&connection).unwrap(); + let insert = |measure: &str, bucket_start: i64, address_side: &str| { + connection + .execute( + "INSERT INTO address_structure_stats ( + source_id, granularity, bucket_start, bucket_end, ip_version, + src_locality, dst_locality, address_side, measure, structure_kind, + values_json, metadata_json + ) VALUES ('r1', '5m', ?2, ?2 + 300, 4, 'all', 'all', ?3, ?1, + 'structure', '[]', '{}')", + params![measure, bucket_start, address_side], + ) + .unwrap(); + }; + + assert!(assert_maad_measures_present(&connection).is_err()); + for measure in ["addresses", "packets", "bytes"] { + insert(measure, 0, "source"); + } + assert!(assert_maad_measures_present(&connection).is_ok()); + insert("addresses", 300, "source"); + insert("packets", 300, "source"); + let error = assert_maad_measures_present(&connection).unwrap_err(); + assert!( + error + .to_string() + .contains("missing bytes structure rows for source r1 5m bucket 300") + ); + assert!( + connection + .execute( + "INSERT INTO address_structure_stats ( + source_id, granularity, bucket_start, bucket_end, ip_version, + src_locality, dst_locality, address_side, measure, structure_kind, + values_json, metadata_json + ) VALUES ('r1', '5m', 0, 300, 4, 'all', 'all', 'source', 'packets', + 'spectrum', '[]', '{}')", + [], + ) + .is_err() + ); + } + + #[test] + fn maad_measure_presence_is_checked_per_scope_key_not_by_global_counts() { + let connection = Connection::open_in_memory().unwrap(); + init_schema(&connection).unwrap(); + let insert = |measure: &str, bucket_start: i64, address_side: &str| { + connection + .execute( + "INSERT INTO address_structure_stats ( + source_id, granularity, bucket_start, bucket_end, ip_version, + src_locality, dst_locality, address_side, measure, structure_kind, + values_json, metadata_json + ) VALUES ('r1', '5m', ?2, ?2 + 300, 4, 'all', 'all', ?3, ?1, + 'structure', '[]', '{}')", + params![measure, bucket_start, address_side], + ) + .unwrap(); + }; + insert("addresses", 0, "source"); + insert("packets", 0, "source"); + insert("bytes", 300, "source"); + let error = assert_maad_measures_present(&connection) + .unwrap_err() + .to_string(); + assert!(error.contains("missing bytes structure rows for source r1 5m bucket 0")); + } + #[test] fn canonical_database_satisfies_web_query_contract() { let directory = tempfile::tempdir().unwrap(); diff --git a/tools/netflow-db/tests/compare_cli.rs b/tools/netflow-db/tests/compare_cli.rs index dc0c0985..4ba7629f 100644 --- a/tools/netflow-db/tests/compare_cli.rs +++ b/tools/netflow-db/tests/compare_cli.rs @@ -114,7 +114,7 @@ fn compare_rejects_ipv6_maad_rows_missing_from_a_reference_bucket() { Connection::open(&candidate) .unwrap() .execute( - "INSERT INTO address_structure_stats VALUES ('r1','5m',0,300,6,'all','all','source','dimension','[]','{\"totalAddrs\":0}')", + "INSERT INTO address_structure_stats VALUES ('r1','5m',0,300,6,'all','all','source','addresses','dimension','[]','{\"totalAddrs\":0}')", [], ) .unwrap(); @@ -137,7 +137,7 @@ fn compare_rejects_ipv6_maad_rows_for_an_unrelated_source() { Connection::open(&candidate) .unwrap() .execute( - "INSERT INTO address_structure_stats VALUES ('r2','5m',0,300,6,'all','all','source','dimension','[]','{\"totalAddrs\":0}')", + "INSERT INTO address_structure_stats VALUES ('r2','5m',0,300,6,'all','all','source','addresses','dimension','[]','{\"totalAddrs\":0}')", [], ) .unwrap(); @@ -237,7 +237,7 @@ fn create_shared_database(path: &std::path::Path, flows: i64, dimension: f64, ex bucket_start INTEGER NOT NULL, bucket_end INTEGER NOT NULL, ip_version INTEGER NOT NULL, src_locality TEXT NOT NULL, dst_locality TEXT NOT NULL, address_side TEXT NOT NULL, - structure_kind TEXT NOT NULL, values_json TEXT NOT NULL, + measure TEXT NOT NULL, structure_kind TEXT NOT NULL, values_json TEXT NOT NULL, metadata_json TEXT NOT NULL ); CREATE TABLE processed_inputs ( @@ -269,7 +269,7 @@ fn create_shared_database(path: &std::path::Path, flows: i64, dimension: f64, ex .unwrap(); connection .execute( - "INSERT INTO address_structure_stats VALUES ('r1','5m',0,300,4,'all','all','source','dimension',?1,'{\"totalAddrs\":2}')", + "INSERT INTO address_structure_stats VALUES ('r1','5m',0,300,4,'all','all','source','addresses','dimension',?1,'{\"totalAddrs\":2}')", [format!("[{{\"q\":1,\"dim\":{dimension}}}]")], ) .unwrap(); @@ -281,7 +281,7 @@ fn create_shared_database(path: &std::path::Path, flows: i64, dimension: f64, ex .unwrap(); connection .execute( - "INSERT INTO address_structure_stats VALUES ('r1','30m',3600,5400,4,'all','all','source','dimension','[]','{\"totalAddrs\":0}')", + "INSERT INTO address_structure_stats VALUES ('r1','30m',3600,5400,4,'all','all','source','addresses','dimension','[]','{\"totalAddrs\":0}')", [], ) .unwrap(); @@ -300,7 +300,7 @@ fn create_shared_database(path: &std::path::Path, flows: i64, dimension: f64, ex .unwrap(); connection .execute( - "INSERT INTO address_structure_stats VALUES ('r1','30m',0,1800,4,'all','all','source','dimension','[]','{\"totalAddrs\":0}')", + "INSERT INTO address_structure_stats VALUES ('r1','30m',0,1800,4,'all','all','source','addresses','dimension','[]','{\"totalAddrs\":0}')", [], ) .unwrap(); diff --git a/tools/netflow-db/tests/pipeline_cli_help.rs b/tools/netflow-db/tests/pipeline_cli_help.rs index f03de27e..ecb4f2c2 100644 --- a/tools/netflow-db/tests/pipeline_cli_help.rs +++ b/tools/netflow-db/tests/pipeline_cli_help.rs @@ -421,3 +421,102 @@ fn config_pipeline_resolves_locality_files_from_config_directory_and_marks_datas .unwrap(); assert_eq!(has_locality, 1); } + +#[test] +fn csv_pipeline_drops_and_logs_zero_packet_flows_but_keeps_coverage() { + let temporary = tempdir().unwrap(); + let csv = temporary.path().join("flows.csv"); + let mapping = temporary.path().join("mapping.json"); + let database = temporary.path().join("zero-packets.sqlite"); + fs::write( + &csv, + concat!( + "received,src,dst,packets,bytes\n", + "0,192.0.2.1,198.51.100.1,4,400\n", + "10,192.0.2.1,198.51.100.9,0,0\n", + "300,192.0.2.3,198.51.100.3,0,40\n", + ), + ) + .unwrap(); + fs::write( + &mapping, + serde_json::to_vec(&serde_json::json!({ + "timestamp_format": "unix", + "timestamp_timezone": "UTC", + "columns": { + "time_received": "received", + "src_ip": "src", + "dst_ip": "dst", + "packets": "packets", + "bytes": "bytes" + }, + "source_id": {"value": "edge"} + })) + .unwrap(), + ) + .unwrap(); + let config = temporary.path().join("csv-pipeline.json"); + fs::write( + &config, + serde_json::to_vec(&serde_json::json!({ + "database_path": database, + "timezone": "UTC", + "run_maad": false, + "inputs": [{ + "input_kind": "csv", + "path": csv, + "mapping_path": mapping + }] + })) + .unwrap(), + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_netflow-db")) + .args([ + "pipeline", + "--config", + config.to_str().unwrap(), + "--no-maad", + ]) + .env("RUST_LOG", "info") + .env("NO_COLOR", "1") + .output() + .unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "stderr={stderr}"); + assert_eq!( + stderr + .lines() + .filter(|line| line.contains("dropped zero-packet flows") + && line.contains("dropped_zero_packet_flows=1")) + .count(), + 2, + "stderr={stderr}" + ); + + let connection = Connection::open(&database).unwrap(); + let rows = connection + .prepare( + "SELECT t.bucket_start, t.flows, t.packets, c.observed_units + FROM traffic_stats t JOIN bucket_coverage c + ON c.source_id = t.source_id AND c.granularity = t.granularity + AND c.bucket_start = t.bucket_start + WHERE t.granularity = '5m' AND t.ip_version = 4 + AND t.src_locality = 'all' AND t.dst_locality = 'all' + ORDER BY t.bucket_start", + ) + .unwrap() + .query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + )) + }) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(rows, [(0, 1, 4, 1), (300, 0, 0, 1)]); +} diff --git a/tools/netflow-db/tests/ten_minute_rollups.rs b/tools/netflow-db/tests/ten_minute_rollups.rs index a81717f1..609c11a4 100644 --- a/tools/netflow-db/tests/ten_minute_rollups.rs +++ b/tools/netflow-db/tests/ten_minute_rollups.rs @@ -170,18 +170,21 @@ fn ten_minute_rollups_match_their_five_minute_children() { assert_eq!(unique_sources("10m", 0), 3); assert_eq!(unique_sources("10m", 600), 1); - let maad_total_addrs = |granularity: &str, bucket_start: i64| -> i64 { + let maad_total_addrs = |granularity: &str, bucket_start: i64, measure: &str| -> i64 { connection .query_row( "SELECT json_extract(metadata_json, '$.totalAddrs') FROM address_structure_stats WHERE source_id = 'edge' AND granularity = ?1 AND bucket_start = ?2 AND ip_version = 4 AND src_locality = 'all' AND dst_locality = 'all' - AND address_side = 'source' AND structure_kind = 'structure'", - params![granularity, bucket_start], + AND address_side = 'source' AND measure = ?3 + AND structure_kind = 'structure'", + params![granularity, bucket_start, measure], |row| row.get(0), ) .unwrap() }; - assert_eq!(maad_total_addrs("10m", 0), 3); - assert_eq!(maad_total_addrs("10m", 600), 1); + for measure in ["addresses", "packets", "bytes"] { + assert_eq!(maad_total_addrs("10m", 0, measure), 3, "{measure}"); + assert_eq!(maad_total_addrs("10m", 600, measure), 1, "{measure}"); + } }