From b987c2d57c9927dd37313492d6abc1ad4296488b Mon Sep 17 00:00:00 2001 From: flamboh Date: Thu, 24 Sep 2026 16:41:09 -0700 Subject: [PATCH 1/8] feat(netflow-db): store packet- and byte-weighted MAAD measures Scope accumulation keeps summed packets and bytes per unique address instead of a bare address set. Rollups merge children by summing, so per-address totals stay additive. Counters are stored unaligned, so each entry grows by exactly 16 bytes. Every MAAD computation now runs three measures for IPv4 and IPv6 at every granularity: addresses (structure, spectrum, dimensions, unchanged), packets and bytes (structure and dimensions only). A weighted measure excludes addresses whose counter sums to 0, records the count as zeroWeightAddrs in the row metadata and logs it. address_structure_stats gains a measure key column with a CHECK that limits spectrum rows to addresses. The table, product schema, pipeline contract, result config and MAAD contract versions bump, and the measure set joins the product identity, so a fresh product database is required. Compare, export, verify and the web-verify queries key on or filter by measure, and verify --require-maad-data checks that every measure is present. --- docs/code/pipeline-contract.md | 16 +- docs/user/querying.md | 52 ++- tools/netflow-db/src/compare.rs | 1 + tools/netflow-db/src/domain.rs | 229 +++++++++- tools/netflow-db/src/export.rs | 1 + tools/netflow-db/src/feed.rs | 4 +- tools/netflow-db/src/lib.rs | 2 +- tools/netflow-db/src/nfdump.rs | 40 +- tools/netflow-db/src/operations.rs | 4 +- tools/netflow-db/src/pipeline.rs | 7 +- tools/netflow-db/src/publish.rs | 416 ++++++++++++++----- tools/netflow-db/src/storage.rs | 20 +- tools/netflow-db/src/verify.rs | 93 ++++- tools/netflow-db/tests/compare_cli.rs | 12 +- tools/netflow-db/tests/ten_minute_rollups.rs | 13 +- 15 files changed, 732 insertions(+), 178 deletions(-) diff --git a/docs/code/pipeline-contract.md b/docs/code/pipeline-contract.md index 35374123..71db2bac 100644 --- a/docs/code/pipeline-contract.md +++ b/docs/code/pipeline-contract.md @@ -11,7 +11,7 @@ The pipeline binds each database to one product identity. The identity contains - The canonical endpoint-locality rules - The pipeline timezone - The native decoder contract -- The MAAD enabled state +- The MAAD enabled state, contract version, configuration, and measure set The pipeline rejects a database when its identity differs. Build a new database for a different product identity. @@ -76,6 +76,20 @@ rules or splitting a prefix does not change the identity. Any change to the matc does, and address-file contents are read on every run. The digest keeps private address lists out of database metadata and export manifests. +## MAAD measures + +Each scope and address side keeps one entry per unique address with its summed packets and bytes. +Rollups merge children by summing those counters per address, so a rollup's per-address totals +equal the sums over its five-minute children. The per-address map adds 16 bytes to each address +entry. + +Every MAAD computation runs three measures on the same entries, for both IP versions and every +granularity. `addresses` weighs each address as 1 and stores structure, spectrum, and dimensions. +`packets` and `bytes` weight moments and D1 entropy by the summed counter and store structure and +dimensions only. Prefix validity always uses distinct-address counts. A weighted measure excludes +addresses whose counter sums to 0 and records the excluded count as `zeroWeightAddrs` in the row +metadata. The measure set is part of the result configuration identity. + ## Coordinated subset runs Repeat `--dataset` for two or more registry entries to build coordinated subset products. Each entry diff --git a/docs/user/querying.md b/docs/user/querying.md index 8f383678..ea999233 100644 --- a/docs/user/querying.md +++ b/docs/user/querying.md @@ -22,16 +22,16 @@ Show a table schema: ## Main tables -| Table | Content | -| ------------------------- | ---------------------------------------------------- | -| `datasets` | Public dataset metadata | -| `source_members` | Logical-source membership | -| `traffic_stats` | Flow, packet, byte, duration, and TTL metrics | -| `protocol_stats` | Unique protocol counts and protocol lists | -| `address_count_stats` | Unique source-address and destination-address counts | -| `port_count_stats` | Unique low-port and high-port counts | -| `address_structure_stats` | MAAD structure, spectrum, and dimension values | -| `processed_inputs` | Input processing state and provenance | +| Table | Content | +| ------------------------- | ---------------------------------------------------------- | +| `datasets` | Public dataset metadata | +| `source_members` | Logical-source membership | +| `traffic_stats` | Flow, packet, byte, duration, and TTL metrics | +| `protocol_stats` | Unique protocol counts and protocol lists | +| `address_count_stats` | Unique source-address and destination-address counts | +| `port_count_stats` | Unique low-port and high-port counts | +| `address_structure_stats` | MAAD structure, spectrum, and dimension values per measure | +| `processed_inputs` | Input processing state and provenance | The `granularity` value is `5m`, `10m`, `30m`, `1h`, or `1d`. @@ -117,6 +117,38 @@ WHERE granularity = '1h' ORDER BY source_id, bucket_start, ip_version, port_side, port_range; ``` +## Query MAAD results + +`address_structure_stats` stores one MAAD analysis per bucket, IP version, locality pair, address +side, and `measure`: + +| `measure` | Weights each address by | Stored `structure_kind` rows | +| ----------- | ---------------------------------------- | ------------------------------------ | +| `addresses` | 1 (distinct addresses) | `structure`, `spectrum`, `dimension` | +| `packets` | Packets summed over the bucket and scope | `structure`, `dimension` | +| `bytes` | Bytes summed over the bucket and scope | `structure`, `dimension` | + +Weighted measures keep the address-count prefix tests and weight only the moments and the entropy. +Addresses with zero summed packets or bytes are left out of that measure. Their count is stored in +`metadata_json` as `zeroWeightAddrs`. Always filter on `measure`. Otherwise results for different +measures mix: + +```sql +SELECT + source_id, + bucket_start, + address_side, + values_json +FROM address_structure_stats +WHERE granularity = '1h' + AND ip_version = 4 + AND src_locality = 'all' + AND dst_locality = 'all' + AND measure = 'packets' + AND structure_kind = 'dimension' +ORDER BY source_id, bucket_start, address_side; +``` + ## Query observation averages The database stores sums and counts for safe rollup calculations. It also stores calculated averages for direct queries. diff --git a/tools/netflow-db/src/compare.rs b/tools/netflow-db/src/compare.rs index 9ce7f516..f3e1e8b7 100644 --- a/tools/netflow-db/src/compare.rs +++ b/tools/netflow-db/src/compare.rs @@ -94,6 +94,7 @@ const TABLES: &[TableSpec] = &[ "src_locality", "dst_locality", "address_side", + "measure", "structure_kind", "bucket_end", ], diff --git a/tools/netflow-db/src/domain.rs b/tools/netflow-db/src/domain.rs index a0ba2069..a7d4deb7 100644 --- a/tools/netflow-db/src/domain.rs +++ b/tools/netflow-db/src/domain.rs @@ -1,7 +1,7 @@ //! Canonical, adapter-independent NetFlow observations and statistical buckets. use std::{ - collections::{BTreeMap, BTreeSet, HashSet}, + collections::{BTreeMap, BTreeSet, HashMap, HashSet, hash_map::Entry}, net::IpAddr, sync::Arc, }; @@ -683,11 +683,152 @@ impl FromIterator for AddressSet { } } +/// Packets and bytes attributed to one address within a scope. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct AddressTraffic { + pub packets: u64, + pub bytes: u64, +} + +impl AddressTraffic { + #[must_use] + pub const fn new(packets: u64, bytes: u64) -> Self { + Self { packets, bytes } + } + + fn add(&mut self, other: Self) { + self.packets = self.packets.saturating_add(other.packets); + self.bytes = self.bytes.saturating_add(other.bytes); + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(C, packed)] +struct PackedAddressTraffic { + packets: u64, + bytes: u64, +} + +impl From for AddressTraffic { + fn from(value: PackedAddressTraffic) -> Self { + Self::new(value.packets, value.bytes) + } +} + +impl From for PackedAddressTraffic { + fn from(value: AddressTraffic) -> Self { + Self { + packets: value.packets, + bytes: value.bytes, + } + } +} + +/// Unique IP addresses with their summed traffic and no iteration-order contract. +/// +/// Counters are stored unaligned so each entry costs the address plus 16 bytes. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AddressTotals(HashMap); + +impl AddressTotals { + pub fn add(&mut self, address: IpAddr, traffic: AddressTraffic) { + match self.0.entry(address) { + Entry::Occupied(mut entry) => { + let mut total = AddressTraffic::from(*entry.get()); + total.add(traffic); + entry.insert(total.into()); + } + Entry::Vacant(entry) => { + entry.insert(traffic.into()); + } + } + } + + /// Sum another scope's totals into this one, keeping rollups additive. + pub fn merge(&mut self, other: &Self) { + self.0.reserve(other.0.len()); + for (address, traffic) in other.iter() { + self.add(address, traffic); + } + } + + pub fn iter(&self) -> impl Iterator + '_ { + self.0 + .iter() + .map(|(address, traffic)| (*address, AddressTraffic::from(*traffic))) + } + + #[must_use] + pub fn get(&self, address: &IpAddr) -> Option { + self.0.get(address).copied().map(AddressTraffic::from) + } + + #[must_use] + pub fn len(&self) -> usize { + self.0.len() + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl Serialize for AddressTotals { + fn serialize(&self, serializer: S) -> Result { + serializer.collect_seq( + self.iter() + .map(|(address, traffic)| (address, traffic.packets, traffic.bytes)), + ) + } +} + +impl<'de> Deserialize<'de> for AddressTotals { + fn deserialize>(deserializer: D) -> Result { + let entries = Vec::<(IpAddr, u64, u64)>::deserialize(deserializer)?; + Ok(entries + .into_iter() + .map(|(address, packets, bytes)| (address, AddressTraffic::new(packets, bytes))) + .collect()) + } +} + +impl FromIterator<(IpAddr, AddressTraffic)> for AddressTotals { + fn from_iter>(entries: T) -> Self { + let mut totals = Self::default(); + for (address, traffic) in entries { + totals.add(address, traffic); + } + totals + } +} + +/// The per-address measure a MAAD analysis weights prefixes by. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum MaadMeasure { + Addresses, + Packets, + Bytes, +} + +impl MaadMeasure { + pub const ALL: [Self; 3] = [Self::Addresses, Self::Packets, Self::Bytes]; + + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Addresses => "addresses", + Self::Packets => "packets", + Self::Bytes => "bytes", + } + } +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct ScopedAddressesFact { pub scope: Scope, pub address_side: AddressSide, - pub addresses: AddressSet, + pub addresses: AddressTotals, } impl ScopedAddressesFact { @@ -695,7 +836,7 @@ impl ScopedAddressesFact { pub fn new( scope: Scope, address_side: AddressSide, - addresses: impl IntoIterator, + addresses: impl IntoIterator, ) -> Self { Self { scope, @@ -911,7 +1052,7 @@ pub struct ScopedProtocols { pub struct ScopedAddresses { pub scope: Scope, pub address_side: AddressSide, - pub addresses: AddressSet, + pub addresses: AddressTotals, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -1078,7 +1219,7 @@ pub struct AddressSetRow<'a> { pub key: BucketKey, pub scope: Scope, pub address_side: AddressSide, - pub addresses: &'a AddressSet, + pub addresses: &'a AddressTotals, } #[derive(Debug, Clone, PartialEq)] @@ -1111,7 +1252,7 @@ pub struct StatisticalBucket { coverage: BucketCoverage, traffic: BTreeMap, protocols: BTreeMap>, - addresses: BTreeMap<(Scope, AddressSide), AddressSet>, + addresses: BTreeMap<(Scope, AddressSide), AddressTotals>, ports: BTreeMap<(Scope, PortSide), FixedBitSet>, five_minute_starts: BTreeSet, } @@ -1148,7 +1289,7 @@ impl StatisticalBucket { for side in [AddressSide::Destination, AddressSide::Source] { bucket .addresses - .insert((scope, side), AddressSet::default()); + .insert((scope, side), AddressTotals::default()); bucket.ports.insert((scope, side), empty_ports()); } } @@ -1194,7 +1335,7 @@ impl StatisticalBucket { self.addresses .entry((entry.scope, entry.address_side)) .or_default() - .extend(entry.addresses.iter().copied()); + .merge(&entry.addresses); } for entry in &child.ports { self.ports @@ -1320,6 +1461,10 @@ impl StatisticalBucket { .locality .ok_or(DomainError::UnclassifiedObservation)?; let scopes = scopes_for(observation.ip_version(), source, destination); + let traffic = AddressTraffic::new( + observation.packets.unsigned_abs(), + observation.bytes.unsigned_abs(), + ); let mut updates = Vec::with_capacity(scopes.len()); for scope in scopes { let mut metrics = self.traffic.get(&scope).cloned().unwrap_or_default(); @@ -1335,11 +1480,11 @@ impl StatisticalBucket { self.addresses .entry((scope, AddressSide::Source)) .or_default() - .insert(observation.src_ip); + .add(observation.src_ip, traffic); self.addresses .entry((scope, AddressSide::Destination)) .or_default() - .insert(observation.dst_ip); + .add(observation.dst_ip, traffic); if let Some(port) = observation.src_port { insert_port( self.ports @@ -1382,7 +1527,7 @@ impl StatisticalBucket { self.addresses .entry((fact.scope, fact.address_side)) .or_default() - .extend(fact.addresses.iter().copied()); + .merge(&fact.addresses); } } @@ -1438,9 +1583,9 @@ mod tests { use serde_json::json; use super::{ - AddressSet, AddressSide, BucketKey, DomainError, EndpointLocality, FlowObservation, - FlowSelection, Granularity, GroupedTrafficFact, IpVersion, Locality, PortRange, Scope, - ScopedAddressesFact, StatisticalBucket, + AddressSet, AddressSide, AddressTotals, AddressTraffic, BucketKey, DomainError, + EndpointLocality, FlowObservation, FlowSelection, Granularity, GroupedTrafficFact, + IpVersion, Locality, PortRange, Scope, ScopedAddressesFact, StatisticalBucket, }; use crate::coverage::{BucketCoverage, CoverageState}; use crate::locality::{LocalityRuleConfig, LocalityRules}; @@ -1488,6 +1633,35 @@ mod tests { assert_eq!(forward.len(), 2); } + #[test] + fn address_totals_sum_traffic_per_address_without_order_semantics() { + let first = "192.0.2.1".parse().unwrap(); + let second = "2001:db8::1".parse().unwrap(); + + let forward: AddressTotals = [ + (first, AddressTraffic::new(1, 10)), + (second, AddressTraffic::new(0, 0)), + (first, AddressTraffic::new(2, 20)), + ] + .into_iter() + .collect(); + let mut reverse: AddressTotals = + [(second, AddressTraffic::new(0, 0))].into_iter().collect(); + reverse.merge(&[(first, AddressTraffic::new(3, 30))].into_iter().collect()); + + assert_eq!(forward, reverse); + assert_eq!(forward.len(), 2); + assert_eq!(forward.get(&first), Some(AddressTraffic::new(3, 30))); + assert_eq!(forward.get(&second), Some(AddressTraffic::new(0, 0))); + let round_trip: AddressTotals = + serde_json::from_value(serde_json::to_value(&forward).unwrap()).unwrap(); + assert_eq!(round_trip, forward); + assert_eq!( + std::mem::size_of::<(IpAddr, super::PackedAddressTraffic)>(), + std::mem::size_of::() + 16 + ); + } + #[test] fn unclassified_observations_are_rejected_by_buckets() { let mut bucket = StatisticalBucket::new(key(Granularity::FiveMinutes, 0, 300)); @@ -1823,7 +1997,10 @@ mod tests { .add(ScopedAddressesFact::new( Scope::new(IpVersion::V4, Locality::External, Locality::Internal), AddressSide::Destination, - [address([203, 0, 113, 1]), address([203, 0, 113, 2])], + [ + (address([203, 0, 113, 1]), AddressTraffic::new(1, 10)), + (address([203, 0, 113, 2]), AddressTraffic::new(0, 0)), + ], )) .unwrap(); let sparse_expected = sparse.finish(); @@ -1947,9 +2124,12 @@ mod tests { }) .unwrap() .addresses, - [address([192, 0, 2, 1]), address([192, 0, 2, 2])] - .into_iter() - .collect::() + [ + (address([192, 0, 2, 1]), AddressTraffic::new(1, 10)), + (address([192, 0, 2, 2]), AddressTraffic::new(2, 20)), + ] + .into_iter() + .collect::() ); let rows = rolled_up.rows(); let counts = rows @@ -1993,7 +2173,10 @@ mod tests { .add(ScopedAddressesFact::new( scope, AddressSide::Destination, - [address([198, 51, 100, 3]), address([198, 51, 100, 2])], + [ + (address([198, 51, 100, 3]), AddressTraffic::new(5, 50)), + (address([198, 51, 100, 2]), AddressTraffic::new(5, 50)), + ], )) .unwrap(); let bucket = bucket.finish(); @@ -2017,12 +2200,12 @@ mod tests { .unwrap() .addresses, &[ - address([198, 51, 100, 1]), - address([198, 51, 100, 2]), - address([198, 51, 100, 3]), + (address([198, 51, 100, 1]), AddressTraffic::new(1, 10)), + (address([198, 51, 100, 2]), AddressTraffic::new(6, 60)), + (address([198, 51, 100, 3]), AddressTraffic::new(5, 50)), ] .into_iter() - .collect::() + .collect::() ); } } diff --git a/tools/netflow-db/src/export.rs b/tools/netflow-db/src/export.rs index 1c7b6e7e..555872ce 100644 --- a/tools/netflow-db/src/export.rs +++ b/tools/netflow-db/src/export.rs @@ -381,6 +381,7 @@ fn required_columns(table: &str) -> Vec<&'static str> { "port_count_stats" => vec!["port_side", "port_range", "unique_port_count"], "address_structure_stats" => vec![ "address_side", + "measure", "structure_kind", "values_json", "metadata_json", diff --git a/tools/netflow-db/src/feed.rs b/tools/netflow-db/src/feed.rs index 45ebb4c5..6416d1a0 100644 --- a/tools/netflow-db/src/feed.rs +++ b/tools/netflow-db/src/feed.rs @@ -414,9 +414,9 @@ fn collect_total_ipv4_addresses(bucket: CanonicalBucket, addresses: &mut BTreeSe { continue; } - for address in scoped.addresses.iter() { + for (address, _) in scoped.addresses.iter() { if let IpAddr::V4(address) = address { - addresses.insert(*address); + addresses.insert(address); } } } diff --git a/tools/netflow-db/src/lib.rs b/tools/netflow-db/src/lib.rs index c3e93f1b..dde34b06 100644 --- a/tools/netflow-db/src/lib.rs +++ b/tools/netflow-db/src/lib.rs @@ -3,7 +3,7 @@ #![forbid(unsafe_code)] /// The persisted pipeline contract version produced by this implementation. -pub const PIPELINE_CONTRACT_VERSION: u32 = 6; +pub const PIPELINE_CONTRACT_VERSION: u32 = 7; pub mod compare; pub mod config; diff --git a/tools/netflow-db/src/nfdump.rs b/tools/netflow-db/src/nfdump.rs index 5971ed12..57a8bc5a 100644 --- a/tools/netflow-db/src/nfdump.rs +++ b/tools/netflow-db/src/nfdump.rs @@ -11,10 +11,10 @@ use fixedbitset::FixedBitSet; use crate::{ coverage::BucketCoverage, domain::{ - AddressSet, AddressSide, BucketKey, CanonicalBucket, DAILY_ACTIVE_MIN_BYTES, - DAILY_ACTIVE_MIN_FLOWS, DAILY_ACTIVE_MIN_PACKETS, EndpointLocality, FlowSelection, - Granularity, IpVersion, Scope, ScopedAddresses, ScopedPorts, ScopedProtocols, - ScopedTraffic, TrafficMetrics, ZERO_FILL_LOCALITY_PAIRS, + AddressSet, AddressSide, AddressTotals, AddressTraffic, BucketKey, CanonicalBucket, + DAILY_ACTIVE_MIN_BYTES, DAILY_ACTIVE_MIN_FLOWS, DAILY_ACTIVE_MIN_PACKETS, EndpointLocality, + FlowSelection, Granularity, IpVersion, Scope, ScopedAddresses, ScopedPorts, + ScopedProtocols, ScopedTraffic, TrafficMetrics, ZERO_FILL_LOCALITY_PAIRS, }, }; @@ -276,8 +276,8 @@ impl std::error::Error for NfdumpError { struct ScopeAccumulator { metrics: [i64; METRIC_COUNT], protocols: [u64; 4], - source_addresses: AddressSet, - destination_addresses: AddressSet, + source_addresses: AddressTotals, + destination_addresses: AddressTotals, source_ports: FixedBitSet, destination_ports: FixedBitSet, } @@ -331,8 +331,10 @@ impl ScopeAccumulator { } self.protocols[usize::from(flow.protocol) / 64] |= 1_u64 << (usize::from(flow.protocol) % 64); - self.source_addresses.insert(flow.source_address); - self.destination_addresses.insert(flow.destination_address); + let traffic = AddressTraffic::new(flow.packets.unsigned_abs(), flow.bytes.unsigned_abs()); + self.source_addresses.add(flow.source_address, traffic); + self.destination_addresses + .add(flow.destination_address, traffic); insert_port(&mut self.source_ports, flow.source_port); insert_port(&mut self.destination_ports, flow.destination_port); } @@ -1131,6 +1133,10 @@ mod tests { use crate::domain::{Granularity, Locality, Scope}; use crate::locality::{LocalityRuleConfig, LocalityRules}; + fn address_keys(totals: &AddressTotals) -> AddressSet { + totals.iter().map(|(address, _)| address).collect() + } + fn rules(configs: &[LocalityRuleConfig]) -> Arc { Arc::new(LocalityRules::from_config(configs, std::path::Path::new("/")).unwrap()) } @@ -1231,13 +1237,19 @@ mod tests { assert_eq!(bucket.protocols[index].protocols, ["6"]); } assert_eq!( - bucket.addresses[0].addresses, + address_keys(&bucket.addresses[0].addresses), [IpAddr::V4(Ipv4Addr::new(198, 51, 100, 2))] .into_iter() .collect::() ); assert_eq!( - bucket.addresses[1].addresses, + bucket.addresses[0] + .addresses + .get(&IpAddr::V4(Ipv4Addr::new(198, 51, 100, 2))), + Some(AddressTraffic::new(2, 128)) + ); + assert_eq!( + address_keys(&bucket.addresses[1].addresses), [IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1))] .into_iter() .collect::() @@ -1303,7 +1315,7 @@ mod tests { assert_eq!(bucket.protocols[index].protocols, ["58"]); } assert_eq!( - bucket.addresses[10].addresses, + address_keys(&bucket.addresses[10].addresses), [IpAddr::V6(Ipv6Addr::new(0x2001, 0x0db8, 0, 0, 0, 0, 0, 2))] .into_iter() .collect::() @@ -1487,11 +1499,11 @@ mod tests { }; assert_eq!(buckets.iter().map(all_v4).collect::>(), [3, 0, 3, 4]); assert_eq!( - buckets[0].addresses[1].addresses, + address_keys(&buckets[0].addresses[1].addresses), [source_a].into_iter().collect::() ); assert_eq!( - buckets[3].addresses[1].addresses, + address_keys(&buckets[3].addresses[1].addresses), [source_b].into_iter().collect::() ); } @@ -1794,7 +1806,7 @@ mod tests { assert_eq!(bucket.traffic[3].metrics.flows, 2); assert_eq!(bucket.protocols[0].protocols, ["17", "6"]); assert_eq!( - bucket.addresses[1].addresses, + address_keys(&bucket.addresses[1].addresses), [ IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)), IpAddr::V4(Ipv4Addr::new(192, 0, 2, 2)), diff --git a/tools/netflow-db/src/operations.rs b/tools/netflow-db/src/operations.rs index a30339a1..a203719a 100644 --- a/tools/netflow-db/src/operations.rs +++ b/tools/netflow-db/src/operations.rs @@ -120,12 +120,12 @@ pub fn select_web_verification_window( ON st.source_id = ts.source_id AND st.granularity = '5m' AND st.bucket_start = ts.bucket_start AND st.ip_version = 4 AND st.src_locality = 'all' AND st.dst_locality = 'all' - AND st.structure_kind = 'structure' + AND st.measure = 'addresses' AND st.structure_kind = 'structure' JOIN address_structure_stats sp ON sp.source_id = ts.source_id AND sp.granularity = '5m' AND sp.bucket_start = ts.bucket_start AND sp.ip_version = 4 AND sp.src_locality = 'all' AND sp.dst_locality = 'all' - AND sp.structure_kind = 'spectrum' + AND sp.measure = 'addresses' AND sp.structure_kind = 'spectrum' WHERE ts.source_id = ?1 AND ts.granularity = '5m' AND ts.src_locality = 'all' AND ts.dst_locality = 'all' ORDER BY ts.bucket_start LIMIT 1 diff --git a/tools/netflow-db/src/pipeline.rs b/tools/netflow-db/src/pipeline.rs index e22b0ea8..316750a2 100644 --- a/tools/netflow-db/src/pipeline.rs +++ b/tools/netflow-db/src/pipeline.rs @@ -1514,7 +1514,7 @@ fn bind_identity( }) }); let result_config = json!({ - "version": 5, + "version": 6, "timezone": pipeline.timezone, "locality": pipeline .selection @@ -1532,8 +1532,9 @@ fn bind_identity( "maad": { "enabled": pipeline.run_maad, "backend": "in-process", - "contract_version": 4, - "config": maad_config + "contract_version": 5, + "config": maad_config, + "measures": crate::domain::MaadMeasure::ALL.map(crate::domain::MaadMeasure::as_str), } }); let identity = ProductIdentity::create( diff --git a/tools/netflow-db/src/publish.rs b/tools/netflow-db/src/publish.rs index 36aa866f..d0bafa36 100644 --- a/tools/netflow-db/src/publish.rs +++ b/tools/netflow-db/src/publish.rs @@ -8,10 +8,14 @@ use std::{ use rayon::prelude::*; use rusqlite::Connection; +use serde::Serialize; use thiserror::Error; use crate::{ - domain::{AddressSetRow, BucketKey, CanonicalBucket, CanonicalRows, DomainError, IpVersion}, + domain::{ + AddressSetRow, AddressTraffic, BucketKey, CanonicalBucket, CanonicalRows, DomainError, + IpVersion, MaadMeasure, + }, maad, storage::{ AddressCountStatsRow, AddressStructureStatsRow, BucketCoverageRow, PortCountStatsRow, @@ -32,6 +36,8 @@ pub enum PublishError { Json(#[from] serde_json::Error), #[error("unable to build MAAD worker pool: {0}")] MaadPool(String), + #[error(transparent)] + Maad(#[from] maad::MaadError), #[error("aggregate bucket lacks complete five-minute coverage: {0:?}")] IncompleteCoverage(BucketKey), } @@ -65,6 +71,7 @@ pub struct WriteBucketsProfile { pub(crate) port_count_rows: u64, pub(crate) maad_address_sets: u64, pub(crate) maad_addresses: u64, + pub(crate) maad_zero_weight_addresses: u64, pub(crate) address_structure_rows: u64, pub(crate) address_structure_json_bytes: u64, } @@ -166,8 +173,18 @@ pub(crate) fn write_buckets_profiled( .map(|addresses| count(addresses.addresses.len())) .sum::(); let maad_started = Instant::now(); - let address_structure = maad_rows(&rows.address_sets)?; + let (address_structure, zero_weight_addresses) = maad_rows(&rows.address_sets)?; profile.maad_elapsed += maad_started.elapsed(); + profile.maad_zero_weight_addresses += zero_weight_addresses; + if zero_weight_addresses > 0 { + tracing::info!( + source_id = %bucket.key.source_id, + granularity = bucket.key.granularity.as_str(), + bucket_start = bucket.key.bucket_start, + zero_weight_addresses, + "excluded zero-weight addresses from weighted MAAD measures" + ); + } profile.address_structure_rows += count(address_structure.len()); profile.address_structure_json_bytes += address_structure .iter() @@ -272,65 +289,150 @@ fn insert_rows( }) } +/// Compute every MAAD measure for each scoped address set. +/// +/// Returns the rows and the number of addresses excluded from weighted measures +/// because their summed weight was zero. fn maad_rows( address_sets: &[AddressSetRow<'_>], -) -> Result, PublishError> { +) -> Result<(Vec, u64), PublishError> { if address_sets.is_empty() { - return Ok(Vec::new()); + return Ok((Vec::new(), 0)); } + let work = address_sets + .iter() + .flat_map(|addresses| MaadMeasure::ALL.map(|measure| (addresses, measure))) + .collect::>(); let pool = maad_pool()?; - let rows = pool.install(|| { - address_sets - .par_iter() - .map(|addresses| { - let result = maad_result(addresses); - let metadata_json = serde_json::to_string(&result.metadata)?; - let dimensions = dimensions(&addresses.key, addresses.scope); - Ok::<_, serde_json::Error>([ - AddressStructureStatsRow { - dimensions: dimensions.clone(), - address_side: addresses.address_side.as_str().to_owned(), - structure_kind: "structure".into(), - values_json: serde_json::to_string(&result.structure)?, - metadata_json: metadata_json.clone(), - }, - AddressStructureStatsRow { - dimensions: dimensions.clone(), - address_side: addresses.address_side.as_str().to_owned(), - structure_kind: "spectrum".into(), - values_json: serde_json::to_string(&result.spectrum)?, - metadata_json: metadata_json.clone(), - }, - AddressStructureStatsRow { - dimensions, - address_side: addresses.address_side.as_str().to_owned(), - structure_kind: "dimension".into(), - values_json: serde_json::to_string(&result.dimensions)?, - metadata_json, - }, - ]) - }) + let measured = pool.install(|| { + work.par_iter() + .map(|&(addresses, measure)| measure_rows(addresses, measure)) .collect::, _>>() })?; - Ok(rows.into_iter().flatten().collect()) + let zero_weight_addresses = measured.iter().map(|(_, excluded)| count(*excluded)).sum(); + Ok(( + measured.into_iter().flat_map(|(rows, _)| rows).collect(), + zero_weight_addresses, + )) } -fn maad_result(addresses: &AddressSetRow<'_>) -> maad::MaadResult { +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct WeightedMetadata<'a> { + #[serde(flatten)] + metadata: &'a maad::MaadMetadata, + zero_weight_addrs: usize, +} + +fn measure_rows( + addresses: &AddressSetRow<'_>, + measure: MaadMeasure, +) -> Result<(Vec, usize), PublishError> { + let dimensions = dimensions(&addresses.key, addresses.scope); + let row = + |structure_kind: &str, values_json: String, metadata_json: &str| AddressStructureStatsRow { + dimensions: dimensions.clone(), + address_side: addresses.address_side.as_str().to_owned(), + measure: measure.as_str().to_owned(), + structure_kind: structure_kind.to_owned(), + values_json, + metadata_json: metadata_json.to_owned(), + }; + let weight: fn(AddressTraffic) -> u64 = match measure { + MaadMeasure::Addresses => { + let result = address_maad_result(addresses); + let metadata_json = serde_json::to_string(&result.metadata)?; + return Ok(( + vec![ + row( + "structure", + serde_json::to_string(&result.structure)?, + &metadata_json, + ), + row( + "spectrum", + serde_json::to_string(&result.spectrum)?, + &metadata_json, + ), + row( + "dimension", + serde_json::to_string(&result.dimensions)?, + &metadata_json, + ), + ], + 0, + )); + } + MaadMeasure::Packets => |traffic| traffic.packets, + MaadMeasure::Bytes => |traffic| traffic.bytes, + }; + let (result, zero_weight_addrs) = weighted_maad_result(addresses, weight)?; + let metadata_json = serde_json::to_string(&WeightedMetadata { + metadata: &result.metadata, + zero_weight_addrs, + })?; + Ok(( + vec![ + row( + "structure", + serde_json::to_string(&result.structure)?, + &metadata_json, + ), + row( + "dimension", + serde_json::to_string(&result.dimensions)?, + &metadata_json, + ), + ], + zero_weight_addrs, + )) +} + +fn address_maad_result(addresses: &AddressSetRow<'_>) -> maad::MaadResult { + let family = addresses.addresses.iter().map(|(address, _)| address); match addresses.scope.ip_version { - IpVersion::V4 => maad::compute(addresses.addresses.iter().filter_map( - |address| match address { - IpAddr::V4(address) => Some(*address), + IpVersion::V4 => maad::compute(family.filter_map(|address| match address { + IpAddr::V4(address) => Some(address), + IpAddr::V6(_) => None, + })), + IpVersion::V6 => maad::compute(family.filter_map(|address| match address { + IpAddr::V6(address) => Some(address), + IpAddr::V4(_) => None, + })), + } +} + +/// Weighted MAAD over addresses with a positive summed weight, and the count of +/// same-family addresses excluded because their weight was zero. +fn weighted_maad_result( + addresses: &AddressSetRow<'_>, + weight: fn(AddressTraffic) -> u64, +) -> Result<(maad::MaadResult, usize), maad::MaadError> { + let mut zero_weight_addrs = 0; + let mut positive = |traffic| match weight(traffic) { + 0 => { + zero_weight_addrs += 1; + None + } + value => Some(value as f64), + }; + let entries = addresses.addresses.iter(); + let result = match addresses.scope.ip_version { + IpVersion::V4 => { + maad::compute_weighted(entries.filter_map(|(address, traffic)| match address { + IpAddr::V4(address) => positive(traffic).map(|value| (address, value)), IpAddr::V6(_) => None, - }, - )), - IpVersion::V6 => maad::compute(addresses.addresses.iter().filter_map( - |address| match address { - IpAddr::V6(address) => Some(*address), + }))? + } + IpVersion::V6 => { + maad::compute_weighted(entries.filter_map(|(address, traffic)| match address { + IpAddr::V6(address) => positive(traffic).map(|value| (address, value)), IpAddr::V4(_) => None, - }, - )), - } + }))? + } + }; + Ok((result, zero_weight_addrs)) } fn maad_pool() -> Result<&'static rayon::ThreadPool, PublishError> { @@ -371,7 +473,7 @@ mod tests { use super::*; use crate::{ domain::{ - AddressSet, AddressSide, EndpointLocality, FlowObservation, Granularity, IpVersion, + AddressSide, AddressTotals, EndpointLocality, FlowObservation, Granularity, IpVersion, Locality, Scope, ScopedAddressesFact, StatisticalBucket, }, storage::init_stats_tables, @@ -416,7 +518,7 @@ mod tests { row.get::<_, i64>(0) }) .unwrap(), - 60 + 140 ); assert_eq!(profile.bucket_keys, 1); assert_eq!(profile.write_calls, 1); @@ -425,14 +527,15 @@ mod tests { assert_eq!(profile.address_count_rows, 20); assert_eq!(profile.port_count_rows, 40); assert_eq!(profile.maad_address_sets, 20); - assert_eq!(profile.address_structure_rows, 60); + assert_eq!(profile.address_structure_rows, 140); + assert_eq!(profile.maad_zero_weight_addresses, 0); assert!(profile.address_structure_json_bytes > 0); assert!(profile.total_elapsed >= profile.other_elapsed()); } #[test] fn address_insertion_order_does_not_change_persisted_products() { - fn persist(addresses: impl IntoIterator) -> Connection { + fn persist(addresses: impl IntoIterator) -> Connection { let connection = Connection::open_in_memory().unwrap(); init_stats_tables(&connection).unwrap(); let mut builder = @@ -466,7 +569,7 @@ mod tests { ), query( connection, - "SELECT printf('%s|%s|%s|%s|%s|%s|%s', ip_version, src_locality, dst_locality, address_side, structure_kind, values_json, metadata_json) FROM address_structure_stats ORDER BY ip_version, src_locality, dst_locality, address_side, structure_kind", + "SELECT printf('%s|%s|%s|%s|%s|%s|%s|%s', ip_version, src_locality, dst_locality, address_side, measure, structure_kind, values_json, metadata_json) FROM address_structure_stats ORDER BY ip_version, src_locality, dst_locality, address_side, measure, structure_kind", ), ) } @@ -474,8 +577,19 @@ mod tests { let first = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)); let second = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 2)); let third = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 3)); - let forward = persist([first, second, first, third]); - let reverse = persist([third, first, second, second]); + let traffic = AddressTraffic::new; + let forward = persist([ + (first, traffic(1, 10)), + (second, traffic(2, 200)), + (first, traffic(3, 30)), + (third, traffic(4, 4_000)), + ]); + let reverse = persist([ + (third, traffic(4, 4_000)), + (first, traffic(4, 40)), + (second, traffic(1, 100)), + (second, traffic(1, 100)), + ]); assert_eq!(product_rows(&forward), product_rows(&reverse)); } @@ -483,18 +597,22 @@ mod tests { #[test] fn maad_rows_preserve_scope_order_and_bytes() { let key = BucketKey::new("r1", Granularity::FiveMinutes, 0, 300); - let first_addresses = AddressSet::from_iter([ - IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)), - IpAddr::V4(Ipv4Addr::new(192, 0, 2, 2)), - IpAddr::V4(Ipv4Addr::new(192, 0, 2, 3)), - IpAddr::V4(Ipv4Addr::new(192, 0, 2, 4)), - ]); - let second_addresses = AddressSet::from_iter([ - IpAddr::V4(Ipv4Addr::new(198, 51, 100, 1)), - IpAddr::V4(Ipv4Addr::new(198, 51, 100, 2)), - IpAddr::V4(Ipv4Addr::new(198, 51, 100, 3)), - IpAddr::V4(Ipv4Addr::new(198, 51, 100, 4)), - ]); + let first_addresses = (0..64_u8) + .map(|index| { + ( + IpAddr::V4(Ipv4Addr::new(192, index % 4, index, 1)), + AddressTraffic::new(u64::from(index % 5 + 1).pow(3), 100), + ) + }) + .collect::(); + let second_addresses = (0..64_u8) + .map(|index| { + ( + IpAddr::V4(Ipv4Addr::new(198, index % 4, index, 1)), + AddressTraffic::new(5, 5 * u64::from(index) + 5), + ) + }) + .collect::(); let rows = [ AddressSetRow { key: key.clone(), @@ -510,39 +628,88 @@ mod tests { }, ]; - let first = maad_rows(&rows).unwrap(); - let second = maad_rows(&rows).unwrap(); + let (first, first_excluded) = maad_rows(&rows).unwrap(); + let (second, second_excluded) = maad_rows(&rows).unwrap(); assert_eq!(first, second); - assert_eq!(first.len(), 6); + assert_eq!((first_excluded, second_excluded), (0, 0)); + let kinds = |src_locality: &str, side: &str| { + [ + ("addresses", "structure"), + ("addresses", "spectrum"), + ("addresses", "dimension"), + ("packets", "structure"), + ("packets", "dimension"), + ("bytes", "structure"), + ("bytes", "dimension"), + ] + .map(|(measure, kind)| { + ( + src_locality.to_owned(), + side.to_owned(), + measure.to_owned(), + kind.to_owned(), + ) + }) + }; assert_eq!( first - .chunks_exact(3) - .map(|rows| ( - rows[0].dimensions.src_locality.as_str(), - rows[0].address_side.as_str(), - rows.iter() - .map(|row| row.structure_kind.as_str()) - .collect::>(), + .iter() + .map(|row| ( + row.dimensions.src_locality.clone(), + row.address_side.clone(), + row.measure.clone(), + row.structure_kind.clone(), )) .collect::>(), - vec![ - ("all", "source", vec!["structure", "spectrum", "dimension"]), - ( - "external", - "destination", - vec!["structure", "spectrum", "dimension"] - ), - ] + [kinds("all", "source"), kinds("external", "destination")].concat() + ); + let packets_structure = |scope_rows: &[AddressStructureStatsRow]| { + scope_rows + .iter() + .find(|row| row.measure == "packets" && row.structure_kind == "structure") + .unwrap() + .values_json + .clone() + }; + let addresses_structure = |scope_rows: &[AddressStructureStatsRow]| { + scope_rows + .iter() + .find(|row| row.measure == "addresses" && row.structure_kind == "structure") + .unwrap() + .values_json + .clone() + }; + assert_ne!( + packets_structure(&first[..7]), + addresses_structure(&first[..7]) ); + let tau = |json: String| -> Vec { + serde_json::from_str::>(&json) + .unwrap() + .iter() + .map(|row| row["tauTilde"].as_f64().unwrap()) + .collect() + }; + let uniform_packets = tau(packets_structure(&first[7..])); + let addresses = tau(addresses_structure(&first[7..])); + assert_eq!(uniform_packets.len(), addresses.len()); + for (packets, addresses) in uniform_packets.iter().zip(&addresses) { + assert!((packets - addresses).abs() < 1e-12); + } } #[test] fn maad_rows_use_the_ipv6_prefix_range_for_ipv6_scopes() { let base = u128::from(Ipv6Addr::new(0x2001, 0x0db8, 0, 0, 0, 0, 0, 0)); - let addresses = AddressSet::from_iter( - (0..=255_u128).map(|subnet| IpAddr::V6(Ipv6Addr::from(base | (subnet << 64)))), - ); + let addresses = (0..=255_u128) + .map(|subnet| { + ( + IpAddr::V6(Ipv6Addr::from(base | (subnet << 64))), + AddressTraffic::new(1 + (subnet % 7) as u64, 1_500), + ) + }) + .collect::(); let rows = [AddressSetRow { key: BucketKey::new("r1", Granularity::FiveMinutes, 0, 300), scope: Scope::new(IpVersion::V6, Locality::All, Locality::All), @@ -550,13 +717,74 @@ mod tests { addresses: &addresses, }]; - let rows = maad_rows(&rows).unwrap(); + let (rows, _) = maad_rows(&rows).unwrap(); - assert_eq!(rows.len(), 3); + assert_eq!(rows.len(), 7); assert!(rows.iter().all(|row| row.dimensions.ip_version == 6)); - let metadata: serde_json::Value = serde_json::from_str(&rows[0].metadata_json).unwrap(); - assert_eq!(metadata["totalAddrs"], 256); - assert_eq!(metadata["minPrefixLength"], 23); - assert_eq!(metadata["maxPrefixLength"], 63); + for row in &rows { + let metadata: serde_json::Value = serde_json::from_str(&row.metadata_json).unwrap(); + assert_eq!(metadata["totalAddrs"], 256, "{}", row.measure); + assert_eq!(metadata["minPrefixLength"], 23, "{}", row.measure); + assert_eq!(metadata["maxPrefixLength"], 63, "{}", row.measure); + } + } + + #[test] + fn weighted_measures_exclude_zero_weight_addresses_and_count_them() { + let addresses = (0..=255_u8) + .map(|last| { + let packets = u64::from(last % 4 != 0); + let bytes = if last < 10 { 0 } else { 40 * u64::from(last) }; + ( + IpAddr::V4(Ipv4Addr::new(10, last % 16, last, 1)), + AddressTraffic::new(packets, bytes), + ) + }) + .chain([( + IpAddr::V6(Ipv6Addr::new(0x2001, 0x0db8, 0, 0, 0, 0, 0, 1)), + AddressTraffic::new(0, 0), + )]) + .collect::(); + let rows = [AddressSetRow { + key: BucketKey::new("r1", Granularity::FiveMinutes, 0, 300), + scope: Scope::new(IpVersion::V4, Locality::All, Locality::All), + address_side: AddressSide::Destination, + addresses: &addresses, + }]; + + let (rows, excluded) = maad_rows(&rows).unwrap(); + + let metadata = |measure: &str| -> serde_json::Value { + serde_json::from_str( + &rows + .iter() + .find(|row| row.measure == measure && row.structure_kind == "structure") + .unwrap() + .metadata_json, + ) + .unwrap() + }; + assert_eq!(excluded, 64 + 10); + assert_eq!(metadata("addresses")["totalAddrs"], 256); + assert!(metadata("addresses").get("zeroWeightAddrs").is_none()); + assert_eq!(metadata("packets")["totalAddrs"], 192); + assert_eq!(metadata("packets")["zeroWeightAddrs"], 64); + assert_eq!(metadata("bytes")["totalAddrs"], 246); + assert_eq!(metadata("bytes")["zeroWeightAddrs"], 10); + let packets = addresses + .iter() + .filter_map(|(address, traffic)| match address { + IpAddr::V4(address) if traffic.packets > 0 => { + Some((address, traffic.packets as f64)) + } + _ => None, + }); + assert_eq!( + rows.iter() + .find(|row| row.measure == "packets" && row.structure_kind == "structure") + .unwrap() + .values_json, + serde_json::to_string(&maad::compute_weighted(packets).unwrap().structure).unwrap() + ); } } diff --git a/tools/netflow-db/src/storage.rs b/tools/netflow-db/src/storage.rs index 7f793947..03e9713e 100644 --- a/tools/netflow-db/src/storage.rs +++ b/tools/netflow-db/src/storage.rs @@ -37,13 +37,13 @@ pub const STATS_TABLE_NAMES: [&str; 6] = [ /// Versioned product schema bound into every pipeline database identity. pub fn product_schema() -> serde_json::Value { serde_json::json!({ - "version": 5, + "version": 6, "tables": [ {"name": "traffic_stats", "version": 4}, {"name": "protocol_stats", "version": 3}, {"name": "address_count_stats", "version": 3}, {"name": "port_count_stats", "version": 3}, - {"name": "address_structure_stats", "version": 3}, + {"name": "address_structure_stats", "version": 4}, {"name": "bucket_coverage", "version": 2} ] }) @@ -1511,18 +1511,20 @@ pub fn init_stats_tables(connection: &Connection) -> Result<(), StorageError> { src_locality TEXT NOT NULL CHECK (src_locality IN ('all', 'internal', 'external')), dst_locality TEXT NOT NULL CHECK (dst_locality IN ('all', 'internal', 'external')), address_side TEXT NOT NULL CHECK (address_side IN ('source', 'destination')), + measure TEXT NOT NULL CHECK (measure IN ('addresses', 'packets', 'bytes')), structure_kind TEXT NOT NULL CHECK (structure_kind IN ('structure', 'spectrum', 'dimension')), values_json TEXT NOT NULL, metadata_json TEXT NOT NULL, processed_at DATETIME DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (source_id, granularity, bucket_start, ip_version, src_locality, dst_locality, address_side, structure_kind) + CHECK (measure = 'addresses' OR structure_kind <> 'spectrum'), + PRIMARY KEY (source_id, granularity, bucket_start, ip_version, src_locality, dst_locality, address_side, measure, structure_kind) ) WITHOUT ROWID; CREATE INDEX IF NOT EXISTS idx_address_structure_stats_query - ON address_structure_stats (granularity, bucket_start, source_id, ip_version, src_locality, dst_locality, address_side, structure_kind); + ON address_structure_stats (granularity, bucket_start, source_id, ip_version, src_locality, dst_locality, address_side, measure, structure_kind); CREATE INDEX IF NOT EXISTS idx_address_structure_stats_timeseries ON address_structure_stats ( source_id, granularity, src_locality, dst_locality, - ip_version, structure_kind, bucket_start + ip_version, measure, structure_kind, bucket_start ); DROP INDEX IF EXISTS idx_protocol_stats_query; @@ -1920,6 +1922,7 @@ pub struct PortCountStatsRow { pub struct AddressStructureStatsRow { pub dimensions: StatsDimensions, pub address_side: String, + pub measure: String, pub structure_kind: String, pub values_json: String, pub metadata_json: String, @@ -2078,9 +2081,9 @@ pub fn insert_address_structure_stats_rows( " INSERT OR REPLACE INTO address_structure_stats ( source_id, granularity, bucket_start, bucket_end, ip_version, - src_locality, dst_locality, address_side, structure_kind, + src_locality, dst_locality, address_side, measure, structure_kind, values_json, metadata_json - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12) ", )?; for row in rows { @@ -2094,6 +2097,7 @@ pub fn insert_address_structure_stats_rows( d.src_locality, d.dst_locality, row.address_side, + row.measure, row.structure_kind, row.values_json, row.metadata_json @@ -2336,6 +2340,7 @@ impl AddressStructureStatsRow { Self { dimensions: StatsDimensions::example(), address_side: "source".into(), + measure: "addresses".into(), structure_kind: "structure".into(), values_json: "[]".into(), metadata_json: "{}".into(), @@ -3298,6 +3303,7 @@ mod tests { "src_locality", "dst_locality", "ip_version", + "measure", "structure_kind", "bucket_start", ] diff --git a/tools/netflow-db/src/verify.rs b/tools/netflow-db/src/verify.rs index c59ada54..a2b7bb55 100644 --- a/tools/netflow-db/src/verify.rs +++ b/tools/netflow-db/src/verify.rs @@ -94,10 +94,8 @@ pub fn verify_database( )); } } - if options.require_maad_data && row_counts["address_structure_stats"] == 0 { - return Err(VerifyError::Incompatible( - "address_structure_stats has no rows".into(), - )); + if options.require_maad_data { + assert_maad_measures_present(&connection)?; } if options.require_processed { assert_processed_inputs_complete(&connection)?; @@ -337,6 +335,7 @@ const REQUIRED_COLUMNS: &[(&str, &[&str])] = &[ "src_locality", "dst_locality", "address_side", + "measure", "structure_kind", "values_json", "metadata_json", @@ -511,6 +510,34 @@ fn ipv4_literal(value: &str) -> Option<&str> { }) } +/// Every scope must carry a structure row for each MAAD measure. +fn assert_maad_measures_present(connection: &Connection) -> Result<(), VerifyError> { + let counts = connection + .prepare( + "SELECT measure, COUNT(*) FROM address_structure_stats + WHERE structure_kind = 'structure' GROUP BY measure", + )? + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) + })? + .collect::, _>>()?; + let expected = counts.get("addresses").copied().unwrap_or_default(); + if expected == 0 { + return Err(VerifyError::Incompatible( + "address_structure_stats has no addresses structure rows".into(), + )); + } + for measure in ["packets", "bytes"] { + let actual = counts.get(measure).copied().unwrap_or_default(); + if actual != expected { + return Err(VerifyError::Incompatible(format!( + "address_structure_stats has {actual} {measure} structure rows; expected {expected}" + ))); + } + } + Ok(()) +} + fn assert_processed_inputs_complete(connection: &Connection) -> Result<(), VerifyError> { let pending: i64 = connection.query_row( "SELECT COUNT(*) FROM processed_inputs WHERE status != 'processed'", @@ -614,7 +641,7 @@ fn assert_ipv6_maad_rows( SELECT 1 FROM address_structure_stats WHERE source_id = ?1 AND granularity = '1h' AND ip_version = 6 AND src_locality = 'all' AND dst_locality = 'all' - AND structure_kind = 'structure' + AND measure = 'addresses' AND structure_kind = 'structure' AND bucket_start >= ?2 AND bucket_start < ?3 )", params![source_id, bucket_start, bucket_end], @@ -704,7 +731,7 @@ const STRUCTURE_QUERY: &str = " WHERE granularity = '1h' AND source_id IN (?) AND bucket_start >= ? AND bucket_start < ? AND ip_version = 4 AND src_locality = 'all' AND dst_locality = 'all' - AND structure_kind = 'structure' + AND measure = 'addresses' AND structure_kind = 'structure' GROUP BY source_id, bucket_start ORDER BY source_id, bucket_start LIMIT 1"; const SPECTRUM_QUERY: &str = " @@ -715,7 +742,7 @@ const SPECTRUM_QUERY: &str = " WHERE granularity = '1h' AND source_id IN (?) AND bucket_start >= ? AND bucket_start < ? AND ip_version = 4 AND src_locality = 'all' AND dst_locality = 'all' - AND structure_kind = 'spectrum' + AND measure = 'addresses' AND structure_kind = 'spectrum' GROUP BY source_id, bucket_start ORDER BY source_id, bucket_start LIMIT 1"; const FILE_DETAILS_QUERY: &str = " @@ -820,16 +847,62 @@ mod tests { .execute( "INSERT INTO address_structure_stats ( source_id, granularity, bucket_start, bucket_end, ip_version, - src_locality, dst_locality, address_side, structure_kind, + src_locality, dst_locality, address_side, measure, structure_kind, values_json, metadata_json - ) VALUES ('r1', '1h', 0, 3600, 6, 'all', 'all', 'source', 'structure', - '[]', '{}')", + ) VALUES ('r1', '1h', 0, 3600, 6, 'all', 'all', 'source', 'addresses', + 'structure', '[]', '{}')", [], ) .unwrap(); assert!(assert_ipv6_maad_rows(&connection, "r1", 0, 3_600).is_ok()); } + #[test] + fn maad_data_requires_a_structure_row_for_every_measure() { + let connection = Connection::open_in_memory().unwrap(); + init_schema(&connection).unwrap(); + let insert = |measure: &str, bucket_start: i64| { + connection + .execute( + "INSERT INTO address_structure_stats ( + source_id, granularity, bucket_start, bucket_end, ip_version, + src_locality, dst_locality, address_side, measure, structure_kind, + values_json, metadata_json + ) VALUES ('r1', '5m', ?2, ?2 + 300, 4, 'all', 'all', 'source', ?1, + 'structure', '[]', '{}')", + params![measure, bucket_start], + ) + .unwrap(); + }; + + assert!(assert_maad_measures_present(&connection).is_err()); + for measure in ["addresses", "packets", "bytes"] { + insert(measure, 0); + } + assert!(assert_maad_measures_present(&connection).is_ok()); + insert("addresses", 300); + insert("packets", 300); + let error = assert_maad_measures_present(&connection).unwrap_err(); + assert!( + error + .to_string() + .contains("1 bytes structure rows; expected 2") + ); + assert!( + connection + .execute( + "INSERT INTO address_structure_stats ( + source_id, granularity, bucket_start, bucket_end, ip_version, + src_locality, dst_locality, address_side, measure, structure_kind, + values_json, metadata_json + ) VALUES ('r1', '5m', 0, 300, 4, 'all', 'all', 'source', 'packets', + 'spectrum', '[]', '{}')", + [], + ) + .is_err() + ); + } + #[test] fn canonical_database_satisfies_web_query_contract() { let directory = tempfile::tempdir().unwrap(); diff --git a/tools/netflow-db/tests/compare_cli.rs b/tools/netflow-db/tests/compare_cli.rs index dc0c0985..4ba7629f 100644 --- a/tools/netflow-db/tests/compare_cli.rs +++ b/tools/netflow-db/tests/compare_cli.rs @@ -114,7 +114,7 @@ fn compare_rejects_ipv6_maad_rows_missing_from_a_reference_bucket() { Connection::open(&candidate) .unwrap() .execute( - "INSERT INTO address_structure_stats VALUES ('r1','5m',0,300,6,'all','all','source','dimension','[]','{\"totalAddrs\":0}')", + "INSERT INTO address_structure_stats VALUES ('r1','5m',0,300,6,'all','all','source','addresses','dimension','[]','{\"totalAddrs\":0}')", [], ) .unwrap(); @@ -137,7 +137,7 @@ fn compare_rejects_ipv6_maad_rows_for_an_unrelated_source() { Connection::open(&candidate) .unwrap() .execute( - "INSERT INTO address_structure_stats VALUES ('r2','5m',0,300,6,'all','all','source','dimension','[]','{\"totalAddrs\":0}')", + "INSERT INTO address_structure_stats VALUES ('r2','5m',0,300,6,'all','all','source','addresses','dimension','[]','{\"totalAddrs\":0}')", [], ) .unwrap(); @@ -237,7 +237,7 @@ fn create_shared_database(path: &std::path::Path, flows: i64, dimension: f64, ex bucket_start INTEGER NOT NULL, bucket_end INTEGER NOT NULL, ip_version INTEGER NOT NULL, src_locality TEXT NOT NULL, dst_locality TEXT NOT NULL, address_side TEXT NOT NULL, - structure_kind TEXT NOT NULL, values_json TEXT NOT NULL, + measure TEXT NOT NULL, structure_kind TEXT NOT NULL, values_json TEXT NOT NULL, metadata_json TEXT NOT NULL ); CREATE TABLE processed_inputs ( @@ -269,7 +269,7 @@ fn create_shared_database(path: &std::path::Path, flows: i64, dimension: f64, ex .unwrap(); connection .execute( - "INSERT INTO address_structure_stats VALUES ('r1','5m',0,300,4,'all','all','source','dimension',?1,'{\"totalAddrs\":2}')", + "INSERT INTO address_structure_stats VALUES ('r1','5m',0,300,4,'all','all','source','addresses','dimension',?1,'{\"totalAddrs\":2}')", [format!("[{{\"q\":1,\"dim\":{dimension}}}]")], ) .unwrap(); @@ -281,7 +281,7 @@ fn create_shared_database(path: &std::path::Path, flows: i64, dimension: f64, ex .unwrap(); connection .execute( - "INSERT INTO address_structure_stats VALUES ('r1','30m',3600,5400,4,'all','all','source','dimension','[]','{\"totalAddrs\":0}')", + "INSERT INTO address_structure_stats VALUES ('r1','30m',3600,5400,4,'all','all','source','addresses','dimension','[]','{\"totalAddrs\":0}')", [], ) .unwrap(); @@ -300,7 +300,7 @@ fn create_shared_database(path: &std::path::Path, flows: i64, dimension: f64, ex .unwrap(); connection .execute( - "INSERT INTO address_structure_stats VALUES ('r1','30m',0,1800,4,'all','all','source','dimension','[]','{\"totalAddrs\":0}')", + "INSERT INTO address_structure_stats VALUES ('r1','30m',0,1800,4,'all','all','source','addresses','dimension','[]','{\"totalAddrs\":0}')", [], ) .unwrap(); diff --git a/tools/netflow-db/tests/ten_minute_rollups.rs b/tools/netflow-db/tests/ten_minute_rollups.rs index a81717f1..609c11a4 100644 --- a/tools/netflow-db/tests/ten_minute_rollups.rs +++ b/tools/netflow-db/tests/ten_minute_rollups.rs @@ -170,18 +170,21 @@ fn ten_minute_rollups_match_their_five_minute_children() { assert_eq!(unique_sources("10m", 0), 3); assert_eq!(unique_sources("10m", 600), 1); - let maad_total_addrs = |granularity: &str, bucket_start: i64| -> i64 { + let maad_total_addrs = |granularity: &str, bucket_start: i64, measure: &str| -> i64 { connection .query_row( "SELECT json_extract(metadata_json, '$.totalAddrs') FROM address_structure_stats WHERE source_id = 'edge' AND granularity = ?1 AND bucket_start = ?2 AND ip_version = 4 AND src_locality = 'all' AND dst_locality = 'all' - AND address_side = 'source' AND structure_kind = 'structure'", - params![granularity, bucket_start], + AND address_side = 'source' AND measure = ?3 + AND structure_kind = 'structure'", + params![granularity, bucket_start, measure], |row| row.get(0), ) .unwrap() }; - assert_eq!(maad_total_addrs("10m", 0), 3); - assert_eq!(maad_total_addrs("10m", 600), 1); + for measure in ["addresses", "packets", "bytes"] { + assert_eq!(maad_total_addrs("10m", 0, measure), 3, "{measure}"); + assert_eq!(maad_total_addrs("10m", 600, measure), 1, "{measure}"); + } } From 2dd7548ff89e0aa6103b37d3e1ae5f71ffb4e287 Mon Sep 17 00:00:00 2001 From: flamboh Date: Thu, 24 Sep 2026 16:41:34 -0700 Subject: [PATCH 2/8] feat(web): choose the MAAD measure on file details Structure, spectrum and file-detail MAAD queries now bind a measure from a shared parser (parseMaadParams / parseMaadStatsParams, default addresses), so the MAX(CASE) pivots never mix measures. Spectrum routes reject packets and bytes with a 400 because no weighted spectrum exists. The file detail page gets a shared MaadMeasureFilter next to the address family control. Packets and bytes hide the spectrum section. A Drizzle migration adds the measure key and CHECKs and keeps existing rows as addresses. --- apps/web/drizzle/0005_maad_measures.sql | 29 + apps/web/drizzle/meta/0005_snapshot.json | 1106 +++++++++++++++++ apps/web/drizzle/meta/_journal.json | 7 + .../filters/MaadMeasureFilter.svelte | 31 + .../netflow/NetflowFileDetailPage.svelte | 56 +- .../netflow/NetflowFileRouterCard.svelte | 20 +- .../netflow/file-detail-loader.svelte.ts | 37 +- apps/web/src/lib/server/db/local-schema.ts | 8 +- apps/web/src/lib/server/db/schema.ts | 12 + apps/web/src/lib/server/netflow-v3.ts | 63 +- apps/web/src/lib/types/types.ts | 16 + .../netflow/files/[slug]/details/+server.ts | 16 +- .../netflow/files/[slug]/spectrum/+server.ts | 23 +- .../netflow/files/[slug]/structure/+server.ts | 14 +- .../api/netflow/spectrum-stats/+server.ts | 20 +- .../api/netflow/structure-stats/+server.ts | 15 +- apps/web/tests/lib/server/migrations.test.ts | 46 + apps/web/tests/lib/server/netflow-v3.test.ts | 46 + apps/web/tests/performance/seed-large-db.ts | 4 +- .../tests/routes/api-aggregate-routes.test.ts | 62 +- .../tests/routes/netflow-file-routes.test.ts | 60 +- 21 files changed, 1609 insertions(+), 82 deletions(-) create mode 100644 apps/web/drizzle/0005_maad_measures.sql create mode 100644 apps/web/drizzle/meta/0005_snapshot.json create mode 100644 apps/web/src/lib/components/filters/MaadMeasureFilter.svelte diff --git a/apps/web/drizzle/0005_maad_measures.sql b/apps/web/drizzle/0005_maad_measures.sql new file mode 100644 index 00000000..06680500 --- /dev/null +++ b/apps/web/drizzle/0005_maad_measures.sql @@ -0,0 +1,29 @@ +PRAGMA foreign_keys=OFF;--> statement-breakpoint +CREATE TABLE `__new_address_structure_stats` ( + `source_id` text NOT NULL, + `granularity` text NOT NULL, + `bucket_start` integer NOT NULL, + `bucket_end` integer NOT NULL, + `ip_version` integer NOT NULL, + `src_locality` text NOT NULL, + `dst_locality` text NOT NULL, + `address_side` text NOT NULL, + `measure` text NOT NULL, + `structure_kind` text NOT NULL, + `values_json` text NOT NULL, + `metadata_json` text NOT NULL, + `processed_at` text DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(`source_id`, `granularity`, `bucket_start`, `ip_version`, `src_locality`, `dst_locality`, `address_side`, `measure`, `structure_kind`), + CONSTRAINT "address_structure_stats_ip_version_check" CHECK("__new_address_structure_stats"."ip_version" IN (4, 6)), + CONSTRAINT "address_structure_stats_src_locality_check" CHECK("__new_address_structure_stats"."src_locality" IN ('all', 'internal', 'external')), + CONSTRAINT "address_structure_stats_dst_locality_check" CHECK("__new_address_structure_stats"."dst_locality" IN ('all', 'internal', 'external')), + CONSTRAINT "address_structure_stats_measure_check" CHECK("__new_address_structure_stats"."measure" IN ('addresses', 'packets', 'bytes')), + CONSTRAINT "address_structure_stats_measure_spectrum_check" CHECK("__new_address_structure_stats"."measure" = 'addresses' OR "__new_address_structure_stats"."structure_kind" <> 'spectrum') +); +--> statement-breakpoint +INSERT INTO `__new_address_structure_stats`("source_id", "granularity", "bucket_start", "bucket_end", "ip_version", "src_locality", "dst_locality", "address_side", "measure", "structure_kind", "values_json", "metadata_json", "processed_at") SELECT "source_id", "granularity", "bucket_start", "bucket_end", "ip_version", "src_locality", "dst_locality", "address_side", 'addresses', "structure_kind", "values_json", "metadata_json", "processed_at" FROM `address_structure_stats`;--> statement-breakpoint +DROP TABLE `address_structure_stats`;--> statement-breakpoint +ALTER TABLE `__new_address_structure_stats` RENAME TO `address_structure_stats`;--> statement-breakpoint +PRAGMA foreign_keys=ON;--> statement-breakpoint +CREATE INDEX `idx_address_structure_stats_query` ON `address_structure_stats` (`granularity`,`bucket_start`,`source_id`,`ip_version`,`src_locality`,`dst_locality`,`address_side`,`measure`,`structure_kind`);--> statement-breakpoint +CREATE INDEX `idx_address_structure_stats_timeseries` ON `address_structure_stats` (`source_id`,`granularity`,`src_locality`,`dst_locality`,`ip_version`,`measure`,`structure_kind`,`bucket_start`); \ No newline at end of file diff --git a/apps/web/drizzle/meta/0005_snapshot.json b/apps/web/drizzle/meta/0005_snapshot.json new file mode 100644 index 00000000..7858b439 --- /dev/null +++ b/apps/web/drizzle/meta/0005_snapshot.json @@ -0,0 +1,1106 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "346de28f-0cfb-4986-8daf-28235ade1fdf", + "prevId": "4a3c1a73-bfe4-4e21-95c9-7ea8a091979f", + "tables": { + "address_count_stats": { + "name": "address_count_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "address_side": { + "name": "address_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_address_count": { + "name": "unique_address_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_address_count_stats_query": { + "name": "idx_address_count_stats_query", + "columns": [ + "granularity", + "bucket_start", + "source_id", + "ip_version", + "src_locality", + "dst_locality", + "address_side" + ], + "isUnique": false + }, + "idx_address_count_stats_timeseries": { + "name": "idx_address_count_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "address_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality", + "address_side" + ], + "name": "address_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "address_count_stats_ip_version_check": { + "name": "address_count_stats_ip_version_check", + "value": "\"address_count_stats\".\"ip_version\" IN (4, 6)" + }, + "address_count_stats_src_locality_check": { + "name": "address_count_stats_src_locality_check", + "value": "\"address_count_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "address_count_stats_dst_locality_check": { + "name": "address_count_stats_dst_locality_check", + "value": "\"address_count_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "address_structure_stats": { + "name": "address_structure_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "address_side": { + "name": "address_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "measure": { + "name": "measure", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "structure_kind": { + "name": "structure_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "values_json": { + "name": "values_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata_json": { + "name": "metadata_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_address_structure_stats_query": { + "name": "idx_address_structure_stats_query", + "columns": [ + "granularity", + "bucket_start", + "source_id", + "ip_version", + "src_locality", + "dst_locality", + "address_side", + "measure", + "structure_kind" + ], + "isUnique": false + }, + "idx_address_structure_stats_timeseries": { + "name": "idx_address_structure_stats_timeseries", + "columns": [ + "source_id", + "granularity", + "src_locality", + "dst_locality", + "ip_version", + "measure", + "structure_kind", + "bucket_start" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "address_structure_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_measure_structure_kind_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality", + "address_side", + "measure", + "structure_kind" + ], + "name": "address_structure_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_address_side_measure_structure_kind_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "address_structure_stats_ip_version_check": { + "name": "address_structure_stats_ip_version_check", + "value": "\"address_structure_stats\".\"ip_version\" IN (4, 6)" + }, + "address_structure_stats_src_locality_check": { + "name": "address_structure_stats_src_locality_check", + "value": "\"address_structure_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "address_structure_stats_dst_locality_check": { + "name": "address_structure_stats_dst_locality_check", + "value": "\"address_structure_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + }, + "address_structure_stats_measure_check": { + "name": "address_structure_stats_measure_check", + "value": "\"address_structure_stats\".\"measure\" IN ('addresses', 'packets', 'bytes')" + }, + "address_structure_stats_measure_spectrum_check": { + "name": "address_structure_stats_measure_spectrum_check", + "value": "\"address_structure_stats\".\"measure\" = 'addresses' OR \"address_structure_stats\".\"structure_kind\" <> 'spectrum'" + } + } + }, + "bucket_coverage": { + "name": "bucket_coverage", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "coverage_state": { + "name": "coverage_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "observed_units": { + "name": "observed_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expected_units": { + "name": "expected_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "rejected_units": { + "name": "rejected_units", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_bucket_coverage_query": { + "name": "idx_bucket_coverage_query", + "columns": ["granularity", "bucket_start", "source_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "bucket_coverage_source_id_granularity_bucket_start_pk": { + "columns": ["source_id", "granularity", "bucket_start"], + "name": "bucket_coverage_source_id_granularity_bucket_start_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "bucket_coverage_interval_check": { + "name": "bucket_coverage_interval_check", + "value": "\"bucket_coverage\".\"bucket_end\" > \"bucket_coverage\".\"bucket_start\"" + }, + "bucket_coverage_expected_check": { + "name": "bucket_coverage_expected_check", + "value": "\"bucket_coverage\".\"expected_units\" > 0" + }, + "bucket_coverage_observed_check": { + "name": "bucket_coverage_observed_check", + "value": "\"bucket_coverage\".\"observed_units\" >= 0 AND \"bucket_coverage\".\"observed_units\" <= \"bucket_coverage\".\"expected_units\"" + }, + "bucket_coverage_rejected_check": { + "name": "bucket_coverage_rejected_check", + "value": "\"bucket_coverage\".\"rejected_units\" >= 0 AND \"bucket_coverage\".\"rejected_units\" <= \"bucket_coverage\".\"expected_units\"" + }, + "bucket_coverage_state_check": { + "name": "bucket_coverage_state_check", + "value": "(\"bucket_coverage\".\"coverage_state\" = 'complete' AND \"bucket_coverage\".\"observed_units\" = \"bucket_coverage\".\"expected_units\" AND \"bucket_coverage\".\"rejected_units\" = 0) OR (\"bucket_coverage\".\"coverage_state\" = 'unknown' AND \"bucket_coverage\".\"observed_units\" = 0 AND \"bucket_coverage\".\"rejected_units\" = 0) OR (\"bucket_coverage\".\"coverage_state\" = 'partial' AND NOT (\"bucket_coverage\".\"observed_units\" = \"bucket_coverage\".\"expected_units\" AND \"bucket_coverage\".\"rejected_units\" = 0) AND NOT (\"bucket_coverage\".\"observed_units\" = 0 AND \"bucket_coverage\".\"rejected_units\" = 0))" + } + } + }, + "datasets": { + "name": "datasets", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "default_start_date": { + "name": "default_start_date", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_mode": { + "name": "source_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'static'" + }, + "discovery_mode": { + "name": "discovery_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'static'" + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "has_locality": { + "name": "has_locality", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "port_count_stats": { + "name": "port_count_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "port_side": { + "name": "port_side", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "port_range": { + "name": "port_range", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_port_count": { + "name": "unique_port_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_port_count_stats_timeseries": { + "name": "idx_port_count_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "port_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_port_side_port_range_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality", + "port_side", + "port_range" + ], + "name": "port_count_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_port_side_port_range_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "port_count_stats_ip_version_check": { + "name": "port_count_stats_ip_version_check", + "value": "\"port_count_stats\".\"ip_version\" IN (4, 6)" + }, + "port_count_stats_src_locality_check": { + "name": "port_count_stats_src_locality_check", + "value": "\"port_count_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "port_count_stats_dst_locality_check": { + "name": "port_count_stats_dst_locality_check", + "value": "\"port_count_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "processed_inputs": { + "name": "processed_inputs", + "columns": { + "input_kind": { + "name": "input_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_locator": { + "name": "input_locator", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "discovered_at": { + "name": "discovered_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "idx_processed_inputs_source_bucket": { + "name": "idx_processed_inputs_source_bucket", + "columns": ["source_id", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "processed_inputs_input_kind_input_locator_source_id_bucket_start_pk": { + "columns": ["input_kind", "input_locator", "source_id", "bucket_start"], + "name": "processed_inputs_input_kind_input_locator_source_id_bucket_start_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "protocol_stats": { + "name": "protocol_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unique_protocols_count": { + "name": "unique_protocols_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "protocols_list": { + "name": "protocols_list", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_protocol_stats_timeseries": { + "name": "idx_protocol_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "protocol_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality" + ], + "name": "protocol_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "protocol_stats_ip_version_check": { + "name": "protocol_stats_ip_version_check", + "value": "\"protocol_stats\".\"ip_version\" IN (4, 6)" + }, + "protocol_stats_src_locality_check": { + "name": "protocol_stats_src_locality_check", + "value": "\"protocol_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "protocol_stats_dst_locality_check": { + "name": "protocol_stats_dst_locality_check", + "value": "\"protocol_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + }, + "source_members": { + "name": "source_members", + "columns": { + "dataset_id": { + "name": "dataset_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "member_id": { + "name": "member_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "source_members_dataset_id_source_id_member_id_pk": { + "columns": ["dataset_id", "source_id", "member_id"], + "name": "source_members_dataset_id_source_id_member_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "traffic_stats": { + "name": "traffic_stats", + "columns": { + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "granularity": { + "name": "granularity", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket_end": { + "name": "bucket_end", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_version": { + "name": "ip_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "src_locality": { + "name": "src_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dst_locality": { + "name": "dst_locality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows": { + "name": "flows", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_tcp": { + "name": "flows_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_udp": { + "name": "flows_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_icmp": { + "name": "flows_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "flows_other": { + "name": "flows_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets": { + "name": "packets", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_tcp": { + "name": "packets_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_udp": { + "name": "packets_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_icmp": { + "name": "packets_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "packets_other": { + "name": "packets_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_tcp": { + "name": "bytes_tcp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_udp": { + "name": "bytes_udp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_icmp": { + "name": "bytes_icmp", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes_other": { + "name": "bytes_other", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_sum_ms": { + "name": "duration_sum_ms", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_count": { + "name": "duration_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_duration_ms": { + "name": "average_duration_ms", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "min_ttl_sum": { + "name": "min_ttl_sum", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "min_ttl_count": { + "name": "min_ttl_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_min_ttl": { + "name": "average_min_ttl", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "max_ttl_sum": { + "name": "max_ttl_sum", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "max_ttl_count": { + "name": "max_ttl_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "average_max_ttl": { + "name": "average_max_ttl", + "type": "real", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "CURRENT_TIMESTAMP" + } + }, + "indexes": { + "idx_traffic_stats_query": { + "name": "idx_traffic_stats_query", + "columns": [ + "granularity", + "bucket_start", + "source_id", + "ip_version", + "src_locality", + "dst_locality" + ], + "isUnique": false + }, + "idx_traffic_stats_timeseries": { + "name": "idx_traffic_stats_timeseries", + "columns": ["source_id", "granularity", "src_locality", "dst_locality", "bucket_start"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "traffic_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk": { + "columns": [ + "source_id", + "granularity", + "bucket_start", + "ip_version", + "src_locality", + "dst_locality" + ], + "name": "traffic_stats_source_id_granularity_bucket_start_ip_version_src_locality_dst_locality_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": { + "traffic_stats_ip_version_check": { + "name": "traffic_stats_ip_version_check", + "value": "\"traffic_stats\".\"ip_version\" IN (4, 6)" + }, + "traffic_stats_src_locality_check": { + "name": "traffic_stats_src_locality_check", + "value": "\"traffic_stats\".\"src_locality\" IN ('all', 'internal', 'external')" + }, + "traffic_stats_dst_locality_check": { + "name": "traffic_stats_dst_locality_check", + "value": "\"traffic_stats\".\"dst_locality\" IN ('all', 'internal', 'external')" + } + } + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/apps/web/drizzle/meta/_journal.json b/apps/web/drizzle/meta/_journal.json index ba24b415..0d4ecbd5 100644 --- a/apps/web/drizzle/meta/_journal.json +++ b/apps/web/drizzle/meta/_journal.json @@ -36,6 +36,13 @@ "when": 1790313292395, "tag": "0004_dataset_locality", "breakpoints": true + }, + { + "idx": 5, + "version": "6", + "when": 1790328400092, + "tag": "0005_maad_measures", + "breakpoints": true } ] } diff --git a/apps/web/src/lib/components/filters/MaadMeasureFilter.svelte b/apps/web/src/lib/components/filters/MaadMeasureFilter.svelte new file mode 100644 index 00000000..67c5aee3 --- /dev/null +++ b/apps/web/src/lib/components/filters/MaadMeasureFilter.svelte @@ -0,0 +1,31 @@ + + + diff --git a/apps/web/src/lib/components/netflow/NetflowFileDetailPage.svelte b/apps/web/src/lib/components/netflow/NetflowFileDetailPage.svelte index eebddb0f..9f0f5eb3 100644 --- a/apps/web/src/lib/components/netflow/NetflowFileDetailPage.svelte +++ b/apps/web/src/lib/components/netflow/NetflowFileDetailPage.svelte @@ -6,11 +6,15 @@ import NetflowFileMessageCard from '$lib/components/netflow/NetflowFileMessageCard.svelte'; import NetflowFileRouterCard from '$lib/components/netflow/NetflowFileRouterCard.svelte'; import SegmentedControl from '$lib/components/common/SegmentedControl.svelte'; + import MaadMeasureFilter from '$lib/components/filters/MaadMeasureFilter.svelte'; import { navigateToNetflowFile } from '$lib/utils/netflow-file-navigation'; import { + DEFAULT_MAAD_MEASURE, MAAD_IP_VERSION_OPTIONS, + maadMeasureHasSpectrum, type FlowDirection, - type MaadIpVersion + type MaadIpVersion, + type MaadMeasure } from '$lib/types/types'; import { createDateFromPSTComponents, @@ -36,6 +40,8 @@ let { data }: { data: NetflowFileDetailData } = $props(); let loader = $state.raw | null>(null); + let maadMeasure = $state(DEFAULT_MAAD_MEASURE); + const showSpectrum = $derived(maadMeasureHasSpectrum(maadMeasure)); const formatCount = (value: number | null | undefined) => typeof value === 'number' && Number.isFinite(value) ? value.toLocaleString() : 'N/A'; @@ -59,7 +65,13 @@ const nextSlug = $derived(getNextSlug(data.slug)); function syncLoader() { - loader = getNetflowFileDetailLoader(data.dataset, data.slug, data.direction, data.ipVersion); + loader = getNetflowFileDetailLoader( + data.dataset, + data.slug, + data.direction, + data.ipVersion, + maadMeasure + ); loader.refresh(); } @@ -78,6 +90,14 @@ }); } + function handleMaadMeasureChange({ measure }: { measure: MaadMeasure }) { + if (measure === maadMeasure) { + return; + } + maadMeasure = measure; + syncLoader(); + } + onMount(() => { syncLoader(); }); @@ -106,18 +126,24 @@ : 'N/A'} /> -
- MAAD address family: - ({ - value: String(option.value), - label: option.label - }))} - value={String(data.ipVersion)} - onValueChange={(value) => handleMaadIpVersionChange(Number(value) as MaadIpVersion)} - ariaLabel="Select MAAD IP address family" - buttonClass="sm:min-w-32" - /> +
+
+ MAAD address family: + ({ + value: String(option.value), + label: option.label + }))} + value={String(data.ipVersion)} + onValueChange={(value) => handleMaadIpVersionChange(Number(value) as MaadIpVersion)} + ariaLabel="Select MAAD IP address family" + buttonClass="sm:min-w-32" + /> +
+
+ MAAD measure: + +
{#if loader?.error && !loader.hasRows} @@ -144,7 +170,7 @@ /> {/if} {#each loader.rows as row (row.key)} - + {/each}
{/if} diff --git a/apps/web/src/lib/components/netflow/NetflowFileRouterCard.svelte b/apps/web/src/lib/components/netflow/NetflowFileRouterCard.svelte index 8967bd95..8a897547 100644 --- a/apps/web/src/lib/components/netflow/NetflowFileRouterCard.svelte +++ b/apps/web/src/lib/components/netflow/NetflowFileRouterCard.svelte @@ -6,10 +6,12 @@ let { row, + showSpectrum = true, formatCount, formatTimestampAsPST }: { row: NetflowFileRouterRow; + showSpectrum?: boolean; formatCount: (value: number | null | undefined) => string; formatTimestampAsPST: (timestamp: number) => string; } = $props(); @@ -35,12 +37,18 @@ source={row.source.structure} destination={row.destination.structure} /> - + {#if showSpectrum} + + {:else} +

+ The spectrum is only computed for the addresses measure. +

+ {/if} diff --git a/apps/web/src/lib/components/netflow/file-detail-loader.svelte.ts b/apps/web/src/lib/components/netflow/file-detail-loader.svelte.ts index 29ecf452..e88164ab 100644 --- a/apps/web/src/lib/components/netflow/file-detail-loader.svelte.ts +++ b/apps/web/src/lib/components/netflow/file-detail-loader.svelte.ts @@ -1,8 +1,10 @@ import { DEFAULT_MAAD_IP_VERSION, + DEFAULT_MAAD_MEASURE, type FileIpCounts, type FlowDirection, type MaadIpVersion, + type MaadMeasure, type NetflowFileDetailsResponse, type NetflowFileSummaryRecord, type SpectrumData, @@ -72,14 +74,16 @@ function buildLoaderKey( dataset: string, slug: string, direction: FlowDirection, - ipVersion: MaadIpVersion + ipVersion: MaadIpVersion, + measure: MaadMeasure ): string { - return `${dataset}:${slug}:${direction}:${ipVersion}`; + return `${dataset}:${slug}:${direction}:${ipVersion}:${measure}`; } function buildScopeSearchParams( direction: FlowDirection, - ipVersion: MaadIpVersion + ipVersion: MaadIpVersion, + measure: MaadMeasure ): Record { const params: Record = {}; @@ -91,6 +95,10 @@ function buildScopeSearchParams( params.ipVersion = String(ipVersion); } + if (measure !== DEFAULT_MAAD_MEASURE) { + params.measure = measure; + } + return params; } @@ -112,6 +120,7 @@ class NetflowFileDetailLoader { readonly slug: string; readonly direction: FlowDirection; readonly ipVersion: MaadIpVersion; + readonly measure: MaadMeasure; readonly key: string; private _loading = $state(false); @@ -124,12 +133,19 @@ class NetflowFileDetailLoader { private _lastAccessedAt = Date.now(); private summarySkeletonDelayId: ReturnType | null = null; - constructor(dataset: string, slug: string, direction: FlowDirection, ipVersion: MaadIpVersion) { + constructor( + dataset: string, + slug: string, + direction: FlowDirection, + ipVersion: MaadIpVersion, + measure: MaadMeasure + ) { this.dataset = dataset; this.slug = slug; this.direction = direction; this.ipVersion = ipVersion; - this.key = buildLoaderKey(dataset, slug, direction, ipVersion); + this.measure = measure; + this.key = buildLoaderKey(dataset, slug, direction, ipVersion, measure); } get loading() { @@ -299,7 +315,7 @@ class NetflowFileDetailLoader { const response = await fetch( `/api/netflow/files/${this.slug}/details?${new URLSearchParams({ dataset: this.dataset, - ...buildScopeSearchParams(this.direction, this.ipVersion) + ...buildScopeSearchParams(this.direction, this.ipVersion, this.measure) }).toString()}` ); @@ -379,7 +395,7 @@ class NetflowFileDetailLoader { return `/api/netflow/files/${this.slug}/${kind === 'ipCounts' ? 'ip-counts' : kind}?${new URLSearchParams( { dataset: this.dataset, - ...buildScopeSearchParams(this.direction, this.ipVersion), + ...buildScopeSearchParams(this.direction, this.ipVersion, this.measure), router, source: String(side === 'source') } @@ -412,13 +428,14 @@ export function getNetflowFileDetailLoader( dataset: string, slug: string, direction: FlowDirection, - ipVersion: MaadIpVersion = DEFAULT_MAAD_IP_VERSION + ipVersion: MaadIpVersion = DEFAULT_MAAD_IP_VERSION, + measure: MaadMeasure = DEFAULT_MAAD_MEASURE ) { - const key = buildLoaderKey(dataset, slug, direction, ipVersion); + const key = buildLoaderKey(dataset, slug, direction, ipVersion, measure); let loader = fileDetailLoaders.get(key); if (!loader) { - loader = new NetflowFileDetailLoader(dataset, slug, direction, ipVersion); + loader = new NetflowFileDetailLoader(dataset, slug, direction, ipVersion, measure); fileDetailLoaders.set(key, loader); } diff --git a/apps/web/src/lib/server/db/local-schema.ts b/apps/web/src/lib/server/db/local-schema.ts index 5eaf4f27..d55a5117 100644 --- a/apps/web/src/lib/server/db/local-schema.ts +++ b/apps/web/src/lib/server/db/local-schema.ts @@ -121,13 +121,15 @@ export const localSchemaSql = ` src_locality TEXT NOT NULL CHECK(src_locality IN ('all', 'internal', 'external')), dst_locality TEXT NOT NULL CHECK(dst_locality IN ('all', 'internal', 'external')), address_side TEXT NOT NULL CHECK(address_side IN ('source', 'destination')), + measure TEXT NOT NULL CHECK(measure IN ('addresses', 'packets', 'bytes')), structure_kind TEXT NOT NULL CHECK(structure_kind IN ('structure', 'spectrum', 'dimension')), values_json TEXT NOT NULL, metadata_json TEXT NOT NULL, processed_at TEXT DEFAULT CURRENT_TIMESTAMP, + CHECK(measure = 'addresses' OR structure_kind <> 'spectrum'), PRIMARY KEY( source_id, granularity, bucket_start, ip_version, - src_locality, dst_locality, address_side, structure_kind + src_locality, dst_locality, address_side, measure, structure_kind ) ); @@ -181,12 +183,12 @@ export const localSchemaSql = ` CREATE INDEX IF NOT EXISTS idx_address_structure_stats_query ON address_structure_stats ( granularity, bucket_start, source_id, ip_version, - src_locality, dst_locality, address_side, structure_kind + src_locality, dst_locality, address_side, measure, structure_kind ); CREATE INDEX IF NOT EXISTS idx_address_structure_stats_timeseries ON address_structure_stats ( source_id, granularity, src_locality, dst_locality, - ip_version, structure_kind, bucket_start + ip_version, measure, structure_kind, bucket_start ); CREATE INDEX IF NOT EXISTS idx_port_count_stats_timeseries ON port_count_stats ( diff --git a/apps/web/src/lib/server/db/schema.ts b/apps/web/src/lib/server/db/schema.ts index 0d5e1bb5..f2b060c6 100644 --- a/apps/web/src/lib/server/db/schema.ts +++ b/apps/web/src/lib/server/db/schema.ts @@ -331,6 +331,7 @@ export const addressStructureStats = sqliteTable( srcLocality: text('src_locality', { enum: ['all', 'internal', 'external'] }).notNull(), dstLocality: text('dst_locality', { enum: ['all', 'internal', 'external'] }).notNull(), addressSide: text('address_side', { enum: ['source', 'destination'] }).notNull(), + measure: text('measure', { enum: ['addresses', 'packets', 'bytes'] }).notNull(), structureKind: text('structure_kind', { enum: ['structure', 'spectrum', 'dimension'] }).notNull(), @@ -348,6 +349,7 @@ export const addressStructureStats = sqliteTable( table.srcLocality, table.dstLocality, table.addressSide, + table.measure, table.structureKind ] }), @@ -359,6 +361,7 @@ export const addressStructureStats = sqliteTable( table.srcLocality, table.dstLocality, table.addressSide, + table.measure, table.structureKind ), index('idx_address_structure_stats_timeseries').on( @@ -367,6 +370,7 @@ export const addressStructureStats = sqliteTable( table.srcLocality, table.dstLocality, table.ipVersion, + table.measure, table.structureKind, table.bucketStart ), @@ -378,6 +382,14 @@ export const addressStructureStats = sqliteTable( check( 'address_structure_stats_dst_locality_check', sql`${table.dstLocality} IN ('all', 'internal', 'external')` + ), + check( + 'address_structure_stats_measure_check', + sql`${table.measure} IN ('addresses', 'packets', 'bytes')` + ), + check( + 'address_structure_stats_measure_spectrum_check', + sql`${table.measure} = 'addresses' OR ${table.structureKind} <> 'spectrum'` ) ] ); diff --git a/apps/web/src/lib/server/netflow-v3.ts b/apps/web/src/lib/server/netflow-v3.ts index 7ea1d653..1613a4c8 100644 --- a/apps/web/src/lib/server/netflow-v3.ts +++ b/apps/web/src/lib/server/netflow-v3.ts @@ -1,14 +1,17 @@ import { DEFAULT_MAAD_IP_VERSION, + DEFAULT_MAAD_MEASURE, FLOW_DIRECTIONS, flowDirectionLocalities, IP_GRANULARITIES, MAAD_IP_VERSIONS, + MAAD_MEASURES, type FlowDirection, type FlowLocality, type FlowLocalityPair, type IpGranularity, - type MaadIpVersion + type MaadIpVersion, + type MaadMeasure } from '$lib/types/types'; import type { SourceDefinition } from '$lib/server/datasets'; import type { StructureFunctionPoint } from '$lib/types/types'; @@ -30,6 +33,13 @@ export interface AggregateStatsParams { dstLocality: FlowLocality; } +export interface MaadParams { + ipVersion: MaadIpVersion; + measure: MaadMeasure; +} + +export type MaadStatsParams = AggregateStatsParams & MaadParams; + export interface RequestValidationError { error: string; status: 400; @@ -41,6 +51,7 @@ export type NetflowSchemaVersion = 'v3'; const VALID_IP_GRANULARITIES = new Set(IP_GRANULARITIES); const VALID_FLOW_DIRECTIONS = new Set(FLOW_DIRECTIONS); +const VALID_MAAD_MEASURES = new Set(MAAD_MEASURES); export function assertNetflowV3Database(): void { return; @@ -117,6 +128,56 @@ export function parseMaadIpVersion(url: URL): MaadIpVersion | RequestValidationE return version; } +export function parseMaadMeasure(url: URL): MaadMeasure | RequestValidationError { + const param = url.searchParams.get('measure'); + if (param === null) { + return DEFAULT_MAAD_MEASURE; + } + + if (!VALID_MAAD_MEASURES.has(param)) { + return { + error: `Invalid measure. Expected one of: ${MAAD_MEASURES.join(', ')}`, + status: 400 + }; + } + + return param as MaadMeasure; +} + +export function parseMaadParams(url: URL): MaadParams | RequestValidationError { + const ipVersion = parseMaadIpVersion(url); + if (typeof ipVersion !== 'number') { + return ipVersion; + } + + const measure = parseMaadMeasure(url); + if (typeof measure !== 'string') { + return measure; + } + + return { ipVersion, measure }; +} + +export function parseMaadStatsParams(url: URL): MaadStatsParams | RequestValidationError { + const aggregate = parseAggregateStatsParams(url); + if ('error' in aggregate) { + return aggregate; + } + + const maad = parseMaadParams(url); + if ('error' in maad) { + return maad; + } + + return { ...aggregate, ...maad }; +} + +export function spectrumMeasureError(measure: MaadMeasure): RequestValidationError | null { + return measure === 'addresses' + ? null + : { error: `Spectrum is only computed for the addresses measure, not ${measure}`, status: 400 }; +} + export function parseFlowDirectionParams( url: URL ): ({ direction: FlowDirection } & FlowLocalityPair) | RequestValidationError { diff --git a/apps/web/src/lib/types/types.ts b/apps/web/src/lib/types/types.ts index c6bb5f99..0db4006e 100644 --- a/apps/web/src/lib/types/types.ts +++ b/apps/web/src/lib/types/types.ts @@ -379,6 +379,22 @@ export const MAAD_IP_VERSION_OPTIONS: MaadIpVersionOption[] = [ { value: 6, label: 'IPv6 (/23–/64)' } ]; +export const MAAD_MEASURES = ['addresses', 'packets', 'bytes'] as const; + +export type MaadMeasure = (typeof MAAD_MEASURES)[number]; + +export const DEFAULT_MAAD_MEASURE: MaadMeasure = 'addresses'; + +export const MAAD_MEASURE_OPTIONS: { value: MaadMeasure; label: string }[] = [ + { value: 'addresses', label: 'Addresses' }, + { value: 'packets', label: 'Packets' }, + { value: 'bytes', label: 'Bytes' } +]; + +export function maadMeasureHasSpectrum(measure: MaadMeasure): boolean { + return measure === 'addresses'; +} + export interface SpectrumPoint { alpha: number; f: number; diff --git a/apps/web/src/routes/api/netflow/files/[slug]/details/+server.ts b/apps/web/src/routes/api/netflow/files/[slug]/details/+server.ts index 2c213b25..233b2f90 100644 --- a/apps/web/src/routes/api/netflow/files/[slug]/details/+server.ts +++ b/apps/web/src/routes/api/netflow/files/[slug]/details/+server.ts @@ -14,7 +14,7 @@ import { getDatasetFromRequest, slugToBucketStart, withDb } from '../utils'; import { normalizeStructurePoints, parseFlowDirectionParams, - parseMaadIpVersion + parseMaadParams } from '$lib/server/netflow-v3'; const FIVE_MINUTES = '5m'; @@ -133,9 +133,9 @@ export const GET: RequestHandler = async ({ params, url, platform }) => { return json({ error: flowDirection.error }, { status: flowDirection.status }); } - const ipVersion = parseMaadIpVersion(url); - if (typeof ipVersion !== 'number') { - return json({ error: ipVersion.error }, { status: ipVersion.status }); + const maad = parseMaadParams(url); + if ('error' in maad) { + return json({ error: maad.error }, { status: maad.status }); } if (!slug || slug.length !== 12 || !/^\d{12}$/.test(slug)) { @@ -210,6 +210,7 @@ export const GET: RequestHandler = async ({ params, url, platform }) => { AND ip_version = ? AND src_locality = ? AND dst_locality = ? + AND measure = ? AND structure_kind = 'structure' GROUP BY source_id, bucket_start ), @@ -225,6 +226,7 @@ export const GET: RequestHandler = async ({ params, url, platform }) => { AND ip_version = ? AND src_locality = ? AND dst_locality = ? + AND measure = ? AND structure_kind = 'spectrum' GROUP BY source_id, bucket_start ) @@ -281,14 +283,16 @@ export const GET: RequestHandler = async ({ params, url, platform }) => { flowDirection.dstLocality, FIVE_MINUTES, bucketStart, - ipVersion, + maad.ipVersion, flowDirection.srcLocality, flowDirection.dstLocality, + maad.measure, FIVE_MINUTES, bucketStart, - ipVersion, + maad.ipVersion, flowDirection.srcLocality, flowDirection.dstLocality, + maad.measure, bucketStart ] ); diff --git a/apps/web/src/routes/api/netflow/files/[slug]/spectrum/+server.ts b/apps/web/src/routes/api/netflow/files/[slug]/spectrum/+server.ts index d08133b4..8d0376fe 100644 --- a/apps/web/src/routes/api/netflow/files/[slug]/spectrum/+server.ts +++ b/apps/web/src/routes/api/netflow/files/[slug]/spectrum/+server.ts @@ -1,7 +1,11 @@ import { json } from '@sveltejs/kit'; import type { RequestHandler } from './$types'; import type { SpectrumData, SpectrumPoint } from '$lib/types/types'; -import { parseFlowDirectionParams, parseMaadIpVersion } from '$lib/server/netflow-v3'; +import { + parseFlowDirectionParams, + parseMaadParams, + spectrumMeasureError +} from '$lib/server/netflow-v3'; import { getDatasetFromRequest, slugToBucketStart, withDb } from '../utils'; const FIVE_MINUTES = '5m'; @@ -21,9 +25,14 @@ export const GET: RequestHandler = async ({ params, url, platform }) => { return json({ error: flowDirection.error }, { status: flowDirection.status }); } - const ipVersion = parseMaadIpVersion(url); - if (typeof ipVersion !== 'number') { - return json({ error: ipVersion.error }, { status: ipVersion.status }); + const maad = parseMaadParams(url); + if ('error' in maad) { + return json({ error: maad.error }, { status: maad.status }); + } + + const measureError = spectrumMeasureError(maad.measure); + if (measureError) { + return json({ error: measureError.error }, { status: measureError.status }); } if (!slug || slug.length !== 12 || !/^\d{12}$/.test(slug)) { @@ -61,16 +70,18 @@ export const GET: RequestHandler = async ({ params, url, platform }) => { AND src_locality = ? AND dst_locality = ? AND address_side = ? + AND measure = ? AND structure_kind = 'spectrum' LIMIT 1`, [ router, FIVE_MINUTES, bucketStart, - ipVersion, + maad.ipVersion, flowDirection.srcLocality, flowDirection.dstLocality, - isSource ? 'source' : 'destination' + isSource ? 'source' : 'destination', + maad.measure ] ); diff --git a/apps/web/src/routes/api/netflow/files/[slug]/structure/+server.ts b/apps/web/src/routes/api/netflow/files/[slug]/structure/+server.ts index b7c2fcb8..c9ed349f 100644 --- a/apps/web/src/routes/api/netflow/files/[slug]/structure/+server.ts +++ b/apps/web/src/routes/api/netflow/files/[slug]/structure/+server.ts @@ -5,7 +5,7 @@ import { getDatasetFromRequest, slugToBucketStart, withDb } from '../utils'; import { normalizeStructurePoints, parseFlowDirectionParams, - parseMaadIpVersion + parseMaadParams } from '$lib/server/netflow-v3'; const FIVE_MINUTES = '5m'; @@ -25,9 +25,9 @@ export const GET: RequestHandler = async ({ params, url, platform }) => { return json({ error: flowDirection.error }, { status: flowDirection.status }); } - const ipVersion = parseMaadIpVersion(url); - if (typeof ipVersion !== 'number') { - return json({ error: ipVersion.error }, { status: ipVersion.status }); + const maad = parseMaadParams(url); + if ('error' in maad) { + return json({ error: maad.error }, { status: maad.status }); } if (!slug || slug.length !== 12 || !/^\d{12}$/.test(slug)) { @@ -65,16 +65,18 @@ export const GET: RequestHandler = async ({ params, url, platform }) => { AND src_locality = ? AND dst_locality = ? AND address_side = ? + AND measure = ? AND structure_kind = 'structure' LIMIT 1`, [ router, FIVE_MINUTES, bucketStart, - ipVersion, + maad.ipVersion, flowDirection.srcLocality, flowDirection.dstLocality, - isSource ? 'source' : 'destination' + isSource ? 'source' : 'destination', + maad.measure ] ); diff --git a/apps/web/src/routes/api/netflow/spectrum-stats/+server.ts b/apps/web/src/routes/api/netflow/spectrum-stats/+server.ts index c9229dc0..cb46a8e4 100644 --- a/apps/web/src/routes/api/netflow/spectrum-stats/+server.ts +++ b/apps/web/src/routes/api/netflow/spectrum-stats/+server.ts @@ -4,11 +4,7 @@ import type { SpectrumPoint } from '$lib/types/types'; import type { SpectrumStatsPayload, SpectrumStatsResponse } from '$lib/types/spectrum-stats'; import { buildCoverageTimelines } from '$lib/server/db/coverage'; import { getRequestedDataset, withDatasetDb } from '$lib/server/datasets'; -import { - parseAggregateStatsParams, - parseMaadIpVersion, - placeholders -} from '$lib/server/netflow-v3'; +import { parseMaadStatsParams, placeholders, spectrumMeasureError } from '$lib/server/netflow-v3'; type SpectrumStatsRow = SpectrumStatsPayload & { router: string; @@ -53,15 +49,15 @@ function parseSpectrumPoints( } export const GET: RequestHandler = async ({ url, platform }) => { - const params = parseAggregateStatsParams(url); + const params = parseMaadStatsParams(url); if ('error' in params) { return json({ error: params.error }, { status: params.status }); } - const ipVersion = parseMaadIpVersion(url); - if (typeof ipVersion !== 'number') { - return json({ error: ipVersion.error }, { status: ipVersion.status }); + const measureError = spectrumMeasureError(params.measure); + if (measureError) { + return json({ error: measureError.error }, { status: measureError.status }); } - const { routers, granularity, start, end, srcLocality, dstLocality } = params; + const { routers, granularity, start, end, srcLocality, dstLocality, ipVersion, measure } = params; try { const dataset = await getRequestedDataset(url, platform); @@ -75,7 +71,8 @@ export const GET: RequestHandler = async ({ url, platform }) => { dstLocality, start, end, - ipVersion + ipVersion, + measure ]; const query = ` @@ -93,6 +90,7 @@ export const GET: RequestHandler = async ({ url, platform }) => { AND bucket_start >= ? AND bucket_start < ? AND ip_version = ? + AND measure = ? AND structure_kind = 'spectrum' GROUP BY ${sourceColumn}, bucket_start `; diff --git a/apps/web/src/routes/api/netflow/structure-stats/+server.ts b/apps/web/src/routes/api/netflow/structure-stats/+server.ts index b606e12f..a1b29996 100644 --- a/apps/web/src/routes/api/netflow/structure-stats/+server.ts +++ b/apps/web/src/routes/api/netflow/structure-stats/+server.ts @@ -6,8 +6,7 @@ import { buildCoverageTimelines } from '$lib/server/db/coverage'; import { getRequestedDataset, withDatasetDb } from '$lib/server/datasets'; import { normalizeStructurePoints, - parseAggregateStatsParams, - parseMaadIpVersion, + parseMaadStatsParams, placeholders } from '$lib/server/netflow-v3'; @@ -64,15 +63,11 @@ function parseStructurePoints( } export const GET: RequestHandler = async ({ url, platform }) => { - const params = parseAggregateStatsParams(url); + const params = parseMaadStatsParams(url); if ('error' in params) { return json({ error: params.error }, { status: params.status }); } - const ipVersion = parseMaadIpVersion(url); - if (typeof ipVersion !== 'number') { - return json({ error: ipVersion.error }, { status: ipVersion.status }); - } - const { routers, granularity, start, end, srcLocality, dstLocality } = params; + const { routers, granularity, start, end, srcLocality, dstLocality, ipVersion, measure } = params; try { const dataset = await getRequestedDataset(url, platform); @@ -86,7 +81,8 @@ export const GET: RequestHandler = async ({ url, platform }) => { dstLocality, start, end, - ipVersion + ipVersion, + measure ]; const query = ` @@ -104,6 +100,7 @@ export const GET: RequestHandler = async ({ url, platform }) => { AND bucket_start >= ? AND bucket_start < ? AND ip_version = ? + AND measure = ? AND structure_kind = 'structure' GROUP BY ${sourceColumn}, bucket_start `; diff --git a/apps/web/tests/lib/server/migrations.test.ts b/apps/web/tests/lib/server/migrations.test.ts index 2f74327f..194b9d48 100644 --- a/apps/web/tests/lib/server/migrations.test.ts +++ b/apps/web/tests/lib/server/migrations.test.ts @@ -162,6 +162,7 @@ describe('D1 migrations', () => { 'src_locality', 'dst_locality', 'ip_version', + 'measure', 'structure_kind', 'bucket_start' ] @@ -342,4 +343,49 @@ describe('D1 migrations', () => { database.close(); } }); + + it('keys MAAD rows by measure and keeps existing rows as the addresses measure', () => { + const database = new Database(':memory:'); + const migrations = migrationFiles(); + const measureMigration = migrations.findIndex((fileName) => + fs + .readFileSync(path.join(migrationsDirectory, fileName), 'utf8') + .includes('address_structure_stats_measure_check') + ); + try { + for (const migration of migrations.slice(0, measureMigration)) { + applyMigration(database, migration); + } + database + .prepare( + `INSERT INTO address_structure_stats ( + source_id, granularity, bucket_start, bucket_end, ip_version, + src_locality, dst_locality, address_side, structure_kind, + values_json, metadata_json + ) VALUES ('edge', '5m', 0, 300, 4, 'all', 'all', 'source', 'spectrum', '[]', '{}')` + ) + .run(); + + for (const migration of migrations.slice(measureMigration)) { + applyMigration(database, migration); + } + + expect( + database.prepare('SELECT measure, structure_kind FROM address_structure_stats').all() + ).toEqual([{ measure: 'addresses', structure_kind: 'spectrum' }]); + const insert = database.prepare( + `INSERT INTO address_structure_stats ( + source_id, granularity, bucket_start, bucket_end, ip_version, + src_locality, dst_locality, address_side, measure, structure_kind, + values_json, metadata_json + ) VALUES ('edge', '5m', 0, 300, 4, 'all', 'all', 'source', ?, ?, '[]', '{}')` + ); + insert.run('packets', 'structure'); + insert.run('bytes', 'structure'); + expect(() => insert.run('packets', 'spectrum')).toThrow(/CHECK constraint failed/); + expect(() => insert.run('flows', 'structure')).toThrow(/CHECK constraint failed/); + } finally { + database.close(); + } + }); }); diff --git a/apps/web/tests/lib/server/netflow-v3.test.ts b/apps/web/tests/lib/server/netflow-v3.test.ts index 74b25884..bee0781f 100644 --- a/apps/web/tests/lib/server/netflow-v3.test.ts +++ b/apps/web/tests/lib/server/netflow-v3.test.ts @@ -9,6 +9,8 @@ import { parseFlowDirection, parseFlowDirectionParams, parseMaadIpVersion, + parseMaadMeasure, + parseMaadStatsParams, parseSourceIds, parseTimestamp, resolveSourceIds @@ -169,6 +171,50 @@ describe('netflow v3 helpers', () => { } }); + it('parses the MAAD measure with an addresses default', () => { + expect(parseMaadMeasure(new URL('http://localhost/api/test'))).toBe('addresses'); + for (const measure of ['addresses', 'packets', 'bytes']) { + expect(parseMaadMeasure(new URL(`http://localhost/api/test?measure=${measure}`))).toBe( + measure + ); + } + expect(parseMaadMeasure(new URL('http://localhost/api/test?measure=Packets'))).toEqual({ + error: 'Invalid measure. Expected one of: addresses, packets, bytes', + status: 400 + }); + }); + + it('extends aggregate stats params with the MAAD ip version and measure', () => { + expect( + parseMaadStatsParams( + new URL( + 'http://localhost/api/test?routers=r1&startDate=100&endDate=200&direction=egress&ipVersion=6&measure=bytes' + ) + ) + ).toEqual({ + routers: ['r1'], + granularity: '1h', + start: 100, + end: 200, + direction: 'egress', + srcLocality: 'internal', + dstLocality: 'external', + ipVersion: 6, + measure: 'bytes' + }); + expect( + parseMaadStatsParams( + new URL('http://localhost/api/test?routers=r1&startDate=100&endDate=200&measure=flows') + ) + ).toEqual({ + error: 'Invalid measure. Expected one of: addresses, packets, bytes', + status: 400 + }); + expect( + parseMaadStatsParams(new URL('http://localhost/api/test?startDate=100&endDate=200')) + ).toEqual({ error: 'No routers selected', status: 400 }); + }); + it('normalizes structure points from MAAD variants', () => { expect(normalizeStructurePoints([{ q: 1, tauTilde: 2, s: 3 }])).toEqual([ { q: 1, tau: 2, sd: 3 } diff --git a/apps/web/tests/performance/seed-large-db.ts b/apps/web/tests/performance/seed-large-db.ts index 74bd165d..b8b410cb 100644 --- a/apps/web/tests/performance/seed-large-db.ts +++ b/apps/web/tests/performance/seed-large-db.ts @@ -195,11 +195,11 @@ db.exec(` INSERT INTO address_structure_stats ( source_id, granularity, bucket_start, bucket_end, ip_version, - src_locality, dst_locality, address_side, structure_kind, + src_locality, dst_locality, address_side, measure, structure_kind, values_json, metadata_json ) SELECT source_id, '1h', bucket_start, bucket_start + 3600, 4, - src_locality, dst_locality, address_side, structure_kind, + src_locality, dst_locality, address_side, 'addresses', structure_kind, CASE structure_kind WHEN 'spectrum' THEN '[{"alpha":0.1,"f":0.2},{"alpha":0.2,"f":0.3},{"alpha":0.3,"f":0.4}]' ELSE '[{"q":1,"tau":0.2,"sd":0.01},{"q":2,"tau":0.4,"sd":0.02}]' diff --git a/apps/web/tests/routes/api-aggregate-routes.test.ts b/apps/web/tests/routes/api-aggregate-routes.test.ts index 84ac33c6..4a65b1be 100644 --- a/apps/web/tests/routes/api-aggregate-routes.test.ts +++ b/apps/web/tests/routes/api-aggregate-routes.test.ts @@ -456,7 +456,8 @@ describe('aggregate API routes', () => { 'all', 100, 200, - 4 + 4, + 'addresses' ]); await expect(response.json()).resolves.toEqual({ timelines: [ @@ -516,7 +517,8 @@ describe('aggregate API routes', () => { 'all', 100, 200, - 6 + 6, + 'addresses' ]); const spectrumResponse = await getSpectrumStats({ @@ -532,7 +534,8 @@ describe('aggregate API routes', () => { 'all', 100, 200, - 6 + 6, + 'addresses' ]); }); @@ -559,4 +562,57 @@ describe('aggregate API routes', () => { error: 'Invalid ipVersion. Expected one of: 4, 6' }); }); + + it('filters structure stats by the requested MAAD measure', async () => { + const all = vi.fn().mockResolvedValue([]); + vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); + mockDatasetSession({ all }); + + for (const measure of ['addresses', 'packets', 'bytes']) { + all.mockClear(); + const response = await getStructureStats({ + url: new URL( + `http://localhost/api/netflow/structure-stats?routers=r1&startDate=100&endDate=200&measure=${measure}` + ) + } as never); + expect(response.status).toBe(200); + expect(all).toHaveBeenNthCalledWith(1, expect.stringContaining('AND measure = ?'), [ + '1h', + 'r1', + 'all', + 'all', + 100, + 200, + 4, + measure + ]); + } + }); + + it('rejects unknown measures and weighted spectrum requests', async () => { + const all = vi.fn().mockResolvedValue([]); + vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); + mockDatasetSession({ all }); + + const invalid = await getStructureStats({ + url: new URL( + 'http://localhost/api/netflow/structure-stats?routers=r1&startDate=100&endDate=200&measure=flows' + ) + } as never); + expect(invalid.status).toBe(400); + await expect(invalid.json()).resolves.toEqual({ + error: 'Invalid measure. Expected one of: addresses, packets, bytes' + }); + + const weightedSpectrum = await getSpectrumStats({ + url: new URL( + 'http://localhost/api/netflow/spectrum-stats?routers=r1&startDate=100&endDate=200&measure=packets' + ) + } as never); + expect(weightedSpectrum.status).toBe(400); + await expect(weightedSpectrum.json()).resolves.toEqual({ + error: 'Spectrum is only computed for the addresses measure, not packets' + }); + expect(all).not.toHaveBeenCalled(); + }); }); diff --git a/apps/web/tests/routes/netflow-file-routes.test.ts b/apps/web/tests/routes/netflow-file-routes.test.ts index 15c444ff..ed63fe2c 100644 --- a/apps/web/tests/routes/netflow-file-routes.test.ts +++ b/apps/web/tests/routes/netflow-file-routes.test.ts @@ -106,7 +106,8 @@ describe('netflow file helpers and routes', () => { 4, 'internal', 'external', - 'destination' + 'destination', + 'addresses' ]); expect(get).toHaveBeenNthCalledWith(3, expect.any(String), [ 'r1', @@ -115,7 +116,8 @@ describe('netflow file helpers and routes', () => { 4, 'internal', 'external', - 'source' + 'source', + 'addresses' ]); await expect(ipResponse.json()).resolves.toEqual({ ipv4Count: 1, ipv6Count: 3 }); await expect(spectrumResponse.json()).resolves.toEqual({ @@ -213,11 +215,13 @@ describe('netflow file helpers and routes', () => { 4, 'internal', 'external', + 'addresses', '5m', bucketStart, 4, 'internal', 'external', + 'addresses', bucketStart ]); await expect(response.json()).resolves.toEqual({ @@ -335,7 +339,8 @@ describe('netflow file helpers and routes', () => { 6, 'all', 'all', - 'destination' + 'destination', + 'addresses' ]); expect(get).toHaveBeenNthCalledWith(2, expect.any(String), [ 'r1', @@ -344,7 +349,8 @@ describe('netflow file helpers and routes', () => { 6, 'all', 'all', - 'source' + 'source', + 'addresses' ]); }); @@ -376,6 +382,50 @@ describe('netflow file helpers and routes', () => { } }); + it('binds the requested MAAD measure into the per-file and details MAAD queries', async () => { + const get = vi.fn().mockResolvedValue({ valuesJson: '[{"q":1,"tauTilde":2,"sd":0.5}]' }); + const all = vi.fn().mockResolvedValue([]); + vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); + mockDatasetSession({ get, all }); + + const bucketStart = slugToBucketStart('202503010005'); + + const structureResponse = await getStructure({ + params: { slug: '202503010005' }, + url: new URL( + 'http://localhost/api/netflow/files/x/structure?router=r1&source=true&measure=packets' + ) + } as never); + expect(structureResponse.status).toBe(200); + expect(get).toHaveBeenCalledWith(expect.stringContaining('AND measure = ?'), [ + 'r1', + '5m', + bucketStart, + 4, + 'all', + 'all', + 'source', + 'packets' + ]); + + const spectrumResponse = await getSpectrum({ + params: { slug: '202503010005' }, + url: new URL( + 'http://localhost/api/netflow/files/x/spectrum?router=r1&source=true&measure=bytes' + ) + } as never); + expect(spectrumResponse.status).toBe(400); + expect(get).toHaveBeenCalledTimes(1); + + await getDetails({ + params: { slug: '202503010005' }, + url: new URL('http://localhost/api/netflow/files/x/details?dataset=alpha&measure=bytes') + } as never); + const [detailsQuery, detailsParams] = all.mock.calls[0] as [string, unknown[]]; + expect(detailsQuery.match(/AND measure = \?/g)).toHaveLength(2); + expect(detailsParams.filter((value) => value === 'bytes')).toHaveLength(2); + }); + it('binds the requested MAAD ip version into the file details MAAD query', async () => { const all = vi.fn().mockResolvedValue([]); vi.mocked(getRequestedDataset).mockResolvedValue('alpha'); @@ -403,11 +453,13 @@ describe('netflow file helpers and routes', () => { 6, 'all', 'all', + 'addresses', '5m', bucketStart, 6, 'all', 'all', + 'addresses', bucketStart ]); }); From 4595418c060bf7743e390910c65f24295aa7377e Mon Sep 17 00:00:00 2001 From: flamboh Date: Thu, 24 Sep 2026 21:11:02 -0700 Subject: [PATCH 3/8] perf(netflow-db): compute a scope's MAAD measures together and in parallel Each scoped address set now runs one shared prefix walk for addresses, packets and bytes, falling back to separate walks only when a weighted measure has zero-weight addresses. The MAAD pool grows from 2 workers to the available parallelism, capped at 8. Rollup merges no longer reserve capacity for every child entry. On one uoregon day the stored tables are identical to the previous build. --- tools/netflow-db/src/domain.rs | 1 - tools/netflow-db/src/publish.rs | 231 +++++++++++++++++--------------- 2 files changed, 123 insertions(+), 109 deletions(-) diff --git a/tools/netflow-db/src/domain.rs b/tools/netflow-db/src/domain.rs index a7d4deb7..46c5d9c1 100644 --- a/tools/netflow-db/src/domain.rs +++ b/tools/netflow-db/src/domain.rs @@ -746,7 +746,6 @@ impl AddressTotals { /// Sum another scope's totals into this one, keeping rollups additive. pub fn merge(&mut self, other: &Self) { - self.0.reserve(other.0.len()); for (address, traffic) in other.iter() { self.add(address, traffic); } diff --git a/tools/netflow-db/src/publish.rs b/tools/netflow-db/src/publish.rs index d0bafa36..dc902b16 100644 --- a/tools/netflow-db/src/publish.rs +++ b/tools/netflow-db/src/publish.rs @@ -42,7 +42,7 @@ pub enum PublishError { IncompleteCoverage(BucketKey), } -const MAAD_WORKERS: usize = 2; +const MAX_MAAD_WORKERS: usize = 8; static MAAD_POOL: OnceLock> = OnceLock::new(); /// Aggregate timings and work counts for one or more `write_buckets` calls. @@ -300,14 +300,11 @@ fn maad_rows( return Ok((Vec::new(), 0)); } - let work = address_sets - .iter() - .flat_map(|addresses| MaadMeasure::ALL.map(|measure| (addresses, measure))) - .collect::>(); let pool = maad_pool()?; let measured = pool.install(|| { - work.par_iter() - .map(|&(addresses, measure)| measure_rows(addresses, measure)) + address_sets + .par_iter() + .map(scope_rows) .collect::, _>>() })?; let zero_weight_addresses = measured.iter().map(|(_, excluded)| count(*excluded)).sum(); @@ -325,120 +322,138 @@ struct WeightedMetadata<'a> { zero_weight_addrs: usize, } -fn measure_rows( +/// One scope's MAAD results: distinct addresses, then packets and bytes with +/// the number of zero-weight addresses each weighted measure excluded. +struct ScopeResults { + addresses: maad::MaadResult, + weighted: [(maad::MaadResult, usize); 2], +} + +fn scope_rows( addresses: &AddressSetRow<'_>, - measure: MaadMeasure, ) -> Result<(Vec, usize), PublishError> { + let results = match addresses.scope.ip_version { + IpVersion::V4 => scope_results(addresses.addresses.iter().filter_map( + |(address, traffic)| match address { + IpAddr::V4(address) => Some((address, traffic)), + IpAddr::V6(_) => None, + }, + ))?, + IpVersion::V6 => scope_results(addresses.addresses.iter().filter_map( + |(address, traffic)| match address { + IpAddr::V6(address) => Some((address, traffic)), + IpAddr::V4(_) => None, + }, + ))?, + }; let dimensions = dimensions(&addresses.key, addresses.scope); - let row = - |structure_kind: &str, values_json: String, metadata_json: &str| AddressStructureStatsRow { - dimensions: dimensions.clone(), - address_side: addresses.address_side.as_str().to_owned(), - measure: measure.as_str().to_owned(), - structure_kind: structure_kind.to_owned(), - values_json, - metadata_json: metadata_json.to_owned(), - }; - let weight: fn(AddressTraffic) -> u64 = match measure { - MaadMeasure::Addresses => { - let result = address_maad_result(addresses); - let metadata_json = serde_json::to_string(&result.metadata)?; - return Ok(( - vec![ - row( - "structure", - serde_json::to_string(&result.structure)?, - &metadata_json, - ), - row( - "spectrum", - serde_json::to_string(&result.spectrum)?, - &metadata_json, - ), - row( - "dimension", - serde_json::to_string(&result.dimensions)?, - &metadata_json, - ), - ], - 0, - )); - } - MaadMeasure::Packets => |traffic| traffic.packets, - MaadMeasure::Bytes => |traffic| traffic.bytes, + let row = |measure: MaadMeasure, + structure_kind: &str, + values_json: String, + metadata_json: &str| AddressStructureStatsRow { + dimensions: dimensions.clone(), + address_side: addresses.address_side.as_str().to_owned(), + measure: measure.as_str().to_owned(), + structure_kind: structure_kind.to_owned(), + values_json, + metadata_json: metadata_json.to_owned(), }; - let (result, zero_weight_addrs) = weighted_maad_result(addresses, weight)?; - let metadata_json = serde_json::to_string(&WeightedMetadata { - metadata: &result.metadata, - zero_weight_addrs, - })?; - Ok(( - vec![ - row( - "structure", - serde_json::to_string(&result.structure)?, - &metadata_json, - ), - row( - "dimension", - serde_json::to_string(&result.dimensions)?, - &metadata_json, - ), - ], - zero_weight_addrs, - )) -} - -fn address_maad_result(addresses: &AddressSetRow<'_>) -> maad::MaadResult { - let family = addresses.addresses.iter().map(|(address, _)| address); - match addresses.scope.ip_version { - IpVersion::V4 => maad::compute(family.filter_map(|address| match address { - IpAddr::V4(address) => Some(address), - IpAddr::V6(_) => None, - })), - IpVersion::V6 => maad::compute(family.filter_map(|address| match address { - IpAddr::V6(address) => Some(address), - IpAddr::V4(_) => None, - })), + let address_metadata = serde_json::to_string(&results.addresses.metadata)?; + let mut rows = vec![ + row( + MaadMeasure::Addresses, + "structure", + serde_json::to_string(&results.addresses.structure)?, + &address_metadata, + ), + row( + MaadMeasure::Addresses, + "spectrum", + serde_json::to_string(&results.addresses.spectrum)?, + &address_metadata, + ), + row( + MaadMeasure::Addresses, + "dimension", + serde_json::to_string(&results.addresses.dimensions)?, + &address_metadata, + ), + ]; + let mut zero_weight_addresses = 0; + for (measure, (result, zero_weight_addrs)) in [MaadMeasure::Packets, MaadMeasure::Bytes] + .into_iter() + .zip(&results.weighted) + { + let metadata_json = serde_json::to_string(&WeightedMetadata { + metadata: &result.metadata, + zero_weight_addrs: *zero_weight_addrs, + })?; + rows.push(row( + measure, + "structure", + serde_json::to_string(&result.structure)?, + &metadata_json, + )); + rows.push(row( + measure, + "dimension", + serde_json::to_string(&result.dimensions)?, + &metadata_json, + )); + zero_weight_addresses += zero_weight_addrs; } + Ok((rows, zero_weight_addresses)) } -/// Weighted MAAD over addresses with a positive summed weight, and the count of -/// same-family addresses excluded because their weight was zero. -fn weighted_maad_result( - addresses: &AddressSetRow<'_>, - weight: fn(AddressTraffic) -> u64, -) -> Result<(maad::MaadResult, usize), maad::MaadError> { - let mut zero_weight_addrs = 0; - let mut positive = |traffic| match weight(traffic) { - 0 => { - zero_weight_addrs += 1; - None - } - value => Some(value as f64), - }; - let entries = addresses.addresses.iter(); - let result = match addresses.scope.ip_version { - IpVersion::V4 => { - maad::compute_weighted(entries.filter_map(|(address, traffic)| match address { - IpAddr::V4(address) => positive(traffic).map(|value| (address, value)), - IpAddr::V6(_) => None, - }))? - } - IpVersion::V6 => { - maad::compute_weighted(entries.filter_map(|(address, traffic)| match address { - IpAddr::V6(address) => positive(traffic).map(|value| (address, value)), - IpAddr::V4(_) => None, - }))? - } +/// Compute all measures over one shared prefix walk. Weighted measures leave out +/// zero-weight addresses, so a scope with any falls back to separate walks. +fn scope_results( + entries: impl Iterator, +) -> Result { + let entries = entries.collect::>(); + let zero_packets = entries + .iter() + .filter(|(_, traffic)| traffic.packets == 0) + .count(); + let zero_bytes = entries + .iter() + .filter(|(_, traffic)| traffic.bytes == 0) + .count(); + if zero_packets == 0 && zero_bytes == 0 { + let (addresses, [packets, bytes]) = + maad::compute_measures(entries.iter().map(|&(address, traffic)| { + (address, [traffic.packets as f64, traffic.bytes as f64]) + }))?; + return Ok(ScopeResults { + addresses, + weighted: [(packets, 0), (bytes, 0)], + }); + } + let weighted = |weight: fn(AddressTraffic) -> u64| { + maad::compute_weighted(entries.iter().filter_map(|&(address, traffic)| { + match weight(traffic) { + 0 => None, + value => Some((address, value as f64)), + } + })) }; - Ok((result, zero_weight_addrs)) + Ok(ScopeResults { + addresses: maad::compute(entries.iter().map(|&(address, _)| address)), + weighted: [ + (weighted(|traffic| traffic.packets)?, zero_packets), + (weighted(|traffic| traffic.bytes)?, zero_bytes), + ], + }) } fn maad_pool() -> Result<&'static rayon::ThreadPool, PublishError> { match MAAD_POOL.get_or_init(|| { rayon::ThreadPoolBuilder::new() - .num_threads(MAAD_WORKERS) + .num_threads( + std::thread::available_parallelism() + .map_or(1, std::num::NonZeroUsize::get) + .min(MAX_MAAD_WORKERS), + ) .thread_name(|index| format!("maad-{index}")) .build() .map_err(|error| error.to_string()) From 6cca5a1dcff60fdfc026e771995b39b7358e8cc5 Mon Sep 17 00:00:00 2001 From: flamboh Date: Thu, 24 Sep 2026 21:21:23 -0700 Subject: [PATCH 4/8] perf(netflow-db): decode the next nfcapd batch while publishing the current one A scoped decode thread works on batch N+1 while the writer publishes batch N, so decode workers no longer wait for MAAD and SQLite writes. At most two decoded batches are held. Revision checks still bracket every decode. --- tools/netflow-db/src/pipeline.rs | 159 ++++++++++++++++++++++--------- 1 file changed, 112 insertions(+), 47 deletions(-) diff --git a/tools/netflow-db/src/pipeline.rs b/tools/netflow-db/src/pipeline.rs index 316750a2..e91b930c 100644 --- a/tools/netflow-db/src/pipeline.rs +++ b/tools/netflow-db/src/pipeline.rs @@ -2576,9 +2576,7 @@ fn process_nfcapd_tree_day( .collect::>() }); let decode_pool = build_nfcapd_decode_pool()?; - let mut next = start; - - while next < end { + let plan_batch = |next: i64| -> Result { let batch_starts = nfcapd_batch_starts( next, end, @@ -2588,7 +2586,7 @@ fn process_nfcapd_tree_day( &tree.member_bounds, tree.extend_gaps_to_window, )?; - next = batch_starts + let following = batch_starts .last() .copied() .map(|last| next_local_five_minute_start(last, &timezone)) @@ -2610,8 +2608,14 @@ fn process_nfcapd_tree_day( .collect::>() .into_iter() .collect::>(); - verify_nfdump_revision(first_pipeline)?; - let decoded = decode_pool.install(|| { + Ok(NfcapdTreeBatch { + following, + prepared, + requests, + }) + }; + let decode_batch = |requests: &[NfcapdTreeRequest]| { + decode_pool.install(|| { requests .par_iter() .map(|((member, bucket_start), path)| { @@ -2634,49 +2638,110 @@ fn process_nfcapd_tree_day( Ok::<_, PipelineError>(((member.clone(), *bucket_start), buckets)) }) .collect::, _>>() - })?; + }) + }; + + if start >= end { + return Ok(()); + } + let mut batch = plan_batch(start)?; + verify_nfdump_revision(first_pipeline)?; + let mut decoded = decode_batch(&batch.requests)?; + verify_nfdump_revision(first_pipeline)?; + // Decoding the next batch overlaps publishing the current one, so decode + // workers keep running while MAAD and SQLite writes for this batch proceed. + // At most two decoded batches are held at once. + loop { + let upcoming = if batch.following < end { + verify_nfdump_revision(first_pipeline)?; + Some(plan_batch(batch.following)?) + } else { + None + }; + let (published, upcoming_decoded) = std::thread::scope(|scope| { + let decoding = upcoming + .as_ref() + .map(|upcoming| scope.spawn(|| decode_batch(&upcoming.requests))); + let published = publish_nfcapd_tree_batch( + &batch.prepared, + &decoded, + sinks, + pending, + aggregates, + &timezone, + ); + let upcoming_decoded = decoding.map(|handle| { + handle + .join() + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) + }); + (published, upcoming_decoded) + }); + published?; + let Some(upcoming) = upcoming else { + return Ok(()); + }; + decoded = upcoming_decoded.expect("upcoming batch was decoded")?; verify_nfdump_revision(first_pipeline)?; + batch = upcoming; + } +} - for (selection_index, sink_index) in pending.iter().copied().enumerate() { - let aggregate = aggregates[sink_index] - .as_mut() - .expect("pending output has aggregate state"); - for timestamp in &prepared { - for job in ×tamp.jobs { - let member_buckets = job - .present - .iter() - .map(|(member, _)| { - &decoded[&(member.clone(), timestamp.bucket_start)][selection_index] - }) - .collect::>(); - let logical = logical_source_bucket( - &job.source_id, - timestamp.bucket_start, - job.expected_units, - &member_buckets, - )?; - aggregate.reject_persisted_siblings( - sinks[sink_index].connection, - &logical, - &timezone, - )?; - publish_nfcapd_bucket( - sinks[sink_index].connection, - &logical, - &job.owners, - &job.absences, - &job.evidence, - true, - sinks[sink_index].pipeline.run_maad, - )?; - aggregate.include(&logical, &timezone)?; - sinks[sink_index].report.rollup_buckets += aggregate.flush_complete( - sinks[sink_index].connection, - sinks[sink_index].pipeline.run_maad, - )?; - sinks[sink_index].report.five_minute_buckets += 1; - } +type NfcapdTreeRequest = ((String, i64), PathBuf); + +struct NfcapdTreeBatch { + following: i64, + prepared: Vec, + requests: Vec, +} + +fn publish_nfcapd_tree_batch( + prepared: &[PreparedTreeTimestamp], + decoded: &BTreeMap<(String, i64), Vec>, + sinks: &mut [ProductSink<'_>], + pending: &[usize], + aggregates: &mut [Option], + timezone: &str, +) -> Result<(), PipelineError> { + for (selection_index, sink_index) in pending.iter().copied().enumerate() { + let aggregate = aggregates[sink_index] + .as_mut() + .expect("pending output has aggregate state"); + for timestamp in prepared { + for job in ×tamp.jobs { + let member_buckets = job + .present + .iter() + .map(|(member, _)| { + &decoded[&(member.clone(), timestamp.bucket_start)][selection_index] + }) + .collect::>(); + let logical = logical_source_bucket( + &job.source_id, + timestamp.bucket_start, + job.expected_units, + &member_buckets, + )?; + aggregate.reject_persisted_siblings( + sinks[sink_index].connection, + &logical, + timezone, + )?; + publish_nfcapd_bucket( + sinks[sink_index].connection, + &logical, + &job.owners, + &job.absences, + &job.evidence, + true, + sinks[sink_index].pipeline.run_maad, + )?; + aggregate.include(&logical, timezone)?; + sinks[sink_index].report.rollup_buckets += aggregate.flush_complete( + sinks[sink_index].connection, + sinks[sink_index].pipeline.run_maad, + )?; + sinks[sink_index].report.five_minute_buckets += 1; } } } From d2d4a5b126aa27a56c7ec55bd8f8c4f78d9b174d Mon Sep 17 00:00:00 2001 From: flamboh Date: Fri, 25 Sep 2026 02:32:17 -0700 Subject: [PATCH 5/8] fix(netflow-db): surface later-batch errors only after the current batch publishes Planning, revision checks and decoding for nfcapd batch N+1 still overlap publishing batch N, but their errors are reported only after batch N is published, restoring publish-then-plan error order. The overlap loop is a small helper with tests for a corrupt capture in the second of several 13-request batches and for a planning failure. The inline comment is removed. --- tools/netflow-db/src/pipeline.rs | 130 ++++++++++++++++++++++++++----- 1 file changed, 109 insertions(+), 21 deletions(-) diff --git a/tools/netflow-db/src/pipeline.rs b/tools/netflow-db/src/pipeline.rs index e91b930c..523bc97a 100644 --- a/tools/netflow-db/src/pipeline.rs +++ b/tools/netflow-db/src/pipeline.rs @@ -2644,32 +2644,54 @@ fn process_nfcapd_tree_day( if start >= end { return Ok(()); } - let mut batch = plan_batch(start)?; + let first = plan_batch(start)?; verify_nfdump_revision(first_pipeline)?; - let mut decoded = decode_batch(&batch.requests)?; - verify_nfdump_revision(first_pipeline)?; - // Decoding the next batch overlaps publishing the current one, so decode - // workers keep running while MAAD and SQLite writes for this batch proceed. - // At most two decoded batches are held at once. - loop { - let upcoming = if batch.following < end { + publish_overlapped( + first, + |batch| { + (batch.following < end).then(|| { + verify_nfdump_revision(first_pipeline)?; + plan_batch(batch.following) + }) + }, + |batch| { + let decoded = decode_batch(&batch.requests)?; verify_nfdump_revision(first_pipeline)?; - Some(plan_batch(batch.following)?) - } else { - None - }; - let (published, upcoming_decoded) = std::thread::scope(|scope| { - let decoding = upcoming - .as_ref() - .map(|upcoming| scope.spawn(|| decode_batch(&upcoming.requests))); - let published = publish_nfcapd_tree_batch( + Ok(decoded) + }, + |batch, decoded| { + publish_nfcapd_tree_batch( &batch.prepared, - &decoded, + decoded, sinks, pending, aggregates, &timezone, - ); + ) + }, + ) +} + +/// Publish batches in order while the following batch decodes on a scoped thread. +/// +/// Planning, revision checks and decoding for batch N+1 report their errors only after +/// batch N is published, and at most two decoded batches are held at once. +fn publish_overlapped( + first: B, + next: impl Fn(&B) -> Option>, + decode: impl Fn(&B) -> Result + Sync, + mut publish: impl FnMut(&B, &D) -> Result<(), PipelineError>, +) -> Result<(), PipelineError> { + let mut batch = first; + let mut decoded = decode(&batch)?; + loop { + let upcoming = next(&batch); + let (published, upcoming_decoded) = std::thread::scope(|scope| { + let decoding = match &upcoming { + Some(Ok(upcoming)) => Some(scope.spawn(|| decode(upcoming))), + _ => None, + }; + let published = publish(&batch, &decoded); let upcoming_decoded = decoding.map(|handle| { handle .join() @@ -2681,8 +2703,8 @@ fn process_nfcapd_tree_day( let Some(upcoming) = upcoming else { return Ok(()); }; - decoded = upcoming_decoded.expect("upcoming batch was decoded")?; - verify_nfdump_revision(first_pipeline)?; + let upcoming = upcoming?; + decoded = upcoming_decoded.expect("a planned batch is decoded")?; batch = upcoming; } } @@ -3658,6 +3680,72 @@ mod tests { fs::set_permissions(executable, fs::Permissions::from_mode(0o755)).unwrap(); } + #[test] + fn overlapped_publishing_finishes_a_batch_before_a_later_batch_fails() { + let requests_per_batch = NFCAPD_DECODE_BATCH_SIZE + 1; + let batches = 3; + let published = std::sync::Mutex::new(Vec::new()); + let decoded = std::sync::Mutex::new(Vec::new()); + + let result = publish_overlapped( + 0_usize, + |&batch| (batch + 1 < batches).then_some(Ok(batch + 1)), + |&batch| { + let requests = (0..requests_per_batch) + .map(|request| batch * requests_per_batch + request) + .collect::>(); + decoded.lock().unwrap().push(batch); + if batch == 1 { + return Err(PipelineError::InvalidConfig(format!( + "corrupt capture in batch {batch}" + ))); + } + Ok(requests) + }, + |&batch, requests| { + assert_eq!(requests.len(), requests_per_batch); + published.lock().unwrap().push(batch); + Ok(()) + }, + ); + + assert!( + result + .unwrap_err() + .to_string() + .contains("corrupt capture in batch 1") + ); + assert_eq!(*published.lock().unwrap(), [0]); + assert_eq!(*decoded.lock().unwrap(), [0, 1]); + } + + #[test] + fn overlapped_publishing_reports_a_planning_error_after_the_current_batch() { + let published = std::sync::Mutex::new(Vec::new()); + + let result = publish_overlapped( + 0_usize, + |_| { + Some(Err(PipelineError::InvalidConfig( + "unplannable batch".into(), + ))) + }, + |&batch| Ok(batch), + |&batch, _| { + published.lock().unwrap().push(batch); + Ok(()) + }, + ); + + assert!( + result + .unwrap_err() + .to_string() + .contains("unplannable batch") + ); + assert_eq!(*published.lock().unwrap(), [0]); + } + fn write_nfcapd_day(root: &Path, member: &str, date: &str) { let mut bucket_start = parse_date_start(date, DEFAULT_TIMEZONE).unwrap(); let end = next_date_start(date, DEFAULT_TIMEZONE).unwrap(); From 1898b006781491a18e546dfacb652cb9ea96ac78 Mon Sep 17 00:00:00 2001 From: flamboh Date: Sat, 26 Sep 2026 14:46:41 -0700 Subject: [PATCH 6/8] fix(netflow-db): drop and log zero-packet flows before aggregation --- docs/code/pipeline-contract.md | 5 + docs/user/querying.md | 3 +- tools/netflow-db/src/domain.rs | 24 ++++ tools/netflow-db/src/ingest.rs | 128 ++++++++++++++++++- tools/netflow-db/src/nfdump.rs | 133 +++++++++++++++++--- tools/netflow-db/src/normalize.rs | 6 +- tools/netflow-db/src/publish.rs | 30 ++++- tools/netflow-db/tests/pipeline_cli_help.rs | 99 +++++++++++++++ 8 files changed, 403 insertions(+), 25 deletions(-) diff --git a/docs/code/pipeline-contract.md b/docs/code/pipeline-contract.md index 71db2bac..cd2dc5ea 100644 --- a/docs/code/pipeline-contract.md +++ b/docs/code/pipeline-contract.md @@ -90,6 +90,11 @@ dimensions only. Prefix validity always uses distinct-address counts. A weighted addresses whose counter sums to 0 and records the excluded count as `zeroWeightAddrs` in the row metadata. The measure set is part of the result configuration identity. +Native and CSV ingestion drop flows whose reported packet count is 0 before any statistic +accumulates, so they add no traffic, protocols, ports, or addresses. The bucket keeps its observed +coverage, and publishing logs the dropped flow count per bucket. A CSV row without a packet count +has an unknown count rather than a reported zero, so it still contributes. + ## Coordinated subset runs Repeat `--dataset` for two or more registry entries to build coordinated subset products. Each entry diff --git a/docs/user/querying.md b/docs/user/querying.md index ea999233..c104c492 100644 --- a/docs/user/querying.md +++ b/docs/user/querying.md @@ -129,7 +129,8 @@ side, and `measure`: | `bytes` | Bytes summed over the bucket and scope | `structure`, `dimension` | Weighted measures keep the address-count prefix tests and weight only the moments and the entropy. -Addresses with zero summed packets or bytes are left out of that measure. Their count is stored in +The pipeline drops flows that report 0 packets before aggregation. Addresses with zero summed +packets or bytes, such as CSV rows without a packet count, are left out of that measure. Their count is stored in `metadata_json` as `zeroWeightAddrs`. Always filter on `measure`. Otherwise results for different measures mix: diff --git a/tools/netflow-db/src/domain.rs b/tools/netflow-db/src/domain.rs index 46c5d9c1..813e988c 100644 --- a/tools/netflow-db/src/domain.rs +++ b/tools/netflow-db/src/domain.rs @@ -200,6 +200,7 @@ pub struct FlowObservation { pub protocol: u8, pub packets: i64, pub bytes: i64, + pub packets_reported: bool, pub src_tos: u8, pub time_received_ms: Option, pub time_end_ms: Option, @@ -234,6 +235,7 @@ impl FlowObservation { protocol, packets, bytes, + packets_reported: true, src_tos, time_received_ms: None, time_end_ms: None, @@ -254,6 +256,12 @@ impl FlowObservation { IpVersion::of(self.src_ip) } + #[must_use] + pub const fn with_unreported_packets(mut self) -> Self { + self.packets_reported = false; + self + } + #[must_use] pub const fn with_locality( mut self, @@ -1074,6 +1082,8 @@ pub struct CanonicalBucket { pub addresses: Vec, pub ports: Vec, pub five_minute_starts: BTreeSet, + #[serde(default)] + pub dropped_zero_packet_flows: u64, } fn serialize_fixed_bitset(ports: &FixedBitSet, serializer: S) -> Result @@ -1254,6 +1264,7 @@ pub struct StatisticalBucket { addresses: BTreeMap<(Scope, AddressSide), AddressTotals>, ports: BTreeMap<(Scope, PortSide), FixedBitSet>, five_minute_starts: BTreeSet, + dropped_zero_packet_flows: u64, } impl StatisticalBucket { @@ -1274,6 +1285,7 @@ impl StatisticalBucket { addresses: BTreeMap::new(), ports: BTreeMap::new(), five_minute_starts, + dropped_zero_packet_flows: 0, } } @@ -1344,6 +1356,9 @@ impl StatisticalBucket { } self.five_minute_starts .extend(child.five_minute_starts.iter().copied()); + self.dropped_zero_packet_flows = self + .dropped_zero_packet_flows + .saturating_add(child.dropped_zero_packet_flows); self.coverage.include(child.coverage)?; Ok(()) } @@ -1395,6 +1410,7 @@ impl StatisticalBucket { }) .collect(), five_minute_starts: self.five_minute_starts.clone(), + dropped_zero_packet_flows: self.dropped_zero_packet_flows, } } @@ -1410,6 +1426,7 @@ impl StatisticalBucket { addresses, ports, five_minute_starts, + dropped_zero_packet_flows, } = self; CanonicalBucket { @@ -1443,6 +1460,7 @@ impl StatisticalBucket { }) .collect(), five_minute_starts, + dropped_zero_packet_flows, } } @@ -1459,6 +1477,12 @@ impl StatisticalBucket { let (source, destination) = observation .locality .ok_or(DomainError::UnclassifiedObservation)?; + if observation.packets_reported && observation.packets == 0 { + self.dropped_zero_packet_flows = self + .dropped_zero_packet_flows + .saturating_add(observation.flow_count.unsigned_abs()); + return Ok(()); + } let scopes = scopes_for(observation.ip_version(), source, destination); let traffic = AddressTraffic::new( observation.packets.unsigned_abs(), diff --git a/tools/netflow-db/src/ingest.rs b/tools/netflow-db/src/ingest.rs index 9f529e88..6b79adbf 100644 --- a/tools/netflow-db/src/ingest.rs +++ b/tools/netflow-db/src/ingest.rs @@ -713,6 +713,7 @@ CREATE TABLE accepted_rows ( protocol INTEGER NOT NULL, packets INTEGER NOT NULL, bytes INTEGER NOT NULL, + packets_reported INTEGER NOT NULL CHECK (packets_reported IN (0, 1)), src_tos INTEGER NOT NULL, time_received_ms INTEGER, time_end_ms INTEGER, @@ -777,7 +778,7 @@ impl UnsortedCsvScanState { "SELECT src_ip, dst_ip, protocol, packets, bytes, src_tos, \ time_received_ms, time_end_ms, time_start_ms, src_port, dst_port, \ dst_tos, duration_ms, min_ttl, max_ttl, flow_count, src_internal, \ - dst_internal \ + dst_internal, packets_reported \ FROM accepted_rows \ WHERE source_id = ?1 AND bucket_start = ?2 \ ORDER BY id", @@ -916,10 +917,11 @@ impl CsvScanAccumulator for UnsortedCsvScanState { "INSERT INTO accepted_rows (\ source_id, bucket_start, src_ip, dst_ip, protocol, packets, bytes, src_tos, \ time_received_ms, time_end_ms, time_start_ms, src_port, dst_port, dst_tos, \ - duration_ms, min_ttl, max_ttl, flow_count, src_internal, dst_internal\ + duration_ms, min_ttl, max_ttl, flow_count, src_internal, dst_internal, \ + packets_reported\ ) VALUES (\ ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, \ - ?19, ?20\ + ?19, ?20, ?21\ )", params![ source_id, @@ -942,6 +944,7 @@ impl CsvScanAccumulator for UnsortedCsvScanState { observation.flow_count, source == EndpointLocality::Internal, destination == EndpointLocality::Internal, + observation.packets_reported, ], )?; Ok(()) @@ -994,6 +997,7 @@ fn staged_observation(row: &rusqlite::Row<'_>) -> Result std::io::Result { #[cfg(test)] mod tests { - use std::{collections::BTreeMap, fs, io::Write, sync::Arc}; + use std::{collections::BTreeMap, fs, io::Write, net::Ipv4Addr, sync::Arc}; use flate2::{Compression, write::GzEncoder}; use serde_json::json; @@ -1591,7 +1595,9 @@ mod tests { use super::*; use crate::config::{CsvSourceConfig, InputOrder}; use crate::coverage::CoverageState; - use crate::domain::{IpVersion, Locality, Scope}; + use crate::domain::{ + AddressSide, AddressTraffic, CanonicalBucket, IpVersion, Locality, Scope, TrafficMetrics, + }; use crate::locality::{LocalityRuleConfig, LocalityRules}; fn tos_rules() -> Arc { @@ -1825,6 +1831,118 @@ mod tests { } } + #[test] + fn csv_scans_drop_zero_packet_flows_but_keep_observed_coverage() { + let directory = tempdir().unwrap(); + let input = directory.path().join("zero-packets.csv"); + fs::write( + &input, + concat!( + "received,src,dst,packets,bytes,protocol\n", + "0,192.0.2.1,198.51.100.1,2,128,TCP\n", + "10,192.0.2.1,198.51.100.9,0,0,TCP\n", + "300,192.0.2.2,198.51.100.3,0,40,UDP\n", + ), + ) + .unwrap(); + let selection = FlowSelection::from_payload(Some(&json!({ + "version": 1, + "kind": "flows", + "ip_prefix": "192.0.2.0/24", + }))) + .unwrap(); + let scope = Scope::new(IpVersion::V4, Locality::All, Locality::All); + let source = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)); + let addresses = |bucket: &CanonicalBucket, side: AddressSide| { + bucket + .addresses + .iter() + .find(|entry| entry.scope == scope && entry.address_side == side) + .unwrap() + .addresses + .clone() + }; + + for input_order in [InputOrder::TimestampAscending, InputOrder::Unsorted] { + let mut config = config(); + config.input_order = input_order; + let mut buckets = Vec::new(); + scan_csv(&input, &config, &selection, |ready| { + buckets.push(ready.bucket); + Ok::<_, ()>(()) + }) + .unwrap(); + + assert_eq!(buckets.len(), 2); + for bucket in &buckets { + assert_eq!(bucket.dropped_zero_packet_flows, 1); + assert_eq!(bucket.coverage.state(), CoverageState::Complete); + } + let metrics = |bucket: &CanonicalBucket| { + bucket + .traffic + .iter() + .find(|entry| entry.scope == scope) + .unwrap() + .metrics + .clone() + }; + assert_eq!(metrics(&buckets[0]).flows, 1); + assert_eq!(metrics(&buckets[0]).packets, 2); + assert_eq!(metrics(&buckets[0]).bytes, 128); + let sources = addresses(&buckets[0], AddressSide::Source); + assert_eq!(sources.len(), 1); + assert_eq!(sources.get(&source), Some(AddressTraffic::new(2, 128))); + let destinations = addresses(&buckets[0], AddressSide::Destination); + assert_eq!(destinations.len(), 1); + assert!( + destinations + .get(&IpAddr::V4(Ipv4Addr::new(198, 51, 100, 1))) + .is_some() + ); + assert_eq!(metrics(&buckets[1]), TrafficMetrics::default()); + assert!(addresses(&buckets[1], AddressSide::Source).is_empty()); + assert!(addresses(&buckets[1], AddressSide::Destination).is_empty()); + } + + let unreported = directory.path().join("no-packets.csv"); + fs::write( + &unreported, + concat!( + "received,src,dst,bytes,protocol\n", + "0,192.0.2.1,198.51.100.1,128,TCP\n", + "10,192.0.2.1,198.51.100.9,,TCP\n", + ), + ) + .unwrap(); + for input_order in [InputOrder::TimestampAscending, InputOrder::Unsorted] { + let mut config = config(); + config.columns.remove("packets"); + config.input_order = input_order; + let mut buckets = Vec::new(); + scan_csv(&unreported, &config, &selection, |ready| { + buckets.push(ready.bucket); + Ok::<_, ()>(()) + }) + .unwrap(); + + assert_eq!(buckets.len(), 1); + assert_eq!(buckets[0].dropped_zero_packet_flows, 0); + assert_eq!( + buckets[0] + .traffic + .iter() + .find(|entry| entry.scope == scope) + .unwrap() + .metrics + .flows, + 2, + "a missing packet count is unknown, not a reported zero" + ); + assert_eq!(addresses(&buckets[0], AddressSide::Destination).len(), 2); + } + } + #[test] fn scan_reads_selected_members_from_compressed_tar() { let directory = tempdir().unwrap(); diff --git a/tools/netflow-db/src/nfdump.rs b/tools/netflow-db/src/nfdump.rs index 57a8bc5a..184457d4 100644 --- a/tools/netflow-db/src/nfdump.rs +++ b/tools/netflow-db/src/nfdump.rs @@ -396,7 +396,7 @@ pub(crate) fn reduce_to_bucket( selection: &FlowSelection, ) -> Result { match reduce_stream(&mut input, selection, None) { - Ok(scopes) => Ok(finish_bucket(scopes, key)), + Ok(reduced) => Ok(finish_bucket(reduced, key)), Err(error) => { drain_to_eof(&mut input); Err(error) @@ -411,7 +411,7 @@ pub(crate) fn reduce_to_bucket_with_active_sources( active_sources: &AddressSet, ) -> Result { match reduce_stream(&mut input, selection, Some(active_sources)) { - Ok(scopes) => Ok(finish_bucket(scopes, key)), + Ok(reduced) => Ok(finish_bucket(reduced, key)), Err(error) => { drain_to_eof(&mut input); Err(error) @@ -425,9 +425,9 @@ pub(crate) fn reduce_to_buckets_with_active_sources( selections_and_active_sources: &[(FlowSelection, Arc)], ) -> Result, NfdumpError> { match reduce_stream_for_active_sources(&mut input, selections_and_active_sources) { - Ok(scopes) => Ok(scopes + Ok(reduced) => Ok(reduced .into_iter() - .map(|scopes| finish_bucket(scopes, key.clone())) + .map(|reduced| finish_bucket(reduced, key.clone())) .collect()), Err(error) => { drain_to_eof(&mut input); @@ -445,6 +445,9 @@ pub(crate) fn reduce_to_daily_source_activities( .map(|_| HashMap::::new()) .collect::>(); let result = visit_stream(&mut input, |flow, _, _| { + if flow.packets == 0 { + return Ok(()); + } for (selection, activity) in selections.iter().zip(&mut activities) { if classify_selected(flow, selection).is_none() { continue; @@ -470,14 +473,14 @@ pub(crate) fn reduce_to_daily_source_activities( fn reduce_stream_for_active_sources( input: &mut impl Read, selections_and_active_sources: &[(FlowSelection, Arc)], -) -> Result, NfdumpError> { +) -> Result, NfdumpError> { validate_daily_active_selection_pairs(selections_and_active_sources)?; - let mut scopes = (0..selections_and_active_sources.len()) - .map(|_| std::array::from_fn(|_| ScopeAccumulator::default())) + let mut reduced = (0..selections_and_active_sources.len()) + .map(|_| ReducedScopes::default()) .collect::>(); visit_stream(input, |flow, block_index, record_ordinal| { - for ((selection, active_sources), scopes) in - selections_and_active_sources.iter().zip(&mut scopes) + for ((selection, active_sources), reduced) in + selections_and_active_sources.iter().zip(&mut reduced) { let Some(locality) = classify_selected(flow, selection) else { continue; @@ -485,11 +488,11 @@ fn reduce_stream_for_active_sources( if !active_sources.contains(&flow.source_address) { continue; } - add_flow_to_scopes(scopes, flow, locality, block_index, record_ordinal)?; + reduced.add(flow, locality, block_index, record_ordinal)?; } Ok(()) })?; - Ok(scopes) + Ok(reduced) } fn validate_daily_active_selections(selections: &[FlowSelection]) -> Result<(), NfdumpError> { @@ -540,7 +543,7 @@ fn reduce_stream( input: &mut R, selection: &FlowSelection, active_sources: Option<&AddressSet>, -) -> Result<[ScopeAccumulator; 10], NfdumpError> { +) -> Result { if selection.selects_daily_active_sources() && active_sources.is_none() { return Err(NfdumpError::new( Phase::Aggregate, @@ -548,7 +551,7 @@ fn reduce_stream( ErrorReason::MissingDailyActiveSources, )); } - let mut scopes = std::array::from_fn(|_| ScopeAccumulator::default()); + let mut reduced = ReducedScopes::default(); visit_stream(input, |flow, block_index, record_ordinal| { let Some(locality) = classify_selected(flow, selection) else { return Ok(()); @@ -556,9 +559,47 @@ fn reduce_stream( if active_sources.is_some_and(|sources| !sources.contains(&flow.source_address)) { return Ok(()); } - add_flow_to_scopes(&mut scopes, flow, locality, block_index, record_ordinal) + reduced.add(flow, locality, block_index, record_ordinal) })?; - Ok(scopes) + Ok(reduced) +} + +struct ReducedScopes { + scopes: [ScopeAccumulator; 10], + dropped_zero_packet_flows: u64, +} + +impl Default for ReducedScopes { + fn default() -> Self { + Self { + scopes: std::array::from_fn(|_| ScopeAccumulator::default()), + dropped_zero_packet_flows: 0, + } + } +} + +impl ReducedScopes { + fn add( + &mut self, + flow: &Flow, + locality: (EndpointLocality, EndpointLocality), + block_index: u64, + record_ordinal: u64, + ) -> Result<(), NfdumpError> { + if flow.packets == 0 { + self.dropped_zero_packet_flows = self + .dropped_zero_packet_flows + .saturating_add(flow.flow_count.unsigned_abs()); + return Ok(()); + } + add_flow_to_scopes( + &mut self.scopes, + flow, + locality, + block_index, + record_ordinal, + ) + } } fn add_flow_to_scopes( @@ -1037,12 +1078,12 @@ const fn record_field_at(offset: usize) -> Field { } } -fn finish_bucket(scopes: [ScopeAccumulator; 10], key: BucketKey) -> CanonicalBucket { +fn finish_bucket(reduced: ReducedScopes, key: BucketKey) -> CanonicalBucket { let mut traffic = Vec::with_capacity(10); let mut protocols = Vec::with_capacity(10); let mut addresses = Vec::with_capacity(20); let mut ports = Vec::with_capacity(20); - for (index, accumulator) in scopes.into_iter().enumerate() { + for (index, accumulator) in reduced.scopes.into_iter().enumerate() { let scope = Scope::new( if index < 5 { IpVersion::V4 @@ -1071,6 +1112,7 @@ fn finish_bucket(scopes: [ScopeAccumulator; 10], key: BucketKey) -> CanonicalBuc addresses, ports, five_minute_starts, + dropped_zero_packet_flows: reduced.dropped_zero_packet_flows, } } @@ -1258,6 +1300,63 @@ mod tests { assert!(bucket.ports[1].ports.contains(443)); } + #[test] + fn zero_packet_flows_are_dropped_and_counted_while_the_bucket_stays_observed() { + let mut zero_packets = base_record(); + zero_packets[16..20].copy_from_slice(&[198, 51, 100, 9]); + zero_packets[32..40].copy_from_slice(&0_u64.to_le_bytes()); + zero_packets[48..56].copy_from_slice(&3_u64.to_le_bytes()); + + let zero_only = reduce_to_bucket( + Cursor::new(stream(&[zero_packets])), + key(), + &FlowSelection::default(), + ) + .unwrap(); + assert_eq!(zero_only.dropped_zero_packet_flows, 3); + assert_eq!(zero_only.coverage, BucketCoverage::complete_unit()); + assert_eq!(zero_only.five_minute_starts, [1_700_000_000].into()); + assert_eq!(zero_only.traffic.len(), 10); + assert!( + zero_only + .traffic + .iter() + .all(|traffic| traffic.metrics == TrafficMetrics::default()) + ); + assert!( + zero_only + .addresses + .iter() + .all(|addresses| addresses.addresses.is_empty()) + ); + + let mixed = reduce_to_bucket( + Cursor::new(stream(&[base_record(), zero_packets])), + key(), + &FlowSelection::default(), + ) + .unwrap(); + assert_eq!(mixed.dropped_zero_packet_flows, 3); + assert_eq!(mixed.traffic[0].metrics.flows, 1); + assert_eq!(mixed.traffic[0].metrics.packets, 2); + assert_eq!(mixed.traffic[0].metrics.bytes, 128); + let source = IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)); + assert_eq!( + address_keys(&mixed.addresses[1].addresses), + [source].into_iter().collect::() + ); + assert_eq!( + mixed.addresses[1].addresses.get(&source), + Some(AddressTraffic::new(2, 128)) + ); + assert_eq!( + address_keys(&mixed.addresses[0].addresses), + [IpAddr::V4(Ipv4Addr::new(198, 51, 100, 2))] + .into_iter() + .collect::() + ); + } + #[test] fn icmp_requires_and_retains_a_zero_destination_port() { let mut invalid = ONE_V4_TEST_STREAM.to_vec(); diff --git a/tools/netflow-db/src/normalize.rs b/tools/netflow-db/src/normalize.rs index c5de9c5d..f9cf6f9f 100644 --- a/tools/netflow-db/src/normalize.rs +++ b/tools/netflow-db/src/normalize.rs @@ -102,7 +102,8 @@ pub fn normalize_csv_values( let src_ip = parse_ip(required(value(src_column)?, src_column)?)?; let dst_ip = parse_ip(required(value(dst_column)?, dst_column)?)?; let protocol = extract_protocol(optional("protocol")?, config)?; - let packets = extract_nonnegative(optional("packets")?, "packets", 0)?; + let raw_packets = optional("packets")?; + let packets = extract_nonnegative(raw_packets, "packets", 0)?; let bytes = extract_nonnegative(optional("bytes")?, "bytes", 0)?; let src_tos = extract_u8(optional("src_tos")?, "src_tos", 0)?; let dst_tos = extract_u8(optional("dst_tos")?, "dst_tos", 0)?; @@ -123,6 +124,9 @@ pub fn normalize_csv_values( .with_ports(src_port, dst_port) .with_measurements(duration_ms, min_ttl, max_ttl)? .with_flow_count(flow_count)?; + if raw_packets.is_none() { + observation = observation.with_unreported_packets(); + } observation.time_received_ms = timestamps.get("time_received").copied(); observation.time_end_ms = timestamps.get("time_end").copied(); observation.time_start_ms = timestamps.get("time_start").copied(); diff --git a/tools/netflow-db/src/publish.rs b/tools/netflow-db/src/publish.rs index dc902b16..41ad5ea8 100644 --- a/tools/netflow-db/src/publish.rs +++ b/tools/netflow-db/src/publish.rs @@ -14,7 +14,7 @@ use thiserror::Error; use crate::{ domain::{ AddressSetRow, AddressTraffic, BucketKey, CanonicalBucket, CanonicalRows, DomainError, - IpVersion, MaadMeasure, + Granularity, IpVersion, MaadMeasure, }, maad, storage::{ @@ -72,6 +72,7 @@ pub struct WriteBucketsProfile { pub(crate) maad_address_sets: u64, pub(crate) maad_addresses: u64, pub(crate) maad_zero_weight_addresses: u64, + pub(crate) dropped_zero_packet_flows: u64, pub(crate) address_structure_rows: u64, pub(crate) address_structure_json_bytes: u64, } @@ -151,6 +152,18 @@ pub(crate) fn write_buckets_profiled( .collect::>(); insert_bucket_coverage_rows(connection, &coverage_rows)?; for bucket in buckets { + if bucket.key.granularity == Granularity::FiveMinutes + && bucket.dropped_zero_packet_flows > 0 + { + profile.dropped_zero_packet_flows += bucket.dropped_zero_packet_flows; + tracing::info!( + source_id = %bucket.key.source_id, + granularity = bucket.key.granularity.as_str(), + bucket_start = bucket.key.bucket_start, + dropped_zero_packet_flows = bucket.dropped_zero_packet_flows, + "dropped zero-packet flows before aggregation" + ); + } let canonical_rows_started = Instant::now(); let rows = bucket.rows(); profile.canonical_rows_elapsed += canonical_rows_started.elapsed(); @@ -514,6 +527,20 @@ mod tests { .with_locality(EndpointLocality::Internal, EndpointLocality::External), ) .unwrap(); + builder + .add( + FlowObservation::new( + IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)), + IpAddr::V4(Ipv4Addr::new(198, 51, 100, 9)), + 6, + 0, + 0, + 0, + ) + .unwrap() + .with_locality(EndpointLocality::Internal, EndpointLocality::External), + ) + .unwrap(); let profile = write_buckets_profiled(&connection, &[builder.finish()], true).unwrap(); @@ -544,6 +571,7 @@ mod tests { assert_eq!(profile.maad_address_sets, 20); assert_eq!(profile.address_structure_rows, 140); assert_eq!(profile.maad_zero_weight_addresses, 0); + assert_eq!(profile.dropped_zero_packet_flows, 1); assert!(profile.address_structure_json_bytes > 0); assert!(profile.total_elapsed >= profile.other_elapsed()); } diff --git a/tools/netflow-db/tests/pipeline_cli_help.rs b/tools/netflow-db/tests/pipeline_cli_help.rs index f03de27e..ecb4f2c2 100644 --- a/tools/netflow-db/tests/pipeline_cli_help.rs +++ b/tools/netflow-db/tests/pipeline_cli_help.rs @@ -421,3 +421,102 @@ fn config_pipeline_resolves_locality_files_from_config_directory_and_marks_datas .unwrap(); assert_eq!(has_locality, 1); } + +#[test] +fn csv_pipeline_drops_and_logs_zero_packet_flows_but_keeps_coverage() { + let temporary = tempdir().unwrap(); + let csv = temporary.path().join("flows.csv"); + let mapping = temporary.path().join("mapping.json"); + let database = temporary.path().join("zero-packets.sqlite"); + fs::write( + &csv, + concat!( + "received,src,dst,packets,bytes\n", + "0,192.0.2.1,198.51.100.1,4,400\n", + "10,192.0.2.1,198.51.100.9,0,0\n", + "300,192.0.2.3,198.51.100.3,0,40\n", + ), + ) + .unwrap(); + fs::write( + &mapping, + serde_json::to_vec(&serde_json::json!({ + "timestamp_format": "unix", + "timestamp_timezone": "UTC", + "columns": { + "time_received": "received", + "src_ip": "src", + "dst_ip": "dst", + "packets": "packets", + "bytes": "bytes" + }, + "source_id": {"value": "edge"} + })) + .unwrap(), + ) + .unwrap(); + let config = temporary.path().join("csv-pipeline.json"); + fs::write( + &config, + serde_json::to_vec(&serde_json::json!({ + "database_path": database, + "timezone": "UTC", + "run_maad": false, + "inputs": [{ + "input_kind": "csv", + "path": csv, + "mapping_path": mapping + }] + })) + .unwrap(), + ) + .unwrap(); + + let output = Command::new(env!("CARGO_BIN_EXE_netflow-db")) + .args([ + "pipeline", + "--config", + config.to_str().unwrap(), + "--no-maad", + ]) + .env("RUST_LOG", "info") + .env("NO_COLOR", "1") + .output() + .unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(output.status.success(), "stderr={stderr}"); + assert_eq!( + stderr + .lines() + .filter(|line| line.contains("dropped zero-packet flows") + && line.contains("dropped_zero_packet_flows=1")) + .count(), + 2, + "stderr={stderr}" + ); + + let connection = Connection::open(&database).unwrap(); + let rows = connection + .prepare( + "SELECT t.bucket_start, t.flows, t.packets, c.observed_units + FROM traffic_stats t JOIN bucket_coverage c + ON c.source_id = t.source_id AND c.granularity = t.granularity + AND c.bucket_start = t.bucket_start + WHERE t.granularity = '5m' AND t.ip_version = 4 + AND t.src_locality = 'all' AND t.dst_locality = 'all' + ORDER BY t.bucket_start", + ) + .unwrap() + .query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + )) + }) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(rows, [(0, 1, 4, 1), (300, 0, 0, 1)]); +} From f32aba1f1f0d234676d64632adca44ba7be8981f Mon Sep 17 00:00:00 2001 From: flamboh Date: Sat, 26 Sep 2026 14:47:06 -0700 Subject: [PATCH 7/8] fix(netflow-db): verify MAAD measure presence per scope key --- tools/netflow-db/src/verify.rs | 108 ++++++++++++++++++++++++--------- 1 file changed, 81 insertions(+), 27 deletions(-) diff --git a/tools/netflow-db/src/verify.rs b/tools/netflow-db/src/verify.rs index a2b7bb55..a12abcfd 100644 --- a/tools/netflow-db/src/verify.rs +++ b/tools/netflow-db/src/verify.rs @@ -512,30 +512,58 @@ fn ipv4_literal(value: &str) -> Option<&str> { /// Every scope must carry a structure row for each MAAD measure. fn assert_maad_measures_present(connection: &Connection) -> Result<(), VerifyError> { - let counts = connection - .prepare( - "SELECT measure, COUNT(*) FROM address_structure_stats - WHERE structure_kind = 'structure' GROUP BY measure", - )? - .query_map([], |row| { - Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?)) - })? - .collect::, _>>()?; - let expected = counts.get("addresses").copied().unwrap_or_default(); - if expected == 0 { + let has_rows: bool = connection.query_row( + "SELECT EXISTS ( + SELECT 1 FROM address_structure_stats WHERE structure_kind = 'structure' + )", + [], + |row| row.get(0), + )?; + if !has_rows { return Err(VerifyError::Incompatible( - "address_structure_stats has no addresses structure rows".into(), + "address_structure_stats has no structure rows".into(), )); } - for measure in ["packets", "bytes"] { - let actual = counts.get(measure).copied().unwrap_or_default(); - if actual != expected { - return Err(VerifyError::Incompatible(format!( - "address_structure_stats has {actual} {measure} structure rows; expected {expected}" - ))); - } + let incomplete = connection + .query_row( + "SELECT source_id, granularity, bucket_start, ip_version, src_locality, + dst_locality, address_side, + SUM(measure = 'addresses'), SUM(measure = 'packets'), SUM(measure = 'bytes') + FROM address_structure_stats + WHERE structure_kind = 'structure' + GROUP BY granularity, bucket_start, source_id, ip_version, src_locality, + dst_locality, address_side + HAVING SUM(measure = 'addresses') = 0 OR SUM(measure = 'packets') = 0 + OR SUM(measure = 'bytes') = 0 + LIMIT 1", + [], + |row| { + let present = [row.get::<_, i64>(7)?, row.get(8)?, row.get(9)?]; + let missing = ["addresses", "packets", "bytes"] + .into_iter() + .zip(present) + .filter(|(_, count)| *count == 0) + .map(|(measure, _)| measure) + .collect::>() + .join(", "); + Ok(format!( + "address_structure_stats is missing {missing} structure rows for \ + source {} {} bucket {} IPv{} {}->{} {}", + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, i64>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + row.get::<_, String>(6)?, + )) + }, + ) + .optional()?; + match incomplete { + Some(message) => Err(VerifyError::Incompatible(message)), + None => Ok(()), } - Ok(()) } fn assert_processed_inputs_complete(connection: &Connection) -> Result<(), VerifyError> { @@ -861,32 +889,32 @@ mod tests { fn maad_data_requires_a_structure_row_for_every_measure() { let connection = Connection::open_in_memory().unwrap(); init_schema(&connection).unwrap(); - let insert = |measure: &str, bucket_start: i64| { + let insert = |measure: &str, bucket_start: i64, address_side: &str| { connection .execute( "INSERT INTO address_structure_stats ( source_id, granularity, bucket_start, bucket_end, ip_version, src_locality, dst_locality, address_side, measure, structure_kind, values_json, metadata_json - ) VALUES ('r1', '5m', ?2, ?2 + 300, 4, 'all', 'all', 'source', ?1, + ) VALUES ('r1', '5m', ?2, ?2 + 300, 4, 'all', 'all', ?3, ?1, 'structure', '[]', '{}')", - params![measure, bucket_start], + params![measure, bucket_start, address_side], ) .unwrap(); }; assert!(assert_maad_measures_present(&connection).is_err()); for measure in ["addresses", "packets", "bytes"] { - insert(measure, 0); + insert(measure, 0, "source"); } assert!(assert_maad_measures_present(&connection).is_ok()); - insert("addresses", 300); - insert("packets", 300); + insert("addresses", 300, "source"); + insert("packets", 300, "source"); let error = assert_maad_measures_present(&connection).unwrap_err(); assert!( error .to_string() - .contains("1 bytes structure rows; expected 2") + .contains("missing bytes structure rows for source r1 5m bucket 300") ); assert!( connection @@ -903,6 +931,32 @@ mod tests { ); } + #[test] + fn maad_measure_presence_is_checked_per_scope_key_not_by_global_counts() { + let connection = Connection::open_in_memory().unwrap(); + init_schema(&connection).unwrap(); + let insert = |measure: &str, bucket_start: i64, address_side: &str| { + connection + .execute( + "INSERT INTO address_structure_stats ( + source_id, granularity, bucket_start, bucket_end, ip_version, + src_locality, dst_locality, address_side, measure, structure_kind, + values_json, metadata_json + ) VALUES ('r1', '5m', ?2, ?2 + 300, 4, 'all', 'all', ?3, ?1, + 'structure', '[]', '{}')", + params![measure, bucket_start, address_side], + ) + .unwrap(); + }; + insert("addresses", 0, "source"); + insert("packets", 0, "source"); + insert("bytes", 300, "source"); + let error = assert_maad_measures_present(&connection) + .unwrap_err() + .to_string(); + assert!(error.contains("missing bytes structure rows for source r1 5m bucket 0")); + } + #[test] fn canonical_database_satisfies_web_query_contract() { let directory = tempfile::tempdir().unwrap(); From da1fe86c996136d7df3ccb81325523159a618144 Mon Sep 17 00:00:00 2001 From: flamboh Date: Sat, 26 Sep 2026 14:47:24 -0700 Subject: [PATCH 8/8] feat(netflow-db): add --maad-workers pipeline option --- docs/user/setup-pipeline.md | 1 + tools/netflow-db/src/main.rs | 35 ++++++++++++++++++++++++ tools/netflow-db/src/publish.rs | 47 ++++++++++++++++++++++++++------- 3 files changed, 74 insertions(+), 9 deletions(-) diff --git a/docs/user/setup-pipeline.md b/docs/user/setup-pipeline.md index 725bb8a1..00dbaa7f 100644 --- a/docs/user/setup-pipeline.md +++ b/docs/user/setup-pipeline.md @@ -213,6 +213,7 @@ On the native path, nfcapd input needs the fork path: set the top-level `"nfdump | `--nfdump` | Names the nfdump executable. | | `--force` | Rewrites selected nfcapd buckets. | | `--no-maad` | Skips the MAAD statistics. | +| `--maad-workers` | Sets MAAD threads (default: CPUs, max 8). | Time limits must align with local-day boundaries. diff --git a/tools/netflow-db/src/main.rs b/tools/netflow-db/src/main.rs index 030e9b26..50e8bfa7 100644 --- a/tools/netflow-db/src/main.rs +++ b/tools/netflow-db/src/main.rs @@ -3,6 +3,7 @@ use std::{ fs::File, io::{self, BufRead, BufReader, Write}, net::{Ipv4Addr, Ipv6Addr}, + num::NonZeroUsize, path::PathBuf, }; @@ -95,6 +96,9 @@ struct PipelineArgs { force: bool, #[arg(long)] no_maad: bool, + /// MAAD worker threads (defaults to available parallelism, capped at 8). + #[arg(long)] + maad_workers: Option, /// Leave partial results in place but fail if requested five-minute coverage is incomplete. #[arg(long)] require_complete: bool, @@ -342,6 +346,9 @@ fn main() -> Result<()> { } fn run_pipeline(args: PipelineArgs) -> Result<()> { + if let Some(workers) = args.maad_workers { + netflow_db::publish::set_maad_workers(workers)?; + } let mut selection = serde_json::Map::new(); if args.daily_active_sources { selection.insert("kind".into(), serde_json::json!("daily_active_sources")); @@ -712,3 +719,31 @@ fn parse_boundary(raw: &str, timezone: &str) -> Result { .timestamp() .as_second()) } + +#[cfg(test)] +mod tests { + use super::*; + + fn pipeline_args(extra: &[&str]) -> Result { + let mut argv = vec!["netflow-db", "pipeline", "--config", "pipeline.json"]; + argv.extend_from_slice(extra); + Cli::try_parse_from(argv).map(|cli| match cli.command { + Command::Pipeline(args) => args, + command => panic!("unexpected command {command:?}"), + }) + } + + #[test] + fn maad_workers_must_be_a_positive_count() { + assert_eq!(pipeline_args(&[]).unwrap().maad_workers, None); + assert_eq!( + pipeline_args(&["--maad-workers", "3"]) + .unwrap() + .maad_workers, + NonZeroUsize::new(3) + ); + for invalid in ["0", "-1", "many"] { + assert!(pipeline_args(&["--maad-workers", invalid]).is_err()); + } + } +} diff --git a/tools/netflow-db/src/publish.rs b/tools/netflow-db/src/publish.rs index 41ad5ea8..8d1369f0 100644 --- a/tools/netflow-db/src/publish.rs +++ b/tools/netflow-db/src/publish.rs @@ -2,6 +2,7 @@ use std::{ net::IpAddr, + num::NonZeroUsize, sync::OnceLock, time::{Duration, Instant}, }; @@ -459,23 +460,44 @@ fn scope_results( }) } +/// Size the MAAD worker pool before its first use; later calls fail. +pub fn set_maad_workers(workers: NonZeroUsize) -> Result<(), PublishError> { + let mut initialized = false; + let pool = MAAD_POOL.get_or_init(|| { + initialized = true; + build_maad_pool(workers.get()) + }); + if !initialized { + return Err(PublishError::MaadPool( + "the worker pool was already initialized".into(), + )); + } + pool.as_ref() + .map(|_| ()) + .map_err(|error| PublishError::MaadPool(error.clone())) +} + fn maad_pool() -> Result<&'static rayon::ThreadPool, PublishError> { match MAAD_POOL.get_or_init(|| { - rayon::ThreadPoolBuilder::new() - .num_threads( - std::thread::available_parallelism() - .map_or(1, std::num::NonZeroUsize::get) - .min(MAX_MAAD_WORKERS), - ) - .thread_name(|index| format!("maad-{index}")) - .build() - .map_err(|error| error.to_string()) + build_maad_pool( + std::thread::available_parallelism() + .map_or(1, NonZeroUsize::get) + .min(MAX_MAAD_WORKERS), + ) }) { Ok(pool) => Ok(pool), Err(error) => Err(PublishError::MaadPool(error.clone())), } } +fn build_maad_pool(workers: usize) -> Result { + rayon::ThreadPoolBuilder::new() + .num_threads(workers) + .thread_name(|index| format!("maad-{index}")) + .build() + .map_err(|error| error.to_string()) +} + fn dimensions(key: &BucketKey, scope: crate::domain::Scope) -> StatsDimensions { StatsDimensions { source_id: key.source_id.clone(), @@ -576,6 +598,13 @@ mod tests { assert!(profile.total_elapsed >= profile.other_elapsed()); } + #[test] + fn maad_workers_cannot_resize_an_initialized_pool() { + assert!(maad_pool().unwrap().current_num_threads() >= 1); + let error = set_maad_workers(NonZeroUsize::new(2).unwrap()).unwrap_err(); + assert!(error.to_string().contains("already initialized")); + } + #[test] fn address_insertion_order_does_not_change_persisted_products() { fn persist(addresses: impl IntoIterator) -> Connection {