diff --git a/crates/waf-ids-core/src/lib.rs b/crates/waf-ids-core/src/lib.rs index f9673e0..33efeab 100644 --- a/crates/waf-ids-core/src/lib.rs +++ b/crates/waf-ids-core/src/lib.rs @@ -13,6 +13,9 @@ pub struct AppData { pub threats: Vec, #[serde(default)] pub operator_threat_keys: Vec, + /// DNSBL identities independently managed through the operator API. + #[serde(default)] + pub operator_dnsbl_keys: Vec, pub dnsbl: Vec, pub events: Vec, pub next_event_id: u64, @@ -47,6 +50,7 @@ impl AppData { ttl_seconds: 86_400, }], operator_threat_keys: Vec::new(), + operator_dnsbl_keys: Vec::new(), dnsbl: vec![DnsblEntry { address: "203.0.113.10".parse().expect("seed IP address is valid"), code: "127.0.0.2".to_string(), @@ -138,6 +142,17 @@ pub struct DnsblEntry { pub prefix_len: Option, } +/// Stable DNSBL ownership identity. It deliberately matches [`upsert_dnsbl`], +/// whose physical row identity is the IP address rather than payload metadata. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] +#[serde(transparent)] +pub struct DnsblEntryKey(pub IpAddr); + +/// Returns the durable ownership identity for a DNSBL entry. +pub fn dnsbl_entry_key(entry: &DnsblEntry) -> DnsblEntryKey { + DnsblEntryKey(entry.address) +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct CommercialProfile { pub tenant_id: String, @@ -189,6 +204,9 @@ pub struct ThreatFeedStatus { pub struct ThreatFeedOwnership { pub feed_id: String, pub threat_keys: Vec, + /// DNSBL rows currently claimed by this feed snapshot. + #[serde(default)] + pub dnsbl_keys: Vec, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Hash)] @@ -573,6 +591,27 @@ pub fn replace_threat_feed_ownership( ownership.push(ThreatFeedOwnership { feed_id, threat_keys, + dnsbl_keys: Vec::new(), + }); + Vec::new() + } +} + +/// Replaces one feed's DNSBL snapshot ownership and returns its prior keys. +pub fn replace_threat_feed_dnsbl_ownership( + ownership: &mut Vec, + feed_id: String, + dnsbl_keys: Vec, +) -> Vec { + if let Some(existing) = ownership.iter_mut().find(|item| item.feed_id == feed_id) { + let previous = existing.dnsbl_keys.clone(); + existing.dnsbl_keys = dnsbl_keys; + previous + } else { + ownership.push(ThreatFeedOwnership { + feed_id, + threat_keys: Vec::new(), + dnsbl_keys, }); Vec::new() } @@ -1776,6 +1815,55 @@ mod tests { assert!(shannon_entropy(b"abcdefgh") > 2.9); } + #[test] + fn replaces_threat_feed_dnsbl_ownership_and_returns_previous_keys() { + let first = DnsblEntryKey("203.0.113.1".parse().unwrap()); + let second = DnsblEntryKey("203.0.113.2".parse().unwrap()); + let mut ownership = Vec::new(); + + assert!( + replace_threat_feed_dnsbl_ownership(&mut ownership, "feed-a".to_string(), vec![first],) + .is_empty() + ); + assert_eq!(ownership[0].dnsbl_keys, vec![first]); + + let previous = + replace_threat_feed_dnsbl_ownership(&mut ownership, "feed-a".to_string(), vec![second]); + assert_eq!(previous, vec![first]); + assert_eq!(ownership[0].dnsbl_keys, vec![second]); + + assert!( + replace_threat_feed_dnsbl_ownership(&mut ownership, "feed-b".to_string(), vec![first],) + .is_empty() + ); + assert_eq!(ownership.len(), 2); + assert!(ownership[1].threat_keys.is_empty()); + } + + #[test] + fn dnsbl_ownership_fields_default_when_deserializing_predecessor_state() { + let key = DnsblEntryKey("203.0.113.3".parse().unwrap()); + let mut data = AppData::seeded(); + data.operator_dnsbl_keys.push(key); + data.threat_feed_ownership.push(ThreatFeedOwnership { + feed_id: "legacy-feed".to_string(), + threat_keys: Vec::new(), + dnsbl_keys: vec![key], + }); + + let mut legacy = serde_json::to_value(data).unwrap(); + let object = legacy.as_object_mut().unwrap(); + object.remove("operator_dnsbl_keys"); + object["threat_feed_ownership"].as_array_mut().unwrap()[0] + .as_object_mut() + .unwrap() + .remove("dnsbl_keys"); + + let loaded: AppData = serde_json::from_value(legacy).unwrap(); + assert!(loaded.operator_dnsbl_keys.is_empty()); + assert!(loaded.threat_feed_ownership[0].dnsbl_keys.is_empty()); + } + #[test] fn records_audit_logs_with_monotonic_ids() { let mut data = AppData::seeded(); diff --git a/docs/doctoring/misp-to-ids-admission.md b/docs/doctoring/misp-to-ids-admission.md new file mode 100644 index 0000000..017fe46 --- /dev/null +++ b/docs/doctoring/misp-to-ids-admission.md @@ -0,0 +1,39 @@ +# MISP `to_ids` admission policy + +## Decision boundary + +Wardnet converts selected MISP attributes into gateway enforcement material (`ThreatIndicator` and, for IPs, `DnsblEntry`). The conversion is therefore an admission decision, not a neutral JSON import. `to_ids` is treated as affirmative evidence that an attribute is intended for detection, and an attribute is admitted only when Wardnet can positively recognize that signal as true and can establish that the attribute has not been withdrawn. + +The MISP object schema defines `to_ids` as a boolean, and the MISP-STIX converter documents that `to_ids=True` is the detection-ready case that becomes an Indicator. MISP REST search can also return attributes regardless of their `to_ids` setting unless the caller filters them. Wardnet consequently cannot infer IDS authorization from mere presence in a MISP document. Missing, `false`, `null`, object, array, and otherwise unrecognized `to_ids` values fail closed and contribute to `skipped_attributes`; they do not produce enforcement rows. The previously supported explicit scalar compatibility spellings (`"1"`, case-insensitive `"true"`, numeric `1`) remain accepted so the repair does not broaden unrelated parser incompatibility. + +MISP's ZeroMQ contract exposes attribute deletion independently of `to_ids`: examples use `deleted: false`/`"0"` for active material and set `deleted` to `1` when an attribute is deleted, with subscribers instructed to inspect the deletion marker. MISP also gives an enclosing Object its own deletion lifecycle; the current event-object rendering path explicitly reads `object['deleted']` and marks the whole object deleted. A positive nested attribute `to_ids` value therefore cannot resurrect either a withdrawn attribute or a withdrawn parent Object. Wardnet treats recognized `deleted=true`/`1` states at either scope as withdrawn; recognized `false`/`0` states and an omitted `deleted` member remain active for compatibility with ordinary exports. A present but structurally unrecognized deletion state at either scope fails closed rather than becoming an implicit active signal. + +This is the security-design application of fail-safe defaults: permission to create enforcement material is established by positive evidence rather than by the absence of a denial. Saltzer and Schroeder (1975) describe that protection principle as defaulting to lack of permission unless conditions for access are established. For the parser boundary here, the analogous safe state is “do not create an enforcement indicator” when the upstream authorization signal is absent, withdrawn, or structurally invalid. + +The hostile regressions intentionally supply valid JSON whose admission members are semantically malformed, absent, or contradictory with lifecycle state. That test style is consistent with the fuzzing literature's treatment of syntactically and semantically malformed inputs as a primary way to expose security-relevant parser behavior (Manès et al., 2021). Wardnet already archives the permitted arXiv version of that survey at [`../papers/fuzzing-art-science-engineering-survey-arxiv-1812.00140.pdf`](../papers/fuzzing-art-science-engineering-survey-arxiv-1812.00140.pdf). The Saltzer–Schroeder paper is linked to the authors' MIT-hosted web rendering rather than copied into the repository because the hosted metadata retains the authors' 1975 copyright; this document does not manufacture a redistribution right. + +## Alternatives considered + +Keeping `unwrap_or(true)` for `to_ids` was rejected because an omitted field would silently become stronger evidence than MISP supplied. Treating arbitrary JSON values as true was rejected because a producer/schema defect would widen enforcement. Treating an unknown or malformed `deleted` value as active was rejected for the same reason: lifecycle ambiguity must not revive enforcement material. Ignoring an enclosing MISP Object's deletion marker was rejected because it allows a nested attribute to outlive the producer's parent-object withdrawal decision. Rejecting the entire MISP document was also rejected for this bounded repair: `skipped_attributes` already provides a per-attribute fail-closed path and allows valid sibling indicators to remain usable while preserving input-quality evidence. A future contract version may remove the legacy scalar compatibility spellings, but that is a separate compatibility decision and is not required to close this defect. + +## Verification contract + +`tests/misp_to_ids_admission.rs` is the focused security regression. It requires object, array, `null`, and missing `to_ids` values to be skipped while an explicit boolean `true` control remains admitted. It also requires attribute-level `deleted=true`, `deleted="1"`, and a structurally malformed deletion marker to be skipped even when `to_ids=true`, while `deleted=false`, `deleted="0"`, and omitted deletion state remain active. The parent-object hostile case requires nested otherwise-actionable attributes to be skipped when the enclosing MISP Object is deleted or has a structurally ambiguous deletion marker; recognized active and omitted parent deletion state remain compatible. Existing unit coverage retains boolean `to_ids=false` rejection and the supported explicit scalar compatibility cases. Merge evidence must come from the exact current PR head and then-live Wardnet CI/fuzz/security/review gates; source reasoning or predecessor runs are not a substitute. + +## Traceability and references + +MISP Project. (n.d.). *misp-objects: Definition, description and relationship types of MISP objects* (`schema_objects.json`). GitHub. https://github.com/MISP/misp-objects/blob/main/schema_objects.json + +MISP Project. (n.d.). *MISP-STIX converter*. GitHub. https://github.com/MISP/misp-stix + +MISP Project. (n.d.). *MISP ZeroMQ documentation*. GitHub. https://github.com/MISP/misp-book/blob/main/misp-zmq/README.md + +MISP Project. (2026). *Event object row rendering* (`app/View/Elements/Events/View/row_object.ctp`, commit `9294667a5b40e59ea42314c2aafa99086ce1d8e6`). GitHub. https://github.com/MISP/MISP/blob/9294667a5b40e59ea42314c2aafa99086ce1d8e6/app/View/Elements/Events/View/row_object.ctp + +Saltzer, J. H., & Schroeder, M. D. (1975). The protection of information in computer systems. *Proceedings of the IEEE, 63*(9), 1278–1308. https://doi.org/10.1109/PROC.1975.9939 + +Author-hosted rendering: https://web.mit.edu/saltzer/www/publications/protection/index.html + +Manès, V. J. M., Han, H., Han, C., Cha, S. K., Egele, M., Schwartz, E. J., & Woo, M. (2021). The art, science, and engineering of fuzzing: A survey. *IEEE Transactions on Software Engineering, 47*(11), 2312–2331. https://doi.org/10.1109/TSE.2019.2946563 + +Repository archive: [`../papers/fuzzing-art-science-engineering-survey-arxiv-1812.00140.pdf`](../papers/fuzzing-art-science-engineering-survey-arxiv-1812.00140.pdf) diff --git a/docs/doctoring/threat-feed-dnsbl-ownership.md b/docs/doctoring/threat-feed-dnsbl-ownership.md new file mode 100644 index 0000000..eef00fa --- /dev/null +++ b/docs/doctoring/threat-feed-dnsbl-ownership.md @@ -0,0 +1,76 @@ +# Threat-feed DNSBL snapshot ownership + +## Decision boundary + +`POST /api/threat-feeds/import` is a snapshot-reconciliation boundary, not an append-only DNSBL ingest path. A feed refresh may withdraw a previously published address. Wardnet therefore distinguishes four facts that the global `Vec` cannot express by itself: + +1. the stable DNSBL identity currently used by `upsert_dnsbl`; +2. which feed snapshots still claim that identity; +3. whether an operator independently owns that identity; and +4. the current effective payload stored for that identity. + +The stable identity is the same identity the mutation primitive already enforces: IP `address`. Introducing a different ownership key would make reconciliation disagree with `upsert_dnsbl` and would permit two logical owners to mutate one physical row under incompatible identities. + +## Implemented state model + +The durable `AppData` authority now carries explicit DNSBL ownership rather than inferring it from source strings, TTL, threat indicators, audit logs, or adapter-specific conventions. + +- `DnsblEntryKey(IpAddr)` is serializable/hashable and matches the global DNSBL upsert identity. +- Each `ThreatFeedOwnership` carries `dnsbl_keys` with `#[serde(default)]`, so persisted predecessor state loads without a destructive rewrite. +- `AppData` carries independent `operator_dnsbl_keys`, also with `#[serde(default)]`. +- `/api/dnsbl` upsert and operator-key registration occur in the same `mutate_and_persist` mutation, so either the effective payload plus ownership persist together or the mutation rolls back. +- A feed import replaces that feed's threat and DNSBL ownership sets as one snapshot mutation, then removes each previously owned DNSBL row only when no other feed and no operator owns the address. +- Feed import skips effective-payload writes for operator-owned addresses and increments `upserted_dnsbl` only for writes it actually performs. + +Ownership metadata is internal control-plane state. It does not become threat intelligence, does not become a synthetic `ThreatIndicator`, and is not encoded into `source`, audit-log text, or another bounded context. Those shortcuts were rejected because they would make authority implicit and break DDD naming and semantic boundaries. + +## Replay, persistence, and idempotency + +A repeated identical feed snapshot leaves ownership and effective DNSBL state semantically unchanged apart from the feed freshness timestamp already owned by the import path. Restarting from persisted `AppData` retains enough ownership to make the next refresh deterministic; an in-memory sidecar is not sufficient. A refresh of feed A cannot delete an address still owned by feed B. A later operator upsert at an address previously owned by a feed survives withdrawal of that feed without payload rollback. + +The repair remains inside Wardnet's shared threat-feed admission/control-plane path. MISP, STIX/TAXII, OpenCTI, KEV, and other adapters provide feed material but do not copy or reimplement reconciliation. This is why the valid review finding was repaired at `apply_threat_feed_import`, not inside `misp_import.rs`. + +## Hostile RED and causal GREEN + +`tests/threat_feed_dnsbl_ownership.rs` is the focused public-API regression. The corrected hostile lineage reached exact RED `28d0ac12d37b4c97ea58b2d55831a6c1e7b9cf98`: the existing implementation failed stale feed withdrawal and operator-overwrite isolation while the valid shared-feed control remained preserved. The refresh requests retain unrelated valid material so they reach snapshot reconciliation without changing the existing contract that rejects a completely empty feed import. + +The hostile contract requires: + +- import feed A with a DNSBL address, refresh A without that DNSBL key, and require the withdrawn row to disappear; +- import the same address from feeds A and B, withdraw it from A, and require the row to remain because B still owns it; +- import an address from a feed, overwrite that address through the operator `/api/dnsbl` surface, withdraw the feed, and require the operator payload to survive at the domain-field level; +- reject feed overwrite of an operator-owned payload and report zero feed DNSBL writes for that skipped key. + +`tests/threat_feed_dnsbl_persistence.rs` additionally covers restart-before-withdrawal, predecessor-state deserialization and persistence-failure rollback/retry. A solution that merely stops gateway scoring while leaving stale `/api/dnsbl` state, relies on TTL expiry, synthesizes hidden threat indicators, or keeps ownership only in process memory does not satisfy the contract. + +The causal source repair was committed as `7042aa19267886e3af9c378dddd879929837877b`. Before that source-only commit was pushed, rescue run `34000662730` executed the resulting working tree: all locked workspace tests passed, including all four DNSBL ownership cases and all three persistence cases, and strict workspace Clippy passed. The workflow then verified that only `crates/waf-ids-core/src/lib.rs` and `src/lib.rs` were modified by the repair, removed both temporary repair workflows, and non-force pushed the causal commit. This is source-GREEN evidence, not a substitute for the required workflows on a later committed head. + +## Protected-base compatibility and operational evidence + +Protected `main` advanced independently through #171 to `a52ccd0a24a727d9349bb32def7713882d8cad1e`. A bounded non-force restack run `34000892973` checked the then-current #167 head, verified both the feature ref and protected-main SHA were unchanged, merged that exact protected head without rewriting history, removed its temporary restack workflow, and ran the full locked workspace tests plus strict workspace Clippy successfully before pushing merge commit `d8b452cf1d609bb6e9c9a8a33f265c0a32dce7c9`. Thus the causal Rust repair is candidate-base compatible with #171's protected truth. + +The first standard required workflows on bot-authored source commit `7042aa1...` terminated `action_required` without jobs because that commit was produced by a `GITHUB_TOKEN` workflow; they are not GREEN evidence. A subsequent human-authored restack-control commit did trigger normal standard workflow materialization, with the Ubuntu CI lane entering the known queued class while the macOS restack runner acquired compute and completed. This narrows the runner evidence already handed to `.github#712`: the observed starvation is not an organization-wide inability to allocate any GitHub-hosted runner. + +Merge remains prohibited until the unchanged final committed head obtains the then-live repository/security/review/governance evidence required by the protected ruleset. Queued, `action_required`, predecessor-head, or working-tree evidence must not be promoted to an exact-head required-gate verdict. + +## Security rationale + +This is a fail-safe lifecycle requirement. Revocation or withdrawal must converge the enforcement set toward less authority, not leave an orphaned deny decision whose producer no longer claims it. The rule also prevents one producer from deleting another producer's still-valid deny state and prevents automated feed refresh from erasing a later human/operator decision. + +The general protection rationale remains the fail-safe-default principle documented for the adjacent MISP admission repair: authority must be established by explicit positive state, and ambiguous or withdrawn authority must not silently continue enforcement. See `docs/doctoring/misp-to-ids-admission.md` and its Saltzer–Schroeder traceability. The hostile regression style follows the repository's existing Manès et al. fuzzing-survey traceability and tests semantic lifecycle corruption rather than malformed JSON alone. + +## Acceptance + +The production behavior and causal source verification now satisfy these implementation criteria: + +- `DnsblEntryKey` and its ownership fields are explicit, persisted, serde-defaulted, and tests cover predecessor-state deserialization; +- ownership replacement and stale-row removal happen inside the same `mutate_and_persist` transaction as the feed snapshot; +- same-address multiple-feed ownership prevents premature deletion; +- operator ownership prevents feed deletion and feed payload overwrite; +- persistence failure rolls ownership and effective state back before retry; +- repeated snapshots are idempotent at the domain-state level; +- `upserted_dnsbl` reports actual feed writes rather than requested input length when an operator-owned row is preserved; +- existing threat ownership semantics remain unchanged; +- no adapter-specific copy of reconciliation logic is introduced. + +Release/merge acceptance remains separate: exact-current CI/Fuzz/security/coverage/SBOM/provenance/review/thread/governance evidence must be terminal-valid before protected merge, followed by fresh protected-head release evidence before any release-ready claim. diff --git a/src/lib.rs b/src/lib.rs index ab902ca..d1c66d8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,17 +21,17 @@ use tokio::{ }; use waf_ids_core::{ AppData, BLOCK_SCORE, buyer_evidence_manifest_at, commercial_readiness_snapshot_at, - enforce_event_limit, kpi_snapshot_at, prometheus_exposition, rate_limit_step, record_audit_log, - replace_threat_feed_ownership, select_route, signature_catalog, threat_feed_freshness_snapshot, - threat_indicator_key, upsert_dnsbl, upsert_route, upsert_threat, upsert_threat_feed, - validate_commercial_profile, validate_dnsbl, validate_route, validate_threat, - validate_threat_feed_import, + dnsbl_entry_key, enforce_event_limit, kpi_snapshot_at, prometheus_exposition, rate_limit_step, + record_audit_log, replace_threat_feed_dnsbl_ownership, replace_threat_feed_ownership, + select_route, signature_catalog, threat_feed_freshness_snapshot, threat_indicator_key, + upsert_dnsbl, upsert_route, upsert_threat, upsert_threat_feed, validate_commercial_profile, + validate_dnsbl, validate_route, validate_threat, validate_threat_feed_import, }; pub use waf_ids_core::{ AuditLogEntry, BuyerEvidenceEndpoint, BuyerEvidenceManifest, BuyerEvidenceRuntimeCounts, - CommercialProfile, CommercialReadiness, DnsblEntry, EnforcementMode, LicenseStatus, - NewAuditLogEntry, ProductEdition, ReadinessCheck, ReadinessStatus, RouteConfig, ScoredRequest, - SecurityEvent, Severity, SignatureInfo, SocKpiSnapshot, TARGET_SALE_VALUE_KRW, + CommercialProfile, CommercialReadiness, DnsblEntry, DnsblEntryKey, EnforcementMode, + LicenseStatus, NewAuditLogEntry, ProductEdition, ReadinessCheck, ReadinessStatus, RouteConfig, + ScoredRequest, SecurityEvent, Severity, SignatureInfo, SocKpiSnapshot, TARGET_SALE_VALUE_KRW, ThreatFeedFreshness, ThreatFeedImport, ThreatFeedImportResult, ThreatFeedStatus, ThreatIndicator, export_dnsbl_zone, ip_in_network, reverse_ipv4_for_dnsbl, score_request, }; @@ -996,6 +996,7 @@ async fn create_dnsbl( match state .mutate_and_persist(|data| { let saved = upsert_dnsbl(&mut data.dnsbl, entry.clone()); + mark_operator_dnsbl_key(data, &saved); record_successful_audit_log( data, actor, @@ -2702,6 +2703,13 @@ fn mark_operator_threat_key(data: &mut AppData, indicator: &ThreatIndicator) { } } +fn mark_operator_dnsbl_key(data: &mut AppData, entry: &DnsblEntry) { + let key = dnsbl_entry_key(entry); + if !data.operator_dnsbl_keys.contains(&key) { + data.operator_dnsbl_keys.push(key); + } +} + async fn apply_threat_feed_import( state: &AppState, actor: String, @@ -2712,6 +2720,8 @@ async fn apply_threat_feed_import( state .mutate_and_persist(|data| { let operator_owned: HashSet<_> = data.operator_threat_keys.iter().cloned().collect(); + let operator_dnsbl_owned: HashSet<_> = + data.operator_dnsbl_keys.iter().copied().collect(); let threat_keys: Vec<_> = feed.threats.iter().map(threat_indicator_key).collect(); let previous_keys: HashSet<_> = replace_threat_feed_ownership( &mut data.threat_feed_ownership, @@ -2720,6 +2730,14 @@ async fn apply_threat_feed_import( ) .into_iter() .collect(); + let dnsbl_keys: Vec<_> = feed.dnsbl.iter().map(dnsbl_entry_key).collect(); + let previous_dnsbl_keys: HashSet<_> = replace_threat_feed_dnsbl_ownership( + &mut data.threat_feed_ownership, + feed.feed_id.clone(), + dnsbl_keys, + ) + .into_iter() + .collect(); if !previous_keys.is_empty() { // A key this feed is dropping might still be owned by another // feed (e.g. two feeds importing the same CVE under a shared @@ -2740,6 +2758,20 @@ async fn apply_threat_feed_import( || operator_owned.contains(&key) }); } + if !previous_dnsbl_keys.is_empty() { + let still_dnsbl_owned: HashSet<_> = data + .threat_feed_ownership + .iter() + .filter(|ownership| ownership.feed_id != feed.feed_id) + .flat_map(|ownership| ownership.dnsbl_keys.iter().copied()) + .collect(); + data.dnsbl.retain(|entry| { + let key = dnsbl_entry_key(entry); + !previous_dnsbl_keys.contains(&key) + || still_dnsbl_owned.contains(&key) + || operator_dnsbl_owned.contains(&key) + }); + } let mut upserted_threats = 0usize; for threat in feed.threats.iter().cloned() { if operator_owned.contains(&threat_indicator_key(&threat)) { @@ -2748,8 +2780,13 @@ async fn apply_threat_feed_import( upsert_threat(&mut data.threats, threat); upserted_threats += 1; } + let mut upserted_dnsbl = 0usize; for entry in feed.dnsbl.iter().cloned() { + if operator_dnsbl_owned.contains(&dnsbl_entry_key(&entry)) { + continue; + } upsert_dnsbl(&mut data.dnsbl, entry); + upserted_dnsbl += 1; } upsert_threat_feed( &mut data.threat_feeds, @@ -2765,7 +2802,7 @@ async fn apply_threat_feed_import( let result = ThreatFeedImportResult { feed_id: feed.feed_id.clone(), upserted_threats, - upserted_dnsbl: feed.dnsbl.len(), + upserted_dnsbl, last_updated_unix: imported_at, }; record_successful_audit_log(data, actor, action, "threat_feed", result.feed_id.clone()); @@ -6500,6 +6537,23 @@ mod tests { ); } + #[test] + fn operator_dnsbl_key_marking_is_idempotent() { + let mut data = AppData::seeded(); + let entry = DnsblEntry { + address: "203.0.113.245".parse().unwrap(), + code: "127.0.0.2".to_string(), + reason: "operator".to_string(), + source: "operator".to_string(), + ttl_seconds: 300, + prefix_len: None, + }; + + mark_operator_dnsbl_key(&mut data, &entry); + mark_operator_dnsbl_key(&mut data, &entry); + assert_eq!(data.operator_dnsbl_keys, vec![dnsbl_entry_key(&entry)]); + } + #[tokio::test] async fn gateway_covers_monitor_proxy_not_found_and_bad_gateway_paths() { let upstream_app = Router::new().route( @@ -6563,6 +6617,7 @@ mod tests { ], threats: Vec::new(), operator_threat_keys: Vec::new(), + operator_dnsbl_keys: Vec::new(), dnsbl: Vec::new(), events: Vec::new(), next_event_id: 1, @@ -7440,6 +7495,7 @@ mod tests { }], threats: Vec::new(), operator_threat_keys: Vec::new(), + operator_dnsbl_keys: Vec::new(), dnsbl: Vec::new(), events: Vec::new(), next_event_id: 1, diff --git a/src/misp_import.rs b/src/misp_import.rs index 65d0707..b12e3d1 100644 --- a/src/misp_import.rs +++ b/src/misp_import.rs @@ -97,8 +97,15 @@ pub fn misp_material_from_value( .or_else(|| event.get("id").and_then(|i| i.as_str())) .unwrap_or("misp-event"); - for attr in collect_event_attributes(event) { - match materialize_attribute(attr, source, ttl_seconds, severity.clone(), event_label) { + for (attr, parent_active) in collect_event_attributes(event) { + match materialize_attribute( + attr, + source, + ttl_seconds, + severity.clone(), + event_label, + parent_active, + ) { AttributeOutcome::Mapped { threats: t, dnsbl: d, @@ -112,7 +119,7 @@ pub fn misp_material_from_value( } for (attr, severity) in loose_attributes { - match materialize_attribute(attr, source, ttl_seconds, severity, "misp-attribute") { + match materialize_attribute(attr, source, ttl_seconds, severity, "misp-attribute", true) { AttributeOutcome::Mapped { threats: t, dnsbl: d, @@ -190,15 +197,27 @@ fn severity_from_event(event: &serde_json::Value) -> Severity { } } -fn collect_event_attributes(event: &serde_json::Value) -> Vec<&serde_json::Value> { +fn active_by_deleted_marker(deleted: Option<&serde_json::Value>) -> bool { + deleted + .map(|value| match value { + serde_json::Value::Bool(is_deleted) => !*is_deleted, + serde_json::Value::String(value) => value == "0" || value.eq_ignore_ascii_case("false"), + serde_json::Value::Number(value) => value.as_u64() == Some(0), + _ => false, + }) + .unwrap_or(true) +} + +fn collect_event_attributes(event: &serde_json::Value) -> Vec<(&serde_json::Value, bool)> { let mut out = Vec::new(); if let Some(attrs) = event.get("Attribute").and_then(|a| a.as_array()) { - out.extend(attrs.iter()); + out.extend(attrs.iter().map(|attr| (attr, true))); } if let Some(objects) = event.get("Object").and_then(|o| o.as_array()) { for object in objects { + let parent_active = active_by_deleted_marker(object.get("deleted")); if let Some(attrs) = object.get("Attribute").and_then(|a| a.as_array()) { - out.extend(attrs.iter()); + out.extend(attrs.iter().map(|attr| (attr, parent_active))); } } } @@ -211,18 +230,26 @@ fn materialize_attribute( ttl_seconds: u64, severity: Severity, event_label: &str, + parent_active: bool, ) -> AttributeOutcome { - // Skip non-IDS attributes when to_ids is explicitly false (MISP convention). + // MISP's `to_ids` contract is affirmative evidence. Preserve the previously supported + // scalar true spellings, but absent or malformed values cannot authorize enforcement. + // See docs/doctoring/misp-to-ids-admission.md. let to_ids = attr .get("to_ids") .map(|v| match v { serde_json::Value::Bool(b) => *b, serde_json::Value::String(s) => s == "1" || s.eq_ignore_ascii_case("true"), serde_json::Value::Number(n) => n.as_u64() == Some(1), - _ => true, + _ => false, }) - .unwrap_or(true); - if !to_ids { + .unwrap_or(false); + + // MISP publishes deletion state independently at both object and attribute scope. A + // nested attribute cannot override a withdrawn parent object. Omission remains active + // for compatibility; any present unrecognized deletion state fails closed. + let active = active_by_deleted_marker(attr.get("deleted")); + if !parent_active || !to_ids || !active { return AttributeOutcome::Skipped; } diff --git a/tests/misp_to_ids_admission.rs b/tests/misp_to_ids_admission.rs new file mode 100644 index 0000000..305777a --- /dev/null +++ b/tests/misp_to_ids_admission.rs @@ -0,0 +1,120 @@ +#[path = "../src/misp_import.rs"] +mod misp_import; + +#[test] +fn malformed_or_missing_to_ids_values_fail_closed() { + // Policy and research traceability: docs/doctoring/misp-to-ids-admission.md. + let raw = r#"[ + {"type":"domain","value":"object.example","to_ids":{"unexpected":true}}, + {"type":"domain","value":"array.example","to_ids":[true]}, + {"type":"domain","value":"null.example","to_ids":null}, + {"type":"domain","value":"missing.example"}, + {"type":"domain","value":"valid.example","to_ids":true} + ]"#; + + let material = misp_import::parse_misp_document(raw, "misp:test", 60).unwrap(); + + assert_eq!(material.threats.len(), 1); + assert_eq!(material.threats[0].value, "valid.example"); + assert_eq!(material.skipped_attributes, 4); +} + +#[test] +fn deleted_misp_attributes_cannot_authorize_enforcement() { + // A positive `to_ids` signal cannot resurrect an attribute MISP marks deleted. + let raw = r#"[ + {"type":"domain","value":"deleted-bool.example","to_ids":true,"deleted":true}, + {"type":"domain","value":"deleted-string.example","to_ids":true,"deleted":"1"}, + {"type":"domain","value":"malformed-deleted.example","to_ids":true,"deleted":{"unexpected":false}}, + {"type":"domain","value":"active-bool.example","to_ids":true,"deleted":false}, + {"type":"domain","value":"active-string.example","to_ids":true,"deleted":"0"}, + {"type":"domain","value":"active-omitted.example","to_ids":true} + ]"#; + + let material = misp_import::parse_misp_document(raw, "misp:test", 60).unwrap(); + + assert_eq!(material.threats.len(), 3); + assert!( + material + .threats + .iter() + .any(|threat| threat.value == "active-bool.example") + ); + assert!( + material + .threats + .iter() + .any(|threat| threat.value == "active-string.example") + ); + assert!( + material + .threats + .iter() + .any(|threat| threat.value == "active-omitted.example") + ); + assert_eq!(material.skipped_attributes, 3); +} + +#[test] +fn deleted_or_ambiguous_misp_objects_cannot_authorize_nested_attributes() { + // MISP objects carry an independent lifecycle marker. A nested attribute's + // `to_ids=true` cannot override a deleted or structurally ambiguous parent object. + let raw = r#"{ + "Event": { + "id": "object-lifecycle", + "Object": [ + { + "name": "deleted-bool", + "deleted": true, + "Attribute": [ + {"type":"domain","value":"deleted-object-bool.example","to_ids":true,"deleted":false} + ] + }, + { + "name": "deleted-string", + "deleted": "1", + "Attribute": [ + {"type":"domain","value":"deleted-object-string.example","to_ids":true} + ] + }, + { + "name": "ambiguous-object", + "deleted": {"unexpected": false}, + "Attribute": [ + {"type":"domain","value":"ambiguous-object.example","to_ids":true} + ] + }, + { + "name": "active-bool", + "deleted": false, + "Attribute": [ + {"type":"domain","value":"active-object-bool.example","to_ids":true} + ] + }, + { + "name": "active-omitted", + "Attribute": [ + {"type":"domain","value":"active-object-omitted.example","to_ids":true} + ] + } + ] + } + }"#; + + let material = misp_import::parse_misp_document(raw, "misp:test", 60).unwrap(); + + assert_eq!(material.threats.len(), 2); + assert!( + material + .threats + .iter() + .any(|threat| threat.value == "active-object-bool.example") + ); + assert!( + material + .threats + .iter() + .any(|threat| threat.value == "active-object-omitted.example") + ); + assert_eq!(material.skipped_attributes, 3); +} diff --git a/tests/threat_feed_dnsbl_ownership.rs b/tests/threat_feed_dnsbl_ownership.rs new file mode 100644 index 0000000..980e264 --- /dev/null +++ b/tests/threat_feed_dnsbl_ownership.rs @@ -0,0 +1,234 @@ +use axum::{ + body::{Body, to_bytes}, + http::{Method, Request, StatusCode}, +}; +use serde_json::json; +use tower::ServiceExt; +use waf_ids_ai_soc::{AppState, DnsblEntry, build_app}; + +fn json_request(method: Method, uri: &str, body: serde_json::Value) -> Request { + Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .header("x-admin-token", "secret") + .body(Body::from(body.to_string())) + .unwrap() +} + +async fn dnsbl(app: &axum::Router) -> Vec { + let response = app + .clone() + .oneshot( + Request::builder() + .method(Method::GET) + .uri("/api/dnsbl") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + serde_json::from_slice(&bytes).unwrap() +} + +fn feed_payload(feed_id: &str, source: &str, addresses: &[&str]) -> serde_json::Value { + json!({ + "feed_id": feed_id, + "source": source, + "ttl_seconds": 3600, + "threats": [], + "dnsbl": addresses.iter().map(|address| json!({ + "address": address, + "code": "127.0.0.2", + "reason": format!("{feed_id} verdict"), + "source": source, + "ttl_seconds": 3600, + "prefix_len": null + })).collect::>() + }) +} + +#[tokio::test] +async fn feed_refresh_reaps_dnsbl_entries_it_withdraws() { + let app = build_app(AppState::seeded(Some("secret".to_string()))); + + let first = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/threat-feeds/import", + feed_payload("feed-a", "feed:a", &["203.0.113.210"]), + )) + .await + .unwrap(); + assert_eq!(first.status(), StatusCode::CREATED); + assert!( + dnsbl(&app) + .await + .iter() + .any(|entry| entry.address.to_string() == "203.0.113.210") + ); + + let refresh = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/threat-feeds/import", + feed_payload("feed-a", "feed:a", &["203.0.113.250"]), + )) + .await + .unwrap(); + assert_eq!(refresh.status(), StatusCode::CREATED); + + assert!( + !dnsbl(&app) + .await + .iter() + .any(|entry| entry.address.to_string() == "203.0.113.210"), + "withdrawn feed DNSBL material must not keep blocking after refresh" + ); +} + +#[tokio::test] +async fn feed_refresh_preserves_dnsbl_still_owned_by_another_feed() { + let app = build_app(AppState::seeded(Some("secret".to_string()))); + let address = "203.0.113.211"; + + for (feed_id, source) in [("feed-a", "feed:a"), ("feed-b", "feed:b")] { + let response = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/threat-feeds/import", + feed_payload(feed_id, source, &[address]), + )) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + } + + let refresh = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/threat-feeds/import", + feed_payload("feed-a", "feed:a", &["203.0.113.250"]), + )) + .await + .unwrap(); + assert_eq!(refresh.status(), StatusCode::CREATED); + + assert!( + dnsbl(&app) + .await + .iter() + .any(|entry| entry.address.to_string() == address), + "one feed cannot delete a DNSBL address still claimed by another feed" + ); +} + +#[tokio::test] +async fn feed_refresh_preserves_operator_managed_dnsbl_payload() { + let app = build_app(AppState::seeded(Some("secret".to_string()))); + let address = "203.0.113.212"; + + let feed = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/threat-feeds/import", + feed_payload("feed-a", "feed:a", &[address]), + )) + .await + .unwrap(); + assert_eq!(feed.status(), StatusCode::CREATED); + + let operator = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/dnsbl", + json!({ + "address": address, + "code": "127.0.0.77", + "reason": "operator-reviewed exception payload", + "source": "operator", + "ttl_seconds": 86400, + "prefix_len": null + }), + )) + .await + .unwrap(); + assert_eq!(operator.status(), StatusCode::CREATED); + + let refresh = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/threat-feeds/import", + feed_payload("feed-a", "feed:a", &["203.0.113.250"]), + )) + .await + .unwrap(); + assert_eq!(refresh.status(), StatusCode::CREATED); + + let entries = dnsbl(&app).await; + let entry = entries + .iter() + .find(|entry| entry.address.to_string() == address) + .expect("operator-owned DNSBL entry must survive feed withdrawal"); + assert_eq!(entry.code, "127.0.0.77"); + assert_eq!(entry.reason, "operator-reviewed exception payload"); + assert_eq!(entry.source, "operator"); + assert_eq!(entry.ttl_seconds, 86400); +} + +#[tokio::test] +async fn feed_import_does_not_overwrite_operator_dnsbl_or_count_a_skipped_write() { + let app = build_app(AppState::seeded(Some("secret".to_string()))); + let address = "203.0.113.213"; + + let operator = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/dnsbl", + json!({ + "address": address, + "code": "127.0.0.88", + "reason": "operator-owned payload", + "source": "operator", + "ttl_seconds": 86400, + "prefix_len": null + }), + )) + .await + .unwrap(); + assert_eq!(operator.status(), StatusCode::CREATED); + + let imported = app + .clone() + .oneshot(json_request( + Method::POST, + "/api/threat-feeds/import", + feed_payload("feed-a", "feed:a", &[address]), + )) + .await + .unwrap(); + assert_eq!(imported.status(), StatusCode::CREATED); + let bytes = to_bytes(imported.into_body(), usize::MAX).await.unwrap(); + let result: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(result["upserted_dnsbl"], 0); + + let entries = dnsbl(&app).await; + let entry = entries + .iter() + .find(|entry| entry.address.to_string() == address) + .expect("operator-owned DNSBL entry must remain present"); + assert_eq!(entry.code, "127.0.0.88"); + assert_eq!(entry.reason, "operator-owned payload"); + assert_eq!(entry.source, "operator"); + assert_eq!(entry.ttl_seconds, 86400); +} diff --git a/tests/threat_feed_dnsbl_persistence.rs b/tests/threat_feed_dnsbl_persistence.rs new file mode 100644 index 0000000..907d5c3 --- /dev/null +++ b/tests/threat_feed_dnsbl_persistence.rs @@ -0,0 +1,195 @@ +use axum::{ + body::{Body, to_bytes}, + http::{Method, Request, StatusCode}, +}; +use serde_json::json; +use std::{ + path::{Path, PathBuf}, + time::{SystemTime, UNIX_EPOCH}, +}; +use tower::ServiceExt; +use waf_ids_ai_soc::{AppConfig, AppState, DnsblEntry, build_app}; + +fn json_request(method: Method, uri: &str, body: serde_json::Value) -> Request { + Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .header("x-admin-token", "secret") + .body(Body::from(body.to_string())) + .unwrap() +} + +fn feed_payload(feed_id: &str, source: &str, addresses: &[&str]) -> serde_json::Value { + json!({ + "feed_id": feed_id, + "source": source, + "ttl_seconds": 3600, + "threats": [], + "dnsbl": addresses.iter().map(|address| json!({ + "address": address, + "code": "127.0.0.2", + "reason": format!("{feed_id} verdict"), + "source": source, + "ttl_seconds": 3600, + "prefix_len": null + })).collect::>() + }) +} + +fn unique_state_path(test_name: &str) -> PathBuf { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + std::env::temp_dir().join(format!( + "wardnet-{test_name}-{}-{nonce}.json", + std::process::id() + )) +} + +fn file_config(path: &Path) -> AppConfig { + AppConfig { + admin_token: Some("secret".to_string()), + state_path: Some(path.to_path_buf()), + dnsbl_origin: AppConfig::DEFAULT_DNSBL_ORIGIN.to_string(), + event_limit: AppConfig::DEFAULT_EVENT_LIMIT, + } +} + +async fn dnsbl(app: &axum::Router) -> Vec { + let response = app + .clone() + .oneshot( + Request::builder() + .method(Method::GET) + .uri("/api/dnsbl") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + serde_json::from_slice(&bytes).unwrap() +} + +async fn import_feed( + app: &axum::Router, + feed_id: &str, + source: &str, + addresses: &[&str], +) -> StatusCode { + app.clone() + .oneshot(json_request( + Method::POST, + "/api/threat-feeds/import", + feed_payload(feed_id, source, addresses), + )) + .await + .unwrap() + .status() +} + +#[tokio::test] +async fn dnsbl_feed_ownership_survives_restart_before_withdrawal() { + let path = unique_state_path("dnsbl-ownership-restart"); + let config = file_config(&path); + let target = "203.0.113.220"; + + let app = build_app(AppState::load(config.clone()).await.unwrap()); + assert_eq!( + import_feed(&app, "feed-a", "feed:a", &[target]).await, + StatusCode::CREATED + ); + drop(app); + + let app = build_app(AppState::load(config).await.unwrap()); + assert_eq!( + import_feed(&app, "feed-a", "feed:a", &["203.0.113.250"]).await, + StatusCode::CREATED + ); + assert!( + !dnsbl(&app) + .await + .iter() + .any(|entry| entry.address.to_string() == target), + "persisted feed ownership must still reap a withdrawn DNSBL address after restart" + ); + + let _ = std::fs::remove_file(path); +} + +#[tokio::test] +async fn predecessor_state_without_dnsbl_ownership_fields_remains_loadable() { + let path = unique_state_path("dnsbl-ownership-legacy-state"); + let config = file_config(&path); + + let state = AppState::load(config.clone()).await.unwrap(); + drop(state); + + let mut legacy: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + let object = legacy.as_object_mut().expect("seeded state is an object"); + // These keys are intentionally absent in the predecessor schema. The GREEN + // implementation must add them with serde defaults rather than requiring a + // one-shot migration that makes an existing Wardnet state file unreadable. + object.remove("operator_dnsbl_keys"); + object.insert( + "threat_feed_ownership".to_string(), + json!([{"feed_id": "legacy-feed", "threat_keys": []}]), + ); + std::fs::write(&path, serde_json::to_vec_pretty(&legacy).unwrap()).unwrap(); + + AppState::load(config) + .await + .expect("pre-DNSBL-ownership state must deserialize with empty ownership defaults"); + + let _ = std::fs::remove_file(path); +} + +#[tokio::test] +async fn failed_persistence_rolls_back_dnsbl_ownership_before_retry() { + let path = unique_state_path("dnsbl-ownership-rollback"); + let backup = path.with_extension("backup.json"); + let config = file_config(&path); + let target = "203.0.113.221"; + let app = build_app(AppState::load(config).await.unwrap()); + + assert_eq!( + import_feed(&app, "feed-a", "feed:a", &[target]).await, + StatusCode::CREATED + ); + + std::fs::rename(&path, &backup).unwrap(); + std::fs::create_dir(&path).unwrap(); + assert_eq!( + import_feed(&app, "feed-a", "feed:a", &["203.0.113.250"]).await, + StatusCode::INTERNAL_SERVER_ERROR, + "replacing a state file with a directory must make persistence fail" + ); + assert!( + dnsbl(&app) + .await + .iter() + .any(|entry| entry.address.to_string() == target), + "failed persistence must restore the complete pre-mutation DNSBL snapshot" + ); + + std::fs::remove_dir(&path).unwrap(); + std::fs::rename(&backup, &path).unwrap(); + assert_eq!( + import_feed(&app, "feed-a", "feed:a", &["203.0.113.250"]).await, + StatusCode::CREATED + ); + assert!( + !dnsbl(&app) + .await + .iter() + .any(|entry| entry.address.to_string() == target), + "retry after rollback must see the original ownership and reap the withdrawn address" + ); + + let _ = std::fs::remove_file(path); + let _ = std::fs::remove_file(backup); +}