From cbeed236e4da98b1e27b864ceedc450c5f33e8a7 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 21:34:17 +0900 Subject: [PATCH 01/23] feat(feeds): refresh official threat sources --- crates/waf-ids-core/src/lib.rs | 97 +++++++ docs/official-threat-feeds.md | 62 ++++ src/credentials.rs | 19 +- src/lib.rs | 499 ++++++++++++++++++++++++++++++++- src/official_feeds.rs | 293 +++++++++++++++++++ 5 files changed, 961 insertions(+), 9 deletions(-) create mode 100644 docs/official-threat-feeds.md create mode 100644 src/official_feeds.rs diff --git a/crates/waf-ids-core/src/lib.rs b/crates/waf-ids-core/src/lib.rs index e378869..8356d93 100644 --- a/crates/waf-ids-core/src/lib.rs +++ b/crates/waf-ids-core/src/lib.rs @@ -22,6 +22,8 @@ pub struct AppData { pub commercial: CommercialProfile, #[serde(default)] pub threat_feeds: Vec, + #[serde(default = "official_threat_feed_registry")] + pub official_threat_feeds: Vec, } impl AppData { @@ -56,10 +58,105 @@ impl AppData { next_audit_log_id: 1, commercial: CommercialProfile::seeded(), threat_feeds: Vec::new(), + official_threat_feeds: official_threat_feed_registry(), } } } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct OfficialThreatFeed { + pub source_id: String, + pub official_url: String, + pub parser: String, + pub indicator_types: Vec, + pub attribution: String, + pub license_url: String, + pub refresh_interval_seconds: u64, + pub ttl_seconds: u64, + #[serde(default)] + pub etag: Option, + #[serde(default)] + pub last_modified: Option, + #[serde(default)] + pub last_attempt_unix: Option, + #[serde(default)] + pub last_success_unix: Option, + #[serde(default)] + pub last_error: Option, + #[serde(default)] + pub source_notice: Option, +} + +pub fn official_threat_feed_registry() -> Vec { + [ + ( + "spamhaus-drop-v4", + "https://www.spamhaus.org/drop/drop_v4.json", + "spamhaus_drop_json", + &["ipv4_cidr"][..], + "The Spamhaus Project", + "https://www.spamhaus.org/blocklists/drop-fair-use-policy/", + ), + ( + "spamhaus-drop-v6", + "https://www.spamhaus.org/drop/drop_v6.json", + "spamhaus_drop_json", + &["ipv6_cidr"][..], + "The Spamhaus Project", + "https://www.spamhaus.org/blocklists/drop-fair-use-policy/", + ), + ( + "urlhaus-online", + "https://urlhaus-api.abuse.ch/v2/files/exports/{AUTH_KEY}/recent.csv", + "urlhaus_recent_csv", + &["url", "domain"][..], + "URLhaus by abuse.ch", + "https://abuse.ch/terms-of-use/", + ), + ( + "threatfox-recent", + "https://threatfox-api.abuse.ch/api/v1/", + "threatfox_json", + &["domain", "ipv4", "ipv6"][..], + "ThreatFox by abuse.ch", + "https://abuse.ch/terms-of-use/", + ), + ] + .into_iter() + .map( + |(source_id, official_url, parser, indicator_types, attribution, license_url)| { + OfficialThreatFeed { + source_id: source_id.to_string(), + official_url: official_url.to_string(), + parser: parser.to_string(), + indicator_types: indicator_types + .iter() + .map(|value| value.to_string()) + .collect(), + attribution: attribution.to_string(), + license_url: license_url.to_string(), + refresh_interval_seconds: if source_id.starts_with("spamhaus") { + 86_400 + } else { + 3_600 + }, + ttl_seconds: if source_id.starts_with("spamhaus") { + 172_800 + } else { + 7_200 + }, + etag: None, + last_modified: None, + last_attempt_unix: None, + last_success_unix: None, + last_error: None, + source_notice: None, + } + }, + ) + .collect() +} + fn initial_audit_log_id() -> u64 { 1 } diff --git a/docs/official-threat-feeds.md b/docs/official-threat-feeds.md new file mode 100644 index 0000000..6d89e17 --- /dev/null +++ b/docs/official-threat-feeds.md @@ -0,0 +1,62 @@ +# Official threat-feed refresh API + +Wardnet keeps operator-supplied `POST /api/threat-feeds/import` separate from a +closed registry of official upstreams. The registry is persisted with source identity, +URL template, parser, supported indicator types, attribution and terms link, HTTP +validators, refresh interval, TTL, last attempt/success/error, and the upstream copyright +notice when supplied. + +| Source id | Upstream contract | Default refresh / TTL | Credential | +| --- | --- | --- | --- | +| `spamhaus-drop-v4` | Spamhaus DROP IPv4 JSON Lines | 24h / 48h | None | +| `spamhaus-drop-v6` | Spamhaus DROP IPv6 JSON Lines | 24h / 48h | None | +| `urlhaus-online` | URLhaus authenticated recent CSV export | 1h / 2h | KV key `urlhaus_auth_key` | +| `threatfox-recent` | ThreatFox `get_iocs`, one-day window | 1h / 2h | KV key `threatfox_auth_key` | + +Spamhaus asks automated users not to fetch more often than hourly and documents daily +refresh as sufficient; Wardnet therefore defaults DROP to daily. abuse.ch exports update +more often, but Wardnet deliberately uses a one-hour floor. A request before the interval +expires returns `429` with `Retry-After`. + +The data is governed by source-specific terms, not by Wardnet's MIT license. Preserve +attribution and review the [Spamhaus DROP fair-use policy](https://www.spamhaus.org/blocklists/drop-fair-use-policy/) +and [abuse.ch terms of use](https://abuse.ch/terms-of-use/) before commercial use or +redistribution. The canonical formats are documented by +[Spamhaus](https://www.spamhaus.org/blocklists/do-not-route-or-peer/), +[URLhaus](https://urlhaus.abuse.ch/api/), and +[ThreatFox](https://threatfox.abuse.ch/api/). + +## Credentials + +Place abuse.ch keys in the JSON credential registry selected by +`WAF_IDS_CREDENTIALS_PATH`; handlers read only `CredentialRegistry::get_credential`. +Keys are never returned by status, written into persisted feed metadata, included in audit +logs, or exposed in request errors. + +```json +{ + "admin_tokens": "operator-token:soc:write,reader-token:auditor:readonly", + "urlhaus_auth_key": "...", + "threatfox_auth_key": "..." +} +``` + +## Endpoints + +`GET /api/official-threat-feeds` requires any authenticated admin principal and returns +registry/status metadata without secrets. + +`POST /api/official-threat-feeds/{source_id}/refresh` requires a write-capable principal. +Unknown/non-canonical sources are rejected. Wardnet sends `If-None-Match` and +`If-Modified-Since` when validators exist. A `304` refreshes freshness evidence without +rewriting indicator content. + +On a successful `200`, Wardnet parses and validates the entire response before one +persistent mutation replaces only rows owned by that source, updates the existing +`ThreatFeedStatus` freshness record, stores validators/status, and writes an audit entry. +Network, HTTP, body, parsing, or validation failure records `last_error` but retains the +last-known-good threat/DNSBL rows unchanged. + +These are threat-intelligence feeds consumed by gateway scoring and the authoritative +DNSBL zone export. They are not recursive DNS resolvers and do not configure an upstream +DNS resolver. diff --git a/src/credentials.rs b/src/credentials.rs index 02b7f39..9b26ec6 100644 --- a/src/credentials.rs +++ b/src/credentials.rs @@ -11,6 +11,8 @@ use std::{collections::HashMap, io::ErrorKind, path::Path}; /// Well-known secret keys loaded into the registry at bootstrap. pub const CRED_ADMIN_TOKEN: &str = "admin_token"; pub const CRED_ADMIN_TOKENS: &str = "admin_tokens"; +pub const CRED_THREATFOX_AUTH_KEY: &str = "threatfox_auth_key"; +pub const CRED_URLHAUS_AUTH_KEY: &str = "urlhaus_auth_key"; /// Where secret-bearing credentials were loaded from (never includes values). #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] @@ -88,7 +90,12 @@ impl CredentialRegistry { path.display() ) })?; - for key in [CRED_ADMIN_TOKEN, CRED_ADMIN_TOKENS] { + for key in [ + CRED_ADMIN_TOKEN, + CRED_ADMIN_TOKENS, + CRED_THREATFOX_AUTH_KEY, + CRED_URLHAUS_AUTH_KEY, + ] { if let Some(raw) = file_map.get(key) { let text = json_value_as_nonempty_string(raw); if let Some(text) = text { @@ -193,7 +200,7 @@ mod tests { let mut file = std::fs::File::create(&path).unwrap(); write!( file, - r#"{{"admin_token":"from-file","admin_tokens":"filetok:operator"}}"# + r#"{{"admin_token":"from-file","admin_tokens":"filetok:operator","urlhaus_auth_key":"urlhaus-secret","threatfox_auth_key":"threatfox-secret"}}"# ) .unwrap(); drop(file); @@ -210,6 +217,14 @@ mod tests { registry.get_credential(CRED_ADMIN_TOKENS), Some("filetok:operator") ); + assert_eq!( + registry.get_credential(CRED_URLHAUS_AUTH_KEY), + Some("urlhaus-secret") + ); + assert_eq!( + registry.get_credential(CRED_THREATFOX_AUTH_KEY), + Some("threatfox-secret") + ); let _ = std::fs::remove_dir_all(&dir); } diff --git a/src/lib.rs b/src/lib.rs index 8f54751..7cea632 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,7 @@ use axum::{ Json, Router, body::Bytes, extract::{DefaultBodyLimit, Path as PathParam, Query, State}, - http::{HeaderMap, Method, StatusCode, Uri}, + http::{HeaderMap, Method, StatusCode, Uri, header}, response::{Html, IntoResponse, Response}, routing::{any, get, post}, }; @@ -29,20 +29,25 @@ use waf_ids_core::{ pub use waf_ids_core::{ AuditLogEntry, BuyerEvidenceEndpoint, BuyerEvidenceManifest, BuyerEvidenceRuntimeCounts, CommercialProfile, CommercialReadiness, DnsblEntry, EnforcementMode, LicenseStatus, - NewAuditLogEntry, ProductEdition, ReadinessCheck, ReadinessStatus, RouteConfig, ScoredRequest, - SecurityEvent, Severity, SignatureInfo, SocKpiSnapshot, TARGET_SALE_VALUE_KRW, - ThreatFeedFreshness, ThreatFeedImport, ThreatFeedImportResult, ThreatFeedStatus, - ThreatIndicator, export_dnsbl_zone, ip_in_network, reverse_ipv4_for_dnsbl, score_request, + NewAuditLogEntry, OfficialThreatFeed, ProductEdition, ReadinessCheck, ReadinessStatus, + RouteConfig, ScoredRequest, SecurityEvent, Severity, SignatureInfo, SocKpiSnapshot, + TARGET_SALE_VALUE_KRW, ThreatFeedFreshness, ThreatFeedImport, ThreatFeedImportResult, + ThreatFeedStatus, ThreatIndicator, export_dnsbl_zone, ip_in_network, reverse_ipv4_for_dnsbl, + score_request, }; mod coraza_audit; mod credentials; mod misp_import; +mod official_feeds; mod opencti_import; mod stix_import; mod suricata_eve; mod taxii; -pub use credentials::{CRED_ADMIN_TOKEN, CRED_ADMIN_TOKENS, CredentialRegistry, CredentialSource}; +pub use credentials::{ + CRED_ADMIN_TOKEN, CRED_ADMIN_TOKENS, CRED_THREATFOX_AUTH_KEY, CRED_URLHAUS_AUTH_KEY, + CredentialRegistry, CredentialSource, +}; #[derive(Clone)] pub struct AppState { @@ -56,6 +61,11 @@ pub struct AppState { admin_tokens: HashMap, /// Where admin secrets were bootstrapped from (file/env/none). Never holds values. credentials_source: CredentialSource, + credentials: CredentialRegistry, + // ponytail: one refresh lock; split per source if concurrent feed refresh throughput matters. + official_feed_refresh_lock: Arc>, + #[cfg(test)] + official_feed_url_overrides: HashMap, state_path: Option, dnsbl_origin: String, event_limit: usize, @@ -126,6 +136,10 @@ impl AppState { admin_token: config.admin_token, admin_tokens: HashMap::new(), credentials_source: CredentialSource::None, + credentials: CredentialRegistry::empty(), + official_feed_refresh_lock: Arc::new(Mutex::new(())), + #[cfg(test)] + official_feed_url_overrides: HashMap::new(), state_path: config.state_path, dnsbl_origin: normalized_origin(&config.dnsbl_origin), event_limit: config.event_limit.max(1), @@ -181,6 +195,19 @@ impl AppState { self } + pub fn with_credential_registry(mut self, credentials: CredentialRegistry) -> Self { + self.credentials_source = credentials.source(); + self.credentials = credentials; + self + } + + #[cfg(test)] + fn with_official_feed_url(mut self, source_id: &str, url: String) -> Self { + self.official_feed_url_overrides + .insert(source_id.to_string(), url); + self + } + /// The principal mapped to the request's `X-Admin-Token`, if configured. fn principal_for_token(&self, headers: &HeaderMap) -> Option<&AdminPrincipal> { headers @@ -463,6 +490,14 @@ pub fn build_app(state: AppState) -> Router { .route("/api/threat-feeds", get(list_threat_feeds)) .route("/api/threat-feeds/freshness", get(threat_feed_freshness)) .route("/api/threat-feeds/import", post(import_threat_feed)) + .route( + "/api/official-threat-feeds", + get(list_official_threat_feeds), + ) + .route( + "/api/official-threat-feeds/{source_id}/refresh", + post(refresh_official_threat_feed), + ) .route( "/api/threat-feeds/import/phishing-database", post(import_phishing_database_feed), @@ -1124,6 +1159,332 @@ async fn import_threat_feed( } } +async fn list_official_threat_feeds(State(state): State, headers: HeaderMap) -> Response { + if !official_feed_authenticated(&state, &headers) { + return error(StatusCode::UNAUTHORIZED, "missing or invalid X-Admin-Token"); + } + Json(state.inner.read().await.official_threat_feeds.clone()).into_response() +} + +#[derive(Debug, Serialize)] +struct OfficialFeedRefreshResult { + source_id: String, + not_modified: bool, + threat_count: usize, + dnsbl_count: usize, + refreshed_at_unix: u64, +} + +async fn refresh_official_threat_feed( + State(state): State, + PathParam(source_id): PathParam, + headers: HeaderMap, +) -> Response { + if !official_feed_authenticated(&state, &headers) { + return error(StatusCode::UNAUTHORIZED, "missing or invalid X-Admin-Token"); + } + if !admin_authorized(&state, &headers) { + return error(StatusCode::FORBIDDEN, "admin principal is read-only"); + } + let _refresh_guard = state.official_feed_refresh_lock.lock().await; + let now = now_unix(); + let feed = { + let data = state.inner.read().await; + data.official_threat_feeds + .iter() + .find(|feed| feed.source_id == source_id) + .cloned() + }; + let Some(feed) = feed else { + return error(StatusCode::NOT_FOUND, "unknown official threat feed source"); + }; + let Some(canonical) = waf_ids_core::official_threat_feed_registry() + .into_iter() + .find(|item| item.source_id == feed.source_id) + else { + return error(StatusCode::NOT_FOUND, "unknown official threat feed source"); + }; + if feed.official_url != canonical.official_url || feed.parser != canonical.parser { + return error( + StatusCode::CONFLICT, + "official feed registry metadata does not match the built-in source", + ); + } + if let Some(last_attempt) = feed.last_attempt_unix + && now.saturating_sub(last_attempt) < feed.refresh_interval_seconds + { + let retry_after = feed + .refresh_interval_seconds + .saturating_sub(now.saturating_sub(last_attempt)); + return ( + StatusCode::TOO_MANY_REQUESTS, + [(header::RETRY_AFTER, retry_after.to_string())], + Json(ErrorBody { + error: "official feed refresh interval has not elapsed".to_string(), + }), + ) + .into_response(); + } + + let configured_url = { + #[cfg(test)] + { + state + .official_feed_url_overrides + .get(&source_id) + .cloned() + .unwrap_or_else(|| feed.official_url.clone()) + } + #[cfg(not(test))] + { + feed.official_url.clone() + } + }; + let (url, auth_header, request_body) = match feed.source_id.as_str() { + "urlhaus-online" => { + let Some(key) = state.credentials.get_credential(CRED_URLHAUS_AUTH_KEY) else { + return official_feed_failure( + &state, + &source_id, + now, + "URLhaus credential is unavailable", + ) + .await; + }; + (configured_url.replace("{AUTH_KEY}", key), None, None) + } + "threatfox-recent" => { + let Some(key) = state.credentials.get_credential(CRED_THREATFOX_AUTH_KEY) else { + return official_feed_failure( + &state, + &source_id, + now, + "ThreatFox credential is unavailable", + ) + .await; + }; + ( + configured_url, + Some(key.to_string()), + Some(serde_json::json!({"query": "get_iocs", "days": 1})), + ) + } + _ => (configured_url, None, None), + }; + + let mut request = if let Some(body) = request_body { + state.feed_http.post(&url).json(&body) + } else { + state.feed_http.get(&url) + }; + if let Some(key) = auth_header { + request = request.header("Auth-Key", key); + } + if let Some(etag) = &feed.etag { + request = request.header(header::IF_NONE_MATCH, etag); + } + if let Some(last_modified) = &feed.last_modified { + request = request.header(header::IF_MODIFIED_SINCE, last_modified); + } + let response = match request.send().await { + Ok(response) => response, + Err(_) => { + return official_feed_failure(&state, &source_id, now, "official feed request failed") + .await; + } + }; + let etag = response + .headers() + .get(header::ETAG) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + let last_modified = response + .headers() + .get(header::LAST_MODIFIED) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + if response.status() == reqwest::StatusCode::NOT_MODIFIED { + return finish_official_feed_not_modified( + &state, + &source_id, + now, + etag, + last_modified, + audit_actor(&state, &headers), + ) + .await; + } + if !response.status().is_success() { + let message = format!("official feed returned HTTP {}", response.status().as_u16()); + return official_feed_failure(&state, &source_id, now, &message).await; + } + let body = match response.text().await { + Ok(body) => body, + Err(_) => { + return official_feed_failure( + &state, + &source_id, + now, + "official feed body read failed", + ) + .await; + } + }; + let parsed = match official_feeds::parse(&feed.parser, &source_id, feed.ttl_seconds, &body) { + Ok(parsed) => parsed, + Err(message) => return official_feed_failure(&state, &source_id, now, &message).await, + }; + let import = ThreatFeedImport { + feed_id: source_id.clone(), + source: feed.attribution.clone(), + ttl_seconds: feed.ttl_seconds, + threats: parsed.threats, + dnsbl: parsed.dnsbl, + }; + if let Err(message) = validate_threat_feed_import(&import) { + return official_feed_failure(&state, &source_id, now, message).await; + } + let actor = audit_actor(&state, &headers); + match state + .mutate_and_persist(|data| { + data.threats.retain(|item| item.source != source_id); + data.dnsbl.retain(|item| item.source != source_id); + let threat_count = import.threats.len(); + let dnsbl_count = import.dnsbl.len(); + for threat in import.threats { + upsert_threat(&mut data.threats, threat); + } + for entry in import.dnsbl { + upsert_dnsbl(&mut data.dnsbl, entry); + } + upsert_threat_feed( + &mut data.threat_feeds, + ThreatFeedStatus { + feed_id: source_id.clone(), + source: feed.attribution.clone(), + last_updated_unix: now, + threat_count, + dnsbl_count, + ttl_seconds: feed.ttl_seconds, + }, + ); + if let Some(status) = data + .official_threat_feeds + .iter_mut() + .find(|item| item.source_id == source_id) + { + status.etag = etag; + status.last_modified = last_modified; + status.last_attempt_unix = Some(now); + status.last_success_unix = Some(now); + status.last_error = None; + status.source_notice = parsed.source_notice; + } + record_successful_audit_log( + data, + actor, + "refresh_official_threat_feed", + "official_threat_feed", + source_id.clone(), + ); + OfficialFeedRefreshResult { + source_id, + not_modified: false, + threat_count, + dnsbl_count, + refreshed_at_unix: now, + } + }) + .await + { + Ok(result) => Json(result).into_response(), + Err(message) => error(StatusCode::INTERNAL_SERVER_ERROR, message), + } +} + +async fn finish_official_feed_not_modified( + state: &AppState, + source_id: &str, + now: u64, + etag: Option, + last_modified: Option, + actor: String, +) -> Response { + match state + .mutate_and_persist(|data| { + let feed_status = data + .threat_feeds + .iter_mut() + .find(|item| item.feed_id == source_id); + let (threat_count, dnsbl_count) = feed_status + .map(|status| { + status.last_updated_unix = now; + (status.threat_count, status.dnsbl_count) + }) + .unwrap_or((0, 0)); + if let Some(status) = data + .official_threat_feeds + .iter_mut() + .find(|item| item.source_id == source_id) + { + status.etag = etag.or_else(|| status.etag.clone()); + status.last_modified = last_modified.or_else(|| status.last_modified.clone()); + status.last_attempt_unix = Some(now); + status.last_success_unix = Some(now); + status.last_error = None; + } + record_successful_audit_log( + data, + actor, + "refresh_official_threat_feed_not_modified", + "official_threat_feed", + source_id.to_string(), + ); + OfficialFeedRefreshResult { + source_id: source_id.to_string(), + not_modified: true, + threat_count, + dnsbl_count, + refreshed_at_unix: now, + } + }) + .await + { + Ok(result) => Json(result).into_response(), + Err(message) => error(StatusCode::INTERNAL_SERVER_ERROR, message), + } +} + +async fn official_feed_failure( + state: &AppState, + source_id: &str, + now: u64, + message: &str, +) -> Response { + let sanitized = message.replace('\n', " "); + let persisted = state + .mutate_and_persist(|data| { + if let Some(status) = data + .official_threat_feeds + .iter_mut() + .find(|item| item.source_id == source_id) + { + status.last_attempt_unix = Some(now); + status.last_error = Some(sanitized.clone()); + } + }) + .await; + match persisted { + Ok(()) => error(StatusCode::BAD_GATEWAY, sanitized), + Err(error_message) => error(StatusCode::INTERNAL_SERVER_ERROR, error_message), + } +} + +fn official_feed_authenticated(state: &AppState, headers: &HeaderMap) -> bool { + (state.admin_token.is_some() || !state.admin_tokens.is_empty()) + && admin_authenticated(state, headers) +} + /// Optional STIX import metadata via query string. #[derive(Debug, Deserialize)] struct StixImportQuery { @@ -3044,7 +3405,7 @@ pub async fn run_from_env( .map_err(|message| std::io::Error::new(std::io::ErrorKind::InvalidData, message))? .with_rate_limit(rate_limit, rate_limit_window) .with_admin_tokens(admin_tokens) - .with_credentials_source(credentials.source()) + .with_credential_registry(credentials) .with_max_body_size(max_body_bytes); let served = axum::serve(listener, build_app(state)) .with_graceful_shutdown(shutdown) @@ -3065,6 +3426,10 @@ mod tests { future::IntoFuture, io::{Read, Write}, net::TcpListener as StdTcpListener, + sync::{ + Arc as StdArc, + atomic::{AtomicUsize, Ordering}, + }, thread, time::{SystemTime, UNIX_EPOCH}, }; @@ -3136,6 +3501,124 @@ mod tests { assert_eq!(ips[1], "203.0.113.5".parse::().unwrap()); } + #[tokio::test] + async fn official_feed_refresh_atomically_keeps_last_known_good() { + let unconfigured = build_app(AppState::seeded(None)); + assert_eq!( + app_request( + &unconfigured, + empty_request(Method::GET, "/api/official-threat-feeds") + ) + .await + .status(), + StatusCode::UNAUTHORIZED + ); + let calls = StdArc::new(AtomicUsize::new(0)); + let mock_calls = calls.clone(); + let upstream = Router::new().route( + "/drop.json", + get(move || { + let call = mock_calls.fetch_add(1, Ordering::SeqCst); + async move { + if call == 0 { + ( + StatusCode::OK, + [(header::ETAG, "\"drop-v1\"")], + "{\"cidr\":\"198.51.100.0/24\",\"sblid\":\"SBL1\"}\n{\"type\":\"metadata\",\"copyright\":\"Copyright Spamhaus\"}\n", + ) + .into_response() + } else { + (StatusCode::OK, "not-json").into_response() + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + tokio::spawn(axum::serve(listener, upstream).into_future()); + + let state = AppState::seeded(Some("secret".to_string())) + .with_official_feed_url("spamhaus-drop-v4", format!("http://{address}/drop.json")); + let inspect = state.clone(); + let app = build_app(state); + assert_eq!( + app_request( + &app, + empty_request(Method::GET, "/api/official-threat-feeds") + ) + .await + .status(), + StatusCode::UNAUTHORIZED + ); + assert_eq!( + app_request( + &app, + authed_empty_request(Method::GET, "/api/official-threat-feeds", "secret") + ) + .await + .status(), + StatusCode::OK + ); + + let refreshed = app_request( + &app, + authed_empty_request( + Method::POST, + "/api/official-threat-feeds/spamhaus-drop-v4/refresh", + "secret", + ), + ) + .await; + assert_eq!(refreshed.status(), StatusCode::OK); + let first = inspect.inner.read().await.clone(); + assert!( + first.dnsbl.iter().any(|entry| { + entry.source == "spamhaus-drop-v4" && entry.prefix_len == Some(24) + }) + ); + let source = first + .official_threat_feeds + .iter() + .find(|feed| feed.source_id == "spamhaus-drop-v4") + .unwrap(); + assert_eq!(source.etag.as_deref(), Some("\"drop-v1\"")); + assert_eq!(source.source_notice.as_deref(), Some("Copyright Spamhaus")); + + inspect + .inner + .write() + .await + .official_threat_feeds + .iter_mut() + .find(|feed| feed.source_id == "spamhaus-drop-v4") + .unwrap() + .last_attempt_unix = None; + let failed = app_request( + &app, + authed_empty_request( + Method::POST, + "/api/official-threat-feeds/spamhaus-drop-v4/refresh", + "secret", + ), + ) + .await; + assert_eq!(failed.status(), StatusCode::BAD_GATEWAY); + let after = inspect.inner.read().await; + assert_eq!( + after.dnsbl, first.dnsbl, + "failed parse must preserve LKG data" + ); + assert!( + after + .official_threat_feeds + .iter() + .find(|feed| feed.source_id == "spamhaus-drop-v4") + .unwrap() + .last_error + .is_some() + ); + } + #[tokio::test] async fn run_from_env_binds_and_serves_until_shutdown() { let _guard = ENV_GUARD.lock().await; @@ -5553,6 +6036,7 @@ mod tests { next_audit_log_id: 1, commercial: CommercialProfile::seeded(), threat_feeds: Vec::new(), + official_threat_feeds: waf_ids_core::official_threat_feed_registry(), }, AppConfig { admin_token: None, @@ -6428,6 +6912,7 @@ mod tests { next_audit_log_id: 1, commercial: CommercialProfile::seeded(), threat_feeds: Vec::new(), + official_threat_feeds: waf_ids_core::official_threat_feed_registry(), }, AppConfig { admin_token: None, diff --git a/src/official_feeds.rs b/src/official_feeds.rs new file mode 100644 index 0000000..52ff84e --- /dev/null +++ b/src/official_feeds.rs @@ -0,0 +1,293 @@ +use crate::{DnsblEntry, Severity, ThreatIndicator}; +use serde_json::Value; +use std::net::IpAddr; + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct ParsedOfficialFeed { + pub threats: Vec, + pub dnsbl: Vec, + pub source_notice: Option, +} + +pub fn parse( + parser: &str, + source_id: &str, + ttl_seconds: u64, + body: &str, +) -> Result { + match parser { + "spamhaus_drop_json" => parse_spamhaus(source_id, ttl_seconds, body), + "urlhaus_recent_csv" => parse_urlhaus(source_id, ttl_seconds, body), + "threatfox_json" => parse_threatfox(source_id, ttl_seconds, body), + _ => Err(format!("unsupported official feed parser: {parser}")), + } +} + +fn parse_spamhaus( + source_id: &str, + ttl_seconds: u64, + body: &str, +) -> Result { + let mut parsed = ParsedOfficialFeed::default(); + for (line_number, line) in body.lines().enumerate() { + let line = line.trim(); + if line.is_empty() { + continue; + } + let value: Value = serde_json::from_str(line).map_err(|error| { + format!("invalid Spamhaus JSON on line {}: {error}", line_number + 1) + })?; + if value.get("type").is_some() { + parsed.source_notice = value + .get("copyright") + .and_then(Value::as_str) + .map(str::to_string); + continue; + } + let cidr = value + .get("cidr") + .and_then(Value::as_str) + .ok_or_else(|| format!("Spamhaus record {} is missing cidr", line_number + 1))?; + let (address, prefix_len) = parse_cidr(cidr)?; + let sblid = value.get("sblid").and_then(Value::as_str).unwrap_or("DROP"); + parsed.dnsbl.push(DnsblEntry { + address, + prefix_len: Some(prefix_len), + code: "127.0.0.2".to_string(), + reason: format!("Spamhaus DROP {sblid}"), + source: source_id.to_string(), + ttl_seconds, + }); + } + require_material(parsed, source_id) +} + +fn parse_urlhaus( + source_id: &str, + ttl_seconds: u64, + body: &str, +) -> Result { + let mut parsed = ParsedOfficialFeed::default(); + for line in body.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let fields = csv_fields(line)?; + let Some(url) = fields + .iter() + .find(|field| field.starts_with("http://") || field.starts_with("https://")) + else { + continue; + }; + parsed + .threats + .push(threat(url, "url", source_id, ttl_seconds)); + let host = reqwest::Url::parse(url) + .ok() + .and_then(|url| url.host_str().map(str::to_ascii_lowercase)); + if let Some(host) = host { + if let Ok(address) = host.parse::() { + parsed + .threats + .push(threat(&host, "client_ip", source_id, ttl_seconds)); + parsed.dnsbl.push(DnsblEntry { + address, + prefix_len: None, + code: "127.0.0.2".to_string(), + reason: "URLhaus malware URL host".to_string(), + source: source_id.to_string(), + ttl_seconds, + }); + } else { + parsed + .threats + .push(threat(&host, "domain", source_id, ttl_seconds)); + } + } + } + require_material(parsed, source_id) +} + +fn parse_threatfox( + source_id: &str, + ttl_seconds: u64, + body: &str, +) -> Result { + let value: Value = + serde_json::from_str(body).map_err(|error| format!("invalid ThreatFox JSON: {error}"))?; + if value.get("query_status").and_then(Value::as_str) != Some("ok") { + return Err(format!( + "ThreatFox query failed: {}", + value + .get("query_status") + .and_then(Value::as_str) + .unwrap_or("missing query_status") + )); + } + let records = value + .get("data") + .and_then(Value::as_array) + .ok_or_else(|| "ThreatFox response is missing data".to_string())?; + let mut parsed = ParsedOfficialFeed::default(); + for record in records { + let Some(ioc) = record.get("ioc").and_then(Value::as_str) else { + continue; + }; + let ioc_type = record.get("ioc_type").and_then(Value::as_str).unwrap_or(""); + if ioc_type == "domain" { + parsed + .threats + .push(threat(ioc, "domain", source_id, ttl_seconds)); + } else if ioc_type.starts_with("ip") { + let Some(address) = threatfox_ip(ioc) else { + continue; + }; + parsed.threats.push(threat( + &address.to_string(), + "client_ip", + source_id, + ttl_seconds, + )); + parsed.dnsbl.push(DnsblEntry { + address, + prefix_len: None, + code: "127.0.0.2".to_string(), + reason: format!("ThreatFox {ioc_type}: {ioc}"), + source: source_id.to_string(), + ttl_seconds, + }); + } + } + require_material(parsed, source_id) +} + +fn threat(value: &str, indicator_type: &str, source: &str, ttl_seconds: u64) -> ThreatIndicator { + ThreatIndicator { + value: value.to_string(), + indicator_type: indicator_type.to_string(), + severity: Severity::High, + source: source.to_string(), + ttl_seconds, + } +} + +fn require_material( + parsed: ParsedOfficialFeed, + source_id: &str, +) -> Result { + if parsed.threats.is_empty() && parsed.dnsbl.is_empty() { + Err(format!( + "official feed {source_id} contained no supported indicators" + )) + } else { + Ok(parsed) + } +} + +fn parse_cidr(value: &str) -> Result<(IpAddr, u8), String> { + let (address, prefix) = value + .split_once('/') + .ok_or_else(|| format!("invalid CIDR: {value}"))?; + let address = address + .parse::() + .map_err(|_| format!("invalid CIDR address: {value}"))?; + let prefix = prefix + .parse::() + .map_err(|_| format!("invalid CIDR prefix: {value}"))?; + let maximum = if address.is_ipv4() { 32 } else { 128 }; + if prefix > maximum { + return Err(format!("invalid CIDR prefix: {value}")); + } + Ok((address, prefix)) +} + +fn threatfox_ip(value: &str) -> Option { + value + .parse() + .ok() + .or_else(|| { + value + .parse::() + .ok() + .map(|socket| socket.ip()) + }) + .or_else(|| value.rsplit_once(':').and_then(|(ip, _)| ip.parse().ok())) +} + +fn csv_fields(line: &str) -> Result, String> { + let mut fields = Vec::new(); + let mut field = String::new(); + let mut quoted = false; + let mut chars = line.chars().peekable(); + while let Some(character) = chars.next() { + match character { + '"' if quoted && chars.peek() == Some(&'"') => { + field.push('"'); + chars.next(); + } + '"' => quoted = !quoted, + ',' if !quoted => fields.push(std::mem::take(&mut field)), + other => field.push(other), + } + } + if quoted { + return Err("unterminated quoted CSV field".to_string()); + } + fields.push(field); + Ok(fields) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_official_feed_shapes_and_rejects_empty_material() { + let spamhaus = parse( + "spamhaus_drop_json", + "spamhaus-drop-v4", + 7200, + "{\"cidr\":\"192.0.2.0/24\",\"sblid\":\"SBL1\"}\n{\"type\":\"metadata\",\"copyright\":\"Copyright Spamhaus\"}\n", + ) + .unwrap(); + assert_eq!(spamhaus.dnsbl[0].prefix_len, Some(24)); + assert_eq!( + spamhaus.source_notice.as_deref(), + Some("Copyright Spamhaus") + ); + + let urlhaus = parse( + "urlhaus_recent_csv", + "urlhaus-online", + 7200, + "# attribution\n1,2026-01-01,https://evil.example/a,online\n", + ) + .unwrap(); + assert!( + urlhaus + .threats + .iter() + .any(|item| item.indicator_type == "domain") + ); + + let threatfox = parse( + "threatfox_json", + "threatfox-recent", + 7200, + r#"{"query_status":"ok","data":[{"ioc":"198.51.100.9:443","ioc_type":"ip:port"},{"ioc":"[2001:db8::9]:443","ioc_type":"ip:port"},{"ioc":"bad.example","ioc_type":"domain"}]}"#, + ) + .unwrap(); + assert_eq!(threatfox.dnsbl[0].address.to_string(), "198.51.100.9"); + assert_eq!(threatfox.dnsbl[1].address.to_string(), "2001:db8::9"); + assert!( + parse( + "threatfox_json", + "x", + 1, + r#"{"query_status":"no_results","data":[]}"# + ) + .is_err() + ); + } +} From 412de1e6339f16efd1636818bc1825a2f619a667 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 22:01:49 +0900 Subject: [PATCH 02/23] fix(feeds): bound refresh failures --- docs/official-threat-feeds.md | 4 +- src/credentials.rs | 36 +++++++++- src/lib.rs | 126 +++++++++++++++++++++++++++++----- src/official_feeds.rs | 10 +++ 4 files changed, 156 insertions(+), 20 deletions(-) diff --git a/docs/official-threat-feeds.md b/docs/official-threat-feeds.md index 6d89e17..a10513e 100644 --- a/docs/official-threat-feeds.md +++ b/docs/official-threat-feeds.md @@ -55,7 +55,9 @@ On a successful `200`, Wardnet parses and validates the entire response before o persistent mutation replaces only rows owned by that source, updates the existing `ThreatFeedStatus` freshness record, stores validators/status, and writes an audit entry. Network, HTTP, body, parsing, or validation failure records `last_error` but retains the -last-known-good threat/DNSBL rows unchanged. +last-known-good threat/DNSBL rows unchanged. Requests have a 15-second total timeout and +responses are streamed with an 8 MiB hard limit. Credential preflight failures do not consume +the source refresh interval, so adding a missing key permits an immediate corrective retry. These are threat-intelligence feeds consumed by gateway scoring and the authoritative DNSBL zone export. They are not recursive DNS resolvers and do not configure an upstream diff --git a/src/credentials.rs b/src/credentials.rs index 9b26ec6..ececdd7 100644 --- a/src/credentials.rs +++ b/src/credentials.rs @@ -22,6 +22,8 @@ pub enum CredentialSource { File, /// Secrets came only from env bootstrap (`ADMIN_TOKEN` / `ADMIN_TOKENS`). Env, + /// Secrets were combined from file and environment bootstrap transports. + Mixed, /// No admin secrets configured. #[default] None, @@ -32,6 +34,7 @@ impl CredentialSource { match self { Self::File => "file", Self::Env => "env", + Self::Mixed => "mixed", Self::None => "none", } } @@ -128,7 +131,9 @@ impl CredentialRegistry { from_env = true; } - let source = if from_file { + let source = if from_file && from_env { + CredentialSource::Mixed + } else if from_file { CredentialSource::File } else if from_env { CredentialSource::Env @@ -249,7 +254,7 @@ mod tests { Some("envtok:bob".to_string()), ) .unwrap(); - assert_eq!(registry.source(), CredentialSource::File); + assert_eq!(registry.source(), CredentialSource::Mixed); assert_eq!(registry.get_credential(CRED_ADMIN_TOKEN), Some("file-only")); assert_eq!( registry.get_credential(CRED_ADMIN_TOKENS), @@ -259,6 +264,33 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + #[test] + fn reports_mixed_when_feed_keys_are_file_and_admin_auth_is_env() { + let dir = std::env::temp_dir().join(format!( + "wardnet-creds-mixed-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("credentials.json"); + std::fs::write(&path, r#"{"threatfox_auth_key":"feed-key"}"#).unwrap(); + + let registry = + CredentialRegistry::bootstrap_secrets(Some(&path), Some("env-admin".to_string()), None) + .unwrap(); + assert_eq!(registry.source(), CredentialSource::Mixed); + assert_eq!(registry.get_credential(CRED_ADMIN_TOKEN), Some("env-admin")); + assert_eq!( + registry.get_credential(CRED_THREATFOX_AUTH_KEY), + Some("feed-key") + ); + + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn missing_credentials_file_falls_back_to_env() { let path = std::env::temp_dir().join(format!( diff --git a/src/lib.rs b/src/lib.rs index 7cea632..3d4d2f5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -417,6 +417,10 @@ const PHISHING_DATABASE_DNSBL_CODE: &str = "127.0.0.66"; const PHISHING_DATABASE_DNSBL_REASON: &str = "phishing.database active IP"; const PHISHING_DATABASE_FETCH_TIMEOUT_SECS: u64 = 15; const PHISHING_DATABASE_MAX_BODY_BYTES: usize = 8 * 1024 * 1024; +#[cfg(not(test))] +const OFFICIAL_FEED_FETCH_TIMEOUT_SECS: u64 = PHISHING_DATABASE_FETCH_TIMEOUT_SECS; +#[cfg(test)] +const OFFICIAL_FEED_FETCH_TIMEOUT_SECS: u64 = 1; const PHISHING_DATABASE_ALLOWED_HOSTS: &[&str] = &["raw.githubusercontent.com", "phish.co.za"]; fn phishing_database_default_feed_id() -> String { @@ -1248,6 +1252,7 @@ async fn refresh_official_threat_feed( &source_id, now, "URLhaus credential is unavailable", + false, ) .await; }; @@ -1260,6 +1265,7 @@ async fn refresh_official_threat_feed( &source_id, now, "ThreatFox credential is unavailable", + false, ) .await; }; @@ -1286,11 +1292,20 @@ async fn refresh_official_threat_feed( if let Some(last_modified) = &feed.last_modified { request = request.header(header::IF_MODIFIED_SINCE, last_modified); } + request = request.timeout(std::time::Duration::from_secs( + OFFICIAL_FEED_FETCH_TIMEOUT_SECS, + )); let response = match request.send().await { Ok(response) => response, Err(_) => { - return official_feed_failure(&state, &source_id, now, "official feed request failed") - .await; + return official_feed_failure( + &state, + &source_id, + now, + "official feed request failed", + true, + ) + .await; } }; let etag = response @@ -1316,23 +1331,19 @@ async fn refresh_official_threat_feed( } if !response.status().is_success() { let message = format!("official feed returned HTTP {}", response.status().as_u16()); - return official_feed_failure(&state, &source_id, now, &message).await; + return official_feed_failure(&state, &source_id, now, &message, true).await; } - let body = match response.text().await { + let body = match read_official_feed_body(response).await { Ok(body) => body, - Err(_) => { - return official_feed_failure( - &state, - &source_id, - now, - "official feed body read failed", - ) - .await; + Err(message) => { + return official_feed_failure(&state, &source_id, now, &message, true).await; } }; let parsed = match official_feeds::parse(&feed.parser, &source_id, feed.ttl_seconds, &body) { Ok(parsed) => parsed, - Err(message) => return official_feed_failure(&state, &source_id, now, &message).await, + Err(message) => { + return official_feed_failure(&state, &source_id, now, &message, true).await; + } }; let import = ThreatFeedImport { feed_id: source_id.clone(), @@ -1342,7 +1353,7 @@ async fn refresh_official_threat_feed( dnsbl: parsed.dnsbl, }; if let Err(message) = validate_threat_feed_import(&import) { - return official_feed_failure(&state, &source_id, now, message).await; + return official_feed_failure(&state, &source_id, now, message, true).await; } let actor = audit_actor(&state, &headers); match state @@ -1460,6 +1471,7 @@ async fn official_feed_failure( source_id: &str, now: u64, message: &str, + attempted_upstream: bool, ) -> Response { let sanitized = message.replace('\n', " "); let persisted = state @@ -1469,7 +1481,9 @@ async fn official_feed_failure( .iter_mut() .find(|item| item.source_id == source_id) { - status.last_attempt_unix = Some(now); + if attempted_upstream { + status.last_attempt_unix = Some(now); + } status.last_error = Some(sanitized.clone()); } }) @@ -1480,6 +1494,30 @@ async fn official_feed_failure( } } +async fn read_official_feed_body(response: reqwest::Response) -> Result { + use futures_util::StreamExt; + + if let Some(length) = response.content_length() + && length as usize > PHISHING_DATABASE_MAX_BODY_BYTES + { + return Err(format!( + "official feed body too large: {length} bytes (limit: {PHISHING_DATABASE_MAX_BODY_BYTES})" + )); + } + let mut bytes = Vec::new(); + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|_| "official feed body read failed".to_string())?; + if bytes.len().saturating_add(chunk.len()) > PHISHING_DATABASE_MAX_BODY_BYTES { + return Err(format!( + "official feed body too large: limit {PHISHING_DATABASE_MAX_BODY_BYTES} bytes exceeded while streaming" + )); + } + bytes.extend_from_slice(&chunk); + } + String::from_utf8(bytes).map_err(|_| "official feed body is not valid UTF-8".to_string()) +} + fn official_feed_authenticated(state: &AppState, headers: &HeaderMap) -> bool { (state.admin_token.is_some() || !state.admin_tokens.is_empty()) && admin_authenticated(state, headers) @@ -3513,6 +3551,55 @@ mod tests { .status(), StatusCode::UNAUTHORIZED ); + let missing_credentials_state = AppState::seeded(Some("secret".to_string())); + let missing_credentials_inspect = missing_credentials_state.clone(); + let missing_credentials_app = build_app(missing_credentials_state); + let missing_credentials = app_request( + &missing_credentials_app, + authed_empty_request( + Method::POST, + "/api/official-threat-feeds/threatfox-recent/refresh", + "secret", + ), + ) + .await; + assert_eq!(missing_credentials.status(), StatusCode::BAD_GATEWAY); + assert!( + missing_credentials_inspect + .inner + .read() + .await + .official_threat_feeds + .iter() + .find(|feed| feed.source_id == "threatfox-recent") + .unwrap() + .last_attempt_unix + .is_none(), + "preflight failures must not throttle the corrective retry" + ); + let stalled_listener = StdTcpListener::bind("127.0.0.1:0").unwrap(); + let stalled_address = stalled_listener.local_addr().unwrap(); + let stalled_thread = thread::spawn(move || { + let (_stream, _) = stalled_listener.accept().unwrap(); + std::thread::sleep(std::time::Duration::from_secs(2)); + }); + let stalled_state = AppState::seeded(Some("secret".to_string())).with_official_feed_url( + "spamhaus-drop-v6", + format!("http://{stalled_address}/drop.json"), + ); + let stalled_app = build_app(stalled_state); + let timed_out = app_request( + &stalled_app, + authed_empty_request( + Method::POST, + "/api/official-threat-feeds/spamhaus-drop-v6/refresh", + "secret", + ), + ) + .await; + assert_eq!(timed_out.status(), StatusCode::BAD_GATEWAY); + assert!(body_text(timed_out).await.contains("request failed")); + stalled_thread.join().unwrap(); let calls = StdArc::new(AtomicUsize::new(0)); let mock_calls = calls.clone(); let upstream = Router::new().route( @@ -3528,7 +3615,11 @@ mod tests { ) .into_response() } else { - (StatusCode::OK, "not-json").into_response() + ( + StatusCode::OK, + Body::from(vec![b'x'; PHISHING_DATABASE_MAX_BODY_BYTES + 1]), + ) + .into_response() } } }), @@ -3603,10 +3694,11 @@ mod tests { ) .await; assert_eq!(failed.status(), StatusCode::BAD_GATEWAY); + assert!(body_text(failed).await.contains("body too large")); let after = inspect.inner.read().await; assert_eq!( after.dnsbl, first.dnsbl, - "failed parse must preserve LKG data" + "failed refresh must preserve LKG data" ); assert!( after diff --git a/src/official_feeds.rs b/src/official_feeds.rs index 52ff84e..138d811 100644 --- a/src/official_feeds.rs +++ b/src/official_feeds.rs @@ -256,6 +256,16 @@ mod tests { spamhaus.source_notice.as_deref(), Some("Copyright Spamhaus") ); + assert!( + parse( + "spamhaus_drop_json", + "spamhaus-drop-v4", + 7200, + "{not-json}\n" + ) + .is_err(), + "malformed upstream records fail closed before the LKG swap" + ); let urlhaus = parse( "urlhaus_recent_csv", From c9ff4ae94c8fdac8675df106ed9d39fdb3a5e0ae Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 22:52:26 +0900 Subject: [PATCH 03/23] fix(feeds): preserve multi-source refresh state --- crates/waf-ids-core/src/lib.rs | 12 +++-- docs/official-threat-feeds.md | 11 +++-- src/lib.rs | 86 ++++++++++++++++++++++++++++++++-- 3 files changed, 97 insertions(+), 12 deletions(-) diff --git a/crates/waf-ids-core/src/lib.rs b/crates/waf-ids-core/src/lib.rs index 8356d93..9224a06 100644 --- a/crates/waf-ids-core/src/lib.rs +++ b/crates/waf-ids-core/src/lib.rs @@ -607,10 +607,11 @@ pub fn upsert_threat( } pub fn upsert_dnsbl(entries: &mut Vec, entry: DnsblEntry) -> DnsblEntry { - if let Some(existing) = entries - .iter_mut() - .find(|item| item.address == entry.address) - { + if let Some(existing) = entries.iter_mut().find(|item| { + item.address == entry.address + && item.prefix_len == entry.prefix_len + && item.source == entry.source + }) { *existing = entry.clone(); } else { entries.push(entry.clone()); @@ -1432,6 +1433,9 @@ pub fn readiness_check(id: &str, passed: bool, evidence: &str) -> ReadinessCheck pub fn export_dnsbl_zone(origin: &str, entries: &[DnsblEntry]) -> String { let mut out = format!("$ORIGIN {}.\n$TTL 300\n", sanitize_zone_origin(origin)); for entry in entries { + if entry.prefix_len.is_some() { + continue; + } if let IpAddr::V4(address) = entry.address { // The response code is emitted as a bare, unquoted A-record token, so // it must be a valid IPv4 loopback literal (RFC 5782: DNSBL answers diff --git a/docs/official-threat-feeds.md b/docs/official-threat-feeds.md index a10513e..7c42e67 100644 --- a/docs/official-threat-feeds.md +++ b/docs/official-threat-feeds.md @@ -16,7 +16,9 @@ notice when supplied. Spamhaus asks automated users not to fetch more often than hourly and documents daily refresh as sufficient; Wardnet therefore defaults DROP to daily. abuse.ch exports update more often, but Wardnet deliberately uses a one-hour floor. A request before the interval -expires returns `429` with `Retry-After`. +expires returns `429` with `Retry-After`. An upstream request that fails also consumes the +interval, preventing operator retries from exceeding the source's fetch policy; credential +preflight failures do not. The data is governed by source-specific terms, not by Wardnet's MIT license. Preserve attribution and review the [Spamhaus DROP fair-use policy](https://www.spamhaus.org/blocklists/drop-fair-use-policy/) @@ -59,6 +61,7 @@ last-known-good threat/DNSBL rows unchanged. Requests have a 15-second total tim responses are streamed with an 8 MiB hard limit. Credential preflight failures do not consume the source refresh interval, so adding a missing key permits an immediate corrective retry. -These are threat-intelligence feeds consumed by gateway scoring and the authoritative -DNSBL zone export. They are not recursive DNS resolvers and do not configure an upstream -DNS resolver. +These are threat-intelligence feeds consumed by gateway scoring. The authoritative DNSBL +zone exports exact IPv4 host entries only; CIDR ranges and IPv6 entries remain available to +inline gateway scoring rather than being expanded into an unbounded zone. Wardnet is not a +recursive DNS resolver and does not configure an upstream DNS resolver. diff --git a/src/lib.rs b/src/lib.rs index 3d4d2f5..d39a2a3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -116,6 +116,7 @@ impl AppState { Some(path) => load_or_seed_state(path).await?, None => AppData::seeded(), }; + backfill_official_threat_feeds(&mut data); let event_limit = config.event_limit.max(1); enforce_event_limit(&mut data, event_limit); if let Some(path) = config.state_path.as_deref() { @@ -286,6 +287,18 @@ impl AppState { } } +fn backfill_official_threat_feeds(data: &mut AppData) { + for feed in waf_ids_core::official_threat_feed_registry() { + if !data + .official_threat_feeds + .iter() + .any(|existing| existing.source_id == feed.source_id) + { + data.official_threat_feeds.push(feed); + } + } +} + #[derive(Debug, Clone)] pub struct AppConfig { pub admin_token: Option, @@ -1384,8 +1397,8 @@ async fn refresh_official_threat_feed( .iter_mut() .find(|item| item.source_id == source_id) { - status.etag = etag; - status.last_modified = last_modified; + status.etag = etag.or_else(|| status.etag.clone()); + status.last_modified = last_modified.or_else(|| status.last_modified.clone()); status.last_attempt_unix = Some(now); status.last_success_unix = Some(now); status.last_error = None; @@ -3614,6 +3627,12 @@ mod tests { "{\"cidr\":\"198.51.100.0/24\",\"sblid\":\"SBL1\"}\n{\"type\":\"metadata\",\"copyright\":\"Copyright Spamhaus\"}\n", ) .into_response() + } else if call == 1 { + ( + StatusCode::OK, + "{\"cidr\":\"198.51.101.0/24\",\"sblid\":\"SBL2\"}\n", + ) + .into_response() } else { ( StatusCode::OK, @@ -3675,6 +3694,37 @@ mod tests { assert_eq!(source.etag.as_deref(), Some("\"drop-v1\"")); assert_eq!(source.source_notice.as_deref(), Some("Copyright Spamhaus")); + inspect + .inner + .write() + .await + .official_threat_feeds + .iter_mut() + .find(|feed| feed.source_id == "spamhaus-drop-v4") + .unwrap() + .last_attempt_unix = None; + let refreshed_without_validator = app_request( + &app, + authed_empty_request( + Method::POST, + "/api/official-threat-feeds/spamhaus-drop-v4/refresh", + "secret", + ), + ) + .await; + assert_eq!(refreshed_without_validator.status(), StatusCode::OK); + let second = inspect.inner.read().await.clone(); + assert_eq!( + second + .official_threat_feeds + .iter() + .find(|feed| feed.source_id == "spamhaus-drop-v4") + .unwrap() + .etag + .as_deref(), + Some("\"drop-v1\"") + ); + inspect .inner .write() @@ -3697,7 +3747,7 @@ mod tests { assert!(body_text(failed).await.contains("body too large")); let after = inspect.inner.read().await; assert_eq!( - after.dnsbl, first.dnsbl, + after.dnsbl, second.dnsbl, "failed refresh must preserve LKG data" ); assert!( @@ -4359,6 +4409,14 @@ mod tests { ttl_seconds: 300, prefix_len: None, }, + DnsblEntry { + address: "198.51.100.0".parse().unwrap(), + code: "127.0.0.2".to_string(), + reason: "cidr inline only".to_string(), + source: "unit".to_string(), + ttl_seconds: 300, + prefix_len: Some(24), + }, ], ); @@ -4366,6 +4424,21 @@ mod tests { assert!(zone.contains("10.2.0.192 IN A 127.0.0.2")); assert!(zone.contains("10.2.0.192 IN TXT \"scanner source=unit\"")); assert!(!zone.contains("ipv6 skip")); + assert!(!zone.contains("cidr inline only")); + } + + #[test] + fn backfills_new_official_feed_sources() { + let mut data = AppData::seeded(); + let missing = data.official_threat_feeds.pop().unwrap(); + + backfill_official_threat_feeds(&mut data); + + assert!( + data.official_threat_feeds + .iter() + .any(|feed| feed.source_id == missing.source_id) + ); } #[test] @@ -6322,7 +6395,7 @@ mod tests { address: "203.0.113.10".parse().unwrap(), code: "127.0.0.3".to_string(), reason: "botnet".to_string(), - source: "feed".to_string(), + source: "unit".to_string(), ttl_seconds: 600, prefix_len: None, }, @@ -6332,6 +6405,11 @@ mod tests { assert_eq!(dnsbl[0].code, "127.0.0.3"); assert_eq!(dnsbl[0].reason, "botnet"); + let mut second_source = dnsbl[0].clone(); + second_source.source = "second-feed".to_string(); + upsert_dnsbl(&mut dnsbl, second_source); + assert_eq!(dnsbl.len(), 2); + let mut feeds = vec![ThreatFeedStatus { feed_id: "feed-a".to_string(), source: "misp://old".to_string(), From 2c3f06f49e699430abd4c493ea363f69aad4fdd6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 23:12:10 +0900 Subject: [PATCH 04/23] fix(feeds): preserve stored DNSBL invariants --- crates/waf-ids-core/src/lib.rs | 9 ++++---- src/lib.rs | 39 +++++++++++++++++++++++++--------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/crates/waf-ids-core/src/lib.rs b/crates/waf-ids-core/src/lib.rs index 9224a06..e96ec3a 100644 --- a/crates/waf-ids-core/src/lib.rs +++ b/crates/waf-ids-core/src/lib.rs @@ -607,11 +607,10 @@ pub fn upsert_threat( } pub fn upsert_dnsbl(entries: &mut Vec, entry: DnsblEntry) -> DnsblEntry { - if let Some(existing) = entries.iter_mut().find(|item| { - item.address == entry.address - && item.prefix_len == entry.prefix_len - && item.source == entry.source - }) { + if let Some(existing) = entries + .iter_mut() + .find(|item| item.address == entry.address) + { *existing = entry.clone(); } else { entries.push(entry.clone()); diff --git a/src/lib.rs b/src/lib.rs index d39a2a3..54fc68b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1373,14 +1373,22 @@ async fn refresh_official_threat_feed( .mutate_and_persist(|data| { data.threats.retain(|item| item.source != source_id); data.dnsbl.retain(|item| item.source != source_id); - let threat_count = import.threats.len(); - let dnsbl_count = import.dnsbl.len(); for threat in import.threats { upsert_threat(&mut data.threats, threat); } for entry in import.dnsbl { upsert_dnsbl(&mut data.dnsbl, entry); } + let threat_count = data + .threats + .iter() + .filter(|item| item.source == source_id) + .count(); + let dnsbl_count = data + .dnsbl + .iter() + .filter(|item| item.source == source_id) + .count(); upsert_threat_feed( &mut data.threat_feeds, ThreatFeedStatus { @@ -1402,7 +1410,9 @@ async fn refresh_official_threat_feed( status.last_attempt_unix = Some(now); status.last_success_unix = Some(now); status.last_error = None; - status.source_notice = parsed.source_notice; + status.source_notice = parsed + .source_notice + .or_else(|| status.source_notice.clone()); } record_successful_audit_log( data, @@ -3630,7 +3640,7 @@ mod tests { } else if call == 1 { ( StatusCode::OK, - "{\"cidr\":\"198.51.101.0/24\",\"sblid\":\"SBL2\"}\n", + "{\"cidr\":\"198.51.101.0/24\",\"sblid\":\"SBL2\"}\n{\"cidr\":\"198.51.101.0/24\",\"sblid\":\"SBL2\"}\n", ) .into_response() } else { @@ -3713,16 +3723,24 @@ mod tests { ) .await; assert_eq!(refreshed_without_validator.status(), StatusCode::OK); + let refresh_body: serde_json::Value = json_body(refreshed_without_validator).await; + assert_eq!(refresh_body["dnsbl_count"], 1); let second = inspect.inner.read().await.clone(); + let source = second + .official_threat_feeds + .iter() + .find(|feed| feed.source_id == "spamhaus-drop-v4") + .unwrap(); + assert_eq!(source.etag.as_deref(), Some("\"drop-v1\"")); + assert_eq!(source.source_notice.as_deref(), Some("Copyright Spamhaus")); assert_eq!( second - .official_threat_feeds + .threat_feeds .iter() - .find(|feed| feed.source_id == "spamhaus-drop-v4") + .find(|feed| feed.feed_id == "spamhaus-drop-v4") .unwrap() - .etag - .as_deref(), - Some("\"drop-v1\"") + .dnsbl_count, + 1 ); inspect @@ -6408,7 +6426,8 @@ mod tests { let mut second_source = dnsbl[0].clone(); second_source.source = "second-feed".to_string(); upsert_dnsbl(&mut dnsbl, second_source); - assert_eq!(dnsbl.len(), 2); + assert_eq!(dnsbl.len(), 1); + assert_eq!(dnsbl[0].source, "second-feed"); let mut feeds = vec![ThreatFeedStatus { feed_id: "feed-a".to_string(), From a1b5540b5aac2f93351189c4c3a1f6e96cd2e07b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 02:01:15 +0900 Subject: [PATCH 05/23] fix: reconcile official feed metadata --- src/lib.rs | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 54fc68b..f9c57ef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -289,11 +289,19 @@ impl AppState { fn backfill_official_threat_feeds(data: &mut AppData) { for feed in waf_ids_core::official_threat_feed_registry() { - if !data + if let Some(existing) = data .official_threat_feeds - .iter() - .any(|existing| existing.source_id == feed.source_id) + .iter_mut() + .find(|existing| existing.source_id == feed.source_id) { + existing.official_url = feed.official_url; + existing.parser = feed.parser; + existing.indicator_types = feed.indicator_types; + existing.attribution = feed.attribution; + existing.license_url = feed.license_url; + existing.refresh_interval_seconds = feed.refresh_interval_seconds; + existing.ttl_seconds = feed.ttl_seconds; + } else { data.official_threat_feeds.push(feed); } } @@ -412,7 +420,7 @@ pub struct HealthStatus { pub persistence: String, pub dnsbl_origin: String, pub event_limit: usize, - /// Bootstrap origin for admin secrets: `file`, `env`, or `none` (never secret values). + /// Bootstrap origin for admin secrets: `file`, `env`, `mixed`, or `none` (never secret values). pub credentials_source: String, /// True when at least one admin write token is configured. pub admin_auth_configured: bool, @@ -4459,6 +4467,31 @@ mod tests { ); } + #[test] + fn backfill_reconciles_registry_metadata_but_preserves_refresh_state() { + let mut data = AppData::seeded(); + let feed = &mut data.official_threat_feeds[0]; + feed.official_url = "https://stale.invalid/feed".to_string(); + feed.parser = "stale".to_string(); + feed.etag = Some("preserved".to_string()); + let source_id = feed.source_id.clone(); + + backfill_official_threat_feeds(&mut data); + + let canonical = waf_ids_core::official_threat_feed_registry() + .into_iter() + .find(|feed| feed.source_id == source_id) + .unwrap(); + let feed = data + .official_threat_feeds + .iter() + .find(|feed| feed.source_id == source_id) + .unwrap(); + assert_eq!(feed.official_url, canonical.official_url); + assert_eq!(feed.parser, canonical.parser); + assert_eq!(feed.etag.as_deref(), Some("preserved")); + } + #[test] fn scores_threat_indicator_matches() { // Uses a site-specific IoC that does NOT overlap a built-in signature, From 3625e07d1f21c30026d9c1086935186e55f654fa Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 02:22:04 +0900 Subject: [PATCH 06/23] fix(dnsbl): retain exact-host prefixes in zone export --- crates/waf-ids-core/src/lib.rs | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/crates/waf-ids-core/src/lib.rs b/crates/waf-ids-core/src/lib.rs index e96ec3a..e0c8afc 100644 --- a/crates/waf-ids-core/src/lib.rs +++ b/crates/waf-ids-core/src/lib.rs @@ -1432,7 +1432,10 @@ pub fn readiness_check(id: &str, passed: bool, evidence: &str) -> ReadinessCheck pub fn export_dnsbl_zone(origin: &str, entries: &[DnsblEntry]) -> String { let mut out = format!("$ORIGIN {}.\n$TTL 300\n", sanitize_zone_origin(origin)); for entry in entries { - if entry.prefix_len.is_some() { + if !matches!( + (entry.address, entry.prefix_len), + (IpAddr::V4(_), None | Some(32)) | (IpAddr::V6(_), None | Some(128)) + ) { continue; } if let IpAddr::V4(address) = entry.address { @@ -1634,6 +1637,33 @@ mod tests { assert_txt_quotes_escaped(&zone); } + #[test] + fn export_dnsbl_zone_keeps_exact_ipv4_hosts_and_omits_subnets() { + let entries = [ + DnsblEntry { + address: "198.51.100.7".parse().unwrap(), + code: "127.0.0.2".to_string(), + reason: "exact host".to_string(), + source: "unit".to_string(), + ttl_seconds: 300, + prefix_len: Some(32), + }, + DnsblEntry { + address: "198.51.100.0".parse().unwrap(), + code: "127.0.0.2".to_string(), + reason: "subnet".to_string(), + source: "unit".to_string(), + ttl_seconds: 300, + prefix_len: Some(24), + }, + ]; + + let zone = export_dnsbl_zone("dnsbl.example", &entries); + + assert!(zone.contains("7.100.51.198 IN A 127.0.0.2")); + assert!(!zone.contains("0.100.51.198")); + } + #[test] fn export_dnsbl_zone_rejects_non_ip_code_injection() { // A `code` that is not a valid IP literal would become a bare A-record From 1db1b96952eddde29ec9b54ac502d33c345636be Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 03:46:02 +0900 Subject: [PATCH 07/23] fix(feeds): preserve official refresh provenance --- Cargo.lock | 96 +++++++++++++++++++++++++++++++++- Cargo.toml | 2 + crates/waf-ids-core/src/lib.rs | 34 ++++++++---- docs/official-threat-feeds.md | 11 +++- src/lib.rs | 37 ++++++++++++- src/official_feeds.rs | 54 +++++++------------ 6 files changed, 186 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c696190..0cee870 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -93,6 +93,15 @@ version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -134,10 +143,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.0", "rand_core 0.10.1", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.0" @@ -147,6 +165,47 @@ dependencies = [ "libc", ] +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "displaydoc" version = "0.2.7" @@ -255,6 +314,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1020,6 +1089,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "shlex" version = "2.0.1" @@ -1292,6 +1372,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unarray" version = "0.1.4" @@ -1334,16 +1420,24 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "waf-ids-ai-soc" version = "0.1.0" dependencies = [ "axum", + "csv", "futures-util", "proptest", "reqwest", "serde", "serde_json", + "sha2", "tokio", "tower", "waf-ids-core", diff --git a/Cargo.toml b/Cargo.toml index b2ec231..ac9d55d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,8 +13,10 @@ resolver = "3" axum = "0.8" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "multipart", "json", "stream"] } futures-util = { version = "0.3", default-features = false, features = ["std"] } +csv = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = "0.10" tokio = { version = "1", features = ["fs", "macros", "net", "rt-multi-thread", "signal", "sync"] } waf-ids-core = { path = "crates/waf-ids-core" } diff --git a/crates/waf-ids-core/src/lib.rs b/crates/waf-ids-core/src/lib.rs index e0c8afc..f8fba7f 100644 --- a/crates/waf-ids-core/src/lib.rs +++ b/crates/waf-ids-core/src/lib.rs @@ -85,6 +85,8 @@ pub struct OfficialThreatFeed { pub last_error: Option, #[serde(default)] pub source_notice: Option, + #[serde(default)] + pub content_sha256: Option, } pub fn official_threat_feed_registry() -> Vec { @@ -96,6 +98,8 @@ pub fn official_threat_feed_registry() -> Vec { &["ipv4_cidr"][..], "The Spamhaus Project", "https://www.spamhaus.org/blocklists/drop-fair-use-policy/", + 86_400, + 172_800, ), ( "spamhaus-drop-v6", @@ -104,6 +108,8 @@ pub fn official_threat_feed_registry() -> Vec { &["ipv6_cidr"][..], "The Spamhaus Project", "https://www.spamhaus.org/blocklists/drop-fair-use-policy/", + 86_400, + 172_800, ), ( "urlhaus-online", @@ -112,6 +118,8 @@ pub fn official_threat_feed_registry() -> Vec { &["url", "domain"][..], "URLhaus by abuse.ch", "https://abuse.ch/terms-of-use/", + 3_600, + 7_200, ), ( "threatfox-recent", @@ -120,11 +128,22 @@ pub fn official_threat_feed_registry() -> Vec { &["domain", "ipv4", "ipv6"][..], "ThreatFox by abuse.ch", "https://abuse.ch/terms-of-use/", + 3_600, + 7_200, ), ] .into_iter() .map( - |(source_id, official_url, parser, indicator_types, attribution, license_url)| { + |( + source_id, + official_url, + parser, + indicator_types, + attribution, + license_url, + refresh_interval_seconds, + ttl_seconds, + )| { OfficialThreatFeed { source_id: source_id.to_string(), official_url: official_url.to_string(), @@ -135,22 +154,15 @@ pub fn official_threat_feed_registry() -> Vec { .collect(), attribution: attribution.to_string(), license_url: license_url.to_string(), - refresh_interval_seconds: if source_id.starts_with("spamhaus") { - 86_400 - } else { - 3_600 - }, - ttl_seconds: if source_id.starts_with("spamhaus") { - 172_800 - } else { - 7_200 - }, + refresh_interval_seconds, + ttl_seconds, etag: None, last_modified: None, last_attempt_unix: None, last_success_unix: None, last_error: None, source_notice: None, + content_sha256: None, } }, ) diff --git a/docs/official-threat-feeds.md b/docs/official-threat-feeds.md index 7c42e67..880e025 100644 --- a/docs/official-threat-feeds.md +++ b/docs/official-threat-feeds.md @@ -4,7 +4,8 @@ Wardnet keeps operator-supplied `POST /api/threat-feeds/import` separate from a closed registry of official upstreams. The registry is persisted with source identity, URL template, parser, supported indicator types, attribution and terms link, HTTP validators, refresh interval, TTL, last attempt/success/error, and the upstream copyright -notice when supplied. +notice when supplied. Each successful `200` also records a SHA-256 digest of the exact +validated response body; this is local provenance evidence, not an upstream signature. | Source id | Upstream contract | Default refresh / TTL | Credential | | --- | --- | --- | --- | @@ -28,6 +29,14 @@ redistribution. The canonical formats are documented by [URLhaus](https://urlhaus.abuse.ch/api/), and [ThreatFox](https://threatfox.abuse.ch/api/). +The current contracts were rechecked against those official pages on 2026-08-27: +Spamhaus publishes the two JSON URLs, requires attribution and preservation of its date/© +notice, re-evaluates DROP daily, and says daily fetching is sufficient. URLhaus documents +the authenticated `recent.csv` export URL. ThreatFox requires `Auth-Key` and documents +`get_iocs` with a one-day minimum window. Neither abuse.ch API documents a detached +checksum for these responses, so Wardnet records its own digest after TLS retrieval and +full-response validation while retaining ETag/Last-Modified when provided. + ## Credentials Place abuse.ch keys in the JSON credential registry selected by diff --git a/src/lib.rs b/src/lib.rs index f9c57ef..829175c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,6 +7,7 @@ use axum::{ routing::{any, get, post}, }; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use std::{ collections::{HashMap, HashSet}, io::ErrorKind, @@ -307,6 +308,18 @@ fn backfill_official_threat_feeds(data: &mut AppData) { } } +fn upsert_official_dnsbl(entries: &mut Vec, entry: DnsblEntry) { + if let Some(existing) = entries.iter_mut().find(|item| { + item.address == entry.address + && item.prefix_len == entry.prefix_len + && item.source == entry.source + }) { + *existing = entry; + } else { + entries.push(entry); + } +} + #[derive(Debug, Clone)] pub struct AppConfig { pub admin_token: Option, @@ -1366,6 +1379,7 @@ async fn refresh_official_threat_feed( return official_feed_failure(&state, &source_id, now, &message, true).await; } }; + let content_sha256 = format!("{:x}", Sha256::digest(body.as_bytes())); let import = ThreatFeedImport { feed_id: source_id.clone(), source: feed.attribution.clone(), @@ -1385,7 +1399,7 @@ async fn refresh_official_threat_feed( upsert_threat(&mut data.threats, threat); } for entry in import.dnsbl { - upsert_dnsbl(&mut data.dnsbl, entry); + upsert_official_dnsbl(&mut data.dnsbl, entry); } let threat_count = data .threats @@ -1421,6 +1435,7 @@ async fn refresh_official_threat_feed( status.source_notice = parsed .source_notice .or_else(|| status.source_notice.clone()); + status.content_sha256 = Some(content_sha256); } record_successful_audit_log( data, @@ -3711,6 +3726,7 @@ mod tests { .unwrap(); assert_eq!(source.etag.as_deref(), Some("\"drop-v1\"")); assert_eq!(source.source_notice.as_deref(), Some("Copyright Spamhaus")); + assert_eq!(source.content_sha256.as_deref().map(str::len), Some(64)); inspect .inner @@ -4492,6 +4508,25 @@ mod tests { assert_eq!(feed.etag.as_deref(), Some("preserved")); } + #[test] + fn official_dnsbl_upsert_preserves_overlapping_source_provenance() { + let mut entries = Vec::new(); + for source in ["spamhaus-drop-v4", "threatfox-recent"] { + upsert_official_dnsbl( + &mut entries, + DnsblEntry { + address: "198.51.100.7".parse().unwrap(), + prefix_len: None, + code: "127.0.0.2".to_string(), + reason: "official evidence".to_string(), + source: source.to_string(), + ttl_seconds: 300, + }, + ); + } + assert_eq!(entries.len(), 2); + } + #[test] fn scores_threat_indicator_matches() { // Uses a site-specific IoC that does NOT overlap a built-in signature, diff --git a/src/official_feeds.rs b/src/official_feeds.rs index 138d811..d80a659 100644 --- a/src/official_feeds.rs +++ b/src/official_feeds.rs @@ -68,16 +68,25 @@ fn parse_urlhaus( body: &str, ) -> Result { let mut parsed = ParsedOfficialFeed::default(); - for line in body.lines() { - let line = line.trim(); - if line.is_empty() || line.starts_with('#') { - continue; - } - let fields = csv_fields(line)?; - let Some(url) = fields - .iter() - .find(|field| field.starts_with("http://") || field.starts_with("https://")) - else { + let csv_body = body + .lines() + .filter(|line| !line.trim_start().starts_with('#')) + .collect::>() + .join("\n"); + let mut reader = csv::ReaderBuilder::new() + .has_headers(true) + .from_reader(csv_body.as_bytes()); + let headers = reader + .headers() + .map_err(|error| format!("invalid URLhaus CSV header: {error}"))? + .clone(); + let url_index = headers + .iter() + .position(|header| header.eq_ignore_ascii_case("url")) + .ok_or_else(|| "URLhaus CSV is missing url header".to_string())?; + for record in reader.records() { + let record = record.map_err(|error| format!("invalid URLhaus CSV record: {error}"))?; + let Some(url) = record.get(url_index).filter(|url| !url.is_empty()) else { continue; }; parsed @@ -215,29 +224,6 @@ fn threatfox_ip(value: &str) -> Option { .or_else(|| value.rsplit_once(':').and_then(|(ip, _)| ip.parse().ok())) } -fn csv_fields(line: &str) -> Result, String> { - let mut fields = Vec::new(); - let mut field = String::new(); - let mut quoted = false; - let mut chars = line.chars().peekable(); - while let Some(character) = chars.next() { - match character { - '"' if quoted && chars.peek() == Some(&'"') => { - field.push('"'); - chars.next(); - } - '"' => quoted = !quoted, - ',' if !quoted => fields.push(std::mem::take(&mut field)), - other => field.push(other), - } - } - if quoted { - return Err("unterminated quoted CSV field".to_string()); - } - fields.push(field); - Ok(fields) -} - #[cfg(test)] mod tests { use super::*; @@ -271,7 +257,7 @@ mod tests { "urlhaus_recent_csv", "urlhaus-online", 7200, - "# attribution\n1,2026-01-01,https://evil.example/a,online\n", + "# attribution\nid,dateadded,url,url_status,reporter\n1,2026-01-01,https://evil.example/a,online,\"analyst\nteam\"\n", ) .unwrap(); assert!( From 5a3bbcf89ea5dfc53429dfdd6e74f4da90d782c5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 04:36:34 +0900 Subject: [PATCH 08/23] fix(feeds): parse official URLhaus CSV headers --- crates/waf-ids-core/src/lib.rs | 19 +++++++++++++++++-- src/official_feeds.rs | 11 +++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/crates/waf-ids-core/src/lib.rs b/crates/waf-ids-core/src/lib.rs index f8fba7f..16c7491 100644 --- a/crates/waf-ids-core/src/lib.rs +++ b/crates/waf-ids-core/src/lib.rs @@ -115,7 +115,7 @@ pub fn official_threat_feed_registry() -> Vec { "urlhaus-online", "https://urlhaus-api.abuse.ch/v2/files/exports/{AUTH_KEY}/recent.csv", "urlhaus_recent_csv", - &["url", "domain"][..], + &["url", "domain", "client_ip"][..], "URLhaus by abuse.ch", "https://abuse.ch/terms-of-use/", 3_600, @@ -125,7 +125,7 @@ pub fn official_threat_feed_registry() -> Vec { "threatfox-recent", "https://threatfox-api.abuse.ch/api/v1/", "threatfox_json", - &["domain", "ipv4", "ipv6"][..], + &["domain", "client_ip"][..], "ThreatFox by abuse.ch", "https://abuse.ch/terms-of-use/", 3_600, @@ -1526,6 +1526,21 @@ fn escape_txt(value: &str) -> String { mod tests { use super::*; + #[test] + fn official_feed_metadata_names_the_emitted_indicator_types() { + let feeds = official_threat_feed_registry(); + let urlhaus = feeds + .iter() + .find(|feed| feed.source_id == "urlhaus-online") + .unwrap(); + assert_eq!(urlhaus.indicator_types, ["url", "domain", "client_ip"]); + let threatfox = feeds + .iter() + .find(|feed| feed.source_id == "threatfox-recent") + .unwrap(); + assert_eq!(threatfox.indicator_types, ["domain", "client_ip"]); + } + #[test] fn score_request_matches_client_ip_threat_indicators() { let threats = vec![ThreatIndicator { diff --git a/src/official_feeds.rs b/src/official_feeds.rs index d80a659..81176c1 100644 --- a/src/official_feeds.rs +++ b/src/official_feeds.rs @@ -70,7 +70,14 @@ fn parse_urlhaus( let mut parsed = ParsedOfficialFeed::default(); let csv_body = body .lines() - .filter(|line| !line.trim_start().starts_with('#')) + .filter_map(|line| { + let trimmed = line.trim_start(); + let header = trimmed + .strip_prefix('#') + .map(str::trim_start) + .filter(|candidate| candidate.starts_with("id,dateadded,url,")); + header.or_else(|| (!trimmed.starts_with('#')).then_some(line)) + }) .collect::>() .join("\n"); let mut reader = csv::ReaderBuilder::new() @@ -257,7 +264,7 @@ mod tests { "urlhaus_recent_csv", "urlhaus-online", 7200, - "# attribution\nid,dateadded,url,url_status,reporter\n1,2026-01-01,https://evil.example/a,online,\"analyst\nteam\"\n", + "# attribution\n# id,dateadded,url,url_status,reporter\n1,2026-01-01,https://evil.example/a,online,\"analyst\nteam\"\n", ) .unwrap(); assert!( From 531943e81df576d60168948b0c53d9228818cc67 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 26 Aug 2026 12:38:32 -0700 Subject: [PATCH 09/23] fix(runtime): register SIGTERM before readiness (#123) --- src/main.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index 2c8fc82..2bab02a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,12 +8,16 @@ async fn main() -> Result<(), Box> { } #[cfg(all(not(test), unix))] -async fn shutdown_signal() { +fn shutdown_signal() -> impl std::future::Future { // Shut down gracefully on SIGTERM (what container runtimes and the e2e test // harness send) so in-flight requests drain and the process exits cleanly. + // Register eagerly before run_from_env publishes listener readiness; an + // async fn would not install the handler until its future was first polled. let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) .expect("install SIGTERM handler"); - term.recv().await; + async move { + term.recv().await; + } } #[cfg(all(not(test), not(unix)))] From 7ab2108c53dbd59f1170c09355148cab0cf2e675 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 05:46:30 +0900 Subject: [PATCH 10/23] fix(feeds): preserve DNSBL provenance and refresh time --- README.md | 2 +- crates/waf-ids-core/src/lib.rs | 9 ++++---- src/lib.rs | 42 +++++++++++++--------------------- 3 files changed, 22 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index d158758..aed6b30 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ Management writes are upserts: - routes are keyed by `id` - threat indicators are keyed by `indicator_type`, `value`, and `source` -- DNSBL entries are keyed by `address` +- DNSBL entries are keyed by `address`, `prefix_len`, and `source` so independent evidence is retained DNSBL response codes must be IPv4 loopback-style values in `127.0.0.0/8`. diff --git a/crates/waf-ids-core/src/lib.rs b/crates/waf-ids-core/src/lib.rs index 16c7491..f727b91 100644 --- a/crates/waf-ids-core/src/lib.rs +++ b/crates/waf-ids-core/src/lib.rs @@ -619,10 +619,11 @@ pub fn upsert_threat( } pub fn upsert_dnsbl(entries: &mut Vec, entry: DnsblEntry) -> DnsblEntry { - if let Some(existing) = entries - .iter_mut() - .find(|item| item.address == entry.address) - { + if let Some(existing) = entries.iter_mut().find(|item| { + item.address == entry.address + && item.prefix_len == entry.prefix_len + && item.source == entry.source + }) { *existing = entry.clone(); } else { entries.push(entry.clone()); diff --git a/src/lib.rs b/src/lib.rs index 829175c..465eed7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -308,18 +308,6 @@ fn backfill_official_threat_feeds(data: &mut AppData) { } } -fn upsert_official_dnsbl(entries: &mut Vec, entry: DnsblEntry) { - if let Some(existing) = entries.iter_mut().find(|item| { - item.address == entry.address - && item.prefix_len == entry.prefix_len - && item.source == entry.source - }) { - *existing = entry; - } else { - entries.push(entry); - } -} - #[derive(Debug, Clone)] pub struct AppConfig { pub admin_token: Option, @@ -1335,7 +1323,7 @@ async fn refresh_official_threat_feed( return official_feed_failure( &state, &source_id, - now, + now_unix(), "official feed request failed", true, ) @@ -1356,7 +1344,7 @@ async fn refresh_official_threat_feed( return finish_official_feed_not_modified( &state, &source_id, - now, + now_unix(), etag, last_modified, audit_actor(&state, &headers), @@ -1365,18 +1353,18 @@ async fn refresh_official_threat_feed( } if !response.status().is_success() { let message = format!("official feed returned HTTP {}", response.status().as_u16()); - return official_feed_failure(&state, &source_id, now, &message, true).await; + return official_feed_failure(&state, &source_id, now_unix(), &message, true).await; } let body = match read_official_feed_body(response).await { Ok(body) => body, Err(message) => { - return official_feed_failure(&state, &source_id, now, &message, true).await; + return official_feed_failure(&state, &source_id, now_unix(), &message, true).await; } }; let parsed = match official_feeds::parse(&feed.parser, &source_id, feed.ttl_seconds, &body) { Ok(parsed) => parsed, Err(message) => { - return official_feed_failure(&state, &source_id, now, &message, true).await; + return official_feed_failure(&state, &source_id, now_unix(), &message, true).await; } }; let content_sha256 = format!("{:x}", Sha256::digest(body.as_bytes())); @@ -1388,8 +1376,9 @@ async fn refresh_official_threat_feed( dnsbl: parsed.dnsbl, }; if let Err(message) = validate_threat_feed_import(&import) { - return official_feed_failure(&state, &source_id, now, message, true).await; + return official_feed_failure(&state, &source_id, now_unix(), message, true).await; } + let completed_at = now_unix(); let actor = audit_actor(&state, &headers); match state .mutate_and_persist(|data| { @@ -1399,7 +1388,7 @@ async fn refresh_official_threat_feed( upsert_threat(&mut data.threats, threat); } for entry in import.dnsbl { - upsert_official_dnsbl(&mut data.dnsbl, entry); + upsert_dnsbl(&mut data.dnsbl, entry); } let threat_count = data .threats @@ -1416,7 +1405,7 @@ async fn refresh_official_threat_feed( ThreatFeedStatus { feed_id: source_id.clone(), source: feed.attribution.clone(), - last_updated_unix: now, + last_updated_unix: completed_at, threat_count, dnsbl_count, ttl_seconds: feed.ttl_seconds, @@ -1429,8 +1418,8 @@ async fn refresh_official_threat_feed( { status.etag = etag.or_else(|| status.etag.clone()); status.last_modified = last_modified.or_else(|| status.last_modified.clone()); - status.last_attempt_unix = Some(now); - status.last_success_unix = Some(now); + status.last_attempt_unix = Some(completed_at); + status.last_success_unix = Some(completed_at); status.last_error = None; status.source_notice = parsed .source_notice @@ -1449,7 +1438,7 @@ async fn refresh_official_threat_feed( not_modified: false, threat_count, dnsbl_count, - refreshed_at_unix: now, + refreshed_at_unix: completed_at, } }) .await @@ -4512,7 +4501,7 @@ mod tests { fn official_dnsbl_upsert_preserves_overlapping_source_provenance() { let mut entries = Vec::new(); for source in ["spamhaus-drop-v4", "threatfox-recent"] { - upsert_official_dnsbl( + upsert_dnsbl( &mut entries, DnsblEntry { address: "198.51.100.7".parse().unwrap(), @@ -6494,8 +6483,9 @@ mod tests { let mut second_source = dnsbl[0].clone(); second_source.source = "second-feed".to_string(); upsert_dnsbl(&mut dnsbl, second_source); - assert_eq!(dnsbl.len(), 1); - assert_eq!(dnsbl[0].source, "second-feed"); + assert_eq!(dnsbl.len(), 2); + assert_eq!(dnsbl[0].source, "unit"); + assert_eq!(dnsbl[1].source, "second-feed"); let mut feeds = vec![ThreatFeedStatus { feed_id: "feed-a".to_string(), From ad41b0f69f95606f2e3833a041fbaac841ec2d44 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 06:37:04 +0900 Subject: [PATCH 11/23] fix(feeds): reset validators when source URL changes --- src/lib.rs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 465eed7..d25fcfc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -295,6 +295,7 @@ fn backfill_official_threat_feeds(data: &mut AppData) { .iter_mut() .find(|existing| existing.source_id == feed.source_id) { + let url_changed = existing.official_url != feed.official_url; existing.official_url = feed.official_url; existing.parser = feed.parser; existing.indicator_types = feed.indicator_types; @@ -302,6 +303,10 @@ fn backfill_official_threat_feeds(data: &mut AppData) { existing.license_url = feed.license_url; existing.refresh_interval_seconds = feed.refresh_interval_seconds; existing.ttl_seconds = feed.ttl_seconds; + if url_changed { + existing.etag = None; + existing.last_modified = None; + } } else { data.official_threat_feeds.push(feed); } @@ -4473,12 +4478,12 @@ mod tests { } #[test] - fn backfill_reconciles_registry_metadata_but_preserves_refresh_state() { + fn backfill_preserves_validators_only_while_registry_url_is_unchanged() { let mut data = AppData::seeded(); let feed = &mut data.official_threat_feeds[0]; - feed.official_url = "https://stale.invalid/feed".to_string(); feed.parser = "stale".to_string(); feed.etag = Some("preserved".to_string()); + feed.last_modified = Some("preserved-date".to_string()); let source_id = feed.source_id.clone(); backfill_official_threat_feeds(&mut data); @@ -4495,6 +4500,21 @@ mod tests { assert_eq!(feed.official_url, canonical.official_url); assert_eq!(feed.parser, canonical.parser); assert_eq!(feed.etag.as_deref(), Some("preserved")); + + let feed = data + .official_threat_feeds + .iter_mut() + .find(|feed| feed.source_id == source_id) + .unwrap(); + feed.official_url = "https://stale.invalid/feed".to_string(); + backfill_official_threat_feeds(&mut data); + let feed = data + .official_threat_feeds + .iter() + .find(|feed| feed.source_id == source_id) + .unwrap(); + assert!(feed.etag.is_none()); + assert!(feed.last_modified.is_none()); } #[test] From fc2ecf12474a5f8ea80c27dc462b5faf611c1110 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 27 Aug 2026 07:06:16 +0900 Subject: [PATCH 12/23] docs(feeds): record scrubbed official runtime evidence --- docs/official-threat-feeds.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/official-threat-feeds.md b/docs/official-threat-feeds.md index 880e025..7375184 100644 --- a/docs/official-threat-feeds.md +++ b/docs/official-threat-feeds.md @@ -74,3 +74,23 @@ These are threat-intelligence feeds consumed by gateway scoring. The authoritati zone exports exact IPv4 host entries only; CIDR ranges and IPv6 entries remain available to inline gateway scoring rather than being expanded into an unbounded zone. Wardnet is not a recursive DNS resolver and does not configure an upstream DNS resolver. + +## Scrubbed runtime evidence (2026-08-27) + +A bounded local run of this PR's production refresh endpoint fetched and parsed the official +Spamhaus TLS sources without publishing indicator rows. `spamhaus-drop-v4` returned HTTP 200 +and atomically installed 1,703 DNSBL ranges; `spamhaus-drop-v6` returned HTTP 200 and installed +92 ranges. Both status records captured a successful attempt timestamp, a response digest, +`Last-Modified`, and the upstream copyright notice. Neither response supplied an ETag. + +The same run exercised URLhaus and ThreatFox through the official registry, but no matching KV +credentials were available. Both failed closed with HTTP 502 before an upstream request; +`last_attempt_unix` and `last_success_unix` remained unset, no digest or indicator counts were +fabricated, and an operator can retry immediately after adding a credential. The source +registry continued to expose attribution, terms links, parser identity, indicator types, and +the one-hour refresh/two-hour TTL policy without exposing credential values. + +Last-known-good behavior is covered by the bounded official refresh integration test: timeout, +oversized body, parse, and validation failures retain the prior source-owned rows while recording +the failure status. The live successful snapshot above establishes the parser and persistence +path against current official content; it does not redistribute or enumerate any IOC row. From 1691f95d65d3974d59eafeef319d71bbb3be1c75 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 00:17:37 +0900 Subject: [PATCH 13/23] fix(feeds): classify URLhaus IPv6 hosts as DNSBL entries --- src/official_feeds.rs | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/official_feeds.rs b/src/official_feeds.rs index 81176c1..75ee10f 100644 --- a/src/official_feeds.rs +++ b/src/official_feeds.rs @@ -103,10 +103,14 @@ fn parse_urlhaus( .ok() .and_then(|url| url.host_str().map(str::to_ascii_lowercase)); if let Some(host) = host { - if let Ok(address) = host.parse::() { - parsed - .threats - .push(threat(&host, "client_ip", source_id, ttl_seconds)); + let ip_candidate = host.trim_start_matches('[').trim_end_matches(']'); + if let Ok(address) = ip_candidate.parse::() { + parsed.threats.push(threat( + &address.to_string(), + "client_ip", + source_id, + ttl_seconds, + )); parsed.dnsbl.push(DnsblEntry { address, prefix_len: None, @@ -273,6 +277,20 @@ mod tests { .iter() .any(|item| item.indicator_type == "domain") ); + let urlhaus_ipv6 = parse( + "urlhaus_recent_csv", + "urlhaus-online", + 7200, + "# attribution\n# id,dateadded,url,url_status,reporter\n1,2026-01-01,https://[2001:db8::7]/a,online,analyst\n", + ) + .unwrap(); + assert!( + urlhaus_ipv6 + .threats + .iter() + .any(|item| item.indicator_type == "client_ip" && item.value == "2001:db8::7") + ); + assert_eq!(urlhaus_ipv6.dnsbl[0].address.to_string(), "2001:db8::7"); let threatfox = parse( "threatfox_json", From 224d56407c2477d3c214008065e27df866cddf00 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 00:19:02 +0900 Subject: [PATCH 14/23] fix(feeds): clear stale provenance on URL changes --- src/lib.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index d25fcfc..500f28f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -306,6 +306,8 @@ fn backfill_official_threat_feeds(data: &mut AppData) { if url_changed { existing.etag = None; existing.last_modified = None; + existing.content_sha256 = None; + existing.last_success_unix = None; } } else { data.official_threat_feeds.push(feed); @@ -4484,6 +4486,8 @@ mod tests { feed.parser = "stale".to_string(); feed.etag = Some("preserved".to_string()); feed.last_modified = Some("preserved-date".to_string()); + feed.content_sha256 = Some("preserved-sha".to_string()); + feed.last_success_unix = Some(123); let source_id = feed.source_id.clone(); backfill_official_threat_feeds(&mut data); @@ -4515,6 +4519,8 @@ mod tests { .unwrap(); assert!(feed.etag.is_none()); assert!(feed.last_modified.is_none()); + assert!(feed.content_sha256.is_none()); + assert!(feed.last_success_unix.is_none()); } #[test] From 682ff42b4f841bdcb1805e04097b5a5719642110 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 00:19:56 +0900 Subject: [PATCH 15/23] fix(feeds): preserve multiline URLhaus CSV rows --- src/official_feeds.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/official_feeds.rs b/src/official_feeds.rs index 75ee10f..f214e61 100644 --- a/src/official_feeds.rs +++ b/src/official_feeds.rs @@ -70,14 +70,22 @@ fn parse_urlhaus( let mut parsed = ParsedOfficialFeed::default(); let csv_body = body .lines() - .filter_map(|line| { + .scan(false, |seen_header, line| { let trimmed = line.trim_start(); - let header = trimmed - .strip_prefix('#') - .map(str::trim_start) - .filter(|candidate| candidate.starts_with("id,dateadded,url,")); - header.or_else(|| (!trimmed.starts_with('#')).then_some(line)) + if !*seen_header { + let header = trimmed + .strip_prefix('#') + .map(str::trim_start) + .filter(|candidate| candidate.starts_with("id,dateadded,url,")); + if let Some(header) = header { + *seen_header = true; + return Some(Some(header)); + } + return Some(None); + } + Some(Some(line)) }) + .flatten() .collect::>() .join("\n"); let mut reader = csv::ReaderBuilder::new() @@ -268,7 +276,7 @@ mod tests { "urlhaus_recent_csv", "urlhaus-online", 7200, - "# attribution\n# id,dateadded,url,url_status,reporter\n1,2026-01-01,https://evil.example/a,online,\"analyst\nteam\"\n", + "# attribution\n# id,dateadded,url,url_status,reporter\n1,2026-01-01,https://evil.example/a,online,\"analyst\n#team\"\n", ) .unwrap(); assert!( From 75d060914efad08f0416579a71fd08e48e8a38a8 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Fri, 28 Aug 2026 06:43:32 +0900 Subject: [PATCH 16/23] fix(feeds): reset refresh throttle on canonical URL drift --- src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 500f28f..8383352 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -307,6 +307,7 @@ fn backfill_official_threat_feeds(data: &mut AppData) { existing.etag = None; existing.last_modified = None; existing.content_sha256 = None; + existing.last_attempt_unix = None; existing.last_success_unix = None; } } else { @@ -4487,6 +4488,7 @@ mod tests { feed.etag = Some("preserved".to_string()); feed.last_modified = Some("preserved-date".to_string()); feed.content_sha256 = Some("preserved-sha".to_string()); + feed.last_attempt_unix = Some(456); feed.last_success_unix = Some(123); let source_id = feed.source_id.clone(); @@ -4520,6 +4522,7 @@ mod tests { assert!(feed.etag.is_none()); assert!(feed.last_modified.is_none()); assert!(feed.content_sha256.is_none()); + assert!(feed.last_attempt_unix.is_none()); assert!(feed.last_success_unix.is_none()); } From 81dd5dfad4ecf4bcb7d37584170d01877a794440 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Fri, 28 Aug 2026 11:02:58 +0900 Subject: [PATCH 17/23] fix(feeds): invalidate cached state on parser drift --- src/lib.rs | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 8383352..a86976c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -295,7 +295,8 @@ fn backfill_official_threat_feeds(data: &mut AppData) { .iter_mut() .find(|existing| existing.source_id == feed.source_id) { - let url_changed = existing.official_url != feed.official_url; + let content_contract_changed = + existing.official_url != feed.official_url || existing.parser != feed.parser; existing.official_url = feed.official_url; existing.parser = feed.parser; existing.indicator_types = feed.indicator_types; @@ -303,12 +304,13 @@ fn backfill_official_threat_feeds(data: &mut AppData) { existing.license_url = feed.license_url; existing.refresh_interval_seconds = feed.refresh_interval_seconds; existing.ttl_seconds = feed.ttl_seconds; - if url_changed { + if content_contract_changed { existing.etag = None; existing.last_modified = None; existing.content_sha256 = None; existing.last_attempt_unix = None; existing.last_success_unix = None; + existing.source_notice = None; } } else { data.official_threat_feeds.push(feed); @@ -4481,7 +4483,7 @@ mod tests { } #[test] - fn backfill_preserves_validators_only_while_registry_url_is_unchanged() { + fn backfill_preserves_official_feed_state_only_while_content_contract_is_unchanged() { let mut data = AppData::seeded(); let feed = &mut data.official_threat_feeds[0]; feed.parser = "stale".to_string(); @@ -4490,6 +4492,7 @@ mod tests { feed.content_sha256 = Some("preserved-sha".to_string()); feed.last_attempt_unix = Some(456); feed.last_success_unix = Some(123); + feed.source_notice = Some("preserved-notice".to_string()); let source_id = feed.source_id.clone(); backfill_official_threat_feeds(&mut data); @@ -4505,7 +4508,38 @@ mod tests { .unwrap(); assert_eq!(feed.official_url, canonical.official_url); assert_eq!(feed.parser, canonical.parser); + assert!(feed.etag.is_none()); + assert!(feed.last_modified.is_none()); + assert!(feed.content_sha256.is_none()); + assert!(feed.last_attempt_unix.is_none()); + assert!(feed.last_success_unix.is_none()); + assert!(feed.source_notice.is_none()); + + let feed = data + .official_threat_feeds + .iter_mut() + .find(|feed| feed.source_id == source_id) + .unwrap(); + feed.etag = Some("preserved".to_string()); + feed.last_modified = Some("preserved-date".to_string()); + feed.content_sha256 = Some("preserved-sha".to_string()); + feed.last_attempt_unix = Some(456); + feed.last_success_unix = Some(123); + feed.source_notice = Some("preserved-notice".to_string()); + + backfill_official_threat_feeds(&mut data); + + let feed = data + .official_threat_feeds + .iter() + .find(|feed| feed.source_id == source_id) + .unwrap(); assert_eq!(feed.etag.as_deref(), Some("preserved")); + assert_eq!(feed.last_modified.as_deref(), Some("preserved-date")); + assert_eq!(feed.content_sha256.as_deref(), Some("preserved-sha")); + assert_eq!(feed.last_attempt_unix, Some(456)); + assert_eq!(feed.last_success_unix, Some(123)); + assert_eq!(feed.source_notice.as_deref(), Some("preserved-notice")); let feed = data .official_threat_feeds @@ -4524,6 +4558,7 @@ mod tests { assert!(feed.content_sha256.is_none()); assert!(feed.last_attempt_unix.is_none()); assert!(feed.last_success_unix.is_none()); + assert!(feed.source_notice.is_none()); } #[test] From 29ce28da43d28fd47964cb8fe7b1f2ba504fb868 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Fri, 28 Aug 2026 12:09:14 +0900 Subject: [PATCH 18/23] fix(feeds): ignore offline URLhaus rows --- src/official_feeds.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/official_feeds.rs b/src/official_feeds.rs index f214e61..953c639 100644 --- a/src/official_feeds.rs +++ b/src/official_feeds.rs @@ -99,8 +99,19 @@ fn parse_urlhaus( .iter() .position(|header| header.eq_ignore_ascii_case("url")) .ok_or_else(|| "URLhaus CSV is missing url header".to_string())?; + let url_status_index = headers + .iter() + .position(|header| header.eq_ignore_ascii_case("url_status")); for record in reader.records() { let record = record.map_err(|error| format!("invalid URLhaus CSV record: {error}"))?; + if let Some(status_index) = url_status_index { + let Some(status) = record.get(status_index) else { + continue; + }; + if !status.eq_ignore_ascii_case("online") { + continue; + } + } let Some(url) = record.get(url_index).filter(|url| !url.is_empty()) else { continue; }; @@ -299,6 +310,22 @@ mod tests { .any(|item| item.indicator_type == "client_ip" && item.value == "2001:db8::7") ); assert_eq!(urlhaus_ipv6.dnsbl[0].address.to_string(), "2001:db8::7"); + let urlhaus_mixed_status = parse( + "urlhaus_recent_csv", + "urlhaus-online", + 7200, + "# attribution\n# id,dateadded,url,url_status,reporter\n1,2026-01-01,https://offline.example/a,offline,analyst\n2,2026-01-01,https://active.example/a,online,analyst\n", + ) + .unwrap(); + assert!( + urlhaus_mixed_status + .threats + .iter() + .any(|item| item.value == "active.example") + ); + assert!(urlhaus_mixed_status.threats.iter().all(|item| item.value + != "https://offline.example/a" + && item.value != "offline.example")); let threatfox = parse( "threatfox_json", From f56c62a9a190c732cd71db25d43d79e619955dbb Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Fri, 28 Aug 2026 13:24:07 +0900 Subject: [PATCH 19/23] fix(feeds): require https for credentialed official sources --- src/lib.rs | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index a86976c..62b03cd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1309,6 +1309,34 @@ async fn refresh_official_threat_feed( } _ => (configured_url, None, None), }; + let url = if auth_header.is_some() { + let parsed = match reqwest::Url::parse(&url) { + Ok(parsed) => parsed, + Err(_) => { + return official_feed_failure( + &state, + &source_id, + now, + "official feed URL is invalid", + false, + ) + .await; + } + }; + if parsed.scheme() != "https" { + return official_feed_failure( + &state, + &source_id, + now, + "official feed URL must use https", + false, + ) + .await; + } + parsed.to_string() + } else { + url + }; let mut request = if let Some(body) = request_body { state.feed_http.post(&url).json(&body) @@ -3542,6 +3570,19 @@ mod tests { } } + fn credentials_file_path(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "wardnet-{name}-{}-{}", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + std::fs::create_dir_all(&dir).unwrap(); + dir.join("credentials.json") + } + #[test] fn parse_event_limit_reads_optional_env() { assert_eq!( @@ -3622,6 +3663,49 @@ mod tests { .is_none(), "preflight failures must not throttle the corrective retry" ); + let credentials_path = credentials_file_path("official-feed-auth"); + std::fs::write( + &credentials_path, + r#"{"admin_token":"secret","threatfox_auth_key":"threatfox-secret"}"#, + ) + .unwrap(); + let credentialed_state = AppState::seeded(Some("secret".to_string())) + .with_credential_registry( + CredentialRegistry::bootstrap_secrets(Some(&credentials_path), None, None).unwrap(), + ) + .with_official_feed_url("threatfox-recent", "http://127.0.0.1:9/api/v1/".to_string()); + let credentialed_inspect = credentialed_state.clone(); + let credentialed_app = build_app(credentialed_state); + let insecure = app_request( + &credentialed_app, + authed_empty_request( + Method::POST, + "/api/official-threat-feeds/threatfox-recent/refresh", + "secret", + ), + ) + .await; + assert_eq!(insecure.status(), StatusCode::BAD_GATEWAY); + assert!(body_text(insecure).await.contains("must use https")); + assert!( + credentialed_inspect + .inner + .read() + .await + .official_threat_feeds + .iter() + .find(|feed| feed.source_id == "threatfox-recent") + .unwrap() + .last_attempt_unix + .is_none(), + "insecure preflight failures must not throttle the corrective retry" + ); + let _ = std::fs::remove_file(&credentials_path); + let _ = std::fs::remove_dir( + credentials_path + .parent() + .expect("credentials path always has a parent"), + ); let stalled_listener = StdTcpListener::bind("127.0.0.1:0").unwrap(); let stalled_address = stalled_listener.local_addr().unwrap(); let stalled_thread = thread::spawn(move || { From 9de0b21916e34a2130c4f648c1a544730320ec43 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Fri, 28 Aug 2026 17:19:06 +0900 Subject: [PATCH 20/23] fix(feeds): cap official feed material imports --- docs/runbooks/operations.md | 4 +- src/official_feeds.rs | 186 +++++++++++++++++++++++++++--------- 2 files changed, 142 insertions(+), 48 deletions(-) diff --git a/docs/runbooks/operations.md b/docs/runbooks/operations.md index 9b6b701..4b3fecc 100644 --- a/docs/runbooks/operations.md +++ b/docs/runbooks/operations.md @@ -50,7 +50,7 @@ WAF_IDS_STATE_PATH=./waf-ids-state.local.json \ cargo run ``` -Health reports `credentials_source` (`file` / `env` / `none`) and +Health reports `credentials_source` (`file` / `env` / `mixed` / `none`) and `admin_auth_configured` (boolean) without exposing secret values. ## Health Check @@ -65,7 +65,7 @@ Expected fields: - `persistence`: `memory` or `file` - `dnsbl_origin`: configured DNSBL origin without a trailing dot - `event_limit`: retained security event count -- `credentials_source`: `file`, `env`, or `none` +- `credentials_source`: `file`, `env`, `mixed`, or `none` - `admin_auth_configured`: whether any admin write token is configured ## Smoke Test diff --git a/src/official_feeds.rs b/src/official_feeds.rs index 953c639..ed9eed2 100644 --- a/src/official_feeds.rs +++ b/src/official_feeds.rs @@ -2,6 +2,8 @@ use crate::{DnsblEntry, Severity, ThreatIndicator}; use serde_json::Value; use std::net::IpAddr; +const MAX_OFFICIAL_FEED_MATERIAL_ITEMS: usize = 50_000; + #[derive(Debug, Default, PartialEq, Eq)] pub struct ParsedOfficialFeed { pub threats: Vec, @@ -14,11 +16,27 @@ pub fn parse( source_id: &str, ttl_seconds: u64, body: &str, +) -> Result { + parse_with_limit( + parser, + source_id, + ttl_seconds, + body, + MAX_OFFICIAL_FEED_MATERIAL_ITEMS, + ) +} + +fn parse_with_limit( + parser: &str, + source_id: &str, + ttl_seconds: u64, + body: &str, + max_material_items: usize, ) -> Result { match parser { - "spamhaus_drop_json" => parse_spamhaus(source_id, ttl_seconds, body), - "urlhaus_recent_csv" => parse_urlhaus(source_id, ttl_seconds, body), - "threatfox_json" => parse_threatfox(source_id, ttl_seconds, body), + "spamhaus_drop_json" => parse_spamhaus(source_id, ttl_seconds, body, max_material_items), + "urlhaus_recent_csv" => parse_urlhaus(source_id, ttl_seconds, body, max_material_items), + "threatfox_json" => parse_threatfox(source_id, ttl_seconds, body, max_material_items), _ => Err(format!("unsupported official feed parser: {parser}")), } } @@ -27,6 +45,7 @@ fn parse_spamhaus( source_id: &str, ttl_seconds: u64, body: &str, + max_material_items: usize, ) -> Result { let mut parsed = ParsedOfficialFeed::default(); for (line_number, line) in body.lines().enumerate() { @@ -50,14 +69,19 @@ fn parse_spamhaus( .ok_or_else(|| format!("Spamhaus record {} is missing cidr", line_number + 1))?; let (address, prefix_len) = parse_cidr(cidr)?; let sblid = value.get("sblid").and_then(Value::as_str).unwrap_or("DROP"); - parsed.dnsbl.push(DnsblEntry { - address, - prefix_len: Some(prefix_len), - code: "127.0.0.2".to_string(), - reason: format!("Spamhaus DROP {sblid}"), - source: source_id.to_string(), - ttl_seconds, - }); + push_dnsbl( + &mut parsed, + source_id, + max_material_items, + DnsblEntry { + address, + prefix_len: Some(prefix_len), + code: "127.0.0.2".to_string(), + reason: format!("Spamhaus DROP {sblid}"), + source: source_id.to_string(), + ttl_seconds, + }, + )?; } require_material(parsed, source_id) } @@ -66,6 +90,7 @@ fn parse_urlhaus( source_id: &str, ttl_seconds: u64, body: &str, + max_material_items: usize, ) -> Result { let mut parsed = ParsedOfficialFeed::default(); let csv_body = body @@ -115,33 +140,44 @@ fn parse_urlhaus( let Some(url) = record.get(url_index).filter(|url| !url.is_empty()) else { continue; }; - parsed - .threats - .push(threat(url, "url", source_id, ttl_seconds)); + push_threat( + &mut parsed, + source_id, + max_material_items, + threat(url, "url", source_id, ttl_seconds), + )?; let host = reqwest::Url::parse(url) .ok() .and_then(|url| url.host_str().map(str::to_ascii_lowercase)); if let Some(host) = host { let ip_candidate = host.trim_start_matches('[').trim_end_matches(']'); if let Ok(address) = ip_candidate.parse::() { - parsed.threats.push(threat( - &address.to_string(), - "client_ip", + push_threat( + &mut parsed, source_id, - ttl_seconds, - )); - parsed.dnsbl.push(DnsblEntry { - address, - prefix_len: None, - code: "127.0.0.2".to_string(), - reason: "URLhaus malware URL host".to_string(), - source: source_id.to_string(), - ttl_seconds, - }); + max_material_items, + threat(&address.to_string(), "client_ip", source_id, ttl_seconds), + )?; + push_dnsbl( + &mut parsed, + source_id, + max_material_items, + DnsblEntry { + address, + prefix_len: None, + code: "127.0.0.2".to_string(), + reason: "URLhaus malware URL host".to_string(), + source: source_id.to_string(), + ttl_seconds, + }, + )?; } else { - parsed - .threats - .push(threat(&host, "domain", source_id, ttl_seconds)); + push_threat( + &mut parsed, + source_id, + max_material_items, + threat(&host, "domain", source_id, ttl_seconds), + )?; } } } @@ -152,6 +188,7 @@ fn parse_threatfox( source_id: &str, ttl_seconds: u64, body: &str, + max_material_items: usize, ) -> Result { let value: Value = serde_json::from_str(body).map_err(|error| format!("invalid ThreatFox JSON: {error}"))?; @@ -175,32 +212,76 @@ fn parse_threatfox( }; let ioc_type = record.get("ioc_type").and_then(Value::as_str).unwrap_or(""); if ioc_type == "domain" { - parsed - .threats - .push(threat(ioc, "domain", source_id, ttl_seconds)); + push_threat( + &mut parsed, + source_id, + max_material_items, + threat(ioc, "domain", source_id, ttl_seconds), + )?; } else if ioc_type.starts_with("ip") { let Some(address) = threatfox_ip(ioc) else { continue; }; - parsed.threats.push(threat( - &address.to_string(), - "client_ip", + push_threat( + &mut parsed, source_id, - ttl_seconds, - )); - parsed.dnsbl.push(DnsblEntry { - address, - prefix_len: None, - code: "127.0.0.2".to_string(), - reason: format!("ThreatFox {ioc_type}: {ioc}"), - source: source_id.to_string(), - ttl_seconds, - }); + max_material_items, + threat(&address.to_string(), "client_ip", source_id, ttl_seconds), + )?; + push_dnsbl( + &mut parsed, + source_id, + max_material_items, + DnsblEntry { + address, + prefix_len: None, + code: "127.0.0.2".to_string(), + reason: format!("ThreatFox {ioc_type}: {ioc}"), + source: source_id.to_string(), + ttl_seconds, + }, + )?; } } require_material(parsed, source_id) } +fn push_threat( + parsed: &mut ParsedOfficialFeed, + source_id: &str, + max_material_items: usize, + indicator: ThreatIndicator, +) -> Result<(), String> { + ensure_material_capacity(parsed, source_id, max_material_items)?; + parsed.threats.push(indicator); + Ok(()) +} + +fn push_dnsbl( + parsed: &mut ParsedOfficialFeed, + source_id: &str, + max_material_items: usize, + entry: DnsblEntry, +) -> Result<(), String> { + ensure_material_capacity(parsed, source_id, max_material_items)?; + parsed.dnsbl.push(entry); + Ok(()) +} + +fn ensure_material_capacity( + parsed: &ParsedOfficialFeed, + source_id: &str, + max_material_items: usize, +) -> Result<(), String> { + if parsed.threats.len().saturating_add(parsed.dnsbl.len()) >= max_material_items { + Err(format!( + "official feed {source_id} exceeded material limit of {max_material_items} items" + )) + } else { + Ok(()) + } +} + fn threat(value: &str, indicator_type: &str, source: &str, ttl_seconds: u64) -> ThreatIndicator { ThreatIndicator { value: value.to_string(), @@ -346,4 +427,17 @@ mod tests { .is_err() ); } + + #[test] + fn official_feed_parse_fails_closed_when_material_limit_is_exceeded() { + let err = parse_with_limit( + "urlhaus_recent_csv", + "urlhaus-online", + 7200, + "# attribution\n# id,dateadded,url,url_status,reporter\n1,2026-01-01,https://[2001:db8::7]/a,online,analyst\n2,2026-01-01,https://evil.example/a,online,analyst\n", + 3, + ) + .unwrap_err(); + assert!(err.contains("exceeded material limit")); + } } From a74dab92f9656cecd610278e818dd2d66829c8f9 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Fri, 28 Aug 2026 21:27:10 +0900 Subject: [PATCH 21/23] test(feeds): pin redirect handling for official sources --- docs/official-threat-feeds.md | 12 +++++-- src/lib.rs | 67 +++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/docs/official-threat-feeds.md b/docs/official-threat-feeds.md index 7375184..960ef9f 100644 --- a/docs/official-threat-feeds.md +++ b/docs/official-threat-feeds.md @@ -29,7 +29,7 @@ redistribution. The canonical formats are documented by [URLhaus](https://urlhaus.abuse.ch/api/), and [ThreatFox](https://threatfox.abuse.ch/api/). -The current contracts were rechecked against those official pages on 2026-08-27: +The current contracts were rechecked against those official pages on 2026-08-28: Spamhaus publishes the two JSON URLs, requires attribution and preservation of its date/© notice, re-evaluates DROP daily, and says daily fetching is sufficient. URLhaus documents the authenticated `recent.csv` export URL. ThreatFox requires `Auth-Key` and documents @@ -37,6 +37,11 @@ the authenticated `recent.csv` export URL. ThreatFox requires `Auth-Key` and doc checksum for these responses, so Wardnet records its own digest after TLS retrieval and full-response validation while retaining ETag/Last-Modified when provided. +The built-in GET endpoints were also rechecked live on 2026-08-28. Spamhaus `drop_v4.json`, +Spamhaus `drop_v6.json`, and URLhaus `csv_recent` each returned direct `HTTP 200` without an +intermediate redirect. ThreatFox uses a POST-only API contract at `/api/v1/`, so a GET/HEAD +probe is not the authoritative success path for that source. + ## Credentials Place abuse.ch keys in the JSON credential registry selected by @@ -72,7 +77,10 @@ the source refresh interval, so adding a missing key permits an immediate correc These are threat-intelligence feeds consumed by gateway scoring. The authoritative DNSBL zone exports exact IPv4 host entries only; CIDR ranges and IPv6 entries remain available to -inline gateway scoring rather than being expanded into an unbounded zone. Wardnet is not a +inline gateway scoring rather than being expanded into an unbounded zone. That host-only +publication rule applies to both official-feed imports and operator-created `DnsblEntry` +records with `prefix_len`; subnet entries remain visible through `GET /api/dnsbl` and active +for inline scoring, but they are intentionally omitted from `/dnsbl/zone`. Wardnet is not a recursive DNS resolver and does not configure an upstream DNS resolver. ## Scrubbed runtime evidence (2026-08-27) diff --git a/src/lib.rs b/src/lib.rs index 62b03cd..dbffdab 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5573,6 +5573,73 @@ mod tests { assert_eq!(response.status(), StatusCode::BAD_GATEWAY); } + #[tokio::test] + async fn official_threat_feed_refresh_rejects_redirected_upstream() { + let target_feed = Router::new().route( + "/drop.json", + get(|| async { + ( + StatusCode::OK, + "{\"cidr\":\"198.51.100.0/24\",\"sblid\":\"SBL1\"}\n", + ) + }), + ); + let target_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let target_addr = target_listener.local_addr().unwrap(); + tokio::spawn(axum::serve(target_listener, target_feed).into_future()); + + let redirect_target = format!("http://{target_addr}/drop.json"); + let redirect_feed = Router::new().route( + "/drop.json", + get(move || { + let location = redirect_target.clone(); + async move { (StatusCode::FOUND, [(header::LOCATION, location)]) } + }), + ); + let redirect_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let redirect_addr = redirect_listener.local_addr().unwrap(); + tokio::spawn(axum::serve(redirect_listener, redirect_feed).into_future()); + + let state = AppState::seeded(Some("secret".to_string())).with_official_feed_url( + "spamhaus-drop-v4", + format!("http://{redirect_addr}/drop.json"), + ); + let inspect = state.clone(); + let app = build_app(state); + + let response = app_request( + &app, + authed_empty_request( + Method::POST, + "/api/official-threat-feeds/spamhaus-drop-v4/refresh", + "secret", + ), + ) + .await; + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + + let body = body_text(response).await; + assert!(body.contains("official feed returned HTTP 302")); + + let state = inspect.inner.read().await; + assert!( + state + .dnsbl + .iter() + .all(|entry| entry.source != "spamhaus-drop-v4") + ); + let feed = state + .official_threat_feeds + .iter() + .find(|feed| feed.source_id == "spamhaus-drop-v4") + .unwrap(); + assert_eq!(feed.last_success_unix, None); + assert_eq!( + feed.last_error.as_deref(), + Some("official feed returned HTTP 302") + ); + } + #[tokio::test] async fn suricata_eve_ingest_maps_alerts_to_security_events() { let app = build_app(AppState::seeded(Some("secret".to_string()))); From 55499513274606233601f6ea8bafc15e6b0bf7bf Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Fri, 28 Aug 2026 22:46:55 +0900 Subject: [PATCH 22/23] fix(auth): read admin credentials from registry --- src/lib.rs | 42 +++++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index dbffdab..caa745f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -210,19 +210,34 @@ impl AppState { self } + fn configured_admin_tokens(&self) -> Option> { + self.credentials + .get_credential(CRED_ADMIN_TOKENS) + .map(parse_admin_tokens) + .filter(|tokens| !tokens.is_empty()) + .or_else(|| (!self.admin_tokens.is_empty()).then(|| self.admin_tokens.clone())) + } + + fn configured_admin_token(&self) -> Option<&str> { + self.credentials + .get_credential(CRED_ADMIN_TOKEN) + .or(self.admin_token.as_deref()) + } + /// The principal mapped to the request's `X-Admin-Token`, if configured. - fn principal_for_token(&self, headers: &HeaderMap) -> Option<&AdminPrincipal> { - headers + fn principal_for_token(&self, headers: &HeaderMap) -> Option { + let presented = headers .get("x-admin-token") - .and_then(|value| value.to_str().ok()) - .and_then(|token| self.admin_tokens.get(token)) + .and_then(|value| value.to_str().ok())?; + self.configured_admin_tokens() + .and_then(|tokens| tokens.get(presented).cloned()) } /// The actor name mapped to the request's `X-Admin-Token`, if that token is a /// configured RBAC token. fn actor_for_token(&self, headers: &HeaderMap) -> Option { self.principal_for_token(headers) - .map(|principal| principal.actor.clone()) + .map(|principal| principal.actor) } /// Records one gateway request for `client_ip` and returns `true` if it is @@ -283,7 +298,9 @@ impl AppState { dnsbl_origin: self.dnsbl_origin.clone(), event_limit: self.event_limit, credentials_source: self.credentials_source.as_str().to_string(), - admin_auth_configured: self.admin_token.is_some() || !self.admin_tokens.is_empty(), + admin_auth_configured: self.credentials.has_admin_auth() + || self.admin_token.is_some() + || !self.admin_tokens.is_empty(), } } } @@ -2815,10 +2832,10 @@ fn admin_authenticated(state: &AppState, headers: &HeaderMap) -> bool { let presented = headers .get("x-admin-token") .and_then(|value| value.to_str().ok()); - if !state.admin_tokens.is_empty() { - return presented.is_some_and(|token| state.admin_tokens.contains_key(token)); + if let Some(tokens) = state.configured_admin_tokens() { + return presented.is_some_and(|token| tokens.contains_key(token)); } - let Some(expected) = state.admin_token.as_deref() else { + let Some(expected) = state.configured_admin_token() else { return true; }; presented.is_some_and(|actual| actual == expected) @@ -2831,16 +2848,15 @@ fn admin_authorized(state: &AppState, headers: &HeaderMap) -> bool { .get("x-admin-token") .and_then(|value| value.to_str().ok()); // RBAC tokens take precedence when configured. - if !state.admin_tokens.is_empty() { + if let Some(tokens) = state.configured_admin_tokens() { return presented.is_some_and(|token| { - state - .admin_tokens + tokens .get(token) .is_some_and(|principal| principal.can_write) }); } // Fallback: single shared token (None means auth is disabled). - let Some(expected) = state.admin_token.as_deref() else { + let Some(expected) = state.configured_admin_token() else { return true; }; presented.is_some_and(|actual| actual == expected) From aad8224ab664861b540bdf818e9580470cbe89fb Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Sat, 29 Aug 2026 00:04:18 +0900 Subject: [PATCH 23/23] fix(feeds): honor registry-backed admin auth --- src/lib.rs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index caa745f..479390e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1609,7 +1609,9 @@ async fn read_official_feed_body(response: reqwest::Response) -> Result bool { - (state.admin_token.is_some() || !state.admin_tokens.is_empty()) + (state.credentials.has_admin_auth() + || state.admin_token.is_some() + || !state.admin_tokens.is_empty()) && admin_authenticated(state, headers) } @@ -4240,6 +4242,20 @@ mod tests { assert_eq!(audit_actor(&state, &named), "carol"); } + #[test] + fn official_feed_auth_accepts_registry_only_admin_token() { + let registry = + CredentialRegistry::bootstrap_secrets(None, Some("registry-admin".to_string()), None) + .unwrap(); + let state = AppState::seeded(None).with_credential_registry(registry); + + let mut headers = HeaderMap::new(); + headers.insert("x-admin-token", "registry-admin".parse().unwrap()); + + assert!(admin_authenticated(&state, &headers)); + assert!(official_feed_authenticated(&state, &headers)); + } + #[tokio::test] async fn readonly_token_can_read_audit_logs_but_cannot_write() { let tokens = parse_admin_tokens("write:ops:admin,read:auditor:readonly");