diff --git a/Cargo.lock b/Cargo.lock index 4ea03ec..72654a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -828,7 +828,7 @@ dependencies = [ [[package]] name = "ant-cli" -version = "0.1.6" +version = "0.2.0-rc.1" dependencies = [ "ant-core", "anyhow", @@ -870,6 +870,7 @@ dependencies = [ "rand 0.8.6", "reqwest 0.12.28", "rmp-serde", + "saorsa-core 0.23.1", "saorsa-pqc 0.5.1", "self-replace", "self_encryption", @@ -904,8 +905,8 @@ dependencies = [ [[package]] name = "ant-node" -version = "0.10.1" -source = "git+https://github.com/withAutonomi/ant-node.git?branch=mick%2Falways-masque-relay-rebased#5d25e1f1ac91cd82d24116a085dfc1c26d0096ae" +version = "0.11.0-rc.1" +source = "git+https://github.com/WithAutonomi/ant-node.git?branch=rc-2026.4.2#2ba556e4af1b606ed2cf52c01674060cb0ab8ce3" dependencies = [ "aes-gcm-siv", "blake3", @@ -930,7 +931,7 @@ dependencies = [ "rand 0.8.6", "reqwest 0.13.2", "rmp-serde", - "saorsa-core", + "saorsa-core 0.24.0-rc.1", "saorsa-pqc 0.5.1", "self-replace", "self_encryption", @@ -4988,9 +4989,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.38" +version = "0.23.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" +checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e" dependencies = [ "aws-lc-rs", "log", @@ -5126,8 +5127,40 @@ dependencies = [ [[package]] name = "saorsa-core" -version = "0.22.0" -source = "git+https://github.com/saorsa-labs/saorsa-core.git?branch=mick%2Falways-masque-relay-rebased#d37491fd2ead4b2e81f85a8d3c43e707bbafacc7" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d459ba0e4f1a3ac918a1d6b17db392cccde2ee74569545d171d7644fc6463cc7" +dependencies = [ + "anyhow", + "async-trait", + "blake3", + "bytes", + "dashmap", + "dirs 6.0.0", + "futures", + "hex", + "lru", + "once_cell", + "parking_lot", + "postcard", + "rand 0.8.6", + "saorsa-pqc 0.5.1", + "saorsa-transport 0.32.0", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "uuid", + "wyz", +] + +[[package]] +name = "saorsa-core" +version = "0.24.0-rc.1" +source = "git+https://github.com/saorsa-labs/saorsa-core.git?branch=rc-2026.4.2#a2813e83526c70048f5a564848b05e8088d786e4" dependencies = [ "anyhow", "async-trait", @@ -5143,7 +5176,7 @@ dependencies = [ "postcard", "rand 0.8.6", "saorsa-pqc 0.5.1", - "saorsa-transport", + "saorsa-transport 0.33.0-rc.1", "serde", "serde_json", "tempfile", @@ -5240,8 +5273,67 @@ dependencies = [ [[package]] name = "saorsa-transport" -version = "0.31.0" -source = "git+https://github.com/saorsa-labs/saorsa-transport.git?branch=mick%2Falways-masque-relay-rebased#ec163406f526f42667f12a1ed4f5b6ec0ed4e02e" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250826f52ac60992947359218d83c21f658aa85407e3980a5dbb258eff4c0cc3" +dependencies = [ + "anyhow", + "async-trait", + "aws-lc-rs", + "blake3", + "bytes", + "chrono", + "clap", + "core-foundation 0.9.4", + "dashmap", + "dirs 5.0.1", + "futures-util", + "hex", + "igd-next", + "indexmap 2.14.0", + "keyring", + "libc", + "lru-slab", + "nix", + "once_cell", + "parking_lot", + "pin-project-lite", + "quinn-udp 0.6.1", + "rand 0.8.6", + "rcgen", + "regex", + "reqwest 0.13.2", + "rustc-hash", + "rustls", + "rustls-native-certs", + "rustls-pemfile", + "rustls-platform-verifier", + "rustls-post-quantum", + "saorsa-pqc 0.4.2", + "serde", + "serde_json", + "serde_yaml", + "slab", + "socket2 0.5.10", + "system-configuration 0.6.1", + "thiserror 2.0.18", + "time", + "tinyvec", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", + "unicode-width", + "uuid", + "windows", + "x25519-dalek", + "zeroize", +] + +[[package]] +name = "saorsa-transport" +version = "0.33.0-rc.1" +source = "git+https://github.com/saorsa-labs/saorsa-transport.git?branch=rc-2026.4.2#8b44b242fd62023c1a96728382ddd72917a63a4a" dependencies = [ "anyhow", "async-trait", diff --git a/ant-cli/Cargo.toml b/ant-cli/Cargo.toml index 92264d2..6c559e5 100644 --- a/ant-cli/Cargo.toml +++ b/ant-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ant-cli" -version = "0.1.6" +version = "0.2.0-rc.1" edition = "2021" [[bin]] diff --git a/ant-core/Cargo.toml b/ant-core/Cargo.toml index 9b2ac3f..ae5d345 100644 --- a/ant-core/Cargo.toml +++ b/ant-core/Cargo.toml @@ -38,7 +38,7 @@ tracing = "0.1" bytes = "1" lru = "0.16" rand = "0.8" -ant-node = { git = "https://github.com/withAutonomi/ant-node.git", branch = "mick/always-masque-relay-rebased" } +ant-node = { git = "https://github.com/WithAutonomi/ant-node.git", branch = "rc-2026.4.2" } saorsa-pqc = "0.5" tracing-subscriber = { version = "0.3", features = ["env-filter"] } @@ -57,6 +57,12 @@ serial_test = "3" anyhow = "1" alloy = { version = "1.6", features = ["node-bindings"] } tokio-test = "0.4" +rmp-serde = "1" +# Direct access to BootstrapManager used by the cold-start-from-disk test, +# which populates a cache via `add_peer_trusted` (bypasses Sybil rate limits) +# and then verifies reload after save. Version tracks ant-node's transitive +# saorsa-core dep. +saorsa-core = "0.23" [[example]] name = "start-local-devnet" diff --git a/ant-core/src/data/client/chunk.rs b/ant-core/src/data/client/chunk.rs index 592e6a0..46174e1 100644 --- a/ant-core/src/data/client/chunk.rs +++ b/ant-core/src/data/client/chunk.rs @@ -3,6 +3,7 @@ //! Chunks are immutable, content-addressed data blocks where the address //! is the BLAKE3 hash of the content. +use crate::data::client::peer_cache::record_peer_outcome; use crate::data::client::Client; use crate::data::error::{Error, Result}; use ant_node::ant_protocol::{ @@ -15,7 +16,7 @@ use ant_node::CLOSE_GROUP_MAJORITY; use bytes::Bytes; use futures::stream::{FuturesUnordered, StreamExt}; use std::future::Future; -use std::time::Duration; +use std::time::{Duration, Instant}; use tracing::{debug, warn}; /// Data type identifier for chunks (used in quote requests). @@ -171,7 +172,7 @@ impl Client { let addr_hex = hex::encode(address); let timeout_secs = self.config().store_timeout_secs; - send_and_await_chunk_response( + let result = send_and_await_chunk_response( node, target_peer, message_bytes, @@ -204,7 +205,15 @@ impl Client { )) }, ) - .await + .await; + + // No RTT recorded on the PUT path: the wall-clock is dominated by + // the ~4 MB payload upload, which reflects the uploader's uplink + // rather than the peer's responsiveness. Quote-path and GET-path + // RTTs still feed quality scoring. + record_peer_outcome(node, *target_peer, peer_addrs, result.is_ok(), None).await; + + result } /// Retrieve a chunk from the Autonomi network. @@ -278,7 +287,8 @@ impl Client { let addr_hex = hex::encode(address); let timeout_secs = self.config().store_timeout_secs; - send_and_await_chunk_response( + let start = Instant::now(); + let result = send_and_await_chunk_response( node, peer, message_bytes, @@ -325,7 +335,13 @@ impl Client { )) }, ) - .await + .await; + + let success = result.is_ok(); + let rtt_ms = success.then(|| start.elapsed().as_millis() as u64); + record_peer_outcome(node, *peer, peer_addrs, success, rtt_ms).await; + + result } /// Check if a chunk exists on the network. diff --git a/ant-core/src/data/client/data.rs b/ant-core/src/data/client/data.rs index 8c588dc..2541b63 100644 --- a/ant-core/src/data/client/data.rs +++ b/ant-core/src/data/client/data.rs @@ -206,6 +206,7 @@ impl Client { prepared_chunks, payment_intent, }, + data_map_address: None, }) } diff --git a/ant-core/src/data/client/file.rs b/ant-core/src/data/client/file.rs index 5478a96..ce0cfe1 100644 --- a/ant-core/src/data/client/file.rs +++ b/ant-core/src/data/client/file.rs @@ -413,6 +413,23 @@ fn check_disk_space_for_spill(file_size: u64) -> Result<()> { Ok(()) } +/// Whether the data map is published to the network for address-based retrieval. +/// +/// A private upload stores only the data chunks and returns the `DataMap` to +/// the caller — only someone holding that `DataMap` can reconstruct the file. +/// A public upload additionally stores the serialized `DataMap` as a chunk on +/// the network, yielding a single chunk address that anyone can use to +/// retrieve the `DataMap` (via [`Client::data_map_fetch`]) and then the file. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Visibility { + /// Keep the data map local; only the holder can retrieve the file. + #[default] + Private, + /// Publish the data map as a network chunk so anyone with the returned + /// address can retrieve and decrypt the file. + Public, +} + /// Estimated cost of uploading a file, returned by /// [`Client::estimate_upload_cost`]. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] @@ -431,8 +448,14 @@ pub struct UploadCostEstimate { pub payment_mode: PaymentMode, } + /// Result of a file upload: the `DataMap` needed to retrieve the file. +/// +/// Marked `#[non_exhaustive]` so adding a new field in future is not a +/// breaking change for downstream consumers that construct or pattern-match +/// on this struct. #[derive(Debug, Clone)] +#[non_exhaustive] pub struct FileUploadResult { /// The data map containing chunk metadata for reconstruction. pub data_map: DataMap, @@ -444,6 +467,14 @@ pub struct FileUploadResult { pub storage_cost_atto: String, /// Total gas cost in wei. 0 if no on-chain transactions were made. pub gas_cost_wei: u128, + /// Chunk address of the serialized `DataMap`, set only for + /// [`Visibility::Public`] uploads. **`Some` means this address is + /// retrievable from the network (via [`Client::data_map_fetch`])**, not + /// necessarily that *this* upload paid to store it — if the serialized + /// `DataMap` hashed to a chunk that was already on the network (same + /// file uploaded before; deterministic via self-encryption), the address + /// is still returned but no storage payment was made for it. + pub data_map_address: Option<[u8; 32]>, } /// Payment information for external signing — either wave-batch or merkle. @@ -476,12 +507,23 @@ pub enum ExternalPaymentInfo { /// Note: This struct stays in Rust memory — only the public fields of /// `payment_info` are sent to the frontend. `PreparedChunk` contains /// non-serializable network types, so the full struct cannot derive `Serialize`. +/// +/// Marked `#[non_exhaustive]` so adding a new field in future is not a +/// breaking change for downstream consumers. #[derive(Debug)] +#[non_exhaustive] pub struct PreparedUpload { /// The data map for later retrieval. pub data_map: DataMap, /// Payment information — either wave-batch or merkle depending on chunk count. pub payment_info: ExternalPaymentInfo, + /// Chunk address of the serialized `DataMap` when this upload was + /// prepared with [`Visibility::Public`]. `Some` means the address is + /// retrievable on the network after finalization — either because this + /// upload paid to store the chunk in `payment_info`, or because the + /// chunk was already on the network (deterministic self-encryption). + /// Carried through to [`FileUploadResult::data_map_address`]. + pub data_map_address: Option<[u8; 32]>, } /// Return type for [`spawn_file_encryption`]: chunk receiver, `DataMap` oneshot, join handle. @@ -771,11 +813,27 @@ impl Client { /// Phase 1 of external-signer upload: encrypt file and prepare chunks. /// + /// Equivalent to [`Client::file_prepare_upload_with_visibility`] with + /// [`Visibility::Private`] — see that method for details. + pub async fn file_prepare_upload(&self, path: &Path) -> Result { + self.file_prepare_upload_with_visibility(path, Visibility::Private) + .await + } + + /// Phase 1 of external-signer upload with explicit [`Visibility`] control. + /// /// Requires an EVM network (for contract price queries) but NOT a wallet. /// Returns a [`PreparedUpload`] containing the data map, prepared chunks, /// and a [`PaymentIntent`] that the external signer uses to construct /// and submit the on-chain payment transaction. /// + /// When `visibility` is [`Visibility::Public`], the serialized `DataMap` + /// is bundled into the payment batch as an additional chunk and its + /// address is recorded on the returned [`PreparedUpload`]. After + /// [`Client::finalize_upload`] (or `_merkle`) succeeds, that address is + /// surfaced via [`FileUploadResult::data_map_address`] so the uploader + /// can share a single address from which anyone can retrieve the file. + /// /// **Memory note:** Encryption uses disk spilling for bounded memory, but /// the returned [`PreparedUpload`] holds all chunk content in memory (each /// [`PreparedChunk`] contains a `Bytes` with the full chunk data). This is @@ -787,9 +845,13 @@ impl Client { /// /// Returns an error if there is insufficient disk space, the file cannot /// be read, encryption fails, or quote collection fails. - pub async fn file_prepare_upload(&self, path: &Path) -> Result { + pub async fn file_prepare_upload_with_visibility( + &self, + path: &Path, + visibility: Visibility, + ) -> Result { debug!( - "Preparing file upload for external signing: {}", + "Preparing file upload for external signing (visibility={visibility:?}): {}", path.display() ); @@ -807,12 +869,35 @@ impl Client { // Read each chunk from disk and collect quotes concurrently. // Note: all PreparedChunks accumulate in memory because the external-signer // protocol requires them for finalize_upload. NOT memory-bounded for large files. - let chunk_data: Vec = spill + let mut chunk_data: Vec = spill .addresses .iter() .map(|addr| spill.read_chunk(addr)) .collect::, _>>()?; + // For public uploads, bundle the serialized DataMap as an extra chunk + // in the same payment batch. This lets the external signer pay for + // the data chunks and the DataMap chunk in one flow, and lets the + // finalize step return the DataMap's chunk address as the shareable + // retrieval address. + let data_map_address = match visibility { + Visibility::Private => None, + Visibility::Public => { + let serialized = rmp_serde::to_vec(&data_map).map_err(|e| { + Error::Serialization(format!("Failed to serialize DataMap: {e}")) + })?; + let bytes = Bytes::from(serialized); + let address = compute_address(&bytes); + info!( + "Public upload: bundling DataMap chunk ({} bytes) at address {}", + bytes.len(), + hex::encode(address) + ); + chunk_data.push(bytes); + Some(address) + } + }; + let chunk_count = chunk_data.len(); let payment_info = if should_use_merkle(chunk_count, PaymentMode::Auto) { @@ -857,6 +942,22 @@ impl Client { } } + // Surface the "DataMap chunk was already on the network" case + // so debugging "why is data_map_address set but no storage cost + // appears for it?" doesn't require reading the source. See the + // `data_map_address` doc comment for why this is still a valid + // `Some(addr)` outcome. + if let Some(addr) = data_map_address { + if !prepared_chunks.iter().any(|c| c.address == addr) { + info!( + "Public upload: DataMap chunk {} was already stored \ + on the network — address is retrievable without a \ + new payment", + hex::encode(addr) + ); + } + } + let payment_intent = PaymentIntent::from_prepared_chunks(&prepared_chunks); info!( @@ -875,6 +976,7 @@ impl Client { Ok(PreparedUpload { data_map, payment_info, + data_map_address, }) } @@ -894,6 +996,7 @@ impl Client { prepared: PreparedUpload, tx_hash_map: &HashMap, ) -> Result { + let data_map_address = prepared.data_map_address; match prepared.payment_info { ExternalPaymentInfo::WaveBatch { prepared_chunks, @@ -921,6 +1024,7 @@ impl Client { payment_mode_used: PaymentMode::Single, storage_cost_atto: "0".into(), gas_cost_wei: 0, + data_map_address, }) } ExternalPaymentInfo::Merkle { .. } => Err(Error::Payment( @@ -947,6 +1051,7 @@ impl Client { prepared: PreparedUpload, winner_pool_hash: [u8; 32], ) -> Result { + let data_map_address = prepared.data_map_address; match prepared.payment_info { ExternalPaymentInfo::Merkle { prepared_batch, @@ -966,6 +1071,7 @@ impl Client { payment_mode_used: PaymentMode::Merkle, storage_cost_atto: "0".into(), gas_cost_wei: 0, + data_map_address, }) } ExternalPaymentInfo::WaveBatch { .. } => Err(Error::Payment( @@ -1055,6 +1161,7 @@ impl Client { payment_mode_used: PaymentMode::Single, storage_cost_atto: sc, gas_cost_wei: gc, + data_map_address: None, }); } Err(e) => return Err(e), @@ -1080,6 +1187,7 @@ impl Client { payment_mode_used: actual_mode, storage_cost_atto, gas_cost_wei, + data_map_address: None, }) } diff --git a/ant-core/src/data/client/mod.rs b/ant-core/src/data/client/mod.rs index 4cc4c00..cda985b 100644 --- a/ant-core/src/data/client/mod.rs +++ b/ant-core/src/data/client/mod.rs @@ -10,6 +10,7 @@ pub mod data; pub mod file; pub mod merkle; pub mod payment; +pub(crate) mod peer_cache; pub mod quote; use crate::data::client::cache::ChunkCache; diff --git a/ant-core/src/data/client/peer_cache.rs b/ant-core/src/data/client/peer_cache.rs new file mode 100644 index 0000000..dc67aa0 --- /dev/null +++ b/ant-core/src/data/client/peer_cache.rs @@ -0,0 +1,137 @@ +//! Bootstrap-cache population helpers. +//! +//! Wires client-side peer contacts into saorsa-core's `BootstrapManager` +//! so the persistent cache reflects real peer quality across sessions. + +use ant_node::core::{MultiAddr, P2PNode, PeerId}; +use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; +use tracing::debug; + +/// Feed a peer contact outcome into the `BootstrapManager` cache so future +/// cold-starts can rank peers by observed latency and success. +/// +/// `success = true`: upserts the peer via `add_discovered_peer` (subject to +/// saorsa-core Sybil checks — rate limit + IP diversity) and records RTT via +/// `update_peer_metrics`. +/// +/// `success = false`: only updates the quality score of peers already in +/// the cache. Unreachable peers are never inserted. +/// +/// Both upstream calls silently discard errors — peer-cache bookkeeping +/// must never abort a user operation. Enable the `saorsa_core::bootstrap` +/// tracing target to see rejection reasons. +pub(crate) async fn record_peer_outcome( + node: &Arc, + peer_id: PeerId, + addrs: &[MultiAddr], + success: bool, + rtt_ms: Option, +) { + if success { + let before = node.cached_peer_count().await; + let _ = node.add_discovered_peer(peer_id, addrs.to_vec()).await; + let after = node.cached_peer_count().await; + if after > before { + debug!("Bootstrap cache grew: {before} -> {after} peers"); + } + } + if let Some(primary) = select_primary_multiaddr(addrs) { + let _ = node + .update_peer_metrics(primary, success, rtt_ms, None) + .await; + } +} + +/// Pick the `MultiAddr` to use as the peer's cache key. +/// +/// Prefers a globally routable socket address over RFC1918 / link-local / +/// loopback. Without this, a peer advertising `[10.0.0.5, 203.0.113.1]` +/// would be keyed under the RFC1918 address, so metrics recorded during +/// a contact over the public address would land on a stale cache entry. +/// Falls back to any socket-addressable `MultiAddr` if none look global. +fn select_primary_multiaddr(addrs: &[MultiAddr]) -> Option<&MultiAddr> { + addrs + .iter() + .find(|a| a.socket_addr().is_some_and(|sa| is_globally_routable(&sa))) + .or_else(|| addrs.iter().find(|a| a.socket_addr().is_some())) +} + +fn is_globally_routable(addr: &SocketAddr) -> bool { + match addr.ip() { + IpAddr::V4(v4) => { + !v4.is_private() + && !v4.is_loopback() + && !v4.is_link_local() + && !v4.is_broadcast() + && !v4.is_documentation() + && !v4.is_unspecified() + } + IpAddr::V6(v6) => { + // Full Ipv6Addr::is_global is unstable; this is the practical + // subset that mirrors the IPv4 checks above. + !v6.is_loopback() + && !v6.is_unspecified() + && !v6.is_multicast() + && !v6.segments()[0].eq(&0xfe80) // link-local fe80::/10 (approx) + && !matches!(v6.segments()[0] & 0xfe00, 0xfc00) // unique-local fc00::/7 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::{Ipv4Addr, Ipv6Addr}; + + #[test] + fn globally_routable_v4() { + // 8.8.8.8 (Google DNS) — genuinely public, not in any reserved range. + assert!(is_globally_routable(&SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8)), + 80 + ))); + assert!(!is_globally_routable(&SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(10, 0, 0, 5)), + 80 + ))); + assert!(!is_globally_routable(&SocketAddr::new( + IpAddr::V4(Ipv4Addr::LOCALHOST), + 80 + ))); + assert!(!is_globally_routable(&SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1)), + 80 + ))); + // 203.0.113.0/24 is TEST-NET-3 documentation — rejected by + // `is_documentation()`, which is the behaviour we want: quality + // metrics should not land on addresses that are never dialed in + // production by spec. + assert!(!is_globally_routable(&SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(203, 0, 113, 1)), + 80 + ))); + } + + #[test] + fn globally_routable_v6() { + // 2606:4700:4700::1111 (Cloudflare DNS) — a real public v6 outside + // the `2001:db8::/32` documentation prefix. + assert!(is_globally_routable(&SocketAddr::new( + IpAddr::V6(Ipv6Addr::new(0x2606, 0x4700, 0x4700, 0, 0, 0, 0, 0x1111)), + 80 + ))); + assert!(!is_globally_routable(&SocketAddr::new( + IpAddr::V6(Ipv6Addr::LOCALHOST), + 80 + ))); + assert!(!is_globally_routable(&SocketAddr::new( + IpAddr::V6(Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1)), + 80 + ))); + assert!(!is_globally_routable(&SocketAddr::new( + IpAddr::V6(Ipv6Addr::new(0xfc00, 0, 0, 0, 0, 0, 0, 1)), + 80 + ))); + } +} diff --git a/ant-core/src/data/client/quote.rs b/ant-core/src/data/client/quote.rs index 8da0115..c0d2ebd 100644 --- a/ant-core/src/data/client/quote.rs +++ b/ant-core/src/data/client/quote.rs @@ -3,6 +3,7 @@ //! Handles requesting storage quotes from network nodes and //! managing payment for data storage. +use crate::data::client::peer_cache::record_peer_outcome; use crate::data::client::Client; use crate::data::error::{Error, Result}; use ant_node::ant_protocol::{ @@ -14,7 +15,7 @@ use ant_node::{CLOSE_GROUP_MAJORITY, CLOSE_GROUP_SIZE}; use evmlib::common::Amount; use evmlib::PaymentQuote; use futures::stream::{FuturesUnordered, StreamExt}; -use std::time::Duration; +use std::time::{Duration, Instant}; use tracing::{debug, info, warn}; /// Compute XOR distance between a peer's ID bytes and a target address. @@ -106,6 +107,7 @@ impl Client { let node_clone = node.clone(); let quote_future = async move { + let start = Instant::now(); let result = send_and_await_chunk_response( &node_clone, &peer_id_clone, @@ -147,6 +149,11 @@ impl Client { ) .await; + let success = result.is_ok(); + let rtt_ms = success.then(|| start.elapsed().as_millis() as u64); + record_peer_outcome(&node_clone, peer_id_clone, &addrs_clone, success, rtt_ms) + .await; + (peer_id_clone, addrs_clone, result) }; diff --git a/ant-core/src/data/mod.rs b/ant-core/src/data/mod.rs index 4fa312f..97a7daf 100644 --- a/ant-core/src/data/mod.rs +++ b/ant-core/src/data/mod.rs @@ -23,7 +23,7 @@ pub use client::batch::{finalize_batch_payment, PaidChunk, PaymentIntent, Prepar pub use client::data::DataUploadResult; pub use client::file::{ DownloadEvent, ExternalPaymentInfo, FileUploadResult, PreparedUpload, UploadCostEstimate, - UploadEvent, + UploadEvent, Visibility, }; pub use client::merkle::{ finalize_merkle_batch, MerkleBatchPaymentResult, PaymentMode, PreparedMerkleBatch, diff --git a/ant-core/tests/e2e_bootstrap_cache.rs b/ant-core/tests/e2e_bootstrap_cache.rs new file mode 100644 index 0000000..80303ef --- /dev/null +++ b/ant-core/tests/e2e_bootstrap_cache.rs @@ -0,0 +1,143 @@ +//! E2E tests for BootstrapManager cache population from real peer interactions. +//! +//! Proves that client-side uploads and downloads feed the BootstrapManager +//! cache via `add_discovered_peer` + `update_peer_metrics`, so that subsequent +//! cold-starts can load quality-scored peers beyond the bundled bootstrap set. +//! +//! ## Why the assertion is "cache grew", not "cache >= 10" +//! +//! saorsa-core gates `add_peer` through two independent Sybil mechanisms: +//! +//! 1. `BootstrapIpLimiter::can_accept` — the IP-diversity limiter. When the +//! node is built with `allow_loopback = true` (as `MiniTestnet` does), +//! this returns early for loopback IPs, so it is NOT the bottleneck here. +//! 2. `JoinRateLimiter::check_join_allowed` — the temporal rate limiter. +//! Defaults cap inserts at 3 per /24 subnet per hour and are NOT exempt +//! for loopback (`saorsa-core/src/rate_limit.rs:254` has no `is_loopback` +//! branch). All testnet nodes bind to `127.0.0.1`, so all ~11 available +//! peers fall in the single `127.0.0.0/24` bucket — the first 3 land in +//! the cache, the rest are rejected with `Subnet24LimitExceeded`. +//! +//! In production, peers span many /24s (typically one per ASN), so the /24 +//! rate limit is never the binding constraint and crossing +//! `min_peers_to_save = 10` is straightforward. +//! +//! Asserting `after > before` is sufficient proof that the client library +//! correctly wires `add_discovered_peer` and `update_peer_metrics` into the +//! upload (and, transitively, download) paths. The threshold-crossing + +//! persistence behavior is an upstream contract covered by saorsa-transport's +//! own tests. + +#![allow(clippy::unwrap_used, clippy::expect_used)] + +mod support; + +use ant_core::data::{Client, ClientConfig}; +use bytes::Bytes; +use serial_test::serial; +use std::sync::Arc; +use support::MiniTestnet; + +const BOOTSTRAP_CACHE_TEST_NODES: usize = 12; + +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn test_bootstrap_cache_grows_after_client_activity() { + let testnet = MiniTestnet::start(BOOTSTRAP_CACHE_TEST_NODES).await; + let node = testnet.node(3).expect("Node 3 should exist"); + + let client = Client::from_node(Arc::clone(&node), ClientConfig::default()) + .with_wallet(testnet.wallet().clone()); + + let before = node.cached_peer_count().await; + + let content = Bytes::from("bootstrap-cache e2e payload"); + let address = client + .chunk_put(content.clone()) + .await + .expect("chunk_put should succeed with payment"); + + // The GET exercises the download-side hook (chunk_get_from_peer), which + // would silently break if record_peer_outcome's signature drifted from + // what chunk.rs expects. The assertion here is just that the round-trip + // works — cache growth from the GET itself is capped by the /24 rate + // limiter which saturated during the PUT. + let retrieved = client + .chunk_get(&address) + .await + .expect("chunk_get should succeed") + .expect("chunk should be retrievable"); + assert_eq!(retrieved.content.as_ref(), content.as_ref()); + + let after = node.cached_peer_count().await; + assert!( + after > before, + "cache should grow after peer interactions: before={before} after={after}" + ); + + drop(client); + testnet.teardown().await; +} + +/// Cold-start-from-disk round-trip. +/// +/// ## What this proves +/// +/// - A populated `BootstrapManager` cache with ≥ `min_peers_to_save` peers +/// is persisted to disk on `save()`. +/// - A *fresh* `BootstrapManager` constructed against the same `cache_dir` +/// reloads the persisted peers on startup. +/// +/// Together with `test_bootstrap_cache_grows_after_client_activity` above +/// (which exercises the add-during-activity hook), this closes the loop on +/// the V2-202 value prop: cold-start clients reload real peers from disk. +/// +/// ## Why `add_peer_trusted` and not `add_discovered_peer` +/// +/// `add_discovered_peer` goes through `BootstrapManager::add_peer`, which +/// runs both the IP-diversity limiter and the temporal `JoinRateLimiter`. +/// The latter caps inserts at 3 per /24 subnet per hour and has no +/// loopback exemption. A real test that populates 15 peers through that +/// path would need peers on distinct /24s — not practical on a single-host +/// testnet. `add_peer_trusted` skips both limiters and talks to the same +/// underlying `BootstrapCache::add_seed` that our hooks ultimately feed, +/// so the persistence path exercised is identical to production's. +#[tokio::test] +async fn test_bootstrap_cache_roundtrip_through_disk() { + use saorsa_core::{BootstrapConfig, BootstrapManager}; + use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + + let cache_dir = tempfile::TempDir::new().expect("create temp cache dir"); + let config = BootstrapConfig { + cache_dir: cache_dir.path().to_path_buf(), + ..BootstrapConfig::default() + }; + + // Populate with peers on distinct /24s (cosmetic — add_peer_trusted + // skips rate limits — but keeps the data realistic if saorsa-transport + // ever tightens its invariants). + let peer_count = 15; + { + let mgr = BootstrapManager::with_config(config.clone()) + .await + .expect("construct populating BootstrapManager"); + for i in 0..peer_count { + let addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(203, 0, 113, i as u8 + 1)), 9000); + mgr.add_peer_trusted(&addr, vec![addr]).await; + } + assert_eq!(mgr.peer_count().await, peer_count, "in-memory populate"); + mgr.save() + .await + .expect("save should succeed above threshold"); + } + + // Fresh manager, same cache_dir: peers should be reloaded. + let reloaded = BootstrapManager::with_config(config) + .await + .expect("construct reloading BootstrapManager"); + let reloaded_count = reloaded.peer_count().await; + assert_eq!( + reloaded_count, peer_count, + "all {peer_count} peers should reload from disk, got {reloaded_count}" + ); +} diff --git a/ant-core/tests/e2e_file.rs b/ant-core/tests/e2e_file.rs index c7b1e03..fb06897 100644 --- a/ant-core/tests/e2e_file.rs +++ b/ant-core/tests/e2e_file.rs @@ -4,8 +4,10 @@ mod support; -use ant_core::data::Client; +use ant_core::data::{compute_address, Client, ExternalPaymentInfo, Visibility}; +use evmlib::common::{QuoteHash, TxHash}; use serial_test::serial; +use std::collections::HashMap; use std::io::Write; use std::path::PathBuf; use std::sync::Arc; @@ -143,3 +145,158 @@ async fn test_file_download_bytes_written() { drop(client); testnet.teardown().await; } + +/// External-signer prepare must bundle the serialized DataMap as one extra +/// paid chunk when `Visibility::Public` is requested, and must record the +/// resulting chunk address on the `PreparedUpload`. Private prepare must +/// leave that address unset. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn test_file_prepare_upload_visibility() { + let (client, testnet) = setup().await; + + let data = vec![0x37u8; 4096]; + let mut input_file = NamedTempFile::new().expect("create temp file"); + input_file.write_all(&data).expect("write temp file"); + input_file.flush().expect("flush temp file"); + + let private = client + .file_prepare_upload_with_visibility(input_file.path(), Visibility::Private) + .await + .expect("private prepare should succeed"); + + assert!( + private.data_map_address.is_none(), + "private uploads must not publish a DataMap address" + ); + + let public = client + .file_prepare_upload_with_visibility(input_file.path(), Visibility::Public) + .await + .expect("public prepare should succeed"); + + let public_addr = public + .data_map_address + .expect("public prepare must record the DataMap chunk address"); + + // The recorded address must match a fresh hash of the serialized DataMap, + // proving the address refers to exactly the chunk that was added to the + // payment batch (and that `data_map_fetch` on this address will later + // yield the same DataMap we're holding). + let expected_bytes = rmp_serde::to_vec(&public.data_map).expect("serialize DataMap"); + let expected_addr = compute_address(&expected_bytes); + assert_eq!( + public_addr, expected_addr, + "data_map_address must equal compute_address(rmp_serde::to_vec(&data_map))" + ); + + // A small file produces a wave-batch payment (well under the merkle + // threshold), and the datamap chunk must appear in that batch. + match (&private.payment_info, &public.payment_info) { + ( + ExternalPaymentInfo::WaveBatch { + prepared_chunks: priv_chunks, + .. + }, + ExternalPaymentInfo::WaveBatch { + prepared_chunks: pub_chunks, + .. + }, + ) => { + assert_eq!( + pub_chunks.len(), + priv_chunks.len() + 1, + "public prepare must add exactly one chunk (the serialized DataMap) to the batch" + ); + assert!( + pub_chunks.iter().any(|c| c.address == public_addr), + "the extra chunk must be the DataMap chunk at the recorded address" + ); + } + other => panic!("expected wave-batch for a 4KB file, got {other:?}"), + } + + drop(client); + testnet.teardown().await; +} + +/// Full public-upload round-trip (wave-batch path). +/// +/// Simulates the external-signer flow end-to-end: prepare → sign payments +/// via the testnet wallet → finalize → `data_map_fetch` using only the +/// returned address → `file_download` → assert recovered bytes equal the +/// original. Proves the data_map_address actually refers to a retrievable +/// DataMap on the network, not just a hash recorded in memory. +#[tokio::test(flavor = "multi_thread")] +#[serial] +async fn test_public_upload_round_trip_wave_batch() { + let (client, testnet) = setup().await; + + let original = vec![0x5au8; 4096]; + let mut input_file = NamedTempFile::new().expect("create temp file"); + input_file.write_all(&original).expect("write temp file"); + input_file.flush().expect("flush temp file"); + + // Phase 1: prepare as public. + let prepared = client + .file_prepare_upload_with_visibility(input_file.path(), Visibility::Public) + .await + .expect("public prepare should succeed"); + let data_map_address = prepared + .data_map_address + .expect("public prepare must record the DataMap address"); + + // Phase 2: simulate an external signer by paying for the quotes with the + // testnet wallet and collecting the resulting (quote_hash, tx_hash) map. + let payments = match &prepared.payment_info { + ExternalPaymentInfo::WaveBatch { payment_intent, .. } => payment_intent.payments.clone(), + other => panic!("expected wave-batch payment for a 4KB file, got {other:?}"), + }; + let (tx_hash_map, _gas) = testnet + .wallet() + .pay_for_quotes(payments) + .await + .expect("testnet wallet should pay for quotes"); + let tx_hash_map: HashMap = tx_hash_map.into_iter().collect(); + + // Phase 3: finalize. The data map chunk is stored alongside the data + // chunks in this single call — no second network trip needed. + let result = client + .finalize_upload(prepared, &tx_hash_map) + .await + .expect("finalize_upload should succeed"); + assert_eq!( + result.data_map_address, + Some(data_map_address), + "FileUploadResult must carry the DataMap address forward from PreparedUpload" + ); + + // Phase 4: a fresh retriever can fetch the data map using only the + // shared address — they did not participate in the upload. + let fetched_data_map = client + .data_map_fetch(&data_map_address) + .await + .expect("data_map_fetch must retrieve the stored DataMap"); + + // Phase 5: download + verify content. + let output_dir = TempDir::new().expect("create output temp dir"); + let output_path = output_dir.path().join("round_trip_out.bin"); + let bytes_written = client + .file_download(&fetched_data_map, &output_path) + .await + .expect("file_download should succeed"); + assert_eq!( + bytes_written, + original.len() as u64, + "bytes_written should equal original size" + ); + + let downloaded = std::fs::read(&output_path).expect("read downloaded file"); + assert_eq!( + downloaded, original, + "downloaded bytes must equal the original file" + ); + + drop(client); + testnet.teardown().await; +}