diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41d7a8c..f28460e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,5 +25,9 @@ jobs: run: cargo fmt --all -- --check - name: Clippy run: cargo clippy --all-targets -- -D warnings + - name: Clippy (all features) + run: cargo clippy --all-targets --all-features -- -D warnings - name: Test run: cargo test --workspace --verbose + - name: Test (all features) + run: cargo test --workspace --all-features --verbose diff --git a/Cargo.lock b/Cargo.lock index 4977c5f..1aa0cb4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -750,7 +750,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots", + "webpki-roots 1.0.8", ] [[package]] @@ -1114,12 +1114,15 @@ dependencies = [ "directories", "ltk_hashdb", "pollster", + "reqwest", "serde", "serde_json", "sha2", "tempfile", "thiserror 2.0.18", "time", + "tokio", + "ureq", ] [[package]] @@ -1132,7 +1135,6 @@ dependencies = [ "ltk_hashdb", "ltk_mimir_cache", "ltk_mimir_gen", - "reqwest", "sha2", "tempfile", "thiserror 2.0.18", @@ -1772,7 +1774,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots", + "webpki-roots 1.0.8", ] [[package]] @@ -1849,6 +1851,7 @@ version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -2242,9 +2245,21 @@ dependencies = [ "mio", "pin-project-lite", "socket2", + "tokio-macros", "windows-sys 0.61.2", ] +[[package]] +name = "tokio-macros" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -2397,6 +2412,22 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + [[package]] name = "url" version = "2.5.8" @@ -2545,6 +2576,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.8", +] + [[package]] name = "webpki-roots" version = "1.0.8" diff --git a/Cargo.toml b/Cargo.toml index 7d0fc07..ff7e1ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ directories = "5" sha2 = "0.10" time = { version = "0.3", features = ["formatting"] } reqwest = { version = "0.12", default-features = false, features = ["blocking", "rustls-tls"] } +ureq = "2" # CLI anyhow = "1" @@ -51,6 +52,7 @@ ltk_meta = "0.6" # dev tempfile = "3" pollster = "0.4" +tokio = { version = "1", features = ["rt", "macros", "net", "time"] } [workspace.dependencies.plotters] version = "0.3" diff --git a/crates/ltk_mimir_cache/Cargo.toml b/crates/ltk_mimir_cache/Cargo.toml index b42e4a8..9b3017e 100644 --- a/crates/ltk_mimir_cache/Cargo.toml +++ b/crates/ltk_mimir_cache/Cargo.toml @@ -16,10 +16,18 @@ serde_json.workspace = true sha2.workspace = true thiserror.workspace = true time.workspace = true +ureq = { workspace = true, optional = true } +reqwest = { workspace = true, optional = true } + +[features] +default = [] +ureq = ["dep:ureq"] +reqwest = ["dep:reqwest"] [dev-dependencies] pollster.workspace = true tempfile.workspace = true +tokio.workspace = true [lints] workspace = true diff --git a/crates/ltk_mimir_cache/src/fetch.rs b/crates/ltk_mimir_cache/src/fetch.rs new file mode 100644 index 0000000..3c2cf33 --- /dev/null +++ b/crates/ltk_mimir_cache/src/fetch.rs @@ -0,0 +1,299 @@ +//! Bundled, feature-gated [`Fetch`](crate::Fetch)/[`AsyncFetch`](crate::AsyncFetch) +//! implementations for the common case: pulling `.lhdb` release assets over +//! HTTP. +//! +//! The core crate ships no HTTP client - [`HashStore::update`](crate::HashStore::update) +//! takes a caller-supplied fetcher. These optional types remove the +//! release-asset glue that every consumer would otherwise rewrite: a +//! [`ReleaseSource`] (a GitHub latest-release layout or an explicit mirror base +//! URL) plus a concrete [`HttpFetchError`], so no consumer writes the error +//! plumbing the [`Fetch::Error`](crate::Fetch::Error) `Sized` bound would +//! otherwise force. +//! +//! - `ureq` feature → [`UreqFetch`], a blocking [`Fetch`](crate::Fetch). +//! - `reqwest` feature → [`ReqwestFetch`], an async +//! [`AsyncFetch`](crate::AsyncFetch). +//! +//! Both fetchers are silent by design; per-file progress output stays with the +//! caller (wrap the fetcher in a closure, which is itself a `Fetch`). + +/// Default `User-Agent` for the bundled fetchers, matching the mimir CLI. +const USER_AGENT: &str = concat!("mimir/", env!("CARGO_PKG_VERSION")); + +/// Where release assets live: a GitHub repo's latest release, or an explicit +/// base URL (a mirror). Shared by the blocking and async fetchers. +#[derive(Debug, Clone)] +pub struct ReleaseSource { + base: String, +} + +impl ReleaseSource { + /// The GitHub latest-release layout: + /// `https://github.com/{owner_repo}/releases/latest/download`. + pub fn github(owner_repo: &str) -> Self { + Self { + base: format!("https://github.com/{owner_repo}/releases/latest/download"), + } + } + + /// An explicit base URL serving `manifest.json` + the `.lhdb` assets (a + /// mirror). A trailing slash is trimmed so asset URLs join cleanly. + pub fn base_url(url: impl Into) -> Self { + let url = url.into(); + Self { + base: url.trim_end_matches('/').to_owned(), + } + } + + /// The full URL for one asset filename under this source. + fn asset_url(&self, filename: &str) -> String { + format!("{}/{filename}", self.base) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum HttpFetchError { + /// The request never produced an HTTP response (DNS, connect, TLS, or a + /// mid-body read failure). + #[error("fetching {url}")] + Transport { + url: String, + + #[source] + source: Box, + }, + + /// The server answered, but with a non-success status. + #[error("unexpected HTTP {status} for {url}")] + Status { status: u16, url: String }, +} + +#[cfg(feature = "ureq")] +mod ureq_impl { + use std::io::Read; + + use super::{HttpFetchError, ReleaseSource, USER_AGENT}; + use crate::Fetch; + + /// A blocking [`Fetch`] over `ureq`, pulling assets from a [`ReleaseSource`]. + pub struct UreqFetch { + source: ReleaseSource, + + agent: ureq::Agent, + } + + impl UreqFetch { + /// A fetcher for `source`, using a fresh agent with the mimir + /// `User-Agent`. + pub fn new(source: ReleaseSource) -> Self { + let agent = ureq::AgentBuilder::new().user_agent(USER_AGENT).build(); + + Self { source, agent } + } + } + + impl Fetch for UreqFetch { + type Error = HttpFetchError; + + fn fetch(&self, filename: &str) -> Result, HttpFetchError> { + let url = self.source.asset_url(filename); + + // `call()` fails on non-2xx, so a 404 arrives as `Error::Status`. + let response = match self.agent.get(&url).call() { + Ok(response) => response, + Err(ureq::Error::Status(status, _)) => { + return Err(HttpFetchError::Status { status, url }) + } + Err(err) => { + return Err(HttpFetchError::Transport { + url, + source: Box::new(err), + }) + } + }; + + let mut bytes = Vec::new(); + response + .into_reader() + .read_to_end(&mut bytes) + .map_err(|err| HttpFetchError::Transport { + url, + source: Box::new(err), + })?; + + Ok(bytes) + } + } +} + +#[cfg(feature = "ureq")] +pub use ureq_impl::UreqFetch; + +#[cfg(feature = "reqwest")] +mod reqwest_impl { + use std::future::Future; + + use super::{HttpFetchError, ReleaseSource, USER_AGENT}; + use crate::AsyncFetch; + + pub struct ReqwestFetch { + source: ReleaseSource, + + client: reqwest::Client, + } + + impl ReqwestFetch { + /// A fetcher for `source`, using a client with the mimir `User-Agent`. + pub fn new(source: ReleaseSource) -> Self { + let client = reqwest::Client::builder() + .user_agent(USER_AGENT) + .build() + .unwrap_or_default(); + + Self { source, client } + } + } + + impl AsyncFetch for ReqwestFetch { + type Error = HttpFetchError; + + fn fetch( + &self, + filename: &str, + ) -> impl Future, HttpFetchError>> + Send { + // Own everything the future needs so it is `'static` and `Send`. + let url = self.source.asset_url(filename); + let client = self.client.clone(); + + async move { + let response = + client + .get(&url) + .send() + .await + .map_err(|err| HttpFetchError::Transport { + url: url.clone(), + source: Box::new(err), + })?; + + let status = response.status(); + if !status.is_success() { + return Err(HttpFetchError::Status { + status: status.as_u16(), + url, + }); + } + + let bytes = response + .bytes() + .await + .map_err(|err| HttpFetchError::Transport { + url: url.clone(), + source: Box::new(err), + })?; + + Ok(bytes.to_vec()) + } + } + } +} + +#[cfg(feature = "reqwest")] +pub use reqwest_impl::ReqwestFetch; + +#[cfg(all(test, any(feature = "ureq", feature = "reqwest")))] +mod tests { + use std::io::{Read, Write}; + use std::net::{TcpListener, TcpStream}; + use std::thread::{self, JoinHandle}; + + use super::*; + #[cfg(feature = "reqwest")] + use crate::AsyncFetch; + #[cfg(feature = "ureq")] + use crate::Fetch; + + /// A throwaway HTTP/1.1 server: serves `payload` at `ok_path`, 404s + /// everything else, and handles exactly `connections` requests (one per + /// connection) before the thread exits. + fn serve(payload: Vec, ok_path: String, connections: usize) -> (String, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}", listener.local_addr().unwrap()); + + let handle = thread::spawn(move || { + for _ in 0..connections { + let (mut stream, _) = listener.accept().unwrap(); + handle_one(&mut stream, &payload, &ok_path); + } + }); + + (base, handle) + } + + /// Route a single request off its request line and write one response. + fn handle_one(stream: &mut TcpStream, payload: &[u8], ok_path: &str) { + let mut buf = [0u8; 1024]; + let n = stream.read(&mut buf).unwrap(); + let request = String::from_utf8_lossy(&buf[..n]); + let path = request.split_whitespace().nth(1).unwrap_or(""); + + if path == ok_path { + let header = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", + payload.len() + ); + stream.write_all(header.as_bytes()).unwrap(); + stream.write_all(payload).unwrap(); + } else { + stream + .write_all(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n") + .unwrap(); + } + + stream.flush().unwrap(); + } + + #[cfg(feature = "ureq")] + #[test] + fn ureq_fetch_returns_bytes_and_maps_404() { + let payload = b"lhdb-bytes".to_vec(); + let (base, server) = serve(payload.clone(), "/game-1.lhdb".to_string(), 2); + + // Two fetchers → two connections, matching the server's count. + let ok = UreqFetch::new(ReleaseSource::base_url(&base)); + assert_eq!(ok.fetch("game-1.lhdb").unwrap(), payload); + + let missing = UreqFetch::new(ReleaseSource::base_url(&base)); + match missing.fetch("nope.lhdb") { + Err(HttpFetchError::Status { status: 404, .. }) => {} + other => panic!("expected a 404 status error, got {other:?}"), + } + + server.join().unwrap(); + } + + #[cfg(feature = "reqwest")] + #[test] + fn reqwest_fetch_returns_bytes_and_maps_404() { + let payload = b"lhdb-bytes".to_vec(); + let (base, server) = serve(payload.clone(), "/game-1.lhdb".to_string(), 2); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + let ok = ReqwestFetch::new(ReleaseSource::base_url(&base)); + assert_eq!(ok.fetch("game-1.lhdb").await.unwrap(), payload); + + let missing = ReqwestFetch::new(ReleaseSource::base_url(&base)); + match missing.fetch("nope.lhdb").await { + Err(HttpFetchError::Status { status: 404, .. }) => {} + other => panic!("expected a 404 status error, got {other:?}"), + } + }); + + server.join().unwrap(); + } +} diff --git a/crates/ltk_mimir_cache/src/lib.rs b/crates/ltk_mimir_cache/src/lib.rs index 0b1c070..f82c78c 100644 --- a/crates/ltk_mimir_cache/src/lib.rs +++ b/crates/ltk_mimir_cache/src/lib.rs @@ -10,6 +10,8 @@ mod dir; mod error; +#[cfg(any(feature = "ureq", feature = "reqwest"))] +mod fetch; mod fsutil; mod lock; mod manifest; @@ -17,6 +19,12 @@ mod store; mod update; pub use error::{CommitError, GcError, ManifestError, NoCacheDirError, OpenError, UpdateError}; +#[cfg(feature = "reqwest")] +pub use fetch::ReqwestFetch; +#[cfg(feature = "ureq")] +pub use fetch::UreqFetch; +#[cfg(any(feature = "ureq", feature = "reqwest"))] +pub use fetch::{HttpFetchError, ReleaseSource}; pub use lock::UpdateLock; pub use manifest::{Manifest, Source, TableEntry, SCHEMA_VERSION}; pub use store::{CommitItem, GcReport, HashStore}; diff --git a/crates/ltk_mimir_cli/Cargo.toml b/crates/ltk_mimir_cli/Cargo.toml index 432b13c..fb87a84 100644 --- a/crates/ltk_mimir_cli/Cargo.toml +++ b/crates/ltk_mimir_cli/Cargo.toml @@ -15,11 +15,10 @@ path = "src/main.rs" [dependencies] ltk_hashdb.workspace = true -ltk_mimir_cache.workspace = true +ltk_mimir_cache = { workspace = true, features = ["ureq"] } ltk_mimir_gen.workspace = true anyhow.workspace = true clap.workspace = true -reqwest.workspace = true zstd.workspace = true indicatif.workspace = true sha2.workspace = true diff --git a/crates/ltk_mimir_cli/src/update.rs b/crates/ltk_mimir_cli/src/update.rs index c20825d..19f63a2 100644 --- a/crates/ltk_mimir_cli/src/update.rs +++ b/crates/ltk_mimir_cli/src/update.rs @@ -4,13 +4,13 @@ //! Tables are built by CI from the canonical txt lists and shipped as GitHub //! release assets, so updating a machine is a download, not a rebuild. The //! whole compare → download → verify → install loop is -//! [`HashStore::update`] in `ltk_mimir_cache`; this module just supplies the -//! reqwest-backed [`Fetch`] and prints what happened. +//! [`HashStore::update`] in `ltk_mimir_cache`; this module just points its +//! bundled [`UreqFetch`] at the right release and prints what happened. use std::path::PathBuf; use anyhow::Result; -use ltk_mimir_cache::{Fetch, HashStore, UpdateOptions, UpdateOutcome}; +use ltk_mimir_cache::{Fetch, HashStore, ReleaseSource, UpdateOptions, UpdateOutcome, UreqFetch}; pub struct Options { /// GitHub `owner/repo` whose latest release ships the tables. @@ -28,22 +28,25 @@ pub struct Options { } pub fn run(opts: &Options) -> Result<()> { - let base = match &opts.url { - Some(url) => url.trim_end_matches('/').to_owned(), - None => format!("https://github.com/{}/releases/latest/download", opts.repo), + let source = match &opts.url { + Some(url) => ReleaseSource::base_url(url.clone()), + None => ReleaseSource::github(&opts.repo), }; let store = match &opts.dir { Some(dir) => HashStore::at(dir), None => HashStore::discover()?, }; - let client = reqwest::blocking::Client::builder() - .user_agent(concat!("mimir/", env!("CARGO_PKG_VERSION"))) - .build()?; - let outcome = store.update( - &HttpFetch { base, client }, - UpdateOptions { force: opts.force }, - )?; + let fetch = UreqFetch::new(source); + let fetch = |filename: &str| { + if filename.ends_with(".lhdb") { + println!("downloading {filename}"); + } + + fetch.fetch(filename) + }; + + let outcome = store.update(&fetch, UpdateOptions { force: opts.force })?; let report = match outcome { UpdateOutcome::Locked => { println!( @@ -79,26 +82,3 @@ pub fn run(opts: &Options) -> Result<()> { Ok(()) } - -/// Release assets over HTTP, with a progress line per table download. -struct HttpFetch { - base: String, - client: reqwest::blocking::Client, -} - -impl Fetch for HttpFetch { - // reqwest's error already names the URL and HTTP status, so it flows - // through `UpdateError::Fetch` untouched instead of being boxed. - type Error = reqwest::Error; - - fn fetch(&self, filename: &str) -> std::result::Result, reqwest::Error> { - if filename.ends_with(".lhdb") { - println!("downloading {filename}"); - } - - let url = format!("{}/{filename}", self.base); - let response = self.client.get(&url).send()?.error_for_status()?; - - Ok(response.bytes()?.to_vec()) - } -}