diff --git a/Cargo.lock b/Cargo.lock index b096bcbc..9d094896 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -734,21 +734,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1830,22 +1815,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.20" @@ -2361,23 +2330,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe 0.1.6", - "openssl-sys", - "schannel", - "security-framework 2.11.1", - "security-framework-sys", - "tempfile", -] - [[package]] name = "nix" version = "0.30.1" @@ -2445,56 +2397,12 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" -[[package]] -name = "openssl" -version = "0.10.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" -dependencies = [ - "bitflags", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - [[package]] name = "openssl-probe" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" -[[package]] -name = "openssl-sys" -version = "0.9.111" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "option-ext" version = "0.2.0" @@ -2959,12 +2867,10 @@ dependencies = [ "http-body-util", "hyper", "hyper-rustls", - "hyper-tls", "hyper-util", "js-sys", "log", "mime", - "native-tls", "percent-encoding", "pin-project-lite", "quinn", @@ -2973,7 +2879,6 @@ dependencies = [ "rustls-platform-verifier", "sync_wrapper", "tokio", - "tokio-native-tls", "tokio-rustls", "tokio-util", "tower", @@ -3066,10 +2971,10 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ - "openssl-probe 0.2.1", + "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.5.1", + "security-framework", ] [[package]] @@ -3097,7 +3002,7 @@ dependencies = [ "rustls-native-certs", "rustls-platform-verifier-android", "rustls-webpki", - "security-framework 3.5.1", + "security-framework", "security-framework-sys", "webpki-root-certs", "windows-sys 0.52.0", @@ -3157,19 +3062,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags", - "core-foundation 0.9.4", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - [[package]] name = "security-framework" version = "3.5.1" @@ -3701,16 +3593,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.4" @@ -3939,12 +3821,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version-ranges" version = "0.1.2" diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml index f6e3e90f..64c355af 100644 --- a/bindings/java/Cargo.toml +++ b/bindings/java/Cargo.toml @@ -31,9 +31,4 @@ url = { version = "2.5.8", default-features = false } tokio = { version = "1.50.0", default-features = false, features = ["rt"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = { version = "1.0.149", default-features = false, features = ["preserve_order"] } - -# Use native TLS only on Windows and Apple OSs -[target.'cfg(any(target_os = "windows", target_vendor = "apple"))'.dependencies] -reqwest = { version = "0.13.2", default-features = false, features = ["native-tls", "http2", "system-proxy"] } -[target.'cfg(not(any(target_os = "windows", target_vendor = "apple")))'.dependencies] reqwest = { version = "0.13.2", features = ["rustls"] } diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs index 84f333df..5207fe87 100644 --- a/bindings/java/src/lib.rs +++ b/bindings/java/src/lib.rs @@ -20,7 +20,7 @@ use sysand_core::{ local_src::{LocalSrcError, LocalSrcProject}, utils::wrapfs, }, - resolve::standard::standard_resolver, + resolve::{net_utils::create_reqwest_client, standard::standard_resolver}, workspace::Workspace, }; @@ -190,7 +190,13 @@ pub extern "system" fn Java_com_sensmetry_sysand_Sysand_info<'local>( let Some(uri) = env.get_str(&uri, "uri") else { return JObjectArray::default(); }; - let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(); + let client = match create_reqwest_client() { + Ok(c) => c, + Err(e) => { + env.throw_exception(ExceptionKind::SysandException, e.to_string()); + return JObjectArray::default(); + } + }; let runtime = { let r = match tokio::runtime::Builder::new_current_thread().build() { diff --git a/bindings/py/Cargo.toml b/bindings/py/Cargo.toml index 6ec4b33e..a4995a8c 100644 --- a/bindings/py/Cargo.toml +++ b/bindings/py/Cargo.toml @@ -29,13 +29,8 @@ pyo3-log = "0.13.3" semver = { version = "1.0.27", default-features = false } typed-path = { version = "0.12.3", default-features = false, features= ["std"] } url = { version = "2.5.8", default-features = false } -reqwest-middleware = "0.5.1" tokio = { version = "1.50.0", default-features = false, features = ["rt"] } - -# Use native TLS only on Windows and Apple OSs -[target.'cfg(any(target_os = "windows", target_vendor = "apple"))'.dependencies] -reqwest = { version = "0.13.2", default-features = false, features = ["native-tls", "http2", "system-proxy"] } -[target.'cfg(not(any(target_os = "windows", target_vendor = "apple")))'.dependencies] +reqwest-middleware = "0.5.1" reqwest = { version = "0.13.2", features = ["rustls"] } [dev-dependencies] diff --git a/bindings/py/src/lib.rs b/bindings/py/src/lib.rs index 372f0cf8..5ac5b35f 100644 --- a/bindings/py/src/lib.rs +++ b/bindings/py/src/lib.rs @@ -36,7 +36,7 @@ use sysand_core::{ utils::wrapfs, }, remove::do_remove, - resolve::standard::standard_resolver, + resolve::{net_utils::create_reqwest_client, standard::standard_resolver}, sources::{do_sources_local_src_project_no_deps, find_project_dependencies}, stdlib::known_std_libs, symbols::Language, @@ -141,7 +141,7 @@ fn do_info_py( py.detach(|| { let mut results = vec![]; - let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(); + let client = create_reqwest_client().map_err(|e| PyRuntimeError::new_err(e.to_string()))?; let runtime = Arc::new( tokio::runtime::Builder::new_current_thread() diff --git a/core/Cargo.toml b/core/Cargo.toml index 129f1e4c..51e80f34 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -67,11 +67,6 @@ tokio = { version = "1.50.0", default-features = false, features = ["rt", "io-ut bytes = { version = "1.11.1", default-features = false } toml_edit = { version = "0.25.4", features = ["serde"] } globset = { version = "0.4.18", default-features = false } - -# Use native TLS only on Windows and Apple OSs -[target.'cfg(any(target_os = "windows", target_vendor = "apple"))'.dependencies] -reqwest = { version = "0.13.2", optional = true, default-features = false, features = ["native-tls", "http2", "system-proxy", "stream"] } -[target.'cfg(not(any(target_os = "windows", target_vendor = "apple")))'.dependencies] reqwest = { version = "0.13.2", optional = true, features = ["rustls", "stream"] } [dev-dependencies] diff --git a/core/src/auth.rs b/core/src/auth.rs index 076fbf44..15405323 100644 --- a/core/src/auth.rs +++ b/core/src/auth.rs @@ -66,7 +66,7 @@ impl HTTPAuthentication for ForceHTTPBasicAuth { F: Fn(&ClientWithMiddleware) -> RequestBuilder + 'static, { request - .basic_auth(self.username.clone(), Some(self.password.clone())) + .basic_auth(&self.username, Some(&self.password)) .send() .await } @@ -99,7 +99,7 @@ pub struct ForceBearerAuth(HeaderAuth); impl ForceBearerAuth { pub fn new>(token: S) -> ForceBearerAuth { ForceBearerAuth(HeaderAuth { - header: "Authorization".to_string(), + header: "Authorization".to_owned(), value: format!("Bearer {}", token.as_ref()), }) } @@ -234,7 +234,7 @@ impl GlobMap { if outcome.is_empty() { GlobMapResult::NotFound } else if outcome.len() == 1 { - GlobMapResult::Found(self.keys[0].clone(), &self.values[outcome[0]]) + GlobMapResult::Found(key.to_owned(), &self.values[outcome[0]]) } else { // Need to do some magic to keep multiple (disjoint) references into a mutable array let mut result = Vec::with_capacity(outcome.len()); @@ -255,7 +255,7 @@ impl GlobMap { if outcome.is_empty() { GlobMapResultMut::NotFound } else if outcome.len() == 1 { - GlobMapResultMut::Found(self.keys[0].clone(), &mut self.values[outcome[0]]) + GlobMapResultMut::Found(key.to_owned(), &mut self.values[outcome[0]]) } else { // Need to do some magic to keep multiple (disjoint) references into a mutable array let mut result = Vec::with_capacity(outcome.len()); @@ -478,13 +478,17 @@ mod tests { panic!("Expected ambiguous result."); } - if let GlobMapResultMut::Found(_, val) = globmap.lookup_mut("axx.com/xxx/xxx/xxx") { + let key = "axx.com/xxx/xxx/xxx"; + if let GlobMapResultMut::Found(k, val) = globmap.lookup_mut(key) { + assert_eq!(k, key); assert_eq!(*val, 2); } else { panic!("Expected unambiguous result."); } - if let GlobMapResultMut::Found(_, val) = globmap.lookup_mut("b.com/xxx") { + let key = "b.com/xxx"; + if let GlobMapResultMut::Found(k, val) = globmap.lookup_mut(key) { + assert_eq!(k, key); assert_eq!(*val, 3); } else { panic!("Expected unambiguous result."); diff --git a/core/src/env/reqwest_http.rs b/core/src/env/reqwest_http.rs index ea2ac11c..b027a6dd 100644 --- a/core/src/env/reqwest_http.rs +++ b/core/src/env/reqwest_http.rs @@ -10,7 +10,6 @@ use std::{ }; use futures::{Stream, TryStreamExt}; -use reqwest_middleware::ClientWithMiddleware; use sha2::Sha256; use thiserror::Error; @@ -24,7 +23,10 @@ use crate::{ project::{ reqwest_kpar_download::ReqwestKparDownloadedProject, reqwest_src::ReqwestSrcProjectAsync, }, - resolve::reqwest_http::HTTPProjectAsync, + resolve::{ + net_utils::{json_head_request, kpar_head_request, text_get_request}, + reqwest_http::HTTPProjectAsync, + }, }; use futures::{AsyncBufReadExt as _, StreamExt as _}; @@ -131,15 +133,9 @@ impl HTTPEnvironmentAsync { let project_url = self.project_src_url(uri, version)?; let src_project_url = Self::url_join(&project_url, ".project.json")?; - let this_url = src_project_url.clone(); - let src_project_request = move |client: &ClientWithMiddleware| { - client - .head(this_url.clone()) - .header("ACCEPT", "application/json, text/plain") - }; let src_project_response = self .auth_policy - .with_authentication(&self.client, &src_project_request) + .with_authentication(&self.client, &json_head_request(src_project_url.clone())) .await?; if !src_project_response.status().is_success() { @@ -160,16 +156,11 @@ impl HTTPEnvironmentAsync { uri: S, version: T, ) -> Result>, HTTPEnvironmentError> { - let project_kpar_url = self.project_kpar_url(&uri, &version)?; - let this_url = project_kpar_url.clone(); - let kpar_project_request = move |client: &ClientWithMiddleware| { - client - .head(this_url.clone()) - .header("ACCEPT", "application/zip, application/octet-stream") - }; + let kpar_project_url = self.project_kpar_url(&uri, &version)?; + let kpar_project_response = self .auth_policy - .with_authentication(&self.client, &kpar_project_request) + .with_authentication(&self.client, &kpar_head_request(kpar_project_url.clone())) .await?; if !kpar_project_response.status().is_success() { @@ -178,7 +169,7 @@ impl HTTPEnvironmentAsync { Ok(Some(HTTPProjectAsync::HTTPKParProjectDownloaded( ReqwestKparDownloadedProject::new_guess_root( - &project_kpar_url, + &kpar_project_url, self.client.clone(), self.auth_policy.clone(), ) @@ -234,11 +225,9 @@ impl ReadEnvironmentAsync for HTTPEnvironmentAsync

; async fn uris_async(&self) -> Result { - let this_url = self.entries_url()?; - let response = self .auth_policy - .with_authentication(&self.client, &move |client| client.get(this_url.clone())) + .with_authentication(&self.client, &text_get_request(self.entries_url()?)) .await?; let inner = if response.status().is_success() { @@ -267,10 +256,12 @@ impl ReadEnvironmentAsync for HTTPEnvironmentAsync

Result { - let this_url = self.versions_url(uri.as_ref())?; let response = self .auth_policy - .with_authentication(&self.client, &move |client| client.get(this_url.clone())) + .with_authentication( + &self.client, + &text_get_request(self.versions_url(uri.as_ref())?), + ) .await?; let inner = if response.status().is_success() { @@ -329,13 +320,13 @@ mod test { use crate::{ auth::Unauthenticated, env::{ReadEnvironment, ReadEnvironmentAsync}, - resolve::reqwest_http::HTTPProjectAsync, + resolve::{net_utils::create_reqwest_client, reqwest_http::HTTPProjectAsync}, }; #[test] fn test_uri_examples() -> Result<(), Box> { let env = super::HTTPEnvironmentAsync { - client: reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(), + client: create_reqwest_client()?, base_url: url::Url::parse("https://www.example.com/a/b")?, prefer_src: true, auth_policy: Arc::new(Unauthenticated {}), @@ -370,7 +361,7 @@ mod test { let host = server.url(); let env = super::HTTPEnvironmentAsync { - client: reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(), + client: create_reqwest_client()?, base_url: url::Url::parse(&host)?, prefer_src: true, auth_policy: Arc::new(Unauthenticated {}), @@ -443,7 +434,7 @@ mod test { let host = server.url(); let env = super::HTTPEnvironmentAsync { - client: reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(), + client: create_reqwest_client()?, base_url: url::Url::parse(&host)?, prefer_src: true, auth_policy: Arc::new(Unauthenticated {}), @@ -483,7 +474,7 @@ mod test { let host = server.url(); let env = super::HTTPEnvironmentAsync { - client: reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(), + client: create_reqwest_client()?, base_url: url::Url::parse(&host)?, prefer_src: false, auth_policy: Arc::new(Unauthenticated {}), @@ -523,7 +514,7 @@ mod test { let host = server.url(); let env = super::HTTPEnvironmentAsync { - client: reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(), + client: create_reqwest_client()?, base_url: url::Url::parse(&host)?, prefer_src: false, auth_policy: Arc::new(Unauthenticated {}), @@ -575,7 +566,7 @@ mod test { let host = server.url(); let env = super::HTTPEnvironmentAsync { - client: reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(), + client: create_reqwest_client()?, base_url: url::Url::parse(&host)?, prefer_src: true, auth_policy: Arc::new(Unauthenticated {}), diff --git a/core/src/lib.rs b/core/src/lib.rs index c3a6f427..b3d63656 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: © 2025 Sysand contributors +// SPDX-FileCopyrightText: © 2026 Sysand contributors // SPDX-License-Identifier: MIT OR Apache-2.0 #![allow(refining_impl_trait)] diff --git a/core/src/project/reqwest_kpar_download.rs b/core/src/project/reqwest_kpar_download.rs index a4df5f62..40a0bfc8 100644 --- a/core/src/project/reqwest_kpar_download.rs +++ b/core/src/project/reqwest_kpar_download.rs @@ -9,7 +9,7 @@ use std::{ }; use futures::AsyncRead; -use reqwest_middleware::{ClientWithMiddleware, RequestBuilder}; +use reqwest::header::HeaderMap; use thiserror::Error; use crate::{ @@ -21,6 +21,7 @@ use crate::{ ProjectRead, ProjectReadAsync, local_kpar::{LocalKParError, LocalKParProject}, }, + resolve::net_utils::kpar_get_request, }; use super::utils::{FsIoError, wrapfs}; @@ -41,10 +42,19 @@ pub struct ReqwestKparDownloadedProject { pub auth_policy: Arc, } +// TODO: reduce size of errors here and elsewhere #[derive(Error, Debug)] pub enum ReqwestKparDownloadedError { - #[error("HTTP request to `{0}` returned status {1}")] - BadHttpStatus(reqwest::Url, reqwest::StatusCode), + #[error( + "HTTP request to `{url}` returned status {} with the following headers: {:#?}", + status, + headers + )] + BadHttpStatus { + url: Box, + status: reqwest::StatusCode, + headers: Box, + }, #[error("failed to parse URL `{0}`: {1}")] ParseUrl(Box, url::ParseError), // TODO: nicer formatting. Debug formatting is used here to include @@ -87,22 +97,17 @@ impl ReqwestKparDownloadedProject { let mut file = wrapfs::File::create(&self.inner.archive_path)?; - let this_url = self.url.clone(); let resp = self .auth_policy - .with_authentication( - &self.client, - &move |client: &ClientWithMiddleware| -> RequestBuilder { - client.get(this_url.clone()) - }, - ) + .with_authentication(&self.client, &kpar_get_request(self.url.clone())) .await?; if !resp.status().is_success() { - return Err(ReqwestKparDownloadedError::BadHttpStatus( - self.url.clone(), - resp.status(), - )); + return Err(ReqwestKparDownloadedError::BadHttpStatus { + url: self.url.as_str().into(), + status: resp.status(), + headers: resp.headers().to_owned().into(), + }); } let mut bytes_stream = resp.bytes_stream(); @@ -187,6 +192,7 @@ mod tests { use crate::{ auth::Unauthenticated, project::{ProjectRead, ProjectReadAsync}, + resolve::net_utils::create_reqwest_client, }; #[test] @@ -228,7 +234,7 @@ mod tests { let project = super::ReqwestKparDownloadedProject::new_guess_root( format!("{}test_basic_download_request.kpar", url,), - reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(), + create_reqwest_client()?, Arc::new(Unauthenticated {}), )? .to_tokio_sync(Arc::new( diff --git a/core/src/project/reqwest_src.rs b/core/src/project/reqwest_src.rs index 526a9e5c..e39bf693 100644 --- a/core/src/project/reqwest_src.rs +++ b/core/src/project/reqwest_src.rs @@ -1,15 +1,12 @@ // SPDX-FileCopyrightText: © 2025 Sysand contributors // SPDX-License-Identifier: MIT OR Apache-2.0 +//! This module implements accessing interchanged projects stored remotely over HTTP. + use std::{io, marker::Send, pin::Pin, sync::Arc}; use futures::{TryStreamExt, join}; -use reqwest_middleware::ClientWithMiddleware; use thiserror::Error; -/// This module implements accessing interchanged projects stored remotely over HTTP. -/// It is currently written using the blocking Reqwest client. Once sysand functionality -/// has stabilised it will be refactored to use the async interface and allow reqwest_middleware. -/// This will enable middleware (such as caching) as well as using reqwest also in WASM. use typed_path::Utf8UnixPath; use crate::{ @@ -18,6 +15,7 @@ use crate::{ lock::Source, model::{InterchangeProjectInfoRaw, InterchangeProjectMetadataRaw}, project::ProjectReadAsync, + resolve::net_utils::{json_get_request, json_head_request, text_get_request}, }; /// Project stored at a remote base-URL such as https://www.example.com/project/ @@ -116,10 +114,9 @@ impl ProjectReadAsync for ReqwestSrcProjectAsync Result, Self::Error> { - let this_url = self.info_url(); let info_resp = self .auth_policy - .with_authentication(&self.client, &move |client| client.get(this_url.clone())) + .with_authentication(&self.client, &json_get_request(self.info_url())) .await .map_err(ReqwestSrcError::ReqwestMiddleware)?; @@ -132,10 +129,9 @@ impl ProjectReadAsync for ReqwestSrcProjectAsync Result, Self::Error> { - let this_url = self.meta_url(); let meta_resp = self .auth_policy - .with_authentication(&self.client, &move |client| client.get(this_url.clone())) + .with_authentication(&self.client, &json_get_request(self.meta_url())) .await .map_err(ReqwestSrcError::ReqwestMiddleware)?; @@ -160,11 +156,9 @@ impl ProjectReadAsync for ReqwestSrcProjectAsync Result, Self::Error> { use futures::StreamExt as _; - let this_url = self.src_url(path); - let resp = self .auth_policy - .with_authentication(&self.client, &move |client| client.get(this_url.clone())) + .with_authentication(&self.client, &text_get_request(self.src_url(path))) .await .map_err(ReqwestSrcError::ReqwestMiddleware)?; @@ -183,17 +177,15 @@ impl ProjectReadAsync for ReqwestSrcProjectAsync bool { - let info_url = self.info_url(); - let info_request = move |client: &ClientWithMiddleware| client.head(info_url.clone()); + let info_request = &json_head_request(self.info_url()); let info_resp = self .auth_policy - .with_authentication(&self.client, &info_request); + .with_authentication(&self.client, info_request); - let meta_url = self.meta_url(); - let var_name = move |client: &ClientWithMiddleware| client.head(meta_url.clone()); + let meta_request = &json_head_request(self.meta_url()); let meta_resp = self .auth_policy - .with_authentication(&self.client, &var_name); + .with_authentication(&self.client, meta_request); match join!(info_resp, meta_resp) { (Ok(info_head), Ok(meta_head)) => { @@ -219,6 +211,7 @@ mod tests { use crate::{ auth::Unauthenticated, project::{ProjectRead, ProjectReadAsync, reqwest_src::ReqwestSrcProjectAsync}, + resolve::net_utils::create_reqwest_client, }; #[test] @@ -227,9 +220,7 @@ mod tests { let url = reqwest::Url::parse(&server.url()).unwrap(); - let client = - reqwest_middleware::ClientBuilder::new(reqwest::ClientBuilder::new().build().unwrap()) - .build(); + let client = create_reqwest_client()?; let project = ReqwestSrcProjectAsync { client, @@ -277,9 +268,7 @@ mod tests { .with_body(src) .create(); - let client = - reqwest_middleware::ClientBuilder::new(reqwest::ClientBuilder::new().build().unwrap()) - .build(); + let client = create_reqwest_client()?; let project = ReqwestSrcProjectAsync { client, diff --git a/core/src/resolve/mod.rs b/core/src/resolve/mod.rs index 57b3c967..d02903e3 100644 --- a/core/src/resolve/mod.rs +++ b/core/src/resolve/mod.rs @@ -17,6 +17,8 @@ pub mod file; #[cfg(all(feature = "filesystem", feature = "networking"))] pub mod gix_git; pub mod memory; +#[cfg(feature = "networking")] +pub mod net_utils; pub mod null; pub mod priority; pub mod remote; diff --git a/core/src/resolve/net_utils.rs b/core/src/resolve/net_utils.rs new file mode 100644 index 00000000..d194720a --- /dev/null +++ b/core/src/resolve/net_utils.rs @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: © 2026 Sysand contributors +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use std::{error::Error, fmt::Display}; + +use reqwest::header::{self, HeaderMap, HeaderValue}; +use reqwest_middleware::{ClientWithMiddleware, RequestBuilder}; +use url::Url; + +// application/vnd.github.raw+json is required for GitHub API to return raw +// file contents +const KPAR_ACCEPT: &str = + "application/zip, application/octet-stream, application/vnd.github.raw+json"; +const JSON_ACCEPT: &str = "application/vnd.github.raw+json, application/json, text/plain"; +// application/octet-stream is included here because `.sysml`/`.kerml` +// file extensions are unusual enough that some servers are likely to +// treat them as binary data +const TEXT_ACCEPT: &str = "application/vnd.github.raw+json, text/plain, application/octet-stream"; + +/// For KPAR and other binary files +pub fn kpar_get_request(url: impl Into) -> impl Fn(&ClientWithMiddleware) -> RequestBuilder { + let this_url = url.into(); + move |client: &ClientWithMiddleware| -> RequestBuilder { + client + .get(this_url.clone()) + .header(header::ACCEPT, KPAR_ACCEPT) + } +} + +pub fn kpar_head_request(url: impl Into) -> impl Fn(&ClientWithMiddleware) -> RequestBuilder { + let this_url = url.into(); + move |client: &ClientWithMiddleware| -> RequestBuilder { + client + .head(this_url.clone()) + .header(header::ACCEPT, KPAR_ACCEPT) + } +} + +/// For JSON files +pub fn json_get_request(url: impl Into) -> impl Fn(&ClientWithMiddleware) -> RequestBuilder { + let this_url = url.into(); + move |client: &ClientWithMiddleware| -> RequestBuilder { + client + .get(this_url.clone()) + .header(header::ACCEPT, JSON_ACCEPT) + } +} + +pub fn json_head_request(url: impl Into) -> impl Fn(&ClientWithMiddleware) -> RequestBuilder { + let this_url = url.into(); + move |client: &ClientWithMiddleware| -> RequestBuilder { + client + .head(this_url.clone()) + .header(header::ACCEPT, JSON_ACCEPT) + } +} + +/// For all text files that are not JSON +pub fn text_get_request(url: impl Into) -> impl Fn(&ClientWithMiddleware) -> RequestBuilder { + let this_url = url.into(); + move |client: &ClientWithMiddleware| -> RequestBuilder { + client + .get(this_url.clone()) + .header(header::ACCEPT, TEXT_ACCEPT) + } +} + +#[derive(Debug)] +pub struct ReqwestClientBuildError { + inner: reqwest::Error, +} + +impl From for ReqwestClientBuildError { + fn from(value: reqwest::Error) -> Self { + Self { inner: value } + } +} + +impl Display for ReqwestClientBuildError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "failed to build reqwest HTTP client: {}", self.inner)?; + match self.inner.source() { + Some(source) => write!(f, "\ncaused by: {}", source), + None => Ok(()), + } + } +} +impl Error for ReqwestClientBuildError {} + +pub fn create_reqwest_client() +-> Result { + const UA: &str = concat!("sysand/", env!("CARGO_PKG_VERSION")); + let mut headers = HeaderMap::new(); + headers.insert(header::USER_AGENT, HeaderValue::from_static(UA)); + + let client = reqwest::Client::builder() + .default_headers(headers) + .build()?; + + Ok(reqwest_middleware::ClientBuilder::new(client).build()) +} diff --git a/core/src/resolve/reqwest_http.rs b/core/src/resolve/reqwest_http.rs index 8076a46c..48a1ae76 100644 --- a/core/src/resolve/reqwest_http.rs +++ b/core/src/resolve/reqwest_http.rs @@ -299,7 +299,9 @@ mod tests { use crate::{ auth::Unauthenticated, project::ProjectRead, - resolve::{ResolutionOutcome, ResolveRead, ResolveReadAsync}, + resolve::{ + ResolutionOutcome, ResolveRead, ResolveReadAsync, net_utils::create_reqwest_client, + }, }; #[test] @@ -322,7 +324,7 @@ mod tests { .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) .create(); - let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(); + let client = create_reqwest_client()?; let resolver = super::HTTPResolverAsync { client, @@ -363,7 +365,7 @@ mod tests { with_slash: bool, //prefer_ranged: bool, ) -> Result<(), Box> { - let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(); + let client = create_reqwest_client()?; let resolver = super::HTTPResolverAsync { client, @@ -486,7 +488,7 @@ mod tests { // .spawn()?; // sleep(Duration::from_millis(1000)); - // let client = reqwest::blocking::ClientBuilder::new().build().unwrap(); + // let client = create_reqwest_client(); // let resolver = super::HTTPResolverAsync { // client, // lax: false, @@ -550,7 +552,7 @@ mod tests { // .with_body(&buf) // .create(); - // let client = reqwest::blocking::ClientBuilder::new().build().unwrap(); + // let client = create_reqwest_client(); // let resolver = super::HTTPResolverAsync { // client, // lax: false, diff --git a/sysand/Cargo.toml b/sysand/Cargo.toml index 2fab36d0..da8698a5 100644 --- a/sysand/Cargo.toml +++ b/sysand/Cargo.toml @@ -43,11 +43,6 @@ pubgrub = { version = "0.3.0", default-features = false } indexmap = "2.13.0" tokio = { version = "1.50.0", default-features = false } reqwest-middleware = { version = "0.5.1" } - -# Use native TLS only on Windows and Apple OSs -[target.'cfg(any(target_os = "windows", target_vendor = "apple"))'.dependencies] -reqwest = { version = "0.13.2", default-features = false, features = ["native-tls", "http2", "system-proxy", "blocking"] } -[target.'cfg(not(any(target_os = "windows", target_vendor = "apple")))'.dependencies] reqwest = { version = "0.13.2", features = ["rustls", "blocking"] } [dev-dependencies] diff --git a/sysand/src/lib.rs b/sysand/src/lib.rs index 01d3367e..bd31b254 100644 --- a/sysand/src/lib.rs +++ b/sysand/src/lib.rs @@ -37,6 +37,7 @@ use sysand_core::{ reference::ProjectReference, utils::wrapfs, }, + resolve::net_utils::create_reqwest_client, stdlib::known_std_libs, workspace::Workspace, }; @@ -163,7 +164,7 @@ pub fn run_cli(args: cli::Args) -> Result<()> { config.merge(auto_config); - let client = reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(); + let client = create_reqwest_client()?; let runtime = Arc::new( tokio::runtime::Builder::new_current_thread()