From 2da44d9892a1fee58b1c9e4a5b363bf83297dd94 Mon Sep 17 00:00:00 2001 From: Tilo Wiklund Date: Thu, 15 Jan 2026 16:08:26 +0100 Subject: [PATCH 01/13] Initial design for auth lib Signed-off-by: Tilo Wiklund --- Cargo.lock | 29 ++-- core/Cargo.toml | 1 + core/src/auth.rs | 366 +++++++++++++++++++++++++++++++++++++++++++++++ core/src/lib.rs | 2 + 4 files changed, 390 insertions(+), 8 deletions(-) create mode 100644 core/src/auth.rs diff --git a/Cargo.lock b/Cargo.lock index 3c298361..0ae30720 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -379,7 +379,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde0e0ec90c9dfb3b4b1a0891a7dcd0e2bffde2f7efed5fe7c9bb00e5bfb915e" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -592,7 +592,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -657,7 +657,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1633,6 +1633,18 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "globset" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3" +dependencies = [ + "aho-corasick", + "bstr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "h2" version = "0.4.12" @@ -2029,7 +2041,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2374,7 +2386,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2999,7 +3011,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3357,6 +3369,7 @@ dependencies = [ "fluent-uri", "futures", "gix", + "globset", "indexmap", "log", "logos", @@ -3474,7 +3487,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4019,7 +4032,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/core/Cargo.toml b/core/Cargo.toml index 548715d3..446377d1 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -60,6 +60,7 @@ tokio = { version = "1.48.0", default-features = false, features = ["rt", "io-ut reqwest-middleware = { version = "0.4.2", default-features = false } bytes = { version = "1.11.0", default-features = false } toml_edit = { version = "0.23.9", features = ["serde"] } +globset = { version = "0.4.18", default-features = false } [dev-dependencies] assert_cmd = "2.1.1" diff --git a/core/src/auth.rs b/core/src/auth.rs new file mode 100644 index 00000000..f861850b --- /dev/null +++ b/core/src/auth.rs @@ -0,0 +1,366 @@ +// SPDX-FileCopyrightText: © 2026 Sysand contributors +// SPDX-License-Identifier: MIT OR Apache-2.0 + +/// This module includes utilities for creating and using authentication policies for requests. +/// +use globset::{GlobBuilder, GlobSetBuilder}; +use reqwest::{Request, Response}; +use reqwest_middleware::{ClientWithMiddleware, RequestBuilder}; + +pub trait HTTPAuthentication { + /// Tries to execute a request with some authentication policy. The request might be retried + /// multiple times and it may generate auxiliary requests (using the provided client). + fn with_authentication( + &mut self, + client: &ClientWithMiddleware, + renew_request: &F, + ) -> impl Future> + where + F: Fn() -> Request + 'static, + { + async { + self.request_with_authentication(client, renew_request(), renew_request) + .await + } + } + + fn request_with_authentication( + &mut self, + client: &ClientWithMiddleware, + request: Request, + renew_request: &F, + ) -> impl Future> + where + F: Fn() -> Request + 'static; +} + +/// Authentication policy that does no authentication +#[derive(Debug, Clone)] +pub struct Unauthenticated {} + +impl HTTPAuthentication for Unauthenticated { + fn request_with_authentication( + &mut self, + client: &ClientWithMiddleware, + request: Request, + _renew_request: &F, + ) -> impl Future> + where + F: Fn() -> Request + 'static, + { + async move { client.execute(request).await } + } +} + +/// Authentication policy that *always* sends a username/password pair +#[derive(Debug, Clone)] +pub struct ForceHTTPBasicAuth { + pub username: String, + pub password: String, +} + +impl HTTPAuthentication for ForceHTTPBasicAuth { + fn request_with_authentication( + &mut self, + client: &ClientWithMiddleware, + request: Request, + _renew_request: &F, + ) -> impl Future> + where + F: Fn() -> Request + 'static, + { + async move { + client + .execute( + RequestBuilder::from_parts(client.clone(), request) + .basic_auth(self.username.clone(), Some(self.password.clone())) + .build()?, + ) + .await + } + } +} + +/// First tries `Higher` priority authentication and then the +/// `Lower` priority one in case the first request results in +/// a response in the 4xx range. +#[derive(Debug, Clone)] +pub struct SequenceAuthentication { + higher: Higher, + lower: Lower, +} + +impl HTTPAuthentication + for SequenceAuthentication +{ + fn request_with_authentication( + &mut self, + client: &ClientWithMiddleware, + request: Request, + renew_request: &F, + ) -> impl Future> + where + F: Fn() -> Request + 'static, + { + async move { + // Always try without authentication first + let initial_response = self + .higher + .request_with_authentication(client, request, renew_request) + .await?; + + // Many servers (e.g. GitLab pages) generate a 404 instead of a 401 or 403 in response + // to lack of authentication. + if initial_response.status().is_client_error() { + self.lower + .request_with_authentication(client, renew_request(), renew_request) + .await + } else { + Ok(initial_response) + } + } + } +} + +#[derive(Debug, Clone)] +pub struct GlobMapBuilder { + keys: Vec, + values: Vec, +} + +#[derive(Debug, Clone)] +pub struct GlobMap { + keys: Vec, + values: Vec, + globset: globset::GlobSet, +} + +impl GlobMapBuilder { + pub fn new() -> Self { + GlobMapBuilder { + keys: vec![], + values: vec![], + } + } + + pub fn add>(&mut self, globstr: S, value: T) { + self.keys.push(globstr.as_ref().to_string()); + self.values.push(value); + } + + pub fn build(self) -> Result, globset::Error> { + let mut builder = GlobSetBuilder::new(); + for globstr in &self.keys { + builder.add(GlobBuilder::new(globstr).literal_separator(true).build()?); + } + Ok(GlobMap { + keys: self.keys, + values: self.values, + globset: builder.build()?, + }) + } +} + +#[derive(Debug)] +pub enum GlobMapResultMut<'a, T> { + /// A unique matching pattern + Found(String, &'a mut T), + /// No matching pattern + NotFound, + /// Multiple matching patterns + Ambiguous(Vec<(String, &'a mut T)>), +} + +impl GlobMap { + pub fn lookup_mut<'a>(&'a mut self, key: &str) -> GlobMapResultMut<'a, T> { + let outcome = self.globset.matches(key); + if outcome.len() == 0 { + GlobMapResultMut::NotFound + } else if outcome.len() == 1 { + GlobMapResultMut::Found(self.keys[0].clone(), &mut self.values[outcome[0]]) + } else { + // Need to do some magic to keep multiple (disjoint) references into a mutable array + let mut result = Vec::with_capacity(outcome.len()); + let mut mut_values_iter = self.values.iter_mut(); + + let mut base = 0; + for idx in outcome { + result.push(( + self.keys[idx].clone(), + mut_values_iter.nth(idx - base).unwrap(), + )); + base = idx + 1; + } + + GlobMapResultMut::Ambiguous(result) + } + } +} + +/// Uses `restricted` authentication only on urls matching one of specified globs, +/// otherwise use `unrestricted`. For an ambiguous match a warning is generated and the +/// ambiguous options are tried, in order, until a non-4xx response is generated. If no +/// option produces a non-4xx response, the *first* response is returned. +pub struct RestrictAuthentication { + pub restricted: GlobMap, + pub unrestricted: Unrestricted, +} + +impl HTTPAuthentication + for RestrictAuthentication +{ + fn request_with_authentication( + &mut self, + client: &ClientWithMiddleware, + request: Request, + renew_request: &F, + ) -> impl Future> + where + F: Fn() -> Request + 'static, + { + async move { + let url = request.url(); + match self.restricted.lookup_mut(url.as_str()) { + GlobMapResultMut::Found(_, restricted) => { + restricted + .request_with_authentication(client, request, renew_request) + .await + } + GlobMapResultMut::NotFound => { + self.unrestricted + .request_with_authentication(client, request, renew_request) + .await + } + GlobMapResultMut::Ambiguous(items) => { + let mut items = items.into_iter(); + let (_, first_restricted) = items.next().unwrap(); + let first_response = first_restricted + .request_with_authentication(client, request, renew_request) + .await?; + if !first_response.status().is_client_error() { + return Ok(first_response); + } else { + for (_, other_restricted) in items { + let other_resonse = other_restricted + .with_authentication(client, renew_request) + .await?; + if !other_resonse.status().is_client_error() { + return Ok(other_resonse); + } + } + return Ok(first_response); + } + } + } + } + } +} + +/// Standard HTTP authentication policy where a restricted set of domains/paths have +/// BasicAuth username/password pairs specified, but they are sent only in response to a +/// 4xx status code. +pub type StandardHTTPAuthentication = RestrictAuthentication< + SequenceAuthentication< + // First try unauthenticated access... + Unauthenticated, + // ... but send username/password in response to 4xx. + // FIXME: Replace by a more general type as more authentication schemes are added + ForceHTTPBasicAuth, + >, + // For all other domains use unauthenticated acceess. + Unauthenticated, +>; + +/// Utility to simplify construction of `StandardHTTPAuthentication` +pub struct StandardHTTPAuthenticationBuilder { + partial: GlobMapBuilder>, +} + +impl StandardHTTPAuthenticationBuilder { + pub fn build(self) -> Result { + Ok(StandardHTTPAuthentication { + restricted: self.partial.build()?, + unrestricted: Unauthenticated {}, + }) + } + + pub fn add_basic_auth, T: AsRef, R: AsRef>( + &mut self, + globstr: S, + username: T, + password: R, + ) { + self.partial.add( + globstr, + SequenceAuthentication { + higher: Unauthenticated {}, + lower: ForceHTTPBasicAuth { + username: username.as_ref().to_string(), + password: password.as_ref().to_string(), + }, + }, + ); + } + + // TODO: For other authentication schemes + // pub fn add_..._auth, ...>(&mut self, globstr: S, ...) +} + +// pub struct GlobsetAuth +#[cfg(test)] +mod tests { + use crate::auth::{GlobMapBuilder, GlobMapResultMut}; + + #[test] + fn basic_globmap_lookup() -> Result<(), Box> { + let mut builder = GlobMapBuilder::new(); + builder.add("a*.com/*", 1); + builder.add("a*.com/**", 2); + builder.add("b.com/*", 3); + builder.add("a*.com/*/*", 4); + let mut globmap = builder.build()?; + + if let GlobMapResultMut::Ambiguous(vals) = globmap.lookup_mut("axx.com/xxx") { + let vals: Vec = vals.into_iter().map(|(_, i)| *i).collect(); + assert_eq!(vals, vec![1, 2]); + } else { + panic!("Expected ambiguous result."); + } + + if let GlobMapResultMut::Ambiguous(vals) = globmap.lookup_mut("axx.com/xxx/xxx") { + let vals: Vec = vals.into_iter().map(|(_, i)| *i).collect(); + assert_eq!(vals, vec![2, 4]); + } else { + panic!("Expected ambiguous result."); + } + + if let GlobMapResultMut::Found(_, val) = globmap.lookup_mut("axx.com/xxx/xxx/xxx") { + assert_eq!(*val, 2); + } else { + panic!("Expected unambiguous result."); + } + + if let GlobMapResultMut::Found(_, val) = globmap.lookup_mut("b.com/xxx") { + assert_eq!(*val, 3); + } else { + panic!("Expected unambiguous result."); + } + + if let GlobMapResultMut::NotFound = globmap.lookup_mut("axx.com") { + } else { + panic!("Expected no result."); + } + + if let GlobMapResultMut::NotFound = globmap.lookup_mut("bxx.com/xxx") { + } else { + panic!("Expected no result."); + } + + if let GlobMapResultMut::NotFound = globmap.lookup_mut("cxx.com/xxx") { + } else { + panic!("Expected no result."); + } + + Ok(()) + } +} diff --git a/core/src/lib.rs b/core/src/lib.rs index 239ced07..a0835a0a 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -8,6 +8,8 @@ pub use commands::*; pub mod model; +#[cfg(feature = "networking")] +pub mod auth; pub mod config; pub mod env; pub mod lock; From 69dd27b1011e54d29fd3aa68381da9acf3cf5cb9 Mon Sep 17 00:00:00 2001 From: Tilo Wiklund Date: Mon, 26 Jan 2026 11:31:17 +0100 Subject: [PATCH 02/13] feat(cli): rudimentary support for http authentication Signed-off-by: Tilo Wiklund --- bindings/java/src/lib.rs | 3 + bindings/py/src/lib.rs | 3 + core/scripts/run_tests.sh | 6 +- core/src/auth.rs | 255 +++++++++------ core/src/config/local_fs.rs | 1 + core/src/config/mod.rs | 12 + core/src/env/reqwest_http.rs | 107 +++--- core/src/project/reqwest_kpar_download.rs | 37 ++- core/src/project/reqwest_src.rs | 100 ++++-- core/src/resolve/reqwest_http.rs | 75 +++-- core/src/resolve/standard.rs | 43 +-- sysand/scripts/run_tests.sh | 2 +- sysand/src/commands/add.rs | 6 +- sysand/src/commands/clone.rs | 6 +- sysand/src/commands/env.rs | 14 +- sysand/src/commands/info.rs | 9 +- sysand/src/commands/lock.rs | 5 +- sysand/src/commands/sync.rs | 11 +- sysand/src/lib.rs | 70 +++- sysand/tests/cfg_base.rs | 1 + sysand/tests/cli_info.rs | 375 +++++++++++++++++++++- sysand/tests/cli_sync.rs | 175 ++++++++++ sysand/tests/common/mod.rs | 51 ++- 23 files changed, 1102 insertions(+), 265 deletions(-) diff --git a/bindings/java/src/lib.rs b/bindings/java/src/lib.rs index e73b60bf..db763503 100644 --- a/bindings/java/src/lib.rs +++ b/bindings/java/src/lib.rs @@ -9,6 +9,7 @@ use jni::{ objects::{JClass, JObject, JObjectArray, JString}, }; use sysand_core::{ + auth::Unauthenticated, build::KParBuildError, commands, env::local_directory::{self, LocalWriteError}, @@ -225,6 +226,8 @@ pub extern "system" fn Java_com_sensmetry_sysand_Sysand_info<'local>( Some(client), index_base_url.map(|x| vec![x]), runtime, + // FIXME: Add Java support for authentication + Arc::new(Unauthenticated {}), ); let results = match commands::info::do_info(&uri, &combined_resolver) { diff --git a/bindings/py/src/lib.rs b/bindings/py/src/lib.rs index df44ced3..176415b9 100644 --- a/bindings/py/src/lib.rs +++ b/bindings/py/src/lib.rs @@ -14,6 +14,7 @@ use pyo3::{ use semver::{Version, VersionReq}; use sysand_core::{ add::do_add, + auth::Unauthenticated, build::{KParBuildError, do_build_kpar}, commands::{ env::{EnvError, do_env_local_dir}, @@ -141,6 +142,8 @@ fn do_info_py( Some(client), index_url, runtime, + // FIXME: Add Python support for authentication + Arc::new(Unauthenticated {}), ); match do_info(&uri, &combined_resolver) { diff --git a/core/scripts/run_tests.sh b/core/scripts/run_tests.sh index 5cfc8d3b..d7e4e901 100755 --- a/core/scripts/run_tests.sh +++ b/core/scripts/run_tests.sh @@ -8,6 +8,6 @@ PACKAGE_DIR=$(dirname "$SCRIPT_DIR") cd "$PACKAGE_DIR" -cargo test --features filesystem,networking,alltests -cargo test --features js -cargo test --features python +cargo test --features filesystem,networking,alltests $@ +cargo test --features js $@ +cargo test --features python $@ diff --git a/core/src/auth.rs b/core/src/auth.rs index f861850b..899a91f1 100644 --- a/core/src/auth.rs +++ b/core/src/auth.rs @@ -4,34 +4,33 @@ /// This module includes utilities for creating and using authentication policies for requests. /// use globset::{GlobBuilder, GlobSetBuilder}; -use reqwest::{Request, Response}; +use reqwest::Response; use reqwest_middleware::{ClientWithMiddleware, RequestBuilder}; pub trait HTTPAuthentication { /// Tries to execute a request with some authentication policy. The request might be retried /// multiple times and it may generate auxiliary requests (using the provided client). fn with_authentication( - &mut self, + &self, client: &ClientWithMiddleware, renew_request: &F, ) -> impl Future> where - F: Fn() -> Request + 'static, + F: Fn(&ClientWithMiddleware) -> RequestBuilder + 'static, { async { - self.request_with_authentication(client, renew_request(), renew_request) + self.request_with_authentication(renew_request(client), renew_request) .await } } fn request_with_authentication( - &mut self, - client: &ClientWithMiddleware, - request: Request, + &self, + request: RequestBuilder, renew_request: &F, ) -> impl Future> where - F: Fn() -> Request + 'static; + F: Fn(&ClientWithMiddleware) -> RequestBuilder + 'static; } /// Authentication policy that does no authentication @@ -39,16 +38,27 @@ pub trait HTTPAuthentication { pub struct Unauthenticated {} impl HTTPAuthentication for Unauthenticated { - fn request_with_authentication( - &mut self, - client: &ClientWithMiddleware, - request: Request, + async fn request_with_authentication( + &self, + request: RequestBuilder, _renew_request: &F, - ) -> impl Future> + ) -> Result where - F: Fn() -> Request + 'static, + F: Fn(&ClientWithMiddleware) -> RequestBuilder + 'static, { - async move { client.execute(request).await } + request.send().await + } + + async fn with_authentication( + &self, + client: &ClientWithMiddleware, + renew_request: &F, + ) -> Result + where + F: Fn(&ClientWithMiddleware) -> RequestBuilder + 'static, + { + self.request_with_authentication(renew_request(client), renew_request) + .await } } @@ -60,24 +70,30 @@ pub struct ForceHTTPBasicAuth { } impl HTTPAuthentication for ForceHTTPBasicAuth { - fn request_with_authentication( - &mut self, - client: &ClientWithMiddleware, - request: Request, + async fn request_with_authentication( + &self, + request: RequestBuilder, _renew_request: &F, - ) -> impl Future> + ) -> Result where - F: Fn() -> Request + 'static, + F: Fn(&ClientWithMiddleware) -> RequestBuilder + 'static, { - async move { - client - .execute( - RequestBuilder::from_parts(client.clone(), request) - .basic_auth(self.username.clone(), Some(self.password.clone())) - .build()?, - ) - .await - } + request + .basic_auth(self.username.clone(), Some(self.password.clone())) + .send() + .await + } + + async fn with_authentication( + &self, + client: &ClientWithMiddleware, + renew_request: &F, + ) -> Result + where + F: Fn(&ClientWithMiddleware) -> RequestBuilder + 'static, + { + self.request_with_authentication(renew_request(client), renew_request) + .await } } @@ -93,31 +109,34 @@ pub struct SequenceAuthentication { impl HTTPAuthentication for SequenceAuthentication { - fn request_with_authentication( - &mut self, - client: &ClientWithMiddleware, - request: Request, + async fn request_with_authentication( + &self, + request: RequestBuilder, renew_request: &F, - ) -> impl Future> + ) -> Result where - F: Fn() -> Request + 'static, + F: Fn(&ClientWithMiddleware) -> RequestBuilder + 'static, { - async move { - // Always try without authentication first - let initial_response = self - .higher - .request_with_authentication(client, request, renew_request) - .await?; - - // Many servers (e.g. GitLab pages) generate a 404 instead of a 401 or 403 in response - // to lack of authentication. - if initial_response.status().is_client_error() { - self.lower - .request_with_authentication(client, renew_request(), renew_request) - .await - } else { - Ok(initial_response) - } + let (client, current_request_result) = request.build_split(); + let current_request = current_request_result?; + + // Always try without authentication first + let initial_response = self + .higher + .request_with_authentication( + RequestBuilder::from_parts(client.clone(), current_request), + renew_request, + ) + .await?; + + // Many servers (e.g. GitLab pages) generate a 404 instead of a 401 or 403 in response + // to lack of authentication. + if initial_response.status().is_client_error() { + self.lower + .request_with_authentication(renew_request(&client), renew_request) + .await + } else { + Ok(initial_response) } } } @@ -135,13 +154,19 @@ pub struct GlobMap { globset: globset::GlobSet, } -impl GlobMapBuilder { - pub fn new() -> Self { +impl Default for GlobMapBuilder { + fn default() -> Self { GlobMapBuilder { keys: vec![], values: vec![], } } +} + +impl GlobMapBuilder { + pub fn new() -> Self { + Self::default() + } pub fn add>(&mut self, globstr: S, value: T) { self.keys.push(globstr.as_ref().to_string()); @@ -161,6 +186,16 @@ impl GlobMapBuilder { } } +#[derive(Debug)] +pub enum GlobMapResult<'a, T> { + /// A unique matching pattern + Found(String, &'a T), + /// No matching pattern + NotFound, + /// Multiple matching patterns + Ambiguous(Vec<(String, &'a T)>), +} + #[derive(Debug)] pub enum GlobMapResultMut<'a, T> { /// A unique matching pattern @@ -172,9 +207,30 @@ pub enum GlobMapResultMut<'a, T> { } impl GlobMap { + pub fn lookup<'a>(&'a self, key: &str) -> GlobMapResult<'a, T> { + let outcome = self.globset.matches(key); + if outcome.is_empty() { + GlobMapResult::NotFound + } else if outcome.len() == 1 { + GlobMapResult::Found(self.keys[0].clone(), &self.values[outcome[0]]) + } else { + // Need to do some magic to keep multiple (disjoint) references into a mutable array + let mut result = Vec::with_capacity(outcome.len()); + let mut values_iter = self.values.iter(); + + let mut base = 0; + for idx in outcome { + result.push((self.keys[idx].clone(), values_iter.nth(idx - base).unwrap())); + base = idx + 1; + } + + GlobMapResult::Ambiguous(result) + } + } + pub fn lookup_mut<'a>(&'a mut self, key: &str) -> GlobMapResultMut<'a, T> { let outcome = self.globset.matches(key); - if outcome.len() == 0 { + if outcome.is_empty() { GlobMapResultMut::NotFound } else if outcome.len() == 1 { GlobMapResultMut::Found(self.keys[0].clone(), &mut self.values[outcome[0]]) @@ -197,6 +253,7 @@ impl GlobMap { } } +#[derive(Debug, Clone)] /// Uses `restricted` authentication only on urls matching one of specified globs, /// otherwise use `unrestricted`. For an ambiguous match a warning is generated and the /// ambiguous options are tried, in order, until a non-4xx response is generated. If no @@ -209,47 +266,56 @@ pub struct RestrictAuthentication { impl HTTPAuthentication for RestrictAuthentication { - fn request_with_authentication( - &mut self, - client: &ClientWithMiddleware, - request: Request, + async fn request_with_authentication( + &self, + request: RequestBuilder, renew_request: &F, - ) -> impl Future> + ) -> Result where - F: Fn() -> Request + 'static, + F: Fn(&ClientWithMiddleware) -> RequestBuilder + 'static, { - async move { - let url = request.url(); - match self.restricted.lookup_mut(url.as_str()) { - GlobMapResultMut::Found(_, restricted) => { - restricted - .request_with_authentication(client, request, renew_request) - .await - } - GlobMapResultMut::NotFound => { - self.unrestricted - .request_with_authentication(client, request, renew_request) - .await - } - GlobMapResultMut::Ambiguous(items) => { - let mut items = items.into_iter(); - let (_, first_restricted) = items.next().unwrap(); - let first_response = first_restricted - .request_with_authentication(client, request, renew_request) - .await?; - if !first_response.status().is_client_error() { - return Ok(first_response); - } else { - for (_, other_restricted) in items { - let other_resonse = other_restricted - .with_authentication(client, renew_request) - .await?; - if !other_resonse.status().is_client_error() { - return Ok(other_resonse); - } + let (client, current_request_result) = request.build_split(); + let current_request = current_request_result?; + + let url = current_request.url(); + match self.restricted.lookup(url.as_str()) { + GlobMapResult::Found(_, restricted) => { + restricted + .request_with_authentication( + RequestBuilder::from_parts(client.clone(), current_request), + renew_request, + ) + .await + } + GlobMapResult::NotFound => { + self.unrestricted + .request_with_authentication( + RequestBuilder::from_parts(client.clone(), current_request), + renew_request, + ) + .await + } + GlobMapResult::Ambiguous(items) => { + let mut items = items.into_iter(); + let (_, first_restricted) = items.next().unwrap(); + let first_response = first_restricted + .request_with_authentication( + RequestBuilder::from_parts(client.clone(), current_request), + renew_request, + ) + .await?; + if !first_response.status().is_client_error() { + Ok(first_response) + } else { + for (_, other_restricted) in items { + let other_resonse = other_restricted + .with_authentication(&client, renew_request) + .await?; + if !other_resonse.status().is_client_error() { + return Ok(other_resonse); } - return Ok(first_response); } + Ok(first_response) } } } @@ -272,11 +338,16 @@ pub type StandardHTTPAuthentication = RestrictAuthentication< >; /// Utility to simplify construction of `StandardHTTPAuthentication` +#[derive(Debug, Default, Clone)] pub struct StandardHTTPAuthenticationBuilder { partial: GlobMapBuilder>, } impl StandardHTTPAuthenticationBuilder { + pub fn new() -> Self { + Self::default() + } + pub fn build(self) -> Result { Ok(StandardHTTPAuthentication { restricted: self.partial.build()?, @@ -303,7 +374,7 @@ impl StandardHTTPAuthenticationBuilder { } // TODO: For other authentication schemes - // pub fn add_..._auth, ...>(&mut self, globstr: S, ...) + // pub fn add_..._auth, ...>(&self, globstr: S, ...) } // pub struct GlobsetAuth diff --git a/core/src/config/local_fs.rs b/core/src/config/local_fs.rs index a6ad6ada..a0232903 100644 --- a/core/src/config/local_fs.rs +++ b/core/src/config/local_fs.rs @@ -70,6 +70,7 @@ mod tests { url: "http://www.example.com".to_string(), ..Default::default() }]), + auth: None, }; config_file .write_all(toml::to_string_pretty(&config).unwrap().as_bytes()) diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs index be7facaf..32e5d582 100644 --- a/core/src/config/mod.rs +++ b/core/src/config/mod.rs @@ -12,6 +12,7 @@ pub struct Config { pub quiet: Option, pub verbose: Option, pub index: Option>, + pub auth: Option>, } impl Config { @@ -19,6 +20,10 @@ impl Config { self.quiet = self.quiet.or(config.quiet); self.verbose = self.verbose.or(config.verbose); extend_option_vec(&mut self.index, config.index); + + if let Some(auth) = config.auth { + self.auth = Some(auth.clone()); + } } pub fn index_urls( @@ -98,6 +103,12 @@ pub struct Index { pub default: Option, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum AuthSource { + EnvVar, + Keyring, +} + #[cfg(test)] mod tests { use url::Url; @@ -133,6 +144,7 @@ mod tests { url: "http://www.example.com".to_string(), ..Default::default() }]), + auth: None, }; defaults.merge(config.clone()); diff --git a/core/src/env/reqwest_http.rs b/core/src/env/reqwest_http.rs index 7da22cc4..a9d483b9 100644 --- a/core/src/env/reqwest_http.rs +++ b/core/src/env/reqwest_http.rs @@ -6,13 +6,16 @@ use std::{ marker::{Send, Unpin}, pin::Pin, string::String, + sync::Arc, }; use futures::{Stream, TryStreamExt}; +use reqwest_middleware::ClientWithMiddleware; use sha2::Sha256; use thiserror::Error; use crate::{ + auth::{HTTPAuthentication, StandardHTTPAuthentication}, env::{ AsSyncEnvironmentTokio, ReadEnvironmentAsync, local_directory::{ENTRIES_PATH, VERSIONS_PATH}, @@ -26,11 +29,12 @@ use crate::{ use futures::{AsyncBufReadExt as _, StreamExt as _}; -pub type HTTPEnvironment = AsSyncEnvironmentTokio; +pub type HTTPEnvironment = AsSyncEnvironmentTokio>; #[derive(Debug)] -pub struct HTTPEnvironmentAsync { +pub struct HTTPEnvironmentAsync { pub client: reqwest_middleware::ClientWithMiddleware, + pub auth_policy: Arc, pub base_url: reqwest::Url, pub prefer_src: bool, // Currently no async implementation of ranged @@ -55,7 +59,7 @@ pub fn path_encode_uri>(uri: S) -> std::vec::IntoIter { segment_uri_generic::(uri) } -impl HTTPEnvironmentAsync { +impl HTTPEnvironmentAsync { pub fn root_url(&self) -> url::Url { let mut result = self.base_url.clone(); @@ -77,12 +81,6 @@ impl HTTPEnvironmentAsync { Self::url_join(&self.root_url(), ENTRIES_PATH) } - pub fn get_entries_request( - &self, - ) -> Result { - Ok(self.client.get(self.entries_url()?)) - } - pub fn iri_url>(&self, iri: S) -> Result { let mut result = self.root_url(); @@ -107,13 +105,6 @@ impl HTTPEnvironmentAsync { self.iri_url_join(iri, VERSIONS_PATH) } - pub fn get_versions_request>( - &self, - iri: S, - ) -> Result { - Ok(self.client.get(self.versions_url(iri)?)) - } - pub fn project_kpar_url, T: AsRef>( &self, iri: S, @@ -136,20 +127,23 @@ impl HTTPEnvironmentAsync { &self, uri: S, version: T, - ) -> Result, HTTPEnvironmentError> { + ) -> Result>, HTTPEnvironmentError> { let project_url = self.project_src_url(uri, version)?; let src_project_url = Self::url_join(&project_url, ".project.json")?; - if !self - .client - .head(src_project_url.clone()) - .header("ACCEPT", "application/json, text/plain") - .send() + let this_url = src_project_url.clone(); + let src_project_request = move |client: &ClientWithMiddleware| { + client + .head(this_url.clone()) + .header("ACCEPT", "application/json, text/plain") + }; + let src_project_response = self + .auth_policy + .with_authentication(&self.client, &src_project_request) .await - .map_err(|e| HTTPEnvironmentError::HTTPRequest(src_project_url.as_str().into(), e))? - .status() - .is_success() - { + .map_err(|e| HTTPEnvironmentError::HTTPRequest(src_project_url.as_str().into(), e))?; + + if !src_project_response.status().is_success() { return Ok(None); } @@ -157,6 +151,7 @@ impl HTTPEnvironmentAsync { ReqwestSrcProjectAsync { client: self.client.clone(), url: src_project_url, + auth_policy: self.auth_policy.clone(), }, ))) } @@ -165,15 +160,21 @@ impl HTTPEnvironmentAsync { &self, uri: S, version: T, - ) -> Result, HTTPEnvironmentError> { + ) -> Result>, HTTPEnvironmentError> { let kpar_project_url = self.project_kpar_url(&uri, &version)?; - if !self - .client - .head(kpar_project_url.clone()) - .header("ACCEPT", "application/zip, application/octet-stream") - .send() - .await + let this_url = kpar_project_url.clone(); + let kpar_project_request = move |client: &ClientWithMiddleware| { + client + .head(this_url.clone()) + .header("ACCEPT", "application/zip, application/octet-stream") + }; + let kpar_project_response = self + .auth_policy + .with_authentication(&self.client, &kpar_project_request) + .await; + + if !kpar_project_response .map_err(|e| HTTPEnvironmentError::HTTPRequest(kpar_project_url.as_str().into(), e))? .status() .is_success() @@ -185,6 +186,7 @@ impl HTTPEnvironmentAsync { ReqwestKparDownloadedProject::new_guess_root( &self.project_kpar_url(&uri, &version)?, self.client.clone(), + self.auth_policy.clone(), ) .expect("internal IO error"), ))) @@ -229,7 +231,9 @@ fn trim_line(line: Result) -> Result { Ok(line?.trim().to_string()) } -impl ReadEnvironmentAsync for HTTPEnvironmentAsync { +impl ReadEnvironmentAsync + for HTTPEnvironmentAsync +{ type ReadError = HTTPEnvironmentError; // This can be made more concrete, but the type is humongous @@ -238,9 +242,15 @@ impl ReadEnvironmentAsync for HTTPEnvironmentAsync { >; async fn uris_async(&self) -> Result { - let response = self.get_entries_request()?.send().await.map_err(|e| { - HTTPEnvironmentError::HTTPRequest(self.entries_url().unwrap().as_str().into(), e) - })?; + let this_url = self.entries_url()?; + + let response = self + .auth_policy + .with_authentication(&self.client, &move |client| client.get(this_url.clone())) + .await + .map_err(|e| { + HTTPEnvironmentError::HTTPRequest(self.entries_url().unwrap().as_str().into(), e) + })?; let inner = if response.status().is_success() { Some( @@ -268,9 +278,17 @@ impl ReadEnvironmentAsync for HTTPEnvironmentAsync { &self, uri: S, ) -> Result { - let response = self.get_versions_request(&uri)?.send().await.map_err(|e| { - HTTPEnvironmentError::HTTPRequest(self.versions_url(uri).unwrap().as_str().into(), e) - })?; + let this_url = self.versions_url(uri.as_ref())?; + let response = self + .auth_policy + .with_authentication(&self.client, &move |client| client.get(this_url.clone())) + .await + .map_err(|e| { + HTTPEnvironmentError::HTTPRequest( + self.versions_url(uri).unwrap().as_str().into(), + e, + ) + })?; let inner = if response.status().is_success() { Some( @@ -290,7 +308,7 @@ impl ReadEnvironmentAsync for HTTPEnvironmentAsync { Ok(Optionally { inner }) } - type InterchangeProjectRead = HTTPProjectAsync; + type InterchangeProjectRead = HTTPProjectAsync; async fn get_project_async, T: AsRef>( &self, @@ -326,6 +344,7 @@ mod test { use std::sync::Arc; use crate::{ + auth::Unauthenticated, env::{ReadEnvironment, ReadEnvironmentAsync}, resolve::reqwest_http::HTTPProjectAsync, }; @@ -336,6 +355,7 @@ mod test { client: reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(), base_url: url::Url::parse("https://www.example.com/a/b")?, prefer_src: true, + auth_policy: Arc::new(Unauthenticated {}), // try_ranged: false, }; @@ -370,6 +390,7 @@ mod test { client: reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(), base_url: url::Url::parse(&host)?, prefer_src: true, + auth_policy: Arc::new(Unauthenticated {}), //try_ranged: false, } .to_tokio_sync(Arc::new( @@ -442,6 +463,7 @@ mod test { client: reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(), base_url: url::Url::parse(&host)?, prefer_src: true, + auth_policy: Arc::new(Unauthenticated {}), //try_ranged: false, } .to_tokio_sync(Arc::new( @@ -481,6 +503,7 @@ mod test { client: reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(), base_url: url::Url::parse(&host)?, prefer_src: false, + auth_policy: Arc::new(Unauthenticated {}), //try_ranged: false, } .to_tokio_sync(Arc::new( @@ -520,6 +543,7 @@ mod test { client: reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(), base_url: url::Url::parse(&host)?, prefer_src: false, + auth_policy: Arc::new(Unauthenticated {}), //try_ranged: false, } .to_tokio_sync(Arc::new( @@ -571,6 +595,7 @@ mod test { client: reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(), base_url: url::Url::parse(&host)?, prefer_src: true, + auth_policy: Arc::new(Unauthenticated {}), //try_ranged: false, } .to_tokio_sync(Arc::new( diff --git a/core/src/project/reqwest_kpar_download.rs b/core/src/project/reqwest_kpar_download.rs index 7f633263..be565c42 100644 --- a/core/src/project/reqwest_kpar_download.rs +++ b/core/src/project/reqwest_kpar_download.rs @@ -5,15 +5,20 @@ use std::{ io::{self, Write as _}, marker::Unpin, pin::Pin, + sync::Arc, }; use futures::AsyncRead; +use reqwest_middleware::{ClientWithMiddleware, RequestBuilder}; use tempfile::tempdir; use thiserror::Error; -use crate::project::{ - ProjectRead, ProjectReadAsync, - local_kpar::{LocalKParError, LocalKParProject}, +use crate::{ + auth::HTTPAuthentication, + project::{ + ProjectRead, ProjectReadAsync, + local_kpar::{LocalKParError, LocalKParProject}, + }, }; use super::utils::{FsIoError, ToPathBuf, wrapfs}; @@ -27,10 +32,11 @@ use super::utils::{FsIoError, ToPathBuf, wrapfs}; /// Downloads the full archive to a temporary directory and then accesses it using /// `LocalKParProject`. #[derive(Debug)] -pub struct ReqwestKparDownloadedProject { +pub struct ReqwestKparDownloadedProject { pub url: reqwest::Url, pub client: reqwest_middleware::ClientWithMiddleware, pub inner: LocalKParProject, + pub auth_policy: Arc, } #[derive(Error, Debug)] @@ -57,10 +63,11 @@ impl From for ReqwestKparDownloadedError { } } -impl ReqwestKparDownloadedProject { +impl ReqwestKparDownloadedProject { pub fn new_guess_root>( url: S, client: reqwest_middleware::ClientWithMiddleware, + auth_policy: Arc, ) -> Result { let tmp_dir = tempdir().map_err(FsIoError::MkTempDir)?; @@ -77,6 +84,7 @@ impl ReqwestKparDownloadedProject { root: None, }, client, + auth_policy, }) } @@ -87,10 +95,15 @@ impl ReqwestKparDownloadedProject { let mut file = wrapfs::File::create(&self.inner.archive_path)?; + let this_url = self.url.clone(); let resp = self - .client - .get(self.url.clone()) - .send() + .auth_policy + .with_authentication( + &self.client, + &move |client: &ClientWithMiddleware| -> RequestBuilder { + client.get(this_url.clone()) + }, + ) .await .map_err(|e| ReqwestKparDownloadedError::Reqwest(self.url.as_str().into(), e))?; @@ -134,7 +147,7 @@ impl AsyncRead for AsAsyncRead { } } -impl ProjectReadAsync for ReqwestKparDownloadedProject { +impl ProjectReadAsync for ReqwestKparDownloadedProject { type Error = ReqwestKparDownloadedError; async fn get_project_async( @@ -182,7 +195,10 @@ mod tests { sync::Arc, }; - use crate::project::{ProjectRead, ProjectReadAsync}; + use crate::{ + auth::Unauthenticated, + project::{ProjectRead, ProjectReadAsync}, + }; #[test] fn test_basic_download_request() -> Result<(), Box> { @@ -224,6 +240,7 @@ mod tests { let project = super::ReqwestKparDownloadedProject::new_guess_root( format!("{}test_basic_download_request.kpar", url,), reqwest_middleware::ClientBuilder::new(reqwest::Client::new()).build(), + Arc::new(Unauthenticated {}), )? .to_tokio_sync(Arc::new( tokio::runtime::Builder::new_current_thread() diff --git a/core/src/project/reqwest_src.rs b/core/src/project/reqwest_src.rs index b2572441..50c40277 100644 --- a/core/src/project/reqwest_src.rs +++ b/core/src/project/reqwest_src.rs @@ -1,9 +1,10 @@ // SPDX-FileCopyrightText: © 2025 Sysand contributors // SPDX-License-Identifier: MIT OR Apache-2.0 -use std::{io, marker::Send, pin::Pin}; +use std::{io, marker::Send, pin::Pin, sync::Arc}; use futures::{TryStreamExt, join}; +use reqwest_middleware::ClientWithMiddleware; use thiserror::Error; /// This module implements accessing interchanged projects stored remotely over HTTP. /// It is currently written using the blocking Reqwest client. Once sysand functionality @@ -12,6 +13,7 @@ use thiserror::Error; use typed_path::Utf8UnixPath; use crate::{ + auth::HTTPAuthentication, model::{InterchangeProjectInfoRaw, InterchangeProjectMetadataRaw}, project::ProjectReadAsync, }; @@ -27,14 +29,15 @@ use crate::{ /// is accessed by /// GET https://www.example.com/project/M%C3%ABkan%C3%AFk/K%C3%B6mmand%C3%B6h.sysml #[derive(Clone, Debug)] -pub struct ReqwestSrcProjectAsync { +pub struct ReqwestSrcProjectAsync { /// (reqwest) HTTP client to use for GET requests pub client: reqwest_middleware::ClientWithMiddleware, // Internally an Arc /// Base-url of the project pub url: reqwest::Url, + pub auth_policy: Arc, } -impl ReqwestSrcProjectAsync { +impl ReqwestSrcProjectAsync { pub fn info_url(&self) -> reqwest::Url { self.url.join(".project.json").expect("internal URL error") } @@ -49,29 +52,29 @@ impl ReqwestSrcProjectAsync { .expect("internal URL error") } - pub fn head_info(&self) -> reqwest_middleware::RequestBuilder { - self.client - .head(self.info_url()) - .header(reqwest::header::ACCEPT, "application/json") - } - - pub fn head_meta(&self) -> reqwest_middleware::RequestBuilder { - self.client - .head(self.meta_url()) - .header(reqwest::header::ACCEPT, "application/json") - } - - pub fn get_info(&self) -> reqwest_middleware::RequestBuilder { - self.client - .get(self.info_url()) - .header(reqwest::header::ACCEPT, "application/json") - } - - pub fn get_meta(&self) -> reqwest_middleware::RequestBuilder { - self.client - .get(self.meta_url()) - .header(reqwest::header::ACCEPT, "application/json") - } + // pub fn head_info(&self) -> reqwest_middleware::RequestBuilder { + // self.client + // .head(self.info_url()) + // .header(reqwest::header::ACCEPT, "application/json") + // } + + // pub fn head_meta(&self) -> reqwest_middleware::RequestBuilder { + // self.client + // .head(self.meta_url()) + // .header(reqwest::header::ACCEPT, "application/json") + // } + + // pub fn get_info(&self) -> reqwest_middleware::RequestBuilder { + // self.client + // .get(self.info_url()) + // .header(reqwest::header::ACCEPT, "application/json") + // } + + // pub fn get_meta(&self) -> reqwest_middleware::RequestBuilder { + // self.client + // .get(self.meta_url()) + // .header(reqwest::header::ACCEPT, "application/json") + // } pub fn reqwest_src>( &self, @@ -93,7 +96,7 @@ pub enum ReqwestSrcError { BadStatus(Box, reqwest::StatusCode), } -impl ProjectReadAsync for ReqwestSrcProjectAsync { +impl ProjectReadAsync for ReqwestSrcProjectAsync { type Error = ReqwestSrcError; async fn get_project_async( @@ -111,9 +114,10 @@ impl ProjectReadAsync for ReqwestSrcProjectAsync { } async fn get_info_async(&self) -> Result, Self::Error> { + let this_url = self.info_url(); let info_resp = self - .get_info() - .send() + .auth_policy + .with_authentication(&self.client, &move |client| client.get(this_url.clone())) .await .map_err(|e| ReqwestSrcError::Reqwest(self.info_url().into(), e))?; @@ -129,9 +133,10 @@ impl ProjectReadAsync for ReqwestSrcProjectAsync { } async fn get_meta_async(&self) -> Result, Self::Error> { + let this_url = self.meta_url(); let meta_resp = self - .get_meta() - .send() + .auth_policy + .with_authentication(&self.client, &move |client| client.get(this_url.clone())) .await .map_err(|e| ReqwestSrcError::Reqwest(self.meta_url().into(), e))?; @@ -180,7 +185,19 @@ impl ProjectReadAsync for ReqwestSrcProjectAsync { } async fn is_definitely_invalid_async(&self) -> bool { - match join!(self.head_info().send(), self.head_meta().send()) { + let info_url = self.info_url(); + let info_request = move |client: &ClientWithMiddleware| client.head(info_url.clone()); + let info_resp = self + .auth_policy + .with_authentication(&self.client, &info_request); + + let meta_url = self.meta_url(); + let var_name = move |client: &ClientWithMiddleware| client.head(meta_url.clone()); + let meta_resp = self + .auth_policy + .with_authentication(&self.client, &var_name); + + match join!(info_resp, meta_resp) { (Ok(info_head), Ok(meta_head)) => { !info_head.status().is_success() || !meta_head.status().is_success() } @@ -201,7 +218,10 @@ mod tests { use typed_path::Utf8UnixPath; - use crate::project::{ProjectRead, ProjectReadAsync, reqwest_src::ReqwestSrcProjectAsync}; + use crate::{ + auth::Unauthenticated, + project::{ProjectRead, ProjectReadAsync, reqwest_src::ReqwestSrcProjectAsync}, + }; #[test] fn empty_remote_definitely_invalid_http_src() -> Result<(), Box> { @@ -213,7 +233,12 @@ mod tests { reqwest_middleware::ClientBuilder::new(reqwest::ClientBuilder::new().build().unwrap()) .build(); - let project = ReqwestSrcProjectAsync { client, url }.to_tokio_sync(Arc::new( + let project = ReqwestSrcProjectAsync { + client, + url, + auth_policy: Arc::new(Unauthenticated {}), + } + .to_tokio_sync(Arc::new( tokio::runtime::Builder::new_current_thread() .enable_all() .build()?, @@ -258,7 +283,12 @@ mod tests { reqwest_middleware::ClientBuilder::new(reqwest::ClientBuilder::new().build().unwrap()) .build(); - let project = ReqwestSrcProjectAsync { client, url }.to_tokio_sync(Arc::new( + let project = ReqwestSrcProjectAsync { + client, + url, + auth_policy: Arc::new(Unauthenticated {}), + } + .to_tokio_sync(Arc::new( tokio::runtime::Builder::new_current_thread() .enable_all() .build()?, diff --git a/core/src/resolve/reqwest_http.rs b/core/src/resolve/reqwest_http.rs index c7ad0e6f..438bd85b 100644 --- a/core/src/resolve/reqwest_http.rs +++ b/core/src/resolve/reqwest_http.rs @@ -1,13 +1,14 @@ // SPDX-FileCopyrightText: © 2025 Sysand contributors // SPDX-License-Identifier: MIT OR Apache-2.0 -use std::{convert::Infallible, io, pin::Pin}; +use std::{convert::Infallible, io, pin::Pin, sync::Arc}; use fluent_uri::component::Scheme; use futures::AsyncRead; use thiserror::Error; use crate::{ + auth::HTTPAuthentication, model::{InterchangeProjectInfoRaw, InterchangeProjectMetadataRaw}, project::{ ProjectRead, ProjectReadAsync, reqwest_kpar_download::ReqwestKparDownloadedProject, @@ -18,9 +19,10 @@ use crate::{ /// Tries to resolve http(s) URLs as direct (resolvable) links to interchange projects. #[derive(Debug)] -pub struct HTTPResolverAsync { +pub struct HTTPResolverAsync { pub client: reqwest_middleware::ClientWithMiddleware, pub lax: bool, + pub auth_policy: Arc, //pub prefer_ranged: bool, } @@ -28,29 +30,29 @@ pub const SCHEME_HTTP: &Scheme = Scheme::new_or_panic("http"); pub const SCHEME_HTTPS: &Scheme = Scheme::new_or_panic("https"); #[derive(Debug)] -pub enum HTTPProjectAsync { - HTTPSrcProject(ReqwestSrcProjectAsync), +pub enum HTTPProjectAsync { + HTTPSrcProject(ReqwestSrcProjectAsync), // HTTPKParProjectRanged(ReqwestKparRangedProject), - HTTPKParProjectDownloaded(ReqwestKparDownloadedProject), + HTTPKParProjectDownloaded(ReqwestKparDownloadedProject), } #[derive(Error, Debug)] -pub enum HTTPProjectError { +pub enum HTTPProjectError { #[error(transparent)] - SrcProject(::Error), + SrcProject( as ProjectReadAsync>::Error), // #[error(transparent)] // KParRanged(::Error), #[error(transparent)] - KparDownloaded(::Error), + KparDownloaded( as ProjectReadAsync>::Error), } -pub enum HTTPProjectAsyncReader<'a> { - SrcProjectReader(::SourceReader<'a>), +pub enum HTTPProjectAsyncReader<'a, Pol: HTTPAuthentication + 'a> { + SrcProjectReader( as ProjectReadAsync>::SourceReader<'a>), //KParRangedReader(::SourceReader<'a>), - KparDownloadedReader(::SourceReader<'a>), + KparDownloadedReader( as ProjectReadAsync>::SourceReader<'a>), } -impl AsyncRead for HTTPProjectAsyncReader<'_> { +impl AsyncRead for HTTPProjectAsyncReader<'_, Pol> { fn poll_read( self: Pin<&mut Self>, cx: &mut std::task::Context<'_>, @@ -64,8 +66,10 @@ impl AsyncRead for HTTPProjectAsyncReader<'_> { } } -impl ProjectReadAsync for HTTPProjectAsync { - type Error = HTTPProjectError; +impl ProjectReadAsync + for HTTPProjectAsync +{ + type Error = HTTPProjectError; async fn get_project_async( &self, @@ -92,7 +96,7 @@ impl ProjectReadAsync for HTTPProjectAsync { } type SourceReader<'a> - = HTTPProjectAsyncReader<'a> + = HTTPProjectAsyncReader<'a, Pol> where Self: 'a; @@ -135,18 +139,19 @@ impl ProjectReadAsync for HTTPProjectAsync { } } -pub struct HTTPProjects { +pub struct HTTPProjects { client: reqwest_middleware::ClientWithMiddleware, url: reqwest::Url, src_done: bool, kpar_done: bool, // See the comments in `try_resolve_as_src`. lax: bool, + auth_policy: Arc, //prefer_ranged: bool, } -impl HTTPProjects { - pub fn try_resolve_as_kpar(&self) -> Option { +impl HTTPProjects { + pub fn try_resolve_as_kpar(&self) -> Option> { // TODO: Decide a policy for KPar vs Src urls let url = if self.url.path() == "" || !self.url.path().ends_with("/") { self.url.clone() @@ -172,12 +177,16 @@ impl HTTPProjects { // } Some(HTTPProjectAsync::HTTPKParProjectDownloaded( - ReqwestKparDownloadedProject::new_guess_root(&url, self.client.clone()) - .expect("internal IO error"), + ReqwestKparDownloadedProject::new_guess_root( + &url, + self.client.clone(), + self.auth_policy.clone(), + ) + .expect("internal IO error"), )) } - pub fn try_resolve_as_src(&self) -> Option { + pub fn try_resolve_as_src(&self, auth_policy: Arc) -> Option> { // These URLs should technically have a path that ends (explicitly or implicitly) // with a slash, due to the way relative references are treated in HTTP. E.g.: // resolving `bar` relative to `http://www.example.com/foo` gives `http://www.example.com/bar` @@ -186,6 +195,7 @@ impl HTTPProjects { Some(HTTPProjectAsync::HTTPSrcProject(ReqwestSrcProjectAsync { client: self.client.clone(), // Already internally an Rc url: self.url.clone(), + auth_policy: auth_policy.clone(), })) // If the resolver is set to be lax, try forcing the terminal slash } else if self.lax { @@ -197,6 +207,7 @@ impl HTTPProjects { Some(HTTPProjectAsync::HTTPSrcProject(ReqwestSrcProjectAsync { client: self.client.clone(), // Already internally an Rc url: lax_url, + auth_policy, })) } else { None @@ -204,14 +215,14 @@ impl HTTPProjects { } } -impl Iterator for HTTPProjects { - type Item = Result; +impl Iterator for HTTPProjects { + type Item = Result, Infallible>; fn next(&mut self) -> Option { if !self.src_done { self.src_done = true; - if let Some(proj) = self.try_resolve_as_src() { + if let Some(proj) = self.try_resolve_as_src(self.auth_policy.clone()) { return Some(Ok(proj)); } } @@ -235,12 +246,14 @@ impl Iterator for HTTPProjects { /// appears to support HTTP Range requests. If successful, it uses `HTTPKparProjectRanged` /// instead of `HTTPKparProjectDownloaded`. In case of *any* failure, or if `prefer_ranged` /// is false, `HTTPKparProjectDownloaded` is used instead. -impl ResolveReadAsync for HTTPResolverAsync { +impl ResolveReadAsync + for HTTPResolverAsync +{ type Error = Infallible; - type ProjectStorage = HTTPProjectAsync; + type ProjectStorage = HTTPProjectAsync; - type ResolvedStorages = futures::stream::Iter; + type ResolvedStorages = futures::stream::Iter>; async fn resolve_read_async( &self, @@ -256,6 +269,7 @@ impl ResolveReadAsync for HTTPResolverAsync { src_done: false, kpar_done: false, lax: self.lax, + auth_policy: self.auth_policy.clone(), // prefer_ranged: self.prefer_ranged, })) } else { @@ -275,6 +289,7 @@ mod tests { use std::sync::Arc; use crate::{ + auth::Unauthenticated, project::ProjectRead, resolve::{ResolutionOutcome, ResolveRead, ResolveReadAsync}, }; @@ -304,7 +319,7 @@ mod tests { let resolver = super::HTTPResolverAsync { client, lax: false, - //prefer_ranged: true, + auth_policy: Arc::new(Unauthenticated {}), //prefer_ranged: true, } .to_tokio_sync(Arc::new( tokio::runtime::Builder::new_current_thread() @@ -345,7 +360,7 @@ mod tests { let resolver = super::HTTPResolverAsync { client, lax: true, - //prefer_ranged, + auth_policy: Arc::new(Unauthenticated {}), //prefer_ranged, } .to_tokio_sync(Arc::new( tokio::runtime::Builder::new_current_thread() @@ -363,7 +378,7 @@ mod tests { let ResolutionOutcome::Resolved(projects) = resolver.resolve_read_raw(url)? else { panic!() }; - let projects: Vec = + let projects: Vec> = projects.into_iter().map(|x| x.unwrap().inner).collect(); assert_eq!(projects.len(), 2); diff --git a/core/src/resolve/standard.rs b/core/src/resolve/standard.rs index c90bfe22..91a557f0 100644 --- a/core/src/resolve/standard.rs +++ b/core/src/resolve/standard.rs @@ -4,6 +4,7 @@ use std::{fmt, path::PathBuf, result::Result, sync::Arc}; use crate::{ + auth::HTTPAuthentication, env::{local_directory::LocalDirectoryEnvironment, reqwest_http::HTTPEnvironmentAsync}, resolve::{ AsSyncResolveTokio, ResolveRead, ResolveReadAsync, @@ -20,29 +21,29 @@ use reqwest_middleware::ClientWithMiddleware; pub type LocalEnvResolver = EnvResolver; -pub type RemoteIndexResolver = SequentialResolver>; +pub type RemoteIndexResolver = SequentialResolver>>; -type StandardResolverInner = CombinedResolver< +type StandardResolverInner = CombinedResolver< FileResolver, LocalEnvResolver, - RemoteResolver, GitResolver>, - AsSyncResolveTokio, + RemoteResolver>, GitResolver>, + AsSyncResolveTokio>, >; -pub struct StandardResolver(StandardResolverInner); +pub struct StandardResolver(StandardResolverInner); -impl fmt::Debug for StandardResolver { +impl fmt::Debug for StandardResolver { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("CliResolver").field(&self.0).finish() } } -impl ResolveRead for StandardResolver { - type Error = ::Error; +impl ResolveRead for StandardResolver { + type Error = as ResolveRead>::Error; - type ProjectStorage = ::ProjectStorage; + type ProjectStorage = as ResolveRead>::ProjectStorage; - type ResolvedStorages = ::ResolvedStorages; + type ResolvedStorages = as ResolveRead>::ResolvedStorages; fn resolve_read( &self, @@ -59,16 +60,17 @@ pub fn standard_file_resolver(cwd: Option) -> FileResolver { } } -pub fn standard_remote_resolver( +pub fn standard_remote_resolver( client: ClientWithMiddleware, runtime: Arc, -) -> RemoteResolver, GitResolver> { + auth_policy: Arc, +) -> RemoteResolver>, GitResolver> { RemoteResolver { http_resolver: Some( HTTPResolverAsync { client, lax: true, - //prefer_ranged: true, + auth_policy, //prefer_ranged: true, } .to_tokio_sync(runtime), ), @@ -85,16 +87,18 @@ pub fn standard_local_resolver(local_env_path: PathBuf) -> LocalEnvResolver { } } -pub fn standard_index_resolver( +pub fn standard_index_resolver( client: ClientWithMiddleware, urls: Vec, runtime: Arc, -) -> AsSyncResolveTokio { + auth_policy: Arc, +) -> AsSyncResolveTokio> { SequentialResolver::new(urls.into_iter().map(|url| EnvResolver { env: HTTPEnvironmentAsync { client: client.clone(), base_url: url.clone(), prefer_src: true, + auth_policy: auth_policy.clone(), //try_ranged: true, }, })) @@ -102,21 +106,22 @@ pub fn standard_index_resolver( } // TODO: Replace most of these arguments by some general CLIOptions object -pub fn standard_resolver( +pub fn standard_resolver( cwd: Option, local_env_path: Option, client: Option, index_urls: Option>, runtime: Arc, -) -> StandardResolver { + auth_policy: Arc, +) -> StandardResolver { let file_resolver = standard_file_resolver(cwd); let remote_resolver = client .clone() - .map(|x| standard_remote_resolver(x, runtime.clone())); + .map(|x| standard_remote_resolver(x, runtime.clone(), auth_policy.clone())); let local_resolver = local_env_path.map(standard_local_resolver); let index_resolver = client .zip(index_urls) - .map(|(client, urls)| standard_index_resolver(client, urls, runtime)); + .map(|(client, urls)| standard_index_resolver(client, urls, runtime, auth_policy)); StandardResolver(CombinedResolver { file_resolver: Some(file_resolver), diff --git a/sysand/scripts/run_tests.sh b/sysand/scripts/run_tests.sh index a15c39bf..b31ad9e1 100755 --- a/sysand/scripts/run_tests.sh +++ b/sysand/scripts/run_tests.sh @@ -8,4 +8,4 @@ PACKAGE_DIR=$(dirname "$SCRIPT_DIR") cd "$PACKAGE_DIR" -cargo test --features alltests +cargo test --features alltests $@ diff --git a/sysand/src/commands/add.rs b/sysand/src/commands/add.rs index fa35e94c..4d7c8ddb 100644 --- a/sysand/src/commands/add.rs +++ b/sysand/src/commands/add.rs @@ -7,6 +7,7 @@ use anyhow::Result; use sysand_core::{ add::do_add, + auth::HTTPAuthentication, config::Config, lock::Lock, project::{local_src::LocalSrcProject, utils::wrapfs}, @@ -16,7 +17,7 @@ use crate::{CliError, cli::ResolutionOptions, command_sync}; // TODO: Collect common arguments #[allow(clippy::too_many_arguments)] -pub fn command_add>( +pub fn command_add, Pol: HTTPAuthentication + std::fmt::Debug + 'static>( iri: S, versions_constraint: Option, no_lock: bool, @@ -26,6 +27,7 @@ pub fn command_add>( current_project: Option, client: reqwest_middleware::ClientWithMiddleware, runtime: Arc, + auth_policy: Arc, ) -> Result<()> { let mut current_project = current_project.ok_or(CliError::MissingProjectCurrentDir)?; let project_root = current_project.root_path(); @@ -50,6 +52,7 @@ pub fn command_add>( config, client.clone(), runtime.clone(), + auth_policy.clone(), )?; if !no_sync { @@ -64,6 +67,7 @@ pub fn command_add>( client, &provided_iris, runtime, + auth_policy, )?; } } diff --git a/sysand/src/commands/clone.rs b/sysand/src/commands/clone.rs index 6e78957e..5ca77d88 100644 --- a/sysand/src/commands/clone.rs +++ b/sysand/src/commands/clone.rs @@ -12,6 +12,7 @@ use std::{ }; use sysand_core::{ + auth::HTTPAuthentication, commands::lock::{DEFAULT_LOCKFILE_NAME, LockOutcome}, config::Config, discover::discover_project, @@ -39,7 +40,7 @@ pub enum ProjectLocator { /// Clones project from `locator` to `target` directory. #[allow(clippy::too_many_arguments)] -pub fn command_clone( +pub fn command_clone( locator: ProjectLocatorArgs, version: Option, target: Option, @@ -48,6 +49,7 @@ pub fn command_clone( config: &Config, client: reqwest_middleware::ClientWithMiddleware, runtime: Arc, + auth_policy: Arc, ) -> Result<()> { let ResolutionOptions { index, @@ -129,6 +131,7 @@ pub fn command_clone( Some(client.clone()), index_urls, runtime.clone(), + auth_policy.clone(), ); match locator { ProjectLocator::Iri(iri) => { @@ -226,6 +229,7 @@ pub fn command_clone( client, &provided_iris, runtime, + auth_policy, )?; } diff --git a/sysand/src/commands/env.rs b/sysand/src/commands/env.rs index 68f5d767..547a1361 100644 --- a/sysand/src/commands/env.rs +++ b/sysand/src/commands/env.rs @@ -12,6 +12,7 @@ use anyhow::{Result, anyhow, bail}; use fluent_uri::Iri; use sysand_core::{ + auth::HTTPAuthentication, commands::{env::do_env_local_dir, lock::LockOutcome}, config::Config, env::local_directory::LocalDirectoryEnvironment, @@ -41,7 +42,7 @@ pub fn command_env>(path: P) -> Result // TODO: Factor out provided_iris logic #[allow(clippy::too_many_arguments)] -pub fn command_env_install( +pub fn command_env_install( iri: Iri, version: Option, install_opts: InstallOptions, @@ -50,6 +51,7 @@ pub fn command_env_install( project_root: Option, client: reqwest_middleware::ClientWithMiddleware, runtime: Arc, + auth_policy: Arc, ) -> Result<()> { let project_root = project_root.unwrap_or(wrapfs::current_dir()?); let mut env = crate::get_or_create_env(project_root.as_path())?; @@ -100,6 +102,7 @@ pub fn command_env_install( Some(client.clone()), index_urls, runtime.clone(), + auth_policy.clone(), ), ); @@ -143,6 +146,7 @@ pub fn command_env_install( client, &provided_iris, runtime, + auth_policy, )?; } @@ -151,7 +155,10 @@ pub fn command_env_install( // TODO: Collect common arguments #[allow(clippy::too_many_arguments)] -pub fn command_env_install_path>( +pub fn command_env_install_path< + S: AsRef, + Pol: HTTPAuthentication + std::fmt::Debug + 'static, +>( iri: S, version: Option, path: String, @@ -161,6 +168,7 @@ pub fn command_env_install_path>( project_root: Option, client: reqwest_middleware::ClientWithMiddleware, runtime: Arc, + auth_policy: Arc, ) -> Result<()> { let project_root = project_root.unwrap_or(wrapfs::current_dir()?); let mut env = crate::get_or_create_env(project_root.as_path())?; @@ -243,6 +251,7 @@ pub fn command_env_install_path>( Some(client.clone()), index_urls, runtime.clone(), + auth_policy.clone(), ), ); let LockOutcome { @@ -256,6 +265,7 @@ pub fn command_env_install_path>( client, &provided_iris, runtime, + auth_policy, )?; } diff --git a/sysand/src/commands/info.rs b/sysand/src/commands/info.rs index 81199b5d..bdbf7978 100644 --- a/sysand/src/commands/info.rs +++ b/sysand/src/commands/info.rs @@ -9,6 +9,7 @@ use crate::{ }, }; use sysand_core::{ + auth::HTTPAuthentication, model::{ InterchangeProjectChecksumRaw, InterchangeProjectInfoRaw, InterchangeProjectMetadataRaw, }, @@ -96,13 +97,14 @@ pub fn command_info_path>(path: P, excluded_iris: &HashSet( uri: Iri, _normalise: bool, client: reqwest_middleware::ClientWithMiddleware, index_urls: Option>, excluded_iris: &HashSet, runtime: Arc, + auth_policy: Arc, ) -> Result<()> { let cwd = wrapfs::current_dir().ok(); @@ -118,6 +120,7 @@ pub fn command_info_uri( Some(client), index_urls, runtime, + auth_policy, ); let mut found = false; @@ -183,13 +186,14 @@ pub fn command_info_verb_path>( } } -pub fn command_info_verb_uri( +pub fn command_info_verb_uri( uri: Iri, verb: InfoCommandVerb, numbered: bool, client: reqwest_middleware::ClientWithMiddleware, index_urls: Option>, runtime: Arc, + auth_policy: Arc, ) -> Result<()> { match verb { InfoCommandVerb::Get(get_verb) => { @@ -208,6 +212,7 @@ pub fn command_info_verb_uri( Some(client), index_urls, runtime, + auth_policy, ); let mut found = false; diff --git a/sysand/src/commands/lock.rs b/sysand/src/commands/lock.rs index dff78128..b0e4a5df 100644 --- a/sysand/src/commands/lock.rs +++ b/sysand/src/commands/lock.rs @@ -9,6 +9,7 @@ use anyhow::{Result, bail}; use pubgrub::Reporter as _; use sysand_core::{ + auth::HTTPAuthentication, commands::lock::{ DEFAULT_LOCKFILE_NAME, LockError, LockOutcome, LockProjectError, do_lock_local_editable, }, @@ -29,12 +30,13 @@ use crate::{DEFAULT_INDEX_URL, cli::ResolutionOptions}; /// `path` must be relative to workspace root. // TODO: this will not work properly if run in subdir of workspace, // as `path` will then refer to a deeper subdir -pub fn command_lock>( +pub fn command_lock, Pol: HTTPAuthentication + std::fmt::Debug + 'static>( path: P, resolution_opts: ResolutionOptions, config: &Config, client: reqwest_middleware::ClientWithMiddleware, runtime: Arc, + auth_policy: Arc, ) -> Result<()> { assert!(path.as_ref().is_relative(), "{}", path.as_ref().display()); let ResolutionOptions { @@ -83,6 +85,7 @@ pub fn command_lock>( Some(client), index_urls, runtime, + auth_policy, ), ); diff --git a/sysand/src/commands/sync.rs b/sysand/src/commands/sync.rs index 8f795a56..ce90a957 100644 --- a/sysand/src/commands/sync.rs +++ b/sysand/src/commands/sync.rs @@ -7,6 +7,7 @@ use anyhow::Result; use url::ParseError; use sysand_core::{ + auth::HTTPAuthentication, env::local_directory::LocalDirectoryEnvironment, lock::Lock, project::{ @@ -16,13 +17,14 @@ use sysand_core::{ }, }; -pub fn command_sync>( +pub fn command_sync, Pol: HTTPAuthentication>( lock: &Lock, project_root: P, env: &mut LocalDirectoryEnvironment, client: reqwest_middleware::ClientWithMiddleware, provided_iris: &HashMap>, runtime: Arc, + auth_policy: Arc, ) -> Result<()> { sysand_core::commands::sync::do_sync( lock, @@ -31,10 +33,11 @@ pub fn command_sync>( project_path: project_root.as_ref().join(src_path), }), Some( - |remote_src: String| -> Result, ParseError> { + |remote_src: String| -> Result>, ParseError> { Ok(ReqwestSrcProjectAsync { client: client.clone(), url: reqwest::Url::parse(&remote_src)?, + auth_policy: auth_policy.clone() } .to_tokio_sync(runtime.clone())) }, @@ -42,11 +45,11 @@ pub fn command_sync>( // TODO: Fix error handling here Some(|kpar_path: String| LocalKParProject::new_guess_root(kpar_path).unwrap()), Some( - |remote_kpar: String| -> Result, ParseError> { + |remote_kpar: String| -> Result>, ParseError> { Ok( ReqwestKparDownloadedProject::new_guess_root(reqwest::Url::parse( &remote_kpar, - )?, client.clone()) + )?, client.clone(), auth_policy.clone()) .unwrap().to_tokio_sync(runtime.clone()), ) }, diff --git a/sysand/src/lib.rs b/sysand/src/lib.rs index 927413cb..26cae74c 100644 --- a/sysand/src/lib.rs +++ b/sysand/src/lib.rs @@ -14,6 +14,7 @@ use std::{ use anyhow::{Result, bail}; use sysand_core::{ + auth::StandardHTTPAuthenticationBuilder, config::{ Config, local_fs::{get_config, load_configs}, @@ -101,6 +102,56 @@ pub fn run_cli(args: cli::Args) -> Result<()> { let _runtime_keepalive = runtime.clone(); + // FIXME: This is a temporary implementation to provide credentials until + // https://github.com/sensmetry/sysand/pull/157 + // gets merged. + let mut basic_auth_patterns = HashMap::new(); + let mut basic_auth_users = HashMap::new(); + let mut basic_auth_passwords = HashMap::new(); + + for (key, value) in std::env::vars() { + if let Some(key_rest) = key.strip_prefix("SYSAND_CRED_") { + if let Some(key_name) = key_rest.strip_suffix("_BASIC_USER") { + basic_auth_users.insert(key_name.to_owned(), value); + } else if let Some(key_name) = key_rest.strip_suffix("_BASIC_PASS") { + basic_auth_passwords.insert(key_name.to_owned(), value); + } else { + basic_auth_patterns.insert(key_rest.to_owned(), value); + } + } + } + + let mut basic_auth_pattern_names = HashSet::new(); + for x in [ + &basic_auth_patterns, + &basic_auth_users, + &basic_auth_passwords, + ] { + for k in x.keys() { + basic_auth_pattern_names.insert(k); + } + } + + let mut basic_auths_builder: StandardHTTPAuthenticationBuilder = + StandardHTTPAuthenticationBuilder::new(); + for k in basic_auth_pattern_names { + match ( + basic_auth_patterns.get(k), + basic_auth_users.get(k), + basic_auth_passwords.get(k), + ) { + (Some(pattern), Some(username), Some(password)) => { + basic_auths_builder.add_basic_auth(pattern, username, password); + } + _ => { + anyhow::bail!( + "Please specify all of SYSAND_CRED_{k}, SYSAND_CRED_{k}_BASIC_USER, SYSAND_CRED_{k}_BASIC_PASS" + ); + } + } + } + let basic_auth_policy = Arc::new(basic_auths_builder.build()?); + match args.command { cli::Command::Init { path, @@ -139,6 +190,7 @@ pub fn run_cli(args: cli::Args) -> Result<()> { project_root, client, runtime, + basic_auth_policy, ) } else { command_env_install( @@ -150,6 +202,7 @@ pub fn run_cli(args: cli::Args) -> Result<()> { project_root, client, runtime, + basic_auth_policy, ) } } @@ -188,8 +241,15 @@ pub fn run_cli(args: cli::Args) -> Result<()> { }, cli::Command::Lock { resolution_opts } => { if project_root.is_some() { - crate::commands::lock::command_lock(".", resolution_opts, &config, client, runtime) - .map(|_| ()) + crate::commands::lock::command_lock( + ".", + resolution_opts, + &config, + client, + runtime, + basic_auth_policy, + ) + .map(|_| ()) } else { bail!("not inside a project") } @@ -216,6 +276,7 @@ pub fn run_cli(args: cli::Args) -> Result<()> { &config, client.clone(), runtime.clone(), + basic_auth_policy.clone(), )?; } let lock = Lock::from_str(&wrapfs::read_to_string(lockfile)?)?; @@ -226,6 +287,7 @@ pub fn run_cli(args: cli::Args) -> Result<()> { client, &provided_iris, runtime, + basic_auth_policy, ) } cli::Command::PrintRoot => command_print_root(cwd), @@ -362,6 +424,7 @@ pub fn run_cli(args: cli::Args) -> Result<()> { index_urls, &excluded_iris, runtime, + basic_auth_policy, ), (Location::Iri(iri), Some(subcommand)) => { let numbered = subcommand.numbered(); @@ -373,6 +436,7 @@ pub fn run_cli(args: cli::Args) -> Result<()> { client, index_urls, runtime, + basic_auth_policy, ) } (Location::Path(path), None) => command_info_path(Path::new(&path), &excluded_iris), @@ -399,6 +463,7 @@ pub fn run_cli(args: cli::Args) -> Result<()> { current_project, client, runtime, + basic_auth_policy, ), cli::Command::Remove { iri } => command_remove(iri, current_project), cli::Command::Include { @@ -474,6 +539,7 @@ pub fn run_cli(args: cli::Args) -> Result<()> { &config, client, runtime, + basic_auth_policy, ), } } diff --git a/sysand/tests/cfg_base.rs b/sysand/tests/cfg_base.rs index 7dadcd38..b5c6bacd 100644 --- a/sysand/tests/cfg_base.rs +++ b/sysand/tests/cfg_base.rs @@ -32,6 +32,7 @@ fn cfg_set_quiet() -> Result<(), Box> { quiet: Some(true), verbose: None, index: None, + auth: None, })?; let out_quiet_local_config = diff --git a/sysand/tests/cli_info.rs b/sysand/tests/cli_info.rs index 235a137c..54ce9151 100644 --- a/sysand/tests/cli_info.rs +++ b/sysand/tests/cli_info.rs @@ -7,6 +7,8 @@ use std::process::Command; use std::{error::Error, io::Write as _}; use assert_cmd::prelude::*; +use indexmap::IndexMap; +use mockito::Matcher; use predicates::prelude::*; // pub due to https://github.com/rust-lang/rust/issues/46379 @@ -131,13 +133,13 @@ fn info_basic_iri_auto() -> Result<(), Box> { } #[test] -fn info_basic_http_url() -> Result<(), Box> { +fn info_basic_http_url_noauth() -> Result<(), Box> { let mut server = mockito::Server::new(); let git_mock = server .mock("GET", "/info/refs?service=git-upload-pack") .with_status(404) - .expect_at_most(2) // TODO: Reduce this to 1 after caching + .expect_at_most(2) // TODO: Reduce this to 1 .create(); let kpar_range_probe = server @@ -157,6 +159,7 @@ fn info_basic_http_url() -> Result<(), Box> { .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"name":"info_basic_http_url","version":"1.2.3","usage":[]}"#) + .expect_at_most(1) // TODO: Reduce this .create(); let info_mock = server @@ -164,7 +167,7 @@ fn info_basic_http_url() -> Result<(), Box> { .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"name":"info_basic_http_url","version":"1.2.3","usage":[]}"#) - .expect_at_most(3) // TODO: Reduce this to 1 after caching + .expect_at_most(3) // TODO: Reduce this to 1 .create(); let meta_mock_head = server @@ -172,6 +175,7 @@ fn info_basic_http_url() -> Result<(), Box> { .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) + .expect_at_most(1) .create(); let meta_mock = server @@ -179,7 +183,7 @@ fn info_basic_http_url() -> Result<(), Box> { .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) - .expect_at_most(3) // TODO: Reduce this to 1 after caching + .expect_at_most(3) // TODO: Reduce this to 1 .create(); let (_, _, out) = run_sysand(["info", "--iri", &server.url()], None)?; @@ -203,6 +207,146 @@ fn info_basic_http_url() -> Result<(), Box> { Ok(()) } +#[test] +fn info_basic_http_url_auth() -> Result<(), Box> { + let mut server = mockito::Server::new(); + + let git_mock = server + .mock("GET", "/info/refs?service=git-upload-pack") + .match_header("authorization", Matcher::Missing) + .with_status(404) + .expect(2) // TODO: Reduce this to 1 + .create(); + + // let kpar_range_probe = server + // .mock("HEAD", "/") + // .match_header("authorization", Matcher::Missing) + // .with_status(404) + // .expect(1) + // .create(); + + let kpar_download_try = server + .mock("GET", "/") + .match_header("authorization", Matcher::Missing) + .with_status(404) + .expect(1) + .create(); + + let kpar_download_try_auth = server + .mock("GET", "/") + .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .with_status(404) + .expect(1) + .create(); + + let info_mock_head = server + .mock("HEAD", "/.project.json") + .match_header("authorization", Matcher::Missing) + .with_status(404) + .with_header("content-type", "application/json") + .with_body(r#"{"name":"info_basic_http_url","version":"1.2.3","usage":[]}"#) + .expect(1) // TODO: Reduce this + .create(); + + let info_mock_head_auth = server + .mock("HEAD", "/.project.json") + .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"name":"info_basic_http_url","version":"1.2.3","usage":[]}"#) + .expect(1) // TODO: Reduce this + .create(); + + let info_mock = server + .mock("GET", "/.project.json") + .match_header("authorization", Matcher::Missing) + .with_status(404) + .with_header("content-type", "application/json") + .with_body(r#"{"name":"info_basic_http_url","version":"1.2.3","usage":[]}"#) + .expect(3) // TODO: Reduce this to 1 + .create(); + + let info_mock_auth = server + .mock("GET", "/.project.json") + .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"name":"info_basic_http_url","version":"1.2.3","usage":[]}"#) + .expect(3) // TODO: Reduce this to 1 + .create(); + + let meta_mock_head = server + .mock("HEAD", "/.meta.json") + .match_header("authorization", Matcher::Missing) + .with_status(404) + .with_header("content-type", "application/json") + .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) + .expect(1) + .create(); + + let meta_mock_head_auth = server + .mock("HEAD", "/.meta.json") + .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) + .expect(1) + .create(); + + let meta_mock = server + .mock("GET", "/.meta.json") + .with_status(404) + .match_header("authorization", Matcher::Missing) + .with_header("content-type", "application/json") + .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) + .expect(3) // TODO: Reduce this to 1 + .create(); + + let meta_mock_auth = server + .mock("GET", "/.meta.json") + .with_status(200) + .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .with_header("content-type", "application/json") + .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) + .expect(3) // TODO: Reduce this to 1 + .create(); + + let (_, _, out) = run_sysand_with( + ["info", "--iri", &server.url()], + None, + &IndexMap::from([ + ("SYSAND_CRED_TEST".to_string(), "http://127.0.0.1:*/**".to_string()), + ("SYSAND_CRED_TEST_BASIC_USER".to_string(), "user_1234".to_string()), + ("SYSAND_CRED_TEST_BASIC_PASS".to_string(), "pass_4321".to_string()), + ]) + )?; + + out.assert() + .success() + .stdout(predicate::str::contains("Name: info_basic_http_url")) + .stdout(predicate::str::contains("Version: 1.2.3")); + + git_mock.assert(); + + info_mock_head.assert(); + info_mock_head_auth.assert(); + meta_mock_head.assert(); + meta_mock_head_auth.assert(); + + // kpar_range_probe.assert(); + kpar_download_try.assert(); + kpar_download_try_auth.assert(); + + info_mock.assert(); + info_mock_auth.assert(); + + meta_mock.assert(); + meta_mock_auth.assert(); + + + Ok(()) +} + // #[test] // fn info_non_ranged_http_kpar() -> Result<(), Box> { // let buf = { @@ -389,7 +533,7 @@ fn info_basic_index_url() -> Result<(), Box> { .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"name":"info_basic_index_url","version":"1.2.3","usage":[]}"#) - .expect_at_most(2) // TODO: Reduce this to 1 after caching + .expect_at_most(2) // TODO: Reduce this to 1 .create(); let meta_mock = server @@ -397,7 +541,7 @@ fn info_basic_index_url() -> Result<(), Box> { .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) - .expect_at_most(2) // TODO: Reduce this to 1 after caching + .expect_at_most(2) // TODO: Reduce this to 1 .create(); let (_, _, out) = run_sysand( @@ -440,7 +584,7 @@ fn info_basic_index_url() -> Result<(), Box> { } #[test] -fn info_multi_index_url() -> Result<(), Box> { +fn info_multi_index_url_noauth() -> Result<(), Box> { let mut server = mockito::Server::new(); let mut server_alt = mockito::Server::new(); @@ -468,7 +612,7 @@ fn info_multi_index_url() -> Result<(), Box> { .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"name":"info_multi_index_url","version":"1.2.3","usage":[]}"#) - .expect_at_most(2) // TODO: Reduce this to 1 after caching + .expect_at_most(2) // TODO: Reduce this to 1 .create(); let meta_mock = server @@ -476,7 +620,7 @@ fn info_multi_index_url() -> Result<(), Box> { .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) - .expect_at_most(2) // TODO: Reduce this to 1 after caching + .expect_at_most(2) // TODO: Reduce this to 1 .create(); let versions_alt_mock = server_alt @@ -503,7 +647,7 @@ fn info_multi_index_url() -> Result<(), Box> { .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"name":"info_multi_index_url_alt","version":"1.2.3","usage":[]}"#) - .expect_at_most(2) // TODO: Reduce this to 1 after caching + .expect_at_most(2) // TODO: Reduce this to 1 .create(); let meta_alt_mock = server_alt @@ -511,7 +655,7 @@ fn info_multi_index_url() -> Result<(), Box> { .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) - .expect_at_most(2) // TODO: Reduce this to 1 after caching + .expect_at_most(2) // TODO: Reduce this to 1 .create(); let (_, _, out) = run_sysand( @@ -578,6 +722,205 @@ fn info_multi_index_url() -> Result<(), Box> { Ok(()) } +#[test] +fn info_multi_index_url_auth() -> Result<(), Box> { + let mut server = mockito::Server::new(); + let mut server_alt = mockito::Server::new(); + + let versions_mock = server + .mock( + "GET", + "/f38ace6666fe279c9e856b2a25b14bf0a03b8c23ff1db524acf1afd78f66b042/versions.txt", + ) + .match_header("authorization", Matcher::Missing) + .with_status(404) + .with_header("content-type", "text/plain") + .with_body("1.2.3\n") + .expect(1) + .create(); + + let versions_mock_auth = server + .mock( + "GET", + "/f38ace6666fe279c9e856b2a25b14bf0a03b8c23ff1db524acf1afd78f66b042/versions.txt", + ) + .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .with_status(200) + .with_header("content-type", "text/plain") + .with_body("1.2.3\n") + .expect(1) + .create(); + + let project_mock_head = server + .mock("HEAD", "/f38ace6666fe279c9e856b2a25b14bf0a03b8c23ff1db524acf1afd78f66b042/1.2.3.kpar/.project.json") + .match_header("authorization", Matcher::Missing) + .with_status(404) + .with_header("content-type", "application/json") + .with_body(r#"{"name":"info_multi_index_url","version":"1.2.3","usage":[]}"#) + .expect(1) + .create(); + + let project_mock_head_auth = server + .mock("HEAD", "/f38ace6666fe279c9e856b2a25b14bf0a03b8c23ff1db524acf1afd78f66b042/1.2.3.kpar/.project.json") + .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"name":"info_multi_index_url","version":"1.2.3","usage":[]}"#) + .expect(1) + .create(); + + let project_mock = server + .mock("GET", "/f38ace6666fe279c9e856b2a25b14bf0a03b8c23ff1db524acf1afd78f66b042/1.2.3.kpar/.project.json") + .match_header("authorization", Matcher::Missing) + .with_status(404) + .with_header("content-type", "application/json") + .with_body(r#"{"name":"info_multi_index_url","version":"1.2.3","usage":[]}"#) + .expect(2) // TODO: Reduce this to 1 + .create(); + + let project_mock_auth = server + .mock("GET", "/f38ace6666fe279c9e856b2a25b14bf0a03b8c23ff1db524acf1afd78f66b042/1.2.3.kpar/.project.json") + .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"name":"info_multi_index_url","version":"1.2.3","usage":[]}"#) + .expect(2) // TODO: Reduce this to 1 + .create(); + + let meta_mock = server + .mock("GET", "/f38ace6666fe279c9e856b2a25b14bf0a03b8c23ff1db524acf1afd78f66b042/1.2.3.kpar/.meta.json") + .match_header("authorization", Matcher::Missing) + .with_status(404) + .with_header("content-type", "application/json") + .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) + .expect(2) // TODO: Reduce this to 1 + .create(); + + let meta_mock_auth = server + .mock("GET", "/f38ace6666fe279c9e856b2a25b14bf0a03b8c23ff1db524acf1afd78f66b042/1.2.3.kpar/.meta.json") + .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) + .expect(2) // TODO: Reduce this to 1 + .create(); + + let versions_alt_mock = server_alt + .mock( + "GET", + "/f0f4203b967855590901dc5c90f525d732015ca10598e333815cc30600874565/versions.txt", + ) + .match_header("authorization", Matcher::Missing) + .with_status(200) + .with_header("content-type", "text/plain") + .with_body("1.2.3\n") + .expect(1) + .create(); + + let project_alt_mock_head = server_alt + .mock("HEAD", "/f0f4203b967855590901dc5c90f525d732015ca10598e333815cc30600874565/1.2.3.kpar/.project.json") + .match_header("authorization", Matcher::Missing) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"name":"info_multi_index_url_alt","version":"1.2.3","usage":[]}"#) + .expect(1) + .create(); + + let project_alt_mock = server_alt + .mock("GET", "/f0f4203b967855590901dc5c90f525d732015ca10598e333815cc30600874565/1.2.3.kpar/.project.json") + .match_header("authorization", Matcher::Missing) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"name":"info_multi_index_url_alt","version":"1.2.3","usage":[]}"#) + .expect(2) // TODO: Reduce this to 1 + .create(); + + let meta_alt_mock = server_alt + .mock("GET", "/f0f4203b967855590901dc5c90f525d732015ca10598e333815cc30600874565/1.2.3.kpar/.meta.json") + .match_header("authorization", Matcher::Missing) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) + .expect(2) // TODO: Reduce this to 1 + .create(); + + let auth_env = IndexMap::from([ + ("SYSAND_CRED_TEST".to_string(), format!("http://{}/**", server.host_with_port())), + ("SYSAND_CRED_TEST_BASIC_USER".to_string(), "user_1234".to_string()), + ("SYSAND_CRED_TEST_BASIC_PASS".to_string(), "pass_4321".to_string()), + ]); + + let (_, _, out) = run_sysand_with( + [ + "info", + "--iri", + "urn:kpar:info_multi_index_url", + "--index", + &server.url(), + "--default-index", + &server_alt.url(), + ], + None, + &auth_env + )?; + + versions_mock.assert(); + versions_mock_auth.assert(); + project_mock_head.assert(); + project_mock_head_auth.assert(); + project_mock.assert(); + project_mock_auth.assert(); + meta_mock.assert(); + meta_mock_auth.assert(); + + out.assert() + .success() + .stdout(predicate::str::contains("Name: info_multi_index_url")) + .stdout(predicate::str::contains("Version: 1.2.3")); + + let (_, _, out) = run_sysand_with( + [ + "info", + "--iri", + "urn:kpar:info_multi_index_url_alt", + "--index", + &server.url(), + "--default-index", + &server_alt.url(), + ], + None, + &auth_env, + )?; + + out.assert() + .success() + .stdout(predicate::str::contains("Name: info_multi_index_url_alt")) + .stdout(predicate::str::contains("Version: 1.2.3")); + + versions_alt_mock.assert(); + project_alt_mock_head.assert(); + project_alt_mock.assert(); + meta_alt_mock.assert(); + + let (_, _, out) = run_sysand_with( + [ + "info", + "--iri", + "urn:kpar:other", + "--default-index", + &server.url(), + ], + None, + &auth_env, + )?; + + out.assert().failure().stderr(predicate::str::contains( + "unable to find interchange project 'urn:kpar:other'", + )); + + Ok(()) +} + #[test] fn info_multi_index_url_config() -> Result<(), Box> { let mut server = mockito::Server::new(); @@ -607,7 +950,7 @@ fn info_multi_index_url_config() -> Result<(), Box> { .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"name":"info_multi_index_url_config","version":"1.2.3","usage":[]}"#) - .expect_at_most(2) // TODO: Reduce this to 1 after caching + .expect_at_most(2) // TODO: Reduce this to 1 .create(); let meta_mock = server @@ -615,7 +958,7 @@ fn info_multi_index_url_config() -> Result<(), Box> { .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) - .expect_at_most(2) // TODO: Reduce this to 1 after caching + .expect_at_most(2) // TODO: Reduce this to 1 .create(); let versions_alt_mock = server_alt @@ -642,7 +985,7 @@ fn info_multi_index_url_config() -> Result<(), Box> { .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"name":"info_multi_index_url_config_alt","version":"1.2.3","usage":[]}"#) - .expect_at_most(2) // TODO: Reduce this to 1 after caching + .expect_at_most(2) // TODO: Reduce this to 1 .create(); let meta_alt_mock = server_alt @@ -650,7 +993,7 @@ fn info_multi_index_url_config() -> Result<(), Box> { .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) - .expect_at_most(2) // TODO: Reduce this to 1 after caching + .expect_at_most(2) // TODO: Reduce this to 1 .create(); let cfg = format!( @@ -908,6 +1251,8 @@ fn info_detailed_verbs() -> Result<(), Box> { #[test] fn info_set_metamodel() -> Result<(), Box> { + let _ = env_logger::try_init(); + let (_tmp, cwd, out) = run_sysand( ["init", "info_custom_metamodel", "--version", "1.2.3"], None, diff --git a/sysand/tests/cli_sync.rs b/sysand/tests/cli_sync.rs index be25b944..44d158c4 100644 --- a/sysand/tests/cli_sync.rs +++ b/sysand/tests/cli_sync.rs @@ -4,6 +4,8 @@ use std::io::Write; use assert_cmd::prelude::*; +use indexmap::IndexMap; +use mockito::Matcher; use predicates::prelude::*; use sysand_core::commands::lock::DEFAULT_LOCKFILE_NAME; @@ -147,3 +149,176 @@ sources = [ Ok(()) } + +#[test] +fn sync_to_remote_auth() -> Result<(), Box> { + let (_temp_dir, cwd) = new_temp_cwd()?; + + let mut server = mockito::Server::new(); + + let info_mock = server + .mock("GET", "/.project.json") + .match_header("authorization", Matcher::Missing) + .with_status(404) + .with_header("content-type", "application/json") + .with_body(r#"{"name":"sync_to_remote","version":"1.2.3","usage":[]}"#) + .expect(4) // TODO: Reduce this to 1 + .create(); + + let info_mock_auth = server + .mock("GET", "/.project.json") + .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"name":"sync_to_remote","version":"1.2.3","usage":[]}"#) + .expect(4) // TODO: Reduce this to 1 + .create(); + + let meta_mock = server + .mock("GET", "/.meta.json") + .match_header("authorization", Matcher::Missing) + .with_status(404) + .with_header("content-type", "application/json") + .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) + .expect(4) // TODO: Reduce this to 1 + .create(); + + let meta_mock_auth = server + .mock("GET", "/.meta.json") + .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) + .expect(4) // TODO: Reduce this to 1 + .create(); + + let mut lockfile = std::fs::File::create_new(cwd.join(DEFAULT_LOCKFILE_NAME))?; + + lockfile.write_all( + format!( + r#"lock_version = "0.2" + +[[project]] +name = "sync_to_remote" +version = "1.2.3" +identifiers = ["urn:kpar:sync_to_remote"] +checksum = "39f49107a084ab27624ee78d4d37f87a1f7606a2b5d242cdcd9374cf20ab1895" +sources = [ + {{ remote_src = "{}" }}, +] +"#, + &server.url() + ) + .as_bytes(), + )?; + + let out = run_sysand_in_with(&cwd, ["sync"], None, &IndexMap::from([ + ("SYSAND_CRED_TEST".to_string(), format!("http://{}/**", server.host_with_port())), + ("SYSAND_CRED_TEST_BASIC_USER".to_string(), "user_1234".to_string()), + ("SYSAND_CRED_TEST_BASIC_PASS".to_string(), "pass_4321".to_string()), + ]))?; + + info_mock.assert(); + info_mock_auth.assert(); + meta_mock.assert(); + meta_mock_auth.assert(); + + out.assert() + .success() + .stderr(predicate::str::contains("Creating")) + .stderr(predicate::str::contains("Syncing")) + .stderr(predicate::str::contains("Installing")); + + let out = run_sysand_in(&cwd, ["env", "list"], None)?; + + out.assert() + .success() + .stdout(predicate::str::contains("`urn:kpar:sync_to_remote` 1.2.3")); + + let out = run_sysand_in(&cwd, ["sync"], None)?; + + out.assert() + .success() + .stderr(predicate::str::contains("env is already up to date")); + + Ok(()) +} + +#[test] +fn sync_to_remote_incorrect_auth() -> Result<(), Box> { + let (_temp_dir, cwd) = new_temp_cwd()?; + + let mut server = mockito::Server::new(); + + let info_mock = server + .mock("GET", "/.project.json") + .match_header("authorization", Matcher::Missing) + .with_status(404) + .with_header("content-type", "application/json") + .with_body(r#"{"name":"sync_to_remote","version":"1.2.3","usage":[]}"#) + .expect(2) // TODO: Reduce this to 1 + .create(); + + let info_mock_auth = server + .mock("GET", "/.project.json") + .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"name":"sync_to_remote","version":"1.2.3","usage":[]}"#) + .expect(0) // TODO: Reduce this to 1 + .create(); + + let meta_mock = server + .mock("GET", "/.meta.json") + .match_header("authorization", Matcher::Missing) + .with_status(404) + .with_header("content-type", "application/json") + .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) + .expect(2) // TODO: Reduce this to 1 + .create(); + + let meta_mock_auth = server + .mock("GET", "/.meta.json") + .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) + .expect(0) // TODO: Reduce this to 1 + .create(); + + let mut lockfile = std::fs::File::create_new(cwd.join(DEFAULT_LOCKFILE_NAME))?; + + lockfile.write_all( + format!( + r#"lock_version = "0.2" + +[[project]] +name = "sync_to_remote" +version = "1.2.3" +identifiers = ["urn:kpar:sync_to_remote"] +checksum = "39f49107a084ab27624ee78d4d37f87a1f7606a2b5d242cdcd9374cf20ab1895" +sources = [ + {{ remote_src = "{}" }}, +] +"#, + &server.url() + ) + .as_bytes(), + )?; + + let out = run_sysand_in_with(&cwd, ["sync"], None, &IndexMap::from([ + ("SYSAND_CRED_TEST".to_string(),"http://127.0.0.1:80/**".to_string()), + ("SYSAND_CRED_TEST_BASIC_USER".to_string(), "user_1234".to_string()), + ("SYSAND_CRED_TEST_BASIC_PASS".to_string(), "pass_4321".to_string()), + ]))?; + + info_mock.assert(); + info_mock_auth.assert(); + meta_mock.assert(); + meta_mock_auth.assert(); + + out.assert() + .failure(); + + Ok(()) +} diff --git a/sysand/tests/common/mod.rs b/sysand/tests/common/mod.rs index 3742336b..0471bb11 100644 --- a/sysand/tests/common/mod.rs +++ b/sysand/tests/common/mod.rs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: © 2025 Sysand contributors // SPDX-License-Identifier: MIT OR Apache-2.0 +use indexmap::IndexMap; #[cfg(not(target_os = "windows"))] use rexpect::session::{PtySession, spawn_command}; #[cfg(not(target_os = "windows"))] @@ -21,10 +22,11 @@ pub fn fixture_path(name: &str) -> PathBuf { path } -pub fn sysand_cmd_in<'a, I: IntoIterator>( +pub fn sysand_cmd_in_with<'a, I: IntoIterator>( cwd: &Path, args: I, cfg: Option<&str>, + env: &IndexMap ) -> Result> { let cfg_args = if let Some(config) = cfg { let config_path = cwd.join("sysand.toml"); @@ -47,6 +49,7 @@ pub fn sysand_cmd_in<'a, I: IntoIterator>( let mut cmd = Command::new(assert_cmd::cargo::cargo_bin!("sysand")); cmd.env("NO_COLOR", "1"); + cmd.envs(env); cmd.args(args); @@ -55,6 +58,14 @@ pub fn sysand_cmd_in<'a, I: IntoIterator>( Ok(cmd) } +pub fn sysand_cmd_in<'a, I: IntoIterator>( + cwd: &Path, + args: I, + cfg: Option<&str>, +) -> Result> { + sysand_cmd_in_with(cwd, args, cfg, &IndexMap::default()) +} + /// Creates a temporary directory and returns the tuple of the temporary /// directory handle and the canonicalised path to it. We need to canonicalise /// the path because tests check the output of CLI to see whether it operated on @@ -70,14 +81,24 @@ pub fn new_temp_cwd() -> Result<(TempDir, PathBuf), Box> { pub fn sysand_cmd<'a, I: IntoIterator>( args: I, cfg: Option<&str>, + env: &IndexMap ) -> Result<(TempDir, PathBuf, Command), Box> { // NOTE had trouble getting test-temp-dir crate working, but would be better let (temp_dir, cwd) = new_temp_cwd()?; - let cmd = sysand_cmd_in(&cwd, args /*, stdin*/, cfg)?; + let cmd = sysand_cmd_in_with(&cwd, args /*, stdin*/, cfg, env)?; Ok((temp_dir, cwd, cmd)) } +pub fn run_sysand_in_with<'a, I: IntoIterator>( + cwd: &Path, + args: I, + cfg: Option<&str>, + env: &IndexMap, +) -> Result> { + Ok(sysand_cmd_in_with(cwd, args, cfg, env)?.output()?) +} + pub fn run_sysand_in<'a, I: IntoIterator>( cwd: &Path, args: I, @@ -86,15 +107,23 @@ pub fn run_sysand_in<'a, I: IntoIterator>( Ok(sysand_cmd_in(cwd, args, cfg)?.output()?) } -pub fn run_sysand<'a, I: IntoIterator>( +pub fn run_sysand_with<'a, I: IntoIterator>( args: I, cfg: Option<&str>, + env: &IndexMap ) -> Result<(TempDir, PathBuf, Output), Box> { - let (temp_dir, cwd, mut cmd) = sysand_cmd(args /*, stdin*/, cfg)?; + let (temp_dir, cwd, mut cmd) = sysand_cmd(args /*, stdin*/, cfg, env)?; Ok((temp_dir, cwd, cmd.output()?)) } +pub fn run_sysand<'a, I: IntoIterator>( + args: I, + cfg: Option<&str>, +) -> Result<(TempDir, PathBuf, Output), Box> { + run_sysand_with(args, cfg, &IndexMap::default()) +} + // TODO: Figure out how to do interactive tests on Windows. #[cfg(not(target_os = "windows"))] pub fn run_sysand_interactive_in<'a, I: IntoIterator>( @@ -110,16 +139,26 @@ pub fn run_sysand_interactive_in<'a, I: IntoIterator>( // TODO: Figure out how to do interactive tests on Windows. #[cfg(not(target_os = "windows"))] -pub fn run_sysand_interactive<'a, I: IntoIterator>( +pub fn run_sysand_interactive_with<'a, I: IntoIterator>( args: I, timeout_ms: Option, cfg: Option<&str>, + env: &IndexMap, ) -> Result<(TempDir, PathBuf, PtySession), Box> { - let (temp_dir, cwd, cmd) = sysand_cmd(args, cfg)?; + let (temp_dir, cwd, cmd) = sysand_cmd(args, cfg, env)?; Ok((temp_dir, cwd, spawn_command(cmd, timeout_ms)?)) } +#[cfg(not(target_os = "windows"))] +pub fn run_sysand_interactive<'a, I: IntoIterator>( + args: I, + timeout_ms: Option, + cfg: Option<&str>, +) -> Result<(TempDir, PathBuf, PtySession), Box> { + run_sysand_interactive_with(args, timeout_ms, cfg, &IndexMap::default()) +} + // TODO: Figure out how to do interactive tests on Windows. #[cfg(not(target_os = "windows"))] pub fn await_exit(p: PtySession) -> Result> { From 8b882f79f1af35f44f0b62f8613735d43be51d51 Mon Sep 17 00:00:00 2001 From: Tilo Wiklund Date: Mon, 26 Jan 2026 11:32:21 +0100 Subject: [PATCH 03/13] chores Signed-off-by: Tilo Wiklund --- sysand/tests/cli_info.rs | 65 ++++++++++++++++++++++++++-------- sysand/tests/cli_sync.rs | 71 +++++++++++++++++++++++++++++--------- sysand/tests/common/mod.rs | 6 ++-- 3 files changed, 108 insertions(+), 34 deletions(-) diff --git a/sysand/tests/cli_info.rs b/sysand/tests/cli_info.rs index 54ce9151..bed56c89 100644 --- a/sysand/tests/cli_info.rs +++ b/sysand/tests/cli_info.rs @@ -234,7 +234,10 @@ fn info_basic_http_url_auth() -> Result<(), Box> { let kpar_download_try_auth = server .mock("GET", "/") - .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .match_header( + "authorization", + Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string()), + ) .with_status(404) .expect(1) .create(); @@ -250,7 +253,10 @@ fn info_basic_http_url_auth() -> Result<(), Box> { let info_mock_head_auth = server .mock("HEAD", "/.project.json") - .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .match_header( + "authorization", + Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string()), + ) .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"name":"info_basic_http_url","version":"1.2.3","usage":[]}"#) @@ -268,7 +274,10 @@ fn info_basic_http_url_auth() -> Result<(), Box> { let info_mock_auth = server .mock("GET", "/.project.json") - .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .match_header( + "authorization", + Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string()), + ) .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"name":"info_basic_http_url","version":"1.2.3","usage":[]}"#) @@ -286,7 +295,10 @@ fn info_basic_http_url_auth() -> Result<(), Box> { let meta_mock_head_auth = server .mock("HEAD", "/.meta.json") - .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .match_header( + "authorization", + Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string()), + ) .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) @@ -305,7 +317,10 @@ fn info_basic_http_url_auth() -> Result<(), Box> { let meta_mock_auth = server .mock("GET", "/.meta.json") .with_status(200) - .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .match_header( + "authorization", + Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string()), + ) .with_header("content-type", "application/json") .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) .expect(3) // TODO: Reduce this to 1 @@ -315,10 +330,19 @@ fn info_basic_http_url_auth() -> Result<(), Box> { ["info", "--iri", &server.url()], None, &IndexMap::from([ - ("SYSAND_CRED_TEST".to_string(), "http://127.0.0.1:*/**".to_string()), - ("SYSAND_CRED_TEST_BASIC_USER".to_string(), "user_1234".to_string()), - ("SYSAND_CRED_TEST_BASIC_PASS".to_string(), "pass_4321".to_string()), - ]) + ( + "SYSAND_CRED_TEST".to_string(), + "http://127.0.0.1:*/**".to_string(), + ), + ( + "SYSAND_CRED_TEST_BASIC_USER".to_string(), + "user_1234".to_string(), + ), + ( + "SYSAND_CRED_TEST_BASIC_PASS".to_string(), + "pass_4321".to_string(), + ), + ]), )?; out.assert() @@ -343,7 +367,6 @@ fn info_basic_http_url_auth() -> Result<(), Box> { meta_mock.assert(); meta_mock_auth.assert(); - Ok(()) } @@ -744,7 +767,10 @@ fn info_multi_index_url_auth() -> Result<(), Box> { "GET", "/f38ace6666fe279c9e856b2a25b14bf0a03b8c23ff1db524acf1afd78f66b042/versions.txt", ) - .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .match_header( + "authorization", + Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string()), + ) .with_status(200) .with_header("content-type", "text/plain") .with_body("1.2.3\n") @@ -845,9 +871,18 @@ fn info_multi_index_url_auth() -> Result<(), Box> { .create(); let auth_env = IndexMap::from([ - ("SYSAND_CRED_TEST".to_string(), format!("http://{}/**", server.host_with_port())), - ("SYSAND_CRED_TEST_BASIC_USER".to_string(), "user_1234".to_string()), - ("SYSAND_CRED_TEST_BASIC_PASS".to_string(), "pass_4321".to_string()), + ( + "SYSAND_CRED_TEST".to_string(), + format!("http://{}/**", server.host_with_port()), + ), + ( + "SYSAND_CRED_TEST_BASIC_USER".to_string(), + "user_1234".to_string(), + ), + ( + "SYSAND_CRED_TEST_BASIC_PASS".to_string(), + "pass_4321".to_string(), + ), ]); let (_, _, out) = run_sysand_with( @@ -861,7 +896,7 @@ fn info_multi_index_url_auth() -> Result<(), Box> { &server_alt.url(), ], None, - &auth_env + &auth_env, )?; versions_mock.assert(); diff --git a/sysand/tests/cli_sync.rs b/sysand/tests/cli_sync.rs index 44d158c4..946c135b 100644 --- a/sysand/tests/cli_sync.rs +++ b/sysand/tests/cli_sync.rs @@ -167,7 +167,10 @@ fn sync_to_remote_auth() -> Result<(), Box> { let info_mock_auth = server .mock("GET", "/.project.json") - .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .match_header( + "authorization", + Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string()), + ) .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"name":"sync_to_remote","version":"1.2.3","usage":[]}"#) @@ -185,7 +188,10 @@ fn sync_to_remote_auth() -> Result<(), Box> { let meta_mock_auth = server .mock("GET", "/.meta.json") - .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .match_header( + "authorization", + Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string()), + ) .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) @@ -212,11 +218,25 @@ sources = [ .as_bytes(), )?; - let out = run_sysand_in_with(&cwd, ["sync"], None, &IndexMap::from([ - ("SYSAND_CRED_TEST".to_string(), format!("http://{}/**", server.host_with_port())), - ("SYSAND_CRED_TEST_BASIC_USER".to_string(), "user_1234".to_string()), - ("SYSAND_CRED_TEST_BASIC_PASS".to_string(), "pass_4321".to_string()), - ]))?; + let out = run_sysand_in_with( + &cwd, + ["sync"], + None, + &IndexMap::from([ + ( + "SYSAND_CRED_TEST".to_string(), + format!("http://{}/**", server.host_with_port()), + ), + ( + "SYSAND_CRED_TEST_BASIC_USER".to_string(), + "user_1234".to_string(), + ), + ( + "SYSAND_CRED_TEST_BASIC_PASS".to_string(), + "pass_4321".to_string(), + ), + ]), + )?; info_mock.assert(); info_mock_auth.assert(); @@ -261,7 +281,10 @@ fn sync_to_remote_incorrect_auth() -> Result<(), Box> { let info_mock_auth = server .mock("GET", "/.project.json") - .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .match_header( + "authorization", + Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string()), + ) .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"name":"sync_to_remote","version":"1.2.3","usage":[]}"#) @@ -279,7 +302,10 @@ fn sync_to_remote_incorrect_auth() -> Result<(), Box> { let meta_mock_auth = server .mock("GET", "/.meta.json") - .match_header("authorization", Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string())) + .match_header( + "authorization", + Matcher::Exact("Basic dXNlcl8xMjM0OnBhc3NfNDMyMQ==".to_string()), + ) .with_status(200) .with_header("content-type", "application/json") .with_body(r#"{"index":{},"created":"0000-00-00T00:00:00.123456789Z"}"#) @@ -306,19 +332,32 @@ sources = [ .as_bytes(), )?; - let out = run_sysand_in_with(&cwd, ["sync"], None, &IndexMap::from([ - ("SYSAND_CRED_TEST".to_string(),"http://127.0.0.1:80/**".to_string()), - ("SYSAND_CRED_TEST_BASIC_USER".to_string(), "user_1234".to_string()), - ("SYSAND_CRED_TEST_BASIC_PASS".to_string(), "pass_4321".to_string()), - ]))?; + let out = run_sysand_in_with( + &cwd, + ["sync"], + None, + &IndexMap::from([ + ( + "SYSAND_CRED_TEST".to_string(), + "http://127.0.0.1:80/**".to_string(), + ), + ( + "SYSAND_CRED_TEST_BASIC_USER".to_string(), + "user_1234".to_string(), + ), + ( + "SYSAND_CRED_TEST_BASIC_PASS".to_string(), + "pass_4321".to_string(), + ), + ]), + )?; info_mock.assert(); info_mock_auth.assert(); meta_mock.assert(); meta_mock_auth.assert(); - out.assert() - .failure(); + out.assert().failure(); Ok(()) } diff --git a/sysand/tests/common/mod.rs b/sysand/tests/common/mod.rs index 0471bb11..1d574527 100644 --- a/sysand/tests/common/mod.rs +++ b/sysand/tests/common/mod.rs @@ -26,7 +26,7 @@ pub fn sysand_cmd_in_with<'a, I: IntoIterator>( cwd: &Path, args: I, cfg: Option<&str>, - env: &IndexMap + env: &IndexMap, ) -> Result> { let cfg_args = if let Some(config) = cfg { let config_path = cwd.join("sysand.toml"); @@ -81,7 +81,7 @@ pub fn new_temp_cwd() -> Result<(TempDir, PathBuf), Box> { pub fn sysand_cmd<'a, I: IntoIterator>( args: I, cfg: Option<&str>, - env: &IndexMap + env: &IndexMap, ) -> Result<(TempDir, PathBuf, Command), Box> { // NOTE had trouble getting test-temp-dir crate working, but would be better let (temp_dir, cwd) = new_temp_cwd()?; @@ -110,7 +110,7 @@ pub fn run_sysand_in<'a, I: IntoIterator>( pub fn run_sysand_with<'a, I: IntoIterator>( args: I, cfg: Option<&str>, - env: &IndexMap + env: &IndexMap, ) -> Result<(TempDir, PathBuf, Output), Box> { let (temp_dir, cwd, mut cmd) = sysand_cmd(args /*, stdin*/, cfg, env)?; From a47cd07c7bb3bab6ac638b9dd414ed3cbf42a29b Mon Sep 17 00:00:00 2001 From: Tilo Wiklund Date: Wed, 28 Jan 2026 09:55:31 +0100 Subject: [PATCH 04/13] add docs Signed-off-by: Tilo Wiklund --- docs/src/SUMMARY.md | 1 + docs/src/authentication.md | 44 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 docs/src/authentication.md diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 69c33ddd..4d03d27c 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -4,6 +4,7 @@ - [Installation](getting_started/installation.md) - [Tutorial](getting_started/tutorial.md) - [Project metadata](metadata.md) +- [Auhtentication](authentication.md) - [Commands](commands.md) - [sysand init](commands/init.md) - [sysand new](commands/new.md) diff --git a/docs/src/authentication.md b/docs/src/authentication.md new file mode 100644 index 00000000..edb4201e --- /dev/null +++ b/docs/src/authentication.md @@ -0,0 +1,44 @@ +# Authentication + +Project indices and remotely stored project kpars (or sources) may require authentication in order +to get authorised access. Sysand currently supports this for: + +- HTTP(S) using the [basic access authentication scheme](https://en.wikipedia.org/wiki/Basic_access_authentication) + +Support is planned for: + +- HTTP(S) with digest access, (fixed) bearer token, and OAuth2 device authentication +- Git with private-key and basic access authentication + +## Configuring + +At the time of writing authentication can only be configured through environment variables. +Providing credentials is done by setting environment variables following the pattern + +``` +SYSAND_CRED_ = +SYSAND_CRED__BASIC_USER = +SYSAND_CRED__BASIC_PASS = +``` + +Where `` is arbitrary, `` is a wildcard (glob) pattern matching URLs, and +`:` are credentials that may be used with URLs matching the pattern. + +Thus, for example, + +``` +SYSAND_CRED_TEST = "https://*.example.com/**" +SYSAND_CRED_TEST_BASIC_USER = "foo" +SYSAND_CRED_TEST_BASIC_PASS = "bar" +``` + +Would tell Sysand that it *may* use the credentials `foo:bar` with URLs such as + +``` +https://www.example.com/projects/project.kpar +https://projects.example.com/entries.txt +https://projects.example.com/projects/myproject/versions.txt +``` + +In the wildcard pattern, `?` matches any single letter, `*` matches any sequence of characters +not containing `/`, and `**` matches any sequence of characters possibly including `/`. \ No newline at end of file From 4cb25f1f63dade594df71122e5d6a646b09c9f2b Mon Sep 17 00:00:00 2001 From: Tilo Wiklund Date: Wed, 28 Jan 2026 10:19:31 +0100 Subject: [PATCH 05/13] clarify docs Signed-off-by: Tilo Wiklund --- docs/src/authentication.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/src/authentication.md b/docs/src/authentication.md index edb4201e..555ad742 100644 --- a/docs/src/authentication.md +++ b/docs/src/authentication.md @@ -41,4 +41,9 @@ https://projects.example.com/projects/myproject/versions.txt ``` In the wildcard pattern, `?` matches any single letter, `*` matches any sequence of characters -not containing `/`, and `**` matches any sequence of characters possibly including `/`. \ No newline at end of file +not containing `/`, and `**` matches any sequence of characters possibly including `/`. + +Credentials will *only* be sent to URLs matching the pattern, and even then only if an +unauthenticated response produces a status in the 4xx range. If multiple patterns match, they will +be tried in an arbitrary order, after the initial unauthenticated attempt, until one results in a +response not in the 4xx range. From b887f8c2e202e2a4f91dd1591e3411d4ef17afb1 Mon Sep 17 00:00:00 2001 From: Tilo Wiklund Date: Mon, 2 Feb 2026 11:20:05 +0100 Subject: [PATCH 06/13] rename Pol to Policy and strengthen trait assumptions Signed-off-by: Tilo Wiklund --- core/src/auth.rs | 5 ++- core/src/env/reqwest_http.rs | 12 +++---- core/src/project/reqwest_kpar_download.rs | 6 ++-- core/src/project/reqwest_src.rs | 2 +- core/src/resolve/reqwest_http.rs | 42 +++++++++++------------ core/src/resolve/standard.rs | 28 +++++++-------- sysand/src/commands/add.rs | 4 +-- sysand/src/commands/clone.rs | 4 +-- sysand/src/commands/env.rs | 11 +++--- sysand/src/commands/info.rs | 8 ++--- sysand/src/commands/lock.rs | 4 +-- sysand/src/commands/sync.rs | 8 ++--- 12 files changed, 63 insertions(+), 71 deletions(-) diff --git a/core/src/auth.rs b/core/src/auth.rs index 899a91f1..abdd30ac 100644 --- a/core/src/auth.rs +++ b/core/src/auth.rs @@ -1,13 +1,12 @@ // SPDX-FileCopyrightText: © 2026 Sysand contributors // SPDX-License-Identifier: MIT OR Apache-2.0 -/// This module includes utilities for creating and using authentication policies for requests. -/// +//! This module includes utilities for creating and using authentication policies for requests. use globset::{GlobBuilder, GlobSetBuilder}; use reqwest::Response; use reqwest_middleware::{ClientWithMiddleware, RequestBuilder}; -pub trait HTTPAuthentication { +pub trait HTTPAuthentication: std::fmt::Debug + 'static { /// Tries to execute a request with some authentication policy. The request might be retried /// multiple times and it may generate auxiliary requests (using the provided client). fn with_authentication( diff --git a/core/src/env/reqwest_http.rs b/core/src/env/reqwest_http.rs index a9d483b9..a1504260 100644 --- a/core/src/env/reqwest_http.rs +++ b/core/src/env/reqwest_http.rs @@ -59,7 +59,7 @@ pub fn path_encode_uri>(uri: S) -> std::vec::IntoIter { segment_uri_generic::(uri) } -impl HTTPEnvironmentAsync { +impl HTTPEnvironmentAsync { pub fn root_url(&self) -> url::Url { let mut result = self.base_url.clone(); @@ -127,7 +127,7 @@ impl HTTPEnvironmentAsync { &self, uri: S, version: T, - ) -> Result>, HTTPEnvironmentError> { + ) -> Result>, HTTPEnvironmentError> { let project_url = self.project_src_url(uri, version)?; let src_project_url = Self::url_join(&project_url, ".project.json")?; @@ -160,7 +160,7 @@ impl HTTPEnvironmentAsync { &self, uri: S, version: T, - ) -> Result>, HTTPEnvironmentError> { + ) -> Result>, HTTPEnvironmentError> { let kpar_project_url = self.project_kpar_url(&uri, &version)?; let this_url = kpar_project_url.clone(); @@ -231,9 +231,7 @@ fn trim_line(line: Result) -> Result { Ok(line?.trim().to_string()) } -impl ReadEnvironmentAsync - for HTTPEnvironmentAsync -{ +impl ReadEnvironmentAsync for HTTPEnvironmentAsync { type ReadError = HTTPEnvironmentError; // This can be made more concrete, but the type is humongous @@ -308,7 +306,7 @@ impl ReadEnvironmentAsync Ok(Optionally { inner }) } - type InterchangeProjectRead = HTTPProjectAsync; + type InterchangeProjectRead = HTTPProjectAsync; async fn get_project_async, T: AsRef>( &self, diff --git a/core/src/project/reqwest_kpar_download.rs b/core/src/project/reqwest_kpar_download.rs index 97038db8..74d61bbc 100644 --- a/core/src/project/reqwest_kpar_download.rs +++ b/core/src/project/reqwest_kpar_download.rs @@ -63,11 +63,11 @@ impl From for ReqwestKparDownloadedError { } } -impl ReqwestKparDownloadedProject { +impl ReqwestKparDownloadedProject { pub fn new_guess_root>( url: S, client: reqwest_middleware::ClientWithMiddleware, - auth_policy: Arc, + auth_policy: Arc, ) -> Result { let tmp_dir = tempdir().map_err(FsIoError::MkTempDir)?; @@ -147,7 +147,7 @@ impl AsyncRead for AsAsyncRead { } } -impl ProjectReadAsync for ReqwestKparDownloadedProject { +impl ProjectReadAsync for ReqwestKparDownloadedProject { type Error = ReqwestKparDownloadedError; async fn get_project_async( diff --git a/core/src/project/reqwest_src.rs b/core/src/project/reqwest_src.rs index 50c40277..3007165d 100644 --- a/core/src/project/reqwest_src.rs +++ b/core/src/project/reqwest_src.rs @@ -96,7 +96,7 @@ pub enum ReqwestSrcError { BadStatus(Box, reqwest::StatusCode), } -impl ProjectReadAsync for ReqwestSrcProjectAsync { +impl ProjectReadAsync for ReqwestSrcProjectAsync { type Error = ReqwestSrcError; async fn get_project_async( diff --git a/core/src/resolve/reqwest_http.rs b/core/src/resolve/reqwest_http.rs index 438bd85b..2754866a 100644 --- a/core/src/resolve/reqwest_http.rs +++ b/core/src/resolve/reqwest_http.rs @@ -37,22 +37,24 @@ pub enum HTTPProjectAsync { } #[derive(Error, Debug)] -pub enum HTTPProjectError { +pub enum HTTPProjectError { #[error(transparent)] - SrcProject( as ProjectReadAsync>::Error), + SrcProject( as ProjectReadAsync>::Error), // #[error(transparent)] // KParRanged(::Error), #[error(transparent)] - KparDownloaded( as ProjectReadAsync>::Error), + KparDownloaded( as ProjectReadAsync>::Error), } -pub enum HTTPProjectAsyncReader<'a, Pol: HTTPAuthentication + 'a> { - SrcProjectReader( as ProjectReadAsync>::SourceReader<'a>), +pub enum HTTPProjectAsyncReader<'a, Policy: HTTPAuthentication> { + SrcProjectReader( as ProjectReadAsync>::SourceReader<'a>), //KParRangedReader(::SourceReader<'a>), - KparDownloadedReader( as ProjectReadAsync>::SourceReader<'a>), + KparDownloadedReader( + as ProjectReadAsync>::SourceReader<'a>, + ), } -impl AsyncRead for HTTPProjectAsyncReader<'_, Pol> { +impl AsyncRead for HTTPProjectAsyncReader<'_, Policy> { fn poll_read( self: Pin<&mut Self>, cx: &mut std::task::Context<'_>, @@ -66,10 +68,8 @@ impl AsyncRead for HTTPProjectAsyncReader<'_, Pol> { } } -impl ProjectReadAsync - for HTTPProjectAsync -{ - type Error = HTTPProjectError; +impl ProjectReadAsync for HTTPProjectAsync { + type Error = HTTPProjectError; async fn get_project_async( &self, @@ -96,7 +96,7 @@ impl ProjectReadAsync } type SourceReader<'a> - = HTTPProjectAsyncReader<'a, Pol> + = HTTPProjectAsyncReader<'a, Policy> where Self: 'a; @@ -150,8 +150,8 @@ pub struct HTTPProjects { //prefer_ranged: bool, } -impl HTTPProjects { - pub fn try_resolve_as_kpar(&self) -> Option> { +impl HTTPProjects { + pub fn try_resolve_as_kpar(&self) -> Option> { // TODO: Decide a policy for KPar vs Src urls let url = if self.url.path() == "" || !self.url.path().ends_with("/") { self.url.clone() @@ -186,7 +186,7 @@ impl HTTPProjects { )) } - pub fn try_resolve_as_src(&self, auth_policy: Arc) -> Option> { + pub fn try_resolve_as_src(&self, auth_policy: Arc) -> Option> { // These URLs should technically have a path that ends (explicitly or implicitly) // with a slash, due to the way relative references are treated in HTTP. E.g.: // resolving `bar` relative to `http://www.example.com/foo` gives `http://www.example.com/bar` @@ -215,8 +215,8 @@ impl HTTPProjects { } } -impl Iterator for HTTPProjects { - type Item = Result, Infallible>; +impl Iterator for HTTPProjects { + type Item = Result, Infallible>; fn next(&mut self) -> Option { if !self.src_done { @@ -246,14 +246,12 @@ impl Iterator for HTTPProjects { /// appears to support HTTP Range requests. If successful, it uses `HTTPKparProjectRanged` /// instead of `HTTPKparProjectDownloaded`. In case of *any* failure, or if `prefer_ranged` /// is false, `HTTPKparProjectDownloaded` is used instead. -impl ResolveReadAsync - for HTTPResolverAsync -{ +impl ResolveReadAsync for HTTPResolverAsync { type Error = Infallible; - type ProjectStorage = HTTPProjectAsync; + type ProjectStorage = HTTPProjectAsync; - type ResolvedStorages = futures::stream::Iter>; + type ResolvedStorages = futures::stream::Iter>; async fn resolve_read_async( &self, diff --git a/core/src/resolve/standard.rs b/core/src/resolve/standard.rs index 7e3fa2f4..5c8f60cb 100644 --- a/core/src/resolve/standard.rs +++ b/core/src/resolve/standard.rs @@ -33,18 +33,18 @@ type StandardResolverInner = CombinedResolver< pub struct StandardResolver(StandardResolverInner); -impl fmt::Debug for StandardResolver { +impl fmt::Debug for StandardResolver { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_tuple("CliResolver").field(&self.0).finish() } } -impl ResolveRead for StandardResolver { - type Error = as ResolveRead>::Error; +impl ResolveRead for StandardResolver { + type Error = as ResolveRead>::Error; - type ProjectStorage = as ResolveRead>::ProjectStorage; + type ProjectStorage = as ResolveRead>::ProjectStorage; - type ResolvedStorages = as ResolveRead>::ResolvedStorages; + type ResolvedStorages = as ResolveRead>::ResolvedStorages; fn resolve_read( &self, @@ -61,11 +61,11 @@ pub fn standard_file_resolver(cwd: Option) -> FileResolver { } } -pub fn standard_remote_resolver( +pub fn standard_remote_resolver( client: ClientWithMiddleware, runtime: Arc, - auth_policy: Arc, -) -> RemoteResolver>, GitResolver> { + auth_policy: Arc, +) -> RemoteResolver>, GitResolver> { RemoteResolver { http_resolver: Some( HTTPResolverAsync { @@ -88,12 +88,12 @@ pub fn standard_local_resolver(local_env_path: Utf8PathBuf) -> LocalEnvResolver } } -pub fn standard_index_resolver( +pub fn standard_index_resolver( client: ClientWithMiddleware, urls: Vec, runtime: Arc, - auth_policy: Arc, -) -> AsSyncResolveTokio> { + auth_policy: Arc, +) -> AsSyncResolveTokio> { SequentialResolver::new(urls.into_iter().map(|url| EnvResolver { env: HTTPEnvironmentAsync { client: client.clone(), @@ -107,14 +107,14 @@ pub fn standard_index_resolver( +pub fn standard_resolver( cwd: Option, local_env_path: Option, client: Option, index_urls: Option>, runtime: Arc, - auth_policy: Arc, -) -> StandardResolver { + auth_policy: Arc, +) -> StandardResolver { let file_resolver = standard_file_resolver(cwd); let remote_resolver = client .clone() diff --git a/sysand/src/commands/add.rs b/sysand/src/commands/add.rs index 4d7c8ddb..35564386 100644 --- a/sysand/src/commands/add.rs +++ b/sysand/src/commands/add.rs @@ -17,7 +17,7 @@ use crate::{CliError, cli::ResolutionOptions, command_sync}; // TODO: Collect common arguments #[allow(clippy::too_many_arguments)] -pub fn command_add, Pol: HTTPAuthentication + std::fmt::Debug + 'static>( +pub fn command_add, Policy: HTTPAuthentication>( iri: S, versions_constraint: Option, no_lock: bool, @@ -27,7 +27,7 @@ pub fn command_add, Pol: HTTPAuthentication + std::fmt::Debug + 's current_project: Option, client: reqwest_middleware::ClientWithMiddleware, runtime: Arc, - auth_policy: Arc, + auth_policy: Arc, ) -> Result<()> { let mut current_project = current_project.ok_or(CliError::MissingProjectCurrentDir)?; let project_root = current_project.root_path(); diff --git a/sysand/src/commands/clone.rs b/sysand/src/commands/clone.rs index 0478b6ca..ad907785 100644 --- a/sysand/src/commands/clone.rs +++ b/sysand/src/commands/clone.rs @@ -34,7 +34,7 @@ pub enum ProjectLocator { /// Clones project from `locator` to `target` directory. #[allow(clippy::too_many_arguments)] -pub fn command_clone( +pub fn command_clone( locator: ProjectLocatorArgs, version: Option, target: Option, @@ -43,7 +43,7 @@ pub fn command_clone( config: &Config, client: reqwest_middleware::ClientWithMiddleware, runtime: Arc, - auth_policy: Arc, + auth_policy: Arc, ) -> Result<()> { let ResolutionOptions { index, diff --git a/sysand/src/commands/env.rs b/sysand/src/commands/env.rs index 5d986810..25eab94e 100644 --- a/sysand/src/commands/env.rs +++ b/sysand/src/commands/env.rs @@ -38,7 +38,7 @@ pub fn command_env>(path: P) -> Result( +pub fn command_env_install( iri: Iri, version: Option, install_opts: InstallOptions, @@ -47,7 +47,7 @@ pub fn command_env_install( project_root: Option, client: reqwest_middleware::ClientWithMiddleware, runtime: Arc, - auth_policy: Arc, + auth_policy: Arc, ) -> Result<()> { let project_root = project_root.unwrap_or(wrapfs::current_dir()?); let mut env = crate::get_or_create_env(project_root.as_path())?; @@ -151,10 +151,7 @@ pub fn command_env_install( // TODO: Collect common arguments #[allow(clippy::too_many_arguments)] -pub fn command_env_install_path< - S: AsRef, - Pol: HTTPAuthentication + std::fmt::Debug + 'static, ->( +pub fn command_env_install_path, Policy: HTTPAuthentication>( iri: S, version: Option, path: Utf8PathBuf, @@ -164,7 +161,7 @@ pub fn command_env_install_path< project_root: Option, client: reqwest_middleware::ClientWithMiddleware, runtime: Arc, - auth_policy: Arc, + auth_policy: Arc, ) -> Result<()> { let project_root = project_root.unwrap_or(wrapfs::current_dir()?); let mut env = crate::get_or_create_env(project_root.as_path())?; diff --git a/sysand/src/commands/info.rs b/sysand/src/commands/info.rs index f9eba550..7bafcac9 100644 --- a/sysand/src/commands/info.rs +++ b/sysand/src/commands/info.rs @@ -99,14 +99,14 @@ pub fn command_info_path>( } } -pub fn command_info_uri( +pub fn command_info_uri( uri: Iri, _normalise: bool, client: reqwest_middleware::ClientWithMiddleware, index_urls: Option>, excluded_iris: &HashSet, runtime: Arc, - auth_policy: Arc, + auth_policy: Arc, ) -> Result<()> { let cwd = wrapfs::current_dir().ok(); @@ -188,14 +188,14 @@ pub fn command_info_verb_path>( } } -pub fn command_info_verb_uri( +pub fn command_info_verb_uri( uri: Iri, verb: InfoCommandVerb, numbered: bool, client: reqwest_middleware::ClientWithMiddleware, index_urls: Option>, runtime: Arc, - auth_policy: Arc, + auth_policy: Arc, ) -> Result<()> { match verb { InfoCommandVerb::Get(get_verb) => { diff --git a/sysand/src/commands/lock.rs b/sysand/src/commands/lock.rs index 8c75aeba..ec5a0eb5 100644 --- a/sysand/src/commands/lock.rs +++ b/sysand/src/commands/lock.rs @@ -31,13 +31,13 @@ use crate::{DEFAULT_INDEX_URL, cli::ResolutionOptions}; /// `path` must be relative to workspace root. // TODO: this will not work properly if run in subdir of workspace, // as `path` will then refer to a deeper subdir -pub fn command_lock, Pol: HTTPAuthentication + std::fmt::Debug + 'static>( +pub fn command_lock, Policy: HTTPAuthentication>( path: P, resolution_opts: ResolutionOptions, config: &Config, client: reqwest_middleware::ClientWithMiddleware, runtime: Arc, - auth_policy: Arc, + auth_policy: Arc, ) -> Result<()> { assert!(path.as_ref().is_relative(), "{}", path.as_ref()); let ResolutionOptions { diff --git a/sysand/src/commands/sync.rs b/sysand/src/commands/sync.rs index 01ad3cbc..5753d62a 100644 --- a/sysand/src/commands/sync.rs +++ b/sysand/src/commands/sync.rs @@ -18,14 +18,14 @@ use sysand_core::{ }, }; -pub fn command_sync, Pol: HTTPAuthentication>( +pub fn command_sync, Policy: HTTPAuthentication>( lock: &Lock, project_root: P, env: &mut LocalDirectoryEnvironment, client: reqwest_middleware::ClientWithMiddleware, provided_iris: &HashMap>, runtime: Arc, - auth_policy: Arc, + auth_policy: Arc, ) -> Result<()> { sysand_core::commands::sync::do_sync( lock, @@ -34,7 +34,7 @@ pub fn command_sync, Pol: HTTPAuthentication>( project_path: project_root.as_ref().join(src_path), }), Some( - |remote_src: String| -> Result>, ParseError> { + |remote_src: String| -> Result>, ParseError> { Ok(ReqwestSrcProjectAsync { client: client.clone(), url: reqwest::Url::parse(&remote_src)?, @@ -46,7 +46,7 @@ pub fn command_sync, Pol: HTTPAuthentication>( // TODO: Fix error handling here Some(|kpar_path: &Utf8Path| LocalKParProject::new_guess_root(kpar_path).unwrap()), Some( - |remote_kpar: String| -> Result>, ParseError> { + |remote_kpar: String| -> Result>, ParseError> { Ok( ReqwestKparDownloadedProject::new_guess_root(reqwest::Url::parse( &remote_kpar, From d74b7c16ead02a8cecb743f50ca3835d7a0ea1b7 Mon Sep 17 00:00:00 2001 From: Tilo Wiklund Date: Mon, 2 Feb 2026 11:47:41 +0100 Subject: [PATCH 07/13] Use text codeblocks instead of bare/plain and fix minor typos Signed-off-by: Tilo Wiklund --- CONTRIBUTING.md | 2 +- DEVELOPMENT.md | 4 ++-- bindings/java/README.md | 2 +- docs/src/authentication.md | 10 +++++----- docs/src/metadata.md | 10 +++++----- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 09c80673..bbc66e96 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,7 +16,7 @@ the code as free software. The text accepted by signing the commit, is the widely adopted DCO, maintained by the Linux Foundation, the full text of which is reproduced below. -```plain +```text Developer Certificate of Origin Version 1.1 diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 53bf6b92..c0d37640 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -155,10 +155,10 @@ Rules to follow: Rules for markdown (Rust doc comments, `.md` files): - always include a language specifier in fenced code blocks, - use `plain` if no language is appropriate: + use `text` if no language is appropriate: ````md - ```plain + ```text ``` ```` diff --git a/bindings/java/README.md b/bindings/java/README.md index 9c7f3340..4576f267 100644 --- a/bindings/java/README.md +++ b/bindings/java/README.md @@ -33,7 +33,7 @@ allow native modules as described in [JEP 472](https://openjdk.org/jeps/472#Description). Currently, the warning looks as follows: - ```plain + ```text WARNING: A restricted method in java.lang.System has been called WARNING: java.lang.System::load has been called by com.sensmetry.sysand.NativeLoader in an unnamed module (file:.../sysand-0.0.4-SNAPSHOT.jar) WARNING: Use --enable-native-access=ALL-UNNAMED to avoid a warning for callers in this module diff --git a/docs/src/authentication.md b/docs/src/authentication.md index 555ad742..41ed2605 100644 --- a/docs/src/authentication.md +++ b/docs/src/authentication.md @@ -1,6 +1,6 @@ # Authentication -Project indices and remotely stored project kpars (or sources) may require authentication in order +Project indices and remotely stored project KPARs (or sources) may require authentication in order to get authorised access. Sysand currently supports this for: - HTTP(S) using the [basic access authentication scheme](https://en.wikipedia.org/wiki/Basic_access_authentication) @@ -12,10 +12,10 @@ Support is planned for: ## Configuring -At the time of writing authentication can only be configured through environment variables. +At the time of writing, authentication can only be configured through environment variables. Providing credentials is done by setting environment variables following the pattern -``` +```text SYSAND_CRED_ = SYSAND_CRED__BASIC_USER = SYSAND_CRED__BASIC_PASS = @@ -26,7 +26,7 @@ Where `` is arbitrary, `` is a wildcard (glob) pattern matching URLs Thus, for example, -``` +```text SYSAND_CRED_TEST = "https://*.example.com/**" SYSAND_CRED_TEST_BASIC_USER = "foo" SYSAND_CRED_TEST_BASIC_PASS = "bar" @@ -34,7 +34,7 @@ SYSAND_CRED_TEST_BASIC_PASS = "bar" Would tell Sysand that it *may* use the credentials `foo:bar` with URLs such as -``` +```text https://www.example.com/projects/project.kpar https://projects.example.com/entries.txt https://projects.example.com/projects/myproject/versions.txt diff --git a/docs/src/metadata.md b/docs/src/metadata.md index a37bd7a9..8fef0d4a 100644 --- a/docs/src/metadata.md +++ b/docs/src/metadata.md @@ -97,7 +97,7 @@ major/minor/patch component is the same. This is different from SemVer which considers [all pre-1.0.0 packages to be incompatible][semver-0]. Examples: -```plain +```text ^1.2.3 := 1.2.3 := >=1.2.3, <2.0.0 ^1.2 := 1.2 := >=1.2.0, <2.0.0 ^1 := 1 := >=1.0.0, <2.0.0 @@ -116,7 +116,7 @@ version is specified, only patch-level changes are allowed. If only a major version is given, then minor- and patch-level changes are allowed. Examples: -```plain +```text ~1.2.3 := >=1.2.3, <1.3.0 ~1.2 := >=1.2.0, <1.3.0 ~1 := >=1.0.0, <2.0.0 @@ -128,7 +128,7 @@ Wildcard operator (`*`) allows for any version where the wildcard is positioned. Examples: -```plain +```text * := >=0.0.0 1.* := >=1.0.0, <2.0.0 1.2.* := >=1.2.0, <1.3.0 @@ -141,7 +141,7 @@ Since the version in a comparator may be partial, only the parts specified are required to match exactly. Examples: -```plain +```text =1.2.3 := >=1.2.3, <1.2.4 =1.2 := >=1.2.0, <1.3.0 =1 := >=1.0.0, <2.0.0 @@ -156,7 +156,7 @@ comparison operator is given, the allowed versions range has no opposite end. Examples: -```plain +```text >=1.2.0 >1 := >=2.0.0 <2 := <2.0.0 From 240ab118c00a5e776e97d501752a74480b84c6e4 Mon Sep 17 00:00:00 2001 From: Tilo Wiklund Date: Mon, 2 Feb 2026 11:48:46 +0100 Subject: [PATCH 08/13] Add warning when multiple auth patterns are matched Signed-off-by: Tilo Wiklund --- core/src/auth.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/core/src/auth.rs b/core/src/auth.rs index abdd30ac..70f5401c 100644 --- a/core/src/auth.rs +++ b/core/src/auth.rs @@ -33,7 +33,7 @@ pub trait HTTPAuthentication: std::fmt::Debug + 'static { } /// Authentication policy that does no authentication -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub struct Unauthenticated {} impl HTTPAuthentication for Unauthenticated { @@ -119,7 +119,6 @@ impl HTTPAuthentication let (client, current_request_result) = request.build_split(); let current_request = current_request_result?; - // Always try without authentication first let initial_response = self .higher .request_with_authentication( @@ -295,6 +294,17 @@ impl HTTPAuthe .await } GlobMapResult::Ambiguous(items) => { + let items: Vec<_> = items.into_iter().collect(); + + let matched_patterns = items + .iter() + .fold(String::new(), |acc, (p, _)| acc + "\n" + p); + log::warn!( + "URL {} matches multiple authentication patterns: {}", + url.as_str(), + matched_patterns + ); + let mut items = items.into_iter(); let (_, first_restricted) = items.next().unwrap(); let first_response = first_restricted From 90f61e1645a03ac1c1e73844f4f12a0c16021add Mon Sep 17 00:00:00 2001 From: Tilo Wiklund Date: Mon, 2 Feb 2026 12:26:31 +0100 Subject: [PATCH 09/13] sign Signed-off-by: Tilo Wiklund --- core/src/auth.rs | 12 ------------ core/src/config/local_fs.rs | 2 +- core/src/config/mod.rs | 10 +++++----- sysand/tests/cfg_base.rs | 2 +- sysand/tests/cli_info.rs | 31 +++++++------------------------ sysand/tests/common/mod.rs | 20 ++++++++++---------- 6 files changed, 24 insertions(+), 53 deletions(-) diff --git a/core/src/auth.rs b/core/src/auth.rs index 70f5401c..0d693bfe 100644 --- a/core/src/auth.rs +++ b/core/src/auth.rs @@ -47,18 +47,6 @@ impl HTTPAuthentication for Unauthenticated { { request.send().await } - - async fn with_authentication( - &self, - client: &ClientWithMiddleware, - renew_request: &F, - ) -> Result - where - F: Fn(&ClientWithMiddleware) -> RequestBuilder + 'static, - { - self.request_with_authentication(renew_request(client), renew_request) - .await - } } /// Authentication policy that *always* sends a username/password pair diff --git a/core/src/config/local_fs.rs b/core/src/config/local_fs.rs index 815a2f85..42f12c59 100644 --- a/core/src/config/local_fs.rs +++ b/core/src/config/local_fs.rs @@ -77,7 +77,7 @@ mod tests { url: "http://www.example.com".to_string(), ..Default::default() }]), - auth: None, + // auth: None, }; config_file .write_all(toml::to_string_pretty(&config).unwrap().as_bytes()) diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs index 32e5d582..6e0d9ae7 100644 --- a/core/src/config/mod.rs +++ b/core/src/config/mod.rs @@ -12,7 +12,7 @@ pub struct Config { pub quiet: Option, pub verbose: Option, pub index: Option>, - pub auth: Option>, + // pub auth: Option>, } impl Config { @@ -21,9 +21,9 @@ impl Config { self.verbose = self.verbose.or(config.verbose); extend_option_vec(&mut self.index, config.index); - if let Some(auth) = config.auth { - self.auth = Some(auth.clone()); - } + // if let Some(auth) = config.auth { + // self.auth = Some(auth.clone()); + // } } pub fn index_urls( @@ -144,7 +144,7 @@ mod tests { url: "http://www.example.com".to_string(), ..Default::default() }]), - auth: None, + // auth: None, }; defaults.merge(config.clone()); diff --git a/sysand/tests/cfg_base.rs b/sysand/tests/cfg_base.rs index b5c6bacd..8c4a37b7 100644 --- a/sysand/tests/cfg_base.rs +++ b/sysand/tests/cfg_base.rs @@ -32,7 +32,7 @@ fn cfg_set_quiet() -> Result<(), Box> { quiet: Some(true), verbose: None, index: None, - auth: None, + // auth: None, })?; let out_quiet_local_config = diff --git a/sysand/tests/cli_info.rs b/sysand/tests/cli_info.rs index e0d4f38e..486ca985 100644 --- a/sysand/tests/cli_info.rs +++ b/sysand/tests/cli_info.rs @@ -330,18 +330,9 @@ fn info_basic_http_url_auth() -> Result<(), Box> { ["info", "--iri", &server.url()], None, &IndexMap::from([ - ( - "SYSAND_CRED_TEST".to_string(), - "http://127.0.0.1:*/**".to_string(), - ), - ( - "SYSAND_CRED_TEST_BASIC_USER".to_string(), - "user_1234".to_string(), - ), - ( - "SYSAND_CRED_TEST_BASIC_PASS".to_string(), - "pass_4321".to_string(), - ), + ("SYSAND_CRED_TEST", "http://127.0.0.1:*/**"), + ("SYSAND_CRED_TEST_BASIC_USER", "user_1234"), + ("SYSAND_CRED_TEST_BASIC_PASS", "pass_4321"), ]), )?; @@ -870,19 +861,11 @@ fn info_multi_index_url_auth() -> Result<(), Box> { .expect(2) // TODO: Reduce this to 1 .create(); + let server_pattern = format!("http://{}/**", server.host_with_port()); let auth_env = IndexMap::from([ - ( - "SYSAND_CRED_TEST".to_string(), - format!("http://{}/**", server.host_with_port()), - ), - ( - "SYSAND_CRED_TEST_BASIC_USER".to_string(), - "user_1234".to_string(), - ), - ( - "SYSAND_CRED_TEST_BASIC_PASS".to_string(), - "pass_4321".to_string(), - ), + ("SYSAND_CRED_TEST", server_pattern.as_ref()), + ("SYSAND_CRED_TEST_BASIC_USER", "user_1234"), + ("SYSAND_CRED_TEST_BASIC_PASS", "pass_4321"), ]); let (_, _, out) = run_sysand_with( diff --git a/sysand/tests/common/mod.rs b/sysand/tests/common/mod.rs index d1d7e37a..52ef4491 100644 --- a/sysand/tests/common/mod.rs +++ b/sysand/tests/common/mod.rs @@ -6,13 +6,13 @@ use camino_tempfile::Utf8TempDir; use indexmap::IndexMap; #[cfg(not(target_os = "windows"))] use rexpect::session::{PtySession, spawn_command}; -#[cfg(not(target_os = "windows"))] -use std::os::unix::process::ExitStatusExt; use std::{ error::Error, io::Write, process::{Command, Output}, }; +#[cfg(not(target_os = "windows"))] +use std::{ffi::OsStr, os::unix::process::ExitStatusExt}; pub fn fixture_path(name: &str) -> Utf8PathBuf { let mut path = Utf8PathBuf::from(env!("CARGO_MANIFEST_DIR")); @@ -26,7 +26,7 @@ pub fn sysand_cmd_in_with<'a, I: IntoIterator>( cwd: &Utf8Path, args: I, cfg: Option<&str>, - env: &IndexMap, + env: &IndexMap, impl AsRef>, ) -> Result> { let cfg_args = if let Some(config) = cfg { let config_path = cwd.join("sysand.toml"); @@ -60,7 +60,7 @@ pub fn sysand_cmd_in<'a, I: IntoIterator>( args: I, cfg: Option<&str>, ) -> Result> { - sysand_cmd_in_with(cwd, args, cfg, &IndexMap::default()) + sysand_cmd_in_with(cwd, args, cfg, &IndexMap::<&str, &str>::default()) } /// Creates a temporary directory and returns the tuple of the temporary @@ -78,7 +78,7 @@ pub fn new_temp_cwd() -> Result<(Utf8TempDir, Utf8PathBuf), Box> { pub fn sysand_cmd<'a, I: IntoIterator>( args: I, cfg: Option<&str>, - env: &IndexMap, + env: &IndexMap, impl AsRef>, ) -> Result<(Utf8TempDir, Utf8PathBuf, Command), Box> { // NOTE had trouble getting test-temp-dir crate working, but would be better let (temp_dir, cwd) = new_temp_cwd()?; @@ -91,7 +91,7 @@ pub fn run_sysand_in_with<'a, I: IntoIterator>( cwd: &Utf8Path, args: I, cfg: Option<&str>, - env: &IndexMap, + env: &IndexMap, impl AsRef>, ) -> Result> { Ok(sysand_cmd_in_with(cwd, args, cfg, env)?.output()?) } @@ -107,7 +107,7 @@ pub fn run_sysand_in<'a, I: IntoIterator>( pub fn run_sysand_with<'a, I: IntoIterator>( args: I, cfg: Option<&str>, - env: &IndexMap, + env: &IndexMap, impl AsRef>, ) -> Result<(Utf8TempDir, Utf8PathBuf, Output), Box> { let (temp_dir, cwd, mut cmd) = sysand_cmd(args /*, stdin*/, cfg, env)?; @@ -118,7 +118,7 @@ pub fn run_sysand<'a, I: IntoIterator>( args: I, cfg: Option<&str>, ) -> Result<(Utf8TempDir, Utf8PathBuf, Output), Box> { - run_sysand_with(args, cfg, &IndexMap::default()) + run_sysand_with(args, cfg, &IndexMap::<&str, &str>::default()) } // TODO: Figure out how to do interactive tests on Windows. @@ -140,7 +140,7 @@ pub fn run_sysand_interactive_with<'a, I: IntoIterator>( args: I, timeout_ms: Option, cfg: Option<&str>, - env: &IndexMap, + env: &IndexMap, impl AsRef>, ) -> Result<(Utf8TempDir, Utf8PathBuf, PtySession), Box> { let (temp_dir, cwd, cmd) = sysand_cmd(args, cfg, env)?; @@ -153,7 +153,7 @@ pub fn run_sysand_interactive<'a, I: IntoIterator>( timeout_ms: Option, cfg: Option<&str>, ) -> Result<(Utf8TempDir, Utf8PathBuf, PtySession), Box> { - run_sysand_interactive_with(args, timeout_ms, cfg, &IndexMap::default()) + run_sysand_interactive_with(args, timeout_ms, cfg, &IndexMap::<&str, &str>::default()) } // TODO: Figure out how to do interactive tests on Windows. From 9a49a0a1f32554d8eca039cdcd4af6b71b22ddbc Mon Sep 17 00:00:00 2001 From: Tilo Wiklund Date: Mon, 2 Feb 2026 12:28:25 +0100 Subject: [PATCH 10/13] fix osstr import on windows Signed-off-by: Tilo Wiklund --- sysand/tests/common/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sysand/tests/common/mod.rs b/sysand/tests/common/mod.rs index 52ef4491..17396ab3 100644 --- a/sysand/tests/common/mod.rs +++ b/sysand/tests/common/mod.rs @@ -6,13 +6,15 @@ use camino_tempfile::Utf8TempDir; use indexmap::IndexMap; #[cfg(not(target_os = "windows"))] use rexpect::session::{PtySession, spawn_command}; +#[cfg(not(target_os = "windows"))] +use std::os::unix::process::ExitStatusExt; use std::{ error::Error, io::Write, process::{Command, Output}, }; -#[cfg(not(target_os = "windows"))] -use std::{ffi::OsStr, os::unix::process::ExitStatusExt}; + +use std::ffi::OsStr; pub fn fixture_path(name: &str) -> Utf8PathBuf { let mut path = Utf8PathBuf::from(env!("CARGO_MANIFEST_DIR")); From 6bab79d1ecd1695b3caf01b83738fcac4f2b0d4e Mon Sep 17 00:00:00 2001 From: Tilo Wiklund <75035892+tilowiklundSensmetry@users.noreply.github.com> Date: Mon, 2 Feb 2026 13:33:21 +0100 Subject: [PATCH 11/13] Update core/src/env/reqwest_http.rs Co-authored-by: Victor Linroth Signed-off-by: Tilo Wiklund <75035892+tilowiklundSensmetry@users.noreply.github.com> --- core/src/env/reqwest_http.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/env/reqwest_http.rs b/core/src/env/reqwest_http.rs index a1504260..466cdaf7 100644 --- a/core/src/env/reqwest_http.rs +++ b/core/src/env/reqwest_http.rs @@ -32,7 +32,7 @@ use futures::{AsyncBufReadExt as _, StreamExt as _}; pub type HTTPEnvironment = AsSyncEnvironmentTokio>; #[derive(Debug)] -pub struct HTTPEnvironmentAsync { +pub struct HTTPEnvironmentAsync { pub client: reqwest_middleware::ClientWithMiddleware, pub auth_policy: Arc, pub base_url: reqwest::Url, From deded908f58b2074b905b58237bc96b8ee38f72b Mon Sep 17 00:00:00 2001 From: Tilo Wiklund Date: Mon, 2 Feb 2026 13:37:20 +0100 Subject: [PATCH 12/13] minor fixes Signed-off-by: Tilo Wiklund --- core/src/auth.rs | 12 ------------ core/src/env/reqwest_http.rs | 2 +- core/src/project/reqwest_kpar_download.rs | 4 ++-- core/src/project/reqwest_src.rs | 6 +++--- core/src/resolve/reqwest_http.rs | 14 +++++++------- core/src/resolve/standard.rs | 10 +++++----- 6 files changed, 18 insertions(+), 30 deletions(-) diff --git a/core/src/auth.rs b/core/src/auth.rs index 0d693bfe..5655d2f7 100644 --- a/core/src/auth.rs +++ b/core/src/auth.rs @@ -70,18 +70,6 @@ impl HTTPAuthentication for ForceHTTPBasicAuth { .send() .await } - - async fn with_authentication( - &self, - client: &ClientWithMiddleware, - renew_request: &F, - ) -> Result - where - F: Fn(&ClientWithMiddleware) -> RequestBuilder + 'static, - { - self.request_with_authentication(renew_request(client), renew_request) - .await - } } /// First tries `Higher` priority authentication and then the diff --git a/core/src/env/reqwest_http.rs b/core/src/env/reqwest_http.rs index 466cdaf7..b79122d4 100644 --- a/core/src/env/reqwest_http.rs +++ b/core/src/env/reqwest_http.rs @@ -34,7 +34,7 @@ pub type HTTPEnvironment = AsSyncEnvironmentTokio { pub client: reqwest_middleware::ClientWithMiddleware, - pub auth_policy: Arc, + pub auth_policy: Arc, pub base_url: reqwest::Url, pub prefer_src: bool, // Currently no async implementation of ranged diff --git a/core/src/project/reqwest_kpar_download.rs b/core/src/project/reqwest_kpar_download.rs index 74d61bbc..ecf75c3d 100644 --- a/core/src/project/reqwest_kpar_download.rs +++ b/core/src/project/reqwest_kpar_download.rs @@ -32,11 +32,11 @@ use super::utils::{FsIoError, wrapfs}; /// Downloads the full archive to a temporary directory and then accesses it using /// `LocalKParProject`. #[derive(Debug)] -pub struct ReqwestKparDownloadedProject { +pub struct ReqwestKparDownloadedProject { pub url: reqwest::Url, pub client: reqwest_middleware::ClientWithMiddleware, pub inner: LocalKParProject, - pub auth_policy: Arc, + pub auth_policy: Arc, } #[derive(Error, Debug)] diff --git a/core/src/project/reqwest_src.rs b/core/src/project/reqwest_src.rs index 3007165d..f4403195 100644 --- a/core/src/project/reqwest_src.rs +++ b/core/src/project/reqwest_src.rs @@ -29,15 +29,15 @@ use crate::{ /// is accessed by /// GET https://www.example.com/project/M%C3%ABkan%C3%AFk/K%C3%B6mmand%C3%B6h.sysml #[derive(Clone, Debug)] -pub struct ReqwestSrcProjectAsync { +pub struct ReqwestSrcProjectAsync { /// (reqwest) HTTP client to use for GET requests pub client: reqwest_middleware::ClientWithMiddleware, // Internally an Arc /// Base-url of the project pub url: reqwest::Url, - pub auth_policy: Arc, + pub auth_policy: Arc, } -impl ReqwestSrcProjectAsync { +impl ReqwestSrcProjectAsync { pub fn info_url(&self) -> reqwest::Url { self.url.join(".project.json").expect("internal URL error") } diff --git a/core/src/resolve/reqwest_http.rs b/core/src/resolve/reqwest_http.rs index 2754866a..e4ac004b 100644 --- a/core/src/resolve/reqwest_http.rs +++ b/core/src/resolve/reqwest_http.rs @@ -19,10 +19,10 @@ use crate::{ /// Tries to resolve http(s) URLs as direct (resolvable) links to interchange projects. #[derive(Debug)] -pub struct HTTPResolverAsync { +pub struct HTTPResolverAsync { pub client: reqwest_middleware::ClientWithMiddleware, pub lax: bool, - pub auth_policy: Arc, + pub auth_policy: Arc, //pub prefer_ranged: bool, } @@ -30,10 +30,10 @@ pub const SCHEME_HTTP: &Scheme = Scheme::new_or_panic("http"); pub const SCHEME_HTTPS: &Scheme = Scheme::new_or_panic("https"); #[derive(Debug)] -pub enum HTTPProjectAsync { - HTTPSrcProject(ReqwestSrcProjectAsync), +pub enum HTTPProjectAsync { + HTTPSrcProject(ReqwestSrcProjectAsync), // HTTPKParProjectRanged(ReqwestKparRangedProject), - HTTPKParProjectDownloaded(ReqwestKparDownloadedProject), + HTTPKParProjectDownloaded(ReqwestKparDownloadedProject), } #[derive(Error, Debug)] @@ -139,14 +139,14 @@ impl ProjectReadAsync for HTTPProjectAsync { } } -pub struct HTTPProjects { +pub struct HTTPProjects { client: reqwest_middleware::ClientWithMiddleware, url: reqwest::Url, src_done: bool, kpar_done: bool, // See the comments in `try_resolve_as_src`. lax: bool, - auth_policy: Arc, + auth_policy: Arc, //prefer_ranged: bool, } diff --git a/core/src/resolve/standard.rs b/core/src/resolve/standard.rs index 5c8f60cb..bd22d54a 100644 --- a/core/src/resolve/standard.rs +++ b/core/src/resolve/standard.rs @@ -22,16 +22,16 @@ use reqwest_middleware::ClientWithMiddleware; pub type LocalEnvResolver = EnvResolver; -pub type RemoteIndexResolver = SequentialResolver>>; +pub type RemoteIndexResolver = SequentialResolver>>; -type StandardResolverInner = CombinedResolver< +type StandardResolverInner = CombinedResolver< FileResolver, LocalEnvResolver, - RemoteResolver>, GitResolver>, - AsSyncResolveTokio>, + RemoteResolver>, GitResolver>, + AsSyncResolveTokio>, >; -pub struct StandardResolver(StandardResolverInner); +pub struct StandardResolver(StandardResolverInner); impl fmt::Debug for StandardResolver { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { From 819b9d94b65cd689edbbcc46650d5fd558b7fe2d Mon Sep 17 00:00:00 2001 From: Tilo Wiklund Date: Mon, 2 Feb 2026 15:38:51 +0100 Subject: [PATCH 13/13] chore Signed-off-by: Tilo Wiklund --- core/src/resolve/standard.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/resolve/standard.rs b/core/src/resolve/standard.rs index bd22d54a..49419b70 100644 --- a/core/src/resolve/standard.rs +++ b/core/src/resolve/standard.rs @@ -22,7 +22,8 @@ use reqwest_middleware::ClientWithMiddleware; pub type LocalEnvResolver = EnvResolver; -pub type RemoteIndexResolver = SequentialResolver>>; +pub type RemoteIndexResolver = + SequentialResolver>>; type StandardResolverInner = CombinedResolver< FileResolver,