From 62dba6fb551b9cda8094276960ca86b69bef7cba Mon Sep 17 00:00:00 2001 From: "adinata.wijaya" Date: Fri, 9 Jan 2026 11:24:56 +0100 Subject: [PATCH 1/6] - create temp file in the cached folder instead of working directory - update dependencies --- Cargo.toml | 12 ++++++------ README.md | 5 +++++ src/repo_tools/mod.rs | 3 ++- src/repo_tools/primitives.rs | 26 ++++++++++++++++++++------ 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c53e70f..17e6c7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lfspull" -version = "0.4.1" +version = "0.4.2" edition = "2021" license = "MIT" authors = ["Volume Graphics GmbH"] @@ -12,25 +12,25 @@ description = "A simple git lfs file pulling implementation in pure rust. Can on [dependencies] clap = { version = "4.1", features = ["derive", "env"] } thiserror = "2" -reqwest = { version="0.12" , features = ["json", "stream"] } -http = "1.3" +reqwest = { version="0.13" , features = ["json", "stream"] } +http = "1.4" serde = {version ="1.0", features=['derive']} serde_json = "1.0" bytes = "1.4" sha2 = "0.10" hex = "0.4" glob = "0.3" -url = "2.3" +url = "2.5" tokio = { version = "1", features = ["macros", "rt-multi-thread", "fs"] } tracing = "0.1" tracing-subscriber = "0.3" vg_errortools = {version="0.1.0", features = ["tokio"]} -enable-ansi-support = "0.2" +enable-ansi-support = "0.3" futures-util = "0.3.30" tempfile = "3.12" [dev-dependencies] -cucumber = "0.21" +cucumber = "0.22" tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] } uuid = { version = "1.2", features = ["serde", "v4"] } diff --git a/README.md b/README.md index f1b58a7..823534b 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,11 @@ Please see our docs.rs for example code and the gherkin tests for how to check t ## Changelog +### 0.4.2 + +- create temp file in the cached folder instead of working directory +- update dependencies + ### 0.4.1 - add rust-toolchain 1.88 diff --git a/src/repo_tools/mod.rs b/src/repo_tools/mod.rs index 25c112e..24a73b5 100644 --- a/src/repo_tools/mod.rs +++ b/src/repo_tools/mod.rs @@ -140,7 +140,7 @@ async fn get_file_cached>( if cache_file.is_file() { Ok((cache_file, FilePullMode::UsedLocalCache)) } else { - fat_io_wrap_tokio(cache_dir, fs::create_dir_all) + fat_io_wrap_tokio(&cache_dir, fs::create_dir_all) .await .map_err(|_| { LFSError::DirectoryTraversalError( @@ -155,6 +155,7 @@ async fn get_file_cached>( max_retry, randomizer_bytes, timeout, + Some(cache_dir), ) .await?; if cache_file.exists() { diff --git a/src/repo_tools/primitives.rs b/src/repo_tools/primitives.rs index bc56b97..6583a1e 100644 --- a/src/repo_tools/primitives.rs +++ b/src/repo_tools/primitives.rs @@ -121,11 +121,12 @@ fn url_with_auth(url: &str, access_token: Option<&str>) -> Result Ok(url) } -pub async fn handle_download( +async fn handle_download( meta_data: &MetaData, repo_remote_url: &str, access_token: Option<&str>, randomizer_bytes: Option, + temp_dir: &Option>, ) -> Result { const MEDIA_TYPE: &str = "application/vnd.git-lfs+json"; let client = Client::builder().build()?; @@ -192,8 +193,14 @@ pub async fn handle_download( debug!("creating temp file in current dir"); const TEMP_SUFFIX: &str = ".lfstmp"; - const TEMP_FOLDER: &str = "./"; - let tmp_path = PathBuf::from(TEMP_FOLDER).join(format!("{}{TEMP_SUFFIX}", &meta_data.oid)); + + let temp_dir = if let Some(dir) = temp_dir { + dir.as_ref() + } else { + Path::new("./") + }; + + let tmp_path = PathBuf::from(temp_dir).join(format!("{}{TEMP_SUFFIX}", &meta_data.oid)); if randomizer_bytes.is_none() && tmp_path.exists() { debug!("temp file exists. Deleting"); fat_io_wrap_tokio(&tmp_path, fs::remove_file).await?; @@ -202,7 +209,7 @@ pub async fn handle_download( .prefix(&meta_data.oid) .suffix(TEMP_SUFFIX) .rand_bytes(randomizer_bytes.unwrap_or_default()) - .tempfile_in(TEMP_FOLDER) + .tempfile_in(temp_dir) .map_err(|e| LFSError::TempFile(e.to_string()))?; debug!("created tempfile: {:?}", &temp_file); @@ -246,11 +253,18 @@ pub async fn download_file( max_retry: u32, randomizer_bytes: Option, connection_timeout: Option, + temp_dir: Option>, ) -> Result { let effective_timeout = get_effective_timeout(connection_timeout, meta_data.size); for attempt in 1..=max_retry { debug!("Download attempt {attempt}"); - let download = handle_download(meta_data, repo_remote_url, access_token, randomizer_bytes); + let download = handle_download( + meta_data, + repo_remote_url, + access_token, + randomizer_bytes, + &temp_dir, + ); let result = if let Some(seconds) = effective_timeout { timeout(Duration::from_secs(seconds), download).await } else { @@ -383,7 +397,7 @@ size 226848"#; #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn try_pull_from_demo_repo() { let parsed = parse_lfs_string(LFS_TEST_DATA).expect("Could not parse demo-string!"); - let temp_file = download_file(&parsed, URL, None, 3, None, Some(0)) + let temp_file = download_file(&parsed, URL, None, 3, None, Some(0), None::<&str>) .await .expect("could not download file"); let temp_size = temp_file From 6165fa2b69b99b68595fc0ed9f08a13348e211c5 Mon Sep 17 00:00:00 2001 From: "adinata.wijaya" Date: Fri, 9 Jan 2026 11:39:28 +0100 Subject: [PATCH 2/6] revert dependencies --- Cargo.toml | 10 +++++----- README.md | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 17e6c7f..7296d93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,25 +12,25 @@ description = "A simple git lfs file pulling implementation in pure rust. Can on [dependencies] clap = { version = "4.1", features = ["derive", "env"] } thiserror = "2" -reqwest = { version="0.13" , features = ["json", "stream"] } -http = "1.4" +reqwest = { version="0.12" , features = ["json", "stream"] } +http = "1.3" serde = {version ="1.0", features=['derive']} serde_json = "1.0" bytes = "1.4" sha2 = "0.10" hex = "0.4" glob = "0.3" -url = "2.5" +url = "2.3" tokio = { version = "1", features = ["macros", "rt-multi-thread", "fs"] } tracing = "0.1" tracing-subscriber = "0.3" vg_errortools = {version="0.1.0", features = ["tokio"]} -enable-ansi-support = "0.3" +enable-ansi-support = "0.2" futures-util = "0.3.30" tempfile = "3.12" [dev-dependencies] -cucumber = "0.22" +cucumber = "0.21" tokio = { version = "1", features = ["macros", "rt-multi-thread", "time"] } uuid = { version = "1.2", features = ["serde", "v4"] } diff --git a/README.md b/README.md index 823534b..a8da2f2 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,6 @@ Please see our docs.rs for example code and the gherkin tests for how to check t ### 0.4.2 - create temp file in the cached folder instead of working directory -- update dependencies ### 0.4.1 From d0f694b63c9bfd94cd146256d262e3674165656d Mon Sep 17 00:00:00 2001 From: "adinata.wijaya" Date: Fri, 9 Jan 2026 13:33:17 +0100 Subject: [PATCH 3/6] test --- src/repo_tools/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/repo_tools/mod.rs b/src/repo_tools/mod.rs index 24a73b5..0a237af 100644 --- a/src/repo_tools/mod.rs +++ b/src/repo_tools/mod.rs @@ -231,6 +231,7 @@ pub async fn pull_file>( lfs_file.as_ref().to_string_lossy() ); fat_io_wrap_tokio(&lfs_file, fs::remove_file).await?; + info!("file deleted"); fs::hard_link(&file_name_cached, lfs_file) .await .map_err(|e| FatIOError::from_std_io_err(e, file_name_cached.clone()))?; From 6af4b2480175b2f7df5c0e1f239a7b5a0d0013c8 Mon Sep 17 00:00:00 2001 From: "adinata.wijaya" Date: Mon, 12 Jan 2026 11:41:05 +0100 Subject: [PATCH 4/6] detect whether cached file and repo are in the same drive/device. If yes, use hard link, if not, file will be copied --- README.md | 1 + src/repo_tools/mod.rs | 79 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 73 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a8da2f2..56f10fd 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ Please see our docs.rs for example code and the gherkin tests for how to check t ### 0.4.2 - create temp file in the cached folder instead of working directory +- detect whether cached file and repo are in the same drive/device. If yes, use hard link, if not, file will be copied ### 0.4.1 diff --git a/src/repo_tools/mod.rs b/src/repo_tools/mod.rs index 0a237af..638df80 100644 --- a/src/repo_tools/mod.rs +++ b/src/repo_tools/mod.rs @@ -201,11 +201,13 @@ pub async fn pull_file>( randomizer_bytes: Option, timeout: Option, ) -> Result { - info!("Pulling file {}", lfs_file.as_ref().to_string_lossy()); + let lfs_file = lfs_file.as_ref(); + + info!("Pulling file {}", lfs_file.to_string_lossy()); if !primitives::is_lfs_node_file(&lfs_file).await? { info!( "File ({}) not an lfs-node file - pulled already.", - lfs_file.as_ref().file_name().unwrap().to_string_lossy() + lfs_file.file_name().unwrap().to_string_lossy() ); return Ok(FilePullMode::WasAlreadyPresent); } @@ -228,13 +230,24 @@ pub async fn pull_file>( info!( "Found file (Origin: {:?}), linking to {}", origin, - lfs_file.as_ref().to_string_lossy() + lfs_file.to_string_lossy() ); + + let is_of_same_root = are_paths_on_same_devices(&file_name_cached, lfs_file).await?; fat_io_wrap_tokio(&lfs_file, fs::remove_file).await?; - info!("file deleted"); - fs::hard_link(&file_name_cached, lfs_file) - .await - .map_err(|e| FatIOError::from_std_io_err(e, file_name_cached.clone()))?; + + if is_of_same_root { + info!("Setting hard link"); + fs::hard_link(&file_name_cached, lfs_file) + .await + .map_err(|e| FatIOError::from_std_io_err(e, file_name_cached.clone()))?; + } else { + info!("Copying file"); + fs::copy(&file_name_cached, lfs_file) + .await + .map_err(|e| FatIOError::from_std_io_err(e, file_name_cached.clone()))?; + } + Ok(origin) } @@ -252,6 +265,58 @@ fn glob_recurse(wildcard_pattern: &str) -> Result, LFSError> { Ok(return_vec) } +#[cfg(windows)] +async fn are_paths_on_same_devices( + source: impl AsRef, + target: impl AsRef, +) -> Result { + use std::path::Component; + + fn get_root(path: &Path) -> Option { + path.components() + .find(|&element| element == Component::RootDir) + } + + let source = source.as_ref().canonicalize().map_err(|e| { + LFSError::DirectoryTraversalError(format!( + "Problem getting the absolute path of {}: {}", + source.as_ref().to_string_lossy(), + e.to_string().as_str() + )) + })?; + let target = target.as_ref().canonicalize().map_err(|e| { + LFSError::DirectoryTraversalError(format!( + "Problem getting the absolute path of {}: {}", + target.as_ref().to_string_lossy(), + e.to_string().as_str() + )) + })?; + + let source_root = get_root(&source); + let target_root = get_root(&target); + + Ok(source_root == target_root) +} + +#[cfg(unix)] +async fn are_paths_on_same_devices( + source: impl AsRef, + target: impl AsRef, +) -> Result { + use std::os::unix::fs::MetadataExt; + + let source = source.as_ref(); + let target = target.as_ref(); + let meta_source = fs::metadata(source).await.map_err(|_| { + LFSError::DirectoryTraversalError("Could not get device information".to_string()) + })?; + let meta_target = fs::metadata(target).await.map_err(|_| { + LFSError::DirectoryTraversalError("Could not get device information".to_string()) + })?; + + Ok(meta_source.dev() == meta_target.dev()) +} + /// Pulls a glob recurse expression /// In addition to the same errors as in `pull_file`, more `LFSError::DirectoryTraversalError` can occur if something is wrong with the pattern /// # Arguments From eb2cf6432676348bd5cabce740c397aa2ace6cc9 Mon Sep 17 00:00:00 2001 From: "adinata.wijaya" Date: Mon, 12 Jan 2026 14:11:50 +0100 Subject: [PATCH 5/6] fix get path's drive --- src/repo_tools/mod.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/repo_tools/mod.rs b/src/repo_tools/mod.rs index 638df80..4425fc1 100644 --- a/src/repo_tools/mod.rs +++ b/src/repo_tools/mod.rs @@ -272,9 +272,11 @@ async fn are_paths_on_same_devices( ) -> Result { use std::path::Component; - fn get_root(path: &Path) -> Option { - path.components() - .find(|&element| element == Component::RootDir) + fn get_root(path: &Path) -> Option { + path.components().find_map(|element| match element { + Component::Prefix(prefix) => Some(prefix.as_os_str().to_string_lossy().to_string()), + _ => None, + }) } let source = source.as_ref().canonicalize().map_err(|e| { @@ -428,4 +430,14 @@ mod tests { remote_url_ssh_to_https(REPO_REMOTE_HTTPS.to_string()).expect("Could not parse url"); assert_eq!(repo_url_https.as_str(), REPO_REMOTE_HTTPS); } + + #[tokio::test] + async fn test_test() { + let _ = are_paths_on_same_devices( + "C:\\Users\\adinata.wijaya\\Downloads\\linux", + "C:\\Users\\adinata.wijaya\\Downloads\\linux", + ) + .await + .unwrap(); + } } From d3ea1b9e20457361e6fc225301accd41d1ceb642 Mon Sep 17 00:00:00 2001 From: "adinata.wijaya" Date: Tue, 13 Jan 2026 07:46:00 +0100 Subject: [PATCH 6/6] remove unused test --- src/repo_tools/mod.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/repo_tools/mod.rs b/src/repo_tools/mod.rs index 4425fc1..ea269bf 100644 --- a/src/repo_tools/mod.rs +++ b/src/repo_tools/mod.rs @@ -430,14 +430,4 @@ mod tests { remote_url_ssh_to_https(REPO_REMOTE_HTTPS.to_string()).expect("Could not parse url"); assert_eq!(repo_url_https.as_str(), REPO_REMOTE_HTTPS); } - - #[tokio::test] - async fn test_test() { - let _ = are_paths_on_same_devices( - "C:\\Users\\adinata.wijaya\\Downloads\\linux", - "C:\\Users\\adinata.wijaya\\Downloads\\linux", - ) - .await - .unwrap(); - } }