diff --git a/Cargo.toml b/Cargo.toml index c53e70f..7296d93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lfspull" -version = "0.4.1" +version = "0.4.2" edition = "2021" license = "MIT" authors = ["Volume Graphics GmbH"] diff --git a/README.md b/README.md index f1b58a7..56f10fd 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,11 @@ Please see our docs.rs for example code and the gherkin tests for how to check t ## Changelog +### 0.4.2 + +- create temp file in the cached folder instead of working directory +- detect whether cached file and repo are in the same drive/device. If yes, use hard link, if not, file will be copied + ### 0.4.1 - add rust-toolchain 1.88 diff --git a/src/repo_tools/mod.rs b/src/repo_tools/mod.rs index 25c112e..ea269bf 100644 --- a/src/repo_tools/mod.rs +++ b/src/repo_tools/mod.rs @@ -140,7 +140,7 @@ async fn get_file_cached>( if cache_file.is_file() { Ok((cache_file, FilePullMode::UsedLocalCache)) } else { - fat_io_wrap_tokio(cache_dir, fs::create_dir_all) + fat_io_wrap_tokio(&cache_dir, fs::create_dir_all) .await .map_err(|_| { LFSError::DirectoryTraversalError( @@ -155,6 +155,7 @@ async fn get_file_cached>( max_retry, randomizer_bytes, timeout, + Some(cache_dir), ) .await?; if cache_file.exists() { @@ -200,11 +201,13 @@ pub async fn pull_file>( randomizer_bytes: Option, timeout: Option, ) -> Result { - info!("Pulling file {}", lfs_file.as_ref().to_string_lossy()); + let lfs_file = lfs_file.as_ref(); + + info!("Pulling file {}", lfs_file.to_string_lossy()); if !primitives::is_lfs_node_file(&lfs_file).await? { info!( "File ({}) not an lfs-node file - pulled already.", - lfs_file.as_ref().file_name().unwrap().to_string_lossy() + lfs_file.file_name().unwrap().to_string_lossy() ); return Ok(FilePullMode::WasAlreadyPresent); } @@ -227,12 +230,24 @@ pub async fn pull_file>( info!( "Found file (Origin: {:?}), linking to {}", origin, - lfs_file.as_ref().to_string_lossy() + lfs_file.to_string_lossy() ); + + let is_of_same_root = are_paths_on_same_devices(&file_name_cached, lfs_file).await?; fat_io_wrap_tokio(&lfs_file, fs::remove_file).await?; - fs::hard_link(&file_name_cached, lfs_file) - .await - .map_err(|e| FatIOError::from_std_io_err(e, file_name_cached.clone()))?; + + if is_of_same_root { + info!("Setting hard link"); + fs::hard_link(&file_name_cached, lfs_file) + .await + .map_err(|e| FatIOError::from_std_io_err(e, file_name_cached.clone()))?; + } else { + info!("Copying file"); + fs::copy(&file_name_cached, lfs_file) + .await + .map_err(|e| FatIOError::from_std_io_err(e, file_name_cached.clone()))?; + } + Ok(origin) } @@ -250,6 +265,60 @@ fn glob_recurse(wildcard_pattern: &str) -> Result, LFSError> { Ok(return_vec) } +#[cfg(windows)] +async fn are_paths_on_same_devices( + source: impl AsRef, + target: impl AsRef, +) -> Result { + use std::path::Component; + + fn get_root(path: &Path) -> Option { + path.components().find_map(|element| match element { + Component::Prefix(prefix) => Some(prefix.as_os_str().to_string_lossy().to_string()), + _ => None, + }) + } + + let source = source.as_ref().canonicalize().map_err(|e| { + LFSError::DirectoryTraversalError(format!( + "Problem getting the absolute path of {}: {}", + source.as_ref().to_string_lossy(), + e.to_string().as_str() + )) + })?; + let target = target.as_ref().canonicalize().map_err(|e| { + LFSError::DirectoryTraversalError(format!( + "Problem getting the absolute path of {}: {}", + target.as_ref().to_string_lossy(), + e.to_string().as_str() + )) + })?; + + let source_root = get_root(&source); + let target_root = get_root(&target); + + Ok(source_root == target_root) +} + +#[cfg(unix)] +async fn are_paths_on_same_devices( + source: impl AsRef, + target: impl AsRef, +) -> Result { + use std::os::unix::fs::MetadataExt; + + let source = source.as_ref(); + let target = target.as_ref(); + let meta_source = fs::metadata(source).await.map_err(|_| { + LFSError::DirectoryTraversalError("Could not get device information".to_string()) + })?; + let meta_target = fs::metadata(target).await.map_err(|_| { + LFSError::DirectoryTraversalError("Could not get device information".to_string()) + })?; + + Ok(meta_source.dev() == meta_target.dev()) +} + /// Pulls a glob recurse expression /// In addition to the same errors as in `pull_file`, more `LFSError::DirectoryTraversalError` can occur if something is wrong with the pattern /// # Arguments diff --git a/src/repo_tools/primitives.rs b/src/repo_tools/primitives.rs index bc56b97..6583a1e 100644 --- a/src/repo_tools/primitives.rs +++ b/src/repo_tools/primitives.rs @@ -121,11 +121,12 @@ fn url_with_auth(url: &str, access_token: Option<&str>) -> Result Ok(url) } -pub async fn handle_download( +async fn handle_download( meta_data: &MetaData, repo_remote_url: &str, access_token: Option<&str>, randomizer_bytes: Option, + temp_dir: &Option>, ) -> Result { const MEDIA_TYPE: &str = "application/vnd.git-lfs+json"; let client = Client::builder().build()?; @@ -192,8 +193,14 @@ pub async fn handle_download( debug!("creating temp file in current dir"); const TEMP_SUFFIX: &str = ".lfstmp"; - const TEMP_FOLDER: &str = "./"; - let tmp_path = PathBuf::from(TEMP_FOLDER).join(format!("{}{TEMP_SUFFIX}", &meta_data.oid)); + + let temp_dir = if let Some(dir) = temp_dir { + dir.as_ref() + } else { + Path::new("./") + }; + + let tmp_path = PathBuf::from(temp_dir).join(format!("{}{TEMP_SUFFIX}", &meta_data.oid)); if randomizer_bytes.is_none() && tmp_path.exists() { debug!("temp file exists. Deleting"); fat_io_wrap_tokio(&tmp_path, fs::remove_file).await?; @@ -202,7 +209,7 @@ pub async fn handle_download( .prefix(&meta_data.oid) .suffix(TEMP_SUFFIX) .rand_bytes(randomizer_bytes.unwrap_or_default()) - .tempfile_in(TEMP_FOLDER) + .tempfile_in(temp_dir) .map_err(|e| LFSError::TempFile(e.to_string()))?; debug!("created tempfile: {:?}", &temp_file); @@ -246,11 +253,18 @@ pub async fn download_file( max_retry: u32, randomizer_bytes: Option, connection_timeout: Option, + temp_dir: Option>, ) -> Result { let effective_timeout = get_effective_timeout(connection_timeout, meta_data.size); for attempt in 1..=max_retry { debug!("Download attempt {attempt}"); - let download = handle_download(meta_data, repo_remote_url, access_token, randomizer_bytes); + let download = handle_download( + meta_data, + repo_remote_url, + access_token, + randomizer_bytes, + &temp_dir, + ); let result = if let Some(seconds) = effective_timeout { timeout(Duration::from_secs(seconds), download).await } else { @@ -383,7 +397,7 @@ size 226848"#; #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn try_pull_from_demo_repo() { let parsed = parse_lfs_string(LFS_TEST_DATA).expect("Could not parse demo-string!"); - let temp_file = download_file(&parsed, URL, None, 3, None, Some(0)) + let temp_file = download_file(&parsed, URL, None, 3, None, Some(0), None::<&str>) .await .expect("could not download file"); let temp_size = temp_file