diff --git a/BUCK b/BUCK index 2401a576..430dc7d5 100644 --- a/BUCK +++ b/BUCK @@ -125,3 +125,62 @@ rust_test( "//third-party/rust/crates/zstd-sys/2.0.16+zstd.1.5.7:zstd-sys", ], ) + +rust_test( + name = "decode-index-pack", + srcs = [":git-internal-vendor"], + crate = "decode_index_pack", + crate_root = "vendor/tests/decode-index-pack.rs", + edition = "2024", + features = [ + "default", + "diff_mydrs", + ], + rustc_flags = ["@$(location :git-internal-manifest[env_flags])"], + visibility = ["PUBLIC"], + deps = [ + "//third-party/rust/crates/ahash/0.8.12:ahash", + "//third-party/rust/crates/async-trait/0.1.89:async-trait", + "//third-party/rust/crates/axum/0.8.7:axum", + "//third-party/rust/crates/bincode/2.0.1:bincode", + "//third-party/rust/crates/bstr/1.12.1:bstr", + "//third-party/rust/crates/byteorder/1.5.0:byteorder", + "//third-party/rust/crates/bytes/1.11.0:bytes", + "//third-party/rust/crates/chrono/0.4.42:chrono", + "//third-party/rust/crates/colored/3.0.0:colored", + "//third-party/rust/crates/crc32fast/1.5.0:crc32fast", + "//third-party/rust/crates/dashmap/6.1.0:dashmap", + "//third-party/rust/crates/diffs/0.5.1:diffs", + "//third-party/rust/crates/encoding_rs/0.8.35:encoding_rs", + "//third-party/rust/crates/flate2/1.1.5:flate2", + "//third-party/rust/crates/futures-util/0.3.31:futures-util", + "//third-party/rust/crates/futures/0.3.31:futures", + "//third-party/rust/crates/hex/0.4.3:hex", + "//third-party/rust/crates/libc/0.2.178:libc", + "//third-party/rust/crates/lru-mem/0.3.0:lru-mem", + "//third-party/rust/crates/memchr/2.7.6:memchr", + "//third-party/rust/crates/natord/1.0.9:natord", + "//third-party/rust/crates/num_cpus/1.17.0:num_cpus", + "//third-party/rust/crates/path-absolutize/3.1.1:path-absolutize", + "//third-party/rust/crates/quickcheck/1.0.3:quickcheck", + "//third-party/rust/crates/rand/0.9.2:rand", + "//third-party/rust/crates/rand_chacha/0.9.0:rand_chacha", + "//third-party/rust/crates/rayon/1.11.0:rayon", + "//third-party/rust/crates/sea-orm/1.1.19:sea-orm", + "//third-party/rust/crates/serde/1.0.228:serde", + "//third-party/rust/crates/sha1/0.10.6:sha1", + "//third-party/rust/crates/sha2/0.10.9:sha2", + "//third-party/rust/crates/similar/2.7.0:similar", + "//third-party/rust/crates/tempfile/3.23.0:tempfile", + "//third-party/rust/crates/thiserror/2.0.17:thiserror", + "//third-party/rust/crates/threadpool/1.8.1:threadpool", + "//third-party/rust/crates/tokio-stream/0.1.17:tokio-stream", + "//third-party/rust/crates/tokio-util/0.7.17:tokio-util", + "//third-party/rust/crates/tokio/1.48.0:tokio", + "//third-party/rust/crates/tracing-subscriber/0.3.22:tracing-subscriber", + "//third-party/rust/crates/tracing/0.1.43:tracing", + "//third-party/rust/crates/uuid/1.19.0:uuid", + "//third-party/rust/crates/zstd-sys/2.0.16+zstd.1.5.7:zstd-sys", + ":git_internal", + ], +) diff --git a/buckal.snap b/buckal.snap index 4f0355cc..2dfc727f 100644 --- a/buckal.snap +++ b/buckal.snap @@ -3,7 +3,7 @@ version = 1 [fingerprints] -"path+file://($WORKSPACE)#0.3.0" = "d17aeff219399002b91ddc0e1bd499f2372cb7cd3f4798a5c1e81c6ca1ba56e0" +"path+file://($WORKSPACE)#0.3.0" = "18dfb2bd253fd84c5b5f9200d75a6a0d585b2f4fb1e53efee1cc3d9cf1897b5a" "registry+https://github.com/rust-lang/crates.io-index#adler2@2.0.1" = "7081ce7693ca45bff0e855374e6b7f386805bba1de9f7c288b1c706b99abef7f" "registry+https://github.com/rust-lang/crates.io-index#ahash@0.7.8" = "263bbc26dec8ade458658cf9dae3ef8fc606e85f04c7b47abb4f9255319689ac" "registry+https://github.com/rust-lang/crates.io-index#ahash@0.8.12" = "a383722a12d8343396b1965bfe77f1cebeafb12ccc571747b2b54614e3e1a1ea" diff --git a/src/internal/metadata/entry_meta.rs b/src/internal/metadata/entry_meta.rs index a265fe72..f5755df7 100644 --- a/src/internal/metadata/entry_meta.rs +++ b/src/internal/metadata/entry_meta.rs @@ -6,6 +6,8 @@ pub struct EntryMeta { /// Offset within the pack file pub pack_offset: Option, + /// CRC32 checksum of the compressed object data (including header) + pub crc32: Option, pub is_delta: Option, } @@ -19,4 +21,9 @@ impl EntryMeta { self.pack_id = Some(id.into()); self } + + pub fn set_crc32(&mut self, crc32: u32) -> &mut Self { + self.crc32 = Some(crc32); + self + } } diff --git a/src/internal/pack/cache.rs b/src/internal/pack/cache.rs index ed422af3..42deadd0 100644 --- a/src/internal/pack/cache.rs +++ b/src/internal/pack/cache.rs @@ -261,6 +261,7 @@ mod test { data_decompressed: vec![0; 800], mem_recorder: None, offset: 0, + crc32: 0, is_delta_in_pack: false, }; let b = CacheObject { @@ -268,6 +269,7 @@ mod test { data_decompressed: vec![0; 800], mem_recorder: None, offset: 0, + crc32: 0, is_delta_in_pack: false, }; // insert a @@ -288,6 +290,7 @@ mod test { data_decompressed: vec![0; 1700], mem_recorder: None, offset: 0, + crc32: 0, is_delta_in_pack: false, }; cache.insert(c.offset, c_hash, c.clone()); @@ -313,6 +316,7 @@ mod test { data_decompressed: vec![0; 1500], mem_recorder: None, offset: 0, + crc32: 0, is_delta_in_pack: false, }; let b = CacheObject { @@ -320,6 +324,7 @@ mod test { data_decompressed: vec![0; 1500], mem_recorder: None, offset: 0, + crc32: 0, is_delta_in_pack: false, }; // insert a @@ -338,6 +343,7 @@ mod test { data_decompressed: vec![0; 3000], mem_recorder: None, offset: 0, + crc32: 0, is_delta_in_pack: false, }; cache.insert(c.offset, c_hash, c.clone()); @@ -366,6 +372,7 @@ mod test { data_decompressed: vec![0; 800], mem_recorder: None, offset: 1, + crc32: 0, is_delta_in_pack: false, }; cache_sha1.insert(obj.offset, hash, obj.clone()); @@ -382,6 +389,7 @@ mod test { data_decompressed: vec![0; 1500], mem_recorder: None, offset: 2, + crc32: 0, is_delta_in_pack: false, }; cache_sha256.insert(obj.offset, hash, obj.clone()); diff --git a/src/internal/pack/cache_object.rs b/src/internal/pack/cache_object.rs index 52a22eb7..1da735fb 100644 --- a/src/internal/pack/cache_object.rs +++ b/src/internal/pack/cache_object.rs @@ -84,6 +84,7 @@ impl CacheObjectInfo { pub struct CacheObject { pub(crate) info: CacheObjectInfo, pub offset: usize, + pub crc32: u32, pub data_decompressed: Vec, pub mem_recorder: Option>, // record mem-size of all CacheObjects of a Pack pub is_delta_in_pack: bool, @@ -94,6 +95,7 @@ impl Clone for CacheObject { let obj = CacheObject { info: self.info.clone(), offset: self.offset, + crc32: self.crc32, data_decompressed: self.data_decompressed.clone(), mem_recorder: self.mem_recorder.clone(), is_delta_in_pack: self.is_delta_in_pack, @@ -181,11 +183,17 @@ impl MemSizeRecorder for CacheObject { impl CacheObject { /// Create a new CacheObject which is neither [`ObjectType::OffsetDelta`] nor [`ObjectType::HashDelta`]. - pub fn new_for_undeltified(obj_type: ObjectType, data: Vec, offset: usize) -> Self { + pub fn new_for_undeltified( + obj_type: ObjectType, + data: Vec, + offset: usize, + crc32: u32, + ) -> Self { let hash = utils::calculate_object_hash(obj_type, &data); CacheObject { info: CacheObjectInfo::BaseObject(obj_type, hash), offset, + crc32, data_decompressed: data, mem_recorder: None, is_delta_in_pack: false, @@ -254,6 +262,7 @@ impl CacheObject { let meta = EntryMeta { // pack_id:Some(pack_id), pack_offset: Some(self.offset), + crc32: Some(self.crc32), is_delta: Some(self.is_delta_in_pack), ..Default::default() }; @@ -377,6 +386,7 @@ mod test { let mut obj = CacheObject { info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), offset: 0, + crc32: 0, data_decompressed: vec![0; 1024], mem_recorder: None, is_delta_in_pack: false, @@ -395,6 +405,7 @@ mod test { let mut obj = CacheObject { info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), offset: 0, + crc32: 0, data_decompressed: vec![0; 2048], mem_recorder: None, is_delta_in_pack: false, @@ -414,6 +425,7 @@ mod test { let a = CacheObject { info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), offset: 0, + crc32: 0, data_decompressed: vec![0; 1024], mem_recorder: None, is_delta_in_pack: false, @@ -430,6 +442,7 @@ mod test { let a = CacheObject { info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), offset: 0, + crc32: 0, data_decompressed: vec![0; 2048], mem_recorder: None, is_delta_in_pack: false, @@ -451,6 +464,7 @@ mod test { let a = CacheObject { info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash_a), offset: 0, + crc32: 0, data_decompressed: vec![0; 1024], mem_recorder: None, is_delta_in_pack: false, @@ -460,6 +474,7 @@ mod test { let b = CacheObject { info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash_b), offset: 0, + crc32: 0, data_decompressed: vec![0; (1024.0 * 1.5) as usize], mem_recorder: None, is_delta_in_pack: false, @@ -505,6 +520,7 @@ mod test { let a = CacheObject { info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash_a), offset: 0, + crc32: 0, data_decompressed: vec![0; 2048], mem_recorder: None, is_delta_in_pack: false, @@ -514,6 +530,7 @@ mod test { let b = CacheObject { info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash_b), offset: 0, + crc32: 0, data_decompressed: vec![0; 3072], mem_recorder: None, is_delta_in_pack: false, @@ -618,6 +635,7 @@ mod test { let a = CacheObject { info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), offset: 0, + crc32: 0, data_decompressed: vec![0; 1024], mem_recorder: None, is_delta_in_pack: false, @@ -636,6 +654,7 @@ mod test { let a = CacheObject { info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()), offset: 0, + crc32: 0, data_decompressed: vec![0; 2048], mem_recorder: None, is_delta_in_pack: false, diff --git a/src/internal/pack/decode.rs b/src/internal/pack/decode.rs index 92bcf666..789dc798 100644 --- a/src/internal/pack/decode.rs +++ b/src/internal/pack/decode.rs @@ -32,6 +32,33 @@ use crate::internal::pack::wrapper::Wrapper; use crate::internal::pack::{DEFAULT_TMP_DIR, Pack, utils}; use crate::utils::CountingReader; +/// A reader that counts bytes read and computes CRC32 checksum. +/// which is used to verify the integrity of decompressed data. +struct CrcCountingReader<'a, R> { + inner: R, + bytes_read: u64, + crc: &'a mut crc32fast::Hasher, +} +impl Read for CrcCountingReader<'_, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let n = self.inner.read(buf)?; + self.bytes_read += n as u64; + self.crc.update(&buf[..n]); + Ok(n) + } +} +impl BufRead for CrcCountingReader<'_, R> { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + self.inner.fill_buf() + } + fn consume(&mut self, amt: usize) { + let buf = self.inner.fill_buf().unwrap_or(&[]); + self.crc.update(&buf[..amt.min(buf.len())]); + self.bytes_read += amt as u64; + self.inner.consume(amt); + } +} + /// For the convenience of passing parameters struct SharedParams { pub pool: Arc, @@ -253,9 +280,16 @@ impl Pack { offset: &mut usize, ) -> Result { let init_offset = *offset; + let mut hasher = crc32fast::Hasher::new(); + let mut reader = CrcCountingReader { + inner: pack, + bytes_read: 0, + crc: &mut hasher, + }; // Attempt to read the type and size, handle potential errors - let (type_bits, size) = match utils::read_type_and_varint_size(pack, offset) { + // Note: read_type_and_varint_size updates the offset manually, but we can rely on reader.bytes_read + let (type_bits, size) = match utils::read_type_and_varint_size(&mut reader, offset) { Ok(result) => result, Err(e) => { // Handle the error e.g., by logging it or converting it to GitError @@ -269,15 +303,21 @@ impl Pack { match t { ObjectType::Commit | ObjectType::Tree | ObjectType::Blob | ObjectType::Tag => { - let (data, raw_size) = Pack::decompress_data(pack, size)?; + let (data, raw_size) = Pack::decompress_data(&mut reader, size)?; *offset += raw_size; - Ok(CacheObject::new_for_undeltified(t, data, init_offset)) + let crc32 = hasher.finalize(); + Ok(CacheObject::new_for_undeltified( + t, + data, + init_offset, + crc32, + )) } ObjectType::OffsetDelta | ObjectType::OffsetZstdelta => { - let (delta_offset, bytes) = utils::read_offset_encoding(pack).unwrap(); + let (delta_offset, bytes) = utils::read_offset_encoding(&mut reader).unwrap(); *offset += bytes; - let (data, raw_size) = Pack::decompress_data(pack, size)?; + let (data, raw_size) = Pack::decompress_data(&mut reader, size)?; *offset += raw_size; // Count the base object offset: the current offset - delta offset @@ -300,9 +340,11 @@ impl Pack { } _ => unreachable!(), }; + let crc32 = hasher.finalize(); Ok(CacheObject { info: obj_info, offset: init_offset, + crc32, data_decompressed: data, mem_recorder: None, is_delta_in_pack: true, @@ -310,19 +352,22 @@ impl Pack { } ObjectType::HashDelta => { // Read hash bytes to get the reference object hash(size depends on hash kind,e.g.,20 for SHA1,32 for SHA256) - let ref_sha = ObjectHash::from_stream(pack).unwrap(); + let ref_sha = ObjectHash::from_stream(&mut reader).unwrap(); // Offset is incremented by 20/32 bytes *offset += get_hash_kind().size(); - let (data, raw_size) = Pack::decompress_data(pack, size)?; + let (data, raw_size) = Pack::decompress_data(&mut reader, size)?; *offset += raw_size; let mut reader = Cursor::new(&data); let (_, final_size) = utils::read_delta_object_size(&mut reader)?; + let crc32 = hasher.finalize(); + Ok(CacheObject { info: CacheObjectInfo::HashDelta(ref_sha, final_size), offset: init_offset, + crc32, data_decompressed: data, mem_recorder: None, is_delta_in_pack: true, @@ -704,6 +749,7 @@ impl Pack { CacheObject { info: CacheObjectInfo::BaseObject(base_obj.object_type(), hash), offset: delta_obj.offset, + crc32: delta_obj.crc32, data_decompressed: result, mem_recorder: None, is_delta_in_pack: delta_obj.is_delta_in_pack, @@ -717,6 +763,7 @@ impl Pack { CacheObject { info: CacheObjectInfo::BaseObject(base_obj.object_type(), hash), offset: delta_obj.offset, + crc32: delta_obj.crc32, data_decompressed: result, mem_recorder: None, is_delta_in_pack: delta_obj.is_delta_in_pack, diff --git a/src/internal/pack/index_entry.rs b/src/internal/pack/index_entry.rs index 053a7deb..15657210 100644 --- a/src/internal/pack/index_entry.rs +++ b/src/internal/pack/index_entry.rs @@ -22,9 +22,15 @@ impl TryFrom<&MetaAttached> for IndexEntry { .ok_or(GitError::ConversionError(String::from( "empty offset in pack entry", )))?; + // Use the CRC32 from metadata if available (calculated from compressed data), + // otherwise fallback to calculating it from decompressed data (which is technically wrong for .idx but handles legacy cases) + let crc32 = pack_entry + .meta + .crc32 + .unwrap_or_else(|| calculate_crc32(&pack_entry.inner.data)); Ok(IndexEntry { hash: pack_entry.inner.hash, - crc32: calculate_crc32(&pack_entry.inner.data), + crc32, offset: offset as u64, }) } diff --git a/src/internal/pack/mod.rs b/src/internal/pack/mod.rs index 9351d73a..fe404072 100644 --- a/src/internal/pack/mod.rs +++ b/src/internal/pack/mod.rs @@ -9,11 +9,10 @@ pub mod decode; pub mod encode; pub mod entry; mod index_entry; -mod pack_index; +pub mod pack_index; pub mod utils; pub mod waitlist; pub mod wrapper; - use std::sync::Arc; use std::sync::atomic::AtomicUsize; use threadpool::ThreadPool; diff --git a/src/internal/pack/pack_index.rs b/src/internal/pack/pack_index.rs index a7f2a07f..28ca9844 100644 --- a/src/internal/pack/pack_index.rs +++ b/src/internal/pack/pack_index.rs @@ -1,6 +1,6 @@ use crate::errors::GitError; use crate::hash::ObjectHash; -use crate::internal::pack::index_entry::IndexEntry; +pub use crate::internal::pack::index_entry::IndexEntry; use crate::utils::HashAlgorithm; use tokio::sync::mpsc; @@ -59,38 +59,13 @@ impl IdxBuilder { self.send_data(v.to_be_bytes().to_vec()).await } - /// support idx v3 - /// The 4-byte pack index signature: \377t0c - /// - /// 4-byte version number: 3 - /// - /// 4-byte length of the header section, including the signature and version number - /// - /// 4-byte number of objects contained in the pack - /// - /// 4-byte number of object formats in this pack index: 2 + /// Write the idx v2 header (Git pack index format, used for both SHA1 and SHA256). + /// The 4-byte pack index signature: \377t0c, followed by 4-byte version number: 2. async fn write_header(&mut self) -> Result<(), GitError> { - match &self.inner_hash { - HashAlgorithm::Sha1(_sha1) => { - //magic: FF 74 4F 63 version=2 - let header: [u8; 8] = [0xFF, 0x74, 0x4F, 0x63, 0, 0, 0, 2]; - self.send_data(header.to_vec()).await - } - HashAlgorithm::Sha256(_sha2) => { - // .idx v3 - - let magic: [u8; 4] = [0xFF, 0x74, 0x4F, 0x63]; - let version: u32 = 3; - let header_size: u32 = 20; // magic(4) + version(4) + header_size(4) + object_count(4) + format_count(4) - let format_count: u32 = 1; // in the project ,SHA1 and SHA256 will not appear in the same pack,right?. - - self.send_data(magic.to_vec()).await?; - self.send_u32(version).await?; - self.send_u32(header_size).await?; - self.send_u32(self.object_number as u32).await?; - self.send_u32(format_count).await - } - } + // .idx v2 header (used for both SHA1 and SHA256) + // magic: FF 74 4F 63, version: 2 + let header: [u8; 8] = [0xFF, 0x74, 0x4F, 0x63, 0, 0, 0, 2]; + self.send_data(header.to_vec()).await } async fn write_fanout(&mut self, entries: &mut [IndexEntry]) -> Result<(), GitError> { diff --git a/tests/data/packs/sha1_delta_84ab8b4e9d42f4567463575c598198458f1dbe3e.pack b/tests/data/packs/sha1_delta_84ab8b4e9d42f4567463575c598198458f1dbe3e.pack new file mode 100644 index 00000000..31ed5b65 --- /dev/null +++ b/tests/data/packs/sha1_delta_84ab8b4e9d42f4567463575c598198458f1dbe3e.pack @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:09266c50021c2eb2a92650048f66c00890f84a683d0f4e8adc999a6151730372 +size 3317 diff --git a/tests/data/packs/sha1_small_6e34dc09a1725167aa5af943a208018d29db3335.pack b/tests/data/packs/sha1_small_6e34dc09a1725167aa5af943a208018d29db3335.pack new file mode 100644 index 00000000..d16bcc3d --- /dev/null +++ b/tests/data/packs/sha1_small_6e34dc09a1725167aa5af943a208018d29db3335.pack @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ed2e00dd403d39ee3617e1f3712a1cc30b200cbc68d1e8c012b5c5338d0e0d7 +size 1825 diff --git a/tests/data/packs/sha256_delta_cf65537eaf062245e49f5042e1f5e830ca7f70a642aa74cf21fed93e318eb4dd.pack b/tests/data/packs/sha256_delta_cf65537eaf062245e49f5042e1f5e830ca7f70a642aa74cf21fed93e318eb4dd.pack new file mode 100644 index 00000000..259fa4c8 --- /dev/null +++ b/tests/data/packs/sha256_delta_cf65537eaf062245e49f5042e1f5e830ca7f70a642aa74cf21fed93e318eb4dd.pack @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:260dc7ca77645aca4d4458d8fdbcb3cc1ded67b96f681b0c0d6d1a49bb654afb +size 3495 diff --git a/tests/data/packs/sha256_small_2f6bd516e9640e5c124e8ddb78a985047e02d69b4121ea1ccece01081a4e36fa.pack b/tests/data/packs/sha256_small_2f6bd516e9640e5c124e8ddb78a985047e02d69b4121ea1ccece01081a4e36fa.pack new file mode 100644 index 00000000..6a131236 --- /dev/null +++ b/tests/data/packs/sha256_small_2f6bd516e9640e5c124e8ddb78a985047e02d69b4121ea1ccece01081a4e36fa.pack @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff6d699a0da99600e6121452b134eeffe1374c318991a80c7da09027bd920a40 +size 1941 diff --git a/tests/decode-index-pack.rs b/tests/decode-index-pack.rs new file mode 100644 index 00000000..7be3222d --- /dev/null +++ b/tests/decode-index-pack.rs @@ -0,0 +1,171 @@ +/// Tests for decoding a pack file and re-encoding its index, +/// verifying that the offsets in the generated index match those in the original pack. +/// This ensures that the decoding and encoding processes are consistent. +use std::collections::HashMap; +use std::convert::TryInto; +use std::fs; +use std::io::BufReader; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use git_internal::errors::GitError; +use git_internal::hash::{HashKind, ObjectHash, set_hash_kind_for_test}; +use git_internal::internal::metadata::{EntryMeta, MetaAttached}; +use git_internal::internal::pack::Pack; +use git_internal::internal::pack::entry::Entry; +use git_internal::internal::pack::pack_index::{IdxBuilder, IndexEntry}; +use tokio::sync::mpsc; + +fn packs_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data/packs") +} + +fn find_pack(prefix: &str) -> PathBuf { + let dir = packs_dir(); + for entry in fs::read_dir(&dir).expect("read packs dir failed") { + let entry = entry.expect("dir entry error"); + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.starts_with(prefix) && name.ends_with(".pack") { + return entry.path(); + } + } + panic!("pack with prefix `{prefix}` not found in {:?}", dir); +} + +fn parse_idx_offsets(idx_bytes: &[u8], kind: HashKind) -> HashMap, u64> { + assert!(idx_bytes.len() >= 8, "idx too short"); + assert_eq!(&idx_bytes[0..4], &[0xFF, 0x74, 0x4F, 0x63], "idx magic"); + let version = u32::from_be_bytes(idx_bytes[4..8].try_into().unwrap()); + + assert_eq!(version, 2, "idx version must be 2 per pack-format spec"); + let mut cursor = 8usize; + + // Fanout + let mut fanout = [0u32; 256]; + for i in 0..256 { + fanout[i] = u32::from_be_bytes( + idx_bytes[cursor + i * 4..cursor + i * 4 + 4] + .try_into() + .unwrap(), + ); + } + cursor += 256 * 4; + + let object_count = fanout[255] as usize; + + let hash_len = kind.size(); + let names_end = cursor + object_count * hash_len; + let names = &idx_bytes[cursor..names_end]; + cursor = names_end; + + // Skip CRCs + cursor += object_count * 4; + + // Offsets table + let offsets_end = cursor + object_count * 4; + let offsets_bytes = &idx_bytes[cursor..offsets_end]; + cursor = offsets_end; + + let large_count = offsets_bytes + .chunks_exact(4) + .filter(|raw| u32::from_be_bytes((*raw).try_into().unwrap()) & 0x8000_0000 != 0) + .count(); + + let mut large_offsets = Vec::with_capacity(large_count); + for _ in 0..large_count { + let v = u64::from_be_bytes(idx_bytes[cursor..cursor + 8].try_into().unwrap()); + large_offsets.push(v); + cursor += 8; + } + + let mut map = HashMap::new(); + for (i, raw) in offsets_bytes.chunks_exact(4).enumerate() { + let raw = u32::from_be_bytes(raw.try_into().unwrap()); + let offset = if raw & 0x8000_0000 == 0 { + raw as u64 + } else { + let idx = (raw & 0x7FFF_FFFF) as usize; + large_offsets[idx] + }; + let hash = names[i * hash_len..(i + 1) * hash_len].to_vec(); + map.insert(hash, offset); + } + map +} + +fn decode_pack( + prefix: &str, +) -> Result<(Vec>, ObjectHash, usize), GitError> { + let pack_path = find_pack(prefix); + let file = fs::File::open(pack_path)?; + let mut reader = BufReader::new(file); + let mut pack = Pack::new(Some(2), Some(64 * 1024 * 1024), None, true); + + let entries = Arc::new(Mutex::new(Vec::new())); + let entries_for_cb = entries.clone(); + pack.decode( + &mut reader, + move |entry| { + if let Ok(mut guard) = entries_for_cb.lock() { + guard.push(entry); + } + }, + None::, + )?; + let pack_hash = pack.signature; + let count = pack.number; + let entries = Arc::try_unwrap(entries).unwrap().into_inner().unwrap(); + Ok((entries, pack_hash, count)) +} + +async fn roundtrip(prefix: &str, kind: HashKind) -> Result<(), GitError> { + let _guard = set_hash_kind_for_test(kind); + let (metas, pack_hash, count) = decode_pack(prefix)?; + assert_eq!(metas.len(), count, "decoded entries count mismatch"); + + let mut idx_entries = Vec::with_capacity(metas.len()); + for m in &metas { + idx_entries.push(IndexEntry::try_from(m)?); + } + + let (tx, mut rx) = mpsc::channel::>(1024); + let mut builder = IdxBuilder::new(idx_entries.len(), tx, pack_hash); + builder.write_idx(idx_entries).await?; + + let mut idx_bytes = Vec::new(); + while let Some(chunk) = rx.recv().await { + idx_bytes.extend_from_slice(&chunk); + } + + let offsets_map = parse_idx_offsets(&idx_bytes, kind); + for meta in metas { + let hash = meta.inner.hash.to_data(); + let expected = meta.meta.pack_offset.expect("missing pack offset") as u64; + let actual = *offsets_map + .get(&hash) + .unwrap_or_else(|| panic!("hash missing in idx: {}", meta.inner.hash)); + assert_eq!(actual, expected, "offset mismatch for {}", meta.inner.hash); + } + Ok(()) +} + +#[tokio::test] +async fn idx_offsets_match_sha1_small() -> Result<(), GitError> { + roundtrip("sha1_small_", HashKind::Sha1).await +} + +#[tokio::test] +async fn idx_offsets_match_sha1_delta() -> Result<(), GitError> { + roundtrip("sha1_delta_", HashKind::Sha1).await +} + +#[tokio::test] +async fn idx_offsets_match_sha256_small() -> Result<(), GitError> { + roundtrip("sha256_small_", HashKind::Sha256).await +} + +#[tokio::test] +async fn idx_offsets_match_sha256_delta() -> Result<(), GitError> { + roundtrip("sha256_delta_", HashKind::Sha256).await +}