feat: build .idx files in decode and fix pack decode bugsFeat/update decode

BUCK

@@ -125,3 +125,62 @@ rust_test(
         "//third-party/rust/crates/zstd-sys/2.0.16+zstd.1.5.7:zstd-sys",
     ],
 )
+
+rust_test(
+    name = "decode-index-pack",
+    srcs = [":git-internal-vendor"],
+    crate = "decode_index_pack",
+    crate_root = "vendor/tests/decode-index-pack.rs",
+    edition = "2024",
+    features = [
+        "default",
+        "diff_mydrs",
+    ],
+    rustc_flags = ["@$(location :git-internal-manifest[env_flags])"],
+    visibility = ["PUBLIC"],
+    deps = [
+        "//third-party/rust/crates/ahash/0.8.12:ahash",
+        "//third-party/rust/crates/async-trait/0.1.89:async-trait",
+        "//third-party/rust/crates/axum/0.8.7:axum",
+        "//third-party/rust/crates/bincode/2.0.1:bincode",
+        "//third-party/rust/crates/bstr/1.12.1:bstr",
+        "//third-party/rust/crates/byteorder/1.5.0:byteorder",
+        "//third-party/rust/crates/bytes/1.11.0:bytes",
+        "//third-party/rust/crates/chrono/0.4.42:chrono",
+        "//third-party/rust/crates/colored/3.0.0:colored",
+        "//third-party/rust/crates/crc32fast/1.5.0:crc32fast",
+        "//third-party/rust/crates/dashmap/6.1.0:dashmap",
+        "//third-party/rust/crates/diffs/0.5.1:diffs",
+        "//third-party/rust/crates/encoding_rs/0.8.35:encoding_rs",
+        "//third-party/rust/crates/flate2/1.1.5:flate2",
+        "//third-party/rust/crates/futures-util/0.3.31:futures-util",
+        "//third-party/rust/crates/futures/0.3.31:futures",
+        "//third-party/rust/crates/hex/0.4.3:hex",
+        "//third-party/rust/crates/libc/0.2.178:libc",
+        "//third-party/rust/crates/lru-mem/0.3.0:lru-mem",
+        "//third-party/rust/crates/memchr/2.7.6:memchr",
+        "//third-party/rust/crates/natord/1.0.9:natord",
+        "//third-party/rust/crates/num_cpus/1.17.0:num_cpus",
+        "//third-party/rust/crates/path-absolutize/3.1.1:path-absolutize",
+        "//third-party/rust/crates/quickcheck/1.0.3:quickcheck",
+        "//third-party/rust/crates/rand/0.9.2:rand",
+        "//third-party/rust/crates/rand_chacha/0.9.0:rand_chacha",
+        "//third-party/rust/crates/rayon/1.11.0:rayon",
+        "//third-party/rust/crates/sea-orm/1.1.19:sea-orm",
+        "//third-party/rust/crates/serde/1.0.228:serde",
+        "//third-party/rust/crates/sha1/0.10.6:sha1",
+        "//third-party/rust/crates/sha2/0.10.9:sha2",
+        "//third-party/rust/crates/similar/2.7.0:similar",
+        "//third-party/rust/crates/tempfile/3.23.0:tempfile",
+        "//third-party/rust/crates/thiserror/2.0.17:thiserror",
+        "//third-party/rust/crates/threadpool/1.8.1:threadpool",
+        "//third-party/rust/crates/tokio-stream/0.1.17:tokio-stream",
+        "//third-party/rust/crates/tokio-util/0.7.17:tokio-util",
+        "//third-party/rust/crates/tokio/1.48.0:tokio",
+        "//third-party/rust/crates/tracing-subscriber/0.3.22:tracing-subscriber",
+        "//third-party/rust/crates/tracing/0.1.43:tracing",
+        "//third-party/rust/crates/uuid/1.19.0:uuid",
+        "//third-party/rust/crates/zstd-sys/2.0.16+zstd.1.5.7:zstd-sys",
+        ":git_internal",
+    ],
+)

buckal.snap

@@ -3,7 +3,7 @@
 version = 1
 
 [fingerprints]
-"path+file://($WORKSPACE)#0.3.0" = "d17aeff219399002b91ddc0e1bd499f2372cb7cd3f4798a5c1e81c6ca1ba56e0"
+"path+file://($WORKSPACE)#0.3.0" = "18dfb2bd253fd84c5b5f9200d75a6a0d585b2f4fb1e53efee1cc3d9cf1897b5a"
 "registry+https://github.com/rust-lang/crates.io-index#adler2@2.0.1" = "7081ce7693ca45bff0e855374e6b7f386805bba1de9f7c288b1c706b99abef7f"
 "registry+https://github.com/rust-lang/crates.io-index#ahash@0.7.8" = "263bbc26dec8ade458658cf9dae3ef8fc606e85f04c7b47abb4f9255319689ac"
 "registry+https://github.com/rust-lang/crates.io-index#ahash@0.8.12" = "a383722a12d8343396b1965bfe77f1cebeafb12ccc571747b2b54614e3e1a1ea"

src/internal/metadata/entry_meta.rs

@@ -6,6 +6,8 @@ pub struct EntryMeta {
 
     /// Offset within the pack file
     pub pack_offset: Option<usize>,
+    /// CRC32 checksum of the compressed object data (including header)
+    pub crc32: Option<u32>,
 
     pub is_delta: Option<bool>,
 }
@@ -19,4 +21,9 @@ impl EntryMeta {
         self.pack_id = Some(id.into());
         self
     }
+
+    pub fn set_crc32(&mut self, crc32: u32) -> &mut Self {
+        self.crc32 = Some(crc32);
+        self
+    }
 }

src/internal/pack/cache.rs

@@ -261,13 +261,15 @@ mod test {
             data_decompressed: vec![0; 800],
             mem_recorder: None,
             offset: 0,
+            crc32: 0,
             is_delta_in_pack: false,
         };
         let b = CacheObject {
             info: CacheObjectInfo::BaseObject(ObjectType::Blob, b_hash),
             data_decompressed: vec![0; 800],
             mem_recorder: None,
             offset: 0,
+            crc32: 0,
             is_delta_in_pack: false,
         };
         // insert a
@@ -288,6 +290,7 @@ mod test {
             data_decompressed: vec![0; 1700],
             mem_recorder: None,
             offset: 0,
+            crc32: 0,
             is_delta_in_pack: false,
         };
         cache.insert(c.offset, c_hash, c.clone());
@@ -313,13 +316,15 @@ mod test {
             data_decompressed: vec![0; 1500],
             mem_recorder: None,
             offset: 0,
+            crc32: 0,
             is_delta_in_pack: false,
         };
         let b = CacheObject {
             info: CacheObjectInfo::BaseObject(ObjectType::Blob, b_hash),
             data_decompressed: vec![0; 1500],
             mem_recorder: None,
             offset: 0,
+            crc32: 0,
             is_delta_in_pack: false,
         };
         // insert a
@@ -338,6 +343,7 @@ mod test {
             data_decompressed: vec![0; 3000],
             mem_recorder: None,
             offset: 0,
+            crc32: 0,
             is_delta_in_pack: false,
         };
         cache.insert(c.offset, c_hash, c.clone());
@@ -366,6 +372,7 @@ mod test {
                 data_decompressed: vec![0; 800],
                 mem_recorder: None,
                 offset: 1,
+                crc32: 0,
                 is_delta_in_pack: false,
             };
             cache_sha1.insert(obj.offset, hash, obj.clone());
@@ -382,6 +389,7 @@ mod test {
                 data_decompressed: vec![0; 1500],
                 mem_recorder: None,
                 offset: 2,
+                crc32: 0,
                 is_delta_in_pack: false,
             };
             cache_sha256.insert(obj.offset, hash, obj.clone());

src/internal/pack/cache_object.rs

@@ -84,6 +84,7 @@ impl CacheObjectInfo {
 pub struct CacheObject {
     pub(crate) info: CacheObjectInfo,
     pub offset: usize,
+    pub crc32: u32,
     pub data_decompressed: Vec<u8>,
     pub mem_recorder: Option<Arc<AtomicUsize>>, // record mem-size of all CacheObjects of a Pack
     pub is_delta_in_pack: bool,
@@ -94,6 +95,7 @@ impl Clone for CacheObject {
         let obj = CacheObject {
             info: self.info.clone(),
             offset: self.offset,
+            crc32: self.crc32,
             data_decompressed: self.data_decompressed.clone(),
             mem_recorder: self.mem_recorder.clone(),
             is_delta_in_pack: self.is_delta_in_pack,
@@ -181,11 +183,17 @@ impl MemSizeRecorder for CacheObject {
 
 impl CacheObject {
     /// Create a new CacheObject which is neither [`ObjectType::OffsetDelta`] nor [`ObjectType::HashDelta`].
-    pub fn new_for_undeltified(obj_type: ObjectType, data: Vec<u8>, offset: usize) -> Self {
+    pub fn new_for_undeltified(
+        obj_type: ObjectType,
+        data: Vec<u8>,
+        offset: usize,
+        crc32: u32,
+    ) -> Self {
         let hash = utils::calculate_object_hash(obj_type, &data);
         CacheObject {
             info: CacheObjectInfo::BaseObject(obj_type, hash),
             offset,
+            crc32,
             data_decompressed: data,
             mem_recorder: None,
             is_delta_in_pack: false,
@@ -254,6 +262,7 @@ impl CacheObject {
                 let meta = EntryMeta {
                     // pack_id:Some(pack_id),
                     pack_offset: Some(self.offset),
+                    crc32: Some(self.crc32),
                     is_delta: Some(self.is_delta_in_pack),
                     ..Default::default()
                 };
@@ -377,6 +386,7 @@ mod test {
         let mut obj = CacheObject {
             info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()),
             offset: 0,
+            crc32: 0,
             data_decompressed: vec![0; 1024],
             mem_recorder: None,
             is_delta_in_pack: false,
@@ -395,6 +405,7 @@ mod test {
         let mut obj = CacheObject {
             info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()),
             offset: 0,
+            crc32: 0,
             data_decompressed: vec![0; 2048],
             mem_recorder: None,
             is_delta_in_pack: false,
@@ -414,6 +425,7 @@ mod test {
         let a = CacheObject {
             info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()),
             offset: 0,
+            crc32: 0,
             data_decompressed: vec![0; 1024],
             mem_recorder: None,
             is_delta_in_pack: false,
@@ -430,6 +442,7 @@ mod test {
         let a = CacheObject {
             info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()),
             offset: 0,
+            crc32: 0,
             data_decompressed: vec![0; 2048],
             mem_recorder: None,
             is_delta_in_pack: false,
@@ -451,6 +464,7 @@ mod test {
         let a = CacheObject {
             info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash_a),
             offset: 0,
+            crc32: 0,
             data_decompressed: vec![0; 1024],
             mem_recorder: None,
             is_delta_in_pack: false,
@@ -460,6 +474,7 @@ mod test {
         let b = CacheObject {
             info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash_b),
             offset: 0,
+            crc32: 0,
             data_decompressed: vec![0; (1024.0 * 1.5) as usize],
             mem_recorder: None,
             is_delta_in_pack: false,
@@ -505,6 +520,7 @@ mod test {
         let a = CacheObject {
             info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash_a),
             offset: 0,
+            crc32: 0,
             data_decompressed: vec![0; 2048],
             mem_recorder: None,
             is_delta_in_pack: false,
@@ -514,6 +530,7 @@ mod test {
         let b = CacheObject {
             info: CacheObjectInfo::BaseObject(ObjectType::Blob, hash_b),
             offset: 0,
+            crc32: 0,
             data_decompressed: vec![0; 3072],
             mem_recorder: None,
             is_delta_in_pack: false,
@@ -618,6 +635,7 @@ mod test {
         let a = CacheObject {
             info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()),
             offset: 0,
+            crc32: 0,
             data_decompressed: vec![0; 1024],
             mem_recorder: None,
             is_delta_in_pack: false,
@@ -636,6 +654,7 @@ mod test {
         let a = CacheObject {
             info: CacheObjectInfo::BaseObject(ObjectType::Blob, ObjectHash::default()),
             offset: 0,
+            crc32: 0,
             data_decompressed: vec![0; 2048],
             mem_recorder: None,
             is_delta_in_pack: false,