From 04293025026bab6ee2844ce3bf562ebafaecc9ac Mon Sep 17 00:00:00 2001 From: matt Date: Mon, 13 Jul 2026 13:00:27 -0700 Subject: [PATCH 1/4] feat!: mount command --- BENCHMARK.md | 34 +- Cargo.lock | 155 +++++- Cargo.toml | 6 +- README.md | 46 +- bench.sh | 29 ++ docs/src/pages/index.astro | 47 +- share/fossil.1 | 31 +- share/fossil.bash | 8 +- share/fossil.fish | 6 + share/fossil.zsh | 9 + src/commands/explain.rs | 9 + src/commands/map.rs | 2 + src/commands/mod.rs | 2 + src/commands/mount.rs | 1010 ++++++++++++++++++++++++++++++++++++ src/commands/take.rs | 44 +- src/core/block.rs | 29 +- src/core/container.rs | 283 +++++++++- src/core/dir.rs | 40 +- src/core/models/bwtm.rs | 98 ++++ src/core/models/huffman.rs | 16 + src/core/models/lz.rs | 2 +- src/core/models/lzr.rs | 151 ++++++ src/main.rs | 33 +- tests/container.rs | 48 ++ tests/dir.rs | 40 +- tests/lazy.rs | 120 +++++ tests/models/bwtm.rs | 35 ++ tests/models/huffman.rs | 27 + tests/models/lzr.rs | 67 +++ 29 files changed, 2335 insertions(+), 92 deletions(-) create mode 100644 src/commands/mount.rs create mode 100644 tests/lazy.rs diff --git a/BENCHMARK.md b/BENCHMARK.md index 9c064ef..c2f4d35 100644 --- a/BENCHMARK.md +++ b/BENCHMARK.md @@ -8,15 +8,33 @@ fossil against `gzip -9` and `zstd -19` on the example files. Run it yourself: | file | original | fossil | gzip -9 | zstd -19 | |---|---|---|---|---| -| z | 100000 | 217 (99.8%) | 134 (99.9%) | 25 (100.0%) | -| mixed.bin | 360448 | 78449 (78.2%) | 93399 (74.1%) | 82787 (77.0%) | -| bigmix.bin | 901120 | 195706 (78.3%) | 228264 (74.7%) | 204357 (77.3%) | -| cat.ppm | 1848015 | 497028 (73.1%) | 904590 (51.1%) | 781646 (57.7%) | -| wave.pcm | 300000 | 57095 (81.0%) | 294794 (1.7%) | 293048 (2.3%) | -| cat.jpg | 60055 | 60073 (-0.0%) | 60059 (-0.0%) | 60069 (-0.0%) | +| z | 100000 | 170 (99.8%) | 134 (99.9%) | 25 (100.0%) | +| mixed.bin | 360448 | 75670 (79.0%) | 93399 (74.1%) | 82787 (77.0%) | +| bigmix.bin | 901120 | 188742 (79.1%) | 228264 (74.7%) | 204357 (77.3%) | +| cat.ppm | 1848015 | 496127 (73.2%) | 904590 (51.1%) | 781646 (57.7%) | +| wave.pcm | 300000 | 56950 (81.0%) | 294794 (1.7%) | 293048 (2.3%) | +| cat.jpg | 60055 | 60065 (-0.0%) | 60059 (-0.0%) | 60069 (-0.0%) | These numbers are only for these files. Other data would land somewhere else. Roughly why each row does what it does: the mixed files do well because fossil picks a model per 4 KB block, which fits data that keeps changing as you read it. The image gets filtered row by row first (PNG-style), so the models see small differences instead of raw pixels. wave.pcm is audio, so fossil fits a predictor to each block the way FLAC does and codes what's left over. gzip and zstd don't predict audio, so they barely touch it. cat.jpg is already compressed, so there's nothing left to take. -The blocks are still 4 KB, but the LZ models can look back up to 64 KB into what they've already seen, so a repeat far from the original only costs a pointer instead of a second copy. One of them, LZR, codes those pointers with a bit of context, like LZMA, which is what helps the mixed files. +The blocks are still 4 KB, but the LZ models can look back up to 256 KB into what they've already seen, so a repeat far from the original only costs a pointer instead of a second copy. One of them, LZR2, codes those pointers like LZMA: each byte gets its own adaptive context, and reusing the last match distance costs one bit. That's what helps the mixed files. -The point of fossil isn't only to win rows. It's that `fossil explain` can tell you which model handled each block, and why. +## Directories + +Directories trade some ratio for random access. Files are packed into one stream, and +the LZ window resets every 256 KB, so `take` and `mount` can decode one file without +touching the rest. Matches can't cross a segment boundary; that's the whole cost. +Compared against a `tar` of the same directory: + +| directory | tar bytes | fossil | gzip -9 | zstd -19 | +|---|---|---|---|---| +| src/ | 337920 | 54645 (83.8%) | 54484 (83.9%) | 46647 (86.2%) | +| share/ | 30720 | 2606 (91.5%) | 3186 (89.6%) | 3037 (90.1%) | + +fossil is level with gzip on source trees and beats both on smaller ones. zstd wins on +directories because its window is much larger than 256 KB, so it dedups across the whole +archive. That's the price of cheap `take` and `mount`. Single-file numbers are +unaffected; only directory fossils are segmented. + +The point of fossil isn't only to win rows. It's that `fossil explain` can tell you which +model handled each block, and why. diff --git a/Cargo.lock b/Cargo.lock index f9e0f98..0028d11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,18 @@ dependencies = [ "objc2", ] +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "clipboard-win" version = "5.4.1" @@ -54,9 +66,11 @@ checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" [[package]] name = "fossil" -version = "0.1.1" +version = "0.1.2" dependencies = [ "clipboard-win", + "fuser", + "libc", "objc2", "objc2-app-kit", "objc2-foundation", @@ -64,6 +78,22 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fuser" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53274f494609e77794b627b1a3cddfe45d675a6b2e9ba9c0fdc8d8eee2184369" +dependencies = [ + "libc", + "log", + "memchr", + "nix", + "page_size", + "pkg-config", + "smallvec", + "zerocopy", +] + [[package]] name = "libc" version = "0.2.186" @@ -76,6 +106,30 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "objc2" version = "0.6.4" @@ -228,6 +282,40 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + [[package]] name = "rustix" version = "1.1.4" @@ -241,6 +329,23 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "terminal_size" version = "0.4.4" @@ -251,6 +356,34 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-link" version = "0.2.1" @@ -265,3 +398,23 @@ checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ "windows-link", ] + +[[package]] +name = "zerocopy" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7cbbc0a705a0fd05cc3676525980d2bf5a9bc4adac6d6475209a7887cf59d19" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e817b7b52d0c7358d3246da9d69935ebb18116b2b102b4230dac079b4862f5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index 9d3055b..0b0c99a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fossil" -version = "0.1.1" +version = "0.1.2" edition = "2024" description = "a compressor that shows its work" repository = "https://github.com/punctuations/fossil" @@ -9,6 +9,10 @@ license = "GPL-3.0-only" [dependencies] terminal_size = "0.4" +[target.'cfg(unix)'.dependencies] +fuser = "0.15" +libc = "0.2" + [target.'cfg(target_os = "macos")'.dependencies] objc2 = "0.6" objc2-foundation = { version = "0.3", features = ["NSString", "NSURL", "NSArray", "NSData"] } diff --git a/README.md b/README.md index 89271b3..3f1acbe 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,9 @@ binary in `target/release`. fossil pack [output] # compress a file or directory → .fossil (no input packs the clipboard) fossil lift # fossilize the clipboard, then copy the .fossil back fossil unpack [out] # restore the original (verifies CRC) +fossil list # list the files inside a directory fossil +fossil take [path] # extract one file straight to stdout (no full decode) +fossil mount # mount a directory fossil as a live filesystem (unix) fossil inspect # per-block analysis: entropy, model, savings fossil map # entropy heatmap, or block models for a .fossil fossil explain # the reconstruction recipe (--block N for one block) @@ -53,20 +56,22 @@ fossil cuts a file into 4 KB blocks and runs a handful of small models on each o keeping whichever output comes out smallest. The choice is written into the file, so `fossil explain` can read it back block by block. -The models so far: RAW, RLE, Huffman, LZ, LZ+Huffman, LZR (LZ tokens range-coded with a -literal context, LZMA-style), BWT+MTF+range, adaptive range, order-1 PPM, a generator for +The models so far: RAW, RLE, Huffman, LZ, LZ+Huffman, LZR2 (LZ tokens range-coded with +literal and pointer contexts, LZMA-style), BWT+MTF+zero-run+range, adaptive range, +order-1 PPM, a generator for ramps and constant fills, a delta filter, CSV transpose, a word dictionary, and a FLAC-style signal model (windowed adaptive LPC, mid/side stereo, partitioned Rice residuals) for 8/16/24-bit audio and sensor data. The LZ -models can look back up to 64 KB into what they've already seen, so a repeat far from its +models can look back up to 256 KB into what they've already seen, so a repeat far from its original only costs a pointer, not a second copy. Raw images (PPM and BMP) get filtered row by row first (PNG-style, each row picks the filter that works best), so the models see small differences instead of raw pixels. -Tiny or random files are stored as-is so they never grow, every file carries a CRC32 so -corruption shows up on unpack, and packing a directory makes one LZ pass over the whole -thing so duplicate files cost almost nothing. See [BENCHMARK.md](BENCHMARK.md) for how it -lands against gzip -9 and zstd -19 on the sample files. +Tiny or random files are stored as-is so they never grow, and every fossil carries a CRC32 +so corruption shows up on unpack. A directory fossil keeps a per-file offset and CRC32 in +its manifest, and the LZ window resets every 256 KB, so `fossil take` and `fossil mount` +can decode one file without touching the rest. See [BENCHMARK.md](BENCHMARK.md) for how it +lands against gzip -9 and zstd -19 on files and directories. ## Format @@ -76,21 +81,25 @@ unsigned LEB128 (7 bits per byte, low byte first). | field | bytes | notes | |---|---|---| | magic | 4 | `FOSL` | -| version | 1 | currently 1 | +| version | 1 | currently 2 | | mode | 1 | 0 = blocks, 1 = stored | | filter | 1 | 0 = none, 1 = image (PNG-style rows) | | ext length | 1 | length of the original extension | -| ext | n | the extension bytes (empty for clipboard input and directories) | +| ext | n | the extension bytes (empty for clipboard input; `/` for directories) | | original size | varint | length of the original input | | crc32 | 4 | little-endian CRC32 of the original input | +| meta length | varint | length of the metadata (0 for a plain file), since version 2 | +| meta | n | directory manifest when present (see below) | Then the body depends on `mode`: - **stored**: the original bytes, verbatim. Used when blocking wouldn't help (tiny, random, or already-compressed input), so a file never grows by more than the header. -- **blocks**: a `varint` block count, then each block as `model` (1 byte), - `orig_len` (varint), `payload_len` (varint), and `payload_len` payload bytes. +- **blocks**: a `varint` block count, then the last block's decoded length + (varint; every other block decodes to exactly 4096 bytes), then each block as + `model` (1 byte), `payload_len` (varint), and `payload_len` payload bytes. + Version 1 archives stored a per-block `orig_len` before `payload_len` instead. To unpack, decode each block with its model and concatenate. If `filter` is set, the result is the image residual stream; reversing the per-row filters gives the @@ -100,8 +109,9 @@ it). Each model has its own payload layout (RLE runs, Huffman and range tables, LZ tokens, LPC coefficients plus Rice residuals for audio, and so on); the exact encodings live in `src/core/models/`. A directory is packed by bundling its files -into one stream, running a single LZ pass over the whole thing, and storing that -as a file with the extension `/`. +into one stream and storing it with the extension `/`; the metadata holds a manifest +(`FDR2` magic, then each file's path, length, and CRC32). The LZ window resets every +256 KB, so a file decodes from its own segment. ## Complexity @@ -115,15 +125,15 @@ and keeps the smallest. |---|---|---| | RAW, RLE, RANGE, PPM, GEN, DELTA, CSV, WORD | $O(n)$ | $O(n)$ | | ENTROPY | $O(n + A)$ | $O(n)$ | -| LZ / LZH / LZR | $O(n\cdot c)$ | $O(n)$ | -| BWTM | $O(n \log(n) + n\cdot A)$ | $O(n\cdot A)$ | +| LZ / LZH / LZR2 | $O(n\cdot c)$ | $O(n)$ | +| BWTM2 | $O(n \log(n) + n\cdot A)$ | $O(n\cdot A)$ | | SIGNAL | $O(n\cdot p)$ per config, gated | $O(n \cdot p)$ | -Since `A`, `c`, and `p` are fixed, every model is linear in `n` except BWTM's -suffix sort, which is `n log n`. In practice the LZ family and BWTM dominate a +Since `A`, `c`, and `p` are fixed, every model is linear in `n` except BWTM2's +suffix sort, which is `n log n`. In practice the LZ family and BWTM2 dominate a block; SIGNAL only runs when the cheaper models leave the block near its original size (audio, sensor data), so its cost stays off the common path. The LZ family -also shares a single match-finder pass across LZ, LZH, and LZR rather than +also shares a single match-finder pass across LZ, LZH, and LZR2 rather than repeating it three times. Run `fossil help` for the full command list. diff --git a/bench.sh b/bench.sh index d34b7ce..4292440 100755 --- a/bench.sh +++ b/bench.sh @@ -40,4 +40,33 @@ for f in "${files[@]}"; do "$zcol" done +printf "\n" +printf "| %-12s | %10s | %10s | %10s | %10s |\n" directory "tar bytes" "fossil" "gzip -9" "zstd -19" +printf "|%s|%s|%s|%s|%s|\n" "--------------" "------------" "------------" "------------" "------------" + +dirs=(src share) + +for d in "${dirs[@]}"; do + [ -d "$d" ] || continue + orig=$(tar -cf - "$d" 2>/dev/null | wc -c | tr -d ' ') + + "$BIN" pack "$d" /tmp/bench >/dev/null 2>&1 + fos=$(wc -c < /tmp/bench.fossil | tr -d ' ') + + gz=$(tar -cf - "$d" 2>/dev/null | gzip -9 | wc -c | tr -d ' ') + + if [ "$have_zstd" -eq 1 ]; then + zs=$(tar -cf - "$d" 2>/dev/null | zstd -19 2>/dev/null | wc -c | tr -d ' ') + zcol="$zs ($(pct "$orig" "$zs"))" + else + zcol="n/a" + fi + + printf "| %-12s | %10s | %s | %s | %s |\n" \ + "$d/" "$orig" \ + "$fos ($(pct "$orig" "$fos"))" \ + "$gz ($(pct "$orig" "$gz"))" \ + "$zcol" +done + rm -f /tmp/bench.fossil diff --git a/docs/src/pages/index.astro b/docs/src/pages/index.astro index fc79c59..d6e4c4a 100644 --- a/docs/src/pages/index.astro +++ b/docs/src/pages/index.astro @@ -179,6 +179,21 @@ const toRows = (s: string) => s.split("\n").map((line) => [...line]); >restore the original (checks the CRC first) + fossil list <dir.fossil>list the files inside a directory fossil + fossil take <file.fossil> [path]pull one file out to stdout without a full decode + fossil mount <dir.fossil> <dir>mount a directory fossil as a live filesystem (unix) fossil inspect <file>per-block breakdown: entropy, model, savings s.split("\n").map((line) => [...line]); Each model looks for a different kind of structure. Run-length squashes long runs of one byte. Huffman and range coding give common bytes shorter codes. LZ replaces repeated chunks with pointers back to - earlier copies, and LZR packs those pointers a bit tighter with some - context, like LZMA. BWT reorders the data so similar contexts line up, + earlier copies, and LZR2 packs those pointers tighter with context, like + LZMA. BWT reorders the data so similar contexts line up, which helps the stages after it. The generator spots simple patterns like counters and gradients and stores the rule instead of the bytes.

@@ -232,12 +247,14 @@ const toRows = (s: string) => s.split("\n").map((line) => [...line]); LZrepeated substrings LZHLZ, then Huffman LZRLZ tokens range-coded with a literal context (LZMA-style)LZR2LZ tokens range-coded with literal, repeat-distance, and positional + contexts (LZMA-style) BWTMBurrows-Wheeler + move-to-front + range codingBWTM2Burrows-Wheeler + move-to-front + zero-run + range coding RANGEadaptive range coding, no stored table @@ -263,14 +280,16 @@ const toRows = (s: string) => s.split("\n").map((line) => [...line]);

A few things about the format. Tiny or random files are stored as-is, - so they never grow. Lengths use varints. Every file gets a CRC32, so + so they never grow. Lengths use varints. Every fossil gets a CRC32, so corruption turns up on unpack. The blocks are still 4 KB, but the LZ - models can look back up to 64 KB into what they've already seen, so a + models can look back up to 256 KB into what they've already seen, so a repeat far from its original only costs a pointer instead of a second copy. Raw images (PPM and BMP) get filtered row by row first (PNG-style), so the models see small differences instead of raw - pixels. Packing a folder runs one LZ pass over everything, so - duplicate files cost almost nothing. + pixels. A folder is packed into one stream with a manifest of each + file’s offset and CRC32. The LZ window resets every 256 KB, so + take and mount can decode one file without + touching the rest.

@@ -283,9 +302,9 @@ const toRows = (s: string) => s.split("\n").map((line) => [...line]);

- - - + + +
filefossilgzip -9zstd -19
mixed.bin78.2%74.1%77.0%
bigmix.bin78.3%74.7%77.3%
cat.ppm73.1%51.1%57.7%
mixed.bin79.0%74.1%77.0%
bigmix.bin79.1%74.7%77.3%
cat.ppm73.2%51.1%57.7%
wave.pcm81.0%1.7%2.3%
cat.jpg~0%~0%~0%
@@ -296,7 +315,9 @@ const toRows = (s: string) => s.split("\n").map((line) => [...line]); small differences instead of raw pixels. wave.pcm is audio, and fossil fits a predictor to each block the way FLAC does. gzip and zstd don't predict audio, so they barely touch it. The jpg is already - compressed, so there's nothing left to take. + compressed, so there's nothing left to take. Directories give up a little + ratio so take and mount can decode one file + at a time.

diff --git a/share/fossil.1 b/share/fossil.1 index 3d09106..d0899f1 100644 --- a/share/fossil.1 +++ b/share/fossil.1 @@ -1,4 +1,4 @@ -.TH FOSSIL 1 "June 2026" "fossil 0.1.0" "User Commands" +.TH FOSSIL 1 "July 2026" "fossil 0.1.2" "User Commands" .SH NAME fossil \- structure-aware compression that explains how it rebuilds your files .SH SYNOPSIS @@ -29,6 +29,20 @@ back to it. .BI "unpack " " [output]" Restore the original (verifies the CRC). .TP +.BI "list " "" +List the files inside a directory fossil. +.TP +.BI "take " " [path]" +Pull one file out to stdout. For a directory fossil, name the inner +.IR path ; +it is decoded from its own segment without touching the rest of the archive, and +its stored CRC is checked unless +.BR --trust . +.TP +.BI "mount " " " +Mount a directory fossil as a live filesystem (unix). Reads decode on demand, +writes are staged in memory, and changes are written back on unmount. +.TP .BI "inspect " "" Per-block analysis: entropy, model, savings. .TP @@ -75,6 +89,15 @@ in the file manager so it is easy to drag elsewhere. .B unpack --trust Skip the CRC check on unpack. .TP +.B take --trust +Skip the per-file CRC check when taking from a directory fossil. +.TP +.B mount --verbose +Show a rolling display of recent filesystem events while serving. +.TP +.B mount --log +Print every filesystem event line-by-line, for diagnosing. +.TP .BI "explain --block " N Deep-dive a single block. .SH EXAMPLES @@ -91,6 +114,12 @@ Restore an archive: Check integrity: .B fossil verify archive.fossil .TP +Extract one file from a directory fossil: +.B fossil take archive.fossil src/main.rs +.TP +Mount a directory fossil: +.B fossil mount archive.fossil mnt/ +.TP Pipe directly into fossil: .B cat foo.png | fossil pack > foo.fossil .SH AUTHOR diff --git a/share/fossil.bash b/share/fossil.bash index f1c4a1b..ffea6d5 100644 --- a/share/fossil.bash +++ b/share/fossil.bash @@ -2,7 +2,7 @@ _fossil() { local cur cur="${COMP_WORDS[COMP_CWORD]}" - local commands="pack lift unpack inspect map explain verify update help" + local commands="pack lift unpack list take mount inspect map explain verify update help" if [ "$COMP_CWORD" -eq 1 ]; then COMPREPLY=( $(compgen -W "$commands --version --help" -- "$cur") ) @@ -19,6 +19,12 @@ _fossil() { unpack|recover|exhume|uncover) COMPREPLY=( $(compgen -W "--trust" -- "$cur") ) ;; + take|from|cat) + COMPREPLY=( $(compgen -W "--trust" -- "$cur") ) + ;; + mount) + COMPREPLY=( $(compgen -W "--verbose --log" -- "$cur") ) + ;; explain|why|describe) COMPREPLY=( $(compgen -W "--block" -- "$cur") ) ;; diff --git a/share/fossil.fish b/share/fossil.fish index 8096b9a..761a3a6 100644 --- a/share/fossil.fish +++ b/share/fossil.fish @@ -3,6 +3,9 @@ complete -c fossil -f complete -c fossil -n __fish_use_subcommand -a pack -d 'compress a file or directory' complete -c fossil -n __fish_use_subcommand -a lift -d 'fossilize the clipboard' complete -c fossil -n __fish_use_subcommand -a unpack -d 'restore the original' +complete -c fossil -n __fish_use_subcommand -a list -d 'list files in a directory fossil' +complete -c fossil -n __fish_use_subcommand -a take -d 'pull one file out to stdout' +complete -c fossil -n __fish_use_subcommand -a mount -d 'mount a directory fossil (unix)' complete -c fossil -n __fish_use_subcommand -a inspect -d 'per-block analysis' complete -c fossil -n __fish_use_subcommand -a map -d 'entropy heatmap or block models' complete -c fossil -n __fish_use_subcommand -a explain -d 'the reconstruction recipe' @@ -19,4 +22,7 @@ complete -c fossil -n '__fish_seen_subcommand_from lift' -l lossy -d 'quantize ( complete -c fossil -n '__fish_seen_subcommand_from lift' -l best-effort -d 'already-compressed inputs lossless' complete -c fossil -n '__fish_seen_subcommand_from lift' -l images-only -d 'lossy on raw images only' complete -c fossil -n '__fish_seen_subcommand_from unpack' -l trust -d 'skip the CRC check' +complete -c fossil -n '__fish_seen_subcommand_from take' -l trust -d 'skip the per-file CRC check' +complete -c fossil -n '__fish_seen_subcommand_from mount' -l verbose -d 'rolling display of recent filesystem events' +complete -c fossil -n '__fish_seen_subcommand_from mount' -l log -d 'print every event line-by-line' complete -c fossil -n '__fish_seen_subcommand_from explain' -l block -d 'deep-dive a single block' diff --git a/share/fossil.zsh b/share/fossil.zsh index 1897568..7de140a 100644 --- a/share/fossil.zsh +++ b/share/fossil.zsh @@ -6,6 +6,9 @@ _fossil() { 'pack:compress a file or directory (no input packs the clipboard)' 'lift:fossilize the clipboard, copy the .fossil back' 'unpack:restore the original (verifies CRC)' + 'list:list the files inside a directory fossil' + 'take:pull one file out to stdout without a full decode' + 'mount:mount a directory fossil as a live filesystem' 'inspect:per-block analysis' 'map:entropy heatmap or block models' 'explain:the reconstruction recipe' @@ -40,6 +43,12 @@ _fossil() { unpack|recover|exhume|uncover) _arguments '--trust[skip the CRC check]' '*:file:_files' ;; + take|from|cat) + _arguments '--trust[skip the per-file CRC check]' '*:file:_files' + ;; + mount) + _arguments '--verbose[rolling display of recent filesystem events]' '--log[print every event line-by-line]' '*:file:_files' + ;; explain|why|describe) _arguments '--block[deep-dive a single block]:block number:' '*:file:_files' ;; diff --git a/src/commands/explain.rs b/src/commands/explain.rs index 48b2563..00525d1 100644 --- a/src/commands/explain.rs +++ b/src/commands/explain.rs @@ -27,7 +27,9 @@ fn reason(model: u8) -> &'static str { block::LZ => "repeated substrings", block::LZH => "LZ, then Huffman", block::LZR => "LZ tokens range-coded with a literal context", + block::LZR2 => "LZ tokens range-coded with repeat-distance and positional contexts", block::BWTM => "Burrows-Wheeler + move-to-front + range coding", + block::BWTM2 => "Burrows-Wheeler + move-to-front + zero-run + range coding", block::RANGE => "adaptive range coding, no stored table", block::PPM => "order-1 context (each byte from the last)", block::GEN => "formulas like constant fills and ramps", @@ -247,7 +249,14 @@ fn model_insight(model: u8, a: &EntropyReport, runs: usize, covered: usize, len: "→ LZR: {:.0}% repeats, tokens range-coded with an order-1 literal context (LZMA-style)", pct ), + block::LZR2 => + format!( + "→ LZR2: {:.0}% repeats, range-coded with repeat-distance and positional contexts (LZMA-style)", + pct + ), block::BWTM => "→ BWT regrouped similar contexts into runs, then entropy-coded".to_string(), + block::BWTM2 => + "→ BWT regrouped similar contexts, zero runs collapsed, then entropy-coded".to_string(), block::RANGE => format!( "→ range: adaptive coding near the {:.1} bits/byte entropy floor (no table)", diff --git a/src/commands/map.rs b/src/commands/map.rs index 5bebe1d..c97555b 100644 --- a/src/commands/map.rs +++ b/src/commands/map.rs @@ -27,7 +27,9 @@ fn model_color(model: u8) -> &'static str { block::LZ => "38;5;77", block::LZH => "38;5;75", block::LZR => "38;5;111", + block::LZR2 => "38;5;117", block::BWTM => "38;5;177", + block::BWTM2 => "38;5;183", block::RANGE => "38;5;221", block::PPM => "38;5;215", block::GEN => "38;5;203", diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 44ee26f..7494c92 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -2,6 +2,8 @@ pub mod explain; pub mod inspect; pub mod list; pub mod map; +#[cfg(unix)] +pub mod mount; pub mod pack; pub mod take; pub mod unpack; diff --git a/src/commands/mount.rs b/src/commands/mount.rs new file mode 100644 index 0000000..6f5f43f --- /dev/null +++ b/src/commands/mount.rs @@ -0,0 +1,1010 @@ +use std::collections::{BTreeMap, HashMap, VecDeque}; +use std::ffi::{CString, OsStr}; +use std::fs; +use std::io::{self, IsTerminal, Write}; +use std::os::unix::ffi::OsStrExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant, SystemTime}; + +use fuser::{ + FileAttr, FileType, Filesystem, MountOption, ReplyAttr, ReplyCreate, ReplyData, ReplyDirectory, + ReplyEmpty, ReplyEntry, ReplyOpen, ReplyWrite, Request, TimeOrNow, +}; + +use fossil::core::container::{self, BlockRef}; +use fossil::core::{crc, dir}; + +use crate::error; +use crate::utils::color::{Color, paint}; + +type EventLog = Arc>>; + +enum Sink { + None, + Ring(EventLog), + Lines(Instant), +} + +const TTL: Duration = Duration::from_secs(1); +const ROOT: u64 = 1; + +enum Body { + Stored { + offset: usize, + len: usize, + crc: Option, + }, + Loaded(Vec), +} + +enum Kind { + Dir(BTreeMap), + File(Body), +} + +struct Node { + parent: u64, + name: String, + kind: Kind, +} + +struct FossilFs { + path: PathBuf, + data: Vec, + blocks: Vec, + seg_blocks: usize, + cache: BTreeMap>, + nodes: HashMap, + next: u64, + dirty: bool, + uid: u32, + gid: u32, + sink: Sink, +} + +impl FossilFs { + fn open(path: &str, sink: Sink) -> io::Result { + let data = fs::read(path)?; + let parts = container::read_lazy(&data)?.into_parts(); + + if parts.ext != "/" { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "not a directory fossil (nothing to mount)", + )); + } + + if parts.meta.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "directory fossil has no manifest", + )); + } + + let entries = dir::read(&parts.meta)?; + + let mut me = FossilFs { + path: PathBuf::from(path), + blocks: parts.blocks, + seg_blocks: parts.seg_blocks, + data, + cache: BTreeMap::new(), + nodes: HashMap::new(), + next: ROOT + 1, + dirty: false, + uid: unsafe { libc::getuid() }, + gid: unsafe { libc::getgid() }, + sink, + }; + + me.nodes.insert( + ROOT, + Node { + parent: ROOT, + name: "/".into(), + kind: Kind::Dir(BTreeMap::new()), + }, + ); + + for e in &entries { + me.insert_file( + &e.path, + Body::Stored { + offset: e.offset, + len: e.len, + crc: e.crc, + }, + ); + } + + Ok(me) + } + + fn ensure_dir(&mut self, parent: u64, name: &str) -> u64 { + if let Some(Node { + kind: Kind::Dir(children), + .. + }) = self.nodes.get(&parent) + { + if let Some(&ino) = children.get(name) { + return ino; + } + } + + let ino = self.next; + self.next += 1; + self.nodes.insert( + ino, + Node { + parent, + name: name.to_string(), + kind: Kind::Dir(BTreeMap::new()), + }, + ); + if let Some(Node { + kind: Kind::Dir(children), + .. + }) = self.nodes.get_mut(&parent) + { + children.insert(name.to_string(), ino); + } + ino + } + + fn insert_file(&mut self, path: &str, body: Body) { + let comps: Vec = path + .split('/') + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + .collect(); + + if comps.is_empty() { + return; + } + + let mut cur = ROOT; + for comp in &comps[..comps.len() - 1] { + cur = self.ensure_dir(cur, comp); + } + + let name = comps.last().unwrap().clone(); + let ino = self.next; + self.next += 1; + self.nodes.insert( + ino, + Node { + parent: cur, + name: name.clone(), + kind: Kind::File(body), + }, + ); + if let Some(Node { + kind: Kind::Dir(children), + .. + }) = self.nodes.get_mut(&cur) + { + children.insert(name, ino); + } + } + + fn path_of(&self, ino: u64) -> String { + let mut parts = Vec::new(); + let mut cur = ino; + while cur != ROOT { + match self.nodes.get(&cur) { + Some(node) => { + parts.push(node.name.clone()); + cur = node.parent; + } + None => break, + } + } + parts.reverse(); + parts.join("/") + } + + fn note(&self, event: String) { + match &self.sink { + Sink::None => {} + Sink::Ring(log) => { + let mut log = log.lock().unwrap(); + log.push_back(event); + while log.len() > 5 { + log.pop_front(); + } + } + Sink::Lines(start) => { + eprintln!(" +{:.3}s {}", start.elapsed().as_secs_f64(), event); + } + } + } + + fn child(&self, parent: u64, name: &str) -> Option { + match self.nodes.get(&parent) { + Some(Node { + kind: Kind::Dir(children), + .. + }) => children.get(name).copied(), + _ => None, + } + } + + fn materialize(&mut self, ino: u64) -> io::Result<()> { + let (offset, len, want) = match self.nodes.get(&ino) { + Some(Node { + kind: Kind::File(Body::Stored { offset, len, crc }), + .. + }) => (*offset, *len, *crc), + _ => return Ok(()), + }; + + let bytes = container::decode_range( + &self.data, + &self.blocks, + self.seg_blocks, + &mut self.cache, + offset, + len, + )?; + + if let Some(want) = want { + if crc::crc32(&bytes) != want { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "file checksum mismatch in archive", + )); + } + } + + self.note(format!("decode {} · {} B", self.path_of(ino), bytes.len())); + + if let Some(node) = self.nodes.get_mut(&ino) { + node.kind = Kind::File(Body::Loaded(bytes)); + } + Ok(()) + } + + fn size_of(&self, ino: u64) -> u64 { + match self.nodes.get(&ino) { + Some(Node { + kind: Kind::File(Body::Stored { len, .. }), + .. + }) => *len as u64, + Some(Node { + kind: Kind::File(Body::Loaded(b)), + .. + }) => b.len() as u64, + _ => 0, + } + } + + fn attr(&self, ino: u64) -> FileAttr { + let is_dir = matches!( + self.nodes.get(&ino), + Some(Node { + kind: Kind::Dir(_), + .. + }) + ); + + let size = self.size_of(ino); + let now = SystemTime::now(); + + FileAttr { + ino, + size, + blocks: size.div_ceil(512), + atime: now, + mtime: now, + ctime: now, + crtime: now, + kind: if is_dir { + FileType::Directory + } else { + FileType::RegularFile + }, + perm: if is_dir { 0o755 } else { 0o644 }, + nlink: if is_dir { 2 } else { 1 }, + uid: self.uid, + gid: self.gid, + rdev: 0, + blksize: 512, + flags: 0, + } + } + + fn collect(&mut self, ino: u64, prefix: &str, out: &mut Vec<(String, Vec)>) { + let children = match self.nodes.get(&ino) { + Some(Node { + kind: Kind::Dir(c), .. + }) => c.clone(), + _ => return, + }; + + for (name, child) in children { + let path = if prefix.is_empty() { + name.clone() + } else { + format!("{prefix}/{name}") + }; + + let is_file = matches!( + self.nodes.get(&child), + Some(Node { + kind: Kind::File(_), + .. + }) + ); + + if is_file { + if self.materialize(child).is_err() { + continue; + } + if let Some(Node { + kind: Kind::File(Body::Loaded(b)), + .. + }) = self.nodes.get(&child) + { + out.push((path, b.clone())); + } + } else { + self.collect(child, &path, out); + } + } + } + + fn repack(&mut self) -> io::Result<()> { + let mut files: Vec<(String, Vec)> = Vec::new(); + self.collect(ROOT, "", &mut files); + files.sort_by(|a, b| a.0.cmp(&b.0)); + + let (meta, payload) = dir::pack(&files); + let bytes = container::write_progress_meta(&payload, "/", &meta, None, false); + + let mut tmp = self.path.clone().into_os_string(); + tmp.push(".tmp"); + let tmp = PathBuf::from(tmp); + + fs::write(&tmp, &bytes)?; + fs::rename(&tmp, &self.path)?; + self.dirty = false; + Ok(()) + } +} + +impl Drop for FossilFs { + fn drop(&mut self) { + if self.dirty { + let _ = self.repack(); + } + } +} + +impl Filesystem for FossilFs { + fn destroy(&mut self) { + if self.dirty { + if let Err(e) = self.repack() { + error!("failed to write archive on unmount: {}", e); + } + } + } + + fn lookup(&mut self, _req: &Request<'_>, parent: u64, name: &OsStr, reply: ReplyEntry) { + match self.child(parent, &name.to_string_lossy()) { + Some(ino) => reply.entry(&TTL, &self.attr(ino), 0), + None => reply.error(libc::ENOENT), + } + } + + fn getattr(&mut self, _req: &Request<'_>, ino: u64, _fh: Option, reply: ReplyAttr) { + if self.nodes.contains_key(&ino) { + reply.attr(&TTL, &self.attr(ino)); + } else { + reply.error(libc::ENOENT); + } + } + + fn setattr( + &mut self, + _req: &Request<'_>, + ino: u64, + _mode: Option, + _uid: Option, + _gid: Option, + size: Option, + _atime: Option, + _mtime: Option, + _ctime: Option, + _fh: Option, + _crtime: Option, + _chgtime: Option, + _bkuptime: Option, + _flags: Option, + reply: ReplyAttr, + ) { + if let Some(new_len) = size { + if self.materialize(ino).is_err() { + reply.error(libc::EIO); + return; + } + if let Some(Node { + kind: Kind::File(Body::Loaded(buf)), + .. + }) = self.nodes.get_mut(&ino) + { + buf.resize(new_len as usize, 0); + self.dirty = true; + } + } + + if self.nodes.contains_key(&ino) { + reply.attr(&TTL, &self.attr(ino)); + } else { + reply.error(libc::ENOENT); + } + } + + fn open(&mut self, _req: &Request<'_>, _ino: u64, _flags: i32, reply: ReplyOpen) { + reply.opened(0, 0); + } + + fn read( + &mut self, + _req: &Request<'_>, + ino: u64, + _fh: u64, + offset: i64, + size: u32, + _flags: i32, + _lock_owner: Option, + reply: ReplyData, + ) { + if self.materialize(ino).is_err() { + reply.error(libc::EIO); + return; + } + + if let Some(Node { + kind: Kind::File(Body::Loaded(b)), + .. + }) = self.nodes.get(&ino) + { + let start = (offset as usize).min(b.len()); + let end = (start + size as usize).min(b.len()); + let chunk = end - start; + reply.data(&b[start..end]); + self.note(format!("read {} · {} B", self.path_of(ino), chunk)); + } else { + reply.error(libc::EISDIR); + } + } + + fn write( + &mut self, + _req: &Request<'_>, + ino: u64, + _fh: u64, + offset: i64, + data: &[u8], + _write_flags: u32, + _flags: i32, + _lock_owner: Option, + reply: ReplyWrite, + ) { + if self.materialize(ino).is_err() { + reply.error(libc::EIO); + return; + } + + let off = offset as usize; + let wrote = if let Some(Node { + kind: Kind::File(Body::Loaded(buf)), + .. + }) = self.nodes.get_mut(&ino) + { + let end = off + data.len(); + if buf.len() < end { + buf.resize(end, 0); + } + buf[off..end].copy_from_slice(data); + true + } else { + false + }; + + if wrote { + self.dirty = true; + self.note(format!("write {} · {} B", self.path_of(ino), data.len())); + reply.written(data.len() as u32); + } else { + reply.error(libc::EISDIR); + } + } + + fn create( + &mut self, + _req: &Request<'_>, + parent: u64, + name: &OsStr, + _mode: u32, + _umask: u32, + _flags: i32, + reply: ReplyCreate, + ) { + let name = name.to_string_lossy().into_owned(); + + if !matches!( + self.nodes.get(&parent), + Some(Node { + kind: Kind::Dir(_), + .. + }) + ) { + reply.error(libc::ENOTDIR); + return; + } + + let ino = match self.child(parent, &name) { + Some(existing) => { + if let Some(node) = self.nodes.get_mut(&existing) { + node.kind = Kind::File(Body::Loaded(Vec::new())); + } + existing + } + None => { + let ino = self.next; + self.next += 1; + self.nodes.insert( + ino, + Node { + parent, + name: name.clone(), + kind: Kind::File(Body::Loaded(Vec::new())), + }, + ); + if let Some(Node { + kind: Kind::Dir(children), + .. + }) = self.nodes.get_mut(&parent) + { + children.insert(name, ino); + } + ino + } + }; + + self.dirty = true; + self.note(format!("create {}", self.path_of(ino))); + reply.created(&TTL, &self.attr(ino), 0, 0, 0); + } + + fn mkdir( + &mut self, + _req: &Request<'_>, + parent: u64, + name: &OsStr, + _mode: u32, + _umask: u32, + reply: ReplyEntry, + ) { + let name = name.to_string_lossy().into_owned(); + + if self.child(parent, &name).is_some() { + reply.error(libc::EEXIST); + return; + } + + let ino = self.ensure_dir(parent, &name); + self.note(format!("mkdir {}", self.path_of(ino))); + reply.entry(&TTL, &self.attr(ino), 0); + } + + fn unlink(&mut self, _req: &Request<'_>, parent: u64, name: &OsStr, reply: ReplyEmpty) { + let name = name.to_string_lossy().into_owned(); + + match self.child(parent, &name) { + Some(ino) + if matches!( + self.nodes.get(&ino), + Some(Node { + kind: Kind::File(_), + .. + }) + ) => + { + self.nodes.remove(&ino); + if let Some(Node { + kind: Kind::Dir(children), + .. + }) = self.nodes.get_mut(&parent) + { + children.remove(&name); + } + self.dirty = true; + self.note(format!("rm {}", name)); + reply.ok(); + } + Some(_) => reply.error(libc::EISDIR), + None => reply.error(libc::ENOENT), + } + } + + fn rmdir(&mut self, _req: &Request<'_>, parent: u64, name: &OsStr, reply: ReplyEmpty) { + let name = name.to_string_lossy().into_owned(); + + match self.child(parent, &name) { + Some(ino) => match self.nodes.get(&ino) { + Some(Node { + kind: Kind::Dir(children), + .. + }) if children.is_empty() => { + self.nodes.remove(&ino); + if let Some(Node { + kind: Kind::Dir(children), + .. + }) = self.nodes.get_mut(&parent) + { + children.remove(&name); + } + self.dirty = true; + reply.ok(); + } + Some(Node { + kind: Kind::Dir(_), .. + }) => reply.error(libc::ENOTEMPTY), + _ => reply.error(libc::ENOTDIR), + }, + None => reply.error(libc::ENOENT), + } + } + + fn rename( + &mut self, + _req: &Request<'_>, + parent: u64, + name: &OsStr, + newparent: u64, + newname: &OsStr, + _flags: u32, + reply: ReplyEmpty, + ) { + let name = name.to_string_lossy().into_owned(); + let newname = newname.to_string_lossy().into_owned(); + + let ino = match self.child(parent, &name) { + Some(ino) => ino, + None => { + reply.error(libc::ENOENT); + return; + } + }; + + if !matches!( + self.nodes.get(&newparent), + Some(Node { + kind: Kind::Dir(_), + .. + }) + ) { + reply.error(libc::ENOTDIR); + return; + } + + if let Some(old) = self.child(newparent, &newname) { + self.nodes.remove(&old); + if let Some(Node { + kind: Kind::Dir(children), + .. + }) = self.nodes.get_mut(&newparent) + { + children.remove(&newname); + } + } + + if let Some(Node { + kind: Kind::Dir(children), + .. + }) = self.nodes.get_mut(&parent) + { + children.remove(&name); + } + if let Some(Node { + kind: Kind::Dir(children), + .. + }) = self.nodes.get_mut(&newparent) + { + children.insert(newname.clone(), ino); + } + if let Some(node) = self.nodes.get_mut(&ino) { + node.parent = newparent; + node.name = newname; + } + + self.dirty = true; + reply.ok(); + } + + fn readdir( + &mut self, + _req: &Request<'_>, + ino: u64, + _fh: u64, + offset: i64, + mut reply: ReplyDirectory, + ) { + let (children, parent) = match self.nodes.get(&ino) { + Some(Node { + kind: Kind::Dir(c), + parent, + .. + }) => (c.clone(), *parent), + _ => { + reply.error(libc::ENOTDIR); + return; + } + }; + + let mut rows: Vec<(u64, FileType, String)> = Vec::new(); + rows.push((ino, FileType::Directory, ".".into())); + rows.push((parent, FileType::Directory, "..".into())); + for (cname, cino) in &children { + let ft = match self.nodes.get(cino) { + Some(Node { + kind: Kind::Dir(_), .. + }) => FileType::Directory, + _ => FileType::RegularFile, + }; + rows.push((*cino, ft, cname.clone())); + } + + for (i, (cino, ft, cname)) in rows.into_iter().enumerate().skip(offset as usize) { + if reply.add(cino, (i + 1) as i64, ft, &cname) { + break; + } + } + reply.ok(); + } + + fn flush( + &mut self, + _req: &Request<'_>, + _ino: u64, + _fh: u64, + _lock_owner: u64, + reply: ReplyEmpty, + ) { + reply.ok(); + } + + fn fsync( + &mut self, + _req: &Request<'_>, + _ino: u64, + _fh: u64, + _datasync: bool, + reply: ReplyEmpty, + ) { + reply.ok(); + } +} + +static STOP: AtomicBool = AtomicBool::new(false); + +extern "C" fn on_signal(_: libc::c_int) { + STOP.store(true, Ordering::SeqCst); +} + +const DOTS: [&str; 3] = [" ·", " ··", "···"]; + +struct Serve { + done: Arc, + handle: Option>, +} + +impl Serve { + fn start(msg: String, log: Option) -> Self { + let done = Arc::new(AtomicBool::new(false)); + + if !io::stderr().is_terminal() { + return Serve { done, handle: None }; + } + + let flag = done.clone(); + let handle = thread::spawn(move || { + let mut i = 0; + let mut drawn = 1usize; + while !flag.load(Ordering::Relaxed) { + if drawn > 1 { + eprint!("\x1b[{}A", drawn - 1); + } + eprint!("\r\x1b[J"); + eprint!( + " {} {}", + paint(DOTS[i % DOTS.len()], "38;5;173"), + paint(&msg, "38;5;173") + ); + + if let Some(log) = &log { + let events: Vec = log.lock().unwrap().iter().cloned().collect(); + for e in &events { + eprint!("\n\x1b[2K {}", e.clone().dim()); + } + for _ in events.len()..5 { + eprint!("\n\x1b[2K"); + } + drawn = 6; + } else { + drawn = 1; + } + + let _ = io::stderr().flush(); + i += 1; + thread::sleep(Duration::from_millis(400)); + } + + if drawn > 1 { + eprint!("\x1b[{}A", drawn - 1); + } + eprint!("\r\x1b[J"); + let _ = io::stderr().flush(); + }); + + Serve { + done, + handle: Some(handle), + } + } + + fn stop(self) { + self.done.store(true, Ordering::Relaxed); + if let Some(h) = self.handle { + let end = Instant::now() + Duration::from_millis(400); + while !h.is_finished() && Instant::now() < end { + thread::sleep(Duration::from_millis(20)); + } + if h.is_finished() { + let _ = h.join(); + } + } + } +} + +fn force_unmount(mountpoint: &Path) { + let Ok(c) = CString::new(mountpoint.as_os_str().as_bytes()) else { + return; + }; + + #[cfg(target_os = "macos")] + unsafe { + libc::unmount(c.as_ptr(), libc::MNT_FORCE); + } + + #[cfg(not(target_os = "macos"))] + unsafe { + libc::umount2(c.as_ptr(), libc::MNT_DETACH); + } +} + +pub fn run(archive: &str, mountpoint: &str, verbose: bool, log_lines: bool) { + let ring: Option = + (verbose && !log_lines).then(|| Arc::new(Mutex::new(VecDeque::new()))); + + let sink = if log_lines { + Sink::Lines(Instant::now()) + } else if let Some(r) = &ring { + Sink::Ring(r.clone()) + } else { + Sink::None + }; + + let fs = match FossilFs::open(archive, sink) { + Ok(fs) => fs, + Err(e) => { + error!("{}", e); + return; + } + }; + + let mp = Path::new(mountpoint); + if !mp.is_dir() { + error!("mount point is not a directory: {}", mountpoint); + return; + } + + #[cfg_attr(not(target_os = "macos"), allow(unused_mut))] + let mut options = vec![ + MountOption::FSName("fossil".into()), + MountOption::Subtype("fossil".into()), + MountOption::DefaultPermissions, + MountOption::AutoUnmount, + ]; + + #[cfg(target_os = "macos")] + options.push(MountOption::CUSTOM("noappledouble".into())); + + unsafe { + libc::signal(libc::SIGINT, on_signal as *const () as libc::sighandler_t); + libc::signal(libc::SIGTERM, on_signal as *const () as libc::sighandler_t); + } + + let session = match fuser::spawn_mount2(fs, mp, &options) { + Ok(s) => s, + Err(e) => { + error!("mount failed: {}", e); + return; + } + }; + + println!( + " {} {} {}", + archive.accent(), + "→".bold(), + mountpoint.accent() + ); + + let serve = if log_lines { + println!(" serving {} · ctrl-c to commit", mountpoint); + None + } else { + Some(Serve::start( + format!("serving {} · ctrl-c to commit", mountpoint), + ring, + )) + }; + + while !STOP.load(Ordering::SeqCst) && !session.guard.is_finished() { + thread::sleep(Duration::from_millis(200)); + } + + if let Some(serve) = serve { + serve.stop(); + } + println!(" unmounting…"); + + let joiner = thread::spawn(move || session.join()); + let watch = mp.to_path_buf(); + let started = Instant::now(); + + while !joiner.is_finished() { + if started.elapsed() >= Duration::from_secs(2) { + force_unmount(&watch); + thread::sleep(Duration::from_millis(500)); + } else { + thread::sleep(Duration::from_millis(100)); + } + } + + let _ = joiner.join(); + println!(" unmounted"); +} + +pub fn help() -> Vec { + vec![ + "fossil mount".header(), + "mount a directory fossil as a live filesystem".bold(), + "".into(), + "usage".header(), + " fossil mount [--verbose] ".into(), + "".into(), + "arguments".header(), + " directory archive to mount".into(), + " empty directory to mount onto".into(), + "".into(), + "options".header(), + " --verbose, -v rolling display of the last few filesystem events".into(), + " --log, -l print every event line-by-line, for diagnosing".into(), + "".into(), + "notes".header(), + " reads decode on demand; writes are staged in memory".into(), + " changes are written back to the archive on unmount".into(), + " stop with ctrl-c (or unmount the volume)".into(), + " rename is unreliable on macFUSE; copy then delete instead".into(), + "".into(), + "examples".header(), + " fossil mount project.fossil mnt/".into(), + " fossil mount --verbose project.fossil mnt/".into(), + ] +} diff --git a/src/commands/take.rs b/src/commands/take.rs index c7fdb91..0283ab7 100644 --- a/src/commands/take.rs +++ b/src/commands/take.rs @@ -44,9 +44,9 @@ fn take(input: &str, inner_path: Option<&str>, trust: bool) -> io::Result, trust: bool) -> io::Result, trust: bool) -> io::Result, trust: bool) -> io::Result, trust: bool) -> io::Result &'static str { match model { @@ -33,13 +35,27 @@ pub fn model_name(model: u8) -> &'static str { WORD => "WORD", LZR => "LZR", SIGNAL => "SIGNAL", + LZR2 => "LZR2", + BWTM2 => "BWTM2", _ => "RAW", } } const FAST_CHAIN: usize = 8; +pub const SEGMENT_BLOCKS: usize = 64; + pub fn encode_block(input: &[u8], start: usize, end: usize, fast: bool) -> (u8, Vec) { + encode_block_seg(input, start, end, 0, fast) +} + +pub fn encode_block_seg( + input: &[u8], + start: usize, + end: usize, + seg: usize, + fast: bool, +) -> (u8, Vec) { let bytes = &input[start..end]; let mut model = RAW; let mut best = bytes.to_vec(); @@ -62,7 +78,8 @@ pub fn encode_block(input: &[u8], start: usize, end: usize, fast: bool) -> (u8, best = huff; } - let wstart = start.saturating_sub(lz::HISTORY); + let floor = if seg == 0 { 0 } else { start - (start % seg) }; + let wstart = start.saturating_sub(lz::HISTORY).max(floor); let combined = &input[wstart..end]; let emit = start - wstart; @@ -83,16 +100,16 @@ pub fn encode_block(input: &[u8], start: usize, end: usize, fast: bool) -> (u8, best = lzh_enc; } - let lzr_enc = lzr::encode_tokens(combined, emit, &tokens); + let lzr_enc = lzr::encode_tokens2(combined, emit, &tokens); if lzr_enc.len() < best.len() { - model = LZR; + model = LZR2; best = lzr_enc; } if !fast { - let bwtm = bwtm::encode(bytes); + let bwtm = bwtm::encode2(bytes); if bwtm.len() < best.len() { - model = BWTM; + model = BWTM2; best = bwtm; } @@ -145,7 +162,9 @@ pub fn decode_block(model: u8, payload: &[u8], orig_len: usize, history: &[u8]) LZ => lz::decode_windowed(payload, orig_len, history), LZH => lzh::decode_windowed(payload, orig_len, history), LZR => lzr::decode_windowed(payload, orig_len, history), + LZR2 => lzr::decode_windowed2(payload, orig_len, history), BWTM => bwtm::decode(payload, orig_len), + BWTM2 => bwtm::decode2(payload, orig_len), RANGE => range::decode(payload, orig_len), PPM => ppm::decode(payload, orig_len), GEN => generator::decode(payload), diff --git a/src/core/container.rs b/src/core/container.rs index a30f7df..7630aea 100644 --- a/src/core/container.rs +++ b/src/core/container.rs @@ -1,7 +1,8 @@ +use std::collections::BTreeMap; use std::io; use std::sync::atomic::{AtomicUsize, Ordering}; -use super::block::{RAW, decode_block, encode_block}; +use super::block::{RAW, SEGMENT_BLOCKS, decode_block, encode_block_seg}; use super::crc; use super::image; use super::varint; @@ -86,7 +87,13 @@ pub fn write_progress_meta( p.total.store(n, Ordering::Relaxed); } - let encoded = encode_blocks(block_src, progress, fast); + let seg = if ext == "/" { + SEGMENT_BLOCKS * BLOCK_SIZE + } else { + 0 + }; + + let encoded = encode_blocks(block_src, seg, progress, fast); let block_lens: Vec = block_src.chunks(BLOCK_SIZE).map(|c| c.len()).collect(); assemble(bytes, ext, filtered.is_some(), meta, &block_lens, &encoded) @@ -107,10 +114,12 @@ pub fn assemble( let mut blocked = header(MODE_BLOCKS, filter, ext_bytes, orig.len(), crc, meta); varint::write(&mut blocked, encoded.len()); + if let Some(last) = block_lens.last() { + varint::write(&mut blocked, *last); + } - for (i, (model, payload)) in encoded.iter().enumerate() { + for (model, payload) in encoded.iter() { blocked.push(*model); - varint::write(&mut blocked, block_lens[i]); varint::write(&mut blocked, payload.len()); blocked.extend_from_slice(payload); } @@ -129,17 +138,23 @@ fn encode_one( input: &[u8], start: usize, end: usize, + seg: usize, progress: Option<&Progress>, fast: bool, ) -> (u8, Vec) { - let out = encode_block(input, start, end, fast); + let out = encode_block_seg(input, start, end, seg, fast); if let Some(p) = progress { p.done.fetch_add(1, Ordering::Relaxed); } out } -fn encode_blocks(input: &[u8], progress: Option<&Progress>, fast: bool) -> Vec<(u8, Vec)> { +fn encode_blocks( + input: &[u8], + seg: usize, + progress: Option<&Progress>, + fast: bool, +) -> Vec<(u8, Vec)> { let n = input.len(); let n_blocks = if n == 0 { 0 } else { n.div_ceil(BLOCK_SIZE) }; @@ -152,7 +167,7 @@ fn encode_blocks(input: &[u8], progress: Option<&Progress>, fast: bool) -> Vec<( .map(|k| { let start = k * BLOCK_SIZE; let end = (start + BLOCK_SIZE).min(n); - encode_one(input, start, end, progress, fast) + encode_one(input, start, end, seg, progress, fast) }) .collect(); } @@ -171,7 +186,7 @@ fn encode_blocks(input: &[u8], progress: Option<&Progress>, fast: bool) -> Vec<( } let start = k * BLOCK_SIZE; let end = (start + BLOCK_SIZE).min(n); - local.push((k, encode_one(input, start, end, progress, fast))); + local.push((k, encode_one(input, start, end, seg, progress, fast))); } local }) @@ -271,10 +286,19 @@ pub fn read(data: &[u8]) -> io::Result { } MODE_BLOCKS => { let n_blocks = c.varint()?; + let last_len = if version >= 2 && n_blocks > 0 { + c.varint()? + } else { + 0 + }; let mut blocks = Vec::with_capacity(n_blocks); - for _ in 0..n_blocks { + for i in 0..n_blocks { let model = c.u8()?; - let orig_len = c.varint()?; + let orig_len = if version >= 2 { + if i == n_blocks - 1 { last_len } else { BLOCK_SIZE } + } else { + c.varint()? + }; let pay_len = c.varint()?; let payload = c.take(pay_len)?.to_vec(); blocks.push(Block { @@ -305,9 +329,20 @@ pub fn read(data: &[u8]) -> io::Result { impl Container { pub fn decode(&self) -> Vec { + let seg = if self.ext == "/" { + SEGMENT_BLOCKS * BLOCK_SIZE + } else { + 0 + }; + let mut out = Vec::with_capacity(self.orig_size); - for b in &self.blocks { - let decoded = decode_block(b.model, &b.payload, b.orig_len, &out); + for (i, b) in self.blocks.iter().enumerate() { + let floor = if seg == 0 { + 0 + } else { + ((i / SEGMENT_BLOCKS) * seg).min(out.len()) + }; + let decoded = decode_block(b.model, &b.payload, b.orig_len, &out[floor..]); out.extend_from_slice(&decoded); } @@ -318,3 +353,227 @@ impl Container { return out; } } + +pub struct BlockRef { + pub model: u8, + pub orig_len: usize, + pub payload_offset: usize, + pub payload_len: usize, +} + +pub struct LazyContainer<'a> { + pub ext: String, + pub orig_size: usize, + pub crc: u32, + pub filter: u8, + pub meta: Vec, + pub blocks: Vec, + data: &'a [u8], + seg_blocks: usize, + cache: BTreeMap>, +} + +pub fn read_lazy(data: &[u8]) -> io::Result> { + let mut c = Cursor { data, pos: 0 }; + + if c.take(4)? != MAGIC { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "not a fossil file (bad magic)", + )); + } + let version = c.u8()?; + if version > VERSION { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("unsupported version {}", version), + )); + } + + let mode = c.u8()?; + let filter = c.u8()?; + let ext_len = c.u8()? as usize; + let ext = String::from_utf8_lossy(c.take(ext_len)?).into_owned(); + let orig_size = c.varint()?; + let crc = c.u32le()?; + + let meta = if version >= 2 { + let meta_len = c.varint()?; + c.take(meta_len)?.to_vec() + } else { + Vec::new() + }; + + let mut blocks = Vec::new(); + + match mode { + MODE_STORED => { + let payload_offset = c.pos; + c.take(orig_size)?; + blocks.reserve(orig_size.div_ceil(BLOCK_SIZE)); + let mut off = 0; + while off < orig_size { + let len = (orig_size - off).min(BLOCK_SIZE); + blocks.push(BlockRef { + model: RAW, + orig_len: len, + payload_offset: payload_offset + off, + payload_len: len, + }); + off += len; + } + } + MODE_BLOCKS => { + let n_blocks = c.varint()?; + let last_len = if version >= 2 && n_blocks > 0 { + c.varint()? + } else { + 0 + }; + blocks.reserve(n_blocks); + for i in 0..n_blocks { + let model = c.u8()?; + let orig_len = if version >= 2 { + if i == n_blocks - 1 { last_len } else { BLOCK_SIZE } + } else { + c.varint()? + }; + let payload_len = c.varint()?; + let payload_offset = c.pos; + c.take(payload_len)?; + blocks.push(BlockRef { + model, + orig_len, + payload_offset, + payload_len, + }); + } + } + other => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("unknown container mode {}", other), + )); + } + } + + let seg_blocks = if ext == "/" { SEGMENT_BLOCKS } else { 0 }; + + Ok(LazyContainer { + ext, + orig_size, + crc, + filter, + meta, + blocks, + data, + seg_blocks, + cache: BTreeMap::new(), + }) +} + +impl<'a> LazyContainer<'a> { + pub fn read_range(&mut self, offset: usize, len: usize) -> io::Result> { + if offset.saturating_add(len) > self.orig_size { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "read range beyond archive", + )); + } + decode_range( + self.data, + &self.blocks, + self.seg_blocks, + &mut self.cache, + offset, + len, + ) + } + + pub fn into_parts(self) -> LazyParts { + LazyParts { + ext: self.ext, + orig_size: self.orig_size, + crc: self.crc, + filter: self.filter, + meta: self.meta, + blocks: self.blocks, + seg_blocks: self.seg_blocks, + } + } +} + +pub struct LazyParts { + pub ext: String, + pub orig_size: usize, + pub crc: u32, + pub filter: u8, + pub meta: Vec, + pub blocks: Vec, + pub seg_blocks: usize, +} + +pub fn decode_range( + data: &[u8], + blocks: &[BlockRef], + seg_blocks: usize, + cache: &mut BTreeMap>, + offset: usize, + len: usize, +) -> io::Result> { + if len == 0 { + return Ok(Vec::new()); + } + + let end = offset + .checked_add(len) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "read range overflow"))?; + + let seg_start = |idx: usize| { + if seg_blocks == 0 { + 0 + } else { + (idx / seg_blocks) * seg_blocks + } + }; + + let first = offset / BLOCK_SIZE; + let last = (end - 1) / BLOCK_SIZE; + + if last >= blocks.len() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "read range beyond archive", + )); + } + + let from = seg_start(first); + let base = from * BLOCK_SIZE; + + let mut buf: Vec = Vec::new(); + + for j in from..=last { + let floor = (seg_start(j) * BLOCK_SIZE - base).min(buf.len()); + + let decoded = if cache.contains_key(&j) { + cache[&j].clone() + } else { + let b = &blocks[j]; + let payload = &data[b.payload_offset..b.payload_offset + b.payload_len]; + let d = decode_block(b.model, payload, b.orig_len, &buf[floor..]); + cache.insert(j, d.clone()); + d + }; + + buf.extend_from_slice(&decoded); + } + + if end - base > buf.len() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "read range beyond archive", + )); + } + + Ok(buf[offset - base..end - base].to_vec()) +} diff --git a/src/core/dir.rs b/src/core/dir.rs index 9062681..380add4 100644 --- a/src/core/dir.rs +++ b/src/core/dir.rs @@ -1,21 +1,24 @@ use std::io; +use super::crc; use super::varint; const MAGIC: &[u8; 4] = b"FDIR"; +const MAGIC_V2: &[u8; 4] = b"FDR2"; #[derive(Debug, Clone)] pub struct Entry { pub path: String, pub offset: usize, pub len: usize, + pub crc: Option, } pub fn pack(files: &[(String, Vec)]) -> (Vec, Vec) { let mut meta = Vec::new(); let mut payload = Vec::new(); - meta.extend_from_slice(MAGIC); + meta.extend_from_slice(MAGIC_V2); varint::write(&mut meta, files.len()); for (path, bytes) in files { @@ -24,6 +27,7 @@ pub fn pack(files: &[(String, Vec)]) -> (Vec, Vec) { varint::write(&mut meta, path_bytes.len()); meta.extend_from_slice(path_bytes); varint::write(&mut meta, bytes.len()); + meta.extend_from_slice(&crc::crc32(bytes).to_le_bytes()); payload.extend_from_slice(bytes); } @@ -32,13 +36,24 @@ pub fn pack(files: &[(String, Vec)]) -> (Vec, Vec) { } pub fn read(meta: &[u8]) -> io::Result> { - if meta.len() < MAGIC.len() || &meta[..MAGIC.len()] != MAGIC { + if meta.len() < MAGIC.len() { return Err(io::Error::new( io::ErrorKind::InvalidData, "missing directory manifest", )); } + let has_crc = match &meta[..MAGIC.len()] { + m if m == MAGIC_V2 => true, + m if m == MAGIC => false, + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "missing directory manifest", + )); + } + }; + let mut pos = MAGIC.len(); let count = varint::read(meta, &mut pos); @@ -60,7 +75,26 @@ pub fn read(meta: &[u8]) -> io::Result> { let len = varint::read(meta, &mut pos); - entries.push(Entry { path, offset, len }); + let crc = if has_crc { + if pos + 4 > meta.len() { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "truncated directory manifest", + )); + } + let c = u32::from_le_bytes([meta[pos], meta[pos + 1], meta[pos + 2], meta[pos + 3]]); + pos += 4; + Some(c) + } else { + None + }; + + entries.push(Entry { + path, + offset, + len, + crc, + }); offset += len; } diff --git a/src/core/models/bwtm.rs b/src/core/models/bwtm.rs index 12df937..b22a367 100644 --- a/src/core/models/bwtm.rs +++ b/src/core/models/bwtm.rs @@ -1,3 +1,4 @@ +use super::range::{Decoder, Encoder, Model}; use super::{bwt, mtf, range}; use crate::core::varint; @@ -11,6 +12,103 @@ pub fn encode(bytes: &[u8]) -> Vec { return out; } +fn encode_run(enc: &mut Encoder, model: &mut Model, mut v: usize) { + loop { + let mut byte = v & 0x7f; + if v >= 0x80 { + byte |= 0x80; + } + enc.encode(model.cum_freq(byte), model.freq(byte), model.total()); + model.update(byte); + if v < 0x80 { + break; + } + v >>= 7; + } +} + +fn decode_run(dec: &mut Decoder, model: &mut Model) -> usize { + let mut v = 0usize; + let mut shift = 0; + loop { + let value = dec.decode_freq(model.total()); + let byte = model.find(value); + dec.update(model.cum_freq(byte), model.freq(byte)); + model.update(byte); + v |= (byte & 0x7f) << shift; + if byte & 0x80 == 0 { + break; + } + shift += 7; + } + v +} + +pub fn encode2(bytes: &[u8]) -> Vec { + let (last, primary) = bwt::forward(bytes); + let moved = mtf::forward(&last); + + let mut enc = Encoder::new(); + let mut sym_model = Model::new(); + let mut run_model = Model::new(); + + let mut i = 0; + while i < moved.len() { + let sym = moved[i] as usize; + enc.encode( + sym_model.cum_freq(sym), + sym_model.freq(sym), + sym_model.total(), + ); + sym_model.update(sym); + + if sym == 0 { + let mut run = 1; + while i + run < moved.len() && moved[i + run] == 0 { + run += 1; + } + encode_run(&mut enc, &mut run_model, run - 1); + i += run; + } else { + i += 1; + } + } + + let mut out = Vec::new(); + varint::write(&mut out, primary); + out.extend_from_slice(&enc.finish()); + return out; +} + +pub fn decode2(data: &[u8], orig_len: usize) -> Vec { + let mut pos = 0; + let primary = varint::read(data, &mut pos); + + let mut dec = Decoder::new(&data[pos..]); + let mut sym_model = Model::new(); + let mut run_model = Model::new(); + + let mut moved = Vec::with_capacity(orig_len); + while moved.len() < orig_len { + let value = dec.decode_freq(sym_model.total()); + let sym = sym_model.find(value); + dec.update(sym_model.cum_freq(sym), sym_model.freq(sym)); + sym_model.update(sym); + + if sym == 0 { + let run = decode_run(&mut dec, &mut run_model) + 1; + for _ in 0..run.min(orig_len - moved.len()) { + moved.push(0); + } + } else { + moved.push(sym as u8); + } + } + + let last = mtf::inverse(&moved); + return bwt::inverse(&last, primary); +} + pub fn decode(data: &[u8], orig_len: usize) -> Vec { let mut pos = 0; let primary = varint::read(data, &mut pos); diff --git a/src/core/models/huffman.rs b/src/core/models/huffman.rs index d5b355c..d47bfb4 100644 --- a/src/core/models/huffman.rs +++ b/src/core/models/huffman.rs @@ -100,6 +100,15 @@ fn write_table(out: &mut Vec, lengths: &[u8; 256]) { sparse.push(lengths[s]); } + let max_len = *lengths.iter().max().unwrap(); + if max_len <= 15 && sparse.len() >= 130 { + out.push(2); + for pair in lengths.chunks(2) { + out.push(pair[0] | (pair[1] << 4)); + } + return; + } + if sparse.len() < 257 { out.extend_from_slice(&sparse); } else { @@ -120,6 +129,13 @@ fn read_table(data: &[u8], pos: &mut usize) -> [u8; 256] { let end = (*pos + 256).min(data.len()); lengths[..end - *pos].copy_from_slice(&data[*pos..end]); *pos = end; + } else if mode == 2 { + let end = (*pos + 128).min(data.len()); + for (i, &b) in data[*pos..end].iter().enumerate() { + lengths[i * 2] = b & 0x0f; + lengths[i * 2 + 1] = b >> 4; + } + *pos = end; } else { let n = varint::read(data, pos); for _ in 0..n { diff --git a/src/core/models/lz.rs b/src/core/models/lz.rs index 2894556..62afe59 100644 --- a/src/core/models/lz.rs +++ b/src/core/models/lz.rs @@ -3,7 +3,7 @@ use crate::core::varint; const MIN_MATCH: usize = 3; const MAX_MATCH: usize = 258; const WINDOW: usize = 4096; -pub const HISTORY: usize = 1 << 16; +pub const HISTORY: usize = 1 << 18; fn longest_match(bytes: &[u8], pos: usize) -> (usize, usize) { let start = pos.saturating_sub(WINDOW); diff --git a/src/core/models/lzr.rs b/src/core/models/lzr.rs index cdcc98a..02febc8 100644 --- a/src/core/models/lzr.rs +++ b/src/core/models/lzr.rs @@ -126,6 +126,157 @@ pub fn encode_tokens(bytes: &[u8], emit_start: usize, tokens: &[Token]) -> Vec= 0x80 { + byte |= 0x80; + } + let m = &mut models[idx.min(models.len() - 1)]; + enc.encode(m.cum_freq(byte), m.freq(byte), m.total()); + m.update(byte); + if v < 0x80 { + break; + } + v >>= 7; + idx += 1; + } +} + +fn decode_value_ctx(dec: &mut Decoder, models: &mut [Model]) -> usize { + let mut v = 0usize; + let mut shift = 0; + let mut idx = 0; + loop { + let m = &mut models[idx.min(models.len() - 1)]; + let value = dec.decode_freq(m.total()); + let byte = m.find(value); + dec.update(m.cum_freq(byte), m.freq(byte)); + m.update(byte); + v |= (byte & 0x7f) << shift; + if byte & 0x80 == 0 { + break; + } + shift += 7; + idx += 1; + } + v +} + +pub fn encode_tokens2(bytes: &[u8], emit_start: usize, tokens: &[Token]) -> Vec { + let mut enc = Encoder::new(); + let mut flag = [Bit::new(), Bit::new()]; + let mut rep = Bit::new(); + let mut lits: Vec = (0..256).map(|_| Model::new()).collect(); + let mut len_models: Vec = (0..2).map(|_| Model::new()).collect(); + let mut dist_models: Vec = (0..3).map(|_| Model::new()).collect(); + + let mut pos = emit_start; + let mut prev_match = 0usize; + let mut last_dist = 0usize; + + for t in tokens { + match t { + Token::Lit(b) => { + flag[prev_match].encode(&mut enc, 0); + let ctx = if pos > 0 { bytes[pos - 1] as usize } else { 0 }; + let sym = *b as usize; + let m = &lits[ctx]; + enc.encode(m.cum_freq(sym), m.freq(sym), m.total()); + lits[ctx].update(sym); + pos += 1; + prev_match = 0; + } + Token::Match { dist, len } => { + flag[prev_match].encode(&mut enc, 1); + if last_dist != 0 { + if *dist == last_dist { + rep.encode(&mut enc, 1); + encode_value_ctx(&mut enc, &mut len_models, len - biglz::MIN_MATCH); + pos += len; + prev_match = 1; + continue; + } + rep.encode(&mut enc, 0); + } + encode_value_ctx(&mut enc, &mut len_models, len - biglz::MIN_MATCH); + encode_value_ctx(&mut enc, &mut dist_models, *dist); + last_dist = *dist; + pos += len; + prev_match = 1; + } + } + } + + enc.finish() +} + +pub fn decode_windowed2(data: &[u8], count: usize, history: &[u8]) -> Vec { + let seed = history.len().min(lz::HISTORY); + let mut out = Vec::with_capacity(seed + count); + out.extend_from_slice(&history[history.len() - seed..]); + + let target = seed + count; + let mut dec = Decoder::new(data); + let mut flag = [Bit::new(), Bit::new()]; + let mut rep = Bit::new(); + let mut lits: Vec = (0..256).map(|_| Model::new()).collect(); + let mut len_models: Vec = (0..2).map(|_| Model::new()).collect(); + let mut dist_models: Vec = (0..3).map(|_| Model::new()).collect(); + let mut prev_match = 0usize; + let mut last_dist = 0usize; + + while out.len() < target { + let is_match = flag[prev_match].decode(&mut dec); + if is_match == 0 { + let ctx = if out.is_empty() { + 0 + } else { + out[out.len() - 1] as usize + }; + let m = &lits[ctx]; + let value = dec.decode_freq(m.total()); + let sym = m.find(value); + dec.update(m.cum_freq(sym), m.freq(sym)); + lits[ctx].update(sym); + out.push(sym as u8); + prev_match = 0; + } else { + let dist = if last_dist != 0 && rep.decode(&mut dec) == 1 { + last_dist + } else { + let len = decode_value_ctx(&mut dec, &mut len_models) + biglz::MIN_MATCH; + let dist = decode_value_ctx(&mut dec, &mut dist_models); + last_dist = dist; + if dist == 0 || dist > out.len() { + break; + } + let start = out.len() - dist; + for x in 0..len { + let b = out[start + x]; + out.push(b); + } + prev_match = 1; + continue; + }; + + let len = decode_value_ctx(&mut dec, &mut len_models) + biglz::MIN_MATCH; + if dist == 0 || dist > out.len() { + break; + } + let start = out.len() - dist; + for x in 0..len { + let b = out[start + x]; + out.push(b); + } + prev_match = 1; + } + } + + out.split_off(seed) +} + pub fn decode(data: &[u8], count: usize) -> Vec { decode_windowed(data, count, &[]) } diff --git a/src/main.rs b/src/main.rs index 0cc1747..1130c87 100644 --- a/src/main.rs +++ b/src/main.rs @@ -143,6 +143,31 @@ fn dispatch() { commands::take::run(&args[2], inner, output, trust); } + #[cfg(unix)] + "mount" => { + let mut verbose = false; + let mut log = false; + let mut pos: Vec<&String> = Vec::new(); + for a in &args[2..] { + match a.as_str() { + "--verbose" | "-v" => verbose = true, + "--log" | "-l" => log = true, + _ => pos.push(a), + } + } + + if pos.len() != 2 { + error!("mount expects an archive and a mountpoint"); + info!( + "fossil mount [--verbose] [--log] ", + usage = true + ); + return; + } + + commands::mount::run(pos[0], pos[1], verbose, log); + } + "explain" | "why" | "whats" | "describe" => { let mut block: Option = None; let mut pos: Vec<&String> = Vec::new(); @@ -213,6 +238,8 @@ fn dispatch() { "unpack" => ui::subcommand(commands::unpack::help()), "list" => ui::subcommand(commands::list::help()), "take" => ui::subcommand(commands::take::help()), + #[cfg(unix)] + "mount" => ui::subcommand(commands::mount::help()), "inspect" => ui::subcommand(commands::inspect::help()), "map" => ui::subcommand(commands::map::help()), "explain" => ui::subcommand(commands::explain::help()), @@ -271,8 +298,8 @@ fn dispatch() { } const COMMANDS: &[&str] = &[ - "pack", "lift", "unpack", "list", "take", "inspect", "map", "explain", "verify", "update", - "help", + "pack", "lift", "unpack", "list", "take", "mount", "inspect", "map", "explain", "verify", + "update", "help", ]; fn closest(input: &str) -> Option<&'static str> { @@ -488,7 +515,7 @@ fn help() { // ); // n!(); println!( - "{}", + " {}", "need help with a specific command? run `fossil help `" .dim() .italic() diff --git a/tests/container.rs b/tests/container.rs index ada42e4..13d3ba2 100644 --- a/tests/container.rs +++ b/tests/container.rs @@ -48,3 +48,51 @@ fn incompressible_data_uses_stored_form() { fn tiny_file_roundtrips() { roundtrip(b"hi", "txt"); } + +#[test] +fn reads_legacy_v1_block_framing() { + let orig = b"abcdefghij"; + let crc = fossil::core::crc::crc32(orig); + + let mut v1 = Vec::new(); + v1.extend_from_slice(b"FOSL"); + v1.push(1); + v1.push(0); + v1.push(0); + v1.push(3); + v1.extend_from_slice(b"txt"); + v1.push(10); + v1.extend_from_slice(&crc.to_le_bytes()); + v1.push(2); + v1.push(0); + v1.push(6); + v1.push(6); + v1.extend_from_slice(b"abcdef"); + v1.push(0); + v1.push(4); + v1.push(4); + v1.extend_from_slice(b"ghij"); + + let c = fossil::core::container::read(&v1).unwrap(); + assert_eq!(c.ext, "txt"); + assert_eq!(c.blocks.len(), 2); + assert_eq!(c.blocks[0].orig_len, 6); + assert_eq!(c.decode(), orig); + + let mut lazy = fossil::core::container::read_lazy(&v1).unwrap(); + assert_eq!(lazy.blocks[1].orig_len, 4); + assert_eq!(lazy.read_range(2, 4).unwrap(), b"cdef"); +} + +#[test] +fn v2_framing_round_trips_multi_block() { + let data: Vec = (0..20000u32).map(|i| (i % 251) as u8).collect(); + let packed = fossil::core::container::write(&data, "bin"); + assert_eq!(packed[4], 2); + + let c = fossil::core::container::read(&packed).unwrap(); + assert_eq!(c.decode(), data); + + let mut lazy = fossil::core::container::read_lazy(&packed).unwrap(); + assert_eq!(lazy.read_range(8000, 5000).unwrap(), &data[8000..13000]); +} diff --git a/tests/dir.rs b/tests/dir.rs index ada0833..ce57708 100644 --- a/tests/dir.rs +++ b/tests/dir.rs @@ -1,6 +1,6 @@ use fossil::core::{ container::{read, write_progress_meta}, - dir, + crc, dir, }; fn sample_files() -> Vec<(String, Vec)> { @@ -112,6 +112,44 @@ fn rejects_bad_manifest_magic() { assert!(dir::read(b"XXXXrest of junk").is_err()); } +#[test] +fn manifest_stores_per_file_crc() { + let files = sample_files(); + + let (meta, _payload) = dir::pack(&files); + let entries = dir::read(&meta).unwrap(); + + for (entry, (_, contents)) in entries.iter().zip(files.iter()) { + assert_eq!(entry.crc, Some(crc::crc32(contents))); + } +} + +#[test] +fn legacy_fdir_manifest_still_reads() { + let mut meta = Vec::new(); + meta.extend_from_slice(b"FDIR"); + meta.push(2); + meta.push(1); + meta.extend_from_slice(b"a"); + meta.push(3); + meta.push(2); + meta.extend_from_slice(b"bb"); + meta.push(5); + + let entries = dir::read(&meta).unwrap(); + + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].path, "a"); + assert_eq!(entries[0].offset, 0); + assert_eq!(entries[0].len, 3); + assert_eq!(entries[0].crc, None); + + assert_eq!(entries[1].path, "bb"); + assert_eq!(entries[1].offset, 3); + assert_eq!(entries[1].len, 5); + assert_eq!(entries[1].crc, None); +} + #[test] fn handles_empty_files() { let files = vec![ diff --git a/tests/lazy.rs b/tests/lazy.rs new file mode 100644 index 0000000..cdf6d13 --- /dev/null +++ b/tests/lazy.rs @@ -0,0 +1,120 @@ +use fossil::core::{container, dir}; + +fn sample_files() -> Vec<(String, Vec)> { + let mut a = Vec::new(); + for i in 0..40000u32 { + a.extend_from_slice(format!("line {i} the quick brown fox jumps\n").as_bytes()); + } + + let mut b = Vec::new(); + for i in 0..20000u32 { + b.extend_from_slice(format!("{},{},{}\n", i, i.wrapping_mul(2), i % 7).as_bytes()); + } + + let c: Vec = (0..50000u32) + .map(|i| (i.wrapping_mul(2654435761) >> 24) as u8) + .collect(); + + vec![ + ("a.txt".to_string(), a), + ("b.csv".to_string(), b), + ("c.bin".to_string(), c), + ] +} + +fn build() -> (Vec, Vec<(String, Vec)>) { + let files = sample_files(); + let (meta, payload) = dir::pack(&files); + let bytes = container::write_progress_meta(&payload, "/", &meta, None, false); + (bytes, files) +} + +#[test] +fn lazy_take_matches_each_file() { + let (bytes, files) = build(); + + let mut lazy = container::read_lazy(&bytes).unwrap(); + assert!( + lazy.blocks.len() > 64, + "test corpus should span several segments, got {} blocks", + lazy.blocks.len() + ); + + let entries = dir::read(&lazy.meta).unwrap(); + for (i, e) in entries.iter().enumerate() { + let got = lazy.read_range(e.offset, e.len).unwrap(); + assert_eq!(got, files[i].1, "file {} mismatch", e.path); + } +} + +#[test] +fn lazy_read_range_matches_full_decode() { + let (bytes, _) = build(); + let full = container::read(&bytes).unwrap().decode(); + let mut lazy = container::read_lazy(&bytes).unwrap(); + + let spots = [ + 0usize, + 1, + 4095, + 4096, + 4097, + 262143, + 262144, + 262145, + 300000, + full.len() - 1, + ]; + let lens = [1usize, 2, 5000, 262146]; + + for &off in &spots { + for &len in &lens { + if off + len <= full.len() { + let got = lazy.read_range(off, len).unwrap(); + assert_eq!(got, &full[off..off + len], "range off={off} len={len}"); + } + } + } +} + +#[test] +fn lazy_take_works_on_stored_directory_fossil() { + let mut state = 0x9e3779b97f4a7c15u64; + let mut noise = |n: usize| -> Vec { + (0..n) + .map(|_| { + state = state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (state >> 33) as u8 + }) + .collect() + }; + + let files = vec![ + ("a.bin".to_string(), noise(20000)), + ("b.bin".to_string(), noise(20000)), + ("c.bin".to_string(), noise(5)), + ]; + + let (meta, payload) = dir::pack(&files); + let bytes = container::write_progress_meta(&payload, "/", &meta, None, false); + assert_eq!(bytes[5], 1, "test corpus should force a stored container"); + + let mut lazy = container::read_lazy(&bytes).unwrap(); + let entries = dir::read(&lazy.meta).unwrap(); + for (i, e) in entries.iter().enumerate() { + let got = lazy.read_range(e.offset, e.len).unwrap(); + assert_eq!(got, files[i].1, "file {} mismatch", e.path); + } +} + +#[test] +fn lazy_read_range_repeated_reads_are_stable() { + let (bytes, _) = build(); + let full = container::read(&bytes).unwrap().decode(); + let mut lazy = container::read_lazy(&bytes).unwrap(); + + for off in [262144usize, 4096, 262144, 300000, 4096] { + let got = lazy.read_range(off, 8192).unwrap(); + assert_eq!(got, &full[off..off + 8192]); + } +} diff --git a/tests/models/bwtm.rs b/tests/models/bwtm.rs index d86aff6..c7ed97f 100644 --- a/tests/models/bwtm.rs +++ b/tests/models/bwtm.rs @@ -30,3 +30,38 @@ fn shrinks_repetitive_text() { let data = b"abracadabra ".repeat(80); assert!(encode(&data).len() < data.len()); } + +use fossil::core::models::bwtm; + +fn roundtrip2(data: &[u8]) { + assert_eq!(bwtm::decode2(&bwtm::encode2(data), data.len()), data); +} + +#[test] +fn bwtm2_roundtrips_text() { + roundtrip2(b"the quick brown fox jumps over the lazy dog"); +} + +#[test] +fn bwtm2_roundtrips_empty() { + roundtrip2(&[]); +} + +#[test] +fn bwtm2_roundtrips_repeats() { + roundtrip2(&[0x42u8; 500]); +} + +#[test] +fn bwtm2_roundtrips_all_bytes() { + let data: Vec = (0..=255).collect(); + roundtrip2(&data); +} + +#[test] +fn bwtm2_beats_bwtm_on_repetitive_text() { + let data = b"abracadabra ".repeat(300); + let old = bwtm::encode(&data); + let new = bwtm::encode2(&data); + assert!(new.len() < old.len(), "bwtm2 {} >= bwtm {}", new.len(), old.len()); +} diff --git a/tests/models/huffman.rs b/tests/models/huffman.rs index 2de6e4b..8c28f4c 100644 --- a/tests/models/huffman.rs +++ b/tests/models/huffman.rs @@ -98,3 +98,30 @@ fn small_alphabet_has_tiny_table() { let enc = encode(b"aaaaabbbbbcccccddddd"); assert!(enc.len() < 30); } + +#[test] +fn dense_alphabet_roundtrips_with_packed_table() { + let mut data = Vec::new(); + for round in 0..20u32 { + for b in 0..=255u8 { + for _ in 0..(1 + (b as u32 + round) % 5) { + data.push(b); + } + } + } + let enc = encode(&data); + assert_eq!(decode(&enc, data.len()), data); +} + +#[test] +fn packed_table_is_smaller_than_dense() { + let mut data = Vec::new(); + for b in 0..=255u8 { + for _ in 0..(1 + (b as usize % 7)) { + data.push(b); + } + } + let enc = encode(&data); + assert_eq!(enc[0], 2); + assert_eq!(decode(&enc, data.len()), data); +} diff --git a/tests/models/lzr.rs b/tests/models/lzr.rs index 6ffbf36..67ff05d 100644 --- a/tests/models/lzr.rs +++ b/tests/models/lzr.rs @@ -38,3 +38,70 @@ fn windowed_matches_into_history() { let enc = lzr::encode_from(&combined, history.len()); assert_eq!(lzr::decode_windowed(&enc, pattern.len(), &history), pattern); } + +use fossil::core::biglz; + +fn roundtrip2(data: &[u8]) { + let tokens = biglz::tokens(data, 0); + let enc = lzr::encode_tokens2(data, 0, &tokens); + assert_eq!(lzr::decode_windowed2(&enc, data.len(), &[]), data); +} + +#[test] +fn lzr2_round_trips_repetitive_text() { + let data = b"the quick brown fox the quick brown fox ".repeat(200); + roundtrip2(&data); +} + +#[test] +fn lzr2_round_trips_strided_records() { + let mut data = Vec::new(); + for i in 0u32..500 { + data.extend_from_slice(&i.to_le_bytes()); + data.extend_from_slice(b"ROW-PADDING-"); + } + roundtrip2(&data); +} + +#[test] +fn lzr2_round_trips_pseudo_random() { + let data: Vec = (0u64..5000) + .map(|i| { + let mut x = i.wrapping_mul(0x9E3779B97F4A7C15); + x ^= x >> 29; + (x >> 24) as u8 + }) + .collect(); + roundtrip2(&data); +} + +#[test] +fn lzr2_round_trips_empty() { + roundtrip2(&[]); +} + +#[test] +fn lzr2_windowed_matches_into_history() { + let pattern = b"abcdefghij".repeat(50); + let history = pattern.clone(); + + let mut combined = history.clone(); + combined.extend_from_slice(&pattern); + + let tokens = biglz::tokens(&combined, history.len()); + let enc = lzr::encode_tokens2(&combined, history.len(), &tokens); + assert_eq!(lzr::decode_windowed2(&enc, pattern.len(), &history), pattern); +} + +#[test] +fn lzr2_beats_lzr_on_strided_data() { + let mut data = Vec::new(); + for i in 0u32..800 { + data.extend_from_slice(b"field,"); + data.extend_from_slice(&(i % 10).to_le_bytes()); + } + let tokens = biglz::tokens(&data, 0); + let old = lzr::encode_tokens(&data, 0, &tokens); + let new = lzr::encode_tokens2(&data, 0, &tokens); + assert!(new.len() <= old.len(), "lzr2 {} > lzr {}", new.len(), old.len()); +} From b25fc2ed1d9fd42f90479368cdb35d65f9781964 Mon Sep 17 00:00:00 2001 From: Matt Date: Tue, 14 Jul 2026 09:24:20 -0700 Subject: [PATCH 2/4] fix: mount issues --- bench.sh | 11 +- src/commands/mount.rs | 442 ++++++++++++++++++++++++++++++++-------- src/commands/pack.rs | 33 ++- src/commands/take.rs | 24 ++- src/commands/unpack.rs | 17 ++ src/core/container.rs | 20 +- src/core/dir.rs | 43 +++- src/core/models/bwtm.rs | 4 +- src/core/models/lzr.rs | 9 +- src/main.rs | 7 + 10 files changed, 498 insertions(+), 112 deletions(-) diff --git a/bench.sh b/bench.sh index 4292440..5ccb7e7 100755 --- a/bench.sh +++ b/bench.sh @@ -10,6 +10,9 @@ files=(examples/z examples/mixed.bin examples/bigmix.bin examples/cat.ppm exampl have_zstd=0 command -v zstd >/dev/null 2>&1 && have_zstd=1 +tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/fossil-bench.XXXXXX")" +trap 'rm -rf "$tmpdir"' EXIT + pct() { awk -v o="$1" -v n="$2" 'BEGIN { if (o == 0) { print "-" } else { printf "%.1f%%", (1 - n/o) * 100 } }' } @@ -21,8 +24,8 @@ for f in "${files[@]}"; do [ -f "$f" ] || continue orig=$(wc -c < "$f" | tr -d ' ') - "$BIN" pack "$f" /tmp/bench >/dev/null 2>&1 - fos=$(wc -c < /tmp/bench.fossil | tr -d ' ') + "$BIN" pack "$f" "$tmpdir/bench" >/dev/null 2>&1 + fos=$(wc -c < "$tmpdir/bench.fossil" | tr -d ' ') gz=$(gzip -9 -c "$f" | wc -c | tr -d ' ') @@ -50,8 +53,8 @@ for d in "${dirs[@]}"; do [ -d "$d" ] || continue orig=$(tar -cf - "$d" 2>/dev/null | wc -c | tr -d ' ') - "$BIN" pack "$d" /tmp/bench >/dev/null 2>&1 - fos=$(wc -c < /tmp/bench.fossil | tr -d ' ') + "$BIN" pack "$d" "$tmpdir/bench" >/dev/null 2>&1 + fos=$(wc -c < "$tmpdir/bench.fossil" | tr -d ' ') gz=$(tar -cf - "$d" 2>/dev/null | gzip -9 | wc -c | tr -d ' ') diff --git a/src/commands/mount.rs b/src/commands/mount.rs index 6f5f43f..ce12044 100644 --- a/src/commands/mount.rs +++ b/src/commands/mount.rs @@ -31,6 +31,18 @@ enum Sink { const TTL: Duration = Duration::from_secs(1); const ROOT: u64 = 1; +const RENAME_NOREPLACE: u32 = 1; +const RENAME_EXCHANGE: u32 = 2; + +const MAX_FILE: u64 = 1 << 32; + +fn resolve_time(t: TimeOrNow) -> SystemTime { + match t { + TimeOrNow::SpecificTime(t) => t, + TimeOrNow::Now => SystemTime::now(), + } +} + enum Body { Stored { offset: usize, @@ -49,6 +61,33 @@ struct Node { parent: u64, name: String, kind: Kind, + perm: u16, + uid: u32, + gid: u32, + atime: SystemTime, + mtime: SystemTime, + ctime: SystemTime, + crtime: SystemTime, +} + +impl Node { + fn new(parent: u64, name: String, kind: Kind, uid: u32, gid: u32) -> Self { + let now = SystemTime::now(); + let perm = if matches!(kind, Kind::Dir(_)) { 0o755 } else { 0o644 }; + + Node { + parent, + name, + kind, + perm, + uid, + gid, + atime: now, + mtime: now, + ctime: now, + crtime: now, + } + } } struct FossilFs { @@ -85,6 +124,7 @@ impl FossilFs { } let entries = dir::read(&parts.meta)?; + let dirs = dir::read_dirs(&parts.meta)?; let mut me = FossilFs { path: PathBuf::from(path), @@ -100,13 +140,10 @@ impl FossilFs { sink, }; + let (uid, gid) = (me.uid, me.gid); me.nodes.insert( ROOT, - Node { - parent: ROOT, - name: "/".into(), - kind: Kind::Dir(BTreeMap::new()), - }, + Node::new(ROOT, "/".into(), Kind::Dir(BTreeMap::new()), uid, gid), ); for e in &entries { @@ -120,9 +157,20 @@ impl FossilFs { ); } + for d in &dirs { + me.insert_dir(d); + } + Ok(me) } + fn insert_dir(&mut self, path: &str) { + let mut cur = ROOT; + for comp in path.split('/').filter(|s| !s.is_empty()) { + cur = self.ensure_dir(cur, comp); + } + } + fn ensure_dir(&mut self, parent: u64, name: &str) -> u64 { if let Some(Node { kind: Kind::Dir(children), @@ -138,11 +186,13 @@ impl FossilFs { self.next += 1; self.nodes.insert( ino, - Node { + Node::new( parent, - name: name.to_string(), - kind: Kind::Dir(BTreeMap::new()), - }, + name.to_string(), + Kind::Dir(BTreeMap::new()), + self.uid, + self.gid, + ), ); if let Some(Node { kind: Kind::Dir(children), @@ -175,11 +225,7 @@ impl FossilFs { self.next += 1; self.nodes.insert( ino, - Node { - parent: cur, - name: name.clone(), - kind: Kind::File(body), - }, + Node::new(cur, name.clone(), Kind::File(body), self.uid, self.gid), ); if let Some(Node { kind: Kind::Dir(children), @@ -232,6 +278,32 @@ impl FossilFs { } } + fn is_dir(&self, ino: u64) -> bool { + matches!( + self.nodes.get(&ino), + Some(Node { + kind: Kind::Dir(_), + .. + }) + ) + } + + fn is_ancestor(&self, ino: u64, of: u64) -> bool { + let mut cur = of; + loop { + if cur == ino { + return true; + } + if cur == ROOT { + return false; + } + match self.nodes.get(&cur) { + Some(node) => cur = node.parent, + None => return false, + } + } + } + fn materialize(&mut self, ino: u64) -> io::Result<()> { let (offset, len, want) = match self.nodes.get(&ino) { Some(Node { @@ -282,46 +354,45 @@ impl FossilFs { } fn attr(&self, ino: u64) -> FileAttr { - let is_dir = matches!( - self.nodes.get(&ino), - Some(Node { - kind: Kind::Dir(_), - .. - }) - ); - + let node = &self.nodes[&ino]; + let is_dir = matches!(node.kind, Kind::Dir(_)); let size = self.size_of(ino); - let now = SystemTime::now(); FileAttr { ino, size, blocks: size.div_ceil(512), - atime: now, - mtime: now, - ctime: now, - crtime: now, + atime: node.atime, + mtime: node.mtime, + ctime: node.ctime, + crtime: node.crtime, kind: if is_dir { FileType::Directory } else { FileType::RegularFile }, - perm: if is_dir { 0o755 } else { 0o644 }, + perm: node.perm, nlink: if is_dir { 2 } else { 1 }, - uid: self.uid, - gid: self.gid, + uid: node.uid, + gid: node.gid, rdev: 0, blksize: 512, flags: 0, } } - fn collect(&mut self, ino: u64, prefix: &str, out: &mut Vec<(String, Vec)>) { + fn collect( + &mut self, + ino: u64, + prefix: &str, + out: &mut Vec<(String, Vec)>, + dirs: &mut Vec, + ) -> io::Result<()> { let children = match self.nodes.get(&ino) { Some(Node { kind: Kind::Dir(c), .. }) => c.clone(), - _ => return, + _ => return Ok(()), }; for (name, child) in children { @@ -331,37 +402,41 @@ impl FossilFs { format!("{prefix}/{name}") }; - let is_file = matches!( - self.nodes.get(&child), + match self.nodes.get(&child) { Some(Node { kind: Kind::File(_), .. - }) - ); - - if is_file { - if self.materialize(child).is_err() { - continue; - } - if let Some(Node { - kind: Kind::File(Body::Loaded(b)), - .. - }) = self.nodes.get(&child) - { - out.push((path, b.clone())); + }) => { + self.materialize(child)?; + if let Some(Node { + kind: Kind::File(Body::Loaded(b)), + .. + }) = self.nodes.get(&child) + { + out.push((path, b.clone())); + } } - } else { - self.collect(child, &path, out); + Some(Node { + kind: Kind::Dir(c), .. + }) if c.is_empty() => dirs.push(path), + Some(Node { + kind: Kind::Dir(_), .. + }) => self.collect(child, &path, out, dirs)?, + None => {} } } + + Ok(()) } fn repack(&mut self) -> io::Result<()> { let mut files: Vec<(String, Vec)> = Vec::new(); - self.collect(ROOT, "", &mut files); + let mut dirs: Vec = Vec::new(); + self.collect(ROOT, "", &mut files, &mut dirs)?; files.sort_by(|a, b| a.0.cmp(&b.0)); + dirs.sort(); - let (meta, payload) = dir::pack(&files); + let (meta, payload) = dir::pack_tree(&files, &dirs); let bytes = container::write_progress_meta(&payload, "/", &meta, None, false); let mut tmp = self.path.clone().into_os_string(); @@ -411,21 +486,34 @@ impl Filesystem for FossilFs { &mut self, _req: &Request<'_>, ino: u64, - _mode: Option, - _uid: Option, - _gid: Option, + mode: Option, + uid: Option, + gid: Option, size: Option, - _atime: Option, - _mtime: Option, - _ctime: Option, + atime: Option, + mtime: Option, + ctime: Option, _fh: Option, - _crtime: Option, + crtime: Option, _chgtime: Option, _bkuptime: Option, _flags: Option, reply: ReplyAttr, ) { + if !self.nodes.contains_key(&ino) { + reply.error(libc::ENOENT); + return; + } + if let Some(new_len) = size { + let Ok(new_len) = usize::try_from(new_len).map_err(|_| ()) else { + reply.error(libc::EFBIG); + return; + }; + if new_len as u64 > MAX_FILE { + reply.error(libc::EFBIG); + return; + } if self.materialize(ino).is_err() { reply.error(libc::EIO); return; @@ -435,16 +523,36 @@ impl Filesystem for FossilFs { .. }) = self.nodes.get_mut(&ino) { - buf.resize(new_len as usize, 0); + buf.resize(new_len, 0); self.dirty = true; } } - if self.nodes.contains_key(&ino) { - reply.attr(&TTL, &self.attr(ino)); - } else { - reply.error(libc::ENOENT); + let node = self.nodes.get_mut(&ino).unwrap(); + + if let Some(mode) = mode { + node.perm = (mode & 0o7777) as u16; + } + if let Some(uid) = uid { + node.uid = uid; + } + if let Some(gid) = gid { + node.gid = gid; + } + if let Some(atime) = atime { + node.atime = resolve_time(atime); } + if let Some(mtime) = mtime { + node.mtime = resolve_time(mtime); + } + if let Some(ctime) = ctime { + node.ctime = ctime; + } + if let Some(crtime) = crtime { + node.crtime = crtime; + } + + reply.attr(&TTL, &self.attr(ino)); } fn open(&mut self, _req: &Request<'_>, _ino: u64, _flags: i32, reply: ReplyOpen) { @@ -462,6 +570,11 @@ impl Filesystem for FossilFs { _lock_owner: Option, reply: ReplyData, ) { + let Ok(offset) = usize::try_from(offset) else { + reply.error(libc::EINVAL); + return; + }; + if self.materialize(ino).is_err() { reply.error(libc::EIO); return; @@ -472,8 +585,8 @@ impl Filesystem for FossilFs { .. }) = self.nodes.get(&ino) { - let start = (offset as usize).min(b.len()); - let end = (start + size as usize).min(b.len()); + let start = offset.min(b.len()); + let end = start.saturating_add(size as usize).min(b.len()); let chunk = end - start; reply.data(&b[start..end]); self.note(format!("read {} · {} B", self.path_of(ino), chunk)); @@ -494,18 +607,31 @@ impl Filesystem for FossilFs { _lock_owner: Option, reply: ReplyWrite, ) { + let Ok(off) = usize::try_from(offset) else { + reply.error(libc::EINVAL); + return; + }; + + let Some(end) = off.checked_add(data.len()) else { + reply.error(libc::EFBIG); + return; + }; + + if end as u64 > MAX_FILE { + reply.error(libc::EFBIG); + return; + } + if self.materialize(ino).is_err() { reply.error(libc::EIO); return; } - let off = offset as usize; let wrote = if let Some(Node { kind: Kind::File(Body::Loaded(buf)), .. }) = self.nodes.get_mut(&ino) { - let end = off + data.len(); if buf.len() < end { buf.resize(end, 0); } @@ -559,11 +685,13 @@ impl Filesystem for FossilFs { self.next += 1; self.nodes.insert( ino, - Node { + Node::new( parent, - name: name.clone(), - kind: Kind::File(Body::Loaded(Vec::new())), - }, + name.clone(), + Kind::File(Body::Loaded(Vec::new())), + self.uid, + self.gid, + ), ); if let Some(Node { kind: Kind::Dir(children), @@ -597,7 +725,13 @@ impl Filesystem for FossilFs { return; } + if !self.is_dir(parent) { + reply.error(libc::ENOTDIR); + return; + } + let ino = self.ensure_dir(parent, &name); + self.dirty = true; self.note(format!("mkdir {}", self.path_of(ino))); reply.entry(&TTL, &self.attr(ino), 0); } @@ -668,12 +802,17 @@ impl Filesystem for FossilFs { name: &OsStr, newparent: u64, newname: &OsStr, - _flags: u32, + flags: u32, reply: ReplyEmpty, ) { let name = name.to_string_lossy().into_owned(); let newname = newname.to_string_lossy().into_owned(); + if flags & RENAME_EXCHANGE != 0 { + reply.error(libc::EOPNOTSUPP); + return; + } + let ino = match self.child(parent, &name) { Some(ino) => ino, None => { @@ -693,14 +832,53 @@ impl Filesystem for FossilFs { return; } + if self.is_ancestor(ino, newparent) { + reply.error(libc::EINVAL); + return; + } + if let Some(old) = self.child(newparent, &newname) { - self.nodes.remove(&old); - if let Some(Node { - kind: Kind::Dir(children), - .. - }) = self.nodes.get_mut(&newparent) - { - children.remove(&newname); + if flags & RENAME_NOREPLACE != 0 { + reply.error(libc::EEXIST); + return; + } + + if old != ino { + let src_is_dir = self.is_dir(ino); + match self.nodes.get(&old) { + Some(Node { + kind: Kind::Dir(children), + .. + }) => { + if !src_is_dir { + reply.error(libc::EISDIR); + return; + } + if !children.is_empty() { + reply.error(libc::ENOTEMPTY); + return; + } + } + Some(Node { + kind: Kind::File(_), + .. + }) => { + if src_is_dir { + reply.error(libc::ENOTDIR); + return; + } + } + None => {} + } + + self.nodes.remove(&old); + if let Some(Node { + kind: Kind::Dir(children), + .. + }) = self.nodes.get_mut(&newparent) + { + children.remove(&newname); + } } } @@ -787,7 +965,18 @@ impl Filesystem for FossilFs { _datasync: bool, reply: ReplyEmpty, ) { - reply.ok(); + if !self.dirty { + reply.ok(); + return; + } + + match self.repack() { + Ok(()) => reply.ok(), + Err(e) => { + self.note(format!("fsync failed · {}", e)); + reply.error(libc::EIO); + } + } } } @@ -877,15 +1066,94 @@ fn force_unmount(mountpoint: &Path) { return; }; - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "linux", target_os = "android"))] unsafe { - libc::unmount(c.as_ptr(), libc::MNT_FORCE); + libc::umount2(c.as_ptr(), libc::MNT_DETACH); } - #[cfg(not(target_os = "macos"))] + #[cfg(any( + target_os = "macos", + target_os = "freebsd", + target_os = "dragonfly", + target_os = "netbsd", + target_os = "openbsd" + ))] unsafe { - libc::umount2(c.as_ptr(), libc::MNT_DETACH); + libc::unmount(c.as_ptr(), libc::MNT_FORCE); } + + #[cfg(not(any( + target_os = "linux", + target_os = "android", + target_os = "macos", + target_os = "freebsd", + target_os = "dragonfly", + target_os = "netbsd", + target_os = "openbsd" + )))] + let _ = c; +} + +#[cfg(any(target_os = "linux", target_os = "android"))] +fn allow_auto_unmount() -> bool { + const CONF: &str = "/etc/fuse.conf"; + + let already_set = fs::read_to_string(CONF).is_ok_and(|conf| { + conf.lines() + .any(|line| line.split('#').next().unwrap_or("").trim() == "user_allow_other") + }); + + if already_set || unsafe { libc::geteuid() } == 0 { + return true; + } + + let declined = std::env::var_os("XDG_CONFIG_HOME") + .map(PathBuf::from) + .or_else(|| std::env::var_os("HOME").map(|h| PathBuf::from(h).join(".config"))) + .map(|base| base.join("fossil").join("no-fuse-conf")); + + if declined.as_ref().is_some_and(|path| path.exists()) || !io::stdin().is_terminal() { + return false; + } + + println!( + " auto-unmount needs {} in {}", + "user_allow_other".accent(), + CONF.accent() + ); + println!(" without it, a crash can leave the mount point stale"); + print!(" add it now (uses sudo)? [y/N] "); + io::stdout().flush().ok(); + + let mut answer = String::new(); + if io::stdin().read_line(&mut answer).is_err() { + return false; + } + + if !matches!(answer.trim(), "y" | "Y" | "yes") { + if let Some(path) = declined { + if let Some(parent) = path.parent() { + let _ = fs::create_dir_all(parent); + } + let _ = fs::write(&path, ""); + } + return false; + } + + let added = std::process::Command::new("sudo") + .args(["sh", "-c", "echo user_allow_other >> /etc/fuse.conf"]) + .status() + .is_ok_and(|s| s.success()); + + if !added { + error!("couldn't write /etc/fuse.conf, mounting without auto-unmount"); + } + added +} + +#[cfg(not(any(target_os = "linux", target_os = "android")))] +fn allow_auto_unmount() -> bool { + true } pub fn run(archive: &str, mountpoint: &str, verbose: bool, log_lines: bool) { @@ -914,14 +1182,16 @@ pub fn run(archive: &str, mountpoint: &str, verbose: bool, log_lines: bool) { return; } - #[cfg_attr(not(target_os = "macos"), allow(unused_mut))] let mut options = vec![ MountOption::FSName("fossil".into()), MountOption::Subtype("fossil".into()), MountOption::DefaultPermissions, - MountOption::AutoUnmount, ]; + if allow_auto_unmount() { + options.push(MountOption::AutoUnmount); + } + #[cfg(target_os = "macos")] options.push(MountOption::CUSTOM("noappledouble".into())); diff --git a/src/commands/pack.rs b/src/commands/pack.rs index 34388e7..f8c2ff0 100644 --- a/src/commands/pack.rs +++ b/src/commands/pack.rs @@ -201,28 +201,40 @@ struct PackReport { lossy: Option, } -fn collect_files(dir: &Path, base: &Path, out: &mut Vec<(String, Vec)>) -> io::Result<()> { +fn collect_files( + dir: &Path, + base: &Path, + out: &mut Vec<(String, Vec)>, + dirs: &mut Vec, +) -> io::Result<()> { let mut entries: Vec<_> = fs::read_dir(dir)?.collect::>>()?; entries.sort_by_key(|e| e.path()); + if entries.is_empty() && dir != base { + dirs.push(rel_path(dir, base)); + return Ok(()); + } + for entry in entries { let path = entry.path(); if path.is_dir() { - collect_files(&path, base, out)?; + collect_files(&path, base, out, dirs)?; } else if path.is_file() { - let rel = path - .strip_prefix(base) - .unwrap_or(&path) - .to_string_lossy() - .replace('\\', "/"); let data = fs::read(&path)?; - out.push((rel, data)); + out.push((rel_path(&path, base), data)); } } Ok(()) } +fn rel_path(path: &Path, base: &Path) -> String { + path.strip_prefix(base) + .unwrap_or(path) + .to_string_lossy() + .replace('\\', "/") +} + fn pack( input: &str, output: &str, @@ -268,7 +280,8 @@ fn pack( (bytes, String::new(), raw, applied, Vec::new()) } else if input_path.is_dir() { let mut files = Vec::new(); - collect_files(input_path, input_path, &mut files)?; + let mut dirs = Vec::new(); + collect_files(input_path, input_path, &mut files, &mut dirs)?; let mut applied = None; if let Some(k) = opts.bits { for (rel, data) in files.iter_mut() { @@ -295,7 +308,7 @@ fn pack( } let raw: u64 = files.iter().map(|(_, d)| d.len() as u64).sum(); - let (meta, payload) = fossil::core::dir::pack(&files); + let (meta, payload) = fossil::core::dir::pack_tree(&files, &dirs); (payload, "/".to_string(), raw, applied, meta) } else if input_path.is_file() { diff --git a/src/commands/take.rs b/src/commands/take.rs index 0283ab7..eb85938 100644 --- a/src/commands/take.rs +++ b/src/commands/take.rs @@ -76,15 +76,27 @@ fn take(input: &str, inner_path: Option<&str>, trust: bool) -> io::Result { + if crc::crc32(&bytes) != want { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "checksum mismatch, file is corrupt (use --trust to skip)", + )); + } + } + None => { + let whole = container::read(&data)?.decode(); + if crc::crc32(&whole) != archive_crc { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "checksum mismatch, fossil is corrupt (use --trust to skip)", + )); + } } } } diff --git a/src/commands/unpack.rs b/src/commands/unpack.rs index b5cbb0c..800cc50 100644 --- a/src/commands/unpack.rs +++ b/src/commands/unpack.rs @@ -136,6 +136,23 @@ fn unpack(input: &str, output: &str, trust: bool) -> io::Result { written += 1; } + for path in dir::read_dirs(&container.meta)? { + let rel_path = Path::new(&path); + + let unsafe_path = rel_path.components().any(|part| { + matches!( + part, + Component::ParentDir | Component::RootDir | Component::Prefix(_) + ) + }); + + if unsafe_path { + continue; + } + + fs::create_dir_all(root.join(rel_path))?; + } + archive_files = Some(written); root.to_path_buf() } else if to_stdout { diff --git a/src/core/container.rs b/src/core/container.rs index 7630aea..67df3f6 100644 --- a/src/core/container.rs +++ b/src/core/container.rs @@ -243,6 +243,16 @@ impl<'a> Cursor<'a> { } } +fn check_last_len(last_len: usize) -> io::Result { + if last_len == 0 || last_len > BLOCK_SIZE { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("invalid final block length {}", last_len), + )); + } + Ok(last_len) +} + pub fn read(data: &[u8]) -> io::Result { let mut c = Cursor { data, pos: 0 }; @@ -287,7 +297,7 @@ pub fn read(data: &[u8]) -> io::Result { MODE_BLOCKS => { let n_blocks = c.varint()?; let last_len = if version >= 2 && n_blocks > 0 { - c.varint()? + check_last_len(c.varint()?)? } else { 0 }; @@ -426,7 +436,7 @@ pub fn read_lazy(data: &[u8]) -> io::Result> { MODE_BLOCKS => { let n_blocks = c.varint()?; let last_len = if version >= 2 && n_blocks > 0 { - c.varint()? + check_last_len(c.varint()?)? } else { 0 }; @@ -474,6 +484,12 @@ pub fn read_lazy(data: &[u8]) -> io::Result> { impl<'a> LazyContainer<'a> { pub fn read_range(&mut self, offset: usize, len: usize) -> io::Result> { + if self.filter != FILTER_NONE { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "range reads are unsupported for filtered containers", + )); + } if offset.saturating_add(len) > self.orig_size { return Err(io::Error::new( io::ErrorKind::InvalidData, diff --git a/src/core/dir.rs b/src/core/dir.rs index 380add4..5513288 100644 --- a/src/core/dir.rs +++ b/src/core/dir.rs @@ -15,6 +15,10 @@ pub struct Entry { } pub fn pack(files: &[(String, Vec)]) -> (Vec, Vec) { + pack_tree(files, &[]) +} + +pub fn pack_tree(files: &[(String, Vec)], dirs: &[String]) -> (Vec, Vec) { let mut meta = Vec::new(); let mut payload = Vec::new(); @@ -32,10 +36,27 @@ pub fn pack(files: &[(String, Vec)]) -> (Vec, Vec) { payload.extend_from_slice(bytes); } + varint::write(&mut meta, dirs.len()); + + for path in dirs { + let path_bytes = path.as_bytes(); + + varint::write(&mut meta, path_bytes.len()); + meta.extend_from_slice(path_bytes); + } + (meta, payload) } pub fn read(meta: &[u8]) -> io::Result> { + Ok(parse(meta)?.0) +} + +pub fn read_dirs(meta: &[u8]) -> io::Result> { + Ok(parse(meta)?.1) +} + +fn parse(meta: &[u8]) -> io::Result<(Vec, Vec)> { if meta.len() < MAGIC.len() { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -98,5 +119,25 @@ pub fn read(meta: &[u8]) -> io::Result> { offset += len; } - Ok(entries) + let mut dirs = Vec::new(); + + if has_crc && pos < meta.len() { + let dir_count = varint::read(meta, &mut pos); + + for _ in 0..dir_count { + let path_len = varint::read(meta, &mut pos); + + if pos + path_len > meta.len() { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "truncated directory manifest", + )); + } + + dirs.push(String::from_utf8_lossy(&meta[pos..pos + path_len]).into_owned()); + pos += path_len; + } + } + + Ok((entries, dirs)) } diff --git a/src/core/models/bwtm.rs b/src/core/models/bwtm.rs index b22a367..c6458db 100644 --- a/src/core/models/bwtm.rs +++ b/src/core/models/bwtm.rs @@ -27,10 +27,12 @@ fn encode_run(enc: &mut Encoder, model: &mut Model, mut v: usize) { } } +const MAX_RUN_CHUNKS: usize = usize::BITS.div_ceil(7) as usize; + fn decode_run(dec: &mut Decoder, model: &mut Model) -> usize { let mut v = 0usize; let mut shift = 0; - loop { + for _ in 0..MAX_RUN_CHUNKS { let value = dec.decode_freq(model.total()); let byte = model.find(value); dec.update(model.cum_freq(byte), model.freq(byte)); diff --git a/src/core/models/lzr.rs b/src/core/models/lzr.rs index 02febc8..05d7257 100644 --- a/src/core/models/lzr.rs +++ b/src/core/models/lzr.rs @@ -66,10 +66,12 @@ fn encode_value(enc: &mut Encoder, model: &mut Model, mut v: usize) { } } +const MAX_VALUE_CHUNKS: usize = usize::BITS.div_ceil(7) as usize; + fn decode_value(dec: &mut Decoder, model: &mut Model) -> usize { let mut v = 0usize; let mut shift = 0; - loop { + for _ in 0..MAX_VALUE_CHUNKS { let value = dec.decode_freq(model.total()); let byte = model.find(value); dec.update(model.cum_freq(byte), model.freq(byte)); @@ -148,7 +150,7 @@ fn decode_value_ctx(dec: &mut Decoder, models: &mut [Model]) -> usize { let mut v = 0usize; let mut shift = 0; let mut idx = 0; - loop { + for _ in 0..MAX_VALUE_CHUNKS { let m = &mut models[idx.min(models.len() - 1)]; let value = dec.decode_freq(m.total()); let byte = m.find(value); @@ -252,6 +254,7 @@ pub fn decode_windowed2(data: &[u8], count: usize, history: &[u8]) -> Vec { if dist == 0 || dist > out.len() { break; } + let len = len.min(target - out.len()); let start = out.len() - dist; for x in 0..len { let b = out[start + x]; @@ -265,6 +268,7 @@ pub fn decode_windowed2(data: &[u8], count: usize, history: &[u8]) -> Vec { if dist == 0 || dist > out.len() { break; } + let len = len.min(target - out.len()); let start = out.len() - dist; for x in 0..len { let b = out[start + x]; @@ -315,6 +319,7 @@ pub fn decode_windowed(data: &[u8], count: usize, history: &[u8]) -> Vec { if dist == 0 || dist > out.len() { break; } + let len = len.min(target - out.len()); let start = out.len() - dist; for x in 0..len { let b = out[start + x]; diff --git a/src/main.rs b/src/main.rs index 1130c87..4f1093d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -297,11 +297,18 @@ fn dispatch() { } } +#[cfg(unix)] const COMMANDS: &[&str] = &[ "pack", "lift", "unpack", "list", "take", "mount", "inspect", "map", "explain", "verify", "update", "help", ]; +#[cfg(not(unix))] +const COMMANDS: &[&str] = &[ + "pack", "lift", "unpack", "list", "take", "inspect", "map", "explain", "verify", "update", + "help", +]; + fn closest(input: &str) -> Option<&'static str> { COMMANDS .iter() From 6ce8dbc40e93aa6a7f767ed87993e301c13eecee Mon Sep 17 00:00:00 2001 From: matt Date: Tue, 14 Jul 2026 12:29:56 -0700 Subject: [PATCH 3/4] fix: conditional mount deps --- .github/workflows/ci.yml | 5 +++ .github/workflows/release.yml | 5 +++ Cargo.toml | 12 +++++-- build.rs | 6 ++++ src/commands/mod.rs | 2 +- src/commands/mount.rs | 67 +++++++++++++++++++++++++++++++++++ src/main.rs | 8 ++--- 7 files changed, 97 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec2834c..5fb3864 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,5 +16,10 @@ jobs: steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable + - name: install macfuse (macos) + if: runner.os == 'macOS' + run: | + brew install --cask macfuse + echo "PKG_CONFIG_PATH=/usr/local/lib/pkgconfig" >> "$GITHUB_ENV" - run: cargo build --verbose - run: cargo test --verbose diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f7bb2e7..2023db8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,6 +24,11 @@ jobs: steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable + - name: install macfuse (macos) + if: runner.os == 'macOS' + run: | + brew install --cask macfuse + echo "PKG_CONFIG_PATH=/usr/local/lib/pkgconfig" >> "$GITHUB_ENV" - run: cargo build --release --verbose - name: stage binary shell: bash diff --git a/Cargo.toml b/Cargo.toml index 0b0c99a..9633b0a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,14 +6,20 @@ description = "a compressor that shows its work" repository = "https://github.com/punctuations/fossil" license = "GPL-3.0-only" +[features] +default = ["mount"] +mount = ["dep:fuser", "dep:libc"] + [dependencies] terminal_size = "0.4" -[target.'cfg(unix)'.dependencies] -fuser = "0.15" -libc = "0.2" +[target.'cfg(all(unix, not(target_os = "macos")))'.dependencies] +fuser = { version = "0.15", default-features = false, optional = true } +libc = { version = "0.2", optional = true } [target.'cfg(target_os = "macos")'.dependencies] +fuser = { version = "0.15", optional = true } +libc = { version = "0.2", optional = true } objc2 = "0.6" objc2-foundation = { version = "0.3", features = ["NSString", "NSURL", "NSArray", "NSData"] } objc2-app-kit = { version = "0.3", features = ["NSPasteboard", "NSPasteboardItem", "NSWorkspace"] } diff --git a/build.rs b/build.rs index 9ca6ce1..e3eb0ba 100644 --- a/build.rs +++ b/build.rs @@ -11,4 +11,10 @@ fn main() { println!("cargo:rustc-env=FOSSIL_COMMIT={commit}"); println!("cargo:rerun-if-changed=.git/HEAD"); + + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("macos") + && std::env::var("CARGO_FEATURE_MOUNT").is_ok() + { + println!("cargo:rustc-link-arg-bins=-Wl,-weak-lfuse"); + } } diff --git a/src/commands/mod.rs b/src/commands/mod.rs index 7494c92..da893bc 100644 --- a/src/commands/mod.rs +++ b/src/commands/mod.rs @@ -2,7 +2,7 @@ pub mod explain; pub mod inspect; pub mod list; pub mod map; -#[cfg(unix)] +#[cfg(all(unix, feature = "mount"))] pub mod mount; pub mod pack; pub mod take; diff --git a/src/commands/mount.rs b/src/commands/mount.rs index ce12044..18089be 100644 --- a/src/commands/mount.rs +++ b/src/commands/mount.rs @@ -1156,7 +1156,74 @@ fn allow_auto_unmount() -> bool { true } +#[cfg(target_os = "macos")] +fn macfuse_present() -> bool { + Path::new("/usr/local/lib/libfuse.2.dylib").exists() +} + +#[cfg(target_os = "macos")] +fn ensure_macfuse() -> bool { + if macfuse_present() { + return true; + } + + let marker = std::env::var("HOME") + .map(|h| PathBuf::from(h).join("Library/Application Support/fossil/macfuse-prompted")); + let already_asked = marker.as_ref().map(|m| m.exists()).unwrap_or(true); + + if already_asked || !io::stdin().is_terminal() || !io::stderr().is_terminal() { + error!("mount needs macFUSE (brew install --cask macfuse)"); + return false; + } + + if let Ok(m) = &marker { + if let Some(dir) = m.parent() { + let _ = fs::create_dir_all(dir); + } + let _ = fs::write(m, b""); + } + + eprintln!( + " {}", + "fossil mount needs macFUSE, which is not installed".bold() + ); + eprint!(" install it now with homebrew? {} ", "[y/N]".dim()); + let _ = io::stderr().flush(); + + let mut line = String::new(); + if io::stdin().read_line(&mut line).is_err() + || !matches!(line.trim(), "y" | "Y" | "yes" | "Yes") + { + error!("mount needs macFUSE (brew install --cask macfuse)"); + return false; + } + + let status = std::process::Command::new("brew") + .args(["install", "--cask", "macfuse"]) + .status(); + + match status { + Ok(s) if s.success() => { + eprintln!( + " {}", + "macFUSE installed · approve the system extension in".bold() + ); + eprintln!( + " {}", + "System Settings › Privacy & Security, then run fossil mount again".bold() + ); + } + _ => error!("install failed; get macFUSE from https://macfuse.github.io"), + } + false +} + pub fn run(archive: &str, mountpoint: &str, verbose: bool, log_lines: bool) { + #[cfg(target_os = "macos")] + if !ensure_macfuse() { + return; + } + let ring: Option = (verbose && !log_lines).then(|| Arc::new(Mutex::new(VecDeque::new()))); diff --git a/src/main.rs b/src/main.rs index 4f1093d..3aa56f2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -143,7 +143,7 @@ fn dispatch() { commands::take::run(&args[2], inner, output, trust); } - #[cfg(unix)] + #[cfg(all(unix, feature = "mount"))] "mount" => { let mut verbose = false; let mut log = false; @@ -238,7 +238,7 @@ fn dispatch() { "unpack" => ui::subcommand(commands::unpack::help()), "list" => ui::subcommand(commands::list::help()), "take" => ui::subcommand(commands::take::help()), - #[cfg(unix)] + #[cfg(all(unix, feature = "mount"))] "mount" => ui::subcommand(commands::mount::help()), "inspect" => ui::subcommand(commands::inspect::help()), "map" => ui::subcommand(commands::map::help()), @@ -297,13 +297,13 @@ fn dispatch() { } } -#[cfg(unix)] +#[cfg(all(unix, feature = "mount"))] const COMMANDS: &[&str] = &[ "pack", "lift", "unpack", "list", "take", "mount", "inspect", "map", "explain", "verify", "update", "help", ]; -#[cfg(not(unix))] +#[cfg(not(all(unix, feature = "mount")))] const COMMANDS: &[&str] = &[ "pack", "lift", "unpack", "list", "take", "inspect", "map", "explain", "verify", "update", "help", From 47ce9e29ce736a67aad4980a7207910cce45cc67 Mon Sep 17 00:00:00 2001 From: matt Date: Tue, 14 Jul 2026 12:42:10 -0700 Subject: [PATCH 4/4] fix: add assertion and dont skip empty dir --- src/commands/mount.rs | 3 +++ src/commands/pack.rs | 2 +- src/core/models/lz.rs | 3 +++ tests/container.rs | 1 + 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/commands/mount.rs b/src/commands/mount.rs index 18089be..5e51c33 100644 --- a/src/commands/mount.rs +++ b/src/commands/mount.rs @@ -433,6 +433,9 @@ impl FossilFs { let mut files: Vec<(String, Vec)> = Vec::new(); let mut dirs: Vec = Vec::new(); self.collect(ROOT, "", &mut files, &mut dirs)?; + if files.is_empty() && dirs.is_empty() { + dirs.push(String::new()); + } files.sort_by(|a, b| a.0.cmp(&b.0)); dirs.sort(); diff --git a/src/commands/pack.rs b/src/commands/pack.rs index f8c2ff0..34ccbe0 100644 --- a/src/commands/pack.rs +++ b/src/commands/pack.rs @@ -210,7 +210,7 @@ fn collect_files( let mut entries: Vec<_> = fs::read_dir(dir)?.collect::>>()?; entries.sort_by_key(|e| e.path()); - if entries.is_empty() && dir != base { + if entries.is_empty() { dirs.push(rel_path(dir, base)); return Ok(()); } diff --git a/src/core/models/lz.rs b/src/core/models/lz.rs index 62afe59..93d7896 100644 --- a/src/core/models/lz.rs +++ b/src/core/models/lz.rs @@ -5,6 +5,9 @@ const MAX_MATCH: usize = 258; const WINDOW: usize = 4096; pub const HISTORY: usize = 1 << 18; +const _: () = + assert!(HISTORY == crate::core::block::SEGMENT_BLOCKS * crate::core::container::BLOCK_SIZE); + fn longest_match(bytes: &[u8], pos: usize) -> (usize, usize) { let start = pos.saturating_sub(WINDOW); let max_len = (bytes.len() - pos).min(MAX_MATCH); diff --git a/tests/container.rs b/tests/container.rs index 13d3ba2..7eeaea3 100644 --- a/tests/container.rs +++ b/tests/container.rs @@ -91,6 +91,7 @@ fn v2_framing_round_trips_multi_block() { assert_eq!(packed[4], 2); let c = fossil::core::container::read(&packed).unwrap(); + assert!(c.blocks.len() > 1); assert_eq!(c.decode(), data); let mut lazy = fossil::core::container::read_lazy(&packed).unwrap();