diff --git a/signatures/bzip2.toml b/signatures/bzip2.toml index a8e59d4..ea25dfe 100644 --- a/signatures/bzip2.toml +++ b/signatures/bzip2.toml @@ -4,9 +4,18 @@ category = "compression" struct = """ skip[3]; u8 level; + u8 bm0; u8 bm1; u8 bm2; u8 bm3; u8 bm4; u8 bm5; """ -# "BZh" then a block-size digit 1-9. -constraints = [ "level in 0x31..0x39" ] +# "BZh" + a block-size digit 1-9, then the 48-bit stream block magic. Checking the +# block magic lifts the test from 4 bytes to 10 and removes the structural false +# positive where a chance "BZh<1-9>" in opaque/compressed data was identified as +# bzip2 (it then failed to decompress). The 6 bytes after the header are the +# first block's magic 0x314159265359 ("1AY&SY", the BCD digits of pi), or, for a +# stream with no blocks, the end-of-stream magic 0x177245385090. +constraints = [ + "level in 0x31..0x39", + "(bm0 == 0x31 && bm1 == 0x41 && bm2 == 0x59 && bm3 == 0x26 && bm4 == 0x53 && bm5 == 0x59) || (bm0 == 0x17 && bm1 == 0x72 && bm2 == 0x45 && bm3 == 0x38 && bm4 == 0x50 && bm5 == 0x90)", +] confidence = "structural" [[magic]] diff --git a/tests/run.sh b/tests/run.sh index 3102837..8845958 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -38,6 +38,8 @@ python3 tests/test_legacy_fs.py python3 tests/test_luks.py # Legacy standalone .lzma identify (verified via trial-decode) + byte-exact extract + FP guards. python3 tests/test_lzma.py +# bzip2 identify requires the 48-bit stream block magic, not just "BZh" (FP guard). +python3 tests/test_bzip2.py # Ingenic jzlzma (hardware LZ77) identify (verified via trial-decode) + byte-exact # extract + recursion + FP guards. python3 tests/test_jzlzma.py diff --git a/tests/test_bzip2.py b/tests/test_bzip2.py new file mode 100644 index 0000000..0cf1b1d --- /dev/null +++ b/tests/test_bzip2.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""bzip2 identify regression: require the stream block magic, not just "BZh". + +The bzip2 signature used to accept any "BZh" followed by an ASCII block-size +digit (1-9) -- a 4-byte test. In a multi-megabyte opaque/compressed blob a +chance "BZh7"-style sequence satisfies that and was reported as bzip2, then +failed to decompress. A real bzip2 stream carries a 48-bit block magic right +after the 4-byte header: 0x314159265359 for the first compressed block, or the +end-of-stream magic 0x177245385090 when there are no blocks. Checking those six +bytes lifts the match from 4 bytes to 10 and removes the false positive. + +Real streams come from Python's stdlib bz2 module, so no external tool is needed. + +Run: python3 tests/test_bzip2.py [path-to-moria] +""" +import bz2 +import json +import os +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +MORIA = sys.argv[1] if len(sys.argv) > 1 else os.path.join(HERE, "..", "build", "moria") + +fails = [] + + +def check(cond, msg): + if not cond: + fails.append(msg) + + +def bzip2_findings(data: bytes): + with tempfile.NamedTemporaryFile(suffix=".bin") as f: + f.write(data) + f.flush() + r = subprocess.run([MORIA, "-j", f.name], capture_output=True, timeout=120) + doc = json.loads(r.stdout) + return [x for x in doc.get("findings", []) if x["type"] == "bzip2"] + + +def main(): + payload = (b"The quick brown fox jumps over the lazy dog. " * 400) + + # 1) A genuine bzip2 stream (has the 0x314159265359 block magic) is identified, + # at offset 0 and at a realistic nonzero offset. + real = bz2.compress(payload) + check(real[:4] == b"BZh9", f"fixture: unexpected header {real[:4]!r}") + check(real[4:10] == bytes.fromhex("314159265359"), + f"fixture: expected block magic, got {real[4:10].hex()}") + gs = bzip2_findings(real) + check(len(gs) == 1 and gs[0]["offset"] == 0, + f"real bzip2 at 0: expected one finding at 0, got {[(g['offset']) for g in gs]}") + gs = bzip2_findings(b"\x00" * 0x400 + real) + check(len(gs) == 1 and gs[0]["offset"] == 0x400, + f"real bzip2 at 0x400: got {[(g['offset']) for g in gs]}") + + # 2) An empty bzip2 stream (no blocks -> end-of-stream magic 0x177245385090 + # directly after the header) is still identified. + empty = bz2.compress(b"") + check(empty[4:10] == bytes.fromhex("177245385090"), + f"fixture: expected EOS magic, got {empty[4:10].hex()}") + gs = bzip2_findings(empty) + check(len(gs) == 1, f"empty bzip2 stream: expected one finding, got {len(gs)}") + + # 3) FALSE-POSITIVE guard: "BZh" + a digit but NOT followed by the block magic + # must NOT be identified as bzip2. (This is the real tc7200 FP shape.) + fp = b"BZh7" + bytes.fromhex("11c0e9878492") + os.urandom(4096) + gs = bzip2_findings(fp) + check(len(gs) == 0, f"FP guard: 'BZh7'+non-magic must not match, got {len(gs)}") + + # 4) FALSE-POSITIVE guard: valid 4-byte header + random where the next six bytes + # are random -> overwhelmingly rejected. + hits = 0 + for i in range(64): + blob = b"BZh" + bytes([0x31 + (i % 9)]) + os.urandom(64) + hits += len(bzip2_findings(blob)) + check(hits == 0, f"FP guard: 64 random post-header blobs yielded {hits} bzip2 matches") + + print("-" * 60) + if fails: + for m in fails: + print("FAIL:", m) + return 1 + print("PASS: bzip2 identify requires block magic (real + empty match; FP shapes rejected)") + return 0 + + +if __name__ == "__main__": + sys.exit(main())