From f4590dfc69838b08ae311f411afb40bf760493f9 Mon Sep 17 00:00:00 2001 From: Matt Brown Date: Sat, 17 Oct 2026 04:05:16 -0400 Subject: [PATCH] lzma: anchor on the dict low-16-bits, not a fixed props byte (#45) signatures/lzma.toml anchored on the fixed prefix 5D 00 00, i.e. props byte 0x5D only (lc3/lp0/pb2, the xz --format=lzma default). A LZMA-alone stream built with any other (lc, lp, pb) was invisible: binwalk found the OpenWrt ath79 kernel at offset 0x200 (props 0x6D = lc1/lp2/pb2) and moria did not, so a user would conclude the kernel is absent rather than that detection failed. The validator is already props-agnostic: its props <= 224 check is exactly the valid (lc<=8, lp<=4, pb<=4) byte set, and it trial-decodes to a clean end-of-stream marker, so false-positive safety never came from the anchor. The matcher is literal (no wildcard), so widen the anchor to the format-invariant 00 00 (the dict_size low 16 bits, always zero because a real dict size is a multiple of 64 KiB) at magic_offset = 1, placing the structure start on the props byte. Same model as the yaffs2 ffff anchor: a common anchor the validator earns. Signature plus two comments only; no validator or decoder logic change. Verification: OpenWrt 0x6D case now reported verified. Corpus sweep over a 375-file slice found 429 real LZMA streams vs 80 with the old anchor (+18 files), 0 crashes, 0 false positives (every new finding re-decodes as valid LZMA-alone with a clean EOF). Full suite green, fuzz smoke clean, ASan/UBSan clean on the lzma path, synthetic, and all gained corpus files. New test_lzma.py section 5 covers props 0x6D/0x5A/0x5C/0xB8 identify + byte-exact 0x6D extract. Also add fuzz_lzx to .gitignore. --- .gitignore | 1 + signatures/lzma.toml | 21 ++++++++++------- src/validators/lzma.cpp | 6 ++--- tests/test_lzma.py | 51 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 66ddebe..ccfb7b7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ build*/ fuzz_scan fuzz_esp32_part fuzz_esp32_nvs +fuzz_lzx *.dSYM/ fuzz/corpus/ crash-* diff --git a/signatures/lzma.toml b/signatures/lzma.toml index 1fe38ae..5d3e975 100644 --- a/signatures/lzma.toml +++ b/signatures/lzma.toml @@ -3,18 +3,23 @@ category = "compression" # Legacy standalone LZMA1 (".lzma alone") has no magic number. Its 13-byte header # is [props u8][dict_size u32 LE][uncompressed_size u64 LE], then a raw LZMA1 -# stream. Anchor on the near-universal firmware header prefix `5D 00 00`: props -# 0x5D (lc3/lp0/pb2, the toolchain default emitted by xz --format=lzma, LZMA SDK, -# and every vendor loader observed) followed by a dict size that is a multiple of -# 64 KiB (so its low 16 bits are 0). The validator then checks the header fields -# and trial-decodes to a clean end-of-stream marker, so a stray `5D 00 00` in -# unrelated data is rejected (no false positive). Identify + extract. -magic_offset = 0 +# stream. The props byte varies with the (lc, lp, pb) parameters, so anchoring on +# a fixed props value (the old `5D 00 00` for the xz --format=lzma default) misses +# streams built with any other parameter set (issue #45: OpenWrt ath79 kernels use +# props 0x6D). Anchor instead on the format-invariant `00 00`: the low 16 bits of +# the dict_size field, which are always zero because every real encoder uses a dict +# size that is a multiple of 64 KiB. magic_offset = 1 places the structure start on +# the props byte that precedes those two zero bytes. Like yaffs2's 0xFFFF anchor, +# the `00 00` pattern is common, so the validator earns the finding: it checks +# props <= 224 (exactly the valid (lc, lp, pb) set), a power-of-two dict in range, +# and trial-decodes to a clean end-of-stream marker, so unrelated `?? 00 00` data +# is rejected (no false positive). Identify + extract. +magic_offset = 1 validator = "lzma" confidence = "structural" [[magic]] -hex = "5d0000" +hex = "0000" endian = "little" [doc] diff --git a/src/validators/lzma.cpp b/src/validators/lzma.cpp index 8d1d132..9fc61aa 100644 --- a/src/validators/lzma.cpp +++ b/src/validators/lzma.cpp @@ -15,8 +15,8 @@ constexpr uint64_t kUnknownSize = UINT64_MAX; // all-0xFF = size not stored // props = (pb * 5 + lp) * 9 + lc, with lc<=8, lp<=4, pb<=4 → max 224. constexpr uint8_t kMaxProps = 224; // Dictionary bounds. Real firmware uses 8/16/32/64 MiB; the format allows less. -// A multiple of 64 KiB (the `5D 00 00` anchor guarantees the low 16 bits are 0), -// required to be a power of two in [4 KiB, 768 MiB]. +// A multiple of 64 KiB (the `00 00` anchor is the dict's low 16 bits, so a match +// guarantees they are 0), required to be a power of two in [4 KiB, 768 MiB]. constexpr uint32_t kMinDict = 1u << 12; constexpr uint32_t kMaxDict = 768u << 20; // Cap the identify-time trial decode. Larger than any plausible firmware LZMA @@ -43,7 +43,7 @@ bool validate_lzma(ValidatorCtx& ctx) { if (known && (*usize == 0 || *usize > kProbeCap)) return false; // implausible declared size // FP-proof gate: trial-decode to a clean end-of-stream marker. A stray - // `5D 00 00 ...` cannot reach the marker, so it is rejected here. + // `?? 00 00 ...` cannot reach the marker, so it is rejected here. auto src = r.bytes(off, r.size() - off); if (!src) return false; size_t consumed = 0; diff --git a/tests/test_lzma.py b/tests/test_lzma.py index b6dfb64..5feeea8 100644 --- a/tests/test_lzma.py +++ b/tests/test_lzma.py @@ -17,6 +17,10 @@ 3. Known-size header: correct declared size passes; wrong declared size rejected. 4. FP guards: valid header + random payload rejected; a truncated stream (no end marker) rejected; a sub-threshold dict rejected. + 5. Non-default props bytes (issue #45): streams built with (lc, lp, pb) other + than the xz default must still be identified + extracted. The anchor is the + format-invariant `00 00` (dict low 16 bits), not a fixed props value, so a + props-0x6D OpenWrt-style stream at an offset is found byte-exact. Run: python3 tests/test_lzma.py """ @@ -50,6 +54,13 @@ def real_alone(payload: bytes) -> bytes: return lzma.compress(payload, format=lzma.FORMAT_ALONE) +def real_alone_props(payload: bytes, lc: int, lp: int, pb: int, dict_size: int = 8 << 20) -> bytes: + """A genuine .lzma-alone stream with explicit (lc, lp, pb) -> arbitrary props byte.""" + filt = [{"id": lzma.FILTER_LZMA1, "dict_size": dict_size, "lc": lc, "lp": lp, "pb": pb}] + c = lzma.LZMACompressor(format=lzma.FORMAT_ALONE, filters=filt) + return c.compress(payload) + c.flush() + + def with_known_size(stream: bytes, size: int) -> bytes: """Patch the 8-byte uncompressed-size field (offset 5) to a definite value.""" b = bytearray(stream) @@ -119,6 +130,46 @@ def check(cond, msg): baddict[2] = 0x00 check(not lzma_findings(bytes(baddict)), "FP: sub-threshold dict size was accepted") + # --- 5. non-default props bytes (issue #45) ----------------------------- + # The old anchor was the fixed prefix `5D 00 00`, so any stream with a props + # byte other than 0x5D was invisible. Confirmed in the wild on OpenWrt ath79 + # (props 0x6D = lc1/lp2/pb2). The anchor is now the format-invariant `00 00` + # (dict low 16 bits), so every valid (lc, lp, pb) is funneled to the validator. + # liblzma's encoder enforces lc + lp <= 4; the validator's props <= 224 check + # still covers the wider decode range, this just picks encoder-producible spread. + props_cases = [ + (1, 2, 2), # 0x6D — OpenWrt ath79 kernel, the reported case + (0, 0, 2), # 0x5A + (2, 0, 2), # 0x5C + (4, 0, 4), # 0xB8 — high props, well off the 0x5D default + ] + for lc, lp, pb in props_cases: + props = (pb * 5 + lp) * 9 + lc + s = real_alone_props(payload, lc, lp, pb) + check(s[0] == props, f"props fixture byte mismatch for ({lc},{lp},{pb})") + # place it at a nonzero offset, as in real firmware (kernel after a header) + blob = b"\x00" * 0x200 + s + gs = lzma_findings(blob) + check(len(gs) == 1, + f"props 0x{props:02x} ({lc},{lp},{pb}): expected one finding, got {len(gs)}") + if gs: + check(gs[0]["confidence_tier"] == "verified", + f"props 0x{props:02x}: tier {gs[0]['confidence_tier']}, want verified") + check(gs[0]["offset"] == 0x200, f"props 0x{props:02x}: offset {gs[0]['offset']:#x}") + + # extract byte-exact for the reported OpenWrt (props 0x6D) case + ow = real_alone_props(payload, 1, 2, 2) + with tempfile.TemporaryDirectory() as t: + img = os.path.join(t, "ow.lzma") + with open(img, "wb") as f: + f.write(ow) + subprocess.run([MORIA, "-e", img], capture_output=True, timeout=120) + dec = os.path.join(img + ".extracted", "0x0-lzma", "decompressed") + check(os.path.exists(dec), "issue #45: props-0x6D extract produced no file") + if os.path.exists(dec): + with open(dec, "rb") as f: + check(f.read() == payload, "issue #45: props-0x6D extract not byte-exact") + print("-" * 60) if fails: for m in fails: