lzma: anchor on the dict low-16-bits, not a fixed props byte (#45)

.gitignore

@@ -2,6 +2,7 @@ build*/
 fuzz_scan
 fuzz_esp32_part
 fuzz_esp32_nvs
+fuzz_lzx
 *.dSYM/
 fuzz/corpus/
 crash-*

signatures/lzma.toml

@@ -3,18 +3,23 @@ category = "compression"
 
 # Legacy standalone LZMA1 (".lzma alone") has no magic number. Its 13-byte header
 # is [props u8][dict_size u32 LE][uncompressed_size u64 LE], then a raw LZMA1
-# stream. Anchor on the near-universal firmware header prefix `5D 00 00`: props
-# 0x5D (lc3/lp0/pb2, the toolchain default emitted by xz --format=lzma, LZMA SDK,
-# and every vendor loader observed) followed by a dict size that is a multiple of
-# 64 KiB (so its low 16 bits are 0). The validator then checks the header fields
-# and trial-decodes to a clean end-of-stream marker, so a stray `5D 00 00` in
-# unrelated data is rejected (no false positive). Identify + extract.
-magic_offset = 0
+# stream. The props byte varies with the (lc, lp, pb) parameters, so anchoring on
+# a fixed props value (the old `5D 00 00` for the xz --format=lzma default) misses
+# streams built with any other parameter set (issue #45: OpenWrt ath79 kernels use
+# props 0x6D). Anchor instead on the format-invariant `00 00`: the low 16 bits of
+# the dict_size field, which are always zero because every real encoder uses a dict
+# size that is a multiple of 64 KiB. magic_offset = 1 places the structure start on
+# the props byte that precedes those two zero bytes. Like yaffs2's 0xFFFF anchor,
+# the `00 00` pattern is common, so the validator earns the finding: it checks
+# props <= 224 (exactly the valid (lc, lp, pb) set), a power-of-two dict in range,
+# and trial-decodes to a clean end-of-stream marker, so unrelated `?? 00 00` data
+# is rejected (no false positive). Identify + extract.
+magic_offset = 1
 validator = "lzma"
 confidence = "structural"
 
 [[magic]]
-hex = "5d0000"
+hex = "0000"
 endian = "little"
 
 [doc]

src/validators/lzma.cpp

@@ -15,8 +15,8 @@ constexpr uint64_t kUnknownSize = UINT64_MAX;  // all-0xFF = size not stored
 // props = (pb * 5 + lp) * 9 + lc, with lc<=8, lp<=4, pb<=4 → max 224.
 constexpr uint8_t kMaxProps = 224;
 // Dictionary bounds. Real firmware uses 8/16/32/64 MiB; the format allows less.
-// A multiple of 64 KiB (the `5D 00 00` anchor guarantees the low 16 bits are 0),
-// required to be a power of two in [4 KiB, 768 MiB].
+// A multiple of 64 KiB (the `00 00` anchor is the dict's low 16 bits, so a match
+// guarantees they are 0), required to be a power of two in [4 KiB, 768 MiB].
 constexpr uint32_t kMinDict = 1u << 12;
 constexpr uint32_t kMaxDict = 768u << 20;
 // Cap the identify-time trial decode. Larger than any plausible firmware LZMA
@@ -43,7 +43,7 @@ bool validate_lzma(ValidatorCtx& ctx) {
     if (known && (*usize == 0 || *usize > kProbeCap)) return false;  // implausible declared size
 
     // FP-proof gate: trial-decode to a clean end-of-stream marker. A stray
-    // `5D 00 00 ...` cannot reach the marker, so it is rejected here.
+    // `?? 00 00 ...` cannot reach the marker, so it is rejected here.
     auto src = r.bytes(off, r.size() - off);
     if (!src) return false;
     size_t consumed = 0;

tests/test_lzma.py

@@ -17,6 +17,10 @@
   3. Known-size header: correct declared size passes; wrong declared size rejected.
   4. FP guards: valid header + random payload rejected; a truncated stream (no end
      marker) rejected; a sub-threshold dict rejected.
+  5. Non-default props bytes (issue #45): streams built with (lc, lp, pb) other
+     than the xz default must still be identified + extracted. The anchor is the
+     format-invariant `00 00` (dict low 16 bits), not a fixed props value, so a
+     props-0x6D OpenWrt-style stream at an offset is found byte-exact.
 
 Run: python3 tests/test_lzma.py
 """
@@ -50,6 +54,13 @@ def real_alone(payload: bytes) -> bytes:
     return lzma.compress(payload, format=lzma.FORMAT_ALONE)
 
 
+def real_alone_props(payload: bytes, lc: int, lp: int, pb: int, dict_size: int = 8 << 20) -> bytes:
+    """A genuine .lzma-alone stream with explicit (lc, lp, pb) -> arbitrary props byte."""
+    filt = [{"id": lzma.FILTER_LZMA1, "dict_size": dict_size, "lc": lc, "lp": lp, "pb": pb}]
+    c = lzma.LZMACompressor(format=lzma.FORMAT_ALONE, filters=filt)
+    return c.compress(payload) + c.flush()
+
+
 def with_known_size(stream: bytes, size: int) -> bytes:
     """Patch the 8-byte uncompressed-size field (offset 5) to a definite value."""
     b = bytearray(stream)
@@ -119,6 +130,46 @@ def check(cond, msg):
     baddict[2] = 0x00
     check(not lzma_findings(bytes(baddict)), "FP: sub-threshold dict size was accepted")
 
+    # --- 5. non-default props bytes (issue #45) -----------------------------
+    # The old anchor was the fixed prefix `5D 00 00`, so any stream with a props
+    # byte other than 0x5D was invisible. Confirmed in the wild on OpenWrt ath79
+    # (props 0x6D = lc1/lp2/pb2). The anchor is now the format-invariant `00 00`
+    # (dict low 16 bits), so every valid (lc, lp, pb) is funneled to the validator.
+    # liblzma's encoder enforces lc + lp <= 4; the validator's props <= 224 check
+    # still covers the wider decode range, this just picks encoder-producible spread.
+    props_cases = [
+        (1, 2, 2),  # 0x6D — OpenWrt ath79 kernel, the reported case
+        (0, 0, 2),  # 0x5A
+        (2, 0, 2),  # 0x5C
+        (4, 0, 4),  # 0xB8 — high props, well off the 0x5D default
+    ]
+    for lc, lp, pb in props_cases:
+        props = (pb * 5 + lp) * 9 + lc
+        s = real_alone_props(payload, lc, lp, pb)
+        check(s[0] == props, f"props fixture byte mismatch for ({lc},{lp},{pb})")
+        # place it at a nonzero offset, as in real firmware (kernel after a header)
+        blob = b"\x00" * 0x200 + s
+        gs = lzma_findings(blob)
+        check(len(gs) == 1,
+              f"props 0x{props:02x} ({lc},{lp},{pb}): expected one finding, got {len(gs)}")
+        if gs:
+            check(gs[0]["confidence_tier"] == "verified",
+                  f"props 0x{props:02x}: tier {gs[0]['confidence_tier']}, want verified")
+            check(gs[0]["offset"] == 0x200, f"props 0x{props:02x}: offset {gs[0]['offset']:#x}")
+
+    # extract byte-exact for the reported OpenWrt (props 0x6D) case
+    ow = real_alone_props(payload, 1, 2, 2)
+    with tempfile.TemporaryDirectory() as t:
+        img = os.path.join(t, "ow.lzma")
+        with open(img, "wb") as f:
+            f.write(ow)
+        subprocess.run([MORIA, "-e", img], capture_output=True, timeout=120)
+        dec = os.path.join(img + ".extracted", "0x0-lzma", "decompressed")
+        check(os.path.exists(dec), "issue #45: props-0x6D extract produced no file")
+        if os.path.exists(dec):
+            with open(dec, "rb") as f:
+                check(f.read() == payload, "issue #45: props-0x6D extract not byte-exact")
+
     print("-" * 60)
     if fails:
         for m in fails: