From e5960281fb81c462ae86f4bf04e17e96df92c43f Mon Sep 17 00:00:00 2001 From: fengqiaoqu <325140621+fengqiaoqu@users.noreply.github.com> Date: Sat, 17 Oct 2026 10:14:26 +0800 Subject: [PATCH] lzma: also anchor on props 0x6D (OpenWrt lc1/lp2/pb2 kernels) The .lzma-alone header has no magic, and the signature only matched props 0x5D. OpenWrt compresses its kernels with lc1/lp2/pb2 (props 0x6D), so the kernel at 0x200 in its factory images went unreported. Add a 6d0000 anchor; the validator's trial decode still filters stray matches. Extend test_lzma.py with an embedded 0x6D stream. Fixes #45 --- signatures/lzma.toml | 18 ++++++++++++------ src/validators/lzma.cpp | 5 +++-- src/validators/lzma.hpp | 9 +++++---- tests/test_lzma.py | 34 +++++++++++++++++++++++++++++----- 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/signatures/lzma.toml b/signatures/lzma.toml index 1fe38ae..d67a6ab 100644 --- a/signatures/lzma.toml +++ b/signatures/lzma.toml @@ -3,12 +3,14 @@ category = "compression" # Legacy standalone LZMA1 (".lzma alone") has no magic number. Its 13-byte header # is [props u8][dict_size u32 LE][uncompressed_size u64 LE], then a raw LZMA1 -# stream. Anchor on the near-universal firmware header prefix `5D 00 00`: props -# 0x5D (lc3/lp0/pb2, the toolchain default emitted by xz --format=lzma, LZMA SDK, -# and every vendor loader observed) followed by a dict size that is a multiple of -# 64 KiB (so its low 16 bits are 0). The validator then checks the header fields -# and trial-decodes to a clean end-of-stream marker, so a stray `5D 00 00` in -# unrelated data is rejected (no false positive). Identify + extract. +# stream. The props byte is a parameter, not a magic, so anchor on the props +# values firmware actually ships, each followed by a dict size that is a multiple +# of 64 KiB (so its low 16 bits are 0): +# 5D 00 00 lc3/lp0/pb2, the default of xz --format=lzma and the LZMA SDK +# 6D 00 00 lc1/lp2/pb2, what OpenWrt's build uses for its kernels (#45) +# The validator then checks the header fields and trial-decodes to a clean +# end-of-stream marker, so a stray anchor in unrelated data is rejected (no +# false positive). Identify + extract. magic_offset = 0 validator = "lzma" confidence = "structural" @@ -17,6 +19,10 @@ confidence = "structural" hex = "5d0000" endian = "little" +[[magic]] +hex = "6d0000" +endian = "little" + [doc] description = "LZMA-compressed data (legacy .lzma alone container, LZMA1)." references = [ diff --git a/src/validators/lzma.cpp b/src/validators/lzma.cpp index 8d1d132..249ed57 100644 --- a/src/validators/lzma.cpp +++ b/src/validators/lzma.cpp @@ -15,7 +15,7 @@ constexpr uint64_t kUnknownSize = UINT64_MAX; // all-0xFF = size not stored // props = (pb * 5 + lp) * 9 + lc, with lc<=8, lp<=4, pb<=4 → max 224. constexpr uint8_t kMaxProps = 224; // Dictionary bounds. Real firmware uses 8/16/32/64 MiB; the format allows less. -// A multiple of 64 KiB (the `5D 00 00` anchor guarantees the low 16 bits are 0), +// A multiple of 64 KiB (the `xx 00 00` anchors guarantee the low 16 bits are 0), // required to be a power of two in [4 KiB, 768 MiB]. constexpr uint32_t kMinDict = 1u << 12; constexpr uint32_t kMaxDict = 768u << 20; @@ -43,7 +43,8 @@ bool validate_lzma(ValidatorCtx& ctx) { if (known && (*usize == 0 || *usize > kProbeCap)) return false; // implausible declared size // FP-proof gate: trial-decode to a clean end-of-stream marker. A stray - // `5D 00 00 ...` cannot reach the marker, so it is rejected here. + // anchor match (`5D 00 00 ...`, `6D 00 00 ...`) cannot reach the marker, so + // it is rejected here. auto src = r.bytes(off, r.size() - off); if (!src) return false; size_t consumed = 0; diff --git a/src/validators/lzma.hpp b/src/validators/lzma.hpp index 15976a4..79e25b2 100644 --- a/src/validators/lzma.hpp +++ b/src/validators/lzma.hpp @@ -4,9 +4,10 @@ namespace ft { // The legacy .lzma "alone" container has no magic number: a 13-byte header // (props byte, 4-byte dict size, 8-byte uncompressed size) then a raw LZMA1 -// stream. Anchored on the near-universal firmware header prefix `5D 00 00` -// (props 0x5D = lc3/lp0/pb2, dict a multiple of 64 KiB), this validates the -// header fields and trial-decodes to a clean end-of-stream marker, so a stray -// `5D 00 00` in unrelated data cannot false-positive. Reports `verified`. +// stream. Anchored on the props values firmware ships (0x5D = lc3/lp0/pb2, the +// toolchain default; 0x6D = lc1/lp2/pb2, OpenWrt's kernels) followed by a dict +// that is a multiple of 64 KiB, this validates the header fields and +// trial-decodes to a clean end-of-stream marker, so a stray anchor match in +// unrelated data cannot false-positive. Reports `verified`. bool validate_lzma(ValidatorCtx& ctx); } // namespace ft diff --git a/tests/test_lzma.py b/tests/test_lzma.py index b6dfb64..6a19e3a 100644 --- a/tests/test_lzma.py +++ b/tests/test_lzma.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -"""Legacy standalone LZMA (".lzma alone") identify + extract regression (issue #43). +"""Legacy standalone LZMA (".lzma alone") identify + extract regression (issues #43, #45). moria must recognize a legacy LZMA1 "alone" stream (no magic number: a 13-byte [props][dict_size][uncompressed_size] header, then a raw LZMA1 stream) and, under --e, decompress it byte-exact. The format is anchored on the near-universal -firmware header prefix `5D 00 00` and confirmed by a trial-decode to a clean -end-of-stream marker, so the false-positive guards below are the point of the -whole exercise. +-e, decompress it byte-exact. The format is anchored on the header prefixes +firmware ships (`5D 00 00`, the toolchain default, and `6D 00 00`, OpenWrt's +lc1/lp2/pb2 kernels) and confirmed by a trial-decode to a clean end-of-stream +marker, so the false-positive guards below are the point of the whole exercise. Real streams here come from Python's stdlib lzma module (FORMAT_ALONE), so the "real stream" cases need no external tool. @@ -17,6 +17,8 @@ 3. Known-size header: correct declared size passes; wrong declared size rejected. 4. FP guards: valid header + random payload rejected; a truncated stream (no end marker) rejected; a sub-threshold dict rejected. + 5. Non-default props: an OpenWrt-style props 0x6D stream embedded at 0x200 is + identified and extracted byte-exact (#45). Run: python3 tests/test_lzma.py """ @@ -119,6 +121,28 @@ def check(cond, msg): baddict[2] = 0x00 check(not lzma_findings(bytes(baddict)), "FP: sub-threshold dict size was accepted") + # --- 5. OpenWrt props 0x6D (lc1/lp2/pb2), not at offset 0 (#45) --------- + owrt = lzma.compress(payload, format=lzma.FORMAT_ALONE, filters=[{ + "id": lzma.FILTER_LZMA1, "lc": 1, "lp": 2, "pb": 2, "dict_size": 8 << 20}]) + check(owrt[:3] == b"\x6d\x00\x00", "props 0x6D fixture header not 6d 00 00") + image = b"\x00" * 0x200 + owrt + b"\xff" * 64 + g6 = lzma_findings(image) + check(len(g6) == 1, f"props 0x6D: expected one lzma finding, got {len(g6)}") + if g6: + check(g6[0]["offset"] == 0x200, f"props 0x6D: offset {g6[0]['offset']:#x}, want 0x200") + check(g6[0]["confidence_tier"] == "verified", + f"props 0x6D: tier not verified: {g6[0]['confidence_tier']}") + with tempfile.TemporaryDirectory() as t: + img = os.path.join(t, "owrt.bin") + with open(img, "wb") as f: + f.write(image) + subprocess.run([MORIA, "-e", img], capture_output=True, timeout=120) + dec = os.path.join(img + ".extracted", "0x200-lzma", "decompressed") + check(os.path.exists(dec), "props 0x6D extract: decompressed file missing") + if os.path.exists(dec): + with open(dec, "rb") as f: + check(f.read() == payload, "props 0x6D extract: not byte-exact") + print("-" * 60) if fails: for m in fails: