gif/png: size findings to their real terminator, not to EOF (#58)

CMakeLists.txt

@@ -124,6 +124,8 @@ add_executable(moria
   src/validators/jffs2.cpp
   src/validators/zip.cpp
   src/validators/jpeg.cpp
+  src/validators/png.cpp
+  src/validators/gif.cpp
   src/validators/otra.cpp
   src/validators/rae_rfp.cpp
   src/validators/wince_rom.cpp
@@ -254,6 +256,7 @@ if(Python3_Interpreter_FOUND)
     test_luks
     test_lzma
     test_jzlzma
+    test_image_size
     test_rae_rfp
     test_otra
     test_wince_rom

signatures/gif.toml

@@ -2,6 +2,14 @@ name = "gif"
 category = "image"
 confidence = "structural"
 
+# The 6-byte magic alone cannot bound the image (a GIF has no length field) and is
+# weak enough to match stray bytes in binary data. The validator walks the block
+# stream to the 0x3B trailer to set the true size (otherwise the finding runs to
+# the next finding or EOF, swallowing unrelated trailing bytes; see issue #58) and
+# rejects false positives whose first block after the header + color table is not a
+# valid introducer. A clean walk upgrades the tier to `consistent`.
+validator = "gif"
+
 [[magic]]
 ascii = "GIF87a"
 endian = "little"

signatures/png.toml

@@ -5,6 +5,12 @@ category = "image"
 # folded into one 16-byte magic -> effectively self-validating.
 confidence = "consistent"
 
+# The magic identifies the image but cannot bound it. The validator walks the
+# chunk list to IEND to set the true size (otherwise the finding runs to the next
+# finding or EOF, swallowing unrelated trailing bytes; see issue #58) and checks
+# each chunk's CRC-32, upgrading the tier to `verified` when they all pass.
+validator = "png"
+
 [[magic]]
 hex = "89504e470d0a1a0a0000000d49484452"
 endian = "big"

src/validators/gif.cpp

@@ -0,0 +1,91 @@
+// gif.cpp — GIF size via block walk (header ... 0x3B trailer). #58.
+#include "validators/gif.hpp"
+
+#include <cstdint>
+#include <optional>
+
+namespace ft {
+
+namespace {
+
+// Skip a GIF sub-block chain: [size u8][size bytes]... ended by a 0x00 size.
+// Returns the offset just past the terminator, or nullopt on EOF / overrun.
+std::optional<size_t> skip_subblocks(const Reader& r, size_t pos) {
+    const size_t limit = r.size();
+    for (int guard = 0; guard < 100000; ++guard) {
+        auto sz = r.bytes(pos, 1);
+        if (!sz) return std::nullopt;
+        uint8_t n = (*sz)[0];
+        pos += 1;
+        if (n == 0) return pos;              // block terminator
+        if (n > limit - pos) return std::nullopt;  // data runs off EOF
+        pos += n;
+    }
+    return std::nullopt;
+}
+
+}  // namespace
+
+bool validate_gif(ValidatorCtx& ctx) {
+    const Reader& r = ctx.reader;
+    const size_t base = ctx.offset;
+
+    // Header (6 bytes) + logical screen descriptor (7 bytes). The packed byte at
+    // base+10 flags a global color table and encodes its size.
+    auto packed = r.bytes(base + 10, 1);
+    if (!packed) return true;                // too short to walk; keep magic tier
+    size_t pos = base + 13;
+    if ((*packed)[0] & 0x80) {
+        int n = (*packed)[0] & 0x07;
+        pos += 3u << (n + 1);                // global color table: 3 * 2^(n+1) bytes
+    }
+
+    // Validity gate (also FP rejection). The 6-byte GIF8[79]a magic alone matches
+    // stray occurrences in binary/compressed data. After the header + GCT, a real
+    // GIF's data stream must begin with a block introducer: image descriptor
+    // (0x2C), extension (0x21), or the trailer (0x3B). Anything else proves this
+    // is not a GIF, so reject the finding. A truncated-but-real GIF still has a
+    // valid first introducer, so it survives this gate and falls through below.
+    auto first = r.bytes(pos, 1);
+    if (!first) return true;                 // header present but body past EOF: keep magic tier
+    if ((*first)[0] != 0x2C && (*first)[0] != 0x21 && (*first)[0] != 0x3B) return false;
+
+    for (int guard = 0; guard < 1000000; ++guard) {
+        auto intro = r.bytes(pos, 1);
+        if (!intro) break;                   // ran off EOF before the trailer
+        uint8_t b = (*intro)[0];
+        if (b == 0x3B) {                     // trailer
+            ctx.out.size = (pos - base) + 1;
+            ctx.out.set_confidence(Confidence::Consistent,
+                                   "GIF blocks walked to trailer, size computed");
+            return true;
+        }
+        if (b == 0x2C) {                     // image descriptor
+            // 0x2C + left,top,width,height (u16 each) + packed(u8) = 10 bytes;
+            // the image-descriptor packed byte sits at pos+9.
+            auto lp = r.bytes(pos + 9, 1);
+            if (!lp) break;
+            size_t p = pos + 10;
+            if ((*lp)[0] & 0x80) {
+                int n = (*lp)[0] & 0x07;
+                p += 3u << (n + 1);          // local color table
+            }
+            auto lzw = r.bytes(p, 1);        // LZW minimum code size
+            if (!lzw) break;
+            auto nxt = skip_subblocks(r, p + 1);  // image-data sub-block chain
+            if (!nxt) break;
+            pos = *nxt;
+        } else if (b == 0x21) {              // extension: label byte, then sub-blocks
+            auto nxt = skip_subblocks(r, pos + 2);
+            if (!nxt) break;
+            pos = *nxt;
+        } else {
+            break;                           // unknown block: malformed, stop
+        }
+    }
+    // No trailer reached: keep the decisive magic-tier finding and let size
+    // inference fall back. Do not reject.
+    return true;
+}
+
+}  // namespace ft

src/validators/gif.hpp

@@ -0,0 +1,15 @@
+#pragma once
+#include "signature.hpp"
+namespace ft {
+// Compute a GIF's true on-disk size by walking its block stream (logical screen
+// descriptor, optional global color table, then image-descriptor / extension
+// blocks and their sub-block chains) to the 0x3B trailer, instead of letting the
+// finding run to the next finding or EOF. A full walk to the trailer promotes the
+// finding to `consistent` (GIF has no checksum to reach `verified`). The 6-byte
+// GIF8[79]a magic is weak, so the validator also rejects clear false positives:
+// after the header + global color table a real GIF must begin with a block
+// introducer (0x2C / 0x21 / 0x3B), and anything else is not a GIF. A truncated
+// but real GIF keeps a valid first introducer, so it survives with its size left
+// for inference. See issue #58.
+bool validate_gif(ValidatorCtx& ctx);
+}  // namespace ft

src/validators/png.cpp

@@ -0,0 +1,50 @@
+// png.cpp — PNG size via chunk walk (IHDR ... IEND), with per-chunk CRC-32. #58.
+#include "validators/png.hpp"
+
+#include <cstdint>
+
+#include "crc32.hpp"
+
+namespace ft {
+
+bool validate_png(ValidatorCtx& ctx) {
+    const Reader& r = ctx.reader;
+    const size_t base = ctx.offset;
+    const size_t limit = r.size();
+
+    // The magic already matched the 8-byte signature plus the IHDR length+type,
+    // so the first chunk starts at base+8. A chunk is length(4 BE) | type(4) |
+    // data[length] | crc(4); the CRC covers type+data. Walk to IEND.
+    size_t pos = base + 8;
+    bool all_crc_ok = true;
+    for (int guard = 0; guard < 100000; ++guard) {
+        auto len = r.at<uint32_t>(pos, Endian::Big);
+        auto type = r.bytes(pos + 4, 4);
+        if (!len || !type) break;             // chunk header runs off EOF
+        if (*len > limit - (pos + 8)) break;  // data runs off EOF
+        const size_t crc_pos = pos + 8 + *len;
+        auto crc = r.at<uint32_t>(crc_pos, Endian::Big);
+        if (!crc) break;                      // crc runs off EOF
+        auto covered = r.bytes(pos + 4, 4 + *len);  // type + data
+        if (covered && crc32_ieee(*covered) != *crc) all_crc_ok = false;
+
+        const auto& t = *type;
+        const bool is_iend = t[0] == 'I' && t[1] == 'E' && t[2] == 'N' && t[3] == 'D';
+        const size_t chunk_end = crc_pos + 4;
+        if (is_iend) {
+            ctx.out.size = chunk_end - base;
+            if (all_crc_ok)
+                ctx.out.set_confidence(Confidence::Verified,
+                                       "PNG chunks walked to IEND, every chunk CRC-32 verified");
+            else
+                ctx.out.set_confidence(Confidence::Consistent, "PNG chunks walked to IEND");
+            return true;
+        }
+        pos = chunk_end;
+    }
+    // No IEND reached (truncated or malformed): keep the decisive magic-tier
+    // finding and let size inference fall back. Do not reject.
+    return true;
+}
+
+}  // namespace ft

src/validators/png.hpp

@@ -0,0 +1,12 @@
+#pragma once
+#include "signature.hpp"
+namespace ft {
+// Compute a PNG's true on-disk size by walking its chunk list from the IHDR
+// (just past the 8-byte signature) to IEND, instead of letting the finding run
+// to the next finding or EOF. Each chunk's CRC-32 is checked: a full walk with
+// every CRC valid promotes the finding to `verified`, a walk that reaches IEND
+// without full CRC coverage stays `consistent`. Never rejects — the 16-byte
+// signature+IHDR magic is already decisive, so a truncated image keeps its
+// finding with size left for inference. See issue #58.
+bool validate_png(ValidatorCtx& ctx);
+}  // namespace ft

src/validators/registry.cpp

@@ -8,6 +8,7 @@
 #include "validators/dtb.hpp"
 #include "validators/elf.hpp"
 #include "validators/esp32_part.hpp"
+#include "validators/gif.hpp"
 #include "validators/esp32_nvs.hpp"
 #include "validators/ihex.hpp"
 #include "validators/jffs2.hpp"
@@ -19,6 +20,7 @@
 #include "validators/lzma.hpp"
 #include "validators/otra.hpp"
 #include "validators/partition.hpp"
+#include "validators/png.hpp"
 #include "validators/rae_rfp.hpp"
 #include "validators/spiffs.hpp"
 #include "validators/squashfs.hpp"
@@ -54,6 +56,8 @@ Validator find_validator(const std::string& name) {
     if (name == "tar") return validate_tar;
     if (name == "zip") return validate_zip;
     if (name == "jpeg") return validate_jpeg;
+    if (name == "png") return validate_png;
+    if (name == "gif") return validate_gif;
     if (name == "rae_rfp") return validate_rae_rfp;
     if (name == "wince_rom") return validate_wince_rom;
     if (name == "wince_hive") return validate_wince_hive;

tests/gen_samples.py

@@ -591,13 +591,32 @@ def ihex():
     return line.encode()
 
 
+def _png_chunk(typ: bytes, data: bytes) -> bytes:
+    import zlib
+    return (struct.pack(">I", len(data)) + typ + data +
+            struct.pack(">I", zlib.crc32(typ + data) & 0xFFFFFFFF))
+
+
 def png():
-    ihdr = struct.pack(">I", 13) + b"IHDR" + struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0)
-    return b"\x89PNG\r\n\x1a\n" + ihdr + b"\0" * 8
+    # A complete 1x1 PNG: signature + IHDR + IDAT + IEND, every chunk CRC valid
+    # (so the validator walks to IEND and reports `verified`). The 16-byte magic
+    # is sig(8) + IHDR length+tag, so this still matches the signature.
+    import zlib
+    sig = b"\x89PNG\r\n\x1a\n"
+    ihdr = _png_chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0))
+    idat = _png_chunk(b"IDAT", zlib.compress(b"\x00\x00\x00\x00"))
+    iend = _png_chunk(b"IEND", b"")
+    return sig + ihdr + idat + iend
 
 
 def gif():
-    return b"GIF89a" + struct.pack("<HH", 1, 1) + b"\x00\x00\x00" + b"\0" * 8
+    # A complete 1x1 GIF89a: header + logical screen descriptor (no GCT) + one
+    # image descriptor (no LCT) + a minimal LZW data sub-block chain + the 0x3B
+    # trailer, so the validator walks to the trailer and reports `consistent`.
+    hdr = b"GIF89a" + struct.pack("<HH", 1, 1) + b"\x00\x00\x00"  # w,h,packed=0,bg,aspect
+    imgdesc = b"\x2c" + struct.pack("<HHHH", 0, 0, 1, 1) + b"\x00"  # left,top,w,h,packed=0
+    img_data = b"\x02" + b"\x01\x4c" + b"\x00"  # LZW min code size, 1-byte sub-block, terminator
+    return hdr + imgdesc + img_data + b"\x3b"
 
 
 def bmp():

tests/run.sh

@@ -41,6 +41,8 @@ python3 tests/test_lzma.py
 # Ingenic jzlzma (hardware LZ77) identify (verified via trial-decode) + byte-exact
 # extract + recursion + FP guards.
 python3 tests/test_jzlzma.py
+# GIF/PNG sized to their real terminator (0x3B / IEND), not to EOF (issue #58).
+python3 tests/test_image_size.py
 # RFP section extraction + LZARI decode round-trip (self-contained hard gate).
 python3 tests/test_rae_rfp.py
 # Windows CE XIP ROM: ROMHDR resolution, module PE rebuild, CECompress decode.