Indexer: Preflight Oversized Images, Increase Watchdog Frequency, and Identify Files That Cause Crashes

Sources/DigUpApp/Engine.swift

@@ -110,6 +110,24 @@ nonisolated final class Engine: @unchecked Sendable {
     private var watchdogStops: [Date] = []
     /// The helper's exit status when its watchdog ended it on a file (`digup`'s Watchdog: EX_TEMPFAIL).
     private static let stoppedOnFile: Int32 = 75
+    /// The queue's pending-file count as the helper last reported it, and what it was at the last stop on a
+    /// file: stops are a stall only when the queue isn't moving between them (issue #182 — 55,000 files
+    /// behind oversized RAWs, where the 4-stops cap froze the index after 4 of them).
+    private var lastPending: Int?
+    private var pendingAtLastStop: Int?
+    /// Exits that mean "the helper died on the file it was reading": fatal signals — SIGILL, SIGTRAP (Swift's
+    /// allocation trap, the "exited (5)" of #182), SIGABRT, SIGFPE, SIGKILL, SIGBUS (mapped-file faults:
+    /// the crash class a truncated picture gives), SIGSEGV. `terminationStatus` carries the signal on
+    /// macOS; the helper's `CrashNotes` leaves a note beside the index, so the next helper marks that
+    /// file. (10 was missing from the list when the pre-push review checked it: SIGBUS crashes left the
+    /// note unread and the helper unrestarted.)
+    static func diedOnFile(_ status: Int32) -> Bool { [4, 5, 6, 8, 9, 10, 11].contains(status) }
+    /// The last file the helper reported reading (its `"file"` events): when it dies on the spot, the
+    /// app names it for the next helper (`DIGUP_STOPPED_FILE`), because the queue's head is not
+    /// necessarily the file it died on — image batches keep all eight files pending till the batch
+    /// ends, and the head-marking fallback would fail an innocent one (pre-push review of #182).
+    private var lastReadFile: String?
+    private var crashFile: String?   // the name to pass on, taken by the next spawn
     /// The next helper drops files under folders that are no longer chosen. At launch too, in case the folders
     /// changed while a helper had no time to; but an app with no folders at all never empties the index by itself.
     private var pruneWanted: Bool
@@ -518,7 +536,12 @@ nonisolated final class Engine: @unchecked Sendable {
             arguments += extra.split(separator: " ").map(String.init)
         }
         process.arguments = arguments + roots.map(\.path)
-        process.environment = ProcessInfo.processInfo.environment.merging(["DIGUP_MODELS": modelsDirectory.path]) { $1 }
+        var environment = ["DIGUP_MODELS": modelsDirectory.path]
+        if let file = crashFile {   // taken once: the next helper marks the file its predecessor died reading
+            environment["DIGUP_STOPPED_FILE"] = file
+            crashFile = nil
+        }
+        process.environment = ProcessInfo.processInfo.environment.merging(environment) { $1 }
         process.qualityOfService = .utility
         let input = Pipe(), output = Pipe()
         process.standardInput = input
@@ -610,6 +633,7 @@ nonisolated final class Engine: @unchecked Sendable {
                 + "\(int("removed")) removed · \(int("unchanged")) unchanged in \(int("ms")) ms")
         case "file":
             let (done, total) = (int("done"), int("total"))
+            lastReadFile = event["path"] as? String
             update { $0.activity = .indexing(done: done, total: total) }
             if event["new"] as? Bool == true, let path = event["path"] as? String {
                 log(String(format: "index: %@ %@ in %.2f s", (path as NSString).lastPathComponent,
@@ -675,6 +699,7 @@ nonisolated final class Engine: @unchecked Sendable {
             log("code sync: \(int("added")) new · \(int("changed")) changed · \(int("moved")) moved · "
                 + "\(int("removed")) removed · \(int("unchanged")) unchanged in \(int("ms")) ms")
         case "file":
+            lastReadFile = event["path"] as? String
             update { $0.code.indexing = .init(done: int("done"), total: int("total")) }
             refreshCodeSearcher(force: false)
         case "summary":
@@ -708,6 +733,7 @@ nonisolated final class Engine: @unchecked Sendable {
 
     private func applyCounts(_ event: [String: Any]) {
         if let files = event["files"] as? Int {
+            lastPending = event["pending"] as? Int
             update {
                 $0.files = files
                 $0.searchable = event["searchable"] as? Int ?? 0
@@ -746,10 +772,24 @@ nonisolated final class Engine: @unchecked Sendable {
             restartHelper = false
             startWanted = false
             startHelper()
-        } else if status == Self.stoppedOnFile {
-            // Its watchdog ended it on a file whose reading ran away: the next one skips that file and goes on. Not
-            // over and over, though, if something else is wrong.
+        } else if status == Self.stoppedOnFile || Self.diedOnFile(status) {
+            // Its watchdog ended it on a file whose reading ran away, or it died on the spot (Swift's
+            // allocation trap — "exited (5)" — which beats the watchdog: `CrashNotes` noted the queue's head,
+            // and the next helper marks it). Not over and over, though, if something else is wrong — but a
+            // queue that moves 100 files between stops is getting past its bad files, not looping (#182:
+            // a photo index of 55,000 froze on this cap after four oversized RAWs).
+            if Self.diedOnFile(status) {
+                log("indexer: it died (exit \(status)) on a file: it will be marked, and indexing goes on")
+                // Name it for the helper that starts next (its `CrashNotes` note carries no path):
+                // the queue's head-mark is a fallback, not the answer — a media batch keeps all its
+                // files pending, and the head of a queue of images is rarely the one being read.
+                crashFile = lastReadFile
+            }
             watchdogStops = watchdogStops.filter { Date().timeIntervalSince($0) < 600 } + [Date()]
+            if let pending = lastPending, let before = pendingAtLastStop, before - pending >= 100 {
+                watchdogStops = []
+            }
+            pendingAtLastStop = lastPending
             if watchdogStops.count <= 3 { startHelper() } else { log("indexer: stopped on files 4 times in 10 min") }
         } else if syncWantedAt != nil, status == 0 {
             // A sync asked for while it was on its way out still has to happen.

Sources/DigUpKit/Extractors.swift

@@ -20,14 +20,124 @@ enum ExtractError: Error, CustomStringConvertible {
 }
 
 /// Pictures for the model: a downscaled copy of each image (`prepare`) and the camera's date. ImageIO on macOS (HEIC,
-/// RAW, WebP and friends); stb_image elsewhere (Platform/).
-enum ImageExtractor {
+/// RAW, WebP and friends); stb_image elsewhere (Platform/). Public for the indexing helper to set its
+/// `decodeMemoryBudget` at startup; `prepare` itself stays internal to the kit.
+public enum ImageExtractor {
     struct Prepared: Sendable {
         let path: String
         let width: Int
         let height: Int
         let taken: String?   // EXIF DateTimeOriginal, as written by the camera
     }
+
+    /// What reading one picture may cost the helper in memory, set at its start from `--memory-limit`
+    /// (`main.swift`: the watchdog's own limit). A picture whose header says its reading would ask for more
+    /// goes into the index as unreadable *without being decoded*: a 45 MP RAW has run to over 8 GB in the
+    /// wild, killing the helper before the watchdog's tick could note the file, and every restart took that
+    /// file first and died on it again (issue #182: 55,000 files stuck behind it). 0 = no preflight, for the
+    /// tools that don't index under a watchdog.
+    nonisolated(unsafe) public static var decodeMemoryBudget: UInt64 = 0
+
+    /// Bytes the reading of one RAW pixel can take. Measured on 2026-11-05 on an Apple M1 with 8 GB RAM
+    /// (macOS 26.5.1): the production thumbnail path on a real 45.4 MP Nikon Z9 NEF peaks at 782 MB, 17 B/px,
+    /// so ImageIO of that macOS decodes RAW safely. The #182 reports — the same camera — are files whose
+    /// reading ran past an 8 GB watchdog, which floors the coefficient at 176 B/px; 192 carries the guard.
+    /// RAW is only read on the ImageIO side. Measured on one camera on one macOS, and taken for all makers.
+    public static let rawDecodeBytesPerPixel = 192
+
+    /// Bytes per pixel of a layered TIFF: every full-resolution layer decoded with its masks. A design
+    /// estimate, not measured — Photoshop's layered files of the #182 reports can't be synthesized from
+    /// nothing. Only files that say so themselves get it: `LayeredTIFF`.
+    static let layeredTIFFBytesPerPixel = 16
+
+    /// Bytes without ImageIO: stb decodes the whole picture, `oriented` copies it, and the scaling copies it
+    /// again (3 B/px decoded, a 4-byte copy, the scaled copy). Estimated, not measured on Linux or Windows.
+    static let portableDecodeBytesPerPixel = 8
+
+    /// A size for the file's note: "4.1 GB", "900 MB".
+    static func decodeSize(_ bytes: UInt64) -> String {
+        bytes >= 1 << 30 ? String(format: "%.1f GB", Double(bytes) / Double(1 << 30)) : "\(bytes >> 20) MB"
+    }
+}
+
+/// The pixels of the layers of a Photoshop-style layered TIFF, nil when the file isn't layered. A layered
+/// TIFF's master declares itself a reduced preview (`SubfileType & 1`) and keeps its full-size layers — the
+/// colours, and alpha masks the same size — in sub-IFDs. A plain TIFF, including a page chain where every
+/// IFD is full size, is not layered: ImageIO decodes those in streaming tiles, measured 2026-11-05 at a
+/// 70 MB peak on a 2,125 MP file, so it gets no preflight. Only the headers are read: 8 bytes, 12 per
+/// entry, 4 per sub-IFD — never pixels, of a file that may be 2 GB.
+enum LayeredTIFF {
+    /// A side above this is not a picture: 16,777,216 px on a side is 280 GB of pixels at the preflight's
+    /// 16 B/px, and no file of it exists. The sizes come from 4-byte LONG tags, so a header saying
+    /// 0xFFFFFFFF is garbage — and two of those multiplied overflowed the sum below, trapping *inside* the
+    /// guard built to stop crashes (issue #182, pre-push review S4). Garbage gets no estimate: nil, meaning
+    /// "not this file's shape", so nothing is marked on it.
+    static let maxDimension = 1 << 24
+
+    /// All pixels of master and layers, counted in `UInt64` for the same reason — and saturating, so a file
+    /// that lies about its size produces a huge estimate, not a trap. `maxLayers`: more sub-IFDs than this
+    /// is not Photoshop (which caps at 50) — a file pulling at the parser is left unguarded, and the
+    /// watchdog ends the helper on it.
+    static func layerPixels(in url: URL, maxLayers: Int = 64) -> UInt64? {
+        guard let file = try? FileHandle(forReadingFrom: url) else { return nil }
+        defer { try? file.close() }
+        func chunk(_ at: Int, _ count: Int) -> Data? {
+            guard at >= 0, count > 0, count <= 1 << 20 else { return nil }
+            do {
+                try file.seek(toOffset: UInt64(at))
+                guard let data = try file.read(upToCount: count), data.count == count else { return nil }
+                return data
+            } catch {
+                return nil   // past the end, a closed file: the caller reads it as "not this file's shape"
+            }
+        }
+        guard let header = chunk(0, 8), header[header.startIndex] == 0x49 || header[header.startIndex] == 0x4D
+        else { return nil }
+        let big = header[header.startIndex] == 0x4D   // "MM": Motorola (big-endian) order
+        func number(_ at: Int, _ width: Int) -> Int? {
+            guard let data = chunk(at, width) else { return nil }
+            let bytes = big ? [UInt8](data) : [UInt8](data).reversed()
+            return bytes.reduce(0) { $0 << 8 | Int($1) }
+        }
+        func ifd(_ at: Int) -> (subfileType: Int, width: Int, height: Int, subIFDs: [Int])? {
+            guard let entries = number(at, 2), entries >= 1, entries <= 1024 else { return nil }
+            var read = (0, 0, 0, [Int]())
+            for i in 0..<entries {
+                let e = at + 2 + i &* 12
+                guard let tag = number(e, 2), let type = number(e + 2, 2), let count = number(e + 4, 4),
+                      count >= 0, count <= 1 << 16 else { return nil }
+                let unit = type == 3 || type == 8 ? 2 : type == 1 || type == 2 || type == 6 || type == 7 ? 1 : 4
+                let total = count &* unit
+                func value(_ index: Int) -> Int? {
+                    if total <= 4 { return index == 0 ? number(e + 8, unit) : nil }
+                    guard let base = number(e + 8, 4) else { return nil }
+                    return number(base + index &* unit, unit)
+                }
+                switch tag {
+                case 0x00FE: read.0 = value(0) ?? 0
+                case 0x0100: read.1 = value(0) ?? 0
+                case 0x0101: read.2 = value(0) ?? 0
+                case 0x0145 where count >= 1 && count <= maxLayers:
+                    read.3 = total <= 4 && count == 1 ? [number(e + 8, 4) ?? 0] : (0..<count).compactMap { value($0) }
+                default: break
+                }
+            }
+            return read
+        }
+        guard number(2, 2) == 42, let first = number(4, 4), first >= 8,
+              let master = ifd(first), master.subfileType & 1 == 1,   // the master says "I am a preview"
+              !master.subIFDs.isEmpty, master.width > 0, master.height > 0,
+              master.width <= maxDimension, master.height <= maxDimension   // a header lying about its size
+        else { return nil }
+        var total = UInt64(master.width) &* UInt64(master.height)   // the preview decodes too
+        for at in master.subIFDs {
+            guard at >= 8, let layer = ifd(at), layer.subfileType & 1 == 0,   // a full-size layer, not a preview
+                  layer.width > 0, layer.height > 0,
+                  layer.width <= maxDimension, layer.height <= maxDimension else { continue }
+            total = total &+ UInt64(layer.width) &* UInt64(layer.height)
+        }
+        return total
+    }
 }
 
 /// Text recognition in screenshots, for keyword search only: Apple Vision on macOS, nothing elsewhere yet.

Sources/DigUpKit/Model.swift

@@ -111,3 +111,40 @@ public enum CodeQuery {
         return trimmed.dropFirst(prefix.count).trimmingCharacters(in: .whitespacesAndNewlines)
     }
 }
+
+/// The file an indexing helper had to stop on (`Watchdog` in the helper, `CrashNotes` for a crash), written beside
+/// the index for the next helper, which marks it as unreadable and goes on. A note with no path says "the helper
+/// died on the file it was reading", and the app that watched it names that file for the next helper
+/// (`DIGUP_STOPPED_FILE`, from the `file` events both apps report): the queue's head is *not* necessarily it —
+/// media runs in batches of eight and the whole batch stays pending till it ends (`Indexer`), so marking the
+/// head of an image queue would fail an innocent file and leave the poison queued (issue #182, pre-push
+/// review S3). The head-mark is the fallback for a spawn that carries no name — a helper started from a
+/// terminal, or a crash before the app saw any file.
+public struct StoppedFile: Codable {
+    /// The exit status that says "a file was stopped: start a new helper" (EX_TEMPFAIL).
+    public static let exitCode: Int32 = 75
+
+    public let path: String?
+    public let why: String
+
+    public init(path: String?, why: String) {
+        self.path = path
+        self.why = why
+    }
+
+    public static func location(in folder: URL) -> URL {
+        folder.appendingPathComponent("stopped-file.json")
+    }
+
+    public func write(to url: URL) {
+        try? JSONEncoder().encode(self).writeReplacing(url)
+    }
+
+    /// The note an earlier helper left, removed as it's read.
+    public static func take(from folder: URL) -> StoppedFile? {
+        let url = location(in: folder)
+        guard let data = try? Data(contentsOf: url) else { return nil }
+        try? FileManager.default.removeItem(at: url)
+        return try? JSONDecoder().decode(StoppedFile.self, from: data)
+    }
+}

Sources/DigUpKit/Platform/Apple/Extractors+Apple.swift

@@ -23,6 +23,40 @@ extension ImageExtractor {
         let width = properties[kCGImagePropertyPixelWidth] as? Int ?? 0
         let height = properties[kCGImagePropertyPixelHeight] as? Int ?? 0
         guard max(width, height) >= minSide else { throw ExtractError.tooSmall(width, height) }
+        // The preflight of issue #182, for the two kinds of picture that ran away in the field: RAW, and
+        // TIFFs whose master says its full-resolution layers live in sub-IFDs. The reading of both has
+        // ballooned past gigabytes on real photos (a 45 MP Nikon RAW ran past an 8 GB watchdog), and the
+        // helper died on them before the watchdog tick could note the file. Both are judged from the
+        // header, which ImageIO has already read for us: pixels, and the master's own declaration.
+        // Plain TIFFs, JPEGs, PNGs get nothing: their scaled decode streams in tiles, measured at 70 MB
+        // for a 2,125 MP TIFF and 18 MB for a 100 MP PNG (2026-11-05, Apple M1 with 8 GB RAM).
+        let budget = decodeMemoryBudget
+        if budget > 0 {
+            let uti = (CGImageSourceGetType(source) as String?).flatMap { UTType($0) }
+            // Pixels and their estimates in `UInt64`, saturating: ImageIO reports the header's own numbers,
+            // and a file lying about them (a 4-byte dimension of 0xFFFFFFFF, a negative one) must give a
+            // huge estimate, not an overflow trap in the guard built to stop traps (#182 review S4). The
+            // budget is a `UInt64` too: comparing in `Int` needed `Int(budget)`, which itself traps above
+            // `Int.max` on an absurd `--memory-limit` (review N1).
+            let pixels = UInt64(max(width, 0)) &* UInt64(max(height, 0))
+            // Every camera RAW UTI (com.nikon.raw-image and friends) conforms to public.raw-image.
+            if uti?.conforms(to: .rawImage) == true {
+                let estimate = pixels &* UInt64(rawDecodeBytesPerPixel)
+                guard estimate <= budget else {
+                    throw ExtractError.unreadable("needed an estimated \(decodeSize(estimate)) of memory to read: "
+                                                  + "a RAW picture of \(pixels / 1_000_000) MP "
+                                                  + "(the helper's limit is \(decodeSize(budget)))")
+                }
+            } else if uti?.conforms(to: .tiff) == true, let layers = LayeredTIFF.layerPixels(in: url) {
+                let estimate = (pixels &+ layers) &* UInt64(layeredTIFFBytesPerPixel)
+                guard estimate <= budget else {
+                    throw ExtractError.unreadable("needed an estimated \(decodeSize(estimate)) of memory to read: "
+                                                  + "a layered TIFF, \(pixels / 1_000_000) MP of preview and "
+                                                  + "\(layers / 1_000_000) MP of layers (the helper's limit is "
+                                                  + "\(decodeSize(budget)))")
+                }
+            }
+        }
         let options: [CFString: Any] = [
             kCGImageSourceCreateThumbnailFromImageAlways: true,
             kCGImageSourceThumbnailMaxPixelSize: maxPixel,