fix: statistical stride sampling for binary detection #181

src/ingest/chunker.ts

@@ -73,16 +73,24 @@ export interface ChunkResult {
 
 /**
  * Check if a chunk's text content appears binary.
- * Encodes text to bytes and checks for null bytes or >30% non-printable chars.
+ *
+ * Uses U+FFFD replacement character density as the signal — TextDecoder
+ * produces U+FFFD for invalid byte sequences, so real text produces zero
+ * U+FFFD while binary source content produces many.
  */
-function isBinaryChunk(text: string): boolean {
-  const bytes = new TextEncoder().encode(text);
-  let nonPrintable = 0;
-  for (let i = 0; i < bytes.length; i++) {
-    if (bytes[i] === 0) return true;
-    if (bytes[i] < 0x20 && bytes[i] !== 0x09 && bytes[i] !== 0x0a && bytes[i] !== 0x0d) nonPrintable++;
+export function isBinaryChunk(text: string): boolean {
+  if (text.length === 0) return false;
+
+  // Count U+FFFD replacement characters — strong signal of binary source
+  let replacementCount = 0;
+  for (let i = 0; i < text.length; i++) {
+    if (text.charCodeAt(i) === 0xFFFD) {
+      replacementCount++;
+    }
   }
-  return bytes.length > 0 && nonPrintable / bytes.length > 0.3;
+
+  // If >10% of characters are replacement chars, the source was likely binary
+  return replacementCount / text.length > 0.1;
 }
 
 // ---------------------------------------------------------------------------

src/ingest/loader.ts

@@ -175,7 +175,6 @@ export function isBinaryExtension(filePath: string): boolean {
 }
 
 const FULL_SCAN_SIZE = 64 * 1024; // 64 KB — files ≤ this get fully scanned
-const SAMPLE_SIZE = 4 * 1024;      // 4 KB per sample region
 const BINARY_RATIO_THRESHOLD = 0.3;
 
 function isPrintable(b: number): boolean {
@@ -201,34 +200,36 @@ function isBinaryRegion(buf: Uint8Array, start: number, end: number): boolean {
  *
  * Strategy:
  * - Files ≤ 64KB: scans the **entire file content**
- * - Files  > 64KB: samples up to 3 regions — first 4KB, middle 4KB, last 4KB
+ * - Files  > 64KB: first 8KB thorough scan + statistical stride sampling + last 8KB thorough scan
  * - Null byte in any region → immediate binary
  */
 export function isBinaryContent(buf: Uint8Array): boolean {
-  if (buf.length === 0) return false; // empty file is not binary
+  if (buf.length === 0) return false;
 
   if (buf.length <= FULL_SCAN_SIZE) {
-    // Small file: scan entire content
+    // Small file: scan entire content (existing behavior)
     return isBinaryRegion(buf, 0, buf.length);
   }
 
-  // Large file: sample 3 regions
-  const half = Math.floor(buf.length / 2);
-  const halfSample = Math.floor(SAMPLE_SIZE / 2);
-
-  const regions = [
-    { start: 0, end: SAMPLE_SIZE },                                       // first 4KB
-    { start: half - halfSample, end: half + Math.ceil(SAMPLE_SIZE / 2) }, // middle 4KB
-    { start: buf.length - SAMPLE_SIZE, end: buf.length },                 // last 4KB
-  ];
-
-  for (const region of regions) {
-    const clampedStart = Math.max(0, Math.min(region.start, buf.length - 1));
-    const clampedEnd = Math.max(clampedStart, Math.min(region.end, buf.length));
-    if (isBinaryRegion(buf, clampedStart, clampedEnd)) {
-      return true;
-    }
+  // Large file (>64KB): first 8KB thorough scan + stride sampling + last 8KB
+
+  // 1. Thorough scan of first 8KB (catches ELF/PE/Mach-O headers)
+  const firstEnd = Math.min(8192, buf.length);
+  if (isBinaryRegion(buf, 0, firstEnd)) return true;
+
+  // 2. Statistical stride sampling: check every 512th byte across the ENTIRE file
+  let nonPrintable = 0;
+  let sampled = 0;
+  for (let i = 0; i < buf.length; i += 512) {
+    if (buf[i] === 0) return true; // null byte → immediate binary
+    if (!isPrintable(buf[i])) nonPrintable++;
+    sampled++;
   }
+  if (nonPrintable / sampled > BINARY_RATIO_THRESHOLD) return true;
+
+  // 3. Thorough scan of last 8KB (catches trailers, symbol tables)
+  const lastStart = Math.max(0, buf.length - 8192);
+  if (isBinaryRegion(buf, lastStart, buf.length)) return true;
 
   return false;
 }

test/ingest.test.ts

@@ -9,6 +9,7 @@ import { join } from "path";
 import { openDatabase, createSchema } from "../src/schema";
 import { ingestPath } from "../src/ingest";
 import { isBinaryContent } from "../src/ingest/loader";
+import { isBinaryChunk } from "../src/ingest/chunker";
 import { sessionCache } from "../src/cache";
 import { brainSearch } from "../src/search/unified";
 
@@ -798,21 +799,6 @@ describe("isBinaryContent", () => {
     expect(isBinaryContent(buf)).toBe(true);
   });
 
-  test("large file (>64KB) with binary region at the end", () => {
-    const size = 70 * 1024;
-    const buf = new Uint8Array(size);
-    for (let i = 0; i < size; i++) {
-      buf[i] = 0x20; // space
-    }
-    // High concentration of non-printable chars in the last 4KB region (>30%)
-    const lastStart = size - 2000;
-    for (let i = lastStart; i < size; i++) {
-      buf[i] = 0x01; // non-printable control char
-    }
-    // 2000/4096 ≈ 49% non-printable in last sample region → exceeds 30% threshold
-    expect(isBinaryContent(buf)).toBe(true);
-  });
-
   test("file with >30% non-printable chars throughout", () => {
     const buf = new Uint8Array(1000);
     // 40% non-printable, 60% printable
@@ -822,6 +808,45 @@ describe("isBinaryContent", () => {
     expect(isBinaryContent(buf)).toBe(true);
   });
 
+  test("large file (>64KB) with sparse null bytes caught by stride sampling", () => {
+    const size = 100 * 1024; // 100KB
+    const buf = new Uint8Array(size);
+    buf.fill(0x20); // printable spaces
+    // Insert null bytes at every 1024th position (stride-aligned: 1024 = 2×512)
+    for (let i = 1024; i < size; i += 1024) {
+      buf[i] = 0;
+    }
+    // Stride sampling at position 1024 catches the first null byte
+    expect(isBinaryContent(buf)).toBe(true);
+  });
+
+  test("large file (>64KB) with binary only in first 8KB region", () => {
+    const size = 70 * 1024;
+    const buf = new Uint8Array(size);
+    buf.fill(0x20); // printable spaces
+    // Insert null byte in the first 8KB region
+    buf[100] = 0;
+    // First 8KB thorough scan catches it
+    expect(isBinaryContent(buf)).toBe(true);
+  });
+
+  test("large file (>64KB) that is all printable text (should NOT be flagged)", () => {
+    const size = 70 * 1024;
+    const buf = new Uint8Array(size);
+    buf.fill(0x20); // space is printable
+    expect(isBinaryContent(buf)).toBe(false);
+  });
+
+  test("large file (>64KB) with null byte at stride position beyond first 8KB", () => {
+    const size = 200 * 1024; // 200KB
+    const buf = new Uint8Array(size);
+    buf.fill(0x41); // printable 'A'
+    // Place null byte at stride position 129×512 = 66048
+    // This is beyond the first 8KB scan (0–8191) and before the last 8KB (196608–204799)
+    buf[66048] = 0;
+    expect(isBinaryContent(buf)).toBe(true);
+  });
+
   test("zero-byte file (edge case — empty = not binary)", () => {
     expect(isBinaryContent(new Uint8Array(0))).toBe(false);
   });
@@ -843,42 +868,27 @@ describe("isBinaryContent", () => {
 // ---------------------------------------------------------------------------
 
 describe("isBinaryChunk (per-chunk validation)", () => {
-  // Re-implement the private function from chunker.ts for testing
-  function isBinaryChunk(text: string): boolean {
-    const bytes = new TextEncoder().encode(text);
-    let nonPrintable = 0;
-    for (let i = 0; i < bytes.length; i++) {
-      if (bytes[i] === 0) return true;
-      if (bytes[i] < 0x20 && bytes[i] !== 0x09 && bytes[i] !== 0x0a && bytes[i] !== 0x0d) nonPrintable++;
-    }
-    return bytes.length > 0 && nonPrintable / bytes.length > 0.3;
-  }
-
-  test("normal text chunk (should pass)", () => {
+  test("normal text chunk (should NOT be flagged)", () => {
     expect(isBinaryChunk("This is a normal chunk of text with some code: fn() => {}")).toBe(false);
   });
 
-  test("chunk with embedded null byte (should be flagged)", () => {
-    const text = "normal text\u0000with null byte";
+  test("text with many U+FFFD replacement chars (>10% — should be flagged)", () => {
+    // 50 replacement chars out of 150 total = 33% → >10% threshold → flagged
+    const text = "\uFFFD".repeat(50) + "A".repeat(100);
     expect(isBinaryChunk(text)).toBe(true);
   });
 
-  test("chunk with >30% non-printable chars (should be flagged)", () => {
-    // Create a string with >30% non-printable chars
-    let text = "";
-    for (let i = 0; i < 10; i++) {
-      text += "\x01"; // non-printable
-      text += "A";    // printable
-    }
-    // 50% non-printable → should be flagged
-    expect(isBinaryChunk(text)).toBe(true);
+  test("text with a few U+FFFD chars (<10% — should NOT be flagged)", () => {
+    // 3 replacement chars out of 103 total ≈ 2.9% → <10% threshold → not flagged
+    const text = "\uFFFD".repeat(3) + "A".repeat(100);
+    expect(isBinaryChunk(text)).toBe(false);
   });
 
   test("empty chunk (edge case — should NOT be flagged)", () => {
     expect(isBinaryChunk("")).toBe(false);
   });
 
-  test("chunk with only whitespace (should NOT be flagged)", () => {
-    expect(isBinaryChunk("   \t\n  \n  ")).toBe(false);
+  test("real text with no replacement chars (should NOT be flagged)", () => {
+    expect(isBinaryChunk("const x = 42;\nexport default x;\n")).toBe(false);
   });
 });