feat: enhanced binary detection with full-content scanning and per-chunk validation #178

src/ingest/chunker.ts

@@ -13,6 +13,7 @@
 import { hashContent } from "../schema";
 import { sessionCache } from "../cache";
 import { extractSymbols, type ExtractedSymbol } from "./symbolExtractor";
+import { log } from "../logger";
 
 // ---------------------------------------------------------------------------
 // Constants
@@ -61,6 +62,29 @@ export interface Chunk {
   tokenCount: number;
 }
 
+export interface ChunkResult {
+  chunks: Chunk[];
+  binarySkipped: number;
+}
+
+// ---------------------------------------------------------------------------
+// Per-chunk binary validation (safety net)
+// ---------------------------------------------------------------------------
+
+/**
+ * Check if a chunk's text content appears binary.
+ * Encodes text to bytes and checks for null bytes or >30% non-printable chars.
+ */
+function isBinaryChunk(text: string): boolean {
+  const bytes = new TextEncoder().encode(text);
+  let nonPrintable = 0;
+  for (let i = 0; i < bytes.length; i++) {
+    if (bytes[i] === 0) return true;
+    if (bytes[i] < 0x20 && bytes[i] !== 0x09 && bytes[i] !== 0x0a && bytes[i] !== 0x0d) nonPrintable++;
+  }
+  return bytes.length > 0 && nonPrintable / bytes.length > 0.3;
+}
+
 // ---------------------------------------------------------------------------
 // LineIndex: O(log n) line-at-offset lookups
 // ---------------------------------------------------------------------------
@@ -375,57 +399,83 @@ async function fallbackChunk(
  * - **Small files (≤1024 tokens)**: single document chunk
  * - **Large files without symbols**: 512-token sliding windows, 77-token overlap
  */
-export async function chunkContent(input: ChunkInput): Promise<Chunk[]> {
-  if (!input.content || input.content.length === 0) return [];
+export async function chunkContent(input: ChunkInput): Promise<ChunkResult> {
+  if (!input.content || input.content.length === 0) return { chunks: [], binarySkipped: 0 };
 
   const { content, documentId, fileId, filePath, language, totalLines } = input;
   const totalTokenCount = estimateTokens(content);
 
+  let chunks: Chunk[];
+
   // Markdown: heading-based chunking
   if (language === "markdown" || language === "text") {
     const headingChunks = chunkByHeadings(content, documentId, fileId);
-    if (headingChunks.length > 1) return headingChunks;
-    // Single heading = whole doc, fall through to document/window logic
+    if (headingChunks.length > 1) {
+      chunks = headingChunks;
+    } else {
+      // Single heading = whole doc, fall through to document/window logic
+      chunks = [];
+    }
+  } else {
+    chunks = [];
   }
 
-  // Small files: single document chunk
-  if (totalTokenCount <= MAX_TOKENS_PER_CHUNK) {
-    const h = hashContentCached(content);
-    return [
-      {
-        id: crypto.randomUUID(),
-        documentId,
-        fileId,
-        chunkIndex: 0,
-        content,
-        contentHash: h,
-        symbol: null,
-        kind: null,
-        startLine: 1,
-        endLine: totalLines,
-        chunkType: "document",
-        tokenCount: totalTokenCount,
-      },
-    ];
+  if (chunks.length === 0) {
+    // Small files: single document chunk
+    if (totalTokenCount <= MAX_TOKENS_PER_CHUNK) {
+      const h = hashContentCached(content);
+      chunks = [
+        {
+          id: crypto.randomUUID(),
+          documentId,
+          fileId,
+          chunkIndex: 0,
+          content,
+          contentHash: h,
+          symbol: null,
+          kind: null,
+          startLine: 1,
+          endLine: totalLines,
+          chunkType: "document",
+          tokenCount: totalTokenCount,
+        },
+      ];
+    } else {
+      // Code files: try symbol extraction
+      if (
+        language === "typescript" ||
+        language === "javascript" ||
+        language === "php" ||
+        language === "rust"
+      ) {
+        try {
+          const symbols = await extractSymbols(content, filePath);
+          if (symbols.length > 0) {
+            chunks = await chunkBySymbols(content, symbols, documentId, fileId, totalTokenCount);
+          } else {
+            chunks = await fallbackChunk(content, documentId, fileId, totalTokenCount);
+          }
+        } catch {
+          chunks = await fallbackChunk(content, documentId, fileId, totalTokenCount);
+        }
+      } else {
+        // Large files without symbols: sliding windows
+        chunks = await fallbackChunk(content, documentId, fileId, totalTokenCount);
+      }
+    }
   }
 
-  // Code files: try symbol extraction
-  if (
-    language === "typescript" ||
-    language === "javascript" ||
-    language === "php" ||
-    language === "rust"
-  ) {
-    try {
-      const symbols = await extractSymbols(content, filePath);
-      if (symbols.length > 0) {
-        return await chunkBySymbols(content, symbols, documentId, fileId, totalTokenCount);
-      }
-    } catch {
-      // Fall through to window chunking
+  // ── Filter out binary chunks (safety net) ────────────────────────────
+  const filtered: Chunk[] = [];
+  let binarySkipped = 0;
+  for (const chunk of chunks) {
+    if (isBinaryChunk(chunk.content)) {
+      log("warn", "chunker", `Binary chunk skipped for ${filePath} (chunk ${chunk.chunkIndex}, type ${chunk.chunkType})`);
+      binarySkipped++;
+    } else {
+      filtered.push(chunk);
     }
   }
 
-  // Large files without symbols: sliding windows
-  return await fallbackChunk(content, documentId, fileId, totalTokenCount);
+  return { chunks: filtered, binarySkipped };
 }

src/ingest/index.ts

@@ -22,7 +22,7 @@ import { generateId, hashBuffer, hashContent, checkpointDatabase } from "../sche
 import { log } from "../logger";
 import { ingestMutex } from "./mutex";
 import { resolveFiles, isBinaryContent, type WalkResult, type WalkedFile } from "./loader";
-import { chunkContent, type Chunk } from "./chunker";
+import { chunkContent, type ChunkResult, type Chunk } from "./chunker";
 import { extractSymbols } from "./symbolExtractor";
 import { embedChunks } from "./embed";
 import { loadVec0 } from "../embed/extensionLoader";
@@ -40,6 +40,7 @@ export interface IngestResult {
   chunksCreated: number;
   chunksEmbedded: number; // chunks successfully embedded into vec0
   documentsCreated: number;
+  binarySkipped: number; // files/chunks skipped due to binary content detection
   errors: string[];
   durationMs: number;
 }
@@ -105,6 +106,7 @@ export async function ingestPath(
       chunksCreated: 0,
       chunksEmbedded: 0,
       documentsCreated: 0,
+      binarySkipped: 0,
       errors: [],
       durationMs: 0,
     };
@@ -207,6 +209,7 @@ export async function ingestPath(
         // Binary content guard: skip files with null bytes (safety net for misnamed binaries)
         if (isBinaryContent(new Uint8Array(buf))) {
           log("debug", "ingest", `Skipped binary content: ${filePath}`);
+          result.binarySkipped++;
           return;
         }
 
@@ -296,15 +299,18 @@ export async function ingestPath(
         const totalLines = content.split("\n").length;
 
         let chunks: Chunk[];
+        let chunkResult: ChunkResult;
         try {
-          chunks = await chunkContent({
+          chunkResult = await chunkContent({
             documentId: docId,
             fileId,
             content,
             filePath,
             language,
             totalLines,
           });
+          chunks = chunkResult.chunks;
+          result.binarySkipped += chunkResult.binarySkipped;
         } catch (err) {
           db.exec(`ROLLBACK TO SAVEPOINT ${spFile}`);
           result.errors.push(`Failed to chunk ${filePath}: ${String(err)}`);

src/ingest/loader.ts

@@ -174,7 +174,8 @@ export function isBinaryExtension(filePath: string): boolean {
   return BINARY_EXTENSIONS.has(ext);
 }
 
-const BINARY_SCAN_BYTES = 256;
+const FULL_SCAN_SIZE = 64 * 1024; // 64 KB — files ≤ this get fully scanned
+const SAMPLE_SIZE = 4 * 1024;      // 4 KB per sample region
 const BINARY_RATIO_THRESHOLD = 0.3;
 
 function isPrintable(b: number): boolean {
@@ -184,15 +185,52 @@ function isPrintable(b: number): boolean {
   return false;
 }
 
-/** Returns true if the buffer appears binary (null bytes or >30% non-printable chars). */
-export function isBinaryContent(buf: Uint8Array): boolean {
-  const limit = Math.min(BINARY_SCAN_BYTES, buf.length);
+/** Check a specific byte range for binary content. */
+function isBinaryRegion(buf: Uint8Array, start: number, end: number): boolean {
   let nonPrintable = 0;
-  for (let i = 0; i < limit; i++) {
+  const len = end - start;
+  for (let i = start; i < end; i++) {
     if (buf[i] === 0) return true; // null byte → binary
     if (!isPrintable(buf[i])) nonPrintable++;
   }
-  return nonPrintable / limit > BINARY_RATIO_THRESHOLD;
+  return len > 0 && nonPrintable / len > BINARY_RATIO_THRESHOLD;
+}
+
+/**
+ * Returns true if the buffer appears binary (null bytes or >30% non-printable chars).
+ *
+ * Strategy:
+ * - Files ≤ 64KB: scans the **entire file content**
+ * - Files  > 64KB: samples up to 3 regions — first 4KB, middle 4KB, last 4KB
+ * - Null byte in any region → immediate binary
+ */
+export function isBinaryContent(buf: Uint8Array): boolean {
+  if (buf.length === 0) return false; // empty file is not binary
+
+  if (buf.length <= FULL_SCAN_SIZE) {
+    // Small file: scan entire content
+    return isBinaryRegion(buf, 0, buf.length);
+  }
+
+  // Large file: sample 3 regions
+  const half = Math.floor(buf.length / 2);
+  const halfSample = Math.floor(SAMPLE_SIZE / 2);
+
+  const regions = [
+    { start: 0, end: SAMPLE_SIZE },                                       // first 4KB
+    { start: half - halfSample, end: half + Math.ceil(SAMPLE_SIZE / 2) }, // middle 4KB
+    { start: buf.length - SAMPLE_SIZE, end: buf.length },                 // last 4KB
+  ];
+
+  for (const region of regions) {
+    const clampedStart = Math.max(0, Math.min(region.start, buf.length - 1));
+    const clampedEnd = Math.max(clampedStart, Math.min(region.end, buf.length));
+    if (isBinaryRegion(buf, clampedStart, clampedEnd)) {
+      return true;
+    }
+  }
+
+  return false;
 }
 
 // ---------------------------------------------------------------------------