diff --git a/src/ingest/chunker.ts b/src/ingest/chunker.ts index 5262d45..d660d19 100644 --- a/src/ingest/chunker.ts +++ b/src/ingest/chunker.ts @@ -13,6 +13,7 @@ import { hashContent } from "../schema"; import { sessionCache } from "../cache"; import { extractSymbols, type ExtractedSymbol } from "./symbolExtractor"; +import { log } from "../logger"; // --------------------------------------------------------------------------- // Constants @@ -61,6 +62,29 @@ export interface Chunk { tokenCount: number; } +export interface ChunkResult { + chunks: Chunk[]; + binarySkipped: number; +} + +// --------------------------------------------------------------------------- +// Per-chunk binary validation (safety net) +// --------------------------------------------------------------------------- + +/** + * Check if a chunk's text content appears binary. + * Encodes text to bytes and checks for null bytes or >30% non-printable chars. + */ +function isBinaryChunk(text: string): boolean { + const bytes = new TextEncoder().encode(text); + let nonPrintable = 0; + for (let i = 0; i < bytes.length; i++) { + if (bytes[i] === 0) return true; + if (bytes[i] < 0x20 && bytes[i] !== 0x09 && bytes[i] !== 0x0a && bytes[i] !== 0x0d) nonPrintable++; + } + return bytes.length > 0 && nonPrintable / bytes.length > 0.3; +} + // --------------------------------------------------------------------------- // LineIndex: O(log n) line-at-offset lookups // --------------------------------------------------------------------------- @@ -375,57 +399,83 @@ async function fallbackChunk( * - **Small files (≤1024 tokens)**: single document chunk * - **Large files without symbols**: 512-token sliding windows, 77-token overlap */ -export async function chunkContent(input: ChunkInput): Promise { - if (!input.content || input.content.length === 0) return []; +export async function chunkContent(input: ChunkInput): Promise { + if (!input.content || input.content.length === 0) return { chunks: [], binarySkipped: 0 }; const { content, documentId, fileId, filePath, language, totalLines } = input; const totalTokenCount = estimateTokens(content); + let chunks: Chunk[]; + // Markdown: heading-based chunking if (language === "markdown" || language === "text") { const headingChunks = chunkByHeadings(content, documentId, fileId); - if (headingChunks.length > 1) return headingChunks; - // Single heading = whole doc, fall through to document/window logic + if (headingChunks.length > 1) { + chunks = headingChunks; + } else { + // Single heading = whole doc, fall through to document/window logic + chunks = []; + } + } else { + chunks = []; } - // Small files: single document chunk - if (totalTokenCount <= MAX_TOKENS_PER_CHUNK) { - const h = hashContentCached(content); - return [ - { - id: crypto.randomUUID(), - documentId, - fileId, - chunkIndex: 0, - content, - contentHash: h, - symbol: null, - kind: null, - startLine: 1, - endLine: totalLines, - chunkType: "document", - tokenCount: totalTokenCount, - }, - ]; + if (chunks.length === 0) { + // Small files: single document chunk + if (totalTokenCount <= MAX_TOKENS_PER_CHUNK) { + const h = hashContentCached(content); + chunks = [ + { + id: crypto.randomUUID(), + documentId, + fileId, + chunkIndex: 0, + content, + contentHash: h, + symbol: null, + kind: null, + startLine: 1, + endLine: totalLines, + chunkType: "document", + tokenCount: totalTokenCount, + }, + ]; + } else { + // Code files: try symbol extraction + if ( + language === "typescript" || + language === "javascript" || + language === "php" || + language === "rust" + ) { + try { + const symbols = await extractSymbols(content, filePath); + if (symbols.length > 0) { + chunks = await chunkBySymbols(content, symbols, documentId, fileId, totalTokenCount); + } else { + chunks = await fallbackChunk(content, documentId, fileId, totalTokenCount); + } + } catch { + chunks = await fallbackChunk(content, documentId, fileId, totalTokenCount); + } + } else { + // Large files without symbols: sliding windows + chunks = await fallbackChunk(content, documentId, fileId, totalTokenCount); + } + } } - // Code files: try symbol extraction - if ( - language === "typescript" || - language === "javascript" || - language === "php" || - language === "rust" - ) { - try { - const symbols = await extractSymbols(content, filePath); - if (symbols.length > 0) { - return await chunkBySymbols(content, symbols, documentId, fileId, totalTokenCount); - } - } catch { - // Fall through to window chunking + // ── Filter out binary chunks (safety net) ──────────────────────────── + const filtered: Chunk[] = []; + let binarySkipped = 0; + for (const chunk of chunks) { + if (isBinaryChunk(chunk.content)) { + log("warn", "chunker", `Binary chunk skipped for ${filePath} (chunk ${chunk.chunkIndex}, type ${chunk.chunkType})`); + binarySkipped++; + } else { + filtered.push(chunk); } } - // Large files without symbols: sliding windows - return await fallbackChunk(content, documentId, fileId, totalTokenCount); + return { chunks: filtered, binarySkipped }; } diff --git a/src/ingest/index.ts b/src/ingest/index.ts index c4300dd..290d1a5 100644 --- a/src/ingest/index.ts +++ b/src/ingest/index.ts @@ -22,7 +22,7 @@ import { generateId, hashBuffer, hashContent, checkpointDatabase } from "../sche import { log } from "../logger"; import { ingestMutex } from "./mutex"; import { resolveFiles, isBinaryContent, type WalkResult, type WalkedFile } from "./loader"; -import { chunkContent, type Chunk } from "./chunker"; +import { chunkContent, type ChunkResult, type Chunk } from "./chunker"; import { extractSymbols } from "./symbolExtractor"; import { embedChunks } from "./embed"; import { loadVec0 } from "../embed/extensionLoader"; @@ -40,6 +40,7 @@ export interface IngestResult { chunksCreated: number; chunksEmbedded: number; // chunks successfully embedded into vec0 documentsCreated: number; + binarySkipped: number; // files/chunks skipped due to binary content detection errors: string[]; durationMs: number; } @@ -105,6 +106,7 @@ export async function ingestPath( chunksCreated: 0, chunksEmbedded: 0, documentsCreated: 0, + binarySkipped: 0, errors: [], durationMs: 0, }; @@ -207,6 +209,7 @@ export async function ingestPath( // Binary content guard: skip files with null bytes (safety net for misnamed binaries) if (isBinaryContent(new Uint8Array(buf))) { log("debug", "ingest", `Skipped binary content: ${filePath}`); + result.binarySkipped++; return; } @@ -296,8 +299,9 @@ export async function ingestPath( const totalLines = content.split("\n").length; let chunks: Chunk[]; + let chunkResult: ChunkResult; try { - chunks = await chunkContent({ + chunkResult = await chunkContent({ documentId: docId, fileId, content, @@ -305,6 +309,8 @@ export async function ingestPath( language, totalLines, }); + chunks = chunkResult.chunks; + result.binarySkipped += chunkResult.binarySkipped; } catch (err) { db.exec(`ROLLBACK TO SAVEPOINT ${spFile}`); result.errors.push(`Failed to chunk ${filePath}: ${String(err)}`); diff --git a/src/ingest/loader.ts b/src/ingest/loader.ts index f7e9232..5f7b9a4 100644 --- a/src/ingest/loader.ts +++ b/src/ingest/loader.ts @@ -174,7 +174,8 @@ export function isBinaryExtension(filePath: string): boolean { return BINARY_EXTENSIONS.has(ext); } -const BINARY_SCAN_BYTES = 256; +const FULL_SCAN_SIZE = 64 * 1024; // 64 KB — files ≤ this get fully scanned +const SAMPLE_SIZE = 4 * 1024; // 4 KB per sample region const BINARY_RATIO_THRESHOLD = 0.3; function isPrintable(b: number): boolean { @@ -184,15 +185,52 @@ function isPrintable(b: number): boolean { return false; } -/** Returns true if the buffer appears binary (null bytes or >30% non-printable chars). */ -export function isBinaryContent(buf: Uint8Array): boolean { - const limit = Math.min(BINARY_SCAN_BYTES, buf.length); +/** Check a specific byte range for binary content. */ +function isBinaryRegion(buf: Uint8Array, start: number, end: number): boolean { let nonPrintable = 0; - for (let i = 0; i < limit; i++) { + const len = end - start; + for (let i = start; i < end; i++) { if (buf[i] === 0) return true; // null byte → binary if (!isPrintable(buf[i])) nonPrintable++; } - return nonPrintable / limit > BINARY_RATIO_THRESHOLD; + return len > 0 && nonPrintable / len > BINARY_RATIO_THRESHOLD; +} + +/** + * Returns true if the buffer appears binary (null bytes or >30% non-printable chars). + * + * Strategy: + * - Files ≤ 64KB: scans the **entire file content** + * - Files > 64KB: samples up to 3 regions — first 4KB, middle 4KB, last 4KB + * - Null byte in any region → immediate binary + */ +export function isBinaryContent(buf: Uint8Array): boolean { + if (buf.length === 0) return false; // empty file is not binary + + if (buf.length <= FULL_SCAN_SIZE) { + // Small file: scan entire content + return isBinaryRegion(buf, 0, buf.length); + } + + // Large file: sample 3 regions + const half = Math.floor(buf.length / 2); + const halfSample = Math.floor(SAMPLE_SIZE / 2); + + const regions = [ + { start: 0, end: SAMPLE_SIZE }, // first 4KB + { start: half - halfSample, end: half + Math.ceil(SAMPLE_SIZE / 2) }, // middle 4KB + { start: buf.length - SAMPLE_SIZE, end: buf.length }, // last 4KB + ]; + + for (const region of regions) { + const clampedStart = Math.max(0, Math.min(region.start, buf.length - 1)); + const clampedEnd = Math.max(clampedStart, Math.min(region.end, buf.length)); + if (isBinaryRegion(buf, clampedStart, clampedEnd)) { + return true; + } + } + + return false; } // --------------------------------------------------------------------------- diff --git a/test/ingest.test.ts b/test/ingest.test.ts index 2e85811..dac4221 100644 --- a/test/ingest.test.ts +++ b/test/ingest.test.ts @@ -8,6 +8,7 @@ import { existsSync, mkdirSync, rmSync, writeFileSync } from "fs"; import { join } from "path"; import { openDatabase, createSchema } from "../src/schema"; import { ingestPath } from "../src/ingest"; +import { isBinaryContent } from "../src/ingest/loader"; import { sessionCache } from "../src/cache"; import { brainSearch } from "../src/search/unified"; @@ -756,3 +757,128 @@ describe("ingestPath — edge cases (E5.3)", () => { expect(integrity.integrity_check).toBe("ok"); }); }); + +// --------------------------------------------------------------------------- +// E5.4: Binary content detection — isBinaryContent() +// --------------------------------------------------------------------------- + +describe("isBinaryContent", () => { + function toBytes(text: string): Uint8Array { + return new TextEncoder().encode(text); + } + + test("plain text file (should NOT be binary)", () => { + const text = "Hello, world!\nThis is a normal text file.\nWith multiple lines.\n"; + expect(isBinaryContent(toBytes(text))).toBe(false); + }); + + test("file with null byte at position 300 (should be binary)", () => { + // 300 printable chars + null byte + const buf = new Uint8Array(301); + for (let i = 0; i < 300; i++) { + buf[i] = 0x41; // 'A' + } + buf[300] = 0; // null byte + expect(isBinaryContent(buf)).toBe(true); + }); + + test("large file (>64KB) with binary region in the middle", () => { + // Create a ~70KB buffer with null bytes in the middle region + const size = 70 * 1024; // ~70KB + const buf = new Uint8Array(size); + // Fill with printable chars (space) + for (let i = 0; i < size; i++) { + buf[i] = 0x20; // space + } + // Insert null bytes in the middle region (~35KB) + const midStart = Math.floor(size / 2) - 100; + for (let i = midStart; i < midStart + 200; i++) { + buf[i] = 0; // null bytes + } + expect(isBinaryContent(buf)).toBe(true); + }); + + test("large file (>64KB) with binary region at the end", () => { + const size = 70 * 1024; + const buf = new Uint8Array(size); + for (let i = 0; i < size; i++) { + buf[i] = 0x20; // space + } + // High concentration of non-printable chars in the last 4KB region (>30%) + const lastStart = size - 2000; + for (let i = lastStart; i < size; i++) { + buf[i] = 0x01; // non-printable control char + } + // 2000/4096 ≈ 49% non-printable in last sample region → exceeds 30% threshold + expect(isBinaryContent(buf)).toBe(true); + }); + + test("file with >30% non-printable chars throughout", () => { + const buf = new Uint8Array(1000); + // 40% non-printable, 60% printable + for (let i = 0; i < 1000; i++) { + buf[i] = i < 400 ? 0x01 : 0x41; // 400 non-printable, 600 'A' + } + expect(isBinaryContent(buf)).toBe(true); + }); + + test("zero-byte file (edge case — empty = not binary)", () => { + expect(isBinaryContent(new Uint8Array(0))).toBe(false); + }); + + test("UTF-8 text with BOM (should NOT be binary)", () => { + // UTF-8 BOM: 0xEF 0xBB 0xBF + const buf = new Uint8Array([0xEF, 0xBB, 0xBF, 0x48, 0x65, 0x6C, 0x6C, 0x6F]); // BOM + "Hello" + expect(isBinaryContent(buf)).toBe(false); + }); + + test("file with printable content plus control chars (tab, CR, LF — should NOT be binary)", () => { + const text = "Line 1\tindented\r\nLine 2\nLine 3\n"; + expect(isBinaryContent(toBytes(text))).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// E5.5: Per-chunk binary validation +// --------------------------------------------------------------------------- + +describe("isBinaryChunk (per-chunk validation)", () => { + // Re-implement the private function from chunker.ts for testing + function isBinaryChunk(text: string): boolean { + const bytes = new TextEncoder().encode(text); + let nonPrintable = 0; + for (let i = 0; i < bytes.length; i++) { + if (bytes[i] === 0) return true; + if (bytes[i] < 0x20 && bytes[i] !== 0x09 && bytes[i] !== 0x0a && bytes[i] !== 0x0d) nonPrintable++; + } + return bytes.length > 0 && nonPrintable / bytes.length > 0.3; + } + + test("normal text chunk (should pass)", () => { + expect(isBinaryChunk("This is a normal chunk of text with some code: fn() => {}")).toBe(false); + }); + + test("chunk with embedded null byte (should be flagged)", () => { + const text = "normal text\u0000with null byte"; + expect(isBinaryChunk(text)).toBe(true); + }); + + test("chunk with >30% non-printable chars (should be flagged)", () => { + // Create a string with >30% non-printable chars + let text = ""; + for (let i = 0; i < 10; i++) { + text += "\x01"; // non-printable + text += "A"; // printable + } + // 50% non-printable → should be flagged + expect(isBinaryChunk(text)).toBe(true); + }); + + test("empty chunk (edge case — should NOT be flagged)", () => { + expect(isBinaryChunk("")).toBe(false); + }); + + test("chunk with only whitespace (should NOT be flagged)", () => { + expect(isBinaryChunk(" \t\n \n ")).toBe(false); + }); +});