From e16f820f4a27482cc5adfa0dd7fec9e40c43c8f8 Mon Sep 17 00:00:00 2001
From: 4 Bytes Robby <robby@4bytes.de>
Date: Tue, 23 Jun 2026 08:44:48 +0200
Subject: [PATCH] fix: also detect NUL bytes in isBinaryChunk per-chunk
 validator

---
 src/ingest/chunker.ts | 6 +++---
 test/ingest.test.ts   | 9 +++++++++
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/ingest/chunker.ts b/src/ingest/chunker.ts
index cd8954b..3d4aa1f 100644
--- a/src/ingest/chunker.ts
+++ b/src/ingest/chunker.ts
@@ -88,9 +88,9 @@ export function isBinaryChunk(text: string): boolean {
   // Count U+FFFD replacement characters — strong signal of binary source
   let replacementCount = 0;
   for (let i = 0; i < text.length; i++) {
-    if (text.charCodeAt(i) === 0xFFFD) {
-      replacementCount++;
-    }
+    const c = text.charCodeAt(i);
+    if (c === 0) return true; // NUL byte → immediate binary signal
+    if (c === 0xFFFD) replacementCount++;
   }
 
   // If >10% of characters are replacement chars, the source was likely binary
diff --git a/test/ingest.test.ts b/test/ingest.test.ts
index e9b8640..602cb36 100644
--- a/test/ingest.test.ts
+++ b/test/ingest.test.ts
@@ -914,4 +914,13 @@ describe("isBinaryChunk (per-chunk validation)", () => {
   test("real text with no replacement chars (should NOT be flagged)", () => {
     expect(isBinaryChunk("const x = 42;\nexport default x;\n")).toBe(false);
   });
+
+  test("chunk with embedded null byte (should be flagged)", () => {
+    const text = "normal text\u0000with null byte";
+    expect(isBinaryChunk(text)).toBe(true);
+  });
+
+  test("chunk with null byte at start", () => {
+    expect(isBinaryChunk("\u0000hello")).toBe(true);
+  });
 });