From a3ea4fbded66c215fc210721e08b6d77a83ce4c6 Mon Sep 17 00:00:00 2001
From: 4 Bytes Robby <robby@4bytes.de>
Date: Tue, 23 Jun 2026 07:47:45 +0200
Subject: [PATCH] fix: skip tree-sitter symbol extraction for files >200KB to
 prevent event-loop block

---
 src/ingest/chunker.ts | 25 ++++++++++++++++++-------
 test/ingest.test.ts   | 23 +++++++++++++++++++++++
 2 files changed, 41 insertions(+), 7 deletions(-)

diff --git a/src/ingest/chunker.ts b/src/ingest/chunker.ts
index 4c8f81e..ec281af 100644
--- a/src/ingest/chunker.ts
+++ b/src/ingest/chunker.ts
@@ -24,6 +24,11 @@ const MAX_TOKENS_PER_CHUNK = 1024;
 const TOKEN_WINDOW = 512;
 const TOKEN_OVERLAP = 77; // ~15 %
 
+/** Maximum file size for tree-sitter symbol extraction (200KB).
+ *  Files larger than this skip symbol extraction entirely and go straight
+ *  to windowed chunking, preventing event-loop blocks from sync WASM parse. */
+const SYMBOL_EXTRACTION_MAX_CHARS = 200 * 1024; // 200 KB
+
 // ---------------------------------------------------------------------------
 // Token estimation
 // ---------------------------------------------------------------------------
@@ -456,15 +461,21 @@ export async function chunkContent(input: ChunkInput): Promise<ChunkResult> {
         language === "php" ||
         language === "rust"
       ) {
-        try {
-          const symbols = await extractSymbols(content, filePath);
-          if (symbols.length > 0) {
-            chunks = await chunkBySymbols(content, symbols, documentId, fileId, totalTokenCount);
-          } else {
+        // Skip tree-sitter for large files — sync WASM parse blocks the event loop
+        if (content.length > SYMBOL_EXTRACTION_MAX_CHARS) {
+          log("info", "chunker", `Skipping symbol extraction for large file (${(content.length / 1024).toFixed(0)}KB): ${filePath}`);
+          chunks = await fallbackChunk(content, documentId, fileId, totalTokenCount);
+        } else {
+          try {
+            const symbols = await extractSymbols(content, filePath);
+            if (symbols.length > 0) {
+              chunks = await chunkBySymbols(content, symbols, documentId, fileId, totalTokenCount);
+            } else {
+              chunks = await fallbackChunk(content, documentId, fileId, totalTokenCount);
+            }
+          } catch {
             chunks = await fallbackChunk(content, documentId, fileId, totalTokenCount);
           }
-        } catch {
-          chunks = await fallbackChunk(content, documentId, fileId, totalTokenCount);
         }
       } else {
         // Large files without symbols: sliding windows
diff --git a/test/ingest.test.ts b/test/ingest.test.ts
index 2e1f8a8..27319a2 100644
--- a/test/ingest.test.ts
+++ b/test/ingest.test.ts
@@ -757,6 +757,29 @@ describe("ingestPath — edge cases (E5.3)", () => {
       .get()!;
     expect(integrity.integrity_check).toBe("ok");
   });
+
+  test("large file (>200KB) skips tree-sitter and uses window chunking", async () => {
+    // Use a dedicated subdirectory so leftover files from other tests don't interfere
+    const largeDir = join(TEST_DIR, "large-file-skip-symbol");
+    if (existsSync(largeDir)) rmSync(largeDir, { recursive: true, force: true });
+    mkdirSync(largeDir, { recursive: true });
+
+    // Generate a large TypeScript file (>200KB) with simple structure
+    let content = "// Large generated TypeScript file\n";
+    while (content.length < 210 * 1024) {
+      content += `export const item${Date.now()}_${Math.random().toString(36).slice(2)} = "value";\n`;
+    }
+    writeFileSync(join(largeDir, "large-generated.ts"), content);
+
+    const result = await ingestPath(db, largeDir, { recursive: false, reIndex: true });
+    expect(result.errors.length).toBe(0);
+    expect(result.chunksCreated).toBeGreaterThan(0);
+    // Verify chunks are window type (not symbol type), proving symbol extraction was skipped
+    const chunks = db.query("SELECT chunk_type FROM chunks WHERE file_id IN (SELECT id FROM files WHERE path LIKE '%large-generated.ts')").all() as { chunk_type: string }[];
+    for (const c of chunks) {
+      expect(c.chunk_type).toBe("window");
+    }
+  }, 30000);
 });
 
 // ---------------------------------------------------------------------------