From db9c2f8d1aa2b0e4879801ebe5a13f9676786e8f Mon Sep 17 00:00:00 2001 From: Robby Date: Fri, 12 Jun 2026 20:40:47 +0200 Subject: [PATCH 1/9] feat: add SKIP_DIRS, file-count guard, lower max size to 2MB #128 --- package.json | 2 +- src/ingest/index.ts | 22 +++++++++++++++++----- src/ingest/loader.ts | 6 ++++++ 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 4e5c0a6..57911c3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@four-bytes/four-opencode-brain", - "version": "1.7.4", + "version": "1.7.5", "description": "Unified brain plugin — single SQLite DB for RAG search, memory, and knowledge base", "license": "Apache-2.0", "type": "module", diff --git a/src/ingest/index.ts b/src/ingest/index.ts index c7a3e24..4536335 100644 --- a/src/ingest/index.ts +++ b/src/ingest/index.ts @@ -4,7 +4,7 @@ // 1. Resolve path // 2. Walk files (recursive if dir, single if file) // 3. Detect language from extension -// 4. 10MB file size cap (skip oversized files, never read content → avoid OOM) +// 4. 2MB file size cap (skip oversized files, never read content → avoid OOM) // 5. Content-hash check against files table → skip unchanged // 6. Upsert files table (FK-preserving: ON CONFLICT DO UPDATE, keeps rowid) // 7. Insert documents table (dedup via BEFORE INSERT trigger) @@ -58,8 +58,8 @@ export interface IngestOptions { // Constants // --------------------------------------------------------------------------- -/** Maximum file size for ingestion (10 MB). Files larger than this are skipped. */ -const MAX_FILE_SIZE = 10 * 1024 * 1024; // 10 MB +/** Maximum file size for ingestion (2 MB). Files larger than this are skipped. */ +const MAX_FILE_SIZE = 2 * 1024 * 1024; // 2 MB /** Per-file processing timeout (10 seconds). */ const FILE_TIMEOUT_MS = 10_000; // 10 seconds per file @@ -147,6 +147,18 @@ export async function ingestPath( return result; } + // ── File-count guard: warn at 500, hard-abort at 5000 ────────────── + const FILE_COUNT_WARN = 500; + const FILE_COUNT_ABORT = 5000; + if (walkedFiles.length > FILE_COUNT_ABORT) { + result.errors.push(`Aborted: ${walkedFiles.length} files found — exceeds hard limit of ${FILE_COUNT_ABORT}. Use a more specific path.`); + result.durationMs = Date.now() - startTime; + return result; + } + if (walkedFiles.length > FILE_COUNT_WARN) { + log("warn", "ingest", `Large ingest: ${walkedFiles.length} files — may take a while`); + } + // Yield so event loop can handle HTTP requests + tool calls before ingest starts await new Promise(r => setTimeout(r, 0)); @@ -161,7 +173,7 @@ export async function ingestPath( file: filePath, }); - // ── 4. 10MB file size cap (before reading content) ─────────────── + // ── 4. 2MB file size cap (before reading content) ─────────────── let fileStats; try { fileStats = await stat(filePath); @@ -172,7 +184,7 @@ export async function ingestPath( if (fileStats.size > MAX_FILE_SIZE) { result.errors.push( - `Skipped ${filePath}: file size ${fileStats.size} exceeds 10MB cap`, + `Skipped ${filePath}: file size ${fileStats.size} exceeds 2MB cap`, ); return; } diff --git a/src/ingest/loader.ts b/src/ingest/loader.ts index 788a812..f7e9232 100644 --- a/src/ingest/loader.ts +++ b/src/ingest/loader.ts @@ -36,6 +36,11 @@ const SKIP_DIRS = new Set([ "target", "zig-cache", "zig-out", + ".opencode", + "docker", + "var", + "bundles", + "db-dumps", ]); const EXTENSION_LANG_MAP: Record = { @@ -119,6 +124,7 @@ const EXTENSION_LANG_MAP: Record = { ".zsh": "shell", ".ps1": "powershell", ".sql": "sql", + ".twig": "twig", }; const BINARY_EXTENSIONS = new Set([ From ce4c572d63d1f6ecadf39cff380ede7273939ce8 Mon Sep 17 00:00:00 2001 From: Robby Date: Fri, 12 Jun 2026 20:54:05 +0200 Subject: [PATCH 2/9] feat: add slow-file warning logging (30s threshold, app.log) #128 --- package.json | 2 +- src/ingest/index.ts | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 57911c3..453896e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@four-bytes/four-opencode-brain", - "version": "1.7.5", + "version": "1.7.6", "description": "Unified brain plugin — single SQLite DB for RAG search, memory, and knowledge base", "license": "Apache-2.0", "type": "module", diff --git a/src/ingest/index.ts b/src/ingest/index.ts index 4536335..a0c501c 100644 --- a/src/ingest/index.ts +++ b/src/ingest/index.ts @@ -62,7 +62,10 @@ export interface IngestOptions { const MAX_FILE_SIZE = 2 * 1024 * 1024; // 2 MB /** Per-file processing timeout (10 seconds). */ -const FILE_TIMEOUT_MS = 10_000; // 10 seconds per file +const FILE_TIMEOUT_MS = 30_000; // 30 seconds per file + +/** Files exceeding this duration get logged to app.log as a warning. */ +const SLOW_FILE_WARN_MS = 30_000; // 30 seconds // --------------------------------------------------------------------------- // Progress event helpers (gated on BRAIN_DEBUG=true) @@ -398,6 +401,12 @@ export async function ingestPath( if (process.env.BRAIN_DEBUG === "true") { log("debug", "ingest", `processed ${result.filesProcessed}/${walkedFiles.length}: ${filePath} (${Date.now() - fileStart}ms)`); } + + // Slow-file warning: log to app.log for diagnostics + const elapsed = Date.now() - fileStart; + if (elapsed > SLOW_FILE_WARN_MS) { + log("warn", `ingest.slow_file.${filePath}`, `${filePath} took ${elapsed}ms`, { path: filePath, durationMs: elapsed, size }); + } } for (const [i, walked] of walkedFiles.entries()) { @@ -409,7 +418,7 @@ export async function ingestPath( } catch (err) { if (err instanceof Error && err.message === 'timeout') { emitProgressEvent("ingest.file_timeout", { file: walked.path }); - result.errors.push(`Timeout processing ${walked.path}: exceeded ${FILE_TIMEOUT_MS}ms`); + log("warn", "ingest.timeout", `Timeout processing ${walked.path}: exceeded ${FILE_TIMEOUT_MS}ms`, { path: walked.path, timeoutMs: FILE_TIMEOUT_MS }); result.filesProcessed++; continue; } From cae2dd206eb69bfa520d72a2a6b9cdf486f9562f Mon Sep 17 00:00:00 2001 From: 4 Bytes Robby Date: Sun, 14 Jun 2026 10:06:56 +0200 Subject: [PATCH 3/9] fix: record timeout errors in IngestResult.errors --- src/ingest/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ingest/index.ts b/src/ingest/index.ts index a0c501c..d8dc842 100644 --- a/src/ingest/index.ts +++ b/src/ingest/index.ts @@ -419,6 +419,7 @@ export async function ingestPath( if (err instanceof Error && err.message === 'timeout') { emitProgressEvent("ingest.file_timeout", { file: walked.path }); log("warn", "ingest.timeout", `Timeout processing ${walked.path}: exceeded ${FILE_TIMEOUT_MS}ms`, { path: walked.path, timeoutMs: FILE_TIMEOUT_MS }); + result.errors.push(`Timeout after ${FILE_TIMEOUT_MS}ms: ${walked.path}`); result.filesProcessed++; continue; } From 54ca790740b82288d208d66dfb0009bfeec2bf2d Mon Sep 17 00:00:00 2001 From: 4 Bytes Robby Date: Sun, 14 Jun 2026 10:09:05 +0200 Subject: [PATCH 4/9] fix: update test to match 2MB cap --- test/ingest.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/ingest.test.ts b/test/ingest.test.ts index 1534a9b..e33d07f 100644 --- a/test/ingest.test.ts +++ b/test/ingest.test.ts @@ -698,16 +698,16 @@ describe("ingestPath — edge cases (E5.3)", () => { expect(result.errors.length).toBe(0); }); - test("skips file over 10MB cap", async () => { - // Create a file just over 10MB - const largeContent = "x".repeat(11 * 1024 * 1024); // ~11MB + test("skips file over 2MB cap", async () => { + // Create a file just over 2MB + const largeContent = "x".repeat(3 * 1024 * 1024); // ~3MB writeFileSync(join(EDGE_DIR, "large.ts"), largeContent, "utf-8"); const result = await ingestPath(db, EDGE_DIR); expect(result.filesFound).toBe(1); expect(result.filesIndexed).toBe(0); expect(result.errors.length).toBeGreaterThanOrEqual(1); - expect(result.errors[0]).toContain("exceeds 10MB cap"); + expect(result.errors[0]).toContain("exceeds 2MB cap"); }); test("concurrent ingests don't corrupt database", async () => { From c5d956a0555fd48f0a08cfd844ce22f1305c809e Mon Sep 17 00:00:00 2001 From: 4 Bytes Robby Date: Sun, 14 Jun 2026 10:20:58 +0200 Subject: [PATCH 5/9] fix: update test comment to reflect actual 3MB size --- test/ingest.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/ingest.test.ts b/test/ingest.test.ts index e33d07f..2e85811 100644 --- a/test/ingest.test.ts +++ b/test/ingest.test.ts @@ -699,7 +699,7 @@ describe("ingestPath — edge cases (E5.3)", () => { }); test("skips file over 2MB cap", async () => { - // Create a file just over 2MB + // Create a file ~3MB (over the 2MB cap) const largeContent = "x".repeat(3 * 1024 * 1024); // ~3MB writeFileSync(join(EDGE_DIR, "large.ts"), largeContent, "utf-8"); From afe261b3f6d14067eb03f05fed583a6d5ad8fcca Mon Sep 17 00:00:00 2001 From: 4 Bytes Robby Date: Sun, 14 Jun 2026 10:23:16 +0200 Subject: [PATCH 6/9] fix: stale timeout comment, type annotation, double-counting --- src/ingest/index.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/ingest/index.ts b/src/ingest/index.ts index d8dc842..852fbd5 100644 --- a/src/ingest/index.ts +++ b/src/ingest/index.ts @@ -21,7 +21,7 @@ import type { Database } from "bun:sqlite"; import { generateId, hashBuffer, hashContent, checkpointDatabase } from "../schema"; import { log } from "../logger"; import { ingestMutex } from "./mutex"; -import { resolveFiles, detectLanguage, isBinaryContent, type WalkResult } from "./loader"; +import { resolveFiles, detectLanguage, isBinaryContent, type WalkResult, type WalkedFile } from "./loader"; import { chunkContent, type Chunk } from "./chunker"; import { extractSymbols } from "./symbolExtractor"; import { embedChunks } from "./embed"; @@ -61,7 +61,7 @@ export interface IngestOptions { /** Maximum file size for ingestion (2 MB). Files larger than this are skipped. */ const MAX_FILE_SIZE = 2 * 1024 * 1024; // 2 MB -/** Per-file processing timeout (10 seconds). */ +/** Per-file processing timeout (30 seconds). */ const FILE_TIMEOUT_MS = 30_000; // 30 seconds per file /** Files exceeding this duration get logged to app.log as a warning. */ @@ -165,7 +165,7 @@ export async function ingestPath( // Yield so event loop can handle HTTP requests + tool calls before ingest starts await new Promise(r => setTimeout(r, 0)); - async function processFile(walked: WalkResult, i: number): Promise { + async function processFile(walked: WalkedFile, i: number): Promise { const filePath = walked.path; const language = walked.language; const fileStart = Date.now(); @@ -420,8 +420,7 @@ export async function ingestPath( emitProgressEvent("ingest.file_timeout", { file: walked.path }); log("warn", "ingest.timeout", `Timeout processing ${walked.path}: exceeded ${FILE_TIMEOUT_MS}ms`, { path: walked.path, timeoutMs: FILE_TIMEOUT_MS }); result.errors.push(`Timeout after ${FILE_TIMEOUT_MS}ms: ${walked.path}`); - result.filesProcessed++; - continue; + continue; // Don't increment filesProcessed — processFile handles it if it completes } throw err; } From 4161dc339ceb3e90a08cec7be26c7046dd6a2256 Mon Sep 17 00:00:00 2001 From: 4 Bytes Robby Date: Sun, 14 Jun 2026 10:30:40 +0200 Subject: [PATCH 7/9] fix: cancel processFile on timeout with abort flag --- src/ingest/index.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/ingest/index.ts b/src/ingest/index.ts index 852fbd5..25f5ac6 100644 --- a/src/ingest/index.ts +++ b/src/ingest/index.ts @@ -165,7 +165,9 @@ export async function ingestPath( // Yield so event loop can handle HTTP requests + tool calls before ingest starts await new Promise(r => setTimeout(r, 0)); - async function processFile(walked: WalkedFile, i: number): Promise { + const abortFlags = new Map(); + + async function processFile(walked: WalkedFile, i: number, aborted: Map): Promise { const filePath = walked.path; const language = walked.language; const fileStart = Date.now(); @@ -390,6 +392,13 @@ export async function ingestPath( } } + // ── Abort guard: if timeout fired, ROLLBACK (don't commit) and skip counters ── + if (aborted.get(i)) { + db.exec(`ROLLBACK TO SAVEPOINT ${spFile}`); + db.exec(`RELEASE SAVEPOINT ${spFile}`); + return; + } + // ── RELEASE per-file SAVEPOINT ───────────────────────────── db.exec(`RELEASE SAVEPOINT ${spFile}`); @@ -412,11 +421,12 @@ export async function ingestPath( for (const [i, walked] of walkedFiles.entries()) { try { await Promise.race([ - processFile(walked, i), + processFile(walked, i, abortFlags), new Promise((_, reject) => setTimeout(() => reject(new Error('timeout')), FILE_TIMEOUT_MS)) ]); } catch (err) { if (err instanceof Error && err.message === 'timeout') { + abortFlags.set(i, true); emitProgressEvent("ingest.file_timeout", { file: walked.path }); log("warn", "ingest.timeout", `Timeout processing ${walked.path}: exceeded ${FILE_TIMEOUT_MS}ms`, { path: walked.path, timeoutMs: FILE_TIMEOUT_MS }); result.errors.push(`Timeout after ${FILE_TIMEOUT_MS}ms: ${walked.path}`); From 9d56d07887bb516f841b3d5780b3e8e493f4b56b Mon Sep 17 00:00:00 2001 From: 4 Bytes Robby Date: Sun, 14 Jun 2026 10:42:10 +0200 Subject: [PATCH 8/9] fix: revert counter increments on abort --- src/ingest/index.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ingest/index.ts b/src/ingest/index.ts index 25f5ac6..18e9d55 100644 --- a/src/ingest/index.ts +++ b/src/ingest/index.ts @@ -265,6 +265,10 @@ export async function ingestPath( const fileName = filePath.split("/").pop() ?? filePath; const filetype = filePath.split(".").pop() ?? "unknown"; + // Snapshot result counters so we can revert this file's increments on abort + const docCountBefore = result.documentsCreated; + const chunkCountBefore = result.chunksCreated; + try { db.run( `INSERT OR IGNORE INTO documents (id, title, content, content_hash, type, path, language, filetype, project_hash) @@ -396,6 +400,8 @@ export async function ingestPath( if (aborted.get(i)) { db.exec(`ROLLBACK TO SAVEPOINT ${spFile}`); db.exec(`RELEASE SAVEPOINT ${spFile}`); + result.chunksCreated -= result.chunksCreated - chunkCountBefore; + result.documentsCreated -= result.documentsCreated - docCountBefore; return; } From f9e3318e1547758a9d9d60d9fa67421d31101202 Mon Sep 17 00:00:00 2001 From: 4 Bytes Robby Date: Sun, 14 Jun 2026 12:24:39 +0200 Subject: [PATCH 9/9] fix: remove unused import, verify SAVEPOINT cleanup --- src/ingest/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ingest/index.ts b/src/ingest/index.ts index 18e9d55..c4300dd 100644 --- a/src/ingest/index.ts +++ b/src/ingest/index.ts @@ -21,7 +21,7 @@ import type { Database } from "bun:sqlite"; import { generateId, hashBuffer, hashContent, checkpointDatabase } from "../schema"; import { log } from "../logger"; import { ingestMutex } from "./mutex"; -import { resolveFiles, detectLanguage, isBinaryContent, type WalkResult, type WalkedFile } from "./loader"; +import { resolveFiles, isBinaryContent, type WalkResult, type WalkedFile } from "./loader"; import { chunkContent, type Chunk } from "./chunker"; import { extractSymbols } from "./symbolExtractor"; import { embedChunks } from "./embed";