diff --git a/CHANGELOG.md b/CHANGELOG.md index e611c15..966d236 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to this project are documented here, following [Keep a Changelog](https://keepachangelog.com/) and semantic versioning. +## [0.1.16] - 2026-09-17 + +### Fixed + +- Treat vendored (`node_modules`, `vendor`, ...) and build output (`dist`, `build`, ...) directories as single trim candidates: record the directory once by name and never read or descend into it. Reported patterns and category totals are unchanged. + ## [0.1.15] - 2026-09-17 ### Fixed diff --git a/README.md b/README.md index 9282944..89e3050 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,8 @@ ctxtrim · my-repo · 412 files Estimates use the widely-cited ~4-chars-per-token rule (great for ranking and relative savings; pass `--price` to match your model). +Vendored dependency and build output directories (`node_modules`, `dist`, `build`, ...) are reported as a single entry keyed by directory name — sized from a cheap stat walk without reading the files inside. Directory-level categories are decided by the directory name, not by reading its contents. + ## Usage ```bash diff --git a/package-lock.json b/package-lock.json index 9c33059..20a1a7d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,20 @@ { "name": "ctxtrim", +<<<<<<< HEAD + "version": "0.1.16", +======= "version": "0.1.15", +>>>>>>> origin/main "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ctxtrim", +<<<<<<< HEAD + "version": "0.1.16", +======= "version": "0.1.15", +>>>>>>> origin/main "license": "MIT", "bin": { "ctxtrim": "bin/ctxtrim.js" diff --git a/package.json b/package.json index be200eb..8f855ca 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ctxtrim", - "version": "0.1.15", + "version": "0.1.16", "description": "Trim what bloats your AI coding context. Scan a repo, find the high-cost/low-value files ballooning your Claude Code / Cursor / Codex context, and write ignore files to cut token cost. Zero dependencies.", "type": "module", "bin": { diff --git a/src/scan.js b/src/scan.js index 23a67b3..cadef51 100644 --- a/src/scan.js +++ b/src/scan.js @@ -1,14 +1,37 @@ // Walk a repo, estimate each file's token cost, classify it, and aggregate. import { readdirSync, readFileSync, statSync, openSync, readSync, closeSync } from "node:fs"; import { join, relative, sep } from "node:path"; -import { classify, classifyPath, ignorePattern } from "./classify.js"; +import { classify, classifyPath, ignorePattern, VENDOR_DIRS, BUILD_DIRS } from "./classify.js"; const ALWAYS_SKIP = new Set([".git"]); +// Directory names that are vendored or build output: record once, never descend. +const VENDOR_SKIP = new Set(VENDOR_DIRS.map((d) => d.toLowerCase())); +const BUILD_SKIP = new Set(BUILD_DIRS.map((d) => d.toLowerCase())); const MAX_READ = 5_000_000; // bytes fully read; larger files are estimated from size /** ~4 chars per token is the widely-cited rule of thumb; good enough to rank files. */ export const estimateTokens = (text) => Math.ceil(text.length / 4); +/** Cheap total token estimate for a directory: sum sibling sizes without reading contents. */ +function dirTokenEstimate(abs) { + let bytes = 0; + const stack = [abs]; + while (stack.length) { + const dir = stack.pop(); + let entries; + try { entries = readdirSync(dir, { withFileTypes: true }); } catch { continue; } + for (const e of entries) { + if (e.isSymbolicLink()) continue; + const child = join(dir, e.name); + try { + if (e.isDirectory()) stack.push(child); + else if (e.isFile()) bytes += statSync(child).size; + } catch { /* skip unreadable entries */ } + } + } + return Math.ceil(bytes / 4); +} + function fileInfo(abs, size) { // Read up to MAX_READ bytes for both token estimate and the generated-marker sample. let text = ""; @@ -48,7 +71,23 @@ export function scanRepo(target, opts = {}) { for (const e of entries) { if (ALWAYS_SKIP.has(e.name)) continue; const abs = join(dir, e.name); - if (e.isDirectory()) { walk(abs); continue; } + if (e.isDirectory()) { + const lower = e.name.toLowerCase(); + if (VENDOR_SKIP.has(lower) || BUILD_SKIP.has(lower)) { + // Record the directory once by name; do not descend or read contents. + const rel = relative(root, abs).split(sep).join("/"); + const reason = VENDOR_SKIP.has(lower) + ? "vendored dependency directory" + : "build / generated output directory"; + files.push({ + rel, size: 0, tokens: dirTokenEstimate(abs), + category: VENDOR_SKIP.has(lower) ? "vendored" : "build", + trim: true, binary: false, reason, + }); + continue; + } + walk(abs); continue; + } if (!e.isFile() && !e.isSymbolicLink()) continue; let size = 0; try { diff --git a/test/ctxtrim.test.js b/test/ctxtrim.test.js index 2f64198..f490c2f 100644 --- a/test/ctxtrim.test.js +++ b/test/ctxtrim.test.js @@ -1,11 +1,11 @@ -import fs, { existsSync, mkdtempSync, rmSync, symlinkSync, truncateSync, writeFileSync } from "node:fs"; +import fs, { existsSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, truncateSync, writeFileSync } from "node:fs"; import { spawnSync } from "node:child_process"; import { syncBuiltinESMExports } from "node:module"; import { tmpdir } from "node:os"; import { test } from "node:test"; import assert from "node:assert/strict"; import { fileURLToPath } from "node:url"; -import { dirname, join } from "node:path"; +import { dirname, join, sep } from "node:path"; import { scanRepo, estimateTokens } from "../src/scan.js"; import { classify, classifyPath } from "../src/classify.js"; import { merge, block } from "../src/ignore.js"; @@ -122,6 +122,55 @@ test("scan includes symlinked source files", (t) => { assert.deepEqual(result.files.map((file) => file.rel).sort(), ["linked.py", "real.py"]); }); +test("scan does not read files inside vendored or build directories", (t) => { + const root = mkdtempSync(join(tmpdir(), "ctxtrim-vendored-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + + // node_modules with a big file that would be read WITHOUT this fix. + const npm = join(root, "node_modules", "dep"); + mkdirSync(join(npm), { recursive: true }); + writeFileSync(join(npm, "index.js"), "x".repeat(200_000)); + // dist with a big bundle. + const dist = join(root, "dist"); + mkdirSync(dist, { recursive: true }); + writeFileSync(join(dist, "bundle.js"), "y".repeat(200_000)); + writeFileSync(join(root, "src.js"), "export const n = 1;\n"); + + const readFiles = []; + const readFileSync = fs.readFileSync; + t.mock.method(fs, "readFileSync", (...args) => { + readFiles.push(String(args[0])); + if (String(args[0]).includes("node_modules") || String(args[0]).includes(sep + "dist")) { + const err = new Error(`should never read vendored/build file: ${args[0]}`); + err.code = "EACCES"; + throw err; + } + return readFileSync(...args); + }); + syncBuiltinESMExports(); + + let result; + try { + result = scanRepo(root); + } finally { + t.mock.restoreAll(); + syncBuiltinESMExports(); + } + + assert.ok( + !readFiles.some((p) => p.includes("node_modules")), + "no file inside node_modules should be read", + ); + assert.ok(!readFiles.some((p) => p.includes(sep + "dist")), "no file inside dist should be read"); + + // The vendored dir still reports under its collapse pattern, and the totals still include it. + assert.ok(result.patterns.includes("node_modules/")); + assert.ok(result.patterns.includes("dist/")); + assert.equal(result.totals.byCategory.vendored.tokens, 200_000 / 4); + assert.equal(result.totals.byCategory.build.tokens, 200_000 / 4); + assert.equal(result.files.find((f) => f.rel === "src.js").category, "source"); +}); + test("scan surfaces unreadable (EACCES) files with estimated tokens, not silent 0", (t) => { const root = mkdtempSync(join(tmpdir(), "ctxtrim-unreadable-")); t.after(() => rmSync(root, { recursive: true, force: true })); @@ -157,11 +206,11 @@ test("scan finds trimmable bloat and keeps source", () => { const s = scanRepo(repo); assert.ok(s.totals.trimTokens > 0); assert.ok(s.totals.wastePct > 50, `expected mostly-junk fixture, got ${s.totals.wastePct}%`); - // the three bloat files are flagged + // the three bloat files are flagged (dist/bundle.js collapses to the dist dir) const trimmed = new Set(s.files.filter((f) => f.trim).map((f) => f.rel)); assert.ok(trimmed.has("package-lock.json")); assert.ok(trimmed.has("data/seed.json")); - assert.ok([...trimmed].some((p) => p.startsWith("dist/"))); + assert.ok(trimmed.has("dist"), "build dir recorded once by name"); // real source is NOT trimmed const src = s.files.find((f) => f.rel === "src/index.js"); assert.equal(src.trim, false);