From afd289d592315f28b405dd546320d4fbe15a7587 Mon Sep 17 00:00:00 2001 From: Tanmay-008 Date: Tue, 14 Jul 2026 12:42:07 +0530 Subject: [PATCH 1/4] perf(mapper): cache full-root filesystem scans across concurrent mappers The Go, C/C++, and Dotnet mappers unconditionally run full-root `walk()` scans in parallel, producing 3x redundant disk I/O in large polyglot monorepos. This introduces a shared Virtual File System (VFS) Cache inside the `MapperContext`. The cache memoizes `readdir`, `lstat`, and `realpath` Promises, ensuring that concurrent mappers hitting the same directories share the exact same I/O operation. Language-specific skip filters now run instantly on the cached memory arrays. --- src/mapper.ts | 12 +++++++---- src/mappers/c-cpp.ts | 6 +++--- src/mappers/dotnet.ts | 6 +++--- src/mappers/go.ts | 14 ++++++------- src/mappers/shared.ts | 31 +++++++++++++++++++---------- src/mappers/types.ts | 2 ++ src/mappers/vfs-cache.ts | 43 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 87 insertions(+), 27 deletions(-) create mode 100644 src/mappers/vfs-cache.ts diff --git a/src/mapper.ts b/src/mapper.ts index dcaf8fb7..58a9b952 100644 --- a/src/mapper.ts +++ b/src/mapper.ts @@ -27,6 +27,7 @@ import { createNearbyTestFinder, PathFilters, pathMatchesFilters } from "./mappe import { swiftSeeds } from "./mappers/swift.js"; import { turboTaskGraph } from "./mappers/turbo.js"; import { FeatureMapper, FeatureSeed, MapperContext } from "./mappers/types.js"; +import { createVfsCache } from "./mappers/vfs-cache.js"; import { FeatureRecord, ProjectRecord } from "./types.js"; export type MapResult = { @@ -291,10 +292,13 @@ async function collectSeeds( project: ProjectRecord, options: MapOptions, ): Promise { - const context: MapperContext = createMapperContext({ - discoverNodeProjects: () => discoverNodeProjects(root), - buildNodeTaskGraph: (projects) => turboTaskGraph(root, projects), - }); + const context: MapperContext = { + ...createMapperContext({ + discoverNodeProjects: () => discoverNodeProjects(root), + buildNodeTaskGraph: (projects) => turboTaskGraph(root, projects), + }), + vfs: createVfsCache(), + }; const runNodeMappers = shouldRunNodeMappers(root, project); const groups = await Promise.all( featureMappers.map(async (mapper) => { diff --git a/src/mappers/c-cpp.ts b/src/mappers/c-cpp.ts index 7dca5a04..15b5e397 100644 --- a/src/mappers/c-cpp.ts +++ b/src/mappers/c-cpp.ts @@ -15,10 +15,10 @@ import { withCudaConcurrency, } from "./shared.js"; import { cCppGroupSeeds } from "./c-cpp-groups.js"; -import { FeatureSeed, SeedFileRef } from "./types.js"; +import { FeatureSeed, MapperContext, SeedFileRef } from "./types.js"; -export async function cCppSeeds(root: string): Promise { - const files = (await walk(root, [""], shouldSkipCOrCppPath)).filter( +export async function cCppSeeds(root: string, context: MapperContext): Promise { + const files = (await walk(root, [""], shouldSkipCOrCppPath, context.vfs)).filter( (path) => !isSampleProjectPath(path) && (isCOrCppSource(path) || isMakefile(path) || isCMake(path)), ); diff --git a/src/mappers/dotnet.ts b/src/mappers/dotnet.ts index 5f75d9ce..348e1a4b 100644 --- a/src/mappers/dotnet.ts +++ b/src/mappers/dotnet.ts @@ -4,7 +4,7 @@ import { shellQuotePath } from "../shell.js"; import { TrustBoundary } from "../types.js"; import { partitionFileGroups } from "./grouping.js"; import { isSampleProjectPath, normalize, pathMatchesPrefix, shouldSkip, walk } from "./shared.js"; -import { FeatureSeed, SeedFileRef, SeedTestRef } from "./types.js"; +import { FeatureSeed, MapperContext, SeedFileRef, SeedTestRef } from "./types.js"; const maxOwnedFiles = 12; const maxTests = 8; @@ -31,8 +31,8 @@ type DotnetSolution = { projectPaths: string[]; }; -export async function dotnetSeeds(root: string): Promise { - const files = await walk(root, [""], shouldSkipDotnetPath); +export async function dotnetSeeds(root: string, context: MapperContext): Promise { + const files = await walk(root, [""], shouldSkipDotnetPath, context.vfs); const fileSet = new Set(files); const solutions = await dotnetSolutions(root, files.filter(isDotnetSolutionPath)); const projectPaths = uniqueStrings([ diff --git a/src/mappers/go.ts b/src/mappers/go.ts index 8fc25934..af0b41ab 100644 --- a/src/mappers/go.ts +++ b/src/mappers/go.ts @@ -3,14 +3,14 @@ import { readdir, readFile, realpath } from "node:fs/promises"; import { isAbsolute, join, relative } from "node:path"; import { pathExists } from "../fs.js"; import { packageKind, packageTrustBoundaries, normalize, shouldSkip, walk } from "./shared.js"; -import { FeatureSeed, SeedFileRef, SeedTestRef } from "./types.js"; +import { FeatureSeed, MapperContext, SeedFileRef, SeedTestRef } from "./types.js"; -export async function goSeeds(root: string): Promise { +export async function goSeeds(root: string, context: MapperContext): Promise { if (!(await pathExists(join(root, "go.mod")))) { return []; } const modulePath = await goModulePath(root); - const packages = await goPackages(root, modulePath); + const packages = await goPackages(root, modulePath, context); const packageByImport = new Map(packages.map((pkg) => [pkg.importPath, pkg])); const seeds: FeatureSeed[] = []; for (const pkg of packages) { @@ -36,12 +36,12 @@ type GoPackageFiles = { generated: string[]; }; -async function goPackages(root: string, modulePath: string | null): Promise { +async function goPackages(root: string, modulePath: string | null, context: MapperContext): Promise { const listed = await goListPackages(root); if (listed.length > 0) { return listed; } - return fallbackGoPackages(root, modulePath); + return fallbackGoPackages(root, modulePath, context); } async function goListPackages(root: string): Promise { @@ -66,9 +66,9 @@ async function goListPackages(root: string): Promise { return packages; } -async function fallbackGoPackages(root: string, modulePath: string | null): Promise { +async function fallbackGoPackages(root: string, modulePath: string | null, context: MapperContext): Promise { const dirs = new Set(); - for (const file of await walk(root, [""])) { + for (const file of await walk(root, [""], shouldSkip, context.vfs)) { if (!file.endsWith(".go")) { continue; } diff --git a/src/mappers/shared.ts b/src/mappers/shared.ts index 08b69c7a..87cfdbed 100644 --- a/src/mappers/shared.ts +++ b/src/mappers/shared.ts @@ -190,23 +190,28 @@ export async function walk( root: string, prefixes: string[], skipPath: (path: string) => boolean = shouldSkip, + vfs?: import("./vfs-cache.js").VfsCache, ): Promise { + const fsLstat = vfs ? vfs.fileStat : lstat; + const fsRealpath = vfs + ? vfs.resolveRealpath + : (p: string) => realpath(p).catch(() => p); const files: string[] = []; const seen = new Set(); const seenRoots = new Set(); - const realRoot = await realpath(root).catch(() => root); + const realRoot = await fsRealpath(root); for (const prefix of prefixes) { const start = join(root, prefix); if (!(await pathExists(start))) { continue; } - let info = await lstat(start); - const canonicalStart = await realpath(start).catch(() => start); + let info = await fsLstat(start); + const canonicalStart = await fsRealpath(start); if (info.isSymbolicLink() && prefix !== "") { continue; } if (info.isSymbolicLink()) { - info = await lstat(canonicalStart).catch(() => info); + info = await fsLstat(canonicalStart).catch(() => info); } if (!pathInsideRoot(realRoot, canonicalStart)) { continue; @@ -223,7 +228,7 @@ export async function walk( continue; } seenRoots.add(canonicalStart); - await walkDir(realRoot, canonicalStart, files, seen, skipPath); + await walkDir(realRoot, canonicalStart, files, seen, skipPath, vfs); } return files.toSorted(); } @@ -234,12 +239,18 @@ async function walkDir( files: string[], seen: Set, skipPath: (path: string) => boolean, + vfs?: import("./vfs-cache.js").VfsCache, ): Promise { - const dirInfo = await lstat(dir); + const fsLstat = vfs ? vfs.fileStat : lstat; + const fsRealpath = vfs + ? vfs.resolveRealpath + : (p: string) => realpath(p).catch(() => p); + const fsReaddir = vfs ? vfs.readDirectory : readdir; + const dirInfo = await fsLstat(dir); if (dirInfo.isSymbolicLink()) { return; } - const realDir = await realpath(dir).catch(() => dir); + const realDir = await fsRealpath(dir); if (!pathInsideRoot(root, realDir)) { return; } @@ -247,7 +258,7 @@ async function walkDir( if (skipPath(relDir)) { return; } - const entries = await readdir(dir); + const entries = await fsReaddir(dir); for (const entry of entries) { const full = join(dir, entry); const rel = normalize(relative(root, full)); @@ -255,12 +266,12 @@ async function walkDir( continue; } seen.add(rel); - const info = await lstat(full); + const info = await fsLstat(full); if (info.isSymbolicLink()) { continue; } if (info.isDirectory()) { - await walkDir(root, full, files, seen, skipPath); + await walkDir(root, full, files, seen, skipPath, vfs); } else if (info.isFile()) { files.push(rel); } diff --git a/src/mappers/types.ts b/src/mappers/types.ts index cdaf5785..4d5742e1 100644 --- a/src/mappers/types.ts +++ b/src/mappers/types.ts @@ -1,6 +1,7 @@ import { FeatureRecord, TrustBoundary } from "../types.js"; import type { NodeProjectInfo } from "./projects.js"; import type { WorkspaceTaskGraph } from "./task-graph.js"; +import type { VfsCache } from "./vfs-cache.js"; export type SeedFileRef = { path: string; @@ -44,4 +45,5 @@ export type FeatureMapper = { export type MapperContext = { nodeProjects(): Promise; nodeTaskGraph(): Promise; + vfs: VfsCache; }; diff --git a/src/mappers/vfs-cache.ts b/src/mappers/vfs-cache.ts new file mode 100644 index 00000000..df147724 --- /dev/null +++ b/src/mappers/vfs-cache.ts @@ -0,0 +1,43 @@ +import { lstat, readdir, realpath } from "node:fs/promises"; +import type { Stats } from "node:fs"; + +export type VfsCache = { + readDirectory(path: string): Promise; + fileStat(path: string): Promise; + resolveRealpath(path: string): Promise; +}; + +export function createVfsCache(): VfsCache { + const dirCache = new Map>(); + const statCache = new Map>(); + const realpathCache = new Map>(); + + return { + readDirectory(path: string): Promise { + let cached = dirCache.get(path); + if (cached === undefined) { + cached = readdir(path).catch(() => []); + dirCache.set(path, cached); + } + return cached; + }, + + fileStat(path: string): Promise { + let cached = statCache.get(path); + if (cached === undefined) { + cached = lstat(path); + statCache.set(path, cached); + } + return cached; + }, + + resolveRealpath(path: string): Promise { + let cached = realpathCache.get(path); + if (cached === undefined) { + cached = realpath(path).catch(() => path); + realpathCache.set(path, cached); + } + return cached; + }, + }; +} From 1af05d54e2e803e6f4f3251ae95c22e7ef255a5e Mon Sep 17 00:00:00 2001 From: Tanmay-008 Date: Tue, 14 Jul 2026 13:15:57 +0530 Subject: [PATCH 2/4] style: format files --- src/mappers/go.ts | 12 ++++++++++-- src/mappers/shared.ts | 8 ++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/mappers/go.ts b/src/mappers/go.ts index af0b41ab..a5f3f690 100644 --- a/src/mappers/go.ts +++ b/src/mappers/go.ts @@ -36,7 +36,11 @@ type GoPackageFiles = { generated: string[]; }; -async function goPackages(root: string, modulePath: string | null, context: MapperContext): Promise { +async function goPackages( + root: string, + modulePath: string | null, + context: MapperContext, +): Promise { const listed = await goListPackages(root); if (listed.length > 0) { return listed; @@ -66,7 +70,11 @@ async function goListPackages(root: string): Promise { return packages; } -async function fallbackGoPackages(root: string, modulePath: string | null, context: MapperContext): Promise { +async function fallbackGoPackages( + root: string, + modulePath: string | null, + context: MapperContext, +): Promise { const dirs = new Set(); for (const file of await walk(root, [""], shouldSkip, context.vfs)) { if (!file.endsWith(".go")) { diff --git a/src/mappers/shared.ts b/src/mappers/shared.ts index 87cfdbed..bdd0e82b 100644 --- a/src/mappers/shared.ts +++ b/src/mappers/shared.ts @@ -193,9 +193,7 @@ export async function walk( vfs?: import("./vfs-cache.js").VfsCache, ): Promise { const fsLstat = vfs ? vfs.fileStat : lstat; - const fsRealpath = vfs - ? vfs.resolveRealpath - : (p: string) => realpath(p).catch(() => p); + const fsRealpath = vfs ? vfs.resolveRealpath : (p: string) => realpath(p).catch(() => p); const files: string[] = []; const seen = new Set(); const seenRoots = new Set(); @@ -242,9 +240,7 @@ async function walkDir( vfs?: import("./vfs-cache.js").VfsCache, ): Promise { const fsLstat = vfs ? vfs.fileStat : lstat; - const fsRealpath = vfs - ? vfs.resolveRealpath - : (p: string) => realpath(p).catch(() => p); + const fsRealpath = vfs ? vfs.resolveRealpath : (p: string) => realpath(p).catch(() => p); const fsReaddir = vfs ? vfs.readDirectory : readdir; const dirInfo = await fsLstat(dir); if (dirInfo.isSymbolicLink()) { From 4a14b65fc67d6cd0c8f5575d83e8541ecbebccd9 Mon Sep 17 00:00:00 2001 From: Tanmay-008 Date: Tue, 14 Jul 2026 19:08:44 +0530 Subject: [PATCH 3/4] fix(mapper): preserve directory-read failures in VFS cache Removes the silent empty-array fallback from readdir so permission errors correctly bubble up. Adds focused cache/error regression coverage. --- src/mappers/vfs-cache.test.ts | 42 +++++++++++++++++++++++++++++++++++ src/mappers/vfs-cache.ts | 2 +- 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 src/mappers/vfs-cache.test.ts diff --git a/src/mappers/vfs-cache.test.ts b/src/mappers/vfs-cache.test.ts new file mode 100644 index 00000000..afbdd858 --- /dev/null +++ b/src/mappers/vfs-cache.test.ts @@ -0,0 +1,42 @@ +import { readdir } from "node:fs/promises"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { createVfsCache } from "./vfs-cache.js"; + +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + readdir: vi.fn(), + }; +}); + +describe("VfsCache", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("memoizes successful readdir calls", async () => { + vi.mocked(readdir).mockResolvedValueOnce(["file1.txt", "file2.txt"] as any); + const vfs = createVfsCache(); + + const first = await vfs.readDirectory("/fake/path"); + const second = await vfs.readDirectory("/fake/path"); + + expect(first).toEqual(["file1.txt", "file2.txt"]); + expect(second).toEqual(["file1.txt", "file2.txt"]); + expect(readdir).toHaveBeenCalledTimes(1); + expect(readdir).toHaveBeenCalledWith("/fake/path"); + }); + + it("preserves and memoizes directory-read failures", async () => { + const error = new Error("EACCES: permission denied, scandir '/fake/secret'"); + vi.mocked(readdir).mockRejectedValueOnce(error); + const vfs = createVfsCache(); + + await expect(vfs.readDirectory("/fake/secret")).rejects.toThrow(error); + await expect(vfs.readDirectory("/fake/secret")).rejects.toThrow(error); + + expect(readdir).toHaveBeenCalledTimes(1); + expect(readdir).toHaveBeenCalledWith("/fake/secret"); + }); +}); diff --git a/src/mappers/vfs-cache.ts b/src/mappers/vfs-cache.ts index df147724..9f1c9993 100644 --- a/src/mappers/vfs-cache.ts +++ b/src/mappers/vfs-cache.ts @@ -16,7 +16,7 @@ export function createVfsCache(): VfsCache { readDirectory(path: string): Promise { let cached = dirCache.get(path); if (cached === undefined) { - cached = readdir(path).catch(() => []); + cached = readdir(path); dirCache.set(path, cached); } return cached; From b4d0c256fa6170b11cf893f47f3e4fa896be0741 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 20 Jul 2026 18:46:41 -0700 Subject: [PATCH 4/4] perf(mapper): share policy-aware root inventory Co-authored-by: Tanmay-008 --- CHANGELOG.md | 1 + src/mapper.ts | 34 ++++++++---- src/mappers/c-cpp.ts | 5 +- src/mappers/context.test.ts | 61 ++++++++++++++++++++-- src/mappers/context.ts | 8 ++- src/mappers/dotnet.ts | 6 +-- src/mappers/go.ts | 4 +- src/mappers/shared.test.ts | 43 ++++++++++++++- src/mappers/shared.ts | 98 +++++++++++++++++++++++------------ src/mappers/types.ts | 6 ++- src/mappers/vfs-cache.test.ts | 42 --------------- src/mappers/vfs-cache.ts | 43 --------------- 12 files changed, 203 insertions(+), 148 deletions(-) delete mode 100644 src/mappers/vfs-cache.test.ts delete mode 100644 src/mappers/vfs-cache.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fa9a87a..821869aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 0.7.1 - Unreleased +- Reduced mapper startup I/O by sharing one root file inventory across Go fallback, C/C++, and .NET mapping, thanks @Tanmay-008. - Fixed revalidation prompts to compact historical and feature metadata and hard-cap metadata lists even when configured file limits are high, preventing provider input overflows, thanks @pai-scaffolde. - Added an opt-in Claude host auth context that preserves the default-deny environment, uses Claude Code safe mode, validates auth through doctor, and reports redacted OAuth failure signals, thanks @grantjayy. diff --git a/src/mapper.ts b/src/mapper.ts index 58a9b952..4d7c3ace 100644 --- a/src/mapper.ts +++ b/src/mapper.ts @@ -5,9 +5,9 @@ import { seedIdentityParts, stableFeatureJson, } from "./mapper-reconciliation.js"; -import { cCppSeeds } from "./mappers/c-cpp.js"; +import { cCppSeeds, shouldSkipCOrCppPath } from "./mappers/c-cpp.js"; import { configSeeds } from "./mappers/config.js"; -import { dotnetSeeds } from "./mappers/dotnet.js"; +import { dotnetSeeds, shouldSkipDotnetPath } from "./mappers/dotnet.js"; import { elixirSeeds } from "./mappers/elixir.js"; import { goSeeds } from "./mappers/go.js"; import { appleSeeds } from "./mappers/apple.js"; @@ -23,11 +23,16 @@ import { createMapperContext } from "./mappers/context.js"; import { discoverNodeProjects, hasFallbackNodeProjectSignal } from "./mappers/projects.js"; import { rubySeeds } from "./mappers/ruby.js"; import { rustSeeds } from "./mappers/rust.js"; -import { createNearbyTestFinder, PathFilters, pathMatchesFilters } from "./mappers/shared.js"; +import { + createNearbyTestFinder, + PathFilters, + pathMatchesFilters, + shouldSkip, + walkByPolicy, +} from "./mappers/shared.js"; import { swiftSeeds } from "./mappers/swift.js"; import { turboTaskGraph } from "./mappers/turbo.js"; import { FeatureMapper, FeatureSeed, MapperContext } from "./mappers/types.js"; -import { createVfsCache } from "./mappers/vfs-cache.js"; import { FeatureRecord, ProjectRecord } from "./types.js"; export type MapResult = { @@ -292,13 +297,20 @@ async function collectSeeds( project: ProjectRecord, options: MapOptions, ): Promise { - const context: MapperContext = { - ...createMapperContext({ - discoverNodeProjects: () => discoverNodeProjects(root), - buildNodeTaskGraph: (projects) => turboTaskGraph(root, projects), - }), - vfs: createVfsCache(), - }; + const context: MapperContext = createMapperContext({ + discoverNodeProjects: () => discoverNodeProjects(root), + buildNodeTaskGraph: (projects) => turboTaskGraph(root, projects), + buildRootFileInventory: () => + walkByPolicy( + root, + [""], + [ + { key: "go-fallback", skipPath: shouldSkip }, + { key: "c-cpp", skipPath: shouldSkipCOrCppPath }, + { key: "dotnet", skipPath: shouldSkipDotnetPath }, + ], + ), + }); const runNodeMappers = shouldRunNodeMappers(root, project); const groups = await Promise.all( featureMappers.map(async (mapper) => { diff --git a/src/mappers/c-cpp.ts b/src/mappers/c-cpp.ts index 15b5e397..aff62188 100644 --- a/src/mappers/c-cpp.ts +++ b/src/mappers/c-cpp.ts @@ -11,14 +11,13 @@ import { shouldSkip, stripLineComments, targetLanguageTag, - walk, withCudaConcurrency, } from "./shared.js"; import { cCppGroupSeeds } from "./c-cpp-groups.js"; import { FeatureSeed, MapperContext, SeedFileRef } from "./types.js"; export async function cCppSeeds(root: string, context: MapperContext): Promise { - const files = (await walk(root, [""], shouldSkipCOrCppPath, context.vfs)).filter( + const files = (await context.rootFiles("c-cpp")).filter( (path) => !isSampleProjectPath(path) && (isCOrCppSource(path) || isMakefile(path) || isCMake(path)), ); @@ -1107,7 +1106,7 @@ function isCOrCppDependencyPath(path: string): boolean { return /(^|\/)(deps|vendor|CMakeFiles|cmake-build-[^/]+)(\/|$)/u.test(path); } -function shouldSkipCOrCppPath(path: string): boolean { +export function shouldSkipCOrCppPath(path: string): boolean { return shouldSkip(path) || isCOrCppDependencyPath(path); } diff --git a/src/mappers/context.test.ts b/src/mappers/context.test.ts index 9abaf158..1ebda0b2 100644 --- a/src/mappers/context.test.ts +++ b/src/mappers/context.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it, vi } from "vitest"; import { createMapperContext } from "./context.js"; import { emptyTaskGraph } from "./task-graph.js"; +import type { RootFileInventory } from "./types.js"; + +const emptyRootFileInventory = async (): Promise => new Map(); describe("createMapperContext", () => { it("shares concurrent first access across all Node consumers", async () => { @@ -8,7 +11,11 @@ describe("createMapperContext", () => { const graph = emptyTaskGraph(); const discoverNodeProjects = vi.fn(async () => projects); const buildNodeTaskGraph = vi.fn(async () => graph); - const context = createMapperContext({ discoverNodeProjects, buildNodeTaskGraph }); + const context = createMapperContext({ + discoverNodeProjects, + buildNodeTaskGraph, + buildRootFileInventory: emptyRootFileInventory, + }); const results = await Promise.all([ context.nodeProjects(), @@ -29,7 +36,11 @@ describe("createMapperContext", () => { throw failure; }); const buildNodeTaskGraph = vi.fn(async () => emptyTaskGraph()); - const context = createMapperContext({ discoverNodeProjects, buildNodeTaskGraph }); + const context = createMapperContext({ + discoverNodeProjects, + buildNodeTaskGraph, + buildRootFileInventory: emptyRootFileInventory, + }); const results = await Promise.allSettled([ context.nodeProjects(), @@ -52,7 +63,11 @@ describe("createMapperContext", () => { const buildNodeTaskGraph = vi.fn(async () => { throw failure; }); - const context = createMapperContext({ discoverNodeProjects, buildNodeTaskGraph }); + const context = createMapperContext({ + discoverNodeProjects, + buildNodeTaskGraph, + buildRootFileInventory: emptyRootFileInventory, + }); const results = await Promise.allSettled([context.nodeTaskGraph(), context.nodeTaskGraph()]); @@ -68,10 +83,46 @@ describe("createMapperContext", () => { const discoverNodeProjects = vi.fn(async () => []); const buildNodeTaskGraph = vi.fn(async () => emptyTaskGraph()); - await createMapperContext({ discoverNodeProjects, buildNodeTaskGraph }).nodeTaskGraph(); - await createMapperContext({ discoverNodeProjects, buildNodeTaskGraph }).nodeTaskGraph(); + await createMapperContext({ + discoverNodeProjects, + buildNodeTaskGraph, + buildRootFileInventory: emptyRootFileInventory, + }).nodeTaskGraph(); + await createMapperContext({ + discoverNodeProjects, + buildNodeTaskGraph, + buildRootFileInventory: emptyRootFileInventory, + }).nodeTaskGraph(); expect(discoverNodeProjects).toHaveBeenCalledTimes(2); expect(buildNodeTaskGraph).toHaveBeenCalledTimes(2); }); + + it("shares one root-file inventory across concurrent mapper consumers", async () => { + const goFiles = ["fallback.go"]; + const cCppFiles = ["main.cpp"]; + const dotnetFiles = ["Program.cs"]; + const buildRootFileInventory = vi.fn( + async (): Promise => + new Map([ + ["go-fallback", goFiles], + ["c-cpp", cCppFiles], + ["dotnet", dotnetFiles], + ]), + ); + const context = createMapperContext({ + discoverNodeProjects: async () => [], + buildNodeTaskGraph: async () => emptyTaskGraph(), + buildRootFileInventory, + }); + + const results = await Promise.all([ + context.rootFiles("go-fallback"), + context.rootFiles("c-cpp"), + context.rootFiles("dotnet"), + ]); + + expect(results).toEqual([goFiles, cCppFiles, dotnetFiles]); + expect(buildRootFileInventory).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/mappers/context.ts b/src/mappers/context.ts index 98d0364c..9dfd2784 100644 --- a/src/mappers/context.ts +++ b/src/mappers/context.ts @@ -1,16 +1,20 @@ import type { NodeProjectInfo } from "./projects.js"; import type { WorkspaceTaskGraph } from "./task-graph.js"; -import type { MapperContext } from "./types.js"; +import type { MapperContext, RootFileInventory } from "./types.js"; export type MapperContextLoaders = { discoverNodeProjects(): Promise; buildNodeTaskGraph(projects: NodeProjectInfo[]): Promise; + buildRootFileInventory(): Promise; }; export function createMapperContext(loaders: MapperContextLoaders): MapperContext { const nodeProjects = memoizeAsync(loaders.discoverNodeProjects); const nodeTaskGraph = memoizeAsync(async () => loaders.buildNodeTaskGraph(await nodeProjects())); - return { nodeProjects, nodeTaskGraph }; + const rootFileInventory = memoizeAsync(loaders.buildRootFileInventory); + const rootFiles: MapperContext["rootFiles"] = async (policy) => + (await rootFileInventory()).get(policy) ?? []; + return { nodeProjects, nodeTaskGraph, rootFiles }; } function memoizeAsync(loader: () => Promise): () => Promise { diff --git a/src/mappers/dotnet.ts b/src/mappers/dotnet.ts index 348e1a4b..423d7e3d 100644 --- a/src/mappers/dotnet.ts +++ b/src/mappers/dotnet.ts @@ -3,7 +3,7 @@ import { basename, dirname, extname, join } from "node:path"; import { shellQuotePath } from "../shell.js"; import { TrustBoundary } from "../types.js"; import { partitionFileGroups } from "./grouping.js"; -import { isSampleProjectPath, normalize, pathMatchesPrefix, shouldSkip, walk } from "./shared.js"; +import { isSampleProjectPath, normalize, pathMatchesPrefix, shouldSkip } from "./shared.js"; import { FeatureSeed, MapperContext, SeedFileRef, SeedTestRef } from "./types.js"; const maxOwnedFiles = 12; @@ -32,7 +32,7 @@ type DotnetSolution = { }; export async function dotnetSeeds(root: string, context: MapperContext): Promise { - const files = await walk(root, [""], shouldSkipDotnetPath, context.vfs); + const files = await context.rootFiles("dotnet"); const fileSet = new Set(files); const solutions = await dotnetSolutions(root, files.filter(isDotnetSolutionPath)); const projectPaths = uniqueStrings([ @@ -1092,7 +1092,7 @@ function dotnetLanguageName(language: DotnetProject["language"]): string { return "C#"; } -function shouldSkipDotnetPath(path: string): boolean { +export function shouldSkipDotnetPath(path: string): boolean { if (shouldSkip(path) || isSampleProjectPath(path)) { return true; } diff --git a/src/mappers/go.ts b/src/mappers/go.ts index a5f3f690..f4cb991a 100644 --- a/src/mappers/go.ts +++ b/src/mappers/go.ts @@ -2,7 +2,7 @@ import { spawn } from "node:child_process"; import { readdir, readFile, realpath } from "node:fs/promises"; import { isAbsolute, join, relative } from "node:path"; import { pathExists } from "../fs.js"; -import { packageKind, packageTrustBoundaries, normalize, shouldSkip, walk } from "./shared.js"; +import { packageKind, packageTrustBoundaries, normalize, shouldSkip } from "./shared.js"; import { FeatureSeed, MapperContext, SeedFileRef, SeedTestRef } from "./types.js"; export async function goSeeds(root: string, context: MapperContext): Promise { @@ -76,7 +76,7 @@ async function fallbackGoPackages( context: MapperContext, ): Promise { const dirs = new Set(); - for (const file of await walk(root, [""], shouldSkip, context.vfs)) { + for (const file of await context.rootFiles("go-fallback")) { if (!file.endsWith(".go")) { continue; } diff --git a/src/mappers/shared.test.ts b/src/mappers/shared.test.ts index 59a70859..fb9cf51e 100644 --- a/src/mappers/shared.test.ts +++ b/src/mappers/shared.test.ts @@ -1,5 +1,46 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; -import { createNearbyTestFinder } from "./shared.js"; +import { createNearbyTestFinder, walkByPolicy } from "./shared.js"; + +describe("shared filesystem inventory", () => { + it("prunes each policy at directory traversal time", async () => { + const root = await mkdtemp(join(tmpdir(), "clawpatch-walk-policy-")); + try { + await Promise.all([ + mkdir(join(root, "src"), { recursive: true }), + mkdir(join(root, "vendor"), { recursive: true }), + mkdir(join(root, "obj"), { recursive: true }), + ]); + await Promise.all([ + writeFile(join(root, "src", "main.txt"), "source"), + writeFile(join(root, "vendor", "dependency.txt"), "dependency"), + writeFile(join(root, "obj", "generated.txt"), "generated"), + ]); + + const files = await walkByPolicy( + root, + [""], + [ + { key: "all", skipPath: () => false }, + { key: "no-vendor", skipPath: (path) => path === "vendor" }, + { key: "no-obj", skipPath: (path) => path === "obj" }, + ], + ); + + expect(files.get("all")).toEqual([ + "obj/generated.txt", + "src/main.txt", + "vendor/dependency.txt", + ]); + expect(files.get("no-vendor")).toEqual(["obj/generated.txt", "src/main.txt"]); + expect(files.get("no-obj")).toEqual(["src/main.txt", "vendor/dependency.txt"]); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); describe("nearby test discovery", () => { it("caches shared directory walks for one mapping run", async () => { diff --git a/src/mappers/shared.ts b/src/mappers/shared.ts index bdd0e82b..86222fa7 100644 --- a/src/mappers/shared.ts +++ b/src/mappers/shared.ts @@ -190,86 +190,116 @@ export async function walk( root: string, prefixes: string[], skipPath: (path: string) => boolean = shouldSkip, - vfs?: import("./vfs-cache.js").VfsCache, ): Promise { - const fsLstat = vfs ? vfs.fileStat : lstat; - const fsRealpath = vfs ? vfs.resolveRealpath : (p: string) => realpath(p).catch(() => p); - const files: string[] = []; - const seen = new Set(); - const seenRoots = new Set(); - const realRoot = await fsRealpath(root); + const files = await walkByPolicy(root, prefixes, [{ key: "default", skipPath }]); + return files.get("default") ?? []; +} + +export type WalkPolicy = { + key: Key; + skipPath(path: string): boolean; +}; + +type WalkPolicyState = WalkPolicy & { + files: string[]; + seen: Set; + seenRoots: Set; +}; + +export async function walkByPolicy( + root: string, + prefixes: string[], + policies: readonly WalkPolicy[], +): Promise> { + const states: WalkPolicyState[] = policies.map((policy) => ({ + ...policy, + files: [], + seen: new Set(), + seenRoots: new Set(), + })); + const realRoot = await realpath(root).catch(() => root); for (const prefix of prefixes) { const start = join(root, prefix); if (!(await pathExists(start))) { continue; } - let info = await fsLstat(start); - const canonicalStart = await fsRealpath(start); + let info = await lstat(start); + const canonicalStart = await realpath(start).catch(() => start); if (info.isSymbolicLink() && prefix !== "") { continue; } if (info.isSymbolicLink()) { - info = await fsLstat(canonicalStart).catch(() => info); + info = await lstat(canonicalStart).catch(() => info); } if (!pathInsideRoot(realRoot, canonicalStart)) { continue; } const rel = normalize(relative(realRoot, canonicalStart)); if (info.isFile()) { - if (!seen.has(rel) && !skipPath(rel)) { - seen.add(rel); - files.push(rel); + for (const state of states) { + if (!state.seen.has(rel) && !state.skipPath(rel)) { + state.seen.add(rel); + state.files.push(rel); + } } continue; } - if (!info.isDirectory() || seenRoots.has(canonicalStart)) { + if (!info.isDirectory()) { + continue; + } + const rootStates = states.filter((state) => !state.seenRoots.has(canonicalStart)); + if (rootStates.length === 0) { continue; } - seenRoots.add(canonicalStart); - await walkDir(realRoot, canonicalStart, files, seen, skipPath, vfs); + for (const state of rootStates) { + state.seenRoots.add(canonicalStart); + } + await walkDirByPolicy(realRoot, canonicalStart, rootStates); } - return files.toSorted(); + return new Map(states.map((state) => [state.key, state.files.toSorted()])); } -async function walkDir( +async function walkDirByPolicy( root: string, dir: string, - files: string[], - seen: Set, - skipPath: (path: string) => boolean, - vfs?: import("./vfs-cache.js").VfsCache, + states: WalkPolicyState[], ): Promise { - const fsLstat = vfs ? vfs.fileStat : lstat; - const fsRealpath = vfs ? vfs.resolveRealpath : (p: string) => realpath(p).catch(() => p); - const fsReaddir = vfs ? vfs.readDirectory : readdir; - const dirInfo = await fsLstat(dir); + const dirInfo = await lstat(dir); if (dirInfo.isSymbolicLink()) { return; } - const realDir = await fsRealpath(dir); + const realDir = await realpath(dir).catch(() => dir); if (!pathInsideRoot(root, realDir)) { return; } const relDir = normalize(relative(root, dir)); - if (skipPath(relDir)) { + const activeStates = states.filter((state) => !state.skipPath(relDir)); + if (activeStates.length === 0) { return; } - const entries = await fsReaddir(dir); + const entries = await readdir(dir); for (const entry of entries) { const full = join(dir, entry); const rel = normalize(relative(root, full)); - if (seen.has(rel) || skipPath(rel)) { + const entryStates = activeStates.filter( + (state) => !state.seen.has(rel) && !state.skipPath(rel), + ); + if (entryStates.length === 0) { continue; } - seen.add(rel); - const info = await fsLstat(full); + for (const state of entryStates) { + state.seen.add(rel); + } + const info = await lstat(full); if (info.isSymbolicLink()) { continue; } if (info.isDirectory()) { - await walkDir(root, full, files, seen, skipPath, vfs); + await walkDirByPolicy(root, full, entryStates); } else if (info.isFile()) { - files.push(rel); + for (const state of entryStates) { + state.files.push(rel); + } } } } diff --git a/src/mappers/types.ts b/src/mappers/types.ts index 4d5742e1..38b56d19 100644 --- a/src/mappers/types.ts +++ b/src/mappers/types.ts @@ -1,7 +1,6 @@ import { FeatureRecord, TrustBoundary } from "../types.js"; import type { NodeProjectInfo } from "./projects.js"; import type { WorkspaceTaskGraph } from "./task-graph.js"; -import type { VfsCache } from "./vfs-cache.js"; export type SeedFileRef = { path: string; @@ -42,8 +41,11 @@ export type FeatureMapper = { map(root: string, context: MapperContext): Promise; }; +export type RootFilePolicy = "go-fallback" | "c-cpp" | "dotnet"; +export type RootFileInventory = Map; + export type MapperContext = { nodeProjects(): Promise; nodeTaskGraph(): Promise; - vfs: VfsCache; + rootFiles(policy: RootFilePolicy): Promise; }; diff --git a/src/mappers/vfs-cache.test.ts b/src/mappers/vfs-cache.test.ts deleted file mode 100644 index afbdd858..00000000 --- a/src/mappers/vfs-cache.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { readdir } from "node:fs/promises"; -import { beforeEach, describe, expect, it, vi } from "vitest"; -import { createVfsCache } from "./vfs-cache.js"; - -vi.mock("node:fs/promises", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - readdir: vi.fn(), - }; -}); - -describe("VfsCache", () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it("memoizes successful readdir calls", async () => { - vi.mocked(readdir).mockResolvedValueOnce(["file1.txt", "file2.txt"] as any); - const vfs = createVfsCache(); - - const first = await vfs.readDirectory("/fake/path"); - const second = await vfs.readDirectory("/fake/path"); - - expect(first).toEqual(["file1.txt", "file2.txt"]); - expect(second).toEqual(["file1.txt", "file2.txt"]); - expect(readdir).toHaveBeenCalledTimes(1); - expect(readdir).toHaveBeenCalledWith("/fake/path"); - }); - - it("preserves and memoizes directory-read failures", async () => { - const error = new Error("EACCES: permission denied, scandir '/fake/secret'"); - vi.mocked(readdir).mockRejectedValueOnce(error); - const vfs = createVfsCache(); - - await expect(vfs.readDirectory("/fake/secret")).rejects.toThrow(error); - await expect(vfs.readDirectory("/fake/secret")).rejects.toThrow(error); - - expect(readdir).toHaveBeenCalledTimes(1); - expect(readdir).toHaveBeenCalledWith("/fake/secret"); - }); -}); diff --git a/src/mappers/vfs-cache.ts b/src/mappers/vfs-cache.ts deleted file mode 100644 index 9f1c9993..00000000 --- a/src/mappers/vfs-cache.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { lstat, readdir, realpath } from "node:fs/promises"; -import type { Stats } from "node:fs"; - -export type VfsCache = { - readDirectory(path: string): Promise; - fileStat(path: string): Promise; - resolveRealpath(path: string): Promise; -}; - -export function createVfsCache(): VfsCache { - const dirCache = new Map>(); - const statCache = new Map>(); - const realpathCache = new Map>(); - - return { - readDirectory(path: string): Promise { - let cached = dirCache.get(path); - if (cached === undefined) { - cached = readdir(path); - dirCache.set(path, cached); - } - return cached; - }, - - fileStat(path: string): Promise { - let cached = statCache.get(path); - if (cached === undefined) { - cached = lstat(path); - statCache.set(path, cached); - } - return cached; - }, - - resolveRealpath(path: string): Promise { - let cached = realpathCache.get(path); - if (cached === undefined) { - cached = realpath(path).catch(() => path); - realpathCache.set(path, cached); - } - return cached; - }, - }; -}