perf(mapper): cache full-root filesystem scans across concurrent mappers

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## 0.7.1 - Unreleased
 
+- Reduced mapper startup I/O by sharing one root file inventory across Go fallback, C/C++, and .NET mapping, thanks @Tanmay-008.
 - Fixed revalidation prompts to compact historical and feature metadata and hard-cap metadata lists even when configured file limits are high, preventing provider input overflows, thanks @pai-scaffolde.
 - Added an opt-in Claude host auth context that preserves the default-deny environment, uses Claude Code safe mode, validates auth through doctor, and reports redacted OAuth failure signals, thanks @grantjayy.
 

src/mapper.ts

@@ -5,9 +5,9 @@ import {
   seedIdentityParts,
   stableFeatureJson,
 } from "./mapper-reconciliation.js";
-import { cCppSeeds } from "./mappers/c-cpp.js";
+import { cCppSeeds, shouldSkipCOrCppPath } from "./mappers/c-cpp.js";
 import { configSeeds } from "./mappers/config.js";
-import { dotnetSeeds } from "./mappers/dotnet.js";
+import { dotnetSeeds, shouldSkipDotnetPath } from "./mappers/dotnet.js";
 import { elixirSeeds } from "./mappers/elixir.js";
 import { goSeeds } from "./mappers/go.js";
 import { appleSeeds } from "./mappers/apple.js";
@@ -23,7 +23,13 @@ import { createMapperContext } from "./mappers/context.js";
 import { discoverNodeProjects, hasFallbackNodeProjectSignal } from "./mappers/projects.js";
 import { rubySeeds } from "./mappers/ruby.js";
 import { rustSeeds } from "./mappers/rust.js";
-import { createNearbyTestFinder, PathFilters, pathMatchesFilters } from "./mappers/shared.js";
+import {
+  createNearbyTestFinder,
+  PathFilters,
+  pathMatchesFilters,
+  shouldSkip,
+  walkByPolicy,
+} from "./mappers/shared.js";
 import { swiftSeeds } from "./mappers/swift.js";
 import { turboTaskGraph } from "./mappers/turbo.js";
 import { FeatureMapper, FeatureSeed, MapperContext } from "./mappers/types.js";
@@ -294,6 +300,16 @@ async function collectSeeds(
   const context: MapperContext = createMapperContext({
     discoverNodeProjects: () => discoverNodeProjects(root),
     buildNodeTaskGraph: (projects) => turboTaskGraph(root, projects),
+    buildRootFileInventory: () =>
+      walkByPolicy(
+        root,
+        [""],
+        [
+          { key: "go-fallback", skipPath: shouldSkip },
+          { key: "c-cpp", skipPath: shouldSkipCOrCppPath },
+          { key: "dotnet", skipPath: shouldSkipDotnetPath },
+        ],
+      ),
   });
   const runNodeMappers = shouldRunNodeMappers(root, project);
   const groups = await Promise.all(

src/mappers/c-cpp.ts

@@ -11,14 +11,13 @@ import {
   shouldSkip,
   stripLineComments,
   targetLanguageTag,
-  walk,
   withCudaConcurrency,
 } from "./shared.js";
 import { cCppGroupSeeds } from "./c-cpp-groups.js";
-import { FeatureSeed, SeedFileRef } from "./types.js";
+import { FeatureSeed, MapperContext, SeedFileRef } from "./types.js";
 
-export async function cCppSeeds(root: string): Promise<FeatureSeed[]> {
-  const files = (await walk(root, [""], shouldSkipCOrCppPath)).filter(
+export async function cCppSeeds(root: string, context: MapperContext): Promise<FeatureSeed[]> {
+  const files = (await context.rootFiles("c-cpp")).filter(
     (path) =>
       !isSampleProjectPath(path) && (isCOrCppSource(path) || isMakefile(path) || isCMake(path)),
   );
@@ -1107,7 +1106,7 @@ function isCOrCppDependencyPath(path: string): boolean {
   return /(^|\/)(deps|vendor|CMakeFiles|cmake-build-[^/]+)(\/|$)/u.test(path);
 }
 
-function shouldSkipCOrCppPath(path: string): boolean {
+export function shouldSkipCOrCppPath(path: string): boolean {
   return shouldSkip(path) || isCOrCppDependencyPath(path);
 }
 

src/mappers/context.test.ts

@@ -1,14 +1,21 @@
 import { describe, expect, it, vi } from "vitest";
 import { createMapperContext } from "./context.js";
 import { emptyTaskGraph } from "./task-graph.js";
+import type { RootFileInventory } from "./types.js";
+
+const emptyRootFileInventory = async (): Promise<RootFileInventory> => new Map();
 
 describe("createMapperContext", () => {
   it("shares concurrent first access across all Node consumers", async () => {
     const projects: [] = [];
     const graph = emptyTaskGraph();
     const discoverNodeProjects = vi.fn(async () => projects);
     const buildNodeTaskGraph = vi.fn(async () => graph);
-    const context = createMapperContext({ discoverNodeProjects, buildNodeTaskGraph });
+    const context = createMapperContext({
+      discoverNodeProjects,
+      buildNodeTaskGraph,
+      buildRootFileInventory: emptyRootFileInventory,
+    });
 
     const results = await Promise.all([
       context.nodeProjects(),
@@ -29,7 +36,11 @@ describe("createMapperContext", () => {
       throw failure;
     });
     const buildNodeTaskGraph = vi.fn(async () => emptyTaskGraph());
-    const context = createMapperContext({ discoverNodeProjects, buildNodeTaskGraph });
+    const context = createMapperContext({
+      discoverNodeProjects,
+      buildNodeTaskGraph,
+      buildRootFileInventory: emptyRootFileInventory,
+    });
 
     const results = await Promise.allSettled([
       context.nodeProjects(),
@@ -52,7 +63,11 @@ describe("createMapperContext", () => {
     const buildNodeTaskGraph = vi.fn(async () => {
       throw failure;
     });
-    const context = createMapperContext({ discoverNodeProjects, buildNodeTaskGraph });
+    const context = createMapperContext({
+      discoverNodeProjects,
+      buildNodeTaskGraph,
+      buildRootFileInventory: emptyRootFileInventory,
+    });
 
     const results = await Promise.allSettled([context.nodeTaskGraph(), context.nodeTaskGraph()]);
 
@@ -68,10 +83,46 @@ describe("createMapperContext", () => {
     const discoverNodeProjects = vi.fn(async () => []);
     const buildNodeTaskGraph = vi.fn(async () => emptyTaskGraph());
 
-    await createMapperContext({ discoverNodeProjects, buildNodeTaskGraph }).nodeTaskGraph();
-    await createMapperContext({ discoverNodeProjects, buildNodeTaskGraph }).nodeTaskGraph();
+    await createMapperContext({
+      discoverNodeProjects,
+      buildNodeTaskGraph,
+      buildRootFileInventory: emptyRootFileInventory,
+    }).nodeTaskGraph();
+    await createMapperContext({
+      discoverNodeProjects,
+      buildNodeTaskGraph,
+      buildRootFileInventory: emptyRootFileInventory,
+    }).nodeTaskGraph();
 
     expect(discoverNodeProjects).toHaveBeenCalledTimes(2);
     expect(buildNodeTaskGraph).toHaveBeenCalledTimes(2);
   });
+
+  it("shares one root-file inventory across concurrent mapper consumers", async () => {
+    const goFiles = ["fallback.go"];
+    const cCppFiles = ["main.cpp"];
+    const dotnetFiles = ["Program.cs"];
+    const buildRootFileInventory = vi.fn(
+      async (): Promise<RootFileInventory> =>
+        new Map([
+          ["go-fallback", goFiles],
+          ["c-cpp", cCppFiles],
+          ["dotnet", dotnetFiles],
+        ]),
+    );
+    const context = createMapperContext({
+      discoverNodeProjects: async () => [],
+      buildNodeTaskGraph: async () => emptyTaskGraph(),
+      buildRootFileInventory,
+    });
+
+    const results = await Promise.all([
+      context.rootFiles("go-fallback"),
+      context.rootFiles("c-cpp"),
+      context.rootFiles("dotnet"),
+    ]);
+
+    expect(results).toEqual([goFiles, cCppFiles, dotnetFiles]);
+    expect(buildRootFileInventory).toHaveBeenCalledTimes(1);
+  });
 });

src/mappers/context.ts

@@ -1,16 +1,20 @@
 import type { NodeProjectInfo } from "./projects.js";
 import type { WorkspaceTaskGraph } from "./task-graph.js";
-import type { MapperContext } from "./types.js";
+import type { MapperContext, RootFileInventory } from "./types.js";
 
 export type MapperContextLoaders = {
   discoverNodeProjects(): Promise<NodeProjectInfo[]>;
   buildNodeTaskGraph(projects: NodeProjectInfo[]): Promise<WorkspaceTaskGraph>;
+  buildRootFileInventory(): Promise<RootFileInventory>;
 };
 
 export function createMapperContext(loaders: MapperContextLoaders): MapperContext {
   const nodeProjects = memoizeAsync(loaders.discoverNodeProjects);
   const nodeTaskGraph = memoizeAsync(async () => loaders.buildNodeTaskGraph(await nodeProjects()));
-  return { nodeProjects, nodeTaskGraph };
+  const rootFileInventory = memoizeAsync(loaders.buildRootFileInventory);
+  const rootFiles: MapperContext["rootFiles"] = async (policy) =>
+    (await rootFileInventory()).get(policy) ?? [];
+  return { nodeProjects, nodeTaskGraph, rootFiles };
 }
 
 function memoizeAsync<T>(loader: () => Promise<T>): () => Promise<T> {

src/mappers/dotnet.ts

@@ -3,8 +3,8 @@ import { basename, dirname, extname, join } from "node:path";
 import { shellQuotePath } from "../shell.js";
 import { TrustBoundary } from "../types.js";
 import { partitionFileGroups } from "./grouping.js";
-import { isSampleProjectPath, normalize, pathMatchesPrefix, shouldSkip, walk } from "./shared.js";
-import { FeatureSeed, SeedFileRef, SeedTestRef } from "./types.js";
+import { isSampleProjectPath, normalize, pathMatchesPrefix, shouldSkip } from "./shared.js";
+import { FeatureSeed, MapperContext, SeedFileRef, SeedTestRef } from "./types.js";
 
 const maxOwnedFiles = 12;
 const maxTests = 8;
@@ -31,8 +31,8 @@ type DotnetSolution = {
   projectPaths: string[];
 };
 
-export async function dotnetSeeds(root: string): Promise<FeatureSeed[]> {
-  const files = await walk(root, [""], shouldSkipDotnetPath);
+export async function dotnetSeeds(root: string, context: MapperContext): Promise<FeatureSeed[]> {
+  const files = await context.rootFiles("dotnet");
   const fileSet = new Set(files);
   const solutions = await dotnetSolutions(root, files.filter(isDotnetSolutionPath));
   const projectPaths = uniqueStrings([
@@ -1092,7 +1092,7 @@ function dotnetLanguageName(language: DotnetProject["language"]): string {
   return "C#";
 }
 
-function shouldSkipDotnetPath(path: string): boolean {
+export function shouldSkipDotnetPath(path: string): boolean {
   if (shouldSkip(path) || isSampleProjectPath(path)) {
     return true;
   }

src/mappers/go.ts

@@ -2,15 +2,15 @@ import { spawn } from "node:child_process";
 import { readdir, readFile, realpath } from "node:fs/promises";
 import { isAbsolute, join, relative } from "node:path";
 import { pathExists } from "../fs.js";
-import { packageKind, packageTrustBoundaries, normalize, shouldSkip, walk } from "./shared.js";
-import { FeatureSeed, SeedFileRef, SeedTestRef } from "./types.js";
+import { packageKind, packageTrustBoundaries, normalize, shouldSkip } from "./shared.js";
+import { FeatureSeed, MapperContext, SeedFileRef, SeedTestRef } from "./types.js";
 
-export async function goSeeds(root: string): Promise<FeatureSeed[]> {
+export async function goSeeds(root: string, context: MapperContext): Promise<FeatureSeed[]> {
   if (!(await pathExists(join(root, "go.mod")))) {
     return [];
   }
   const modulePath = await goModulePath(root);
-  const packages = await goPackages(root, modulePath);
+  const packages = await goPackages(root, modulePath, context);
   const packageByImport = new Map(packages.map((pkg) => [pkg.importPath, pkg]));
   const seeds: FeatureSeed[] = [];
   for (const pkg of packages) {
@@ -36,12 +36,16 @@ type GoPackageFiles = {
   generated: string[];
 };
 
-async function goPackages(root: string, modulePath: string | null): Promise<GoPackage[]> {
+async function goPackages(
+  root: string,
+  modulePath: string | null,
+  context: MapperContext,
+): Promise<GoPackage[]> {
   const listed = await goListPackages(root);
   if (listed.length > 0) {
     return listed;
   }
-  return fallbackGoPackages(root, modulePath);
+  return fallbackGoPackages(root, modulePath, context);
 }
 
 async function goListPackages(root: string): Promise<GoPackage[]> {
@@ -66,9 +70,13 @@ async function goListPackages(root: string): Promise<GoPackage[]> {
   return packages;
 }
 
-async function fallbackGoPackages(root: string, modulePath: string | null): Promise<GoPackage[]> {
+async function fallbackGoPackages(
+  root: string,
+  modulePath: string | null,
+  context: MapperContext,
+): Promise<GoPackage[]> {
   const dirs = new Set<string>();
-  for (const file of await walk(root, [""])) {
+  for (const file of await context.rootFiles("go-fallback")) {
     if (!file.endsWith(".go")) {
       continue;
     }

src/mappers/shared.test.ts

@@ -1,5 +1,46 @@
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { describe, expect, it, vi } from "vitest";
-import { createNearbyTestFinder } from "./shared.js";
+import { createNearbyTestFinder, walkByPolicy } from "./shared.js";
+
+describe("shared filesystem inventory", () => {
+  it("prunes each policy at directory traversal time", async () => {
+    const root = await mkdtemp(join(tmpdir(), "clawpatch-walk-policy-"));
+    try {
+      await Promise.all([
+        mkdir(join(root, "src"), { recursive: true }),
+        mkdir(join(root, "vendor"), { recursive: true }),
+        mkdir(join(root, "obj"), { recursive: true }),
+      ]);
+      await Promise.all([
+        writeFile(join(root, "src", "main.txt"), "source"),
+        writeFile(join(root, "vendor", "dependency.txt"), "dependency"),
+        writeFile(join(root, "obj", "generated.txt"), "generated"),
+      ]);
+
+      const files = await walkByPolicy(
+        root,
+        [""],
+        [
+          { key: "all", skipPath: () => false },
+          { key: "no-vendor", skipPath: (path) => path === "vendor" },
+          { key: "no-obj", skipPath: (path) => path === "obj" },
+        ],
+      );
+
+      expect(files.get("all")).toEqual([
+        "obj/generated.txt",
+        "src/main.txt",
+        "vendor/dependency.txt",
+      ]);
+      expect(files.get("no-vendor")).toEqual(["obj/generated.txt", "src/main.txt"]);
+      expect(files.get("no-obj")).toEqual(["src/main.txt", "vendor/dependency.txt"]);
+    } finally {
+      await rm(root, { recursive: true, force: true });
+    }
+  });
+});
 
 describe("nearby test discovery", () => {
   it("caches shared directory walks for one mapping run", async () => {