diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fa9a87..821869a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 0.7.1 - Unreleased +- Reduced mapper startup I/O by sharing one root file inventory across Go fallback, C/C++, and .NET mapping, thanks @Tanmay-008. - Fixed revalidation prompts to compact historical and feature metadata and hard-cap metadata lists even when configured file limits are high, preventing provider input overflows, thanks @pai-scaffolde. - Added an opt-in Claude host auth context that preserves the default-deny environment, uses Claude Code safe mode, validates auth through doctor, and reports redacted OAuth failure signals, thanks @grantjayy. diff --git a/src/mapper.ts b/src/mapper.ts index dcaf8fb..4d7c3ac 100644 --- a/src/mapper.ts +++ b/src/mapper.ts @@ -5,9 +5,9 @@ import { seedIdentityParts, stableFeatureJson, } from "./mapper-reconciliation.js"; -import { cCppSeeds } from "./mappers/c-cpp.js"; +import { cCppSeeds, shouldSkipCOrCppPath } from "./mappers/c-cpp.js"; import { configSeeds } from "./mappers/config.js"; -import { dotnetSeeds } from "./mappers/dotnet.js"; +import { dotnetSeeds, shouldSkipDotnetPath } from "./mappers/dotnet.js"; import { elixirSeeds } from "./mappers/elixir.js"; import { goSeeds } from "./mappers/go.js"; import { appleSeeds } from "./mappers/apple.js"; @@ -23,7 +23,13 @@ import { createMapperContext } from "./mappers/context.js"; import { discoverNodeProjects, hasFallbackNodeProjectSignal } from "./mappers/projects.js"; import { rubySeeds } from "./mappers/ruby.js"; import { rustSeeds } from "./mappers/rust.js"; -import { createNearbyTestFinder, PathFilters, pathMatchesFilters } from "./mappers/shared.js"; +import { + createNearbyTestFinder, + PathFilters, + pathMatchesFilters, + shouldSkip, + walkByPolicy, +} from "./mappers/shared.js"; import { swiftSeeds } from "./mappers/swift.js"; import { turboTaskGraph } from "./mappers/turbo.js"; import { FeatureMapper, FeatureSeed, MapperContext } from "./mappers/types.js"; @@ -294,6 +300,16 @@ async function collectSeeds( const context: MapperContext = createMapperContext({ discoverNodeProjects: () => discoverNodeProjects(root), buildNodeTaskGraph: (projects) => turboTaskGraph(root, projects), + buildRootFileInventory: () => + walkByPolicy( + root, + [""], + [ + { key: "go-fallback", skipPath: shouldSkip }, + { key: "c-cpp", skipPath: shouldSkipCOrCppPath }, + { key: "dotnet", skipPath: shouldSkipDotnetPath }, + ], + ), }); const runNodeMappers = shouldRunNodeMappers(root, project); const groups = await Promise.all( diff --git a/src/mappers/c-cpp.ts b/src/mappers/c-cpp.ts index 7dca5a0..aff6218 100644 --- a/src/mappers/c-cpp.ts +++ b/src/mappers/c-cpp.ts @@ -11,14 +11,13 @@ import { shouldSkip, stripLineComments, targetLanguageTag, - walk, withCudaConcurrency, } from "./shared.js"; import { cCppGroupSeeds } from "./c-cpp-groups.js"; -import { FeatureSeed, SeedFileRef } from "./types.js"; +import { FeatureSeed, MapperContext, SeedFileRef } from "./types.js"; -export async function cCppSeeds(root: string): Promise { - const files = (await walk(root, [""], shouldSkipCOrCppPath)).filter( +export async function cCppSeeds(root: string, context: MapperContext): Promise { + const files = (await context.rootFiles("c-cpp")).filter( (path) => !isSampleProjectPath(path) && (isCOrCppSource(path) || isMakefile(path) || isCMake(path)), ); @@ -1107,7 +1106,7 @@ function isCOrCppDependencyPath(path: string): boolean { return /(^|\/)(deps|vendor|CMakeFiles|cmake-build-[^/]+)(\/|$)/u.test(path); } -function shouldSkipCOrCppPath(path: string): boolean { +export function shouldSkipCOrCppPath(path: string): boolean { return shouldSkip(path) || isCOrCppDependencyPath(path); } diff --git a/src/mappers/context.test.ts b/src/mappers/context.test.ts index 9abaf15..1ebda0b 100644 --- a/src/mappers/context.test.ts +++ b/src/mappers/context.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it, vi } from "vitest"; import { createMapperContext } from "./context.js"; import { emptyTaskGraph } from "./task-graph.js"; +import type { RootFileInventory } from "./types.js"; + +const emptyRootFileInventory = async (): Promise => new Map(); describe("createMapperContext", () => { it("shares concurrent first access across all Node consumers", async () => { @@ -8,7 +11,11 @@ describe("createMapperContext", () => { const graph = emptyTaskGraph(); const discoverNodeProjects = vi.fn(async () => projects); const buildNodeTaskGraph = vi.fn(async () => graph); - const context = createMapperContext({ discoverNodeProjects, buildNodeTaskGraph }); + const context = createMapperContext({ + discoverNodeProjects, + buildNodeTaskGraph, + buildRootFileInventory: emptyRootFileInventory, + }); const results = await Promise.all([ context.nodeProjects(), @@ -29,7 +36,11 @@ describe("createMapperContext", () => { throw failure; }); const buildNodeTaskGraph = vi.fn(async () => emptyTaskGraph()); - const context = createMapperContext({ discoverNodeProjects, buildNodeTaskGraph }); + const context = createMapperContext({ + discoverNodeProjects, + buildNodeTaskGraph, + buildRootFileInventory: emptyRootFileInventory, + }); const results = await Promise.allSettled([ context.nodeProjects(), @@ -52,7 +63,11 @@ describe("createMapperContext", () => { const buildNodeTaskGraph = vi.fn(async () => { throw failure; }); - const context = createMapperContext({ discoverNodeProjects, buildNodeTaskGraph }); + const context = createMapperContext({ + discoverNodeProjects, + buildNodeTaskGraph, + buildRootFileInventory: emptyRootFileInventory, + }); const results = await Promise.allSettled([context.nodeTaskGraph(), context.nodeTaskGraph()]); @@ -68,10 +83,46 @@ describe("createMapperContext", () => { const discoverNodeProjects = vi.fn(async () => []); const buildNodeTaskGraph = vi.fn(async () => emptyTaskGraph()); - await createMapperContext({ discoverNodeProjects, buildNodeTaskGraph }).nodeTaskGraph(); - await createMapperContext({ discoverNodeProjects, buildNodeTaskGraph }).nodeTaskGraph(); + await createMapperContext({ + discoverNodeProjects, + buildNodeTaskGraph, + buildRootFileInventory: emptyRootFileInventory, + }).nodeTaskGraph(); + await createMapperContext({ + discoverNodeProjects, + buildNodeTaskGraph, + buildRootFileInventory: emptyRootFileInventory, + }).nodeTaskGraph(); expect(discoverNodeProjects).toHaveBeenCalledTimes(2); expect(buildNodeTaskGraph).toHaveBeenCalledTimes(2); }); + + it("shares one root-file inventory across concurrent mapper consumers", async () => { + const goFiles = ["fallback.go"]; + const cCppFiles = ["main.cpp"]; + const dotnetFiles = ["Program.cs"]; + const buildRootFileInventory = vi.fn( + async (): Promise => + new Map([ + ["go-fallback", goFiles], + ["c-cpp", cCppFiles], + ["dotnet", dotnetFiles], + ]), + ); + const context = createMapperContext({ + discoverNodeProjects: async () => [], + buildNodeTaskGraph: async () => emptyTaskGraph(), + buildRootFileInventory, + }); + + const results = await Promise.all([ + context.rootFiles("go-fallback"), + context.rootFiles("c-cpp"), + context.rootFiles("dotnet"), + ]); + + expect(results).toEqual([goFiles, cCppFiles, dotnetFiles]); + expect(buildRootFileInventory).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/mappers/context.ts b/src/mappers/context.ts index 98d0364..9dfd278 100644 --- a/src/mappers/context.ts +++ b/src/mappers/context.ts @@ -1,16 +1,20 @@ import type { NodeProjectInfo } from "./projects.js"; import type { WorkspaceTaskGraph } from "./task-graph.js"; -import type { MapperContext } from "./types.js"; +import type { MapperContext, RootFileInventory } from "./types.js"; export type MapperContextLoaders = { discoverNodeProjects(): Promise; buildNodeTaskGraph(projects: NodeProjectInfo[]): Promise; + buildRootFileInventory(): Promise; }; export function createMapperContext(loaders: MapperContextLoaders): MapperContext { const nodeProjects = memoizeAsync(loaders.discoverNodeProjects); const nodeTaskGraph = memoizeAsync(async () => loaders.buildNodeTaskGraph(await nodeProjects())); - return { nodeProjects, nodeTaskGraph }; + const rootFileInventory = memoizeAsync(loaders.buildRootFileInventory); + const rootFiles: MapperContext["rootFiles"] = async (policy) => + (await rootFileInventory()).get(policy) ?? []; + return { nodeProjects, nodeTaskGraph, rootFiles }; } function memoizeAsync(loader: () => Promise): () => Promise { diff --git a/src/mappers/dotnet.ts b/src/mappers/dotnet.ts index 5f75d9c..423d7e3 100644 --- a/src/mappers/dotnet.ts +++ b/src/mappers/dotnet.ts @@ -3,8 +3,8 @@ import { basename, dirname, extname, join } from "node:path"; import { shellQuotePath } from "../shell.js"; import { TrustBoundary } from "../types.js"; import { partitionFileGroups } from "./grouping.js"; -import { isSampleProjectPath, normalize, pathMatchesPrefix, shouldSkip, walk } from "./shared.js"; -import { FeatureSeed, SeedFileRef, SeedTestRef } from "./types.js"; +import { isSampleProjectPath, normalize, pathMatchesPrefix, shouldSkip } from "./shared.js"; +import { FeatureSeed, MapperContext, SeedFileRef, SeedTestRef } from "./types.js"; const maxOwnedFiles = 12; const maxTests = 8; @@ -31,8 +31,8 @@ type DotnetSolution = { projectPaths: string[]; }; -export async function dotnetSeeds(root: string): Promise { - const files = await walk(root, [""], shouldSkipDotnetPath); +export async function dotnetSeeds(root: string, context: MapperContext): Promise { + const files = await context.rootFiles("dotnet"); const fileSet = new Set(files); const solutions = await dotnetSolutions(root, files.filter(isDotnetSolutionPath)); const projectPaths = uniqueStrings([ @@ -1092,7 +1092,7 @@ function dotnetLanguageName(language: DotnetProject["language"]): string { return "C#"; } -function shouldSkipDotnetPath(path: string): boolean { +export function shouldSkipDotnetPath(path: string): boolean { if (shouldSkip(path) || isSampleProjectPath(path)) { return true; } diff --git a/src/mappers/go.ts b/src/mappers/go.ts index 8fc2593..f4cb991 100644 --- a/src/mappers/go.ts +++ b/src/mappers/go.ts @@ -2,15 +2,15 @@ import { spawn } from "node:child_process"; import { readdir, readFile, realpath } from "node:fs/promises"; import { isAbsolute, join, relative } from "node:path"; import { pathExists } from "../fs.js"; -import { packageKind, packageTrustBoundaries, normalize, shouldSkip, walk } from "./shared.js"; -import { FeatureSeed, SeedFileRef, SeedTestRef } from "./types.js"; +import { packageKind, packageTrustBoundaries, normalize, shouldSkip } from "./shared.js"; +import { FeatureSeed, MapperContext, SeedFileRef, SeedTestRef } from "./types.js"; -export async function goSeeds(root: string): Promise { +export async function goSeeds(root: string, context: MapperContext): Promise { if (!(await pathExists(join(root, "go.mod")))) { return []; } const modulePath = await goModulePath(root); - const packages = await goPackages(root, modulePath); + const packages = await goPackages(root, modulePath, context); const packageByImport = new Map(packages.map((pkg) => [pkg.importPath, pkg])); const seeds: FeatureSeed[] = []; for (const pkg of packages) { @@ -36,12 +36,16 @@ type GoPackageFiles = { generated: string[]; }; -async function goPackages(root: string, modulePath: string | null): Promise { +async function goPackages( + root: string, + modulePath: string | null, + context: MapperContext, +): Promise { const listed = await goListPackages(root); if (listed.length > 0) { return listed; } - return fallbackGoPackages(root, modulePath); + return fallbackGoPackages(root, modulePath, context); } async function goListPackages(root: string): Promise { @@ -66,9 +70,13 @@ async function goListPackages(root: string): Promise { return packages; } -async function fallbackGoPackages(root: string, modulePath: string | null): Promise { +async function fallbackGoPackages( + root: string, + modulePath: string | null, + context: MapperContext, +): Promise { const dirs = new Set(); - for (const file of await walk(root, [""])) { + for (const file of await context.rootFiles("go-fallback")) { if (!file.endsWith(".go")) { continue; } diff --git a/src/mappers/shared.test.ts b/src/mappers/shared.test.ts index 59a7085..fb9cf51 100644 --- a/src/mappers/shared.test.ts +++ b/src/mappers/shared.test.ts @@ -1,5 +1,46 @@ +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; -import { createNearbyTestFinder } from "./shared.js"; +import { createNearbyTestFinder, walkByPolicy } from "./shared.js"; + +describe("shared filesystem inventory", () => { + it("prunes each policy at directory traversal time", async () => { + const root = await mkdtemp(join(tmpdir(), "clawpatch-walk-policy-")); + try { + await Promise.all([ + mkdir(join(root, "src"), { recursive: true }), + mkdir(join(root, "vendor"), { recursive: true }), + mkdir(join(root, "obj"), { recursive: true }), + ]); + await Promise.all([ + writeFile(join(root, "src", "main.txt"), "source"), + writeFile(join(root, "vendor", "dependency.txt"), "dependency"), + writeFile(join(root, "obj", "generated.txt"), "generated"), + ]); + + const files = await walkByPolicy( + root, + [""], + [ + { key: "all", skipPath: () => false }, + { key: "no-vendor", skipPath: (path) => path === "vendor" }, + { key: "no-obj", skipPath: (path) => path === "obj" }, + ], + ); + + expect(files.get("all")).toEqual([ + "obj/generated.txt", + "src/main.txt", + "vendor/dependency.txt", + ]); + expect(files.get("no-vendor")).toEqual(["obj/generated.txt", "src/main.txt"]); + expect(files.get("no-obj")).toEqual(["src/main.txt", "vendor/dependency.txt"]); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); describe("nearby test discovery", () => { it("caches shared directory walks for one mapping run", async () => { diff --git a/src/mappers/shared.ts b/src/mappers/shared.ts index 08b69c7..86222fa 100644 --- a/src/mappers/shared.ts +++ b/src/mappers/shared.ts @@ -191,9 +191,32 @@ export async function walk( prefixes: string[], skipPath: (path: string) => boolean = shouldSkip, ): Promise { - const files: string[] = []; - const seen = new Set(); - const seenRoots = new Set(); + const files = await walkByPolicy(root, prefixes, [{ key: "default", skipPath }]); + return files.get("default") ?? []; +} + +export type WalkPolicy = { + key: Key; + skipPath(path: string): boolean; +}; + +type WalkPolicyState = WalkPolicy & { + files: string[]; + seen: Set; + seenRoots: Set; +}; + +export async function walkByPolicy( + root: string, + prefixes: string[], + policies: readonly WalkPolicy[], +): Promise> { + const states: WalkPolicyState[] = policies.map((policy) => ({ + ...policy, + files: [], + seen: new Set(), + seenRoots: new Set(), + })); const realRoot = await realpath(root).catch(() => root); for (const prefix of prefixes) { const start = join(root, prefix); @@ -213,27 +236,33 @@ export async function walk( } const rel = normalize(relative(realRoot, canonicalStart)); if (info.isFile()) { - if (!seen.has(rel) && !skipPath(rel)) { - seen.add(rel); - files.push(rel); + for (const state of states) { + if (!state.seen.has(rel) && !state.skipPath(rel)) { + state.seen.add(rel); + state.files.push(rel); + } } continue; } - if (!info.isDirectory() || seenRoots.has(canonicalStart)) { + if (!info.isDirectory()) { + continue; + } + const rootStates = states.filter((state) => !state.seenRoots.has(canonicalStart)); + if (rootStates.length === 0) { continue; } - seenRoots.add(canonicalStart); - await walkDir(realRoot, canonicalStart, files, seen, skipPath); + for (const state of rootStates) { + state.seenRoots.add(canonicalStart); + } + await walkDirByPolicy(realRoot, canonicalStart, rootStates); } - return files.toSorted(); + return new Map(states.map((state) => [state.key, state.files.toSorted()])); } -async function walkDir( +async function walkDirByPolicy( root: string, dir: string, - files: string[], - seen: Set, - skipPath: (path: string) => boolean, + states: WalkPolicyState[], ): Promise { const dirInfo = await lstat(dir); if (dirInfo.isSymbolicLink()) { @@ -244,25 +273,33 @@ async function walkDir( return; } const relDir = normalize(relative(root, dir)); - if (skipPath(relDir)) { + const activeStates = states.filter((state) => !state.skipPath(relDir)); + if (activeStates.length === 0) { return; } const entries = await readdir(dir); for (const entry of entries) { const full = join(dir, entry); const rel = normalize(relative(root, full)); - if (seen.has(rel) || skipPath(rel)) { + const entryStates = activeStates.filter( + (state) => !state.seen.has(rel) && !state.skipPath(rel), + ); + if (entryStates.length === 0) { continue; } - seen.add(rel); + for (const state of entryStates) { + state.seen.add(rel); + } const info = await lstat(full); if (info.isSymbolicLink()) { continue; } if (info.isDirectory()) { - await walkDir(root, full, files, seen, skipPath); + await walkDirByPolicy(root, full, entryStates); } else if (info.isFile()) { - files.push(rel); + for (const state of entryStates) { + state.files.push(rel); + } } } } diff --git a/src/mappers/types.ts b/src/mappers/types.ts index cdaf578..38b56d1 100644 --- a/src/mappers/types.ts +++ b/src/mappers/types.ts @@ -41,7 +41,11 @@ export type FeatureMapper = { map(root: string, context: MapperContext): Promise; }; +export type RootFilePolicy = "go-fallback" | "c-cpp" | "dotnet"; +export type RootFileInventory = Map; + export type MapperContext = { nodeProjects(): Promise; nodeTaskGraph(): Promise; + rootFiles(policy: RootFilePolicy): Promise; };