From ce1eaa0e91e8079aef8c6f56a9babc8388e96f10 Mon Sep 17 00:00:00 2001 From: Yannic Charlon <52761674+JustYannicc@users.noreply.github.com> Date: Fri, 3 Jul 2026 21:21:59 +0200 Subject: [PATCH] Add file search perf harness --- package.json | 1 + scripts/file-search-perf-harness.mjs | 478 ++++++++++++++++++++++ scripts/test-file-search-perf-harness.mjs | 52 +++ src/main/file-search-index.ts | 54 +++ 4 files changed, 585 insertions(+) create mode 100644 scripts/file-search-perf-harness.mjs create mode 100644 scripts/test-file-search-perf-harness.mjs diff --git a/package.json b/package.json index f262a8c1..11a1701d 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "build:main": "tsc -p tsconfig.main.json && cp src/main/emoji-data.json dist/main/emoji-data.json", "build:renderer": "vite build", "check:i18n": "node scripts/check-i18n.mjs", + "perf:file-search": "node scripts/file-search-perf-harness.mjs", "test": "node --test 'scripts/test-*.mjs'", "build:native": "node scripts/build-native.mjs", "ensure:cross-arch-esbuild": "node scripts/ensure-cross-arch-esbuild.mjs", diff --git a/scripts/file-search-perf-harness.mjs b/scripts/file-search-perf-harness.mjs new file mode 100644 index 00000000..904e5564 --- /dev/null +++ b/scripts/file-search-perf-harness.mjs @@ -0,0 +1,478 @@ +#!/usr/bin/env node + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { performance } from 'node:perf_hooks'; +import { importTs } from './lib/ts-import.mjs'; + +const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.resolve(SCRIPT_DIR, '..'); +const FILE_SEARCH_INDEX_TS = path.join(REPO_ROOT, 'src/main/file-search-index.ts'); + +const DEFAULT_CONFIG = { + projects: 24, + modulesPerProject: 4, + filesPerModule: 8, + queryRuns: 5, + updateCount: 32, + deleteCount: 32, + limit: 12, + keep: false, + json: false, + output: '', + includeProtectedHomeRoots: true, + thresholds: {}, +}; + +const FILE_NAME_PATTERNS = [ + 'command-palette-{project}-{module}-{file}.ts', + 'launch-plan-alpha-{project}-{module}-{file}.md', + 'invoice-report-{project}-{module}-{file}.json', + 'notes-search-index-{project}-{module}-{file}.txt', + 'shortcut-resolver-{project}-{module}-{file}.tsx', + 'settings-migration-{project}-{module}-{file}.ts', + 'quick-action-workflow-{project}-{module}-{file}.md', + 'browser-history-cache-{project}-{module}-{file}.json', +]; + +function pad(value, width = 3) { + return String(value).padStart(width, '0'); +} + +function formatPattern(pattern, values) { + return pattern + .replaceAll('{project}', values.project) + .replaceAll('{module}', values.module) + .replaceAll('{file}', values.file); +} + +function readNumber(value, fallback, min = 1) { + const parsed = Number(value); + if (!Number.isFinite(parsed)) return fallback; + return Math.max(min, Math.floor(parsed)); +} + +function readOptionalNumber(value) { + if (value === undefined || value === '') return undefined; + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : undefined; +} + +function getArgValue(args, name) { + const inline = args.find((arg) => arg.startsWith(`${name}=`)); + if (inline) return inline.slice(name.length + 1); + const index = args.indexOf(name); + if (index >= 0 && index + 1 < args.length) return args[index + 1]; + return undefined; +} + +function hasFlag(args, name) { + return args.includes(name); +} + +function getUsage() { + return [ + 'Usage: node scripts/file-search-perf-harness.mjs [options]', + '', + 'Options:', + ' --projects Number of synthetic projects', + ' --modules Modules per project', + ' --files-per-module Files per module', + ' --query-runs Query repetitions per query', + ' --updates Watcher-style added paths', + ' --deletes Deleted paths in the tombstone batch', + ' --limit Search result limit', + ' --json Print JSON output', + ' --output Write the same output to a file', + ' --keep Keep the temp fixture for inspection', + ' --threshold-index-ms Fail if initial indexing exceeds n ms', + ' --threshold-normal-query-p95-ms Fail if normal query p95 exceeds n ms', + ' --threshold-path-query-p95-ms Fail if path-like query p95 exceeds n ms', + ' --threshold-watch-update-ms Fail if watcher-style batch exceeds n ms', + ' --threshold-delete-ms Fail if delete batch exceeds n ms', + ].join('\n'); +} + +export function parseFileSearchPerfHarnessArgs(args = process.argv.slice(2)) { + return { + projects: readNumber(getArgValue(args, '--projects'), DEFAULT_CONFIG.projects), + modulesPerProject: readNumber(getArgValue(args, '--modules'), DEFAULT_CONFIG.modulesPerProject), + filesPerModule: readNumber(getArgValue(args, '--files-per-module'), DEFAULT_CONFIG.filesPerModule), + queryRuns: readNumber(getArgValue(args, '--query-runs'), DEFAULT_CONFIG.queryRuns), + updateCount: readNumber(getArgValue(args, '--updates'), DEFAULT_CONFIG.updateCount), + deleteCount: readNumber(getArgValue(args, '--deletes'), DEFAULT_CONFIG.deleteCount), + limit: readNumber(getArgValue(args, '--limit'), DEFAULT_CONFIG.limit), + keep: hasFlag(args, '--keep'), + json: hasFlag(args, '--json'), + output: getArgValue(args, '--output') || '', + includeProtectedHomeRoots: !hasFlag(args, '--exclude-protected-home-roots'), + thresholds: { + initialIndexMs: readOptionalNumber(getArgValue(args, '--threshold-index-ms')), + normalQueryP95Ms: readOptionalNumber(getArgValue(args, '--threshold-normal-query-p95-ms')), + pathQueryP95Ms: readOptionalNumber(getArgValue(args, '--threshold-path-query-p95-ms')), + watchUpdateBatchMs: readOptionalNumber(getArgValue(args, '--threshold-watch-update-ms')), + deleteBatchMs: readOptionalNumber(getArgValue(args, '--threshold-delete-ms')), + }, + }; +} + +function mergeConfig(overrides = {}) { + return { + ...DEFAULT_CONFIG, + ...overrides, + thresholds: { + ...DEFAULT_CONFIG.thresholds, + ...(overrides.thresholds || {}), + }, + }; +} + +async function writeFile(filePath, contents) { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, contents); +} + +async function createSyntheticHome(config) { + const tempRootRaw = await fs.mkdtemp(path.join(os.tmpdir(), 'supercmd-file-search-perf-')); + const tempRoot = await fs.realpath(tempRootRaw); + const homeDir = path.join(tempRoot, 'home'); + const benchRoot = path.join(homeDir, 'BenchRoot'); + const projectsRoot = path.join(benchRoot, 'Projects'); + const archiveRoot = path.join(benchRoot, 'Archive'); + const referenceRoot = path.join(benchRoot, 'References'); + const filePaths = []; + const deleteCandidates = []; + + await fs.mkdir(projectsRoot, { recursive: true }); + await fs.mkdir(archiveRoot, { recursive: true }); + await fs.mkdir(referenceRoot, { recursive: true }); + + for (let projectIndex = 0; projectIndex < config.projects; projectIndex += 1) { + const project = `project-${pad(projectIndex)}`; + for (let moduleIndex = 0; moduleIndex < config.modulesPerProject; moduleIndex += 1) { + const moduleName = `module-${pad(moduleIndex, 2)}`; + const srcDir = path.join(projectsRoot, project, moduleName, 'src'); + const docsDir = path.join(projectsRoot, project, moduleName, 'docs'); + + for (let fileIndex = 0; fileIndex < config.filesPerModule; fileIndex += 1) { + const file = pad(fileIndex, 2); + const pattern = FILE_NAME_PATTERNS[fileIndex % FILE_NAME_PATTERNS.length]; + const targetDir = fileIndex % 2 === 0 ? srcDir : docsDir; + const fileName = formatPattern(pattern, { project, module: moduleName, file }); + const filePath = path.join(targetDir, fileName); + await writeFile( + filePath, + [ + `project=${project}`, + `module=${moduleName}`, + `file=${file}`, + `intent=${fileName}`, + '', + ].join('\n') + ); + filePaths.push(filePath); + if (fileName.startsWith('launch-plan-alpha') || fileName.startsWith('invoice-report')) { + deleteCandidates.push(filePath); + } + } + } + + await writeFile( + path.join(archiveRoot, `release-notes-command-palette-${project}.md`), + `release notes for ${project}\n` + ); + await writeFile( + path.join(referenceRoot, `path-query-reference-${project}.txt`), + `path reference for ${project}\n` + ); + } + + return { + tempRoot, + homeDir, + benchRoot, + projectsRoot, + filePaths, + deleteCandidates, + }; +} + +function percentile(values, percentileValue) { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil((percentileValue / 100) * sorted.length) - 1)); + return sorted[index]; +} + +function summarizeDurations(samples) { + const totalMs = samples.reduce((sum, sample) => sum + sample.durationMs, 0); + const durations = samples.map((sample) => sample.durationMs); + return { + runs: samples.length, + minMs: Number(Math.min(...durations).toFixed(3)), + meanMs: Number((totalMs / Math.max(1, samples.length)).toFixed(3)), + medianMs: Number(percentile(durations, 50).toFixed(3)), + p95Ms: Number(percentile(durations, 95).toFixed(3)), + maxMs: Number(Math.max(...durations).toFixed(3)), + totalResults: samples.reduce((sum, sample) => sum + sample.resultCount, 0), + }; +} + +async function measureDuration(fn) { + const startedAt = performance.now(); + const result = await fn(); + return { + durationMs: performance.now() - startedAt, + result, + }; +} + +async function measureQueries(searchIndexedFiles, queries, config) { + const samples = []; + for (let run = 0; run < config.queryRuns; run += 1) { + for (const query of queries) { + const measured = await measureDuration(() => searchIndexedFiles(query, { limit: config.limit })); + samples.push({ + query, + run, + durationMs: measured.durationMs, + resultCount: measured.result.length, + }); + } + } + return { + ...summarizeDurations(samples), + samples: samples.map((sample) => ({ + ...sample, + durationMs: Number(sample.durationMs.toFixed(3)), + })), + }; +} + +function buildQueries(homeDir) { + return { + normal: [ + 'command palette', + 'launch plan alpha', + 'invoice report', + 'notes search index', + ], + pathLike: [ + '~/BenchRoot/Projects/project-', + '~/BenchRoot/Archive/release-notes-command', + `${homeDir}/BenchRoot/References/path-query-reference`, + ], + }; +} + +async function createUpdateFiles(fixture, config) { + const updatePaths = []; + for (let index = 0; index < config.updateCount; index += 1) { + const project = `project-${pad(index % config.projects)}`; + const moduleName = `module-${pad(index % config.modulesPerProject, 2)}`; + const filePath = path.join( + fixture.projectsRoot, + project, + moduleName, + 'src', + `watcher-added-command-${pad(index)}.ts` + ); + await writeFile(filePath, `watcher added command ${index}\n`); + updatePaths.push(filePath); + } + return updatePaths; +} + +async function deleteFixtureFiles(fixture, config) { + const deletePaths = fixture.deleteCandidates.slice(0, config.deleteCount); + for (const filePath of deletePaths) { + await fs.rm(filePath, { force: true }); + } + return deletePaths; +} + +function evaluateThresholds(metrics, thresholds = {}) { + const checks = [ + ['initialIndexMs', metrics.initialIndexMs, thresholds.initialIndexMs], + ['normalQueryP95Ms', metrics.normalQueries.p95Ms, thresholds.normalQueryP95Ms], + ['pathQueryP95Ms', metrics.pathLikeQueries.p95Ms, thresholds.pathQueryP95Ms], + ['watchUpdateBatchMs', metrics.watchUpdateBatchMs, thresholds.watchUpdateBatchMs], + ['deleteBatchMs', metrics.deleteBatchMs, thresholds.deleteBatchMs], + ]; + const failures = checks + .filter(([, actual, threshold]) => typeof threshold === 'number' && actual > threshold) + .map(([name, actual, threshold]) => `${name} ${actual.toFixed(3)}ms exceeded ${threshold}ms`); + + return { + passed: failures.length === 0, + failures, + applied: Object.fromEntries( + checks + .filter(([, , threshold]) => typeof threshold === 'number') + .map(([name, , threshold]) => [name, threshold]) + ), + }; +} + +function roundMetric(value) { + return Number(value.toFixed(3)); +} + +function toDisplayLines(summary) { + const thresholdLine = summary.thresholds.applied && Object.keys(summary.thresholds.applied).length > 0 + ? summary.thresholds.passed + ? 'Thresholds: passed' + : `Thresholds: failed (${summary.thresholds.failures.join('; ')})` + : 'Thresholds: not applied'; + + return [ + 'File search perf harness', + `Home fixture: ${summary.fixture.homeDir}`, + `Indexed entries: ${summary.fixture.initialEntryCount}`, + `Initial indexing: ${summary.metrics.initialIndexMs.toFixed(3)}ms`, + `Normal queries: mean ${summary.metrics.normalQueries.meanMs.toFixed(3)}ms, p95 ${summary.metrics.normalQueries.p95Ms.toFixed(3)}ms, results ${summary.metrics.normalQueries.totalResults}`, + `Path-like queries: mean ${summary.metrics.pathLikeQueries.meanMs.toFixed(3)}ms, p95 ${summary.metrics.pathLikeQueries.p95Ms.toFixed(3)}ms, results ${summary.metrics.pathLikeQueries.totalResults}`, + `Watcher-style updates: ${summary.metrics.watchUpdateBatchMs.toFixed(3)}ms for ${summary.fixture.updateCount} paths`, + `Delete batch: ${summary.metrics.deleteBatchMs.toFixed(3)}ms for ${summary.fixture.deleteCount} paths`, + `Post-update query: ${summary.metrics.postUpdateQueryMs.toFixed(3)}ms (${summary.verification.postUpdateResultCount} results)`, + `Deleted exact matches after tombstone: ${summary.verification.deletedExactMatches}`, + thresholdLine, + ]; +} + +export async function runFileSearchPerfHarness(overrides = {}) { + const config = mergeConfig(overrides); + const fixture = await createSyntheticHome(config); + let fileSearchIndexPerfHarness = null; + let summary = null; + let cleanupCompleted = false; + try { + const fileSearchIndex = await importTs(FILE_SEARCH_INDEX_TS); + const { + __fileSearchIndexPerfHarness, + getFileSearchIndexStatus, + searchIndexedFiles, + } = fileSearchIndex; + fileSearchIndexPerfHarness = __fileSearchIndexPerfHarness; + + if (!fileSearchIndexPerfHarness) { + throw new Error('Missing __fileSearchIndexPerfHarness export from file-search-index.ts'); + } + + const queries = buildQueries(fixture.homeDir); + const initialIndex = await measureDuration(() => + fileSearchIndexPerfHarness.rebuild({ + homeDir: fixture.homeDir, + includeProtectedHomeRoots: config.includeProtectedHomeRoots, + }) + ); + const statusAfterIndex = getFileSearchIndexStatus(); + + const normalQueries = await measureQueries(searchIndexedFiles, queries.normal, config); + const pathLikeQueries = await measureQueries(searchIndexedFiles, queries.pathLike, config); + + const updatePaths = await createUpdateFiles(fixture, config); + const watchUpdate = await measureDuration(() => + fileSearchIndexPerfHarness.applyWatchEventBatch(updatePaths) + ); + const postUpdateQuery = await measureDuration(() => + searchIndexedFiles('watcher added command', { limit: config.limit }) + ); + + const deletePaths = await deleteFixtureFiles(fixture, config); + const deleteBatch = await measureDuration(() => + fileSearchIndexPerfHarness.applyWatchEventBatch(deletePaths) + ); + const deletedNeedle = deletePaths[0] ? path.basename(deletePaths[0]) : ''; + const postDeleteQuery = await measureDuration(() => + deletedNeedle ? searchIndexedFiles(deletedNeedle, { limit: config.limit }) : Promise.resolve([]) + ); + const deletedExactMatches = deletedNeedle + ? postDeleteQuery.result.filter((result) => result.name === deletedNeedle).length + : 0; + + const metrics = { + initialIndexMs: roundMetric(initialIndex.durationMs), + normalQueries, + pathLikeQueries, + watchUpdateBatchMs: roundMetric(watchUpdate.durationMs), + postUpdateQueryMs: roundMetric(postUpdateQuery.durationMs), + deleteBatchMs: roundMetric(deleteBatch.durationMs), + postDeleteQueryMs: roundMetric(postDeleteQuery.durationMs), + }; + const realTempDir = await fs.realpath(os.tmpdir()); + + summary = { + config: { + projects: config.projects, + modulesPerProject: config.modulesPerProject, + filesPerModule: config.filesPerModule, + queryRuns: config.queryRuns, + updateCount: config.updateCount, + deleteCount: config.deleteCount, + limit: config.limit, + }, + fixture: { + tempRoot: fixture.tempRoot, + homeDir: fixture.homeDir, + initialFileCount: fixture.filePaths.length + config.projects * 2, + initialEntryCount: statusAfterIndex.indexedEntryCount, + updateCount: updatePaths.length, + deleteCount: deletePaths.length, + }, + metrics, + verification: { + postUpdateResultCount: postUpdateQuery.result.length, + postDeleteResultCount: postDeleteQuery.result.length, + deletedExactMatches, + homeDirectoryUsed: statusAfterIndex.homeDirectory, + homeIsTempDirectory: fixture.homeDir.startsWith(realTempDir), + }, + thresholds: evaluateThresholds(metrics, config.thresholds), + cleanup: { + keptFixture: Boolean(config.keep), + completed: false, + }, + }; + + return summary; + } finally { + fileSearchIndexPerfHarness?.reset(); + if (!config.keep) { + await fs.rm(fixture.tempRoot, { recursive: true, force: true }); + cleanupCompleted = true; + } + if (summary) { + summary.cleanup.completed = cleanupCompleted; + } + } +} + +async function runCli() { + if (hasFlag(process.argv, '--help')) { + process.stdout.write(`${getUsage()}\n`); + return; + } + + const config = parseFileSearchPerfHarnessArgs(); + const summary = await runFileSearchPerfHarness(config); + const output = config.json ? `${JSON.stringify(summary, null, 2)}\n` : `${toDisplayLines(summary).join('\n')}\n`; + + if (config.output) { + await fs.writeFile(path.resolve(config.output), output); + } + process.stdout.write(output); + + if (!summary.thresholds.passed) { + process.exitCode = 1; + } +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + runCli().catch((error) => { + console.error(error); + process.exitCode = 1; + }); +} diff --git a/scripts/test-file-search-perf-harness.mjs b/scripts/test-file-search-perf-harness.mjs new file mode 100644 index 00000000..5ff1ddde --- /dev/null +++ b/scripts/test-file-search-perf-harness.mjs @@ -0,0 +1,52 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import { runFileSearchPerfHarness } from './file-search-perf-harness.mjs'; + +async function pathExists(filePath) { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +test('file search perf harness covers deterministic temp-home scenarios', async () => { + const summary = await runFileSearchPerfHarness({ + projects: 8, + modulesPerProject: 3, + filesPerModule: 6, + queryRuns: 2, + updateCount: 10, + deleteCount: 10, + limit: 8, + thresholds: { + initialIndexMs: 15_000, + normalQueryP95Ms: 2_000, + pathQueryP95Ms: 2_000, + watchUpdateBatchMs: 5_000, + deleteBatchMs: 5_000, + }, + }); + + assert.equal(summary.thresholds.passed, true, summary.thresholds.failures.join('\n')); + assert.equal(summary.verification.homeDirectoryUsed, summary.fixture.homeDir); + assert.equal(summary.verification.homeIsTempDirectory, true); + assert.ok( + summary.fixture.homeDir.startsWith(await fs.realpath(os.tmpdir())), + 'harness must use an OS temp home' + ); + assert.equal(summary.cleanup.completed, true); + assert.equal(await pathExists(summary.fixture.tempRoot), false, 'temp fixture should be cleaned up'); + + assert.ok(summary.fixture.initialEntryCount >= summary.fixture.initialFileCount); + assert.ok(summary.metrics.initialIndexMs >= 0); + assert.ok(summary.metrics.normalQueries.totalResults > 0); + assert.ok(summary.metrics.pathLikeQueries.totalResults > 0); + assert.ok(summary.verification.postUpdateResultCount > 0); + assert.equal(summary.verification.deletedExactMatches, 0); +}); diff --git a/src/main/file-search-index.ts b/src/main/file-search-index.ts index 9065be73..4e94ac0a 100644 --- a/src/main/file-search-index.ts +++ b/src/main/file-search-index.ts @@ -882,6 +882,60 @@ export function stopFileSearchIndexing(): void { stopFileSearchWatcher(); } +function resetFileSearchIndexForPerfHarness(options?: { + homeDir?: string; + includeProtectedHomeRoots?: boolean; +}): void { + stopFileSearchIndexing(); + activeIndex = null; + rebuildPromise = null; + configuredHomeDir = ''; + includeRoots = []; + refreshIntervalMs = DEFAULT_REFRESH_INTERVAL_MS; + includeProtectedHomeRoots = Boolean(options?.includeProtectedHomeRoots); + indexing = false; + lastIndexError = null; + lastBuildStartedAt = 0; + pendingWatchEvents.clear(); + if (options?.homeDir) { + ensureConfigured(options.homeDir); + } +} + +async function rebuildFileSearchIndexForPerfHarness(options: { + homeDir: string; + includeProtectedHomeRoots?: boolean; +}): Promise { + resetFileSearchIndexForPerfHarness(options); + if (includeRoots.length === 0) return; + + indexing = true; + try { + activeIndex = await buildIndexSnapshot(configuredHomeDir); + lastIndexError = null; + } catch (error) { + lastIndexError = error instanceof Error ? error.message : String(error || 'Unknown indexing error'); + throw error; + } finally { + indexing = false; + rebuildPromise = null; + lastBuildStartedAt = 0; + } +} + +async function applyFileSearchWatchEventBatchForPerfHarness(paths: string[]): Promise { + if (!activeIndex) { + throw new Error('File search perf harness requires an active index before applying watch events.'); + } + await applyWatchEventBatch(paths.map((candidatePath) => path.resolve(candidatePath))); +} + +export const __fileSearchIndexPerfHarness = { + reset: resetFileSearchIndexForPerfHarness, + rebuild: rebuildFileSearchIndexForPerfHarness, + applyWatchEventBatch: applyFileSearchWatchEventBatchForPerfHarness, +}; + export async function searchIndexedFiles( rawQuery: string, options?: { limit?: number }