From 6f68b654c6b6e1bac2c1d348d4df3644d0c321d8 Mon Sep 17 00:00:00 2001 From: David Dodlek Date: Tue, 6 Oct 2026 18:14:33 +0200 Subject: [PATCH 1/2] feat(codex): Codex uses mgrep for conceptual questions and keeps rg for exact lookups Replace the Codex integration, which appended a "MANDATORY / NEVER use grep" skill to ~/.codex/AGENTS.md and registered an MCP server without tools. Codex then stopped searching any other way. The integration is now a native Codex plugin from the same repository marketplace as the Claude plugin: - MCP server (`mgrep mcp --expose-tools`) with one read-only `search` tool. It runs outside Codex's network sandbox, so searches need no approval prompt, and it syncs the session directory in the background for the lifetime of the session. - SessionStart hook that only adds neutral guidance: mgrep for conceptual questions, rg/grep for exact strings and symbols. - Short neutral skill. - `install-codex` adds the marketplace and installs the plugin; `uninstall-codex` reverses it. Both clean up the legacy AGENTS.md block and MCP registration. Supporting changes the MCP tool needs: search logic extracted to src/lib/search.ts and shared by the CLI and the tool, non-interactive store creation for the MCP server, version read from package.json, and startWatch no longer calls process.exit and reports whether watching started. --- .agents/plugins/marketplace.json | 20 + README.md | 38 ++ package.json | 2 +- plugins/mgrep/.codex-plugin/plugin.json | 17 + plugins/mgrep/codex/hooks.json | 16 + plugins/mgrep/codex/mcp.json | 9 + plugins/mgrep/codex/session_start.py | 46 ++ plugins/mgrep/codex/skills/mgrep/SKILL.md | 28 ++ src/commands/search.ts | 176 +------ src/commands/watch.ts | 37 +- src/commands/watch_mcp.ts | 298 +++++++++--- src/index.ts | 13 +- src/install/codex.ts | 136 +++--- src/lib/context.ts | 17 +- src/lib/search.ts | 189 ++++++++ src/lib/update-notice.ts | 2 +- src/lib/version.ts | 11 + test/codex.bats | 347 ++++++++++++++ test/helpers/mcp-client.mjs | 109 +++++ test/mcp.bats | 535 ++++++++++++++++++++++ test/test.bats | 2 +- 21 files changed, 1754 insertions(+), 294 deletions(-) create mode 100644 .agents/plugins/marketplace.json create mode 100644 plugins/mgrep/.codex-plugin/plugin.json create mode 100644 plugins/mgrep/codex/hooks.json create mode 100644 plugins/mgrep/codex/mcp.json create mode 100644 plugins/mgrep/codex/session_start.py create mode 100644 plugins/mgrep/codex/skills/mgrep/SKILL.md create mode 100644 src/lib/search.ts create mode 100644 src/lib/version.ts create mode 100644 test/codex.bats create mode 100644 test/helpers/mcp-client.mjs create mode 100644 test/mcp.bats diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json new file mode 100644 index 0000000..d2591e3 --- /dev/null +++ b/.agents/plugins/marketplace.json @@ -0,0 +1,20 @@ +{ + "name": "Mixedbread-Grep", + "interface": { + "displayName": "Mixedbread" + }, + "plugins": [ + { + "name": "mgrep", + "source": { + "source": "local", + "path": "./plugins/mgrep" + }, + "policy": { + "installation": "AVAILABLE", + "authentication": "ON_INSTALL" + }, + "category": "Developer Tools" + } + ] +} diff --git a/README.md b/README.md index 6adffe6..e2d4ea6 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,44 @@ These commands sign you in (if needed) and add Mixedbread `mgrep` support to the agent. After that you only have to start the agent in your project folder, thats it. +### Codex + +With the Codex CLI installed, run: + +```bash +mgrep install-codex +``` + +This authenticates with Mixedbread, adds the `Mixedbread-Grep` marketplace, and +installs `mgrep@Mixedbread-Grep`. The plugin includes an MCP server with a +read-only `search` tool and a short skill. The MCP server syncs the session's +project directory in the background while Codex runs. + +A SessionStart hook adds search guidance on startup and resume, helping Codex +choose semantic search for conceptual or cross-file questions and rg or grep for +exact matches. On first start, Codex asks you to review the hook ("Hooks need +review"); you can also review it with `/hooks`. This one-time review allows the +hook to run. The hook only adds context; the MCP server owns background sync. + +Use MCP `search` for conceptual questions; `path` is optional and defaults to the +session's project directory. Use rg or grep for exact strings, symbol names, and +regexes. The MCP server runs `mgrep mcp --expose-tools`. Searches within the +project wait for the watcher's initial sync. If the watcher cannot start, or the +search path is outside the project, the first search syncs that root once. + +The plugin forwards `MXBAI_API_KEY` and `MXBAI_STORE` from Codex's environment. +You can also authenticate with `mgrep login`. No API key is written into plugin +configuration. To install from a local checkout or another Git URL, set +`MGREP_CODEX_MARKETPLACE` before running `mgrep install-codex`. + +```bash +mgrep uninstall-codex +``` + +Uninstall removes the plugin, its marketplace, and the legacy MCP registration. +Both commands remove the old mgrep block from `$CODEX_HOME/AGENTS.md` and +`~/.codex/AGENTS.md`, preserving surrounding instructions. + ### More Agents Coming Soon More agents (Cursor, Windsurf, etc.) are on the way—this section will grow as soon as each integration lands. diff --git a/package.json b/package.json index 81e653b..fa9e5de 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "build": "tsc", "postbuild": "chmod +x dist/index.js && cp -r plugins dist/", "dev": "npx tsc && node dist/index.js", - "test": "bats test/test.bats --filter-tags !long-running", + "test": "bats test/ --filter-tags !long-running", "format": "biome check --write .", "format:check": "biome check .", "lint": "biome lint --check .", diff --git a/plugins/mgrep/.codex-plugin/plugin.json b/plugins/mgrep/.codex-plugin/plugin.json new file mode 100644 index 0000000..b43b255 --- /dev/null +++ b/plugins/mgrep/.codex-plugin/plugin.json @@ -0,0 +1,17 @@ +{ + "name": "mgrep", + "version": "0.1.13", + "description": "Search your local files using Mixedbread", + "author": { + "name": "Joel Dierkes" + }, + "skills": "./codex/skills/", + "mcpServers": "./codex/mcp.json", + "hooks": "./codex/hooks.json", + "interface": { + "displayName": "mgrep", + "shortDescription": "Semantic search for your project files", + "developerName": "Mixedbread", + "category": "Developer Tools" + } +} diff --git a/plugins/mgrep/codex/hooks.json b/plugins/mgrep/codex/hooks.json new file mode 100644 index 0000000..b386a64 --- /dev/null +++ b/plugins/mgrep/codex/hooks.json @@ -0,0 +1,16 @@ +{ + "hooks": { + "SessionStart": [ + { + "matcher": "startup|resume", + "hooks": [ + { + "type": "command", + "command": "python3 \"${CLAUDE_PLUGIN_ROOT:-$PLUGIN_ROOT}/codex/session_start.py\"", + "timeout": 5 + } + ] + } + ] + } +} diff --git a/plugins/mgrep/codex/mcp.json b/plugins/mgrep/codex/mcp.json new file mode 100644 index 0000000..563b998 --- /dev/null +++ b/plugins/mgrep/codex/mcp.json @@ -0,0 +1,9 @@ +{ + "mcpServers": { + "mgrep": { + "command": "mgrep", + "args": ["mcp", "--expose-tools"], + "env_vars": ["MXBAI_API_KEY", "MXBAI_STORE"] + } + } +} diff --git a/plugins/mgrep/codex/session_start.py b/plugins/mgrep/codex/session_start.py new file mode 100644 index 0000000..ebe5ff0 --- /dev/null +++ b/plugins/mgrep/codex/session_start.py @@ -0,0 +1,46 @@ +import json +import os +import sys +from pathlib import Path + + +def read_update_notice(): + try: + cache_path = Path.home() / ".mgrep" / "update-check.json" + with open(cache_path, "r", encoding="utf-8") as handle: + cache = json.load(handle) + latest = cache.get("latestVersion") + if not isinstance(latest, str): + return "" + return f" An mgrep update is available (latest: {latest}). Tell the user to run `mgrep update` to upgrade." + except Exception: + return "" + + +def main(): + try: + payload = json.load(sys.stdin) + except (ValueError, OSError): + payload = {} + cwd = payload.get("cwd") if isinstance(payload, dict) else None + if not isinstance(cwd, str) or not cwd or "\0" in cwd: + cwd = os.getcwd() + cwd = os.path.abspath(cwd) + context = ( + "mgrep semantic search is available through the `search` tool of the mgrep MCP server. " + "Use it for conceptual or cross-file questions where you don't know the exact identifier, " + "such as 'where is authentication handled'. " + f"It searches {cwd} by default. " + "Use rg or grep for exact strings, symbol names and regexes. " + "Both are fine; pick whichever fits the question." + ) + print(json.dumps({ + "hookSpecificOutput": { + "hookEventName": "SessionStart", + "additionalContext": context + read_update_notice(), + } + })) + + +if __name__ == "__main__": + main() diff --git a/plugins/mgrep/codex/skills/mgrep/SKILL.md b/plugins/mgrep/codex/skills/mgrep/SKILL.md new file mode 100644 index 0000000..1c58ca2 --- /dev/null +++ b/plugins/mgrep/codex/skills/mgrep/SKILL.md @@ -0,0 +1,28 @@ +--- +name: mgrep +description: Semantic search for conceptual questions about project files, alongside rg and grep for exact matches. +--- + +Use the mgrep MCP server's `search` tool for conceptual or cross-file questions +when you do not know the exact identifier, such as "where is authentication +handled". Provide a natural-language `query`; `path` is optional and defaults +to the session's project directory. Results contain file paths and line ranges; +read only those ranges. The MCP server keeps the session's project directory synced. + +Use rg or grep for exact strings, symbol names, and regexes. Both approaches are +fine; choose whichever fits the question. + +Outside MCP, use the CLI from your project directory: + +```bash +mgrep search "how are failed uploads retried" +mgrep search --sync "where is authentication handled" ./src +mgrep search --web --answer "how does this API work" +``` + +Prefer the MCP tool inside Codex to avoid shell network approval prompts. The CLI +works in a terminal or when MCP is unavailable. Set `web` and `answer` to true +for an MCP web lookup. Use `maxCount` (CLI `-m`) to limit results. + +`content: true` and CLI `-c` include full matching content and can produce large +output. Start with paths and line ranges, then read the relevant ranges. diff --git a/src/commands/search.ts b/src/commands/search.ts index 880379e..a9621ca 100644 --- a/src/commands/search.ts +++ b/src/commands/search.ts @@ -1,4 +1,3 @@ -import { join, normalize } from "node:path"; import type { Command } from "commander"; import { Command as CommanderCommand, InvalidArgumentError } from "commander"; import { @@ -8,13 +7,8 @@ import { } from "../lib/config.js"; import { createFileSystem, createStore } from "../lib/context.js"; import { DEFAULT_IGNORE_PATTERNS } from "../lib/file.js"; -import type { - AskResponse, - ChunkType, - FileMetadata, - SearchResponse, - Store, -} from "../lib/store.js"; +import { resolveSearchPath, runSearch } from "../lib/search.js"; +import type { Store } from "../lib/store.js"; import { createIndexingSpinner, formatDryRunSummary, @@ -26,107 +20,6 @@ import { QuotaExceededError, } from "../lib/utils.js"; -function extractSources(response: AskResponse): { [key: number]: ChunkType } { - const sources: { [key: number]: ChunkType } = {}; - const answer = response.answer; - - // Match ALL cite tags and capture the i="..." - const citeTags = answer.match(/= 0 && - end >= start && - end < response.sources.length - ) { - for (let i = start; i <= end; i++) { - sources[i] = response.sources[i]; - } - } - } - - return sources; -} - -function formatAskResponse(response: AskResponse, show_content: boolean) { - const sources = extractSources(response); - const sourceEntries = Object.entries(sources).map( - ([index, chunk]) => `${index}: ${formatChunk(chunk, show_content)}`, - ); - return `${response.answer}\n\n${sourceEntries.join("\n")}`; -} - -function formatSearchResponse(response: SearchResponse, show_content: boolean) { - return response.data - .map((chunk) => formatChunk(chunk, show_content)) - .join("\n"); -} - -function isWebResult(chunk: ChunkType): boolean { - return ( - "filename" in chunk && - typeof chunk.filename === "string" && - chunk.filename.startsWith("http") - ); -} - -function formatChunk(chunk: ChunkType, show_content: boolean) { - const pwd = process.cwd(); - - if (isWebResult(chunk) && chunk.type === "text") { - const url = "filename" in chunk ? chunk.filename : "Unknown URL"; - const content = show_content ? chunk.text : ""; - return `${url} (${(chunk.score * 100).toFixed(2)}% match)${content ? `\n${content}` : ""}`; - } - - const path = - (chunk.metadata as FileMetadata)?.path?.replace(pwd, "") ?? "Unknown path"; - let line_range = ""; - let content = ""; - switch (chunk.type) { - case "text": { - const start_line = (chunk.generated_metadata?.start_line as number) + 1; - const end_line = - start_line + (chunk.generated_metadata?.num_lines as number); - line_range = `:${start_line}-${end_line}`; - content = show_content ? chunk.text : ""; - break; - } - case "image_url": - line_range = - chunk.generated_metadata?.type === "pdf" - ? `, page ${chunk.chunk_index + 1}` - : ""; - break; - case "audio_url": - line_range = ""; - break; - case "video_url": - line_range = ""; - break; - } - - return `.${path}${line_range} (${(chunk.score * 100).toFixed(2)}% match)${content ? `\n${content}` : ""}`; -} - function parseBooleanEnv( envVar: string | undefined, defaultValue: boolean, @@ -138,9 +31,14 @@ function parseBooleanEnv( /** * Syncs local files to the store with progress indication. + * @param store The authenticated store. + * @param storeName The store to sync. + * @param root The absolute directory to sync. + * @param dryRun Whether to report changes without applying them. + * @param config The project indexing limits and settings. * @returns true if the caller should return early (dry-run mode), false otherwise */ -async function syncFiles( +export async function syncFiles( store: Store, storeName: string, root: string, @@ -285,9 +183,7 @@ export const search: Command = new CommanderCommand("search") }; const config = loadConfig(root, cliOptions); - const search_path = exec_path?.startsWith("/") - ? exec_path - : normalize(join(root, exec_path ?? "")); + const search_path = resolveSearchPath(root, exec_path); if (options.sync && isAtOrAboveHomeDirectory(search_path)) { console.error( @@ -316,45 +212,21 @@ export const search: Command = new CommanderCommand("search") } } - const storeIds = options.web - ? [options.store, "mixedbread/web"] - : [options.store]; - - const filters = { - all: [ - { - key: "path", - operator: "starts_with" as const, - value: search_path, - }, - ], - }; - - const searchOptions = { - rerank: options.rerank, - ...(options.agentic && { agentic: true }), - }; - - let response: string; - if (!options.answer) { - const results = await store.search( - storeIds, - pattern, - parseInt(options.maxCount, 10), - searchOptions, - filters, - ); - response = formatSearchResponse(results, options.content); - } else { - const results = await store.ask( - storeIds, - pattern, - parseInt(options.maxCount, 10), - searchOptions, - filters, - ); - response = formatAskResponse(results, options.content); - } + const response = await runSearch( + store, + { + store: options.store, + query: pattern, + path: search_path, + maxCount: parseInt(options.maxCount, 10), + content: options.content, + answer: options.answer, + web: options.web, + rerank: options.rerank, + agentic: options.agentic, + }, + root, + ); console.log(response); } catch (error) { diff --git a/src/commands/watch.ts b/src/commands/watch.ts index 50e9d3d..cd39db0 100644 --- a/src/commands/watch.ts +++ b/src/commands/watch.ts @@ -4,6 +4,7 @@ import { Command, InvalidArgumentError } from "commander"; import { type CliConfigOptions, loadConfig } from "../lib/config.js"; import { createFileSystem, createStore } from "../lib/context.js"; import { DEFAULT_IGNORE_PATTERNS } from "../lib/file.js"; +import type { Store } from "../lib/store.js"; import { createIndexingSpinner, formatDryRunSummary, @@ -22,13 +23,20 @@ export interface WatchOptions { dryRun: boolean; maxFileSize?: number; maxFileCount?: number; + createStore?: () => Promise; } -export async function startWatch(options: WatchOptions): Promise { +/** + * Starts in-process synchronization and file watchers for the current directory. + * @param options Store, sync limits, and optional non-interactive store factory. + * @returns Whether initial synchronization completed and file watchers were set up, + * regardless of per-file upload errors. + */ +export async function startWatch(options: WatchOptions): Promise { let refreshInterval: NodeJS.Timeout | undefined; try { - const store = await createStore(); + const store = await (options.createStore ?? createStore)(); // Refresh JWT token every 5 minutes (before 15-minute expiration) if (!options.dryRun) { @@ -60,7 +68,7 @@ export async function startWatch(options: WatchOptions): Promise { "Please run this command from within a specific project subdirectory.", ); process.exitCode = 1; - return; + return false; } const cliOptions: CliConfigOptions = { @@ -112,7 +120,7 @@ export async function startWatch(options: WatchOptions): Promise { includeTotal: true, }), ); - return; + return false; } } catch (e) { if (e instanceof QuotaExceededError) { @@ -123,7 +131,9 @@ export async function startWatch(options: WatchOptions): Promise { console.error( " Upgrade your plan at https://platform.mixedbread.com to continue syncing.\n", ); - process.exit(1); + clearInterval(refreshInterval); + process.exitCode = 1; + return false; } if (e instanceof MaxFileCountExceededError) { spinner.fail("File count exceeded"); @@ -131,7 +141,9 @@ export async function startWatch(options: WatchOptions): Promise { console.error( " Increase the limit with --max-file-count or MGREP_MAX_FILE_COUNT environment variable.\n", ); - process.exit(1); + clearInterval(refreshInterval); + process.exitCode = 1; + return false; } spinner.fail("Initial upload failed"); throw e; @@ -146,6 +158,7 @@ export async function startWatch(options: WatchOptions): Promise { // the callback filter runs, which exhausts the kernel watcher limit on // large repos. const watchers = new Map(); + let watchSetupSucceeded = true; function isMissingPathError(error: unknown): boolean { return ( @@ -163,6 +176,7 @@ export async function startWatch(options: WatchOptions): Promise { return null; } + watchSetupSucceeded = false; console.warn(`Warning: failed to inspect path ${filePath}:`, error); return undefined; } @@ -270,8 +284,14 @@ export async function startWatch(options: WatchOptions): Promise { handleFileEvent(eventType, dirPath, name); }); + watcher.on("error", (error) => { + watchSetupSucceeded = false; + console.error(`Failed to watch ${dirPath}:`, error); + closeWatcher(dirPath); + }); watchers.set(dirPath, watcher); } catch (error) { + watchSetupSucceeded = false; console.warn(`Warning: failed to watch ${dirPath}:`, error); return; } @@ -285,11 +305,15 @@ export async function startWatch(options: WatchOptions): Promise { watchDirectory(path.join(dirPath, entry.name)); } } catch (error) { + watchSetupSucceeded = false; console.warn(`Warning: failed to read directory ${dirPath}:`, error); } } watchDirectory(watchRoot); + // fs.watch can report startup failures asynchronously. + await new Promise((resolve) => setImmediate(resolve)); + return watchSetupSucceeded && watchers.has(watchRoot); } catch (error) { if (refreshInterval) { clearInterval(refreshInterval); @@ -297,6 +321,7 @@ export async function startWatch(options: WatchOptions): Promise { const message = error instanceof Error ? error.message : "Unknown error"; console.error("Failed to start watcher:", message); process.exitCode = 1; + return false; } } diff --git a/src/commands/watch_mcp.ts b/src/commands/watch_mcp.ts index fe992e6..7758adc 100644 --- a/src/commands/watch_mcp.ts +++ b/src/commands/watch_mcp.ts @@ -1,94 +1,276 @@ +import { realpathSync } from "node:fs"; +import { resolve, sep } from "node:path"; +import { fileURLToPath } from "node:url"; import { Server } from "@modelcontextprotocol/sdk/server/index.js"; import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; import { CallToolRequestSchema, ListToolsRequestSchema, + ResultSchema, } from "@modelcontextprotocol/sdk/types.js"; import { Command } from "commander"; +import { z } from "zod"; +import { loadConfig } from "../lib/config.js"; +import { createStore } from "../lib/context.js"; +import { runSearch } from "../lib/search.js"; +import { isAtOrAboveHomeDirectory } from "../lib/utils.js"; +import { getPackageVersion } from "../lib/version.js"; +import { syncFiles } from "./search.js"; import { startWatch } from "./watch.js"; +// The SDK's default roots schema rejects non-file URIs before we can skip them. +const rootsResult = z.object({ + roots: z.array(z.object({ uri: z.string() })), +}); + +const searchArguments = z.strictObject({ + query: z.string().min(1), + path: z.string().optional(), + maxCount: z.number().int().positive().default(10), + content: z.boolean().default(false), + answer: z.boolean().default(false), + web: z.boolean().default(false), +}); + export const watchMcp = new Command("mcp") .description("Start MCP server for mgrep") + .option("--expose-tools", "Expose search tools via MCP", false) + .option("--no-watch", "Skip background file sync") .action(async (_options, cmd) => { - process.on("SIGINT", () => { - console.error("Received SIGINT, shutting down gracefully..."); - process.exit(0); - }); - - process.on("SIGTERM", () => { - console.error("Received SIGTERM, shutting down gracefully..."); - process.exit(0); - }); - - // Prevent unhandled promise rejections from crashing the MCP server - process.on("unhandledRejection", (reason, promise) => { - console.error( - "[ERROR] Unhandled Rejection at:", - promise, - "reason:", - reason, - ); - }); - - // The MCP server is writing to stdout, so all logs are written to stderr + // The MCP server is writing to stdout, so all logs are written to stderr. console.log = (...args: unknown[]) => { process.stderr.write(`[LOG] ${args.join(" ")}\n`); }; - console.error = (...args: unknown[]) => { process.stderr.write(`[ERROR] ${args.join(" ")}\n`); }; - console.debug = (...args: unknown[]) => { process.stderr.write(`[DEBUG] ${args.join(" ")}\n`); }; + console.info = console.log; - const options: { - store: string; - } = cmd.optsWithGlobals(); + process.on("unhandledRejection", (reason) => { + console.error("Unhandled rejection:", reason); + }); + const options: { store: string; exposeTools: boolean; watch: boolean } = + cmd.optsWithGlobals(); + const configuredSyncWaitMs = Number( + process.env.MGREP_MCP_SYNC_WAIT_MS?.trim() || "20000", + ); + const syncWaitMs = + Number.isInteger(configuredSyncWaitMs) && + configuredSyncWaitMs >= 0 && + configuredSyncWaitMs <= 2147483647 + ? configuredSyncWaitMs + : 20000; + const root = process.cwd(); const transport = new StdioServerTransport(); const server = new Server( + { name: "mgrep", version: getPackageVersion() }, { - name: "mgrep", - version: "0.1.3", - }, - { - capabilities: { - tools: {}, - }, + capabilities: { tools: {} }, + instructions: `mgrep semantic search is available through this server's \`search\` tool. Use it for conceptual or cross-file questions where you don't know the exact identifier, such as 'where is authentication handled'. It searches ${root} by default. Use rg or grep for exact strings, symbol names and regexes. Both are fine; pick whichever fits the question.`, }, ); - server.setRequestHandler(ListToolsRequestSchema, async () => { - return { - tools: [], - }; + let syncTimer: NodeJS.Timeout | undefined; + let closing = false; + const firstSyncs = new Map>(); + const serverRoot = realpathSync(root); + const serverRootPrefix = serverRoot.endsWith(sep) + ? serverRoot + : `${serverRoot}${sep}`; + let completeWatch: (watching: boolean) => void = () => {}; + // Requests can arrive before the delayed watcher startup completes. + const watcherReady = new Promise((resolve) => { + completeWatch = resolve; }); + if (!options.watch) completeWatch(false); - server.setRequestHandler(CallToolRequestSchema, async (_request) => { - return { - result: "Not implemented", - }; - }); + const resolveRoot = async (searchPath?: string): Promise => { + if (searchPath !== undefined) + return realpathSync(resolve(root, searchPath)); + if (server.getClientCapabilities()?.roots) { + const { roots } = rootsResult.parse( + await server.request( + { method: "roots/list" }, + ResultSchema.passthrough(), + ), + ); + const fileRoot = roots.find((entry) => entry.uri.startsWith("file://")); + if (fileRoot) return realpathSync(fileURLToPath(fileRoot.uri)); + } + if (options.watch) return root; + throw new Error( + "An absolute project path is required; pass the absolute project directory as path.", + ); + }; - await server.connect(transport); + const shutdown = async (): Promise => { + if (closing) return; + closing = true; + clearTimeout(syncTimer); + completeWatch(false); + try { + await server.close(); + } finally { + // In-process fs.watch handles must not outlive the MCP client. + process.exit(0); + } + }; - const startBackgroundSync = async () => { - console.log("[SYNC] Scheduling initial sync in 5 seconds..."); + server.onclose = () => void shutdown(); + process.stdin.once("end", () => void shutdown()); + process.stdin.once("close", () => void shutdown()); + for (const signal of ["SIGINT", "SIGTERM"] as const) { + process.once(signal, () => { + console.error(`Received ${signal}, shutting down gracefully...`); + void shutdown(); + }); + } - setTimeout(async () => { - console.log("[SYNC] Starting file sync..."); - try { - await startWatch({ store: options.store, dryRun: false }); - } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - console.error("[SYNC] Sync failed:", errorMessage); - } - }, 1000); - }; + server.setRequestHandler(ListToolsRequestSchema, async () => ({ + tools: options.exposeTools + ? [ + { + name: "search", + description: + "Semantic search over the project's files, indexed by mgrep. Use it for conceptual or cross-file questions where you don't know the exact identifier, such as 'where is authentication handled' or 'how are failed uploads retried'. For exact strings, symbol names or regexes, use rg or grep instead. path is optional and defaults to the session's project directory. Results are file paths with line ranges; read only those ranges. Set web and answer to true for a web lookup.", + inputSchema: { + type: "object", + properties: { + query: { + type: "string", + minLength: 1, + description: "A full natural-language question.", + }, + path: { + type: "string", + description: + "A path relative to the server cwd or an absolute path.", + }, + maxCount: { + type: "integer", + minimum: 1, + multipleOf: 1, + default: 10, + }, + content: { type: "boolean", default: false }, + answer: { type: "boolean", default: false }, + web: { type: "boolean", default: false }, + }, + required: options.watch ? ["query"] : ["query", "path"], + additionalProperties: false, + }, + annotations: { + title: "mgrep semantic search", + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: false, + }, + }, + ] + : [], + })); - startBackgroundSync().catch((error) => { - console.error("[SYNC] Background sync setup failed:", error); + server.setRequestHandler(CallToolRequestSchema, async (request) => { + try { + if (!options.exposeTools || request.params.name !== "search") { + throw new Error(`Unknown tool: ${request.params.name}`); + } + const args = searchArguments.parse(request.params.arguments ?? {}); + const searchRoot = await resolveRoot(args.path); + let watcherStatus: boolean | undefined = false; + if ( + options.watch && + (searchRoot === serverRoot || searchRoot.startsWith(serverRootPrefix)) + ) { + let waitTimer: NodeJS.Timeout | undefined; + try { + watcherStatus = await Promise.race([ + watcherReady, + new Promise((resolve) => { + waitTimer = setTimeout(() => resolve(undefined), syncWaitMs); + }), + ]); + } finally { + clearTimeout(waitTimer); + } + } + const store = await createStore({ interactive: false }); + if (watcherStatus === false && !isAtOrAboveHomeDirectory(searchRoot)) { + let firstSync = firstSyncs.get(searchRoot); + if (!firstSync) { + firstSync = syncFiles( + store, + options.store, + searchRoot, + false, + loadConfig(searchRoot), + ).then(() => undefined); + firstSyncs.set(searchRoot, firstSync); + } + try { + await firstSync; + } catch (error) { + firstSyncs.delete(searchRoot); + throw error; + } + } + let text = await runSearch( + store, + { + ...args, + store: options.store, + path: searchRoot, + rerank: true, + agentic: false, + }, + searchRoot, + ); + if (watcherStatus === undefined) { + text = [ + text, + "Note: the initial mgrep sync is still running; results may be incomplete.", + ] + .filter(Boolean) + .join("\n"); + } + return { content: [{ type: "text", text }] }; + } catch (error) { + const text = error instanceof Error ? error.message : String(error); + return { content: [{ type: "text", text }], isError: true }; + } }); + + await server.connect(transport); + if (closing || !options.watch) return; + + console.log("[SYNC] Scheduling initial sync in 1 second..."); + syncTimer = setTimeout(async () => { + if (isAtOrAboveHomeDirectory(root)) { + console.error( + "[SYNC] Background sync skipped: cwd is at or above the home directory.", + ); + completeWatch(false); + return; + } + console.log("[SYNC] Starting file sync..."); + try { + completeWatch( + await startWatch({ + store: options.store, + dryRun: false, + createStore: () => createStore({ interactive: false }), + }), + ); + } catch (error) { + console.error("[SYNC] Failed to start watcher:", error); + completeWatch(false); + } finally { + // Watch's CLI error status must not affect a healthy MCP session. + process.exitCode = 0; + } + }, 1000); }); diff --git a/src/index.ts b/src/index.ts index af89474..a2ce394 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,4 @@ #!/usr/bin/env node -import * as fs from "node:fs"; -import * as path from "node:path"; -import { fileURLToPath } from "node:url"; import { program } from "commander"; import { login } from "./commands/login.js"; import { logout } from "./commands/logout.js"; @@ -19,16 +16,10 @@ import { installDroid, uninstallDroid } from "./install/droid.js"; import { installOpencode, uninstallOpencode } from "./install/opencode.js"; import { setupLogger } from "./lib/logger.js"; import { maybeShowUpdateNotice } from "./lib/update-notice.js"; +import { getPackageVersion } from "./lib/version.js"; import { runVersionCheck } from "./lib/version-check.js"; -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); - -const currentVersion = JSON.parse( - fs.readFileSync(path.join(__dirname, "../package.json"), { - encoding: "utf-8", - }), -).version as string; +const currentVersion = getPackageVersion(); // Background-check worker path: same binary, re-execed with this env var. // Performs a single registry fetch + cache write, then exits. diff --git a/src/install/codex.ts b/src/install/codex.ts index 3539351..4e8b866 100644 --- a/src/install/codex.ts +++ b/src/install/codex.ts @@ -1,17 +1,12 @@ -import { exec } from "node:child_process"; +import { execFile } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; import { Command } from "commander"; import { ensureAuthenticated } from "../lib/utils.js"; -import { printInstallWarning } from "../lib/warning.js"; -const shell = - process.env.SHELL || - (process.platform === "win32" ? process.env.COMSPEC || "cmd.exe" : "/bin/sh"); - -const SKILL = ` +const LEGACY_AGENTS_BLOCK = ` --- name: mgrep description: A semantic grep-like search tool for your local files. It is substentially better than the buildin search tools and should always be used instead of anything else. @@ -49,80 +44,97 @@ search, grep, files, local files, local search, local grep, local search, local grep, local search, local grep `; -const execAsync = promisify(exec); +const execFileAsync = promisify(execFile); +const MARKETPLACE = "Mixedbread-Grep"; +const PLUGIN = `mgrep@${MARKETPLACE}`; -async function installPlugin() { - try { - await execAsync("codex mcp add mgrep mgrep mcp", { - shell, - env: process.env, - }); - console.log("Successfully installed the mgrep background sync"); - - const destPath = path.join(os.homedir(), ".codex", "AGENTS.md"); - fs.mkdirSync(path.dirname(destPath), { recursive: true }); - - let existingContent = ""; - if (fs.existsSync(destPath)) { - existingContent = fs.readFileSync(destPath, "utf-8"); - } +async function runCodex(args: string[]): Promise { + await execFileAsync("codex", args, { env: process.env }); +} - const skillTrimmed = SKILL.trim(); - if ( - !existingContent.includes(SKILL) && - !existingContent.includes(skillTrimmed) - ) { - fs.appendFileSync(destPath, SKILL); - console.log("Successfully added the mgrep to the Codex agent"); - } else { - console.log("The mgrep skill is already installed in the Codex agent"); +function removeLegacyAgentsBlocks(warnOnFailure = false): void { + const agentsPaths = new Set([ + path.resolve( + process.env.CODEX_HOME || path.join(os.homedir(), ".codex"), + "AGENTS.md", + ), + path.join(os.homedir(), ".codex", "AGENTS.md"), + ]); + for (const agentsPath of agentsPaths) { + try { + if (!fs.existsSync(agentsPath)) continue; + const existingContent = fs.readFileSync(agentsPath, "utf-8"); + let updatedContent = existingContent; + let previousContent = ""; + while (updatedContent !== previousContent) { + previousContent = updatedContent; + updatedContent = updatedContent.replace(LEGACY_AGENTS_BLOCK, ""); + updatedContent = updatedContent.replace(LEGACY_AGENTS_BLOCK.trim(), ""); + } + if (updatedContent === existingContent) continue; + if (updatedContent.trim() === "") fs.unlinkSync(agentsPath); + else fs.writeFileSync(agentsPath, updatedContent); + } catch (error) { + if (!warnOnFailure) throw error; + console.warn( + `Warning: Could not clean legacy instructions in ${agentsPath}: ${error}`, + ); } - - printInstallWarning("Codex", "mgrep uninstall-codex"); - } catch (error) { - console.error(`Error installing plugin: ${error}`); - process.exit(1); } } -async function uninstallPlugin() { +async function installPlugin(): Promise { try { - await execAsync("codex mcp remove mgrep", { shell, env: process.env }); + await runCodex(["mcp", "remove", "mgrep"]).catch(() => undefined); + removeLegacyAgentsBlocks(); + const source = + process.env.MGREP_CODEX_MARKETPLACE || + "https://github.com/mixedbread-ai/mgrep.git"; + try { + await runCodex(["plugin", "marketplace", "add", source]); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!/already (?:added|exists|registered|configured)/i.test(message)) + throw error; + } + await runCodex(["plugin", "add", PLUGIN]); + console.log("Successfully installed the mgrep plugin for Codex"); + console.log( + "On first start, Codex asks you to review the plugin's SessionStart hook (or use /hooks); it only adds search guidance. The plugin's MCP server syncs the session's project directory in the background while Codex runs.", + ); } catch (error) { - console.error(`Error uninstalling plugin: ${error}`); - process.exit(1); + console.error(`Error installing plugin: ${error}`); + process.exitCode = 1; } +} - const destPath = path.join(os.homedir(), ".codex", "AGENTS.md"); - if (fs.existsSync(destPath)) { - const existingContent = fs.readFileSync(destPath, "utf-8"); - let updatedContent = existingContent; - let previousContent = ""; - - while (updatedContent !== previousContent) { - previousContent = updatedContent; - updatedContent = updatedContent.replace(SKILL, ""); - updatedContent = updatedContent.replace(SKILL.trim(), ""); - } - - if (updatedContent.trim() === "") { - fs.unlinkSync(destPath); - } else { - fs.writeFileSync(destPath, updatedContent); +async function uninstallPlugin(): Promise { + for (const args of [ + ["plugin", "remove", PLUGIN], + ["plugin", "marketplace", "remove", MARKETPLACE], + ["mcp", "remove", "mgrep"], + ]) { + try { + await runCodex(args); + } catch (error) { + console.warn(`Warning: Could not run codex ${args.join(" ")}: ${error}`); } } - console.log("Successfully removed the mgrep from the Codex agent"); + removeLegacyAgentsBlocks(true); + console.log("Finished removing the mgrep Codex integration"); } +/** Installs the native mgrep Codex plugin and removes legacy agent instructions. */ export const installCodex = new Command("install-codex") - .description("Install the Codex agent") - .action(async () => { + .description("Install the mgrep plugin for Codex") + .action(async (): Promise => { await ensureAuthenticated(); await installPlugin(); }); +/** Removes the native mgrep Codex plugin and legacy agent instructions. */ export const uninstallCodex = new Command("uninstall-codex") - .description("Uninstall the Codex agent") - .action(async () => { + .description("Uninstall the mgrep plugin for Codex") + .action(async (): Promise => { await uninstallPlugin(); }); diff --git a/src/lib/context.ts b/src/lib/context.ts index 52db3ef..220a4ea 100644 --- a/src/lib/context.ts +++ b/src/lib/context.ts @@ -13,16 +13,29 @@ const BASE_URL = isDevelopment() ? "http://localhost:8000" : "https://api.mixedbread.com"; +/** Options controlling store authentication. */ +export interface CreateStoreOptions { + interactive?: boolean; +} + /** * Creates an authenticated Store instance * Supports authentication via MXBAI_API_KEY env var or OAuth token + * @param options Authentication options; interactive defaults to true. + * Set interactive to false to skip the login prompt and let getJWTToken throw + * when credentials are missing. Test mode always returns a TestStore first. + * @returns The authenticated store, or a TestStore in test mode. */ -export async function createStore(): Promise { +export async function createStore( + options: CreateStoreOptions = {}, +): Promise { if (isTest) { return new TestStore(); } - await ensureAuthenticated(); + if (options.interactive ?? true) { + await ensureAuthenticated(); + } async function createClient() { const jwtToken = await getJWTToken(); diff --git a/src/lib/search.ts b/src/lib/search.ts new file mode 100644 index 0000000..ad0de55 --- /dev/null +++ b/src/lib/search.ts @@ -0,0 +1,189 @@ +import { isAbsolute, join, normalize, relative } from "node:path"; +import type { + AskResponse, + ChunkType, + FileMetadata, + SearchResponse, + Store, +} from "./store.js"; + +export interface RunSearchOptions { + store: string; + query: string; + path: string; + maxCount: number; + content: boolean; + answer: boolean; + web: boolean; + rerank: boolean; + agentic: boolean; +} + +/** + * Resolves the search filter path without changing absolute paths. + * @param root The absolute search root. + * @param execPath An optional relative or absolute search path. + * @returns The absolute path used by the starts_with filter. + */ +export function resolveSearchPath(root: string, execPath?: string): string { + return execPath !== undefined && isAbsolute(execPath) + ? execPath + : normalize(join(root, execPath ?? "")); +} + +/** + * Searches indexed files or generates an answer without syncing files. + * @param store The authenticated store implementation. + * @param options Search, filter, and output options. + * @param root The absolute root used to make result paths relative. + * @returns The formatted text shared by the CLI and MCP tool. + */ +export async function runSearch( + store: Store, + options: RunSearchOptions, + root: string, +): Promise { + const storeIds = options.web + ? [options.store, "mixedbread/web"] + : [options.store]; + const filters = { + all: [ + { key: "path", operator: "starts_with" as const, value: options.path }, + ], + }; + const searchOptions = { + rerank: options.rerank, + ...(options.agentic && { agentic: true }), + }; + + if (options.answer) { + const results = await store.ask( + storeIds, + options.query, + options.maxCount, + searchOptions, + filters, + ); + return formatAskResponse(results, options.content, root); + } + + const results = await store.search( + storeIds, + options.query, + options.maxCount, + searchOptions, + filters, + ); + return formatSearchResponse(results, options.content, root); +} + +function extractSources(response: AskResponse): { [key: number]: ChunkType } { + const sources: { [key: number]: ChunkType } = {}; + const answer = response.answer; + + // Match ALL cite tags and capture the i="..." + const citeTags = answer.match(/= 0 && + end >= start && + end < response.sources.length + ) { + for (let i = start; i <= end; i++) { + sources[i] = response.sources[i]; + } + } + } + + return sources; +} + +function formatAskResponse( + response: AskResponse, + show_content: boolean, + root: string, +): string { + const sources = extractSources(response); + const sourceEntries = Object.entries(sources).map( + ([index, chunk]) => `${index}: ${formatChunk(chunk, show_content, root)}`, + ); + return `${response.answer}\n\n${sourceEntries.join("\n")}`; +} + +function formatSearchResponse( + response: SearchResponse, + show_content: boolean, + root: string, +): string { + return response.data + .map((chunk) => formatChunk(chunk, show_content, root)) + .join("\n"); +} + +function isWebResult(chunk: ChunkType): boolean { + return ( + "filename" in chunk && + typeof chunk.filename === "string" && + chunk.filename.startsWith("http") + ); +} + +function formatChunk( + chunk: ChunkType, + show_content: boolean, + root: string, +): string { + if (isWebResult(chunk) && chunk.type === "text") { + const url = "filename" in chunk ? chunk.filename : "Unknown URL"; + const content = show_content ? chunk.text : ""; + return `${url} (${(chunk.score * 100).toFixed(2)}% match)${content ? `\n${content}` : ""}`; + } + + const filePath = (chunk.metadata as FileMetadata)?.path; + const path = filePath ? `./${relative(root, filePath)}` : ".Unknown path"; + let line_range = ""; + let content = ""; + switch (chunk.type) { + case "text": { + const start_line = (chunk.generated_metadata?.start_line as number) + 1; + const end_line = + start_line + (chunk.generated_metadata?.num_lines as number); + line_range = `:${start_line}-${end_line}`; + content = show_content ? chunk.text : ""; + break; + } + case "image_url": + line_range = + chunk.generated_metadata?.type === "pdf" + ? `, page ${chunk.chunk_index + 1}` + : ""; + break; + case "audio_url": + line_range = ""; + break; + case "video_url": + line_range = ""; + break; + } + + return `${path}${line_range} (${(chunk.score * 100).toFixed(2)}% match)${content ? `\n${content}` : ""}`; +} diff --git a/src/lib/update-notice.ts b/src/lib/update-notice.ts index 589b52e..04e9476 100644 --- a/src/lib/update-notice.ts +++ b/src/lib/update-notice.ts @@ -17,7 +17,7 @@ import { const SUPPRESSED_COMMANDS = new Set([ "watch", - "watch-mcp", + "mcp", "update", "login", "logout", diff --git a/src/lib/version.ts b/src/lib/version.ts new file mode 100644 index 0000000..fe6f6e2 --- /dev/null +++ b/src/lib/version.ts @@ -0,0 +1,11 @@ +import { readFileSync } from "node:fs"; + +/** + * Reads the version from the installed package relative to this module. + * @returns The package version used by the CLI and MCP server. + */ +export function getPackageVersion(): string { + return JSON.parse( + readFileSync(new URL("../../package.json", import.meta.url), "utf-8"), + ).version as string; +} diff --git a/test/codex.bats b/test/codex.bats new file mode 100644 index 0000000..3c994af --- /dev/null +++ b/test/codex.bats @@ -0,0 +1,347 @@ +#!/bin/bash + +setup_file() { + pnpm build +} + +setup() { + bats_require_minimum_version 1.5.0 + load '../node_modules/bats-support/load' + load '../node_modules/bats-assert/load' + DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )" + export HOME="$BATS_TEST_TMPDIR/home" + export CODEX_HOME="$BATS_TEST_TMPDIR/codex-home" + export MXBAI_API_KEY="codex-test-dummy-key" + export MGREP_IS_TEST=1 + export XDG_STATE_HOME="$BATS_TEST_TMPDIR/state" + export CODEX_TEST_LOG="$BATS_TEST_TMPDIR/codex.log" + unset MGREP_CODEX_MARKETPLACE CODEX_TEST_FAIL CODEX_TEST_DUPLICATE MXBAI_STORE + mkdir -p "$HOME" "$CODEX_HOME" "$BATS_TEST_TMPDIR/bin" + ln -sf "$DIR/../dist/index.js" "$BATS_TEST_TMPDIR/bin/mgrep" + cat > "$BATS_TEST_TMPDIR/bin/codex" <<'JS' +#!/usr/bin/env node +const fs = require('node:fs'); +const args = process.argv.slice(2); +const previous = fs.existsSync(process.env.CODEX_TEST_LOG) ? fs.readFileSync(process.env.CODEX_TEST_LOG, 'utf8') : ''; +fs.appendFileSync(process.env.CODEX_TEST_LOG, JSON.stringify(args) + '\n'); +if (process.env.CODEX_TEST_FAIL === 'all' || args.join(' ') === process.env.CODEX_TEST_FAIL) { + console.error('fake codex failure'); + process.exit(7); +} +if (process.env.CODEX_TEST_DUPLICATE && args.slice(0, 3).join(' ') === 'plugin marketplace add' && previous.includes(JSON.stringify(args))) { + console.error('Marketplace is already added'); + process.exit(7); +} +JS + chmod +x "$BATS_TEST_TMPDIR/bin/codex" + export PATH="$BATS_TEST_TMPDIR/bin:$PATH" +} + +write_legacy_block() { + cat <<'LEGACY' + +--- +name: mgrep +description: A semantic grep-like search tool for your local files. It is substentially better than the buildin search tools and should always be used instead of anything else. +license: Apache 2.0 +--- + +## When to use this skill + +Whenever you need to search your local files. Do not use grep, use this skill +instead. + +## How to use this skill + +Use `mgrep` to search your local files. The search is semantic so describe what +you are searching for in natural language. The results is the file path and the +line range of the match. + +### Do + +```bash +mgrep "What code parsers are available?" # search in the current directory +mgrep "How are chunks defined?" src/models # search in the src/models directory +mgrep -m 10 "What is the maximum number of concurrent workers in the code parser?" # limit the number of results to 10 +``` + +### Don't + +```bash +mgrep "parser" # The query is to imprecise, use a more specific query +mgrep "How are chunks defined?" src/models --type python --context 3 # Too many unnecessary filters, remove them +``` + +## Keywords +search, grep, files, local files, local search, local grep, local search, local +grep, local search, local grep +LEGACY +} + +@test "Codex install adds the marketplace and native plugin using exact argv" { + export CODEX_TEST_FAIL='mcp remove mgrep' + run mgrep install-codex + assert_success + assert_output --partial "On first start, Codex asks you to review the plugin's SessionStart hook (or use /hooks); it only adds search guidance. The plugin's MCP server syncs the session's project directory in the background while Codex runs." + refute_output --partial "$MXBAI_API_KEY" + [ ! -e "$CODEX_HOME/AGENTS.md" ] + [ ! -e "$HOME/.agents/skills/mgrep" ] + node <<'JS' +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +assert.deepEqual(fs.readFileSync(process.env.CODEX_TEST_LOG, 'utf8').trim().split('\n').map(JSON.parse), [ + ['mcp', 'remove', 'mgrep'], + ['plugin', 'marketplace', 'add', 'https://github.com/mixedbread-ai/mgrep.git'], + ['plugin', 'add', 'mgrep@Mixedbread-Grep'], +]); +JS +} + +@test "Codex install passes marketplace overrides literally as one argument" { + for source in '/local checkout; $(exit 99)' 'https://example.com/team/mgrep.git'; do + export MGREP_CODEX_MARKETPLACE="$source" + run mgrep install-codex + assert_success + node <<'JS' +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const calls = fs.readFileSync(process.env.CODEX_TEST_LOG, 'utf8').trim().split('\n').map(JSON.parse); +assert.deepEqual(calls.at(-2), ['plugin', 'marketplace', 'add', process.env.MGREP_CODEX_MARKETPLACE]); +JS + done +} + +@test "Codex install removes raw and trimmed legacy blocks in both homes preserving surrounding content" { + write_legacy_block > "$BATS_TEST_TMPDIR/legacy.md" + mkdir -p "$HOME/.codex" + node <<'JS' +const fs = require('node:fs'); +const path = require('node:path'); +const legacy = fs.readFileSync(path.join(process.env.BATS_TEST_TMPDIR, 'legacy.md'), 'utf8'); +for (const root of [process.env.CODEX_HOME, path.join(process.env.HOME, '.codex')]) { + fs.writeFileSync(path.join(root, 'AGENTS.md'), 'Before\n' + legacy + legacy.trim() + '\nBetween\n' + legacy + legacy.trim() + '\nAfter\n'); +} +JS + run mgrep install-codex + assert_success + printf 'Before\n\nBetween\n\nAfter\n' > "$BATS_TEST_TMPDIR/expected.md" + cmp "$CODEX_HOME/AGENTS.md" "$BATS_TEST_TMPDIR/expected.md" + cmp "$HOME/.codex/AGENTS.md" "$BATS_TEST_TMPDIR/expected.md" +} + +@test "Codex install deletes only legacy-only AGENTS and preserves untouched whitespace" { + write_legacy_block > "$CODEX_HOME/AGENTS.md" + mkdir -p "$HOME/.codex" + printf ' \n\t\n' > "$HOME/.codex/AGENTS.md" + cp "$HOME/.codex/AGENTS.md" "$BATS_TEST_TMPDIR/expected.md" + run mgrep install-codex + assert_success + [ ! -e "$CODEX_HOME/AGENTS.md" ] + cmp "$HOME/.codex/AGENTS.md" "$BATS_TEST_TMPDIR/expected.md" +} + +@test "Codex install is idempotent when an already-added marketplace reports failure" { + export CODEX_TEST_DUPLICATE=1 + write_legacy_block > "$CODEX_HOME/AGENTS.md" + run mgrep install-codex + assert_success + run mgrep install-codex + assert_success + [ ! -e "$CODEX_HOME/AGENTS.md" ] + node <<'JS' +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const calls = fs.readFileSync(process.env.CODEX_TEST_LOG, 'utf8').trim().split('\n').map(JSON.parse); +assert.equal(calls.length, 6); +assert.deepEqual(calls.slice(0, 3), calls.slice(3)); +JS +} + +@test "Codex install reports genuine marketplace and plugin failures" { + for step in 'plugin marketplace add https://github.com/mixedbread-ai/mgrep.git' 'plugin add mgrep@Mixedbread-Grep'; do + export CODEX_TEST_FAIL="$step" + run mgrep install-codex + assert_failure 1 + assert_output --partial 'fake codex failure' + done +} + +@test "Codex uninstall invokes every step and cleans both homes after failures" { + export CODEX_TEST_FAIL=all + mkdir -p "$HOME/.codex" + write_legacy_block > "$CODEX_HOME/AGENTS.md" + { printf 'Keep this\n'; write_legacy_block; } > "$HOME/.codex/AGENTS.md" + run mgrep uninstall-codex + assert_success + assert_output --partial 'Warning: Could not run codex plugin remove' + assert_output --partial 'Warning: Could not run codex plugin marketplace remove' + assert_output --partial 'Warning: Could not run codex mcp remove' + [ ! -e "$CODEX_HOME/AGENTS.md" ] + printf 'Keep this\n' > "$BATS_TEST_TMPDIR/expected.md" + cmp "$HOME/.codex/AGENTS.md" "$BATS_TEST_TMPDIR/expected.md" + node <<'JS' +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +assert.deepEqual(fs.readFileSync(process.env.CODEX_TEST_LOG, 'utf8').trim().split('\n').map(JSON.parse), [ + ['plugin', 'remove', 'mgrep@Mixedbread-Grep'], + ['plugin', 'marketplace', 'remove', 'Mixedbread-Grep'], + ['mcp', 'remove', 'mgrep'], +]); +JS +} + +@test "Codex uninstall warns for one failed cleanup file and continues with the other" { + mkdir -p "$CODEX_HOME/AGENTS.md" "$HOME/.codex" + write_legacy_block > "$HOME/.codex/AGENTS.md" + run mgrep uninstall-codex + assert_success + assert_output --partial "Warning: Could not clean legacy instructions in $CODEX_HOME/AGENTS.md" + [ ! -e "$HOME/.codex/AGENTS.md" ] +} + +@test "Codex plugin manifest version and metadata match the package and Claude manifest" { + node - "$DIR/.." <<'JS' +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const read = (name) => JSON.parse(fs.readFileSync(path.join(process.argv[2], name), 'utf8')); +const plugin = read('plugins/mgrep/.codex-plugin/plugin.json'); +const claude = read('plugins/mgrep/.claude-plugin/plugin.json'); +assert.equal(plugin.version, read('package.json').version); +assert.equal(plugin.description, claude.description); +assert.deepEqual(plugin.author, claude.author); +assert.equal(plugin.skills, './codex/skills/'); +assert.equal(plugin.mcpServers, './codex/mcp.json'); +assert.equal(plugin.hooks, './codex/hooks.json'); +assert.deepEqual(read('plugins/mgrep/codex/hooks.json'), { + hooks: { + SessionStart: [{ + matcher: 'startup|resume', + hooks: [{ + type: 'command', + command: 'python3 "${CLAUDE_PLUGIN_ROOT:-$PLUGIN_ROOT}/codex/session_start.py"', + timeout: 5, + }], + }], + }, +}); +assert.equal(fs.existsSync(path.join(process.argv[2], 'plugins/mgrep/hooks/hooks.json')), false); +assert.deepEqual(read('plugins/mgrep/codex/mcp.json').mcpServers.mgrep, { + command: 'mgrep', args: ['mcp', '--expose-tools'], + env_vars: ['MXBAI_API_KEY', 'MXBAI_STORE'], +}); + +JS +} + +assert_session_start_context() { + python3 - "$output" "$1" "${2:-}" <<'PY' +import json +import sys + +raw, cwd, suffix = sys.argv[1:] +assert len(raw.splitlines()) == 1 +assert json.loads(raw) == { + 'hookSpecificOutput': { + 'hookEventName': 'SessionStart', + 'additionalContext': ( + "mgrep semantic search is available through the `search` tool of the mgrep MCP server. " + "Use it for conceptual or cross-file questions where you don't know the exact identifier, " + "such as 'where is authentication handled'. " + f"It searches {cwd} by default. " + "Use rg or grep for exact strings, symbol names and regexes. " + "Both are fine; pick whichever fits the question." + ) + suffix, + }, +} +PY + [ -z "$stderr" ] +} + +@test "Codex SessionStart hook prints one JSON object with the payload cwd" { + local project="$BATS_TEST_TMPDIR/project with spaces" + mkdir -p "$project" + run --separate-stderr python3 "$DIR/../plugins/mgrep/codex/session_start.py" <<< "{\"cwd\":\"$project\"}" + assert_success + assert_session_start_context "$project" +} + +@test "Codex SessionStart hook falls back to the process cwd on empty stdin" { + local project="$BATS_TEST_TMPDIR/project" + mkdir -p "$project" + cd "$project" + run --separate-stderr python3 "$DIR/../plugins/mgrep/codex/session_start.py" < /dev/null + assert_success + assert_session_start_context "$(pwd -P)" +} + +@test "Codex SessionStart hook falls back on invalid payloads and unusable cwd values" { + local project="$BATS_TEST_TMPDIR/project" + mkdir -p "$project" + cd "$project" + for payload in 'not JSON' '{' 'null' '[]' '42' '"text"' '{}' '{"cwd":null}' '{"cwd":42}' '{"cwd":""}' '{"cwd":"\u0000"}'; do + run --separate-stderr python3 "$DIR/../plugins/mgrep/codex/session_start.py" <<< "$payload" + assert_success + assert_session_start_context "$(pwd -P)" + done +} + +@test "Codex SessionStart hook makes a relative payload cwd absolute" { + local project="$BATS_TEST_TMPDIR/project" + mkdir -p "$project" + cd "$project" + run --separate-stderr python3 "$DIR/../plugins/mgrep/codex/session_start.py" <<< '{"cwd":"nested/../source"}' + assert_success + assert_session_start_context "$(pwd -P)/source" +} + +@test "Codex SessionStart hook includes the cached update notice" { + mkdir -p "$HOME/.mgrep" + printf '%s\n' '{"latestVersion":"9.9.9"}' > "$HOME/.mgrep/update-check.json" + run --separate-stderr python3 "$DIR/../plugins/mgrep/codex/session_start.py" < /dev/null + assert_success + assert_session_start_context "$(pwd -P)" ' An mgrep update is available (latest: 9.9.9). Tell the user to run `mgrep update` to upgrade.' +} + +@test "Codex SessionStart hook ignores malformed update notices" { + mkdir -p "$HOME/.mgrep" + for cache in 'not JSON' 'null' '[]' '{}' '{"latestVersion":42}'; do + printf '%s\n' "$cache" > "$HOME/.mgrep/update-check.json" + run --separate-stderr python3 "$DIR/../plugins/mgrep/codex/session_start.py" < /dev/null + assert_success + assert_session_start_context "$(pwd -P)" + done +} + +@test "Codex SessionStart hook creates no files or changes to existing files" { + python3 - "$DIR/../plugins/mgrep/codex/session_start.py" <<'PY' +import json +import os +from pathlib import Path +import subprocess +import sys + +hook = Path(sys.argv[1]).resolve() +root = Path(os.environ['BATS_TEST_TMPDIR']) +project = root / 'project' +project.mkdir() +cache = Path.home() / '.mgrep' / 'update-check.json' +cache.parent.mkdir() +cache.write_text(json.dumps({'latestVersion': '9.9.9'})) + +def snapshot(): + return { + str(path): path.read_bytes() if path.is_file() else None + for tree in [root, hook.parent] + for path in tree.rglob('*') + } + +before = snapshot() +for payload in ['', 'not JSON', json.dumps({'cwd': str(project), 'session_id': 'steering-only'})]: + result = subprocess.run([sys.executable, str(hook)], input=payload, cwd=project, text=True, capture_output=True) + assert result.returncode == 0, result + assert result.stderr == '', result.stderr + assert json.loads(result.stdout)['hookSpecificOutput']['hookEventName'] == 'SessionStart' + assert snapshot() == before +PY +} diff --git a/test/helpers/mcp-client.mjs b/test/helpers/mcp-client.mjs new file mode 100644 index 0000000..32922ee --- /dev/null +++ b/test/helpers/mcp-client.mjs @@ -0,0 +1,109 @@ +import assert from "node:assert/strict"; +import { writeFileSync } from "node:fs"; +import { setTimeout as delay } from "node:timers/promises"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"; +import { ListRootsRequestSchema } from "@modelcontextprotocol/sdk/types.js"; + +const args = process.argv.slice(2); +const separator = args.indexOf("--"); +if (separator < 0) { + throw new Error( + "Usage: mcp-client.mjs initialize|list|call |calls |close|eof -- ", + ); +} +const [action, name, json] = args.slice(0, separator); +const transport = new StdioClientTransport({ + command: "mgrep", + args: args.slice(separator + 1), + env: process.env, + stderr: "pipe", +}); +const roots = process.env.MGREP_MCP_ROOTS; +const client = new Client( + { name: "mgrep-test", version: "1.0.0" }, + { + capabilities: roots !== undefined ? { roots: {} } : {}, + }, +); +if (roots !== undefined) { + client.setRequestHandler(ListRootsRequestSchema, async () => { + if (process.env.MGREP_MCP_FORBID_ROOTS) + throw new Error("Unexpected roots/list"); + return { roots: JSON.parse(roots) }; + }); +} +let stderr = ""; +transport.stderr.on("data", (chunk) => { + stderr += chunk.toString(); +}); +const timeout = setTimeout(() => { + console.error(`MCP client timed out\n${stderr}`); + void client.close().finally(() => process.exit(1)); +}, 15000); + +try { + await client.connect(transport); + const waitFor = process.env.MGREP_MCP_WAIT_FOR; + while (waitFor && !stderr.includes(waitFor)) { + if (transport.pid === null) throw new Error("MCP server exited early"); + await delay(25); + } + + let result; + switch (action) { + case "initialize": + result = { instructions: client.getInstructions() }; + break; + case "list": + result = await client.listTools(); + break; + case "call-with-logs": + case "call": + result = await client.callTool({ name, arguments: JSON.parse(json) }); + await client.ping(); + break; + case "calls-with-logs": + case "calls": + result = []; + for (const step of JSON.parse(name)) { + if (step.write) writeFileSync(step.write.path, step.write.text); + else + result.push( + await client.callTool({ name: "search", arguments: step }), + ); + } + await client.ping(); + break; + case "idle": + await delay(1500); + await client.ping(); + result = { stderr }; + break; + case "close": + case "eof": { + // SDK 1.22 close() sends SIGTERM; use its child stdin to test EOF separately. + const child = transport._process; + const exited = new Promise((resolve) => { + child.once("exit", (code, signal) => resolve([code, signal])); + }); + if (action === "eof") child.stdin.end(); + else await client.close(); + const [code, signal] = await exited; + assert.equal(code, 0); + assert.equal(signal, null); + result = { exited: true, code, signal }; + break; + } + default: + throw new Error(`Unknown action: ${action}`); + } + if (action.endsWith("-with-logs")) result = { result, stderr }; + console.log(JSON.stringify(result)); +} catch (error) { + console.error(stderr); + throw error; +} finally { + clearTimeout(timeout); + await client.close(); +} diff --git a/test/mcp.bats b/test/mcp.bats new file mode 100644 index 0000000..e431543 --- /dev/null +++ b/test/mcp.bats @@ -0,0 +1,535 @@ +#!/bin/bash + +setup_file() { + pnpm build +} + +setup() { + load '../node_modules/bats-support/load' + load '../node_modules/bats-assert/load' + + DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )" + mkdir -p "$BATS_TEST_TMPDIR/bin" + ln -sf "$DIR/../dist/index.js" "$BATS_TEST_TMPDIR/bin/mgrep" + export PATH="$BATS_TEST_TMPDIR/bin:$PATH" + export MGREP_IS_TEST=1 + export MGREP_TEST_STORE_PATH="$BATS_TEST_TMPDIR/mgrep-test-store.json" + export XDG_STATE_HOME="$BATS_TEST_TMPDIR/state" + unset MGREP_MCP_WAIT_FOR MGREP_MCP_ROOTS MGREP_MCP_FORBID_ROOTS MGREP_MCP_SYNC_WAIT_MS + + mkdir -p "$BATS_TEST_TMPDIR/test-store" + printf 'Hello, world!\nThis is a test file.\n' > "$BATS_TEST_TMPDIR/test-store/test.txt" + printf 'Hello, world!\nA second one.\n' > "$BATS_TEST_TMPDIR/test-store/test-2.txt" + printf 'Hello, world!\nA third one.\n' > "$BATS_TEST_TMPDIR/test-store/test-3.txt" + cd "$BATS_TEST_TMPDIR/test-store" + mgrep search --sync test +} + +mcp_client() { + node "$DIR/helpers/mcp-client.mjs" "$@" +} + +@test "MCP lists no tools without opt-in" { + run mcp_client list -- mcp + assert_success + assert_equal "$output" '{"tools":[]}' +} + +@test "MCP exposes one read-only search tool with the specified schema" { + run mcp_client list -- mcp --expose-tools + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const { tools } = JSON.parse(process.argv[2]); +assert.equal(tools.length, 1); +assert.equal(tools[0].name, 'search'); +assert.deepEqual(tools[0].annotations, { + title: 'mgrep semantic search', readOnlyHint: true, destructiveHint: false, + idempotentHint: true, openWorldHint: false, +}); +assert.deepEqual(tools[0].inputSchema.required, ['query']); +const properties = tools[0].inputSchema.properties; +assert.equal(Object.hasOwn(properties.path, "default"), false); +assert.equal(properties.maxCount.type, "integer"); +assert.equal(tools[0].description, "Semantic search over the project's files, indexed by mgrep. Use it for conceptual or cross-file questions where you don't know the exact identifier, such as 'where is authentication handled' or 'how are failed uploads retried'. For exact strings, symbol names or regexes, use rg or grep instead. path is optional and defaults to the session's project directory. Results are file paths with line ranges; read only those ranges. Set web and answer to true for a web lookup."); +assert.equal(properties.maxCount.default, 10); +for (const key of ['content', 'answer', 'web']) assert.equal(properties[key].default, false); +JS +} + +@test "MCP search returns fixture text and matches CLI output" { + local expected + expected="$(mgrep search --content test)" + run mcp_client call search '{"query":"test","content":true}' -- mcp --expose-tools + assert_success + assert_output --partial 'test.txt' + node - "$output" "$expected" <<'JS' +const assert = require('node:assert/strict'); +assert.deepEqual(JSON.parse(process.argv[2]), { content: [{ type: 'text', text: process.argv[3] }] }); +JS +} + +@test "MCP search supports answer, web, relative and absolute paths and maxCount" { + local expected + expected="$(mgrep search --answer --content --web test .)" + run mcp_client call search '{"query":"test","path":".","answer":true,"content":true,"web":true}' -- --store custom-store mcp --expose-tools + assert_success + node - "$output" "$expected" <<'JS' +const assert = require('node:assert/strict'); +assert.equal(JSON.parse(process.argv[2]).content[0].text, process.argv[3]); +JS + run mcp_client call search "{\"query\":\"Hello\",\"path\":\"$PWD\",\"maxCount\":1}" -- mcp --expose-tools + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +assert.equal(JSON.parse(process.argv[2]).content[0].text.split('\n').length, 1); +JS +} + +@test "MCP unknown and disabled tools return errors without crashing" { + run mcp_client call unknown '{}' -- mcp --expose-tools + assert_success + assert_output --partial '"isError":true' + assert_output --partial 'Unknown tool: unknown' + run mcp_client call search '{"query":"test"}' -- mcp + assert_success + assert_output --partial '"isError":true' + assert_output --partial 'Unknown tool: search' +} + +@test "MCP invalid arguments return errors without crashing" { + local arguments + for arguments in '{}' '{"query":4}' '{"query":""}' '{"query":"test","path":4}' '{"query":"test","maxCount":0}' '{"query":"test","maxCount":1.5}' '{"query":"test","content":"true"}' '{"query":"test","sync":true}'; do + run mcp_client call search "$arguments" -- mcp --expose-tools + assert_success + assert_output --partial '"isError":true' + done +} + +@test "MCP first search syncs files without a watcher" { + printf 'unsynced-needle\n' > new.txt + run mcp_client call search "{\"query\":\"unsynced-needle\",\"path\":\"$PWD\"}" -- mcp --expose-tools --no-watch + assert_success + assert_output --partial './new.txt' +} + +@test "MCP survives watcher file count failure" { + printf 'new file\n' > new.txt + printf 'another new file\n' > another.txt + export MGREP_MAX_FILE_COUNT=1 + run mgrep watch + assert_failure 1 + assert_output --partial '❌ Files to sync (2) exceeds the maximum allowed (1). No files were synced.' + refute_output --partial 'Failed to start watcher:' + + export MGREP_MCP_WAIT_FOR='❌ Files to sync' + run mcp_client call search '{"query":"test"}' -- mcp --expose-tools + assert_success + assert_output --partial '"isError":true' + assert_output --partial 'exceeds the maximum allowed' +} + +@test "MCP skips background sync at and above home while serving search" { + export HOME="$(pwd -P)" + export MGREP_MCP_WAIT_FOR='Background sync skipped:' + run mcp_client call search '{"query":"test"}' -- mcp --expose-tools + assert_success + assert_output --partial 'test.txt' + export HOME="$(pwd -P)/child-home" + run mcp_client call search '{"query":"test"}' -- mcp --expose-tools + assert_success + assert_output --partial 'test.txt' +} + +@test "MCP authentication failure returns an error and watcher failure keeps serving" { + export MGREP_IS_TEST=0 + export HOME="$BATS_TEST_TMPDIR/empty-home" + mkdir -p "$HOME" + unset MXBAI_API_KEY + export MGREP_MCP_WAIT_FOR='Failed to start watcher:' + run mcp_client call search '{"query":"test"}' -- mcp --expose-tools + assert_success + assert_output --partial '"isError":true' + assert_output --partial 'No authentication token found' +} + +@test "MCP exits cleanly after client.close with active watchers" { + export MGREP_MCP_WAIT_FOR='[LOG] Watching for file changes in' + run mcp_client close -- mcp --expose-tools + assert_success + assert_equal "$output" '{"exited":true,"code":0,"signal":null}' +} + +@test "MCP exits cleanly on stdin EOF with active watchers" { + export MGREP_MCP_WAIT_FOR='[LOG] Watching for file changes in' + run mcp_client eof -- mcp --expose-tools + assert_success + assert_equal "$output" '{"exited":true,"code":0,"signal":null}' +} + +@test "MCP exits cleanly on stdin EOF before the watcher starts" { + run mcp_client eof -- mcp + assert_success + assert_equal "$output" '{"exited":true,"code":0,"signal":null}' +} + +@test "MCP survives watcher quota failure and returns search quota errors" { + local preload + preload="$(node --input-type=module - "$DIR/../dist" <<'JS' +import { pathToFileURL } from 'node:url'; +const base = pathToFileURL(`${process.argv[2]}/`).href; +const code = ` +import { TestStore } from ${JSON.stringify(`${base}lib/store.js`)}; +import ${JSON.stringify(`${base}lib/auth.js`)}; +import { QuotaExceededError } from ${JSON.stringify(`${base}lib/utils.js`)}; +TestStore.prototype.uploadFile = async () => { throw new QuotaExceededError('quota test'); }; +const search = TestStore.prototype.search; +TestStore.prototype.search = async function (...args) { + if (args[1] === 'quota') throw new QuotaExceededError('search quota test'); + return search.apply(this, args); +};`; +console.log(`data:text/javascript;base64,${Buffer.from(code).toString('base64')}`); +JS +)" + printf 'new file\n' > new.txt + export NODE_OPTIONS="--import=$preload" + run mgrep watch + assert_failure 1 + assert_output --partial "❌ Free tier quota exceeded. You've reached the monthly limit of 2,000,000 store tokens." + refute_output --partial 'Failed to start watcher:' + + export MGREP_MCP_WAIT_FOR='❌ Free tier quota exceeded.' + run mcp_client call search '{"query":"test"}' -- mcp --expose-tools + assert_success + assert_output --partial '"isError":true' + assert_output --partial 'quota test' + unset MGREP_MCP_WAIT_FOR + export HOME="$(pwd -P)" + run mcp_client call search '{"query":"quota"}' -- mcp --expose-tools + assert_success + assert_equal "$output" '{"content":[{"type":"text","text":"search quota test"}],"isError":true}' +} + +@test "Shared search forwards stores, filters, options and preserves formatting" { + node --input-type=module - "$DIR/../dist/lib/search.js" <<'JS' +import assert from 'node:assert/strict'; +import { pathToFileURL } from 'node:url'; +const { resolveSearchPath, runSearch } = await import(pathToFileURL(process.argv[2])); +const root = process.cwd(); +assert.equal(resolveSearchPath(root), root); +assert.equal(resolveSearchPath(root, 'a/../b'), `${root}/b`); +assert.equal(resolveSearchPath(root, '/absolute/../unchanged'), '/absolute/../unchanged'); +const chunk = { + type: 'text', text: 'fixture', score: 0.75, metadata: { path: `${root}/test.txt` }, + generated_metadata: { start_line: 0, num_lines: 2 }, +}; +const web = { type: 'text', text: 'web fixture', score: 1, filename: 'https://example.com' }; +let received; +const store = { + search: async (...args) => { received = args; return { data: [chunk, web] }; }, + ask: async (...args) => { received = args; return { answer: '', sources: [chunk, web] }; }, +}; +const options = { + store: 'custom', query: 'question', path: root, maxCount: 3, content: true, + answer: false, web: true, rerank: true, agentic: false, +}; +assert.equal(await runSearch(store, options, root), './test.txt:1-3 (75.00% match)\nfixture\nhttps://example.com (100.00% match)\nweb fixture'); +assert.deepEqual(received, [['custom', 'mixedbread/web'], 'question', 3, { rerank: true }, + { all: [{ key: 'path', operator: 'starts_with', value: root }] }]); +assert.equal(await runSearch(store, { ...options, answer: true, web: false, rerank: false, agentic: true }, root), + '\n\n0: ./test.txt:1-3 (75.00% match)\nfixture\n1: https://example.com (100.00% match)\nweb fixture'); +assert.deepEqual(received.slice(0, 4), [['custom'], 'question', 3, { rerank: false, agentic: true }]); +JS +} + +@test "MCP no-watch requires path in its schema and errors without a root" { + run mcp_client list -- mcp --expose-tools --no-watch + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const schema = JSON.parse(process.argv[2]).tools[0].inputSchema; +assert.deepEqual(schema.required, ['query', 'path']); +assert.equal(Object.hasOwn(schema.properties.path, 'default'), false); +JS + run mcp_client call search '{"query":"test"}' -- mcp --expose-tools --no-watch + assert_success + assert_output --partial '"isError":true' + assert_output --partial 'An absolute project path is required' +} + +@test "MCP no-watch never schedules or starts background sync" { + run mcp_client idle -- mcp --expose-tools --no-watch + assert_success + assert_equal "$output" '{"stderr":""}' +} + +@test "MCP uses the first file root and makes results relative to it" { + mkdir -p "$BATS_TEST_TMPDIR/project/src" "$BATS_TEST_TMPDIR/other" + printf 'root-needle\n' > "$BATS_TEST_TMPDIR/project/src/x.ts" + printf 'root-needle\n' > "$BATS_TEST_TMPDIR/other/wrong.txt" + export MGREP_MCP_ROOTS="[{\"uri\":\"https://example.com\"},{\"uri\":\"file://$BATS_TEST_TMPDIR/project\"},{\"uri\":\"file://$BATS_TEST_TMPDIR/other\"}]" + run mcp_client call search '{"query":"root-needle"}' -- mcp --expose-tools --no-watch + assert_success + assert_output --partial './src/x.ts:1-2' + refute_output --partial 'wrong.txt' + refute_output --partial "$BATS_TEST_TMPDIR" +} + +@test "MCP explicit absolute and relative paths win over client roots" { + mkdir -p alternate/src + printf 'explicit-needle\n' > alternate/src/x.ts + export MGREP_MCP_ROOTS="[{\"uri\":\"file://$BATS_TEST_TMPDIR/missing\"}]" + export MGREP_MCP_FORBID_ROOTS=1 + for root in './alternate' "$PWD/alternate"; do + run mcp_client call search "{\"query\":\"explicit-needle\",\"path\":\"$root\"}" -- mcp --expose-tools --no-watch + assert_success + assert_output --partial './src/x.ts:1-2' + refute_output --partial '"isError":true' + done +} + +@test "MCP roots with no file URI fall back to cwd only with watch enabled" { + export MGREP_MCP_ROOTS='[{"uri":"https://example.com"}]' + run mcp_client call search '{"query":"test"}' -- mcp --expose-tools + assert_success + assert_output --partial './test.txt' + run mcp_client call search '{"query":"test"}' -- mcp --expose-tools --no-watch + assert_success + assert_output --partial 'An absolute project path is required' +} + +@test "MCP first-search sync runs once per resolved root during a server lifetime" { + mkdir -p alternate + run mcp_client calls "[{\"query\":\"test\",\"path\":\"$PWD\"},{\"write\":{\"path\":\"$PWD/later.txt\",\"text\":\"later-needle\"}},{\"query\":\"later-needle\",\"path\":\".\"},{\"write\":{\"path\":\"$PWD/alternate/second.txt\",\"text\":\"second-root-needle\"}},{\"query\":\"second-root-needle\",\"path\":\"alternate\"}]" -- mcp --expose-tools --no-watch + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const results = JSON.parse(process.argv[2]); +assert.match(results[0].content[0].text, /test.txt/); +assert.equal(results[1].content[0].text, ''); +assert.match(results[2].content[0].text, /^\.\/second.txt:1-2/); +JS +} + +@test "MCP first-search sync skips roots at and above home with no watcher" { + printf 'unsynced-home-needle\n' > new.txt + for root in "$PWD" "$PWD/.."; do + export HOME="$(pwd -P)" + run mcp_client call search "{\"query\":\"unsynced-home-needle\",\"path\":\"$root\"}" -- mcp --expose-tools --no-watch + assert_success + assert_equal "$output" '{"content":[{"type":"text","text":""}]}' + done +} + +@test "MCP initialize instructions describe semantic search and include the server cwd" { + run mcp_client initialize -- mcp --expose-tools + assert_success + node - "$output" "$(pwd -P)" <<'JS' +const assert = require('node:assert/strict'); +assert.equal(JSON.parse(process.argv[2]).instructions, `mgrep semantic search is available through this server's \`search\` tool. Use it for conceptual or cross-file questions where you don't know the exact identifier, such as 'where is authentication handled'. It searches ${process.argv[3]} by default. Use rg or grep for exact strings, symbol names and regexes. Both are fine; pick whichever fits the question.`); +JS +} + +@test "MCP optional path waits for the initial watcher sync before searching cwd" { + printf 'watcher-needle\n' > new.txt + run mcp_client call-with-logs search '{"query":"watcher-needle"}' -- mcp --expose-tools + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const { result, stderr } = JSON.parse(process.argv[2]); +assert.match(result.content[0].text, /\.\/new.txt/); +assert.equal(result.isError, undefined); +assert.equal((stderr.match(/Initial sync complete/g) || []).length, 1); +assert.doesNotMatch(stderr, /Indexing complete/); +JS +} + +@test "MCP watcher per-file upload errors do not trigger a second sync" { + local preload + preload="$(node --input-type=module - "$DIR/../dist" <<'JS' +import { pathToFileURL } from 'node:url'; +const module = pathToFileURL(`${process.argv[2]}/lib/store.js`).href; +const code = `import { TestStore } from ${JSON.stringify(module)}; +const listFiles = TestStore.prototype.listFiles; +TestStore.prototype.listFiles = async function* (...args) { + console.error('test sync pass'); + yield* listFiles.apply(this, args); +}; +TestStore.prototype.uploadFile = async (_storeId, file) => { + for await (const chunk of file) {} + throw new Error('per-file upload test'); +};`; +console.log(`data:text/javascript;base64,${Buffer.from(code).toString('base64')}`); +JS +)" + printf 'failed upload\n' > new.txt + export NODE_OPTIONS="--import=$preload" + run mcp_client call-with-logs search '{"query":"test"}' -- mcp --expose-tools + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const { result, stderr } = JSON.parse(process.argv[2]); +assert.match(result.content[0].text, /\.\/test.txt/); +assert.equal(result.isError, undefined); +assert.match(stderr, /Initial sync complete.*errors 1/); +assert.match(stderr, /1 file\(s\) failed to upload/); +assert.match(stderr, /\[LOG\] Watching for file changes in/); +assert.equal((stderr.match(/test sync pass/g) || []).length, 1); +assert.doesNotMatch(stderr, /Indexing complete/); +assert.doesNotMatch(result.content[0].text, /Note:/); +JS +} + +@test "MCP search times out waiting for initial sync and searches the existing index without syncing again" { + local preload + preload="$(node --input-type=module - "$DIR/../dist" <<'JS' +import { pathToFileURL } from 'node:url'; +const module = pathToFileURL(`${process.argv[2]}/lib/store.js`).href; +const code = `import { TestStore } from ${JSON.stringify(module)}; +const listFiles = TestStore.prototype.listFiles; +TestStore.prototype.listFiles = async function* (...args) { + console.error('test sync pass'); + yield* listFiles.apply(this, args); +}; +TestStore.prototype.uploadFile = async (_storeId, file) => { + for await (const chunk of file) {} + console.error('test upload blocked'); + await new Promise(() => {}); +};`; +console.log(`data:text/javascript;base64,${Buffer.from(code).toString('base64')}`); +JS +)" + printf 'not-indexed-needle\n' > new.txt + export NODE_OPTIONS="--import=$preload" + export MGREP_MCP_WAIT_FOR='test upload blocked' + export MGREP_MCP_SYNC_WAIT_MS=50 + run mcp_client calls-with-logs '[{"query":"test"},{"query":"not-indexed-needle","path":"."}]' -- mcp --expose-tools + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const { result, stderr } = JSON.parse(process.argv[2]); +const note = 'Note: the initial mgrep sync is still running; results may be incomplete.'; +assert.match(result[0].content[0].text, /\.\/test.txt/); +for (const entry of result) { + assert.equal(entry.isError, undefined); + assert.equal(entry.content[0].text.split('\n').at(-1), note); +} +assert.equal(result[1].content[0].text, note); +assert.equal((stderr.match(/test sync pass/g) || []).length, 1); +assert.equal((stderr.match(/test upload blocked/g) || []).length, 1); +assert.doesNotMatch(stderr, /Initial sync complete|Indexing complete/); +JS +} + +@test "MCP searches inside cwd do not sync again once the watcher is active" { + mkdir -p nested + printf 'nested-needle\n' > nested/child.txt + export MGREP_MCP_WAIT_FOR='[LOG] Watching for file changes in' + run mcp_client calls-with-logs '[{"query":"nested-needle","path":"nested"},{"query":"nested-needle"},{"query":"nested-needle","path":"."}]' -- mcp --expose-tools + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const { result, stderr } = JSON.parse(process.argv[2]); +assert.match(result[0].content[0].text, /^\.\/child.txt/); +for (const entry of result.slice(1)) assert.match(entry.content[0].text, /^\.\/nested\/child.txt/); +assert.equal((stderr.match(/Initial sync complete/g) || []).length, 1); +assert.doesNotMatch(stderr, /Indexing complete/); +JS +} + +@test "MCP searches outside cwd keep first-search sync per root with a watcher" { + mkdir -p ../test-store-sibling + printf 'outside-needle\n' > ../test-store-sibling/outside.txt + export MGREP_MCP_WAIT_FOR='[LOG] Watching for file changes in' + run mcp_client calls-with-logs '[{"query":"outside-needle","path":"../test-store-sibling"},{"query":"outside-needle","path":"../test-store-sibling"}]' -- mcp --expose-tools + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const { result, stderr } = JSON.parse(process.argv[2]); +for (const entry of result) assert.match(entry.content[0].text, /^\.\/outside.txt/); +assert.equal((stderr.match(/Initial sync complete/g) || []).length, 1); +assert.equal((stderr.match(/Indexing complete/g) || []).length, 1); +JS +} + +@test "MCP failed watcher falls back to first-search sync once per root" { + local preload + preload="$(node --input-type=module - "$DIR/../dist" <<'JS' +import { pathToFileURL } from 'node:url'; +const module = pathToFileURL(`${process.argv[2]}/lib/store.js`).href; +const code = `import { TestStore } from ${JSON.stringify(module)}; +TestStore.prototype.retrieve = async () => { throw new Error('watcher startup test'); }; +TestStore.prototype.create = async () => { throw new Error('watcher startup test'); };`; +console.log(`data:text/javascript;base64,${Buffer.from(code).toString('base64')}`); +JS +)" + printf 'fallback-needle\n' > new.txt + export NODE_OPTIONS="--import=$preload" + run mcp_client calls-with-logs '[{"query":"fallback-needle"},{"query":"fallback-needle","path":"."}]' -- mcp --expose-tools + assert_success + node - "$output" <<'JS' +const assert = require('node:assert/strict'); +const { result, stderr } = JSON.parse(process.argv[2]); +for (const entry of result) { + assert.match(entry.content[0].text, /^\.\/new.txt/); + assert.equal(entry.isError, undefined); +} +assert.match(stderr, /Failed to start watcher: watcher startup test/); +assert.equal((stderr.match(/Indexing complete/g) || []).length, 1); +JS +} + +@test "startWatch reports readiness after complete sync and returns false for startup failures or dry run" { + local scenario + for scenario in ready dry-run home quota file-count store-error watch-error async-watch-error partial-sync; do + run node --input-type=module - "$DIR/../dist" "$scenario" <<'JS' +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import fs from 'node:fs'; +import { syncBuiltinESMExports } from 'node:module'; +import { pathToFileURL } from 'node:url'; +const base = pathToFileURL(`${process.argv[2]}/`).href; +const { startWatch } = await import(`${base}commands/watch.js`); +const { TestStore } = await import(`${base}lib/store.js`); +const { QuotaExceededError } = await import(`${base}lib/utils.js`); +const scenario = process.argv[3]; +const store = new TestStore(); +const watchers = []; +fs.watch = () => { + if (scenario === 'watch-error') throw new Error('watch setup test'); + const watcher = new EventEmitter(); + watcher.close = () => {}; + watchers.push(watcher); + if (scenario === 'async-watch-error') process.nextTick(() => watcher.emit('error', new Error('async watch setup test'))); + return watcher; +}; +syncBuiltinESMExports(); +if (scenario === 'home') process.env.HOME = process.cwd(); +if (['quota', 'file-count', 'partial-sync'].includes(scenario)) { + fs.writeFileSync('contract-a.txt', 'contract fixture'); + fs.writeFileSync('contract-b.txt', 'contract fixture'); +} +if (['quota', 'partial-sync'].includes(scenario)) store.uploadFile = async (_storeId, file) => { + for await (const chunk of file) {} + if (scenario === 'quota') throw new QuotaExceededError('contract quota'); + throw new Error('contract upload'); +}; +const result = await startWatch({ + store: 'test-store', dryRun: scenario === 'dry-run', + maxFileCount: scenario === 'file-count' ? 1 : undefined, + createStore: async () => { + if (scenario === 'store-error') throw new Error('contract store'); + return store; + }, +}); +const watching = ['ready', 'partial-sync'].includes(scenario); +assert.equal(result, watching, scenario); +if (watching) assert.ok(watchers.length > 0); +assert.equal(process.exitCode || 0, ['home', 'quota', 'file-count', 'store-error'].includes(scenario) ? 1 : 0, scenario); +process.exitCode = 0; +fs.rmSync('contract-a.txt', { force: true }); +fs.rmSync('contract-b.txt', { force: true }); +JS + assert_success + done +} diff --git a/test/test.bats b/test/test.bats index d3cf846..c8d4df8 100755 --- a/test/test.bats +++ b/test/test.bats @@ -46,7 +46,7 @@ teardown() { run mgrep --version assert_success - assert_output --regexp '^[0-9]+\.[0-9]+\.[0-9]+$' + assert_equal "$output" "$(node -p "JSON.parse(require('node:fs').readFileSync('$DIR/../package.json', 'utf8')).version")" } @test "Search" { From 175350eb6987a6e767ca6bbaa4ae21deb6f2d97c Mon Sep 17 00:00:00 2001 From: David Dodlek Date: Tue, 6 Oct 2026 19:42:34 +0200 Subject: [PATCH 2/2] feat(watch): one active watcher per directory tree across sessions Every agent session starts its own `mgrep watch` (Claude hook) or MCP server watcher (Codex). Sessions in /repo and /repo/services ran two watchers that both hashed and uploaded everything under /repo/services. Watchers on the same store now coordinate through a claim registry in ~/.mgrep/watchers (O_EXCL claim files, building on the lock design of the unmerged prevent-multiple-watches-on-same-dir branch): - The first started watcher wins. A watcher whose directory is covered by an earlier active watcher stands by: no initial sync, no fs.watch, no uploads. It takes over when that watcher exits. - An active watcher excludes subtrees owned by earlier active watchers inside its tree, and gives them back when they exit. - A directory with its own .mgreprc.yaml/.yml is a project boundary: it is never covered by an ancestor, and ancestors neither upload nor delete files inside it. - Claims of dead processes, and unparsable claims older than 30 s, are pruned. Claims are released on exit and SIGTERM; the Claude SessionEnd hook now sends SIGTERM instead of SIGKILL. In the MCP server a covered watcher resolves immediately, so searches do not wait for a sync that another session already owns. A signalled CLI watcher exits immediately, also during a sync. After a quota error, poll-time re-syncs back off for 10 minutes (MGREP_WATCH_QUOTA_BACKOFF_MS) and the quota message prints once. --- README.md | 14 + plugins/mgrep/hooks/mgrep_watch_kill.py | 2 +- src/commands/watch.ts | 482 +++++++++++++----- src/commands/watch_mcp.ts | 1 + src/lib/config.ts | 12 +- src/lib/lock.ts | 247 +++++++++ src/lib/utils.ts | 54 +- test/mcp.bats | 143 ++++++ test/watch-coordination.bats | 644 ++++++++++++++++++++++++ 9 files changed, 1473 insertions(+), 126 deletions(-) create mode 100644 src/lib/lock.ts create mode 100644 test/watch-coordination.bats diff --git a/README.md b/README.md index e2d4ea6..970ccbd 100644 --- a/README.md +++ b/README.md @@ -283,6 +283,20 @@ mgrep watch --max-file-size 1048576 # limit uploads to files under 1MB mgrep watch --max-file-count 5000 # limit sync to 5000 changed files or fewer ``` +#### Multiple sessions + +Watchers using the same store coordinate across sessions: the first started +watcher owns each directory tree. Later sessions covered by it stand by without +syncing or watching files, and take over when it exits. A parent started after a +child watcher leaves that child's subtree to the child, then takes it over if +the child exits. + +A folder containing `.mgreprc.yaml` or `.mgreprc.yml` is a configuration boundary: +ancestor watchers always exclude it, so a session in that folder gets its own +watcher. Boundaries are computed when a watcher becomes active, including on +takeover; restart watchers after adding or removing local config files. +Dry runs skip coordination. + ## Mixedbread under the hood - Every file is pushed into a Mixedbread Store using the same SDK your apps get. diff --git a/plugins/mgrep/hooks/mgrep_watch_kill.py b/plugins/mgrep/hooks/mgrep_watch_kill.py index 0bea2a7..fb5b461 100644 --- a/plugins/mgrep/hooks/mgrep_watch_kill.py +++ b/plugins/mgrep/hooks/mgrep_watch_kill.py @@ -50,7 +50,7 @@ def read_hook_input(): sys.exit(0) debug_log(f"Killing mgrep watch process: {pid}") try: - os.kill(pid, signal.SIGKILL) + os.kill(pid, signal.SIGTERM) debug_log(f"Killed mgrep watch process: {pid}") except ProcessLookupError: debug_log(f"Process {pid} already exited") diff --git a/src/commands/watch.ts b/src/commands/watch.ts index cd39db0..2533fef 100644 --- a/src/commands/watch.ts +++ b/src/commands/watch.ts @@ -1,9 +1,21 @@ import * as fs from "node:fs"; import * as path from "node:path"; import { Command, InvalidArgumentError } from "commander"; -import { type CliConfigOptions, loadConfig } from "../lib/config.js"; +import { + type CliConfigOptions, + hasLocalConfig, + loadConfig, +} from "../lib/config.js"; import { createFileSystem, createStore } from "../lib/context.js"; import { DEFAULT_IGNORE_PATTERNS } from "../lib/file.js"; +import { + claimWatch, + ensureClaim, + findCoveringWatch, + listActiveWatches, + releaseWatch, + type WatchClaim, +} from "../lib/lock.js"; import type { Store } from "../lib/store.js"; import { createIndexingSpinner, @@ -13,6 +25,7 @@ import { deleteFile, initialSync, isAtOrAboveHomeDirectory, + isSubpath, MaxFileCountExceededError, QuotaExceededError, uploadFile, @@ -24,42 +37,42 @@ export interface WatchOptions { maxFileSize?: number; maxFileCount?: number; createStore?: () => Promise; + handleSignals?: boolean; } /** - * Starts in-process synchronization and file watchers for the current directory. - * @param options Store, sync limits, and optional non-interactive store factory. - * @returns Whether initial synchronization completed and file watchers were set up, - * regardless of per-file upload errors. + * Starts a watcher, standing by while an earlier session covers its directory. + * @param options - Store, dry-run mode, file limits, store factory, and signal handling + * @returns True when covered by another watcher or after initial sync and watcher + * setup, regardless of per-file errors; false for dry runs or startup failures */ export async function startWatch(options: WatchOptions): Promise { let refreshInterval: NodeJS.Timeout | undefined; - - try { - const store = await (options.createStore ?? createStore)(); - - // Refresh JWT token every 5 minutes (before 15-minute expiration) - if (!options.dryRun) { - const REFRESH_INTERVAL = 5 * 60 * 1000; - refreshInterval = setInterval(async () => { - try { - await store.refreshClient?.(); - } catch (err) { - console.error( - "Failed to refresh JWT token:", - err instanceof Error ? err.message : "Unknown error", - ); - } - }, REFRESH_INTERVAL); - // Allow process to exit even if interval is active (fs.watch keeps it alive anyway) - refreshInterval.unref(); + let coordinationInterval: NodeJS.Timeout | undefined; + let claim: WatchClaim | null = null; + const watchers = new Map(); + + let stopped = false; + const cleanup = (): void => { + stopped = true; + process.removeListener("SIGINT", stop); + process.removeListener("SIGTERM", stop); + if (refreshInterval) clearInterval(refreshInterval); + if (coordinationInterval) clearInterval(coordinationInterval); + for (const watcher of watchers.values()) watcher.close(); + watchers.clear(); + if (claim) { + releaseWatch(claim); + claim = null; } + }; + const stop = (): void => { + cleanup(); + process.exit(0); + }; - const fileSystem = createFileSystem({ - ignorePatterns: [...DEFAULT_IGNORE_PATTERNS], - }); - const watchRoot = process.cwd(); - + try { + const watchRoot = fs.realpathSync(process.cwd()); if (isAtOrAboveHomeDirectory(watchRoot)) { console.error( "Error: Cannot watch home directory or any parent directory.", @@ -71,95 +84,236 @@ export async function startWatch(options: WatchOptions): Promise { return false; } - const cliOptions: CliConfigOptions = { - maxFileSize: options.maxFileSize, - maxFileCount: options.maxFileCount, - }; - const config = loadConfig(watchRoot, cliOptions); - console.debug("Watching for file changes in", watchRoot); - - const { spinner, onProgress } = createIndexingSpinner(watchRoot); - try { - try { - await store.retrieve(options.store); - } catch { - await store.create({ - name: options.store, - description: - "mgrep store - Mixedbreads multimodal multilingual magic search", - }); - } - const result = await initialSync( - store, - fileSystem, - options.store, - watchRoot, - options.dryRun, - onProgress, - config, - ); - const deletedInfo = - result.deleted > 0 ? ` • deleted ${result.deleted}` : ""; - const errorsInfo = result.errors > 0 ? ` • errors ${result.errors}` : ""; - if (result.errors > 0) { - spinner.warn( - `Initial sync complete (${result.processed}/${result.total}) • uploaded ${result.uploaded}${deletedInfo}${errorsInfo}`, - ); - console.error( - `\n⚠️ ${result.errors} file(s) failed to upload. Run with DEBUG=mgrep* for more details.`, - ); - } else { - spinner.succeed( - `Initial sync complete (${result.processed}/${result.total}) • uploaded ${result.uploaded}${deletedInfo}`, - ); + const mine = { startedAt: Date.now(), pid: process.pid }; + const configuredPollMs = Number(process.env.MGREP_WATCH_POLL_MS ?? 5000); + const pollMs = + Number.isFinite(configuredPollMs) && configuredPollMs > 0 + ? configuredPollMs + : 5000; + + let loggedStandby = false; + function acquireClaim(): boolean { + if (stopped) return false; + const claims = listActiveWatches(options.store); + let owner = findCoveringWatch(watchRoot, mine, claims); + if (!owner) { + claim = claimWatch(options.store, watchRoot, mine.startedAt); + if (claim) { + // A competing ancestor may have published between listing and claiming. + owner = findCoveringWatch( + watchRoot, + mine, + listActiveWatches(options.store), + ); + if (!owner) { + loggedStandby = false; + return true; + } + releaseWatch(claim); + claim = null; + } } - if (options.dryRun) { + owner ??= claims.find((entry) => entry.root === watchRoot); + if (owner && !loggedStandby) { console.log( - formatDryRunSummary(result, { - actionDescription: "found", - includeTotal: true, - }), + `Covered by mgrep watch (pid ${owner.pid}) for ${owner.root}; standing by`, ); - return false; + loggedStandby = true; } - } catch (e) { - if (e instanceof QuotaExceededError) { - spinner.fail("Quota exceeded"); - console.error( - "\n❌ Free tier quota exceeded. You've reached the monthly limit of 2,000,000 store tokens.", - ); - console.error( - " Upgrade your plan at https://platform.mixedbread.com to continue syncing.\n", - ); - clearInterval(refreshInterval); - process.exitCode = 1; - return false; + return false; + } + + let active = options.dryRun; + if (!options.dryRun) { + process.on("exit", cleanup); + if (options.handleSignals !== false) { + process.on("SIGINT", stop); + process.on("SIGTERM", stop); } - if (e instanceof MaxFileCountExceededError) { - spinner.fail("File count exceeded"); - console.error(`\n❌ ${e.message}`); - console.error( - " Increase the limit with --max-file-count or MGREP_MAX_FILE_COUNT environment variable.\n", - ); - clearInterval(refreshInterval); - process.exitCode = 1; - return false; + active = acquireClaim(); + } + + let store: Store; + let config: ReturnType; + async function prepare(): Promise { + if (store) return; + const nextStore = await (options.createStore ?? createStore)(); + if (stopped) return; + config = loadConfig(watchRoot, { + maxFileSize: options.maxFileSize, + maxFileCount: options.maxFileCount, + } satisfies CliConfigOptions); + store = nextStore; + console.debug("Watching for file changes in", watchRoot); + if (!options.dryRun) { + const REFRESH_INTERVAL = 5 * 60 * 1000; + refreshInterval = setInterval(async () => { + try { + await store.refreshClient?.(); + } catch (err) { + console.error( + "Failed to refresh JWT token:", + err instanceof Error ? err.message : "Unknown error", + ); + } + }, REFRESH_INTERVAL); + refreshInterval.unref(); + } + } + + const fileSystem = createFileSystem({ + ignorePatterns: [...DEFAULT_IGNORE_PATTERNS], + }); + let boundaryRoots: string[] = []; + let excludedRoots: string[] = []; + const configuredQuotaBackoffMs = Number( + process.env.MGREP_WATCH_QUOTA_BACKOFF_MS ?? 600000, + ); + const QUOTA_BACKOFF_MS = + Number.isSafeInteger(configuredQuotaBackoffMs) && + configuredQuotaBackoffMs > 0 + ? configuredQuotaBackoffMs + : 600000; + let quotaRetryAt = 0; + let quotaReported = false; + + function collectBoundaries(dir: string): void { + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch (error) { + console.warn(`Warning: failed to read directory ${dir}:`, error); + return; + } + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const childPath = path.join(dir, entry.name); + let child: string; + try { + child = fs.realpathSync(childPath); + } catch (error) { + console.warn(`Warning: failed to inspect path ${childPath}:`, error); + continue; + } + if (fileSystem.isIgnored(child, watchRoot)) continue; + if (hasLocalConfig(child)) boundaryRoots.push(child); + collectBoundaries(child); } - spinner.fail("Initial upload failed"); - throw e; } - console.log("Watching for file changes in", watchRoot); - fileSystem.loadMgrepignore(watchRoot); + function getExcludedRoots(claims: WatchClaim[]): string[] { + return [ + ...new Set([ + ...boundaryRoots, + ...claims + .filter( + (entry) => + entry.pid !== mine.pid && + (entry.startedAt < mine.startedAt || + (entry.startedAt === mine.startedAt && + entry.pid < mine.pid)) && + isSubpath(watchRoot, entry.root), + ) + .map((entry) => entry.root), + ]), + ]; + } + + function isExcluded(filePath: string): boolean { + let realPath = filePath; + try { + realPath = fs.realpathSync(filePath); + } catch { + // A deleted path retains its canonical parent watcher path. + } + return excludedRoots.some( + (root) => realPath === root || isSubpath(root, realPath), + ); + } + + function computeBoundaries(): void { + boundaryRoots = []; + collectBoundaries(watchRoot); + excludedRoots = getExcludedRoots(listActiveWatches(options.store)); + } + async function sync(): Promise { + await prepare(); + if (stopped) return; + const { spinner, onProgress } = createIndexingSpinner(watchRoot); + try { + try { + await store.retrieve(options.store); + } catch { + await store.create({ + name: options.store, + description: + "mgrep store - Mixedbreads multimodal multilingual magic search", + }); + } + const result = await initialSync( + store, + fileSystem, + options.store, + watchRoot, + options.dryRun, + onProgress, + config, + excludedRoots, + ); + const deletedInfo = + result.deleted > 0 ? ` • deleted ${result.deleted}` : ""; + const errorsInfo = + result.errors > 0 ? ` • errors ${result.errors}` : ""; + if (result.errors > 0) { + spinner.warn( + `Initial sync complete (${result.processed}/${result.total}) • uploaded ${result.uploaded}${deletedInfo}${errorsInfo}`, + ); + console.error( + `\n⚠️ ${result.errors} file(s) failed to upload. Run with DEBUG=mgrep* for more details.`, + ); + } else { + spinner.succeed( + `Initial sync complete (${result.processed}/${result.total}) • uploaded ${result.uploaded}${deletedInfo}`, + ); + } + if (options.dryRun) { + console.log( + formatDryRunSummary(result, { + actionDescription: "found", + includeTotal: true, + }), + ); + return; + } + } catch (e) { + if (e instanceof QuotaExceededError) { + spinner.fail("Quota exceeded"); + if (!quotaReported) { + console.error( + "\n❌ Free tier quota exceeded. You've reached the monthly limit of 2,000,000 store tokens.", + ); + console.error( + " Upgrade your plan at https://platform.mixedbread.com to continue syncing.\n", + ); + } + throw e; + } + if (e instanceof MaxFileCountExceededError) { + spinner.fail("File count exceeded"); + console.error(`\n❌ ${e.message}`); + console.error( + " Increase the limit with --max-file-count or MGREP_MAX_FILE_COUNT environment variable.\n", + ); + throw e; + } + spinner.fail("Initial upload failed"); + throw e; + } + } - // Use per-directory non-recursive watchers to avoid watching ignored - // directories like .git/, node_modules/, target/, etc. fs.watch with - // recursive: true allocates inotify watches on ALL subdirectories before - // the callback filter runs, which exhausts the kernel watcher limit on - // large repos. - const watchers = new Map(); let watchSetupSucceeded = true; + // Per-directory watchers avoid allocating handles for ignored subtrees. function isMissingPathError(error: unknown): boolean { return ( error instanceof Error && @@ -221,7 +375,7 @@ export async function startWatch(options: WatchOptions): Promise { ): void { const filePath = path.join(dirPath, name); - if (fileSystem.isIgnored(filePath, watchRoot)) { + if (isExcluded(filePath) || fileSystem.isIgnored(filePath, watchRoot)) { return; } @@ -262,7 +416,8 @@ export async function startWatch(options: WatchOptions): Promise { } function watchDirectory(dirPath: string): void { - if (watchers.has(dirPath)) { + if (stopped) return; + if (isExcluded(dirPath) || watchers.has(dirPath)) { return; } @@ -284,7 +439,7 @@ export async function startWatch(options: WatchOptions): Promise { handleFileEvent(eventType, dirPath, name); }); - watcher.on("error", (error) => { + watcher.on("error", (error: Error) => { watchSetupSucceeded = false; console.error(`Failed to watch ${dirPath}:`, error); closeWatcher(dirPath); @@ -310,16 +465,107 @@ export async function startWatch(options: WatchOptions): Promise { } } - watchDirectory(watchRoot); - // fs.watch can report startup failures asynchronously. - await new Promise((resolve) => setImmediate(resolve)); - return watchSetupSucceeded && watchers.has(watchRoot); - } catch (error) { - if (refreshInterval) { - clearInterval(refreshInterval); + if (active) { + if (!options.dryRun) computeBoundaries(); + await sync(); + if (options.dryRun || stopped) return false; + console.log("Watching for file changes in", watchRoot); + fileSystem.loadMgrepignore(watchRoot); + watchDirectory(watchRoot); + // fs.watch can report startup failures asynchronously. + await new Promise((resolve) => setImmediate(resolve)); + if (!watchSetupSucceeded || !watchers.has(watchRoot)) { + cleanup(); + return false; + } } + let polling = false; + let needsBoundaries = !active; + let needsResync = !active; + const releasedRoots = new Set(); + coordinationInterval = setInterval(() => { + if (stopped || polling) return; + polling = true; + void poll() + .catch((error: unknown) => { + console.error( + "Failed to coordinate watcher:", + error instanceof Error ? error.message : "Unknown error", + ); + }) + .finally(() => { + polling = false; + }); + }, pollMs); + + async function poll(): Promise { + if (stopped) return; + if (!claim && !acquireClaim()) return; + if (!claim) return; + ensureClaim(claim); + let claims = listActiveWatches(options.store); + const covering = findCoveringWatch(watchRoot, mine, claims); + const otherOwner = claims.find( + (entry) => entry.root === watchRoot && entry.pid !== mine.pid, + ); + if (covering || otherOwner) { + needsBoundaries = true; + needsResync = true; + for (const watcher of watchers.values()) watcher.close(); + watchers.clear(); + releaseWatch(claim); + claim = null; + if (!acquireClaim()) return; + } + if (needsBoundaries) { + computeBoundaries(); + needsBoundaries = false; + claims = listActiveWatches(options.store); + } + const nextExcluded = getExcludedRoots(claims); + const removed = excludedRoots.filter( + (root) => !nextExcluded.includes(root), + ); + excludedRoots = nextExcluded; + for (const root of excludedRoots) closeWatcherSubtree(root); + if (removed.length > 0) { + needsResync = true; + for (const root of removed) releasedRoots.add(root); + } + if (needsResync) { + if (Date.now() < quotaRetryAt) return; + try { + await sync(); + } catch (error) { + if (error instanceof QuotaExceededError) { + quotaRetryAt = Date.now() + QUOTA_BACKOFF_MS; + quotaReported = true; + } + throw error; + } + quotaRetryAt = 0; + quotaReported = false; + if (stopped) return; + if (!watchers.has(watchRoot)) { + console.log("Watching for file changes in", watchRoot); + fileSystem.loadMgrepignore(watchRoot); + } + watchDirectory(watchRoot); + for (const root of releasedRoots) watchDirectory(root); + releasedRoots.clear(); + needsResync = false; + } + } + return true; + } catch (error) { + cleanup(); const message = error instanceof Error ? error.message : "Unknown error"; - console.error("Failed to start watcher:", message); + if ( + !(error instanceof QuotaExceededError) && + !(error instanceof MaxFileCountExceededError) + ) { + console.error("Failed to start watcher:", message); + } process.exitCode = 1; return false; } diff --git a/src/commands/watch_mcp.ts b/src/commands/watch_mcp.ts index 7758adc..9c3d85a 100644 --- a/src/commands/watch_mcp.ts +++ b/src/commands/watch_mcp.ts @@ -262,6 +262,7 @@ export const watchMcp = new Command("mcp") await startWatch({ store: options.store, dryRun: false, + handleSignals: false, createStore: () => createStore({ interactive: false }), }), ); diff --git a/src/lib/config.ts b/src/lib/config.ts index c33c8cf..fefec3f 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -4,7 +4,17 @@ import * as path from "node:path"; import YAML from "yaml"; import { z } from "zod"; -const LOCAL_CONFIG_FILES = [".mgreprc.yaml", ".mgreprc.yml"] as const; +/** Names of per-project configuration files. */ +export const LOCAL_CONFIG_FILES = [".mgreprc.yaml", ".mgreprc.yml"] as const; + +/** + * Checks whether a directory defines a project configuration boundary. + * @param dir - Directory to inspect + * @returns Whether either local configuration file exists + */ +export function hasLocalConfig(dir: string): boolean { + return LOCAL_CONFIG_FILES.some((file) => fs.existsSync(path.join(dir, file))); +} const GLOBAL_CONFIG_DIR = ".config/mgrep"; const GLOBAL_CONFIG_FILES = ["config.yaml", "config.yml"] as const; const ENV_PREFIX = "MGREP_"; diff --git a/src/lib/lock.ts b/src/lib/lock.ts new file mode 100644 index 0000000..7142bb6 --- /dev/null +++ b/src/lib/lock.ts @@ -0,0 +1,247 @@ +import { execFileSync } from "node:child_process"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { hasLocalConfig } from "./config.js"; +import { isSubpath } from "./utils.js"; + +/** An active watcher's exclusive directory claim. */ +export interface WatchClaim { + pid: number; + root: string; + store: string; + startedAt: number; +} + +const INVALID_CLAIM_STALE_MS = 30_000; + +function registryDir(): string { + const dir = + process.env.MGREP_WATCH_REGISTRY_DIR ?? + path.join(os.homedir(), ".mgrep", "watchers"); + fs.mkdirSync(dir, { recursive: true }); + return dir; +} + +function claimPath(store: string, root: string): string { + const name = Buffer.from(`${store}\0${root}`).toString("base64url"); + return path.join(registryDir(), `${name}.json`); +} + +function readClaim(file: string): WatchClaim | undefined { + try { + const claim: unknown = JSON.parse(fs.readFileSync(file, "utf8")); + if ( + typeof claim === "object" && + claim !== null && + "pid" in claim && + typeof claim.pid === "number" && + Number.isInteger(claim.pid) && + claim.pid > 0 && + "root" in claim && + typeof claim.root === "string" && + path.isAbsolute(claim.root) && + "store" in claim && + typeof claim.store === "string" && + "startedAt" in claim && + typeof claim.startedAt === "number" && + Number.isFinite(claim.startedAt) + ) { + return claim as WatchClaim; + } + } catch { + // A concurrent O_EXCL writer may not have finished writing its JSON yet. + } + return undefined; +} + +function isMgrepProcess(pid: number): boolean { + try { + process.kill(pid, 0); + } catch { + return false; + } + try { + return execFileSync("ps", ["-o", "command=", "-p", String(pid)], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }).includes("mgrep"); + } catch { + return true; + } +} + +function unlinkClaim(file: string, claim: WatchClaim): void { + try { + const current = readClaim(file); + if (current?.pid === claim.pid && current.startedAt === claim.startedAt) { + fs.unlinkSync(file); + } + } catch { + // Another process may already have removed the claim. + } +} + +function unlinkStaleInvalidClaim(file: string): boolean { + try { + const before = fs.statSync(file); + if (Date.now() - before.mtimeMs <= INVALID_CLAIM_STALE_MS) return false; + if (readClaim(file)) return false; + const current = fs.statSync(file); + // Preserve a concurrent writer's replacement or newly updated claim. + if ( + current.dev !== before.dev || + current.ino !== before.ino || + current.mtimeMs !== before.mtimeMs || + current.size !== before.size + ) { + return false; + } + fs.unlinkSync(file); + return true; + } catch { + // Another process may already have removed the claim. + return false; + } +} + +/** + * Claims a canonical directory using the upstream lock design's O_EXCL write. + * @param store - Store identifier + * @param root - Directory to watch (resolved to its realpath) + * @param startedAt - Session start timestamp + * @returns The owned claim, or null when a live process holds it + */ +export function claimWatch( + store: string, + root: string, + startedAt: number, +): WatchClaim | null { + const claim = { + pid: process.pid, + root: fs.realpathSync(root), + store, + startedAt, + }; + const file = claimPath(store, claim.root); + for (let attempt = 0; attempt < 2; attempt++) { + try { + fs.writeFileSync(file, JSON.stringify(claim), { flag: "wx" }); + return claim; + } catch (error) { + if ( + !(error instanceof Error && "code" in error && error.code === "EEXIST") + ) { + throw error; + } + const existing = readClaim(file); + if (!existing) { + if (!unlinkStaleInvalidClaim(file)) return null; + } else { + if (isMgrepProcess(existing.pid)) return null; + unlinkClaim(file, existing); + } + } + } + return null; +} + +/** + * Releases a claim only while its file still belongs to this pid. + * @param claim - Owned directory claim + * @returns Nothing + */ +export function releaseWatch(claim: WatchClaim): void { + try { + if (claim.pid === process.pid) { + unlinkClaim(claimPath(claim.store, claim.root), claim); + } + } catch { + // Cleanup must remain safe even if the registry directory becomes unavailable. + } +} + +/** + * Restores a missing claim with O_EXCL, without replacing another owner. + * @param claim - Owned directory claim + * @returns Nothing + */ +export function ensureClaim(claim: WatchClaim): void { + try { + fs.writeFileSync( + claimPath(claim.store, claim.root), + JSON.stringify(claim), + { + flag: "wx", + }, + ); + } catch (error) { + if ( + !(error instanceof Error && "code" in error && error.code === "EEXIST") + ) { + throw error; + } + } +} + +/** + * Lists live watchers and prunes dead pids and invalid claims older than 30 seconds. + * @param store - Store identifier + * @returns Live claims for the store + */ +export function listActiveWatches(store: string): WatchClaim[] { + const dir = registryDir(); + const claims: WatchClaim[] = []; + for (const name of fs.readdirSync(dir)) { + if (!name.endsWith(".json")) continue; + const file = path.join(dir, name); + const claim = readClaim(file); + if (!claim) { + unlinkStaleInvalidClaim(file); + continue; + } + if (!isMgrepProcess(claim.pid)) { + unlinkClaim(file, claim); + } else if (claim.store === store) { + claims.push(claim); + } + } + return claims; +} + +/** + * Finds the earliest watcher covering a directory without crossing local config. + * @param root - Canonical directory to cover + * @param mine - This session's timestamp and pid, used to order claims + * @param claims - Live claims on the same store + * @returns The covering earlier watcher, if any + */ +export function findCoveringWatch( + root: string, + mine: { startedAt: number; pid: number }, + claims: WatchClaim[], +): WatchClaim | undefined { + const realRoot = fs.realpathSync(root); + return claims + .filter((claim) => { + if ( + claim.pid === mine.pid || + claim.startedAt > mine.startedAt || + (claim.startedAt === mine.startedAt && claim.pid >= mine.pid) + ) + return false; + let claimRoot: string; + try { + claimRoot = fs.realpathSync(claim.root); + } catch { + return false; + } + if (claimRoot === realRoot) return true; + if (!isSubpath(claimRoot, realRoot)) return false; + for (let dir = realRoot; dir !== claimRoot; dir = path.dirname(dir)) { + if (hasLocalConfig(dir)) return false; + } + return true; + }) + .sort((a, b) => a.startedAt - b.startedAt || a.pid - b.pid)[0]; +} diff --git a/src/lib/utils.ts b/src/lib/utils.ts index abd0b5e..8fc3637 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -6,14 +6,11 @@ import { cancel, confirm, isCancel } from "@clack/prompts"; import { isText } from "istextorbinary"; import pLimit from "p-limit"; import xxhashWasm from "xxhash-wasm"; -import { loginAction } from "../commands/login.js"; import { exceedsMaxFileSize, type MgrepConfig } from "./config.js"; import type { FileSystem } from "./file.js"; import type { Store } from "./store.js"; import type { InitialSyncProgress, InitialSyncResult } from "./sync-helpers.js"; -import { getStoredToken } from "./token.js"; - export const isTest = process.env.MGREP_IS_TEST === "1"; /** Error thrown when the free tier quota is exceeded */ @@ -43,7 +40,13 @@ function isQuotaError(errorMessage: string): boolean { ); } -function isSubpath(parent: string, child: string): boolean { +/** + * Checks strict containment using path separators (callers supply real paths). + * @param parent - Parent directory + * @param child - Candidate descendant + * @returns Whether child is strictly inside parent + */ +export function isSubpath(parent: string, child: string): boolean { const parentPath = path.resolve(parent); const childPath = path.resolve(child); @@ -180,6 +183,8 @@ export async function ensureAuthenticated(): Promise { } // Check for stored OAuth token + // Load authentication lazily so path utilities can be imported without the auth cycle. + const { getStoredToken } = await import("./token.js"); const token = await getStoredToken(); if (token) { return; @@ -195,6 +200,7 @@ export async function ensureAuthenticated(): Promise { process.exit(0); } + const { loginAction } = await import("../commands/login.js"); await loginAction(); } @@ -275,6 +281,18 @@ export async function uploadFile( return true; } +/** + * Syncs files while preserving subtrees owned by other watchers. + * @param store - Store instance + * @param fileSystem - File discovery and ignore rules + * @param storeId - Store identifier + * @param repoRoot - Directory to sync + * @param dryRun - Report changes without uploading or deleting + * @param onProgress - Progress callback + * @param config - File size and count limits + * @param excludedRoots - Subtree roots to leave untouched + * @returns Sync counts + */ export async function initialSync( store: Store, fileSystem: FileSystem, @@ -283,17 +301,41 @@ export async function initialSync( dryRun?: boolean, onProgress?: (info: InitialSyncProgress) => void, config?: MgrepConfig, + excludedRoots: string[] = [], ): Promise { + const canonicalExcludedRoots = excludedRoots.map((root) => { + try { + return fs.realpathSync(root); + } catch { + return path.resolve(root); + } + }); + const isExcluded = (filePath: string): boolean => { + if (canonicalExcludedRoots.length === 0) return false; + let realPath = filePath; + try { + realPath = fs.realpathSync(filePath); + } catch { + // Deleted store files already use the canonical watcher root. + } + return canonicalExcludedRoots.some( + (root) => realPath === root || isSubpath(root, realPath), + ); + }; const storeMetadata = await listStoreFileMetadata(store, storeId, repoRoot); const allFiles = Array.from(fileSystem.getFiles(repoRoot)); const repoFiles = allFiles.filter( - (filePath) => !fileSystem.isIgnored(filePath, repoRoot), + (filePath) => + !isExcluded(filePath) && !fileSystem.isIgnored(filePath, repoRoot), ); const repoFileSet = new Set(repoFiles); const filesToDelete = Array.from(storeMetadata.keys()).filter( - (filePath) => isSubpath(repoRoot, filePath) && !repoFileSet.has(filePath), + (filePath) => + !isExcluded(filePath) && + isSubpath(repoRoot, filePath) && + !repoFileSet.has(filePath), ); // Check files that potentially need uploading (new or modified) diff --git a/test/mcp.bats b/test/mcp.bats index e431543..162ad6a 100644 --- a/test/mcp.bats +++ b/test/mcp.bats @@ -15,6 +15,9 @@ setup() { export MGREP_IS_TEST=1 export MGREP_TEST_STORE_PATH="$BATS_TEST_TMPDIR/mgrep-test-store.json" export XDG_STATE_HOME="$BATS_TEST_TMPDIR/state" + export MGREP_WATCH_REGISTRY_DIR="$BATS_TEST_TMPDIR/registry" + export MGREP_WATCH_POLL_MS=100 + PIDS=() unset MGREP_MCP_WAIT_FOR MGREP_MCP_ROOTS MGREP_MCP_FORBID_ROOTS MGREP_MCP_SYNC_WAIT_MS mkdir -p "$BATS_TEST_TMPDIR/test-store" @@ -25,6 +28,14 @@ setup() { mgrep search --sync test } +teardown() { + if [ -f "$BATS_TEST_TMPDIR/mcp.pid" ]; then + PIDS+=("$(cat "$BATS_TEST_TMPDIR/mcp.pid")") + fi + for pid in "${PIDS[@]}"; do kill -TERM "$pid" 2>/dev/null || true; done + for pid in "${PIDS[@]}"; do wait "$pid" 2>/dev/null || true; done +} + mcp_client() { node "$DIR/helpers/mcp-client.mjs" "$@" } @@ -529,7 +540,139 @@ assert.equal(process.exitCode || 0, ['home', 'quota', 'file-count', 'store-error process.exitCode = 0; fs.rmSync('contract-a.txt', { force: true }); fs.rmSync('contract-b.txt', { force: true }); +if (watching) process.emit('SIGTERM'); JS assert_success done } + +@test "MCP covered watcher searches promptly without syncing and takes over after SIGTERM" { + export MGREP_WATCH_QUOTA_BACKOFF_MS=300 + local failure + for failure in none quota file-count; do + mgrep watch > "$BATS_TEST_TMPDIR/active.log" 2>&1 3>&- & + local active=$! + PIDS+=("$active") + for _ in {1..100}; do + if grep -q 'Initial sync complete' "$BATS_TEST_TMPDIR/active.log"; then break; fi + sleep 0.05 + done + grep -q 'Initial sync complete' "$BATS_TEST_TMPDIR/active.log" + export MGREP_MCP_SYNC_WAIT_MS=30000 + run node --input-type=module - "$DIR/.." "$active" "$BATS_TEST_TMPDIR/mcp.pid" "$failure" <<'JS' +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { performance } from 'node:perf_hooks'; +import { setTimeout as delay } from 'node:timers/promises'; +import { pathToFileURL } from 'node:url'; +const base = pathToFileURL(`${process.argv[2]}/`).href; +const { Client } = await import(`${base}node_modules/@modelcontextprotocol/sdk/dist/esm/client/index.js`); +const { StdioClientTransport } = await import(`${base}node_modules/@modelcontextprotocol/sdk/dist/esm/client/stdio.js`); +const failure = process.argv[5]; +const preload = `import { TestStore } from ${JSON.stringify(`${base}dist/lib/store.js`)}; +import { QuotaExceededError, MaxFileCountExceededError } from ${JSON.stringify(`${base}dist/lib/utils.js`)}; +const list = TestStore.prototype.listFiles; +let fail = ${JSON.stringify(failure)}; +TestStore.prototype.listFiles = async function* (...args) { + console.error('coordination test sync pass'); + const current = fail; + fail = 'none'; + if (current === 'quota') throw new QuotaExceededError('takeover quota test'); + if (current === 'file-count') throw new MaxFileCountExceededError(2, 1); + yield* list.apply(this, args); +};`; +const transport = new StdioClientTransport({ + command: 'mgrep', args: ['mcp', '--expose-tools'], stderr: 'pipe', + env: {...process.env, NODE_OPTIONS: `--import=data:text/javascript;base64,${Buffer.from(preload).toString('base64')}`}, +}); +const client = new Client({name: 'coordination-test', version: '1.0.0'}); +let stderr = ''; +transport.stderr.on('data', chunk => { stderr += chunk.toString(); }); +const deadline = setTimeout(() => { + console.error(stderr); + if (transport.pid) process.kill(transport.pid, 'SIGTERM'); + process.exit(1); +}, 15000); +const waitFor = async predicate => { + const end = performance.now() + 5000; + while (!predicate()) { + assert.ok(performance.now() < end, stderr); + await delay(25); + } +}; +try { + await client.connect(transport); + const pid = transport.pid; + fs.writeFileSync(process.argv[4], String(pid)); + const start = performance.now(); + const result = await client.callTool({name: 'search', arguments: {query: 'test', path: '.'}}); + const elapsedMs = performance.now() - start; + assert.ok(elapsedMs < 5000, `search took ${elapsedMs}ms`); + assert.equal(result.isError, undefined); + assert.match(result.content[0].text, /test.txt/); + assert.doesNotMatch(result.content[0].text, /still running/); + assert.ok(stderr.includes(`Covered by mgrep watch (pid ${process.argv[3]})`), stderr); + assert.doesNotMatch(stderr, /coordination test sync pass|Initial sync complete|Indexing complete/); + const root = fs.realpathSync(process.cwd()); + const claimFile = `${process.env.MGREP_WATCH_REGISTRY_DIR}/${Buffer.from(`mgrep\0${root}`).toString('base64url')}.json`; + assert.equal(JSON.parse(fs.readFileSync(claimFile)).pid, Number(process.argv[3])); + process.kill(Number(process.argv[3]), 'SIGTERM'); + await waitFor(() => { + try { return JSON.parse(fs.readFileSync(claimFile)).pid === pid; } + catch { return false; } + }); + await waitFor(() => stderr.includes('[LOG] Watching for file changes in')); + assert.equal((stderr.match(/coordination test sync pass/g) || []).length, failure === 'none' ? 1 : 2); + if (failure !== 'none') assert.match(stderr, /Failed to coordinate watcher:/); + assert.equal((stderr.match(/Initial sync complete/g) || []).length, 1); + fs.writeFileSync('takeover.txt', 'mcp-takeover-needle'); + await waitFor(() => stderr.includes(`${root}/takeover.txt`)); + const after = await client.callTool({name: 'search', arguments: {query: 'mcp-takeover-needle'}}); + assert.match(after.content[0].text, /takeover.txt/); + console.log(JSON.stringify({elapsedMs: Math.round(elapsedMs), takeover: true})); +} finally { + clearTimeout(deadline); + const child = transport._process; + const exited = child && child.exitCode === null && child.signalCode === null + ? new Promise(resolve => child.once('exit', resolve)) : Promise.resolve(); + await client.close(); + await exited; +} +assert.equal(fs.existsSync(`${process.env.MGREP_WATCH_REGISTRY_DIR}/${Buffer.from(`mgrep\0${fs.realpathSync(process.cwd())}`).toString('base64url')}.json`), false); +JS + assert_success + assert_output --partial '"takeover":true' + wait "$active" + done +} + +@test "startWatch can leave signal handling to MCP and still releases its claim on exit" { + run node --title=mgrep-signal-test --input-type=module - "$DIR/../dist" <<'JS' +import assert from 'node:assert/strict'; +import { EventEmitter } from 'node:events'; +import fs from 'node:fs'; +import { syncBuiltinESMExports } from 'node:module'; +import { pathToFileURL } from 'node:url'; +const base = pathToFileURL(`${process.argv[2]}/`).href; +const { startWatch } = await import(`${base}commands/watch.js`); +const { listActiveWatches } = await import(`${base}lib/lock.js`); +const signals = ['SIGINT', 'SIGTERM']; +const before = signals.map(signal => process.listenerCount(signal)); +const exitCount = process.listenerCount('exit'); +let closed = 0; +fs.watch = () => { + const watcher = new EventEmitter(); + watcher.close = () => { closed++; }; + return watcher; +}; +syncBuiltinESMExports(); +assert.equal(await startWatch({store: 'mgrep', dryRun: false, handleSignals: false}), true); +assert.deepEqual(signals.map(signal => process.listenerCount(signal)), before); +assert.equal(process.listenerCount('exit'), exitCount + 1); +assert.ok(listActiveWatches('mgrep').some(claim => claim.pid === process.pid)); +process.emit('exit'); +assert.ok(closed > 0); +assert.equal(listActiveWatches('mgrep').length, 0); +JS + assert_success +} diff --git a/test/watch-coordination.bats b/test/watch-coordination.bats new file mode 100644 index 0000000..ef07ead --- /dev/null +++ b/test/watch-coordination.bats @@ -0,0 +1,644 @@ +#!/bin/bash + +setup_file() { + pnpm build +} + +setup() { + load '../node_modules/bats-support/load' + load '../node_modules/bats-assert/load' + DIR="$( cd "$( dirname "$BATS_TEST_FILENAME" )" >/dev/null 2>&1 && pwd )" + FIXTURE="$(mktemp -d "$BATS_TMPDIR/mgrep-coordination.XXXXXX")" + export MGREP_IS_TEST=1 + export MGREP_TEST_STORE_PATH="$FIXTURE/store.json" + export MGREP_WATCH_REGISTRY_DIR="$FIXTURE/registry" + export MGREP_WATCH_POLL_MS=100 + export MGREP_AUTO_UPDATE=disabled + mkdir -p "$FIXTURE/bin" "$FIXTURE/repo/services" "$MGREP_WATCH_REGISTRY_DIR" + ln -s "$DIR/../dist/index.js" "$FIXTURE/bin/mgrep" + export PATH="$FIXTURE/bin:$PATH" + ROOT="$(cd "$FIXTURE/repo" && pwd -P)" + CHILD="$ROOT/services" + PIDS=() + cd "$ROOT" + echo 'parent content' > parent.txt + echo 'child content' > services/child.txt + mgrep search --sync test > "$FIXTURE/setup.log" 2>&1 +} + +teardown() { + for pid in "${PIDS[@]}"; do + kill -CONT "$pid" 2>/dev/null || true + kill -TERM "$pid" 2>/dev/null || true + done + # A broken signal handler must not leave a slow upload alive in teardown. + for pid in "${PIDS[@]}"; do kill -KILL "$pid" 2>/dev/null || true; done + for pid in "${PIDS[@]}"; do wait "$pid" 2>/dev/null || true; done + if [ "${BATS_TEST_COMPLETED:-}" != 1 ]; then + for log in "$FIXTURE"/*.log; do cat "$log" >&3; done + fi + rm -rf "$FIXTURE" +} + +start_watcher() { + local root="$1" log="$2" + shift 2 + (cd "$root" && exec mgrep watch "$@") > "$log" 2>&1 3>&- & + WATCH_PID=$! + PIDS+=("$WATCH_PID") +} + +start_injected_watcher() { + local root="$1" log="$2" failure="$3" + (cd "$root" && exec node --title=mgrep-error-test --input-type=module - "$DIR/../dist" "$failure" "$FIXTURE/reject" "$CHILD") > "$log" 2>&1 3>&- <<'JS' & +import fs from 'node:fs'; +import { syncBuiltinESMExports } from 'node:module'; +import { pathToFileURL } from 'node:url'; +const [base, failure, marker, child] = process.argv.slice(2); +const { TestStore } = await import(pathToFileURL(`${base}/lib/store.js`)); +const { QuotaExceededError, MaxFileCountExceededError } = + await import(pathToFileURL(`${base}/lib/utils.js`)); +if (failure.startsWith('sync-quota')) { + const now = Date.now; + Date.now = () => now() + (fs.existsSync(`${marker}.advance`) + ? Number(fs.readFileSync(`${marker}.advance`, 'utf8')) : 0); +} +const upload = TestStore.prototype.uploadFile; +TestStore.prototype.uploadFile = async function (...args) { + if (failure === 'upload' && fs.existsSync(marker)) throw new Error('injected upload failure'); + return upload.apply(this, args); +}; +const remove = TestStore.prototype.deleteFile; +TestStore.prototype.deleteFile = async function (...args) { + if (failure === 'delete' && fs.existsSync(marker)) throw new Error('injected delete failure'); + return remove.apply(this, args); +}; +const list = TestStore.prototype.listFiles; +TestStore.prototype.listFiles = async function* (...args) { + if (failure === 'sync-quota-repeat') console.error('test poll sync attempt'); + if (failure.startsWith('sync') && fs.existsSync(marker)) { + if (failure !== 'sync-quota-repeat') fs.unlinkSync(marker); + if (failure.startsWith('sync-quota')) throw new QuotaExceededError('injected sync failure'); + if (failure === 'sync-count') throw new MaxFileCountExceededError(2, 1); + throw new Error('injected sync failure'); + } + yield* list.apply(this, args); +}; +if (failure === 'directory') { + const watch = fs.watch; + fs.watch = function (dir, ...args) { + const watcher = watch.call(this, dir, ...args); + if (dir === child) { + const close = watcher.close.bind(watcher); + watcher.close = () => { console.log(`closed directory: ${dir}`); close(); }; + const timer = setInterval(() => { + if (!fs.existsSync(marker)) return; + fs.unlinkSync(marker); + clearInterval(timer); + watcher.emit('error', new Error('injected directory failure')); + }, 10); + timer.unref(); + } + return watcher; + }; +} +if (failure === 'poll') { + const read = fs.readdirSync; + fs.readdirSync = function (dir, ...args) { + if (dir === process.env.MGREP_WATCH_REGISTRY_DIR && fs.existsSync(marker)) { + fs.unlinkSync(marker); + throw new Error('injected poll failure'); + } + return read.call(this, dir, ...args); + }; +} +syncBuiltinESMExports(); +const { startWatch } = await import(pathToFileURL(`${base}/commands/watch.js`)); +await startWatch({store: 'mgrep', dryRun: false}); +JS + WATCH_PID=$! + PIDS+=("$WATCH_PID") +} + +wait_log() { + local log="$1" text="$2" + for _ in {1..80}; do + if grep -Fq "$text" "$log"; then return 0; fi + sleep 0.05 + done + cat "$log" >&3 + return 1 +} + +wait_active() { + wait_log "$1" 'Initial sync complete' + # Let the startup continuation install its directory watchers. + sleep 0.15 +} + +claim_count() { + node -e 'const fs = require("fs"); console.log(fs.readdirSync(process.env.MGREP_WATCH_REGISTRY_DIR).filter(n => n.endsWith(".json")).length)' +} + +assert_claim() { + node - "$1" "$2" <<'JS' +const fs = require('fs'); +const claims = fs.readdirSync(process.env.MGREP_WATCH_REGISTRY_DIR) + .filter(n => n.endsWith('.json')) + .map(n => JSON.parse(fs.readFileSync(`${process.env.MGREP_WATCH_REGISTRY_DIR}/${n}`, 'utf8'))); +if (!claims.some(c => c.pid === Number(process.argv[2]) && c.root === process.argv[3])) process.exit(1); +JS +} + +@test "second watcher on the same directory stands by with one claim" { + start_watcher "$ROOT" "$FIXTURE/first.log" + first=$WATCH_PID + wait_active "$FIXTURE/first.log" + start_watcher "$ROOT" "$FIXTURE/second.log" + second=$WATCH_PID + wait_log "$FIXTURE/second.log" "Covered by mgrep watch (pid $first) for $ROOT; standing by" + assert [ "$(claim_count)" = 1 ] + assert_claim "$first" "$ROOT" + kill -0 "$second" + ! grep -q 'Initial sync\|Watching for file changes' "$FIXTURE/second.log" + echo changed >> "$CHILD/child.txt" + wait_log "$FIXTURE/first.log" ": $CHILD/child.txt" + sleep 0.3 + ! grep -q 'change:\|rename:' "$FIXTURE/second.log" +} + +@test "child started after parent stands by and SIGTERM triggers takeover" { + start_watcher "$ROOT" "$FIXTURE/parent.log" + parent=$WATCH_PID + wait_active "$FIXTURE/parent.log" + start_watcher "$CHILD" "$FIXTURE/child.log" + child=$WATCH_PID + wait_log "$FIXTURE/child.log" 'standing by' + assert [ "$(claim_count)" = 1 ] + ! grep -q 'Initial sync' "$FIXTURE/child.log" + kill -TERM "$parent" + wait "$parent" + # Bound takeover latency to a few polling intervals plus startup sync. + for _ in {1..10}; do + if grep -q 'Initial sync complete' "$FIXTURE/child.log"; then break; fi + sleep 0.1 + done + grep -q 'Initial sync complete' "$FIXTURE/child.log" + assert_claim "$child" "$CHILD" + assert [ "$(claim_count)" = 1 ] + sleep 0.15 + echo takeover >> "$CHILD/child.txt" + wait_log "$FIXTURE/child.log" ": $CHILD/child.txt" +} + +@test "SIGTERM exits a CLI watcher during slow initial sync within two seconds" { + local preload + preload="$(node --input-type=module - "$DIR/../dist" <<'JS' +import { pathToFileURL } from 'node:url'; +const module = pathToFileURL(`${process.argv[2]}/lib/store.js`).href; +const code = `import { TestStore } from ${JSON.stringify(module)}; +const upload = TestStore.prototype.uploadFile; +TestStore.prototype.uploadFile = async function (...args) { + console.error('test slow upload started'); + await new Promise(resolve => setTimeout(resolve, 10000)); + return upload.apply(this, args); +};`; +console.log(`data:text/javascript;base64,${Buffer.from(code).toString('base64')}`); +JS +)" + echo slow-upload > "$ROOT/new.txt" + NODE_OPTIONS="--import=$preload" start_watcher "$ROOT" "$FIXTURE/slow.log" + wait_log "$FIXTURE/slow.log" 'test slow upload started' + assert_claim "$WATCH_PID" "$ROOT" + run node - "$WATCH_PID" <<'JS' +const assert = require('node:assert/strict'); +const { performance } = require('node:perf_hooks'); +const pid = Number(process.argv[2]); +const start = performance.now(); +process.kill(pid, 'SIGTERM'); +const timer = setInterval(() => { + try { process.kill(pid, 0); } + catch (error) { + assert.equal(error.code, 'ESRCH'); + assert.ok(performance.now() - start < 2000); + clearInterval(timer); + return; + } + assert.ok(performance.now() - start < 2000, 'watcher did not exit within two seconds'); +}, 10); +JS + assert_success + wait "$WATCH_PID" + assert [ "$(claim_count)" = 0 ] + ! grep -q 'Initial sync complete' "$FIXTURE/slow.log" +} + +@test "poll-time quota takeover backs off ten minutes and prints one banner per series" { + start_watcher "$ROOT" "$FIXTURE/parent.log" + parent=$WATCH_PID + wait_active "$FIXTURE/parent.log" + touch "$FIXTURE/reject" + start_injected_watcher "$CHILD" "$FIXTURE/child.log" sync-quota-repeat + child=$WATCH_PID + wait_log "$FIXTURE/child.log" 'standing by' + kill -TERM "$parent" + wait "$parent" + wait_log "$FIXTURE/child.log" 'Failed to coordinate watcher:' + # Five 100ms poll intervals must not attempt another sync or print a banner. + sleep 0.5 + assert [ "$(grep -c 'test poll sync attempt' "$FIXTURE/child.log")" = 1 ] + assert [ "$(grep -c 'Free tier quota exceeded' "$FIXTURE/child.log")" = 1 ] + assert_claim "$child" "$CHILD" + echo 540000 > "$FIXTURE/reject.advance" + sleep 0.5 + assert [ "$(grep -c 'test poll sync attempt' "$FIXTURE/child.log")" = 1 ] + echo 600001 > "$FIXTURE/reject.advance" + for _ in {1..40}; do + if [ "$(grep -c 'test poll sync attempt' "$FIXTURE/child.log")" -ge 2 ]; then break; fi + sleep 0.05 + done + sleep 0.5 + assert [ "$(grep -c 'test poll sync attempt' "$FIXTURE/child.log")" = 2 ] + assert [ "$(grep -c 'Free tier quota exceeded' "$FIXTURE/child.log")" = 1 ] + rm "$FIXTURE/reject" + echo 1200002 > "$FIXTURE/reject.advance" + wait_active "$FIXTURE/child.log" + assert [ "$(grep -c 'test poll sync attempt' "$FIXTURE/child.log")" = 3 ] + echo recovered >> "$CHILD/child.txt" + wait_log "$FIXTURE/child.log" ": $CHILD/child.txt" + assert_claim "$child" "$CHILD" +} + +@test "parent started after child excludes child uploads then takes over its subtree" { + start_watcher "$CHILD" "$FIXTURE/child.log" + child=$WATCH_PID + wait_active "$FIXTURE/child.log" + kill -STOP "$child" + echo child-change >> "$CHILD/child.txt" + echo parent-change >> "$ROOT/parent.txt" + start_watcher "$ROOT" "$FIXTURE/parent.log" --max-file-count 1 + wait_active "$FIXTURE/parent.log" + assert [ "$(claim_count)" = 2 ] + grep -q 'uploaded 1' "$FIXTURE/parent.log" + grep -q "Initial sync complete (1/1)" "$FIXTURE/parent.log" + kill -CONT "$child" + wait_log "$FIXTURE/child.log" ": $CHILD/child.txt" + sleep 0.3 + ! grep -Fq "change: $CHILD/child.txt" "$FIXTURE/parent.log" + ! grep -Fq "rename: $CHILD/child.txt" "$FIXTURE/parent.log" + kill -TERM "$child" + wait "$child" + # Resync is visible even if every file is already up to date. + for _ in {1..30}; do + if [ "$(grep -c 'Initial sync complete' "$FIXTURE/parent.log")" -ge 2 ]; then break; fi + sleep 0.1 + done + assert [ "$(grep -c 'Initial sync complete' "$FIXTURE/parent.log")" -ge 2 ] + sleep 0.15 + echo parent-takeover >> "$CHILD/child.txt" + wait_log "$FIXTURE/parent.log" ": $CHILD/child.txt" +} + +@test "yaml config child remains independent and parent preserves excluded store files" { + echo 'maxFileSize: 1' > "$CHILD/.mgreprc.yaml" + # This absent child file would be deleted by an ancestor without exclusions. + rm "$CHILD/child.txt" + echo changed >> "$ROOT/parent.txt" + start_watcher "$ROOT" "$FIXTURE/parent.log" --max-file-count 1 + wait_active "$FIXTURE/parent.log" + grep -q 'uploaded 1' "$FIXTURE/parent.log" + ! grep -q 'deleted' "$FIXTURE/parent.log" + node - "$CHILD/child.txt" <<'JS' +const fs = require('fs'); +if (!JSON.parse(fs.readFileSync(process.env.MGREP_TEST_STORE_PATH)).files[process.argv[2]]) process.exit(1); +JS + # Restore before the child starts; mutations are sequential because TestStore + # only serializes writes within a process, not between concurrent processes. + echo 'child content' > "$CHILD/child.txt" + start_watcher "$CHILD" "$FIXTURE/child.log" + child=$WATCH_PID + wait_active "$FIXTURE/child.log" + assert_claim "$child" "$CHILD" + assert [ "$(claim_count)" = 2 ] + ! grep -q 'standing by' "$FIXTURE/child.log" + kill -TERM "$child" + wait "$child" + sleep 0.3 + echo boundary-stays >> "$CHILD/new.txt" + sleep 0.3 + ! grep -Fq "$CHILD/new.txt" "$FIXTURE/parent.log" + assert [ "$(grep -c 'Initial sync complete' "$FIXTURE/parent.log")" = 1 ] +} + +@test "yml boundary between ancestor and grandchild prevents coverage" { + echo 'maxFileCount: 5' > "$CHILD/.mgreprc.yml" + mkdir "$CHILD/nested" + start_watcher "$ROOT" "$FIXTURE/parent.log" + wait_active "$FIXTURE/parent.log" + start_watcher "$CHILD/nested" "$FIXTURE/grandchild.log" + wait_active "$FIXTURE/grandchild.log" + assert [ "$(claim_count)" = 2 ] + ! grep -q 'standing by' "$FIXTURE/grandchild.log" +} + +@test "dead pid claim is pruned and directory is claimable" { + node - "$ROOT" <<'JS' +const fs = require('fs'); +const root = process.argv[2]; +const file = `${process.env.MGREP_WATCH_REGISTRY_DIR}/${Buffer.from(`mgrep\0${root}`).toString('base64url')}.json`; +fs.writeFileSync(file, JSON.stringify({pid: 99999999, root, store: 'mgrep', startedAt: 1})); +JS + start_watcher "$ROOT" "$FIXTURE/watch.log" + wait_active "$FIXTURE/watch.log" + assert_claim "$WATCH_PID" "$ROOT" + assert [ "$(claim_count)" = 1 ] +} + +@test "live unrelated pid claim is pruned and a missing owned claim self heals" { + sleep 30 3>&- & + unrelated=$! + PIDS+=("$unrelated") + node - "$ROOT" "$unrelated" <<'JS' +const fs = require('fs'); +const root = process.argv[2]; +const file = `${process.env.MGREP_WATCH_REGISTRY_DIR}/${Buffer.from(`mgrep\0${root}`).toString('base64url')}.json`; +fs.writeFileSync(file, JSON.stringify({pid: Number(process.argv[3]), root, store: 'mgrep', startedAt: 1})); +JS + start_watcher "$ROOT" "$FIXTURE/watch.log" + wait_active "$FIXTURE/watch.log" + assert_claim "$WATCH_PID" "$ROOT" + rm "$MGREP_WATCH_REGISTRY_DIR"/*.json + sleep 0.3 + assert_claim "$WATCH_PID" "$ROOT" + kill -0 "$unrelated" +} + +@test "dry run skips coordination and failed initial sync releases its claim" { + start_watcher "$ROOT" "$FIXTURE/watch.log" + active=$WATCH_PID + wait_active "$FIXTURE/watch.log" + run mgrep watch --dry-run + assert_success + assert_output --partial 'Initial sync complete' + refute_output --partial 'standing by' + assert [ "$(claim_count)" = 1 ] + kill -TERM "$active" + wait "$active" + echo new > "$ROOT/extra.txt" + echo new > "$ROOT/extra-2.txt" + run mgrep watch --max-file-count 1 + assert_failure + assert_output --partial 'File count exceeded' + assert [ "$(claim_count)" = 0 ] +} + +@test "realpath aliases share claims but path-prefix siblings and stores do not" { + ln -s "$ROOT" "$FIXTURE/alias" + mkdir "$ROOT/services-other" + start_watcher "$CHILD" "$FIXTURE/child.log" + wait_active "$FIXTURE/child.log" + start_watcher "$FIXTURE/alias/services" "$FIXTURE/alias.log" + wait_log "$FIXTURE/alias.log" 'standing by' + start_watcher "$ROOT/services-other" "$FIXTURE/sibling.log" + wait_active "$FIXTURE/sibling.log" + start_watcher "$CHILD" "$FIXTURE/other-store.log" --store another-store + wait_active "$FIXTURE/other-store.log" + assert [ "$(claim_count)" = 3 ] +} + +@test "claim ownership, coverage ordering and pid tie breaks are enforced" { + run node --title=mgrep-registry-test --input-type=module - "$DIR/../dist" "$ROOT" <<'JS' +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { pathToFileURL } from 'node:url'; +const { claimWatch, ensureClaim, releaseWatch, listActiveWatches, findCoveringWatch } = + await import(pathToFileURL(`${process.argv[2]}/lib/lock.js`)); +const root = process.argv[3]; +const claim = claimWatch('test', root, 10); +assert.ok(claim); +assert.equal(claimWatch('test', root, 11), null); +assert.equal(listActiveWatches('other').length, 0); +assert.deepEqual(listActiveWatches('test'), [claim]); +const file = `${process.env.MGREP_WATCH_REGISTRY_DIR}/${Buffer.from(`test\0${root}`).toString('base64url')}.json`; +fs.unlinkSync(file); +ensureClaim(claim); +assert.deepEqual(JSON.parse(fs.readFileSync(file)), claim); +const replacement = {...claim, pid: 99999999}; +fs.writeFileSync(file, JSON.stringify(replacement)); +ensureClaim(claim); +releaseWatch(claim); +assert.deepEqual(JSON.parse(fs.readFileSync(file)), replacement); +assert.equal(listActiveWatches('test').length, 0); +assert.equal(fs.existsSync(file), false); +const mine = {pid: 30, startedAt: 10}; +const earlier = {...claim, pid: 20}; +const later = {...claim, pid: 40}; +assert.equal(findCoveringWatch(`${root}/services`, mine, [later, earlier])?.pid, 20); +assert.equal(findCoveringWatch(`${root}/services`, mine, [later]), undefined); +assert.equal(findCoveringWatch(root, mine, [{...claim, ...mine}]), undefined); +fs.writeFileSync(`${root}/services/.mgreprc.yml`, 'maxFileCount: 1'); +assert.equal(findCoveringWatch(`${root}/services`, mine, [earlier]), undefined); +assert.equal(findCoveringWatch(`${root}/services`, mine, [{...earlier, root: `${root}/services`}])?.pid, 20); +JS + assert_success + assert [ "$(claim_count)" = 0 ] +} + +@test "quota and thrown initialSync failures leave no claim" { + echo new-file > "$ROOT/new.txt" + for failure in quota thrown; do + run node --title=mgrep-failure-test --input-type=module - "$DIR/../dist" "$failure" <<'JS' +import { pathToFileURL } from 'node:url'; +const base = process.argv[2]; +const { TestStore } = await import(pathToFileURL(`${base}/lib/store.js`)); +TestStore.prototype.uploadFile = async function () { + throw new Error('Free tier quota exceeded'); +}; +if (process.argv[3] === 'thrown') { + TestStore.prototype.listFiles = async function* () { throw new Error('initialSync threw'); }; +} +const { startWatch } = await import(pathToFileURL(`${base}/commands/watch.js`)); +await startWatch({store: 'mgrep', dryRun: false}); +JS + assert_failure + if [ "$failure" = quota ]; then + assert_output --partial 'Quota exceeded' + else + assert_output --partial 'Failed to start watcher: initialSync threw' + fi + assert [ "$(claim_count)" = 0 ] + done +} + +@test "per-file initial upload errors keep the claim and keep watching" { + echo new-file > "$ROOT/new.txt" + touch "$FIXTURE/reject" + start_injected_watcher "$ROOT" "$FIXTURE/watch.log" upload + wait_active "$FIXTURE/watch.log" + wait_log "$FIXTURE/watch.log" '1 file(s) failed to upload' + assert_claim "$WATCH_PID" "$ROOT" + kill -0 "$WATCH_PID" + rm "$FIXTURE/reject" + echo recovered >> "$ROOT/new.txt" + wait_log "$FIXTURE/watch.log" ": $ROOT/new.txt" + assert_claim "$WATCH_PID" "$ROOT" +} + +@test "changed-file upload failure keeps watching and a later change uploads" { + start_injected_watcher "$ROOT" "$FIXTURE/watch.log" upload + wait_active "$FIXTURE/watch.log" + touch "$FIXTURE/reject" + echo failed-change >> "$ROOT/parent.txt" + wait_log "$FIXTURE/watch.log" "Failed to upload changed file: $ROOT/parent.txt" + assert_claim "$WATCH_PID" "$ROOT" + kill -0 "$WATCH_PID" + rm "$FIXTURE/reject" + echo recovered-change >> "$ROOT/parent.txt" + wait_log "$FIXTURE/watch.log" ": $ROOT/parent.txt" + node - "$ROOT/parent.txt" <<'JS' +const fs = require('fs'); +const file = JSON.parse(fs.readFileSync(process.env.MGREP_TEST_STORE_PATH)).files[process.argv[2]]; +if (!file.content.includes('recovered-change')) process.exit(1); +JS + assert_claim "$WATCH_PID" "$ROOT" +} + +@test "changed-file delete failure keeps watching and claimed" { + start_injected_watcher "$ROOT" "$FIXTURE/watch.log" delete + wait_active "$FIXTURE/watch.log" + touch "$FIXTURE/reject" + rm "$ROOT/parent.txt" + wait_log "$FIXTURE/watch.log" "Failed to delete file: $ROOT/parent.txt" + assert_claim "$WATCH_PID" "$ROOT" + kill -0 "$WATCH_PID" + rm "$FIXTURE/reject" + echo recovered >> "$CHILD/child.txt" + wait_log "$FIXTURE/watch.log" ": $CHILD/child.txt" + assert_claim "$WATCH_PID" "$ROOT" +} + +@test "directory watcher error closes only that directory and keeps the claim" { + mkdir "$CHILD/nested" + echo nested > "$CHILD/nested/file.txt" + start_injected_watcher "$ROOT" "$FIXTURE/watch.log" directory + wait_active "$FIXTURE/watch.log" + touch "$FIXTURE/reject" + wait_log "$FIXTURE/watch.log" "Failed to watch $CHILD:" + wait_log "$FIXTURE/watch.log" "closed directory: $CHILD" + assert_claim "$WATCH_PID" "$ROOT" + kill -0 "$WATCH_PID" + echo recovered >> "$ROOT/parent.txt" + wait_log "$FIXTURE/watch.log" ": $ROOT/parent.txt" + echo recovered >> "$CHILD/nested/file.txt" + wait_log "$FIXTURE/watch.log" ": $CHILD/nested/file.txt" + assert_claim "$WATCH_PID" "$ROOT" +} + +@test "coordination poll failure is logged and retried with the claim intact" { + start_injected_watcher "$ROOT" "$FIXTURE/watch.log" poll + wait_active "$FIXTURE/watch.log" + touch "$FIXTURE/reject" + wait_log "$FIXTURE/watch.log" 'injected poll failure' + assert_claim "$WATCH_PID" "$ROOT" + rm "$MGREP_WATCH_REGISTRY_DIR"/*.json + sleep 0.3 + assert_claim "$WATCH_PID" "$ROOT" + echo recovered >> "$ROOT/parent.txt" + wait_log "$FIXTURE/watch.log" ": $ROOT/parent.txt" +} + +@test "failed released-subtree resync is retried and installs subtree watchers" { + for failure in sync sync-quota sync-count; do + start_watcher "$CHILD" "$FIXTURE/child.log" + child=$WATCH_PID + wait_active "$FIXTURE/child.log" + start_injected_watcher "$ROOT" "$FIXTURE/parent.log" "$failure" + parent=$WATCH_PID + wait_active "$FIXTURE/parent.log" + touch "$FIXTURE/reject" + kill -TERM "$child" + wait "$child" + wait_log "$FIXTURE/parent.log" 'Failed to coordinate watcher:' + if [ "$failure" = sync-quota ]; then + sleep 0.5 + assert [ "$(grep -c 'Initial sync complete' "$FIXTURE/parent.log")" = 1 ] + echo 600001 > "$FIXTURE/reject.advance" + fi + for _ in {1..30}; do + if [ "$(grep -c 'Initial sync complete' "$FIXTURE/parent.log")" -ge 2 ]; then break; fi + sleep 0.1 + done + assert [ "$(grep -c 'Initial sync complete' "$FIXTURE/parent.log")" -ge 2 ] + assert_claim "$parent" "$ROOT" + sleep 0.15 + echo recovered >> "$CHILD/child.txt" + wait_log "$FIXTURE/parent.log" ": $CHILD/child.txt" + kill -TERM "$parent" + wait "$parent" + done +} + +@test "store retrieve and create failure releases the startup claim" { + run node --title=mgrep-store-failure-test --input-type=module - "$DIR/../dist" <<'JS' +import { pathToFileURL } from 'node:url'; +const base = process.argv[2]; +const { TestStore } = await import(pathToFileURL(`${base}/lib/store.js`)); +TestStore.prototype.retrieve = async function () { throw new Error('retrieve failed'); }; +TestStore.prototype.create = async function () { throw new Error('create failed'); }; +const { startWatch } = await import(pathToFileURL(`${base}/commands/watch.js`)); +await startWatch({store: 'mgrep', dryRun: false}); +JS + assert_failure + assert_output --partial 'Failed to start watcher: create failed' + assert [ "$(claim_count)" = 0 ] +} + +@test "old invalid claims are pruned by claiming and listing but fresh claims block" { + run node --title=mgrep-stale-test --input-type=module - "$DIR/../dist" "$ROOT" <<'JS' +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { pathToFileURL } from 'node:url'; +const { claimWatch, releaseWatch, listActiveWatches } = + await import(pathToFileURL(`${process.argv[2]}/lib/lock.js`)); +const root = process.argv[3]; +const file = `${process.env.MGREP_WATCH_REGISTRY_DIR}/${Buffer.from(`mgrep\0${root}`).toString('base64url')}.json`; +for (const content of ['{', JSON.stringify({pid: 'invalid'})]) { + fs.writeFileSync(file, content); + assert.equal(claimWatch('mgrep', root, Date.now()), null); + assert.deepEqual(listActiveWatches('mgrep'), []); + assert.equal(fs.readFileSync(file, 'utf8'), content); + const old = new Date(Date.now() - 31000); + fs.utimesSync(file, old, old); + const claim = claimWatch('mgrep', root, Date.now()); + assert.ok(claim); + releaseWatch(claim); + fs.writeFileSync(file, content); + fs.utimesSync(file, old, old); + assert.deepEqual(listActiveWatches('mgrep'), []); + assert.equal(fs.existsSync(file), false); + const afterPrune = claimWatch('mgrep', root, Date.now()); + assert.ok(afterPrune); + releaseWatch(afterPrune); +} +JS + assert_success + assert [ "$(claim_count)" = 0 ] +} + +@test "initialSync realpaths excluded aliases before counting uploads or deletions" { + ln -s "$CHILD" "$FIXTURE/excluded-alias" + rm "$CHILD/child.txt" + echo untouched > "$CHILD/new.txt" + run node --title=mgrep-sync-test --input-type=module - "$DIR/../dist" "$ROOT" "$FIXTURE/excluded-alias" <<'JS' +import assert from 'node:assert/strict'; +import { pathToFileURL } from 'node:url'; +const base = process.argv[2]; +const { initialSync } = await import(pathToFileURL(`${base}/lib/utils.js`)); +const { createFileSystem, createStore } = await import(pathToFileURL(`${base}/lib/context.js`)); +const { loadConfig } = await import(pathToFileURL(`${base}/lib/config.js`)); +const result = await initialSync(await createStore(), createFileSystem(), 'mgrep', process.argv[3], + false, undefined, loadConfig(process.argv[3], {maxFileCount: 1}), [process.argv[4]]); +assert.deepEqual(result, {processed: 1, uploaded: 0, deleted: 0, errors: 0, total: 1}); +JS + assert_success +}