diff --git a/scripts/measure-tool-schema.mjs b/scripts/measure-tool-schema.mjs index 1669f85..dd42c80 100755 --- a/scripts/measure-tool-schema.mjs +++ b/scripts/measure-tool-schema.mjs @@ -49,13 +49,34 @@ const SERVER = fileURLToPath(new URL("../dist/index.js", import.meta.url)); const PROFILES = userCmd.length ? [null] : ["full", "media", "trading", "research", "chat"]; const PREFIX = process.env.MCP_PREFIX ?? (userCmd.length ? "" : "mcp__blockrun__"); -/** One MCP handshake over stdio. Returns the tools array verbatim. */ -function listTools(cmd) { +/** + * One MCP handshake over stdio. Returns the tools array verbatim. + * + * A server that dies before answering is the common case when you point this + * at someone else's command — a typo'd package, a missing bin, a cold `npx` + * that 404s. Report THAT, with its stderr, instead of waiting out the timeout + * and blaming time: the first version of this reported "timed out" after a + * silent minute for what was actually `sh: foo: command not found` in the + * first 200ms. + */ +export function listTools(cmd, { timeoutMs = 120_000 } = {}) { return new Promise((resolve, reject) => { - const child = spawn(cmd[0], cmd.slice(1), { stdio: ["pipe", "pipe", "ignore"] }); + // stderr is piped, not ignored, so a failure can explain itself. Only the + // tail is kept — a server that logs a lot should not be buffered whole. + const child = spawn(cmd[0], cmd.slice(1), { stdio: ["pipe", "pipe", "pipe"] }); const pending = new Map(); let buf = ""; + let stderr = ""; let id = 0; + let settled = false; + + const finish = (fn, arg) => { + if (settled) return; + settled = true; + clearTimeout(timer); + child.kill(); + fn(arg); + }; const send = (m) => child.stdin.write(`${JSON.stringify(m)}\n`); const rpc = (method, params) => @@ -64,11 +85,23 @@ function listTools(cmd) { send({ jsonrpc: "2.0", id, method, params }); }); - child.on("error", reject); - const timer = setTimeout(() => { - child.kill(); - reject(new Error(`timed out waiting for ${cmd.join(" ")}`)); - }, 60_000); + const timer = setTimeout( + () => finish(reject, new Error( + `${cmd.join(" ")} did not answer within ${timeoutMs / 1000}s. ` + + `A cold \`npx\` install of a large package can exceed this — run it once first.`, + )), + timeoutMs, + ); + + child.stderr.on("data", (d) => { stderr = (stderr + d).slice(-2000); }); + child.on("error", (e) => finish(reject, e)); + child.on("exit", (code) => finish(reject, new Error( + `${cmd.join(" ")} exited with code ${code} before completing the handshake.\n` + + (stderr.trim() || "(no stderr output)"), + ))); + + // EPIPE rather than an unhandled crash when the child is already gone. + child.stdin.on("error", () => {}); child.stdout.on("data", (chunk) => { buf += chunk; @@ -92,10 +125,8 @@ function listTools(cmd) { }); send({ jsonrpc: "2.0", method: "notifications/initialized" }); const { result } = await rpc("tools/list", {}); - clearTimeout(timer); - child.kill(); - resolve(result?.tools ?? []); - })().catch(reject); + finish(resolve, result?.tools ?? []); + })().catch((e) => finish(reject, e)); }); } diff --git a/test/schema-tokens.test.ts b/test/schema-tokens.test.ts index 4f036f0..ad39247 100644 --- a/test/schema-tokens.test.ts +++ b/test/schema-tokens.test.ts @@ -24,8 +24,8 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; import { initializeMcpServer } from "../src/mcp-handler.js"; -// @ts-expect-error -- plain .mjs, no types; measure()/asK() are the contract. -import { measure, asK } from "../scripts/measure-tool-schema.mjs"; +// @ts-expect-error -- plain .mjs, no types; measure()/asK()/listTools() are the contract. +import { measure, asK, listTools } from "../scripts/measure-tool-schema.mjs"; const README = readFileSync(new URL("../README.md", import.meta.url), "utf8"); const PREFIX = "mcp__blockrun__"; @@ -99,3 +99,20 @@ test("descriptions are the majority of the cost, which is why the table ranks th `descriptions ${full.descriptions} should exceed half of ${full.total}`, ); }); + +test("a server that dies reports why, instead of waiting out the timeout", async () => { + // We tell people to point this at other people's servers, where the common + // failure is a typo'd command. Reporting "timed out" after a silent minute + // for what was `command not found` in the first 200ms is the difference + // between a tool someone uses twice and one they use once. + const started = Date.now(); + await assert.rejects( + () => listTools(["node", "/nonexistent-mcp-server.js"], { timeoutMs: 30_000 }), + (err: Error) => { + assert.match(err.message, /exited with code/, "names the exit"); + assert.match(err.message, /Cannot find module/, "carries the child's stderr"); + return true; + }, + ); + assert.ok(Date.now() - started < 10_000, "should fail fast, not on the timeout"); +});