From 198ea6787fb0f9a6b1860762183c17878abd77ac Mon Sep 17 00:00:00 2001 From: VickyXAI <115643921+VickyXAI@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:42:45 -0500 Subject: [PATCH] fix(measure): report why a server died instead of waiting out the timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pointing the harness at a command that fails to start gave you a silent 60-second wait and then "timed out" — for what was `sh: blockrun-mcp: command not found` in the first 200ms. Since we advertise this for measuring OTHER people's servers, where a mistyped command is the common case, that is the difference between a tool someone uses twice and one they use once. stderr is now piped rather than ignored (tail only, so a chatty server is not buffered whole) and a child that exits before the handshake completes rejects immediately with its exit code and its own error text. Failure went from 60s and a wrong reason to 0.25s and the right one. The timeout is also raised 60s -> 120s and now says what to do about it: a cold `npx` install of a package this size genuinely exceeds a minute, which is how the bad message got found. Found while verifying the published 0.45.1 end-to-end. Worth recording that `npx -y @blockrun/mcp@` run from inside this repo resolves to the local checkout and fails that way — measure published packages from a neutral directory. From one, 0.45.1 measures 12,900, matching the card. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01WUL3ExR4Nz7uebxKaKwjDi --- scripts/measure-tool-schema.mjs | 55 ++++++++++++++++++++++++++------- test/schema-tokens.test.ts | 21 +++++++++++-- 2 files changed, 62 insertions(+), 14 deletions(-) diff --git a/scripts/measure-tool-schema.mjs b/scripts/measure-tool-schema.mjs index 1669f85..dd42c80 100755 --- a/scripts/measure-tool-schema.mjs +++ b/scripts/measure-tool-schema.mjs @@ -49,13 +49,34 @@ const SERVER = fileURLToPath(new URL("../dist/index.js", import.meta.url)); const PROFILES = userCmd.length ? [null] : ["full", "media", "trading", "research", "chat"]; const PREFIX = process.env.MCP_PREFIX ?? (userCmd.length ? "" : "mcp__blockrun__"); -/** One MCP handshake over stdio. Returns the tools array verbatim. */ -function listTools(cmd) { +/** + * One MCP handshake over stdio. Returns the tools array verbatim. + * + * A server that dies before answering is the common case when you point this + * at someone else's command — a typo'd package, a missing bin, a cold `npx` + * that 404s. Report THAT, with its stderr, instead of waiting out the timeout + * and blaming time: the first version of this reported "timed out" after a + * silent minute for what was actually `sh: foo: command not found` in the + * first 200ms. + */ +export function listTools(cmd, { timeoutMs = 120_000 } = {}) { return new Promise((resolve, reject) => { - const child = spawn(cmd[0], cmd.slice(1), { stdio: ["pipe", "pipe", "ignore"] }); + // stderr is piped, not ignored, so a failure can explain itself. Only the + // tail is kept — a server that logs a lot should not be buffered whole. + const child = spawn(cmd[0], cmd.slice(1), { stdio: ["pipe", "pipe", "pipe"] }); const pending = new Map(); let buf = ""; + let stderr = ""; let id = 0; + let settled = false; + + const finish = (fn, arg) => { + if (settled) return; + settled = true; + clearTimeout(timer); + child.kill(); + fn(arg); + }; const send = (m) => child.stdin.write(`${JSON.stringify(m)}\n`); const rpc = (method, params) => @@ -64,11 +85,23 @@ function listTools(cmd) { send({ jsonrpc: "2.0", id, method, params }); }); - child.on("error", reject); - const timer = setTimeout(() => { - child.kill(); - reject(new Error(`timed out waiting for ${cmd.join(" ")}`)); - }, 60_000); + const timer = setTimeout( + () => finish(reject, new Error( + `${cmd.join(" ")} did not answer within ${timeoutMs / 1000}s. ` + + `A cold \`npx\` install of a large package can exceed this — run it once first.`, + )), + timeoutMs, + ); + + child.stderr.on("data", (d) => { stderr = (stderr + d).slice(-2000); }); + child.on("error", (e) => finish(reject, e)); + child.on("exit", (code) => finish(reject, new Error( + `${cmd.join(" ")} exited with code ${code} before completing the handshake.\n` + + (stderr.trim() || "(no stderr output)"), + ))); + + // EPIPE rather than an unhandled crash when the child is already gone. + child.stdin.on("error", () => {}); child.stdout.on("data", (chunk) => { buf += chunk; @@ -92,10 +125,8 @@ function listTools(cmd) { }); send({ jsonrpc: "2.0", method: "notifications/initialized" }); const { result } = await rpc("tools/list", {}); - clearTimeout(timer); - child.kill(); - resolve(result?.tools ?? []); - })().catch(reject); + finish(resolve, result?.tools ?? []); + })().catch((e) => finish(reject, e)); }); } diff --git a/test/schema-tokens.test.ts b/test/schema-tokens.test.ts index 4f036f0..ad39247 100644 --- a/test/schema-tokens.test.ts +++ b/test/schema-tokens.test.ts @@ -24,8 +24,8 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { Client } from "@modelcontextprotocol/sdk/client/index.js"; import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; import { initializeMcpServer } from "../src/mcp-handler.js"; -// @ts-expect-error -- plain .mjs, no types; measure()/asK() are the contract. -import { measure, asK } from "../scripts/measure-tool-schema.mjs"; +// @ts-expect-error -- plain .mjs, no types; measure()/asK()/listTools() are the contract. +import { measure, asK, listTools } from "../scripts/measure-tool-schema.mjs"; const README = readFileSync(new URL("../README.md", import.meta.url), "utf8"); const PREFIX = "mcp__blockrun__"; @@ -99,3 +99,20 @@ test("descriptions are the majority of the cost, which is why the table ranks th `descriptions ${full.descriptions} should exceed half of ${full.total}`, ); }); + +test("a server that dies reports why, instead of waiting out the timeout", async () => { + // We tell people to point this at other people's servers, where the common + // failure is a typo'd command. Reporting "timed out" after a silent minute + // for what was `command not found` in the first 200ms is the difference + // between a tool someone uses twice and one they use once. + const started = Date.now(); + await assert.rejects( + () => listTools(["node", "/nonexistent-mcp-server.js"], { timeoutMs: 30_000 }), + (err: Error) => { + assert.match(err.message, /exited with code/, "names the exit"); + assert.match(err.message, /Cannot find module/, "carries the child's stderr"); + return true; + }, + ); + assert.ok(Date.now() - started < 10_000, "should fail fast, not on the timeout"); +});