diff --git a/README.md b/README.md index 2f07173..d9088ec 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Agents can only sign transactions.


20 tools  +12.9K context tokens  Agent native  No API keys  Read and trade Polymarket  @@ -173,6 +174,49 @@ codex mcp add blockrun-trading -- npx -y @blockrun/mcp@latest --profile trading ``` An unknown profile name falls back to `full`. `modal` and `phone` are `full`-profile only. + +#### What each profile costs your context + +Installing an MCP server spends context on **every turn**, whether or not you call the tools โ€” +the client loads each tool's schema into the model's prompt and re-sends it for the whole session. +Package managers have shown install size for decades. Almost no MCP server shows this. Ours: + +| Profile | Tools | Context | +|---------|-------|---------| +| `full` *(default)* | 20 | 12,900 | +| `trading` | 9 | 5,554 | +| `media` | 7 | 5,436 | +| `research` | 6 | 3,024 | +| `chat` | 3 | 1,924 | + +Running `--profile trading` instead of the default costs **57% less context** for the same trading +workflow. If you only ever ask about markets, that is the single cheapest change you can make. + +Measure it yourself โ€” against us, or against any other stdio MCP server: + +```bash +npm i gpt-tokenizer +node scripts/measure-tool-schema.mjs # this server, every profile +node scripts/measure-tool-schema.mjs -- npx -y @some/other-mcp-server +``` + +
+How the number is computed, and why it is a slight under-count + +It counts the **model-visible projection** โ€” `{name, description, input_schema}` per tool, with the +`mcp__blockrun__` prefix the host prepends โ€” because that is what lands in the API `tools` array. +It excludes `annotations`, `_meta` and `outputSchema`, which the host consumes and never forwards +to the model (a further ~3.7% on the wire). + +The tokenizer is `o200k_base`. Claude's tokenizer is not public and runs a few percent higher on +JSON, so **every figure here is a slight under-count**, never an over-count. + +Two caveats worth stating plainly. Tool schemas sit at the front of the prompt and are covered by +prompt caching, so after the first turn they re-send at cache-read rates โ€” the *context-window* +cost is 100% every turn, the *dollar* cost is roughly a tenth of that. And 54% of our own cost is +tool **descriptions**, not schemas, which is where the remaining work is. + +
For a complete live signal โ†’ order-preview presentation, use [`skills/signal-to-trade-demo/SKILL.md`](skills/signal-to-trade-demo/SKILL.md) with the [Stanford runbook](docs/stanford-trading-demo.md). diff --git a/package-lock.json b/package-lock.json index 17ef54e..75d387b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,6 +32,7 @@ "@modelcontextprotocol/ext-apps": "^1.7.5", "@types/node": "^20.0.0", "@types/qrcode": "^1.5.6", + "gpt-tokenizer": "^4.0.0", "tsup": "^8.0.0", "tsx": "^4.0.0", "typescript": "^5.0.0", @@ -4520,6 +4521,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gpt-tokenizer": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/gpt-tokenizer/-/gpt-tokenizer-4.0.0.tgz", + "integrity": "sha512-YAWIyzvuVUHEfW7tFfFAxH8qQb+Q3RU9nYOTy7skMNX5qzU6Q8jxTHZLyO56ug1vYvCR7wndzpd3jwD86/mhjQ==", + "dev": true, + "license": "MIT" + }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", diff --git a/package.json b/package.json index b8809d0..5dee569 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "test": "tsx --experimental-test-module-mocks --test test/*.test.ts", "prepublishOnly": "npm run build", "verify:prices": "tsx scripts/verify-prices.ts", + "measure:schema": "node scripts/measure-tool-schema.mjs", "e2e:polymarket:readonly": "tsx scripts/polymarket-e2e-readonly.ts", "e2e:polymarket:approvals": "tsx scripts/polymarket-e2e-verify-approvals.ts", "e2e:polymarket:approve": "tsx scripts/polymarket-e2e-approve.ts", @@ -86,6 +87,7 @@ "@modelcontextprotocol/ext-apps": "^1.7.5", "@types/node": "^20.0.0", "@types/qrcode": "^1.5.6", + "gpt-tokenizer": "^4.0.0", "tsup": "^8.0.0", "tsx": "^4.0.0", "typescript": "^5.0.0", diff --git a/scripts/measure-tool-schema.mjs b/scripts/measure-tool-schema.mjs new file mode 100755 index 0000000..9a4bdd3 --- /dev/null +++ b/scripts/measure-tool-schema.mjs @@ -0,0 +1,160 @@ +#!/usr/bin/env node +/** + * Measure what an MCP server costs a model's context window. + * + * Installing an MCP server puts every tool's schema into the model's prompt, + * on every turn, whether or not the tools are ever called. Package managers + * have shown install size for decades; nothing shows this. So: measure it. + * + * node scripts/measure-tool-schema.mjs this repo, every profile + * node scripts/measure-tool-schema.mjs --json machine-readable + * node scripts/measure-tool-schema.mjs -- npx -y @foo/bar ANY stdio MCP server + * + * Requires `gpt-tokenizer` (a devDependency here; `npm i gpt-tokenizer` elsewhere). + * + * WHAT IS COUNTED: the model-visible projection โ€” `{name, description, + * input_schema}` per tool, with the host's name prefix โ€” because that is what + * lands in the API `tools` array. Not counted: `annotations`, `_meta` and + * `outputSchema`, which the host consumes and does not forward to the model. + * On this server that wire overhead is a further ~3.7%. + * + * TWO WAYS TO GET THIS WRONG, both of which cost us a retraction: + * + * 1. Reading your own source instead of the wire. The schema the model + * receives is GENERATED โ€” the SDK adds fields your source never mentions. + * Always measure a live `tools/list`. + * 2. Re-encoding before counting. `JSON.stringify` is correct because it + * leaves non-ASCII alone. Python's `json.dumps` defaults to + * `ensure_ascii=True`, which turns every em-dash into `\uXXXX` โ€” six + * characters where the model sees one glyph. That inflated our first + * numbers by 4.4%. If your count disagrees with a colleague's while your + * DESCRIPTION totals match to the token, this is why. + * + * The tokenizer is o200k_base. Claude's tokenizer is not public and runs a few + * percent higher on JSON, so every number here is a slight UNDER-count. + */ +import { spawn } from "node:child_process"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { encode } from "gpt-tokenizer/encoding/o200k_base"; + +const argv = process.argv.slice(2); +const sep = argv.indexOf("--"); +const userCmd = sep === -1 ? [] : argv.slice(sep + 1); +const flags = new Set(sep === -1 ? argv : argv.slice(0, sep)); +const asJson = flags.has("--json"); + +const SERVER = fileURLToPath(new URL("../dist/index.js", import.meta.url)); +// Only this repo's own server has profiles to sweep; a foreign server is one run. +const PROFILES = userCmd.length ? [null] : ["full", "media", "trading", "research", "chat"]; +const PREFIX = process.env.MCP_PREFIX ?? (userCmd.length ? "" : "mcp__blockrun__"); + +/** One MCP handshake over stdio. Returns the tools array verbatim. */ +function listTools(cmd) { + return new Promise((resolve, reject) => { + const child = spawn(cmd[0], cmd.slice(1), { stdio: ["pipe", "pipe", "ignore"] }); + const pending = new Map(); + let buf = ""; + let id = 0; + + const send = (m) => child.stdin.write(`${JSON.stringify(m)}\n`); + const rpc = (method, params) => + new Promise((res) => { + pending.set(++id, res); + send({ jsonrpc: "2.0", id, method, params }); + }); + + child.on("error", reject); + const timer = setTimeout(() => { + child.kill(); + reject(new Error(`timed out waiting for ${cmd.join(" ")}`)); + }, 60_000); + + child.stdout.on("data", (chunk) => { + buf += chunk; + let i; + while ((i = buf.indexOf("\n")) !== -1) { + const line = buf.slice(0, i).trim(); + buf = buf.slice(i + 1); + if (!line) continue; + let msg; + try { msg = JSON.parse(line); } catch { continue; } + const done = pending.get(msg.id); + if (done) { pending.delete(msg.id); done(msg); } + } + }); + + (async () => { + await rpc("initialize", { + protocolVersion: "2025-06-18", + capabilities: {}, + clientInfo: { name: "measure-tool-schema", version: "1.0.0" }, + }); + send({ jsonrpc: "2.0", method: "notifications/initialized" }); + const { result } = await rpc("tools/list", {}); + clearTimeout(timer); + child.kill(); + resolve(result?.tools ?? []); + })().catch(reject); + }); +} + +/** The projection the model actually receives, tokenized. */ +export function measure(tools, prefix = "") { + const rows = tools.map((t) => { + const schema = t.inputSchema ?? {}; + const description = t.description ?? ""; + return { + tool: t.name, + tokens: encode(JSON.stringify({ + name: prefix + t.name, + description, + input_schema: schema, + })).length, + description: encode(description).length, + schema: encode(JSON.stringify(schema)).length, + }; + }); + rows.sort((a, b) => b.tokens - a.tokens); + const total = rows.reduce((n, r) => n + r.tokens, 0); + return { + tools: rows.length, + total, + descriptions: rows.reduce((n, r) => n + r.description, 0), + schemas: rows.reduce((n, r) => n + r.schema, 0), + perTool: rows.length ? Math.round(total / rows.length) : 0, + rows, + }; +} + +/** 12900 -> "12.9K". The form the README badge publishes. */ +export const asK = (n) => `${(Math.round(n / 100) / 10).toFixed(1)}K`; + +// Importable: test/schema-tokens.test.ts reuses measure()/asK() to pin the +// published number, so the CLI half must not run on import. +const isCli = process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href; +if (!isCli) { /* library use */ } else await main(); + +async function main() { +const results = {}; +for (const profile of PROFILES) { + const cmd = userCmd.length + ? userCmd + : ["node", SERVER, ...(profile && profile !== "full" ? ["--profile", profile] : [])]; + results[profile ?? userCmd.join(" ")] = measure(await listTools(cmd), PREFIX); +} + +if (asJson) { + console.log(JSON.stringify(results, null, 2)); +} else { + const [headline] = Object.values(results); + for (const [name, r] of Object.entries(results)) { + const pct = (n) => `${((n / r.total) * 100).toFixed(1)}%`; + console.log( + `\n${name} โ€” ${r.total.toLocaleString()} tokens across ${r.tools} tools ` + + `(${r.perTool}/tool ยท descriptions ${pct(r.descriptions)} ยท schemas ${pct(r.schemas)})`, + ); + if (name === "full" || PROFILES.length === 1) console.table(r.rows); + } + console.log(`\nbadge: ${asK(headline.total)}`); +} +} diff --git a/test/schema-tokens.test.ts b/test/schema-tokens.test.ts new file mode 100644 index 0000000..1ea48ce --- /dev/null +++ b/test/schema-tokens.test.ts @@ -0,0 +1,87 @@ +// Run with: npm test (tsx --test) +// +// Pins the context-window cost this repo publishes. +// +// Installing an MCP server spends the user's context on every turn, whether or +// not the tools are called. We put that number on the README the way a package +// manager puts install size on a download โ€” which only means anything if it +// cannot quietly go stale. +// +// So this measures the live server and fails HERE, in the repo that can fix it, +// rather than letting the badge make a claim that stopped being true three +// description edits ago. +// +// The assertion is on the PUBLISHED form ("12.9K"), not the raw token count. +// Pinning 12,900 exactly would fail CI on every wording tweak; pinning the +// rounded figure fails exactly when the published claim becomes wrong. +// +// When it fails: run `npm run measure:schema`, confirm the change is intended, +// and update the README badge and the profile table to what it prints. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { initializeMcpServer } from "../src/mcp-handler.js"; +// @ts-expect-error -- plain .mjs, no types; measure()/asK() are the contract. +import { measure, asK } from "../scripts/measure-tool-schema.mjs"; + +const README = readFileSync(new URL("../README.md", import.meta.url), "utf8"); +const PREFIX = "mcp__blockrun__"; + +/** Same projection the CLI harness measures, over an in-process handshake. */ +async function measureProfile(profile: string) { + const server = new McpServer({ name: "test", version: "0.0.0" }); + initializeMcpServer(server, { argv: ["--profile", profile], env: {} }); + const [clientSide, serverSide] = InMemoryTransport.createLinkedPair(); + const client = new Client({ name: "test-client", version: "0.0.0" }); + await Promise.all([server.connect(serverSide), client.connect(clientSide)]); + const { tools } = await client.listTools(); + await client.close(); + return measure(tools, PREFIX) as { + tools: number; total: number; descriptions: number; perTool: number; + }; +} + +test("the README badge states the measured context cost", async () => { + const full = await measureProfile("full"); + const shown = asK(full.total) as string; + assert.ok( + README.includes(`badge/๐Ÿงฎ_${shown}_Context_Tokens`), + `badge URL should show ${shown} (measured ${full.total} tokens)`, + ); + assert.ok(README.includes(`alt="${shown} context tokens"`), "badge alt text"); +}); + +test("the profile table states each profile's measured cost", async () => { + // Every row is a public claim; a trimmed profile that quietly grew would + // otherwise keep advertising a saving it no longer delivers. + for (const profile of ["full", "media", "trading", "research", "chat"]) { + const { total, tools } = await measureProfile(profile); + // Tolerant of row decoration (`full` carries a *(default)* marker), strict + // on both numbers โ€” the decoration is prose, the numbers are the claim. + const row = new RegExp( + `\\| \`${profile}\`[^|]*\\| ${tools} \\| ${total.toLocaleString()} \\|`, + ); + assert.match(README, row, + `README row for ${profile} should read ${tools} tools / ${total.toLocaleString()} tokens`); + } +}); + +test("the advertised profile saving is the one measured", async () => { + const full = await measureProfile("full"); + const trading = await measureProfile("trading"); + const cut = Math.round((1 - trading.total / full.total) * 100); + assert.ok(README.includes(`${cut}% less context`), `saving should read ${cut}%`); +}); + +test("descriptions are the majority of the cost, which is why the table ranks them", async () => { + // Not decoration: it is the finding the whole measurement exists to surface, + // and if schemas ever overtake descriptions the README's advice is wrong. + const full = await measureProfile("full"); + assert.ok( + full.descriptions > full.total / 2, + `descriptions ${full.descriptions} should exceed half of ${full.total}`, + ); +});