diff --git a/.github/ISSUE_TEMPLATE/audit_false_positive.md b/.github/ISSUE_TEMPLATE/audit_false_positive.md new file mode 100644 index 0000000..f13426a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/audit_false_positive.md @@ -0,0 +1,25 @@ +--- +name: Audit false positive +about: Report a finding that does not fit your MCP server's intended behavior +title: "" +labels: audit-false-positive +assignees: "" +--- + +## Finding + +- Finding id: +- Tool name: +- MCPLens version: + +## Why this looks false positive + + +## Intended tool behavior + + +## Sanitized input + +Please include the relevant sanitized `tools/list` entry, config, and report snippet. Do +not include secrets, customer data, private URLs, or production payloads. + diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..34b7b24 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,31 @@ +--- +name: Bug report +about: Report a reproducible problem with MCPLens +title: "" +labels: bug +assignees: "" +--- + +## What happened? + + +## How to reproduce + +```sh + +``` + +## Expected behavior + + +## Environment + +- MCPLens version: +- Node.js version: +- OS: + +## Logs or input + +Please include sanitized `tools/list`, manifest, config, or command output snippets when +they are relevant. Do not include secrets or production data. + diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..4b61a4f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an improvement for MCPLens +title: "" +labels: enhancement +assignees: "" +--- + +## Problem + + +## Proposed behavior + + +## Alternatives considered + + +## Additional context + + diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..afb31e6 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,11 @@ +## Summary + + +## Testing + +- [ ] `npm test` +- [ ] `npm run smoke:pack` if package contents, CLI entry points, or release metadata changed + +## Notes + + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..7b9603a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,23 @@ +name: CI + +on: + pull_request: + branches: [main] + push: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + + - run: npm ci + - run: npm run build + - run: npm test + - run: npm run smoke:pack diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ce6290e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,18 @@ +# Changelog + +All notable changes to MCPLens will be documented in this file. + +## 0.1.2 - 2026-07-01 + +### Added + +- Local/offline `audit-mcp` command for MCP `tools/list` exports, usage logs, missed prompts, baseline comparison, CI summaries, Markdown reports, JSON reports, and capability recommendations. +- Browser MCP audit profile for browser/session/page action tools, including checks for mutation contracts, preconditions, and trace artifacts. +- OpenAPI-to-MCP `compile` and `build` flow for generating lean TypeScript MCP servers from curated manifests. +- Example API fixtures and impact reports for GitHub, Stripe, Slack, Google Calendar, and Notion. +- Pack smoke test that installs the npm tarball, verifies both `mcplens` and legacy `agentify` binaries, and runs an installed `audit-mcp` command. + +### Changed + +- Published package contents now include the guide, impact report, CI documentation, examples, changelog, and license alongside the built CLI. + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..c22cbb2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,35 @@ +# Contributing + +Thanks for helping improve MCPLens. This project is still early, so small, focused changes +are easiest to review. + +## Development Setup + +```sh +npm ci +npm run build +npm test +npm run smoke:pack +``` + +MCPLens requires Node.js 20 or newer. + +## Pull Requests + +- Keep changes focused on one behavior or documentation update. +- Add or update tests when behavior changes. +- Run `npm test` before opening a pull request. +- Run `npm run smoke:pack` when package contents, CLI entry points, or release metadata change. +- Avoid committing generated `dist/`, `node_modules/`, local reports, secrets, or `.env` files. + +## Local Audit Behavior + +The `audit-mcp` command is local and offline by design. If you add audit features, preserve +that contract unless the change explicitly introduces an opt-in network path and documents +it clearly. + +## Reporting Issues + +Please include the command you ran, your Node.js version, sanitized input snippets when +possible, and the actual output or stack trace. + diff --git a/README.md b/README.md index 2dfece4..1716380 100644 --- a/README.md +++ b/README.md @@ -1,96 +1,154 @@ # MCPLens -MCPLens helps MCP server builders see whether agents can actually discover and use the -right tools. +MCPLens is an open-source CLI for auditing whether agents can discover, choose, and +complete MCP tool workflows. -The main command, `audit-mcp`, runs locally against an MCP `tools/list` export and -optional usage logs. It writes a report that helps prevent tool-surface drift: overlap -between tools, unclear primary vs follow-up flows, bloated descriptions, weak profile -boundaries, and missing instrumentation. +It focuses on tool-surface drift: as MCP servers grow, tools start to overlap, +descriptions get wordier, follow-up flows become unclear, and safety or traceability +signals go missing. MCPLens turns a local `tools/list` export into a practical review +report for maintainers. -MCPLens can also generate a lean TypeScript MCP server from an OpenAPI-backed REST API. -That path is useful when you want to turn a large API surface into a smaller, -agent-oriented MCP server with slimmer responses. +The audit runs locally and offline. It does not send tool definitions, logs, prompts, or +reports to a hosted service. ## Quick Start -Run an audit with `npx`: - ```sh -npx mcplens-cli audit-mcp --tools-list tools.json --logs events.jsonl --out report.md +npx mcplens-cli audit-mcp --tools-list tools.json --out report.md ``` Requires Node.js 20 or newer. -Common audit inputs: +Common options: -- `--tools-list `: an MCP `tools/list` response or a bare tool array. Required. -- `--logs `: JSONL file of MCP/session events. Optional. +- `--tools-list `: MCP `tools/list` response or a bare tool array. Required. +- `--logs `: JSONL MCP/session events. Optional. - `--missed-prompts `: JSON or JSONL prompts where a tool should have fired. Optional. -- `--config `: `mcplens.config.json` policy file for profiles, severities, and thresholds. Optional. +- `--config `: `mcplens.config.json` policy file for profiles and thresholds. Optional. - `--baseline `: previous audit JSON for regression comparison. Optional. - `--out `: Markdown report output. Without it, the report prints to stdout. - `--json `: machine-readable audit report. -- `--capabilities `: machine-readable capability recommendations. -- `--ci`: print a concise CI summary and return nonzero when configured failure rules trigger. -- `--warn-only`: advisory CI mode; print warnings and return `0` even when findings exist. +- `--capabilities `: machine-readable capability/profile recommendations. +- `--ci --warn-only`: advisory CI summary that exits `0`. + +## Export A `tools/list` File + +MCPLens accepts either a full MCP JSON-RPC `tools/list` response: + +```json +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "tools": [ + { + "name": "search_docs", + "description": "Use when: the user needs to find internal docs by keyword, title, owner, or path. Returns matching documents with short snippets and canonical URLs.", + "inputSchema": { + "type": "object", + "properties": { + "query": { "type": "string" } + }, + "required": ["query"] + } + } + ] + } +} +``` -For browser-control MCPs, set `"profile": "browser"` in `mcplens.config.json`. The -browser profile checks that browser/session/page action tools state the state they -mutate, the preconditions required before calling, and the trace output available -afterward. For example, `navigate` should name URL/history mutation and final URL or -screenshot trace; `act` should name DOM/page/application mutation plus action trace; and -`extract` should explicitly say it does not mutate page state and returns extracted data. +Or a bare array: -## Privacy +```json +[ + { + "name": "search_docs", + "description": "Use when: the user needs to find internal docs by keyword, title, owner, or path.", + "inputSchema": { "type": "object", "properties": {} } + } +] +``` -The audit path is local and offline by design: +Most maintainers export this from their MCP server test harness, MCP client logs, or an +inspector session. Keep the export sanitized; tool definitions often reveal internal +system names, URLs, or workflows. -- No network requests. -- No LLM calls. -- No telemetry. -- Only the files you pass in are read. -- Only the output paths you name are written. +## Example Reports -Your tool definitions, logs, and prompts stay on your machine. +Checked-in synthetic examples show the kind of review MCPLens produces: -## What It Does Today +- [`examples/generic-mcp/report.md`](examples/generic-mcp/report.md): short descriptions, + unsafe destructive tools, contextual confirmation helpers, and catch-all API tools. +- [`examples/browser-mcp/report.md`](examples/browser-mcp/report.md): browser-profile checks + for mutation contracts, preconditions, and trace artifacts. +- [`examples/large-mcp/report.md`](examples/large-mcp/report.md): a larger surface with + overlapping search/list/get tools, follow-up helpers, and admin/destructive tools. -`audit-mcp` writes reports and recommendation artifacts. It does not modify an existing -MCP server in place. +Each example includes its `tools-list.json` input so you can rerun the audit. -The audit report can identify: +## What The Audit Catches -- Tools that are hard for agents to discover. -- Tool descriptions that are vague or overlapping. -- Confirm/reject fanout and related workflow friction. -- Which tools should be primary, contextual follow-up helpers, or admin capabilities. -- Shorter, more decisive naming and description rewrites. -- An implementation-plan section that can be handed to Cursor, Claude, or another coding agent. -- Proof metrics for case studies: more tool usage, better correct-tool selection, fewer failed attempts, token/time savings, and less guesswork when changing the tool surface. +MCPLens looks for issues that make agents less likely to pick the right tool or complete a +workflow: -To emit structured artifacts alongside the Markdown report: +- Missing, vague, overly short, or overly long descriptions. +- Overlapping tools that compete for the same prompt. +- Catch-all tools that hide unclear side effects behind generic inputs. +- Write and destructive tools without safety, confirmation, or review wording. +- Confirm/reject helpers that should be contextual instead of always visible. +- New or regressed findings compared with a baseline report. +- Browser tools that do not say what page/session state they mutate, what must be true + before calling, or what trace artifact is available afterward. + +The score is secondary. Treat the report as a concrete review checklist, not as proof that +tool usage will improve. + +## CI + +Use warn-only CI by default so drift becomes visible in pull requests without blocking an +urgent deploy: ```sh npx mcplens-cli audit-mcp \ --tools-list tools.json \ - --logs events.jsonl \ - --missed-prompts missed-prompts.json \ - --out activation-report.md \ - --json activation-report.json \ - --capabilities mcp-capabilities.json + --out mcplens-report.md \ + --json mcplens-report.json \ + --capabilities mcplens-capabilities.json \ + --ci \ + --warn-only ``` -For CI, prefer advisory mode so tool-surface drift gets a PR comment without blocking -urgent deploys by default: +Strict CI is available when a team explicitly wants findings to fail the job: ```sh -npx mcplens-cli audit-mcp --tools-list tools.json --out report.md --json report.json --ci --warn-only +npx mcplens-cli audit-mcp --tools-list tools.json --out report.md --json report.json --ci ``` -Strict CI is still available for teams that explicitly want findings to fail the job: -use `--ci` without `--warn-only`. For `mcplens.config.json`, severities, baseline -regression checks, and a GitHub Actions example, see [docs/audit-mcp-ci.md](docs/audit-mcp-ci.md). +For policy config, baselines, browser profile details, and a GitHub Actions example, see +[`docs/audit-mcp-ci.md`](docs/audit-mcp-ci.md). + +## Privacy + +The audit path is local and offline by design: + +- No network requests. +- No LLM calls. +- No telemetry. +- Only the files you pass in are read. +- Only the output paths you name are written. + +Your tool definitions, logs, prompts, and reports stay on your machine. + +## Limitations + +- Without logs or missed-prompt files, MCPLens is a static audit of the exposed tool + surface. +- It cannot prove agents will use tools more often or complete workflows more reliably. +- Scores and findings are heuristics, not ground truth. +- Some findings will be false positives, especially for domain-specific tools with unusual + naming or safety models. +- The best validation is before/after task completion, tool-call traces, first-tool-call + latency, failed-call rates, and user workflow outcomes. ## Install @@ -110,10 +168,29 @@ mcplens audit-mcp --tools-list tools.json --out report.md The package also installs an `agentify` binary as a compatibility alias for older scripts. +## Browser MCP Profile + +For browser-control MCPs, set `"profile": "browser"` in `mcplens.config.json`: + +```json +{ + "profile": "browser" +} +``` + +Browser action tool descriptions should include: + +- `Mutates:` active session, URL/history, DOM/application state, form values, cookies/auth, + focus/scroll/viewport, or explicitly no page-state mutation. +- `Preconditions:` active session, loaded page, prior observe call, known target + element/selector, authenticated state, user gesture, or page readiness. +- `Available afterward:` session id, final URL, replay URL, screenshot, action result, + extracted payload, console logs, network logs, or another trace artifact. + ## OpenAPI To MCP -MCPLens includes an advanced flow for turning an OpenAPI spec into a standalone MCP -server: +MCPLens also includes an advanced flow for turning an OpenAPI spec into a standalone +TypeScript MCP server: ```sh npx mcplens-cli compile \ @@ -126,50 +203,24 @@ npx mcplens-cli compile \ npx mcplens-cli build --manifest agentify.manifest.json --out ./stripe-mcp ``` -`compile` writes an editable manifest describing the tools, upstream requests, response -maps, and hidden endpoints. With `ANTHROPIC_API_KEY` set, it can use an LLM once at build -time for better curation. With `--offline`, it uses deterministic heuristics. +`compile` writes an editable manifest describing tools, upstream requests, response maps, +and hidden endpoints. With `ANTHROPIC_API_KEY` set, it can use an LLM once at build time +for better curation. With `--offline`, it uses deterministic heuristics. `build` generates a standalone TypeScript MCP server, installs dependencies, type-checks -the project, and smoke-tests MCP `tools/list` over stdio. The generated server makes no -LLM calls at runtime. - -Generated projects include: - -- `README.md` with setup, stdio, HTTP, Docker Compose, and client connection notes. -- `ACTIVATE.md` with activation steps for the generated output directory. -- `mcp-client.config.json` with a ready-to-copy `mcpServers` stdio config. -- `mcp-activation.json` with machine-readable activation metadata. -- `.env.example` for transport, HTTP token, upstream base URL, and credentials. -- `Dockerfile` and `docker-compose.yml` for self-hosting. - -See [GUIDE.md](GUIDE.md) for the full OpenAPI-to-MCP walkthrough and [IMPACT.md](IMPACT.md) -for measured token and tool-surface reductions on representative API fixtures. +the project, and smoke-tests MCP `tools/list` over stdio. The generated server makes no LLM +calls at runtime. -## Examples - -The [`examples/`](examples/) directory contains small, reviewable fixtures modeled on -public API documentation for GitHub, Stripe, Slack, Google Calendar, and Notion. - -Regenerate the example reports: - -```sh -for api in github stripe slack google-calendar notion; do - npx mcplens-cli compile \ - --spec "examples/$api/openapi.json" \ - --samples "examples/$api/samples" \ - --impact-report "examples/$api/impact-report.json" \ - --out "examples/$api/agentify.manifest.json" \ - --offline -done -``` +See [`GUIDE.md`](GUIDE.md) for the full OpenAPI-to-MCP walkthrough and +[`IMPACT.md`](IMPACT.md) for token and tool-surface reductions on representative API +fixtures. ## Development From a repo checkout: ```sh -npm install +npm ci npm run build npm test npm run smoke:pack diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..6194b68 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,28 @@ +# Security Policy + +## Supported Versions + +Security fixes are accepted for the latest published version of MCPLens. + +## Reporting a Vulnerability + +Please report security issues privately through GitHub Security Advisories for this +repository. If advisories are unavailable, open a minimal public issue that says you need a +private security contact, but do not include exploit details or sensitive data. + +Useful details include: + +- Affected version or commit. +- Reproduction steps. +- Impact and affected command path. +- Whether any secrets, credentials, local files, or generated MCP servers are exposed. + +## Security Expectations + +- `audit-mcp` must remain local and offline unless a future feature is explicitly + documented as opt-in network behavior. +- Do not include real API tokens, production payloads, private tool definitions, or + customer data in examples, tests, screenshots, or issue reports. +- Generated MCP servers should keep credentials in environment variables and `.env` + files that are excluded from git. + diff --git a/docs/audit-mcp-ci.md b/docs/audit-mcp-ci.md index e401a30..4157ac0 100644 --- a/docs/audit-mcp-ci.md +++ b/docs/audit-mcp-ci.md @@ -168,6 +168,7 @@ jobs: --baseline mcplens-baseline.json \ --out mcplens-report.md \ --json mcplens-report.json \ + --capabilities mcplens-capabilities.json \ --ci \ --warn-only @@ -179,4 +180,5 @@ jobs: path: | mcplens-report.md mcplens-report.json + mcplens-capabilities.json ``` diff --git a/examples/README.md b/examples/README.md index 3fea127..e2c1617 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,7 +1,26 @@ -# MCPLens examples +# MCPLens Examples -Real-world-shaped API fixtures used to measure MCPLens impact. See the aggregated -results in [`../IMPACT.md`](../IMPACT.md). +This directory contains two kinds of examples: + +- Synthetic MCP `tools/list` exports and generated audit reports for `audit-mcp`. +- Real-world-shaped OpenAPI fixtures used by the OpenAPI-to-MCP generator and impact + report. + +## Audit Report Examples + +These examples are synthetic and safe to inspect. They are meant to show the report shape +without requiring access to a private MCP server. + +| Example | Shows | Rerun | +|---------|-------|-------| +| [`generic-mcp/`](generic-mcp/) | Short descriptions, destructive tools, contextual confirmation helpers, catch-all tools | `npx mcplens-cli audit-mcp --tools-list examples/generic-mcp/tools-list.json --out examples/generic-mcp/report.md --offline` | +| [`browser-mcp/`](browser-mcp/) | Browser profile checks for mutation, preconditions, and trace artifacts | `npx mcplens-cli audit-mcp --tools-list examples/browser-mcp/tools-list.json --config examples/browser-mcp/mcplens.config.json --out examples/browser-mcp/report.md --offline` | +| [`large-mcp/`](large-mcp/) | Larger tool surface with overlap, default/admin/contextual exposure pressure | `npx mcplens-cli audit-mcp --tools-list examples/large-mcp/tools-list.json --out examples/large-mcp/report.md --offline` | + +## OpenAPI Fixtures + +Real-world-shaped API fixtures used to measure MCPLens impact. See the aggregated results +in [`../IMPACT.md`](../IMPACT.md). Each directory contains: @@ -43,5 +62,5 @@ for api in github stripe slack google-calendar notion; do done ``` -Drop `--offline` with `ANTHROPIC_API_KEY` set to use LLM curation (better field selection -and endpoint hiding). +Drop `--offline` with `ANTHROPIC_API_KEY` set to use LLM curation for the OpenAPI fixtures +(better field selection and endpoint hiding). diff --git a/examples/browser-mcp/mcplens.config.json b/examples/browser-mcp/mcplens.config.json new file mode 100644 index 0000000..9c09345 --- /dev/null +++ b/examples/browser-mcp/mcplens.config.json @@ -0,0 +1,3 @@ +{ + "profile": "browser" +} diff --git a/examples/browser-mcp/report.md b/examples/browser-mcp/report.md new file mode 100644 index 0000000..7e90b1b --- /dev/null +++ b/examples/browser-mcp/report.md @@ -0,0 +1,345 @@ +# MCP Activation Audit + +Purpose: prevent tool-surface drift. As an MCP server grows, overlapping tools, wordier descriptions, and unclear follow-up flows make agents less likely to call the right tool. + +## Prevent Tool-Surface Drift + +Start by deciding the activation model, then rewrite the concrete tool descriptions below. The score is secondary metadata; the main goal is to make each tool easy for an agent to choose, skip, or handle safely. + +- Surface shape: 5 exposed tools collapse into 5 workflow groups. +- Primary surface: 5 default-visible tools; 0 follow-up/helper tools; 0 admin/destructive tools. +- Overlap diagnosis: 12 overlapping tool findings need merge, rename, or sharper boundaries. +- Flow diagnosis: 0 confirm/reject helper tools should be exposed after a pending action exists, not as primary choices. +- Follow-up distinction: 0 low-risk helper tools reduce default-surface clutter; 0 contribution/submission gate tools may reduce workflow completion. +- Observed activation: not measured yet; add initialize and tools/call logs to prove whether the drift fix increases usage. +- Review priority: Rewrite browser_navigate with concrete "Use when" trigger language and a clearer safe path. +- Tools needing description review: 5 of 5 +- Findings to triage: 6 fail, 17 warn, 0 info +- Preferred rewrite shape: short, decisive `Use when` / `Returns` / `Do not use when` / `Safety`. + +## Implementation Plan For Cursor/Claude + +Feed this section to a coding agent as the concrete fix plan: + +- [ ] Export the current MCP tools/list and keep this report plus the JSON audit as the baseline for future PR checks. +- [ ] Define the activation model: default-visible primary tools are `browser_click`, `browser_extract`, `browser_navigate`, `browser_screenshot`, `browser_type`; low-risk contextual follow-up tools are none; contribution/submission gates are none; admin/destructive tools are none. +- [ ] Update server registration so contextual follow-up tools are only advertised after a pending action exists, or move them behind a separate admin/profile configuration if the client cannot do contextual exposure. +- [ ] For contribution/submission workflows, add draft/confirmation/posting completion metrics before introducing new safety gates. +- [ ] For every item in Actionable Tool Findings, apply the suggested name/description or an equivalent shorter rewrite with a decisive trigger, return shape, exclusion rule, and safety note. +- [ ] For browser action tools, add explicit `Mutates`, `Preconditions`, and `Available afterward` lines so agents know what browser/session/page state changes, what must already be true, and what trace output they can inspect after the call. +- [ ] Resolve each Merge/Hide/Split recommendation by merging overlapping capabilities, renaming tools that compete for the same prompt, or making the boundary explicit in `Do not use when` wording. +- [ ] Add or update standard MCP ToolAnnotations (`readOnlyHint`, `destructiveHint`, `idempotentHint`, `openWorldHint`) so safety is machine-readable instead of buried in prose. +- [ ] Wire warn-only CI on pull requests with a baseline audit so new tools, overlap, score regressions, and missing descriptions get an advisory PR comment before drift ships. +- [ ] Instrument proof metrics: initialized sessions, tools/list payload bytes, tools/call success/error, missed-prompt replay results, first-tool-call latency, and task completion turns. + +## Browser MCP Profile + +For browser action tools, state the operational contract before an agent calls the tool: + +- `Mutates:` the browser state changed by the action, such as active session, page URL/history, DOM/application state, form values, cookies/auth, focus/scroll/viewport, or no page-state mutation. +- `Preconditions:` required active session, loaded page, prior observe call, known target element/selector, authenticated state, user gesture, or page-readiness state. +- `Available afterward:` trace output such as session id, final URL, replay URL, screenshot, action result, observation result, extracted structured payload, console logs, or network logs. + +Examples: + +- `navigate`: mutates current page URL/history; precondition: active session; trace: final URL, screenshot, session ID. +- `act`: mutates DOM/page/application state; precondition: page loaded and target/action known, ideally after observe; trace: action result, screenshot, replay. +- `extract`: does not mutate page state; precondition: page loaded; trace: extracted structured payload. + +## Actionable Tool Findings + +### browser_navigate +- WARN description_too_short: Description is too short for reliable model selection. +- WARN weak_required_input: Required inputs need clearer descriptions or examples: url. +- FAIL browser_action_missing_mutation: Browser action tool does not state which browser state it mutates. +- FAIL browser_action_missing_preconditions: Browser action tool does not state preconditions that must be true before calling. +- FAIL browser_action_missing_artifact: Browser action tool does not state the trace or debug artifact available afterward. +- WARN tool_overlap: Overlaps heavily with browser_type. +- WARN tool_overlap: Overlaps heavily with browser_extract. +- WARN tool_overlap: Overlaps heavily with browser_screenshot. +- Current description: + +```text +Navigate to a URL. +``` + +- Suggested rewrite: + +```text +Use when: the agent must perform the browser navigate browser action in an existing browser session. +Mutates: state whether this changes the active session, page URL/history, DOM/application state, form values, cookies/auth, focus/scroll/viewport, or explicitly does not mutate page state. +Preconditions: state the required active session, loaded page, prior observe call, known target element/selector, authenticated state, user gesture, or page-readiness state before calling. +Available afterward: state the trace output, such as session id, final URL, replay URL, screenshot, action result, observation result, extracted structured payload, console logs, or network logs. +``` + +### browser_click +- WARN description_too_short: Description is too short for reliable model selection. +- WARN weak_required_input: Required inputs need clearer descriptions or examples: selector. +- FAIL browser_action_missing_mutation: Browser action tool does not state which browser state it mutates. +- FAIL browser_action_missing_preconditions: Browser action tool does not state preconditions that must be true before calling. +- FAIL browser_action_missing_artifact: Browser action tool does not state the trace or debug artifact available afterward. +- Current description: + +```text +Click an element. +``` + +- Suggested rewrite: + +```text +Use when: the agent must perform the browser click browser action in an existing browser session. +Mutates: state whether this changes the active session, page URL/history, DOM/application state, form values, cookies/auth, focus/scroll/viewport, or explicitly does not mutate page state. +Preconditions: state the required active session, loaded page, prior observe call, known target element/selector, authenticated state, user gesture, or page-readiness state before calling. +Available afterward: state the trace output, such as session id, final URL, replay URL, screenshot, action result, observation result, extracted structured payload, console logs, or network logs. +``` + +### browser_type +- WARN weak_required_input: Required inputs need clearer descriptions or examples: selector, text. +- WARN tool_overlap: Overlaps heavily with browser_navigate. +- WARN tool_overlap: Overlaps heavily with browser_extract. +- WARN tool_overlap: Overlaps heavily with browser_screenshot. +- Current description: + +```text +Use when: the user needs to enter text into a focused or selected browser field. Mutates: DOM form values and focus state. Preconditions: active browser session, loaded page, and a known selector or focused input. Available afterward: session id, final URL, action result, and screenshot. +``` + +- Suggested rewrite: + +```text +Use when: the user needs to enter text into a focused or selected browser field. Mutates: DOM form values and focus state. Preconditions: active browser session, loaded page, and a known selector or focused input. Available afterward: session id, final URL, action result, and screenshot. +``` + +### browser_extract +- WARN tool_overlap: Overlaps heavily with browser_navigate. +- WARN tool_overlap: Overlaps heavily with browser_type. +- WARN tool_overlap: Overlaps heavily with browser_screenshot. +- Current description: + +```text +Use when: the user needs structured data from the currently loaded page. Mutates: no page state. Preconditions: active browser session and loaded page. Available afterward: extracted structured payload and source URL. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the browser extract workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### browser_screenshot +- WARN tool_overlap: Overlaps heavily with browser_navigate. +- WARN tool_overlap: Overlaps heavily with browser_type. +- WARN tool_overlap: Overlaps heavily with browser_extract. +- Current description: + +```text +Use when: the user needs a visual trace of the current page. Mutates: no page state. Preconditions: active browser session and loaded page. Available afterward: screenshot artifact, session id, and current URL. +``` + +- Suggested rewrite: + +```text +Use when: the user needs a visual trace of the current page. Mutates: no page state. Preconditions: active browser session and loaded page. Available afterward: screenshot artifact, session id, and current URL. +``` + +## Recommended Description Format + +Use this structure for tool descriptions, especially for write, destructive, public-posting, or workflow-helper tools: + +```text +Use when: the concrete user situation or agent state that should trigger this tool. +Returns: the result shape or decision the agent can expect. +Do not use when: nearby tasks where another tool or no tool is a better choice. +Safety: side effects, confirmation requirements, redaction rules, auth scope, or why the tool is read-only. +``` + +For browser action tools, use this browser-specific structure: + +```text +Use when: the concrete browser interaction the agent should perform. +Mutates: session, URL/history, DOM/application state, form values, cookies/auth, or explicitly no page-state mutation. +Preconditions: active session, loaded page, prior observe call, known target element, authenticated state, or other readiness requirements. +Available afterward: session id, final URL, replay URL, screenshot, action result, observation result, extracted data, console logs, or network logs. +``` + +## Rewritten Tool Descriptions + +Keep these rewrites short and activation-oriented. Avoid implementation detail unless it helps the agent choose the correct tool. + +### browser_click -> browser_click +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 0.7 + +Use when: the agent must perform the browser click browser action in an existing browser session. +Mutates: state whether this changes the active session, page URL/history, DOM/application state, form values, cookies/auth, focus/scroll/viewport, or explicitly does not mutate page state. +Preconditions: state the required active session, loaded page, prior observe call, known target element/selector, authenticated state, user gesture, or page-readiness state before calling. +Available afterward: state the trace output, such as session id, final URL, replay URL, screenshot, action result, observation result, extracted structured payload, console logs, or network logs. + +### browser_extract -> browser_extract +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 0.7 + +Use when: the user needs the browser extract workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +### browser_navigate -> browser_navigate +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 0.7 + +Use when: the agent must perform the browser navigate browser action in an existing browser session. +Mutates: state whether this changes the active session, page URL/history, DOM/application state, form values, cookies/auth, focus/scroll/viewport, or explicitly does not mutate page state. +Preconditions: state the required active session, loaded page, prior observe call, known target element/selector, authenticated state, user gesture, or page-readiness state before calling. +Available afterward: state the trace output, such as session id, final URL, replay URL, screenshot, action result, observation result, extracted structured payload, console logs, or network logs. + +### browser_screenshot -> browser_screenshot +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 0.7 + +Use when: the user needs a visual trace of the current page. Mutates: no page state. Preconditions: active browser session and loaded page. Available afterward: screenshot artifact, session id, and current URL. + +### browser_type -> browser_type +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 0.7 + +Use when: the user needs to enter text into a focused or selected browser field. Mutates: DOM form values and focus state. Preconditions: active browser session, loaded page, and a known selector or focused input. Available afterward: session id, final URL, action result, and screenshot. + +## Merge/Hide/Split Recommendations + +- No high-confidence hide or merge recommendations. + +## Recommended Tool Set + +Use this activation model to keep primary tools from competing with after-action helpers: + +### core +Default surface for ordinary agent sessions. Read/feedback/contribution tools stay visible; confirm/reject helpers belong here too but should be exposed contextually, not in the default tools/list. + +- `browser_click` +- `browser_extract` +- `browser_navigate` +- `browser_screenshot` +- `browser_type` + +### admin +Maintenance and destructive capabilities should not compete with high-value default workflows. + + +## Secondary Summary And CI Metadata + +- Tools exposed: 5 +- Average discoverability score: 56 +- Default-visible tools (shown in every session): 5 +- Core profile: 5 (5 default-visible + 0 contextual helpers) +- Low-risk follow-up helpers: 0 +- Contribution/submission gates to measure: 0 +- Admin profile (kept out of the default surface): 0 +- Workflow groups: 5 +- tools/list payload: 2 KB +- Confirm/reject helpers: 0 (0%) +- Top recommendation: Rewrite browser_navigate with concrete "Use when" trigger language and a clearer safe path. +- CI status: FAIL +- CI findings: 6 fail, 17 warn, 0 info +- Recommended CI posture: advisory PR comment or warn-only check by default; strict failure only for teams that explicitly want blocking policy. + +### Strict CI Failures + +| Rule | Tool | Message | +| --- | --- | --- | +| browser_action_missing_mutation | `browser_click` | Browser action tool does not state which browser state it mutates. | +| browser_action_missing_preconditions | `browser_click` | Browser action tool does not state preconditions that must be true before calling. | +| browser_action_missing_artifact | `browser_click` | Browser action tool does not state the trace or debug artifact available afterward. | +| browser_action_missing_mutation | `browser_navigate` | Browser action tool does not state which browser state it mutates. | +| browser_action_missing_preconditions | `browser_navigate` | Browser action tool does not state preconditions that must be true before calling. | +| browser_action_missing_artifact | `browser_navigate` | Browser action tool does not state the trace or debug artifact available afterward. | + +## Proof To Collect Before Monetization + +- Tool usage: compare sessions with any tool call before and after the surface change. +- Correct-tool selection: replay missed prompts and track whether the expected tool becomes the top match. +- Failed attempts: compare `tools/call` errors and prompt retries after descriptions are shortened. +- Token/time savings: compare `tools/list` payload size, first-tool-call latency, and total turns to task completion. +- Change confidence: use PR baseline diffs to show which new or edited tools would have caused surface drift. + +## Missed-Prompt Coverage Analysis + +- No missed prompts were provided. + +## Activation And Contribution Funnel + +- Solved problem events: not measured +- Draft created events: not measured +- Confirmation shown events: not measured +- Public post events: not measured +- Contribution completion rate: not measured +- generic_problem_solved: not measured - Logs do not include solved-problem events, so the contribution funnel cannot be measured from the true eligibility point. +- draft_created: not measured - No draft-created events were observed. Add instrumentation or make the safe draft tool more visible. +- user_confirmation_shown: not measured - No user-confirmation events were observed. Track when a draft is shown for explicit public-post confirmation. +- public_post_created: not measured - No public-post events were observed. + +## Current Tool Surface + +| Tool | Workflow | Role | Score | Calls | Errors | Declared priorityHint | +| --- | --- | --- | ---: | ---: | ---: | ---: | +| `browser_click` | browser_click | write | 43 | 0 | 0 | | +| `browser_extract` | browser_extract | write | 76 | 0 | 0 | | +| `browser_navigate` | browser_navigate | write | 19 | 0 | 0 | | +| `browser_screenshot` | browser_screenshot | write | 76 | 0 | 0 | | +| `browser_type` | browser_type | write | 66 | 0 | 0 | | + +## Tool-Level Findings + +### browser_click +- WARN description_too_short: Description is too short for reliable model selection. +- WARN weak_required_input: Required inputs need clearer descriptions or examples: selector. +- FAIL browser_action_missing_mutation: Browser action tool does not state which browser state it mutates. +- FAIL browser_action_missing_preconditions: Browser action tool does not state preconditions that must be true before calling. +- FAIL browser_action_missing_artifact: Browser action tool does not state the trace or debug artifact available afterward. + +### browser_extract +- WARN tool_overlap: Overlaps heavily with browser_navigate. +- WARN tool_overlap: Overlaps heavily with browser_type. +- WARN tool_overlap: Overlaps heavily with browser_screenshot. + +### browser_navigate +- WARN description_too_short: Description is too short for reliable model selection. +- WARN weak_required_input: Required inputs need clearer descriptions or examples: url. +- FAIL browser_action_missing_mutation: Browser action tool does not state which browser state it mutates. +- FAIL browser_action_missing_preconditions: Browser action tool does not state preconditions that must be true before calling. +- FAIL browser_action_missing_artifact: Browser action tool does not state the trace or debug artifact available afterward. +- WARN tool_overlap: Overlaps heavily with browser_type. +- WARN tool_overlap: Overlaps heavily with browser_extract. +- WARN tool_overlap: Overlaps heavily with browser_screenshot. + +### browser_screenshot +- WARN tool_overlap: Overlaps heavily with browser_navigate. +- WARN tool_overlap: Overlaps heavily with browser_type. +- WARN tool_overlap: Overlaps heavily with browser_extract. + +### browser_type +- WARN weak_required_input: Required inputs need clearer descriptions or examples: selector, text. +- WARN tool_overlap: Overlaps heavily with browser_navigate. +- WARN tool_overlap: Overlaps heavily with browser_extract. +- WARN tool_overlap: Overlaps heavily with browser_screenshot. + +## A/B Test Plan + +- Expose a core profile with only search, feedback, low-friction usage tracking, and safe draft/start contribution tools. +- Rewrite contribution descriptions to separate draft creation from confirmed public publishing. +- Move admin/destructive and confirm/reject helper tools to an admin or contextual surface, then compare first-tool-call and draft-created rates. +- Add setup instructions telling agents to search existing shared solutions before answering from memory. +- Track initialized sessions -> useful tool-call sessions -> draft created -> confirmation shown -> public post created. + +## Exact Next Instrumentation To Add + +- Emit `initialize`, `tools/list`, and `tools/call` with stable `sessionId` values. +- Emit `solved_problem` when an agent has fixed a generic reusable problem. +- Emit `draft_created`, `user_confirmation_shown`, `public_post_created`, and `policy_block` for the contribution flow. +- Track tool-call errors with `tool_error` or `tools/call` events where `ok` is false. + +## Privacy/Safety Friction Review + +- Keep public posting behind explicit user confirmation and terms acceptance. +- Make the draft tool safe to call without confirmation because it does not publish. +- Require the draft step to remove secrets, personal data, company/customer names, private URLs, and incident-specific details. +- Set the standard MCP ToolAnnotations (readOnlyHint, destructiveHint, idempotentHint, openWorldHint) on each tool so clients can reason about safety. +- `advisoryPriority` is not a standard MCP annotation and most clients ignore it; the main fix is a smaller default surface with clearer trigger language. diff --git a/examples/browser-mcp/tools-list.json b/examples/browser-mcp/tools-list.json new file mode 100644 index 0000000..3d1a640 --- /dev/null +++ b/examples/browser-mcp/tools-list.json @@ -0,0 +1,70 @@ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "tools": [ + { + "name": "browser_navigate", + "description": "Navigate to a URL.", + "inputSchema": { + "type": "object", + "properties": { + "url": { + "type": "string" + } + }, + "required": ["url"] + } + }, + { + "name": "browser_click", + "description": "Click an element.", + "inputSchema": { + "type": "object", + "properties": { + "selector": { + "type": "string" + } + }, + "required": ["selector"] + } + }, + { + "name": "browser_type", + "description": "Use when: the user needs to enter text into a focused or selected browser field. Mutates: DOM form values and focus state. Preconditions: active browser session, loaded page, and a known selector or focused input. Available afterward: session id, final URL, action result, and screenshot.", + "inputSchema": { + "type": "object", + "properties": { + "selector": { + "type": "string" + }, + "text": { + "type": "string" + } + }, + "required": ["selector", "text"] + } + }, + { + "name": "browser_extract", + "description": "Use when: the user needs structured data from the currently loaded page. Mutates: no page state. Preconditions: active browser session and loaded page. Available afterward: extracted structured payload and source URL.", + "inputSchema": { + "type": "object", + "properties": { + "schema": { + "type": "object" + } + } + } + }, + { + "name": "browser_screenshot", + "description": "Use when: the user needs a visual trace of the current page. Mutates: no page state. Preconditions: active browser session and loaded page. Available afterward: screenshot artifact, session id, and current URL.", + "inputSchema": { + "type": "object", + "properties": {} + } + } + ] + } +} diff --git a/examples/generic-mcp/report.md b/examples/generic-mcp/report.md new file mode 100644 index 0000000..35e5724 --- /dev/null +++ b/examples/generic-mcp/report.md @@ -0,0 +1,407 @@ +# MCP Activation Audit + +Purpose: prevent tool-surface drift. As an MCP server grows, overlapping tools, wordier descriptions, and unclear follow-up flows make agents less likely to call the right tool. + +## Prevent Tool-Surface Drift + +Start by deciding the activation model, then rewrite the concrete tool descriptions below. The score is secondary metadata; the main goal is to make each tool easy for an agent to choose, skip, or handle safely. + +- Surface shape: 8 exposed tools collapse into 8 workflow groups. +- Primary surface: 6 default-visible tools; 1 follow-up/helper tools; 1 admin/destructive tools. +- Overlap diagnosis: 4 overlapping tool findings need merge, rename, or sharper boundaries. +- Flow diagnosis: 1 confirm/reject helper tool should be exposed after a pending action exists, not as primary choices. +- Follow-up distinction: 1 low-risk helper tool reduce default-surface clutter; 0 contribution/submission gate tools may reduce workflow completion. +- Observed activation: not measured yet; add initialize and tools/call logs to prove whether the drift fix increases usage. +- Review priority: Rewrite create_task with concrete "Use when" trigger language and a clearer safe path. +- Tools needing description review: 8 of 8 +- Findings to triage: 1 fail, 14 warn, 0 info +- Preferred rewrite shape: short, decisive `Use when` / `Returns` / `Do not use when` / `Safety`. + +## Implementation Plan For Cursor/Claude + +Feed this section to a coding agent as the concrete fix plan: + +- [ ] Export the current MCP tools/list and keep this report plus the JSON audit as the baseline for future PR checks. +- [ ] Define the activation model: default-visible primary tools are `create_task`, `get_doc`, `list_docs`, `run_any_request`, `search_docs`, `update_task`; low-risk contextual follow-up tools are `confirm_task_creation`; contribution/submission gates are none; admin/destructive tools are `delete_task`. +- [ ] Update server registration so contextual follow-up tools are only advertised after a pending action exists, or move them behind a separate admin/profile configuration if the client cannot do contextual exposure. +- [ ] For contribution/submission workflows, add draft/confirmation/posting completion metrics before introducing new safety gates. +- [ ] For every item in Actionable Tool Findings, apply the suggested name/description or an equivalent shorter rewrite with a decisive trigger, return shape, exclusion rule, and safety note. +- [ ] Resolve each Merge/Hide/Split recommendation by merging overlapping capabilities, renaming tools that compete for the same prompt, or making the boundary explicit in `Do not use when` wording. +- [ ] Add or update standard MCP ToolAnnotations (`readOnlyHint`, `destructiveHint`, `idempotentHint`, `openWorldHint`) so safety is machine-readable instead of buried in prose. +- [ ] Wire warn-only CI on pull requests with a baseline audit so new tools, overlap, score regressions, and missing descriptions get an advisory PR comment before drift ships. +- [ ] Instrument proof metrics: initialized sessions, tools/list payload bytes, tools/call success/error, missed-prompt replay results, first-tool-call latency, and task completion turns. + +## Actionable Tool Findings + +### delete_task +- Exposure: Admin, maintenance, or destructive capability should not be exposed to every ordinary agent session by default. Preferred action: admin_profile. +- WARN description_too_short: Description is too short for reliable model selection. +- WARN weak_required_input: Required inputs need clearer descriptions or examples: taskId. +- FAIL unsafe_destructive_tool: Destructive tool lacks explicit confirmation, safety, or review wording. +- WARN tool_overlap: Overlaps heavily with update_task. +- Recommendation: Move to an admin profile unless ordinary users need this in the default surface. +- Current description: + +```text +Delete a task. +``` + +- Suggested rewrite: + +```text +Use when: an administrator intentionally needs delete_task maintenance. +Returns: the maintenance result. +Do not use when: serving ordinary user workflows or when a read-only inspection tool would be enough. +Safety: admin/destructive capability; keep out of the default user-facing profile and require explicit operator intent. +``` + +### confirm_task_creation +- Exposure: Follow-up helper tools are useful after a pending action exists and reduce default-surface clutter without changing the primary completion path. Preferred action: contextual_exposure. +- WARN weak_required_input: Required inputs need clearer descriptions or examples: pendingId. +- Recommendation: Keep for safety, but consider contextual exposure only when there is a pending action. +- Current description: + +```text +Confirm a drafted task. +``` + +- Suggested rewrite: + +```text +Use when: a pending task_creation action has already been shown to the user and the user explicitly confirmed it. +Returns: the confirmed result. +Do not use when: starting a new workflow, guessing user intent, or bypassing a preview/draft step. +Safety: confirmation helper; expose contextually only after there is a pending action. +``` + +### create_task +- WARN description_too_short: Description is too short for reliable model selection. +- WARN weak_required_input: Required inputs need clearer descriptions or examples: title. +- WARN unsafe_write_tool: Write-like tool lacks safety, draft, redaction, or explicit-confirmation wording. +- WARN tool_overlap: Overlaps heavily with update_task. +- Recommendation: Prefer a verb plus concrete domain object in the name. +- Current description: + +```text +Create a task. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the task workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### run_any_request +- WARN weak_required_input: Required inputs need clearer descriptions or examples: method, path. +- WARN unsafe_write_tool: Write-like tool lacks safety, draft, redaction, or explicit-confirmation wording. +- Recommendation: Prefer a verb plus concrete domain object in the name. +- Current description: + +```text +Run any request against the workspace API. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the run any request workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### update_task +- WARN unsafe_write_tool: Write-like tool lacks safety, draft, redaction, or explicit-confirmation wording. +- WARN tool_overlap: Overlaps heavily with create_task. +- WARN tool_overlap: Overlaps heavily with delete_task. +- Current description: + +```text +Use when: the user asks to change an existing task's title, body, assignee, due date, or status. Returns the updated task. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the update task workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### list_docs +- WARN description_too_short: Description is too short for reliable model selection. +- Current description: + +```text +List docs. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the list docs workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### search_docs +- Recommendation: Prefer a verb plus concrete domain object in the name. +- Current description: + +```text +Use when: the user needs to find internal docs by keyword, title, owner, or path. Returns matching documents with short snippets and canonical URLs. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the search workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### get_doc +- Current description: + +```text +Use when: the user has a specific document id or URL and needs the full current document body plus metadata. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the doc workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +## Recommended Description Format + +Use this structure for tool descriptions, especially for write, destructive, public-posting, or workflow-helper tools: + +```text +Use when: the concrete user situation or agent state that should trigger this tool. +Returns: the result shape or decision the agent can expect. +Do not use when: nearby tasks where another tool or no tool is a better choice. +Safety: side effects, confirmation requirements, redaction rules, auth scope, or why the tool is read-only. +``` + +## Rewritten Tool Descriptions + +Keep these rewrites short and activation-oriented. Avoid implementation detail unless it helps the agent choose the correct tool. + +### confirm_task_creation -> confirm_task_creation +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 0.2 + +Use when: a pending task_creation action has already been shown to the user and the user explicitly confirmed it. +Returns: the confirmed result. +Do not use when: starting a new workflow, guessing user intent, or bypassing a preview/draft step. +Safety: confirmation helper; expose contextually only after there is a pending action. + +### create_task -> create_task +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 0.7 + +Use when: the user needs the task workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +### delete_task -> delete_task +Profile: admin; advisory priority (non-standard MCP hint, most clients ignore): 0.1 + +Use when: an administrator intentionally needs delete_task maintenance. +Returns: the maintenance result. +Do not use when: serving ordinary user workflows or when a read-only inspection tool would be enough. +Safety: admin/destructive capability; keep out of the default user-facing profile and require explicit operator intent. + +### get_doc -> get_doc +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 1 + +Use when: the user needs the doc workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +### list_docs -> list_docs +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 1 + +Use when: the user needs the list docs workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +### run_any_request -> run_any_request +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 0.7 + +Use when: the user needs the run any request workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +### search_docs -> search_docs +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 1 + +Use when: the user needs the search workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +### update_task -> update_task +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 0.7 + +Use when: the user needs the update task workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +## Merge/Hide/Split Recommendations + +- Hide or move `confirm_task_creation`: Follow-up helper tools are useful after a pending action exists and reduce default-surface clutter without changing the primary completion path. Preferred action: contextual_exposure. +- Hide or move `delete_task`: Admin, maintenance, or destructive capability should not be exposed to every ordinary agent session by default. Preferred action: admin_profile. + +## Recommended Tool Set + +Use this activation model to keep primary tools from competing with after-action helpers: + +### core +Default surface for ordinary agent sessions. Read/feedback/contribution tools stay visible; confirm/reject helpers belong here too but should be exposed contextually, not in the default tools/list. + +- `confirm_task_creation` - contextual follow-up (reduces default-surface clutter) +- `create_task` +- `get_doc` +- `list_docs` +- `run_any_request` +- `search_docs` +- `update_task` + +### admin +Maintenance and destructive capabilities should not compete with high-value default workflows. + +- `delete_task` + +## Secondary Summary And CI Metadata + +- Tools exposed: 8 +- Average discoverability score: 76.4 +- Default-visible tools (shown in every session): 6 +- Core profile: 7 (6 default-visible + 1 contextual helpers) +- Low-risk follow-up helpers: 1 +- Contribution/submission gates to measure: 0 +- Admin profile (kept out of the default surface): 1 +- Workflow groups: 8 +- tools/list payload: 3 KB +- Confirm/reject helpers: 1 (13%) +- Top recommendation: Rewrite create_task with concrete "Use when" trigger language and a clearer safe path. +- CI status: FAIL +- CI findings: 1 fail, 14 warn, 0 info +- Recommended CI posture: advisory PR comment or warn-only check by default; strict failure only for teams that explicitly want blocking policy. + +### Strict CI Failures + +| Rule | Tool | Message | +| --- | --- | --- | +| unsafe_destructive_tool | `delete_task` | Destructive tool lacks explicit confirmation, safety, or review wording. | + +## Proof To Collect Before Monetization + +- Tool usage: compare sessions with any tool call before and after the surface change. +- Correct-tool selection: replay missed prompts and track whether the expected tool becomes the top match. +- Failed attempts: compare `tools/call` errors and prompt retries after descriptions are shortened. +- Token/time savings: compare `tools/list` payload size, first-tool-call latency, and total turns to task completion. +- Change confidence: use PR baseline diffs to show which new or edited tools would have caused surface drift. + +## Missed-Prompt Coverage Analysis + +- No missed prompts were provided. + +## Activation And Contribution Funnel + +- Solved problem events: not measured +- Draft created events: not measured +- Confirmation shown events: not measured +- Public post events: not measured +- Contribution completion rate: not measured +- generic_problem_solved: not measured - Logs do not include solved-problem events, so the contribution funnel cannot be measured from the true eligibility point. +- draft_created: not measured - No draft-created events were observed. Add instrumentation or make the safe draft tool more visible. +- user_confirmation_shown: not measured - No user-confirmation events were observed. Track when a draft is shown for explicit public-post confirmation. +- public_post_created: not measured - No public-post events were observed. + +## Current Tool Surface + +| Tool | Workflow | Role | Score | Calls | Errors | Declared priorityHint | +| --- | --- | --- | ---: | ---: | ---: | ---: | +| `confirm_task_creation` | task_creation | confirm | 95 | 0 | 0 | | +| `create_task` | task | write | 49 | 0 | 0 | | +| `delete_task` | delete_task | destructive | 53 | 0 | 0 | | +| `get_doc` | doc | read | 100 | 0 | 0 | | +| `list_docs` | list_docs | read | 84 | 0 | 0 | | +| `run_any_request` | run_any_request | write | 68 | 0 | 0 | | +| `search_docs` | search | read | 92 | 0 | 0 | | +| `update_task` | update_task | write | 70 | 0 | 0 | | + +## Tool-Level Findings + +### confirm_task_creation +- WARN weak_required_input: Required inputs need clearer descriptions or examples: pendingId. +- Recommendation: Keep for safety, but consider contextual exposure only when there is a pending action. + +### create_task +- WARN description_too_short: Description is too short for reliable model selection. +- WARN weak_required_input: Required inputs need clearer descriptions or examples: title. +- WARN unsafe_write_tool: Write-like tool lacks safety, draft, redaction, or explicit-confirmation wording. +- WARN tool_overlap: Overlaps heavily with update_task. +- Recommendation: Prefer a verb plus concrete domain object in the name. + +### delete_task +- WARN description_too_short: Description is too short for reliable model selection. +- WARN weak_required_input: Required inputs need clearer descriptions or examples: taskId. +- FAIL unsafe_destructive_tool: Destructive tool lacks explicit confirmation, safety, or review wording. +- WARN tool_overlap: Overlaps heavily with update_task. +- Recommendation: Move to an admin profile unless ordinary users need this in the default surface. + +### list_docs +- WARN description_too_short: Description is too short for reliable model selection. + +### run_any_request +- WARN weak_required_input: Required inputs need clearer descriptions or examples: method, path. +- WARN unsafe_write_tool: Write-like tool lacks safety, draft, redaction, or explicit-confirmation wording. +- Recommendation: Prefer a verb plus concrete domain object in the name. + +### search_docs +- Recommendation: Prefer a verb plus concrete domain object in the name. + +### update_task +- WARN unsafe_write_tool: Write-like tool lacks safety, draft, redaction, or explicit-confirmation wording. +- WARN tool_overlap: Overlaps heavily with create_task. +- WARN tool_overlap: Overlaps heavily with delete_task. + +## A/B Test Plan + +- Expose a core profile with only search, feedback, low-friction usage tracking, and safe draft/start contribution tools. +- Rewrite contribution descriptions to separate draft creation from confirmed public publishing. +- Move admin/destructive and confirm/reject helper tools to an admin or contextual surface, then compare first-tool-call and draft-created rates. +- Add setup instructions telling agents to search existing shared solutions before answering from memory. +- Track initialized sessions -> useful tool-call sessions -> draft created -> confirmation shown -> public post created. + +## Exact Next Instrumentation To Add + +- Emit `initialize`, `tools/list`, and `tools/call` with stable `sessionId` values. +- Emit `solved_problem` when an agent has fixed a generic reusable problem. +- Emit `draft_created`, `user_confirmation_shown`, `public_post_created`, and `policy_block` for the contribution flow. +- Track tool-call errors with `tool_error` or `tools/call` events where `ok` is false. + +## Privacy/Safety Friction Review + +- Keep public posting behind explicit user confirmation and terms acceptance. +- Make the draft tool safe to call without confirmation because it does not publish. +- Require the draft step to remove secrets, personal data, company/customer names, private URLs, and incident-specific details. +- Set the standard MCP ToolAnnotations (readOnlyHint, destructiveHint, idempotentHint, openWorldHint) on each tool so clients can reason about safety. +- `advisoryPriority` is not a standard MCP annotation and most clients ignore it; the main fix is a smaller default surface with clearer trigger language. diff --git a/examples/generic-mcp/tools-list.json b/examples/generic-mcp/tools-list.json new file mode 100644 index 0000000..9a7e8b1 --- /dev/null +++ b/examples/generic-mcp/tools-list.json @@ -0,0 +1,127 @@ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "tools": [ + { + "name": "search_docs", + "description": "Use when: the user needs to find internal docs by keyword, title, owner, or path. Returns matching documents with short snippets and canonical URLs.", + "inputSchema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Keyword, title, owner, or path fragment to search for." + } + }, + "required": ["query"] + } + }, + { + "name": "list_docs", + "description": "List docs.", + "inputSchema": { + "type": "object", + "properties": { + "space": { + "type": "string" + } + } + } + }, + { + "name": "get_doc", + "description": "Use when: the user has a specific document id or URL and needs the full current document body plus metadata.", + "inputSchema": { + "type": "object", + "properties": { + "docId": { + "type": "string", + "description": "Document id or canonical document URL." + } + }, + "required": ["docId"] + } + }, + { + "name": "create_task", + "description": "Create a task.", + "inputSchema": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "body": { + "type": "string" + } + }, + "required": ["title"] + } + }, + { + "name": "update_task", + "description": "Use when: the user asks to change an existing task's title, body, assignee, due date, or status. Returns the updated task.", + "inputSchema": { + "type": "object", + "properties": { + "taskId": { + "type": "string", + "description": "Existing task id." + }, + "fields": { + "type": "object", + "description": "Fields to update." + } + }, + "required": ["taskId", "fields"] + } + }, + { + "name": "delete_task", + "description": "Delete a task.", + "inputSchema": { + "type": "object", + "properties": { + "taskId": { + "type": "string" + } + }, + "required": ["taskId"] + } + }, + { + "name": "confirm_task_creation", + "description": "Confirm a drafted task.", + "inputSchema": { + "type": "object", + "properties": { + "pendingId": { + "type": "string" + } + }, + "required": ["pendingId"] + } + }, + { + "name": "run_any_request", + "description": "Run any request against the workspace API.", + "inputSchema": { + "type": "object", + "properties": { + "method": { + "type": "string" + }, + "path": { + "type": "string" + }, + "body": { + "type": "object" + } + }, + "required": ["method", "path"] + } + } + ] + } +} diff --git a/examples/large-mcp/report.md b/examples/large-mcp/report.md new file mode 100644 index 0000000..82eb475 --- /dev/null +++ b/examples/large-mcp/report.md @@ -0,0 +1,704 @@ +# MCP Activation Audit + +Purpose: prevent tool-surface drift. As an MCP server grows, overlapping tools, wordier descriptions, and unclear follow-up flows make agents less likely to call the right tool. + +## Prevent Tool-Surface Drift + +Start by deciding the activation model, then rewrite the concrete tool descriptions below. The score is secondary metadata; the main goal is to make each tool easy for an agent to choose, skip, or handle safely. + +- Surface shape: 16 exposed tools collapse into 14 workflow groups. +- Primary surface: 12 default-visible tools; 2 follow-up/helper tools; 2 admin/destructive tools. +- Overlap diagnosis: 24 overlapping tool findings need merge, rename, or sharper boundaries. +- Flow diagnosis: 2 confirm/reject helper tools should be exposed after a pending action exists, not as primary choices. +- Follow-up distinction: 2 low-risk helper tools reduce default-surface clutter; 0 contribution/submission gate tools may reduce workflow completion. +- Observed activation: not measured yet; add initialize and tools/call logs to prove whether the drift fix increases usage. +- Review priority: Rewrite delete_project with concrete "Use when" trigger language and a clearer safe path. +- Tools needing description review: 16 of 16 +- Findings to triage: 1 fail, 44 warn, 1 info +- Preferred rewrite shape: short, decisive `Use when` / `Returns` / `Do not use when` / `Safety`. + +## Implementation Plan For Cursor/Claude + +Feed this section to a coding agent as the concrete fix plan: + +- [ ] Export the current MCP tools/list and keep this report plus the JSON audit as the baseline for future PR checks. +- [ ] Define the activation model: default-visible primary tools are `comment_on_task`, `complete_task`, `create_task`, `find_projects`, `get_project`, `get_user`, `list_project_tasks`, `list_users`, `raw_workspace_api`, `search_projects`, `search_tasks`, `update_task`; low-risk contextual follow-up tools are `confirm_task_update`, `reject_task_update`; contribution/submission gates are none; admin/destructive tools are `admin_export_workspace`, `delete_project`. +- [ ] Update server registration so contextual follow-up tools are only advertised after a pending action exists, or move them behind a separate admin/profile configuration if the client cannot do contextual exposure. +- [ ] For contribution/submission workflows, add draft/confirmation/posting completion metrics before introducing new safety gates. +- [ ] For every item in Actionable Tool Findings, apply the suggested name/description or an equivalent shorter rewrite with a decisive trigger, return shape, exclusion rule, and safety note. +- [ ] Resolve each Merge/Hide/Split recommendation by merging overlapping capabilities, renaming tools that compete for the same prompt, or making the boundary explicit in `Do not use when` wording. +- [ ] Add or update standard MCP ToolAnnotations (`readOnlyHint`, `destructiveHint`, `idempotentHint`, `openWorldHint`) so safety is machine-readable instead of buried in prose. +- [ ] Wire warn-only CI on pull requests with a baseline audit so new tools, overlap, score regressions, and missing descriptions get an advisory PR comment before drift ships. +- [ ] Instrument proof metrics: initialized sessions, tools/list payload bytes, tools/call success/error, missed-prompt replay results, first-tool-call latency, and task completion turns. + +## Actionable Tool Findings + +### delete_project +- Exposure: Admin, maintenance, or destructive capability should not be exposed to every ordinary agent session by default. Preferred action: admin_profile. +- WARN description_too_short: Description is too short for reliable model selection. +- WARN weak_required_input: Required inputs need clearer descriptions or examples: projectId. +- FAIL unsafe_destructive_tool: Destructive tool lacks explicit confirmation, safety, or review wording. +- WARN tool_overlap: Overlaps heavily with get_project. +- WARN tool_overlap: Overlaps heavily with list_project_tasks. +- WARN tool_overlap: Overlaps heavily with create_task. +- Recommendation: Move to an admin profile unless ordinary users need this in the default surface. +- Current description: + +```text +Delete a project. +``` + +- Suggested rewrite: + +```text +Use when: an administrator intentionally needs delete_project maintenance. +Returns: the maintenance result. +Do not use when: serving ordinary user workflows or when a read-only inspection tool would be enough. +Safety: admin/destructive capability; keep out of the default user-facing profile and require explicit operator intent. +``` + +### confirm_task_update +- Exposure: Follow-up helper tools are useful after a pending action exists and reduce default-surface clutter without changing the primary completion path. Preferred action: contextual_exposure. +- WARN weak_required_input: Required inputs need clearer descriptions or examples: pendingId. +- WARN tool_overlap: Overlaps heavily with reject_task_update. +- Recommendation: Keep for safety, but consider contextual exposure only when there is a pending action. +- Current description: + +```text +Confirm a pending task update. +``` + +- Suggested rewrite: + +```text +Use when: a pending task_update action has already been shown to the user and the user explicitly confirmed it. +Returns: the confirmed result. +Do not use when: starting a new workflow, guessing user intent, or bypassing a preview/draft step. +Safety: confirmation helper; expose contextually only after there is a pending action. +``` + +### reject_task_update +- Exposure: Follow-up helper tools are useful after a pending action exists and reduce default-surface clutter without changing the primary completion path. Preferred action: contextual_exposure. +- WARN weak_required_input: Required inputs need clearer descriptions or examples: pendingId. +- WARN tool_overlap: Overlaps heavily with confirm_task_update. +- Recommendation: Keep for safety, but consider contextual exposure only when there is a pending action. +- Current description: + +```text +Reject a pending task update. +``` + +- Suggested rewrite: + +```text +Use when: a pending task_update action has been shown and the user rejects it or asks to discard it. +Returns: the canceled pending action. +Do not use when: starting a new workflow or silently canceling an action without user intent. +Safety: rejection helper; expose contextually only after there is a pending action. +``` + +### create_task +- WARN weak_required_input: Required inputs need clearer descriptions or examples: projectId, title. +- WARN unsafe_write_tool: Write-like tool lacks safety, draft, redaction, or explicit-confirmation wording. +- WARN tool_overlap: Overlaps heavily with delete_project. +- Recommendation: Prefer a verb plus concrete domain object in the name. +- Current description: + +```text +Create a task in a project. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the task workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### search_tasks +- WARN weak_required_input: Required inputs need clearer descriptions or examples: query. +- WARN tool_overlap: Overlaps heavily with search_projects. +- WARN tool_overlap: Overlaps heavily with find_projects. +- WARN tool_overlap: Overlaps heavily with list_project_tasks. +- Recommendation: Prefer a verb plus concrete domain object in the name. +- Current description: + +```text +Use when: the user needs to find tasks across projects by keyword, assignee, label, or status. Returns task ids, project ids, titles, status, and assignees. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the search workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### search_projects +- WARN tool_overlap: Overlaps heavily with find_projects. +- WARN tool_overlap: Overlaps heavily with get_project. +- WARN tool_overlap: Overlaps heavily with search_tasks. +- Recommendation: Prefer a verb plus concrete domain object in the name. +- Current description: + +```text +Use when: the user needs to find projects by name, owner, status, or keyword. Returns project ids, names, owners, status, and short summaries. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the search workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### list_project_tasks +- WARN weak_required_input: Required inputs need clearer descriptions or examples: projectId. +- WARN tool_overlap: Overlaps heavily with get_project. +- WARN tool_overlap: Overlaps heavily with search_tasks. +- WARN tool_overlap: Overlaps heavily with list_users. +- WARN tool_overlap: Overlaps heavily with delete_project. +- Current description: + +```text +Use when: the user needs tasks in a known project, optionally filtered by status or assignee. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the list project task workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### comment_on_task +- WARN weak_required_input: Required inputs need clearer descriptions or examples: taskId, body. +- WARN unsafe_write_tool: Write-like tool lacks safety, draft, redaction, or explicit-confirmation wording. +- WARN tool_overlap: Overlaps heavily with update_task. +- Current description: + +```text +Use when: the user asks to add a comment or status note to an existing task. Returns the created comment id and task link. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the comment on task workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### find_projects +- WARN description_too_short: Description is too short for reliable model selection. +- WARN tool_overlap: Overlaps heavily with search_projects. +- WARN tool_overlap: Overlaps heavily with search_tasks. +- Current description: + +```text +Find projects. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the find project workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### raw_workspace_api +- INFO implementation_oriented_description: Description is implementation-oriented instead of user-intent-oriented. +- WARN weak_required_input: Required inputs need clearer descriptions or examples: method, path. +- WARN unsafe_write_tool: Write-like tool lacks safety, draft, redaction, or explicit-confirmation wording. +- Current description: + +```text +Call any workspace API endpoint. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the raw workspace api workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### update_task +- WARN weak_required_input: Required inputs need clearer descriptions or examples: taskId, fields. +- WARN unsafe_write_tool: Write-like tool lacks safety, draft, redaction, or explicit-confirmation wording. +- WARN tool_overlap: Overlaps heavily with comment_on_task. +- Current description: + +```text +Use when: the user asks to edit an existing task's title, body, assignee, labels, status, or due date. Returns the updated task. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the update task workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### get_project +- WARN weak_required_input: Required inputs need clearer descriptions or examples: projectId. +- WARN tool_overlap: Overlaps heavily with search_projects. +- WARN tool_overlap: Overlaps heavily with list_project_tasks. +- WARN tool_overlap: Overlaps heavily with delete_project. +- Current description: + +```text +Use when: the user has a project id and needs the current project summary, owner, status, milestones, and links. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the project workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### list_users +- WARN description_too_short: Description is too short for reliable model selection. +- WARN tool_overlap: Overlaps heavily with list_project_tasks. +- Current description: + +```text +List users. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the list user workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### complete_task +- WARN weak_required_input: Required inputs need clearer descriptions or examples: taskId. +- WARN unsafe_write_tool: Write-like tool lacks safety, draft, redaction, or explicit-confirmation wording. +- Current description: + +```text +Mark a task complete. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the complete task workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### get_user +- WARN weak_required_input: Required inputs need clearer descriptions or examples: userId. +- Current description: + +```text +Use when: the user needs profile details for a known user id or email. Returns display name, email, team, and active status. +``` + +- Suggested rewrite: + +```text +Use when: the user needs the user workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. +``` + +### admin_export_workspace +- Exposure: Admin, maintenance, or destructive capability should not be exposed to every ordinary agent session by default. Preferred action: admin_profile. +- Recommendation: Move to an admin profile unless ordinary users need this in the default surface. +- Current description: + +```text +Export the entire workspace. +``` + +- Suggested rewrite: + +```text +Use when: an administrator intentionally needs admin_export_workspace maintenance. +Returns: the maintenance result. +Do not use when: serving ordinary user workflows or when a read-only inspection tool would be enough. +Safety: admin/destructive capability; keep out of the default user-facing profile and require explicit operator intent. +``` + +## Recommended Description Format + +Use this structure for tool descriptions, especially for write, destructive, public-posting, or workflow-helper tools: + +```text +Use when: the concrete user situation or agent state that should trigger this tool. +Returns: the result shape or decision the agent can expect. +Do not use when: nearby tasks where another tool or no tool is a better choice. +Safety: side effects, confirmation requirements, redaction rules, auth scope, or why the tool is read-only. +``` + +## Rewritten Tool Descriptions + +Keep these rewrites short and activation-oriented. Avoid implementation detail unless it helps the agent choose the correct tool. + +### admin_export_workspace -> admin_export_workspace +Profile: admin; advisory priority (non-standard MCP hint, most clients ignore): 0.1 + +Use when: an administrator intentionally needs admin_export_workspace maintenance. +Returns: the maintenance result. +Do not use when: serving ordinary user workflows or when a read-only inspection tool would be enough. +Safety: admin/destructive capability; keep out of the default user-facing profile and require explicit operator intent. + +### comment_on_task -> comment_on_task +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 0.7 + +Use when: the user needs the comment on task workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +### complete_task -> complete_task +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 0.7 + +Use when: the user needs the complete task workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +### confirm_task_update -> confirm_task_update +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 0.2 + +Use when: a pending task_update action has already been shown to the user and the user explicitly confirmed it. +Returns: the confirmed result. +Do not use when: starting a new workflow, guessing user intent, or bypassing a preview/draft step. +Safety: confirmation helper; expose contextually only after there is a pending action. + +### create_task -> create_task +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 0.7 + +Use when: the user needs the task workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +### delete_project -> delete_project +Profile: admin; advisory priority (non-standard MCP hint, most clients ignore): 0.1 + +Use when: an administrator intentionally needs delete_project maintenance. +Returns: the maintenance result. +Do not use when: serving ordinary user workflows or when a read-only inspection tool would be enough. +Safety: admin/destructive capability; keep out of the default user-facing profile and require explicit operator intent. + +### find_projects -> find_projects +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 1 + +Use when: the user needs the find project workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +### get_project -> get_project +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 1 + +Use when: the user needs the project workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +### get_user -> get_user +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 1 + +Use when: the user needs the user workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +### list_project_tasks -> list_project_tasks +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 1 + +Use when: the user needs the list project task workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +### list_users -> list_users +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 1 + +Use when: the user needs the list user workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +### raw_workspace_api -> raw_workspace_api +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 0.7 + +Use when: the user needs the raw workspace api workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +### reject_task_update -> reject_task_update +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 0.2 + +Use when: a pending task_update action has been shown and the user rejects it or asks to discard it. +Returns: the canceled pending action. +Do not use when: starting a new workflow or silently canceling an action without user intent. +Safety: rejection helper; expose contextually only after there is a pending action. + +### search_projects -> search_projects +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 1 + +Use when: the user needs the search workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +### search_tasks -> search_tasks +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 1 + +Use when: the user needs the search workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +### update_task -> update_task +Profile: core; advisory priority (non-standard MCP hint, most clients ignore): 0.7 + +Use when: the user needs the update task workflow. +Returns: the relevant result. +Do not use when: the task belongs to another workflow or no tool call is needed. +Safety: document whether this tool is read-only, writes data, calls external systems, or requires confirmation. + +## Merge/Hide/Split Recommendations + +- Hide or move `admin_export_workspace`: Admin, maintenance, or destructive capability should not be exposed to every ordinary agent session by default. Preferred action: admin_profile. +- Hide or move `confirm_task_update`: Follow-up helper tools are useful after a pending action exists and reduce default-surface clutter without changing the primary completion path. Preferred action: contextual_exposure. +- Hide or move `delete_project`: Admin, maintenance, or destructive capability should not be exposed to every ordinary agent session by default. Preferred action: admin_profile. +- Hide or move `reject_task_update`: Follow-up helper tools are useful after a pending action exists and reduce default-surface clutter without changing the primary completion path. Preferred action: contextual_exposure. +- task_update is split into preview/confirm/reject helpers. Preserve safety, but expose helpers contextually or behind a lower-priority profile. Tools: `confirm_task_update`, `reject_task_update`. + +## Recommended Tool Set + +Use this activation model to keep primary tools from competing with after-action helpers: + +### core +Default surface for ordinary agent sessions. Read/feedback/contribution tools stay visible; confirm/reject helpers belong here too but should be exposed contextually, not in the default tools/list. + +- `comment_on_task` +- `complete_task` +- `confirm_task_update` - contextual follow-up (reduces default-surface clutter) +- `create_task` +- `find_projects` +- `get_project` +- `get_user` +- `list_project_tasks` +- `list_users` +- `raw_workspace_api` +- `reject_task_update` - contextual follow-up (reduces default-surface clutter) +- `search_projects` +- `search_tasks` +- `update_task` + +### admin +Maintenance and destructive capabilities should not compete with high-value default workflows. + +- `admin_export_workspace` +- `delete_project` + +## Secondary Summary And CI Metadata + +- Tools exposed: 16 +- Average discoverability score: 72.5 +- Default-visible tools (shown in every session): 12 +- Core profile: 14 (12 default-visible + 2 contextual helpers) +- Low-risk follow-up helpers: 2 +- Contribution/submission gates to measure: 0 +- Admin profile (kept out of the default surface): 2 +- Workflow groups: 14 +- tools/list payload: 6 KB +- Confirm/reject helpers: 2 (13%) +- Top recommendation: Rewrite delete_project with concrete "Use when" trigger language and a clearer safe path. +- CI status: FAIL +- CI findings: 1 fail, 44 warn, 1 info +- Recommended CI posture: advisory PR comment or warn-only check by default; strict failure only for teams that explicitly want blocking policy. + +### Strict CI Failures + +| Rule | Tool | Message | +| --- | --- | --- | +| unsafe_destructive_tool | `delete_project` | Destructive tool lacks explicit confirmation, safety, or review wording. | + +## Proof To Collect Before Monetization + +- Tool usage: compare sessions with any tool call before and after the surface change. +- Correct-tool selection: replay missed prompts and track whether the expected tool becomes the top match. +- Failed attempts: compare `tools/call` errors and prompt retries after descriptions are shortened. +- Token/time savings: compare `tools/list` payload size, first-tool-call latency, and total turns to task completion. +- Change confidence: use PR baseline diffs to show which new or edited tools would have caused surface drift. + +## Missed-Prompt Coverage Analysis + +- No missed prompts were provided. + +## Activation And Contribution Funnel + +- Solved problem events: not measured +- Draft created events: not measured +- Confirmation shown events: not measured +- Public post events: not measured +- Contribution completion rate: not measured +- generic_problem_solved: not measured - Logs do not include solved-problem events, so the contribution funnel cannot be measured from the true eligibility point. +- draft_created: not measured - No draft-created events were observed. Add instrumentation or make the safe draft tool more visible. +- user_confirmation_shown: not measured - No user-confirmation events were observed. Track when a draft is shown for explicit public-post confirmation. +- public_post_created: not measured - No public-post events were observed. + +## Current Tool Surface + +| Tool | Workflow | Role | Score | Calls | Errors | Declared priorityHint | +| --- | --- | --- | ---: | ---: | ---: | ---: | +| `admin_export_workspace` | admin_export_workspace | admin | 100 | 0 | 0 | | +| `comment_on_task` | comment_on_task | write | 68 | 0 | 0 | | +| `complete_task` | complete_task | write | 81 | 0 | 0 | | +| `confirm_task_update` | task_update | confirm | 87 | 0 | 0 | | +| `create_task` | task | write | 60 | 0 | 0 | | +| `delete_project` | delete_project | destructive | 37 | 0 | 0 | | +| `find_projects` | find_project | read | 68 | 0 | 0 | | +| `get_project` | project | read | 71 | 0 | 0 | | +| `get_user` | user | read | 95 | 0 | 0 | | +| `list_project_tasks` | list_project_task | read | 63 | 0 | 0 | | +| `list_users` | list_user | read | 76 | 0 | 0 | | +| `raw_workspace_api` | raw_workspace_api | write | 68 | 0 | 0 | | +| `reject_task_update` | task_update | reject | 87 | 0 | 0 | | +| `search_projects` | search | read | 68 | 0 | 0 | | +| `search_tasks` | search | read | 63 | 0 | 0 | | +| `update_task` | update_task | write | 68 | 0 | 0 | | + +## Tool-Level Findings + +### admin_export_workspace +- Recommendation: Move to an admin profile unless ordinary users need this in the default surface. + +### comment_on_task +- WARN weak_required_input: Required inputs need clearer descriptions or examples: taskId, body. +- WARN unsafe_write_tool: Write-like tool lacks safety, draft, redaction, or explicit-confirmation wording. +- WARN tool_overlap: Overlaps heavily with update_task. + +### complete_task +- WARN weak_required_input: Required inputs need clearer descriptions or examples: taskId. +- WARN unsafe_write_tool: Write-like tool lacks safety, draft, redaction, or explicit-confirmation wording. + +### confirm_task_update +- WARN weak_required_input: Required inputs need clearer descriptions or examples: pendingId. +- WARN tool_overlap: Overlaps heavily with reject_task_update. +- Recommendation: Keep for safety, but consider contextual exposure only when there is a pending action. + +### create_task +- WARN weak_required_input: Required inputs need clearer descriptions or examples: projectId, title. +- WARN unsafe_write_tool: Write-like tool lacks safety, draft, redaction, or explicit-confirmation wording. +- WARN tool_overlap: Overlaps heavily with delete_project. +- Recommendation: Prefer a verb plus concrete domain object in the name. + +### delete_project +- WARN description_too_short: Description is too short for reliable model selection. +- WARN weak_required_input: Required inputs need clearer descriptions or examples: projectId. +- FAIL unsafe_destructive_tool: Destructive tool lacks explicit confirmation, safety, or review wording. +- WARN tool_overlap: Overlaps heavily with get_project. +- WARN tool_overlap: Overlaps heavily with list_project_tasks. +- WARN tool_overlap: Overlaps heavily with create_task. +- Recommendation: Move to an admin profile unless ordinary users need this in the default surface. + +### find_projects +- WARN description_too_short: Description is too short for reliable model selection. +- WARN tool_overlap: Overlaps heavily with search_projects. +- WARN tool_overlap: Overlaps heavily with search_tasks. + +### get_project +- WARN weak_required_input: Required inputs need clearer descriptions or examples: projectId. +- WARN tool_overlap: Overlaps heavily with search_projects. +- WARN tool_overlap: Overlaps heavily with list_project_tasks. +- WARN tool_overlap: Overlaps heavily with delete_project. + +### get_user +- WARN weak_required_input: Required inputs need clearer descriptions or examples: userId. + +### list_project_tasks +- WARN weak_required_input: Required inputs need clearer descriptions or examples: projectId. +- WARN tool_overlap: Overlaps heavily with get_project. +- WARN tool_overlap: Overlaps heavily with search_tasks. +- WARN tool_overlap: Overlaps heavily with list_users. +- WARN tool_overlap: Overlaps heavily with delete_project. + +### list_users +- WARN description_too_short: Description is too short for reliable model selection. +- WARN tool_overlap: Overlaps heavily with list_project_tasks. + +### raw_workspace_api +- INFO implementation_oriented_description: Description is implementation-oriented instead of user-intent-oriented. +- WARN weak_required_input: Required inputs need clearer descriptions or examples: method, path. +- WARN unsafe_write_tool: Write-like tool lacks safety, draft, redaction, or explicit-confirmation wording. + +### reject_task_update +- WARN weak_required_input: Required inputs need clearer descriptions or examples: pendingId. +- WARN tool_overlap: Overlaps heavily with confirm_task_update. +- Recommendation: Keep for safety, but consider contextual exposure only when there is a pending action. + +### search_projects +- WARN tool_overlap: Overlaps heavily with find_projects. +- WARN tool_overlap: Overlaps heavily with get_project. +- WARN tool_overlap: Overlaps heavily with search_tasks. +- Recommendation: Prefer a verb plus concrete domain object in the name. + +### search_tasks +- WARN weak_required_input: Required inputs need clearer descriptions or examples: query. +- WARN tool_overlap: Overlaps heavily with search_projects. +- WARN tool_overlap: Overlaps heavily with find_projects. +- WARN tool_overlap: Overlaps heavily with list_project_tasks. +- Recommendation: Prefer a verb plus concrete domain object in the name. + +### update_task +- WARN weak_required_input: Required inputs need clearer descriptions or examples: taskId, fields. +- WARN unsafe_write_tool: Write-like tool lacks safety, draft, redaction, or explicit-confirmation wording. +- WARN tool_overlap: Overlaps heavily with comment_on_task. + +## A/B Test Plan + +- Expose a core profile with only search, feedback, low-friction usage tracking, and safe draft/start contribution tools. +- Rewrite contribution descriptions to separate draft creation from confirmed public publishing. +- Move admin/destructive and confirm/reject helper tools to an admin or contextual surface, then compare first-tool-call and draft-created rates. +- Add setup instructions telling agents to search existing shared solutions before answering from memory. +- Track initialized sessions -> useful tool-call sessions -> draft created -> confirmation shown -> public post created. + +## Exact Next Instrumentation To Add + +- Emit `initialize`, `tools/list`, and `tools/call` with stable `sessionId` values. +- Emit `solved_problem` when an agent has fixed a generic reusable problem. +- Emit `draft_created`, `user_confirmation_shown`, `public_post_created`, and `policy_block` for the contribution flow. +- Track tool-call errors with `tool_error` or `tools/call` events where `ok` is false. + +## Privacy/Safety Friction Review + +- Keep public posting behind explicit user confirmation and terms acceptance. +- Make the draft tool safe to call without confirmation because it does not publish. +- Require the draft step to remove secrets, personal data, company/customer names, private URLs, and incident-specific details. +- Set the standard MCP ToolAnnotations (readOnlyHint, destructiveHint, idempotentHint, openWorldHint) on each tool so clients can reason about safety. +- `advisoryPriority` is not a standard MCP annotation and most clients ignore it; the main fix is a smaller default surface with clearer trigger language. diff --git a/examples/large-mcp/tools-list.json b/examples/large-mcp/tools-list.json new file mode 100644 index 0000000..a667968 --- /dev/null +++ b/examples/large-mcp/tools-list.json @@ -0,0 +1,235 @@ +{ + "jsonrpc": "2.0", + "id": 1, + "result": { + "tools": [ + { + "name": "search_projects", + "description": "Use when: the user needs to find projects by name, owner, status, or keyword. Returns project ids, names, owners, status, and short summaries.", + "inputSchema": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search phrase or project attribute." + } + }, + "required": ["query"] + } + }, + { + "name": "find_projects", + "description": "Find projects.", + "inputSchema": { + "type": "object", + "properties": { + "q": { + "type": "string" + } + } + } + }, + { + "name": "get_project", + "description": "Use when: the user has a project id and needs the current project summary, owner, status, milestones, and links.", + "inputSchema": { + "type": "object", + "properties": { + "projectId": { + "type": "string", + "description": "Project id." + } + }, + "required": ["projectId"] + } + }, + { + "name": "list_project_tasks", + "description": "Use when: the user needs tasks in a known project, optionally filtered by status or assignee.", + "inputSchema": { + "type": "object", + "properties": { + "projectId": { + "type": "string" + }, + "status": { + "type": "string" + }, + "assignee": { + "type": "string" + } + }, + "required": ["projectId"] + } + }, + { + "name": "search_tasks", + "description": "Use when: the user needs to find tasks across projects by keyword, assignee, label, or status. Returns task ids, project ids, titles, status, and assignees.", + "inputSchema": { + "type": "object", + "properties": { + "query": { + "type": "string" + } + }, + "required": ["query"] + } + }, + { + "name": "create_task", + "description": "Create a task in a project.", + "inputSchema": { + "type": "object", + "properties": { + "projectId": { + "type": "string" + }, + "title": { + "type": "string" + }, + "body": { + "type": "string" + } + }, + "required": ["projectId", "title"] + } + }, + { + "name": "update_task", + "description": "Use when: the user asks to edit an existing task's title, body, assignee, labels, status, or due date. Returns the updated task.", + "inputSchema": { + "type": "object", + "properties": { + "taskId": { + "type": "string" + }, + "fields": { + "type": "object" + } + }, + "required": ["taskId", "fields"] + } + }, + { + "name": "complete_task", + "description": "Mark a task complete.", + "inputSchema": { + "type": "object", + "properties": { + "taskId": { + "type": "string" + } + }, + "required": ["taskId"] + } + }, + { + "name": "comment_on_task", + "description": "Use when: the user asks to add a comment or status note to an existing task. Returns the created comment id and task link.", + "inputSchema": { + "type": "object", + "properties": { + "taskId": { + "type": "string" + }, + "body": { + "type": "string" + } + }, + "required": ["taskId", "body"] + } + }, + { + "name": "confirm_task_update", + "description": "Confirm a pending task update.", + "inputSchema": { + "type": "object", + "properties": { + "pendingId": { + "type": "string" + } + }, + "required": ["pendingId"] + } + }, + { + "name": "reject_task_update", + "description": "Reject a pending task update.", + "inputSchema": { + "type": "object", + "properties": { + "pendingId": { + "type": "string" + } + }, + "required": ["pendingId"] + } + }, + { + "name": "list_users", + "description": "List users.", + "inputSchema": { + "type": "object", + "properties": {} + } + }, + { + "name": "get_user", + "description": "Use when: the user needs profile details for a known user id or email. Returns display name, email, team, and active status.", + "inputSchema": { + "type": "object", + "properties": { + "userId": { + "type": "string" + } + }, + "required": ["userId"] + } + }, + { + "name": "admin_export_workspace", + "description": "Export the entire workspace.", + "inputSchema": { + "type": "object", + "properties": { + "format": { + "type": "string" + } + } + } + }, + { + "name": "delete_project", + "description": "Delete a project.", + "inputSchema": { + "type": "object", + "properties": { + "projectId": { + "type": "string" + } + }, + "required": ["projectId"] + } + }, + { + "name": "raw_workspace_api", + "description": "Call any workspace API endpoint.", + "inputSchema": { + "type": "object", + "properties": { + "method": { + "type": "string" + }, + "path": { + "type": "string" + }, + "body": { + "type": "object" + } + }, + "required": ["method", "path"] + } + } + ] + } +} diff --git a/package.json b/package.json index 88741c4..d15a996 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,11 @@ { "name": "mcplens-cli", "version": "0.1.2", - "description": "Audit MCP tool activation locally and generate lean MCP servers from OpenAPI specs", + "description": "Audit whether agents can discover, choose, and complete MCP tool workflows", "license": "MIT", "author": "MCPLens contributors", + "main": "dist/cli.js", + "types": "dist/cli.d.ts", "type": "module", "repository": { "type": "git", @@ -17,13 +19,18 @@ "mcplens", "mcp", "mcp-audit", + "mcp-tools", "tool-activation", + "agent-reliability", "agent", "openapi", "api", "codegen", "typescript" ], + "publishConfig": { + "access": "public" + }, "engines": { "node": ">=20" }, @@ -33,8 +40,14 @@ }, "files": [ "dist", + "docs/audit-mcp-ci.md", + "examples", "src/mapping/runtime.ts", - "README.md" + "README.md", + "GUIDE.md", + "IMPACT.md", + "CHANGELOG.md", + "LICENSE" ], "scripts": { "build": "tsc", diff --git a/tests/scripts/pack-smoke.mjs b/tests/scripts/pack-smoke.mjs index 6fb9b60..030a7fb 100644 --- a/tests/scripts/pack-smoke.mjs +++ b/tests/scripts/pack-smoke.mjs @@ -18,7 +18,21 @@ try { if (!packed?.filename || !Array.isArray(packed.files)) throw new Error(`Unexpected npm pack output: ${pack.stdout}`) const packedFiles = new Set(packed.files.map((file) => file.path)) - for (const required of ['dist/cli.js', 'dist/codegen/generate.js', 'src/mapping/runtime.ts', 'README.md', 'package.json']) { + for (const required of [ + 'dist/cli.js', + 'dist/codegen/generate.js', + 'src/mapping/runtime.ts', + 'README.md', + 'GUIDE.md', + 'IMPACT.md', + 'CHANGELOG.md', + 'LICENSE', + 'docs/audit-mcp-ci.md', + 'examples/generic-mcp/report.md', + 'examples/browser-mcp/report.md', + 'examples/large-mcp/report.md', + 'package.json' + ]) { if (!packedFiles.has(required)) throw new Error(`npm pack did not include ${required}`) }