From d5b73385c6962afbc2470f8529931623ca32b0ff Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 3 Sep 2026 22:43:43 -0700 Subject: [PATCH 01/19] Add passthrough MCP mode --- .changeset/mcp-passthrough-mode.md | 9 + apps/cli/src/main.ts | 51 +- apps/cloud/src/mcp/agent-handler.ts | 12 +- apps/cloud/src/mcp/session-durable-object.ts | 18 +- apps/cloud/src/mcp/session-meta.ts | 2 + apps/host-cloudflare/src/mcp/agent-handler.ts | 10 +- .../src/mcp/session-durable-object.ts | 16 +- apps/local/src/main.ts | 3 + apps/local/src/mcp.ts | 10 +- e2e/cloud/passthrough-scale.test.ts | 122 +++++ e2e/scenarios/mcp-passthrough.test.ts | 332 ++++++++++++ e2e/src/surfaces/mcp.ts | 53 +- packages/core/api/src/server/mcp-build.ts | 4 + packages/core/sdk/src/executor.test.ts | 32 ++ packages/core/sdk/src/executor.ts | 103 +++- packages/core/sdk/src/index.ts | 2 +- packages/core/sdk/src/policies.test.ts | 64 +++ packages/core/sdk/src/tool.ts | 4 + packages/core/sdk/src/types.ts | 28 + .../src/mcp/agent-session-durable-object.ts | 15 + .../hosts/cloudflare/src/mcp/do-headers.ts | 3 + packages/hosts/mcp/src/browser-approval.ts | 36 ++ .../hosts/mcp/src/in-memory-session-store.ts | 40 +- .../hosts/mcp/src/passthrough-tools.test.ts | 485 ++++++++++++++++++ packages/hosts/mcp/src/passthrough-tools.ts | 231 +++++++++ packages/hosts/mcp/src/tool-server.ts | 467 ++++++++++++++--- packages/plugins/graphql/src/sdk/plugin.ts | 1 + packages/plugins/mcp/src/sdk/plugin.ts | 4 + packages/plugins/openapi/src/sdk/invoke.ts | 8 +- packages/react/src/api/analytics.tsx | 1 + .../react/src/components/mcp-install-card.tsx | 34 ++ 31 files changed, 2085 insertions(+), 115 deletions(-) create mode 100644 .changeset/mcp-passthrough-mode.md create mode 100644 e2e/cloud/passthrough-scale.test.ts create mode 100644 e2e/scenarios/mcp-passthrough.test.ts create mode 100644 packages/hosts/mcp/src/passthrough-tools.test.ts create mode 100644 packages/hosts/mcp/src/passthrough-tools.ts diff --git a/.changeset/mcp-passthrough-mode.md b/.changeset/mcp-passthrough-mode.md new file mode 100644 index 0000000000..104225ee82 --- /dev/null +++ b/.changeset/mcp-passthrough-mode.md @@ -0,0 +1,9 @@ +--- +"@executor-js/sdk": minor +"@executor-js/plugin-openapi": patch +"@executor-js/plugin-graphql": patch +"@executor-js/plugin-mcp": patch +"executor": minor +--- + +Add a passthrough MCP mode (`?mode=passthrough`, `executor mcp --mode passthrough`) that serves every connected integration tool as its own MCP tool, with workspace policy folded into each tool's annotations so the client's native approval flow applies. Adds `executor.tools.describeAll()` and a `readOnly` tool annotation. diff --git a/apps/cli/src/main.ts b/apps/cli/src/main.ts index fc2a9bf391..a3101b2ff4 100644 --- a/apps/cli/src/main.ts +++ b/apps/cli/src/main.ts @@ -1363,6 +1363,8 @@ const mcpUrlForActiveLocalServer = (input: { readonly elicitationMode: "browser" | "model"; readonly artifacts: boolean; readonly searchTools: boolean; + readonly toolMode: "codemode" | "passthrough"; + readonly integrations: readonly string[]; }): URL => { const url = new URL("/mcp", input.connection.origin); if (input.elicitationMode === "browser") { @@ -1378,6 +1380,14 @@ const mcpUrlForActiveLocalServer = (input: { if (input.searchTools) { url.searchParams.set("search_tools", "true"); } + // Passthrough is the non-default surface; only it is spelled out, and the + // integration filter only means anything alongside it. + if (input.toolMode === "passthrough") { + url.searchParams.set("mode", "passthrough"); + if (input.integrations.length > 0) { + url.searchParams.set("integrations", input.integrations.join(",")); + } + } return url; }; @@ -1394,6 +1404,8 @@ const runMcpHttpBridge = async (input: { readonly elicitationMode: "browser" | "model"; readonly artifacts: boolean; readonly searchTools: boolean; + readonly toolMode: "codemode" | "passthrough"; + readonly integrations: readonly string[]; }): Promise => { const stdio = new StdioServerTransport(); const authorization = getExecutorServerAuthorizationHeader(input.manifest.connection); @@ -1403,6 +1415,8 @@ const runMcpHttpBridge = async (input: { elicitationMode: input.elicitationMode, artifacts: input.artifacts, searchTools: input.searchTools, + toolMode: input.toolMode, + integrations: input.integrations, }), authorization ? { requestInit: { headers: { Authorization: authorization } } } : undefined, ); @@ -1482,6 +1496,8 @@ const runStdioMcpSession = (input: { readonly elicitationMode: "browser" | "model"; readonly artifacts: boolean; readonly searchTools: boolean; + readonly toolMode: "codemode" | "passthrough"; + readonly integrations: readonly string[]; }) => Effect.gen(function* () { // `executor mcp` never owns the local database. If a local server is already @@ -1499,6 +1515,8 @@ const runStdioMcpSession = (input: { elicitationMode: input.elicitationMode, artifacts: input.artifacts, searchTools: input.searchTools, + toolMode: input.toolMode, + integrations: input.integrations, }), ); return; @@ -1526,6 +1544,8 @@ const runStdioMcpSession = (input: { elicitationMode: input.elicitationMode, artifacts: input.artifacts, searchTools: input.searchTools, + toolMode: input.toolMode, + integrations: input.integrations, }), ); }); @@ -2898,11 +2918,38 @@ const mcpCommand = Command.make( "Serve one search_ tool per connected integration. Off by default; each routes through the same flow as tools.search inside execute.", ), ), + toolMode: Options.choice("mode", ["codemode", "passthrough"] as const) + .pipe(Options.withDefault("codemode")) + .pipe( + Options.withDescription( + "codemode (default) serves the execute tool; passthrough serves every connected integration tool directly, with policy folded into each tool's annotations and no execute, skills, or resume.", + ), + ), + integrations: Options.string("integrations") + .pipe(Options.optional) + .pipe( + Options.withDescription( + "Passthrough only: comma-separated integration slugs to serve. Omit for every connected integration.", + ), + ), }, - ({ scope, elicitationMode, noArtifacts, searchTools }) => + ({ scope, elicitationMode, noArtifacts, searchTools, toolMode, integrations }) => Effect.gen(function* () { applyScope(scope); - yield* runStdioMcpSession({ elicitationMode, artifacts: !noArtifacts, searchTools }); + yield* runStdioMcpSession({ + elicitationMode, + artifacts: !noArtifacts, + searchTools, + toolMode, + integrations: Option.match(integrations, { + onNone: () => [], + onSome: (value) => + value + .split(",") + .map((slug) => slug.trim()) + .filter((slug) => slug.length > 0), + }), + }); }), ).pipe(Command.withDescription("Start an MCP server over stdio")); diff --git a/apps/cloud/src/mcp/agent-handler.ts b/apps/cloud/src/mcp/agent-handler.ts index 05c2107ed0..6ddc1b37bb 100644 --- a/apps/cloud/src/mcp/agent-handler.ts +++ b/apps/cloud/src/mcp/agent-handler.ts @@ -15,7 +15,9 @@ import { currentPropagationHeaders, readArtifactsEnabled, readElicitationMode, + readPassthroughIntegrations, readSearchToolsEnabled, + readToolMode, withVerifiedIdentityHeaders, } from "@executor-js/cloudflare/mcp/do-headers"; import type { McpSessionProps } from "@executor-js/cloudflare/mcp/agent-durable-object"; @@ -187,8 +189,16 @@ const propsForPrincipal = ( ...sessionOrgRoleMetadata(principal), userId: principal.accountId, elicitationMode: readElicitationMode(request), - artifactsEnabled: readArtifactsEnabled(request), + // Forwarded only when spelled out, so the factory applies the tool + // mode's own default to an absent `?artifacts=`. + ...(new URL(request.url).searchParams.has("artifacts") + ? { artifactsEnabled: readArtifactsEnabled(request) } + : {}), searchToolsEnabled: readSearchToolsEnabled(request), + toolMode: readToolMode(request), + ...(readPassthroughIntegrations(request) + ? { passthroughIntegrations: readPassthroughIntegrations(request) } + : {}), resource, webOrigin: new URL(request.url).origin, }, diff --git a/apps/cloud/src/mcp/session-durable-object.ts b/apps/cloud/src/mcp/session-durable-object.ts index 2646a785a0..9afa758d39 100644 --- a/apps/cloud/src/mcp/session-durable-object.ts +++ b/apps/cloud/src/mcp/session-durable-object.ts @@ -384,13 +384,23 @@ export class McpSessionDOSqlite extends McpAgentSessionDOBase { @@ -132,6 +133,7 @@ export const createServerHandlers = async (token: string): Promise tool.static !== true, + ); + expect(visible.length, "the seeded catalog is large").toBeGreaterThan(3000); + + const session = mcp.session(identity, { mode: "passthrough" }); + const startedAt = Date.now(); + const served = yield* session.describeTools(); + const elapsedMs = Date.now() - startedAt; + + expect( + elapsedMs, + `a ${visible.length}-tool passthrough connect stays bounded (took ${elapsedMs}ms)`, + ).toBeLessThan(MAX_PASSTHROUGH_CONNECT_MS); + + // Completeness: one served tool per visible tool, no more, no less. + expect(served.length, "every visible tool is served").toBe(visible.length); + const names = served.map((tool) => tool.name); + expect(new Set(names).size, "every served name is unique").toBe(names.length); + for (const name of names) { + expect(name, "every name fits the MCP grammar").toMatch(/^[A-Za-z0-9_-]{1,64}$/); + } + // Every integration in the catalog is represented under its prefix. + for (const slug of seeded.integrationSlugs) { + expect( + names.some((name) => name.startsWith(`${slug}__`)), + `integration ${slug} is served`, + ).toBe(true); + } + // The codemode surface is absent. + expect(names, "no execute").not.toContain("execute"); + expect(names, "no resume").not.toContain("resume"); + + // Every served tool carries explicit hints and a usable schema: the + // real Vercel spec has POST/DELETE operations (destructive) and GETs + // (read-only), so both values must appear. + const hints = new Set(); + for (const tool of served) { + const annotations = tool.annotations ?? {}; + expect( + typeof annotations.destructiveHint, + `${tool.name} sets destructiveHint explicitly`, + ).toBe("boolean"); + expect(typeof annotations.readOnlyHint, `${tool.name} sets readOnlyHint`).toBe( + "boolean", + ); + hints.add(`${annotations.readOnlyHint}/${annotations.destructiveHint}`); + expect(tool.inputSchema, `${tool.name} advertises an input schema`).toBeDefined(); + } + expect(hints.has("true/false"), "some tools are read-only").toBe(true); + expect(hints.has("false/true"), "some tools require approval").toBe(true); + + // `?integrations=` trims the surface to one integration. + const one = seeded.integrationSlugs[0]!; + const narrowed = mcp.session(identity, { mode: "passthrough", integrations: [one] }); + const narrowedNames = yield* narrowed.listTools(); + expect(narrowedNames.length, "the filter serves only one integration").toBe( + visible.filter((tool) => String(tool.integration) === one).length, + ); + for (const name of narrowedNames) { + expect(name.startsWith(`${one}__`), `${name} belongs to ${one}`).toBe(true); + } + }), + seeded.cleanup, + ); + }), + ), +); diff --git a/e2e/scenarios/mcp-passthrough.test.ts b/e2e/scenarios/mcp-passthrough.test.ts new file mode 100644 index 0000000000..ce730d0c30 --- /dev/null +++ b/e2e/scenarios/mcp-passthrough.test.ts @@ -0,0 +1,332 @@ +// The passthrough tool surface. A plain MCP endpoint serves `execute` +// (codemode); a connection that says `?mode=passthrough` instead serves every +// visible integration tool as its own MCP tool, with the workspace policy +// folded into each tool's annotations at list time, and NONE of `execute`, +// `skills` or `resume`. The harness's own approval flow reads the hints; the +// server never pauses. The proof is comparative where it can be (two sessions, +// same identity, differing only in the query) and end-to-end where it must be: +// a real call reaches a real upstream, and a `block` rule is enforced on both +// the list and the call. +import { randomBytes } from "node:crypto"; +import { createServer, type IncomingMessage } from "node:http"; + +import { expect } from "@effect/vitest"; +import { Effect } from "effect"; +import { composePluginApi } from "@executor-js/api/server"; +import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; +import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared"; + +import { scenario } from "../src/scenario"; +import { Api, Mcp, Target } from "../src/services"; + +const api = composePluginApi([openApiHttpPlugin()] as const); + +const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`; + +/** A two-operation API: a read and a write, so the surface carries one tool + * per policy outcome. The write takes a JSON body with a shared `$ref`, so + * the advertised schema must be self-contained to be usable. */ +const spec = (baseUrl: string): string => + JSON.stringify({ + openapi: "3.0.3", + info: { title: "Passthrough API", version: "1.0.0" }, + servers: [{ url: baseUrl }], + components: { + schemas: { + NewNote: { + type: "object", + properties: { text: { type: "string" } }, + required: ["text"], + }, + }, + }, + paths: { + "/notes": { + get: { + operationId: "listNotes", + summary: "List notes", + responses: { "200": { description: "ok" } }, + }, + post: { + operationId: "createNote", + summary: "Create a note", + requestBody: { + required: true, + content: { + "application/json": { schema: { $ref: "#/components/schemas/NewNote" } }, + }, + }, + responses: { "200": { description: "ok" } }, + }, + }, + }, + }); + +interface RecordedRequest { + readonly method: string; + readonly path: string; + readonly authorization: string | undefined; + readonly body: string; +} + +/** A real upstream that records what reached it, so a passthrough call can be + * proven to have gone over the wire with the connection's credential. */ +const serveRecordingUpstream = Effect.acquireRelease( + Effect.callback<{ + readonly url: string; + readonly requests: RecordedRequest[]; + close: () => void; + }>((resume) => { + const requests: RecordedRequest[] = []; + const readBody = (request: IncomingMessage) => + new Promise((resolve) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => resolve(Buffer.concat(chunks).toString("utf8"))); + }); + const server = createServer((request, response) => { + void readBody(request).then((body) => { + requests.push({ + method: request.method ?? "", + path: request.url ?? "", + authorization: request.headers.authorization, + body, + }); + response.writeHead(200, { "content-type": "application/json" }); + response.end( + JSON.stringify( + request.method === "POST" + ? { id: "note_1", ...(body ? (JSON.parse(body) as object) : {}) } + : { notes: [{ id: "note_0", text: "existing" }] }, + ), + ); + }); + }); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + resume( + Effect.succeed({ + url: `http://127.0.0.1:${port}`, + requests, + close: () => { + server.close(); + server.closeAllConnections(); + }, + }), + ); + }); + }), + (upstream) => Effect.sync(() => upstream.close()), +); + +const rawResultOf = (result: { readonly raw: unknown }) => + result.raw as { + content?: ReadonlyArray<{ type: string; text?: string }>; + structuredContent?: Record; + isError?: boolean; + }; + +scenario( + "Passthrough · a session connected with mode=passthrough serves the catalog as tools, no execute, no resume", + { timeout: 180_000 }, + Effect.scoped( + Effect.gen(function* () { + const target = yield* Target; + const mcp = yield* Mcp; + const { client: makeClient } = yield* Api; + + const identity = yield* target.newIdentity(); + const client = yield* makeClient(api, identity); + const upstream = yield* serveRecordingUpstream; + const slug = unique("ptapi"); + const otherSlug = unique("ptother"); + + yield* Effect.ensuring( + Effect.gen(function* () { + // Two integrations, one connection each: the filter scenario below + // needs a second one to leave out. + for (const s of [slug, otherSlug]) { + yield* client.openapi.addSpec({ + payload: { + spec: { kind: "blob", value: spec(upstream.url) }, + slug: s, + baseUrl: upstream.url, + authenticationTemplate: [ + { + slug: "apiKey", + type: "apiKey", + headers: { authorization: ["Bearer ", { type: "variable", name: "token" }] }, + }, + ], + }, + }); + yield* client.connections.create({ + payload: { + owner: "org", + name: ConnectionName.make("main"), + integration: IntegrationSlug.make(s), + template: AuthTemplateSlug.make("apiKey"), + value: `tok_${s}`, + }, + }); + } + + // The OpenAPI plugin names an operation `.` (group from + // the tag or path, here `notes`), and passthrough mangles the dot to + // `_`, so the wire name is `__notes_listNotes`. Resolve by + // suffix rather than hardcode the group: the mangling is what this + // scenario proves, the grouping is the plugin's own contract. + const toolNamed = (names: readonly string[], integration: string, leaf: string) => + names.find((name) => name.startsWith(`${integration}__`) && name.endsWith(`_${leaf}`)); + + // --- The default is untouched: codemode, no per-tool surface. --- + const codemode = mcp.session(identity); + const codemodeTools = yield* codemode.listTools(); + expect(codemodeTools, "a plain session still serves execute").toContain("execute"); + expect( + toolNamed(codemodeTools, slug, "listNotes"), + "a plain session serves no passthrough tools", + ).toBeUndefined(); + + // --- Passthrough: same identity, `?mode=passthrough`. --- + const passthrough = mcp.session(identity, { mode: "passthrough" }); + const described = yield* passthrough.describeTools(); + const names = described.map((tool) => tool.name); + const listTool = toolNamed(names, slug, "listNotes"); + const createTool = toolNamed(names, slug, "createNote"); + expect( + listTool, + `the read operation is served as its own tool (got ${names.join(", ")})`, + ).toBeDefined(); + expect(createTool, "the write operation is served as its own tool").toBeDefined(); + expect( + toolNamed(names, otherSlug, "listNotes"), + "the second integration is served too", + ).toBeDefined(); + // Names obey the MCP grammar and carry the integration prefix. + for (const name of names) expect(name, "MCP-safe name").toMatch(/^[A-Za-z0-9_-]+$/); + expect(names, "execute is not served in passthrough").not.toContain("execute"); + expect(names, "skills is not served in passthrough").not.toContain("skills"); + expect(names, "resume is not served in passthrough").not.toContain("resume"); + expect( + names.filter((name) => name.startsWith("search_")), + "no search_ tools either", + ).toEqual([]); + + if (!listTool || !createTool) return; // narrowed above; keeps TS honest + // --- Policy is advertised, not enforced by a pause. --- + // The OpenAPI plugin derives approval from the HTTP method: GET is + // read-only and free, POST requires approval by default. + const listDef = described.find((tool) => tool.name === listTool); + const createDef = described.find((tool) => tool.name === createTool); + expect(listDef?.annotations, "GET advertises read-only, non-destructive").toMatchObject({ + readOnlyHint: true, + destructiveHint: false, + }); + expect(createDef?.annotations, "POST advertises destructive").toMatchObject({ + readOnlyHint: false, + destructiveHint: true, + }); + // The body schema arrives self-contained: the `$ref` and its target + // both ride along, so a client can validate without a second fetch. + expect( + JSON.stringify(createDef?.inputSchema), + "the write tool's schema names the body field", + ).toContain("text"); + + // --- A read call reaches the upstream with the connection's credential. --- + const listed = yield* passthrough.call(listTool, {}); + expect(listed.ok, `the read call completes: ${listed.text}`).toBe(true); + expect(listed.text, "the upstream payload comes back").toContain("existing"); + const listReq = upstream.requests.find((r) => r.method === "GET"); + expect(listReq, "the GET reached the upstream").toBeDefined(); + expect(listReq?.authorization, "the connection's credential was applied").toBe( + `Bearer tok_${slug}`, + ); + + // --- An approval-gated call runs to completion: no pause, no resume. --- + const created = yield* passthrough.call(createTool, { body: { text: "hello" } }); + expect(created.ok, `the gated call completes without a pause: ${created.text}`).toBe( + true, + ); + expect(created.text, "the call did not pause").not.toContain("Execution paused"); + expect(created.text, "the call did not ask for a resume").not.toContain("executionId"); + const createReq = upstream.requests.find((r) => r.method === "POST"); + expect(createReq, "the POST reached the upstream").toBeDefined(); + expect(createReq?.body, "the JSON body went over the wire").toContain('"text":"hello"'); + expect( + rawResultOf(created).structuredContent?.status, + "the result is a completed execution", + ).toBe("completed"); + + // --- Arguments are validated against the advertised schema. --- + // The server answers with a JSON-RPC invalid-params error, which the + // client library surfaces as a rejection (`ok: false` here). Either + // way the proof is the same: the upstream never saw a second POST. + const invalid = yield* passthrough.call(createTool, { body: { wrong: 1 } }).pipe( + Effect.map((r) => r.ok), + Effect.catchCause(() => Effect.succeed(false)), + ); + expect(invalid, "a body missing its required field is refused").toBe(false); + expect( + upstream.requests.filter((r) => r.method === "POST").length, + "the invalid call never reached the upstream", + ).toBe(1); + + // --- `?integrations=` narrows the surface. --- + const narrowed = mcp.session(identity, { + mode: "passthrough", + integrations: [otherSlug], + }); + const narrowedNames = yield* narrowed.listTools(); + expect( + toolNamed(narrowedNames, otherSlug, "listNotes"), + "the requested integration is served", + ).toBeDefined(); + expect(narrowedNames, "the other integration is left out").not.toContain(listTool); + + // --- `block` is enforced on the list AND the call. --- + const blockRule = yield* client.policies.create({ + payload: { owner: "org", pattern: `${slug}.*.*.*.createNote`, action: "block" }, + }); + yield* Effect.ensuring( + Effect.gen(function* () { + const afterBlock = mcp.session(identity, { mode: "passthrough" }); + const afterNames = yield* afterBlock.listTools(); + expect(afterNames, "a blocked tool is not listed").not.toContain(createTool); + expect(afterNames, "the unblocked sibling still is").toContain(listTool); + // A client that cached the old name cannot call it either: the + // executor refuses the call at invoke time, which passthrough + // renders as an MCP error result. + const stale = yield* passthrough.call(createTool, { body: { text: "again" } }); + expect(stale.ok, "a blocked tool fails when called").toBe(false); + expect(stale.text, "the failure names the policy").toContain("tool_blocked"); + expect( + upstream.requests.filter((r) => r.method === "POST").length, + "the blocked call never reached the upstream", + ).toBe(1); + }), + client.policies + .remove({ params: { policyId: blockRule.id }, payload: { owner: "org" } }) + .pipe(Effect.ignore), + ); + }), + Effect.gen(function* () { + for (const s of [slug, otherSlug]) { + yield* client.connections + .remove({ + params: { + owner: "org", + integration: IntegrationSlug.make(s), + name: ConnectionName.make("main"), + }, + }) + .pipe(Effect.ignore); + yield* client.openapi.removeSpec({ params: { slug: s } }).pipe(Effect.ignore); + } + }), + ); + }), + ), +); diff --git a/e2e/src/surfaces/mcp.ts b/e2e/src/surfaces/mcp.ts index e1fa12c5f1..047bb953f8 100644 --- a/e2e/src/surfaces/mcp.ts +++ b/e2e/src/surfaces/mcp.ts @@ -100,6 +100,11 @@ export interface McpCallResult { export interface McpToolDef { readonly name: string; readonly description: string; + /** The advertised MCP annotations (`readOnlyHint`, `destructiveHint`, …), + * for scenarios that assert on what a harness's native approval reads. */ + readonly annotations?: Record; + /** The advertised input JSON Schema, verbatim. */ + readonly inputSchema?: unknown; } /** How a connection surfaces a paused (approval-gated) execution. `browser` is @@ -172,6 +177,11 @@ export interface McpSurface { * `search_` tools (`?search_tools=true`). Omitted means * the product default: none. */ readonly searchTools?: boolean; + /** `passthrough` serves every visible integration tool directly + * (`?mode=passthrough`). Omitted means the product default: codemode. */ + readonly mode?: "codemode" | "passthrough"; + /** Passthrough only: `?integrations=a,b`. */ + readonly integrations?: readonly string[]; readonly url?: string; }, ) => McpSession; @@ -315,6 +325,10 @@ export const makeMcpSurface = (target: Target, runDir?: string): McpSurface => ( ...(options?.elicitationMode ? [`elicitation_mode=${options.elicitationMode}`] : []), ...(options?.artifacts === false ? ["artifacts=false"] : []), ...(options?.searchTools === true ? ["search_tools=true"] : []), + ...(options?.mode === "passthrough" ? ["mode=passthrough"] : []), + ...(options?.integrations && options.integrations.length > 0 + ? [`integrations=${encodeURIComponent(options.integrations.join(","))}`] + : []), ].join("&"); const sessionUrl = sessionQuery ? `${mcpUrl}?${sessionQuery}` : mcpUrl; @@ -358,6 +372,10 @@ export const makeMcpSurface = (target: Target, runDir?: string): McpSurface => ( return listed.tools.map((tool) => ({ name: tool.name, description: tool.description ?? "", + ...(tool.annotations + ? { annotations: tool.annotations as Record } + : {}), + inputSchema: tool.inputSchema, })); }), call, @@ -408,14 +426,37 @@ export const makeMcpSurface = (target: Target, runDir?: string): McpSurface => ( return defs.map((tool: { name: string }) => tool.name); }); + // mcporter's `listTools` projects annotations away, so read the raw + // client it holds: the same connection (and cached OAuth), the full tool + // definition. const describeTools = () => Effect.promise(async (): Promise> => { - const defs = await (await runtime()).listTools(serverName, callOptions); - connected = true; - return defs.map((tool: { name: string; description?: string }) => ({ - name: tool.name, - description: tool.description ?? "", - })); + const rt = await runtime(); + if (!connected) { + await rt.listTools(serverName, callOptions); + connected = true; + } + const context = await rt.connect(serverName, { + allowCachedAuth: true, + oauthSessionOptions: callOptions.oauthSessionOptions, + }); + const out: McpToolDef[] = []; + let cursor: string | undefined; + do { + const page = await context.client.listTools(cursor ? { cursor } : undefined); + for (const tool of page.tools) { + out.push({ + name: tool.name, + description: tool.description ?? "", + ...(tool.annotations + ? { annotations: tool.annotations as Record } + : {}), + inputSchema: tool.inputSchema, + }); + } + cursor = page.nextCursor ?? undefined; + } while (cursor); + return out; }); const call = (name: string, args: Record = {}) => diff --git a/packages/core/api/src/server/mcp-build.ts b/packages/core/api/src/server/mcp-build.ts index 2512436557..455d15c24d 100644 --- a/packages/core/api/src/server/mcp-build.ts +++ b/packages/core/api/src/server/mcp-build.ts @@ -71,6 +71,7 @@ export const makeMcpBuildServer = engine, artifacts: executor.artifacts, connections: executor.connections, + tools: executor.tools, ...(hostOptions?.loadAppShellHtml ? { loadAppShellHtml: hostOptions.loadAppShellHtml } : {}), @@ -88,6 +89,9 @@ export const makeMcpBuildServer = ...(options ?? {}), }).pipe( Effect.withSpan("mcp.server.create"), + // A passthrough session with no catalog is a build failure like any + // other: the client gets the same retryable envelope. + Effect.mapError((cause) => new McpEngineBuildError({ cause })), Effect.map((mcpServer) => ({ mcpServer, engine })), ), ), diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index fb2835a529..f3a9e905ec 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -775,6 +775,38 @@ describe("createExecutor", () => { }), ); + it.effect("tools.describeAll inlines only the reachable input definitions per tool", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ + plugins: [demoPlugin] as const, + }); + yield* executor.demo.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { + provider: ProviderKey.make("memory"), + id: ProviderItemId.make("v"), + }, + }); + + const all = yield* executor.tools.describeAll(); + const inspect = all.find((tool) => tool.name === "inspect"); + const run = all.find((tool) => tool.name === "run"); + expect(inspect).toBeDefined(); + expect(run).toBeDefined(); + // The INPUT schema's transitive `$ref` closure rides along under `$defs`, + // so the schema is self-contained on the wire. `Owner` is only reachable + // from the output schema and `Unused` from nothing, so neither appears. + const inlined = inspect?.inputSchema as { $defs?: Record }; + expect(Object.keys(inlined.$defs ?? {}).sort()).toEqual(["Cat", "Collar", "Dog", "Pet"]); + // A tool with no declared input carries no schema at all. + expect(run?.inputSchema).toBeUndefined(); + }), + ); + it.effect("execute dispatches a connection-produced tool to the owning plugin", () => Effect.gen(function* () { const executor = yield* makeTestExecutor({ diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 6916233cb5..b7ed445aea 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -199,10 +199,10 @@ import { ORG_SUBJECT, type ExecutorOwnerPolicyContext, } from "./owner-policy"; -import { ToolSchemaView, type IntegrationDetectionResult } from "./types"; +import { ToolSchemaView, type IntegrationDetectionResult, type ToolProjection } from "./types"; import { type Tool, type ToolAnnotations, type ToolDef, type ToolListFilter } from "./tool"; import { buildToolTypeScriptPreview } from "./schema-types"; -import { collectReferencedDefinitions } from "./schema-refs"; +import { collectReferencedDefinitions, reattachDefs } from "./schema-refs"; import { refreshAccessToken, exchangeClientCredentials, @@ -469,6 +469,19 @@ export type Executor = { readonly tools: { readonly list: (filter?: ToolListFilter) => Effect.Effect; readonly schema: (address: ToolAddress) => Effect.Effect; + /** + * Every visible dynamic tool with its self-contained input schema and + * resolved policy, in ONE pass: the tool rows, the shared `$defs`, and the + * policy rule set are each read once, then joined in memory. Built for + * surfaces that must advertise the whole catalog at once (the passthrough + * MCP mode), where a per-tool `schema()` round-trip would be an N+1 over + * thousands of rows. Blocked tools are omitted; static (plugin + * configuration) tools are omitted too — they are codemode affordances, + * not integration tools. + */ + readonly describeAll: ( + filter?: ToolListFilter, + ) => Effect.Effect; }; readonly providers: { @@ -5758,6 +5771,91 @@ export const createExecutor = => + Effect.gen(function* () { + if (toolsSyncGraceMs === null) { + yield* syncStaleConnectionTools; + } else { + yield* awaitStaleSyncWithinGrace(toolsSyncGraceMs); + } + const integrationWhere = (b: AnyCb) => + b.and( + filter?.integration === undefined + ? true + : b("integration", "=", String(filter.integration)), + filter?.owner === undefined ? true : b("owner", "=", filter.owner), + filter?.connection === undefined + ? true + : b("connection", "=", String(filter.connection)), + ); + // Unlike `toolsList`, this read NEEDS `input_schema`: it is the schema + // the surface advertises. `output_schema` stays out — nothing here + // serves it, and it is the other half of the per-row weight. + const rows = yield* core.findMany("tool", { + where: integrationWhere, + select: [...TOOL_INVOCATION_COLUMNS, "input_schema"], + }); + const policyRules = yield* listActivePolicyRuleSet(); + + // Shared definitions, loaded once and keyed per connection: `$ref`s are + // connection-local (`#/$defs/` resolves against the producing + // connection's `definition` rows), so the join key is the same + // (owner, integration, connection) triple the tool row carries. + const definitionRows = yield* core.findMany("definition", { + where: integrationWhere, + }); + const defsByConnection = new Map>(); + for (const def of definitionRows) { + const key = `${def.owner}\u0000${def.integration}\u0000${def.connection}`; + let bucket = defsByConnection.get(key); + if (!bucket) { + bucket = new Map(); + defsByConnection.set(key, bucket); + } + bucket.set(def.name, decodeJsonColumn(def.schema)); + } + const EMPTY_DEFS: ReadonlyMap = new Map(); + + const projections: ToolProjection[] = []; + for (const row of rows) { + const tool = rowToTool(row); + if (!matchesToolFilter(tool, filter)) continue; + const effective = yield* resolvePolicyFromRuleSet( + normalizedPolicyId(tool), + policyRules, + tool.annotations?.requiresApproval, + ); + if (effective.action === "block") continue; + // The rule set already folded the plugin default in (`liftPlugin`), + // and an explicit user `approve` overrides it — the same answer + // `approvalRequired` gives on the invoke path. + const policy = + effective.action === "require_approval" + ? ("require_approval" as const) + : ("approve" as const); + const defs = + defsByConnection.get(`${row.owner}\u0000${row.integration}\u0000${row.connection}`) ?? + EMPTY_DEFS; + const inputSchema = + tool.inputSchema === undefined ? undefined : reattachDefs(tool.inputSchema, defs); + const readOnly = tool.annotations?.readOnly; + projections.push({ + address: tool.address, + integration: String(tool.integration), + owner: tool.owner, + connection: String(tool.connection), + name: String(tool.name), + description: tool.description, + ...(inputSchema === undefined ? {} : { inputSchema }), + policy, + ...(typeof readOnly === "boolean" ? { readOnly } : {}), + }); + } + return projections; + }).pipe(Effect.withSpan("executor.tools.describe_all")); + // ------------------------------------------------------------------ // Providers // ------------------------------------------------------------------ @@ -7056,6 +7154,7 @@ export const createExecutor = { }), ); }); + +// --------------------------------------------------------------------------- +// tools.describeAll — the one-pass projection the passthrough MCP surface +// serves: every visible tool with its resolved policy, in a single read of the +// catalog + rule set. The fixture plugin marks `vercel.delete` as requiring +// approval by annotation; user rules layer on top exactly as they do for +// `tools.list` and `execute`. +// --------------------------------------------------------------------------- + +describe("executor.tools.describeAll", () => { + it.effect("returns every visible tool with the effective policy folded in", () => + Effect.gen(function* () { + const executor = yield* setupExecutor(); + const all = yield* executor.tools.describeAll(); + const byAddress = new Map(all.map((tool) => [String(tool.address), tool])); + expect([...byAddress.keys()].sort()).toEqual([ + String(addr(GITHUB, "list")), + String(addr(VERCEL, "delete")), + String(addr(VERCEL, "deploy")), + ]); + // Plugin default: `delete` requires approval, `deploy` does not. + expect(byAddress.get(String(addr(VERCEL, "delete")))?.policy).toBe("require_approval"); + expect(byAddress.get(String(addr(VERCEL, "deploy")))?.policy).toBe("approve"); + expect(byAddress.get(String(addr(GITHUB, "list")))?.policy).toBe("approve"); + // The projection carries the routing triple a name mangler needs. + const deploy = byAddress.get(String(addr(VERCEL, "deploy")))!; + expect(deploy.integration).toBe("vercel"); + expect(deploy.owner).toBe("org"); + expect(deploy.connection).toBe(String(CONN)); + expect(deploy.name).toBe("deploy"); + }), + ); + + it.effect("omits blocked tools and honours require_approval / approve rules", () => + Effect.gen(function* () { + const executor = yield* setupExecutor(); + yield* executor.policies.create({ owner: "org", pattern: "github.*", action: "block" }); + yield* executor.policies.create({ + owner: "org", + pattern: "vercel.*.*.deploy", + action: "require_approval", + }); + // An explicit approve overrides the plugin's requiresApproval default. + yield* executor.policies.create({ + owner: "org", + pattern: "vercel.*.*.delete", + action: "approve", + }); + const all = yield* executor.tools.describeAll(); + const byName = new Map(all.map((tool) => [`${tool.integration}.${tool.name}`, tool.policy])); + expect(byName.has("github.list")).toBe(false); + expect(byName.get("vercel.deploy")).toBe("require_approval"); + expect(byName.get("vercel.delete")).toBe("approve"); + }), + ); + + it.effect("narrows by integration like tools.list", () => + Effect.gen(function* () { + const executor = yield* setupExecutor(); + const only = yield* executor.tools.describeAll({ integration: GITHUB }); + expect(only.map((tool) => tool.name)).toEqual(["list"]); + }), + ); +}); diff --git a/packages/core/sdk/src/tool.ts b/packages/core/sdk/src/tool.ts index c3bdb64f65..7e3aa0ab3d 100644 --- a/packages/core/sdk/src/tool.ts +++ b/packages/core/sdk/src/tool.ts @@ -12,6 +12,10 @@ export interface ToolAnnotations { readonly requiresApproval?: boolean; readonly approvalDescription?: string; readonly mayElicit?: boolean; + /** The tool never mutates upstream state (a GET, a GraphQL query, an + * upstream MCP `readOnlyHint`). Set only by plugins that can tell; absent + * means unknown, and a surface must not read it as "mutating". */ + readonly readOnly?: boolean; } /** A tool as produced by a plugin — the definition, no address yet (the SDK diff --git a/packages/core/sdk/src/types.ts b/packages/core/sdk/src/types.ts index ad5d2bb0d4..ab0007527b 100644 --- a/packages/core/sdk/src/types.ts +++ b/packages/core/sdk/src/types.ts @@ -32,6 +32,34 @@ export const ToolSchemaView = Schema.Struct({ }); export type ToolSchemaView = typeof ToolSchemaView.Type; +// --------------------------------------------------------------------------- +// ToolProjection — one visible tool as an MCP passthrough surface serves it: +// the address, the resolved policy, and a SELF-CONTAINED input schema (shared +// `$defs` re-attached), returned by `executor.tools.describeAll()`. Built in +// one pass over the catalog so a workspace with thousands of tools costs a +// handful of reads, not one round-trip per tool. +// --------------------------------------------------------------------------- + +export const ToolProjection = Schema.Struct({ + address: ToolAddress, + integration: Schema.String, + owner: Schema.Literals(["org", "user"]), + connection: Schema.String, + name: Schema.String, + description: Schema.String, + /** JSON Schema with every referenced `$def` inlined under `$defs`. Absent + * when the tool declares no input. */ + inputSchema: Schema.optional(Schema.Unknown), + /** The effective policy action for this caller. `block`ed tools are never + * returned, so this is `approve` or `require_approval`. */ + policy: Schema.Literals(["approve", "require_approval"]), + /** Whether the plugin marks the tool as never mutating upstream state. Only + * plugins that know (HTTP method, GraphQL operation kind, upstream MCP hint) + * set it; absent means unknown. */ + readOnly: Schema.optional(Schema.Boolean), +}); +export type ToolProjection = typeof ToolProjection.Type; + // --------------------------------------------------------------------------- // Integration detection — optional capability on `PluginSpec.detect`. When a // user pastes a URL in the onboarding UI, `executor.integrations.detect(url)` diff --git a/packages/hosts/cloudflare/src/mcp/agent-session-durable-object.ts b/packages/hosts/cloudflare/src/mcp/agent-session-durable-object.ts index 993fab3783..e711b98472 100644 --- a/packages/hosts/cloudflare/src/mcp/agent-session-durable-object.ts +++ b/packages/hosts/cloudflare/src/mcp/agent-session-durable-object.ts @@ -23,6 +23,7 @@ import { defaultMcpResource, type McpResource } from "@executor-js/host-mcp"; import { ResumeResponsePayload, decodeResumeResponse, + type McpToolMode, } from "@executor-js/host-mcp/browser-approval"; import type { IncomingPropagationHeaders, McpElicitationMode } from "./do-headers"; @@ -77,6 +78,12 @@ interface McpSessionInitBase { * tools, read off `?search_tools=` at connect time. Absent means the * default (disabled). */ readonly searchToolsEnabled?: boolean; + /** The tool surface, read off `?mode=` at connect time. Absent means the + * default (codemode). */ + readonly toolMode?: McpToolMode; + /** Passthrough only: the `?integrations=` filter. Absent means every + * visible integration. */ + readonly passthroughIntegrations?: readonly string[]; /** The MCP resource the session was minted against (`/mcp` default vs a * `/mcp/toolkits/` toolkit), so the tool catalog is scoped to it. */ readonly resource: McpResource; @@ -151,6 +158,14 @@ interface SessionMetaBase { * {@link McpSessionInit}). Absent — including for sessions persisted before * the flag existed — means the default (disabled). */ readonly searchToolsEnabled?: boolean; + /** The tool surface (carried from {@link McpSessionInit}). Absent — + * including for sessions persisted before the field existed — means + * codemode. A cold restore MUST rebuild the same surface the client first + * saw, or its cached tool names stop resolving mid-conversation. */ + readonly toolMode?: McpToolMode; + /** Passthrough only: the integration filter (carried from + * {@link McpSessionInit}). */ + readonly passthroughIntegrations?: readonly string[]; /** The MCP resource the session serves (carried from {@link McpSessionInit}); * `buildMcpServer` scopes the tool catalog to it. */ readonly resource: McpResource; diff --git a/packages/hosts/cloudflare/src/mcp/do-headers.ts b/packages/hosts/cloudflare/src/mcp/do-headers.ts index 02c4b196d0..4af7c74b94 100644 --- a/packages/hosts/cloudflare/src/mcp/do-headers.ts +++ b/packages/hosts/cloudflare/src/mcp/do-headers.ts @@ -132,6 +132,9 @@ export const withMcpResponseHeaders = (response: Response): Response => { export { readArtifactsEnabled, readElicitationMode, + readPassthroughIntegrations, readSearchToolsEnabled, + readToolMode, type McpElicitationMode, + type McpToolMode, } from "@executor-js/host-mcp/browser-approval"; diff --git a/packages/hosts/mcp/src/browser-approval.ts b/packages/hosts/mcp/src/browser-approval.ts index a524b89fbb..fb7d4317fd 100644 --- a/packages/hosts/mcp/src/browser-approval.ts +++ b/packages/hosts/mcp/src/browser-approval.ts @@ -79,6 +79,42 @@ export const readSearchToolsEnabled = (request: Request): boolean => { return TRUE_QUERY_VALUES.has(value.toLowerCase()); }; +export type McpToolMode = "codemode" | "passthrough"; + +/** + * Read the tool surface mode off an MCP request's `?mode=` query. The default, + * `codemode`, serves `execute` (the model writes sandboxed TypeScript against + * `tools.*`). `?mode=passthrough` instead serves every visible integration + * tool as its own MCP tool, with no `execute`, `skills`, or `resume`: the + * harness's native approval reads the advertised annotations. Any other value + * reads as the default. + */ +export const readToolMode = (request: Request): McpToolMode => { + const value = new URL(request.url).searchParams.get("mode"); + return value === "passthrough" ? "passthrough" : "codemode"; +}; + +/** + * Read the optional integration filter for passthrough mode off `?integrations=`, + * a comma-separated list of integration slugs. Absent or empty means every + * visible integration. Only meaningful with `?mode=passthrough`; codemode has + * `tools.search` for scoping. Slugs are trimmed and de-duplicated; order is + * not significant. + */ +export const readPassthroughIntegrations = (request: Request): readonly string[] | undefined => { + const value = new URL(request.url).searchParams.get("integrations"); + if (value === null) return undefined; + const slugs = Array.from( + new Set( + value + .split(",") + .map((slug) => slug.trim()) + .filter((slug) => slug.length > 0), + ), + ); + return slugs.length === 0 ? undefined : slugs; +}; + /** * Build the console approval URL for a paused execution: * `//resume/?mcp_session_id=` diff --git a/packages/hosts/mcp/src/in-memory-session-store.ts b/packages/hosts/mcp/src/in-memory-session-store.ts index af9e149741..07782471f9 100644 --- a/packages/hosts/mcp/src/in-memory-session-store.ts +++ b/packages/hosts/mcp/src/in-memory-session-store.ts @@ -11,7 +11,10 @@ import { formatResumeAcknowledgement, readArtifactsEnabled, readElicitationMode, + readPassthroughIntegrations, readSearchToolsEnabled, + readToolMode, + type McpToolMode, } from "./browser-approval"; import { makeInProcessBrowserApprovalStore, @@ -31,7 +34,7 @@ import { type Principal, type McpResource, } from "./seams"; -import type { BrowserApprovalStore } from "./tool-server"; +import type { BrowserApprovalStore, McpPassthroughUnavailableError } from "./tool-server"; // --------------------------------------------------------------------------- // In-process McpSessionStore — the single-node serving store, shared by every @@ -115,13 +118,17 @@ export interface McpBuildServerOptions { /** Whether this session serves the per-integration `search_` * tools. False unless the client connected with `?search_tools=true`. */ readonly searchToolsEnabled?: boolean; + /** The tool surface (`?mode=`): codemode (default) or passthrough. */ + readonly mode?: McpToolMode; + /** Passthrough only: the `?integrations=` filter. */ + readonly passthroughIntegrations?: readonly string[]; } /** Build the per-session `McpServer` + engine for a principal (the host's engine + tools). */ export type McpBuildServer = ( principal: Principal, options?: McpBuildServerOptions, -) => Effect.Effect; +) => Effect.Effect; export interface InMemoryMcpSessionStore { /** The `McpSessionStore` seam value to hand to `inMemoryMcpSessionsLayer`. */ @@ -395,15 +402,27 @@ export const makeInMemoryMcpSessionStore = ( request: Request, sessionId: () => string | null, ): McpBuildServerOptions => { - const artifactsEnabled = readArtifactsEnabled(request); + // `?artifacts=` is only forwarded when the URL spells it out, so the + // factory can apply the mode's own default (on for codemode, off for + // passthrough) to an absent value. + const artifactsEnabled = new URL(request.url).searchParams.has("artifacts") + ? readArtifactsEnabled(request) + : undefined; const searchToolsEnabled = readSearchToolsEnabled(request); + const toolMode = readToolMode(request); + const passthroughIntegrations = readPassthroughIntegrations(request); + const surface = { + ...(artifactsEnabled === undefined ? {} : { artifactsEnabled }), + searchToolsEnabled, + mode: toolMode, + ...(passthroughIntegrations ? { passthroughIntegrations } : {}), + }; const mode = readElicitationMode(request); if (mode !== "browser") { - return { artifactsEnabled, searchToolsEnabled, elicitationMode: { mode } }; + return { ...surface, elicitationMode: { mode } }; } return { - artifactsEnabled, - searchToolsEnabled, + ...surface, elicitationMode: { mode: "browser", // Prefer the pinned public origin; fall back to the request URL (correct @@ -468,9 +487,12 @@ export const makeInMemoryMcpSessionStore = ( }), ), // A build failure has nowhere typed to go in the envelope; render a 500. - Effect.catchTag("McpEngineBuildError", () => - Effect.succeed(jsonRpcError(500, -32603, "Internal server error")), - ), + Effect.catchTags({ + McpEngineBuildError: () => + Effect.succeed(jsonRpcError(500, -32603, "Internal server error")), + McpPassthroughUnavailableError: () => + Effect.succeed(jsonRpcError(500, -32603, "Internal server error")), + }), ); }; diff --git a/packages/hosts/mcp/src/passthrough-tools.test.ts b/packages/hosts/mcp/src/passthrough-tools.test.ts new file mode 100644 index 0000000000..6154763e3f --- /dev/null +++ b/packages/hosts/mcp/src/passthrough-tools.test.ts @@ -0,0 +1,485 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import type * as Cause from "effect/Cause"; + +import type { ExecutionEngine } from "@executor-js/execution"; +import { ToolAddress, type ToolProjection } from "@executor-js/sdk"; + +import { readPassthroughIntegrations, readToolMode } from "./browser-approval"; +import { + MAX_TOOL_NAME_LENGTH, + assignPassthroughNames, + passthroughAnnotations, + passthroughCallCode, + preferredToolName, +} from "./passthrough-tools"; +import { + createExecutorMcpServer, + McpPassthroughUnavailableError, + type ExecutorMcpServerConfig, +} from "./tool-server"; + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const projection = ( + input: Partial & { readonly integration: string; readonly name: string }, +): ToolProjection => { + const owner = input.owner ?? "org"; + const connection = input.connection ?? "main"; + return { + address: ToolAddress.make(`tools.${input.integration}.${owner}.${connection}.${input.name}`), + integration: input.integration, + owner, + connection, + name: input.name, + description: input.description ?? `${input.integration} ${input.name}`, + policy: input.policy ?? "approve", + ...(input.inputSchema === undefined ? {} : { inputSchema: input.inputSchema }), + ...(input.readOnly === undefined ? {} : { readOnly: input.readOnly }), + }; +}; + +/** A stub engine that records every executed code string and answers with a + * fixed value, so a test can prove a passthrough call became the expected + * single-call code and took `execute` (never `executeWithPause`). */ +const makeRecordingEngine = (result: unknown = { ok: true, data: { hello: "world" } }) => { + const executed: string[] = []; + let pausedCalls = 0; + const engine: ExecutionEngine = { + execute: (code) => + Effect.sync(() => { + executed.push(code); + return { result }; + }), + executeWithPause: () => + Effect.sync(() => { + pausedCalls += 1; + return { status: "completed" as const, result: { result } }; + }), + resume: () => Effect.succeed(null), + isExecutionSettled: undefined, + getPausedExecution: () => Effect.succeed(null), + pausedExecutionCount: () => Effect.succeed(0), + hasPausedExecutions: () => Effect.succeed(false), + getDescription: Effect.succeed("test executor"), + shutdown: Effect.void, + }; + return { engine, executed, pausedCalls: () => pausedCalls }; +}; + +const withClient = async ( + config: ExecutorMcpServerConfig, + fn: (client: Client) => Promise, +) => { + const mcpServer = await Effect.runPromise(createExecutorMcpServer(config)); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + const client = new Client({ name: "test-client", version: "1.0.0" }, { capabilities: {} }); + await mcpServer.connect(serverTransport); + await client.connect(clientTransport); + // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: test helper must close MCP transports after async client assertions + try { + await fn(client); + } finally { + await clientTransport.close(); + await serverTransport.close(); + } +}; + +const CATALOG: readonly ToolProjection[] = [ + projection({ + integration: "github", + name: "issues.create", + policy: "require_approval", + readOnly: false, + inputSchema: { + type: "object", + properties: { title: { type: "string" }, body: { $ref: "#/$defs/Body" } }, + required: ["title"], + $defs: { Body: { type: "string" } }, + }, + }), + projection({ integration: "github", name: "issues.list", readOnly: true }), + projection({ integration: "linear", name: "issueCreate", policy: "require_approval" }), +]; + +// --------------------------------------------------------------------------- +// Pure: naming +// --------------------------------------------------------------------------- + +describe("passthrough naming", () => { + it("joins integration and tool with a double underscore and flattens dots", () => { + expect( + preferredToolName( + { integration: "github", connection: "main", name: "issues.create" }, + false, + ), + ).toBe("github__issues_create"); + }); + + it("spells the connection out only when an integration has several", () => { + const named = assignPassthroughNames([ + projection({ integration: "vercel", connection: "personal", name: "domains.list" }), + projection({ integration: "vercel", connection: "work", name: "domains.list" }), + projection({ integration: "linear", name: "issueCreate" }), + ]); + expect(named.map((tool) => tool.name).sort()).toEqual([ + "linear__issueCreate", + "vercel__personal__domains_list", + "vercel__work__domains_list", + ]); + }); + + it("treats the same connection name under two owners as two connections", () => { + const named = assignPassthroughNames([ + projection({ integration: "slack", owner: "org", connection: "main", name: "post" }), + projection({ integration: "slack", owner: "user", connection: "main", name: "post" }), + ]); + // Both are `main`, so the connection segment alone would collide; the + // hash suffix keeps them apart rather than dropping one. + expect(new Set(named.map((tool) => tool.name)).size).toBe(2); + for (const tool of named) expect(tool.name.startsWith("slack__main__post")).toBe(true); + }); + + it("caps names at the provider limit with a stable hash suffix", () => { + const long = "a".repeat(80); + const [tool] = assignPassthroughNames([projection({ integration: "svc", name: long })]); + expect(tool!.name.length).toBeLessThanOrEqual(MAX_TOOL_NAME_LENGTH); + expect(tool!.name).toMatch(/^svc__a+-[0-9a-f]{7}$/); + // Stable across runs: the suffix is a function of the address. + const [again] = assignPassthroughNames([projection({ integration: "svc", name: long })]); + expect(again!.name).toBe(tool!.name); + }); + + it("keeps two long names that share a prefix distinct", () => { + const base = "operation_".repeat(8); + const named = assignPassthroughNames([ + projection({ integration: "svc", name: `${base}one` }), + projection({ integration: "svc", name: `${base}two` }), + ]); + expect(new Set(named.map((tool) => tool.name)).size).toBe(2); + }); + + it("is deterministic regardless of input order", () => { + const a = assignPassthroughNames(CATALOG).map((tool) => tool.name); + const b = assignPassthroughNames([...CATALOG].reverse()).map((tool) => tool.name); + expect(a).toEqual(b); + }); + + it("replaces every character outside the MCP grammar", () => { + const [tool] = assignPassthroughNames([ + projection({ integration: "my api", name: "get:thing/by id" }), + ]); + expect(tool!.name).toMatch(/^[A-Za-z0-9_-]+$/); + }); +}); + +// --------------------------------------------------------------------------- +// Pure: annotations + code +// --------------------------------------------------------------------------- + +describe("passthrough annotations", () => { + it("maps require_approval to destructiveHint and sets both hints explicitly", () => { + expect(passthroughAnnotations({ name: "x", policy: "require_approval" })).toEqual({ + title: "x", + readOnlyHint: false, + destructiveHint: true, + openWorldHint: true, + }); + expect(passthroughAnnotations({ name: "x", policy: "approve", readOnly: true })).toEqual({ + title: "x", + readOnlyHint: true, + destructiveHint: false, + openWorldHint: true, + }); + }); + + it("never advertises read-only for a tool no plugin vouched for", () => { + expect(passthroughAnnotations({ name: "x", policy: "approve" }).readOnlyHint).toBe(false); + }); + + it("emits exactly one awaited tool call with a JSON-literal argument", () => { + expect(passthroughCallCode("tools.github.org.main.issues.create", { title: "hi" })).toBe( + 'return await tools.github.org.main.issues.create({"title":"hi"});', + ); + expect(passthroughCallCode("linear.org.main.issueCreate", undefined)).toBe( + "return await tools.linear.org.main.issueCreate({});", + ); + }); +}); + +// --------------------------------------------------------------------------- +// Wire flags +// --------------------------------------------------------------------------- + +describe("readToolMode / readPassthroughIntegrations", () => { + const request = (query: string) => new Request(`https://example.test/mcp${query}`); + + it("defaults to codemode and only accepts the exact passthrough spelling", () => { + expect(readToolMode(request(""))).toBe("codemode"); + expect(readToolMode(request("?mode=passthrough"))).toBe("passthrough"); + expect(readToolMode(request("?mode=Passthrough"))).toBe("codemode"); + expect(readToolMode(request("?mode=direct"))).toBe("codemode"); + }); + + it("parses, trims and de-duplicates the integration filter", () => { + expect(readPassthroughIntegrations(request(""))).toBeUndefined(); + expect(readPassthroughIntegrations(request("?integrations="))).toBeUndefined(); + expect(readPassthroughIntegrations(request("?integrations=github,%20linear,github,"))).toEqual([ + "github", + "linear", + ]); + }); +}); + +// --------------------------------------------------------------------------- +// Server: the served surface +// --------------------------------------------------------------------------- + +describe("passthrough mode server", () => { + it("serves the catalog and none of the codemode tools", async () => { + const { engine } = makeRecordingEngine(); + await withClient( + { + engine, + mode: "passthrough", + searchToolsEnabled: true, + tools: { describeAll: () => Effect.succeed(CATALOG) }, + }, + async (client) => { + const listed = await client.listTools(); + const names = listed.tools.map((tool) => tool.name).sort(); + expect(names).toEqual([ + "github__issues_create", + "github__issues_list", + "linear__issueCreate", + ]); + // The fixed surface is gone — including the opt-in search tools. + expect(names).not.toContain("execute"); + expect(names).not.toContain("skills"); + expect(names).not.toContain("resume"); + expect(names.some((name) => name.startsWith("search_"))).toBe(false); + }, + ); + }); + + it("advertises policy as annotations and the stored JSON Schema verbatim", async () => { + const { engine } = makeRecordingEngine(); + await withClient( + { + engine, + mode: "passthrough", + tools: { describeAll: () => Effect.succeed(CATALOG) }, + }, + async (client) => { + const listed = await client.listTools(); + const create = listed.tools.find((tool) => tool.name === "github__issues_create"); + expect(create?.annotations).toEqual({ + title: "issues.create", + readOnlyHint: false, + destructiveHint: true, + openWorldHint: true, + }); + expect(create?.inputSchema).toEqual({ + type: "object", + properties: { title: { type: "string" }, body: { $ref: "#/$defs/Body" } }, + required: ["title"], + $defs: { Body: { type: "string" } }, + }); + const list = listed.tools.find((tool) => tool.name === "github__issues_list"); + expect(list?.annotations).toEqual({ + title: "issues.list", + readOnlyHint: true, + destructiveHint: false, + openWorldHint: true, + }); + // No declared input → the permissive empty object, so `{}` is callable. + expect(list?.inputSchema).toEqual({ type: "object", properties: {} }); + }, + ); + }); + + it("runs a call as one execute of single-call code, never a pause", async () => { + const recording = makeRecordingEngine({ ok: true, data: { number: 7 } }); + await withClient( + { + engine: recording.engine, + mode: "passthrough", + tools: { describeAll: () => Effect.succeed(CATALOG) }, + }, + async (client) => { + const result = await client.callTool({ + name: "github__issues_create", + arguments: { title: "hello" }, + }); + expect(recording.executed).toEqual([ + 'return await tools.github.org.main.issues.create({"title":"hello"});', + ]); + expect(recording.pausedCalls()).toBe(0); + // The tool's `data` is the result: nothing sits between the tool and + // the client to unwrap the `{ ok, data }` envelope for it. + expect(result.isError ?? false).toBe(false); + expect(result.structuredContent).toEqual({ + status: "completed", + result: { number: 7 }, + logs: [], + }); + }, + ); + }); + + it("surfaces an expected tool failure as an MCP error result", async () => { + const recording = makeRecordingEngine({ + ok: false, + error: { code: "tool_blocked", message: "Tool blocked by policy: github.org.main.x" }, + }); + await withClient( + { + engine: recording.engine, + mode: "passthrough", + tools: { describeAll: () => Effect.succeed(CATALOG) }, + }, + async (client) => { + const result = await client.callTool({ name: "github__issues_list", arguments: {} }); + expect(result.isError).toBe(true); + expect(result.structuredContent).toEqual({ + status: "error", + error: { code: "tool_blocked", message: "Tool blocked by policy: github.org.main.x" }, + logs: [], + }); + const text = (result.content as Array<{ type: string; text?: string }>)[0]?.text ?? ""; + expect(text).toContain("tool_blocked"); + }, + ); + }); + + it("accepts the approval gate inline but forwards a real input form", async () => { + // An engine whose tool raises an elicitation: first the executor's + // approval-only form (no fields), then a real form asking for a value. + const seen: string[] = []; + const engine: ExecutionEngine = { + ...makeRecordingEngine().engine, + execute: (_code, options) => + Effect.gen(function* () { + const approval = yield* options.onElicitation({ + address: CATALOG[0]!.address, + args: {}, + request: { + _tag: "FormElicitation", + message: "Approve?", + requestedSchema: { type: "object", properties: {} }, + }, + }); + seen.push(`approval:${approval.action}`); + const form = yield* options.onElicitation({ + address: CATALOG[0]!.address, + args: {}, + request: { + _tag: "FormElicitation", + message: "Which project?", + requestedSchema: { + type: "object", + properties: { project: { type: "string" } }, + required: ["project"], + }, + }, + }); + seen.push(`form:${form.action}`); + return { result: { ok: true, data: null } }; + }), + }; + await withClient( + { + engine, + mode: "passthrough", + tools: { describeAll: () => Effect.succeed(CATALOG) }, + }, + async (client) => { + await client.callTool({ name: "github__issues_create", arguments: { title: "x" } }); + // The test client advertises no elicitation capability, so the real + // form is declined rather than silently accepted with no data. + expect(seen).toEqual(["approval:accept", "form:decline"]); + }, + ); + }); + + it("rejects arguments that fail the advertised schema before running anything", async () => { + const recording = makeRecordingEngine(); + await withClient( + { + engine: recording.engine, + mode: "passthrough", + tools: { describeAll: () => Effect.succeed(CATALOG) }, + }, + async (client) => { + await expect( + client.callTool({ name: "github__issues_create", arguments: { body: "no title" } }), + ).rejects.toThrow(/Invalid arguments for tool github__issues_create/); + expect(recording.executed).toEqual([]); + }, + ); + }); + + it("answers an unknown tool name with a not-found error", async () => { + const { engine } = makeRecordingEngine(); + await withClient( + { + engine, + mode: "passthrough", + tools: { describeAll: () => Effect.succeed(CATALOG) }, + }, + async (client) => { + await expect(client.callTool({ name: "github__nope", arguments: {} })).rejects.toThrow( + /not found/, + ); + }, + ); + }); + + it("narrows to the requested integrations and says which were not connected", async () => { + const { engine } = makeRecordingEngine(); + await withClient( + { + engine, + mode: "passthrough", + passthroughIntegrations: ["linear", "notion"], + tools: { describeAll: () => Effect.succeed(CATALOG) }, + }, + async (client) => { + const listed = await client.listTools(); + expect(listed.tools.map((tool) => tool.name)).toEqual(["linear__issueCreate"]); + const instructions = client.getInstructions() ?? ""; + expect(instructions).toContain("1 integration tool"); + expect(instructions).toContain("Requested but not connected (no tools served): notion"); + }, + ); + }); + + it("leaves codemode untouched when the mode is absent", async () => { + const { engine } = makeRecordingEngine(); + await withClient( + { + engine, + description: "Execute TypeScript in a sandboxed runtime.", + tools: { describeAll: () => Effect.succeed(CATALOG) }, + }, + async (client) => { + const names = (await client.listTools()).tools.map((tool) => tool.name); + expect(names).toContain("execute"); + expect(names).toContain("skills"); + expect(names).not.toContain("github__issues_create"); + }, + ); + }); + + it("fails the build, not the session, when the host provides no catalog", async () => { + const { engine } = makeRecordingEngine(); + const outcome = await Effect.runPromise( + createExecutorMcpServer({ engine, mode: "passthrough" }).pipe(Effect.flip), + ); + expect(outcome).toBeInstanceOf(McpPassthroughUnavailableError); + }); +}); diff --git a/packages/hosts/mcp/src/passthrough-tools.ts b/packages/hosts/mcp/src/passthrough-tools.ts new file mode 100644 index 0000000000..3f64eff6bf --- /dev/null +++ b/packages/hosts/mcp/src/passthrough-tools.ts @@ -0,0 +1,231 @@ +// --------------------------------------------------------------------------- +// Passthrough mode — the pure half. +// +// `?mode=passthrough` serves every visible integration tool as its own MCP +// tool instead of the single `execute` codemode tool. This module owns the +// two decisions that make that surface deterministic and safe, with no I/O so +// they can be pinned by unit tests: +// +// 1. NAMING. An executor address (`tools.... +// `) does not fit the MCP tool-name grammar: the tool segment carries +// dots (`aliases.deleteAlias`) and can be long. Names are mangled to +// `__` (dots → `_`), and when an integration has more +// than one connection to `____`, so the +// surface stays fully transparent — there is never a hidden routing +// parameter. Anything over the length cap is truncated and suffixed with +// a short hash of the full address; a residual collision gets the same +// treatment. The map from MCP name back to address lives in the session, +// so the wire never carries an address the client chose. +// +// 2. ANNOTATIONS. Policy is evaluated ONCE, while the list is built, and +// surfaced as MCP `ToolAnnotations` for the harness's native approval: +// `require_approval` → `destructiveHint: true`, `approve` → `false`. Both +// `destructiveHint` and `readOnlyHint` are always set explicitly, because +// the MCP spec defaults an absent `destructiveHint` to TRUE — leaving it +// off would make every tool prompt. `block`ed tools never reach here. +// +// The consequence, stated plainly: in passthrough mode `require_approval` is +// advisory. A harness that auto-approves calls the tool and the server runs it. +// `block` stays enforced server-side, on both the list and the call. +// --------------------------------------------------------------------------- + +import type { ToolProjection } from "@executor-js/sdk"; + +/** The MCP tool-name grammar the SDK and every major client accept. */ +const TOOL_NAME_SAFE = /^[A-Za-z0-9_-]+$/; + +/** The Anthropic API caps tool names at 64 characters; other providers are + * looser, so this is the binding constraint. */ +export const MAX_TOOL_NAME_LENGTH = 64; + +/** Separator between the integration, the optional connection, and the tool. */ +const SEGMENT_SEPARATOR = "__"; + +/** Length of the hash suffix (`-` + hex) appended to a truncated or + * colliding name. Seven hex digits is plenty of headroom for a workspace + * catalog while leaving the human-readable prefix as long as possible. */ +const HASH_HEX_LENGTH = 7; + +/** MCP annotations a passthrough tool advertises. Mirrors the SDK's + * `ToolAnnotations` fields this surface sets, without importing its Zod type. */ +export interface PassthroughAnnotations { + readonly title: string; + readonly readOnlyHint: boolean; + readonly destructiveHint: boolean; + /** Integration tools reach external systems by definition. */ + readonly openWorldHint: true; +} + +export interface PassthroughTool { + /** The MCP tool name, unique within the session. */ + readonly name: string; + readonly projection: ToolProjection; + readonly annotations: PassthroughAnnotations; +} + +/** Replace every character outside the MCP name grammar with `_`. Dots in a + * tool name are structural (`group.leaf`), so they become `_` too rather than + * being dropped, keeping `aliases.deleteAlias` distinguishable from + * `aliasesdeleteAlias`. */ +const sanitizeSegment = (segment: string): string => + Array.from(segment, (char) => (TOOL_NAME_SAFE.test(char) ? char : "_")).join(""); + +/** FNV-1a over UTF-16 code units — stable, dependency-free, and good enough to + * spread a few thousand addresses across 28 bits. This is a disambiguator, + * not a security primitive. */ +const fnv1a = (input: string): number => { + let hash = 0x811c9dc5; + for (let i = 0; i < input.length; i++) { + hash ^= input.charCodeAt(i); + hash = Math.imul(hash, 0x01000193) >>> 0; + } + return hash >>> 0; +}; + +const hashSuffix = (address: string): string => + `-${fnv1a(address).toString(16).padStart(8, "0").slice(0, HASH_HEX_LENGTH)}`; + +/** Fit `name` under the cap by truncating and appending the address hash, so + * two long names that share a prefix still differ. */ +const fitWithHash = (name: string, address: string): string => { + const suffix = hashSuffix(address); + return `${name.slice(0, MAX_TOOL_NAME_LENGTH - suffix.length)}${suffix}`; +}; + +/** + * The preferred (pre-collision) MCP name for a projection. `multiConnection` + * says whether the integration has more than one visible connection in this + * session, which decides whether the connection segment is spelled out. + */ +export const preferredToolName = ( + projection: Pick, + multiConnection: boolean, +): string => { + const segments = multiConnection + ? [projection.integration, projection.connection, projection.name] + : [projection.integration, projection.name]; + return segments.map(sanitizeSegment).join(SEGMENT_SEPARATOR); +}; + +/** + * Map policy + plugin annotations onto MCP annotations. `readOnly` is a plugin + * fact (HTTP method, GraphQL kind, upstream hint); absent means unknown, which + * must read as `false` — advertising read-only for a tool nobody vouched for + * would let a harness skip a prompt it should show. + */ +export const passthroughAnnotations = ( + projection: Pick, +): PassthroughAnnotations => ({ + title: projection.name, + readOnlyHint: projection.readOnly === true, + destructiveHint: projection.policy === "require_approval", + openWorldHint: true, +}); + +/** + * Assign a unique MCP name to every projection. Deterministic for a given + * input order: the first claimant of a name keeps it, later ones get the hash + * suffix. Sorting the input by address first makes the assignment stable + * across sessions, which matters because a client caches tool names. + */ +export const assignPassthroughNames = ( + projections: readonly ToolProjection[], +): readonly PassthroughTool[] => { + const sorted = [...projections].sort((a, b) => + String(a.address) < String(b.address) ? -1 : String(a.address) > String(b.address) ? 1 : 0, + ); + + // Which integrations are served through more than one connection. + const connectionsByIntegration = new Map>(); + for (const projection of sorted) { + let set = connectionsByIntegration.get(projection.integration); + if (!set) { + set = new Set(); + connectionsByIntegration.set(projection.integration, set); + } + set.add(`${projection.owner}/${projection.connection}`); + } + + const taken = new Set(); + const out: PassthroughTool[] = []; + for (const projection of sorted) { + const multi = (connectionsByIntegration.get(projection.integration)?.size ?? 0) > 1; + const address = String(projection.address); + let name = preferredToolName(projection, multi); + if (name.length > MAX_TOOL_NAME_LENGTH || taken.has(name)) { + name = fitWithHash(name, address); + } + // A hash collision on top of a name collision is astronomically unlikely + // but not impossible; keep extending until unique rather than silently + // dropping a tool from the surface. + let salt = 0; + while (taken.has(name)) { + salt += 1; + name = fitWithHash(name, `${address}#${salt}`); + } + taken.add(name); + out.push({ name, projection, annotations: passthroughAnnotations(projection) }); + } + return out; +}; + +/** + * The sandbox code a passthrough call runs. Built HERE from the session's + * resolved address and a JSON-encoded argument — never concatenated from raw + * model input — and shaped exactly like the artifact `execute-action` grammar + * (`return await tools.()`), so it takes the same engine path as + * every other execution: billing, rate limits, shape memory and analytics all + * see it as one execution. + */ +export const passthroughCallCode = (address: string, args: unknown): string => { + const path = address.startsWith("tools.") ? address : `tools.${address}`; + return `return await ${path}(${JSON.stringify(args ?? {})});`; +}; + +/** + * Narrow a projection list to the requested integrations (`?integrations=`). + * Unknown slugs match nothing rather than failing the session: a client that + * names an integration the caller has not connected simply gets no tools for + * it, and the server instructions say so. + */ +export const filterPassthroughIntegrations = ( + projections: readonly ToolProjection[], + integrations: readonly string[] | undefined, +): readonly ToolProjection[] => { + if (!integrations || integrations.length === 0) return projections; + const wanted = new Set(integrations); + return projections.filter((projection) => wanted.has(projection.integration)); +}; + +/** + * The server `instructions` a passthrough session advertises. Names the count + * so a client that truncates its tool list at least sees why, and points at + * the filter for trimming it. + */ +export const passthroughInstructions = (input: { + readonly toolCount: number; + readonly integrations: readonly string[]; + readonly requested: readonly string[] | undefined; +}): string => { + const lines = [ + `This server exposes ${input.toolCount} integration tool${input.toolCount === 1 ? "" : "s"} directly (passthrough mode).`, + "Each tool is named __, or ____ when an integration has several connections.", + "Tools marked destructiveHint require the user's approval in your client before you call them; readOnlyHint marks tools that never mutate upstream state.", + ]; + if (input.integrations.length > 0) { + lines.push(`Integrations served: ${input.integrations.join(", ")}.`); + } + if (input.requested && input.requested.length > 0) { + const missing = input.requested.filter((slug) => !input.integrations.includes(slug)); + if (missing.length > 0) { + lines.push( + `Requested but not connected (no tools served): ${missing.join(", ")}. Connect them in the Executor console.`, + ); + } + } else { + lines.push( + "To narrow this list, connect with ?integrations=, on the endpoint URL.", + ); + } + return lines.join("\n"); +}; diff --git a/packages/hosts/mcp/src/tool-server.ts b/packages/hosts/mcp/src/tool-server.ts index 730c5bfd89..4a4fa5f00c 100644 --- a/packages/hosts/mcp/src/tool-server.ts +++ b/packages/hosts/mcp/src/tool-server.ts @@ -2,7 +2,11 @@ import { Data, Duration, Effect, Match, Option, Predicate, Result, Schema } from import * as Cause from "effect/Cause"; import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { + CallToolRequestSchema, ContentBlockSchema, + ErrorCode, + ListToolsRequestSchema, + McpError, type ClientCapabilities, type ContentBlock, } from "@modelcontextprotocol/sdk/types.js"; @@ -23,6 +27,7 @@ import * as z from "zod/v4"; import { CurrentOrgWriteAccess, isToolFile, + isToolResult, makeOrgWriteAccessState, sanitizeArtifactPreviewMarkup, type OrgWriteAccess, @@ -35,8 +40,11 @@ import type { ElicitationHandler, ElicitationContext, ElicitationRequest, + FormElicitation, SaveArtifactInput, ToolFileValue, + ToolListFilter, + ToolProjection, } from "@executor-js/sdk"; import type * as Tracer from "effect/Tracer"; import { @@ -74,6 +82,14 @@ import { type BindableConnection, } from "./artifact-bindings"; import { MCP_ORG_WRITE_ACCESS_HEADER } from "./seams"; +import { + assignPassthroughNames, + filterPassthroughIntegrations, + passthroughCallCode, + passthroughInstructions, + type PassthroughTool, +} from "./passthrough-tools"; +import type { McpToolMode } from "./browser-approval"; // --------------------------------------------------------------------------- // Workers-compatible JSON Schema validator (replaces Ajv which uses new Function()) @@ -183,6 +199,27 @@ type SharedMcpServerConfig = { * results match what code-side search returns. */ readonly searchToolsEnabled?: boolean; + /** + * The tool surface this connection serves. `codemode` (the default) is the + * `execute` tool plus `skills`/`resume` and the artifact surface. + * `passthrough` (`?mode=passthrough`) registers every visible integration + * tool as its own MCP tool and serves NONE of `execute`, `skills`, or + * `resume`: policy is folded into each tool's annotations at list time and + * the client's own approval flow takes it from there. Requires `tools`. + */ + readonly mode?: McpToolMode; + /** + * Passthrough only: restrict the served tools to these integration slugs + * (`?integrations=a,b`). Absent means every visible integration. + */ + readonly passthroughIntegrations?: readonly string[]; + /** + * The scoped executor's tool catalog, for passthrough mode. Structurally + * satisfied by `executor.tools`. Hosts that never serve passthrough may + * leave it unset; a passthrough session without it fails at build time + * rather than silently serving an empty surface. + */ + readonly tools?: McpToolsPort; /** * Renders an artifact once, server-side, before it is saved — so a component * that throws on its first render is refused at create time with the real @@ -272,6 +309,21 @@ export type McpConnectionsPort = { readonly list: () => Effect.Effect; }; +/** The catalog read passthrough mode needs: every visible tool with its + * self-contained schema and resolved policy, in one pass. Structurally + * satisfied by `Executor["tools"]`. */ +export type McpToolsPort = { + readonly describeAll: ( + filter?: ToolListFilter, + ) => Effect.Effect; +}; + +/** A passthrough session was requested but the host gave the factory no + * catalog to serve. A configuration defect, not a runtime condition. */ +export class McpPassthroughUnavailableError extends Data.TaggedError( + "McpPassthroughUnavailableError", +)<{ readonly reason: string }> {} + export type ExecutorMcpServerConfig = | (ExecutionEngineConfig & SharedMcpServerConfig) | ({ readonly engine: ExecutionEngine } & SharedMcpServerConfig) @@ -363,6 +415,17 @@ const elicitationRequestTag = (request: ElicitationRequest): ElicitationRequest[ Match.exhaustive, ); +/** The executor's approval gate asks for consent with a form that collects + * nothing: `{ type: "object", properties: {} }` (or a bare `{}`). Anything + * with a declared field is a tool asking the user for input. */ +const isApprovalOnlyForm = (request: FormElicitation): boolean => { + const schema = request.requestedSchema; + const properties = schema.properties; + const declaresFields = isRecord(properties) && Object.keys(properties).length > 0; + const requiresFields = Array.isArray(schema.required) && schema.required.length > 0; + return !declaresFields && !requiresFields; +}; + const requestedSchemaIsNonEmpty = (request: ElicitationRequest): boolean => Match.value(request).pipe( Match.tag("FormElicitation", (req) => Object.keys(req.requestedSchema).length > 0), @@ -638,6 +701,32 @@ const toMcpResult = (result: FormattedExecuteInput): McpToolResult => { }; }; +/** + * A passthrough call's result IS the tool's `ToolResult`. Inside `execute` + * the model reads `{ ok, data | error }` and branches; here nothing runs + * between the tool and the client, so an expected failure (`ok: false` — a + * 4xx wall, a blocked policy, a validation miss) has to be an MCP error + * result, and a success unwraps to the tool's `data`. Everything else + * (sandbox error, emitted output) keeps the codemode rendering. + */ +const toPassthroughResult = (outcome: FormattedExecuteInput): McpToolResult => { + const value = outcome.result; + if (outcome.error || !isToolResult(value)) return toMcpResult(outcome); + if (value.ok) { + return toMcpResult({ ...outcome, result: value.data }); + } + const message = `${value.error.code}: ${value.error.message}`; + return { + content: [{ type: "text", text: `Error: ${message}` }], + structuredContent: { + status: "error", + error: value.error, + logs: outcome.logs ?? [], + }, + isError: true, + }; +}; + const toMcpPausedResult = (formatted: ReturnType): McpToolResult => ({ content: [{ type: "text", text: formatted.text }], structuredContent: formatted.structured, @@ -1104,13 +1193,132 @@ const parseJsonContent = (raw: string): Record | undefined => { return Option.isSome(parsed) ? parsed.value : undefined; }; +// --------------------------------------------------------------------------- +// Passthrough surface +// --------------------------------------------------------------------------- + +/** Read the catalog once and assign every visible tool its MCP name. */ +const loadPassthroughTools = ( + config: SharedMcpServerConfig, +): Effect.Effect => + Effect.gen(function* () { + const port = config.tools; + if (!port) { + return yield* new McpPassthroughUnavailableError({ + reason: "passthrough mode requested but the host provided no tool catalog", + }); + } + const projections = yield* port.describeAll().pipe( + Effect.mapError( + (cause) => + new McpPassthroughUnavailableError({ + reason: `tool catalog read failed: ${formatBoundaryError(cause).message}`, + }), + ), + ); + const scoped = filterPassthroughIntegrations(projections, config.passthroughIntegrations); + return assignPassthroughNames(scoped); + }).pipe(Effect.withSpan("mcp.host.passthrough.load")); + +/** An input schema the SDK's validator accepts. Tools that declare none get + * the permissive empty object, so a client can still call them with `{}`. */ +const passthroughInputSchema = (projection: ToolProjection): Record => { + const schema = projection.inputSchema; + if (schema && typeof schema === "object" && !Array.isArray(schema)) { + return schema as Record; + } + return { type: "object", properties: {} }; +}; + +/** + * Register one MCP tool per passthrough entry. The SDK's `registerTool` + * only accepts Zod schemas, but its low-level `tools/list` and `tools/call` + * handlers are the whole contract, so the catalog is served by installing + * those two handlers directly: the JSON Schema the plugin stored goes on the + * wire verbatim (no Zod round-trip, no schema loss), and arguments are + * validated against it with the same validator the SDK is configured with. + */ +const registerPassthroughTools = ( + server: McpServer, + tools: readonly PassthroughTool[], + run: ( + tool: PassthroughTool, + args: unknown, + extra: McpRequestJoinKeys, + ) => Effect.Effect, +): Effect.Effect => + Effect.gen(function* () { + const context = yield* Effect.context(); + const byName = new Map(tools.map((tool) => [tool.name, tool] as const)); + const validator = new CfWorkerJsonSchemaValidator(); + const validators = new Map>(); + const validate = (tool: PassthroughTool, args: unknown) => { + let fn = validators.get(tool.name); + if (!fn) { + fn = validator.getValidator( + passthroughInputSchema(tool.projection) as JsonSchemaType, + ); + validators.set(tool.name, fn); + } + return fn(args); + }; + + yield* Effect.sync(() => { + server.server.registerCapabilities({ tools: { listChanged: true } }); + server.server.setRequestHandler(ListToolsRequestSchema, () => ({ + tools: tools.map((tool) => ({ + name: tool.name, + title: tool.annotations.title, + description: tool.projection.description, + inputSchema: passthroughInputSchema(tool.projection) as { + type: "object"; + [key: string]: unknown; + }, + annotations: tool.annotations, + })), + })); + server.server.setRequestHandler(CallToolRequestSchema, async (request, extra) => { + const tool = byName.get(request.params.name); + if (!tool) { + // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: the MCP SDK's request handler contract renders a thrown McpError as the JSON-RPC error the client expects (same as its own registerTool path) + throw new McpError(ErrorCode.InvalidParams, `Tool ${request.params.name} not found`); + } + const args = request.params.arguments ?? {}; + const checked = validate(tool, args); + if (!checked.valid) { + // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: same JSON-RPC error contract; the message shape matches the SDK's own input-validation error + throw new McpError( + ErrorCode.InvalidParams, + `Input validation error: Invalid arguments for tool ${tool.name}: ${checked.errorMessage ?? "invalid"}`, + ); + } + return Effect.runPromiseWith(context)( + run(tool, args, extra).pipe( + Effect.provideService( + CurrentOrgWriteAccess, + makeOrgWriteAccessState(requestOrgWriteAccess(extra)), + ), + Effect.catchCause((cause) => Effect.succeed(toMcpFailureResult(cause))), + ), + ); + }); + }); + }).pipe( + Effect.withSpan("mcp.host.register_tool", { + attributes: { + "mcp.tool.name": "", + "mcp.passthrough.count": tools.length, + }, + }), + ); + // --------------------------------------------------------------------------- // Server factory // --------------------------------------------------------------------------- export const createExecutorMcpServer = ( config: ExecutorMcpServerConfig, -): Effect.Effect => +): Effect.Effect => Effect.gen(function* () { const engine = "engine" in config ? config.engine : createExecutionEngine(config); const description = @@ -1122,11 +1330,26 @@ export const createExecutorMcpServer = ( // Artifacts are on unless this connection opted out (`?artifacts=false`). // One flag decides the whole surface: the tools, the shell resource, and // the skills catalog below. - const artifactsEnabled = config.artifactsEnabled ?? true; + // Passthrough is a plain tool surface: artifacts are OFF there unless the + // connection spelled out `?artifacts=true`, the reverse of codemode's + // default. `config.artifactsEnabled` is what the host read off the URL, so + // an explicit true survives; an absent value takes the mode's default. + const artifactsEnabled = + config.artifactsEnabled ?? (config.mode === "passthrough" ? false : true); const skillCatalog: readonly Skill[] = skillCatalogFor({ artifacts: artifactsEnabled }); // Per-integration search tools are off unless this connection opted in // (`?search_tools=true`). const searchToolsEnabled = config.searchToolsEnabled ?? false; + // Passthrough (`?mode=passthrough`) replaces the codemode surface + // wholesale. The flag is read once here and every codemode-only + // registration below is gated on it, so the two surfaces cannot leak into + // each other. + const mode: McpToolMode = config.mode ?? "codemode"; + const passthrough = mode === "passthrough"; + // Built before the server exists so the instructions can name the count. + const passthroughTools: readonly PassthroughTool[] = passthrough + ? yield* loadPassthroughTools(config) + : []; // Captured at construction time. SDK callbacks fire later (often // deferred past the outer Effect's await), so we use the runtime to @@ -1227,6 +1450,17 @@ export const createExecutorMcpServer = ( // per host. capabilities: { resources: {}, tools: {} }, jsonSchemaValidator: new CfWorkerJsonSchemaValidator(), + ...(passthrough + ? { + instructions: passthroughInstructions({ + toolCount: passthroughTools.length, + integrations: Array.from( + new Set(passthroughTools.map((tool) => tool.projection.integration)), + ).sort(), + requested: config.passthroughIntegrations, + }), + } + : {}), }, ), ).pipe(Effect.withSpan("mcp.host.create_server")); @@ -1545,103 +1779,172 @@ export const createExecutorMcpServer = ( Effect.annotateSpans(joinKeyAttributes(extra)), ); + // --- passthrough call path --- + // + // One passthrough tool call is ONE execution of synthesized single-call + // code, through the same `engine.execute` every other execution takes — + // so billing, rate limits, shape memory and analytics see nothing new. + // It never pauses: there is no `resume` in this mode. Approval was already + // decided by the harness from the advertised annotations, so an approval + // form elicitation is accepted inline; a URL elicitation (reauth, OAuth) + // has nowhere to go and becomes a failed result carrying the URL, for the + // model to relay and retry after. + const executePassthroughCall = ( + tool: PassthroughTool, + args: unknown, + extra: McpRequestJoinKeys, + ): Effect.Effect => + Effect.gen(function* () { + const address = String(tool.projection.address); + yield* startMarker("mcp.host.tool.execute.start", { + "mcp.tool.name": tool.name, + "mcp.tool.mode": "passthrough", + "executor.tool.address": address, + }); + const { url: supportsUrl } = getElicitationSupport(server); + const native = makeMcpElicitationHandler(server, extra.requestId, debugLog); + const onElicitation: ElicitationHandler = (ctx) => + Match.value(ctx.request).pipe( + // The harness already prompted (or chose not to) from the + // annotations; a second server-side gate would double-prompt. + Match.tag("FormElicitation", (req) => + isApprovalOnlyForm(req) + ? Effect.succeed({ action: "accept" as const, content: {} }) + : // A real form (a tool asking for input) still needs a human. + // Forward it natively when the client can take it; otherwise + // decline, which the invoker turns into a clear failure. + getElicitationSupport(server).form + ? native(ctx) + : Effect.succeed({ action: "decline" as const }), + ), + Match.tag("UrlElicitation", () => + supportsUrl ? native(ctx) : Effect.succeed({ action: "decline" as const }), + ), + Match.exhaustive, + ); + const outcome = yield* engine.execute(passthroughCallCode(address, args), { + onElicitation, + }); + return toPassthroughResult(outcome); + }).pipe( + Effect.withSpan("mcp.host.tool.execute", { + attributes: { + "mcp.tool.name": tool.name, + "mcp.tool.mode": "passthrough", + "executor.integration": tool.projection.integration, + }, + }), + Effect.annotateSpans(joinKeyAttributes(extra)), + ); + // --- tools --- - yield* Effect.sync(() => - server.registerTool( - "execute", - { - description, - inputSchema: { code: z.string().trim().min(1) }, - }, - ({ code }, extra) => runToolEffect(executeCode(code, extra), extra), - ), - ).pipe( - Effect.withSpan("mcp.host.register_tool", { - attributes: { "mcp.tool.name": "execute" }, - }), - ); + // Passthrough serves the catalog itself in place of everything below. + if (passthrough) { + yield* registerPassthroughTools(server, passthroughTools, executePassthroughCall); + } - yield* Effect.sync(() => - server.registerTool( - "skills", - { - description: [ - "Documentation for THIS server's own tools. Not a general skill reader: it serves a short, fixed set of how-to docs about using `execute` and artifacts here, and it cannot reach your harness's skills, a SKILL.md on disk, or any user- or project-authored skill. The argument is a name from its own catalog, never a path or an outside skill's id.", - "These docs hold the long-form guidance that would otherwise bloat another tool's always-loaded description.", - 'Call `skills({ name: "execute" })` for the full guide to writing code for the `execute` tool (search the catalog, call tools, emit results, resume paused runs).', - "Call with no name to list the few docs available.", - ].join("\n"), - inputSchema: { - name: z - .string() - .optional() - .describe( - 'A doc from this server\'s own catalog, e.g. "execute" — not a path or an outside skill name. Omit to list the catalog.', - ), + if (!passthrough) + yield* Effect.sync(() => + server.registerTool( + "execute", + { + description, + inputSchema: { code: z.string().trim().min(1) }, }, - }, - ({ name }, extra) => - runToolEffect(Effect.succeed(skillsResult(name, executeInventory, skillCatalog)), extra), - ), - ).pipe( - Effect.withSpan("mcp.host.register_tool", { - attributes: { "mcp.tool.name": "skills" }, - }), - ); - - yield* Effect.sync(() => { - if (elicitationMode.mode === "native") { - return undefined; - } + ({ code }, extra) => runToolEffect(executeCode(code, extra), extra), + ), + ).pipe( + Effect.withSpan("mcp.host.register_tool", { + attributes: { "mcp.tool.name": "execute" }, + }), + ); - if (elicitationMode.mode === "model") { - return server.registerTool( - "resume", + if (!passthrough) + yield* Effect.sync(() => + server.registerTool( + "skills", { description: [ - "Resume a paused execution using the executionId returned by execute.", - "This connection explicitly allows model-side resume via elicitation_mode=model.", + "Documentation for THIS server's own tools. Not a general skill reader: it serves a short, fixed set of how-to docs about using `execute` and artifacts here, and it cannot reach your harness's skills, a SKILL.md on disk, or any user- or project-authored skill. The argument is a name from its own catalog, never a path or an outside skill's id.", + "These docs hold the long-form guidance that would otherwise bloat another tool's always-loaded description.", + 'Call `skills({ name: "execute" })` for the full guide to writing code for the `execute` tool (search the catalog, call tools, emit results, resume paused runs).', + "Call with no name to list the few docs available.", ].join("\n"), inputSchema: { - executionId: z.string().describe("The execution ID from the paused result"), - action: z - .enum(["accept", "decline", "cancel"]) - .describe("How to respond to the interaction"), - content: z + name: z .string() - .describe("Optional JSON-encoded response content for form elicitations") - .default("{}"), + .optional() + .describe( + 'A doc from this server\'s own catalog, e.g. "execute" — not a path or an outside skill name. Omit to list the catalog.', + ), }, }, - ({ executionId, action, content: rawContent }, extra) => + ({ name }, extra) => runToolEffect( - resumeExecution(executionId, action, parseJsonContent(rawContent), extra), + Effect.succeed(skillsResult(name, executeInventory, skillCatalog)), extra, ), - ); - } + ), + ).pipe( + Effect.withSpan("mcp.host.register_tool", { + attributes: { "mcp.tool.name": "skills" }, + }), + ); - return server.registerTool( - "resume", - { - description: [ - "Request user approval to resume a paused execution.", - "Call this with the executionId returned by execute. If the user has not approved in the browser yet, tell them to open the returned approval URL. If they have approved, this returns the resumed execution result.", - "This connection does not allow the model to choose accept, decline, cancel, or content.", - ].join("\n"), - inputSchema: { - executionId: z.string().describe("The execution ID from the paused result"), + if (!passthrough) + yield* Effect.sync(() => { + if (elicitationMode.mode === "native") { + return undefined; + } + + if (elicitationMode.mode === "model") { + return server.registerTool( + "resume", + { + description: [ + "Resume a paused execution using the executionId returned by execute.", + "This connection explicitly allows model-side resume via elicitation_mode=model.", + ].join("\n"), + inputSchema: { + executionId: z.string().describe("The execution ID from the paused result"), + action: z + .enum(["accept", "decline", "cancel"]) + .describe("How to respond to the interaction"), + content: z + .string() + .describe("Optional JSON-encoded response content for form elicitations") + .default("{}"), + }, + }, + ({ executionId, action, content: rawContent }, extra) => + runToolEffect( + resumeExecution(executionId, action, parseJsonContent(rawContent), extra), + extra, + ), + ); + } + + return server.registerTool( + "resume", + { + description: [ + "Request user approval to resume a paused execution.", + "Call this with the executionId returned by execute. If the user has not approved in the browser yet, tell them to open the returned approval URL. If they have approved, this returns the resumed execution result.", + "This connection does not allow the model to choose accept, decline, cancel, or content.", + ].join("\n"), + inputSchema: { + executionId: z.string().describe("The execution ID from the paused result"), + }, }, - }, - ({ executionId }, extra) => - runToolEffect(resumeAfterBrowserApproval(executionId, extra), extra), + ({ executionId }, extra) => + runToolEffect(resumeAfterBrowserApproval(executionId, extra), extra), + ); + }).pipe( + Effect.withSpan("mcp.host.register_tool", { + attributes: { "mcp.tool.name": "resume" }, + }), ); - }).pipe( - Effect.withSpan("mcp.host.register_tool", { - attributes: { "mcp.tool.name": "resume" }, - }), - ); // --- per-integration search tools (opt-in, `?search_tools=true`) --- // @@ -1659,7 +1962,7 @@ export const createExecutorMcpServer = ( // would only repeat the name) and a single bare `query` parameter — no // paging knobs, because anything past the first page belongs in `execute`. // `namespace-search-tools.test.ts` pins the serialized size. - if (searchToolsEnabled) { + if (searchToolsEnabled && !passthrough) { // The MCP tool-name grammar ([A-Za-z0-9_-]). Integration slugs already // conform (they are `tools.` property names in sandbox code); one // that somehow doesn't is skipped rather than failing the whole session. diff --git a/packages/plugins/graphql/src/sdk/plugin.ts b/packages/plugins/graphql/src/sdk/plugin.ts index 176e6d7721..74196278c3 100644 --- a/packages/plugins/graphql/src/sdk/plugin.ts +++ b/packages/plugins/graphql/src/sdk/plugin.ts @@ -570,6 +570,7 @@ const annotationsFor = (binding: OperationBinding): ToolAnnotations => { return { requiresApproval: true, approvalDescription: `mutation ${binding.fieldName}`, + readOnly: false, }; } return {}; diff --git a/packages/plugins/mcp/src/sdk/plugin.ts b/packages/plugins/mcp/src/sdk/plugin.ts index 4af0c4bf58..77aafdb628 100644 --- a/packages/plugins/mcp/src/sdk/plugin.ts +++ b/packages/plugins/mcp/src/sdk/plugin.ts @@ -543,9 +543,13 @@ const toToolDef = (entry: McpToolManifestEntry): ToolDef => { ...(entry.annotations ? { upstream: entry.annotations } : {}), ...(entry._meta ? { _meta: entry._meta } : {}), }; + const readOnly = entry.annotations?.readOnlyHint; const annotations: StampedAnnotations = { requiresApproval: destructive, ...(destructive ? { approvalDescription: entry.annotations?.title ?? entry.toolName } : {}), + // Carry the upstream read-only hint through verbatim: a passthrough MCP + // surface re-advertises it, and only the upstream server knows. + ...(typeof readOnly === "boolean" ? { readOnly } : {}), mcp: stamp, }; return { diff --git a/packages/plugins/openapi/src/sdk/invoke.ts b/packages/plugins/openapi/src/sdk/invoke.ts index 148b4a052a..271e8cc877 100644 --- a/packages/plugins/openapi/src/sdk/invoke.ts +++ b/packages/plugins/openapi/src/sdk/invoke.ts @@ -1427,11 +1427,15 @@ export const REQUIRE_APPROVAL = new Set(["post", "put", "patch", "delete"]); export const annotationsForOperation = ( method: string, pathTemplate: string, -): { requiresApproval?: boolean; approvalDescription?: string } => { +): { requiresApproval?: boolean; approvalDescription?: string; readOnly?: boolean } => { const m = method.toLowerCase(); - if (!REQUIRE_APPROVAL.has(m)) return {}; + // A safe method (GET/HEAD/OPTIONS) is read-only by HTTP semantics; a + // passthrough MCP surface advertises that as `readOnlyHint` so a harness can + // skip its own confirmation for it. + if (!REQUIRE_APPROVAL.has(m)) return { readOnly: true }; return { requiresApproval: true, approvalDescription: `${method.toUpperCase()} ${pathTemplate}`, + readOnly: false, }; }; diff --git a/packages/react/src/api/analytics.tsx b/packages/react/src/api/analytics.tsx index 465e7b88c3..c637b59bd6 100644 --- a/packages/react/src/api/analytics.tsx +++ b/packages/react/src/api/analytics.tsx @@ -169,6 +169,7 @@ export interface AnalyticsEvents { mcp_install_elicitation_mode_changed: { elicitation_mode: string }; mcp_install_artifacts_toggled: { artifacts: boolean }; mcp_install_search_tools_toggled: { search_tools: boolean }; + mcp_install_tool_mode_changed: { tool_mode: "codemode" | "passthrough" }; // ── Command palette ────────────────────────────────────────────────────── command_palette_navigated: { diff --git a/packages/react/src/components/mcp-install-card.tsx b/packages/react/src/components/mcp-install-card.tsx index cbefa82644..4a169c97e4 100644 --- a/packages/react/src/components/mcp-install-card.tsx +++ b/packages/react/src/components/mcp-install-card.tsx @@ -26,6 +26,9 @@ const McpInstallPreferencesSchema = Schema.Struct({ httpElicitationMode: Schema.Literals(["browser", "model", "native"]), artifacts: Schema.Boolean, searchTools: Schema.Boolean, + // Added after v1 shipped; optional so a stored preference from before it + // existed still decodes and takes the default (codemode). + toolMode: Schema.optional(Schema.Literals(["codemode", "passthrough"])), }); type McpInstallPreferences = typeof McpInstallPreferencesSchema.Type; @@ -52,6 +55,7 @@ const DEFAULT_MCP_INSTALL_PREFERENCES: McpInstallPreferences = { httpElicitationMode: "model", artifacts: true, searchTools: false, + toolMode: "codemode", }; const decodeMcpInstallPreferences = Schema.decodeUnknownOption( Schema.fromJsonString(McpInstallPreferencesSchema), @@ -116,6 +120,9 @@ export const buildMcpHttpEndpoint = (input: { /** Per-integration search tools are off by default, so only the opt-in is * spelled out on the URL (`&search_tools=true`). */ readonly searchTools?: boolean; + /** Codemode is the default, so only passthrough is spelled out on the URL + * (`&mode=passthrough`). */ + readonly toolMode?: "codemode" | "passthrough"; // Cloud only: pins the URL to `//mcp` (the server also accepts the // legacy `//mcp` form). Desktop/local pass nothing and get the bare // `/mcp` path. @@ -138,6 +145,7 @@ export const buildMcpHttpEndpoint = (input: { } if (input.artifacts === false) params.push(["artifacts", "false"]); if (input.searchTools === true) params.push(["search_tools", "true"]); + if (input.toolMode === "passthrough") params.push(["mode", "passthrough"]); if (params.length === 0) return endpoint; const query = params.map(([key, value]) => `${key}=${value}`).join("&"); @@ -160,6 +168,7 @@ export const buildMcpInstallCommand = (input: { readonly elicitationMode?: McpElicitationMode; readonly artifacts?: boolean; readonly searchTools?: boolean; + readonly toolMode?: "codemode" | "passthrough"; readonly devCliCwd?: string; readonly organizationSlug?: string | null; }): string => { @@ -170,6 +179,7 @@ export const buildMcpInstallCommand = (input: { elicitationMode: input.elicitationMode, artifacts: input.artifacts, searchTools: input.searchTools, + toolMode: input.toolMode, organizationSlug: input.organizationSlug, }); const headerFlags: string[] = []; @@ -200,6 +210,9 @@ export const buildMcpInstallCommand = (input: { if (input.searchTools === true) { innerArgs.push("--search-tools"); } + if (input.toolMode === "passthrough") { + innerArgs.push("--mode", "passthrough"); + } return `npx add-mcp ${shellQuoteWord(innerArgs.map(shellQuoteWord).join(" "))} --name executor`; }; @@ -227,6 +240,7 @@ export function McpInstallCard(props: { className?: string }) { } const [advancedOpen, setAdvancedOpen] = useState(false); const { mode, httpElicitationMode, artifacts, searchTools } = preferences; + const toolMode = preferences.toolMode ?? "codemode"; useEffect(() => { writeMcpInstallPreferences(storageKey, preferences); @@ -268,6 +282,7 @@ export function McpInstallCard(props: { className?: string }) { elicitationMode, artifacts, searchTools, + toolMode, devCliCwd, organizationSlug, }); @@ -290,6 +305,25 @@ export function McpInstallCard(props: { className?: string }) {
+
+
Expose tools directly
+
+ {toolMode === "passthrough" + ? "Every connected tool is its own MCP tool. Your client handles approvals from each tool's annotations; there is no execute or resume." + : "Disabled: agents write code against your tools through one execute tool."} +
+
+ { + const nextMode = next ? "passthrough" : "codemode"; + setPreferences((current) => ({ ...current, toolMode: nextMode })); + trackEvent("mcp_install_tool_mode_changed", { tool_mode: nextMode }); + }} + aria-label="Expose tools directly" + /> +
+
Artifacts
From d64685c393e56ebd597c7581de40ebb20d16bc55 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 3 Sep 2026 23:10:19 -0700 Subject: [PATCH 02/19] Add OpenCode codemode passthrough scenario at 10,200 tools --- .../passthrough-opencode-codemode.test.ts | 346 ++++++++++++++++++ 1 file changed, 346 insertions(+) create mode 100644 e2e/cloud/passthrough-opencode-codemode.test.ts diff --git a/e2e/cloud/passthrough-opencode-codemode.test.ts b/e2e/cloud/passthrough-opencode-codemode.test.ts new file mode 100644 index 0000000000..2d26db5318 --- /dev/null +++ b/e2e/cloud/passthrough-opencode-codemode.test.ts @@ -0,0 +1,346 @@ +// Cloud: the REAL OpenCode binary, with ITS OWN codemode on, over an Executor +// passthrough endpoint serving 10,200 tools. +// +// This is the pairing passthrough exists for. Executor stops being the +// codemode layer and becomes a plain MCP server that advertises every tool; +// OpenCode's experimental codemode (`OPENCODE_EXPERIMENTAL_CODE_MODE`) takes +// the whole catalog into its confined interpreter and shows the model ONE +// `execute` tool with a token-budgeted catalog plus `tools.$codemode.search`. +// So the things to prove are on both sides of the wire: +// +// - Executor serves all 10,200 tools to OpenCode within OpenCode's connect +// timeout (30s), so `mcp list` reads "connected" and codemode has a +// catalog to work with. +// - OpenCode does NOT flatten those 10,200 tools into the model's tool list: +// the replay brain is offered `execute`, not 10,200 functions. +// - Discovery works end to end: a program that calls +// `tools.$codemode.search` finds an Executor tool by name, and a second +// program calls it, which lands on a real upstream with the connection's +// credential and returns the payload through OpenCode's interpreter. +// +// The catalog is seeded over the public API as a user would build it (the +// large-catalog seeder, sized to 10,198 tools, plus a 2-operation API backed by +// a recording upstream). The model is the replay brain (scripted turns, real +// agent); OpenCode's OAuth, MCP connect, tool listing and codemode run are all +// its own code. The whole terminal session is recorded to terminal.cast. +import { randomBytes } from "node:crypto"; +import { createServer } from "node:http"; +import { join } from "node:path"; + +import { expect } from "@effect/vitest"; +import { Effect } from "effect"; +import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared"; + +import { serveReplayBrain } from "../src/clients/replay-brain"; +import { scenario } from "../src/scenario"; +import { Api, Cli, OpenCode, RunDir, Target } from "../src/services"; +import { catalogApi, seedLargeCatalog } from "../scenarios/support/large-catalog"; + +const SERVER_NAME = "executor"; + +// 322 (the real Vercel fixture) + 12 × 823 synthetic = 10,198, plus the two +// callable operations below = exactly 10,200 tools on the wire. +const SYNTHETIC_INTEGRATIONS = 12; +const OPS_PER_INTEGRATION = 823; +const EXPECTED_TOOL_COUNT = 10_200; + +// OpenCode's MCP connect timeout (`DEFAULT_TIMEOUT` in its mcp service). The +// tools/list of 10,200 definitions has to fit inside it, or the server reads +// "failed" and codemode sees nothing. +const OPENCODE_CONNECT_TIMEOUT_MS = 30_000; + +const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`; + +/** The one integration that is actually callable: a read and a write against + * the recording upstream. Everything else in the catalog is discovery mass. */ +const notesSpec = (baseUrl: string): string => + JSON.stringify({ + openapi: "3.0.3", + info: { title: "Notes API", version: "1.0.0" }, + servers: [{ url: baseUrl }], + paths: { + "/notes": { + get: { + operationId: "listNotes", + summary: "List every note in the notebook", + responses: { "200": { description: "ok" } }, + }, + post: { + operationId: "createNote", + summary: "Create a note in the notebook", + requestBody: { + required: true, + content: { + "application/json": { + schema: { + type: "object", + properties: { text: { type: "string" } }, + required: ["text"], + }, + }, + }, + }, + responses: { "200": { description: "ok" } }, + }, + }, + }, + }); + +interface RecordedRequest { + readonly method: string; + readonly authorization: string | undefined; +} + +const serveRecordingUpstream = Effect.acquireRelease( + Effect.callback<{ + readonly url: string; + readonly requests: RecordedRequest[]; + close: () => void; + }>((resume) => { + const requests: RecordedRequest[] = []; + const server = createServer((request, response) => { + request.on("data", () => undefined); + request.on("end", () => { + requests.push({ + method: request.method ?? "", + authorization: request.headers.authorization, + }); + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ notes: [{ id: "note_0", text: "existing note" }] })); + }); + }); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + resume( + Effect.succeed({ + url: `http://127.0.0.1:${port}`, + requests, + close: () => { + server.close(); + server.closeAllConnections(); + }, + }), + ); + }); + }), + (upstream) => Effect.sync(() => upstream.close()), +); + +scenario( + "Passthrough · the real OpenCode binary runs its own codemode over 10,200 Executor tools", + { timeout: 900_000 }, + Effect.scoped( + Effect.gen(function* () { + const target = yield* Target; + const opencode = yield* OpenCode; + const runDir = yield* RunDir; + const cli = yield* Cli; + const { client: makeClient } = yield* Api; + + const identity = yield* target.newIdentity(); + const email = identity.credentials?.email ?? identity.label; + const client = yield* makeClient(catalogApi, identity); + const upstream = yield* serveRecordingUpstream; + const notesSlug = unique("notes"); + + // --- Seed: 10,198 discovery-mass tools + 2 callable ones. --- + const seeded = yield* seedLargeCatalog(client, { + syntheticIntegrations: SYNTHETIC_INTEGRATIONS, + opsPerIntegration: OPS_PER_INTEGRATION, + }); + const cleanup = Effect.gen(function* () { + yield* client.connections + .remove({ + params: { + owner: "org", + integration: IntegrationSlug.make(notesSlug), + name: ConnectionName.make("main"), + }, + }) + .pipe(Effect.ignore); + yield* client.openapi.removeSpec({ params: { slug: notesSlug } }).pipe(Effect.ignore); + yield* seeded.cleanup; + }); + + yield* Effect.ensuring( + Effect.gen(function* () { + yield* client.openapi.addSpec({ + payload: { + spec: { kind: "blob", value: notesSpec(upstream.url) }, + slug: notesSlug, + baseUrl: upstream.url, + authenticationTemplate: [ + { + slug: "apiKey", + type: "apiKey", + headers: { authorization: ["Bearer ", { type: "variable", name: "token" }] }, + }, + ], + }, + }); + yield* client.connections.create({ + payload: { + owner: "org", + name: ConnectionName.make("main"), + integration: IntegrationSlug.make(notesSlug), + template: AuthTemplateSlug.make("apiKey"), + value: "tok_notes", + }, + }); + const visible = (yield* client.tools.list({ query: {} })).filter( + (tool) => tool.static !== true, + ); + expect(visible.length, "the catalog is exactly the advertised size").toBe( + EXPECTED_TOOL_COUNT, + ); + + // --- The replay brain: two scripted turns of OpenCode codemode. --- + // Turn 0: discover the notes tool through OpenCode's own search. + // Turn 1: call the path search returned. Turn 2: summarize, stop. + let discoveredPath: string | undefined; + const brain = yield* serveReplayBrain((ctx) => { + // OpenCode also asks the model for a session title, with no tools + // offered. Answer it with text and keep it out of the script. + if (ctx.toolNames.length === 0) return { text: "Notebook" }; + if (ctx.lastRole === "user") { + return { + text: "Searching the connected tools.", + tool: { + name: "execute", + args: { + code: `return await tools.$codemode.search({ query: "list notes notebook", limit: 5 });`, + }, + }, + }; + } + if (discoveredPath === undefined) { + const result = ctx.lastToolResult ?? ""; + // codemode returns pretty-printed JSON; the path is the exact + // expression the next program must call. + const match = new RegExp( + `"path":\\s*"(tools\\.[A-Za-z0-9_$.\\[\\]"-]*${notesSlug}[^"]*listNotes)"`, + ).exec(result); + if (!match) { + throw new Error(`search did not surface the notes tool: ${result.slice(0, 600)}`); + } + discoveredPath = match[1]!; + return { + text: "Found it. Listing the notes.", + tool: { name: "execute", args: { code: `return await ${discoveredPath}({});` } }, + }; + } + return { text: "The notebook has one existing note." }; + }); + + const passthroughUrl = new URL("/mcp?mode=passthrough", target.baseUrl).toString(); + const home = opencode.makeHome(SERVER_NAME, passthroughUrl, { + chatBrainUrl: brain.baseUrl, + }); + const env = { + ...home.env, + OPENCODE_EXPERIMENTAL_CODE_MODE: "true", + PS1: "$ ", + BASH_SILENCE_DEPRECATION_WARNING: "1", + }; + // First-run database migration happens off camera. + yield* Effect.sync(() => opencode.warmUp(home)); + + let connectMs = -1; + yield* cli.session( + ["bash", "--norc"], + async (term) => { + await term.screen.waitForText("$", { timeoutMs: 10_000 }); + + const outputAfter = (text: string, line: string): string | null => { + const echoed = text.lastIndexOf(line); + if (echoed === -1) return null; + const after = text.slice(echoed + line.length); + return after.trimEnd().endsWith("\n$") ? after : null; + }; + const sh = async (line: string, timeoutMs: number) => { + await term.keyboard.type(line); + await term.keyboard.press("Enter"); + const snapshot = await term.screen.waitUntil( + (current) => outputAfter(current.text, line) !== null, + { timeoutMs }, + ); + return outputAfter(snapshot.text, line) ?? ""; + }; + + // OpenCode's own OAuth against the target: discovery, DCR, PKCE. + const consent = opencode.completeOAuthConsent(home, email, home.openedUrls().length); + const auth = await sh(`opencode mcp auth ${SERVER_NAME}`, 90_000); + await consent; + expect(auth, "opencode mcp auth completes").not.toContain("failed"); + + // The load-bearing connect: tools/list of 10,200 definitions + // inside OpenCode's own 30s connect timeout. + const startedAt = Date.now(); + const listed = await sh("opencode mcp list", 120_000); + connectMs = Date.now() - startedAt; + expect( + listed, + `OpenCode connects to the 10,200-tool passthrough endpoint (took ${connectMs}ms)`, + ).toContain("connected"); + + // A real agent turn: OpenCode's codemode over our catalog. + const ran = await sh(`opencode run "List the notes in my notebook"`, 300_000); + expect(ran, "the run did not error").not.toContain("UnknownError"); + }, + { + cwd: home.projectDir, + env, + record: join(runDir, "terminal.cast"), + viewport: { cols: 100, rows: 40 }, + }, + ); + + // --- What OpenCode showed the model, and what came back. --- + expect(brain.errors(), "the scripted brain hit no surprises").toEqual([]); + // Only the turns that carried tools are the agent loop; the + // title request is OpenCode housekeeping. + const requests = brain.requests().filter((request) => request.toolNames.length > 0); + expect(requests.length, "three model turns: search, call, summary").toBe(3); + + // Codemode: the model sees ONE execute tool, not 10,200 functions. + const offered = requests[0]!.toolNames; + expect(offered, "OpenCode offers its codemode execute tool").toContain("execute"); + expect( + offered.filter((name) => name.startsWith(`${SERVER_NAME}_`)), + "no MCP tool is flattened into the model's tool list", + ).toEqual([]); + expect( + offered.length, + "the model's tool list stays small in front of a 10,200-tool server", + ).toBeLessThan(40); + + // Discovery surfaced OUR tool by its passthrough name. + expect(discoveredPath, "search returned a callable path").toBeDefined(); + expect(discoveredPath, "the path names the passthrough tool").toContain(notesSlug); + + // The call went over the wire with the connection's credential and + // the payload came back through OpenCode's interpreter. + const lastToolResult = [...requests[2]!.messages] + .reverse() + .find((message) => message.role === "tool")?.content; + expect(lastToolResult, "the executed program returned the upstream payload").toContain( + "existing note", + ); + const upstreamGet = upstream.requests.find((request) => request.method === "GET"); + expect(upstreamGet, "the GET reached the upstream").toBeDefined(); + expect(upstreamGet?.authorization, "the connection's credential was applied").toBe( + "Bearer tok_notes", + ); + + // Recorded for the run report; the hard bound is OpenCode's own + // connect timeout, which `mcp list` above already proved. + expect(connectMs, "connect stays inside OpenCode's timeout").toBeLessThan( + OPENCODE_CONNECT_TIMEOUT_MS * 4, + ); + }), + cleanup, + ); + }), + ), +); From db18de59fe6d5b3ee20cc64ac828c27ffc701c56 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 3 Sep 2026 23:35:56 -0700 Subject: [PATCH 03/19] Harden passthrough: provenance-gated approvals, injection-safe call code, exclusive artifacts --- apps/cloud/src/mcp/agent-handler.ts | 6 +- apps/cloud/src/mcp/session-durable-object.ts | 11 +- apps/host-cloudflare/src/mcp/agent-handler.ts | 4 +- .../src/mcp/session-durable-object.ts | 9 +- apps/local/src/mcp.ts | 5 +- packages/core/sdk/src/elicitation.ts | 10 + packages/core/sdk/src/executor.ts | 33 +-- packages/core/sdk/src/index.ts | 1 + .../hosts/mcp/src/in-memory-session-store.ts | 9 +- .../hosts/mcp/src/passthrough-tools.test.ts | 199 ++++++++++++++---- packages/hosts/mcp/src/passthrough-tools.ts | 13 +- packages/hosts/mcp/src/tool-server.ts | 109 ++++++---- packages/plugins/graphql/src/sdk/plugin.ts | 3 +- .../react/src/components/mcp-install-card.tsx | 11 +- 14 files changed, 294 insertions(+), 129 deletions(-) diff --git a/apps/cloud/src/mcp/agent-handler.ts b/apps/cloud/src/mcp/agent-handler.ts index 6ddc1b37bb..fec9c2605f 100644 --- a/apps/cloud/src/mcp/agent-handler.ts +++ b/apps/cloud/src/mcp/agent-handler.ts @@ -189,11 +189,7 @@ const propsForPrincipal = ( ...sessionOrgRoleMetadata(principal), userId: principal.accountId, elicitationMode: readElicitationMode(request), - // Forwarded only when spelled out, so the factory applies the tool - // mode's own default to an absent `?artifacts=`. - ...(new URL(request.url).searchParams.has("artifacts") - ? { artifactsEnabled: readArtifactsEnabled(request) } - : {}), + artifactsEnabled: readArtifactsEnabled(request), searchToolsEnabled: readSearchToolsEnabled(request), toolMode: readToolMode(request), ...(readPassthroughIntegrations(request) diff --git a/apps/cloud/src/mcp/session-durable-object.ts b/apps/cloud/src/mcp/session-durable-object.ts index 9afa758d39..e3222084c9 100644 --- a/apps/cloud/src/mcp/session-durable-object.ts +++ b/apps/cloud/src/mcp/session-durable-object.ts @@ -385,13 +385,10 @@ export class McpSessionDOSqlite extends McpAgentSessionDOBase` resolves against the producing + // connection's `definition` rows), so the join key is the same + // (owner, integration, connection) triple the tool row carries. + definitionRows: core.findMany("definition", { where: integrationWhere }), + }), + ); const policyRules = yield* listActivePolicyRuleSet(); - - // Shared definitions, loaded once and keyed per connection: `$ref`s are - // connection-local (`#/$defs/` resolves against the producing - // connection's `definition` rows), so the join key is the same - // (owner, integration, connection) triple the tool row carries. - const definitionRows = yield* core.findMany("definition", { - where: integrationWhere, - }); const defsByConnection = new Map>(); for (const def of definitionRows) { const key = `${def.owner}\u0000${def.integration}\u0000${def.connection}`; @@ -6233,6 +6239,7 @@ export const createExecutor = string | null, ): McpBuildServerOptions => { - // `?artifacts=` is only forwarded when the URL spells it out, so the - // factory can apply the mode's own default (on for codemode, off for - // passthrough) to an absent value. - const artifactsEnabled = new URL(request.url).searchParams.has("artifacts") - ? readArtifactsEnabled(request) - : undefined; + const artifactsEnabled = readArtifactsEnabled(request); const searchToolsEnabled = readSearchToolsEnabled(request); const toolMode = readToolMode(request); const passthroughIntegrations = readPassthroughIntegrations(request); const surface = { - ...(artifactsEnabled === undefined ? {} : { artifactsEnabled }), + artifactsEnabled, searchToolsEnabled, mode: toolMode, ...(passthroughIntegrations ? { passthroughIntegrations } : {}), diff --git a/packages/hosts/mcp/src/passthrough-tools.test.ts b/packages/hosts/mcp/src/passthrough-tools.test.ts index 6154763e3f..173bb9d2bb 100644 --- a/packages/hosts/mcp/src/passthrough-tools.test.ts +++ b/packages/hosts/mcp/src/passthrough-tools.test.ts @@ -201,14 +201,43 @@ describe("passthrough annotations", () => { expect(passthroughAnnotations({ name: "x", policy: "approve" }).readOnlyHint).toBe(false); }); - it("emits exactly one awaited tool call with a JSON-literal argument", () => { + it("emits exactly one awaited tool call with every segment as a string literal", () => { expect(passthroughCallCode("tools.github.org.main.issues.create", { title: "hi" })).toBe( - 'return await tools.github.org.main.issues.create({"title":"hi"});', + 'return await tools["github"]["org"]["main"]["issues"]["create"]({"title":"hi"});', ); expect(passthroughCallCode("linear.org.main.issueCreate", undefined)).toBe( - "return await tools.linear.org.main.issueCreate({});", + 'return await tools["linear"]["org"]["main"]["issueCreate"]({});', ); }); + + it("keeps a hostile tool segment as data, never as code", () => { + // An OpenAPI spec controls its tool paths (`x-executor-toolPath`), so a + // segment can contain anything. It must land inside a JSON string. + const hostile = "x(await tools.victim.org.main.destroy({}))"; + const code = passthroughCallCode(`tools.evil.org.main.${hostile}`, {}); + expect(code).toBe( + `return await tools["evil"]["org"]["main"]["x(await tools"]["victim"]["org"]["main"]["destroy({}))"]({});`, + ); + // Structural proof the payload never escapes a string literal: the source + // is exactly `return await tools` + N bracket-quoted segments + one call. + // Every quoted segment round-trips through JSON.parse to the raw text, so + // whatever the segment contains is data to the interpreter. + const shape = /^return await tools((?:\["(?:[^"\\]|\\.)*"\])+)\((\{.*\})\);$/s.exec(code); + expect(shape).not.toBeNull(); + const segments = [...shape![1]!.matchAll(/\["((?:[^"\\]|\\.)*)"\]/g)].map((m) => + JSON.parse(`"${m[1]}"`), + ); + expect(segments).toEqual([ + "evil", + "org", + "main", + "x(await tools", + "victim", + "org", + "main", + "destroy({}))", + ]); + }); }); // --------------------------------------------------------------------------- @@ -316,7 +345,7 @@ describe("passthrough mode server", () => { arguments: { title: "hello" }, }); expect(recording.executed).toEqual([ - 'return await tools.github.org.main.issues.create({"title":"hello"});', + 'return await tools["github"]["org"]["main"]["issues"]["create"]({"title":"hello"});', ]); expect(recording.pausedCalls()).toBe(0); // The tool's `data` is the result: nothing sits between the tool and @@ -356,52 +385,144 @@ describe("passthrough mode server", () => { ); }); - it("accepts the approval gate inline but forwards a real input form", async () => { - // An engine whose tool raises an elicitation: first the executor's - // approval-only form (no fields), then a real form asking for a value. - const seen: string[] = []; - const engine: ExecutionEngine = { - ...makeRecordingEngine().engine, - execute: (_code, options) => - Effect.gen(function* () { - const approval = yield* options.onElicitation({ - address: CATALOG[0]!.address, - args: {}, - request: { - _tag: "FormElicitation", - message: "Approve?", - requestedSchema: { type: "object", properties: {} }, - }, - }); - seen.push(`approval:${approval.action}`); - const form = yield* options.onElicitation({ + /** An engine whose tool raises the given elicitations in order and records + * each answer. `source` is what the executor stamps: `policy` for its own + * approval gate, `tool` for anything the tool asked for itself. */ + const elicitingEngine = ( + requests: ReadonlyArray<{ readonly source: "policy" | "tool"; readonly request: any }>, + seen: string[], + ): ExecutionEngine => ({ + ...makeRecordingEngine().engine, + execute: (_code, options) => + Effect.gen(function* () { + for (const { source, request } of requests) { + const answer = yield* options.onElicitation({ address: CATALOG[0]!.address, args: {}, - request: { - _tag: "FormElicitation", - message: "Which project?", - requestedSchema: { - type: "object", - properties: { project: { type: "string" } }, - required: ["project"], - }, - }, + request, + source, }); - seen.push(`form:${form.action}`); - return { result: { ok: true, data: null } }; - }), + seen.push(`${source}:${answer.action}`); + if (answer.action !== "accept") { + return { result: { ok: false, error: { code: "declined", message: "declined" } } }; + } + } + return { result: { ok: true, data: null } }; + }), + }); + + const approvalGate = { + _tag: "FormElicitation" as const, + message: "Approve github.org.main.issues.create?", + requestedSchema: { type: "object", properties: {} }, + }; + + it("accepts the executor's own approval gate inline", async () => { + const seen: string[] = []; + await withClient( + { + engine: elicitingEngine([{ source: "policy", request: approvalGate }], seen), + mode: "passthrough", + tools: { describeAll: () => Effect.succeed(CATALOG) }, + }, + async (client) => { + const result = await client.callTool({ + name: "github__issues_create", + arguments: { title: "x" }, + }); + expect(seen).toEqual(["policy:accept"]); + expect(result.isError ?? false).toBe(false); + }, + ); + }); + + it("never auto-accepts a tool-raised prompt, even one with an empty schema", async () => { + // Same wire shape as the approval gate, but raised by the TOOL: a + // per-site grant whose terms live in `meta`. Provenance, not shape, + // decides. With no elicitation capability on the client, the call fails + // and says so — it is not silently granted. + const seen: string[] = []; + const siteGrant = { + _tag: "FormElicitation" as const, + message: "Allow Browser use to access example.com?", + requestedSchema: {}, + meta: { persist: "always", origin: "https://example.com" }, }; + await withClient( + { + engine: elicitingEngine([{ source: "tool", request: siteGrant }], seen), + mode: "passthrough", + tools: { describeAll: () => Effect.succeed(CATALOG) }, + }, + async (client) => { + const result = await client.callTool({ + name: "github__issues_create", + arguments: { title: "x" }, + }); + expect(seen).toEqual(["tool:decline"]); + expect(result.isError).toBe(true); + expect(result.structuredContent).toMatchObject({ + status: "error", + error: { code: "elicitation_unsupported", request: siteGrant.message }, + }); + }, + ); + }); + + it("reports an unanswerable URL request with the URL, not as a user decline", async () => { + const seen: string[] = []; + const reconnect = { + _tag: "UrlElicitation" as const, + message: "Reconnect GitHub", + url: "https://example.test/oauth/start", + elicitationId: "elic_1", + }; + await withClient( + { + engine: elicitingEngine([{ source: "tool", request: reconnect }], seen), + mode: "passthrough", + tools: { describeAll: () => Effect.succeed(CATALOG) }, + }, + async (client) => { + const result = await client.callTool({ + name: "github__issues_create", + arguments: { title: "x" }, + }); + expect(seen).toEqual(["tool:decline"]); + expect(result.isError).toBe(true); + const text = (result.content as Array<{ text?: string }>)[0]?.text ?? ""; + expect(text).toContain("does not support elicitation"); + expect(text).toContain("https://example.test/oauth/start"); + expect(text).not.toContain("declined by the user"); + expect(result.structuredContent).toMatchObject({ + error: { code: "elicitation_unsupported", url: reconnect.url }, + }); + }, + ); + }); + + it("serves no artifact tools in passthrough even when artifacts are requested", async () => { + const { engine } = makeRecordingEngine(); await withClient( { engine, mode: "passthrough", + artifactsEnabled: true, + loadAppShellHtml: async () => "", + artifacts: { + list: () => Effect.succeed([]), + get: () => Effect.die("unused"), + save: () => Effect.die("unused"), + }, tools: { describeAll: () => Effect.succeed(CATALOG) }, }, async (client) => { - await client.callTool({ name: "github__issues_create", arguments: { title: "x" } }); - // The test client advertises no elicitation capability, so the real - // form is declined rather than silently accepted with no data. - expect(seen).toEqual(["approval:accept", "form:decline"]); + const names = (await client.listTools()).tools.map((tool) => tool.name); + expect(names.sort()).toEqual([ + "github__issues_create", + "github__issues_list", + "linear__issueCreate", + ]); }, ); }); diff --git a/packages/hosts/mcp/src/passthrough-tools.ts b/packages/hosts/mcp/src/passthrough-tools.ts index 3f64eff6bf..29b3e2aa20 100644 --- a/packages/hosts/mcp/src/passthrough-tools.ts +++ b/packages/hosts/mcp/src/passthrough-tools.ts @@ -178,8 +178,17 @@ export const assignPassthroughNames = ( * see it as one execution. */ export const passthroughCallCode = (address: string, args: unknown): string => { - const path = address.startsWith("tools.") ? address : `tools.${address}`; - return `return await ${path}(${JSON.stringify(args ?? {})});`; + // Every segment is a JSON string literal in bracket notation, never a bare + // identifier: the tool segment is customer-controlled (an OpenAPI spec may + // set `x-executor-toolPath`), so it must be data in the generated source, + // not syntax. `tools["a"]["b"]` resolves through the sandbox proxy exactly + // as `tools.a.b` does. + const bare = address.startsWith("tools.") ? address.slice("tools.".length) : address; + const accessor = bare + .split(".") + .map((segment) => `[${JSON.stringify(segment)}]`) + .join(""); + return `return await tools${accessor}(${JSON.stringify(args ?? {})});`; }; /** diff --git a/packages/hosts/mcp/src/tool-server.ts b/packages/hosts/mcp/src/tool-server.ts index 4a4fa5f00c..46ca66d9e9 100644 --- a/packages/hosts/mcp/src/tool-server.ts +++ b/packages/hosts/mcp/src/tool-server.ts @@ -40,7 +40,6 @@ import type { ElicitationHandler, ElicitationContext, ElicitationRequest, - FormElicitation, SaveArtifactInput, ToolFileValue, ToolListFilter, @@ -203,9 +202,10 @@ type SharedMcpServerConfig = { * The tool surface this connection serves. `codemode` (the default) is the * `execute` tool plus `skills`/`resume` and the artifact surface. * `passthrough` (`?mode=passthrough`) registers every visible integration - * tool as its own MCP tool and serves NONE of `execute`, `skills`, or - * `resume`: policy is folded into each tool's annotations at list time and - * the client's own approval flow takes it from there. Requires `tools`. + * tool as its own MCP tool and serves NONE of `execute`, `skills`, `resume`, + * or the artifact tools (`artifactsEnabled` is ignored): policy is folded + * into each tool's annotations at list time and the client's own approval + * flow takes it from there. Requires `tools`. */ readonly mode?: McpToolMode; /** @@ -415,17 +415,6 @@ const elicitationRequestTag = (request: ElicitationRequest): ElicitationRequest[ Match.exhaustive, ); -/** The executor's approval gate asks for consent with a form that collects - * nothing: `{ type: "object", properties: {} }` (or a bare `{}`). Anything - * with a declared field is a tool asking the user for input. */ -const isApprovalOnlyForm = (request: FormElicitation): boolean => { - const schema = request.requestedSchema; - const properties = schema.properties; - const declaresFields = isRecord(properties) && Object.keys(properties).length > 0; - const requiresFields = Array.isArray(schema.required) && schema.required.length > 0; - return !declaresFields && !requiresFields; -}; - const requestedSchemaIsNonEmpty = (request: ElicitationRequest): boolean => Match.value(request).pipe( Match.tag("FormElicitation", (req) => Object.keys(req.requestedSchema).length > 0), @@ -727,6 +716,38 @@ const toPassthroughResult = (outcome: FormattedExecuteInput): McpToolResult => { }; }; +/** + * A passthrough tool asked the user for something and the connected client + * advertises no elicitation capability, so nobody could answer. Say exactly + * that, and carry the request — a reconnect/OAuth URL is the usual content — + * so the model can relay it and the user can act outside the client. + */ +const elicitationUnsupportedResult = ( + toolName: string, + request: ElicitationRequest, +): McpToolResult => { + const url = elicitationRequestUrl(request); + const lines = [ + `Tool ${toolName} needs input from the user, but this MCP client does not support elicitation, so the call could not complete.`, + `Request: ${request.message}`, + ...(url ? [`Open this URL to continue, then retry the call: ${url}`] : []), + ]; + return { + content: [{ type: "text", text: `Error: ${lines.join("\n")}` }], + structuredContent: { + status: "error", + error: { + code: "elicitation_unsupported", + message: lines[0]!, + request: request.message, + ...(url ? { url } : {}), + }, + logs: [], + }, + isError: true, + }; +}; + const toMcpPausedResult = (formatted: ReturnType): McpToolResult => ({ content: [{ type: "text", text: formatted.text }], structuredContent: formatted.structured, @@ -1330,12 +1351,13 @@ export const createExecutorMcpServer = ( // Artifacts are on unless this connection opted out (`?artifacts=false`). // One flag decides the whole surface: the tools, the shell resource, and // the skills catalog below. - // Passthrough is a plain tool surface: artifacts are OFF there unless the - // connection spelled out `?artifacts=true`, the reverse of codemode's - // default. `config.artifactsEnabled` is what the host read off the URL, so - // an explicit true survives; an absent value takes the mode's default. + // Passthrough is a plain tool surface and serves NO artifact tools, whatever + // the URL says: the artifact tools are codemode affordances (they run + // sandboxed component code), and the SDK's `registerTool` path would try to + // install the same `tools/list` + `tools/call` handlers passthrough owns. + // The two surfaces are exclusive by construction, not merged. const artifactsEnabled = - config.artifactsEnabled ?? (config.mode === "passthrough" ? false : true); + config.mode === "passthrough" ? false : (config.artifactsEnabled ?? true); const skillCatalog: readonly Skill[] = skillCatalogFor({ artifacts: artifactsEnabled }); // Per-integration search tools are off unless this connection opted in // (`?search_tools=true`). @@ -1803,28 +1825,37 @@ export const createExecutorMcpServer = ( }); const { url: supportsUrl } = getElicitationSupport(server); const native = makeMcpElicitationHandler(server, extra.requestId, debugLog); - const onElicitation: ElicitationHandler = (ctx) => - Match.value(ctx.request).pipe( - // The harness already prompted (or chose not to) from the - // annotations; a second server-side gate would double-prompt. - Match.tag("FormElicitation", (req) => - isApprovalOnlyForm(req) - ? Effect.succeed({ action: "accept" as const, content: {} }) - : // A real form (a tool asking for input) still needs a human. - // Forward it natively when the client can take it; otherwise - // decline, which the invoker turns into a clear failure. - getElicitationSupport(server).form - ? native(ctx) - : Effect.succeed({ action: "decline" as const }), - ), - Match.tag("UrlElicitation", () => - supportsUrl ? native(ctx) : Effect.succeed({ action: "decline" as const }), - ), - Match.exhaustive, - ); + const { form: supportsForm } = getElicitationSupport(server); + // Set when the tool asked the user for something this client cannot + // relay. The handler has no error channel (a non-accept is a decline + // to the executor), so the request is kept here and the whole call is + // reported as unanswerable below — with what was asked, URL included — + // instead of as "declined by the user", which nobody did. + let unanswerable: ElicitationRequest | undefined; + const onElicitation: ElicitationHandler = (ctx) => { + // The executor's OWN approval gate is the only thing accepted + // inline: the harness already prompted (or chose not to) from the + // advertised annotations, so a second server-side gate would + // double-prompt. Provenance is stamped by the executor, not read + // off the request shape — a tool-raised prompt with an empty schema + // can still carry terms of its own (a permanent site grant), and + // must reach a human or fail, never be answered for them. + if (ctx.source === "policy") { + return Effect.succeed({ action: "accept" as const, content: {} }); + } + // Anything the tool itself asked for goes to the client natively + // when it can take it; the native bridge already turns a URL + // request into a form for form-only clients. + if (supportsForm || (supportsUrl && ctx.request._tag === "UrlElicitation")) { + return native(ctx); + } + unanswerable = ctx.request; + return Effect.succeed({ action: "decline" as const }); + }; const outcome = yield* engine.execute(passthroughCallCode(address, args), { onElicitation, }); + if (unanswerable) return elicitationUnsupportedResult(tool.name, unanswerable); return toPassthroughResult(outcome); }).pipe( Effect.withSpan("mcp.host.tool.execute", { diff --git a/packages/plugins/graphql/src/sdk/plugin.ts b/packages/plugins/graphql/src/sdk/plugin.ts index 74196278c3..fd3adfdcbc 100644 --- a/packages/plugins/graphql/src/sdk/plugin.ts +++ b/packages/plugins/graphql/src/sdk/plugin.ts @@ -573,7 +573,8 @@ const annotationsFor = (binding: OperationBinding): ToolAnnotations => { readOnly: false, }; } - return {}; + // A query is the read side of GraphQL by definition. + return { readOnly: true }; }; // --------------------------------------------------------------------------- diff --git a/packages/react/src/components/mcp-install-card.tsx b/packages/react/src/components/mcp-install-card.tsx index 4a169c97e4..655cb2f7ac 100644 --- a/packages/react/src/components/mcp-install-card.tsx +++ b/packages/react/src/components/mcp-install-card.tsx @@ -327,13 +327,16 @@ export function McpInstallCard(props: { className?: string }) {
Artifacts
- {artifacts - ? "Generated UI components are saved to your workspace." - : "Disabled: this connection serves no artifact tools."} + {toolMode === "passthrough" + ? "Not available when tools are exposed directly." + : artifacts + ? "Generated UI components are saved to your workspace." + : "Disabled: this connection serves no artifact tools."}
{ setPreferences((current) => ({ ...current, artifacts: next })); trackEvent("mcp_install_artifacts_toggled", { artifacts: next }); From 2082c84ba51cae9e3fe0060c5007b2d2e4a45a24 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 3 Sep 2026 23:45:57 -0700 Subject: [PATCH 04/19] Passthrough: generation-checked catalog read, single-key tool access --- packages/core/sdk/src/executor.ts | 78 ++++++++++++++----- .../hosts/mcp/src/passthrough-tools.test.ts | 43 ++++------ packages/hosts/mcp/src/passthrough-tools.ts | 21 ++--- 3 files changed, 87 insertions(+), 55 deletions(-) diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 2490c11c55..cd00c2dcd3 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -907,6 +907,12 @@ const validateExecutorDbTables = (required: FumaTables, actual: FumaTables): voi }); }; +/** How many times `tools.describeAll` re-reads a catalog whose tool rows and + * definitions came from different rebuild generations. One rebuild landing + * mid-read needs one retry; the bound only exists so a pathological + * rebuild storm fails loudly instead of spinning. */ +const DESCRIBE_ALL_GENERATION_RETRIES = 3; + const storageFailureFromUnknown = (message: string, cause: unknown): StorageFailure => isStorageFailure(cause) ? cause : new StorageError({ message, cause }); @@ -5794,27 +5800,63 @@ export const createExecutor = ` resolves against the producing - // connection's `definition` rows), so the join key is the same - // (owner, integration, connection) triple the tool row carries. - definitionRows: core.findMany("definition", { where: integrationWhere }), + // Tool rows and their shared `$defs` must come from the SAME catalog + // generation: a detached stale-sync rebuild replaces both tables for a + // connection, and two reads that straddle it would pair an old + // schema's `$ref` with a refreshed definition of the same name. A + // transaction alone does not promise that on every backend (Postgres + // runs READ COMMITTED, D1 has no interactive transactions), so the + // join is checked, not assumed: a rebuild stamps every row it writes + // with one `created_at`, which is the generation marker. When the two + // reads disagree for a connection, they are re-read; a bounded number + // of attempts covers a rebuild landing mid-read without ever serving a + // mixed generation. + const readCatalog = Effect.all({ + rows: core.findMany("tool", { + where: integrationWhere, + select: [...TOOL_INVOCATION_COLUMNS, "input_schema"], + }), + // Shared definitions, keyed per connection below: `$ref`s are + // connection-local (`#/$defs/` resolves against the producing + // connection's `definition` rows), so the join key is the same + // (owner, integration, connection) triple the tool row carries. + definitionRows: core.findMany("definition", { where: integrationWhere }), + }); + const connectionKey = (row: { + readonly owner: string; + readonly integration: string; + readonly connection: string; + }): string => `${row.owner}\u0000${row.integration}\u0000${row.connection}`; + const generationOf = (value: unknown): number => + value instanceof Date ? value.getTime() : new Date(String(value)).getTime(); + const { rows, definitionRows } = yield* readCatalog.pipe( + Effect.flatMap((snapshot) => { + // Per connection: the generation the tool rows carry vs the one + // the definition rows carry. A connection with no definitions has + // nothing to disagree about. + const toolGeneration = new Map(); + for (const row of snapshot.rows) { + toolGeneration.set(connectionKey(row), generationOf(row.created_at)); + } + for (const def of snapshot.definitionRows) { + const expected = toolGeneration.get(connectionKey(def)); + if (expected !== undefined && expected !== generationOf(def.created_at)) { + return Effect.fail( + new StorageError({ + message: "tool catalog changed between reads", + cause: undefined, + }), + ); + } + } + return Effect.succeed(snapshot); }), + Effect.retry({ times: DESCRIBE_ALL_GENERATION_RETRIES }), ); const policyRules = yield* listActivePolicyRuleSet(); const defsByConnection = new Map>(); for (const def of definitionRows) { - const key = `${def.owner}\u0000${def.integration}\u0000${def.connection}`; + const key = connectionKey(def); let bucket = defsByConnection.get(key); if (!bucket) { bucket = new Map(); @@ -5841,9 +5883,7 @@ export const createExecutor = { expect(passthroughAnnotations({ name: "x", policy: "approve" }).readOnlyHint).toBe(false); }); - it("emits exactly one awaited tool call with every segment as a string literal", () => { + it("emits exactly one awaited tool call with the whole address as one string literal", () => { expect(passthroughCallCode("tools.github.org.main.issues.create", { title: "hi" })).toBe( - 'return await tools["github"]["org"]["main"]["issues"]["create"]({"title":"hi"});', + 'return await tools["github.org.main.issues.create"]({"title":"hi"});', ); expect(passthroughCallCode("linear.org.main.issueCreate", undefined)).toBe( - 'return await tools["linear"]["org"]["main"]["issueCreate"]({});', + 'return await tools["linear.org.main.issueCreate"]({});', + ); + // `then` is reserved by every sandbox proxy; as part of one key it is + // just text, so such a tool stays callable. + expect(passthroughCallCode("tools.svc.org.main.items.then", {})).toBe( + 'return await tools["svc.org.main.items.then"]({});', ); }); it("keeps a hostile tool segment as data, never as code", () => { // An OpenAPI spec controls its tool paths (`x-executor-toolPath`), so a // segment can contain anything. It must land inside a JSON string. - const hostile = "x(await tools.victim.org.main.destroy({}))"; + const hostile = 'x"](await tools.victim.org.main.destroy({}))["'; const code = passthroughCallCode(`tools.evil.org.main.${hostile}`, {}); - expect(code).toBe( - `return await tools["evil"]["org"]["main"]["x(await tools"]["victim"]["org"]["main"]["destroy({}))"]({});`, - ); - // Structural proof the payload never escapes a string literal: the source - // is exactly `return await tools` + N bracket-quoted segments + one call. - // Every quoted segment round-trips through JSON.parse to the raw text, so - // whatever the segment contains is data to the interpreter. - const shape = /^return await tools((?:\["(?:[^"\\]|\\.)*"\])+)\((\{.*\})\);$/s.exec(code); + // Structural proof the payload never escapes the string literal: the + // source is exactly `return await tools[]();`, and + // that one string round-trips through JSON.parse to the raw address. + const shape = /^return await tools\[("(?:[^"\\]|\\.)*")\]\((\{.*\})\);$/s.exec(code); expect(shape).not.toBeNull(); - const segments = [...shape![1]!.matchAll(/\["((?:[^"\\]|\\.)*)"\]/g)].map((m) => - JSON.parse(`"${m[1]}"`), - ); - expect(segments).toEqual([ - "evil", - "org", - "main", - "x(await tools", - "victim", - "org", - "main", - "destroy({}))", - ]); + expect(JSON.parse(shape![1]!)).toBe(`evil.org.main.${hostile}`); + // And the call's argument is the JSON we passed, untouched by the address. + expect(JSON.parse(shape![2]!)).toEqual({}); }); }); @@ -345,7 +336,7 @@ describe("passthrough mode server", () => { arguments: { title: "hello" }, }); expect(recording.executed).toEqual([ - 'return await tools["github"]["org"]["main"]["issues"]["create"]({"title":"hello"});', + 'return await tools["github.org.main.issues.create"]({"title":"hello"});', ]); expect(recording.pausedCalls()).toBe(0); // The tool's `data` is the result: nothing sits between the tool and diff --git a/packages/hosts/mcp/src/passthrough-tools.ts b/packages/hosts/mcp/src/passthrough-tools.ts index 29b3e2aa20..d30e06e73d 100644 --- a/packages/hosts/mcp/src/passthrough-tools.ts +++ b/packages/hosts/mcp/src/passthrough-tools.ts @@ -178,17 +178,18 @@ export const assignPassthroughNames = ( * see it as one execution. */ export const passthroughCallCode = (address: string, args: unknown): string => { - // Every segment is a JSON string literal in bracket notation, never a bare - // identifier: the tool segment is customer-controlled (an OpenAPI spec may - // set `x-executor-toolPath`), so it must be data in the generated source, - // not syntax. `tools["a"]["b"]` resolves through the sandbox proxy exactly - // as `tools.a.b` does. + // The whole dotted address is ONE JSON string literal in bracket notation: + // `tools["github.org.main.items.then"](...)`. Two reasons it is not a chain + // of property accesses. The tool segment is customer-controlled (an OpenAPI + // spec may set `x-executor-toolPath`), so it must be data in the generated + // source, never syntax. And every sandbox proxy reserves the property name + // `then` (a thenable check would otherwise await the proxy itself), so a + // per-segment chain could never reach a tool whose path contains `then`. + // Each proxy joins the accessed keys with `.` to form the dispatch path, so + // a single key holding the dotted address reassembles to exactly the same + // path the chain would have. const bare = address.startsWith("tools.") ? address.slice("tools.".length) : address; - const accessor = bare - .split(".") - .map((segment) => `[${JSON.stringify(segment)}]`) - .join(""); - return `return await tools${accessor}(${JSON.stringify(args ?? {})});`; + return `return await tools[${JSON.stringify(bare)}](${JSON.stringify(args ?? {})});`; }; /** From 7eb987e03177772b08b204c56403f39d863324b7 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 4 Sep 2026 00:07:49 -0700 Subject: [PATCH 05/19] Stamp catalog builds with a generation id; describeAll refuses a mixed join --- apps/cloud/drizzle/0018_tool_generation.sql | 2 + apps/cloud/drizzle/meta/0018_snapshot.json | 1516 +++++++++++++++++++ apps/cloud/drizzle/meta/_journal.json | 7 + apps/cloud/src/db/executor-schema.ts | 2 + apps/local/src/db/executor-schema.ts | 2 + packages/core/sdk/src/core-schema.ts | 20 +- packages/core/sdk/src/executor.test.ts | 72 + packages/core/sdk/src/executor.ts | 90 +- 8 files changed, 1687 insertions(+), 24 deletions(-) create mode 100644 apps/cloud/drizzle/0018_tool_generation.sql create mode 100644 apps/cloud/drizzle/meta/0018_snapshot.json diff --git a/apps/cloud/drizzle/0018_tool_generation.sql b/apps/cloud/drizzle/0018_tool_generation.sql new file mode 100644 index 0000000000..7ff8756f81 --- /dev/null +++ b/apps/cloud/drizzle/0018_tool_generation.sql @@ -0,0 +1,2 @@ +ALTER TABLE "definition" ADD COLUMN "generation" text;--> statement-breakpoint +ALTER TABLE "tool" ADD COLUMN "generation" text; \ No newline at end of file diff --git a/apps/cloud/drizzle/meta/0018_snapshot.json b/apps/cloud/drizzle/meta/0018_snapshot.json new file mode 100644 index 0000000000..544c655bca --- /dev/null +++ b/apps/cloud/drizzle/meta/0018_snapshot.json @@ -0,0 +1,1516 @@ +{ + "id": "72cab41c-f919-4f73-812f-77f72e87cdaf", + "prevId": "42251aa3-ae24-4010-ac65-9f41e26cdc20", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memberships": { + "name": "memberships", + "schema": "", + "columns": { + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "memberships_account_id_accounts_id_fk": { + "name": "memberships_account_id_accounts_id_fk", + "tableFrom": "memberships", + "tableTo": "accounts", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memberships_organization_id_organizations_id_fk": { + "name": "memberships_organization_id_organizations_id_fk", + "tableFrom": "memberships", + "tableTo": "organizations", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "memberships_account_id_organization_id_pk": { + "name": "memberships_account_id_organization_id_pk", + "columns": ["account_id", "organization_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organizations": { + "name": "organizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organizations_slug_unique": { + "name": "organizations_slug_unique", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.artifact": { + "name": "artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bindings": { + "name": "bindings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "preview": { + "name": "preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "artifact_uidx": { + "name": "artifact_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blob": { + "name": "blob", + "schema": "", + "columns": { + "namespace": { + "name": "namespace", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "blob_id_uidx": { + "name": "blob_id_uidx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.connection": { + "name": "connection", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "item_ids": { + "name": "item_ids", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_health": { + "name": "last_health", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "tools_synced_at": { + "name": "tools_synced_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "oauth_client": { + "name": "oauth_client", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_client_owner": { + "name": "oauth_client_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_item_id": { + "name": "refresh_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "oauth_scope": { + "name": "oauth_scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_token_url": { + "name": "oauth_token_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_state": { + "name": "provider_state", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "connection_uidx": { + "name": "connection_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.definition": { + "name": "definition", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema": { + "name": "schema", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "definition_uidx": { + "name": "definition_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration": { + "name": "integration", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "health_check": { + "name": "health_check", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "config_revised_at": { + "name": "config_revised_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "can_remove": { + "name": "can_remove", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "can_refresh": { + "name": "can_refresh", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "integration_uidx": { + "name": "integration_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "authorization_url": { + "name": "authorization_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_url": { + "name": "token_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "grant": { + "name": "grant", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret_item_id": { + "name": "client_secret_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_kind": { + "name": "origin_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_integration": { + "name": "origin_integration", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_issuer": { + "name": "origin_issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_redirect_uri": { + "name": "origin_redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_client_uidx": { + "name": "oauth_client_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_session": { + "name": "oauth_session", + "schema": "", + "columns": { + "state": { + "name": "state", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "client_slug": { + "name": "client_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "integration": { + "name": "integration", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_url": { + "name": "redirect_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pkce_verifier": { + "name": "pkce_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_session_uidx": { + "name": "oauth_session_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_storage": { + "name": "plugin_storage", + "schema": "", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "collection": { + "name": "collection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "plugin_storage_uidx": { + "name": "plugin_storage_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "plugin_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "collection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.private_executor_cloud_settings": { + "name": "private_executor_cloud_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "default": "'1.0.0'" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.subject": { + "name": "subject", + "schema": "", + "columns": { + "external_id": { + "name": "external_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "subject_uidx": { + "name": "subject_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool": { + "name": "tool", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_schema": { + "name": "input_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "output_schema": { + "name": "output_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "annotations": { + "name": "annotations", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "generation": { + "name": "generation", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_uidx": { + "name": "tool_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool_policy": { + "name": "tool_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "pattern": { + "name": "pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_policy_uidx": { + "name": "tool_policy_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/apps/cloud/drizzle/meta/_journal.json b/apps/cloud/drizzle/meta/_journal.json index 375397ceca..c166335db2 100644 --- a/apps/cloud/drizzle/meta/_journal.json +++ b/apps/cloud/drizzle/meta/_journal.json @@ -127,6 +127,13 @@ "when": 1788287088210, "tag": "0017_lush_thunderbolts", "breakpoints": true + }, + { + "idx": 18, + "version": "7", + "when": 1788505381630, + "tag": "0018_tool_generation", + "breakpoints": true } ] } diff --git a/apps/cloud/src/db/executor-schema.ts b/apps/cloud/src/db/executor-schema.ts index 0db709b884..dd8b16a7d5 100644 --- a/apps/cloud/src/db/executor-schema.ts +++ b/apps/cloud/src/db/executor-schema.ts @@ -156,6 +156,7 @@ export const tool = pgTable( input_schema: json("input_schema"), output_schema: json("output_schema"), annotations: json("annotations"), + generation: text("generation"), created_at: timestamp("created_at").notNull(), updated_at: timestamp("updated_at").notNull(), row_id: varchar("row_id", { length: 255 }) @@ -186,6 +187,7 @@ export const definition = pgTable( plugin_id: text("plugin_id").notNull(), name: text("name").notNull(), schema: json("schema").notNull(), + generation: text("generation"), created_at: timestamp("created_at").notNull(), row_id: varchar("row_id", { length: 255 }) .primaryKey() diff --git a/apps/local/src/db/executor-schema.ts b/apps/local/src/db/executor-schema.ts index a7f6ceb48e..e12029d8b6 100644 --- a/apps/local/src/db/executor-schema.ts +++ b/apps/local/src/db/executor-schema.ts @@ -124,6 +124,7 @@ export const tool = sqliteTable( input_schema: text("input_schema"), output_schema: text("output_schema"), annotations: text("annotations"), + generation: text("generation"), created_at: integer("created_at").notNull(), updated_at: integer("updated_at").notNull(), row_id: text("row_id").primaryKey().notNull(), @@ -151,6 +152,7 @@ export const definition = sqliteTable( plugin_id: text("plugin_id").notNull(), name: text("name").notNull(), schema: text("schema").notNull(), + generation: text("generation"), created_at: integer("created_at").notNull(), row_id: text("row_id").primaryKey().notNull(), tenant: text("tenant").notNull(), diff --git a/packages/core/sdk/src/core-schema.ts b/packages/core/sdk/src/core-schema.ts index 8014584695..7d2f22ff47 100644 --- a/packages/core/sdk/src/core-schema.ts +++ b/packages/core/sdk/src/core-schema.ts @@ -336,6 +336,14 @@ export const coreTables = defineTables({ input_schema: nullableJsonColumn("input_schema"), output_schema: nullableJsonColumn("output_schema"), annotations: nullableJsonColumn("annotations"), + // One opaque id per catalog (re)build, shared by every tool AND + // definition row that build wrote for the connection. A reader that + // must join the two tables (`tools.describeAll`) compares it to prove + // both halves came from the same build — a wall-clock stamp cannot + // (SQLite stores it at second resolution; two builds can share it). + // Nullable only for rows written before the column existed; the next + // rebuild stamps them. + generation: nullableTextColumn("generation"), created_at: dateColumn("created_at"), updated_at: dateColumn("updated_at"), }, @@ -356,6 +364,8 @@ export const coreTables = defineTables({ // rows (22001) — that drift broke cloud migration 0013 once already. name: textColumn("name"), schema: jsonColumn("schema"), + /** Same value as the `tool` rows written by the same build; see there. */ + generation: nullableTextColumn("generation"), created_at: dateColumn("created_at"), }, ["tenant", "owner", "subject", "integration", "connection", "name"], @@ -449,10 +459,12 @@ export type OAuthClientRow = FumaRow; export type OAuthSessionRow = FumaRow; export type ToolRow = FumaRow; /** The tool-row projection the invoke/list hot paths load: everything except - * the heavy `input_schema`/`output_schema` JSON, which only `tools.schema` - * (describe) needs. Plugin `invokeTool` receives this shape — operation - * details ride in plugin storage or `annotations`, not the row schemas. */ -export type ToolInvocationRow = Omit; + * the heavy `input_schema`/`output_schema` JSON (which only `tools.schema` + * needs) and the build `generation` marker (which only the + * tools-to-definitions join in `tools.describeAll` reads). Plugin + * `invokeTool` receives this shape — operation details ride in plugin + * storage or `annotations`, not the row schemas. */ +export type ToolInvocationRow = Omit; /** The columns backing {@link ToolInvocationRow}, for `select` projections. */ export const TOOL_INVOCATION_COLUMNS = [ "tenant", diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index f3a9e905ec..2531c41c37 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -807,6 +807,78 @@ describe("createExecutor", () => { }), ); + // The tools-to-definitions join in `describeAll` must never serve one + // build's schemas with another build's `$defs`. On a backend with no + // interactive transactions (D1) a rebuild is visible statement by statement, + // so the read is proven consistent by the `generation` stamp each build + // writes on every row, not by a transaction. Modelled here by a storage + // proxy that lets a rebuild commit BETWEEN the tool read and the definition + // read, exactly the interleaving a snapshot would have hidden. + it.effect("tools.describeAll never joins tool rows to definitions from another build", () => + Effect.gen(function* () { + const config = makeTestConfig({ plugins: [demoPlugin] as const }); + // Fires once: after the tool read of a describeAll, before its + // definition read, run the armed rebuild to completion. + const race: { rebuild: (() => Promise) | null } = { rebuild: null }; + const wrap = (inner: FumaDb): FumaDb => + new Proxy(inner, { + get(target, prop) { + if (prop === "withContext") { + return (context: unknown) => + wrap((target.withContext as (c: unknown) => FumaDb)(context)); + } + if (prop === "transaction") { + return (run: (tx: FumaDb) => Promise) => + (target.transaction as (r: (tx: FumaDb) => Promise) => Promise)( + (tx) => run(wrap(tx)), + ); + } + if (prop === "findMany") { + return async (table: unknown, query: unknown) => { + const result = await ( + target.findMany as (t: unknown, q: unknown) => Promise + )(table, query); + if (table === "tool" && race.rebuild) { + const rebuild = race.rebuild; + race.rebuild = null; + await rebuild(); + } + return result; + }; + } + return Reflect.get(target, prop); + }, + }); + const executor = yield* createExecutor({ ...config, db: wrap(config.db) }); + yield* executor.demo.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + const before = yield* executor.tools.describeAll(); + const beforeInspect = before.find((tool) => tool.name === "inspect"); + expect(beforeInspect, "the seeded build is served whole").toBeDefined(); + + // Arm: a full rebuild of the same connection lands between the two reads. + race.rebuild = () => + Effect.runPromise( + executor.connections.refresh({ owner: "org", integration: INTEG, name: CONN }), + ).then(() => undefined); + const after = yield* executor.tools.describeAll(); + expect(race.rebuild, "the rebuild ran mid-read").toBeNull(); + + // Whatever was served is ONE build: the same reachable `$defs` as a + // clean read, never an old schema against new (or missing) definitions. + const inspect = after.find((tool) => tool.name === "inspect"); + expect(inspect, "the tool is still served").toBeDefined(); + const inlined = inspect?.inputSchema as { $defs?: Record }; + expect(Object.keys(inlined.$defs ?? {}).sort()).toEqual(["Cat", "Collar", "Dog", "Pet"]); + }), + ); + it.effect("execute dispatches a connection-produced tool to the owning plugin", () => Effect.gen(function* () { const executor = yield* makeTestExecutor({ diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index cd00c2dcd3..448aa74589 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -3662,6 +3662,10 @@ export const createExecutor = ({ tenant: keys.tenant, owner: keys.owner, @@ -3674,6 +3678,7 @@ export const createExecutor = ` resolves against the producing @@ -5827,30 +5833,74 @@ export const createExecutor = `${row.owner}\u0000${row.integration}\u0000${row.connection}`; - const generationOf = (value: unknown): number => - value instanceof Date ? value.getTime() : new Date(String(value)).getTime(); + // Per connection, both halves must come from ONE build. A build stamps + // every row it writes with the same opaque `generation`, so the proof + // is: the tool rows carry exactly one generation, the definition rows + // carry exactly one, and they are equal. That also catches the windows + // a non-transactional backend (D1 auto-commits each statement) can + // expose mid-rebuild — tools deleted but not reinserted (zero tool + // generations while definitions still carry the old one), or + // definitions deleted but not reinserted (tools carry the NEW + // generation, definitions carry none) — because a build with + // definitions always writes both. A connection whose build wrote no + // definitions legitimately has tools and no definition rows; only a + // definition-bearing build can be caught half-written, and it is, + // by the mismatch. Rows from before the column existed carry null on + // both sides and compare equal, which is exactly right: they were + // written together. + const mixedGenerations = (snapshot: { + readonly rows: ReadonlyArray<{ + readonly owner: string; + readonly integration: string; + readonly connection: string; + readonly generation: string | null; + }>; + readonly definitionRows: ReadonlyArray<{ + readonly owner: string; + readonly integration: string; + readonly connection: string; + readonly generation: string | null; + }>; + }): boolean => { + const toolGenerations = new Map>(); + for (const row of snapshot.rows) { + const key = connectionKey(row); + (toolGenerations.get(key) ?? toolGenerations.set(key, new Set()).get(key)!).add( + row.generation, + ); + } + const definitionGenerations = new Map>(); + for (const def of snapshot.definitionRows) { + const key = connectionKey(def); + ( + definitionGenerations.get(key) ?? definitionGenerations.set(key, new Set()).get(key)! + ).add(def.generation); + } + for (const [key, tools] of toolGenerations) { + if (tools.size !== 1) return true; + const defs = definitionGenerations.get(key); + if (defs === undefined) continue; + if (defs.size !== 1) return true; + if ([...tools][0] !== [...defs][0]) return true; + } + // Definitions whose tools are absent: a build that wrote definitions + // also wrote tools, so this is the tools-deleted window. + for (const key of definitionGenerations.keys()) { + if (!toolGenerations.has(key)) return true; + } + return false; + }; const { rows, definitionRows } = yield* readCatalog.pipe( - Effect.flatMap((snapshot) => { - // Per connection: the generation the tool rows carry vs the one - // the definition rows carry. A connection with no definitions has - // nothing to disagree about. - const toolGeneration = new Map(); - for (const row of snapshot.rows) { - toolGeneration.set(connectionKey(row), generationOf(row.created_at)); - } - for (const def of snapshot.definitionRows) { - const expected = toolGeneration.get(connectionKey(def)); - if (expected !== undefined && expected !== generationOf(def.created_at)) { - return Effect.fail( + Effect.flatMap((snapshot) => + mixedGenerations(snapshot) + ? Effect.fail( new StorageError({ message: "tool catalog changed between reads", cause: undefined, }), - ); - } - } - return Effect.succeed(snapshot); - }), + ) + : Effect.succeed(snapshot), + ), Effect.retry({ times: DESCRIBE_ALL_GENERATION_RETRIES }), ); const policyRules = yield* listActivePolicyRuleSet(); From ded81de2a1049b20c8ee1a43b3b09379ed2b410e Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 4 Sep 2026 00:22:09 -0700 Subject: [PATCH 06/19] Catalog manifest per connection; passthrough refuses tightened policy; search tools are codemode-only --- apps/cli/src/main.ts | 7 + .../0019_connection_tools_manifest.sql | 1 + apps/cloud/drizzle/meta/0019_snapshot.json | 1522 +++++++++++++++++ apps/cloud/drizzle/meta/_journal.json | 7 + apps/cloud/src/db/executor-schema.ts | 1 + apps/local/src/db/executor-schema.ts | 1 + packages/core/sdk/src/core-schema.ts | 12 + packages/core/sdk/src/executor.test.ts | 30 + packages/core/sdk/src/executor.ts | 188 +- .../hosts/mcp/src/passthrough-tools.test.ts | 28 + packages/hosts/mcp/src/tool-server.ts | 54 +- .../react/src/components/mcp-install-card.tsx | 19 +- 12 files changed, 1796 insertions(+), 74 deletions(-) create mode 100644 apps/cloud/drizzle/0019_connection_tools_manifest.sql create mode 100644 apps/cloud/drizzle/meta/0019_snapshot.json diff --git a/apps/cli/src/main.ts b/apps/cli/src/main.ts index a3101b2ff4..d14aad0f21 100644 --- a/apps/cli/src/main.ts +++ b/apps/cli/src/main.ts @@ -2936,6 +2936,13 @@ const mcpCommand = Command.make( ({ scope, elicitationMode, noArtifacts, searchTools, toolMode, integrations }) => Effect.gen(function* () { applyScope(scope); + if (toolMode === "passthrough" && searchTools) { + return yield* Effect.fail( + new Error( + "--search-tools is a codemode option; passthrough already lists every tool. Drop --search-tools or --mode passthrough.", + ), + ); + } yield* runStdioMcpSession({ elicitationMode, artifacts: !noArtifacts, diff --git a/apps/cloud/drizzle/0019_connection_tools_manifest.sql b/apps/cloud/drizzle/0019_connection_tools_manifest.sql new file mode 100644 index 0000000000..ac5b347cb7 --- /dev/null +++ b/apps/cloud/drizzle/0019_connection_tools_manifest.sql @@ -0,0 +1 @@ +ALTER TABLE "connection" ADD COLUMN "tools_manifest" json; \ No newline at end of file diff --git a/apps/cloud/drizzle/meta/0019_snapshot.json b/apps/cloud/drizzle/meta/0019_snapshot.json new file mode 100644 index 0000000000..0358929a2b --- /dev/null +++ b/apps/cloud/drizzle/meta/0019_snapshot.json @@ -0,0 +1,1522 @@ +{ + "id": "54aeebc1-2d84-4b48-be95-cef046b1004b", + "prevId": "72cab41c-f919-4f73-812f-77f72e87cdaf", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memberships": { + "name": "memberships", + "schema": "", + "columns": { + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "memberships_account_id_accounts_id_fk": { + "name": "memberships_account_id_accounts_id_fk", + "tableFrom": "memberships", + "tableTo": "accounts", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memberships_organization_id_organizations_id_fk": { + "name": "memberships_organization_id_organizations_id_fk", + "tableFrom": "memberships", + "tableTo": "organizations", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "memberships_account_id_organization_id_pk": { + "name": "memberships_account_id_organization_id_pk", + "columns": ["account_id", "organization_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organizations": { + "name": "organizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organizations_slug_unique": { + "name": "organizations_slug_unique", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.artifact": { + "name": "artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bindings": { + "name": "bindings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "preview": { + "name": "preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "artifact_uidx": { + "name": "artifact_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blob": { + "name": "blob", + "schema": "", + "columns": { + "namespace": { + "name": "namespace", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "blob_id_uidx": { + "name": "blob_id_uidx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.connection": { + "name": "connection", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "item_ids": { + "name": "item_ids", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_health": { + "name": "last_health", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "tools_synced_at": { + "name": "tools_synced_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "tools_manifest": { + "name": "tools_manifest", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "oauth_client": { + "name": "oauth_client", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_client_owner": { + "name": "oauth_client_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_item_id": { + "name": "refresh_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "oauth_scope": { + "name": "oauth_scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_token_url": { + "name": "oauth_token_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_state": { + "name": "provider_state", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "connection_uidx": { + "name": "connection_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.definition": { + "name": "definition", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema": { + "name": "schema", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "definition_uidx": { + "name": "definition_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration": { + "name": "integration", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "health_check": { + "name": "health_check", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "config_revised_at": { + "name": "config_revised_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "can_remove": { + "name": "can_remove", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "can_refresh": { + "name": "can_refresh", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "integration_uidx": { + "name": "integration_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "authorization_url": { + "name": "authorization_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_url": { + "name": "token_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "grant": { + "name": "grant", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret_item_id": { + "name": "client_secret_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_kind": { + "name": "origin_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_integration": { + "name": "origin_integration", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_issuer": { + "name": "origin_issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_redirect_uri": { + "name": "origin_redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_client_uidx": { + "name": "oauth_client_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_session": { + "name": "oauth_session", + "schema": "", + "columns": { + "state": { + "name": "state", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "client_slug": { + "name": "client_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "integration": { + "name": "integration", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_url": { + "name": "redirect_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pkce_verifier": { + "name": "pkce_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_session_uidx": { + "name": "oauth_session_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_storage": { + "name": "plugin_storage", + "schema": "", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "collection": { + "name": "collection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "plugin_storage_uidx": { + "name": "plugin_storage_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "plugin_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "collection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.private_executor_cloud_settings": { + "name": "private_executor_cloud_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "default": "'1.0.0'" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.subject": { + "name": "subject", + "schema": "", + "columns": { + "external_id": { + "name": "external_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "subject_uidx": { + "name": "subject_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool": { + "name": "tool", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_schema": { + "name": "input_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "output_schema": { + "name": "output_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "annotations": { + "name": "annotations", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "generation": { + "name": "generation", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_uidx": { + "name": "tool_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool_policy": { + "name": "tool_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "pattern": { + "name": "pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_policy_uidx": { + "name": "tool_policy_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/apps/cloud/drizzle/meta/_journal.json b/apps/cloud/drizzle/meta/_journal.json index c166335db2..e0bb0fcdda 100644 --- a/apps/cloud/drizzle/meta/_journal.json +++ b/apps/cloud/drizzle/meta/_journal.json @@ -134,6 +134,13 @@ "when": 1788505381630, "tag": "0018_tool_generation", "breakpoints": true + }, + { + "idx": 19, + "version": "7", + "when": 1788506387005, + "tag": "0019_connection_tools_manifest", + "breakpoints": true } ] } diff --git a/apps/cloud/src/db/executor-schema.ts b/apps/cloud/src/db/executor-schema.ts index dd8b16a7d5..0ffd5e5ba0 100644 --- a/apps/cloud/src/db/executor-schema.ts +++ b/apps/cloud/src/db/executor-schema.ts @@ -62,6 +62,7 @@ export const connection = pgTable( description: text("description"), last_health: json("last_health"), tools_synced_at: bigint("tools_synced_at", { mode: "bigint" }), + tools_manifest: json("tools_manifest"), oauth_client: text("oauth_client"), oauth_client_owner: text("oauth_client_owner"), refresh_item_id: text("refresh_item_id"), diff --git a/apps/local/src/db/executor-schema.ts b/apps/local/src/db/executor-schema.ts index e12029d8b6..97f2bd462d 100644 --- a/apps/local/src/db/executor-schema.ts +++ b/apps/local/src/db/executor-schema.ts @@ -39,6 +39,7 @@ export const connection = sqliteTable( item_ids: text("item_ids").notNull(), credential_write: text("credential_write"), identity_label: text("identity_label"), + tools_manifest: text("tools_manifest"), oauth_client: text("oauth_client"), oauth_client_owner: text("oauth_client_owner"), refresh_item_id: text("refresh_item_id"), diff --git a/packages/core/sdk/src/core-schema.ts b/packages/core/sdk/src/core-schema.ts index 7d2f22ff47..7fb334bb99 100644 --- a/packages/core/sdk/src/core-schema.ts +++ b/packages/core/sdk/src/core-schema.ts @@ -227,6 +227,18 @@ export const coreTables = defineTables({ // Epoch ms of the last tool (re)production for this connection. Stale // vs the integration's `config_revised_at` → re-produced on next read. tools_synced_at: nullableBigintColumn("tools_synced_at"), + // The catalog MANIFEST: which build is active for this connection and + // how many tool + definition rows it wrote, as JSON + // `{ generation, tools, definitions }`. Written LAST by every rebuild, + // after all rows are in. A reader joining tools to definitions + // (`tools.describeAll`) accepts the rows only when every one carries + // this generation and the counts match — which is what proves the + // catalog is whole on a backend that commits each statement on its own + // (D1) and across isolates the per-executor write lock cannot see. + // Null for connections built before the manifest existed; such a + // catalog is served on the (weaker) same-generation check alone until + // its next rebuild stamps it. + tools_manifest: nullableJsonColumn("tools_manifest"), oauth_client: nullableTextColumn("oauth_client"), // The OWNER of `oauth_client` (a Personal connection may be minted through // a shared Workspace app), set together with `oauth_client`; null for diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 2531c41c37..ffd51964ee 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -879,6 +879,36 @@ describe("createExecutor", () => { }), ); + // On D1 every statement of a rebuild commits on its own, so a reader can + // land after the definitions were deleted and before they were reinserted + // (or after only some tool batches landed). The connection's manifest — + // written LAST — is what lets `describeAll` tell that apart from a finished + // build. Modelled here by deleting the definitions out from under a stamped + // catalog: the row counts no longer match the manifest. + it.effect("tools.describeAll refuses a catalog whose rows do not match its manifest", () => + Effect.gen(function* () { + const config = makeTestConfig({ plugins: [demoPlugin] as const }); + const executor = yield* createExecutor(config); + yield* executor.demo.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + const whole = yield* executor.tools.describeAll(); + expect(whole.length).toBeGreaterThan(0); + + // The half-written window: definitions gone, manifest still says N. + yield* Effect.promise(() => + config.db.deleteMany("definition", { where: (b) => b("integration", "=", String(INTEG)) }), + ); + const outcome = yield* Effect.result(executor.tools.describeAll()); + expect(Result.isFailure(outcome), "a partial catalog is refused, not served").toBe(true); + }), + ); + it.effect("execute dispatches a connection-produced tool to the owning plugin", () => Effect.gen(function* () { const executor = yield* makeTestExecutor({ diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 448aa74589..9d684a25b4 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -913,6 +913,23 @@ const validateExecutorDbTables = (required: FumaTables, actual: FumaTables): voi * rebuild storm fails loudly instead of spinning. */ const DESCRIBE_ALL_GENERATION_RETRIES = 3; +/** What a finished catalog build leaves on its connection row (see the + * `tools_manifest` column): the active build and the row counts it wrote. */ +const CatalogManifest = Schema.Struct({ + /** Null for a build that wrote no rows at all (every row it would have + * written is absent, so there is no generation to match). */ + generation: Schema.NullOr(Schema.String), + tools: Schema.Number, + definitions: Schema.Number, +}); +type CatalogManifest = typeof CatalogManifest.Type; +const decodeCatalogManifest = Schema.decodeUnknownOption(CatalogManifest); +const emptyCatalogManifest = (): CatalogManifest => ({ + generation: null, + tools: 0, + definitions: 0, +}); + const storageFailureFromUnknown = (message: string, cause: unknown): StorageFailure => isStorageFailure(cause) ? cause : new StorageError({ message, cause }); @@ -3499,25 +3516,29 @@ export const createExecutor = + b.and( + filter?.integration === undefined + ? true + : b("integration", "=", String(filter.integration)), + filter?.owner === undefined ? true : b("owner", "=", filter.owner), + filter?.connection === undefined ? true : b("name", "=", String(filter.connection)), + ), + select: ["owner", "integration", "name", "tools_manifest"], + }), }); const connectionKey = (row: { readonly owner: string; readonly integration: string; readonly connection: string; }): string => `${row.owner}\u0000${row.integration}\u0000${row.connection}`; - // Per connection, both halves must come from ONE build. A build stamps - // every row it writes with the same opaque `generation`, so the proof - // is: the tool rows carry exactly one generation, the definition rows - // carry exactly one, and they are equal. That also catches the windows - // a non-transactional backend (D1 auto-commits each statement) can - // expose mid-rebuild — tools deleted but not reinserted (zero tool - // generations while definitions still carry the old one), or - // definitions deleted but not reinserted (tools carry the NEW - // generation, definitions carry none) — because a build with - // definitions always writes both. A connection whose build wrote no - // definitions legitimately has tools and no definition rows; only a - // definition-bearing build can be caught half-written, and it is, - // by the mismatch. Rows from before the column existed carry null on - // both sides and compare equal, which is exactly right: they were - // written together. + // Per connection, the rows served must be exactly ONE finished build. + // A build stamps every row it writes with one opaque `generation`, and + // writes the connection's manifest — that generation plus the row + // counts — as its LAST statement. So with a manifest the proof is + // complete: every tool row and every definition row carries the + // manifest's generation, and there are exactly as many of each as it + // says. That holds on a backend that commits each statement on its + // own (D1): a read that lands mid-rebuild sees either the old + // manifest with some rows already replaced (generation mismatch), or + // fewer rows than the manifest counts (still inserting), or the new + // manifest with everything in place. A competing writer in another + // isolate, which the per-executor write lock cannot serialize, is + // caught the same way: whatever it leaves behind does not match the + // manifest that was written last. A connection with no manifest + // (built before the column existed) falls back to the weaker check — + // all rows of one generation, both tables agreeing — until its next + // rebuild stamps it. + type GenerationRow = { + readonly owner: string; + readonly integration: string; + readonly connection: string; + readonly generation: string | null; + }; const mixedGenerations = (snapshot: { - readonly rows: ReadonlyArray<{ + readonly rows: ReadonlyArray; + readonly definitionRows: ReadonlyArray; + readonly manifests: ReadonlyArray<{ readonly owner: string; readonly integration: string; - readonly connection: string; - readonly generation: string | null; - }>; - readonly definitionRows: ReadonlyArray<{ - readonly owner: string; - readonly integration: string; - readonly connection: string; - readonly generation: string | null; + readonly name: string; + readonly tools_manifest: unknown; }>; }): boolean => { - const toolGenerations = new Map>(); - for (const row of snapshot.rows) { - const key = connectionKey(row); - (toolGenerations.get(key) ?? toolGenerations.set(key, new Set()).get(key)!).add( - row.generation, + type Tally = { readonly generations: Set; count: number }; + const tally = (rows: ReadonlyArray): Map => { + const out = new Map(); + for (const row of rows) { + const key = connectionKey(row); + const entry = out.get(key) ?? { generations: new Set(), count: 0 }; + entry.generations.add(row.generation); + entry.count += 1; + out.set(key, entry); + } + return out; + }; + const tools = tally(snapshot.rows); + const definitions = tally(snapshot.definitionRows); + const manifests = new Map(); + for (const row of snapshot.manifests) { + manifests.set( + connectionKey({ + owner: row.owner, + integration: row.integration, + connection: row.name, + }), + Option.getOrNull(decodeCatalogManifest(decodeJsonColumn(row.tools_manifest))), ); } - const definitionGenerations = new Map>(); - for (const def of snapshot.definitionRows) { - const key = connectionKey(def); - ( - definitionGenerations.get(key) ?? definitionGenerations.set(key, new Set()).get(key)! - ).add(def.generation); - } - for (const [key, tools] of toolGenerations) { - if (tools.size !== 1) return true; - const defs = definitionGenerations.get(key); - if (defs === undefined) continue; - if (defs.size !== 1) return true; - if ([...tools][0] !== [...defs][0]) return true; - } - // Definitions whose tools are absent: a build that wrote definitions - // also wrote tools, so this is the tools-deleted window. - for (const key of definitionGenerations.keys()) { - if (!toolGenerations.has(key)) return true; + const keys = new Set([...tools.keys(), ...definitions.keys(), ...manifests.keys()]); + for (const key of keys) { + const t = tools.get(key); + const d = definitions.get(key); + const manifest = manifests.get(key) ?? null; + if (manifest) { + // Strong check: exact generation and exact counts on both sides. + if ((t?.count ?? 0) !== manifest.tools) return true; + if ((d?.count ?? 0) !== manifest.definitions) return true; + if (t && (t.generations.size !== 1 || !t.generations.has(manifest.generation))) { + return true; + } + if (d && (d.generations.size !== 1 || !d.generations.has(manifest.generation))) { + return true; + } + continue; + } + // Weak check (no manifest yet): one generation per table, agreeing. + if (t && t.generations.size !== 1) return true; + if (d && d.generations.size !== 1) return true; + if (t && d && [...t.generations][0] !== [...d.generations][0]) return true; + // Definitions with no tools: a build that writes definitions also + // writes tools, so this is a half-replaced catalog. + if (d && !t) return true; } return false; }; diff --git a/packages/hosts/mcp/src/passthrough-tools.test.ts b/packages/hosts/mcp/src/passthrough-tools.test.ts index 408d696a9d..ead728db1c 100644 --- a/packages/hosts/mcp/src/passthrough-tools.test.ts +++ b/packages/hosts/mcp/src/passthrough-tools.test.ts @@ -427,6 +427,34 @@ describe("passthrough mode server", () => { ); }); + it("refuses a call when policy tightened after the list and the client cannot prompt", async () => { + // `issues.list` was advertised `destructiveHint: false`. An admin then + // adds a require_approval rule; the executor now raises its approval gate + // on invoke. The session never told the client to ask, so the gate must + // NOT be accepted on the client's behalf: with no elicitation capability + // the call fails closed and says to reconnect. + const seen: string[] = []; + await withClient( + { + engine: elicitingEngine([{ source: "policy", request: approvalGate }], seen), + mode: "passthrough", + tools: { describeAll: () => Effect.succeed(CATALOG) }, + }, + async (client) => { + const listed = await client.listTools(); + const list = listed.tools.find((tool) => tool.name === "github__issues_list"); + expect(list?.annotations?.destructiveHint).toBe(false); + const result = await client.callTool({ name: "github__issues_list", arguments: {} }); + expect(seen).toEqual(["policy:decline"]); + expect(result.isError).toBe(true); + expect(result.structuredContent).toMatchObject({ + status: "error", + error: { code: "approval_required_after_list" }, + }); + }, + ); + }); + it("never auto-accepts a tool-raised prompt, even one with an empty schema", async () => { // Same wire shape as the approval gate, but raised by the TOOL: a // per-site grant whose terms live in `meta`. Provenance, not shape, diff --git a/packages/hosts/mcp/src/tool-server.ts b/packages/hosts/mcp/src/tool-server.ts index 46ca66d9e9..2e566830aa 100644 --- a/packages/hosts/mcp/src/tool-server.ts +++ b/packages/hosts/mcp/src/tool-server.ts @@ -195,7 +195,9 @@ type SharedMcpServerConfig = { * the `execute` description lists). The tools exist to carry the namespaces * into the model's context as tool names; each call routes through the same * execution flow as `tools.search({ namespace })` inside `execute`, so the - * results match what code-side search returns. + * results match what code-side search returns. Codemode only: passthrough + * ignores it (its whole catalog is already on the tool list, and search + * results point at an `execute` tool passthrough does not serve). */ readonly searchToolsEnabled?: boolean; /** @@ -748,6 +750,25 @@ const elicitationUnsupportedResult = ( }; }; +/** + * A passthrough tool this session advertised as not needing approval now + * does (a policy was added after the list was built), and the client cannot + * take a native approval prompt. The call is refused — never run without the + * approval the new policy demands — and the fix is a fresh tool list. + */ +const policyTightenedResult = (toolName: string): McpToolResult => { + const message = `Tool ${toolName} now requires the user's approval, but this session advertised it as not requiring approval. Reconnect to refresh the tool list, then call it again so your client can ask for approval.`; + return { + content: [{ type: "text", text: `Error: ${message}` }], + structuredContent: { + status: "error", + error: { code: "approval_required_after_list", message }, + logs: [], + }, + isError: true, + }; +}; + const toMcpPausedResult = (formatted: ReturnType): McpToolResult => ({ content: [{ type: "text", text: formatted.text }], structuredContent: formatted.structured, @@ -1832,16 +1853,30 @@ export const createExecutorMcpServer = ( // reported as unanswerable below — with what was asked, URL included — // instead of as "declined by the user", which nobody did. let unanswerable: ElicitationRequest | undefined; + // Set when the executor's approval gate fired for a tool this session + // advertised as NOT needing approval, and the client cannot take a + // native prompt: policy tightened after the list was built. + let policyTightened = false; const onElicitation: ElicitationHandler = (ctx) => { - // The executor's OWN approval gate is the only thing accepted - // inline: the harness already prompted (or chose not to) from the - // advertised annotations, so a second server-side gate would - // double-prompt. Provenance is stamped by the executor, not read - // off the request shape — a tool-raised prompt with an empty schema - // can still carry terms of its own (a permanent site grant), and - // must reach a human or fail, never be answered for them. + // The executor's OWN approval gate is accepted inline ONLY when the + // tool was advertised as needing approval: the harness prompted (or + // chose not to) from that annotation, so a second server-side gate + // would double-prompt. If policy has since tightened — the session + // still advertises `destructiveHint: false` but the executor now + // requires approval — the client never had the chance to ask, so + // the gate goes to the client natively, or fails the call when it + // cannot, with a message that says to reconnect for the new list. + // Provenance is stamped by the executor, not read off the request + // shape — a tool-raised prompt with an empty schema can still carry + // terms of its own (a permanent site grant), and must reach a human + // or fail, never be answered for them. if (ctx.source === "policy") { - return Effect.succeed({ action: "accept" as const, content: {} }); + if (tool.projection.policy === "require_approval") { + return Effect.succeed({ action: "accept" as const, content: {} }); + } + if (supportsForm) return native(ctx); + policyTightened = true; + return Effect.succeed({ action: "decline" as const }); } // Anything the tool itself asked for goes to the client natively // when it can take it; the native bridge already turns a URL @@ -1855,6 +1890,7 @@ export const createExecutorMcpServer = ( const outcome = yield* engine.execute(passthroughCallCode(address, args), { onElicitation, }); + if (policyTightened) return policyTightenedResult(tool.name); if (unanswerable) return elicitationUnsupportedResult(tool.name, unanswerable); return toPassthroughResult(outcome); }).pipe( diff --git a/packages/react/src/components/mcp-install-card.tsx b/packages/react/src/components/mcp-install-card.tsx index 655cb2f7ac..2203e9fc26 100644 --- a/packages/react/src/components/mcp-install-card.tsx +++ b/packages/react/src/components/mcp-install-card.tsx @@ -144,7 +144,11 @@ export const buildMcpHttpEndpoint = (input: { params.push(["elicitation_mode", input.elicitationMode]); } if (input.artifacts === false) params.push(["artifacts", "false"]); - if (input.searchTools === true) params.push(["search_tools", "true"]); + // Search tools and artifacts are codemode affordances; passthrough serves + // neither, so the URL never claims them alongside it. + if (input.searchTools === true && input.toolMode !== "passthrough") { + params.push(["search_tools", "true"]); + } if (input.toolMode === "passthrough") params.push(["mode", "passthrough"]); if (params.length === 0) return endpoint; @@ -207,7 +211,7 @@ export const buildMcpInstallCommand = (input: { if (input.artifacts === false) { innerArgs.push("--no-artifacts"); } - if (input.searchTools === true) { + if (input.searchTools === true && input.toolMode !== "passthrough") { innerArgs.push("--search-tools"); } if (input.toolMode === "passthrough") { @@ -348,13 +352,16 @@ export function McpInstallCard(props: { className?: string }) {
Integration search tools
- {searchTools - ? "One search tool per connected integration, so agents see your integrations as tool names." - : "Disabled: agents discover tools through search inside execute."} + {toolMode === "passthrough" + ? "Not needed when tools are exposed directly: every tool is already on the list." + : searchTools + ? "One search tool per connected integration, so agents see your integrations as tool names." + : "Disabled: agents discover tools through search inside execute."}
{ setPreferences((current) => ({ ...current, searchTools: next })); trackEvent("mcp_install_search_tools_toggled", { search_tools: next }); From a21589746d154a7593b45b74e29e07bad15f72ce Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 4 Sep 2026 00:30:39 -0700 Subject: [PATCH 07/19] Catalog without a manifest is refused and rescanned; rebuilds clear the sync stamp first --- packages/core/sdk/src/core-schema.ts | 9 ++-- packages/core/sdk/src/executor.test.ts | 43 +++++++++++++++++ packages/core/sdk/src/executor.ts | 65 ++++++++++++++++---------- 3 files changed, 89 insertions(+), 28 deletions(-) diff --git a/packages/core/sdk/src/core-schema.ts b/packages/core/sdk/src/core-schema.ts index 7fb334bb99..dfdde10088 100644 --- a/packages/core/sdk/src/core-schema.ts +++ b/packages/core/sdk/src/core-schema.ts @@ -235,9 +235,12 @@ export const coreTables = defineTables({ // this generation and the counts match — which is what proves the // catalog is whole on a backend that commits each statement on its own // (D1) and across isolates the per-executor write lock cannot see. - // Null for connections built before the manifest existed; such a - // catalog is served on the (weaker) same-generation check alone until - // its next rebuild stamps it. + // Null for connections built before the manifest existed. A null + // manifest means "no proven-whole catalog": `tools.describeAll` refuses + // the connection, and the stale-catalog scan treats it as needing a + // rebuild, which stamps it. Rows written by a rebuild that died before + // its stamp are refused the same way, and `tools_synced_at` is cleared + // FIRST on every rebuild so such a death is also rescanned. tools_manifest: nullableJsonColumn("tools_manifest"), oauth_client: nullableTextColumn("oauth_client"), // The OWNER of `oauth_client` (a Personal connection may be minted through diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index ffd51964ee..cbb0095078 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -909,6 +909,49 @@ describe("createExecutor", () => { }), ); + // A catalog written before the manifest column existed has rows and no + // manifest — exactly what a rebuild that died before its stamp leaves too. + // Nothing proves it whole, so it is not served; and because a null manifest + // marks the connection stale, the read that trips on it is the read that + // rebuilds and stamps it. Modelled by wiping the manifest under a live + // catalog. + it.effect( + "tools.describeAll refuses a catalog with no manifest and the next read rebuilds it", + () => + Effect.gen(function* () { + const config = makeTestConfig({ plugins: [demoPlugin] as const }); + const executor = yield* createExecutor(config); + yield* executor.demo.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + expect((yield* executor.tools.describeAll()).length).toBeGreaterThan(0); + + // Pre-upgrade shape: rows present, manifest absent, stamp present. + yield* Effect.promise(() => + config.db.updateMany("connection", { + where: (b) => b("integration", "=", String(INTEG)), + set: { tools_manifest: null }, + }), + ); + // `describeAll` runs the stale scan first, which sees the null manifest + // and rebuilds — so the served catalog is the freshly stamped one, and + // the manifest is back. + const served = yield* executor.tools.describeAll(); + expect(served.map((tool) => tool.name).sort()).toEqual(["inspect", "run"]); + const [row] = yield* Effect.promise(() => + config.db.findMany("connection", { + where: (b) => b("integration", "=", String(INTEG)), + }), + ); + expect(row?.tools_manifest, "the rebuild stamped a manifest").not.toBeNull(); + }), + ); + it.effect("execute dispatches a connection-produced tool to the owning plugin", () => Effect.gen(function* () { const executor = yield* makeTestExecutor({ diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 9d684a25b4..e8d433c7b9 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -3540,6 +3540,18 @@ export const createExecutor = staleBefore === null - ? b.isNull("tools_synced_at") - : b.or(b.isNull("tools_synced_at"), b("tools_synced_at", "<", staleBefore)), + ? b.or(b.isNull("tools_synced_at"), b.isNull("tools_manifest")) + : b.or( + b.isNull("tools_synced_at"), + b.isNull("tools_manifest"), + b("tools_synced_at", "<", staleBefore), + ), }); // Each rebuild is an independent upstream listing, so they run together // rather than one after another: a host with many stale remote-catalog @@ -5531,7 +5553,7 @@ export const createExecutor = Date: Fri, 4 Sep 2026 00:39:55 -0700 Subject: [PATCH 08/19] Rebuilds claim an ownership token; a build that lost its claim never stamps --- .../drizzle/0020_connection_tools_rebuild.sql | 1 + apps/cloud/drizzle/meta/0020_snapshot.json | 1528 +++++++++++++++++ apps/cloud/drizzle/meta/_journal.json | 7 + apps/cloud/src/db/executor-schema.ts | 1 + apps/local/src/db/executor-schema.ts | 1 + packages/core/sdk/src/core-schema.ts | 9 + packages/core/sdk/src/executor.test.ts | 99 ++ packages/core/sdk/src/executor.ts | 69 +- 8 files changed, 1701 insertions(+), 14 deletions(-) create mode 100644 apps/cloud/drizzle/0020_connection_tools_rebuild.sql create mode 100644 apps/cloud/drizzle/meta/0020_snapshot.json diff --git a/apps/cloud/drizzle/0020_connection_tools_rebuild.sql b/apps/cloud/drizzle/0020_connection_tools_rebuild.sql new file mode 100644 index 0000000000..f24b275afc --- /dev/null +++ b/apps/cloud/drizzle/0020_connection_tools_rebuild.sql @@ -0,0 +1 @@ +ALTER TABLE "connection" ADD COLUMN "tools_rebuild" text; \ No newline at end of file diff --git a/apps/cloud/drizzle/meta/0020_snapshot.json b/apps/cloud/drizzle/meta/0020_snapshot.json new file mode 100644 index 0000000000..ce29f0cd5c --- /dev/null +++ b/apps/cloud/drizzle/meta/0020_snapshot.json @@ -0,0 +1,1528 @@ +{ + "id": "2f2214a9-8d41-43ca-9447-1525d35d9de8", + "prevId": "54aeebc1-2d84-4b48-be95-cef046b1004b", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.memberships": { + "name": "memberships", + "schema": "", + "columns": { + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "memberships_account_id_accounts_id_fk": { + "name": "memberships_account_id_accounts_id_fk", + "tableFrom": "memberships", + "tableTo": "accounts", + "columnsFrom": ["account_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "memberships_organization_id_organizations_id_fk": { + "name": "memberships_organization_id_organizations_id_fk", + "tableFrom": "memberships", + "tableTo": "organizations", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "memberships_account_id_organization_id_pk": { + "name": "memberships_account_id_organization_id_pk", + "columns": ["account_id", "organization_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.organizations": { + "name": "organizations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "organizations_slug_unique": { + "name": "organizations_slug_unique", + "columns": [ + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.artifact": { + "name": "artifact", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bindings": { + "name": "bindings", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "preview": { + "name": "preview", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "artifact_uidx": { + "name": "artifact_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.blob": { + "name": "blob", + "schema": "", + "columns": { + "namespace": { + "name": "namespace", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "blob_id_uidx": { + "name": "blob_id_uidx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.connection": { + "name": "connection", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "item_ids": { + "name": "item_ids", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_health": { + "name": "last_health", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "tools_synced_at": { + "name": "tools_synced_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "tools_manifest": { + "name": "tools_manifest", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "tools_rebuild": { + "name": "tools_rebuild", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_client": { + "name": "oauth_client", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_client_owner": { + "name": "oauth_client_owner", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_item_id": { + "name": "refresh_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "oauth_scope": { + "name": "oauth_scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "oauth_token_url": { + "name": "oauth_token_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_state": { + "name": "provider_state", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "connection_uidx": { + "name": "connection_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.definition": { + "name": "definition", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "schema": { + "name": "schema", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "generation": { + "name": "generation", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "definition_uidx": { + "name": "definition_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.integration": { + "name": "integration", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "config": { + "name": "config", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "health_check": { + "name": "health_check", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "config_revised_at": { + "name": "config_revised_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "can_remove": { + "name": "can_remove", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "can_refresh": { + "name": "can_refresh", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "integration_uidx": { + "name": "integration_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_client": { + "name": "oauth_client", + "schema": "", + "columns": { + "slug": { + "name": "slug", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "authorization_url": { + "name": "authorization_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_url": { + "name": "token_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "grant": { + "name": "grant", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_secret_item_id": { + "name": "client_secret_item_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "credential_write": { + "name": "credential_write", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "token_endpoint_auth_method": { + "name": "token_endpoint_auth_method", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_kind": { + "name": "origin_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_integration": { + "name": "origin_integration", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_issuer": { + "name": "origin_issuer", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_redirect_uri": { + "name": "origin_redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_client_uidx": { + "name": "oauth_client_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "slug", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_session": { + "name": "oauth_session", + "schema": "", + "columns": { + "state": { + "name": "state", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "client_slug": { + "name": "client_slug", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "integration": { + "name": "integration", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "template": { + "name": "template", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_url": { + "name": "redirect_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "pkce_verifier": { + "name": "pkce_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "identity_label": { + "name": "identity_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_session_uidx": { + "name": "oauth_session_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.plugin_storage": { + "name": "plugin_storage", + "schema": "", + "columns": { + "plugin_id": { + "name": "plugin_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "collection": { + "name": "collection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "plugin_storage_uidx": { + "name": "plugin_storage_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "plugin_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "collection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.private_executor_cloud_settings": { + "name": "private_executor_cloud_settings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "version": { + "name": "version", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "default": "'1.0.0'" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.subject": { + "name": "subject", + "schema": "", + "columns": { + "external_id": { + "name": "external_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "subject_uidx": { + "name": "subject_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool": { + "name": "tool", + "schema": "", + "columns": { + "integration": { + "name": "integration", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "connection": { + "name": "connection", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "plugin_id": { + "name": "plugin_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "input_schema": { + "name": "input_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "output_schema": { + "name": "output_schema", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "annotations": { + "name": "annotations", + "type": "json", + "primaryKey": false, + "notNull": false + }, + "generation": { + "name": "generation", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_uidx": { + "name": "tool_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "integration", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "connection", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tool_policy": { + "name": "tool_policy", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "pattern": { + "name": "pattern", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "position": { + "name": "position", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "row_id": { + "name": "row_id", + "type": "varchar(255)", + "primaryKey": true, + "notNull": true + }, + "tenant": { + "name": "tenant", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "tool_policy_uidx": { + "name": "tool_policy_uidx", + "columns": [ + { + "expression": "tenant", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "owner", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "subject", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/apps/cloud/drizzle/meta/_journal.json b/apps/cloud/drizzle/meta/_journal.json index e0bb0fcdda..38dbd78453 100644 --- a/apps/cloud/drizzle/meta/_journal.json +++ b/apps/cloud/drizzle/meta/_journal.json @@ -141,6 +141,13 @@ "when": 1788506387005, "tag": "0019_connection_tools_manifest", "breakpoints": true + }, + { + "idx": 20, + "version": "7", + "when": 1788507464393, + "tag": "0020_connection_tools_rebuild", + "breakpoints": true } ] } diff --git a/apps/cloud/src/db/executor-schema.ts b/apps/cloud/src/db/executor-schema.ts index 0ffd5e5ba0..0137559578 100644 --- a/apps/cloud/src/db/executor-schema.ts +++ b/apps/cloud/src/db/executor-schema.ts @@ -63,6 +63,7 @@ export const connection = pgTable( last_health: json("last_health"), tools_synced_at: bigint("tools_synced_at", { mode: "bigint" }), tools_manifest: json("tools_manifest"), + tools_rebuild: text("tools_rebuild"), oauth_client: text("oauth_client"), oauth_client_owner: text("oauth_client_owner"), refresh_item_id: text("refresh_item_id"), diff --git a/apps/local/src/db/executor-schema.ts b/apps/local/src/db/executor-schema.ts index 97f2bd462d..81afacb562 100644 --- a/apps/local/src/db/executor-schema.ts +++ b/apps/local/src/db/executor-schema.ts @@ -40,6 +40,7 @@ export const connection = sqliteTable( credential_write: text("credential_write"), identity_label: text("identity_label"), tools_manifest: text("tools_manifest"), + tools_rebuild: text("tools_rebuild"), oauth_client: text("oauth_client"), oauth_client_owner: text("oauth_client_owner"), refresh_item_id: text("refresh_item_id"), diff --git a/packages/core/sdk/src/core-schema.ts b/packages/core/sdk/src/core-schema.ts index dfdde10088..1b9ca8ace8 100644 --- a/packages/core/sdk/src/core-schema.ts +++ b/packages/core/sdk/src/core-schema.ts @@ -242,6 +242,15 @@ export const coreTables = defineTables({ // its stamp are refused the same way, and `tools_synced_at` is cleared // FIRST on every rebuild so such a death is also rescanned. tools_manifest: nullableJsonColumn("tools_manifest"), + // The rebuild OWNERSHIP token: the id of the build currently replacing + // this connection's catalog, or null when no build is in flight. A + // rebuild claims it as its first statement (together with clearing + // `tools_synced_at`) and its final stamp is conditioned on still holding + // it. Two builds in different isolates cannot both finish: whichever + // claims last owns the token, and the other's stamp finds it changed + // and writes nothing — so a manifest can never describe rows another + // build has since replaced. Cleared by the stamp that wins. + tools_rebuild: nullableTextColumn("tools_rebuild"), oauth_client: nullableTextColumn("oauth_client"), // The OWNER of `oauth_client` (a Personal connection may be minted through // a shared Workspace app), set together with `oauth_client`; null for diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index cbb0095078..021757b256 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -952,6 +952,105 @@ describe("createExecutor", () => { }), ); + // Two builds of one connection in two isolates, on a backend where every + // statement commits on its own. B claims the rebuild token after A has + // started; A finishes and tries to stamp; B then dies mid-replacement. A's + // stamp must NOT land (A no longer owns the connection), so the row keeps a + // null sync stamp and the stale scan rebuilds it — instead of A's manifest + // describing rows B has since torn out, refused forever. Modelled with a + // storage proxy that lets "B" claim the token between A's mark and A's + // stamp, then simulates B's death by deleting the rows and never stamping. + it.effect( + "a rebuild that lost its claim to a competing build does not stamp, and the row stays stale", + () => + Effect.gen(function* () { + const config = makeTestConfig({ plugins: [demoPlugin] as const }); + const raceState: { armed: boolean } = { armed: false }; + const wrap = (inner: FumaDb): FumaDb => + new Proxy(inner, { + get(target, prop) { + if (prop === "withContext") { + return (context: unknown) => + wrap((target.withContext as (c: unknown) => FumaDb)(context)); + } + if (prop === "transaction") { + return (run: (tx: FumaDb) => Promise) => + (target.transaction as (r: (tx: FumaDb) => Promise) => Promise)( + (tx) => run(wrap(tx)), + ); + } + if (prop === "createMany") { + return async (table: unknown, rows: unknown) => { + const out = await ( + target.createMany as (t: unknown, r: unknown) => Promise + )(table, rows); + // After A has written its definitions (its last rows) and + // before A stamps: B claims the connection, replaces the + // rows, and dies. + if (raceState.armed && table === "definition") { + raceState.armed = false; + await (target.updateMany as (t: unknown, q: unknown) => Promise)( + "connection", + { + where: (b: { (c: string, op: string, v: unknown): unknown }) => + b("integration", "=", String(INTEG)), + set: { tools_synced_at: null, tools_rebuild: "build-B" }, + }, + ); + await (target.deleteMany as (t: unknown, q: unknown) => Promise)( + "definition", + { + where: (b: { (c: string, op: string, v: unknown): unknown }) => + b("integration", "=", String(INTEG)), + }, + ); + } + return out; + }; + } + return Reflect.get(target, prop); + }, + }); + const executor = yield* createExecutor({ ...config, db: wrap(config.db) }); + yield* executor.demo.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + expect((yield* executor.tools.describeAll()).length).toBeGreaterThan(0); + + // "A" rebuilds; the proxy plays "B" in the middle of it. + raceState.armed = true; + yield* executor.connections.refresh({ owner: "org", integration: INTEG, name: CONN }); + expect(raceState.armed, "B interleaved").toBe(false); + + const [row] = yield* Effect.promise(() => + config.db.findMany("connection", { + where: (b) => b("integration", "=", String(INTEG)), + }), + ); + // A did not stamp: the token is still B's and the sync stamp is null, + // so the stale scan will rebuild this connection. + expect(row?.tools_rebuild, "A's stamp did not clear B's claim").toBe("build-B"); + expect(row?.tools_synced_at, "the connection stays stale-marked").toBeNull(); + + // And the next read, which runs the stale scan, rebuilds and serves + // a whole catalog — never A's manifest over B's torn-out rows. + const served = yield* executor.tools.describeAll(); + expect(served.map((tool) => tool.name).sort()).toEqual(["inspect", "run"]); + const [after] = yield* Effect.promise(() => + config.db.findMany("connection", { + where: (b) => b("integration", "=", String(INTEG)), + }), + ); + expect(after?.tools_rebuild, "the recovering build released the token").toBeNull(); + expect(after?.tools_manifest).not.toBeNull(); + }), + ); + it.effect("execute dispatches a connection-produced tool to the owning plugin", () => Effect.gen(function* () { const executor = yield* makeTestExecutor({ diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index e8d433c7b9..3e2f922a3e 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -3522,11 +3522,16 @@ export const createExecutor = core.updateMany("connection", { - where: connectionWhere, + where: (b: AnyCb) => b.and(connectionWhere(b), b("tools_rebuild", "=", buildId)), set: syncedSet(row, manifest), }); - // The FIRST statement of every replacement: clear the sync stamp while - // keeping the previous manifest. On a backend that commits statement - // by statement (D1), a rebuild that dies after its deletes but before - // `stampSynced` leaves rows that no longer match the old manifest — - // refused by `tools.describeAll` — and, because the stamp is already - // null, the next stale-catalog scan rebuilds it instead of leaving it - // refused until someone refreshes by hand. Interactive backends roll - // the whole transaction back, so there this is a no-op. + // The FIRST statement of every replacement: claim the rebuild token + // and clear the sync stamp, keeping the previous manifest. On a + // backend that commits statement by statement (D1), a rebuild that + // dies after its deletes but before `stampSynced` leaves rows that no + // longer match the old manifest — refused by `tools.describeAll` — + // and, because the stamp is already null, the next stale-catalog scan + // rebuilds it instead of leaving it refused until someone refreshes by + // hand. Interactive backends roll the whole transaction back, so + // there this is a no-op. const markRebuilding = core.updateMany("connection", { where: connectionWhere, - set: { tools_synced_at: null }, + set: { tools_synced_at: null, tools_rebuild: buildId }, }); // A failing sync must not bury a recorded dead grant's `expired` // verdict: this sync's own credential resolution is what discovers @@ -3588,6 +3601,11 @@ export const createExecutor = ({ tenant: keys.tenant, owner: keys.owner, @@ -5987,6 +6005,29 @@ export const createExecutor = + core + .updateMany("connection", { + where: (b: AnyCb) => + b.and( + filter?.integration === undefined + ? true + : b("integration", "=", String(filter.integration)), + filter?.owner === undefined ? true : b("owner", "=", filter.owner), + filter?.connection === undefined + ? true + : b("name", "=", String(filter.connection)), + ), + set: { tools_synced_at: null }, + }) + .pipe(Effect.ignore), + ), ); const policyRules = yield* listActivePolicyRuleSet(); const defsByConnection = new Map>(); From c12285b8cc6f12cc874f99011294f94ec472d838 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 4 Sep 2026 00:58:46 -0700 Subject: [PATCH 09/19] Catalog replacement is one fenced atomic unit; recovery stale-marks exact connections --- .../core/fumadb/src/adapters/drizzle/query.ts | 113 ++++++++ .../core/fumadb/src/adapters/memory/index.ts | 21 ++ packages/core/fumadb/src/query/index.ts | 23 ++ packages/core/fumadb/src/query/orm/index.ts | 56 ++++ packages/core/sdk/src/executor.test.ts | 71 ++--- packages/core/sdk/src/executor.ts | 255 +++++++++++------- packages/core/sdk/src/fuma-runtime.ts | 6 + 7 files changed, 418 insertions(+), 127 deletions(-) diff --git a/packages/core/fumadb/src/adapters/drizzle/query.ts b/packages/core/fumadb/src/adapters/drizzle/query.ts index 21c7bc1e95..e213ed0a26 100644 --- a/packages/core/fumadb/src/adapters/drizzle/query.ts +++ b/packages/core/fumadb/src/adapters/drizzle/query.ts @@ -12,6 +12,15 @@ import { import type { SQLProvider } from "../../shared/providers"; import { type ColumnType, parseDrizzle, type TableType } from "./shared"; +/** Thrown inside a `replaceMany` transaction to abort it when the guard + * matched no row; caught at the boundary and reported as `applied: false`. */ +class ReplaceGuardMiss extends Error { + constructor() { + super("replaceMany guard matched no row"); + this.name = "ReplaceGuardMiss"; + } +} + type P_TableType = PostgreSQL.PgTableWithColumns; type P_ColumnType = PostgreSQL.AnyPgColumn; type P_DBType = PostgreSQL.PgDatabase< @@ -635,6 +644,110 @@ export function fromDrizzle( await query; }, + async replaceMany(plan) { + // Every statement of the plan, built against one handle: the guard + // update first, then the deletes, then parameter-bounded insert batches. + const buildStatements = (handle: typeof db): unknown[] => { + const statements: unknown[] = []; + if (plan.guard) { + const guardTable = toDrizzle(plan.guard.table); + let update = handle.update(guardTable).set(mapValues(plan.guard.set, plan.guard.table)); + if (plan.guard.where) { + update = update.where(buildWhere(toDrizzleColumn, plan.guard.where)) as any; + } + statements.push(update); + } + for (const del of plan.deletes) { + const drizzleTable = toDrizzle(del.table); + let query = handle.delete(drizzleTable); + if (del.where) query = query.where(buildWhere(toDrizzleColumn, del.where)) as any; + statements.push(query); + } + for (const ins of plan.inserts) { + if (ins.values.length === 0) continue; + const drizzleTable = toDrizzle(ins.table); + const values = ins.values.map((v) => mapValues(v, ins.table)); + const columnsPerRow = Math.max(1, Object.keys(values[0]!).length); + const batchSize = parameterBoundedBatchSize(ins.table, columnsPerRow, 0, maxBoundParameters); + for (let i = 0; i < values.length; i += batchSize) { + statements.push(handle.insert(drizzleTable).values(values.slice(i, i + batchSize))); + } + } + return statements; + }; + + // How many rows the guard matched. Drizzle returns driver-specific + // shapes; normalize the common ones (`rowsAffected`, `changes`, + // `rowCount`, `meta.changes`) and treat unknown as "matched" so an + // exotic driver never turns every replace into a no-op. + const guardMatched = (result: unknown): boolean => { + if (!plan.guard) return true; + if (result && typeof result === "object") { + const r = result as Record; + for (const key of ["rowsAffected", "changes", "rowCount"]) { + if (typeof r[key] === "number") return (r[key] as number) > 0; + } + const meta = r["meta"]; + if (meta && typeof meta === "object" && typeof (meta as Record)["changes"] === "number") { + return ((meta as Record)["changes"] as number) > 0; + } + } + return true; + }; + + // D1: no interactive transactions, but the driver's native batch runs + // every statement in ONE transaction. The guard is the first statement; + // if it matched nothing, the rest still ran — so the fence is enforced + // by making the deletes/inserts themselves conditional is not possible + // here. Instead, batch the guard ALONE first (atomic, tells us whether + // we own the row), and only then batch the mutations. Between the two + // batches another writer can re-claim, but then ITS stamp is the one + // conditioned on ownership and ours will not land; the rows we wrote + // are attributed to nobody's manifest and refused until it finishes — + // exactly the fail-closed behaviour the reader is built for. + const nativeBatch = db as unknown as { + readonly batch?: (statements: readonly unknown[]) => Promise; + }; + if (!interactiveTransactions) { + if (plan.guard) { + const guardTable = toDrizzle(plan.guard.table); + let update = db.update(guardTable).set(mapValues(plan.guard.set, plan.guard.table)); + if (plan.guard.where) { + update = update.where(buildWhere(toDrizzleColumn, plan.guard.where)) as any; + } + const result = await update; + if (!guardMatched(result)) return { applied: false }; + } + const rest = buildStatements(db).slice(plan.guard ? 1 : 0); + if (rest.length === 0) return { applied: true }; + if (nativeBatch.batch) { + await nativeBatch.batch(rest); + } else { + for (const statement of rest) await statement; + } + return { applied: true }; + } + + // Interactive engines: one transaction, guard first; a guard that + // matched nothing rolls the transaction back untouched. + return runAtomically(async (handle) => { + const statements = buildStatements(handle); + if (plan.guard) { + const result = await statements[0]; + if (!guardMatched(result)) { + // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: abort the driver transaction so nothing after a failed guard commits + throw new ReplaceGuardMiss(); + } + for (const statement of statements.slice(1)) await statement; + return { applied: true }; + } + for (const statement of statements) await statement; + return { applied: true }; + }).catch((error: unknown) => { + if (error instanceof ReplaceGuardMiss) return { applied: false }; + throw error; + }); + }, async transaction(run) { // Some SQLite-compatible engines (Cloudflare D1) reject interactive // transactions — both raw BEGIN/COMMIT and the driver's `.transaction()`. diff --git a/packages/core/fumadb/src/adapters/memory/index.ts b/packages/core/fumadb/src/adapters/memory/index.ts index 205655e9fa..826b121461 100644 --- a/packages/core/fumadb/src/adapters/memory/index.ts +++ b/packages/core/fumadb/src/adapters/memory/index.ts @@ -222,6 +222,27 @@ export function memoryAdapter(options: MemoryAdapterOptions = {}): FumaDBAdapter const rows = tableRows(db, table); db[table.ormName] = rows.filter((row) => !matchesCondition(row, v.where)); }, + async replaceMany(plan) { + // In-memory: apply the guard, and only if it matched a row apply the + // deletes and inserts. Synchronous over one object, so atomic by + // construction. + if (plan.guard) { + let matched = 0; + for (const row of tableRows(db, plan.guard.table)) { + if (!matchesCondition(row, plan.guard.where)) continue; + Object.assign(row, cloneValue(plan.guard.set)); + matched += 1; + } + if (matched === 0) return { applied: false }; + } + for (const del of plan.deletes) { + await this.deleteMany(del.table, { where: del.where }); + } + for (const ins of plan.inserts) { + await this.createMany(ins.table, ins.values); + } + return { applied: true }; + }, async transaction(run: (transactionInstance: AbstractQuery) => Promise) { const snapshot = cloneValue(db); try { diff --git a/packages/core/fumadb/src/query/index.ts b/packages/core/fumadb/src/query/index.ts index 45633dfb94..9a1dff021c 100644 --- a/packages/core/fumadb/src/query/index.ts +++ b/packages/core/fumadb/src/query/index.ts @@ -224,4 +224,27 @@ export interface AbstractQuery { eb: ConditionBuilder ) => Condition | boolean; }) => Promise; + + /** + * Delete + insert across tables as ONE atomic unit, optionally fenced by a + * guard update that must match a row for any of it to apply. Atomic on + * every engine, including those without interactive transactions (the + * adapter uses the driver's native batch there). Returns whether the guard + * matched; with no guard, always `applied: true`. + */ + replaceMany: (plan: { + readonly guard?: { + readonly table: keyof S["tables"]; + readonly where?: (eb: ConditionBuilder) => Condition | boolean; + readonly set: Record; + }; + readonly deletes: readonly { + readonly table: keyof S["tables"]; + readonly where?: (eb: ConditionBuilder) => Condition | boolean; + }[]; + readonly inserts: readonly { + readonly table: keyof S["tables"]; + readonly values: readonly Record[]; + }[]; + }) => Promise<{ readonly applied: boolean }>; } diff --git a/packages/core/fumadb/src/query/orm/index.ts b/packages/core/fumadb/src/query/orm/index.ts index 1136b5e318..0801c3ff03 100644 --- a/packages/core/fumadb/src/query/orm/index.ts +++ b/packages/core/fumadb/src/query/orm/index.ts @@ -383,6 +383,27 @@ export interface ORMAdapter { }, ) => Promise; + /** + * Run a set of deletes and inserts as ONE atomic unit, with an optional + * guard: a conditional update that must match at least one row for the + * rest to apply. On engines without interactive transactions (Cloudflare + * D1) adapters implement this with the driver's native batch, which D1 + * executes as one transaction; elsewhere it is an ordinary transaction. + * The guard makes the whole unit fenced: an adapter that cannot express + * "apply only if the guard matched" atomically must reject the guard. + */ + replaceMany?: ( + plan: { + readonly guard?: { + readonly table: AnyTable; + readonly where: Condition | undefined; + readonly set: Record; + }; + readonly deletes: readonly { readonly table: AnyTable; readonly where: Condition | undefined }[]; + readonly inserts: readonly { readonly table: AnyTable; readonly values: Record[] }[]; + }, + ) => Promise<{ readonly applied: boolean }>; + /** * Override this to support native transaction, otherwise use soft transaction. */ @@ -610,6 +631,41 @@ export function toORM( if (constrainedWhere === false) return; return internal.updateMany(table, { set, where: constrainedWhere }); }, + async replaceMany(plan) { + if (!internal.replaceMany) { + // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: public query rejects an adapter without atomic replace + throw new Error("[FumaDB] This adapter does not support replaceMany."); + } + let guard: { table: AnyTable; where: Condition | undefined; set: Record } | undefined; + if (plan.guard) { + const table = toTable(plan.guard.table); + let conditions = plan.guard.where ? buildCondition(table.columns, plan.guard.where) : undefined; + if (conditions === true) conditions = undefined; + if (conditions === false) return { applied: false }; + const constrained = await applyUpdatePolicies(table, conditions, plan.guard.set, context, "update"); + if (constrained === false) return { applied: false }; + guard = { table, where: constrained, set: plan.guard.set }; + } + const deletes: { table: AnyTable; where: Condition | undefined }[] = []; + for (const del of plan.deletes) { + const table = toTable(del.table); + let conditions = del.where ? buildCondition(table.columns, del.where) : undefined; + if (conditions === true) conditions = undefined; + if (conditions === false) continue; + const constrained = await applyDeletePolicies(table, conditions, context); + if (constrained === false) continue; + deletes.push({ table, where: constrained }); + } + const inserts: { table: AnyTable; values: Record[] }[] = []; + for (const ins of plan.inserts) { + const table = toTable(ins.table); + for (const value of ins.values) { + await runCreatePolicies(table, value, context); + } + if (ins.values.length > 0) inserts.push({ table, values: [...ins.values] }); + } + return internal.replaceMany({ ...(guard ? { guard } : {}), deletes, inserts }); + }, async transaction(run) { return internal.transaction((transactionInstance) => run(withQueryContext(transactionInstance, context)), diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 021757b256..0c3ab77d11 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -952,20 +952,22 @@ describe("createExecutor", () => { }), ); - // Two builds of one connection in two isolates, on a backend where every - // statement commits on its own. B claims the rebuild token after A has - // started; A finishes and tries to stamp; B then dies mid-replacement. A's - // stamp must NOT land (A no longer owns the connection), so the row keeps a - // null sync stamp and the stale scan rebuilds it — instead of A's manifest - // describing rows B has since torn out, refused forever. Modelled with a - // storage proxy that lets "B" claim the token between A's mark and A's - // stamp, then simulates B's death by deleting the rows and never stamping. + // Two builds of one connection in two isolates. A claims the rebuild + // token; B claims it after A (so B owns the connection); A's fenced + // replacement then runs. Its guard is "tools_rebuild is still A", which + // no longer holds, so the WHOLE unit — deletes, inserts, stamp — must be + // discarded, leaving B's claim and a null sync stamp for the stale scan to + // settle. Modelled with a storage proxy that lets "B" re-claim between A's + // claim and A's `replaceMany`, then simulates B dying by never stamping. it.effect( - "a rebuild that lost its claim to a competing build does not stamp, and the row stays stale", + "a rebuild that lost its claim to a competing build writes nothing, and the row stays stale", () => Effect.gen(function* () { const config = makeTestConfig({ plugins: [demoPlugin] as const }); - const raceState: { armed: boolean } = { armed: false }; + const raceState: { armed: boolean; applied: boolean | undefined } = { + armed: false, + applied: undefined, + }; const wrap = (inner: FumaDb): FumaDb => new Proxy(inner, { get(target, prop) { @@ -979,15 +981,10 @@ describe("createExecutor", () => { (tx) => run(wrap(tx)), ); } - if (prop === "createMany") { - return async (table: unknown, rows: unknown) => { - const out = await ( - target.createMany as (t: unknown, r: unknown) => Promise - )(table, rows); - // After A has written its definitions (its last rows) and - // before A stamps: B claims the connection, replaces the - // rows, and dies. - if (raceState.armed && table === "definition") { + if (prop === "replaceMany") { + return async (plan: unknown) => { + // A has claimed; before A's fenced unit runs, B claims. + if (raceState.armed) { raceState.armed = false; await (target.updateMany as (t: unknown, q: unknown) => Promise)( "connection", @@ -997,14 +994,11 @@ describe("createExecutor", () => { set: { tools_synced_at: null, tools_rebuild: "build-B" }, }, ); - await (target.deleteMany as (t: unknown, q: unknown) => Promise)( - "definition", - { - where: (b: { (c: string, op: string, v: unknown): unknown }) => - b("integration", "=", String(INTEG)), - }, - ); } + const out = (await ( + target.replaceMany as (p: unknown) => Promise<{ applied: boolean }> + )(plan)) as { applied: boolean }; + raceState.applied = out.applied; return out; }; } @@ -1020,25 +1014,36 @@ describe("createExecutor", () => { template: TEMPLATE, from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, }); - expect((yield* executor.tools.describeAll()).length).toBeGreaterThan(0); + const before = yield* executor.tools.describeAll(); + expect(before.length).toBeGreaterThan(0); + const rowsBefore = yield* Effect.promise(() => + config.db.findMany("tool", { where: (b) => b("integration", "=", String(INTEG)) }), + ); - // "A" rebuilds; the proxy plays "B" in the middle of it. + // "A" rebuilds; the proxy plays "B" between A's claim and A's unit. raceState.armed = true; yield* executor.connections.refresh({ owner: "org", integration: INTEG, name: CONN }); expect(raceState.armed, "B interleaved").toBe(false); + expect(raceState.applied, "A's fenced unit was discarded").toBe(false); + // Nothing of A's landed: the rows are still the earlier build's, the + // token is B's, and the sync stamp is null for the stale scan. + const rowsAfter = yield* Effect.promise(() => + config.db.findMany("tool", { where: (b) => b("integration", "=", String(INTEG)) }), + ); + expect(rowsAfter.map((row) => row.generation).sort(), "A did not replace the rows").toEqual( + rowsBefore.map((row) => row.generation).sort(), + ); const [row] = yield* Effect.promise(() => config.db.findMany("connection", { where: (b) => b("integration", "=", String(INTEG)), }), ); - // A did not stamp: the token is still B's and the sync stamp is null, - // so the stale scan will rebuild this connection. - expect(row?.tools_rebuild, "A's stamp did not clear B's claim").toBe("build-B"); + expect(row?.tools_rebuild, "A's unit did not clear B's claim").toBe("build-B"); expect(row?.tools_synced_at, "the connection stays stale-marked").toBeNull(); - // And the next read, which runs the stale scan, rebuilds and serves - // a whole catalog — never A's manifest over B's torn-out rows. + // The next read runs the stale scan, rebuilds, and serves one whole + // catalog with the token released. const served = yield* executor.tools.describeAll(); expect(served.map((tool) => tool.name).sort()).toEqual(["inspect", "run"]); const [after] = yield* Effect.promise(() => diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 3e2f922a3e..9647e2d577 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -1,5 +1,6 @@ import { Cause, + Data, Deferred, Duration, Effect, @@ -930,6 +931,16 @@ const emptyCatalogManifest = (): CatalogManifest => ({ definitions: 0, }); +/** `tools.describeAll` found connections whose rows are not one finished + * build. Carries exactly which, so recovery can stale-mark those alone. */ +class CatalogMismatch extends Data.TaggedError("CatalogMismatch")<{ + readonly connections: ReadonlyArray<{ + readonly owner: string; + readonly integration: string; + readonly connection: string; + }>; +}> {} + const storageFailureFromUnknown = (message: string, cause: unknown): StorageFailure => isStorageFailure(cause) ? cause : new StorageError({ message, cause }); @@ -1305,6 +1316,18 @@ type LooseStorageDb = { }, ) => Promise; readonly deleteMany: (tableName: string, options?: unknown) => Promise; + readonly replaceMany: (plan: { + readonly guard?: { + readonly table: string; + readonly where?: unknown; + readonly set: Record; + }; + readonly deletes: readonly { readonly table: string; readonly where?: unknown }[]; + readonly inserts: readonly { + readonly table: string; + readonly values: readonly Record[]; + }[]; + }) => Promise<{ readonly applied: boolean }>; readonly findFirst: ( tableName: string, options?: unknown, @@ -1362,6 +1385,21 @@ const makeCoreDb = (fuma: ReturnType) => ({ fuma.use(`${tableName}.deleteMany`, (db) => asLooseStorageDb(db).deleteMany(tableName, options), ), + /** Delete + insert across tables as one atomic unit, fenced by an optional + * guard update. See `AbstractQuery.replaceMany`. */ + replaceMany: (plan: { + readonly guard?: { + readonly table: CoreTableName; + readonly where?: CoreWhere; + readonly set: Record; + }; + readonly deletes: readonly { readonly table: CoreTableName; readonly where?: CoreWhere }[]; + readonly inserts: readonly { + readonly table: CoreTableName; + readonly values: readonly Record[]; + }[]; + }): Effect.Effect<{ readonly applied: boolean }, StorageFailure> => + fuma.use("replaceMany", (db) => asLooseStorageDb(db).replaceMany(plan)), findFirst: >( tableName: TName, options: TOptions, @@ -3540,31 +3578,59 @@ export const createExecutor = - core.updateMany("connection", { - where: (b: AnyCb) => b.and(connectionWhere(b), b("tools_rebuild", "=", buildId)), - set: syncedSet(row, manifest), + // Replace this connection's catalog as ONE fenced atomic unit: + // + // 1. claim — `tools_rebuild := buildId`, `tools_synced_at := null` + // 2. delete every tool + definition row + // 3. insert this build's rows + // 4. stamp — manifest + sync time, `tools_rebuild := null`, + // conditioned on still holding the claim from step 1 + // + // `replaceMany` runs 2–4 atomically on every backend (a driver + // transaction, or D1's native batch, which is one transaction), so a + // reader can never see a half-replaced catalog and a build can never + // stamp over rows another build wrote: the stamp's condition and the + // rows it describes commit together or not at all. Step 1 is its own + // statement on purpose — it is the durable "a rebuild is in flight" + // marker that survives a death anywhere after it and makes the stale + // scan rebuild the connection. Two builds racing: both claim; the one + // whose 2–4 commits first wins; the other's guard (step 4's condition) + // finds the token changed and its whole unit is discarded, rows + // included. Interactive backends make even step 1 part of the unit. + const replaceCatalog = ( + existingRow: ConnectionRow | null, + toolRows: readonly Record[], + definitionRows: readonly Record[], + manifest: CatalogManifest, + ) => + Effect.gen(function* () { + yield* core.updateMany("connection", { + where: connectionWhere, + set: { tools_synced_at: null, tools_rebuild: buildId }, + }); + const { applied } = yield* core.replaceMany({ + guard: { + table: "connection", + where: (b: AnyCb) => b.and(connectionWhere(b), b("tools_rebuild", "=", buildId)), + set: syncedSet(existingRow, manifest), + }, + deletes: [ + { table: "tool", where }, + { table: "definition", where }, + ], + inserts: [ + { table: "tool", values: toolRows }, + { table: "definition", values: definitionRows }, + ], + }); + if (!applied) { + yield* Effect.logInfo("executor tool sync lost its claim to a newer build", { + integration: String(ref.integration), + connection: String(ref.name), + }); + } + return applied; }); - // The FIRST statement of every replacement: claim the rebuild token - // and clear the sync stamp, keeping the previous manifest. On a - // backend that commits statement by statement (D1), a rebuild that - // dies after its deletes but before `stampSynced` leaves rows that no - // longer match the old manifest — refused by `tools.describeAll` — - // and, because the stamp is already null, the next stale-catalog scan - // rebuilds it instead of leaving it refused until someone refreshes by - // hand. Interactive backends roll the whole transaction back, so - // there this is a no-op. - const markRebuilding = core.updateMany("connection", { - where: connectionWhere, - set: { tools_synced_at: null, tools_rebuild: buildId }, - }); // A failing sync must not bury a recorded dead grant's `expired` // verdict: this sync's own credential resolution is what discovers // invalid_grant (refresh → recorder), so by failure time the row @@ -3604,7 +3670,11 @@ export const createExecutor = ; readonly definitionRows: ReadonlyArray; @@ -5948,7 +5998,11 @@ export const createExecutor = ; - }): boolean => { + }): ReadonlyArray<{ + readonly owner: string; + readonly integration: string; + readonly connection: string; + }> => { type Tally = { readonly generations: Set; count: number }; const tally = (rows: ReadonlyArray): Map => { const out = new Map(); @@ -5975,58 +6029,71 @@ export const createExecutor = - mixedGenerations(snapshot) - ? Effect.fail( + Effect.flatMap((snapshot) => { + const mixed = mixedGenerations(snapshot); + return mixed.length === 0 + ? Effect.succeed(snapshot) + : Effect.fail(new CatalogMismatch({ connections: mixed })); + }), + Effect.retry({ times: DESCRIBE_ALL_GENERATION_RETRIES }), + // Recovery backstop. A catalog that is still inconsistent after the + // retries is not a rebuild landing mid-read; it is one that landed + // WRONG — a build that died mid-claim, or rows a dead build left + // behind. Nothing else would rescan it (its stamp may be non-null), + // so stale-mark EXACTLY the inconsistent connections before + // surfacing: the next read rebuilds those, and only those. Ordinary + // storage failures are not catalog damage and trigger nothing. + Effect.catchTag("CatalogMismatch", (mismatch) => + Effect.forEach( + mismatch.connections, + (key) => + core + .updateMany("connection", { + where: (b: AnyCb) => + b.and( + byOwner(key.owner as Owner)(b), + b("integration", "=", key.integration), + b("name", "=", key.connection), + ), + set: { tools_synced_at: null }, + }) + .pipe(Effect.ignore), + { discard: true }, + ).pipe( + Effect.flatMap(() => + Effect.fail( new StorageError({ message: "tool catalog changed between reads", cause: undefined, }), - ) - : Effect.succeed(snapshot), - ), - Effect.retry({ times: DESCRIBE_ALL_GENERATION_RETRIES }), - // Recovery backstop. A catalog that is still inconsistent after the - // retries is not a rebuild landing mid-read; it is one that landed - // WRONG — a build that died, or lost its claim to a competing build - // that then died. Nothing else would rescan it (its stamp may be - // non-null), so stale-mark every connection in scope before - // surfacing: the next read rebuilds instead of refusing forever. - Effect.tapError(() => - core - .updateMany("connection", { - where: (b: AnyCb) => - b.and( - filter?.integration === undefined - ? true - : b("integration", "=", String(filter.integration)), - filter?.owner === undefined ? true : b("owner", "=", filter.owner), - filter?.connection === undefined - ? true - : b("name", "=", String(filter.connection)), - ), - set: { tools_synced_at: null }, - }) - .pipe(Effect.ignore), + ), + ), + ), ), ); const policyRules = yield* listActivePolicyRuleSet(); diff --git a/packages/core/sdk/src/fuma-runtime.ts b/packages/core/sdk/src/fuma-runtime.ts index ef8adf79c6..394bc3efa8 100644 --- a/packages/core/sdk/src/fuma-runtime.ts +++ b/packages/core/sdk/src/fuma-runtime.ts @@ -341,6 +341,12 @@ const makeSafeFumaQuery = ( updateMany: (name, value) => db.updateMany(table(name), value), upsert: (name, value) => db.upsert(table(name), value), upsertMany: (name, value) => db.upsertMany(table(name), value), + replaceMany: (plan) => + db.replaceMany({ + ...(plan.guard ? { guard: { ...plan.guard, table: table(plan.guard.table) } } : {}), + deletes: plan.deletes.map((del) => ({ ...del, table: table(del.table) })), + inserts: plan.inserts.map((ins) => ({ ...ins, table: table(ins.table) })), + }), }; return Object.freeze(query); From 829b193be5149e608bca8629919941fc14119b0a Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 4 Sep 2026 01:08:52 -0700 Subject: [PATCH 10/19] replaceMany guard reads every driver's row count or refuses to fence; collision-free recovery keys --- .../core/fumadb/src/adapters/drizzle/query.ts | 38 +++-- .../drizzle/replace-many-guard.test.ts | 142 ++++++++++++++++++ .../core/fumadb/src/adapters/memory/index.ts | 18 ++- packages/core/sdk/src/executor.ts | 60 ++++---- 4 files changed, 206 insertions(+), 52 deletions(-) create mode 100644 packages/core/fumadb/src/adapters/drizzle/replace-many-guard.test.ts diff --git a/packages/core/fumadb/src/adapters/drizzle/query.ts b/packages/core/fumadb/src/adapters/drizzle/query.ts index e213ed0a26..896c175c23 100644 --- a/packages/core/fumadb/src/adapters/drizzle/query.ts +++ b/packages/core/fumadb/src/adapters/drizzle/query.ts @@ -676,15 +676,17 @@ export function fromDrizzle( return statements; }; - // How many rows the guard matched. Drizzle returns driver-specific - // shapes; normalize the common ones (`rowsAffected`, `changes`, - // `rowCount`, `meta.changes`) and treat unknown as "matched" so an - // exotic driver never turns every replace into a no-op. + // How many rows the guard matched. Drizzle hands back the driver's own + // result: libsql `rowsAffected`, better-sqlite3 `changes`, node-postgres + // `rowCount`, postgres.js `count` (a RowList), D1 `meta.changes`. A + // result with NONE of these is a driver this fence does not know, and + // a fence that cannot read its own guard is not a fence — so that is a + // hard error, never a silent "matched". const guardMatched = (result: unknown): boolean => { if (!plan.guard) return true; if (result && typeof result === "object") { const r = result as Record; - for (const key of ["rowsAffected", "changes", "rowCount"]) { + for (const key of ["rowsAffected", "changes", "rowCount", "count"]) { if (typeof r[key] === "number") return (r[key] as number) > 0; } const meta = r["meta"]; @@ -692,19 +694,25 @@ export function fromDrizzle( return ((meta as Record)["changes"] as number) > 0; } } - return true; + // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: adapter refuses to fence on a driver whose update result it cannot read + throw new Error( + "[FumaDB Drizzle] replaceMany guard: the driver's update result carries no affected-row count.", + ); }; // D1: no interactive transactions, but the driver's native batch runs - // every statement in ONE transaction. The guard is the first statement; - // if it matched nothing, the rest still ran — so the fence is enforced - // by making the deletes/inserts themselves conditional is not possible - // here. Instead, batch the guard ALONE first (atomic, tells us whether - // we own the row), and only then batch the mutations. Between the two - // batches another writer can re-claim, but then ITS stamp is the one - // conditioned on ownership and ours will not land; the rows we wrote - // are attributed to nobody's manifest and refused until it finishes — - // exactly the fail-closed behaviour the reader is built for. + // every statement in ONE transaction. A batch cannot make its later + // statements conditional on an earlier one's row count, so the guard + // runs ALONE first (one statement, atomic, tells us whether we own the + // row) and only then do the deletes + inserts go through one batch. + // That is guard-then-batch, not one unit: between the two another + // writer can re-claim. What a reader sees at each step stays + // consistent — after our guard the row's manifest names OUR build + // while the table still holds the old rows, which the reader refuses + // (count/generation mismatch); once our batch lands, manifest and rows + // agree and are served; if a re-claimer's guard lands in between, its + // manifest over our rows is refused until its own batch lands. No + // half-built or mismatched catalog is ever served. const nativeBatch = db as unknown as { readonly batch?: (statements: readonly unknown[]) => Promise; }; diff --git a/packages/core/fumadb/src/adapters/drizzle/replace-many-guard.test.ts b/packages/core/fumadb/src/adapters/drizzle/replace-many-guard.test.ts new file mode 100644 index 0000000000..af25d7987f --- /dev/null +++ b/packages/core/fumadb/src/adapters/drizzle/replace-many-guard.test.ts @@ -0,0 +1,142 @@ +import { expect, test } from "@effect/vitest"; + +import { column, idColumn, schema, table } from "../../schema"; +import { fromDrizzle } from "./query"; + +// `replaceMany`'s fence is only as good as its reading of the guard update's +// affected-row count, and every driver spells that differently. A recording +// fake of the Drizzle handle plays each driver's result shape so the fence +// is proven to hold — and to refuse to fence at all — without a live server. +const v1 = schema({ + version: "1.0.0", + tables: { + owners: table("owners", { + id: idColumn("id", "varchar(255)"), + token: column("token", "string"), + }), + rows: table("rows", { + id: idColumn("id", "varchar(255)"), + value: column("value", "string"), + }), + }, +}); + +interface FakeOptions { + /** What the guard UPDATE resolves to. */ + readonly updateResult: unknown; +} + +const createFakePgDb = (options: FakeOptions) => { + const events: string[] = []; + const fakeTables = { + owners: { id: { name: "id" }, token: { name: "token" } }, + rows: { id: { name: "id" }, value: { name: "value" } }, + }; + const makeHandle = (label: string) => ({ + _: { fullSchema: fakeTables }, + update: () => ({ + set: () => { + const builder = { + where: () => builder, + then: (resolve: (value: unknown) => void) => { + events.push(`${label}:update`); + resolve(options.updateResult); + }, + }; + return builder; + }, + }), + delete: () => { + const builder = { + where: () => builder, + then: (resolve: (value: unknown) => void) => { + events.push(`${label}:delete`); + resolve(undefined); + }, + }; + return builder; + }, + insert: () => ({ + values: () => ({ + then: (resolve: (value: unknown) => void) => { + events.push(`${label}:insert`); + resolve(undefined); + }, + }), + }), + transaction: async (callback: (tx: unknown) => Promise): Promise => { + events.push("transaction:begin"); + // oxlint-disable-next-line executor/no-try-catch-or-throw -- fake driver mirrors commit/rollback + try { + const result = await callback(makeHandle("tx")); + events.push("transaction:commit"); + return result; + } catch (error) { + events.push("transaction:rollback"); + throw error; + } + }, + }); + return { db: makeHandle("root"), events }; +}; + +const replace = (db: unknown) => { + const orm = fromDrizzle(v1, db, "postgresql"); + const owners = v1.tables.owners; + const rows = v1.tables.rows; + expect(orm.internal.replaceMany).toBeDefined(); + return orm.internal.replaceMany!({ + guard: { table: owners, where: undefined, set: { token: null } }, + deletes: [{ table: rows, where: undefined }], + inserts: [{ table: rows, values: [{ id: "r1", value: "a" }] }], + }); +}; + +test("postgres.js `count: 0` is a guard miss: the transaction rolls back and nothing applies", async () => { + // postgres.js hands Drizzle a RowList whose affected count is `count`. + const { db, events } = createFakePgDb({ updateResult: Object.assign([], { count: 0 }) }); + const out = await replace(db); + expect(out).toEqual({ applied: false }); + expect(events).toEqual(["transaction:begin", "tx:update", "transaction:rollback"]); +}); + +test("postgres.js `count: 1` is a guard hit: deletes and inserts run in the same transaction", async () => { + const { db, events } = createFakePgDb({ updateResult: Object.assign([], { count: 1 }) }); + const out = await replace(db); + expect(out).toEqual({ applied: true }); + expect(events).toEqual([ + "transaction:begin", + "tx:update", + "tx:delete", + "tx:insert", + "transaction:commit", + ]); +}); + +test("node-postgres `rowCount`, libsql `rowsAffected`, better-sqlite3 `changes`, D1 `meta.changes` are all read", async () => { + for (const updateResult of [ + { rowCount: 0 }, + { rowsAffected: 0 }, + { changes: 0 }, + { meta: { changes: 0 } }, + ]) { + const { db } = createFakePgDb({ updateResult }); + expect(await replace(db), JSON.stringify(updateResult)).toEqual({ applied: false }); + } + for (const updateResult of [ + { rowCount: 2 }, + { rowsAffected: 1 }, + { changes: 1 }, + { meta: { changes: 1 } }, + ]) { + const { db } = createFakePgDb({ updateResult }); + expect(await replace(db), JSON.stringify(updateResult)).toEqual({ applied: true }); + } +}); + +test("a driver result with no affected-row count refuses to fence rather than silently matching", async () => { + const { db, events } = createFakePgDb({ updateResult: { ok: true } }); + await expect(replace(db)).rejects.toThrow(/affected-row count/); + // The transaction was aborted: nothing after the guard ran. + expect(events).toEqual(["transaction:begin", "tx:update", "transaction:rollback"]); +}); diff --git a/packages/core/fumadb/src/adapters/memory/index.ts b/packages/core/fumadb/src/adapters/memory/index.ts index 826b121461..9dbb351163 100644 --- a/packages/core/fumadb/src/adapters/memory/index.ts +++ b/packages/core/fumadb/src/adapters/memory/index.ts @@ -223,12 +223,13 @@ export function memoryAdapter(options: MemoryAdapterOptions = {}): FumaDBAdapter db[table.ormName] = rows.filter((row) => !matchesCondition(row, v.where)); }, async replaceMany(plan) { - // In-memory: apply the guard, and only if it matched a row apply the - // deletes and inserts. Synchronous over one object, so atomic by - // construction. + // In-memory: stage the whole result on a clone, then swap it in + // with no suspension point — so a concurrent reader or writer sees + // either the old state or the new one, never a partial plan. + const staged = cloneValue(db); if (plan.guard) { let matched = 0; - for (const row of tableRows(db, plan.guard.table)) { + for (const row of tableRows(staged, plan.guard.table)) { if (!matchesCondition(row, plan.guard.where)) continue; Object.assign(row, cloneValue(plan.guard.set)); matched += 1; @@ -236,11 +237,16 @@ export function memoryAdapter(options: MemoryAdapterOptions = {}): FumaDBAdapter if (matched === 0) return { applied: false }; } for (const del of plan.deletes) { - await this.deleteMany(del.table, { where: del.where }); + staged[del.table.ormName] = tableRows(staged, del.table).filter( + (row) => !matchesCondition(row, del.where), + ); } for (const ins of plan.inserts) { - await this.createMany(ins.table, ins.values); + const rows = tableRows(staged, ins.table); + for (const value of ins.values) rows.push(applyDefaults(ins.table, value)); } + for (const key of Object.keys(db)) delete db[key]; + Object.assign(db, staged); return { applied: true }; }, async transaction(run: (transactionInstance: AbstractQuery) => Promise) { diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 9647e2d577..51a42f26b2 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -3586,17 +3586,17 @@ export const createExecutor = [], @@ -5959,11 +5959,16 @@ export const createExecutor = `${row.owner}\u0000${row.integration}\u0000${row.connection}`; + }; + // Slugs and names are opaque strings; a tuple encoding is the only + // key that cannot collide, and the original triple rides in a side + // map so recovery never has to parse a key back apart. + const connectionKey = (row: ConnectionTriple): string => + JSON.stringify([row.owner, row.integration, row.connection]); // Per connection, the rows served must be exactly ONE finished build. // A build stamps every row it writes with one opaque `generation`, and // writes the connection's manifest — that generation plus the row @@ -5998,16 +6003,18 @@ export const createExecutor = ; - }): ReadonlyArray<{ - readonly owner: string; - readonly integration: string; - readonly connection: string; - }> => { + }): ReadonlyArray => { type Tally = { readonly generations: Set; count: number }; + const triples = new Map(); + const remember = (row: ConnectionTriple): string => { + const key = connectionKey(row); + if (!triples.has(key)) triples.set(key, row); + return key; + }; const tally = (rows: ReadonlyArray): Map => { const out = new Map(); for (const row of rows) { - const key = connectionKey(row); + const key = remember(row); const entry = out.get(key) ?? { generations: new Set(), count: 0 }; entry.generations.add(row.generation); entry.count += 1; @@ -6020,22 +6027,13 @@ export const createExecutor = (); for (const row of snapshot.manifests) { manifests.set( - connectionKey({ - owner: row.owner, - integration: row.integration, - connection: row.name, - }), + remember({ owner: row.owner, integration: row.integration, connection: row.name }), Option.getOrNull(decodeCatalogManifest(decodeJsonColumn(row.tools_manifest))), ); } const keys = new Set([...tools.keys(), ...definitions.keys(), ...manifests.keys()]); - const mixed: { owner: string; integration: string; connection: string }[] = []; + const mixed: ConnectionTriple[] = []; for (const key of keys) { - const [owner, integration, connection] = key.split("\u0000") as [ - string, - string, - string, - ]; const t = tools.get(key); const d = definitions.get(key); const manifest = manifests.get(key) ?? null; @@ -6049,7 +6047,7 @@ export const createExecutor = Date: Fri, 4 Sep 2026 01:20:09 -0700 Subject: [PATCH 11/19] Approval dominates read-only; replaceMany reads mysql2 counts; honest atomicity contract --- .../core/fumadb/src/adapters/drizzle/query.ts | 16 +++++++++---- .../drizzle/replace-many-guard.test.ts | 4 +++- packages/core/fumadb/src/query/index.ts | 15 ++++++++---- packages/core/fumadb/src/query/orm/index.ts | 13 +++++------ packages/core/sdk/src/core-schema.ts | 12 ++++++---- packages/core/sdk/src/executor.test.ts | 10 ++++---- packages/core/sdk/src/executor.ts | 15 ++++++------ .../hosts/mcp/src/passthrough-tools.test.ts | 22 ++++++++++++++---- packages/hosts/mcp/src/passthrough-tools.ts | 23 ++++++++++++++----- 9 files changed, 87 insertions(+), 43 deletions(-) diff --git a/packages/core/fumadb/src/adapters/drizzle/query.ts b/packages/core/fumadb/src/adapters/drizzle/query.ts index 896c175c23..d3dbe1c52c 100644 --- a/packages/core/fumadb/src/adapters/drizzle/query.ts +++ b/packages/core/fumadb/src/adapters/drizzle/query.ts @@ -678,12 +678,20 @@ export function fromDrizzle( // How many rows the guard matched. Drizzle hands back the driver's own // result: libsql `rowsAffected`, better-sqlite3 `changes`, node-postgres - // `rowCount`, postgres.js `count` (a RowList), D1 `meta.changes`. A - // result with NONE of these is a driver this fence does not know, and - // a fence that cannot read its own guard is not a fence — so that is a - // hard error, never a silent "matched". + // `rowCount`, postgres.js `count` (a RowList), D1 `meta.changes`, and + // mysql2 a `[ResultSetHeader, FieldPacket[]]` tuple whose header + // carries `affectedRows`. A result with NONE of these is a driver this + // fence does not know, and a fence that cannot read its own guard is + // not a fence — so that is a hard error, never a silent "matched". const guardMatched = (result: unknown): boolean => { if (!plan.guard) return true; + const header = + Array.isArray(result) && result.length > 0 && result[0] && typeof result[0] === "object" + ? (result[0] as Record) + : undefined; + if (header && typeof header["affectedRows"] === "number") { + return (header["affectedRows"] as number) > 0; + } if (result && typeof result === "object") { const r = result as Record; for (const key of ["rowsAffected", "changes", "rowCount", "count"]) { diff --git a/packages/core/fumadb/src/adapters/drizzle/replace-many-guard.test.ts b/packages/core/fumadb/src/adapters/drizzle/replace-many-guard.test.ts index af25d7987f..828144835e 100644 --- a/packages/core/fumadb/src/adapters/drizzle/replace-many-guard.test.ts +++ b/packages/core/fumadb/src/adapters/drizzle/replace-many-guard.test.ts @@ -113,12 +113,13 @@ test("postgres.js `count: 1` is a guard hit: deletes and inserts run in the same ]); }); -test("node-postgres `rowCount`, libsql `rowsAffected`, better-sqlite3 `changes`, D1 `meta.changes` are all read", async () => { +test("node-postgres `rowCount`, libsql `rowsAffected`, better-sqlite3 `changes`, D1 `meta.changes`, mysql2 `[header].affectedRows` are all read", async () => { for (const updateResult of [ { rowCount: 0 }, { rowsAffected: 0 }, { changes: 0 }, { meta: { changes: 0 } }, + [{ affectedRows: 0, fieldCount: 0 }, []], ]) { const { db } = createFakePgDb({ updateResult }); expect(await replace(db), JSON.stringify(updateResult)).toEqual({ applied: false }); @@ -128,6 +129,7 @@ test("node-postgres `rowCount`, libsql `rowsAffected`, better-sqlite3 `changes`, { rowsAffected: 1 }, { changes: 1 }, { meta: { changes: 1 } }, + [{ affectedRows: 1, fieldCount: 0 }, []], ]) { const { db } = createFakePgDb({ updateResult }); expect(await replace(db), JSON.stringify(updateResult)).toEqual({ applied: true }); diff --git a/packages/core/fumadb/src/query/index.ts b/packages/core/fumadb/src/query/index.ts index 9a1dff021c..d369df9506 100644 --- a/packages/core/fumadb/src/query/index.ts +++ b/packages/core/fumadb/src/query/index.ts @@ -226,11 +226,18 @@ export interface AbstractQuery { }) => Promise; /** - * Delete + insert across tables as ONE atomic unit, optionally fenced by a - * guard update that must match a row for any of it to apply. Atomic on - * every engine, including those without interactive transactions (the - * adapter uses the driver's native batch there). Returns whether the guard + * Delete + insert across tables, optionally fenced by a guard update that + * must match a row for the rest to apply. Returns whether the guard * matched; with no guard, always `applied: true`. + * + * Atomicity depends on the engine. With interactive transactions the whole + * plan — guard included — is one transaction: a guard miss or any failure + * rolls everything back. WITHOUT them (Cloudflare D1) the guard is its own + * committed statement and the deletes + inserts follow in one native batch + * (itself atomic): if the batch fails or the process dies between the two, + * the guard's update stays. Callers on such engines must be able to tell a + * guard-without-rows state apart on read (the executor's tool catalog does + * so with a per-connection manifest checked against row counts). */ replaceMany: (plan: { readonly guard?: { diff --git a/packages/core/fumadb/src/query/orm/index.ts b/packages/core/fumadb/src/query/orm/index.ts index 0801c3ff03..06550b47c6 100644 --- a/packages/core/fumadb/src/query/orm/index.ts +++ b/packages/core/fumadb/src/query/orm/index.ts @@ -384,13 +384,12 @@ export interface ORMAdapter { ) => Promise; /** - * Run a set of deletes and inserts as ONE atomic unit, with an optional - * guard: a conditional update that must match at least one row for the - * rest to apply. On engines without interactive transactions (Cloudflare - * D1) adapters implement this with the driver's native batch, which D1 - * executes as one transaction; elsewhere it is an ordinary transaction. - * The guard makes the whole unit fenced: an adapter that cannot express - * "apply only if the guard matched" atomically must reject the guard. + * Run a set of deletes and inserts, with an optional guard: a conditional + * update that must match at least one row for the rest to apply. With + * interactive transactions the whole plan is one transaction. Without them + * (Cloudflare D1) the guard is its own committed statement, followed by the + * deletes + inserts in one native batch — see `AbstractQuery.replaceMany` + * for the exact guarantee a caller gets on such engines. */ replaceMany?: ( plan: { diff --git a/packages/core/sdk/src/core-schema.ts b/packages/core/sdk/src/core-schema.ts index 1b9ca8ace8..77c2522553 100644 --- a/packages/core/sdk/src/core-schema.ts +++ b/packages/core/sdk/src/core-schema.ts @@ -229,11 +229,13 @@ export const coreTables = defineTables({ tools_synced_at: nullableBigintColumn("tools_synced_at"), // The catalog MANIFEST: which build is active for this connection and // how many tool + definition rows it wrote, as JSON - // `{ generation, tools, definitions }`. Written LAST by every rebuild, - // after all rows are in. A reader joining tools to definitions - // (`tools.describeAll`) accepts the rows only when every one carries - // this generation and the counts match — which is what proves the - // catalog is whole on a backend that commits each statement on its own + // `{ generation, tools, definitions }`. Written by every rebuild in the + // same transaction as its rows where the engine has one; on D1 it may + // commit BEFORE the row batch. Either way a reader joining tools to + // definitions (`tools.describeAll`) accepts the rows only when every + // one carries this generation and the counts match exactly — that + // agreement, not commit order, is what proves the catalog is whole on + // a backend that commits each statement on its own // (D1) and across isolates the per-executor write lock cannot see. // Null for connections built before the manifest existed. A null // manifest means "no proven-whole catalog": `tools.describeAll` refuses diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 0c3ab77d11..0221095feb 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -879,11 +879,11 @@ describe("createExecutor", () => { }), ); - // On D1 every statement of a rebuild commits on its own, so a reader can - // land after the definitions were deleted and before they were reinserted - // (or after only some tool batches landed). The connection's manifest — - // written LAST — is what lets `describeAll` tell that apart from a finished - // build. Modelled here by deleting the definitions out from under a stamped + // On D1 the manifest and the row batch are separate commits, so a reader + // can land with a manifest whose rows are not (all) there. The manifest's + // exact generation + row counts are what let `describeAll` tell that apart + // from a finished build. Modelled here by deleting the definitions out from + // under a stamped // catalog: the row counts no longer match the manifest. it.effect("tools.describeAll refuses a catalog whose rows do not match its manifest", () => Effect.gen(function* () { diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 51a42f26b2..beac51b4af 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -5942,11 +5942,11 @@ export const createExecutor = b.and( @@ -5972,8 +5972,9 @@ export const createExecutor = ( } }; +const decodeJsonString = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.String)); +const decodeJsonRecord = Schema.decodeUnknownSync( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), +); + const CATALOG: readonly ToolProjection[] = [ projection({ integration: "github", @@ -201,6 +206,15 @@ describe("passthrough annotations", () => { expect(passthroughAnnotations({ name: "x", policy: "approve" }).readOnlyHint).toBe(false); }); + it("approval dominates read-only: a gated GET is not advertised read-only", () => { + // The MCP spec lets a client skip its prompt for a read-only tool, and + // the server accepts its own gate on the strength of the advertisement — + // so a require_approval tool must never be advertised read-only. + expect( + passthroughAnnotations({ name: "x", policy: "require_approval", readOnly: true }), + ).toEqual({ title: "x", readOnlyHint: false, destructiveHint: true, openWorldHint: true }); + }); + it("emits exactly one awaited tool call with the whole address as one string literal", () => { expect(passthroughCallCode("tools.github.org.main.issues.create", { title: "hi" })).toBe( 'return await tools["github.org.main.issues.create"]({"title":"hi"});', @@ -222,12 +236,12 @@ describe("passthrough annotations", () => { const code = passthroughCallCode(`tools.evil.org.main.${hostile}`, {}); // Structural proof the payload never escapes the string literal: the // source is exactly `return await tools[]();`, and - // that one string round-trips through JSON.parse to the raw address. + // that one string decodes back to the raw address. const shape = /^return await tools\[("(?:[^"\\]|\\.)*")\]\((\{.*\})\);$/s.exec(code); expect(shape).not.toBeNull(); - expect(JSON.parse(shape![1]!)).toBe(`evil.org.main.${hostile}`); + expect(decodeJsonString(shape![1]!)).toBe(`evil.org.main.${hostile}`); // And the call's argument is the JSON we passed, untouched by the address. - expect(JSON.parse(shape![2]!)).toEqual({}); + expect(decodeJsonRecord(shape![2]!)).toEqual({}); }); }); diff --git a/packages/hosts/mcp/src/passthrough-tools.ts b/packages/hosts/mcp/src/passthrough-tools.ts index d30e06e73d..3d3c228a9b 100644 --- a/packages/hosts/mcp/src/passthrough-tools.ts +++ b/packages/hosts/mcp/src/passthrough-tools.ts @@ -112,15 +112,26 @@ export const preferredToolName = ( * fact (HTTP method, GraphQL kind, upstream hint); absent means unknown, which * must read as `false` — advertising read-only for a tool nobody vouched for * would let a harness skip a prompt it should show. + * + * Approval DOMINATES read-only. The MCP spec says `destructiveHint` is only + * meaningful when `readOnlyHint` is false, so a client may skip its prompt + * for any read-only tool. A tool a policy says needs approval must therefore + * never be advertised read-only, whatever the plugin knows about it — the + * server accepts its own approval gate on the strength of this advertisement + * (see the passthrough call path), so the advertisement is what the user's + * consent rests on. */ export const passthroughAnnotations = ( projection: Pick, -): PassthroughAnnotations => ({ - title: projection.name, - readOnlyHint: projection.readOnly === true, - destructiveHint: projection.policy === "require_approval", - openWorldHint: true, -}); +): PassthroughAnnotations => { + const requiresApproval = projection.policy === "require_approval"; + return { + title: projection.name, + readOnlyHint: !requiresApproval && projection.readOnly === true, + destructiveHint: requiresApproval, + openWorldHint: true, + }; +}; /** * Assign a unique MCP name to every projection. Deterministic for a given From 01e29433113c54153379c334777451af804f7945 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 4 Sep 2026 01:34:17 -0700 Subject: [PATCH 12/19] Passthrough integration filter narrows the catalog read; refresh reports persisted rows --- packages/core/sdk/src/executor.test.ts | 16 ++++++++- packages/core/sdk/src/executor.ts | 10 +++++- .../hosts/mcp/src/passthrough-tools.test.ts | 12 ++++++- packages/hosts/mcp/src/tool-server.ts | 33 +++++++++++++------ 4 files changed, 58 insertions(+), 13 deletions(-) diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 0221095feb..18c51b14a7 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -1022,7 +1022,11 @@ describe("createExecutor", () => { // "A" rebuilds; the proxy plays "B" between A's claim and A's unit. raceState.armed = true; - yield* executor.connections.refresh({ owner: "org", integration: INTEG, name: CONN }); + const reported = yield* executor.connections.refresh({ + owner: "org", + integration: INTEG, + name: CONN, + }); expect(raceState.armed, "B interleaved").toBe(false); expect(raceState.applied, "A's fenced unit was discarded").toBe(false); @@ -1034,6 +1038,16 @@ describe("createExecutor", () => { expect(rowsAfter.map((row) => row.generation).sort(), "A did not replace the rows").toEqual( rowsBefore.map((row) => row.generation).sort(), ); + // And A reported the persisted rows, not the ones it discovered and + // failed to write — so its caller cannot disagree with the next list. + expect( + reported.map((tool) => String(tool.address)).sort(), + "refresh reports what is persisted", + ).toEqual( + rowsAfter + .map((row) => `tools.${row.integration}.${row.owner}.${row.connection}.${row.name}`) + .sort(), + ); const [row] = yield* Effect.promise(() => config.db.findMany("connection", { where: (b) => b("integration", "=", String(INTEG)), diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index beac51b4af..119f74fc38 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -3805,13 +3805,21 @@ export const createExecutor = rowToTool(row as ConnectionToolRow)); + } return result.tools.map((tool: ToolDef) => rowToTool( diff --git a/packages/hosts/mcp/src/passthrough-tools.test.ts b/packages/hosts/mcp/src/passthrough-tools.test.ts index 5ea85d7220..2d102a0dce 100644 --- a/packages/hosts/mcp/src/passthrough-tools.test.ts +++ b/packages/hosts/mcp/src/passthrough-tools.test.ts @@ -600,7 +600,17 @@ describe("passthrough mode server", () => { engine, mode: "passthrough", passthroughIntegrations: ["linear", "notion"], - tools: { describeAll: () => Effect.succeed(CATALOG) }, + // The filter is pushed into the READ: the port is asked per slug, so + // a real catalog never describes tools the session will not serve. + tools: { + describeAll: (filter) => + Effect.succeed( + CATALOG.filter( + (tool) => + filter?.integration === undefined || tool.integration === filter.integration, + ), + ), + }, }, async (client) => { const listed = await client.listTools(); diff --git a/packages/hosts/mcp/src/tool-server.ts b/packages/hosts/mcp/src/tool-server.ts index 2e566830aa..1b4dac183c 100644 --- a/packages/hosts/mcp/src/tool-server.ts +++ b/packages/hosts/mcp/src/tool-server.ts @@ -26,6 +26,7 @@ import * as z from "zod/v4"; import { CurrentOrgWriteAccess, + IntegrationSlug, isToolFile, isToolResult, makeOrgWriteAccessState, @@ -1250,15 +1251,27 @@ const loadPassthroughTools = ( reason: "passthrough mode requested but the host provided no tool catalog", }); } - const projections = yield* port.describeAll().pipe( - Effect.mapError( - (cause) => - new McpPassthroughUnavailableError({ - reason: `tool catalog read failed: ${formatBoundaryError(cause).message}`, - }), - ), - ); - const scoped = filterPassthroughIntegrations(projections, config.passthroughIntegrations); + // `?integrations=` narrows the READ, not just the result: a catalog + // that is torn for an unrelated integration must not fail a session that + // never asked for it, and no work is spent describing tools that will be + // dropped. One read per requested slug; the whole workspace otherwise. + const requested = config.passthroughIntegrations; + const read = (filter?: { readonly integration: IntegrationSlug }) => + port.describeAll(filter).pipe( + Effect.mapError( + (cause) => + new McpPassthroughUnavailableError({ + reason: `tool catalog read failed: ${formatBoundaryError(cause).message}`, + }), + ), + ); + const projections = + requested && requested.length > 0 + ? (yield* Effect.forEach(requested, (slug) => + read({ integration: IntegrationSlug.make(slug) }), + )).flat() + : yield* read(); + const scoped = filterPassthroughIntegrations(projections, requested); return assignPassthroughNames(scoped); }).pipe(Effect.withSpan("mcp.host.passthrough.load")); @@ -1881,7 +1894,7 @@ export const createExecutorMcpServer = ( // Anything the tool itself asked for goes to the client natively // when it can take it; the native bridge already turns a URL // request into a form for form-only clients. - if (supportsForm || (supportsUrl && ctx.request._tag === "UrlElicitation")) { + if (supportsForm || (supportsUrl && Predicate.isTagged(ctx.request, "UrlElicitation"))) { return native(ctx); } unanswerable = ctx.request; From d2dcf248d86ff44b49df0ad8b996d3e355bcacac Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 4 Sep 2026 01:46:53 -0700 Subject: [PATCH 13/19] Scope the stale catalog sync to the requested integration; harden the narrowed-read and lost-claim tests; release fumadb --- .changeset/mcp-passthrough-mode.md | 1 + packages/core/sdk/src/executor.test.ts | 30 ++- packages/core/sdk/src/executor.ts | 254 ++++++++++-------- .../hosts/mcp/src/passthrough-tools.test.ts | 19 +- 4 files changed, 176 insertions(+), 128 deletions(-) diff --git a/.changeset/mcp-passthrough-mode.md b/.changeset/mcp-passthrough-mode.md index 104225ee82..04deb1dcbc 100644 --- a/.changeset/mcp-passthrough-mode.md +++ b/.changeset/mcp-passthrough-mode.md @@ -1,5 +1,6 @@ --- "@executor-js/sdk": minor +"@executor-js/fumadb": minor "@executor-js/plugin-openapi": patch "@executor-js/plugin-graphql": patch "@executor-js/plugin-mcp": patch diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 18c51b14a7..3e0bd3a278 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -54,6 +54,10 @@ const addr = (tool: string): ToolAddress => ToolAddress.make(`tools.${INTEG}.org // resolveTools (with shared $defs), and supports ctx.transaction rollback. // --------------------------------------------------------------------------- +/** Toggled by a test so the demo plugin's next discovery differs from what + * is persisted. Module-level because the plugin closure is created once. */ +const demoDiscoversExtra = { value: false }; + const demoPlugin = definePlugin(() => ({ id: "demo" as const, credentialProviders: [memoryProvider()], @@ -68,6 +72,11 @@ const demoPlugin = definePlugin(() => ({ resolveTools: () => Effect.succeed({ tools: [ + // A test may flip this to make one discovery differ from the last + // persisted catalog (see the lost-claim rebuild test). + ...(demoDiscoversExtra.value + ? [{ name: ToolName.make("extra"), description: "extra" }] + : []), { name: ToolName.make("inspect"), description: "inspect", @@ -1020,13 +1029,15 @@ describe("createExecutor", () => { config.db.findMany("tool", { where: (b) => b("integration", "=", String(INTEG)) }), ); - // "A" rebuilds; the proxy plays "B" between A's claim and A's unit. + // "A" rebuilds — and discovers something NEW (`extra`) that the + // persisted catalog does not have, so a caller handed A's discovery + // instead of the persisted rows is distinguishable. The proxy plays + // "B" between A's claim and A's unit. + demoDiscoversExtra.value = true; raceState.armed = true; - const reported = yield* executor.connections.refresh({ - owner: "org", - integration: INTEG, - name: CONN, - }); + const reported = yield* executor.connections + .refresh({ owner: "org", integration: INTEG, name: CONN }) + .pipe(Effect.ensuring(Effect.sync(() => void (demoDiscoversExtra.value = false)))); expect(raceState.armed, "B interleaved").toBe(false); expect(raceState.applied, "A's fenced unit was discarded").toBe(false); @@ -1040,9 +1051,14 @@ describe("createExecutor", () => { ); // And A reported the persisted rows, not the ones it discovered and // failed to write — so its caller cannot disagree with the next list. + // A discovered `extra`; persisted has no `extra`; the report must not. expect( - reported.map((tool) => String(tool.address)).sort(), + reported.map((tool) => String(tool.name)).sort(), "refresh reports what is persisted", + ).toEqual(["inspect", "run"]); + expect( + reported.map((tool) => String(tool.address)).sort(), + "refresh reports the persisted addresses", ).toEqual( rowsAfter .map((row) => `tools.${row.integration}.${row.owner}.${row.connection}.${row.name}`) diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 119f74fc38..5ddaafea16 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -5555,123 +5555,137 @@ export const createExecutor = [row.slug, row] as const)); - // The TTL only matters when a loaded plugin actually lists a live remote - // catalog; otherwise skip it so age alone never widens the stale query. - const anyRemoteCatalog = Array.from(runtimes.values()).some( - (runtime) => runtime.plugin.remoteToolCatalog === true, - ); - const cutoff = - toolsSyncTtlMs == null || !anyRemoteCatalog ? null : Date.now() - toolsSyncTtlMs; - - // Bound the scan to potentially-stale rows: stale-marked (NULL stamp) or - // synced before the latest instant any trigger could fire at (the TTL - // cutoff / the newest config revision). Per-row trigger checks below - // re-verify against each row's own integration; in steady state this - // query returns nothing and the read pays one indexed lookup. - const latestRevision = integrations.reduce( - (max, row) => - row.config_revised_at == null - ? max - : Math.max(max ?? Number(row.config_revised_at), Number(row.config_revised_at)), - null, - ); - const staleBefore = - cutoff === null && latestRevision === null - ? null - : Math.max(cutoff ?? Number.MIN_SAFE_INTEGER, latestRevision ?? Number.MIN_SAFE_INTEGER); - - // A connection with no catalog manifest (built before the manifest - // existed) is stale too: `tools.describeAll` refuses it until a rebuild - // stamps one, so the read that first sees it is the read that fixes it. - const connections = yield* core.findMany("connection", { - where: (b: AnyCb) => - staleBefore === null - ? b.or(b.isNull("tools_synced_at"), b.isNull("tools_manifest")) - : b.or( - b.isNull("tools_synced_at"), - b.isNull("tools_manifest"), - b("tools_synced_at", "<", staleBefore), - ), - }); - // Each rebuild is an independent upstream listing, so they run together - // rather than one after another: a host with many stale remote-catalog - // connections otherwise pays the sum of every server's latency on the - // read that trips the TTL. Only the listings overlap — `persistCatalog` - // keeps the catalog writes in a single-file queue, so this fan-out never - // opens two transactions on a one-connection database. - const rebuilds: Effect.Effect[] = []; - for (const connection of connections) { - const integrationRow = integrationBySlug.get(connection.integration); - if (!integrationRow) continue; - const runtime = runtimes.get(integrationRow.plugin_id); - // Only re-produce catalogs this executor can actually re-list — - // rebuilding under an unloaded plugin would clear a working catalog. - // (A loaded plugin without `resolveTools` still flows through: - // `produceConnectionTools` runs its clear-and-stamp cleanup path.) - if (!runtime) continue; - - const syncedAt = - connection.tools_synced_at == null ? null : Number(connection.tools_synced_at); - const revisedTime = - integrationRow.config_revised_at == null + // `scope` narrows the sync to one integration's connections. A read that + // only asked for that integration (`describeAll({ integration })`) must not + // scan, dial, or wait on every other stale connection in the workspace. + const syncStaleConnectionToolsScoped = (scope?: { readonly integration: IntegrationSlug }) => + Effect.gen(function* () { + // The platform view can never persist a rebuilt catalog (writes are + // denied at the storage boundary), so attempting the sync would only + // fire upstream `resolveTools` calls whose results are thrown away — + // network side effects on a read-only credential. Skip it entirely: + // read-only-ness of the platform read path is a stated invariant here, + // not an accident of the best-effort catch below. + if (config.platformView === true) return; + const integrations = yield* core.findMany("integration", { + where: (b: AnyCb) => + scope === undefined ? true : b("slug", "=", String(scope.integration)), + }); + if (integrations.length === 0) return; + const integrationBySlug = new Map(integrations.map((row) => [row.slug, row] as const)); + // The TTL only matters when a loaded plugin actually lists a live remote + // catalog; otherwise skip it so age alone never widens the stale query. + const anyRemoteCatalog = Array.from(runtimes.values()).some( + (runtime) => runtime.plugin.remoteToolCatalog === true, + ); + const cutoff = + toolsSyncTtlMs == null || !anyRemoteCatalog ? null : Date.now() - toolsSyncTtlMs; + + // Bound the scan to potentially-stale rows: stale-marked (NULL stamp) or + // synced before the latest instant any trigger could fire at (the TTL + // cutoff / the newest config revision). Per-row trigger checks below + // re-verify against each row's own integration; in steady state this + // query returns nothing and the read pays one indexed lookup. + const latestRevision = integrations.reduce( + (max, row) => + row.config_revised_at == null + ? max + : Math.max(max ?? Number(row.config_revised_at), Number(row.config_revised_at)), + null, + ); + const staleBefore = + cutoff === null && latestRevision === null ? null - : Number(integrationRow.config_revised_at); - - const staleMarked = syncedAt === null || connection.tools_manifest == null; - const configRevised = revisedTime !== null && (syncedAt ?? 0) < revisedTime; - const expired = - cutoff !== null && - runtime.plugin.remoteToolCatalog === true && - syncedAt !== null && - syncedAt < cutoff; - if (!staleMarked && !configRevised && !expired) continue; + : Math.max( + cutoff ?? Number.MIN_SAFE_INTEGER, + latestRevision ?? Number.MIN_SAFE_INTEGER, + ); - rebuilds.push( - produceConnectionTools( - integrationRow, - { - owner: connection.owner as Owner, - integration: IntegrationSlug.make(connection.integration), - name: ConnectionName.make(connection.name), - }, - "background", - ).pipe( - // Best-effort, but never silent: the read still succeeds on the - // stale-but-working catalog and the peer rebuilds still finish, - // while the operator gets the connection that failed and why. - // Without this a connection whose upstream is permanently broken - // re-fails on every read and leaves no trace anywhere. - Effect.catch((error) => - Effect.logWarning("executor stale tool sync failed", { - integration: connection.integration, - connection: connection.name, - error: describeSyncFailure(error), - }).pipe(Effect.as([] as readonly Tool[])), + // A connection with no catalog manifest (built before the manifest + // existed) is stale too: `tools.describeAll` refuses it until a rebuild + // stamps one, so the read that first sees it is the read that fixes it. + const connections = yield* core.findMany("connection", { + where: (b: AnyCb) => + b.and( + scope === undefined ? true : b("integration", "=", String(scope.integration)), + staleBefore === null + ? b.or(b.isNull("tools_synced_at"), b.isNull("tools_manifest")) + : b.or( + b.isNull("tools_synced_at"), + b.isNull("tools_manifest"), + b("tools_synced_at", "<", staleBefore), + ), ), - Effect.withSpan("executor.tools.sync_stale", { - attributes: { - "executor.integration": connection.integration, - "executor.connection": connection.name, + }); + // Each rebuild is an independent upstream listing, so they run together + // rather than one after another: a host with many stale remote-catalog + // connections otherwise pays the sum of every server's latency on the + // read that trips the TTL. Only the listings overlap — `persistCatalog` + // keeps the catalog writes in a single-file queue, so this fan-out never + // opens two transactions on a one-connection database. + const rebuilds: Effect.Effect[] = []; + for (const connection of connections) { + const integrationRow = integrationBySlug.get(connection.integration); + if (!integrationRow) continue; + const runtime = runtimes.get(integrationRow.plugin_id); + // Only re-produce catalogs this executor can actually re-list — + // rebuilding under an unloaded plugin would clear a working catalog. + // (A loaded plugin without `resolveTools` still flows through: + // `produceConnectionTools` runs its clear-and-stamp cleanup path.) + if (!runtime) continue; + + const syncedAt = + connection.tools_synced_at == null ? null : Number(connection.tools_synced_at); + const revisedTime = + integrationRow.config_revised_at == null + ? null + : Number(integrationRow.config_revised_at); + + const staleMarked = syncedAt === null || connection.tools_manifest == null; + const configRevised = revisedTime !== null && (syncedAt ?? 0) < revisedTime; + const expired = + cutoff !== null && + runtime.plugin.remoteToolCatalog === true && + syncedAt !== null && + syncedAt < cutoff; + if (!staleMarked && !configRevised && !expired) continue; + + rebuilds.push( + produceConnectionTools( + integrationRow, + { + owner: connection.owner as Owner, + integration: IntegrationSlug.make(connection.integration), + name: ConnectionName.make(connection.name), }, - }), - ), - ); - } - yield* Effect.all(rebuilds, { - concurrency: STALE_TOOLS_SYNC_CONCURRENCY, + "background", + ).pipe( + // Best-effort, but never silent: the read still succeeds on the + // stale-but-working catalog and the peer rebuilds still finish, + // while the operator gets the connection that failed and why. + // Without this a connection whose upstream is permanently broken + // re-fails on every read and leaves no trace anywhere. + Effect.catch((error) => + Effect.logWarning("executor stale tool sync failed", { + integration: connection.integration, + connection: connection.name, + error: describeSyncFailure(error), + }).pipe(Effect.as([] as readonly Tool[])), + ), + Effect.withSpan("executor.tools.sync_stale", { + attributes: { + "executor.integration": connection.integration, + "executor.connection": connection.name, + }, + }), + ), + ); + } + yield* Effect.all(rebuilds, { + concurrency: STALE_TOOLS_SYNC_CONCURRENCY, + }); }); - }); + const syncStaleConnectionTools = syncStaleConnectionToolsScoped(); // How long a tools read waits for the stale sync before answering from // the persisted rows (`ExecutorConfig.toolsSyncGraceMs`; `null` blocks @@ -5689,10 +5703,13 @@ export const createExecutor = + const awaitStaleSyncWithinGrace = ( + graceMs: number, + scope?: { readonly integration: IntegrationSlug }, + ) => Effect.gen(function* () { const fiber = yield* Effect.forkDetach( - syncStaleConnectionTools.pipe( + syncStaleConnectionToolsScoped(scope).pipe( Effect.catch((error) => Effect.logWarning("executor stale tool sync scan failed", { error: describeSyncFailure(error), @@ -5909,10 +5926,15 @@ export const createExecutor = => Effect.gen(function* () { + // The stale sync is scoped to the requested integration, so a + // narrowed read never scans, dials, or waits on unrelated + // connections. + const scope = + filter?.integration === undefined ? undefined : { integration: filter.integration }; if (toolsSyncGraceMs === null) { - yield* syncStaleConnectionTools; + yield* syncStaleConnectionToolsScoped(scope); } else { - yield* awaitStaleSyncWithinGrace(toolsSyncGraceMs); + yield* awaitStaleSyncWithinGrace(toolsSyncGraceMs, scope); } const integrationWhere = (b: AnyCb) => b.and( diff --git a/packages/hosts/mcp/src/passthrough-tools.test.ts b/packages/hosts/mcp/src/passthrough-tools.test.ts index 2d102a0dce..0bca6183ec 100644 --- a/packages/hosts/mcp/src/passthrough-tools.test.ts +++ b/packages/hosts/mcp/src/passthrough-tools.test.ts @@ -595,25 +595,34 @@ describe("passthrough mode server", () => { it("narrows to the requested integrations and says which were not connected", async () => { const { engine } = makeRecordingEngine(); + // The filter is pushed into the READ: the port is asked once per slug and + // never for the whole workspace. Recording the calls is what proves it — + // a stub that merely honoured the filter would also pass an unfiltered + // read followed by host-side filtering. + const reads: (string | undefined)[] = []; await withClient( { engine, mode: "passthrough", passthroughIntegrations: ["linear", "notion"], - // The filter is pushed into the READ: the port is asked per slug, so - // a real catalog never describes tools the session will not serve. tools: { - describeAll: (filter) => - Effect.succeed( + describeAll: (filter) => { + reads.push(filter?.integration === undefined ? undefined : String(filter.integration)); + return Effect.succeed( CATALOG.filter( (tool) => filter?.integration === undefined || tool.integration === filter.integration, ), - ), + ); + }, }, }, async (client) => { const listed = await client.listTools(); + expect(reads.sort(), "one read per requested slug, none unfiltered").toEqual([ + "linear", + "notion", + ]); expect(listed.tools.map((tool) => tool.name)).toEqual(["linear__issueCreate"]); const instructions = client.getInstructions() ?? ""; expect(instructions).toContain("1 integration tool"); From 5e0fd2a2dcda4a5ec13dcd14ef277e9fa0514d05 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 4 Sep 2026 01:57:19 -0700 Subject: [PATCH 14/19] A lost rebuild claim reports the persisted catalog only when it is whole --- packages/core/sdk/src/executor.test.ts | 78 +++++++++++++++++++++++++- packages/core/sdk/src/executor.ts | 34 +++++++++-- 2 files changed, 104 insertions(+), 8 deletions(-) diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 3e0bd3a278..97fbb0333a 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -1049,9 +1049,10 @@ describe("createExecutor", () => { expect(rowsAfter.map((row) => row.generation).sort(), "A did not replace the rows").toEqual( rowsBefore.map((row) => row.generation).sort(), ); - // And A reported the persisted rows, not the ones it discovered and - // failed to write — so its caller cannot disagree with the next list. - // A discovered `extra`; persisted has no `extra`; the report must not. + // And A reported the persisted rows, VALIDATED against the manifest, + // not the ones it discovered and failed to write — so its caller + // cannot disagree with the next list. A discovered `extra`; persisted + // has no `extra`; the report must not. expect( reported.map((tool) => String(tool.name)).sort(), "refresh reports what is persisted", @@ -1086,6 +1087,77 @@ describe("createExecutor", () => { }), ); + // D1 commits the winner's manifest before its row batch. A loser that + // reads the rows in that window must NOT report the previous build's rows + // as if they were current: `describeAll` refuses that state (manifest names + // the new generation, rows carry the old), so the loser reports nothing. + it.effect("a lost claim reports nothing while the winner's rows have not landed", () => + Effect.gen(function* () { + const config = makeTestConfig({ plugins: [demoPlugin] as const }); + const raceState: { armed: boolean } = { armed: false }; + const wrap = (inner: FumaDb): FumaDb => + new Proxy(inner, { + get(target, prop) { + if (prop === "withContext") { + return (context: unknown) => + wrap((target.withContext as (c: unknown) => FumaDb)(context)); + } + if (prop === "transaction") { + return (run: (tx: FumaDb) => Promise) => + (target.transaction as (r: (tx: FumaDb) => Promise) => Promise)( + (tx) => run(wrap(tx)), + ); + } + if (prop === "replaceMany") { + return async (plan: unknown) => { + if (raceState.armed) { + raceState.armed = false; + // B claims AND has already committed its manifest for a + // generation whose rows are not in the table yet — the D1 + // guard-then-batch window. + await (target.updateMany as (t: unknown, q: unknown) => Promise)( + "connection", + { + where: (b: { (c: string, op: string, v: unknown): unknown }) => + b("integration", "=", String(INTEG)), + set: { + tools_synced_at: null, + tools_rebuild: "build-B", + tools_manifest: { generation: "gen-B", tools: 2, definitions: 6 }, + }, + }, + ); + } + return (target.replaceMany as (p: unknown) => Promise<{ applied: boolean }>)(plan); + }; + } + return Reflect.get(target, prop); + }, + }); + const executor = yield* createExecutor({ ...config, db: wrap(config.db) }); + yield* executor.demo.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + expect((yield* executor.tools.describeAll()).length).toBeGreaterThan(0); + + raceState.armed = true; + const reported = yield* executor.connections.refresh({ + owner: "org", + integration: INTEG, + name: CONN, + }); + expect(raceState.armed, "B interleaved").toBe(false); + // The rows in the table are the OLD build's; the manifest is B's. That + // is not a servable catalog, and the loser must not pretend it is. + expect(reported, "nothing is reported while the winner lands").toEqual([]); + }), + ); + it.effect("execute dispatches a connection-produced tool to the owning plugin", () => Effect.gen(function* () { const executor = yield* makeTestExecutor({ diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 5ddaafea16..7879f2278d 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -3813,12 +3813,36 @@ export const createExecutor = rowToTool(row as ConnectionToolRow)); + const [rows, definitionRows, connectionRow] = yield* Effect.all([ + core.findMany("tool", { where, select: [...TOOL_INVOCATION_COLUMNS, "generation"] }), + core.findMany("definition", { where }), + findConnectionRow(ref), + ]); + const manifest = connectionRow + ? Option.getOrNull( + decodeCatalogManifest(decodeJsonColumn(connectionRow.tools_manifest)), + ) + : null; + if ( + !manifest || + rows.length !== manifest.tools || + definitionRows.length !== manifest.definitions || + rows.some((row) => row.generation !== manifest.generation) || + definitionRows.some((row) => row.generation !== manifest.generation) + ) { + return []; + } + return rows.map((row) => rowToTool(row)); } return result.tools.map((tool: ToolDef) => From f2e9b949875971142b8885644bb73590a2f44866 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:06:42 -0700 Subject: [PATCH 15/19] Every rebuild reports only a whole persisted catalog and stale-marks a torn one --- packages/core/sdk/src/executor.test.ts | 79 ++++++++++++++++++++++ packages/core/sdk/src/executor.ts | 90 +++++++++++--------------- 2 files changed, 118 insertions(+), 51 deletions(-) diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 97fbb0333a..5102cf6d25 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -1158,6 +1158,85 @@ describe("createExecutor", () => { }), ); + // The D1 race the guard-then-batch protocol cannot prevent: A's guard + // commits (A "won"), B claims + guards + lands its rows, THEN A's delayed + // batch lands rows A under manifest B. A got `applied: true` and must still + // not report its discovery: the persisted state is manifest B over rows A, + // which every list refuses. A reports nothing and stale-marks the row so + // the next read rebuilds. Modelled by rewriting the manifest to B's after + // A's `replaceMany` returns. + it.effect("a build whose rows landed under another build's manifest reports nothing", () => + Effect.gen(function* () { + const config = makeTestConfig({ plugins: [demoPlugin] as const }); + const raceState: { armed: boolean } = { armed: false }; + const wrap = (inner: FumaDb): FumaDb => + new Proxy(inner, { + get(target, prop) { + if (prop === "withContext") { + return (context: unknown) => + wrap((target.withContext as (c: unknown) => FumaDb)(context)); + } + if (prop === "transaction") { + return (run: (tx: FumaDb) => Promise) => + (target.transaction as (r: (tx: FumaDb) => Promise) => Promise)( + (tx) => run(wrap(tx)), + ); + } + if (prop === "replaceMany") { + return async (plan: unknown) => { + const out = await ( + target.replaceMany as (p: unknown) => Promise<{ applied: boolean }> + )(plan); + if (raceState.armed && out.applied) { + raceState.armed = false; + // B's manifest overwrites A's after A's rows are in. + await (target.updateMany as (t: unknown, q: unknown) => Promise)( + "connection", + { + where: (b: { (c: string, op: string, v: unknown): unknown }) => + b("integration", "=", String(INTEG)), + set: { + tools_synced_at: Date.now(), + tools_rebuild: null, + tools_manifest: { generation: "gen-B", tools: 2, definitions: 6 }, + }, + }, + ); + } + return out; + }; + } + return Reflect.get(target, prop); + }, + }); + const executor = yield* createExecutor({ ...config, db: wrap(config.db) }); + yield* executor.demo.seed(); + yield* executor.connections.create({ + owner: "org", + name: CONN, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + expect((yield* executor.tools.describeAll()).length).toBeGreaterThan(0); + + raceState.armed = true; + const reported = yield* executor.connections.refresh({ + owner: "org", + integration: INTEG, + name: CONN, + }); + expect(raceState.armed, "B overwrote the manifest after A's rows landed").toBe(false); + expect(reported, "A reports nothing: its rows sit under B's manifest").toEqual([]); + const [row] = yield* Effect.promise(() => + config.db.findMany("connection", { + where: (b) => b("integration", "=", String(INTEG)), + }), + ); + expect(row?.tools_synced_at, "the connection is stale-marked for the next read").toBeNull(); + }), + ); + it.effect("execute dispatches a connection-produced tool to the owning plugin", () => Effect.gen(function* () { const executor = yield* makeTestExecutor({ diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 7879f2278d..c5858ead48 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -3812,59 +3812,47 @@ export const createExecutor = row.generation !== manifest.generation) || - definitionRows.some((row) => row.generation !== manifest.generation) - ) { - return []; - } - return rows.map((row) => rowToTool(row)); + yield* Effect.logInfo("executor tool sync lost its claim to a newer build", { + integration: String(ref.integration), + connection: String(ref.name), + }); } - - return result.tools.map((tool: ToolDef) => - rowToTool( - { - tenant: keys.tenant, - owner: keys.owner, - subject: keys.subject, - integration: String(ref.integration), - connection: String(ref.name), - plugin_id: integrationRow.plugin_id, - name: String(tool.name), - description: tool.description ?? "", - input_schema: tool.inputSchema ?? null, - output_schema: tool.outputSchema ?? null, - annotations: tool.annotations ?? null, - created_at: now, - updated_at: now, - } as ConnectionToolRow, - tool.annotations, - ), - ); + // Report what is PERSISTED, never what this build discovered — and + // only if what is persisted is one whole build, checked the way every + // list checks it (manifest vs rows). That holds for the loser (it + // wrote nothing) and for the apparent winner alike: on D1 the guard + // commits before the row batch, so a build can see `applied: true` + // and still have its rows land AFTER a later build's manifest, leaving + // manifest B over rows A. Returning this build's local discovery + // there would hand the caller a catalog the next list refuses. When + // the persisted state is not whole, stale-mark the connection so the + // next read rebuilds it, and report an empty catalog rather than a + // wrong one. This cannot go through `describeAll`: that runs the + // stale sync, which would re-enter this very connection's in-flight + // production. + const [rows, persistedDefinitions, connectionRow] = yield* Effect.all([ + core.findMany("tool", { where, select: [...TOOL_INVOCATION_COLUMNS, "generation"] }), + core.findMany("definition", { where }), + findConnectionRow(ref), + ]); + const manifest = connectionRow + ? Option.getOrNull(decodeCatalogManifest(decodeJsonColumn(connectionRow.tools_manifest))) + : null; + const whole = + manifest !== null && + rows.length === manifest.tools && + persistedDefinitions.length === manifest.definitions && + rows.every((row) => row.generation === manifest.generation) && + persistedDefinitions.every((row) => row.generation === manifest.generation); + if (!whole) { + yield* core.updateMany("connection", { + where: connectionWhere, + set: { tools_synced_at: null }, + }); + return []; + } + return rows.map((row) => rowToTool(row)); }); type ToolProductionError = IntegrationNotFoundError | StorageFailure; From cbd416b8b96da9ab23b1e29280727bcbdb0f5573 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:17:07 -0700 Subject: [PATCH 16/19] A refused torn catalog is proven recovered on the next read; drop a duplicate log --- packages/core/sdk/src/executor.test.ts | 20 ++++++++++++++++++++ packages/core/sdk/src/executor.ts | 11 ++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 5102cf6d25..4e57bd2f5e 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -915,6 +915,26 @@ describe("createExecutor", () => { ); const outcome = yield* Effect.result(executor.tools.describeAll()); expect(Result.isFailure(outcome), "a partial catalog is refused, not served").toBe(true); + // The refusal is not the end: it stale-marks exactly this connection, + // and the very next read's stale scan rebuilds it and serves it whole. + const [marked] = yield* Effect.promise(() => + config.db.findMany("connection", { where: (b) => b("integration", "=", String(INTEG)) }), + ); + expect(marked?.tools_synced_at, "the torn connection is stale-marked").toBeNull(); + const recovered = yield* executor.tools.describeAll(); + expect( + recovered.map((tool) => tool.name).sort(), + "the next read rebuilds and serves", + ).toEqual(["inspect", "run"]); + const inlined = recovered.find((tool) => tool.name === "inspect")?.inputSchema as { + $defs?: Record; + }; + expect(Object.keys(inlined.$defs ?? {}).sort(), "definitions are back").toEqual([ + "Cat", + "Collar", + "Dog", + "Pet", + ]); }), ); diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index c5858ead48..efcb0e24fb 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -3805,19 +3805,16 @@ export const createExecutor = Date: Fri, 4 Sep 2026 02:31:56 -0700 Subject: [PATCH 17/19] Kysely replaceMany; describeAll ignores torn catalogs the projection blocks --- .../core/fumadb/src/adapters/kysely/query.ts | 52 ++++++++++++++++++ packages/core/sdk/src/executor.test.ts | 55 +++++++++++++++++++ packages/core/sdk/src/executor.ts | 34 +++++++++--- 3 files changed, 134 insertions(+), 7 deletions(-) diff --git a/packages/core/fumadb/src/adapters/kysely/query.ts b/packages/core/fumadb/src/adapters/kysely/query.ts index 8965c0a659..ac395aa1c8 100644 --- a/packages/core/fumadb/src/adapters/kysely/query.ts +++ b/packages/core/fumadb/src/adapters/kysely/query.ts @@ -28,6 +28,15 @@ import { deserialize, serialize } from "../../schema/serialize"; import type { KyselyConfig } from "../../shared/config"; import type { SQLProvider } from "../../shared/providers"; +/** Thrown inside a `replaceMany` transaction to abort it when the guard + * matched no row; caught at the boundary and reported as `applied: false`. */ +class ReplaceGuardMiss extends Error { + constructor() { + super("replaceMany guard matched no row"); + this.name = "ReplaceGuardMiss"; + } +} + function fullSQLName(column: AnyColumn) { return `${column.table.names.sql}.${column.names.sql}`; } @@ -508,6 +517,49 @@ export function fromKysely( } await query.execute(); }, + replaceMany(plan) { + // Every Kysely dialect has interactive transactions, so the whole plan + // is one transaction: the guard update first, and a guard that matched + // no row aborts it (rollback) before any delete or insert runs. + return kysely + .transaction() + .execute(async (tx) => { + if (plan.guard) { + let update = tx + .updateTable(plan.guard.table.names.sql) + .set(encodeValues(plan.guard.set, plan.guard.table, false)); + if (plan.guard.where) { + const where = plan.guard.where; + update = update.where((eb) => buildWhere(where, eb, provider)); + } + const result = await update.executeTakeFirst(); + if (Number(result.numUpdatedRows) === 0) { + // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: abort the driver transaction so nothing after a failed guard commits + throw new ReplaceGuardMiss(); + } + } + for (const del of plan.deletes) { + let query = tx.deleteFrom(del.table.names.sql); + if (del.where) { + const where = del.where; + query = query.where((eb) => buildWhere(where, eb, provider)); + } + await query.execute(); + } + for (const ins of plan.inserts) { + if (ins.values.length === 0) continue; + await tx + .insertInto(ins.table.names.sql) + .values(ins.values.map((v) => encodeValues(v, ins.table, true))) + .execute(); + } + return { applied: true as const }; + }) + .catch((error: unknown) => { + if (error instanceof ReplaceGuardMiss) return { applied: false as const }; + throw error; + }); + }, transaction(run) { return kysely.transaction().execute((ctx) => { const tx = fromKysely(schema, { diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 4e57bd2f5e..39865c1b1f 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -1257,6 +1257,61 @@ describe("createExecutor", () => { }), ); + // A toolkit endpoint is a policy projection: connections it does not + // grant can never contribute a tool. A torn (or simply unrebuildable) + // catalog on one of THOSE must not fail the read for the connection the + // toolkit does grant. Modelled with a provider that blocks everything but + // `main`, and a torn manifest on `other`. + it.effect("tools.describeAll ignores a torn catalog on a connection the projection blocks", () => + Effect.gen(function* () { + const mainOnly = definePlugin(() => ({ + id: "main-only" as const, + storage: () => ({}), + toolPolicyProvider: () => ({ + list: () => + Effect.succeed([ + { + id: "grant-main", + pattern: `${INTEG}.org.${CONN}.*`, + action: "approve" as const, + position: "a0", + }, + ]), + }), + }))(); + const config = makeTestConfig({ plugins: [demoPlugin, mainOnly] as const }); + const executor = yield* createExecutor(config); + yield* executor.demo.seed(); + for (const name of [CONN, ConnectionName.make("other")]) { + yield* executor.connections.create({ + owner: "org", + name, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + } + const before = yield* executor.tools.describeAll(); + expect(before.map((tool) => String(tool.connection)).sort(), "only main is served").toEqual([ + "main", + "main", + ]); + + // Tear `other`: its manifest says N definitions, none are there, and it + // is stamped synced so the stale scan will not quietly repair it. + yield* Effect.promise(() => + config.db.deleteMany("definition", { + where: (b) => b.and(b("integration", "=", String(INTEG)), b("connection", "=", "other")), + }), + ); + const served = yield* executor.tools.describeAll(); + expect(served.map((tool) => String(tool.connection)).sort(), "main still served").toEqual([ + "main", + "main", + ]); + }), + ); + it.effect("execute dispatches a connection-produced tool to the owning plugin", () => Effect.gen(function* () { const executor = yield* makeTestExecutor({ diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index efcb0e24fb..0d0847af72 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -6091,13 +6091,33 @@ export const createExecutor = => + activeToolPolicyProvider === null + ? Effect.succeed(false) + : resolvePolicyFromRuleSet( + `${triple.integration}.${triple.owner}.${triple.connection}.*`, + policyRulesForScope, + undefined, + ).pipe(Effect.map((policy) => policy.action === "block")); const { rows, definitionRows } = yield* readCatalog.pipe( - Effect.flatMap((snapshot) => { - const mixed = mixedGenerations(snapshot); - return mixed.length === 0 - ? Effect.succeed(snapshot) - : Effect.fail(new CatalogMismatch({ connections: mixed })); - }), + Effect.flatMap((snapshot) => + Effect.gen(function* () { + const mixed: ConnectionTriple[] = []; + for (const triple of mixedGenerations(snapshot)) { + if (!(yield* outOfScope(triple))) mixed.push(triple); + } + return mixed.length === 0 + ? snapshot + : yield* Effect.fail(new CatalogMismatch({ connections: mixed })); + }), + ), Effect.retry({ times: DESCRIBE_ALL_GENERATION_RETRIES }), // Recovery backstop. A catalog that is still inconsistent after the // retries is not a rebuild landing mid-read; it is one that landed @@ -6134,7 +6154,7 @@ export const createExecutor = >(); for (const def of definitionRows) { const key = connectionKey(def); From 6bab5b20fb4d703f789f04cb40f906b36c28b7c4 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:43:10 -0700 Subject: [PATCH 18/19] Toolkit connection scope from real grant overlap; Kysely inserts are parameter-budgeted --- .../core/fumadb/src/adapters/kysely/query.ts | 32 ++++++++-- packages/core/sdk/src/executor.test.ts | 58 ++++++++++++++++++- packages/core/sdk/src/executor.ts | 48 +++++++-------- packages/core/sdk/src/plugin.ts | 18 ++++++ packages/plugins/toolkits/src/server.test.ts | 43 ++++++++++++++ packages/plugins/toolkits/src/server.ts | 53 ++++++++++++++++- 6 files changed, 221 insertions(+), 31 deletions(-) diff --git a/packages/core/fumadb/src/adapters/kysely/query.ts b/packages/core/fumadb/src/adapters/kysely/query.ts index ac395aa1c8..b28d306571 100644 --- a/packages/core/fumadb/src/adapters/kysely/query.ts +++ b/packages/core/fumadb/src/adapters/kysely/query.ts @@ -28,6 +28,16 @@ import { deserialize, serialize } from "../../schema/serialize"; import type { KyselyConfig } from "../../shared/config"; import type { SQLProvider } from "../../shared/providers"; +/** Row cap per insert statement in `replaceMany`, before the parameter + * budget is applied. */ +const REPLACE_MANY_INSERT_BATCH_ROWS = 500; + +/** Conservative bound-parameter budget per statement for each provider. MSSQL + * caps at 2100; SQLite's historical floor is 999; Postgres and MySQL allow + * far more, but the same floor keeps a wide table from overflowing anywhere. */ +const replaceManyParameterBudget = (provider: SQLProvider): number => + provider === "mssql" ? 2000 : 999; + /** Thrown inside a `replaceMany` transaction to abort it when the guard * matched no row; caught at the boundary and reported as `applied: false`. */ class ReplaceGuardMiss extends Error { @@ -548,10 +558,24 @@ export function fromKysely( } for (const ins of plan.inserts) { if (ins.values.length === 0) continue; - await tx - .insertInto(ins.table.names.sql) - .values(ins.values.map((v) => encodeValues(v, ins.table, true))) - .execute(); + const encoded = ins.values.map((v) => encodeValues(v, ins.table, true)); + // Engines cap bound parameters per statement (MSSQL 2100, older + // SQLite 999): chunk so `rows * columns` stays under the budget, + // all inside this one transaction. + const columnsPerRow = Math.max(1, Object.keys(encoded[0]!).length); + const rowsPerStatement = Math.max( + 1, + Math.min( + REPLACE_MANY_INSERT_BATCH_ROWS, + Math.floor(replaceManyParameterBudget(provider) / columnsPerRow), + ), + ); + for (let i = 0; i < encoded.length; i += rowsPerStatement) { + await tx + .insertInto(ins.table.names.sql) + .values(encoded.slice(i, i + rowsPerStatement)) + .execute(); + } } return { applied: true as const }; }) diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 39865c1b1f..82585099c3 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -1260,8 +1260,10 @@ describe("createExecutor", () => { // A toolkit endpoint is a policy projection: connections it does not // grant can never contribute a tool. A torn (or simply unrebuildable) // catalog on one of THOSE must not fail the read for the connection the - // toolkit does grant. Modelled with a provider that blocks everything but - // `main`, and a torn manifest on `other`. + // toolkit does grant. Only a provider that can PROVE it (a prepared + // connection-scope predicate) gets to exclude a connection; a bare + // rule-list provider keeps every connection strict. Modelled with a + // provider that grants only `main`, and a torn manifest on `other`. it.effect("tools.describeAll ignores a torn catalog on a connection the projection blocks", () => Effect.gen(function* () { const mainOnly = definePlugin(() => ({ @@ -1277,6 +1279,10 @@ describe("createExecutor", () => { position: "a0", }, ]), + prepareConnectionScope: () => + Effect.succeed( + (connection: { readonly name: string }) => connection.name === String(CONN), + ), }), }))(); const config = makeTestConfig({ plugins: [demoPlugin, mainOnly] as const }); @@ -1312,6 +1318,54 @@ describe("createExecutor", () => { }), ); + // The inverse: a provider with no connection-scope predicate cannot prove + // anything about `other`, so its torn catalog fails the read — strict by + // default, never silently lenient. + it.effect( + "tools.describeAll stays strict under a provider without a connection-scope predicate", + () => + Effect.gen(function* () { + const mainOnlyNoScope = definePlugin(() => ({ + id: "main-only-noscope" as const, + storage: () => ({}), + toolPolicyProvider: () => ({ + list: () => + Effect.succeed([ + { + id: "grant-main", + pattern: `${INTEG}.org.${CONN}.*`, + action: "approve" as const, + position: "a0", + }, + ]), + }), + }))(); + const config = makeTestConfig({ plugins: [demoPlugin, mainOnlyNoScope] as const }); + const executor = yield* createExecutor(config); + yield* executor.demo.seed(); + for (const name of [CONN, ConnectionName.make("other")]) { + yield* executor.connections.create({ + owner: "org", + name, + integration: INTEG, + template: TEMPLATE, + from: { provider: ProviderKey.make("memory"), id: ProviderItemId.make("v") }, + }); + } + yield* executor.tools.describeAll(); + yield* Effect.promise(() => + config.db.deleteMany("definition", { + where: (b) => + b.and(b("integration", "=", String(INTEG)), b("connection", "=", "other")), + }), + ); + const outcome = yield* Effect.result(executor.tools.describeAll()); + expect(Result.isFailure(outcome), "without proof, a torn connection fails the read").toBe( + true, + ); + }), + ); + it.effect("execute dispatches a connection-produced tool to the owning plugin", () => Effect.gen(function* () { const executor = yield* makeTestExecutor({ diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 0d0847af72..4a1b5216d6 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -6092,32 +6092,32 @@ export const createExecutor = => - activeToolPolicyProvider === null - ? Effect.succeed(false) - : resolvePolicyFromRuleSet( - `${triple.integration}.${triple.owner}.${triple.connection}.*`, - policyRulesForScope, - undefined, - ).pipe(Effect.map((policy) => policy.action === "block")); + const canServeConnection = + activeToolPolicyProvider?.prepareConnectionScope === undefined + ? null + : yield* activeToolPolicyProvider.prepareConnectionScope(); const { rows, definitionRows } = yield* readCatalog.pipe( - Effect.flatMap((snapshot) => - Effect.gen(function* () { - const mixed: ConnectionTriple[] = []; - for (const triple of mixedGenerations(snapshot)) { - if (!(yield* outOfScope(triple))) mixed.push(triple); - } - return mixed.length === 0 - ? snapshot - : yield* Effect.fail(new CatalogMismatch({ connections: mixed })); - }), - ), + Effect.flatMap((snapshot) => { + const mixed = mixedGenerations(snapshot).filter( + (triple) => + canServeConnection === null || + canServeConnection({ + integration: triple.integration, + owner: triple.owner, + name: triple.connection, + }), + ); + return mixed.length === 0 + ? Effect.succeed(snapshot) + : Effect.fail(new CatalogMismatch({ connections: mixed })); + }), Effect.retry({ times: DESCRIBE_ALL_GENERATION_RETRIES }), // Recovery backstop. A catalog that is still inconsistent after the // retries is not a rebuild landing mid-read; it is one that landed diff --git a/packages/core/sdk/src/plugin.ts b/packages/core/sdk/src/plugin.ts index 2ace32f891..5b4dddc533 100644 --- a/packages/core/sdk/src/plugin.ts +++ b/packages/core/sdk/src/plugin.ts @@ -138,6 +138,24 @@ export interface ToolPolicyProvider { }) => EffectivePolicy, StorageFailure >; + /** + * Prepared connection-scope predicate: can ANY tool of this connection be + * visible through this provider? Providers that are capability allowlists + * over connection patterns (toolkits) answer from real pattern overlap — + * a grant of `acme.org.main.issues.*` means yes for `acme/org/main` even + * though the connection-wide wildcard id would not match it. Core uses + * this to leave a connection's catalog out of a consistency check when + * nothing under it could ever be served; a provider that does not + * implement it keeps every connection under strict validation. + */ + readonly prepareConnectionScope?: () => Effect.Effect< + (connection: { + readonly integration: string; + readonly owner: string; + readonly name: string; + }) => boolean, + StorageFailure + >; } // --------------------------------------------------------------------------- diff --git a/packages/plugins/toolkits/src/server.test.ts b/packages/plugins/toolkits/src/server.test.ts index bab67eb0e9..3b8bb0dcf7 100644 --- a/packages/plugins/toolkits/src/server.test.ts +++ b/packages/plugins/toolkits/src/server.test.ts @@ -199,4 +199,47 @@ describe("toolkitsPlugin", () => { ).toContain("google_docs.org.* approve"); }), ); + + // "Can any tool of this connection be visible?" must come from real grant + // overlap. A grant NARROWER than the connection (`github.org.main.issues.*`) + // still makes the connection servable, even though the connection-wide + // wildcard id `github.org.main.*` would not match that pattern — which is + // exactly the trap a synthetic-id probe falls into. + it.effect("prepareConnectionScope answers from grant overlap, not a synthetic wildcard id", () => + Effect.gen(function* () { + const executor = yield* makeTestExecutor({ + plugins: [toolkitsPlugin()] as const, + }); + const toolkit = yield* executor.toolkits.create({ owner: "org", name: "Narrow Kit" }); + yield* executor.toolkits.createConnection(toolkit.id, { + pattern: "github.org.main.issues.*", + }); + const canServe = yield* executor.toolkits.prepareConnectionScopeForSlug(toolkit.slug); + + // The narrowly granted connection IS servable. + expect(canServe({ integration: "github", owner: "org", name: "main" })).toBe(true); + // Sibling connections and other integrations are not. + expect(canServe({ integration: "github", owner: "org", name: "other" })).toBe(false); + expect(canServe({ integration: "linear", owner: "org", name: "main" })).toBe(false); + // An org toolkit never serves a personal connection. + expect(canServe({ integration: "github", owner: "user", name: "main" })).toBe(false); + + // Segment wildcards in the grant head still resolve. + const wide = yield* executor.toolkits.create({ owner: "org", name: "Wide Kit" }); + yield* executor.toolkits.createConnection(wide.id, { pattern: "github.*.*.issues.*" }); + const canServeWide = yield* executor.toolkits.prepareConnectionScopeForSlug(wide.slug); + expect(canServeWide({ integration: "github", owner: "org", name: "anything" })).toBe(true); + expect(canServeWide({ integration: "linear", owner: "org", name: "main" })).toBe(false); + + // The universal grant serves everything the owner model allows. + const all = yield* executor.toolkits.create({ owner: "org", name: "All Kit" }); + yield* executor.toolkits.createConnection(all.id, { pattern: "*" }); + const canServeAll = yield* executor.toolkits.prepareConnectionScopeForSlug(all.slug); + expect(canServeAll({ integration: "linear", owner: "org", name: "main" })).toBe(true); + + // An unknown toolkit serves nothing. + const none = yield* executor.toolkits.prepareConnectionScopeForSlug("no-such-kit"); + expect(none({ integration: "github", owner: "org", name: "main" })).toBe(false); + }), + ); }); diff --git a/packages/plugins/toolkits/src/server.ts b/packages/plugins/toolkits/src/server.ts index 7dacc3e456..713a8ec905 100644 --- a/packages/plugins/toolkits/src/server.ts +++ b/packages/plugins/toolkits/src/server.ts @@ -542,6 +542,52 @@ const makeToolkitsExtension = (ctx: PluginCtx) => { }; }); + // Can any tool of this connection be visible through the toolkit? True + // when any granting pattern (a connection grant, or a legacy policy that + // acts as one) is UNDER the connection — i.e. begins with + // `..` followed by `.` or is exactly that prefix + // with a trailing wildcard — or is the universal `*`. A pattern rooted + // elsewhere can never match a tool id of this connection, whatever its + // remaining segments say. + const prepareConnectionScopeForSlug = ( + slug: string, + ): Effect.Effect< + (connection: { + readonly integration: string; + readonly owner: string; + readonly name: string; + }) => boolean, + StorageFailure + > => + Effect.gen(function* () { + const toolkit = yield* getBySlugEntry(slug); + if (!toolkit) return () => false; + const isOrg = toolkit.owner === "org"; + const policies = yield* listPoliciesForRecord(toolkit.data.id); + const connections = yield* listConnectionsForRecord(toolkit.data.id); + const legacyPolicyIds = legacyConnectionPolicyIds(policies, connections); + const grants = [ + ...connections.map((connection) => connection.pattern), + ...policies.filter((policy) => legacyPolicyIds.has(policy.id)).map((p) => p.pattern), + ]; + return (connection: { + readonly integration: string; + readonly owner: string; + readonly name: string; + }) => { + if (isOrg && connection.owner === "user") return false; + const root = `${connection.integration}.${connection.owner}.${connection.name}`; + return grants.some((pattern) => { + if (pattern === "*") return true; + // A grant under the connection root, or a segment-wildcarded grant + // that still matches the root's three segments. + if (pattern === root || pattern.startsWith(`${root}.`)) return true; + const head = pattern.split(".").slice(0, 3).join("."); + return matchPattern(`${head}.*`, `${root}.x`); + }); + }; + }); + return { list, create, @@ -561,6 +607,7 @@ const makeToolkitsExtension = (ctx: PluginCtx) => { policyRulesForSlug, resolvePolicyForSlug, preparePolicyResolverForSlug, + prepareConnectionScopeForSlug, }; }; @@ -674,7 +721,10 @@ const ToolkitsHandlers = HttpApiBuilder.group(ExecutorApiWithToolkits, "toolkits const makePolicyProvider = ( extension: Pick< ToolkitsExtension, - "policyRulesForSlug" | "resolvePolicyForSlug" | "preparePolicyResolverForSlug" + | "policyRulesForSlug" + | "resolvePolicyForSlug" + | "preparePolicyResolverForSlug" + | "prepareConnectionScopeForSlug" >, slug: string, ): ToolPolicyProvider => ({ @@ -684,6 +734,7 @@ const makePolicyProvider = ( // Preferred path: core calls this once per operation, so the toolkit's // policies + connections are fetched once instead of once per tool. prepare: () => extension.preparePolicyResolverForSlug(slug), + prepareConnectionScope: () => extension.prepareConnectionScopeForSlug(slug), }); export const toolkitsPlugin = definePlugin((options: ToolkitsPluginOptions = {}) => { From 75651f9f250e93ae7502d1f3963bf600c3884aa6 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Fri, 4 Sep 2026 02:52:23 -0700 Subject: [PATCH 19/19] Policy resolver and connection scope come from one prepared snapshot; batch inserts by widest row --- .../core/fumadb/src/adapters/drizzle/query.ts | 4 +- .../core/fumadb/src/adapters/kysely/query.ts | 4 +- packages/core/sdk/src/executor.test.ts | 15 ++- packages/core/sdk/src/executor.ts | 24 ++-- packages/core/sdk/src/index.ts | 1 + packages/core/sdk/src/plugin.ts | 51 ++++---- packages/plugins/toolkits/src/server.ts | 117 +++++++++--------- 7 files changed, 114 insertions(+), 102 deletions(-) diff --git a/packages/core/fumadb/src/adapters/drizzle/query.ts b/packages/core/fumadb/src/adapters/drizzle/query.ts index d3dbe1c52c..73e84ce5ac 100644 --- a/packages/core/fumadb/src/adapters/drizzle/query.ts +++ b/packages/core/fumadb/src/adapters/drizzle/query.ts @@ -667,7 +667,9 @@ export function fromDrizzle( if (ins.values.length === 0) continue; const drizzleTable = toDrizzle(ins.table); const values = ins.values.map((v) => mapValues(v, ins.table)); - const columnsPerRow = Math.max(1, Object.keys(values[0]!).length); + // Drizzle builds a multi-row INSERT over the UNION of every row's + // columns, so the widest row sets the per-row parameter count. + const columnsPerRow = Math.max(1, ...values.map((row) => Object.keys(row).length)); const batchSize = parameterBoundedBatchSize(ins.table, columnsPerRow, 0, maxBoundParameters); for (let i = 0; i < values.length; i += batchSize) { statements.push(handle.insert(drizzleTable).values(values.slice(i, i + batchSize))); diff --git a/packages/core/fumadb/src/adapters/kysely/query.ts b/packages/core/fumadb/src/adapters/kysely/query.ts index b28d306571..e20a47d6a0 100644 --- a/packages/core/fumadb/src/adapters/kysely/query.ts +++ b/packages/core/fumadb/src/adapters/kysely/query.ts @@ -562,7 +562,9 @@ export function fromKysely( // Engines cap bound parameters per statement (MSSQL 2100, older // SQLite 999): chunk so `rows * columns` stays under the budget, // all inside this one transaction. - const columnsPerRow = Math.max(1, Object.keys(encoded[0]!).length); + // Kysely builds a multi-row INSERT over the UNION of every row's + // columns, so the widest row sets the per-row parameter count. + const columnsPerRow = Math.max(1, ...encoded.map((row) => Object.keys(row).length)); const rowsPerStatement = Math.max( 1, Math.min( diff --git a/packages/core/sdk/src/executor.test.ts b/packages/core/sdk/src/executor.test.ts index 82585099c3..4df22cb892 100644 --- a/packages/core/sdk/src/executor.test.ts +++ b/packages/core/sdk/src/executor.test.ts @@ -4,6 +4,7 @@ import { Data, Effect, Inspectable, Logger, Predicate, Result, Scheduler } from import { ElicitationResponse, type ElicitationHandler } from "./elicitation"; import { ToolNotFoundError } from "./errors"; import { createExecutor } from "./executor"; +import { matchPattern } from "./policies"; import { StorageError, type FumaDb } from "./fuma-runtime"; import { AuthTemplateSlug, @@ -1279,10 +1280,16 @@ describe("createExecutor", () => { position: "a0", }, ]), - prepareConnectionScope: () => - Effect.succeed( - (connection: { readonly name: string }) => connection.name === String(CONN), - ), + // One snapshot yields both the resolver and the scope predicate. + prepare: () => + Effect.succeed({ + resolve: ({ toolId }: { readonly toolId: string }) => + matchPattern(`${INTEG}.org.${CONN}.*`, toolId) + ? { action: "approve" as const, source: "user" as const } + : { action: "block" as const, source: "user" as const }, + canServeConnection: (connection: { readonly name: string }) => + connection.name === String(CONN), + }), }), }))(); const config = makeTestConfig({ plugins: [demoPlugin, mainOnly] as const }); diff --git a/packages/core/sdk/src/executor.ts b/packages/core/sdk/src/executor.ts index 4a1b5216d6..0ea003f5ff 100644 --- a/packages/core/sdk/src/executor.ts +++ b/packages/core/sdk/src/executor.ts @@ -177,6 +177,7 @@ import type { OwnerBinding, PluginCtx, PluginExtensions, + PreparedToolPolicy, ResolveToolsResult, StaticIntegrationDecl, StaticToolDecl, @@ -5440,13 +5441,7 @@ export const createExecutor = EffectivePolicy; - }; + | { readonly kind: "prepared"; readonly prepared: PreparedToolPolicy }; const compareProviderPolicyRule = ( a: ToolPolicyProviderRule, @@ -5486,9 +5481,9 @@ export const createExecutor = ({ + Effect.map((prepared) => ({ kind: "prepared" as const, - resolve, + prepared, })), ) : activeToolPolicyProvider.resolve @@ -5514,7 +5509,7 @@ export const createExecutor = => ruleSet.kind === "prepared" - ? Effect.succeed(ruleSet.resolve({ toolId, defaultRequiresApproval })) + ? Effect.succeed(ruleSet.prepared.resolve({ toolId, defaultRequiresApproval })) : ruleSet.kind === "provider" ? ruleSet.provider.resolve ? ruleSet.provider.resolve({ toolId, defaultRequiresApproval }) @@ -6098,11 +6093,14 @@ export const createExecutor = { const mixed = mixedGenerations(snapshot).filter( diff --git a/packages/core/sdk/src/index.ts b/packages/core/sdk/src/index.ts index a4535c0b33..46f41fed23 100644 --- a/packages/core/sdk/src/index.ts +++ b/packages/core/sdk/src/index.ts @@ -388,6 +388,7 @@ export { type OwnerBinding, type ToolPolicyProvider, type ToolPolicyProviderRule, + type PreparedToolPolicy, type IntegrationRecord, type StaticIntegrationDecl, type StaticToolDecl, diff --git a/packages/core/sdk/src/plugin.ts b/packages/core/sdk/src/plugin.ts index 5b4dddc533..a400f4b1e0 100644 --- a/packages/core/sdk/src/plugin.ts +++ b/packages/core/sdk/src/plugin.ts @@ -131,31 +131,36 @@ export interface ToolPolicyProvider { * requests), so caching on it would serve stale policy state. Each operation * gets a fresh snapshot. */ - readonly prepare?: () => Effect.Effect< - (input: { - readonly toolId: string; - readonly defaultRequiresApproval?: boolean; - }) => EffectivePolicy, - StorageFailure - >; + readonly prepare?: () => Effect.Effect; +} + +/** + * What one `prepare()` call hands back: a pure per-tool resolver and, for + * providers that can answer it, a pure connection-scope predicate — BOTH + * derived from the same storage snapshot, so a grant that changes between + * two reads cannot make the resolver serve a connection the predicate has + * just declared unservable (or vice versa). + */ +export interface PreparedToolPolicy { + readonly resolve: (input: { + readonly toolId: string; + readonly defaultRequiresApproval?: boolean; + }) => EffectivePolicy; /** - * Prepared connection-scope predicate: can ANY tool of this connection be - * visible through this provider? Providers that are capability allowlists - * over connection patterns (toolkits) answer from real pattern overlap — - * a grant of `acme.org.main.issues.*` means yes for `acme/org/main` even - * though the connection-wide wildcard id would not match it. Core uses - * this to leave a connection's catalog out of a consistency check when - * nothing under it could ever be served; a provider that does not - * implement it keeps every connection under strict validation. + * Can ANY tool of this connection be visible through this provider? + * Providers that are capability allowlists over connection patterns + * (toolkits) answer from real pattern overlap — a grant of + * `acme.org.main.issues.*` means yes for `acme/org/main` even though the + * connection-wide wildcard id would not match it. Core uses this to leave + * a connection's catalog out of a consistency check when nothing under it + * could ever be served; a provider that omits it keeps every connection + * under strict validation. */ - readonly prepareConnectionScope?: () => Effect.Effect< - (connection: { - readonly integration: string; - readonly owner: string; - readonly name: string; - }) => boolean, - StorageFailure - >; + readonly canServeConnection?: (connection: { + readonly integration: string; + readonly owner: string; + readonly name: string; + }) => boolean; } // --------------------------------------------------------------------------- diff --git a/packages/plugins/toolkits/src/server.ts b/packages/plugins/toolkits/src/server.ts index 713a8ec905..9aadaee4f7 100644 --- a/packages/plugins/toolkits/src/server.ts +++ b/packages/plugins/toolkits/src/server.ts @@ -13,6 +13,7 @@ import { type PluginStorageFacade, type PluginStorageCollectionFacade, type StorageFailure, + type PreparedToolPolicy, type ToolPolicyAction, type ToolPolicyProvider, type ToolPolicyProviderRule, @@ -516,32 +517,6 @@ const makeToolkitsExtension = (ctx: PluginCtx) => { // policies + connections on every tool, which is the per-tool N+1 that scales // with the whole catalog on the list surface. This is byte-for-byte the same // resolution, just hoisted out of the loop. - const preparePolicyResolverForSlug = ( - slug: string, - ): Effect.Effect< - (input: { - readonly toolId: string; - readonly defaultRequiresApproval?: boolean; - }) => EffectivePolicy, - StorageFailure - > => - Effect.gen(function* () { - const toolkit = yield* getBySlugEntry(slug); - if (!toolkit) return () => blockedPolicy(); - const isOrg = toolkit.owner === "org"; - const policies = yield* listPoliciesForRecord(toolkit.data.id); - const connections = yield* listConnectionsForRecord(toolkit.data.id); - return (input: { readonly toolId: string; readonly defaultRequiresApproval?: boolean }) => { - if (isOrg && isPersonalDynamicToolId(input.toolId)) return blockedPolicy(); - return resolveToolkitPolicy( - input.toolId, - connections, - policies, - input.defaultRequiresApproval, - ); - }; - }); - // Can any tool of this connection be visible through the toolkit? True // when any granting pattern (a connection grant, or a legacy policy that // acts as one) is UNDER the connection — i.e. begins with @@ -549,45 +524,70 @@ const makeToolkitsExtension = (ctx: PluginCtx) => { // with a trailing wildcard — or is the universal `*`. A pattern rooted // elsewhere can never match a tool id of this connection, whatever its // remaining segments say. - const prepareConnectionScopeForSlug = ( - slug: string, - ): Effect.Effect< - (connection: { + const connectionScopeFrom = ( + isOrg: boolean, + policies: readonly ToolkitPolicyRecord[], + connections: readonly ToolkitConnectionRecord[], + ) => { + const legacyPolicyIds = legacyConnectionPolicyIds(policies, connections); + const grants = [ + ...connections.map((connection) => connection.pattern), + ...policies.filter((policy) => legacyPolicyIds.has(policy.id)).map((p) => p.pattern), + ]; + return (connection: { readonly integration: string; readonly owner: string; readonly name: string; - }) => boolean, - StorageFailure - > => + }): boolean => { + if (isOrg && connection.owner === "user") return false; + const root = `${connection.integration}.${connection.owner}.${connection.name}`; + return grants.some((pattern) => { + if (pattern === "*") return true; + // A grant under the connection root, or a segment-wildcarded grant + // that still matches the root's three segments. + if (pattern === root || pattern.startsWith(`${root}.`)) return true; + const head = pattern.split(".").slice(0, 3).join("."); + return matchPattern(`${head}.*`, `${root}.x`); + }); + }; + }; + + // ONE read of the toolkit's policies + connections yields BOTH the per-tool + // resolver and the connection-scope predicate, so the two can never + // disagree about a grant that changed between reads. + const preparePolicyResolverForSlug = ( + slug: string, + ): Effect.Effect => Effect.gen(function* () { const toolkit = yield* getBySlugEntry(slug); - if (!toolkit) return () => false; + if (!toolkit) return { resolve: () => blockedPolicy(), canServeConnection: () => false }; const isOrg = toolkit.owner === "org"; const policies = yield* listPoliciesForRecord(toolkit.data.id); const connections = yield* listConnectionsForRecord(toolkit.data.id); - const legacyPolicyIds = legacyConnectionPolicyIds(policies, connections); - const grants = [ - ...connections.map((connection) => connection.pattern), - ...policies.filter((policy) => legacyPolicyIds.has(policy.id)).map((p) => p.pattern), - ]; - return (connection: { - readonly integration: string; - readonly owner: string; - readonly name: string; - }) => { - if (isOrg && connection.owner === "user") return false; - const root = `${connection.integration}.${connection.owner}.${connection.name}`; - return grants.some((pattern) => { - if (pattern === "*") return true; - // A grant under the connection root, or a segment-wildcarded grant - // that still matches the root's three segments. - if (pattern === root || pattern.startsWith(`${root}.`)) return true; - const head = pattern.split(".").slice(0, 3).join("."); - return matchPattern(`${head}.*`, `${root}.x`); - }); + return { + resolve: (input: { + readonly toolId: string; + readonly defaultRequiresApproval?: boolean; + }) => { + if (isOrg && isPersonalDynamicToolId(input.toolId)) return blockedPolicy(); + return resolveToolkitPolicy( + input.toolId, + connections, + policies, + input.defaultRequiresApproval, + ); + }, + canServeConnection: connectionScopeFrom(isOrg, policies, connections), }; }); + /** The scope predicate alone, for callers that want only it. Same snapshot + * rules as `preparePolicyResolverForSlug`, from which it is derived. */ + const prepareConnectionScopeForSlug = (slug: string) => + preparePolicyResolverForSlug(slug).pipe( + Effect.map((prepared) => prepared.canServeConnection ?? (() => false)), + ); + return { list, create, @@ -721,10 +721,7 @@ const ToolkitsHandlers = HttpApiBuilder.group(ExecutorApiWithToolkits, "toolkits const makePolicyProvider = ( extension: Pick< ToolkitsExtension, - | "policyRulesForSlug" - | "resolvePolicyForSlug" - | "preparePolicyResolverForSlug" - | "prepareConnectionScopeForSlug" + "policyRulesForSlug" | "resolvePolicyForSlug" | "preparePolicyResolverForSlug" >, slug: string, ): ToolPolicyProvider => ({ @@ -732,9 +729,9 @@ const makePolicyProvider = ( resolve: ({ toolId, defaultRequiresApproval }) => extension.resolvePolicyForSlug(slug, toolId, defaultRequiresApproval), // Preferred path: core calls this once per operation, so the toolkit's - // policies + connections are fetched once instead of once per tool. + // policies + connections are fetched once instead of once per tool — and + // the connection-scope predicate rides on the same snapshot. prepare: () => extension.preparePolicyResolverForSlug(slug), - prepareConnectionScope: () => extension.prepareConnectionScopeForSlug(slug), }); export const toolkitsPlugin = definePlugin((options: ToolkitsPluginOptions = {}) => {