fix(openapi): reduce large import memory usage

.changeset/openapi-large-catalog-memory.md

@@ -0,0 +1,6 @@
+---
+"@executor-js/sdk": patch
+"@executor-js/plugin-openapi": patch
+---
+
+Reduce large OpenAPI import memory by compiling operation bindings in chunks and filtering integration storage prefixes in SQL.

e2e/scenarios/openapi-large-catalog.test.ts

@@ -0,0 +1,163 @@
+import { randomUUID } from "node:crypto";
+import { assert, expect } from "@effect/vitest";
+import { Effect } from "effect";
+import { composePluginApi } from "@executor-js/api/server";
+import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api";
+import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared";
+import { scenario } from "../src/scenario";
+import { Api, Browser, Target } from "../src/services";
+
+import { visit } from "../src/surfaces/browser";
+
+const api = composePluginApi([openApiHttpPlugin()] as const);
+const spec = (count: number) =>
+  JSON.stringify({
+    openapi: "3.0.3",
+    info: { title: "Large catalog", version: "1" },
+    paths: Object.fromEntries(
+      Array.from({ length: count }, (_, i) => [
+        `/items/${i}`,
+        {
+          get: {
+            operationId: `item${i}`,
+            responses: {
+              "200": {
+                description: "Item details",
+                content: { "application/json": { schema: { $ref: "#/components/schemas/Item" } } },
+              },
+            },
+          },
+        },
+      ]),
+    ),
+    components: {
+      schemas: {
+        Item: {
+          type: "object",
+          properties: {
+            id: { type: "string" },
+            description: { type: "string", description: "Item description. ".repeat(2000) },
+          },
+        },
+      },
+    },
+  });
+
+scenario(
+  "OpenAPI · large imports report complete counts and preserve unrelated catalogs",
+  {},
+  Effect.gen(function* () {
+    const target = yield* Target;
+    const { client } = yield* Api;
+    const browser = yield* Browser;
+    const identity = yield* target.newIdentity();
+    const http = yield* client(api, identity);
+    const slug = IntegrationSlug.make(`large-${randomUUID()}`);
+    const neighbor = IntegrationSlug.make(`neighbor-${randomUUID()}`);
+    yield* Effect.gen(function* () {
+      const add = (integration: string, count: number) =>
+        http.openapi.addSpec({
+          payload: {
+            slug: integration,
+            spec: { kind: "blob", value: spec(count) },
+            baseUrl: "https://example.invalid",
+            authenticationTemplate: [],
+          },
+        });
+      expect((yield* add(neighbor, 1001)).toolCount).toBe(1001);
+      expect((yield* add(slug, 1201)).toolCount).toBe(1201);
+      yield* http.connections.create({
+        payload: {
+          owner: "org",
+          integration: slug,
+          name: ConnectionName.make("main"),
+          template: AuthTemplateSlug.make("none"),
+          value: "catalog-fixture",
+        },
+      });
+      expect(yield* http.tools.list({ query: { integration: slug } })).toHaveLength(1201);
+      const updated = yield* http.openapi.updateSpec({
+        params: { slug },
+        payload: {
+          spec: { kind: "blob", value: spec(1002) },
+        },
+      });
+      expect(updated.toolCount).toBe(1002);
+      expect(updated.removedTools).toHaveLength(199);
+      expect(updated.addedTools).toEqual([]);
+      const failed = yield* http.openapi
+        .updateSpec({
+          params: { slug },
+          payload: {
+            spec: { kind: "blob", value: "invalid" },
+          },
+        })
+        .pipe(Effect.result);
+      expect(failed._tag).toBe("Failure");
+      const missingPaths = yield* http.openapi
+        .updateSpec({
+          params: { slug },
+          payload: {
+            spec: {
+              kind: "blob",
+              value: JSON.stringify({
+                openapi: "3.1.0",
+                info: { title: "No paths", version: "1" },
+              }),
+            },
+          },
+        })
+        .pipe(Effect.result);
+      expect(missingPaths._tag).toBe("Failure");
+      yield* http.connections.refresh({
+        params: {
+          owner: "org",
+          integration: slug,
+          name: ConnectionName.make("main"),
+        },
+      });
+      expect(yield* http.tools.list({ query: { integration: slug } })).toHaveLength(1002);
+      const intact = yield* http.openapi.updateSpec({
+        params: { slug: neighbor },
+        payload: {
+          spec: { kind: "blob", value: spec(1001) },
+        },
+      });
+      expect(intact.toolCount).toBe(1001);
+      expect(intact.removedTools).toEqual([]);
+      const retry = yield* http.openapi.updateSpec({
+        params: { slug },
+        payload: {
+          spec: { kind: "blob", value: spec(1002) },
+        },
+      });
+      expect(retry.toolCount).toBe(1002);
+      expect(retry.addedTools).toEqual([]);
+      expect(retry.removedTools).toEqual([]);
+      const catalog = yield* http.tools.list({ query: { integration: slug } });
+      expect(catalog).toHaveLength(1002);
+      const first = catalog[0];
+      assert(first);
+      const schema = yield* http.tools.schema({
+        query: { address: first.address },
+      });
+      expect(JSON.stringify(schema)).toContain("description");
+      yield* browser.session(identity, async ({ page, step }) => {
+        await step("Open the imported catalog after refresh", async () => {
+          await visit(page, `/integrations/${slug}?tab=tools`);
+          await page.getByPlaceholder("Filter 1002 tools…").waitFor();
+        });
+      });
+    }).pipe(
+      Effect.ensuring(
+        Effect.forEach(
+          [slug, neighbor],
+          (slug) => http.openapi.removeSpec({ params: { slug } }).pipe(Effect.orDie),
+          {
+            discard: true,
+          },
+        ),
+      ),
+    );
+  }),
+);

e2e/vitest.config.ts

@@ -49,6 +49,7 @@ export default defineConfig({
         include: [
           "scenarios/browser-approval.test.ts",
           "scenarios/microsoft-graph-full.test.ts",
+          "scenarios/openapi-large-catalog.test.ts",
           "scenarios/toolkits-mcp.test.ts",
           "cloudflare/**/*.test.ts",
         ],

packages/core/sdk/src/executor.ts

@@ -2389,7 +2389,7 @@ const makePluginStorageFacade = (input: {
     list: (storageInput) =>
       Effect.gen(function* () {
         const rows = yield* input.core.findMany("plugin_storage", {
-          where: whereFor(storageInput.collection),
+          where: whereForPrefix(storageInput.collection, storageInput.keyPrefix),
         });
         return sortByOwnerPrecedence(rows)
           .filter((row) =>

packages/core/sdk/src/plugin-storage.test.ts

@@ -76,6 +76,8 @@ const executionHistoryPlugin = definePlugin(() => ({
         owner,
         entries: keys.map((key) => ({ collection: toolCalls.name, key })),
       }),
+    list: (keyPrefix?: string) =>
+      ctx.storage.pluginStorage.list({ collection: toolCalls.name, keyPrefix }),
     get: (key: string) => ctx.storage.toolCalls.get({ key }),
     getMany: (keys: readonly string[]) => ctx.storage.toolCalls.getMany({ keys }),
     getForOwner: (owner: Owner, key: string) => ctx.storage.toolCalls.getForOwner({ owner, key }),
@@ -164,6 +166,66 @@ const failPluginStorageBulkWriteAfterFirstRow = (db: FumaDb): FumaDb => {
 };
 
 describe("plugin storage collections", () => {
+  it.effect("filters list prefixes in the database before materializing unrelated rows", () =>
+    Effect.gen(function* () {
+      const config = makeTestConfig({
+        backend: "sqlite",
+        plugins: [executionHistoryPlugin] as const,
+      });
+      const reads: number[] = [];
+      const wrap = (db: FumaDb): FumaDb =>
+        new Proxy(db, {
+          get(target, property, receiver) {
+            if (property === "withContext") {
+              const withContext = target.withContext;
+              return withContext === undefined
+                ? undefined
+                : (context: unknown) => wrap(withContext(context));
+            }
+            if (property === "findMany") {
+              const findMany: FumaDb["findMany"] = async (table, options) => {
+                const rows = await target.findMany(table, options);
+                if (table === "plugin_storage") reads.push(rows.length);
+                return rows;
+              };
+              return findMany;
+            }
+            return Reflect.get(target, property, receiver);
+          },
+        });
+      const executor = yield* Effect.acquireRelease(
+        createExecutor({ ...config, db: wrap(config.db) }),
+        (instance) =>
+          instance
+            .close()
+            .pipe(Effect.orDie, Effect.ensuring(Effect.promise(() => config.testDb.close()))),
+      );
+      const data = call({
+        runId: "run",
+        toolId: "tool",
+        status: "ok",
+        startedAt: "2026-01-01T00:00:00Z",
+      });
+      yield* executor.executionHistory.recordMany("org", [
+        { key: "selected.one", data },
+        ...Array.from({ length: 100 }, (_, i) => ({ key: `unrelated.${i}`, data })),
+      ]);
+      reads.length = 0;
+      expect((yield* executor.executionHistory.list("selected.")).map((row) => row.key)).toEqual([
+        "selected.one",
+      ]);
+      expect(reads).toEqual([1]);
+      yield* executor.executionHistory.recordMany("org", [
+        { key: "literal_%.one", data },
+        { key: "literalXX.one", data },
+        { key: "LITERAL_%.one", data },
+      ]);
+      expect((yield* executor.executionHistory.list("literal_%.")).map((row) => row.key)).toEqual([
+        "literal_%.one",
+      ]);
+    }),
+  );
+
   it.effect("queries declared indexes through the executor's SQLite FumaDB target", () =>
     Effect.gen(function* () {
       const executor = yield* makeTestExecutor({

packages/plugins/openapi/src/sdk/plugin.test.ts

@@ -38,6 +38,7 @@ import {
 } from "@executor-js/sdk/testing";
 
 import { openApiPlugin } from "./plugin";
+import { makeDefaultOpenapiStore } from "./store";
 import { type AuthenticationInput } from "./types";
 import {
   addOpenApiTestConnection,
@@ -1125,7 +1126,26 @@ paths:
     "addSpec accepts Microsoft Graph-scale operation catalogs from one spec",
     () =>
       Effect.gen(function* () {
-        const executor = yield* createExecutor(makeTestConfig({ plugins: testPlugins() }));
+        const batchSizes: number[] = [];
+        const executor = yield* createExecutor(
+          makeTestConfig({
+            plugins: [
+              openApiPlugin({
+                storage: (deps) =>
+                  makeDefaultOpenapiStore({
+                    ...deps,
+                    pluginStorage: {
+                      ...deps.pluginStorage,
+                      putMany: (input) => {
+                        batchSizes.push(input.entries.length);
+                        return deps.pluginStorage.putMany(input);
+                      },
+                    },
+                  }),
+              }),
+            ] as const,
+          }),
+        );
 
         const added = yield* executor.openapi.addSpec({
           spec: { kind: "blob", value: microsoftGraphScaleSpecText() },
@@ -1134,6 +1154,21 @@ paths:
         });
 
         expect(added.toolCount).toBe(MICROSOFT_GRAPH_V1_OPERATION_COUNT);
+        expect(batchSizes.length).toBeGreaterThan(1);
+        expect(Math.max(...batchSizes)).toBeLessThanOrEqual(500);
+        expect(batchSizes.reduce((sum, count) => sum + count, 0)).toBe(
+          MICROSOFT_GRAPH_V1_OPERATION_COUNT,
+        );
+        batchSizes.length = 0;
+        const updated = yield* executor.openapi.updateSpec("microsoft_graph_scale", {
+          spec: { kind: "blob", value: microsoftGraphScaleSpecText() },
+        });
+        expect(updated.toolCount).toBe(MICROSOFT_GRAPH_V1_OPERATION_COUNT);
+        expect(batchSizes.length).toBeGreaterThan(1);
+        expect(Math.max(...batchSizes)).toBeLessThanOrEqual(500);
+        expect(batchSizes.reduce((sum, count) => sum + count, 0)).toBe(
+          MICROSOFT_GRAPH_V1_OPERATION_COUNT,
+        );
       }),
     30_000,
   );