Paginate tools.search and tools.executor.sources.list (#492)

PR #265 reported a real symptom — `tools.executor.sources.list()` silently
returns the first 12 sources, so a model that calls it past alphabetical
position 12 mistakes the page for the full inventory. The proposed fix
returned `{ sources, totalSources }` from `sources.list` only, which:

- Tells the model truncation happened, but provides no way to act on it —
  no offset/cursor parameter exists, so the model can only retry with
  `limit: 999` (defeating pagination).
- Leaves `tools.search` untouched — same truncation hazard, same model.
- Introduces an inconsistent shape: `search` returns a bare array,
  `sources.list` returns an object.

This change paginates both with a single shared shape:

  { items, total, hasMore, nextOffset }

Both helpers accept `{ limit?, offset? }`. `nextOffset` is concrete (the
exact number to pass next) so the model doesn't have to compute
`offset + items.length`. Engine arg validation rejects negative offsets.
The `tools.search` description now teaches the paginated shape and shows
the second-page call.

Default `limit` for `sources.list` drops from 200 to 50; the engine layer
still applies 12 by default for both. The off-by-one ergonomics matter
more than the headroom — most users have well under 50 sources.

Tests cover:
- single-page and empty-query results carry the new shape
- pagination across pages with limit + offset, including offset past end
- source listing pagination via the engine
- negative offsets rejected with a clear message

Closes #265.

Author: RhysSullivan

packages/core/execution/src/description.ts

@@ -34,7 +34,7 @@ const formatDescription = (sources: readonly Source[]): string => {
     "",
     "## Workflow",
     "",
-    '1. `const matches = await tools.search({ query: "<intent + key nouns>", limit: 12 });`',
+    '1. `const { items: matches } = await tools.search({ query: "<intent + key nouns>", limit: 12 });`',
     '2. `const path = matches[0]?.path; if (!path) return "No matching tools found.";`',
     "3. `const details = await tools.describe.tool({ path });`",
     "4. Use `details.inputTypeScript` / `details.outputTypeScript` and `details.typeScriptDefinitions` for compact shapes.",
@@ -43,9 +43,10 @@ const formatDescription = (sources: readonly Source[]): string => {
     "",
     "## Rules",
     "",
-    "- `tools.search()` returns ranked matches, best-first. Use short intent phrases like `github issues`, `repo details`, or `create calendar event`.",
+    "- `tools.search()` returns paginated, ranked matches: `{ items, total, hasMore, nextOffset }`. Best-first. Use short intent phrases like `github issues`, `repo details`, or `create calendar event`.",
     '- When you already know the namespace, narrow with `tools.search({ namespace: "github", query: "issues" })`.',
-    "- Use `tools.executor.sources.list()` to inspect configured sources and their tool counts. Returns `[{ id, toolCount, ... }]`.",
+    "- `tools.executor.sources.list()` returns the same paged shape: `{ items: [{ id, toolCount, ... }], total, hasMore, nextOffset }`.",
+    "- If `hasMore` is true and you didn't find what you need, fetch the next page: `tools.search({ query, offset: nextOffset, limit })`. Same `offset` parameter on `tools.executor.sources.list({ offset, limit })`.",
     "- Always use the namespace prefix when calling tools: `tools.<namespace>.<tool>(args)`. Example: `tools.home_assistant_rest_api.states.getState(...)` — not `tools.states.getState(...)`.",
     "- The `tools` object is a lazy proxy — `Object.keys(tools)` won't work. Use `tools.search()` or `tools.executor.sources.list()` instead.",
     '- Pass an object to system tools, e.g. `tools.search({ query: "..." })`, `tools.executor.sources.list()`, and `tools.describe.tool({ path })`.',

packages/core/execution/src/engine.ts

@@ -157,6 +157,20 @@ const readOptionalLimit = (value: unknown, toolName: string): number | Execution
   return Math.floor(value);
 };
 
+const readOptionalOffset = (value: unknown, toolName: string): number | ExecutionToolError => {
+  if (value === undefined) {
+    return 0;
+  }
+
+  if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
+    return new ExecutionToolError({
+      message: `${toolName} offset must be a non-negative number when provided`,
+    });
+  }
+
+  return Math.floor(value);
+};
+
 const makeFullInvoker = (executor: Executor, invokeOptions: InvokeOptions): SandboxToolInvoker => {
   const base = makeExecutorToolInvoker(executor, { invokeOptions });
   return {
@@ -166,7 +180,7 @@ const makeFullInvoker = (executor: Executor, invokeOptions: InvokeOptions): Sand
           return Effect.fail(
             new ExecutionToolError({
               message:
-                "tools.search expects an object: { query?: string; namespace?: string; limit?: number }",
+                "tools.search expects an object: { query?: string; namespace?: string; limit?: number; offset?: number }",
             }),
           );
         }
@@ -192,8 +206,14 @@ const makeFullInvoker = (executor: Executor, invokeOptions: InvokeOptions): Sand
           return Effect.fail(limit);
         }
 
+        const offset = readOptionalOffset(args.offset, "tools.search");
+        if (offset instanceof ExecutionToolError) {
+          return Effect.fail(offset);
+        }
+
         return searchTools(executor, args.query ?? "", limit, {
           namespace: args.namespace,
+          offset,
         }).pipe(
           Effect.withSpan("mcp.tool.dispatch", {
             attributes: { "mcp.tool.name": path, "executor.tool.builtin": true },
@@ -205,7 +225,7 @@ const makeFullInvoker = (executor: Executor, invokeOptions: InvokeOptions): Sand
           return Effect.fail(
             new ExecutionToolError({
               message:
-                "tools.executor.sources.list expects an object: { query?: string; limit?: number }",
+                "tools.executor.sources.list expects an object: { query?: string; limit?: number; offset?: number }",
             }),
           );
         }
@@ -226,9 +246,18 @@ const makeFullInvoker = (executor: Executor, invokeOptions: InvokeOptions): Sand
           return Effect.fail(limit);
         }
 
+        const offset = readOptionalOffset(
+          isRecord(args) ? args.offset : undefined,
+          "tools.executor.sources.list",
+        );
+        if (offset instanceof ExecutionToolError) {
+          return Effect.fail(offset);
+        }
+
         return listExecutorSources(executor, {
           query: isRecord(args) && typeof args.query === "string" ? args.query : undefined,
           limit,
+          offset,
         }).pipe(
           Effect.withSpan("mcp.tool.dispatch", {
             attributes: { "mcp.tool.name": path, "executor.tool.builtin": true },

packages/core/execution/src/tool-invoker.test.ts

@@ -143,23 +143,71 @@ describe("tool discovery", () => {
       const executor = yield* makeSearchExecutor();
 
       const githubMatches = yield* searchTools(executor, "github issues", 5);
-      expect(githubMatches.map((match) => match.path)).toEqual(["github.listRepositoryIssues"]);
-      expect(githubMatches[0]?.score ?? 0).toBeGreaterThan(0);
+      expect(githubMatches.items.map((match) => match.path)).toEqual([
+        "github.listRepositoryIssues",
+      ]);
+      expect(githubMatches.items[0]?.score ?? 0).toBeGreaterThan(0);
+      expect(githubMatches.hasMore).toBe(false);
+      expect(githubMatches.nextOffset).toBeNull();
 
       const repoMatches = yield* searchTools(executor, "repo details", 5);
-      expect(repoMatches[0]?.path).toBe("github.getRepositoryDetails");
+      expect(repoMatches.items[0]?.path).toBe("github.getRepositoryDetails");
 
       const crmMatches = yield* searchTools(executor, "crm create contact", 5);
-      expect(crmMatches[0]?.path).toBe("crm.createContact");
-      expect(crmMatches[0]?.score ?? 0).toBeGreaterThan(crmMatches[1]?.score ?? 0);
+      expect(crmMatches.items[0]?.path).toBe("crm.createContact");
+      expect(crmMatches.items[0]?.score ?? 0).toBeGreaterThan(
+        crmMatches.items[1]?.score ?? 0,
+      );
     }),
   );
 
   it.effect("returns no matches for empty queries instead of listing arbitrary tools", () =>
     Effect.gen(function* () {
       const executor = yield* makeSearchExecutor();
       const matches = yield* searchTools(executor, "", 5);
-      expect(matches).toEqual([]);
+      expect(matches.items).toEqual([]);
+      expect(matches.total).toBe(0);
+      expect(matches.hasMore).toBe(false);
+      expect(matches.nextOffset).toBeNull();
+    }),
+  );
+
+  it.effect("paginates ranked matches via limit + offset with hasMore + nextOffset", () =>
+    Effect.gen(function* () {
+      const executor = yield* makeSearchExecutor();
+
+      // "list" matches `listRepositoryIssues`, `searchDocs` (description has
+      // "documentation" which tokenises adjacent), `listContacts`, etc.
+      // The exact match set isn't important — the pagination invariants are.
+      const all = yield* searchTools(executor, "list", 100);
+      expect(all.items.length).toBeGreaterThan(1);
+      expect(all.total).toBe(all.items.length);
+      expect(all.hasMore).toBe(false);
+      expect(all.nextOffset).toBeNull();
+
+      // First page (limit 1) — matches truncate, hasMore + nextOffset surface.
+      const firstPage = yield* searchTools(executor, "list", 1);
+      expect(firstPage.items).toEqual([all.items[0]]);
+      expect(firstPage.total).toBe(all.total);
+      expect(firstPage.hasMore).toBe(true);
+      expect(firstPage.nextOffset).toBe(1);
+
+      // Second page using nextOffset — order matches the un-paginated rank.
+      const secondPage = yield* searchTools(executor, "list", 1, {
+        offset: firstPage.nextOffset!,
+      });
+      expect(secondPage.items).toEqual([all.items[1]]);
+      expect(secondPage.total).toBe(all.total);
+      // Whether hasMore is true depends on total; at minimum it's consistent.
+      expect(secondPage.hasMore).toBe(all.total > 2);
+      expect(secondPage.nextOffset).toBe(secondPage.hasMore ? 2 : null);
+
+      // Offset past the end — empty page, no more.
+      const past = yield* searchTools(executor, "list", 5, { offset: all.total + 10 });
+      expect(past.items).toEqual([]);
+      expect(past.total).toBe(all.total);
+      expect(past.hasMore).toBe(false);
+      expect(past.nextOffset).toBeNull();
     }),
   );
 
@@ -170,21 +218,28 @@ describe("tool discovery", () => {
       const githubOnly = yield* searchTools(executor, "list", 5, {
         namespace: "github",
       });
-      expect(githubOnly.map((match) => match.path)).toEqual(["github.listRepositoryIssues"]);
+      expect(githubOnly.items.map((match) => match.path)).toEqual([
+        "github.listRepositoryIssues",
+      ]);
 
       const crmOnly = yield* searchTools(executor, "list", 5, {
         namespace: "crm",
       });
-      expect(crmOnly.map((match) => match.path)).toEqual(["crm.listContacts"]);
+      expect(crmOnly.items.map((match) => match.path)).toEqual(["crm.listContacts"]);
 
       const sandboxResult = yield* createExecutionEngine({ executor, codeExecutor }).execute(
         'return await tools.search({ namespace: "crm", query: "create contact", limit: 5 });',
         { onElicitation: acceptAll },
       );
       expect(sandboxResult.error).toBeUndefined();
-      expect(sandboxResult.result).toEqual([
-        expect.objectContaining({ path: "crm.createContact" }),
-      ]);
+      expect(sandboxResult.result).toEqual(
+        expect.objectContaining({
+          items: [expect.objectContaining({ path: "crm.createContact" })],
+          total: 1,
+          hasMore: false,
+          nextOffset: null,
+        }),
+      );
     }),
   );
 
@@ -198,18 +253,102 @@ describe("tool discovery", () => {
       );
       expect(listed.error).toBeUndefined();
       expect(listed.result).toEqual(
-        expect.arrayContaining([
-          expect.objectContaining({ id: "github", toolCount: 3 }),
-          expect.objectContaining({ id: "crm", toolCount: 2 }),
-        ]),
+        expect.objectContaining({
+          items: expect.arrayContaining([
+            expect.objectContaining({ id: "github", toolCount: 3 }),
+            expect.objectContaining({ id: "crm", toolCount: 2 }),
+          ]),
+          total: 2,
+          hasMore: false,
+          nextOffset: null,
+        }),
       );
 
       const searched = yield* createExecutionEngine({ executor, codeExecutor }).execute(
         'return await tools.search({ query: "list contacts", namespace: "crm", limit: 5 });',
         { onElicitation: acceptAll },
       );
       expect(searched.error).toBeUndefined();
-      expect(searched.result).toEqual([expect.objectContaining({ path: "crm.listContacts" })]);
+      expect(searched.result).toEqual(
+        expect.objectContaining({
+          items: [expect.objectContaining({ path: "crm.listContacts" })],
+        }),
+      );
+    }),
+  );
+
+  it.effect("paginates source listings via limit + offset", () =>
+    Effect.gen(function* () {
+      const executor = yield* makeSearchExecutor();
+      const engine = createExecutionEngine({ executor, codeExecutor });
+
+      // total = 2 (github, crm), sorted by name ("CRM" < "GitHub")
+      const firstPage = yield* engine.execute(
+        "return await tools.executor.sources.list({ limit: 1 });",
+        { onElicitation: acceptAll },
+      );
+      expect(firstPage.error).toBeUndefined();
+      expect(firstPage.result).toEqual(
+        expect.objectContaining({
+          items: [expect.objectContaining({ id: "crm" })],
+          total: 2,
+          hasMore: true,
+          nextOffset: 1,
+        }),
+      );
+
+      const secondPage = yield* engine.execute(
+        "return await tools.executor.sources.list({ limit: 1, offset: 1 });",
+        { onElicitation: acceptAll },
+      );
+      expect(secondPage.error).toBeUndefined();
+      expect(secondPage.result).toEqual(
+        expect.objectContaining({
+          items: [expect.objectContaining({ id: "github" })],
+          total: 2,
+          hasMore: false,
+          nextOffset: null,
+        }),
+      );
+    }),
+  );
+
+  it.effect("rejects negative offsets via the engine validator", () =>
+    Effect.gen(function* () {
+      const executor = yield* makeSearchExecutor();
+      const engine = createExecutionEngine({ executor, codeExecutor });
+
+      const badSearch = yield* engine.execute(
+        [
+          "try {",
+          '  await tools.search({ query: "list", offset: -1 });',
+          '  return "unexpected";',
+          "} catch (error) {",
+          "  return error instanceof Error ? error.message : String(error);",
+          "}",
+        ].join("\n"),
+        { onElicitation: acceptAll },
+      );
+      expect(badSearch.error).toBeUndefined();
+      expect(String(badSearch.result)).toContain(
+        "tools.search offset must be a non-negative number when provided",
+      );
+
+      const badList = yield* engine.execute(
+        [
+          "try {",
+          "  await tools.executor.sources.list({ offset: -5 });",
+          '  return "unexpected";',
+          "} catch (error) {",
+          "  return error instanceof Error ? error.message : String(error);",
+          "}",
+        ].join("\n"),
+        { onElicitation: acceptAll },
+      );
+      expect(badList.error).toBeUndefined();
+      expect(String(badList.result)).toContain(
+        "tools.executor.sources.list offset must be a non-negative number when provided",
+      );
     }),
   );
 
@@ -245,15 +384,20 @@ describe("tool discovery", () => {
       );
       expect(invalid.error).toBeUndefined();
       expect(String(invalid.result)).toContain(
-        "tools.search expects an object: { query?: string; namespace?: string; limit?: number }",
+        "tools.search expects an object: { query?: string; namespace?: string; limit?: number; offset?: number }",
       );
 
       const emptyQuery = yield* engine.execute(
         'return await tools.search({ query: "", limit: 5 });',
         { onElicitation: acceptAll },
       );
       expect(emptyQuery.error).toBeUndefined();
-      expect(emptyQuery.result).toEqual([]);
+      expect(emptyQuery.result).toEqual({
+        items: [],
+        total: 0,
+        hasMore: false,
+        nextOffset: null,
+      });
 
       const invalidDescribe = yield* engine.execute(
         [