developer: use server passage budget from search#843

README.md

@@ -368,22 +368,25 @@ firecrawl developer "axum middleware ordering"
 
 #### Options
 
-| Option                | Description                               |
-| --------------------- | ----------------------------------------- |
-| `--limit <n>`         | Number of results (default: 10, max: 100) |
-| `--skills-only`       | Search only agent-skill files             |
-| `-o, --output <path>` | Save to file                              |
-| `--json`              | Output as compact JSON                    |
-| `--pretty`            | Pretty print JSON output                  |
+| Option                      | Description                                               |
+| --------------------------- | --------------------------------------------------------- |
+| `--limit <n>`               | Number of results (default: 10, max: 100)                 |
+| `--skills-only`             | Search only agent-skill files                             |
+| `--passage-budget <tokens>` | Approximate-token budget for all passages (default: 4096) |
+| `-o, --output <path>`       | Save to file                                              |
+| `--json`                    | Output as compact JSON                                    |
+| `--pretty`                  | Pretty print JSON output                                  |
+
+The passage budget accepts 256–16384 tokens and is allocated by the search server across all results. The 4096-token default preserves the intent of the previous readable-output cap: 1200 characters were roughly 300 tokens per result, or about 3000 passage tokens across the default 10 results, with additional allocation headroom.
 
 #### Examples
 
 ```bash
 # Investigate a known bug
 firecrawl developer "tokio spawn_blocking panics thread limit" --limit 10
 
-# Keep the full passages for an agent
-firecrawl developer "tokio select cancellation safety" --json -o results.json
+# Give the server more passage space for an agent
+firecrawl developer "tokio select cancellation safety" --passage-budget 8192 --json -o results.json
 ```
 
 ---

src/__tests__/cli-argv.test.ts

@@ -31,9 +31,28 @@ describe('CLI argv parsing', () => {
     expect(result.stdout).toContain('Usage: firecrawl developer');
     expect(result.stdout).toContain('--limit');
     expect(result.stdout).toContain('--skills-only');
+    expect(result.stdout).toContain('--passage-budget');
     expect(result.stderr).not.toContain('unknown command');
   });
 
+  testWithBuiltCli('rejects an invalid developer passage budget', () => {
+    for (const budget of ['not-a-number', '255', '16385']) {
+      const result = spawnSync(
+        process.execPath,
+        [cliPath, 'developer', 'query', '--passage-budget', budget],
+        {
+          cwd: process.cwd(),
+          encoding: 'utf8',
+        }
+      );
+
+      expect(result.status).not.toBe(0);
+      expect(result.stderr).toContain(
+        'must be an integer between 256 and 16384'
+      );
+    }
+  });
+
   testWithBuiltCli('lists the research command in root help output', () => {
     const result = spawnSync(process.execPath, [cliPath, '--help'], {
       cwd: process.cwd(),

src/__tests__/commands/developer.test.ts

@@ -25,8 +25,17 @@ describe('handleDeveloperSearchCommand', () => {
   // Wrap a payload in the axios envelope returned by `client.http.get`.
   // Mirrors the `/v2/search/developer` response shape:
   //   { success, results: [{ id, type, url, title, passages: [{ text }] }] }
-  const mockDeveloperResponse = (results: any[]) => ({
-    data: { success: true, results },
+  const mockDeveloperResponse = (
+    results: any[],
+    passageBudgetApplied?: number
+  ) => ({
+    data: {
+      success: true,
+      results,
+      ...(passageBudgetApplied == null
+        ? {}
+        : { passage_budget_applied: passageBudgetApplied }),
+    },
   });
 
   const sampleResult = {
@@ -63,7 +72,7 @@ describe('handleDeveloperSearchCommand', () => {
 
       expect(mockHttpGet).toHaveBeenCalledTimes(1);
       expect(mockHttpGet).toHaveBeenCalledWith(
-        '/v2/search/developer?query=tokio+spawn_blocking&integration=cli'
+        '/v2/search/developer?query=tokio+spawn_blocking&passage_budget=4096&integration=cli'
       );
     });
 
@@ -76,7 +85,7 @@ describe('handleDeveloperSearchCommand', () => {
       });
 
       expect(mockHttpGet).toHaveBeenCalledWith(
-        '/v2/search/developer?query=tokio+spawn_blocking&skills=only&integration=cli'
+        '/v2/search/developer?query=tokio+spawn_blocking&skills=only&passage_budget=4096&integration=cli'
       );
     });
 
@@ -89,7 +98,20 @@ describe('handleDeveloperSearchCommand', () => {
       });
 
       expect(mockHttpGet).toHaveBeenCalledWith(
-        '/v2/search/developer?query=tokio+spawn_blocking&k=5&integration=cli'
+        '/v2/search/developer?query=tokio+spawn_blocking&k=5&passage_budget=4096&integration=cli'
+      );
+    });
+
+    it('passes a custom passage budget through verbatim', async () => {
+      mockHttpGet.mockResolvedValue(mockDeveloperResponse([sampleResult], 768));
+
+      await handleDeveloperSearchCommand({
+        query: 'tokio spawn_blocking',
+        passageBudget: 768,
+      });
+
+      expect(mockHttpGet).toHaveBeenCalledWith(
+        '/v2/search/developer?query=tokio+spawn_blocking&passage_budget=768&integration=cli'
       );
     });
 
@@ -125,7 +147,7 @@ describe('handleDeveloperSearchCommand', () => {
       expect(content).toContain('It will panic if this limit is too low.');
     });
 
-    it('joins multiple passages and clips long content', async () => {
+    it('keeps the legacy local cut when the server omits budget metadata', async () => {
       mockHttpGet.mockResolvedValue(
         mockDeveloperResponse([
           {
@@ -143,6 +165,21 @@ describe('handleDeveloperSearchCommand', () => {
       expect(body.length).toBeLessThanOrEqual(1200);
     });
 
+    it('does not cut content after the server applies the passage budget', async () => {
+      const passage = 'x'.repeat(5000);
+      mockHttpGet.mockResolvedValue(
+        mockDeveloperResponse(
+          [{ ...sampleResult, passages: [{ text: passage }] }],
+          4096
+        )
+      );
+
+      await handleDeveloperSearchCommand({ query: 'tokio spawn_blocking' });
+
+      const [content] = vi.mocked(writeOutput).mock.calls[0] as [string];
+      expect(content).toContain(passage);
+    });
+
     it('prints a placeholder when there are no results', async () => {
       mockHttpGet.mockResolvedValue(mockDeveloperResponse([]));
 

src/commands/developer.ts

@@ -5,7 +5,8 @@ import type { DeveloperItem, DeveloperSearchOptions } from '../types/developer';
 // The other mount, /v2/developer/search, rejects keyless callers and may be
 // withdrawn.
 const BASE = '/v2/search/developer';
-const MAX_PASSAGE_CHARS = 1200;
+const DEFAULT_PASSAGE_BUDGET = 4096;
+const LEGACY_MAX_PASSAGE_CHARS = 1200;
 
 async function getDeveloper<T>(
   path: string,
@@ -22,7 +23,10 @@ async function getDeveloper<T>(
   return (response?.data ?? {}) as T;
 }
 
-function fmtDeveloper(results?: DeveloperItem[]): string {
+function fmtDeveloper(
+  results?: DeveloperItem[],
+  passageBudgetApplied?: number
+): string {
   if (!results || results.length === 0) return '(no results)';
 
   return results
@@ -36,7 +40,13 @@ function fmtDeveloper(results?: DeveloperItem[]): string {
         .map((passage) => passage.text ?? '')
         .join('\n---\n')
         .trim();
-      lines.push(body ? body.slice(0, MAX_PASSAGE_CHARS) : '(no content)');
+      // TODO(search#843): Remove this fallback after server passage budgeting
+      // is fully enabled.
+      const renderedBody =
+        passageBudgetApplied == null
+          ? body.slice(0, LEGACY_MAX_PASSAGE_CHARS)
+          : body;
+      lines.push(renderedBody || '(no content)');
       return lines.join('\n');
     })
     .join('\n\n');
@@ -72,11 +82,19 @@ export async function handleDeveloperSearchCommand(
     params.append('query', options.query);
     if (options.k != null) params.append('k', String(options.k));
     if (options.skillsOnly) params.append('skills', 'only');
-    const data = await getDeveloper<{ results?: DeveloperItem[] }>(
-      `${BASE}?${params.toString()}`,
+    params.append(
+      'passage_budget',
+      String(options.passageBudget ?? DEFAULT_PASSAGE_BUDGET)
+    );
+    const data = await getDeveloper<{
+      results?: DeveloperItem[];
+      passage_budget_applied?: number;
+    }>(`${BASE}?${params.toString()}`, options);
+    writeDeveloperOutput(
+      data,
+      fmtDeveloper(data.results, data.passage_budget_applied),
       options
     );
-    writeDeveloperOutput(data, fmtDeveloper(data.results), options);
   } catch (error) {
     handleError(error);
   }

src/index.ts

@@ -5,7 +5,7 @@
  * Entry point for the CLI application
  */
 
-import { Command, Option } from 'commander';
+import { Command, InvalidArgumentError, Option } from 'commander';
 import { readFileSync } from 'fs';
 import {
   handleScrapeCommand,
@@ -226,6 +226,14 @@ function researchLimit(options: {
   return options.k ?? options.limit;
 }
 
+function parsePassageBudget(value: string): number {
+  const budget = Number(value);
+  if (!Number.isInteger(budget) || budget < 256 || budget > 16384) {
+    throw new InvalidArgumentError('must be an integer between 256 and 16384');
+  }
+  return budget;
+}
+
 function parseAgentWebhookOption(
   raw: string | undefined,
   label: string
@@ -1064,6 +1072,12 @@ function createDeveloperCommand(): Command {
     )
     .addOption(new Option('--k <number>').argParser(parseInt).hideHelp())
     .option('--skills-only', 'Search only agent-skill files', false)
+    .option(
+      '--passage-budget <tokens>',
+      'Approximate-token budget for all passage text (default: 4096, range: 256-16384)',
+      parsePassageBudget,
+      4096
+    )
     .option(
       '-k, --api-key <key>',
       'Firecrawl API key (overrides global --api-key)'
@@ -1085,6 +1099,7 @@ Examples:
         query,
         k: researchLimit(options),
         skillsOnly: options.skillsOnly,
+        passageBudget: options.passageBudget,
         apiKey: options.apiKey,
         apiUrl: options.apiUrl,
         output: options.output,

src/types/developer.ts

@@ -2,6 +2,7 @@ export interface DeveloperSearchOptions {
   query: string;
   k?: number;
   skillsOnly?: boolean;
+  passageBudget?: number;
   apiKey?: string;
   apiUrl?: string;
   output?: string;