diff --git a/packages/dsh-web-search/README.md b/packages/dsh-web-search/README.md index 63aa818..2317af5 100644 --- a/packages/dsh-web-search/README.md +++ b/packages/dsh-web-search/README.md @@ -11,7 +11,8 @@ You will need: - Node.js 22.19 or later in the 22.x series, or Node.js 24 or newer; - pnpm 10 or newer; - a [DeepSeek API key](https://platform.deepseek.com/); and -- a [Parallel API key](https://platform.parallel.ai/). +- optionally, a [Parallel API key](https://platform.parallel.ai/) for + authenticated Search API access. Check your installed versions: @@ -55,8 +56,10 @@ is stable, you can leave the suffix off. ### 3. Start DeepSeek Harness -Make your Parallel API key available in the terminal where you will run -Harness: +The plugin works without a Parallel API key through the free Search MCP +endpoint at `https://search.parallel.ai/mcp`. To use authenticated Search API +access instead, make your Parallel API key available in the terminal where you +will run Harness: ```sh export PARALLEL_API_KEY="your-key" @@ -108,13 +111,15 @@ logging: PARALLEL_LOG=info npx --yes @deepseek-ai/dsh@0.1.0-rc.6 web ``` -After a `web_search` call, the terminal should show a successful request to -`https://api.parallel.ai/v1/search`. Review logs before sharing them. +With a Parallel API key, a `web_search` call should show a successful request to +`https://api.parallel.ai/v1/search`. Without a key, search requests use +`https://search.parallel.ai/mcp` instead. Review logs before sharing them. ## Optional settings -The defaults work without extra configuration. If you want to tune the search, -edit `~/.dsh/profiles/web/cordis.patch.yml`: +The defaults work without extra configuration. To tune excerpt limits, edit +`~/.dsh/profiles/web/cordis.patch.yml`. The `mode` setting applies only to +authenticated Search API access: ```yaml - id: web-search-parallel @@ -130,10 +135,14 @@ edit `~/.dsh/profiles/web/cordis.patch.yml`: | `maxCharsTotal` | `25000` | Total excerpt characters returned to Harness | | `maxCharsPerResult` | no limit | Excerpt characters kept for each result | +Anonymous search applies excerpt limits locally, including the separators +between excerpts. Authenticated search sends these limits to the Search API. + Keep `PARALLEL_API_KEY` in the environment rather than putting it in this file, which is stored as readable text. -The plugin always sends requests to `https://api.parallel.ai` and ignores +Authenticated requests always use `https://api.parallel.ai`, and anonymous +requests always use `https://search.parallel.ai/mcp`. The plugin ignores `PARALLEL_BASE_URL`. ## If something goes wrong @@ -141,8 +150,9 @@ The plugin always sends requests to `https://api.parallel.ai` and ignores - **`pnpm` is not found:** run `npm install --global pnpm@10`. - **Port 3080 is already in use:** stop the older Harness process, then start Harness again. -- **Parallel Search is unavailable:** confirm `PARALLEL_API_KEY` is set in the - same terminal that starts Harness. +- **Authenticated Parallel Search is unavailable:** confirm `PARALLEL_API_KEY` + is set in the same terminal that starts Harness. Remove the variable to use + free anonymous search instead. ## Remove diff --git a/packages/dsh-web-search/src/provider.ts b/packages/dsh-web-search/src/provider.ts index f3b86d1..fbea374 100644 --- a/packages/dsh-web-search/src/provider.ts +++ b/packages/dsh-web-search/src/provider.ts @@ -1,3 +1,4 @@ +import { randomUUID } from 'node:crypto'; import Parallel, { APIUserAbortError, type Parallel as ParallelTypes, @@ -12,6 +13,7 @@ import type { export const PARALLEL_PROVIDER_ID = 'parallel'; export const PARALLEL_API_ORIGIN = 'https://api.parallel.ai'; +const PARALLEL_SEARCH_MCP_URL = 'https://search.parallel.ai/mcp'; export const DEFAULT_MAX_CHARS_TOTAL = 25_000; export const PARALLEL_SEARCH_MODES = ['turbo', 'basic', 'advanced'] as const; @@ -46,6 +48,7 @@ const createProductionClient: SearchClientFactory = (apiKey) => export class ParallelSearchProvider implements WebSearchProvider { readonly id = PARALLEL_PROVIDER_ID; private client: SearchClient | undefined; + private readonly sessionId = randomUUID(); constructor( private readonly options: ParallelSearchProviderOptions, @@ -54,7 +57,6 @@ export class ParallelSearchProvider implements WebSearchProvider { available(): boolean { return ( - this.options.apiKey.length > 0 && isMode(this.options.mode) && isPositiveInteger(this.options.maxCharsTotal) && (this.options.maxCharsPerResult === undefined || @@ -70,15 +72,21 @@ export class ParallelSearchProvider implements WebSearchProvider { let payload: unknown; try { - payload = await this.getClient().search( - buildSearchBody(request, this.options), - { - signal, - maxRetries: 0, - timeout: 60_000, - fetchOptions: { redirect: 'error' }, - } - ); + payload = + this.options.apiKey.length === 0 + ? await this.searchFreeMcp(request, signal) + : await this.getClient().search( + { + ...buildSearchBody(request, this.options), + session_id: this.sessionId, + }, + { + signal, + maxRetries: 0, + timeout: 60_000, + fetchOptions: { redirect: 'error' }, + } + ); } catch (error: unknown) { if (signal?.aborted || error instanceof APIUserAbortError) throw abortedError(error, this.options.apiKey); @@ -90,7 +98,10 @@ export class ParallelSearchProvider implements WebSearchProvider { } try { - return mapParallelResponse(payload); + return mapParallelResponse( + payload, + this.options.apiKey.length === 0 ? this.options : undefined + ); } catch (error: unknown) { throw providerError( 'Parallel returned an invalid search response', @@ -103,6 +114,55 @@ export class ParallelSearchProvider implements WebSearchProvider { private getClient(): SearchClient { return (this.client ??= this.createClient(this.options.apiKey)); } + + private async searchFreeMcp( + request: WebSearchRequest, + signal?: AbortSignal + ): Promise { + const timeout = AbortSignal.timeout(60_000); + const response = await fetch(PARALLEL_SEARCH_MCP_URL, { + method: 'POST', + headers: { + Accept: 'application/json, text/event-stream', + 'Content-Type': 'application/json', + }, + redirect: 'error', + signal: + signal === undefined ? timeout : AbortSignal.any([signal, timeout]), + body: JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'web_search', + arguments: { + objective: request.query, + search_queries: [request.query], + session_id: this.sessionId, + }, + }, + }), + }); + + if (!response.ok) + throw new Error(`Parallel Search MCP HTTP ${response.status}`); + + const payload: unknown = await response.json(); + if (!isRecord(payload)) + throw new TypeError('MCP response must be an object'); + if (isRecord(payload.error)) { + throw new Error( + typeof payload.error.message === 'string' + ? payload.error.message + : 'Parallel Search MCP returned an error' + ); + } + if (!isRecord(payload.result) || payload.result.isError === true) { + throw new Error('Parallel Search MCP tool call failed'); + } + + return payload.result.structuredContent; + } } export function buildSearchBody( @@ -139,13 +199,31 @@ export function buildSearchBody( }; } -export function mapParallelResponse(payload: unknown): WebSearchResult { +export function mapParallelResponse( + payload: unknown, + excerptLimits?: Pick< + ParallelSearchProviderOptions, + 'maxCharsTotal' | 'maxCharsPerResult' + > +): WebSearchResult { if (!isRecord(payload) || !Array.isArray(payload.results)) { throw new TypeError('response.results must be an array'); } + // MCP has no excerpt controls. Bound the normalized snippets locally, + // including separators, while leaving source-count truncation to Harness. + let remaining = excerptLimits?.maxCharsTotal ?? Infinity; return { - sources: payload.results.map(mapParallelResult), + sources: payload.results.map((value) => { + const { snippet, ...source } = mapParallelResult(value); + if (snippet === undefined) return source; + const bounded = snippet.slice( + 0, + Math.min(remaining, excerptLimits?.maxCharsPerResult ?? Infinity) + ); + remaining -= bounded.length; + return bounded.length === 0 ? source : { ...source, snippet: bounded }; + }), truncated: false, }; } diff --git a/packages/dsh-web-search/tests/plugin.spec.ts b/packages/dsh-web-search/tests/plugin.spec.ts index 5148e32..e8230e4 100644 --- a/packages/dsh-web-search/tests/plugin.spec.ts +++ b/packages/dsh-web-search/tests/plugin.spec.ts @@ -71,6 +71,48 @@ describe('Parallel plugin config', () => { }); describe('Parallel plugin registration', () => { + it.each(['', 'parallel_test_plugin'])( + 'reuses a session per provider with API key %j', + async (apiKey) => { + const sessionIds: string[] = []; + vi.spyOn(globalThis, 'fetch').mockImplementation(async (_url, init) => { + const body = JSON.parse(init?.body as string); + const sessionId = + apiKey === '' ? body.params.arguments.session_id : body.session_id; + sessionIds.push(sessionId); + const result = { results: [], session_id: sessionId }; + return new Response( + JSON.stringify( + apiKey === '' + ? { jsonrpc: '2.0', id: 1, result: { structuredContent: result } } + : result + ), + { headers: { 'content-type': 'application/json' } } + ); + }); + const ctx = new Context(); + await ctx.plugin(WebRuntime, { searchProvider: 'parallel' }); + const fiber = await ctx.plugin(parallelPlugin, { apiKey }); + try { + await ctx.web.search({ query: 'first query' }); + await ctx.web.search({ query: 'second query' }); + expect(sessionIds[0]).toMatch(/^[0-9a-f-]{36}$/); + expect(sessionIds[1]).toBe(sessionIds[0]); + } finally { + await fiber.dispose(); + } + + const next = await ctx.plugin(parallelPlugin, { apiKey }); + try { + await ctx.web.search({ query: 'new provider' }); + expect(sessionIds[2]).toMatch(/^[0-9a-f-]{36}$/); + expect(sessionIds[2]).not.toBe(sessionIds[0]); + } finally { + await next.dispose(); + } + } + ); + it('registers, selects, and disposes through the real WebRuntime', async () => { mockSearch(); const ctx = new Context(); @@ -106,8 +148,17 @@ describe('Parallel plugin registration', () => { expect(search).toHaveBeenCalledOnce(); }); - it('lets an explicit empty key suppress environment fallback', async () => { + it('lets an explicit empty key choose free search over an environment key', async () => { const search = mockSearch(); + const fetch = vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response( + JSON.stringify({ + jsonrpc: '2.0', + id: 1, + result: { structuredContent: { results: [] } }, + }) + ) + ); const ctx = new Context(); ctx.provide( 'launchEnvironment', @@ -120,21 +171,48 @@ describe('Parallel plugin registration', () => { ); await ctx.plugin(WebRuntime, { searchProvider: 'parallel' }); await ctx.plugin(parallelPlugin, { apiKey: '' }); - await expect(ctx.web.search({ query: 'q' })).rejects.toMatchObject({ - code: 'WEB_PROVIDER_CONFIGURED_UNAVAILABLE', + await expect(ctx.web.search({ query: 'q' })).resolves.toEqual({ + sources: [], + truncated: false, }); + expect(fetch).toHaveBeenCalledOnce(); expect(search).not.toHaveBeenCalled(); }); - it('is unavailable without a key and makes no network call', async () => { + it('uses free MCP search when no API key is configured', async () => { const search = mockSearch(); + const fetch = vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response( + JSON.stringify({ + jsonrpc: '2.0', + id: 1, + result: { + structuredContent: { + results: [ + { + url: 'https://example.test', + excerpts: ['Free search works'], + }, + ], + }, + }, + }) + ) + ); const ctx = new Context(); ctx.provide('launchEnvironment', createLaunchEnvironmentSnapshot([])); await ctx.plugin(WebRuntime, { searchProvider: 'parallel' }); await ctx.plugin(parallelPlugin, {}); - await expect(ctx.web.search({ query: 'q' })).rejects.toMatchObject({ - code: 'WEB_PROVIDER_CONFIGURED_UNAVAILABLE', + await expect(ctx.web.search({ query: 'q' })).resolves.toEqual({ + sources: [ + { + url: 'https://example.test', + snippet: 'Free search works', + }, + ], + truncated: false, }); + expect(fetch).toHaveBeenCalledOnce(); expect(search).not.toHaveBeenCalled(); }); diff --git a/packages/dsh-web-search/tests/provider.spec.ts b/packages/dsh-web-search/tests/provider.spec.ts index c2be9b4..9b9b613 100644 --- a/packages/dsh-web-search/tests/provider.spec.ts +++ b/packages/dsh-web-search/tests/provider.spec.ts @@ -1,6 +1,6 @@ import { APIConnectionTimeoutError, APIUserAbortError } from 'parallel-web'; import { errorChain } from '@deepseek-ai/dsh-llm'; -import { describe, expect, it, vi } from 'vitest'; +import { afterEach, describe, expect, it, vi } from 'vitest'; import { DEFAULT_MAX_CHARS_TOTAL, ParallelSearchProvider, @@ -15,6 +15,10 @@ const baseOptions: ParallelSearchProviderOptions = { maxCharsTotal: DEFAULT_MAX_CHARS_TOTAL, }; +afterEach(() => { + vi.restoreAllMocks(); +}); + function providerWithSearch( search: SearchClient['search'], options: Partial = {} @@ -137,12 +141,186 @@ describe('Parallel response mapping', () => { }); }); +describe('free Parallel MCP search', () => { + it.each([ + { maxCharsTotal: 4, snippets: ['abcd', undefined] }, + { maxCharsTotal: 9, maxCharsPerResult: 6, snippets: ['abcd\n\n', 'ijk'] }, + { maxCharsTotal: 12, snippets: ['abcd\n\nefgh', 'ij'] }, + { maxCharsTotal: 30, snippets: ['abcd\n\nefgh', 'ijklmnop'] }, + ])( + 'bounds normalized excerpts with $maxCharsTotal total characters', + async ({ snippets, ...limits }) => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response( + JSON.stringify({ + jsonrpc: '2.0', + id: 1, + result: { + structuredContent: { + results: [ + { url: 'https://empty.test', excerpts: [] }, + { + url: 'https://a.test', + title: 'A', + excerpts: ['abcd', 'efgh'], + }, + { url: 'https://b.test', excerpts: ['ijklmnop'] }, + ], + }, + }, + }) + ) + ); + const provider = new ParallelSearchProvider({ apiKey: '', ...limits }); + + await expect(provider.search({ query: 'q' })).resolves.toEqual({ + sources: [ + { url: 'https://empty.test' }, + { url: 'https://a.test', title: 'A', snippet: snippets[0] }, + { + url: 'https://b.test', + ...(snippets[1] === undefined ? {} : { snippet: snippets[1] }), + }, + ], + truncated: false, + }); + } + ); + + it('searches without credentials and reuses one session across free searches', async () => { + const fetch = vi.spyOn(globalThis, 'fetch').mockImplementation( + async () => + new Response( + JSON.stringify({ + jsonrpc: '2.0', + id: 1, + result: { + structuredContent: { + results: [ + { + url: 'https://example.test', + title: 'Example', + excerpts: ['Useful excerpt'], + }, + ], + }, + }, + }), + { headers: { 'content-type': 'application/json' } } + ) + ); + const clientFactory = vi.fn(); + const provider = new ParallelSearchProvider( + { ...baseOptions, apiKey: '' }, + clientFactory + ); + + await expect(provider.search({ query: 'first query' })).resolves.toEqual({ + sources: [ + { + url: 'https://example.test', + title: 'Example', + snippet: 'Useful excerpt', + }, + ], + truncated: false, + }); + await provider.search({ query: 'second query' }); + + expect(fetch).toHaveBeenCalledTimes(2); + const [url, options] = fetch.mock.calls[0]!; + expect(url).toBe('https://search.parallel.ai/mcp'); + expect(options).toMatchObject({ + method: 'POST', + redirect: 'error', + headers: { + Accept: 'application/json, text/event-stream', + 'Content-Type': 'application/json', + }, + }); + expect(options?.headers).not.toHaveProperty('Authorization'); + expect(options?.headers).not.toHaveProperty('x-api-key'); + + const firstBody = JSON.parse(options?.body as string); + const secondBody = JSON.parse(fetch.mock.calls[1]?.[1]?.body as string); + expect(firstBody).toMatchObject({ + jsonrpc: '2.0', + id: 1, + method: 'tools/call', + params: { + name: 'web_search', + arguments: { + objective: 'first query', + search_queries: ['first query'], + session_id: expect.any(String), + }, + }, + }); + expect(secondBody.params.arguments.session_id).toBe( + firstBody.params.arguments.session_id + ); + expect(clientFactory).not.toHaveBeenCalled(); + }); + + it('maps free MCP HTTP failures to WEB_PROVIDER_ERROR', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response('rate limited', { status: 429 }) + ); + const provider = new ParallelSearchProvider({ + ...baseOptions, + apiKey: '', + }); + + await expect(provider.search({ query: 'q' })).rejects.toMatchObject({ + code: 'WEB_PROVIDER_ERROR', + cause: { message: expect.stringContaining('429') }, + }); + }); + + it('maps free MCP JSON-RPC failures to WEB_PROVIDER_ERROR', async () => { + vi.spyOn(globalThis, 'fetch').mockResolvedValue( + new Response( + JSON.stringify({ + jsonrpc: '2.0', + id: 1, + error: { code: -32603, message: 'Search temporarily unavailable' }, + }) + ) + ); + const provider = new ParallelSearchProvider({ + ...baseOptions, + apiKey: '', + }); + + await expect(provider.search({ query: 'q' })).rejects.toMatchObject({ + code: 'WEB_PROVIDER_ERROR', + cause: { message: 'Search temporarily unavailable' }, + }); + }); + + it('maps free MCP caller aborts to WEB_ABORTED', async () => { + const controller = new AbortController(); + vi.spyOn(globalThis, 'fetch').mockImplementation(async () => { + controller.abort(); + throw new Error('transport stopped'); + }); + const provider = new ParallelSearchProvider({ + ...baseOptions, + apiKey: '', + }); + + await expect( + provider.search({ query: 'q' }, controller.signal) + ).rejects.toMatchObject({ code: 'WEB_ABORTED' }); + }); +}); + describe('Parallel provider availability and errors', () => { - it('is available only for a key and valid locked options', () => { + it('is available with or without a key when locked options are valid', () => { expect(new ParallelSearchProvider(baseOptions).available()).toBe(true); expect( new ParallelSearchProvider({ ...baseOptions, apiKey: '' }).available() - ).toBe(false); + ).toBe(true); expect( new ParallelSearchProvider({ ...baseOptions,