diff --git a/src/cache/store.ts b/src/cache/store.ts index de3706ff6..77b282cc0 100644 --- a/src/cache/store.ts +++ b/src/cache/store.ts @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'; import { getDatabase } from './db.js'; import { getConfig } from '../config.js'; import { createLogger } from '../logger.js'; +import { normaliseEngineList } from '../util/engine-list.js'; import type { RawFetchResult, ExtractionResult, CachedContent, SearchResultItem, CacheStats, ContentCompleteness } from '../types.js'; const log = createLogger('cache'); @@ -353,6 +354,7 @@ export interface SearchCacheFilters { exact_match?: boolean | null; search_depth?: string | null; reranker?: string | null; + search_engines?: string[] | null; } function normaliseDomainList(list?: string[] | null): string[] | null { @@ -362,6 +364,10 @@ function normaliseDomainList(list?: string[] | null): string[] | null { return [...new Set(lower)].sort(); } +// Shared with the orchestrator's allowlist gates (src/util/engine-list.ts) +// so the cache-key fingerprint and dispatch matching can never drift apart. +export { normaliseEngineList } from '../util/engine-list.js'; + function hasAnyFilter(filters?: SearchCacheFilters): boolean { if (!filters) return false; return ( @@ -375,7 +381,8 @@ function hasAnyFilter(filters?: SearchCacheFilters): boolean { filters.time_range != null || filters.exact_match != null || filters.search_depth != null || - filters.reranker != null + filters.reranker != null || + normaliseEngineList(filters.search_engines) != null ); } @@ -400,6 +407,7 @@ export function buildSearchCacheKey( exact_match: filters!.exact_match ?? null, search_depth: filters!.search_depth ?? null, reranker: filters!.reranker ?? null, + search_engines: normaliseEngineList(filters!.search_engines), }; return `${query}${JSON.stringify(fingerprint)}`; } diff --git a/src/search/core/core-provider.ts b/src/search/core/core-provider.ts index a966047e2..d8355ea61 100644 --- a/src/search/core/core-provider.ts +++ b/src/search/core/core-provider.ts @@ -203,6 +203,7 @@ export class CoreSearchProvider implements SearchProvider { exact_match: input.exact_match, search_depth: depth, reranker: getConfig().reranker, + search_engines: input.search_engines, }); let items: SearchResultItem[] = []; @@ -395,6 +396,7 @@ export class CoreSearchProvider implements SearchProvider { country: input.country, timeRange: input.time_range, exactMatch: input.exact_match, + engineFilter: input.search_engines, }), ), ); @@ -452,6 +454,7 @@ export class CoreSearchProvider implements SearchProvider { country: input.country, timeRange: input.time_range, exactMatch: input.exact_match, + engineFilter: input.search_engines, }); // RRF-merge the retry results on top of the initial dispatch so // we keep ranking signal from both passes. diff --git a/src/search/core/orchestrator.ts b/src/search/core/orchestrator.ts index b31feb15b..c52f4dcfb 100644 --- a/src/search/core/orchestrator.ts +++ b/src/search/core/orchestrator.ts @@ -6,6 +6,7 @@ import type { SearchEngineOptions, } from '../../types.js'; import { createLogger } from '../../logger.js'; +import { normaliseEngineList } from '../../util/engine-list.js'; import { classifyIntentDetailed, extractErrorTokens, @@ -211,6 +212,10 @@ export interface OrchestratorInput { * result whose title+snippet does not contain the unquoted query as a * case-insensitive substring is dropped post-rerank. */ exactMatch?: boolean; + /** Caller-supplied engine allowlist. When non-empty, only engines whose + * name matches an entry (case-insensitive) are dispatched. Wired from + * SearchInput.search_engines via the MCP schema and CLI --search-engines. */ + engineFilter?: string[]; } export interface OrchestratorOutput { @@ -303,6 +308,12 @@ interface RunV1SearchOptions { _isFallback?: boolean; } +function applyEngineAllowlist(entries: EngineEntry[], allowlist: string[]): EngineEntry[] { + const lowered = allowlist.map((n) => n.toLowerCase()); + const filtered = entries.filter((e) => lowered.includes(e.engine.name.toLowerCase())); + return filtered; +} + export async function runV1Search( input: OrchestratorInput, opts: RunV1SearchOptions = {}, @@ -365,9 +376,43 @@ export async function runV1Search( // Probe-only engines are held back from the primary wave: they are a // per-call latency/failure tax on the happy path but still an independent // signal the degraded-recovery wave can pull in when the pool collapses. - const entries = allEntries.filter((e) => e.probeOnly !== true); + let entries = allEntries.filter((e) => e.probeOnly !== true); const probeEntries = allEntries.filter((e) => e.probeOnly === true); + // Apply caller-supplied engine allowlist (SearchInput.search_engines). + // Normalise ONCE up front (trim, lowercase, dedupe, sort) and use that + // normalised list at every gate below — primary, probe fallback, recovery, + // and starvation backfill. Normalising at the gates (rather than raw-matching) + // keeps the orchestrator consistent with the cache-key fingerprint in + // cache/store.ts, which trims the same value: a whitespace-padded valid name + // like [' duckduckgo '] must dispatch ONLY that engine, not miss the + // allowlist and dispatch the full roster (which would then be cached under + // the trimmed single-engine key). An all-blank list normalises to null, + // i.e. treated as "no filter". Case-insensitive match against engine name. + // For the primary wave, if no entries match the allowlist, fall back to the + // full roster (the caller likely made a typo or passed an unknown engine + // name). For recovery and backfill waves, an empty result means those waves + // run nothing — which is correct: if the caller explicitly filtered out all + // probe/backfill engines, we don't secretly re-introduce them. + const engineAllowlist = normaliseEngineList(input.engineFilter); + if (engineAllowlist && engineAllowlist.length > 0) { + const allowlisted = applyEngineAllowlist(entries, engineAllowlist); + if (allowlisted.length > 0) { + entries = allowlisted; + } else { + // No NON-probe entry matched. Distinguish a caller typo from an explicit + // probe-only selection: if the filter names a configured probe-only + // engine (e.g. Mojeek with searchMojeekProbeOnly enabled), dispatch those + // probe-only engines rather than silently restoring the full primary + // roster and dispatching unselected engines. Fall back to the full roster + // ONLY when the filter matches no configured engine at all. + const probeAllowlisted = applyEngineAllowlist(probeEntries, engineAllowlist); + if (probeAllowlisted.length > 0) { + entries = probeAllowlisted; + } + } + } + const options: SearchEngineOptions = { maxResults: input.maxResults ?? DEFAULT_MAX_RESULTS, timeoutMs: input.timeoutMs ?? DEFAULT_TIMEOUT_MS, @@ -623,10 +668,30 @@ export async function runV1Search( const skippedPrimary = outcomes .filter((o) => o.skipped) .map((o) => o.engine); - const recoveryEntries = [ - ...probeEntries, + // Engines that received an ATTEMPTED (non-skipped) primary dispatch. A + // probe-only engine selected via engineFilter and dispatched as the primary + // wave that returns zero results must NOT re-enter the recovery roster via + // probeEntries — that would fire a second external request and recovery wait + // against the same engine without probing a new one. Engines SKIPPED in the + // primary wave (breaker open) stay eligible: recovery is their retry path. + const attemptedPrimary = new Set( + outcomes.filter((o) => !o.skipped).map((o) => o.engine), + ); + let recoveryEntries = [ + ...probeEntries.filter((e) => !attemptedPrimary.has(e.engine.name)), ...entries.filter((e) => skippedPrimary.includes(e.engine.name)), ]; + // Dedupe by engine name: a probe-only engine selected via engineFilter and + // skipped (breaker open) appears in both lists above. + const seenRecovery = new Set(); + recoveryEntries = recoveryEntries.filter((e) => { + if (seenRecovery.has(e.engine.name)) return false; + seenRecovery.add(e.engine.name); + return true; + }); + if (engineAllowlist && engineAllowlist.length > 0) { + recoveryEntries = applyEngineAllowlist(recoveryEntries, engineAllowlist); + } if ( outcomes.length > 0 && primaryHealthy < poolHealthFloor(outcomes.length) && @@ -684,7 +749,10 @@ export async function runV1Search( vertical !== 'images' && !opts._isFallback ) { - const generalEntries = getGeneralEngines(); + let generalEntries = getGeneralEngines(); + if (engineAllowlist && engineAllowlist.length > 0) { + generalEntries = applyEngineAllowlist(generalEntries, engineAllowlist); + } if (generalEntries.length > 0) { log.info('vertical starved below floor, backfilling from general', { from: vertical, diff --git a/src/util/engine-list.ts b/src/util/engine-list.ts new file mode 100644 index 000000000..163ceb81b --- /dev/null +++ b/src/util/engine-list.ts @@ -0,0 +1,16 @@ +/** Normalise a caller-supplied engine list: trim, lowercase, dedupe, sort. + * Mirrors the orchestrator's case-insensitive engine matching, so + * `['DuckDuckGo']`, `['duckduckgo']`, whitespace-padded, reordered, or + * duplicate-name lists — all of which dispatch the same engine set — hit + * the same allowlist gates AND produce identical cache keys. Shared by the + * cache-key fingerprint (cache/store.ts) and every engineFilter gate in the + * orchestrator so the two can never drift apart (a padded value that misses + * the dispatch allowlist but trims into the cache key would file a + * full-roster response under a single-engine key). An all-blank list + * normalises to null, i.e. "no filter". */ +export function normaliseEngineList(list?: string[] | null): string[] | null { + if (!list || list.length === 0) return null; + const lower = list.map((e) => e.toLowerCase().trim()).filter((e) => e.length > 0); + if (lower.length === 0) return null; + return [...new Set(lower)].sort(); +} diff --git a/tests/unit/cache/store-search-key.test.ts b/tests/unit/cache/store-search-key.test.ts index b518b54d8..f5442bf06 100644 --- a/tests/unit/cache/store-search-key.test.ts +++ b/tests/unit/cache/store-search-key.test.ts @@ -62,6 +62,26 @@ describe('buildSearchCacheKey', () => { expect(balanced).not.toBe(noRerank); expect(balanced).not.toBe('q'); // depth always present -> always fingerprinted }); + + it('normalises search_engines: casing, order, and duplicates share one key', () => { + const a = buildSearchCacheKey('q', { search_engines: ['DuckDuckGo'] }); + const b = buildSearchCacheKey('q', { search_engines: ['duckduckgo'] }); + const c = buildSearchCacheKey('q', { search_engines: ['brave', 'duckduckgo'] }); + const d = buildSearchCacheKey('q', { search_engines: ['DuckDuckGo', 'brave'] }); + const e = buildSearchCacheKey('q', { search_engines: ['duckduckgo', 'duckduckgo'] }); + expect(a).toBe(b); // case-insensitive + expect(c).toBe(d); // order- and case-insensitive + expect(a).toBe(e); // duplicates collapse + expect(a).not.toBe(c); // different engine sets stay distinct + }); + + it('treats whitespace-only or empty search_engines as no filter', () => { + const bare = buildSearchCacheKey('q'); + const empty = buildSearchCacheKey('q', { search_engines: [] }); + const blanks = buildSearchCacheKey('q', { search_engines: [' ', ''] }); + expect(bare).toBe(empty); + expect(bare).toBe(blanks); + }); }); describe('cache miss on filter mismatch', () => { diff --git a/tests/unit/search/v1/orchestrator.test.ts b/tests/unit/search/v1/orchestrator.test.ts index 0596e6d7a..3aeee1afb 100644 --- a/tests/unit/search/v1/orchestrator.test.ts +++ b/tests/unit/search/v1/orchestrator.test.ts @@ -103,7 +103,7 @@ function makeMockEngine(cfg: MockEngineConfig): { } function makeEntry( - cfg: MockEngineConfig & { weight?: number; supportsDateFilter?: boolean }, + cfg: MockEngineConfig & { weight?: number; supportsDateFilter?: boolean; probeOnly?: boolean }, ): { entry: EngineEntry; spy: ReturnType } { const { engine, spy } = makeMockEngine(cfg); return { @@ -111,6 +111,7 @@ function makeEntry( engine, weight: cfg.weight, supportsDateFilter: cfg.supportsDateFilter, + probeOnly: cfg.probeOnly, }, spy, }; @@ -1352,3 +1353,251 @@ describe('runV1Search — recency boost', () => { ]); }); }); + +describe('runV1Search — engineFilter (search_engines parameter)', () => { + it('filters engines by name when engineFilter is provided', async () => { + const { entry: bing, spy: bingSpy } = makeEntry({ + name: 'bing', + results: [makeResult('bing', 'https://bing.test/x')], + }); + const { entry: ddg, spy: ddgSpy } = makeEntry({ + name: 'duckduckgo', + results: [makeResult('duckduckgo', 'https://ddg.test/y')], + }); + verticalState.general = [bing, ddg]; + + const out = await runV1Search({ + query: 'test query', + engineFilter: ['duckduckgo'], + }); + expect(ddgSpy).toHaveBeenCalledOnce(); + expect(bingSpy).not.toHaveBeenCalled(); + expect(out.enginesUsed).toEqual(['duckduckgo']); + }); + + it('matches engine names case-insensitively', async () => { + const { entry: bing, spy: bingSpy } = makeEntry({ + name: 'bing', + results: [makeResult('bing', 'https://bing.test/x')], + }); + const { entry: ddg, spy: ddgSpy } = makeEntry({ + name: 'duckduckgo', + results: [makeResult('duckduckgo', 'https://ddg.test/y')], + }); + verticalState.general = [bing, ddg]; + + const out = await runV1Search({ + query: 'test query', + engineFilter: ['DuckDuckGo'], + }); + expect(ddgSpy).toHaveBeenCalledOnce(); + expect(bingSpy).not.toHaveBeenCalled(); + expect(out.enginesUsed).toEqual(['duckduckgo']); + }); + + it('falls back to full roster when filter matches nothing', async () => { + const { entry: bing, spy: bingSpy } = makeEntry({ + name: 'bing', + results: [makeResult('bing', 'https://bing.test/x')], + }); + const { entry: ddg, spy: ddgSpy } = makeEntry({ + name: 'duckduckgo', + results: [makeResult('duckduckgo', 'https://ddg.test/y')], + }); + verticalState.general = [bing, ddg]; + + await runV1Search({ + query: 'test query', + engineFilter: ['nonexistent-engine'], + }); + // Both engines dispatched when filter matches nothing + expect(bingSpy).toHaveBeenCalledOnce(); + expect(ddgSpy).toHaveBeenCalledOnce(); + }); + + it('does not filter when engineFilter is empty', async () => { + const { entry: bing, spy: bingSpy } = makeEntry({ + name: 'bing', + results: [makeResult('bing', 'https://bing.test/x')], + }); + const { entry: ddg, spy: ddgSpy } = makeEntry({ + name: 'duckduckgo', + results: [makeResult('duckduckgo', 'https://ddg.test/y')], + }); + verticalState.general = [bing, ddg]; + + await runV1Search({ + query: 'test query', + engineFilter: [], + }); + expect(bingSpy).toHaveBeenCalledOnce(); + expect(ddgSpy).toHaveBeenCalledOnce(); + }); + + it('does not filter when engineFilter is undefined', async () => { + const { entry: bing, spy: bingSpy } = makeEntry({ + name: 'bing', + results: [makeResult('bing', 'https://bing.test/x')], + }); + const { entry: ddg, spy: ddgSpy } = makeEntry({ + name: 'duckduckgo', + results: [makeResult('duckduckgo', 'https://ddg.test/y')], + }); + verticalState.general = [bing, ddg]; + + await runV1Search({ query: 'test query' }); + expect(bingSpy).toHaveBeenCalledOnce(); + expect(ddgSpy).toHaveBeenCalledOnce(); + }); + + it('dispatches a configured probe-only engine when it is the only filter match (no full-roster fallback)', async () => { + const { entry: bing, spy: bingSpy } = makeEntry({ + name: 'bing', + results: [makeResult('bing', 'https://bing.test/x')], + }); + const { entry: ddg, spy: ddgSpy } = makeEntry({ + name: 'duckduckgo', + results: [makeResult('duckduckgo', 'https://ddg.test/y')], + }); + // Mojeek configured probe-only (searchMojeekProbeOnly enabled): held back + // from the primary wave, but an explicit selection must dispatch it — not + // be treated as "unknown engine" and restore the whole primary roster. + const { entry: mojeek, spy: mojeekSpy } = makeEntry({ + name: 'mojeek', + probeOnly: true, + results: [makeResult('mojeek', 'https://mojeek.test/z')], + }); + verticalState.general = [bing, ddg, mojeek]; + + const out = await runV1Search({ + query: 'test query', + engineFilter: ['mojeek'], + }); + expect(mojeekSpy).toHaveBeenCalledOnce(); + expect(bingSpy).not.toHaveBeenCalled(); + expect(ddgSpy).not.toHaveBeenCalled(); + expect(out.enginesUsed).toEqual(['mojeek']); + }); + + it('keeps the full-roster fallback when the filter matches no configured engine at all', async () => { + const { entry: bing, spy: bingSpy } = makeEntry({ + name: 'bing', + results: [makeResult('bing', 'https://bing.test/x')], + }); + const { entry: mojeek, spy: mojeekSpy } = makeEntry({ + name: 'mojeek', + probeOnly: true, + results: [makeResult('mojeek', 'https://mojeek.test/z')], + }); + verticalState.general = [bing, mojeek]; + + await runV1Search({ + query: 'test query', + engineFilter: ['nonexistent-engine'], + }); + // Unknown name: full primary roster restored (probe-only still held back). + expect(bingSpy).toHaveBeenCalledOnce(); + expect(mojeekSpy).not.toHaveBeenCalled(); + }); + + it('holds probe-only engines back when the filter also matches a primary engine', async () => { + const { entry: bing, spy: bingSpy } = makeEntry({ + name: 'bing', + results: [makeResult('bing', 'https://bing.test/x')], + }); + const { entry: ddg, spy: ddgSpy } = makeEntry({ + name: 'duckduckgo', + results: [makeResult('duckduckgo', 'https://ddg.test/y')], + }); + const { entry: mojeek, spy: mojeekSpy } = makeEntry({ + name: 'mojeek', + probeOnly: true, + results: [makeResult('mojeek', 'https://mojeek.test/z')], + }); + verticalState.general = [bing, ddg, mojeek]; + + const out = await runV1Search({ + query: 'test query', + engineFilter: ['duckduckgo', 'mojeek'], + }); + expect(ddgSpy).toHaveBeenCalledOnce(); + expect(bingSpy).not.toHaveBeenCalled(); + // Probe-only selection is honoured via its intended wave, not the primary. + expect(out.enginesUsed).not.toContain('bing'); + }); + + it('treats a whitespace-padded valid engine name as its trimmed value', async () => { + // The cache-key fingerprint trims filter values, so the dispatch gates + // must too — otherwise [' duckduckgo '] misses the allowlist, dispatches + // the FULL roster, and that response gets cached under the trimmed + // single-engine key. + const { entry: bing, spy: bingSpy } = makeEntry({ + name: 'bing', + results: [makeResult('bing', 'https://bing.test/x')], + }); + const { entry: ddg, spy: ddgSpy } = makeEntry({ + name: 'duckduckgo', + results: [makeResult('duckduckgo', 'https://ddg.test/y')], + }); + verticalState.general = [bing, ddg]; + + const out = await runV1Search({ + query: 'test query', + engineFilter: [' duckduckgo '], + }); + expect(ddgSpy).toHaveBeenCalledOnce(); + expect(bingSpy).not.toHaveBeenCalled(); + expect(out.enginesUsed).toEqual(['duckduckgo']); + }); + + it('treats an all-blank engineFilter as no filter', async () => { + const { entry: bing, spy: bingSpy } = makeEntry({ + name: 'bing', + results: [makeResult('bing', 'https://bing.test/x')], + }); + const { entry: ddg, spy: ddgSpy } = makeEntry({ + name: 'duckduckgo', + results: [makeResult('duckduckgo', 'https://ddg.test/y')], + }); + verticalState.general = [bing, ddg]; + + await runV1Search({ + query: 'test query', + engineFilter: [' ', ''], + }); + // Blank-only list normalises to null: both engines dispatched. + expect(bingSpy).toHaveBeenCalledOnce(); + expect(ddgSpy).toHaveBeenCalledOnce(); + }); + + it('does not re-dispatch a zero-result probe-only engine selected as the primary wave', async () => { + // Selecting a probe-only engine dispatches it as the primary wave. If it + // returns zero results, the pool is below the health floor and the + // recovery wave fires — but it must NOT re-dispatch the same engine + // (second external request + recovery wait without probing a new one). + const { entry: bing, spy: bingSpy } = makeEntry({ + name: 'bing', + results: [makeResult('bing', 'https://bing.test/x')], + }); + const { entry: ddg, spy: ddgSpy } = makeEntry({ + name: 'duckduckgo', + results: [makeResult('duckduckgo', 'https://ddg.test/y')], + }); + const { entry: mojeek, spy: mojeekSpy } = makeEntry({ + name: 'mojeek', + probeOnly: true, + results: [], + }); + verticalState.general = [bing, ddg, mojeek]; + + await runV1Search({ + query: 'test query', + engineFilter: ['mojeek'], + }); + // Exactly one dispatch: the primary attempt. The recovery wave (which + // triggers on 0 healthy < floor 1) excludes the already-attempted probe. + expect(mojeekSpy).toHaveBeenCalledOnce(); + expect(bingSpy).not.toHaveBeenCalled(); + expect(ddgSpy).not.toHaveBeenCalled(); + }); +});