Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions src/search/core/engine-quality.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ const ENGINE_QUALITY: Record<string, EngineQualityTier> = {
// crates.io: structured JSON API with reliable Cargo.toml-sourced
// descriptions → high, same class as wikipedia/mdn.
'crates-io': 'high',
// npm registry: same class as crates-io — structured JSON API with
// reliable descriptions from package metadata.
'npm-registry': 'high',
bing: 'medium',
bing_news: 'medium',
duckduckgo: 'medium',
Expand Down
9 changes: 9 additions & 0 deletions src/search/core/verticals/code.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { DevDocsEngine } from '../../engines/devdocs.js';
import { DuckDuckGoEngine } from '../../engines/duckduckgo.js';
import { BraveEngine } from '../../engines/brave.js';
import { CratesIoEngine } from '../../engines/crates-io.js';
import { NpmRegistryEngine } from '../../engines/npm-registry.js';
import { wrapWithRetryAndBreaker, type EngineEntry } from '../engine-base.js';
import { getConfig } from '../../../config.js';

Expand Down Expand Up @@ -76,6 +77,14 @@ export function getCodeEngines(): EngineEntry[] {
quality: 'high',
});

entries.push({
engine: wrapWithRetryAndBreaker(new NpmRegistryEngine()),
weight: 0.3,
supportsDateFilter: false,
secondary: true,
quality: 'high',
});

cached = entries;
return cached;
}
Expand Down
121 changes: 121 additions & 0 deletions src/search/engines/npm-registry.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
import type { SearchEngine, SearchEngineOptions, RawSearchResult } from '../../types.js';
import { createLogger } from '../../logger.js';

const log = createLogger('search');

interface NpmPublisher {
username?: unknown;
}

interface NpmPackage {
name?: unknown;
version?: unknown;
description?: unknown;
date?: unknown;
publisher?: NpmPublisher;
}

interface NpmSearchObject {
package?: NpmPackage;
}

interface NpmSearchResponse {
objects?: NpmSearchObject[];
}

function asString(v: unknown): string | undefined {
return typeof v === 'string' && v.length > 0 ? v : undefined;
}

// npm's /-/v1/search endpoint caps the `size` parameter at 250; larger values
// are silently truncated server-side. Cap the request and enforce the caller's
// requested limit locally so results never exceed maxResults.
const MAX_NPM_PAGE_SIZE = 250;

// npm's public package-search API: free, no key, returns name/version/
// description/date/publisher for matching packages. Adds a canonical
// JavaScript-package-registry signal to the code vertical — useful when a
// query names or resembles an npm package (e.g. "fastify schema validation")
// so the ecosystem's own metadata (not just blog posts or Stack Overflow)
// surfaces directly. Complements crates-io, which plays the same role for Rust.
export class NpmRegistryEngine implements SearchEngine {
name = 'npm-registry';

async search(query: string, options: SearchEngineOptions = {}): Promise<RawSearchResult[]> {
const timeoutMs = options.timeoutMs ?? 10000;
const maxResults = options.maxResults ?? 10;

const params = new URLSearchParams({
text: query,
size: String(Math.min(maxResults, MAX_NPM_PAGE_SIZE)),
});

const url = `https://registry.npmjs.org/-/v1/search?${params}`;
log.debug('npm registry search', { query });

const response = await fetch(url, {
signal: AbortSignal.timeout(timeoutMs),
headers: {
'User-Agent': 'wigolo/0.1 (https://github.com/KnockOutEZ/wigolo)',
Accept: 'application/json',
},
});

if (!response.ok) throw new Error(`npm registry returned ${response.status}`);

const data: unknown = await response.json();
// Guard against non-object payloads (null, primitive, or missing objects) so the
// engine returns [] instead of throwing on property access.
const objects =
data !== null &&
typeof data === 'object' &&
Array.isArray((data as NpmSearchResponse).objects)
? (data as NpmSearchResponse).objects
: [];
return this.parseObjects(objects, maxResults);
}

private parseObjects(objects: NpmSearchObject[], maxResults: number): RawSearchResult[] {
if (maxResults <= 0) return [];
const results: RawSearchResult[] = [];
const total = objects.length;

for (let i = 0; i < total; i++) {
const entry = objects[i] as NpmSearchObject | null | undefined;
if (entry === null || typeof entry !== 'object') continue;
const pkg = entry.package;
const name = asString(pkg?.name);
if (!name) continue;

const description = asString(pkg?.description) ?? '';
const version = asString(pkg?.version);
const publisher = asString(pkg?.publisher?.username);

const meta: string[] = [];
if (version) meta.push(`v${version}`);
if (publisher) meta.push(`by ${publisher}`);
const suffix = meta.length ? ` (${meta.join(', ')})` : '';
const snippet = `${description}${suffix}`;

// Construct the canonical npmjs URL from the name rather than trusting
// the registry-supplied links.npm (untrusted JSON; could point anywhere).
const url = `https://www.npmjs.com/package/${name}`;
const published_date = asString(pkg?.date);

results.push({
title: name,
url,
snippet,
relevance_score: 1 - i / Math.max(total, 1),
engine: 'npm-registry',
...(published_date ? { published_date } : {}),
});

// Cap after mapping valid packages so nameless/invalid rows do not
// count against maxResults.
if (results.length >= maxResults) break;
}

return results;
}
}
241 changes: 241 additions & 0 deletions tests/unit/search/engines/npm-registry.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,241 @@
import { describe, it, expect, vi, afterEach } from 'vitest';
import { NpmRegistryEngine } from '../../../../src/search/engines/npm-registry.js';

interface FetchCall {
url: string;
init?: RequestInit;
}

function captureFetch(body: unknown, ok = true, status = 200): {
calls: FetchCall[];
restore: () => void;
} {
const calls: FetchCall[] = [];
const spy = vi.spyOn(global, 'fetch').mockImplementation(async (input, init) => {
const url = typeof input === 'string' ? input : input instanceof URL ? input.toString() : input.url;
calls.push({ url, init });
return {
ok,
status,
json: async () => body,
text: async () => JSON.stringify(body),
} as Response;
});
return { calls, restore: () => spy.mockRestore() };
}

describe('NpmRegistryEngine', () => {
afterEach(() => {
vi.restoreAllMocks();
});

it('has name set to npm-registry', () => {
expect(new NpmRegistryEngine().name).toBe('npm-registry');
});

it('maps a successful response to RawSearchResult fields', async () => {
const body = {
objects: [
{
package: {
name: 'express',
version: '5.2.1',
description: 'Fast, unopinionated, minimalist web framework',
date: '2025-12-01T20:49:43.268Z',
publisher: { username: 'jonchurch' },
links: { npm: 'https://www.npmjs.com/package/express' },
},
},
],
};
captureFetch(body);
const results = await new NpmRegistryEngine().search('express');

expect(results).toHaveLength(1);
expect(results[0].title).toBe('express');
expect(results[0].url).toBe('https://www.npmjs.com/package/express');
expect(results[0].engine).toBe('npm-registry');
expect(results[0].snippet).toBe(
'Fast, unopinionated, minimalist web framework (v5.2.1, by jonchurch)',
);
expect(results[0].relevance_score).toBe(1);
expect(results[0].published_date).toBe('2025-12-01T20:49:43.268Z');
});

it('falls back to npmjs package URL when links.npm is missing', async () => {
const body = {
objects: [{ package: { name: 'left-pad', version: '1.3.0', description: 'pad' } }],
};
captureFetch(body);
const results = await new NpmRegistryEngine().search('left-pad');
expect(results[0].url).toBe('https://www.npmjs.com/package/left-pad');
});

it('constructs the canonical npmjs URL instead of trusting links.npm', async () => {
const body = {
objects: [
{
package: {
name: '@types/node',
description: 'typed node',
links: { npm: 'https://evil.example/not-npmjs' },
},
},
],
};
captureFetch(body);
const results = await new NpmRegistryEngine().search('types node');
expect(results[0].url).toBe('https://www.npmjs.com/package/@types/node');
});

it('sets a descriptive User-Agent header', async () => {
const { calls } = captureFetch({ objects: [] });
await new NpmRegistryEngine().search('q');
const headers = calls[0].init?.headers as Record<string, string>;
expect(headers['User-Agent']).toContain('wigolo');
expect(headers['User-Agent']).toContain('https://github.com/KnockOutEZ/wigolo');
});

it('builds snippet without version/publisher metadata when absent', async () => {
const body = {
objects: [{ package: { name: 'foo', description: 'a thing' } }],
};
captureFetch(body);
const results = await new NpmRegistryEngine().search('foo');
expect(results[0].snippet).toBe('a thing');
});

it('skips packages without a name', async () => {
const body = {
objects: [
{ package: { name: null, description: 'no name' } },
{ package: { name: 'valid', description: 'ok', version: '1.0.0' } },
],
};
captureFetch(body);
const results = await new NpmRegistryEngine().search('q');
expect(results).toHaveLength(1);
expect(results[0].title).toBe('valid');
});

it('omits published_date when date is missing', async () => {
const body = { objects: [{ package: { name: 'x', description: 'y' } }] };
captureFetch(body);
const results = await new NpmRegistryEngine().search('q');
expect(results[0].published_date).toBeUndefined();
});

it('passes size matching maxResults', async () => {
const { calls } = captureFetch({ objects: [] });
await new NpmRegistryEngine().search('q', { maxResults: 25 });
expect(calls[0].url).toContain('size=25');
});

it('caps request size at 250 when maxResults exceeds the npm limit', async () => {
const { calls } = captureFetch({ objects: [] });
await new NpmRegistryEngine().search('q', { maxResults: 500 });
expect(calls[0].url).toContain('size=250');
});

it('slices results down to maxResults for local enforcement', async () => {
const objects = Array.from({ length: 5 }, (_, i) => ({
package: { name: `pkg-${i}`, description: 'd' },
}));
captureFetch({ objects });
const results = await new NpmRegistryEngine().search('q', { maxResults: 3 });
expect(results).toHaveLength(3);
expect(results.map((r) => r.title)).toEqual(['pkg-0', 'pkg-1', 'pkg-2']);
});

it('counts maxResults against valid packages, not raw objects', async () => {
const body = {
objects: [
{ package: { name: null, description: 'no name' } },
{ package: { name: 'first-valid', description: 'a' } },
{ package: { name: 'second-valid', description: 'b' } },
],
};
captureFetch(body);
const results = await new NpmRegistryEngine().search('q', { maxResults: 1 });
expect(results).toHaveLength(1);
expect(results[0].title).toBe('first-valid');
});

it('returns empty array when objects is a non-array value', async () => {
captureFetch({ objects: {} });
const results = await new NpmRegistryEngine().search('q');
expect(results).toEqual([]);
captureFetch({ objects: 'not-an-array' });
const results2 = await new NpmRegistryEngine().search('q');
expect(results2).toEqual([]);
});

it('encodes the query text parameter', async () => {
const { calls } = captureFetch({ objects: [] });
await new NpmRegistryEngine().search('fastify schema');
expect(calls[0].url).toContain('text=fastify+schema');
});

it('throws when HTTP response is not ok', async () => {
captureFetch({}, false, 503);
await expect(new NpmRegistryEngine().search('q')).rejects.toThrow(/npm registry returned 503/);
});

it('returns empty array on empty objects', async () => {
captureFetch({ objects: [] });
const results = await new NpmRegistryEngine().search('q');
expect(results).toEqual([]);
});

it('returns empty array when maxResults is 0', async () => {
const body = {
objects: [{ package: { name: 'foo', description: 'a thing' } }],
};
captureFetch(body);
const results = await new NpmRegistryEngine().search('q', { maxResults: 0 });
expect(results).toEqual([]);
});

it('returns empty array when objects field is absent', async () => {
captureFetch({});
const results = await new NpmRegistryEngine().search('q');
expect(results).toEqual([]);
});

it('returns empty array when top-level JSON response is null', async () => {
captureFetch(null);
const results = await new NpmRegistryEngine().search('q');
expect(results).toEqual([]);
});

it('returns empty array when top-level JSON response is a primitive', async () => {
captureFetch('unexpected string');
const results = await new NpmRegistryEngine().search('q');
expect(results).toEqual([]);
captureFetch(42);
const results2 = await new NpmRegistryEngine().search('q');
expect(results2).toEqual([]);
});

it('skips null and non-object entries in objects array', async () => {
const body = {
objects: [
null,
{ package: { name: 'valid', description: 'ok' } },
'not-an-object',
42,
{ package: { name: 'also-valid', description: 'fine' } },
],
};
captureFetch(body);
const results = await new NpmRegistryEngine().search('q');
expect(results).toHaveLength(2);
expect(results[0].title).toBe('valid');
expect(results[1].title).toBe('also-valid');
});

it('propagates fetch errors (timeout/network)', async () => {
vi.spyOn(global, 'fetch').mockRejectedValueOnce(new Error('aborted'));
await expect(new NpmRegistryEngine().search('q')).rejects.toThrow(/aborted/);
});
});
Loading