From dabd89e3932a35390e3e6b094e183469fb506eee Mon Sep 17 00:00:00 2001 From: yc2bgr8 <119500159+yc2bgr8@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:59:34 +0800 Subject: [PATCH] =?UTF-8?q?feat(research):=20=E5=A2=9E=E5=8A=A0=E6=96=B0?= =?UTF-8?q?=E9=97=BB=E6=9D=A5=E6=BA=90=E6=8E=92=E5=BA=8F=E4=B8=8E=E5=8E=BB?= =?UTF-8?q?=E9=87=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/retrieval-ranking.md | 42 ++ package.json | 1 + packages/shared/package.json | 1 + packages/shared/src/index.ts | 1 + packages/shared/src/research/runner.test.ts | 77 ++++ packages/shared/src/research/runner.ts | 87 +++- packages/shared/src/retrieval/index.ts | 13 + packages/shared/src/retrieval/ranking.test.ts | 206 +++++++++ packages/shared/src/retrieval/ranking.ts | 435 ++++++++++++++++++ scripts/eval/retrieval-ranking-smoke.ts | 88 ++++ 10 files changed, 946 insertions(+), 5 deletions(-) create mode 100644 docs/retrieval-ranking.md create mode 100644 packages/shared/src/retrieval/index.ts create mode 100644 packages/shared/src/retrieval/ranking.test.ts create mode 100644 packages/shared/src/retrieval/ranking.ts create mode 100644 scripts/eval/retrieval-ranking-smoke.ts diff --git a/docs/retrieval-ranking.md b/docs/retrieval-ranking.md new file mode 100644 index 0000000..dc1e989 --- /dev/null +++ b/docs/retrieval-ranking.md @@ -0,0 +1,42 @@ +# Retrieval ranking and deduplication + +Folio applies deterministic source normalization before news results enter +Deep Research synthesis. The policy implementation is in +`packages/shared/src/retrieval` and is versioned as `folio-retrieval-v1`. + +The pipeline: + +1. canonicalizes URLs, removes tracking parameters and fragments, and sorts + meaningful query parameters; +2. normalizes titles and fingerprints excerpts; +3. clusters same-URL, same-title, same-content, and near-duplicate results; +4. scores cluster representatives using source class, freshness, query + relevance, and availability; +5. selects Top-K with a per-domain diversity limit. + +Regulators, exchanges, and configured issuer domains receive primary-source +priority. Established news outlets remain useful for independent reporting; +aggregators receive a lower deterministic base score. Every selected or +dropped result records its original rank, cluster id, decision reason, and the +ranking policy version in the Research data bundle. + +## Live smoke run + +The live smoke command calls the public Hacker News Algolia search API, then +runs the same ranking module used by `ResearchRunner`: + +```sh +bun run eval:retrieval-ranking-live -- "NVIDIA earnings" +``` + +It prints machine-readable JSON containing the search endpoint, raw count, +clusters, selected source set, and dropped reasons. The command intentionally +fails when search returns no usable sources. This is an explicit live check, +not part of ordinary unit tests or CI. + +## Boundaries + +Version 1 does not fetch article bodies or resolve network redirects. Providers +may supply a resolved `canonicalUrl`; otherwise Folio uses deterministic URL, +title, and excerpt signals. UI display of clusters and claim-level citation +verification remain separate integration work. diff --git a/package.json b/package.json index 4ff9240..c6a9e54 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "eval:smoke": "bun scripts/eval/run.ts --smoke", "eval:full": "bun scripts/eval/run.ts", "eval:langfuse": "bun scripts/eval/langfuse-live.ts", + "eval:retrieval-ranking-live": "bun scripts/eval/retrieval-ranking-smoke.ts", "test:ui": "bun test packages/ui", "test:e2e": "cd apps/electron && bun run test:e2e", "test:e2e:visible": "cd apps/electron && FINAGENT_E2E_VISIBLE=1 bun run test:e2e", diff --git a/packages/shared/package.json b/packages/shared/package.json index c97b69e..73447c8 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -14,6 +14,7 @@ "./portfolio-risk": "./src/portfolio-risk/index.ts", "./portfolio-import": "./src/portfolio-import/index.ts", "./diagnostics": "./src/diagnostics/index.ts", + "./retrieval": "./src/retrieval/index.ts", "./providers/longbridge": "./src/providers/longbridge/index.ts", "./providers/massive": "./src/providers/massive/index.ts" }, diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index cd924ad..ee7f64c 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -153,3 +153,4 @@ export * from './calibration/index.ts'; export * from './pulse/index.ts'; export * from './export/index.ts'; export * from './evaluation/index.ts'; +export * from './retrieval/index.ts'; diff --git a/packages/shared/src/research/runner.test.ts b/packages/shared/src/research/runner.test.ts index 97ad64f..20d6492 100644 --- a/packages/shared/src/research/runner.test.ts +++ b/packages/shared/src/research/runner.test.ts @@ -107,6 +107,83 @@ describe('ResearchRunner', () => { expect(result.report!.runStatus).toBe('failed'); }); + it('deduplicates and records ranked news before synthesis', async () => { + const news = fakeCap('research.news'); + news.execute = async () => ({ + data: [ + { + id: 'wire', + title: 'NVIDIA announces quarterly results', + summary: 'NVIDIA reported quarterly revenue and guidance.', + url: 'https://reuters.com/technology/nvidia-results?utm_source=feed', + timestamp: 1_700_000_000, + symbols: ['NVDA.US'], + }, + { + id: 'wire-copy', + title: 'NVIDIA announces quarterly results', + summary: 'NVIDIA reported quarterly revenue and guidance.', + url: 'https://copy.test/nvidia-results', + timestamp: 1_700_000_000, + symbols: ['NVDA.US'], + }, + { + id: 'filing', + title: 'NVIDIA quarterly report', + summary: 'Official filing for NVIDIA quarterly results.', + url: 'https://sec.gov/Archives/edgar/data/1/report', + timestamp: 1_700_000_000, + symbols: ['NVDA.US'], + }, + ], + provenance: { + provider: 'test', + fetchedAt: 1_700_000_000_000, + stale: false, + }, + }); + const registry = createCapabilityRegistry([news]); + const local = new LocalResearchSynthesizer(); + let observed: Record = {}; + const runner = new ResearchRunner({ + registry, + synthesizer: { + async synthesize(input, signal) { + observed = JSON.parse(input.dataBundle) as Record; + return local.synthesize(input, signal); + }, + }, + now: () => 1_700_000_000_000, + }); + + await runner.run({ symbol: 'NVDA.US', runId: 'ranked-news' }); + + const selected = observed['research.news'] as Array>; + const metadata = observed['research.news:ranking'] as { + policyVersion: string; + independentSourceCount: number; + decisions: Array<{ + sourceId: string; + originalRank: number; + clusterId: string; + selected: boolean; + reason: string; + }>; + }; + expect(selected).toHaveLength(2); + expect(selected[0].id).toBe('filing'); + expect(selected[0].sourceClass).toBe('regulator'); + expect(metadata.policyVersion).toBe('folio-retrieval-v1'); + expect(metadata.independentSourceCount).toBe(2); + expect(metadata.decisions).toContainEqual({ + sourceId: 'wire-copy', + originalRank: 1, + clusterId: expect.any(String), + selected: false, + reason: 'duplicate_of:wire', + }); + }); + it('cancels mid-fetch when the signal aborts', async () => { const runner = makeRunner([ ['company.profile', 'success'], diff --git a/packages/shared/src/research/runner.ts b/packages/shared/src/research/runner.ts index 28f8c8f..8477421 100644 --- a/packages/shared/src/research/runner.ts +++ b/packages/shared/src/research/runner.ts @@ -20,6 +20,10 @@ import { planForStrategy, type PlannedCapability, } from './planner.ts'; +import { + rankRetrievalSources, + type RetrievalSourceInput, +} from '../retrieval/index.ts'; const CONCURRENCY = 4; const TIMEOUT_MS = 20000; @@ -127,7 +131,7 @@ export class ResearchRunner { }); const runs = buildRuns(plan, outcomes); - const dataBundle = buildDataBundle(outcomes); + const dataBundle = buildDataBundle(outcomes, symbol, this.now()); let synthesis: ResearchSynthesis; try { @@ -189,10 +193,49 @@ function buildRuns(plan: PlannedCapability[], outcomes: RunOutcome[]) { }); } -function buildDataBundle(outcomes: RunOutcome[]): string { +function buildDataBundle( + outcomes: RunOutcome[], + query: string, + now: number +): string { const bundle: Record = {}; for (const outcome of outcomes) { if (outcome.record.status === 'success' && outcome.result) { + if ( + outcome.record.capabilityId === 'research.news' && + Array.isArray(outcome.result.data) + ) { + const sources = toRetrievalSources( + outcome.result.data, + outcome.result.provenance.provider + ); + const ranking = rankRetrievalSources(sources, { + query, + now, + topK: 10, + maxPerDomain: 2, + }); + bundle[outcome.record.capabilityId] = ranking.selected.map((item) => ({ + ...item.source, + canonicalUrl: item.canonicalUrl, + sourceClass: item.sourceClass, + clusterId: item.clusterId, + qualityScore: item.qualityScore, + })); + bundle['research.news:ranking'] = { + policyVersion: ranking.policyVersion, + independentSourceCount: ranking.independentSourceCount, + clusters: ranking.clusters, + decisions: [...ranking.selected, ...ranking.dropped].map((item) => ({ + sourceId: item.source.id, + originalRank: item.originalRank, + clusterId: item.clusterId, + selected: item.selected, + reason: item.decisionReason, + })), + }; + continue; + } bundle[outcome.record.capabilityId] = truncateData( outcome.record.capabilityId, outcome.result.data @@ -202,14 +245,48 @@ function buildDataBundle(outcomes: RunOutcome[]): string { return JSON.stringify(bundle); } +function toRetrievalSources( + data: unknown[], + provider: string +): RetrievalSourceInput[] { + const sources: RetrievalSourceInput[] = []; + for (let index = 0; index < data.length; index += 1) { + const item = data[index]; + if (item === null || typeof item !== 'object') continue; + const record = item as Record; + if (typeof record.title !== 'string' || typeof record.url !== 'string') { + continue; + } + const rawTimestamp = + typeof record.timestamp === 'number' ? record.timestamp : undefined; + const publishedAt = + rawTimestamp === undefined + ? undefined + : rawTimestamp < 10_000_000_000 + ? rawTimestamp * 1000 + : rawTimestamp; + sources.push({ + id: + typeof record.id === 'string' && record.id.length > 0 + ? record.id + : provider + ':' + index, + title: record.title, + url: record.url, + excerpt: + typeof record.summary === 'string' ? record.summary : undefined, + publishedAt, + provider, + available: true, + }); + } + return sources; +} + function truncateData(capabilityId: string, data: unknown): unknown { if (!Array.isArray(data)) return data; if (capabilityId === 'market.kline' || capabilityId === 'market.intraday') { return data.slice(-60); } - if (capabilityId === 'research.news') { - return data.slice(0, 10); - } return data; } diff --git a/packages/shared/src/retrieval/index.ts b/packages/shared/src/retrieval/index.ts new file mode 100644 index 0000000..ec702d9 --- /dev/null +++ b/packages/shared/src/retrieval/index.ts @@ -0,0 +1,13 @@ +export { + RETRIEVAL_RANKING_POLICY_VERSION, + canonicalizeSourceUrl, + normalizeSourceTitle, + rankRetrievalSources, + type DuplicateRelation, + type RankedSource, + type RetrievalRankingOptions, + type RetrievalRankingResult, + type RetrievalSourceInput, + type SourceClass, + type SourceCluster, +} from './ranking.ts'; diff --git a/packages/shared/src/retrieval/ranking.test.ts b/packages/shared/src/retrieval/ranking.test.ts new file mode 100644 index 0000000..7c2948e --- /dev/null +++ b/packages/shared/src/retrieval/ranking.test.ts @@ -0,0 +1,206 @@ +import { describe, expect, it } from 'bun:test'; +import { + RETRIEVAL_RANKING_POLICY_VERSION, + canonicalizeSourceUrl, + normalizeSourceTitle, + rankRetrievalSources, + type RetrievalSourceInput, +} from './ranking.ts'; + +const NOW = Date.UTC(2026, 8, 11); + +function source( + id: string, + url: string, + title: string, + excerpt?: string, + publishedAt = NOW - 3_600_000 +): RetrievalSourceInput { + return { id, url, title, excerpt, publishedAt, available: true }; +} + +describe('canonicalizeSourceUrl', () => { + it('removes tracking and fragments and sorts meaningful parameters', () => { + expect( + canonicalizeSourceUrl( + 'HTTPS://WWW.Example.com/story/?utm_source=x&b=2&a=1#comments' + ) + ).toBe('https://example.com/story?a=1&b=2'); + }); + + it('keeps malformed input stable instead of throwing', () => { + expect(canonicalizeSourceUrl(' not a url ')).toBe('not a url'); + }); +}); + +describe('normalizeSourceTitle', () => { + it('normalizes case, punctuation, unicode width and whitespace', () => { + expect(normalizeSourceTitle('ＮＶＩＤＩＡ： Earnings—Beat!')).toBe( + 'nvidia earnings beat' + ); + }); +}); + +describe('rankRetrievalSources', () => { + it('clusters tracking variants as one independent source', () => { + const result = rankRetrievalSources( + [ + source('a', 'https://example.com/news?id=1&utm_source=feed', 'Result A'), + source('b', 'https://www.example.com/news?id=1#top', 'Result A copied'), + ], + { now: NOW } + ); + expect(result.clusters).toHaveLength(1); + expect(result.independentSourceCount).toBe(1); + expect(result.dropped[0].decisionReason).toBe('duplicate_of:a'); + expect(result.clusters[0].duplicateRelations).toEqual([ + { sourceId: 'b', relation: 'same_url' }, + ]); + }); + + it('clusters exact content copies published under different URLs', () => { + const result = rankRetrievalSources( + [ + source('wire', 'https://wire.test/a', 'Company announces results', 'Revenue rose 20 percent.'), + source('copy', 'https://copy.test/b', 'Results update', 'Revenue rose 20 percent.'), + ], + { now: NOW } + ); + expect(result.clusters).toHaveLength(1); + expect(result.clusters[0].duplicateRelations[0].relation).toBe( + 'same_content' + ); + }); + + it('clusters near-duplicate syndicated excerpts', () => { + const result = rankRetrievalSources( + [ + source( + 'wire', + 'https://wire.test/a', + 'Chipmaker raises outlook', + 'The chipmaker raised its annual revenue outlook after strong demand for data center products' + ), + source( + 'copy', + 'https://copy.test/b', + 'Chipmaker lifts forecast', + 'After strong demand for data center products the chipmaker raised its annual revenue outlook' + ), + ], + { now: NOW, nearDuplicateThreshold: 0.75 } + ); + expect(result.clusters).toHaveLength(1); + expect(result.clusters[0].duplicateRelations[0].relation).toBe( + 'near_duplicate' + ); + }); + + it('does not merge same-title reports from different periods', () => { + const result = rankRetrievalSources( + [ + source( + 'q1', + 'https://issuer.test/q1', + 'Quarterly earnings release', + undefined, + Date.UTC(2025, 0, 1) + ), + source( + 'q2', + 'https://issuer.test/q2', + 'Quarterly earnings release', + undefined, + Date.UTC(2025, 3, 1) + ), + ], + { now: NOW } + ); + expect(result.clusters).toHaveLength(2); + expect(result.independentSourceCount).toBe(2); + }); + + it('prefers primary sources in a relevant financial query', () => { + const result = rankRetrievalSources( + [ + source('blog', 'https://random-blog.test/post', 'NVIDIA files quarterly results'), + source( + 'filing', + 'https://www.sec.gov/Archives/edgar/data/1/report', + 'NVIDIA quarterly results filing' + ), + source('news', 'https://www.reuters.com/technology/report', 'NVIDIA quarterly results'), + ], + { now: NOW, query: 'NVIDIA quarterly results', topK: 3 } + ); + expect(result.selected.map((item) => item.source.id)).toEqual([ + 'filing', + 'news', + 'blog', + ]); + expect(result.selected[0].sourceClass).toBe('regulator'); + }); + + it('recognizes configured issuer domains as primary sources', () => { + const result = rankRetrievalSources( + [source('ir', 'https://investor.nvidia.com/news/1', 'NVIDIA results')], + { now: NOW, issuerDomains: ['investor.nvidia.com'] } + ); + expect(result.selected[0].sourceClass).toBe('issuer'); + }); + + it('enforces domain diversity in Top-K', () => { + const result = rankRetrievalSources( + [ + source('r1', 'https://reuters.com/a', 'Company result one'), + source('r2', 'https://reuters.com/b', 'Company result two'), + source('r3', 'https://reuters.com/c', 'Company result three'), + source('other', 'https://independent.test/a', 'Independent analysis'), + ], + { now: NOW, topK: 3, maxPerDomain: 2 } + ); + expect(result.selected.map((item) => item.source.id)).toEqual([ + 'r1', + 'r2', + 'other', + ]); + expect( + result.dropped.find((item) => item.source.id === 'r3')?.decisionReason + ).toBe('domain_diversity_limit'); + }); + + it('penalizes unavailable sources without dropping their provenance', () => { + const unavailable = source( + 'blocked', + 'https://reuters.com/blocked', + 'Company result' + ); + unavailable.available = false; + const result = rankRetrievalSources( + [ + unavailable, + source('open', 'https://open.test/result', 'Company result explained'), + ], + { now: NOW, topK: 1 } + ); + expect(result.selected[0].source.id).toBe('open'); + expect(result.dropped.map((item) => item.source.id)).toContain('blocked'); + }); + + it('records a machine-readable policy version and all decisions', () => { + const result = rankRetrievalSources( + [ + source('a', 'https://a.test/1', 'One'), + source('b', 'https://b.test/2', 'Two'), + ], + { now: NOW, topK: 1 } + ); + expect(result.policyVersion).toBe(RETRIEVAL_RANKING_POLICY_VERSION); + expect(result.selected).toHaveLength(1); + expect(result.dropped).toHaveLength(1); + expect(result.selected[0].decisionReason).toBe( + 'selected_by_quality_and_diversity' + ); + expect(result.dropped[0].decisionReason).toBe('top_k_limit'); + }); +}); diff --git a/packages/shared/src/retrieval/ranking.ts b/packages/shared/src/retrieval/ranking.ts new file mode 100644 index 0000000..0845218 --- /dev/null +++ b/packages/shared/src/retrieval/ranking.ts @@ -0,0 +1,435 @@ +/** + * Deterministic retrieval normalization, deduplication and diversity ranking. + * + * LLMs may enrich source metadata later, but they are never the only ranking + * signal. Every decision records the policy version and an explainable reason. + */ + +export const RETRIEVAL_RANKING_POLICY_VERSION = 'folio-retrieval-v1'; + +export type SourceClass = + | 'regulator' + | 'exchange' + | 'issuer' + | 'established_news' + | 'aggregator' + | 'other'; + +export type DuplicateRelation = + | 'same_url' + | 'same_title' + | 'same_content' + | 'near_duplicate'; + +export interface RetrievalSourceInput { + id: string; + title: string; + url: string; + excerpt?: string; + publishedAt?: number; + provider?: string; + canonicalUrl?: string; + available?: boolean; +} + +export interface RankedSource { + source: RetrievalSourceInput; + originalRank: number; + canonicalUrl: string; + domain: string; + normalizedTitle: string; + contentHash?: string; + sourceClass: SourceClass; + qualityScore: number; + clusterId: string; + selected: boolean; + decisionReason: string; +} + +export interface SourceCluster { + id: string; + representativeId: string; + memberIds: string[]; + duplicateRelations: Array<{ + sourceId: string; + relation: DuplicateRelation; + }>; + sourceClass: SourceClass; + qualityScore: number; +} + +export interface RetrievalRankingResult { + policyVersion: string; + selected: RankedSource[]; + dropped: RankedSource[]; + clusters: SourceCluster[]; + independentSourceCount: number; +} + +export interface RetrievalRankingOptions { + query?: string; + now?: number; + topK?: number; + maxPerDomain?: number; + issuerDomains?: string[]; + nearDuplicateThreshold?: number; +} + +const TRACKING_PARAMS = new Set([ + 'fbclid', + 'gclid', + 'mc_cid', + 'mc_eid', + 'ref', + 'ref_src', + 'source', +]); + +const REGULATOR_DOMAINS = [ + 'sec.gov', + 'fca.org.uk', + 'sfc.hk', + 'mas.gov.sg', + 'csrc.gov.cn', +]; + +const EXCHANGE_DOMAINS = [ + 'hkex.com.hk', + 'hkexnews.hk', + 'nasdaq.com', + 'nyse.com', + 'sgx.com', + 'sse.com.cn', + 'szse.cn', +]; + +const ESTABLISHED_NEWS_DOMAINS = [ + 'reuters.com', + 'bloomberg.com', + 'ft.com', + 'wsj.com', + 'apnews.com', + 'cnbc.com', +]; + +const AGGREGATOR_DOMAINS = [ + 'finance.yahoo.com', + 'news.google.com', + 'msn.com', + 'marketscreener.com', +]; + +const CLASS_SCORE: Record = { + regulator: 50, + exchange: 48, + issuer: 45, + established_news: 35, + aggregator: 15, + other: 20, +}; + +function isDomain(domain: string, candidate: string): boolean { + return domain === candidate || domain.endsWith('.' + candidate); +} + +export function canonicalizeSourceUrl(rawUrl: string): string { + try { + const url = new URL(rawUrl); + url.hash = ''; + url.hostname = url.hostname.toLowerCase().replace(/^www\./, ''); + for (const key of [...url.searchParams.keys()]) { + const lower = key.toLowerCase(); + if (lower.startsWith('utm_') || TRACKING_PARAMS.has(lower)) { + url.searchParams.delete(key); + } + } + const sorted = [...url.searchParams.entries()].sort(([left], [right]) => + left.localeCompare(right) + ); + url.search = ''; + for (const [key, value] of sorted) url.searchParams.append(key, value); + if (url.pathname !== '/') url.pathname = url.pathname.replace(/\/+$/, ''); + return url.toString(); + } catch { + return rawUrl.trim(); + } +} + +export function normalizeSourceTitle(title: string): string { + return title + .normalize('NFKC') + .toLowerCase() + .replace(/[\p{P}\p{S}]+/gu, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function tokens(text: string): Set { + return new Set( + normalizeSourceTitle(text) + .split(' ') + .filter((token) => token.length > 1) + ); +} + +function similarity(left: string, right: string): number { + const a = tokens(left); + const b = tokens(right); + if (a.size === 0 || b.size === 0) return 0; + let intersection = 0; + for (const token of a) if (b.has(token)) intersection += 1; + return intersection / (a.size + b.size - intersection); +} + +function stableHash(value: string): string { + let hash = 0x811c9dc5; + for (let index = 0; index < value.length; index += 1) { + hash ^= value.charCodeAt(index); + hash = Math.imul(hash, 0x01000193); + } + return (hash >>> 0).toString(16).padStart(8, '0'); +} + +function domainOf(url: string): string { + try { + return new URL(url).hostname.toLowerCase().replace(/^www\./, ''); + } catch { + return ''; + } +} + +function classifySource( + domain: string, + issuerDomains: readonly string[] +): SourceClass { + if (REGULATOR_DOMAINS.some((candidate) => isDomain(domain, candidate))) { + return 'regulator'; + } + if (EXCHANGE_DOMAINS.some((candidate) => isDomain(domain, candidate))) { + return 'exchange'; + } + if (issuerDomains.some((candidate) => isDomain(domain, candidate.toLowerCase()))) { + return 'issuer'; + } + if (ESTABLISHED_NEWS_DOMAINS.some((candidate) => isDomain(domain, candidate))) { + return 'established_news'; + } + if (AGGREGATOR_DOMAINS.some((candidate) => isDomain(domain, candidate))) { + return 'aggregator'; + } + return 'other'; +} + +function qualityScore( + source: RetrievalSourceInput, + sourceClass: SourceClass, + query: string, + now: number +): number { + let score = CLASS_SCORE[sourceClass]; + if (source.available === false) score -= 30; + if (source.publishedAt !== undefined && Number.isFinite(source.publishedAt)) { + const ageDays = Math.max(0, now - source.publishedAt) / 86_400_000; + if (ageDays <= 1) score += 15; + else if (ageDays <= 7) score += 10; + else if (ageDays <= 30) score += 5; + } + const queryTokens = tokens(query); + const sourceTokens = tokens(source.title + ' ' + (source.excerpt ?? '')); + let matches = 0; + for (const token of queryTokens) if (sourceTokens.has(token)) matches += 1; + score += Math.min(15, matches * 5); + return score; +} + +function duplicateRelation( + left: RankedSource, + right: RankedSource, + threshold: number +): DuplicateRelation | undefined { + if (left.canonicalUrl === right.canonicalUrl) return 'same_url'; + if ( + left.normalizedTitle.length > 0 && + left.normalizedTitle === right.normalizedTitle && + isWithinSyndicationWindow(left.source.publishedAt, right.source.publishedAt) + ) { + return 'same_title'; + } + if ( + left.contentHash !== undefined && + left.contentHash === right.contentHash + ) { + return 'same_content'; + } + const leftText = left.source.excerpt || left.source.title; + const rightText = right.source.excerpt || right.source.title; + if ( + similarity(leftText, rightText) >= threshold && + isWithinSyndicationWindow(left.source.publishedAt, right.source.publishedAt) + ) { + return 'near_duplicate'; + } + return undefined; +} + +function isWithinSyndicationWindow( + left: number | undefined, + right: number | undefined +): boolean { + if ( + left === undefined || + right === undefined || + !Number.isFinite(left) || + !Number.isFinite(right) + ) { + return true; + } + return Math.abs(left - right) <= 7 * 86_400_000; +} + +function makeRanked( + source: RetrievalSourceInput, + originalRank: number, + options: Required> & + Pick +): RankedSource { + const canonicalUrl = canonicalizeSourceUrl(source.canonicalUrl ?? source.url); + const domain = domainOf(canonicalUrl); + const normalizedTitle = normalizeSourceTitle(source.title); + const content = source.excerpt?.trim(); + const sourceClass = classifySource(domain, options.issuerDomains ?? []); + return { + source, + originalRank, + canonicalUrl, + domain, + normalizedTitle, + contentHash: content ? stableHash(normalizeSourceTitle(content)) : undefined, + sourceClass, + qualityScore: qualityScore(source, sourceClass, options.query, options.now), + clusterId: '', + selected: false, + decisionReason: '', + }; +} + +/** + * Cluster duplicate sources, rank one representative per independent source, + * then apply a deterministic per-domain diversity limit. + */ +export function rankRetrievalSources( + sources: readonly RetrievalSourceInput[], + options: RetrievalRankingOptions = {} +): RetrievalRankingResult { + const query = options.query ?? ''; + const now = options.now ?? Date.now(); + const topK = Math.max(1, Math.floor(options.topK ?? 10)); + const maxPerDomain = Math.max(1, Math.floor(options.maxPerDomain ?? 2)); + const threshold = options.nearDuplicateThreshold ?? 0.82; + const ranked = sources.map((source, index) => + makeRanked(source, index, { + query, + now, + issuerDomains: options.issuerDomains, + }) + ); + + const groups: Array<{ + members: RankedSource[]; + relations: Array<{ sourceId: string; relation: DuplicateRelation }>; + }> = []; + for (const candidate of ranked) { + let match: + | { + group: (typeof groups)[number]; + relation: DuplicateRelation; + } + | undefined; + for (const group of groups) { + for (const member of group.members) { + const relation = duplicateRelation(candidate, member, threshold); + if (relation) { + match = { group, relation }; + break; + } + } + if (match) break; + } + if (match) { + match.group.members.push(candidate); + match.group.relations.push({ + sourceId: candidate.source.id, + relation: match.relation, + }); + } else { + groups.push({ members: [candidate], relations: [] }); + } + } + + const clusters: SourceCluster[] = []; + const representatives: RankedSource[] = []; + for (const group of groups) { + group.members.sort( + (left, right) => + right.qualityScore - left.qualityScore || + left.originalRank - right.originalRank + ); + const representative = group.members[0]; + const clusterId = + 'cluster-' + + stableHash( + representative.canonicalUrl || + representative.normalizedTitle || + representative.source.id + ); + for (const member of group.members) member.clusterId = clusterId; + for (const duplicate of group.members.slice(1)) { + duplicate.decisionReason = 'duplicate_of:' + representative.source.id; + } + representatives.push(representative); + clusters.push({ + id: clusterId, + representativeId: representative.source.id, + memberIds: group.members.map((member) => member.source.id), + duplicateRelations: group.relations, + sourceClass: representative.sourceClass, + qualityScore: representative.qualityScore, + }); + } + + representatives.sort( + (left, right) => + right.qualityScore - left.qualityScore || + left.originalRank - right.originalRank + ); + + const domainCounts = new Map(); + const selected: RankedSource[] = []; + for (const candidate of representatives) { + const domainCount = domainCounts.get(candidate.domain) ?? 0; + if (selected.length >= topK) { + candidate.decisionReason = 'top_k_limit'; + continue; + } + if (candidate.domain && domainCount >= maxPerDomain) { + candidate.decisionReason = 'domain_diversity_limit'; + continue; + } + candidate.selected = true; + candidate.decisionReason = 'selected_by_quality_and_diversity'; + selected.push(candidate); + if (candidate.domain) domainCounts.set(candidate.domain, domainCount + 1); + } + + const dropped = ranked + .filter((source) => !source.selected) + .sort((left, right) => left.originalRank - right.originalRank); + + return { + policyVersion: RETRIEVAL_RANKING_POLICY_VERSION, + selected, + dropped, + clusters, + independentSourceCount: clusters.length, + }; +} diff --git a/scripts/eval/retrieval-ranking-smoke.ts b/scripts/eval/retrieval-ranking-smoke.ts new file mode 100644 index 0000000..68e6966 --- /dev/null +++ b/scripts/eval/retrieval-ranking-smoke.ts @@ -0,0 +1,88 @@ +import { + rankRetrievalSources, + type RetrievalSourceInput, +} from '../../packages/shared/src/retrieval/index.ts'; + +interface AlgoliaHit { + objectID?: unknown; + url?: unknown; + title?: unknown; + created_at_i?: unknown; +} + +async function main(): Promise { + const query = process.argv.slice(2).join(' ').trim() || 'NVIDIA earnings'; + const endpoint = new URL('https://hn.algolia.com/api/v1/search'); + endpoint.searchParams.set('query', query); + endpoint.searchParams.set('tags', 'story'); + endpoint.searchParams.set('hitsPerPage', '50'); + + const response = await fetch(endpoint, { + headers: { 'user-agent': 'Folio retrieval-ranking smoke test' }, + }); + if (!response.ok) { + throw new Error( + 'Algolia search failed with HTTP ' + response.status + ' ' + response.statusText + ); + } + const payload = (await response.json()) as { hits?: AlgoliaHit[] }; + const hits = Array.isArray(payload.hits) ? payload.hits : []; + const sources: RetrievalSourceInput[] = []; + for (let index = 0; index < hits.length; index += 1) { + const hit = hits[index]; + if (typeof hit.url !== 'string' || typeof hit.title !== 'string') continue; + sources.push({ + id: + typeof hit.objectID === 'string' + ? 'hn-' + hit.objectID + : 'hn-' + index, + title: hit.title.trim(), + url: hit.url, + publishedAt: + typeof hit.created_at_i === 'number' + ? hit.created_at_i * 1000 + : undefined, + provider: 'hn-algolia', + available: true, + }); + } + + const searchedAt = Date.now(); + const result = rankRetrievalSources(sources, { + query, + now: searchedAt, + topK: 10, + maxPerDomain: 2, + }); + const evidence = { + query, + endpoint: endpoint.toString(), + searchedAt, + rawCount: sources.length, + policyVersion: result.policyVersion, + independentSourceCount: result.independentSourceCount, + clusterCount: result.clusters.length, + selectedCount: result.selected.length, + selected: result.selected.map((item) => ({ + id: item.source.id, + title: item.source.title, + url: item.canonicalUrl, + domain: item.domain, + sourceClass: item.sourceClass, + qualityScore: item.qualityScore, + clusterId: item.clusterId, + reason: item.decisionReason, + })), + dropped: result.dropped.map((item) => ({ + id: item.source.id, + clusterId: item.clusterId, + reason: item.decisionReason, + })), + }; + console.log(JSON.stringify(evidence, null, 2)); + if (sources.length === 0 || result.selected.length === 0) { + process.exitCode = 1; + } +} + +await main();