feat(research): rank and deduplicate news sources

docs/retrieval-ranking.md

@@ -0,0 +1,42 @@
+# Retrieval ranking and deduplication
+
+Folio applies deterministic source normalization before news results enter
+Deep Research synthesis. The policy implementation is in
+`packages/shared/src/retrieval` and is versioned as `folio-retrieval-v1`.
+
+The pipeline:
+
+1. canonicalizes URLs, removes tracking parameters and fragments, and sorts
+   meaningful query parameters;
+2. normalizes titles and fingerprints excerpts;
+3. clusters same-URL, same-title, same-content, and near-duplicate results;
+4. scores cluster representatives using source class, freshness, query
+   relevance, and availability;
+5. selects Top-K with a per-domain diversity limit.
+
+Regulators, exchanges, and configured issuer domains receive primary-source
+priority. Established news outlets remain useful for independent reporting;
+aggregators receive a lower deterministic base score. Every selected or
+dropped result records its original rank, cluster id, decision reason, and the
+ranking policy version in the Research data bundle.
+
+## Live smoke run
+
+The live smoke command calls the public Hacker News Algolia search API, then
+runs the same ranking module used by `ResearchRunner`:
+
+```sh
+bun run eval:retrieval-ranking-live -- "NVIDIA earnings"
+```
+
+It prints machine-readable JSON containing the search endpoint, raw count,
+clusters, selected source set, and dropped reasons. The command intentionally
+fails when search returns no usable sources. This is an explicit live check,
+not part of ordinary unit tests or CI.
+
+## Boundaries
+
+Version 1 does not fetch article bodies or resolve network redirects. Providers
+may supply a resolved `canonicalUrl`; otherwise Folio uses deterministic URL,
+title, and excerpt signals. UI display of clusters and claim-level citation
+verification remain separate integration work.

package.json

@@ -19,6 +19,7 @@
     "eval:smoke": "bun scripts/eval/run.ts --smoke",
     "eval:full": "bun scripts/eval/run.ts",
     "eval:langfuse": "bun scripts/eval/langfuse-live.ts",
+    "eval:retrieval-ranking-live": "bun scripts/eval/retrieval-ranking-smoke.ts",
     "test:ui": "bun test packages/ui",
     "test:e2e": "cd apps/electron && bun run test:e2e",
     "test:e2e:visible": "cd apps/electron && FINAGENT_E2E_VISIBLE=1 bun run test:e2e",

packages/shared/package.json

@@ -14,6 +14,7 @@
     "./portfolio-risk": "./src/portfolio-risk/index.ts",
     "./portfolio-import": "./src/portfolio-import/index.ts",
     "./diagnostics": "./src/diagnostics/index.ts",
+    "./retrieval": "./src/retrieval/index.ts",
     "./providers/longbridge": "./src/providers/longbridge/index.ts",
     "./providers/massive": "./src/providers/massive/index.ts"
   },

packages/shared/src/index.ts

@@ -153,3 +153,4 @@ export * from './calibration/index.ts';
 export * from './pulse/index.ts';
 export * from './export/index.ts';
 export * from './evaluation/index.ts';
+export * from './retrieval/index.ts';

packages/shared/src/research/runner.test.ts

@@ -107,6 +107,83 @@ describe('ResearchRunner', () => {
     expect(result.report!.runStatus).toBe('failed');
   });
 
+  it('deduplicates and records ranked news before synthesis', async () => {
+    const news = fakeCap('research.news');
+    news.execute = async () => ({
+      data: [
+        {
+          id: 'wire',
+          title: 'NVIDIA announces quarterly results',
+          summary: 'NVIDIA reported quarterly revenue and guidance.',
+          url: 'https://reuters.com/technology/nvidia-results?utm_source=feed',
+          timestamp: 1_700_000_000,
+          symbols: ['NVDA.US'],
+        },
+        {
+          id: 'wire-copy',
+          title: 'NVIDIA announces quarterly results',
+          summary: 'NVIDIA reported quarterly revenue and guidance.',
+          url: 'https://copy.test/nvidia-results',
+          timestamp: 1_700_000_000,
+          symbols: ['NVDA.US'],
+        },
+        {
+          id: 'filing',
+          title: 'NVIDIA quarterly report',
+          summary: 'Official filing for NVIDIA quarterly results.',
+          url: 'https://sec.gov/Archives/edgar/data/1/report',
+          timestamp: 1_700_000_000,
+          symbols: ['NVDA.US'],
+        },
+      ],
+      provenance: {
+        provider: 'test',
+        fetchedAt: 1_700_000_000_000,
+        stale: false,
+      },
+    });
+    const registry = createCapabilityRegistry([news]);
+    const local = new LocalResearchSynthesizer();
+    let observed: Record<string, unknown> = {};
+    const runner = new ResearchRunner({
+      registry,
+      synthesizer: {
+        async synthesize(input, signal) {
+          observed = JSON.parse(input.dataBundle) as Record<string, unknown>;
+          return local.synthesize(input, signal);
+        },
+      },
+      now: () => 1_700_000_000_000,
+    });
+
+    await runner.run({ symbol: 'NVDA.US', runId: 'ranked-news' });
+
+    const selected = observed['research.news'] as Array<Record<string, unknown>>;
+    const metadata = observed['research.news:ranking'] as {
+      policyVersion: string;
+      independentSourceCount: number;
+      decisions: Array<{
+        sourceId: string;
+        originalRank: number;
+        clusterId: string;
+        selected: boolean;
+        reason: string;
+      }>;
+    };
+    expect(selected).toHaveLength(2);
+    expect(selected[0].id).toBe('filing');
+    expect(selected[0].sourceClass).toBe('regulator');
+    expect(metadata.policyVersion).toBe('folio-retrieval-v1');
+    expect(metadata.independentSourceCount).toBe(2);
+    expect(metadata.decisions).toContainEqual({
+      sourceId: 'wire-copy',
+      originalRank: 1,
+      clusterId: expect.any(String),
+      selected: false,
+      reason: 'duplicate_of:wire',
+    });
+  });
+
   it('cancels mid-fetch when the signal aborts', async () => {
     const runner = makeRunner([
       ['company.profile', 'success'],

packages/shared/src/research/runner.ts

@@ -20,6 +20,10 @@ import {
   planForStrategy,
   type PlannedCapability,
 } from './planner.ts';
+import {
+  rankRetrievalSources,
+  type RetrievalSourceInput,
+} from '../retrieval/index.ts';
 
 const CONCURRENCY = 4;
 const TIMEOUT_MS = 20000;
@@ -127,7 +131,7 @@ export class ResearchRunner {
     });
 
     const runs = buildRuns(plan, outcomes);
-    const dataBundle = buildDataBundle(outcomes);
+    const dataBundle = buildDataBundle(outcomes, symbol, this.now());
 
     let synthesis: ResearchSynthesis;
     try {
@@ -189,10 +193,49 @@ function buildRuns(plan: PlannedCapability[], outcomes: RunOutcome[]) {
   });
 }
 
-function buildDataBundle(outcomes: RunOutcome[]): string {
+function buildDataBundle(
+  outcomes: RunOutcome[],
+  query: string,
+  now: number
+): string {
   const bundle: Record<string, unknown> = {};
   for (const outcome of outcomes) {
     if (outcome.record.status === 'success' && outcome.result) {
+      if (
+        outcome.record.capabilityId === 'research.news' &&
+        Array.isArray(outcome.result.data)
+      ) {
+        const sources = toRetrievalSources(
+          outcome.result.data,
+          outcome.result.provenance.provider
+        );
+        const ranking = rankRetrievalSources(sources, {
+          query,
+          now,
+          topK: 10,
+          maxPerDomain: 2,
+        });
+        bundle[outcome.record.capabilityId] = ranking.selected.map((item) => ({
+          ...item.source,
+          canonicalUrl: item.canonicalUrl,
+          sourceClass: item.sourceClass,
+          clusterId: item.clusterId,
+          qualityScore: item.qualityScore,
+        }));
+        bundle['research.news:ranking'] = {
+          policyVersion: ranking.policyVersion,
+          independentSourceCount: ranking.independentSourceCount,
+          clusters: ranking.clusters,
+          decisions: [...ranking.selected, ...ranking.dropped].map((item) => ({
+            sourceId: item.source.id,
+            originalRank: item.originalRank,
+            clusterId: item.clusterId,
+            selected: item.selected,
+            reason: item.decisionReason,
+          })),
+        };
+        continue;
+      }
       bundle[outcome.record.capabilityId] = truncateData(
         outcome.record.capabilityId,
         outcome.result.data
@@ -202,14 +245,48 @@ function buildDataBundle(outcomes: RunOutcome[]): string {
   return JSON.stringify(bundle);
 }
 
+function toRetrievalSources(
+  data: unknown[],
+  provider: string
+): RetrievalSourceInput[] {
+  const sources: RetrievalSourceInput[] = [];
+  for (let index = 0; index < data.length; index += 1) {
+    const item = data[index];
+    if (item === null || typeof item !== 'object') continue;
+    const record = item as Record<string, unknown>;
+    if (typeof record.title !== 'string' || typeof record.url !== 'string') {
+      continue;
+    }
+    const rawTimestamp =
+      typeof record.timestamp === 'number' ? record.timestamp : undefined;
+    const publishedAt =
+      rawTimestamp === undefined
+        ? undefined
+        : rawTimestamp < 10_000_000_000
+          ? rawTimestamp * 1000
+          : rawTimestamp;
+    sources.push({
+      id:
+        typeof record.id === 'string' && record.id.length > 0
+          ? record.id
+          : provider + ':' + index,
+      title: record.title,
+      url: record.url,
+      excerpt:
+        typeof record.summary === 'string' ? record.summary : undefined,
+      publishedAt,
+      provider,
+      available: true,
+    });
+  }
+  return sources;
+}
+
 function truncateData(capabilityId: string, data: unknown): unknown {
   if (!Array.isArray(data)) return data;
   if (capabilityId === 'market.kline' || capabilityId === 'market.intraday') {
     return data.slice(-60);
   }
-  if (capabilityId === 'research.news') {
-    return data.slice(0, 10);
-  }
   return data;
 }
 

packages/shared/src/retrieval/index.ts

@@ -0,0 +1,13 @@
+export {
+  RETRIEVAL_RANKING_POLICY_VERSION,
+  canonicalizeSourceUrl,
+  normalizeSourceTitle,
+  rankRetrievalSources,
+  type DuplicateRelation,
+  type RankedSource,
+  type RetrievalRankingOptions,
+  type RetrievalRankingResult,
+  type RetrievalSourceInput,
+  type SourceClass,
+  type SourceCluster,
+} from './ranking.ts';