From 815021e50151ad5c7442a810cb25965b703e8a71 Mon Sep 17 00:00:00 2001 From: israel Date: Thu, 3 Sep 2026 13:08:13 +0100 Subject: [PATCH] fix(platform): stop a search returning one passage twice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same text can sit in the corpus twice — one file uploaded as two documents, or a paragraph two documents share. Both copies match, and a bounded result set then spends two of its slots saying one thing while a different answer falls off the end. A caller asking for eight passages can receive seven distinct ones. Deduped on the text a caller actually reads, keyed with the corpus so the same wording found in a document and on a crawled page stays two answers. Whitespace is normalized, so two copies differing only in how their source wrapped lines count as one. Fusion has already sorted by score, so the first occurrence is the best one. The order is load-bearing: this runs AFTER the retrievability filter. Deduping first could keep a copy the caller cannot read and drop the readable one, and the gate would then remove what was kept — losing the passage entirely rather than showing it once. Closes #3061. Supersedes #3105, whose two files #3125 deleted. --- .../backend/core/knowledge/search.test.ts | 109 ++++++++++++++++++ .../platform/backend/core/knowledge/search.ts | 39 ++++++- 2 files changed, 146 insertions(+), 2 deletions(-) diff --git a/services/platform/backend/core/knowledge/search.test.ts b/services/platform/backend/core/knowledge/search.test.ts index 171bdcc127..4855ce5331 100644 --- a/services/platform/backend/core/knowledge/search.test.ts +++ b/services/platform/backend/core/knowledge/search.test.ts @@ -97,3 +97,112 @@ describe('searchKnowledge live document validation', () => { }); }); }); + +describe('searchKnowledge — the same passage twice', () => { + beforeEach(() => { + retrieveMock.mockReset(); + }); + + /** Two refs holding identical text — the same file indexed twice. */ + function duplicatePair(text: string) { + return [ + { ...hit('documents', 'copy_a'), text, fusedScore: 0.9 }, + { ...hit('documents', 'copy_b'), text, fusedScore: 0.4 }, + ]; + } + const ACCESS = { teamIds: [], projectIds: [], includeHub: true }; + + it('returns one copy, keeping the higher-scoring ref', async () => { + // A bounded result set spending two slots on one passage pushes a + // different answer off the end. + retrieveMock.mockResolvedValueOnce({ + hits: duplicatePair('Refunds within 30 days.'), + diagnostics: {}, + }); + const runQuery = vi.fn(async () => ['copy_a', 'copy_b']); + const result = await searchKnowledge({ runQuery } as never, { + organizationId: 'org_1', + orgSlug: 'acme', + query: 'refunds', + access: ACCESS, + }); + expect(result.hits.map((entry) => entry.source.ref)).toEqual(['copy_a']); + }); + + it('keeps a distinct passage from the same document', async () => { + // Deduping is per passage, not per document — a second chunk of the same + // file is a different answer. + retrieveMock.mockResolvedValueOnce({ + hits: [ + { ...hit('documents', 'doc'), text: 'First passage.' }, + { ...hit('documents', 'doc'), text: 'Second passage.' }, + ], + diagnostics: {}, + }); + const runQuery = vi.fn(async () => ['doc']); + const result = await searchKnowledge({ runQuery } as never, { + organizationId: 'org_1', + orgSlug: 'acme', + query: 'passages', + access: ACCESS, + }); + expect(result.hits.map((entry) => entry.text)).toEqual([ + 'First passage.', + 'Second passage.', + ]); + }); + + it('treats two copies that only wrap differently as one', async () => { + retrieveMock.mockResolvedValueOnce({ + hits: [ + { ...hit('documents', 'copy_a'), text: 'Refunds within\n30 days.' }, + { ...hit('documents', 'copy_b'), text: 'Refunds within 30 days.' }, + ], + diagnostics: {}, + }); + const runQuery = vi.fn(async () => ['copy_a', 'copy_b']); + const result = await searchKnowledge({ runQuery } as never, { + organizationId: 'org_1', + orgSlug: 'acme', + query: 'refunds', + access: ACCESS, + }); + expect(result.hits).toHaveLength(1); + }); + + it('keeps the readable copy when the other is filtered out', async () => { + // The order matters: deduping BEFORE the retrievability gate could keep + // an unreadable copy and drop the readable one, and the gate would then + // remove what was kept — losing the passage entirely. + retrieveMock.mockResolvedValueOnce({ + hits: duplicatePair('Refunds within 30 days.'), + diagnostics: {}, + }); + const runQuery = vi.fn(async () => ['copy_b']); + const result = await searchKnowledge({ runQuery } as never, { + organizationId: 'org_1', + orgSlug: 'acme', + query: 'refunds', + access: ACCESS, + }); + expect(result.hits.map((entry) => entry.source.ref)).toEqual(['copy_b']); + }); + + it('does not collapse the same text across different corpora', async () => { + retrieveMock.mockResolvedValueOnce({ + hits: [ + { ...hit('documents', 'doc'), text: 'Shared wording.' }, + { ...hit('web', 'https://example.com'), text: 'Shared wording.' }, + ], + diagnostics: {}, + }); + const runQuery = vi.fn(async () => ['doc']); + const result = await searchKnowledge({ runQuery } as never, { + organizationId: 'org_1', + orgSlug: 'acme', + query: 'shared', + access: ACCESS, + }); + expect(result.hits).toHaveLength(2); + }); +}); diff --git a/services/platform/backend/core/knowledge/search.ts b/services/platform/backend/core/knowledge/search.ts index ab5c71301c..36eac2e9df 100644 --- a/services/platform/backend/core/knowledge/search.ts +++ b/services/platform/backend/core/knowledge/search.ts @@ -51,6 +51,7 @@ import type { import { PRIVATE_KNOWLEDGE_SCHEMA, corporaFor, + type KnowledgeHit, type KnowledgeQuery, type KnowledgeResult, } from '../../../lib/knowledge/types'; @@ -147,12 +148,46 @@ export async function searchKnowledge( const allowed = new Set(retrievable); return { ...result, - hits: result.hits.filter( - (hit) => hit.corpus !== 'documents' || allowed.has(hit.source.ref), + hits: dropRepeatedPassages( + result.hits.filter( + (hit) => hit.corpus !== 'documents' || allowed.has(hit.source.ref), + ), ), }; } +/** + * Drop a passage the caller has already been given, keeping the best-scoring + * one. + * + * The same text can sit in the corpus twice — the same file uploaded as two + * documents, or a paragraph two documents share. Both copies match, and a + * bounded result set then spends two of its slots saying one thing while a + * different answer falls off the end. + * + * Runs AFTER the retrievability filter, deliberately. Deduping first could + * keep a copy the caller cannot read and drop the readable one, and the gate + * would then remove what was kept — losing the passage entirely rather than + * showing it once. Fusion has already sorted by score, so the first + * occurrence is the best one. + */ +function dropRepeatedPassages( + hits: readonly Hit[], +): Hit[] { + const seen = new Set(); + const kept: Hit[] = []; + for (const hit of hits) { + // Keyed on the text a caller actually reads. Whitespace is normalized so + // two copies that differ only in how their source wrapped lines still + // count as one. + const key = `${hit.corpus}\u0000${hit.text.replace(/\s+/g, ' ').trim()}`; + if (seen.has(key)) continue; + seen.add(key); + kept.push(hit); + } + return kept; +} + /** * A `knowledge.search` backend bound to one organization. *