diff --git a/packages/ui/__tests__/chat/source-citations.test.tsx b/packages/ui/__tests__/chat/source-citations.test.tsx new file mode 100644 index 000000000..f9b990e26 --- /dev/null +++ b/packages/ui/__tests__/chat/source-citations.test.tsx @@ -0,0 +1,65 @@ +import { describe, it, expect } from "vitest"; +import { extractSources } from "../../src/chat/source-citations.js"; +import type { ChatUIToolCall } from "@repowise-dev/types/chat"; + +function searchCall(results: Array>): ChatUIToolCall { + return { + id: "tc1", + name: "search_codebase", + arguments: { query: "where is auth handled" }, + result: { results }, + status: "done", + }; +} + +describe("extractSources", () => { + it("reads the rank-normalized confidence, not the raw backend score", () => { + // BM25 fallback scores are unbounded — reading relevance_score here is what + // rendered "1808%" in the sources list. + const sources = extractSources( + [ + searchCall([ + { + page_id: "file_page:auth.py", + title: "auth.py", + relevance_score: 18.08, + confidence_score: 1, + }, + ]), + ], + "repo1", + ); + + expect(sources).toHaveLength(1); + expect(sources[0]?.confidence).toBe(1); + }); + + it("keeps every confidence renderable as a 0-100% badge", () => { + const sources = extractSources( + [ + searchCall([ + { page_id: "a", title: "a", relevance_score: 18.08, confidence_score: 1 }, + { page_id: "b", title: "b", relevance_score: 14.75, confidence_score: 0.82 }, + { page_id: "c", title: "c", relevance_score: 0.43, confidence_score: 0.02 }, + ]), + ], + "repo1", + ); + + expect(sources).toHaveLength(3); + for (const s of sources) { + expect(s.confidence).toBeGreaterThanOrEqual(0); + expect(s.confidence).toBeLessThanOrEqual(1); + } + }); + + it("omits confidence when the server sends no confidence_score", () => { + const sources = extractSources( + [searchCall([{ page_id: "file_page:a.py", title: "a.py", relevance_score: 9.1 }])], + "repo1", + ); + + // Badge hides rather than falling back to the unbounded raw score. + expect(sources[0]?.confidence).toBeUndefined(); + }); +}); diff --git a/packages/ui/src/chat/source-citations.tsx b/packages/ui/src/chat/source-citations.tsx index 56a5a72c0..fb6bf50c9 100644 --- a/packages/ui/src/chat/source-citations.tsx +++ b/packages/ui/src/chat/source-citations.tsx @@ -11,7 +11,12 @@ export interface SourceReference { title: string; pageType: string; targetPath: string; - score?: number; + /** + * Rank-normalized 0–1 confidence, never the raw backend score: embedding + * hits are cosine (0–1) but the BM25 fallback is unbounded, so the raw + * score has no fixed scale to render as a percentage. + */ + confidence?: number | undefined; toolName: string; } @@ -39,7 +44,7 @@ export function extractSources( title: (r.title as string) ?? pageId, pageType: (r.page_type as string) ?? "file_page", targetPath: (r.target_path as string) ?? "", - score: r.relevance_score as number ?? r.score as number, + confidence: r.confidence_score as number | undefined, toolName: tc.name, }); } @@ -176,9 +181,9 @@ export function SourceCitations({ {source.title} - {source.score != null && ( + {source.confidence != null && ( - {(source.score * 100).toFixed(0)}% + {(source.confidence * 100).toFixed(0)}% )}