diff --git a/packages/shared/src/research/claim-verifier.test.ts b/packages/shared/src/research/claim-verifier.test.ts new file mode 100644 index 0000000..4598a13 --- /dev/null +++ b/packages/shared/src/research/claim-verifier.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from 'bun:test'; +import type { JudgeClient } from '../evaluation/judge-client.ts'; +import { + CLAIM_VERIFIER_VERSION, + createClaimVerifier, + type ClaimVerificationInput, +} from './claim-verifier.ts'; + +function input( + claimText: string, + evidence: ClaimVerificationInput['evidence'], +): ClaimVerificationInput { + return { claimId: 'claim-1', claimText, evidence }; +} + +function judgeReturning(reply: string): JudgeClient & { calls: Array<{ system: string; user: string }> } { + const calls: Array<{ system: string; user: string }> = []; + return { + provider: 'test', + model: 'test-judge', + calls, + async complete(system, user) { + calls.push({ system, user }); + return reply; + }, + }; +} + +describe('claim verifier', () => { + it('returns insufficient_evidence without calling the judge when evidence is empty', async () => { + const judge = judgeReturning('{"status":"supported","reason":"invented"}'); + const result = await createClaimVerifier(judge).verify(input('Revenue increased.', [])); + + expect(result).toEqual({ + claimId: 'claim-1', + status: 'insufficient_evidence', + evidenceIds: [], + reason: 'No source evidence was provided for this claim.', + verifierVersion: CLAIM_VERIFIER_VERSION, + }); + expect(judge.calls).toHaveLength(0); + }); + + it('returns supported when the supplied evidence directly supports the claim', async () => { + const judge = judgeReturning('{"status":"supported","reason":"The filing reports the same revenue increase."}'); + const result = await createClaimVerifier(judge).verify( + input('Revenue increased by 12%.', [{ id: 'e-1', content: 'The filing reports revenue increased by 12%.' }]), + ); + + expect(result.status).toBe('supported'); + expect(result.evidenceIds).toEqual(['e-1']); + }); + + it('returns contradicted when the supplied evidence directly contradicts the claim', async () => { + const judge = judgeReturning('{"status":"contradicted","reason":"The filing reports a decline, not an increase."}'); + const result = await createClaimVerifier(judge).verify( + input('Revenue increased by 12%.', [{ id: 'e-1', content: 'Revenue declined by 12%.' }]), + ); + + expect(result.status).toBe('contradicted'); + }); + + it('returns insufficient_evidence when evidence is related but does not establish the claim', async () => { + const judge = judgeReturning('{"status":"insufficient_evidence","reason":"The evidence discusses demand but gives no revenue result."}'); + const result = await createClaimVerifier(judge).verify( + input('Revenue increased by 12%.', [{ id: 'e-1', content: 'Customer demand remained strong.' }]), + ); + + expect(result.status).toBe('insufficient_evidence'); + }); + + it('does not accept a plausible root-cause claim without causal evidence', async () => { + const judge = judgeReturning('{"status":"insufficient_evidence","reason":"The evidence shows both events but no causal link."}'); + const result = await createClaimVerifier(judge).verify( + input('The share-price decline was caused by supply constraints.', [ + { id: 'e-1', content: 'The share price declined after the earnings release.' }, + { id: 'e-2', content: 'The company separately reported supply constraints.' }, + ]), + ); + + expect(result.status).toBe('insufficient_evidence'); + expect(result.evidenceIds).toEqual(['e-1', 'e-2']); + }); + + it('supports a claim using multiple supplied evidence items', async () => { + const judge = judgeReturning('{"status":"supported","reason":"The two filings jointly establish the claim."}'); + const result = await createClaimVerifier(judge).verify( + input('Revenue and operating margin both increased.', [ + { id: 'e-revenue', content: 'Revenue increased by 12%.' }, + { id: 'e-margin', content: 'Operating margin increased by 2 percentage points.' }, + ]), + ); + + expect(result).toEqual({ + claimId: 'claim-1', + status: 'supported', + evidenceIds: ['e-revenue', 'e-margin'], + reason: 'The two filings jointly establish the claim.', + verifierVersion: CLAIM_VERIFIER_VERSION, + }); + }); + + it('fails closed when the judge returns malformed JSON or an unknown status', async () => { + for (const reply of [ + 'not json', + '{"status":"probably_supported","reason":"maybe"}', + '{"status":"supported"}', + ]) { + const result = await createClaimVerifier(judgeReturning(reply)).verify( + input('Revenue increased.', [{ id: 'e-1', content: 'Revenue increased.' }]), + ); + + expect(result.status).toBe('insufficient_evidence'); + expect(result.reason).toStartWith('judge_error:'); + } + }); + + it('fails closed when judge transport fails', async () => { + const failingJudge: JudgeClient = { + provider: 'test', + model: 'test-judge', + async complete() { + throw new Error('network unavailable'); + }, + }; + + const result = await createClaimVerifier(failingJudge).verify( + input('Revenue increased.', [{ id: 'e-1', content: 'Revenue increased.' }]), + ); + + expect(result.status).toBe('insufficient_evidence'); + expect(result.reason).toBe('judge_error: network unavailable'); + }); + + it('instructs the judge to use only supplied evidence and never promotes reason to evidence', async () => { + const judge = judgeReturning('{"status":"supported","reason":"Explanation only."}'); + const result = await createClaimVerifier(judge).verify( + input('Revenue increased.', [{ id: 'source-evidence', content: 'Revenue increased.' }]), + ); + + expect(judge.calls[0].system).toContain('ONLY the supplied source evidence'); + expect(judge.calls[0].system).toContain('Do not use background knowledge'); + expect(result.evidenceIds).toEqual(['source-evidence']); + expect(result.evidenceIds).not.toContain('Explanation only.'); + }); +}); diff --git a/packages/shared/src/research/claim-verifier.ts b/packages/shared/src/research/claim-verifier.ts new file mode 100644 index 0000000..eb2afc7 --- /dev/null +++ b/packages/shared/src/research/claim-verifier.ts @@ -0,0 +1,129 @@ +import type { JudgeClient } from '../evaluation/judge-client.ts'; + +export const CLAIM_VERIFIER_VERSION = 'claim-verifier-v1'; + +export type ClaimVerificationStatus = + | 'supported' + | 'contradicted' + | 'insufficient_evidence'; + +export interface ClaimVerifierEvidence { + id: string; + content: string; +} + +/** Standalone Phase 1 input; integration adapters may map domain claims to it later. */ +export interface ClaimVerificationInput { + claimId: string; + claimText: string; + evidence: ClaimVerifierEvidence[]; +} + +export interface ClaimVerificationResult { + claimId: string; + status: ClaimVerificationStatus; + evidenceIds: string[]; + /** Explanation only. This value is never treated as source evidence. */ + reason: string; + verifierVersion: typeof CLAIM_VERIFIER_VERSION; +} + +export interface ClaimVerifier { + verify(input: ClaimVerificationInput, signal?: AbortSignal): Promise; +} + +interface JudgeResult { + status: ClaimVerificationStatus; + reason: string; +} + +const STATUSES = new Set([ + 'supported', + 'contradicted', + 'insufficient_evidence', +]); + +const SYSTEM_PROMPT = `You verify one claim against ONLY the supplied source evidence. +Do not use background knowledge, assumptions, or facts absent from the supplied evidence. +A plausible explanation is not evidence. A causal or root-cause claim requires evidence that establishes the causal link. +Return supported only when the supplied evidence directly supports the entire claim. +Return contradicted when the supplied evidence directly conflicts with the claim. +Otherwise return insufficient_evidence. +Reply with ONLY strict JSON in this exact shape: +{"status":"supported|contradicted|insufficient_evidence","reason":"brief explanation"}`; + +function userPrompt(input: ClaimVerificationInput): string { + const evidence = input.evidence + .map((item) => `[evidence id=${JSON.stringify(item.id)}]\n${item.content}`) + .join('\n\n'); + return [ + 'CLAIM', + input.claimText, + '', + 'SUPPLIED SOURCE EVIDENCE', + evidence, + ].join('\n'); +} + +function parseJudgeResult(reply: string): JudgeResult | undefined { + let value: unknown; + try { + value = JSON.parse(reply.trim()); + } catch { + return undefined; + } + if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined; + const record = value as Record; + if (typeof record.status !== 'string' || !STATUSES.has(record.status as ClaimVerificationStatus)) { + return undefined; + } + if (typeof record.reason !== 'string' || record.reason.trim() === '') return undefined; + return { + status: record.status as ClaimVerificationStatus, + reason: record.reason.trim(), + }; +} + +function result( + input: ClaimVerificationInput, + status: ClaimVerificationStatus, + reason: string, +): ClaimVerificationResult { + return { + claimId: input.claimId, + status, + evidenceIds: input.evidence.map((item) => item.id), + reason, + verifierVersion: CLAIM_VERIFIER_VERSION, + }; +} + +export function createClaimVerifier(client: JudgeClient): ClaimVerifier { + return { + async verify(input, signal) { + if (input.evidence.length === 0) { + return result( + input, + 'insufficient_evidence', + 'No source evidence was provided for this claim.', + ); + } + + try { + const reply = await client.complete(SYSTEM_PROMPT, userPrompt(input), signal); + const parsed = parseJudgeResult(reply); + if (parsed === undefined) { + return result( + input, + 'insufficient_evidence', + 'judge_error: malformed reply; expected strict JSON with a known status and non-empty reason', + ); + } + return result(input, parsed.status, parsed.reason); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + return result(input, 'insufficient_evidence', `judge_error: ${message}`); + } + }, + }; +}