TSA-297: Return contextWords in the verify response

apps/backend/src/modules/documents/document.repository.ts

@@ -164,12 +164,15 @@ class DocumentRepository {
 		return documents.map((document) => DocumentEntity.initialize(document));
 	}
 
-	public async findLexiconByIds(ids: number[]): Promise<LexiconRow[]> {
+	public async findLexiconByIds(
+		ids: number[],
+		trx?: Transaction,
+	): Promise<LexiconRow[]> {
 		if (ids.length === EMPTY_COLLECTION_LENGTH) {
 			return [];
 		}
 
-		return await LexiconEntryModel.query()
+		return await LexiconEntryModel.query(trx)
 			.select("id", "valueDisplay", "distinctPages")
 			.whereIn("id", ids)
 			.castTo<LexiconRow[]>();

apps/backend/src/modules/documents/document.service.ts

@@ -4,7 +4,6 @@ import {
 	type DocumentCreateResponseDto,
 	type DocumentGetByIdBudgetResponseDto,
 	type DocumentGetLexiconResponseDto,
-	type DocumentGetPagesContextWordResponseDto,
 	type DocumentGetPagesResponseDto,
 	DocumentValidationMessage,
 	EMPTY_LENGTH,
@@ -23,6 +22,11 @@ import { type PageTranscribeQueue } from "~/libs/modules/queue/page-transcribe-q
 import { type BaseStorage } from "~/libs/modules/storage/base-storage.module.js";
 import { StorageBucket } from "~/libs/modules/storage/storage.js";
 import { type PageWithTranscriptionRow } from "~/modules/pages/libs/types/types.js";
+import {
+	buildContextWords,
+	buildPageLexiconMap,
+	extractLexiconIds,
+} from "~/modules/transcription/libs/helpers/helpers.js";
 
 import { PageEntity } from "../pages/page.entity.js";
 import { type PageRepository } from "../pages/page.repository.js";
@@ -34,7 +38,6 @@ import {
 	EMPTY_COLLECTION_LENGTH,
 	MAX_DOCUMENT_PAGES,
 	NON_DELETABLE_DOCUMENT_STATUSES,
-	NOT_FOUND_INDEX,
 	PAGES_TO_QUEUE,
 } from "./libs/constants/constants.js";
 import {
@@ -70,59 +73,6 @@ class DocumentService {
 		this.pageTranscribeQueue = pageTranscribeQueue;
 	}
 
-	private buildContextWords({
-		lexiconById,
-		text,
-	}: {
-		lexiconById: Map<number, { distinctPages: number; valueDisplay: string }>;
-		text: string;
-	}): DocumentGetPagesContextWordResponseDto[] {
-		const contextWords: DocumentGetPagesContextWordResponseDto[] = [];
-
-		for (const [lexiconId, lexicon] of lexiconById) {
-			const { valueDisplay } = lexicon;
-
-			if (valueDisplay.length === EMPTY_COLLECTION_LENGTH) {
-				continue;
-			}
-
-			let searchFrom = 0;
-
-			while (searchFrom <= text.length) {
-				const start = text.indexOf(valueDisplay, searchFrom);
-
-				if (start === NOT_FOUND_INDEX) {
-					break;
-				}
-
-				contextWords.push({
-					end: start + valueDisplay.length,
-					lexiconId,
-					seenOnPages: lexicon.distinctPages,
-					start,
-					word: valueDisplay,
-				});
-
-				searchFrom = start + valueDisplay.length;
-			}
-		}
-
-		return contextWords;
-	}
-
-	private buildPageLexiconMap(
-		contextUsed: null | Record<string, unknown>,
-		lexiconById: Map<number, { distinctPages: number; valueDisplay: string }>,
-	): Map<number, { distinctPages: number; valueDisplay: string }> {
-		return new Map(
-			this.extractLexiconIds(contextUsed).flatMap((id) => {
-				const lexicon = lexiconById.get(id);
-
-				return lexicon ? [[id, lexicon] as const] : [];
-			}),
-		);
-	}
-
 	private buildSourceKey(documentId: number): string {
 		return `uploads/${documentId.toString()}/original.pdf`;
 	}
@@ -131,7 +81,7 @@ class DocumentService {
 		const lexiconIds = new Set<number>();
 
 		for (const page of pages) {
-			for (const id of this.extractLexiconIds(page.transcriptionContextUsed)) {
+			for (const id of extractLexiconIds(page.transcriptionContextUsed)) {
 				lexiconIds.add(id);
 			}
 		}
@@ -230,18 +180,6 @@ class DocumentService {
 		);
 	}
 
-	private extractLexiconIds(
-		contextUsed: null | Record<string, unknown>,
-	): number[] {
-		const ids = contextUsed?.["lexiconIds"];
-
-		if (!Array.isArray(ids)) {
-			return [];
-		}
-
-		return ids.filter((id): id is number => typeof id === "number");
-	}
-
 	private async finalizeIngest(
 		documentId: number,
 		userId: number,
@@ -695,7 +633,7 @@ class DocumentService {
 
 				const text =
 					page.transcriptionEditedText ?? page.transcriptionText ?? "";
-				const pageLexiconById = this.buildPageLexiconMap(
+				const pageLexiconById = buildPageLexiconMap(
 					page.transcriptionContextUsed,
 					lexiconById,
 				);
@@ -712,7 +650,7 @@ class DocumentService {
 						page.transcriptionId === null
 							? null
 							: {
-									contextWords: this.buildContextWords({
+									contextWords: buildContextWords({
 										lexiconById: pageLexiconById,
 										text,
 									}),

apps/backend/src/modules/documents/libs/constants/constants.ts

@@ -5,5 +5,4 @@ export { EMPTY_COLLECTION_LENGTH } from "./empty-collection-length.constant.js";
 export { MAX_DOCUMENT_PAGES } from "./max-document-pages.constant.js";
 export { MIN_NUMBER_OF_FAILED_PAGES } from "./min-number-of-failed-pages.js";
 export { NON_DELETABLE_DOCUMENT_STATUSES } from "./non-deletable-document-statuses.constant.js";
-export { NOT_FOUND_INDEX } from "./not-found-index.constant.js";
 export { PAGES_TO_QUEUE } from "./pages-to-queue.constant.js";

apps/backend/src/modules/pages/page.service.ts

@@ -11,6 +11,11 @@ import { type Transaction, UniqueViolationError } from "objection";
 
 import { type Logger } from "~/libs/modules/logger/logger.js";
 import { type PageTranscribeQueue } from "~/libs/modules/queue/page-transcribe-queue.module.js";
+import {
+	buildContextWords,
+	buildPageLexiconMap,
+	extractLexiconIds,
+} from "~/modules/transcription/libs/helpers/helpers.js";
 
 import { DocumentModel } from "../documents/document.model.js";
 import { type DocumentRepository } from "../documents/document.repository.js";
@@ -91,6 +96,14 @@ class PageService {
 
 		const nextTranscription =
 			await this.transcriptionRepository.findCurrentByPageId(nextPage.id, trx);
+		const text = nextTranscription?.editedText ?? nextTranscription?.text ?? "";
+		const contextUsed = nextTranscription?.contextUsed ?? null;
+		const lexiconRows = await this.documentRepository.findLexiconByIds(
+			extractLexiconIds(contextUsed),
+			trx,
+		);
+		const lexiconById = new Map(lexiconRows.map((row) => [row.id, row]));
+		const pageLexiconById = buildPageLexiconMap(contextUsed, lexiconById);
 
 		return {
 			lexiconAdded: [],
@@ -100,8 +113,11 @@ class PageService {
 				status: nextPage.status,
 				transcription: nextTranscription
 					? {
-							contextWords: [],
-							text: nextTranscription.editedText ?? nextTranscription.text,
+							contextWords: buildContextWords({
+								lexiconById: pageLexiconById,
+								text,
+							}),
+							text,
 						}
 					: null,
 			},

apps/backend/src/modules/transcription/libs/constants/constants.ts

@@ -1,4 +1,5 @@
 export { AMAZON_ID_MARKER } from "./amazon-id-marker.constant.js";
 export { ANTHROPIC_DIRECT_PREFIX } from "./anthropic-direct-prefix.constant.js";
 export { MODEL_RATES } from "./model-rates.constant.js";
+export { NOT_FOUND_INDEX } from "./not-found-index.constant.js";
 export { SYSTEM_PROMPT } from "./system-prompt.constant.js";

apps/backend/src/modules/transcription/libs/helpers/build-context-words.helper.ts

@@ -0,0 +1,46 @@
+import {
+	type DocumentGetPagesContextWordResponseDto,
+	EMPTY_LENGTH,
+} from "@transcripta/shared";
+
+import { NOT_FOUND_INDEX } from "../constants/constants.js";
+import { type BuildContextWordsPayload } from "../types/types.js";
+
+const buildContextWords = ({
+	lexiconById,
+	text,
+}: BuildContextWordsPayload): DocumentGetPagesContextWordResponseDto[] => {
+	const contextWords: DocumentGetPagesContextWordResponseDto[] = [];
+
+	for (const [lexiconId, lexicon] of lexiconById) {
+		const { valueDisplay } = lexicon;
+
+		if (valueDisplay.length === EMPTY_LENGTH) {
+			continue;
+		}
+
+		let searchFrom = 0;
+
+		while (searchFrom <= text.length) {
+			const start = text.indexOf(valueDisplay, searchFrom);
+
+			if (start === NOT_FOUND_INDEX) {
+				break;
+			}
+
+			contextWords.push({
+				end: start + valueDisplay.length,
+				lexiconId,
+				seenOnPages: lexicon.distinctPages,
+				start,
+				word: valueDisplay,
+			});
+
+			searchFrom = start + valueDisplay.length;
+		}
+	}
+
+	return contextWords;
+};
+
+export { buildContextWords };

apps/backend/src/modules/transcription/libs/helpers/build-page-lexicon-map.helper.ts

@@ -0,0 +1,17 @@
+import { type ContextLexiconMap } from "../types/types.js";
+import { extractLexiconIds } from "./extract-lexicon-ids.helper.js";
+
+const buildPageLexiconMap = (
+	contextUsed: null | Record<string, unknown>,
+	lexiconById: ContextLexiconMap,
+): ContextLexiconMap => {
+	return new Map(
+		extractLexiconIds(contextUsed).flatMap((id) => {
+			const lexicon = lexiconById.get(id);
+
+			return lexicon ? [[id, lexicon] as const] : [];
+		}),
+	);
+};
+
+export { buildPageLexiconMap };

apps/backend/src/modules/transcription/libs/helpers/extract-lexicon-ids.helper.ts

@@ -0,0 +1,13 @@
+const extractLexiconIds = (
+	contextUsed: null | Record<string, unknown>,
+): number[] => {
+	const ids = contextUsed?.["lexiconIds"];
+
+	if (!Array.isArray(ids)) {
+		return [];
+	}
+
+	return ids.filter((id): id is number => typeof id === "number");
+};
+
+export { extractLexiconIds };

apps/backend/src/modules/transcription/libs/helpers/helpers.ts

@@ -1,3 +1,6 @@
+export { buildContextWords } from "./build-context-words.helper.js";
+export { buildPageLexiconMap } from "./build-page-lexicon-map.helper.js";
+export { extractLexiconIds } from "./extract-lexicon-ids.helper.js";
 export { createOutputValidator } from "./output-validator.helper.js";
 export { calculateTokenCost } from "./pricing.helper.js";
 export { resolveModelProvider } from "./resolve-model-provider.helper.js";

apps/backend/src/modules/transcription/libs/types/build-context-words-payload.type.ts

@@ -0,0 +1,8 @@
+import { type ContextLexiconMap } from "./context-lexicon-map.type.js";
+
+type BuildContextWordsPayload = {
+	lexiconById: ContextLexiconMap;
+	text: string;
+};
+
+export { type BuildContextWordsPayload };

apps/backend/src/modules/transcription/libs/types/context-lexicon-map.type.ts

@@ -0,0 +1,6 @@
+type ContextLexiconMap = Map<
+	number,
+	{ distinctPages: number; valueDisplay: string }
+>;
+
+export { type ContextLexiconMap };

apps/backend/src/modules/transcription/libs/types/types.ts

@@ -1,3 +1,5 @@
+export { type BuildContextWordsPayload } from "./build-context-words-payload.type.js";
+export { type ContextLexiconMap } from "./context-lexicon-map.type.js";
 export { type ModelProviderValue } from "./model-provider-value.type.js";
 export { type ModelRate } from "./model-rate.type.js";
 export { type TranscriptionDebugRow } from "./transcription-debug-row.type.js";