From 8e86a14a4b6b723d4a8a4b6f26af9398766eeb2e Mon Sep 17 00:00:00 2001 From: Dekshad Date: Thu, 17 Sep 2026 19:15:37 +0300 Subject: [PATCH 1/5] TSA-297: * extract shared context word helpers --- .../src/modules/documents/document.service.ts | 78 ++----------------- .../documents/libs/constants/constants.ts | 1 - .../transcription/libs/constants/constants.ts | 1 + .../constants/not-found-index.constant.ts | 0 .../helpers/build-context-words.helper.ts | 48 ++++++++++++ .../helpers/build-page-lexicon-map.helper.ts | 16 ++++ .../helpers/extract-lexicon-ids.helper.ts | 13 ++++ .../transcription/libs/helpers/helpers.ts | 3 + 8 files changed, 89 insertions(+), 71 deletions(-) rename apps/backend/src/modules/{documents => transcription}/libs/constants/not-found-index.constant.ts (100%) create mode 100644 apps/backend/src/modules/transcription/libs/helpers/build-context-words.helper.ts create mode 100644 apps/backend/src/modules/transcription/libs/helpers/build-page-lexicon-map.helper.ts create mode 100644 apps/backend/src/modules/transcription/libs/helpers/extract-lexicon-ids.helper.ts diff --git a/apps/backend/src/modules/documents/document.service.ts b/apps/backend/src/modules/documents/document.service.ts index 5147c714..5324f06a 100644 --- a/apps/backend/src/modules/documents/document.service.ts +++ b/apps/backend/src/modules/documents/document.service.ts @@ -3,7 +3,6 @@ import { type DocumentCreateRequestDto, type DocumentCreateResponseDto, type DocumentGetByIdBudgetResponseDto, - type DocumentGetPagesContextWordResponseDto, type DocumentGetPagesResponseDto, DocumentValidationMessage, EMPTY_LENGTH, @@ -22,6 +21,11 @@ import { type PageTranscribeQueue } from "~/libs/modules/queue/page-transcribe-q import { type BaseStorage } from "~/libs/modules/storage/base-storage.module.js"; import { StorageBucket } from "~/libs/modules/storage/storage.js"; import { type PageWithTranscriptionRow } from "~/modules/pages/libs/types/types.js"; +import { + buildContextWords, + buildPageLexiconMap, + extractLexiconIds, +} from "~/modules/transcription/libs/helpers/helpers.js"; import { PageEntity } from "../pages/page.entity.js"; import { type PageRepository } from "../pages/page.repository.js"; @@ -33,7 +37,6 @@ import { EMPTY_COLLECTION_LENGTH, MAX_DOCUMENT_PAGES, NON_DELETABLE_DOCUMENT_STATUSES, - NOT_FOUND_INDEX, PAGES_TO_QUEUE, } from "./libs/constants/constants.js"; import { @@ -69,59 +72,6 @@ class DocumentService { this.pageTranscribeQueue = pageTranscribeQueue; } - private buildContextWords({ - lexiconById, - text, - }: { - lexiconById: Map; - text: string; - }): DocumentGetPagesContextWordResponseDto[] { - const contextWords: DocumentGetPagesContextWordResponseDto[] = []; - - for (const [lexiconId, lexicon] of lexiconById) { - const { valueDisplay } = lexicon; - - if (valueDisplay.length === EMPTY_COLLECTION_LENGTH) { - continue; - } - - let searchFrom = 0; - - while (searchFrom <= text.length) { - const start = text.indexOf(valueDisplay, searchFrom); - - if (start === NOT_FOUND_INDEX) { - break; - } - - contextWords.push({ - end: start + valueDisplay.length, - lexiconId, - seenOnPages: lexicon.distinctPages, - start, - word: valueDisplay, - }); - - searchFrom = start + valueDisplay.length; - } - } - - return contextWords; - } - - private buildPageLexiconMap( - contextUsed: null | Record, - lexiconById: Map, - ): Map { - return new Map( - this.extractLexiconIds(contextUsed).flatMap((id) => { - const lexicon = lexiconById.get(id); - - return lexicon ? [[id, lexicon] as const] : []; - }), - ); - } - private buildSourceKey(documentId: number): string { return `uploads/${documentId.toString()}/original.pdf`; } @@ -130,7 +80,7 @@ class DocumentService { const lexiconIds = new Set(); for (const page of pages) { - for (const id of this.extractLexiconIds(page.transcriptionContextUsed)) { + for (const id of extractLexiconIds(page.transcriptionContextUsed)) { lexiconIds.add(id); } } @@ -229,18 +179,6 @@ class DocumentService { ); } - private extractLexiconIds( - contextUsed: null | Record, - ): number[] { - const ids = contextUsed?.["lexiconIds"]; - - if (!Array.isArray(ids)) { - return []; - } - - return ids.filter((id): id is number => typeof id === "number"); - } - private async finalizeIngest( documentId: number, userId: number, @@ -669,7 +607,7 @@ class DocumentService { ]); const text = page.transcriptionText ?? ""; - const pageLexiconById = this.buildPageLexiconMap( + const pageLexiconById = buildPageLexiconMap( page.transcriptionContextUsed, lexiconById, ); @@ -686,7 +624,7 @@ class DocumentService { page.transcriptionId === null ? null : { - contextWords: this.buildContextWords({ + contextWords: buildContextWords({ lexiconById: pageLexiconById, text, }), diff --git a/apps/backend/src/modules/documents/libs/constants/constants.ts b/apps/backend/src/modules/documents/libs/constants/constants.ts index 4b588bba..28b9600e 100644 --- a/apps/backend/src/modules/documents/libs/constants/constants.ts +++ b/apps/backend/src/modules/documents/libs/constants/constants.ts @@ -4,5 +4,4 @@ export { DOCUMENT_OWNER_ID_FOREIGN } from "./document-owner-id-foreign.constant. export { EMPTY_COLLECTION_LENGTH } from "./empty-collection-length.constant.js"; export { MAX_DOCUMENT_PAGES } from "./max-document-pages.constant.js"; export { NON_DELETABLE_DOCUMENT_STATUSES } from "./non-deletable-document-statuses.constant.js"; -export { NOT_FOUND_INDEX } from "./not-found-index.constant.js"; export { PAGES_TO_QUEUE } from "./pages-to-queue.constant.js"; diff --git a/apps/backend/src/modules/transcription/libs/constants/constants.ts b/apps/backend/src/modules/transcription/libs/constants/constants.ts index 84e925f7..8be2e7d0 100644 --- a/apps/backend/src/modules/transcription/libs/constants/constants.ts +++ b/apps/backend/src/modules/transcription/libs/constants/constants.ts @@ -1,3 +1,4 @@ export { AMAZON_ID_MARKER } from "./amazon-id-marker.constant.js"; export { ANTHROPIC_DIRECT_PREFIX } from "./anthropic-direct-prefix.constant.js"; export { MODEL_RATES } from "./model-rates.constant.js"; +export { NOT_FOUND_INDEX } from "./not-found-index.constant.js"; diff --git a/apps/backend/src/modules/documents/libs/constants/not-found-index.constant.ts b/apps/backend/src/modules/transcription/libs/constants/not-found-index.constant.ts similarity index 100% rename from apps/backend/src/modules/documents/libs/constants/not-found-index.constant.ts rename to apps/backend/src/modules/transcription/libs/constants/not-found-index.constant.ts diff --git a/apps/backend/src/modules/transcription/libs/helpers/build-context-words.helper.ts b/apps/backend/src/modules/transcription/libs/helpers/build-context-words.helper.ts new file mode 100644 index 00000000..bc37cf35 --- /dev/null +++ b/apps/backend/src/modules/transcription/libs/helpers/build-context-words.helper.ts @@ -0,0 +1,48 @@ +import { + type DocumentGetPagesContextWordResponseDto, + EMPTY_LENGTH, +} from "@transcripta/shared"; + +import { NOT_FOUND_INDEX } from "../constants/constants.js"; + +const buildContextWords = ({ + lexiconById, + text, +}: { + lexiconById: Map; + text: string; +}): DocumentGetPagesContextWordResponseDto[] => { + const contextWords: DocumentGetPagesContextWordResponseDto[] = []; + + for (const [lexiconId, lexicon] of lexiconById) { + const { valueDisplay } = lexicon; + + if (valueDisplay.length === EMPTY_LENGTH) { + continue; + } + + let searchFrom = 0; + + while (searchFrom <= text.length) { + const start = text.indexOf(valueDisplay, searchFrom); + + if (start === NOT_FOUND_INDEX) { + break; + } + + contextWords.push({ + end: start + valueDisplay.length, + lexiconId, + seenOnPages: lexicon.distinctPages, + start, + word: valueDisplay, + }); + + searchFrom = start + valueDisplay.length; + } + } + + return contextWords; +}; + +export { buildContextWords }; diff --git a/apps/backend/src/modules/transcription/libs/helpers/build-page-lexicon-map.helper.ts b/apps/backend/src/modules/transcription/libs/helpers/build-page-lexicon-map.helper.ts new file mode 100644 index 00000000..e4881623 --- /dev/null +++ b/apps/backend/src/modules/transcription/libs/helpers/build-page-lexicon-map.helper.ts @@ -0,0 +1,16 @@ +import { extractLexiconIds } from "./extract-lexicon-ids.helper.js"; + +const buildPageLexiconMap = ( + contextUsed: null | Record, + lexiconById: Map, +): Map => { + return new Map( + extractLexiconIds(contextUsed).flatMap((id) => { + const lexicon = lexiconById.get(id); + + return lexicon ? [[id, lexicon] as const] : []; + }), + ); +}; + +export { buildPageLexiconMap }; diff --git a/apps/backend/src/modules/transcription/libs/helpers/extract-lexicon-ids.helper.ts b/apps/backend/src/modules/transcription/libs/helpers/extract-lexicon-ids.helper.ts new file mode 100644 index 00000000..99477e7e --- /dev/null +++ b/apps/backend/src/modules/transcription/libs/helpers/extract-lexicon-ids.helper.ts @@ -0,0 +1,13 @@ +const extractLexiconIds = ( + contextUsed: null | Record, +): number[] => { + const ids = contextUsed?.["lexiconIds"]; + + if (!Array.isArray(ids)) { + return []; + } + + return ids.filter((id): id is number => typeof id === "number"); +}; + +export { extractLexiconIds }; diff --git a/apps/backend/src/modules/transcription/libs/helpers/helpers.ts b/apps/backend/src/modules/transcription/libs/helpers/helpers.ts index 1f75a0a6..386690c1 100644 --- a/apps/backend/src/modules/transcription/libs/helpers/helpers.ts +++ b/apps/backend/src/modules/transcription/libs/helpers/helpers.ts @@ -1,3 +1,6 @@ +export { buildContextWords } from "./build-context-words.helper.js"; +export { buildPageLexiconMap } from "./build-page-lexicon-map.helper.js"; +export { extractLexiconIds } from "./extract-lexicon-ids.helper.js"; export { createOutputValidator } from "./output-validator.helper.js"; export { calculateTokenCost } from "./pricing.helper.js"; export { resolveModelProvider } from "./resolve-model-provider.helper.js"; From 69609f553f70514f11231596d2537f376ecbdd76 Mon Sep 17 00:00:00 2001 From: Dekshad Date: Thu, 17 Sep 2026 19:21:09 +0300 Subject: [PATCH 2/5] TSA-297: + return context words for the next page on verify --- .../modules/documents/document.repository.ts | 7 +++++-- apps/backend/src/modules/pages/page.service.ts | 17 ++++++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/apps/backend/src/modules/documents/document.repository.ts b/apps/backend/src/modules/documents/document.repository.ts index 4bde3953..cffc8e20 100644 --- a/apps/backend/src/modules/documents/document.repository.ts +++ b/apps/backend/src/modules/documents/document.repository.ts @@ -153,12 +153,15 @@ class DocumentRepository { return documents.map((document) => DocumentEntity.initialize(document)); } - public async findLexiconByIds(ids: number[]): Promise { + public async findLexiconByIds( + ids: number[], + trx?: Transaction, + ): Promise { if (ids.length === EMPTY_COLLECTION_LENGTH) { return []; } - return await LexiconEntryModel.query() + return await LexiconEntryModel.query(trx) .select("id", "valueDisplay", "distinctPages") .whereIn("id", ids) .castTo(); diff --git a/apps/backend/src/modules/pages/page.service.ts b/apps/backend/src/modules/pages/page.service.ts index 9311b207..8eb8b79d 100644 --- a/apps/backend/src/modules/pages/page.service.ts +++ b/apps/backend/src/modules/pages/page.service.ts @@ -11,6 +11,11 @@ import { type Transaction, UniqueViolationError } from "objection"; import { type Logger } from "~/libs/modules/logger/logger.js"; import { type PageTranscribeQueue } from "~/libs/modules/queue/page-transcribe-queue.module.js"; import { TRANSCRIBABLE_STATUSES } from "~/modules/jobs/libs/constants/constants.js"; +import { + buildContextWords, + buildPageLexiconMap, + extractLexiconIds, +} from "~/modules/transcription/libs/helpers/helpers.js"; import { DocumentModel } from "../documents/document.model.js"; import { type DocumentRepository } from "../documents/document.repository.js"; @@ -87,6 +92,13 @@ class PageService { const nextTranscription = await this.transcriptionRepository.findCurrentByPageId(nextPage.id, trx); + const contextUsed = nextTranscription?.contextUsed ?? null; + const lexiconRows = await this.documentRepository.findLexiconByIds( + extractLexiconIds(contextUsed), + trx, + ); + const lexiconById = new Map(lexiconRows.map((row) => [row.id, row])); + const pageLexiconById = buildPageLexiconMap(contextUsed, lexiconById); return { lexiconAdded: [], @@ -96,7 +108,10 @@ class PageService { status: nextPage.status, transcription: nextTranscription ? { - contextWords: [], + contextWords: buildContextWords({ + lexiconById: pageLexiconById, + text: nextTranscription.text, + }), text: nextTranscription.text, } : null, From 528b7fdd92b6fbfc610421535499f39b7fbf6132 Mon Sep 17 00:00:00 2001 From: Dekshad Date: Fri, 18 Sep 2026 14:54:16 +0300 Subject: [PATCH 3/5] TSA-297: * extract context word helper types --- .../libs/helpers/build-context-words.helper.ts | 6 ++---- .../libs/helpers/build-page-lexicon-map.helper.ts | 5 +++-- .../libs/types/build-context-words-payload.type.ts | 8 ++++++++ .../transcription/libs/types/context-lexicon-map.type.ts | 6 ++++++ .../backend/src/modules/transcription/libs/types/types.ts | 2 ++ 5 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 apps/backend/src/modules/transcription/libs/types/build-context-words-payload.type.ts create mode 100644 apps/backend/src/modules/transcription/libs/types/context-lexicon-map.type.ts diff --git a/apps/backend/src/modules/transcription/libs/helpers/build-context-words.helper.ts b/apps/backend/src/modules/transcription/libs/helpers/build-context-words.helper.ts index bc37cf35..e0c38050 100644 --- a/apps/backend/src/modules/transcription/libs/helpers/build-context-words.helper.ts +++ b/apps/backend/src/modules/transcription/libs/helpers/build-context-words.helper.ts @@ -4,14 +4,12 @@ import { } from "@transcripta/shared"; import { NOT_FOUND_INDEX } from "../constants/constants.js"; +import { type BuildContextWordsPayload } from "../types/types.js"; const buildContextWords = ({ lexiconById, text, -}: { - lexiconById: Map; - text: string; -}): DocumentGetPagesContextWordResponseDto[] => { +}: BuildContextWordsPayload): DocumentGetPagesContextWordResponseDto[] => { const contextWords: DocumentGetPagesContextWordResponseDto[] = []; for (const [lexiconId, lexicon] of lexiconById) { diff --git a/apps/backend/src/modules/transcription/libs/helpers/build-page-lexicon-map.helper.ts b/apps/backend/src/modules/transcription/libs/helpers/build-page-lexicon-map.helper.ts index e4881623..a7f46427 100644 --- a/apps/backend/src/modules/transcription/libs/helpers/build-page-lexicon-map.helper.ts +++ b/apps/backend/src/modules/transcription/libs/helpers/build-page-lexicon-map.helper.ts @@ -1,9 +1,10 @@ +import { type ContextLexiconMap } from "../types/types.js"; import { extractLexiconIds } from "./extract-lexicon-ids.helper.js"; const buildPageLexiconMap = ( contextUsed: null | Record, - lexiconById: Map, -): Map => { + lexiconById: ContextLexiconMap, +): ContextLexiconMap => { return new Map( extractLexiconIds(contextUsed).flatMap((id) => { const lexicon = lexiconById.get(id); diff --git a/apps/backend/src/modules/transcription/libs/types/build-context-words-payload.type.ts b/apps/backend/src/modules/transcription/libs/types/build-context-words-payload.type.ts new file mode 100644 index 00000000..1baa3df3 --- /dev/null +++ b/apps/backend/src/modules/transcription/libs/types/build-context-words-payload.type.ts @@ -0,0 +1,8 @@ +import { type ContextLexiconMap } from "./context-lexicon-map.type.js"; + +type BuildContextWordsPayload = { + lexiconById: ContextLexiconMap; + text: string; +}; + +export { type BuildContextWordsPayload }; diff --git a/apps/backend/src/modules/transcription/libs/types/context-lexicon-map.type.ts b/apps/backend/src/modules/transcription/libs/types/context-lexicon-map.type.ts new file mode 100644 index 00000000..0a76e8e7 --- /dev/null +++ b/apps/backend/src/modules/transcription/libs/types/context-lexicon-map.type.ts @@ -0,0 +1,6 @@ +type ContextLexiconMap = Map< + number, + { distinctPages: number; valueDisplay: string } +>; + +export { type ContextLexiconMap }; diff --git a/apps/backend/src/modules/transcription/libs/types/types.ts b/apps/backend/src/modules/transcription/libs/types/types.ts index e5e3cb75..e3e8352f 100644 --- a/apps/backend/src/modules/transcription/libs/types/types.ts +++ b/apps/backend/src/modules/transcription/libs/types/types.ts @@ -1,3 +1,5 @@ +export { type BuildContextWordsPayload } from "./build-context-words-payload.type.js"; +export { type ContextLexiconMap } from "./context-lexicon-map.type.js"; export { type ModelProviderValue } from "./model-provider-value.type.js"; export { type ModelRate } from "./model-rate.type.js"; export { type TranscriptionDebugRow } from "./transcription-debug-row.type.js"; From 69634a1e2465655e7f979a002433664238cac5a0 Mon Sep 17 00:00:00 2001 From: Dekshad Date: Fri, 18 Sep 2026 14:56:56 +0300 Subject: [PATCH 4/5] TSA-297: * reuse next transcription text in verify response --- apps/backend/src/modules/pages/page.service.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/backend/src/modules/pages/page.service.ts b/apps/backend/src/modules/pages/page.service.ts index 8eb8b79d..233ce3b3 100644 --- a/apps/backend/src/modules/pages/page.service.ts +++ b/apps/backend/src/modules/pages/page.service.ts @@ -92,6 +92,7 @@ class PageService { const nextTranscription = await this.transcriptionRepository.findCurrentByPageId(nextPage.id, trx); + const text = nextTranscription?.text ?? ""; const contextUsed = nextTranscription?.contextUsed ?? null; const lexiconRows = await this.documentRepository.findLexiconByIds( extractLexiconIds(contextUsed), @@ -110,9 +111,9 @@ class PageService { ? { contextWords: buildContextWords({ lexiconById: pageLexiconById, - text: nextTranscription.text, + text, }), - text: nextTranscription.text, + text, } : null, }, From b80be8bcdcdd20e65344586c0fe49d234e2263b3 Mon Sep 17 00:00:00 2001 From: Dekshad Date: Mon, 21 Sep 2026 23:09:09 +0300 Subject: [PATCH 5/5] TSA-297: * fix verify response formatting --- apps/backend/src/modules/pages/page.service.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/backend/src/modules/pages/page.service.ts b/apps/backend/src/modules/pages/page.service.ts index a7ee5641..98fd2abb 100644 --- a/apps/backend/src/modules/pages/page.service.ts +++ b/apps/backend/src/modules/pages/page.service.ts @@ -92,8 +92,7 @@ class PageService { const nextTranscription = await this.transcriptionRepository.findCurrentByPageId(nextPage.id, trx); - const text = - nextTranscription?.editedText ?? nextTranscription?.text ?? ""; + const text = nextTranscription?.editedText ?? nextTranscription?.text ?? ""; const contextUsed = nextTranscription?.contextUsed ?? null; const lexiconRows = await this.documentRepository.findLexiconByIds( extractLexiconIds(contextUsed),