From 22aade898141883420be2e94e7606033eb5e853d Mon Sep 17 00:00:00 2001 From: Alex Ultra <111590326+alex-mextner@users.noreply.github.com> Date: Wed, 4 Nov 2026 15:03:43 +0100 Subject: [PATCH] fix(memory): the 555K-entry stress dictionary loads on first use instead of at every boot (Refs #800, #963) Production parsed data/dictionaries/stress-dict.json at every start although it never speaks Russian through Silero (SILERO_PYTHON_PATH unset) and the MTProto service tier is off: in a bare bun process that file takes RSS from 51 MB to 362 MB at the peak and leaves 190 MB, on a container that shares an overcommitted, swapping 3.7 GB odroid. StressDictionary.lazyFromFile registers the file and ensureLoaded() reads it once, shared by concurrent callers; lookup_stress, the voice reply, the settings voice route and the voice-call TTS await it before marking stress, and an unloaded dictionary throws instead of reporting every word unknown. With SILERO_PYTHON_PATH set the file is read at startup as before, so an unreadable file keeps Silero off. The runbook records the measured steady state of the container (flat at 449-461 MiB, not a leak, same module-load footprint on ff93220a and e4185ddf). Review: single-seat Opus via omp (two rounds; round 1 found the Silero request-time stall, the unstressed-Silero case, a stale README and a missing Silero-path test, all fixed; round 2 none). This is a single-seat review: the full-roster pass is owed in #961. --- data/dictionaries/README.md | 10 +- docs/reference/deploy-runbook.md | 1 + pyproject.toml | 2 +- src/bootstrap/speech.ts | 24 +++- src/bootstrap/voice-calls.ts | 3 +- src/bot/handlers/callback/settings-routes.ts | 1 + src/bot/handlers/message/voice-message.ts | 1 + src/services/ai/tool-handlers/meta.ts | 4 +- src/services/voice/stress-dictionary.ts | 69 ++++++++++- src/services/voice/stress-marker.ts | 3 +- .../speech-stress-dictionary.test.ts | 72 +++++++++++ test/bootstrap/voice-calls.test.ts | 22 ++++ .../handlers/message-handler-units.test.ts | 2 +- test/bot/handlers/voice-prompt.test.ts | 15 ++- test/services/ai/lookup-stress.test.ts | 61 ++++++++++ test/services/voice/stress-dictionary.test.ts | 115 +++++++++++++++++- uv.lock | 2 +- 17 files changed, 383 insertions(+), 24 deletions(-) create mode 100644 test/bootstrap/speech-stress-dictionary.test.ts create mode 100644 test/services/ai/lookup-stress.test.ts diff --git a/data/dictionaries/README.md b/data/dictionaries/README.md index a143015cb..049ff2e21 100644 --- a/data/dictionaries/README.md +++ b/data/dictionaries/README.md @@ -34,7 +34,9 @@ bun scripts/generate-stress-dict.ts ## Usage -Loaded at bot startup by `StressDictionary.loadFromFile()` in -`src/services/voice/stress-dictionary.ts`. Used by `markStress()` in -`src/services/voice/stress-marker.ts` to add stress marks to Russian text -before feeding it to Silero TTS. +Registered at bot startup by `loadStressDictionary()` in `src/bootstrap/speech.ts` and read through +`StressDictionary.lazyFromFile()` in `src/services/voice/stress-dictionary.ts`: at startup when +`SILERO_PYTHON_PATH` is set, otherwise on first use (the `lookup_stress` tool), because parsing it costs about +80 MB retained and over 300 MB at the peak. Used by `markStress()` in `src/services/voice/stress-marker.ts` to +add stress marks to Russian text before feeding it to Silero TTS. The scripts under `scripts/` read it +directly with `StressDictionary.loadFromFile()`. diff --git a/docs/reference/deploy-runbook.md b/docs/reference/deploy-runbook.md index c0d3e77b6..c23ba2710 100644 --- a/docs/reference/deploy-runbook.md +++ b/docs/reference/deploy-runbook.md @@ -253,6 +253,7 @@ If a release that the override accepted with `rollback-compatible: no` fails ver - Images are linux/arm64 (the odroid is aarch64). The host never pulls from GHCR: every deploy loads a checksummed `docker save` archive. - `docker compose` requires root (neither `www-data` nor `hcb-runner` is in the docker group). - Resource limits: bot 1G/1.5cpu with a 384M reservation (cgroup `memory.low`), redis 256M/0.5cpu. The odroid has 6 cores and 3.7 GB RAM shared with the neighbours listed above and is overcommitted (on 2026-10-10 about 2 GB sat in swap and the idle bot was down to 71 MB resident); the reservation keeps the bot's working set out of reclaim so a message is not served from swap. Keep the bot well below the whole box. +- Bot container memory, measured 2026-11-04 (#963, #800). The image before the 0.3.1-0.3.3 deploys (ff93220a) and the current one (e4185ddf) have the same dependencies, Dockerfile, compose limits and module-load footprint (importing every startup module takes a bun process from 18 MB to 131-133 MB RSS on both, 29 MB of heap), so the jump from the 138-163 MiB seen on the old container is not a code change. A fresh e4185ddf container showed 413 MiB in `docker stats` one minute after start and 449-461 MiB over the next 16 minutes: flat, no growth, while the kernel swapped cold pages out (cgroup anon 405 MB falling to 363 MB, `memory.swap.current` 108 MB at minute 20, `memory.low` 384 MiB). Inside it the `bun` process held 328 MB anon plus 93 MB swapped; the five Chromium processes of the render pool (`playwrightPool.initialize()` starts them at boot) are mostly shared binary pages (the two measured held about 5 MB anon each). A container that has run for 28 h looks small because the host swapped its idle part out (the 71 MB of 2026-10-10 is the same effect): compare containers of the same age. The largest single item was the stress dictionary, 555,461 entries parsed at boot: in a bare bun process it takes RSS from 51 MB to 362 MB at the peak and leaves 190 MB, and production never uses it unless a model calls `lookup_stress` (Silero and the MTProto service tier are off there). It now loads on first use (#963), or at startup when `SILERO_PYTHON_PATH` is set (every Russian voice reply needs it then, and an unreadable file keeps Silero off as before); the load is one synchronous parse, so the first `lookup_stress` after a restart stalls the bot for about 2 s on a fast PC and longer on the odroid. Measure with `docker stats --no-stream hypercal-bot`, `grep -E 'RssAnon|VmSwap' /proc//status` inside the container and `/sys/fs/cgroup/memory.current`, `memory.swap.current`, `memory.low`. - No deploy secrets: the SSH secrets `SSH_HOST`, `SSH_USER`, `SSH_KEY`, `DEPLOY_PATH` are unused since #784. ### Image identity diff --git a/pyproject.toml b/pyproject.toml index 54da915c3..3ac46c541 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "hypercalendarbot" -version = "0.3.3" +version = "0.3.4" requires-python = ">=3.12" dependencies = [ "pyrofork>=2.3", diff --git a/src/bootstrap/speech.ts b/src/bootstrap/speech.ts index 3de3f7450..8fb6de089 100644 --- a/src/bootstrap/speech.ts +++ b/src/bootstrap/speech.ts @@ -6,8 +6,11 @@ // Ordering invariants: // - It runs AFTER the voice-call phase and BEFORE createBot(). The voice-call TTS adapter reads // `ctx.speech` at call time, so it sees these services even though they are created later. -// - The stress dictionary loads before Silero: Silero is only created when a dictionary exists, -// because Russian text is stress-marked with it before synthesis. +// - The stress dictionary is registered before Silero: Silero is only created when a dictionary exists, +// because Russian text is stress-marked with it before synthesis. With Silero configured the file is +// read right here, as it always was (every Russian reply needs it, and an unreadable file must keep +// Silero off instead of speaking unstressed text); without Silero it is only registered and its entries +// load when a model first calls `lookup_stress` (see loadStressDictionary). // - Every service is optional and keyed on its own env variable; a missing dictionary is logged // and never stops startup. // - Modules are imported lazily (runtime-selected by configuration, same as before the split) so a @@ -20,10 +23,21 @@ import type { StartupContext } from './runtime.ts'; const STRESS_DICTIONARY_PATH = 'data/dictionaries/stress-dict.json'; -async function loadStressDictionary(): Promise { +/** + * Registers the stress dictionary. The 555K-entry file costs about 80 MB retained (over 300 MB at the peak), and + * without Silero the only reader is the `lookup_stress` tool, so it is registered unread and every consumer calls + * `ensureLoaded()` first. `readNow` (Silero configured) reads it here: a file that cannot be read then leaves the + * dictionary absent, which keeps Silero off, as before. A missing file leaves it absent either way. + */ +export async function loadStressDictionary( + path: string = STRESS_DICTIONARY_PATH, + readNow = false, +): Promise { try { const { StressDictionary: StressDictionaryClass } = await import('../services/voice/stress-dictionary.ts'); - return await StressDictionaryClass.loadFromFile(STRESS_DICTIONARY_PATH); + const dictionary = await StressDictionaryClass.lazyFromFile(path); + if (readNow && !(await dictionary.ensureLoaded())) return undefined; + return dictionary; } catch (error) { botLogger.warn({ err: error }, 'Stress dictionary not loaded'); return undefined; @@ -72,7 +86,7 @@ export async function initSpeechServices(ctx: StartupContext): Promise { const { config, speech } = ctx; speech.transcriptionService = await createTranscriptionService(config); - speech.stressDictionary = await loadStressDictionary(); + speech.stressDictionary = await loadStressDictionary(STRESS_DICTIONARY_PATH, Boolean(config.SILERO_PYTHON_PATH)); speech.fallbackTts = await createFallbackTts(); speech.kokoroTts = await createKokoroTts(config); speech.nliClassifier = await createNliClassifier(config); diff --git a/src/bootstrap/voice-calls.ts b/src/bootstrap/voice-calls.ts index 9f235b301..e622d1c6b 100644 --- a/src/bootstrap/voice-calls.ts +++ b/src/bootstrap/voice-calls.ts @@ -55,7 +55,8 @@ function selectTtsEngine(speech: SpeechServices, tools: StressTextTools, lang: s if (lang === 'ru' && sileroTts && stressDictionary) { return { failureMessage: 'Silero TTS failed, falling back to Google', - synthesize: (clean) => { + synthesize: async (clean) => { + await stressDictionary.ensureLoaded(); const stressedText = tools.markStress(tools.numbersToWords(clean), stressDictionary); botLogger.info({ engine: 'silero', lang }, 'Voice call TTS'); return sileroTts.synthesize(stressedText); diff --git a/src/bot/handlers/callback/settings-routes.ts b/src/bot/handlers/callback/settings-routes.ts index dba9d2ef3..6f3951970 100644 --- a/src/bot/handlers/callback/settings-routes.ts +++ b/src/bot/handlers/callback/settings-routes.ts @@ -44,6 +44,7 @@ function lastAssistantText(history: ChatHistoryMessage[]): string { async function synthesizeVoice(voiceDeps: VoiceDeps, isRu: boolean, text: string): Promise { if (isRu && voiceDeps.sileroTts && voiceDeps.stressDictionary) { const withNumbers = numbersToWords(fixDateOrdinals(text)); + await voiceDeps.stressDictionary.ensureLoaded(); const withStress = markStress(withNumbers, voiceDeps.stressDictionary); return voiceDeps.sileroTts.synthesize(transliterateEnglish(withStress)); } diff --git a/src/bot/handlers/message/voice-message.ts b/src/bot/handlers/message/voice-message.ts index 907253d07..e47d0af3e 100644 --- a/src/bot/handlers/message/voice-message.ts +++ b/src/bot/handlers/message/voice-message.ts @@ -40,6 +40,7 @@ async function synthesizePrimary(user: User, plainText: string, deps: MessageHan if (isRu && deps.sileroTts && deps.stressDictionary) { const withOrdinals = fixDateOrdinals(plainText); const withNumbers = numbersToWords(withOrdinals); + await deps.stressDictionary.ensureLoaded(); const withStress = markStress(withNumbers, deps.stressDictionary); const stressedText = transliterateEnglish(withStress); cmdLogger.info({ userId: user.telegram_id, textLen: stressedText.length }, 'Synthesizing RU voice reply'); diff --git a/src/services/ai/tool-handlers/meta.ts b/src/services/ai/tool-handlers/meta.ts index 6d0135393..589ab265c 100644 --- a/src/services/ai/tool-handlers/meta.ts +++ b/src/services/ai/tool-handlers/meta.ts @@ -252,8 +252,8 @@ export function handleListGoogleCalendars(ctx: AgentContext): ToolResult { } handleListGoogleCalendars.meta = { readonly: true, skipActionLog: true } satisfies ToolHandlerMeta; -export function handleLookupStress(ctx: AgentContext, input: { words: string[] }): ToolResult { - if (!ctx.voice) { +export async function handleLookupStress(ctx: AgentContext, input: { words: string[] }): Promise { + if (!ctx.voice || !(await ctx.voice.stressDictionary.ensureLoaded())) { return { success: false, error: 'Stress dictionary not loaded' }; } diff --git a/src/services/voice/stress-dictionary.ts b/src/services/voice/stress-dictionary.ts index 46f7b126e..bd199cfdb 100644 --- a/src/services/voice/stress-dictionary.ts +++ b/src/services/voice/stress-dictionary.ts @@ -6,18 +6,69 @@ import { voiceLogger } from './types'; * In-memory stress dictionary for Russian words. * Format: bare lowercase word -> stressed form with + before stressed vowel. * Example: "молоко" -> "молок+о" + * + * The production file holds 555K entries: loading it costs about 80 MB of retained heap and over 300 MB at + * the peak, on a container that shares a swapping 1 GiB budget. `lazyFromFile` therefore reads it on the + * first `ensureLoaded()`, so a process that never asks for a stress mark never pays for the dictionary. */ export class StressDictionary { - private dict: Map; - private sortedKeys: string[]; + private dict = new Map(); + private sortedKeysCache: string[] | undefined; + /** Where a lazy dictionary reads from; undefined when the entries were handed to the constructor. */ + private readonly lazyPath: string | undefined; + private lazyLoad: Promise | undefined; + private ready: boolean; - constructor(data: Record) { + /** Pass `lazyPath` only through `lazyFromFile`; with it the entries stay unread until `ensureLoaded()`. */ + constructor(data: Record, lazyPath?: string) { + this.lazyPath = lazyPath; + this.ready = lazyPath === undefined; + if (lazyPath === undefined) this.fill(data); + } + + private fill(data: Record): void { this.dict = new Map(Object.entries(data)); - this.sortedKeys = [...this.dict.keys()].sort(); + this.sortedKeysCache = undefined; voiceLogger.info({ size: this.dict.size }, 'Stress dictionary loaded'); } + /** Sorted only when `findSimilar` first needs it: a plain lookup never pays for 555K sorted keys. */ + private get sortedKeys(): string[] { + this.sortedKeysCache ??= [...this.dict.keys()].sort(); + return this.sortedKeysCache; + } + + /** + * Reads a lazy dictionary's file once (concurrent callers share the read). Resolves true when lookups can be + * answered; false when the file could not be read or parsed, logged once and not retried, and every lookup + * then finds nothing. An eager dictionary is always ready. + */ + ensureLoaded(): Promise { + const path = this.lazyPath; + if (path === undefined) return Promise.resolve(true); + this.lazyLoad ??= this.loadLazyFile(path); + return this.lazyLoad; + } + + private async loadLazyFile(path: string): Promise { + try { + this.fill(z.record(z.string(), z.string()).parse(await Bun.file(path).json())); + return true; + } catch (error) { + voiceLogger.warn({ err: error, path }, 'Stress dictionary not loaded'); + return false; + } finally { + this.ready = true; + } + } + + /** A lazy dictionary answered before `ensureLoaded()` settled would report every word as unknown. */ + private assertReady(): void { + if (!this.ready) throw new Error('StressDictionary used before ensureLoaded() settled'); + } + lookup(word: string): string | null { + this.assertReady(); return this.dict.get(word.toLowerCase()) ?? null; } @@ -36,6 +87,7 @@ export class StressDictionary { * Uses binary search on sorted keys for O(log n) prefix lookup. */ findSimilar(word: string, limit: number): string[] { + this.assertReady(); const results: string[] = []; const minPrefix = Math.max(3, Math.ceil(word.length * 0.6)); @@ -76,4 +128,13 @@ export class StressDictionary { const data = z.record(z.string(), z.string()).parse(await file.json()); return new StressDictionary(data); } + + /** + * A dictionary that reads `path` on its first `ensureLoaded()`. A missing file fails here, at startup, as + * before; a file that exists but cannot be parsed fails at the first load instead (logged, lookups find nothing). + */ + static async lazyFromFile(path: string): Promise { + if (!(await Bun.file(path).exists())) throw new Error(`Stress dictionary file not found: ${path}`); + return new StressDictionary({}, path); + } } diff --git a/src/services/voice/stress-marker.ts b/src/services/voice/stress-marker.ts index a48d6343e..9f13ae53c 100644 --- a/src/services/voice/stress-marker.ts +++ b/src/services/voice/stress-marker.ts @@ -83,7 +83,8 @@ export function numbersToWords(text: string): string { /** * Adds stress marks (+) to Russian text using the stress dictionary. * Words not found in the dictionary are left unchanged (Silero has auto-stress). - * Preserves original punctuation and spacing. + * Preserves original punctuation and spacing. The dictionary may load on first use: `await dict.ensureLoaded()` + * before calling (an unloaded one throws instead of treating every word as unknown). */ export function markStress(text: string, dict: StressDictionary): string { return text.replace(RUSSIAN_WORD, (word) => { diff --git a/test/bootstrap/speech-stress-dictionary.test.ts b/test/bootstrap/speech-stress-dictionary.test.ts new file mode 100644 index 000000000..4616d08b7 --- /dev/null +++ b/test/bootstrap/speech-stress-dictionary.test.ts @@ -0,0 +1,72 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { loadStressDictionary } from '../../src/bootstrap/speech.ts'; +import { StressDictionary } from '../../src/services/voice/stress-dictionary.ts'; +import { voiceLogger } from '../../src/services/voice/types.ts'; +import { botLogger } from '../../src/utils/logger.ts'; + +describe('loadStressDictionary at startup', () => { + let dir: string; + let path: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'hcb-speech-')); + path = join(dir, 'stress-dict.json'); + writeFileSync(path, JSON.stringify({ молоко: 'молок+о' })); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + test('registers an existing file without reading its entries', async () => { + const dictionary = await loadStressDictionary(path); + // A startup that parsed the file would hold this entry; the real one holds 555K and costs about 80 MB. + writeFileSync(path, JSON.stringify({ календарь: 'календ+арь' })); + + expect(dictionary).toBeInstanceOf(StressDictionary); + expect(dictionary?.size).toBe(0); + expect(await dictionary?.ensureLoaded()).toBe(true); + expect(dictionary?.lookup('календарь')).toBe('календ+арь'); + expect(dictionary?.lookup('молоко')).toBeNull(); + }); + + test('a missing file leaves the dictionary absent and logs the warning startup always logged', async () => { + const warn = spyOn(botLogger, 'warn').mockImplementation(() => {}); + try { + const dictionary = await loadStressDictionary(join(dir, 'absent.json')); + + expect(dictionary).toBeUndefined(); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]?.[1]).toBe('Stress dictionary not loaded'); + } finally { + warn.mockRestore(); + } + }); + + test('with Silero configured the file is read at startup, as it always was', async () => { + const dictionary = await loadStressDictionary(path, true); + // Changed after startup: only an eager read would hold the original entry already. + writeFileSync(path, JSON.stringify({ календарь: 'календ+арь' })); + + expect(dictionary?.size).toBe(1); + expect(await dictionary?.ensureLoaded()).toBe(true); + expect(dictionary?.lookup('молоко')).toBe('молок+о'); + expect(dictionary?.lookup('календарь')).toBeNull(); + }); + + test('with Silero configured a file that cannot be parsed leaves no dictionary, so Silero stays off', async () => { + const warn = spyOn(voiceLogger, 'warn').mockImplementation(() => {}); + try { + writeFileSync(path, '{ not json'); + + expect(await loadStressDictionary(path, true)).toBeUndefined(); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]?.[1]).toBe('Stress dictionary not loaded'); + } finally { + warn.mockRestore(); + } + }); +}); diff --git a/test/bootstrap/voice-calls.test.ts b/test/bootstrap/voice-calls.test.ts index 4b3b7e452..d4591365f 100644 --- a/test/bootstrap/voice-calls.test.ts +++ b/test/bootstrap/voice-calls.test.ts @@ -1,9 +1,13 @@ import { describe, expect, spyOn, test } from 'bun:test'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import type { SpeechServices } from '../../src/bootstrap/runtime.ts'; import { createVoiceCallTts, type StressTextTools } from '../../src/bootstrap/voice-calls.ts'; import { KokoroTtsService } from '../../src/services/voice/kokoro-tts-service.ts'; import { SileroTtsService } from '../../src/services/voice/silero-tts-service.ts'; import { StressDictionary } from '../../src/services/voice/stress-dictionary.ts'; +import { markStress, numbersToWords } from '../../src/services/voice/stress-marker.ts'; import { TtsService } from '../../src/services/voice/tts-service.ts'; const tools: StressTextTools = { @@ -81,6 +85,24 @@ describe('createVoiceCallTts', () => { expect(spies.silero).not.toHaveBeenCalled(); }); + test('Russian with a stress dictionary that is not read yet loads it first, so Silero gets the stressed text', async () => { + const { speech, spies } = services(); + const dir = mkdtempSync(join(tmpdir(), 'hcb-call-stress-')); + try { + const path = join(dir, 'stress-dict.json'); + writeFileSync(path, JSON.stringify({ молоко: 'молок+о' })); + speech.stressDictionary = await StressDictionary.lazyFromFile(path); + + const result = await createVoiceCallTts(speech, { numbersToWords, markStress }).synthesize('молоко', 'ru'); + + expect(result.toString()).toBe('silero'); + expect(spies.silero).toHaveBeenCalledWith('молок+о'); + expect(spies.fallback).not.toHaveBeenCalled(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + test('any other language uses the Google fallback', async () => { const { speech, spies } = services(); diff --git a/test/bot/handlers/message-handler-units.test.ts b/test/bot/handlers/message-handler-units.test.ts index 8c87b19b5..536ffffcf 100644 --- a/test/bot/handlers/message-handler-units.test.ts +++ b/test/bot/handlers/message-handler-units.test.ts @@ -109,7 +109,7 @@ describe('voice reply TTS fallback', () => { const audioBuffer = Buffer.from('fake-audio'); const fakeAudioDownload = mock(() => Promise.resolve(Buffer.from('fake-voice-download'))); const voiceUser = makeUserRow({ telegram_id: 1, language: 'ru', timezone: 'UTC', voice_response_enabled: 1 }); - const noStress = stub({ lookup: () => null }); + const noStress = stub({ lookup: () => null, ensureLoaded: () => Promise.resolve(true) }); function makeVoiceDeps(overrides: Partial = {}) { return makeHandlerDeps({ diff --git a/test/bot/handlers/voice-prompt.test.ts b/test/bot/handlers/voice-prompt.test.ts index 1b1b3523a..639a05e6b 100644 --- a/test/bot/handlers/voice-prompt.test.ts +++ b/test/bot/handlers/voice-prompt.test.ts @@ -169,7 +169,10 @@ describe('voice response prompt', () => { const deps = makeVoiceDeps({ sileroTts: { synthesize } as unknown as MessageHandlerDeps['sileroTts'], sendVoice, - stressDictionary: { lookup: () => null } as unknown as MessageHandlerDeps['stressDictionary'], + stressDictionary: { + lookup: () => null, + ensureLoaded: () => Promise.resolve(true), + } as unknown as MessageHandlerDeps['stressDictionary'], }); globalThis.fetch = makeFetch(); @@ -191,7 +194,10 @@ describe('voice response prompt', () => { const deps = makeVoiceDeps({ sileroTts: { synthesize } as unknown as MessageHandlerDeps['sileroTts'], sendVoice, - stressDictionary: { lookup: () => null } as unknown as MessageHandlerDeps['stressDictionary'], + stressDictionary: { + lookup: () => null, + ensureLoaded: () => Promise.resolve(true), + } as unknown as MessageHandlerDeps['stressDictionary'], }); globalThis.fetch = makeFetch(); @@ -213,7 +219,10 @@ describe('voice response prompt', () => { const deps = makeVoiceDeps({ sileroTts: { synthesize } as unknown as MessageHandlerDeps['sileroTts'], sendVoice, - stressDictionary: { lookup: () => null } as unknown as MessageHandlerDeps['stressDictionary'], + stressDictionary: { + lookup: () => null, + ensureLoaded: () => Promise.resolve(true), + } as unknown as MessageHandlerDeps['stressDictionary'], }); globalThis.fetch = makeFetch(); diff --git a/test/services/ai/lookup-stress.test.ts b/test/services/ai/lookup-stress.test.ts new file mode 100644 index 000000000..1087e5c43 --- /dev/null +++ b/test/services/ai/lookup-stress.test.ts @@ -0,0 +1,61 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { handleLookupStress } from '../../../src/services/ai/tool-handlers/meta.ts'; +import type { AgentContext } from '../../../src/services/ai/types.ts'; +import { StressDictionary } from '../../../src/services/voice/stress-dictionary.ts'; +import { voiceLogger } from '../../../src/services/voice/types.ts'; +import { partialOf } from './partial-stubs.ts'; + +describe('lookup_stress with a dictionary that loads on first use', () => { + let dir: string; + let path: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'hcb-lookup-stress-')); + path = join(dir, 'stress-dict.json'); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + function contextWith(dictionary: StressDictionary): AgentContext { + return partialOf('ctx', { voice: { stressDictionary: dictionary } }); + } + + test('the first call loads the file and answers from it', async () => { + writeFileSync(path, JSON.stringify({ молоко: 'молок+о' })); + const dictionary = await StressDictionary.lazyFromFile(path); + expect(dictionary.size).toBe(0); + + const result = await handleLookupStress(contextWith(dictionary), { words: ['молоко'] }); + + expect(result).toEqual({ success: true, output: 'молоко → молок+о' }); + expect(dictionary.size).toBe(1); + }); + + test('an unreadable file is reported as not loaded, never as every word missing', async () => { + const warn = spyOn(voiceLogger, 'warn').mockImplementation(() => {}); + try { + writeFileSync(path, JSON.stringify({ молоко: 'молок+о' })); + const dictionary = await StressDictionary.lazyFromFile(path); + writeFileSync(path, '{ not json'); + + const result = await handleLookupStress(contextWith(dictionary), { words: ['молоко'] }); + + expect(result).toEqual({ success: false, error: 'Stress dictionary not loaded' }); + } finally { + warn.mockRestore(); + } + }); + + test('without a dictionary the tool says so', async () => { + const result = await handleLookupStress(partialOf('ctx', { voice: undefined }), { + words: ['молоко'], + }); + + expect(result).toEqual({ success: false, error: 'Stress dictionary not loaded' }); + }); +}); diff --git a/test/services/voice/stress-dictionary.test.ts b/test/services/voice/stress-dictionary.test.ts index 75a37e6c6..7d3299453 100644 --- a/test/services/voice/stress-dictionary.test.ts +++ b/test/services/voice/stress-dictionary.test.ts @@ -1,5 +1,9 @@ -import { describe, expect, test } from 'bun:test'; +import { afterEach, beforeEach, describe, expect, spyOn, test } from 'bun:test'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { StressDictionary } from '../../../src/services/voice/stress-dictionary.ts'; +import { voiceLogger } from '../../../src/services/voice/types.ts'; describe('StressDictionary', () => { const dict = new StressDictionary({ @@ -56,3 +60,112 @@ describe('StressDictionary', () => { expect(dict.size).toBe(7); }); }); + +describe('StressDictionary loaded on first use', () => { + const WORDS: Record = { + молоко: 'молок+о', + молока: 'молок+а', + молоком: 'молок+ом', + встреча: 'встр+еча', + }; + let dir: string; + let path: string; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'hcb-stress-')); + path = join(dir, 'stress-dict.json'); + writeFileSync(path, JSON.stringify(WORDS)); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + test('registering the file reads nothing: what is loaded is what the file holds at the first ensureLoaded', async () => { + const dict = await StressDictionary.lazyFromFile(path); + // Changed after registration. An eager read at registration would have kept the old entries. + writeFileSync(path, JSON.stringify({ календарь: 'календ+арь' })); + + expect(dict.size).toBe(0); + expect(await dict.ensureLoaded()).toBe(true); + + expect(dict.lookup('календарь')).toBe('календ+арь'); + expect(dict.lookup('молоко')).toBeNull(); + expect(dict.size).toBe(1); + }); + + test('answering before the first load throws instead of reporting every word as unknown', async () => { + const dict = await StressDictionary.lazyFromFile(path); + + expect(() => dict.lookup('молоко')).toThrow(/ensureLoaded/); + expect(() => dict.lookupMany(['молоко'])).toThrow(/ensureLoaded/); + expect(() => dict.findSimilar('молоки', 5)).toThrow(/ensureLoaded/); + }); + + test('after the load it answers exactly like the eager dictionary', async () => { + const eager = new StressDictionary(WORDS); + const lazy = await StressDictionary.lazyFromFile(path); + await lazy.ensureLoaded(); + + expect(lazy.size).toBe(eager.size); + expect(lazy.lookup('Молоко')).toBe(eager.lookup('Молоко')); + expect(lazy.lookupMany(['молоко', 'молоки', 'алексом'])).toEqual(eager.lookupMany(['молоко', 'молоки', 'алексом'])); + expect(lazy.findSimilar('молоки', 5)).toEqual(eager.findSimilar('молоки', 5)); + }); + + test('concurrent callers share one load and later callers get the same answer', async () => { + const dict = await StressDictionary.lazyFromFile(path); + const first = dict.ensureLoaded(); + const second = dict.ensureLoaded(); + + expect(second).toBe(first); + expect(await first).toBe(true); + writeFileSync(path, JSON.stringify({})); + expect(await dict.ensureLoaded()).toBe(true); + expect(dict.size).toBe(Object.keys(WORDS).length); + }); + + test('an eager dictionary is ready without a load', async () => { + const dict = new StressDictionary(WORDS); + + expect(await dict.ensureLoaded()).toBe(true); + expect(dict.lookup('молоко')).toBe('молок+о'); + }); + + test('a missing file is refused at registration, as the eager read was at startup', async () => { + await expect(StressDictionary.lazyFromFile(join(dir, 'absent.json'))).rejects.toThrow(/not found/); + }); + + test('a file that cannot be parsed logs once, finds nothing and is not read again', async () => { + const warn = spyOn(voiceLogger, 'warn').mockImplementation(() => {}); + try { + const dict = await StressDictionary.lazyFromFile(path); + writeFileSync(path, '{ not json'); + + expect(await dict.ensureLoaded()).toBe(false); + expect(dict.lookup('молоко')).toBeNull(); + expect(dict.findSimilar('молоки', 5)).toEqual([]); + + writeFileSync(path, JSON.stringify(WORDS)); + expect(await dict.ensureLoaded()).toBe(false); + expect(dict.lookup('молоко')).toBeNull(); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]?.[1]).toBe('Stress dictionary not loaded'); + } finally { + warn.mockRestore(); + } + }); + + test('a file whose values are not strings is refused like a parse failure', async () => { + const warn = spyOn(voiceLogger, 'warn').mockImplementation(() => {}); + try { + writeFileSync(path, JSON.stringify({ молоко: 5 })); + const dict = await StressDictionary.lazyFromFile(path); + + expect(await dict.ensureLoaded()).toBe(false); + expect(dict.lookup('молоко')).toBeNull(); + } finally { + warn.mockRestore(); + } + }); +}); diff --git a/uv.lock b/uv.lock index 494edacd7..d4db434e2 100644 --- a/uv.lock +++ b/uv.lock @@ -335,7 +335,7 @@ wheels = [ [[package]] name = "hypercalendarbot" -version = "0.3.3" +version = "0.3.4" source = { virtual = "." } dependencies = [ { name = "numpy" },