From bb39609e117d29ef1cd9d4c2e0bfd739a5ada57c Mon Sep 17 00:00:00 2001 From: Yannic Charlon <52761674+JustYannicc@users.noreply.github.com> Date: Fri, 3 Jul 2026 23:29:54 +0200 Subject: [PATCH] perf(ai): cache local model status probes --- .../test-local-model-status-probe-cache.mjs | 146 ++++++++++++++++++ src/main/local-model-status-probe.ts | 60 +++++++ src/main/main.ts | 90 ++++++++--- 3 files changed, 272 insertions(+), 24 deletions(-) create mode 100644 scripts/test-local-model-status-probe-cache.mjs create mode 100644 src/main/local-model-status-probe.ts diff --git a/scripts/test-local-model-status-probe-cache.mjs b/scripts/test-local-model-status-probe-cache.mjs new file mode 100644 index 00000000..7bb75fc1 --- /dev/null +++ b/scripts/test-local-model-status-probe-cache.mjs @@ -0,0 +1,146 @@ +#!/usr/bin/env node + +import test from 'node:test'; +import assert from 'node:assert/strict'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import { importTs } from './lib/ts-import.mjs'; + +const { + createLocalAsrHelperStatusProbeCache, + isCacheableLocalAsrHelperStatus, +} = await importTs(path.resolve('src/main/local-model-status-probe.ts')); + +function makeStatus(overrides = {}) { + return { + state: 'downloaded', + modelName: 'parakeet-tdt-0.6b-v3', + path: '/models/parakeet', + progress: 1, + ...overrides, + }; +} + +function spinForMs(durationMs) { + const end = performance.now() + durationMs; + while (performance.now() < end) {} +} + +test('local ASR helper status cache reuses stable statuses within the TTL', () => { + let now = 1_000; + let probeCalls = 0; + const cache = createLocalAsrHelperStatusProbeCache({ + ttlMs: 5_000, + now: () => now, + readStatus() { + probeCalls += 1; + return makeStatus({ path: `/models/parakeet-${probeCalls}` }); + }, + }); + + assert.equal(cache.getStatus().path, '/models/parakeet-1'); + assert.equal(cache.getStatus().path, '/models/parakeet-1'); + assert.equal(probeCalls, 1); + + now += 5_001; + assert.equal(cache.getStatus().path, '/models/parakeet-2'); + assert.equal(probeCalls, 2); +}); + +test('local ASR helper status cache bypasses cached stable status when force refreshing', () => { + let probeCalls = 0; + const cache = createLocalAsrHelperStatusProbeCache({ + ttlMs: 5_000, + readStatus() { + probeCalls += 1; + return makeStatus({ path: `/models/qwen3-${probeCalls}` }); + }, + }); + + assert.equal(cache.getStatus().path, '/models/qwen3-1'); + assert.equal(cache.getStatus({ forceRefresh: true }).path, '/models/qwen3-2'); + assert.equal(cache.getStatus().path, '/models/qwen3-2'); + assert.equal(probeCalls, 2); +}); + +test('local ASR helper status cache does not retain transient statuses', () => { + const statuses = [ + makeStatus({ state: 'downloading', path: '', progress: 0.5 }), + makeStatus({ state: 'error', path: '', progress: 0, error: 'failed' }), + makeStatus({ state: 'not-downloaded', path: '', progress: 0 }), + ]; + let probeCalls = 0; + const cache = createLocalAsrHelperStatusProbeCache({ + ttlMs: 5_000, + readStatus() { + const status = statuses[Math.min(probeCalls, statuses.length - 1)]; + probeCalls += 1; + return status; + }, + }); + + assert.equal(cache.getStatus().state, 'downloading'); + assert.equal(cache.getStatus().state, 'error'); + assert.equal(cache.getStatus().state, 'not-downloaded'); + assert.equal(cache.getStatus().state, 'not-downloaded'); + assert.equal(probeCalls, 3); +}); + +test('local ASR helper status cache can remember a fresh post-download status', () => { + let probeCalls = 0; + const cache = createLocalAsrHelperStatusProbeCache({ + ttlMs: 5_000, + readStatus() { + probeCalls += 1; + return makeStatus({ state: 'not-downloaded', path: '', progress: 0 }); + }, + }); + + assert.equal(cache.getStatus().state, 'not-downloaded'); + cache.rememberStatus(makeStatus({ state: 'downloaded', path: '/models/fresh', progress: 1 })); + + const status = cache.getStatus(); + assert.equal(status.state, 'downloaded'); + assert.equal(status.path, '/models/fresh'); + assert.equal(probeCalls, 1); +}); + +test('local ASR helper status cache measurement avoids repeated fake sync probes', () => { + const iterations = 60; + const fakeProbeMs = 1.75; + let legacyProbeCalls = 0; + const legacyStart = performance.now(); + for (let index = 0; index < iterations; index += 1) { + legacyProbeCalls += 1; + spinForMs(fakeProbeMs); + } + const legacyMs = performance.now() - legacyStart; + + let cachedProbeCalls = 0; + const cache = createLocalAsrHelperStatusProbeCache({ + ttlMs: 5_000, + readStatus() { + cachedProbeCalls += 1; + spinForMs(fakeProbeMs); + return makeStatus(); + }, + }); + + const cachedStart = performance.now(); + for (let index = 0; index < iterations; index += 1) { + cache.getStatus(); + } + const cachedMs = performance.now() - cachedStart; + + console.log( + `[local-model-status probe-cache] iterations=${iterations} ` + + `legacyCalls=${legacyProbeCalls} legacyMs=${legacyMs.toFixed(3)} ` + + `cachedCalls=${cachedProbeCalls} cachedMs=${cachedMs.toFixed(3)} ` + + `avoidedCalls=${legacyProbeCalls - cachedProbeCalls}` + ); + + assert.equal(legacyProbeCalls, iterations); + assert.equal(cachedProbeCalls, 1); + assert.equal(isCacheableLocalAsrHelperStatus(makeStatus()), true); + assert.ok(cachedMs < legacyMs / 4); +}); diff --git a/src/main/local-model-status-probe.ts b/src/main/local-model-status-probe.ts new file mode 100644 index 00000000..767d8f15 --- /dev/null +++ b/src/main/local-model-status-probe.ts @@ -0,0 +1,60 @@ +export const LOCAL_ASR_HELPER_STATUS_CACHE_TTL_MS = 5_000; + +export type LocalAsrHelperStatus = { + state: string; +}; + +export type LocalAsrHelperStatusProbeOptions = { + forceRefresh?: boolean; +}; + +export type LocalAsrHelperStatusProbeCache = { + getStatus: (options?: LocalAsrHelperStatusProbeOptions) => TStatus; + rememberStatus: (status: TStatus) => TStatus; + clear: () => void; +}; + +type CreateLocalAsrHelperStatusProbeCacheOptions = { + readStatus: () => TStatus; + ttlMs?: number; + now?: () => number; +}; + +export function isCacheableLocalAsrHelperStatus(status: LocalAsrHelperStatus | null | undefined): boolean { + return status?.state === 'downloaded' || status?.state === 'not-downloaded'; +} + +export function createLocalAsrHelperStatusProbeCache({ + readStatus, + ttlMs = LOCAL_ASR_HELPER_STATUS_CACHE_TTL_MS, + now = Date.now, +}: CreateLocalAsrHelperStatusProbeCacheOptions): LocalAsrHelperStatusProbeCache { + let cachedStatus: TStatus | null = null; + let cacheExpiresAt = 0; + + function rememberStatus(status: TStatus): TStatus { + if (isCacheableLocalAsrHelperStatus(status)) { + cachedStatus = status; + cacheExpiresAt = now() + ttlMs; + } else { + cachedStatus = null; + cacheExpiresAt = 0; + } + return status; + } + + return { + getStatus(options = {}) { + const currentTime = now(); + if (!options.forceRefresh && cachedStatus && currentTime < cacheExpiresAt) { + return cachedStatus; + } + return rememberStatus(readStatus()); + }, + rememberStatus, + clear() { + cachedStatus = null; + cacheExpiresAt = 0; + }, + }; +} diff --git a/src/main/main.ts b/src/main/main.ts index cf510437..72776c25 100644 --- a/src/main/main.ts +++ b/src/main/main.ts @@ -19,6 +19,7 @@ import * as fs from 'fs'; import * as os from 'os'; import { fork, execFileSync, type ChildProcess } from 'child_process'; import { getNativeBinaryPath, resolvePackagedUnpackedPath } from './native-binary'; +import { createLocalAsrHelperStatusProbeCache } from './local-model-status-probe'; import { getAvailableCommands, executeCommand, invalidateCache, initCommandsCache, getInflightDiscovery, refreshCommandsNow } from './commands'; import { loadSettings, @@ -255,6 +256,7 @@ type ParakeetModelStatus = { }; let parakeetModelStatus: ParakeetModelStatus | null = null; let parakeetModelEnsurePromise: Promise | null = null; +let parakeetModelStatusProbeCache: ReturnType> | null = null; // Persistent serve-mode process for fast transcription (models stay loaded in memory) let parakeetServerProcess: any = null; // ChildProcess @@ -393,15 +395,7 @@ function getParakeetTranscriberBinaryPath(): string { return getNativeBinaryPath('parakeet-transcriber'); } -function getParakeetModelStatus(): ParakeetModelStatus { - if (parakeetModelStatus?.state === 'downloading') { - return { ...parakeetModelStatus }; - } - if (parakeetModelStatus?.state === 'error') { - return { ...parakeetModelStatus }; - } - - // Ask the binary for the real status +function readParakeetModelStatusFromHelper(): ParakeetModelStatus { const binaryPath = getParakeetTranscriberBinaryPath(); try { if (!fs.existsSync(binaryPath)) { @@ -446,9 +440,36 @@ function getParakeetModelStatus(): ParakeetModelStatus { return parakeetModelStatus; } +function getParakeetModelStatusProbeCache(): ReturnType> { + if (!parakeetModelStatusProbeCache) { + parakeetModelStatusProbeCache = createLocalAsrHelperStatusProbeCache({ + readStatus: readParakeetModelStatusFromHelper, + }); + } + return parakeetModelStatusProbeCache; +} + +function rememberParakeetModelStatus(status: ParakeetModelStatus): ParakeetModelStatus { + parakeetModelStatus = status; + getParakeetModelStatusProbeCache().rememberStatus(status); + return status; +} + +function getParakeetModelStatus(options: { forceRefresh?: boolean } = {}): ParakeetModelStatus { + if (parakeetModelStatus?.state === 'downloading') { + return { ...parakeetModelStatus }; + } + if (parakeetModelStatus?.state === 'error') { + return { ...parakeetModelStatus }; + } + + parakeetModelStatus = getParakeetModelStatusProbeCache().getStatus(options); + return parakeetModelStatus; +} + async function ensureParakeetModelDownloaded(): Promise { // Check if already downloaded - const status = getParakeetModelStatus(); + const status = getParakeetModelStatus({ forceRefresh: true }); if (status.state === 'downloaded' && status.path) { return status.path; } @@ -522,12 +543,12 @@ async function ensureParakeetModelDownloaded(): Promise { }); }); - parakeetModelStatus = { + rememberParakeetModelStatus({ state: 'downloaded', modelName: 'parakeet-tdt-0.6b-v3', path: modelPath, progress: 1, - }; + }); console.log(`[Parakeet] Models ready at ${modelPath}`); return modelPath; } catch (error) { @@ -573,7 +594,7 @@ async function transcribeAudioWithParakeet(opts: { language?: string; mimeType?: string; }): Promise { - const status = getParakeetModelStatus(); + const status = getParakeetModelStatus({ forceRefresh: true }); if (status.state === 'downloading') { throw new Error('Parakeet models are still downloading. Finish setup from onboarding or Settings -> AI -> SuperCmd Whisper.'); } @@ -611,6 +632,7 @@ type Qwen3ModelStatus = { }; let qwen3ModelStatus: Qwen3ModelStatus | null = null; let qwen3ModelEnsurePromise: Promise | null = null; +let qwen3ModelStatusProbeCache: ReturnType> | null = null; let qwen3ServerProcess: any = null; let qwen3ServerReady = false; @@ -743,10 +765,7 @@ function sendQwen3Request(request: Record): Promise { }); } -function getQwen3ModelStatus(): Qwen3ModelStatus { - if (qwen3ModelStatus?.state === 'downloading') return { ...qwen3ModelStatus }; - if (qwen3ModelStatus?.state === 'error') return { ...qwen3ModelStatus }; - +function readQwen3ModelStatusFromHelper(): Qwen3ModelStatus { const binaryPath = getParakeetTranscriberBinaryPath(); try { if (!fs.existsSync(binaryPath)) { @@ -771,8 +790,31 @@ function getQwen3ModelStatus(): Qwen3ModelStatus { return qwen3ModelStatus; } +function getQwen3ModelStatusProbeCache(): ReturnType> { + if (!qwen3ModelStatusProbeCache) { + qwen3ModelStatusProbeCache = createLocalAsrHelperStatusProbeCache({ + readStatus: readQwen3ModelStatusFromHelper, + }); + } + return qwen3ModelStatusProbeCache; +} + +function rememberQwen3ModelStatus(status: Qwen3ModelStatus): Qwen3ModelStatus { + qwen3ModelStatus = status; + getQwen3ModelStatusProbeCache().rememberStatus(status); + return status; +} + +function getQwen3ModelStatus(options: { forceRefresh?: boolean } = {}): Qwen3ModelStatus { + if (qwen3ModelStatus?.state === 'downloading') return { ...qwen3ModelStatus }; + if (qwen3ModelStatus?.state === 'error') return { ...qwen3ModelStatus }; + + qwen3ModelStatus = getQwen3ModelStatusProbeCache().getStatus(options); + return qwen3ModelStatus; +} + async function ensureQwen3ModelDownloaded(): Promise { - const status = getQwen3ModelStatus(); + const status = getQwen3ModelStatus({ forceRefresh: true }); if (status.state === 'downloaded' && status.path) return status.path; if (qwen3ModelEnsurePromise) return await qwen3ModelEnsurePromise; @@ -816,7 +858,7 @@ async function ensureQwen3ModelDownloaded(): Promise { }); }); - qwen3ModelStatus = { state: 'downloaded', modelName: 'qwen3-asr-0.6b', path: modelPath, progress: 1 }; + rememberQwen3ModelStatus({ state: 'downloaded', modelName: 'qwen3-asr-0.6b', path: modelPath, progress: 1 }); console.log(`[Qwen3] Models ready at ${modelPath}`); return modelPath; } catch (error) { @@ -835,7 +877,7 @@ async function transcribeAudioWithQwen3(opts: { language?: string; mimeType?: string; }): Promise { - const status = getQwen3ModelStatus(); + const status = getQwen3ModelStatus({ forceRefresh: true }); if (status.state === 'downloading') throw new Error('Qwen3 models are still downloading.'); if (status.state !== 'downloaded') throw new Error('Qwen3 models have not been downloaded yet. Download them from Settings -> AI -> SuperCmd Whisper.'); @@ -17778,11 +17820,11 @@ if let tiff = image?.tiffRepresentation { ipcMain.handle('parakeet-download-model', async () => { await ensureParakeetModelDownloaded(); - return getParakeetModelStatus(); + return getParakeetModelStatus({ forceRefresh: true }); }); ipcMain.handle('parakeet-warmup', async () => { - const status = getParakeetModelStatus(); + const status = getParakeetModelStatus({ forceRefresh: true }); if (status.state !== 'downloaded') { return { ready: false, error: 'Models not downloaded' }; } @@ -17803,11 +17845,11 @@ if let tiff = image?.tiffRepresentation { ipcMain.handle('qwen3-download-model', async () => { await ensureQwen3ModelDownloaded(); - return getQwen3ModelStatus(); + return getQwen3ModelStatus({ forceRefresh: true }); }); ipcMain.handle('qwen3-warmup', async () => { - const status = getQwen3ModelStatus(); + const status = getQwen3ModelStatus({ forceRefresh: true }); if (status.state !== 'downloaded') { return { ready: false, error: 'Models not downloaded' }; }