diff --git a/.gitignore b/.gitignore index c44bbc869..7cdf53361 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,9 @@ docs/DEVICE_SESSION_COMMENTARY.md # Device wire-capture logs (raw, large, session-specific — kept locally, not versioned) docs/wire-captures/ + +# Loop simulator: generated fixtures and audio (949 MB). Rebuilt by sim/loop/*.mjs. +sim/ + +# Screenshots pasted in during device sessions. +image*.png diff --git a/__tests__/integration/models/sttResidency.test.ts b/__tests__/integration/models/sttResidency.test.ts index 9b78ace50..090c51bd4 100644 --- a/__tests__/integration/models/sttResidency.test.ts +++ b/__tests__/integration/models/sttResidency.test.ts @@ -33,6 +33,9 @@ jest.mock('../../../src/services/whisperService', () => ({ isModelDownloaded: jest.fn(async () => true), deleteModel: jest.fn(async () => {}), downloadModel: jest.fn(async () => '/models/x'), + // Residency asks before evicting. Nothing is decoding in these tests, so it is always free to go; + // the veto itself is covered where transcription is actually in flight. + isFileTranscribing: () => false, }, WHISPER_MODELS: [{ id: 'base', size: 142 }], })); diff --git a/ios/OffgridMobile/Info.plist b/ios/OffgridMobile/Info.plist index a5fff7a7b..52553950e 100644 --- a/ios/OffgridMobile/Info.plist +++ b/ios/OffgridMobile/Info.plist @@ -101,6 +101,14 @@ arm64 + + UIBackgroundModes + + audio + UISupportedInterfaceOrientations UIInterfaceOrientationPortrait diff --git a/ios/Podfile b/ios/Podfile index bf037c631..3d0395cd7 100644 --- a/ios/Podfile +++ b/ios/Podfile @@ -16,7 +16,13 @@ end target 'OffgridMobile' do config = use_native_modules! - + + # onnxruntime-objc ships no module map, and OffgridPro is a Swift pod that imports it for the + # recorder's Silero VAD. Without this, pod install fails outright with "The Swift pod `OffgridPro` + # depends upon `onnxruntime-objc`, which does not define modules" - so this line is what makes the + # ambient recorder buildable on iOS at all, not a tuning preference. + pod 'onnxruntime-objc', :modular_headers => true + use_react_native!( :path => config[:reactNativePath], # An absolute path to your application root. diff --git a/ios/Podfile.lock b/ios/Podfile.lock index 743f99b4f..8874a5df5 100644 --- a/ios/Podfile.lock +++ b/ios/Podfile.lock @@ -41,6 +41,40 @@ PODS: - MMKV (2.4.0): - MMKVCore (~> 2.4.0) - MMKVCore (2.4.0) + - OffgridPro (0.0.1): + - boost + - DoubleConversion + - fast_float + - fmt + - glog + - hermes-engine + - onnxruntime-objc + - RCT-Folly + - RCT-Folly/Fabric + - RCTRequired + - RCTTypeSafety + - React-Core + - React-debug + - React-Fabric + - React-featureflags + - React-graphics + - React-ImageManager + - React-jsi + - React-NativeModulesApple + - React-RCTFabric + - React-renderercss + - React-rendererdebug + - React-utils + - ReactCodegen + - ReactCommon/turbomodule/bridging + - ReactCommon/turbomodule/core + - SocketRocket + - Yoga + - onnxruntime-c (1.27.0) + - onnxruntime-objc (1.27.0): + - onnxruntime-objc/Core (= 1.27.0) + - onnxruntime-objc/Core (1.27.0): + - onnxruntime-c (= 1.27.0) - op-sqlite (15.2.5): - boost - DoubleConversion @@ -3091,6 +3125,11 @@ PODS: - ReactCommon/turbomodule/core - SocketRocket - Yoga + - RNNotifee (9.1.8): + - React-Core + - RNNotifee/NotifeeCore (= 9.1.8) + - RNNotifee/NotifeeCore (9.1.8): + - React-Core - RNReactNativeHapticFeedback (2.3.3): - boost - DoubleConversion @@ -3494,6 +3533,8 @@ DEPENDENCIES: - glog (from `../node_modules/react-native/third-party-podspecs/glog.podspec`) - hermes-engine (from `../node_modules/react-native/sdks/hermes-engine/hermes-engine.podspec`) - llama-rn (from `../node_modules/llama.rn`) + - OffgridPro (from `../pro`) + - onnxruntime-objc - "op-sqlite (from `../node_modules/@op-engineering/op-sqlite`)" - RCT-Folly (from `../node_modules/react-native/third-party-podspecs/RCT-Folly.podspec`) - RCTDeprecation (from `../node_modules/react-native/ReactApple/Libraries/RCTFoundation/RCTDeprecation`) @@ -3583,6 +3624,7 @@ DEPENDENCIES: - RNGestureHandler (from `../node_modules/react-native-gesture-handler`) - RNInAppBrowser (from `../node_modules/react-native-inappbrowser-reborn`) - RNKeychain (from `../node_modules/react-native-keychain`) + - "RNNotifee (from `../node_modules/@notifee/react-native`)" - RNReactNativeHapticFeedback (from `../node_modules/react-native-haptic-feedback`) - RNReanimated (from `../node_modules/react-native-reanimated`) - RNScreens (from `../node_modules/react-native-screens`) @@ -3598,6 +3640,8 @@ SPEC REPOS: trunk: - MMKV - MMKVCore + - onnxruntime-c + - onnxruntime-objc - opencv-rne - SocketRocket - SSZipArchive @@ -3620,6 +3664,8 @@ EXTERNAL SOURCES: :tag: hermes-v0.14.0 llama-rn: :path: "../node_modules/llama.rn" + OffgridPro: + :path: "../pro" op-sqlite: :path: "../node_modules/@op-engineering/op-sqlite" RCT-Folly: @@ -3796,6 +3842,8 @@ EXTERNAL SOURCES: :path: "../node_modules/react-native-inappbrowser-reborn" RNKeychain: :path: "../node_modules/react-native-keychain" + RNNotifee: + :path: "../node_modules/@notifee/react-native" RNReactNativeHapticFeedback: :path: "../node_modules/react-native-haptic-feedback" RNReanimated: @@ -3822,10 +3870,13 @@ SPEC CHECKSUMS: FBLazyVector: 309703e71d3f2f1ed7dc7889d58309c9d77a95a4 fmt: a40bb5bd0294ea969aaaba240a927bd33d878cdd glog: 5683914934d5b6e4240e497e0f4a3b42d1854183 - hermes-engine: 3de70ea2100f1780402cf146bb8110a0cdb2f34e + hermes-engine: 8c6be38f94b3bf8b864981980e64e55f08e467ec llama-rn: e6be4084699f0237fe0156c5f826cdaeb59e399e MMKV: 86859fdfa2b0b21db1fd6e48788474a6416a2c77 MMKVCore: 3d16ce9f7d411e135020915fde98a056859a1efa + OffgridPro: f187e1bf18351f0c92a1c101bfcfee75bb818807 + onnxruntime-c: 412ab51682e622e3d77ddc9176aa92517d171820 + onnxruntime-objc: f88cca350e7603f81c31b5823f3ddfaa67da9f84 op-sqlite: bafff369cecaee4fe65c89eec47deaba26f2db95 opencv-rne: 2305807573b6e29c8c87e3416ab096d09047a7a0 RCT-Folly: 846fda9475e61ec7bcbf8a3fe81edfcaeb090669 @@ -3867,7 +3918,7 @@ SPEC CHECKSUMS: react-native-background-downloader: b02d12c3961322ce1c85fa0f8b3e4adb5b652106 react-native-document-picker: dc2d83366e47e89e7c51e8a41eab99c1d54e941c react-native-document-viewer: 8c6ed07e7e27352743fa98e8dd6d288ad925b884 - react-native-executorch: 9a44ee2b18773cbe5ad2e6d7376eb76f347e2935 + react-native-executorch: 65df20362342afff0040d227d270a1b9a59f0c54 react-native-get-random-values: d16467cf726c618e9c7a8c3c39c31faa2244bbba react-native-image-picker: 0314366753615115fa55c3cc937ac44cb7e75702 react-native-keyboard-controller: 7534b5a39d1e8b2b79f86e8e998ed71c7154f69f @@ -3915,6 +3966,7 @@ SPEC CHECKSUMS: RNGestureHandler: cd4be101cfa17ea6bbd438710caa02e286a84381 RNInAppBrowser: 904d24dc75e8e6c6c98a3160329192608946f9df RNKeychain: a2c134ab796272c3d605e035ab727591000b30f3 + RNNotifee: 5e3b271e8ea7456a36eec994085543c9adca9168 RNReactNativeHapticFeedback: be4f1b4bf0398c30b59b76ed92ecb0a2ff3a69c6 RNReanimated: 292cd58688552a22b3fc1cefcfbc49b336dfed68 RNScreens: 714e10b6b554f7dc7ad9f78dcf36dc8e3fc73415 @@ -3927,6 +3979,6 @@ SPEC CHECKSUMS: whisper-rn: 7566faf9b7d78e39ab9fc634cb90fdee81177793 Yoga: 5456bb010373068fc92221140921b09d126b116e -PODFILE CHECKSUM: f66f810a788ead15881075527443239e49b50db1 +PODFILE CHECKSUM: 5e9c8ded79cfe55241281a1a60dfe0b1a8404d77 COCOAPODS: 1.16.2 diff --git a/jest.config.js b/jest.config.js index ee31e606d..a97f35a76 100644 --- a/jest.config.js +++ b/jest.config.js @@ -35,14 +35,34 @@ module.exports = { preset: 'react-native', setupFilesAfterEnv: ['/jest.setup.ts'], testMatch: ['**/__tests__/**/*.test.ts', '**/__tests__/**/*.test.tsx'], + transform: { + // pro/ has its own package.json, which makes it a separate Babel package - so a test file living + // inside it resolved no Babel config at all and TypeScript syntax (`import type`) failed to parse. + // Naming the config crosses that boundary. + // + // SCOPED to pro's own tests on purpose. Setting `configFile` globally also disables per-package + // .babelrc lookup inside node_modules, which several React Native libraries rely on. + '/pro/__tests__/.*\\.(js|jsx|ts|tsx)$': + ['babel-jest', { configFile: require.resolve('./babel.config.js') }], + '^.+\\.(js|jsx|ts|tsx|mjs)$': 'babel-jest', + // NOT optional. `transform` REPLACES the preset's map rather than merging with it, and + // react-native/jest-preset defines two entries. Overriding with only the code one silently drops + // the asset transformer, so requiring a png or svg returns its raw bytes - which surfaces as + // unrelated rendered tests failing an assertion, nowhere near the cause. + '^.+\\.(bmp|gif|jpg|jpeg|mp4|png|psd|svg|webp)$': + require.resolve('react-native/jest/assetFileTransformer.js'), + }, testPathIgnorePatterns: [ '/node_modules/', '/android/', '/ios/', '/e2e/', 'App.test.tsx', - // pro/ ships its own suite run in the pro repo's CI — never run those from here. - // Anchored to /pro/ so it ignores ONLY the submodule's own tests, NOT this - // repo's __tests__/pro/** pro-dependent suites (a bare '/pro/' matched both). - // The pro-DEPENDENT suites under this repo's __tests__ DO run against the real pro - // when it's checked out, and are ignored only when pro is genuinely absent. - '/pro/', + // Everything inside the submodule EXCEPT its own test suite. + // + // Anchored to /pro/ so it ignores only the submodule, NOT this repo's __tests__/pro/** + // pro-dependent suites (a bare '/pro/' matched both). The negative lookahead is the correction: + // `/pro/` on its own also skipped pro/__tests__, and pro has no separate runner — + // `pro`'s own `npm test` is `cd .. && npx jest`, i.e. THIS config. So a test moved into the + // submodule to keep pro source out of the public repo silently stopped executing. Two had been + // sitting there unrun. Pro-only tests belong in pro; they still have to run from here. + '/pro/(?!__tests__)', ...(proExists ? [] : proDependentTestPaths), ], // Stale agent git-worktrees under .claude/worktrees/ each carry a full repo copy (incl. their own diff --git a/package-lock.json b/package-lock.json index 2665cac7a..7f48f2c86 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@dr.pogodin/react-native-fs": "^2.38.1", "@kesha-antonov/react-native-background-downloader": "^4.5.6", "@modelcontextprotocol/sdk": "^1.29.0", + "@notifee/react-native": "^9.1.8", "@op-engineering/op-sqlite": "^15.2.5", "@react-native-async-storage/async-storage": "^2.2.0", "@react-native-community/slider": "^5.1.2", @@ -4335,6 +4336,15 @@ "node": ">= 8" } }, + "node_modules/@notifee/react-native": { + "version": "9.1.8", + "resolved": "https://registry.npmjs.org/@notifee/react-native/-/react-native-9.1.8.tgz", + "integrity": "sha512-Az/dueoPerJsbbjRxu8a558wKY+gONUrfoy3Hs++5OqbeMsR0dYe6P+4oN6twrLFyzAhEA1tEoZRvQTFDRmvQg==", + "license": "Apache-2.0", + "peerDependencies": { + "react-native": "*" + } + }, "node_modules/@op-engineering/op-sqlite": { "version": "15.2.5", "resolved": "https://registry.npmjs.org/@op-engineering/op-sqlite/-/op-sqlite-15.2.5.tgz", diff --git a/package.json b/package.json index c1995200b..10a553db2 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "@dr.pogodin/react-native-fs": "^2.38.1", "@kesha-antonov/react-native-background-downloader": "^4.5.6", "@modelcontextprotocol/sdk": "^1.29.0", + "@notifee/react-native": "^9.1.8", "@op-engineering/op-sqlite": "^15.2.5", "@react-native-async-storage/async-storage": "^2.2.0", "@react-native-community/slider": "^5.1.2", diff --git a/pro b/pro index ff0d87423..f24d7bbec 160000 --- a/pro +++ b/pro @@ -1 +1 @@ -Subproject commit ff0d874234c23d3dd2a781b77baafe8102c3fad7 +Subproject commit f24d7bbec266cb4a96f9a45188897b5cc64761e6 diff --git a/react-native.config.js b/react-native.config.js new file mode 100644 index 000000000..4405df095 --- /dev/null +++ b/react-native.config.js @@ -0,0 +1,23 @@ +const fs = require('fs'); +const path = require('path'); + +// Autolink the pro submodule's native library ONLY when it is actually on disk. Mirrors the +// fs.existsSync(pro) guard metro.config.js uses for the pro JS: a public clone without the private +// submodule sees an empty/absent pro/ dir, this entry is omitted, and the open build compiles with no +// pro native. +// +// This file says only WHERE pro is. What it contains - the Kotlin package to register, the source +// dir, the podspec - is declared by pro/react-native.config.js, which the CLI reads from this `root` +// and merges underneath. That split is the point: this repo is public, and it should not have to +// carry the private package's class names to link it. +// +// IMPORTANT: check a real file inside pro/, never just the pro/ directory - an uninitialised +// submodule leaves an empty pro/ folder behind. +const proRoot = path.resolve(__dirname, 'pro'); +const proHasNative = fs.existsSync(path.join(proRoot, 'react-native.config.js')); + +module.exports = { + dependencies: { + ...(proHasNative ? { '@offgrid/pro': { root: proRoot } } : {}), + }, +}; diff --git a/src/bootstrap/slotRegistry.ts b/src/bootstrap/slotRegistry.ts index 5866a3b17..9c9371eb5 100644 --- a/src/bootstrap/slotRegistry.ts +++ b/src/bootstrap/slotRegistry.ts @@ -53,6 +53,9 @@ function _clearSlotsForTesting(): void { /** Known slot names, centralised so core and pro stay in sync. */ export const SLOTS = { + /** A card on the Home screen, above recent conversations. Pro registers the recorder / + * conversation surface here; a free build leaves it empty and Home is unchanged. */ + homeRecorder: 'home.recorder', /** Always-mounted root component(s) rendered near the app root (e.g. the TTS * engine bridge). Mounted regardless of screen. */ appRoot: 'app.root', diff --git a/src/screens/HomeScreen/index.tsx b/src/screens/HomeScreen/index.tsx index 160f88483..2ebf17b2e 100644 --- a/src/screens/HomeScreen/index.tsx +++ b/src/screens/HomeScreen/index.tsx @@ -1,5 +1,6 @@ import React from 'react'; import { View, Text, ScrollView, TouchableOpacity } from 'react-native'; +import { useSlot, SLOTS } from '../../bootstrap/slotRegistry'; import { SafeAreaView } from 'react-native-safe-area-context'; import { Button, Card, CustomAlert, hideAlert } from '../../components'; import { AnimatedEntry } from '../../components/AnimatedEntry'; @@ -135,6 +136,8 @@ export const HomeScreen: React.FC = ({ navigation }) => { action?.(); }; + const HomeRecorderCard = useSlot(SLOTS.homeRecorder); + return ( @@ -200,6 +203,9 @@ export const HomeScreen: React.FC = ({ navigation }) => { ) } + {/* Pro's conversation/recorder card. Renders nothing when the slot is empty (free build). */} + {HomeRecorderCard ? : null} + {/* Recent Conversations */} { recentConversations.length > 0 && ( diff --git a/src/services/activeModelService/index.ts b/src/services/activeModelService/index.ts index 74d716742..eb769bcf8 100644 --- a/src/services/activeModelService/index.ts +++ b/src/services/activeModelService/index.ts @@ -105,21 +105,31 @@ class ActiveModelService { * below missed and re-loaded it (unload+load) every time the chat called * loadTextModel — the "second loader" seen only for LiteRT models. */ - private isTextModelCurrent(modelId: string): boolean { + private isTextModelCurrent(modelId: string, needsVision = false): boolean { if (this.loadedTextModelId !== modelId) return false; const model = useAppStore.getState().downloadedModels.find(m => m.id === modelId); - return model?.engine === 'litert' - ? liteRTService.isModelLoaded() - : llmService.isModelLoaded(); + if (model?.engine === 'litert') return liteRTService.isModelLoaded(); + if (!llmService.isModelLoaded()) return false; + // A text-only load of a model that HAS a projector is not "current" for a caller that wants + // vision. Without this, a background load that skipped the projector to save memory would leave + // chat unable to see images and no path back: the id matches, so every later load short-circuits + // and the projector is never initialised. + return !(needsVision && model?.mmProjPath && !llmService.getMultimodalSupport()?.vision); } + /** + * @param opts.skipVision Load WITHOUT the multimodal projector, even when the model has one. + * For background work that only needs text: a projector is hundreds of megabytes of CLIP weights + * on a device already holding a speech model. A later caller that does want vision reloads, which + * `isTextModelCurrent` now allows for. + */ async loadTextModel( modelId: string, timeoutMs: number = 120000, - opts?: { override?: boolean }, + opts?: { override?: boolean; skipVision?: boolean }, ): Promise { // Fast path — model already loaded (no lock; just sync the store). - if (this.isTextModelCurrent(modelId)) { + if (this.isTextModelCurrent(modelId, !opts?.skipVision)) { const store = useAppStore.getState(); if (store.activeModelId !== modelId) { store.setActiveModelId(modelId); @@ -135,10 +145,10 @@ class ActiveModelService { private async doLoadTextModelLocked( modelId: string, timeoutMs: number, - opts?: { override?: boolean }, + opts?: { override?: boolean; skipVision?: boolean }, ): Promise { // Re-check after acquiring — a queued call may have loaded it already. - if (this.isTextModelCurrent(modelId)) { + if (this.isTextModelCurrent(modelId, !opts?.skipVision)) { const store = useAppStore.getState(); if (store.activeModelId !== modelId) { store.setActiveModelId(modelId); @@ -177,6 +187,7 @@ class ActiveModelService { store, timeoutMs, override: !!opts?.override || modelResidencyManager.hasSessionOverride(modelId), + skipVision: opts?.skipVision, loadedTextModelId: this.loadedTextModelId, onLoaded: id => { this.setLoadedText(id); diff --git a/src/services/activeModelService/loaders.ts b/src/services/activeModelService/loaders.ts index 02a8da970..e9cc29ab5 100644 --- a/src/services/activeModelService/loaders.ts +++ b/src/services/activeModelService/loaders.ts @@ -86,6 +86,10 @@ export interface TextLoadContext { /** User forced this load ("Load Anyway"/continue) — skip the conservative native * memory gate so the loader's own fallbacks try instead of a hard block. */ override?: boolean; + /** Load WITHOUT the multimodal projector even when the model ships one. For background work that + * only needs text: the projector is hundreds of megabytes of CLIP weights that will never be read, + * on a device that is usually also holding a speech model. */ + skipVision?: boolean; onLoaded: (modelId: string) => void; onError: () => void; onFinally: () => void; @@ -185,7 +189,10 @@ export async function doLoadTextModel(ctx: TextLoadContext): Promise { ctx.onError(); // resets loadedTextModelId to null before reassignment } - const mmProjPath = await resolveMmProjPath(ctx.model, ctx.modelId); + // Resolving the projector also PERSISTS the link on first discovery, so skipping the resolve + // entirely is deliberate: a text-only load must not be the thing that decides which projector this + // model is paired with. That belongs to a load that actually intends to use it. + const mmProjPath = ctx.skipVision ? undefined : await resolveMmProjPath(ctx.model, ctx.modelId); let timeoutId: ReturnType | null = null; const timeoutPromise = new Promise((_, reject) => { diff --git a/src/services/llm.ts b/src/services/llm.ts index 35e95103d..bea0d6094 100644 --- a/src/services/llm.ts +++ b/src/services/llm.ts @@ -13,6 +13,7 @@ import { validateModelFile, checkMemoryForModel, safeCompletion, resolveSafeContext, describeGpuFallback, isTruncatedResult, } from './llmHelpers'; +import { buildConstrainedCompletionParams, type ConstrainedCompletionOpts } from './llmConstraints'; import { awaitMemoryReclaim, effectiveAvailableMB } from './memoryBudget'; import { modelResidencyManager } from './modelResidency'; import { hardwareService } from './hardware'; @@ -400,8 +401,8 @@ class LLMService { if (!this.multimodalInitialized) return false; return messages.some(m => m.attachments?.some(a => a.type === 'image')); } - /** Generate a completion with a hard token cap (used for summarization, not user-facing). */ - async generateWithMaxTokens(messages: Message[], maxTokens: number): Promise { + /** Capped completion for background work. `opts` constrains the shape; omitting it changes nothing. */ + async generateWithMaxTokens(messages: Message[], maxTokens: number, opts?: ConstrainedCompletionOpts): Promise { if (!this.context) throw new Error('No model loaded'); if (this.isGenerating) throw new Error('Generation already in progress'); this.isGenerating = true; @@ -410,13 +411,12 @@ class LLMService { let fullResponse = ''; const ctx = this.context; const completionWork = safeCompletion(ctx, () => ctx.completion( - { messages: oaiMessages, ...buildCompletionParams(settings, { disableCtxShift: this.shouldDisableCtxShift() }), n_predict: maxTokens }, + { messages: oaiMessages, ...buildCompletionParams(settings, { disableCtxShift: this.shouldDisableCtxShift() }), n_predict: maxTokens, ...buildConstrainedCompletionParams(opts) }, (data) => { if (this.isGenerating && data.token) fullResponse += data.token; }, ), 'generateWithMaxTokens'); this.activeCompletionPromise = completionWork.then(() => { }, () => { }); try { await completionWork; return fullResponse.trim(); } finally { this.isGenerating = false; this.activeCompletionPromise = null; } } - /** Ephemeral, tools-free routing pass for two-pass tool selection (not user-facing). */ async generateToolSelection(systemPrompt: string, userText: string): Promise { const messages: Message[] = [ diff --git a/src/services/llmConstraints.ts b/src/services/llmConstraints.ts new file mode 100644 index 000000000..2c5930936 --- /dev/null +++ b/src/services/llmConstraints.ts @@ -0,0 +1,35 @@ +import { buildThinkingCompletionParams } from './llmHelpers'; + +/** + * Per-call overrides for a constrained, capped generation. + * + * Separate from `llmHelpers` because these are not general helpers: they exist for background, + * structured work (summaries, extraction) where the SHAPE of the output matters more than its prose, + * and where the caller is a machine rather than a person watching tokens arrive. + */ + +/** Opts for a constrained, capped generation (see llmService.generateWithMaxTokens). */ +export interface ConstrainedCompletionOpts { + /** GBNF grammar. Constrained decoding, so the output SHAPE is guaranteed rather than hoped for. + * Measured on a 270M model: 0-2% parseable from the prompt alone, 100% under grammar. + * llama.rn only - LiteRT and remote backends ignore it, so a lenient parser is still required. */ + grammar?: string; + /** Overrides the user's global repeat penalty for one call. Small models loop on noisy input. */ + repeatPenalty?: number; + /** Reasoning OFF. It competes with a hard output cap: verified on gemma4-e2b, a capped structured + * request spent its whole budget reasoning and returned an EMPTY response (done_reason: length). */ + disableThinking?: boolean; +} + +/** + * The per-call overrides for a constrained generation, as llama.cpp completion params. Applied AFTER + * the settings spread by the caller, so an omitted field leaves the user's own setting untouched. + */ +export function buildConstrainedCompletionParams(opts?: ConstrainedCompletionOpts): Record { + if (!opts) return {}; + return { + ...(opts.grammar ? { grammar: opts.grammar } : {}), + ...(opts.repeatPenalty != null ? { penalty_repeat: opts.repeatPenalty } : {}), + ...(opts.disableThinking ? buildThinkingCompletionParams(false) : {}), + }; +} diff --git a/src/services/whisperService.ts b/src/services/whisperService.ts index 1be1f66f6..87732264a 100644 --- a/src/services/whisperService.ts +++ b/src/services/whisperService.ts @@ -29,6 +29,13 @@ class WhisperService { private isTranscribing: boolean = false; private stopFn: (() => void) | null = null; private isReleasingContext: boolean = false; + /** In-flight load, shared by concurrent callers so only ONE native context is ever created. */ + private loadPromise: Promise | null = null; + private loadingModelPath: string | null = null; + /** File transcriptions in flight. Residency's eviction veto: unloading mid-decode cancels the native + * job and whisper.rn returns empty, which reads as a clip that transcribed to nothing. */ + private fileTranscribes = 0; + isFileTranscribing(): boolean { return this.fileTranscribes > 0; } private contextReleasePromise: Promise = Promise.resolve(); private transcriptionFullyStopped: Promise = Promise.resolve(); private activeDownloadId: string | null = null; @@ -55,12 +62,9 @@ class WhisperService { // Content-Length once the download starts. const totalBytes = model.size * 1024 * 1024; const modelKey = makeModelKey(`whisper-${modelId}`, fileName); - // Publish a QUEUED row to the CANONICAL store IMMEDIATELY, before the (possibly - // slot-limited) native start — the same pattern text/image use (startModelDownload). - // Previously the store entry was only added AFTER a concurrency slot opened, so a - // queued STT download had no canonical entry and the Transcription tab fell back to - // the whisper store's progress=0 and rendered "0%" instead of "Queued". Every card - // now reads this one store, so queued looks identical across Text/Image/STT. + // Publish a QUEUED row to the CANONICAL store IMMEDIATELY, before the (possibly slot-limited) + // native start — the pattern text/image use. Added only AFTER a slot opened, a queued STT download + // had no canonical entry and the Transcription tab rendered "0%" instead of "Queued". const QUEUED_PLACEHOLDER_ID = `queued:${modelKey}`; useDownloadStore.getState().add({ modelKey, @@ -177,17 +181,34 @@ class WhisperService { } async loadModel(modelPath: string): Promise { + // SHARE an in-flight load. The checks below are correct sequentially, useless concurrently: a + // 400MB model takes seconds, so a second caller sees context===null, runs its own initWhisper and + // overwrites this.context, leaking a native allocation nothing can release. Observed on device: + // 4 contexts, none removed, 4 x 406MB, kernel then killed the foreground app (1.85GB swap). + if (this.loadPromise) { + if (this.loadingModelPath === modelPath) return this.loadPromise; + await this.loadPromise.catch(() => {}); // other model: let it settle, path check below swaps + } if (this.context && this.currentModelPath !== modelPath) await this.unloadModel(); if (this.context && this.currentModelPath === modelPath) return; if (this.isReleasingContext) { logger.log('[WhisperService] Waiting for context release to finish before loading'); await this.contextReleasePromise; } + this.loadingModelPath = modelPath; + this.loadPromise = this.doLoadModel(modelPath); + try { + await this.loadPromise; + } finally { + this.loadPromise = null; + this.loadingModelPath = null; + } + } - // Validate model file before passing to native layer. - // Native initWithModelPath calls abort() on invalid files, crashing the app. + /** The actual load. Only ever reached through `loadModel`, which serialises it. */ + private async doLoadModel(modelPath: string): Promise { + // Native initWithModelPath calls abort() on invalid files, crashing the app, so validate first. await this.validateModelFile(modelPath); - logger.log(`[Whisper] Loading model: ${modelPath}`); try { this.context = await initWhisper({ filePath: modelPath }); @@ -358,9 +379,7 @@ class WhisperService { // [WIRE] raw realtime transcription event shape from-device (voice-mode STT path) — full result + // segments + timing, so we can ground the realtime-transcript fixtures (distinct from file transcribe). logger.log(`[WIRE-STT-REALTIME] ${JSON.stringify(evt)}`); - const { isCapturing, data, processTime, recordingTime } = evt; - if (isCapturing) { // Live partial — surface immediately for the "listening…" preview. onResult({ @@ -371,7 +390,6 @@ class WhisperService { }); return; } - // FINAL: the utterance ended. Deliver the authoritative transcript — the realtime result if // it captured anything, else the file transcript (B26 fix). Emit it as the single final event. logger.log('[WhisperService] Recording finished'); @@ -448,27 +466,32 @@ class WhisperService { isCurrentlyTranscribing(): boolean { return this.isTranscribing; } - // Transcribe a single audio file + /** Transcribe one file. `maxThreads` is not optional in practice - omitted, whisper.rn resolved to + * one core of eight on device. `prompt` biases decoding toward names it would otherwise guess at. */ async transcribeFile( filePath: string, options?: { language?: string; onProgress?: (progress: number) => void; + maxThreads?: number; + prompt?: string; } ): Promise { - if (!this.context) { - throw new Error('No Whisper model loaded'); - } - + if (!this.context) throw new Error('No Whisper model loaded'); const { promise } = this.context.transcribe(filePath, { language: options?.language || 'en', onProgress: options?.onProgress, + ...(options?.maxThreads ? { maxThreads: options.maxThreads } : {}), + ...(options?.prompt ? { prompt: options.prompt } : {}), }); - - const __res = await promise; - logger.log(`[WIRE-STT] ${JSON.stringify(__res)}`); // [WIRE] raw whisper.rn transcribe result (segments/text) from-device - const { result } = __res; - return cleanTranscription(result); + this.fileTranscribes += 1; // residency's veto, released in the finally below + try { + const __res = await promise; + logger.log(`[WIRE-STT] ${JSON.stringify(__res)}`); // [WIRE] raw whisper.rn transcribe result from-device + return cleanTranscription(__res.result); + } finally { + this.fileTranscribes -= 1; + } } } diff --git a/src/stores/whisperStore.ts b/src/stores/whisperStore.ts index 8e4f9b4f1..9fa037f0a 100644 --- a/src/stores/whisperStore.ts +++ b/src/stores/whisperStore.ts @@ -149,7 +149,16 @@ export const useWhisperStore = create()( } await whisperService.loadModel(modelPath); modelResidencyManager.register( - { key: 'whisper', type: 'whisper', sizeMB }, + { + key: 'whisper', + type: 'whisper', + sizeMB, + // Residency owns eviction, but it must not evict the context out from under a running + // decode: unloading cancels the native job and whisper.rn returns an empty result, so + // the clip reads as transcribed-to-nothing rather than as interrupted. This veto is how + // the owner is told "not right now" instead of the caller second-guessing the policy. + canEvict: () => !whisperService.isFileTranscribing(), + }, () => get().unloadModel(), ); return true;