diff --git a/.agents/skills/add-api-tests/SKILL.md b/.agents/skills/add-api-tests/SKILL.md index 57cffd9276..2b69ff3693 100644 --- a/.agents/skills/add-api-tests/SKILL.md +++ b/.agents/skills/add-api-tests/SKILL.md @@ -47,7 +47,7 @@ fakeJsi.registerModel('/models/task.pte', { execute: writesOutputs([1, 0, 2]), }); -const runner = tracked(await createMyTask({ modelPath: '/models/task.pte', modelOpts })); +const runner = tracked(await createMyTask({ modelSource: '/models/task.pte', modelOpts })); expect(await runner.runTask(imageBuffer(8, 8))).toEqual(/* ... */); ``` diff --git a/.agents/skills/add-task-pipeline/SKILL.md b/.agents/skills/add-task-pipeline/SKILL.md index dd5a3d4a50..0ba47426ca 100644 --- a/.agents/skills/add-task-pipeline/SKILL.md +++ b/.agents/skills/add-task-pipeline/SKILL.md @@ -38,7 +38,7 @@ When implementing task constructors like `create` (e.g. `createClassifier` const dispose = scope.dispose; try { - const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + const model = scope.track(await wrapAsync(loadModel, runtime)(modelSource)); const { dims } = validateSpec(model.schema, { ... }); // may throw const tensors = [scope.track(tensor('float32', shapeA)), scope.track(tensor('float32', shapeB))] as const; @@ -94,6 +94,10 @@ When implementing task constructors like `create` (e.g. `createClassifier` - **Corollary (a quick smell test):** if every variant in `models.ts` passes an _identical_ options object, those fields are not configuration — move them into the task file as constants and shrink the model type to the paths that actually differ. - Do not keep a loop, parameter, or code path solely because it _looks_ more general. If the surrounding math is only valid for one value (e.g. coefficients pinned to a single distilled timestep), the generality is fake and the parameter is a correctness trap. +Task configs use `modelSource: ModelSource`, or `modelSources` for multiple models. Pass each source +unchanged to `loadModel`; tokenizer and other file-only resources keep their path fields. Callback +loaders passed to a worklet runtime must have worklet-compatible methods and captures. + ## 🚫 Avoid / Anti-Patterns - **Do NOT access tensors by index:** Avoid using `tensors[0]` or `tensors[1]` throughout the function body. Always destructure and name them explicitly. @@ -123,7 +127,7 @@ returns scores; models returning logits may need a softmax or another decoder. import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; -import { loadModel } from '../../../core/model'; +import { loadModel, type ModelSource } from '../../../core/model'; import { validateSpec, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import { createResourceScope } from '../../../core/lifetime'; @@ -135,7 +139,7 @@ export type MyTaskOptions = ImagePreprocessorOptions & { }; export type MyTaskModel = { - readonly modelPath: string; + readonly modelSource: ModelSource; readonly modelOpts: MyTaskOptions; }; @@ -168,8 +172,8 @@ export async function createMyTask( const dispose = scope.dispose; try { - const { modelPath, modelOpts } = config; - const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + const { modelSource, modelOpts } = config; + const model = scope.track(await wrapAsync(loadModel, runtime)(modelSource)); // Validate model spec const { variant, dims } = validateSpec(model.schema, { diff --git a/.agents/skills/core-guidelines/SKILL.md b/.agents/skills/core-guidelines/SKILL.md index fa0e5a06ac..d4f8406594 100644 --- a/.agents/skills/core-guidelines/SKILL.md +++ b/.agents/skills/core-guidelines/SKILL.md @@ -51,6 +51,13 @@ registry options. The [task skill](../add-task-pipeline/SKILL.md) owns that dist ## Shared conventions +- `loadModel` accepts local paths, `{ programPath, dataMapPath }` (one or multiple `.ptd` paths), + or `{ programLoader, dataMapLoader? }`. Callback sources require eager initialization. Their JS + handles are owned by the scoped `cpp/core/data_loader` session and released before loading returns; + copied ranges have native ownership. They do not support backend reads after initialization or on + worker threads. Keep model execution/disposal independent of the loading runtime; do not retain JS + callbacks in `ModelHostObject` or materialize a complete source as a fallback. + - Name options parameters `options`. Existing suffixes such as `ModelOpts` or `modelOpts` are allowed. - Native JSI wrappers use `rnexecutorchJsi` from the bridge and start with `'worklet';`. Helpers called from worklets must also be compatible with that runtime. See the native extension skill for bindings. diff --git a/apps/benchmarks/src/drivers.ts b/apps/benchmarks/src/drivers.ts index 09bf50c84f..337c186794 100644 --- a/apps/benchmarks/src/drivers.ts +++ b/apps/benchmarks/src/drivers.ts @@ -316,7 +316,7 @@ function llmDriver(): Driver { }) ); const runner = scope.track( - await llmApi.createLLMRunner(config.modelPath, config.tokenizerPath, config.modalities) + await llmApi.createLLMRunner(config.modelSource, config.tokenizerPath, config.modalities) ); return { runner, preprocessor, dispose: scope.dispose }; } catch (error) { diff --git a/apps/nlp/app/llm/index.tsx b/apps/nlp/app/llm/index.tsx index 0ea542540b..f60f07d497 100644 --- a/apps/nlp/app/llm/index.tsx +++ b/apps/nlp/app/llm/index.tsx @@ -210,11 +210,19 @@ function LLMContent() { style: 'destructive', onPress: async () => { try { - const pathsToUnlink = [ - resource.modelPath, - resource.tokenizerPath, - resource.tokenizerConfigPath, - ].filter(Boolean); + const pathsToUnlink = [resource.tokenizerPath, resource.tokenizerConfigPath]; + const { modelSource } = resource; + + if (typeof modelSource === 'string') { + pathsToUnlink.push(modelSource); + } else if ('programPath' in modelSource) { + pathsToUnlink.push(modelSource.programPath); + if (typeof modelSource.dataMapPath === 'string') { + pathsToUnlink.push(modelSource.dataMapPath); + } else { + pathsToUnlink.push(...modelSource.dataMapPath); + } + } for (const path of pathsToUnlink) { if (path && (await RNBlobUtil.fs.exists(path))) { diff --git a/docs/docs/02-extensions/computer-vision/02-image-classification.md b/docs/docs/02-extensions/computer-vision/02-image-classification.md index e428358000..97be5f57bc 100644 --- a/docs/docs/02-extensions/computer-vision/02-image-classification.md +++ b/docs/docs/02-extensions/computer-vision/02-image-classification.md @@ -143,7 +143,7 @@ configuration object to [`useClassifier`](../../06-api-reference/functions/useCl ```typescript const customClassifier = await createClassifier({ - modelPath: 'https://example.com/my-model.pte', + modelSource: 'https://example.com/my-model.pte', modelOpts: { resizeMode: 'stretch', interpolation: 'linear', diff --git a/docs/docs/02-extensions/computer-vision/03-object-detection.md b/docs/docs/02-extensions/computer-vision/03-object-detection.md index 182ee77ba7..35ba4f920a 100644 --- a/docs/docs/02-extensions/computer-vision/03-object-detection.md +++ b/docs/docs/02-extensions/computer-vision/03-object-detection.md @@ -154,7 +154,7 @@ To use your own fine-tuned object detection `.pte` model, pass an [`ObjectDetect ```typescript const customDetector = await createObjectDetector({ - modelPath: 'https://example.com/my-detector.pte', + modelSource: 'https://example.com/my-detector.pte', modelOpts: { labels: ['hardhat', 'vest', 'boots'], boxFormat: 'xyxy', diff --git a/docs/docs/02-extensions/computer-vision/04-pose-and-keypoints.md b/docs/docs/02-extensions/computer-vision/04-pose-and-keypoints.md index e1bab45f69..baba984157 100644 --- a/docs/docs/02-extensions/computer-vision/04-pose-and-keypoints.md +++ b/docs/docs/02-extensions/computer-vision/04-pose-and-keypoints.md @@ -230,7 +230,7 @@ To use your own fine-tuned pose or landmark detection `.pte` model, pass a [`Key ```typescript const customDetector = await createKeypointDetector({ - modelPath: 'https://example.com/my-pose-model.pte', + modelSource: 'https://example.com/my-pose-model.pte', modelOpts: { landmarks: ['head', 'leftHand', 'rightHand'], boxFormat: 'xyxy', diff --git a/docs/docs/02-extensions/computer-vision/05-optical-character-recognition.md b/docs/docs/02-extensions/computer-vision/05-optical-character-recognition.md index 8131467340..88250c2720 100644 --- a/docs/docs/02-extensions/computer-vision/05-optical-character-recognition.md +++ b/docs/docs/02-extensions/computer-vision/05-optical-character-recognition.md @@ -156,7 +156,7 @@ To use your own custom-trained PaddleOCR `.pte` model and character set, pass a ```typescript const customOcr = await createPaddleOcr({ - modelPath: 'https://example.com/my-ocr.pte', + modelSource: 'https://example.com/my-ocr.pte', charsetPath: 'https://example.com/charset.json', modelOpts: { defaultConfidenceThreshold: 0.5, @@ -179,7 +179,7 @@ The pipeline automatically verifies that the model's exported input and output s - [`PaddleOcr`](../../06-api-reference/type-aliases/PaddleOcr.md) — OCR runner interface ([`recognizeCharacters`](../../06-api-reference/type-aliases/PaddleOcr.md#recognizecharacters), [`recognizeCharactersWorklet`](../../06-api-reference/type-aliases/PaddleOcr.md#recognizecharactersworklet), `dispose`). - [`OcrDetection`](../../06-api-reference/type-aliases/OcrDetection.md) — Single recognized text line with [`text`](../../06-api-reference/type-aliases/OcrDetection.md#text), [`confidence`](../../06-api-reference/type-aliases/OcrDetection.md#confidence), and [`quad`](../../06-api-reference/type-aliases/OcrDetection.md#quad). - [`RecognizeCharactersOptions`](../../06-api-reference/type-aliases/RecognizeCharactersOptions.md) — Inference options (`confidenceThreshold`). -- [`PaddleOcrModel`](../../06-api-reference/type-aliases/PaddleOcrModel.md) — Model configuration spec with `modelPath` and `charsetPath`. +- [`PaddleOcrModel`](../../06-api-reference/type-aliases/PaddleOcrModel.md) — Model configuration spec with `modelSource` and `charsetPath`. - [`PaddleOcrModelOptions`](../../06-api-reference/type-aliases/PaddleOcrModelOptions.md) — Model options (`defaultConfidenceThreshold`). - [`Quad`](../../06-api-reference/react-native-executorch/namespaces/cv/type-aliases/Quad.md) — Oriented 4-corner polygon tuple `[Point, Point, Point, Point]` in pixel coordinates (`top-left`, `top-right`, `bottom-right`, `bottom-left`). - [`ImageBuffer`](../../06-api-reference/react-native-executorch/namespaces/cv/type-aliases/ImageBuffer.md) — Input image buffer structure. diff --git a/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md b/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md index 2ff8ad84eb..b1c132bdbc 100644 --- a/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md +++ b/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md @@ -140,7 +140,7 @@ To use your own fine-tuned semantic segmentation `.pte` model, pass a [`Semantic ```typescript const customSegmenter = await createSemanticSegmenter({ - modelPath: 'https://example.com/my-segmentation.pte', + modelSource: 'https://example.com/my-segmentation.pte', modelOpts: { labels: ['background', 'road', 'sidewalk', 'building'], resizeMode: 'stretch', diff --git a/docs/docs/02-extensions/computer-vision/07-instance-segmentation.md b/docs/docs/02-extensions/computer-vision/07-instance-segmentation.md index 8d4150353f..e3defd8ffa 100644 --- a/docs/docs/02-extensions/computer-vision/07-instance-segmentation.md +++ b/docs/docs/02-extensions/computer-vision/07-instance-segmentation.md @@ -154,7 +154,7 @@ To use your own fine-tuned instance segmentation `.pte` model, pass an [`Instanc ```typescript const customSegmenter = await createInstanceSegmenter({ - modelPath: 'https://example.com/my-instance-seg.pte', + modelSource: 'https://example.com/my-instance-seg.pte', modelOpts: { labels: ['bottle', 'cup', 'can'], boxFormat: 'xyxy', diff --git a/docs/docs/02-extensions/computer-vision/08-style-transfer.md b/docs/docs/02-extensions/computer-vision/08-style-transfer.md index b373a362b7..ccd8bb506b 100644 --- a/docs/docs/02-extensions/computer-vision/08-style-transfer.md +++ b/docs/docs/02-extensions/computer-vision/08-style-transfer.md @@ -123,7 +123,7 @@ To use your own trained feed-forward style transfer `.pte` model, pass a [`Style ```typescript const customStyler = await createStyleTransfer({ - modelPath: 'https://example.com/my-style.pte', + modelSource: 'https://example.com/my-style.pte', modelOpts: { resizeMode: 'stretch', interpolation: 'linear', diff --git a/docs/docs/02-extensions/computer-vision/09-image-embeddings.md b/docs/docs/02-extensions/computer-vision/09-image-embeddings.md index 596b524fa6..fd2f5088ca 100644 --- a/docs/docs/02-extensions/computer-vision/09-image-embeddings.md +++ b/docs/docs/02-extensions/computer-vision/09-image-embeddings.md @@ -148,7 +148,7 @@ To use your own fine-tuned vision encoder `.pte` model, pass an [`ImageEmbedderM ```typescript const customEmbedder = await createImageEmbedder({ - modelPath: 'https://example.com/my-vision-encoder.pte', + modelSource: 'https://example.com/my-vision-encoder.pte', modelOpts: { resizeMode: 'stretch', interpolation: 'linear', diff --git a/docs/docs/02-extensions/computer-vision/10-text-to-image.md b/docs/docs/02-extensions/computer-vision/10-text-to-image.md index f19af54da9..7804df244d 100644 --- a/docs/docs/02-extensions/computer-vision/10-text-to-image.md +++ b/docs/docs/02-extensions/computer-vision/10-text-to-image.md @@ -128,7 +128,7 @@ To use your own fine-tuned SDXS `.pte` model and CLIP tokenizer, pass a [`SdxsTe ```typescript const customGenerator = await createSdxsTextToImage({ - modelPath: 'https://example.com/my-sdxs.pte', + modelSource: 'https://example.com/my-sdxs.pte', tokenizerPath: 'https://example.com/tokenizer.json', }); ``` @@ -146,7 +146,7 @@ The pipeline automatically verifies that the model's exported methods (`encode`, ### Types & Options - [`SdxsTextToImage`](../../06-api-reference/type-aliases/SdxsTextToImage.md) — Text-to-image generator runner interface ([`generate`](../../06-api-reference/type-aliases/SdxsTextToImage.md#generate), [`generateWorklet`](../../06-api-reference/type-aliases/SdxsTextToImage.md#generateworklet)). -- [`SdxsTextToImageModel`](../../06-api-reference/type-aliases/SdxsTextToImageModel.md) — Model configuration spec with `modelPath` and `tokenizerPath`. +- [`SdxsTextToImageModel`](../../06-api-reference/type-aliases/SdxsTextToImageModel.md) — Model configuration spec with `modelSource` and `tokenizerPath`. - [`ImageBuffer`](../../06-api-reference/react-native-executorch/namespaces/cv/type-aliases/ImageBuffer.md) — Generated RGBA output image buffer structure. ### Model Presets diff --git a/docs/docs/02-extensions/computer-vision/11-image-inpainting.md b/docs/docs/02-extensions/computer-vision/11-image-inpainting.md index 33fb7f010b..1b5bd608ad 100644 --- a/docs/docs/02-extensions/computer-vision/11-image-inpainting.md +++ b/docs/docs/02-extensions/computer-vision/11-image-inpainting.md @@ -152,7 +152,7 @@ To use your own inpainting `.pte` model, pass an [`InpainterModel`](../../06-api ```typescript const customInpainter = await createInpainter({ - modelPath: 'https://example.com/my-inpainter.pte', + modelSource: 'https://example.com/my-inpainter.pte', modelOpts: { resizeMode: 'stretch', interpolation: 'linear', diff --git a/docs/docs/02-extensions/computer-vision/12-depth-estimation.md b/docs/docs/02-extensions/computer-vision/12-depth-estimation.md index e0a6648768..2eeba0f702 100644 --- a/docs/docs/02-extensions/computer-vision/12-depth-estimation.md +++ b/docs/docs/02-extensions/computer-vision/12-depth-estimation.md @@ -141,7 +141,7 @@ Pass a [`DepthEstimatorModel`](../../06-api-reference/type-aliases/DepthEstimato import { createDepthEstimator, download, IMAGENET_NORM } from 'react-native-executorch'; const model = await download({ - modelPath: 'https://example.com/my-depth-model.pte', + modelSource: 'https://example.com/my-depth-model.pte', modelOpts: { resizeMode: 'stretch' as const, interpolation: 'cubic' as const, diff --git a/docs/docs/02-extensions/natural-language/02-llm-chat-and-generation.md b/docs/docs/02-extensions/natural-language/02-llm-chat-and-generation.md index f16d5c1a13..ea020c1808 100644 --- a/docs/docs/02-extensions/natural-language/02-llm-chat-and-generation.md +++ b/docs/docs/02-extensions/natural-language/02-llm-chat-and-generation.md @@ -350,7 +350,7 @@ const { chatTemplate, bosToken, eosToken } = llm.parseTokenizerConfig( JSON.parse(await RNBlobUtil.fs.readFile(model.tokenizerConfigPath, 'utf8')) ); const preprocessor = llm.createChatPreprocessor({ chatTemplate, bosToken, eosToken }); -const runner = await llm.createLLMRunner(model.modelPath, model.tokenizerPath); +const runner = await llm.createLLMRunner(model.modelSource, model.tokenizerPath); // Runs on a background worklet thread; `generate` is synchronous. const complete = wrapAsync((r: llm.LLMRunner, prompt: llm.Prompt) => { @@ -405,7 +405,7 @@ To use your own fine-tuned LLM `.pte` model, pass an [`LLMModel`](../../06-api-r ```typescript const customSession = await createLLMChatSession({ - modelPath: 'https://example.com/my-llm.pte', + modelSource: 'https://example.com/my-llm.pte', tokenizerPath: 'https://example.com/tokenizer.json', tokenizerConfigPath: 'https://example.com/tokenizer_config.json', }); diff --git a/docs/docs/02-extensions/natural-language/03-text-embeddings.md b/docs/docs/02-extensions/natural-language/03-text-embeddings.md index ae4e151956..25652bd74f 100644 --- a/docs/docs/02-extensions/natural-language/03-text-embeddings.md +++ b/docs/docs/02-extensions/natural-language/03-text-embeddings.md @@ -168,7 +168,7 @@ To use your own fine-tuned sentence transformer `.pte` model, pass a [`TextEmbed ```typescript const customEmbedder = await createTextEmbedder({ - modelPath: 'https://example.com/my-sentence-transformer.pte', + modelSource: 'https://example.com/my-sentence-transformer.pte', tokenizerPath: 'https://example.com/tokenizer.json', defaultPrompt: 'passage: ', // Optional default prefix }); @@ -188,7 +188,7 @@ The pipeline automatically verifies that the model's exported input and output s ### Types & Options - [`TextEmbedder`](../../06-api-reference/type-aliases/TextEmbedder.md) — Text embedder runner interface ([`embed`](../../06-api-reference/type-aliases/TextEmbedder.md#embed), [`embedWorklet`](../../06-api-reference/type-aliases/TextEmbedder.md#embedworklet)). -- [`TextEmbedderModel`](../../06-api-reference/type-aliases/TextEmbedderModel.md) — Model configuration spec with `modelPath`, `tokenizerPath`, and `defaultPrompt`. +- [`TextEmbedderModel`](../../06-api-reference/type-aliases/TextEmbedderModel.md) — Model configuration spec with `modelSource`, `tokenizerPath`, and `defaultPrompt`. - [`createVectorStore()`](../../06-api-reference/functions/createVectorStore.md) — In-memory vector store with nearest-neighbor search. ### Model Presets diff --git a/docs/docs/02-extensions/natural-language/04-privacy-filter.md b/docs/docs/02-extensions/natural-language/04-privacy-filter.md index 0c57ac6118..62c8e911a4 100644 --- a/docs/docs/02-extensions/natural-language/04-privacy-filter.md +++ b/docs/docs/02-extensions/natural-language/04-privacy-filter.md @@ -163,7 +163,7 @@ To use your own fine-tuned token classification `.pte` model, pass a [`PrivacyFi ```typescript const customFilter = await createPrivacyFilter({ - modelPath: 'https://example.com/my-pii-model.pte', + modelSource: 'https://example.com/my-pii-model.pte', tokenizerPath: 'https://example.com/tokenizer.json', modelOpts: { labelNames: ['O', 'B-NAME', 'I-NAME', 'E-NAME', 'S-NAME'], diff --git a/docs/docs/02-extensions/natural-language/06-colbert-retrieval.md b/docs/docs/02-extensions/natural-language/06-colbert-retrieval.md index 43ab9c2d6e..ea330e1000 100644 --- a/docs/docs/02-extensions/natural-language/06-colbert-retrieval.md +++ b/docs/docs/02-extensions/natural-language/06-colbert-retrieval.md @@ -99,7 +99,7 @@ Any ColBERT model exported with a `[1, S]` token-id and attention-mask input and ```typescript const colbert = await createColbertEmbedder({ - modelPath: 'https://example.com/my-colbert.pte', + modelSource: 'https://example.com/my-colbert.pte', tokenizerPath: 'https://example.com/tokenizer.json', modelOpts: { queryPrefixToken: '[Q] ', diff --git a/docs/docs/03-core-and-advanced/02-models-and-tensors.md b/docs/docs/03-core-and-advanced/02-models-and-tensors.md index 9250088c08..3bc6589eac 100644 --- a/docs/docs/03-core-and-advanced/02-models-and-tensors.md +++ b/docs/docs/03-core-and-advanced/02-models-and-tensors.md @@ -337,8 +337,8 @@ tears it all down. The tensors are reused on every run rather than reallocated. ```typescript import { loadModel, tensor, wrapAsync } from 'react-native-executorch'; -export async function createSimpleClassifier(modelPath: string) { - const model = await wrapAsync(loadModel)(modelPath); +export async function createSimpleClassifier(modelSource: string) { + const model = await wrapAsync(loadModel)(modelSource); const tensors = [ tensor('float32', [1, 3, 224, 224]), // tInput @@ -405,12 +405,12 @@ normal disposal: ```typescript import { createResourceScope, loadModel, tensor, wrapAsync } from 'react-native-executorch'; -export async function createClassifier(modelPath: string) { +export async function createClassifier(modelSource: string) { const scope = createResourceScope(); const dispose = scope.dispose; try { - const model = scope.track(await wrapAsync(loadModel)(modelPath)); + const model = scope.track(await wrapAsync(loadModel)(modelSource)); const tInput = scope.track(tensor('float32', [1, 3, 224, 224])); const tOutput = scope.track(tensor('float32', [1, 1000])); diff --git a/docs/docs/03-core-and-advanced/06-worklets-and-threading.md b/docs/docs/03-core-and-advanced/06-worklets-and-threading.md index e49a0e9c74..99cb92b0ea 100644 --- a/docs/docs/03-core-and-advanced/06-worklets-and-threading.md +++ b/docs/docs/03-core-and-advanced/06-worklets-and-threading.md @@ -92,8 +92,8 @@ High-level pipelines often need to support both async calls from React component import { loadModel, tensor, wrapAsync } from 'react-native-executorch'; import type { WorkletRuntime } from 'react-native-worklets'; -export async function createClassifier(modelPath: string, runtime?: WorkletRuntime) { - const model = await wrapAsync(loadModel, runtime)(modelPath); +export async function createClassifier(modelSource: string, runtime?: WorkletRuntime) { + const model = await wrapAsync(loadModel, runtime)(modelSource); const tInput = tensor('float32', [1, 3, 224, 224]); const tOutput = tensor('float32', [1, 1000]); diff --git a/packages/react-native-executorch/__tests__/api/modelRegistry.test.ts b/packages/react-native-executorch/__tests__/api/modelRegistry.test.ts index bbe17e37c9..4aae756ecd 100644 --- a/packages/react-native-executorch/__tests__/api/modelRegistry.test.ts +++ b/packages/react-native-executorch/__tests__/api/modelRegistry.test.ts @@ -26,20 +26,20 @@ const isObject = (value: unknown): value is Node => */ const isConfig = (value: unknown): value is Node => isObject(value) && - (typeof value.modelPath === 'string' || - (isObject(value.modelPaths) && - Object.values(value.modelPaths).every((path) => typeof path === 'string'))); + (typeof value.modelSource === 'string' || + (isObject(value.modelSources) && + Object.values(value.modelSources).every((path) => typeof path === 'string'))); /** * The model file(s) a config names, as one comparable identity. */ -const modelPathsOf = (config: Node): string => - typeof config.modelPath === 'string' - ? config.modelPath - : Object.values(config.modelPaths as Node) - .map(String) - .sort() - .join('|'); +const modelPathsOf = (config: Node): string => { + if (typeof config.modelSource === 'string') return config.modelSource; + return Object.values(config.modelSources as Node) + .map(String) + .sort() + .join('|'); +}; /** * A backend variant key, e.g. `XNNPACK_FP32`. Size keys (`TINY`) look the same. @@ -249,7 +249,7 @@ describe('models registry — task configs', () => { '$label bundles a tokenizer, a VAD model and its supported languages', ({ config }) => { expect(typeof config.tokenizerPath).toBe('string'); - expect(config.vadModel).toMatchObject({ modelPath: expect.any(String) }); + expect(config.vadModel).toMatchObject({ modelSource: expect.any(String) }); expect(config.supportedLanguages).toEqual(expect.any(Array)); expect((config.supportedLanguages as unknown[]).length).toBeGreaterThan(0); } diff --git a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts index 8f53d1f19a..7e8d30be79 100644 --- a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts +++ b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts @@ -20,15 +20,15 @@ const isObject = (value: unknown): value is Node => const isConfig = (value: unknown): value is Node => isObject(value) && - (typeof value.modelPath === 'string' || - (isObject(value.modelPaths) && - Object.values(value.modelPaths).every((path) => typeof path === 'string'))); + (typeof value.modelSource === 'string' || + (isObject(value.modelSources) && + Object.values(value.modelSources).every((path) => typeof path === 'string'))); /** The `.pte` file(s) a config names, as one comparable string. */ const modelPathsOf = (config: Node): string => - typeof config.modelPath === 'string' - ? config.modelPath - : Object.values(config.modelPaths as Node) + typeof config.modelSource === 'string' + ? config.modelSource + : Object.values(config.modelSources as Node) .map(String) .sort() .join('|'); diff --git a/packages/react-native-executorch/__tests__/core/model.test.ts b/packages/react-native-executorch/__tests__/core/model.test.ts index 5e263e016d..33f3b3ebdf 100644 --- a/packages/react-native-executorch/__tests__/core/model.test.ts +++ b/packages/react-native-executorch/__tests__/core/model.test.ts @@ -1,8 +1,9 @@ -import { loadModel } from '../../src/core/model'; +import { loadModel, type ModelSource } from '../../src/core/model'; import { tensor } from '../../src/core/tensor'; import { f32, i64, method } from '../../src/core/schema'; import { fakeJsi } from '../support/fakeJsi'; import { exported, writesOutputs } from '../support/fixtures'; +import { tracked } from '../support/lifetime'; const PATH = '/models/fixture.pte'; @@ -38,6 +39,26 @@ describe('loadModel', () => { model.dispose(); }); + it.each([ + { programPath: PATH, dataMapPath: '/models/weights.ptd' }, + { programPath: PATH, dataMapPath: ['/models/weights-0.ptd', '/models/weights-1.ptd'] }, + { programLoader: { size: () => 0, load: () => new ArrayBuffer(0) } }, + { + programLoader: { size: () => 0, load: () => new ArrayBuffer(0) }, + dataMapLoader: { size: () => 0, load: () => new ArrayBuffer(0) }, + }, + ])('executes a model supplied through an object source: %o', (source) => { + fakeJsi.registerModel(source, { + schema: exported(method('value', [], [{ kind: 'Int' }])), + execute: () => [42], + }); + const model = tracked(loadModel(source)); + expect(model.path).toBe( + typeof source === 'string' || 'programPath' in source ? PATH : undefined + ); + expect(model.execute('value', [], [])).toEqual([42]); + }); + it('propagates a load failure as a thrown error', () => { expect(() => loadModel('/models/missing.pte')).toThrow(/missing.pte/); }); diff --git a/packages/react-native-executorch/__tests__/fetcher/download.test.ts b/packages/react-native-executorch/__tests__/fetcher/download.test.ts index 702ebd563f..a271f41aca 100644 --- a/packages/react-native-executorch/__tests__/fetcher/download.test.ts +++ b/packages/react-native-executorch/__tests__/fetcher/download.test.ts @@ -54,7 +54,7 @@ describe('download — resolution', () => { fakeNet.serve(URL_B); const config = { - modelPath: URL_A, + modelSource: URL_A, tokenizerPath: URL_B, localPath: '/already/local.pte', modelOpts: { labels: ['cat', 'dog'], normalizeOpts: { alpha: 0.5 } }, @@ -62,21 +62,21 @@ describe('download — resolution', () => { const resolved = await download(config); - expect(resolved.modelPath).toBe(cachedPath('model.pte')); + expect(resolved.modelSource).toBe(cachedPath('model.pte')); expect(resolved.tokenizerPath).toBe(cachedPath('tokenizer.json')); expect(resolved.localPath).toBe('/already/local.pte'); expect(resolved.modelOpts).toEqual(config.modelOpts); }); it('returns the original object identity when nothing needed downloading', async () => { - const config = { modelPath: '/local/model.pte', modelOpts: { labels: ['a'] } }; + const config = { modelSource: '/local/model.pte', modelOpts: { labels: ['a'] } }; const resolved = await download(config); expect(resolved).toBe(config); }); it('keeps untouched sub-objects referentially stable', async () => { fakeNet.serve(URL_A); - const config = { modelPath: URL_A, modelOpts: { labels: ['a'] } }; + const config = { modelSource: URL_A, modelOpts: { labels: ['a'] } }; const resolved = await download(config); diff --git a/packages/react-native-executorch/__tests__/hooks/taskHooks.test.ts b/packages/react-native-executorch/__tests__/hooks/taskHooks.test.ts index 8ba8e4fa89..904fa94bb8 100644 --- a/packages/react-native-executorch/__tests__/hooks/taskHooks.test.ts +++ b/packages/react-native-executorch/__tests__/hooks/taskHooks.test.ts @@ -5,6 +5,7 @@ */ import { renderHook, waitFor } from '@testing-library/react-native'; +import type { ModelSource } from '../../src/core/model'; import { RangeDim, constraint, f32, i64, method } from '../../src/core/schema'; import { setTelemetryEnabled } from '../../src/fetcher/telemetry'; import { useClassifier } from '../../src/hooks/useClassifier'; @@ -20,7 +21,7 @@ const TOKENIZER_URL = 'https://huggingface.co/software-mansion/model/resolve/v1/ const HF_COUNTER = 'https://huggingface.co/software-mansion/model/resolve/main/config.json'; const LABELS = ['cat', 'dog'] as const; -const config = { modelPath: MODEL_URL, modelOpts: { ...STRETCH_PREPROCESSING, labels: LABELS } }; +const config = { modelSource: MODEL_URL, modelOpts: { ...STRETCH_PREPROCESSING, labels: LABELS } }; /** * A hook downloads and loads in one uninterrupted pass, so the fake program has @@ -44,6 +45,55 @@ afterEach(() => { }); describe('useClassifier', () => { + it.each([ + { programPath: MODEL_URL, dataMapPath: MODEL_URL.replace('.pte', '.ptd') }, + { + programPath: MODEL_URL, + dataMapPath: [MODEL_URL.replace('.pte', '.0.ptd'), MODEL_URL.replace('.pte', '.1.ptd')], + }, + { programLoader: { size: () => 0, load: () => new ArrayBuffer(0) } }, + { + programLoader: { size: () => 0, load: () => new ArrayBuffer(0) }, + dataMapLoader: { size: () => 0, load: () => new ArrayBuffer(0) }, + }, + ])('loads, executes, and disposes model source %p', async (modelSource) => { + if (typeof modelSource === 'object' && 'programPath' in modelSource) { + const paths = + typeof modelSource.dataMapPath === 'string' + ? [modelSource.dataMapPath] + : modelSource.dataMapPath; + for (const path of paths) fakeNet.serve(path); + registerModelAtCachePath(); + } else { + fakeJsi.registerModel(modelSource, { + schema: exported(method('forward', [f32(1, 3, 4, 4)], [f32(1, 2)])), + execute: writesOutputs([0, 5]), + }); + } + const { result, unmount } = await renderHook(() => useClassifier({ ...config, modelSource })); + await waitFor(() => expect(result.current.isReady).toBe(true)); + expect((await result.current.classify!(imageBuffer(8, 8))).map(({ label }) => label)).toEqual([ + 'dog', + 'cat', + ]); + + if (typeof modelSource === 'object' && 'programPath' in modelSource) { + const resolved = result.current.resource?.modelSource; + expect(resolved).toEqual({ + programPath: cachePathFor(MODEL_URL), + dataMapPath: + typeof modelSource.dataMapPath === 'string' + ? cachePathFor(modelSource.dataMapPath) + : modelSource.dataMapPath.map(cachePathFor), + }); + } else { + expect(result.current.resource?.modelSource).toBe(modelSource); + } + await unmount(); + expect(fakeJsi.liveModels()).toEqual([]); + expect(fakeJsi.liveTensors()).toBe(0); + }); + it('downloads the model to the path the pipeline then loads from', async () => { // Guards the duplicated derivation in `support/cachePath.ts`. registerModelAtCachePath(); @@ -51,7 +101,7 @@ describe('useClassifier', () => { await waitFor(() => expect(result.current.resource).toBeDefined()); - expect(result.current.resource?.modelPath).toBe(cachePathFor(MODEL_URL)); + expect(result.current.resource?.modelSource).toBe(cachePathFor(MODEL_URL)); }); it('reports not-ready while the download is still in flight', async () => { @@ -145,13 +195,13 @@ describe('useClassifier', () => { }); const { result, rerender } = await renderHook( - ({ modelPath }: { modelPath: string }) => useClassifier({ ...config, modelPath }), - { initialProps: { modelPath: MODEL_URL } } + ({ modelSource }: { modelSource: string }) => useClassifier({ ...config, modelSource }), + { initialProps: { modelSource: MODEL_URL } } ); await waitFor(() => expect(result.current.isReady).toBe(true)); - await rerender({ modelPath: otherUrl }); - await waitFor(() => expect(result.current.resource?.modelPath).toBe(cachePathFor(otherUrl))); + await rerender({ modelSource: otherUrl }); + await waitFor(() => expect(result.current.resource?.modelSource).toBe(cachePathFor(otherUrl))); expect(fakeJsi.liveModels()).toEqual([cachePathFor(otherUrl)]); }); @@ -185,7 +235,7 @@ describe('useTokenizer', () => { describe('useColbertEmbedder', () => { const seq = RangeDim(2, 16); const colbertConfig = { - modelPath: MODEL_URL, + modelSource: MODEL_URL, tokenizerPath: TOKENIZER_URL, modelOpts: { queryPrefixToken: '[Q]', diff --git a/packages/react-native-executorch/__tests__/hooks/useModel.test.ts b/packages/react-native-executorch/__tests__/hooks/useModel.test.ts index 5f1b7c45f3..45adb62c6f 100644 --- a/packages/react-native-executorch/__tests__/hooks/useModel.test.ts +++ b/packages/react-native-executorch/__tests__/hooks/useModel.test.ts @@ -1,3 +1,4 @@ +import type { DataLoader } from '../../src/core/model'; import { act, renderHook, waitFor } from '@testing-library/react-native'; import { useModel } from '../../src/hooks/useModel'; @@ -16,6 +17,33 @@ const factoryOf = (instances: Instance[]) => }); describe('useModel', () => { + it('does not rebuild when only reader function identity changes', async () => { + const reader = (): DataLoader => ({ size: () => 0, load: () => new ArrayBuffer(0) }); + const first = reader(); + const second = reader(); + const factory = jest.fn(async ({ modelSource }: { modelSource: DataLoader }) => ({ + modelSource, + dispose: jest.fn(), + })); + const { result, rerender, unmount } = await renderHook( + ({ loader }: { loader: DataLoader }) => useModel(factory, { modelSource: loader }), + { initialProps: { loader: first } } + ); + await waitFor(() => expect(result.current.model?.modelSource).toBe(first)); + const previous = result.current.model!; + + await rerender({ loader: first }); + expect(factory).toHaveBeenCalledTimes(1); + + await rerender({ loader: second }); + expect(factory).toHaveBeenCalledTimes(1); + expect(result.current.model).toBe(previous); + expect(previous.dispose).not.toHaveBeenCalled(); + + await unmount(); + expect(previous.dispose).toHaveBeenCalledTimes(1); + }); + it('starts with no model and no error', async () => { const { result } = await renderHook(() => useModel(factoryOf([]), undefined)); expect(result.current).toEqual({ model: undefined, error: undefined }); diff --git a/packages/react-native-executorch/__tests__/hooks/useResourceDownload.test.ts b/packages/react-native-executorch/__tests__/hooks/useResourceDownload.test.ts index fafedadaaa..15913860d6 100644 --- a/packages/react-native-executorch/__tests__/hooks/useResourceDownload.test.ts +++ b/packages/react-native-executorch/__tests__/hooks/useResourceDownload.test.ts @@ -45,7 +45,7 @@ describe('useResourceDownload', () => { fakeNet.serve(MODEL_URL); fakeNet.serve(TOKENIZER_URL); const config = { - modelPath: MODEL_URL, + modelSource: MODEL_URL, tokenizerPath: TOKENIZER_URL, modelOpts: { labels: ['a', 'b'] }, }; @@ -54,7 +54,7 @@ describe('useResourceDownload', () => { await waitFor(() => expect(result.current.resource).toBeDefined()); expect(result.current.resource).toEqual({ - modelPath: cachedPath('model.pte'), + modelSource: cachedPath('model.pte'), tokenizerPath: cachedPath('tokenizer.json'), modelOpts: { labels: ['a', 'b'] }, }); @@ -120,13 +120,13 @@ describe('useResourceDownload', () => { it('keys the config by value, so an equal inline object does not re-download', async () => { fakeNet.serve(MODEL_URL); const { result, rerender } = await renderHook( - ({ config }: { config: { modelPath: string } }) => useResourceDownload(config), - { initialProps: { config: { modelPath: MODEL_URL } } } + ({ config }: { config: { modelSource: string } }) => useResourceDownload(config), + { initialProps: { config: { modelSource: MODEL_URL } } } ); await waitFor(() => expect(result.current.resource).toBeDefined()); const resolved = result.current.resource; - await rerender({ config: { modelPath: MODEL_URL } }); + await rerender({ config: { modelSource: MODEL_URL } }); expect(result.current.resource).toBe(resolved); }); diff --git a/packages/react-native-executorch/__tests__/support/fakeJsi.ts b/packages/react-native-executorch/__tests__/support/fakeJsi.ts index 3dbc9062e5..039b4ca51b 100644 --- a/packages/react-native-executorch/__tests__/support/fakeJsi.ts +++ b/packages/react-native-executorch/__tests__/support/fakeJsi.ts @@ -14,6 +14,7 @@ */ import { TextDecoder } from 'util'; import type { ConcreteDim, ModelSpec } from '../../src/core/schema'; +import type { LoadModelOptions, ModelSource } from '../../src/core/model'; import type { DType } from '../../src/core/tensor'; import { FakeTensor, tensorTracker } from './fakeTensor'; import { cv, fakePhonemizer, llm, math, speech } from './fakeOps'; @@ -43,25 +44,25 @@ export type FakeProgram = { execute?: FakeExecute; }; -const programs = new Map(); -const liveModels = new Set(); +const programs = new Map(); +const liveModels = new Set(); const executions: { path: string; methodName: string }[] = []; class FakeModel { - readonly path: string; + readonly path?: string; readonly schema: ModelSpec; readonly backends: Record; private readonly program: FakeProgram; private disposed = false; - constructor(path: string, program: FakeProgram) { + constructor(path: string | undefined, program: FakeProgram) { this.path = path; this.program = program; this.schema = program.schema; this.backends = program.backends ?? Object.fromEntries(Object.keys(program.schema).map((m) => [m, ['XnnpackBackend']])); - liveModels.add(path); + liveModels.add(this); } execute(methodName: string, inputs: unknown[], outputTensors: FakeTensor[]): unknown[] { @@ -69,7 +70,7 @@ class FakeModel { if (!this.schema[methodName]) { throw new Error(`execute: method '${methodName}' is not exported by '${this.path}'`); } - executions.push({ path: this.path, methodName }); + executions.push({ path: this.path ?? '(callback source)', methodName }); const values = this.program.execute?.(methodName, inputs, outputTensors); return values ? [...values] : outputTensors; } @@ -77,7 +78,7 @@ class FakeModel { dispose(): void { if (this.disposed) return; this.disposed = true; - liveModels.delete(this.path); + liveModels.delete(this); } } @@ -227,8 +228,16 @@ const jsi = { createTensor: (shape: number[], dtype: DType) => new FakeTensor(dtype, shape), - loadModel: (path: string, _options?: { eagerLoadMethods?: boolean }) => { - const program = programs.get(path); + loadModel: (source: ModelSource, _options?: LoadModelOptions) => { + // Callback sources are registered by identity. Native byte reads and the + // initialization-only lifetime contract are exercised by the C++ suites. + const path = + typeof source === 'string' + ? source + : 'programPath' in source + ? source.programPath + : undefined; + const program = programs.get(source) ?? (path === undefined ? undefined : programs.get(path)); if (!program) { throw new Error(`loadModel: no program registered at '${path}' (register one via fakeJsi)`); } @@ -266,12 +275,12 @@ export function installFakeJsi(): void { export const fakeJsi = { /** - * Makes `loadModel(path)` succeed and return `program`. - * @param path The model path a pipeline will be pointed at. + * Makes `loadModel(source)` succeed and return `program`. + * @param source The model source a pipeline will be pointed at. * @param program The schema, backends and `execute` behavior to serve. */ - registerModel(path: string, program: FakeProgram): void { - programs.set(path, program); + registerModel(source: ModelSource, program: FakeProgram): void { + programs.set(source, program); }, /** @@ -311,7 +320,7 @@ export const fakeJsi = { /** @returns Paths of models that were loaded and not disposed. */ liveModels(): string[] { - return [...liveModels].sort(); + return [...liveModels].map((model) => model.path ?? '(callback source)').sort(); }, /** @returns Paths of tokenizers that were loaded and not disposed. */ diff --git a/packages/react-native-executorch/__tests__/support/fakeLLM.ts b/packages/react-native-executorch/__tests__/support/fakeLLM.ts index 0dc4c4b2e1..4665c7f4d9 100644 --- a/packages/react-native-executorch/__tests__/support/fakeLLM.ts +++ b/packages/react-native-executorch/__tests__/support/fakeLLM.ts @@ -15,6 +15,7 @@ * the next scripted one — so prefilling the user message and the generation * prompt of the same turn does not skip a response. */ +import type { ModelSource } from '../../src/core/model'; import type { ConcreteDim, ModelSpec } from '../../src/core/schema'; import { fakeJsi } from './fakeJsi'; import type { FakeTensor } from './fakeTensor'; @@ -48,13 +49,17 @@ export type FakePromptRead = { text: string; startPos: number; numTokens: number const VOCAB_SIZE = 4096; /** - * Registers a scripted LLM at `modelPath` and its tokenizer at `tokenizerPath`. - * @param modelPath The `.pte` path the runner will load. + * Registers a scripted LLM at `modelSource` and its tokenizer at `tokenizerPath`. + * @param modelSource The model source the runner will load. * @param tokenizerPath The tokenizer path the runner will load. * @param program The context window and the responses to generate. * @returns Accessors for what the model was asked to read. */ -export function registerFakeLLM(modelPath: string, tokenizerPath: string, program: FakeLLMProgram) { +export function registerFakeLLM( + modelSource: ModelSource, + tokenizerPath: string, + program: FakeLLMProgram +) { const { maxSeqLen = 128, maxContextLen = 1024, eosToken, generations = [], stripStart } = program; const tokens: string[] = []; @@ -121,7 +126,7 @@ export function registerFakeLLM(modelPath: string, tokenizerPath: string, progra } as ModelSpec; /* eslint-enable camelcase */ - fakeJsi.registerModel(modelPath, { + fakeJsi.registerModel(modelSource, { schema, execute: (methodName, inputs, outputs) => { switch (methodName) { diff --git a/packages/react-native-executorch/__tests__/tasks/classification.test.ts b/packages/react-native-executorch/__tests__/tasks/classification.test.ts index 20f48bc510..fad589d30a 100644 --- a/packages/react-native-executorch/__tests__/tasks/classification.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/classification.test.ts @@ -18,7 +18,7 @@ const registerBatched = (labels: readonly string[] = LABELS) => { }; const config = (labels: readonly string[] = LABELS) => ({ - modelPath: MODEL_PATH, + modelSource: MODEL_PATH, modelOpts: { ...STRETCH_PREPROCESSING, labels }, }); @@ -56,7 +56,7 @@ describe('createClassifier — model acceptance', () => { it('surfaces a load failure', async () => { await expect( - createClassifier({ ...config(), modelPath: '/models/absent.pte' }) + createClassifier({ ...config(), modelSource: '/models/absent.pte' }) ).rejects.toThrow(/absent.pte/); }); }); diff --git a/packages/react-native-executorch/__tests__/tasks/colbertEmbedding.test.ts b/packages/react-native-executorch/__tests__/tasks/colbertEmbedding.test.ts index a359d87a28..8ad174e3b6 100644 --- a/packages/react-native-executorch/__tests__/tasks/colbertEmbedding.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/colbertEmbedding.test.ts @@ -35,7 +35,7 @@ const OPTIONS: ColbertEmbedderOptions = { // '?' is not in the vocabulary and must be ignored rather than fail. skiplistTokens: ['.', ',', '?'], }; -const config = { modelPath: MODEL_PATH, tokenizerPath: TOKENIZER_PATH, modelOpts: OPTIONS }; +const config = { modelSource: MODEL_PATH, tokenizerPath: TOKENIZER_PATH, modelOpts: OPTIONS }; const SEQ_SHARED = [ constraint.equality( diff --git a/packages/react-native-executorch/__tests__/tasks/constructionFailure.test.ts b/packages/react-native-executorch/__tests__/tasks/constructionFailure.test.ts index 7a2fb895a9..a330d41d4a 100644 --- a/packages/react-native-executorch/__tests__/tasks/constructionFailure.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/constructionFailure.test.ts @@ -64,10 +64,10 @@ const CV_OPTS = { defaultConfidenceThreshold: 0.5, } as const; -const cv = { modelPath: MODEL_PATH, modelOpts: CV_OPTS } as const; -const tokenized = { modelPath: MODEL_PATH, tokenizerPath: TOKENIZER_PATH } as const; +const cv = { modelSource: MODEL_PATH, modelOpts: CV_OPTS } as const; +const tokenized = { modelSource: MODEL_PATH, tokenizerPath: TOKENIZER_PATH } as const; const vadModel = { - modelPath: MODEL_PATH, + modelSource: MODEL_PATH, defaultOptions: { speechThreshold: 0.5, minSpeechDurationMs: 100, @@ -96,7 +96,7 @@ const factories: Record Promise> = { keypointDetector: () => createKeypointDetector(cv), paddleOcr: () => createPaddleOcr({ - modelPath: MODEL_PATH, + modelSource: MODEL_PATH, charsetPath: CHARSET_PATH, modelOpts: { defaultConfidenceThreshold: 0.5 }, }), @@ -128,7 +128,7 @@ const factories: Record Promise> = { supertonicTextToSpeech: () => createSupertonicTextToSpeech({ name: 'supertonic', - modelPaths: { + modelSources: { durationPredictor: MODEL_PATH, vectorEstimator: MODEL_PATH, textEncoder: MODEL_PATH, @@ -139,14 +139,14 @@ const factories: Record Promise> = { kokoroTextToSpeech: () => createKokoroTextToSpeech({ name: 'kokoro', - modelPaths: { durationPredictor: MODEL_PATH, synthesizer: MODEL_PATH }, + modelSources: { durationPredictor: MODEL_PATH, synthesizer: MODEL_PATH }, phonemizer: { lang: 'en-us' }, voices: { v: VOICE_PATH }, } as never), llmRunner: () => createLLMRunner(MODEL_PATH, TOKENIZER_PATH), llmChatSession: () => createLLMChatSession({ - modelPath: MODEL_PATH, + modelSource: MODEL_PATH, tokenizerPath: TOKENIZER_PATH, tokenizerConfigPath: TOKENIZER_CONFIG_PATH, }), @@ -203,7 +203,7 @@ describe('create — schema validation failure', () => { // factories cleaned up, this leaked one model per attempt. for (const path of ['/a.pte', '/b.pte', '/c.pte']) { fakeJsi.registerModel(path, { schema: MISMATCHED }); - await expect(createClassifier({ ...cv, modelPath: path })).rejects.toThrow(); + await expect(createClassifier({ ...cv, modelSource: path })).rejects.toThrow(); } expect(stillAllocated()).toEqual(NOTHING); diff --git a/packages/react-native-executorch/__tests__/tasks/depthEstimation.test.ts b/packages/react-native-executorch/__tests__/tasks/depthEstimation.test.ts index ccb7609f5d..3bf416a0ec 100644 --- a/packages/react-native-executorch/__tests__/tasks/depthEstimation.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/depthEstimation.test.ts @@ -8,7 +8,7 @@ import type { FakeTensor } from '../support/fakeTensor'; const MODEL_PATH = '/models/depth.pte'; const config = { - modelPath: MODEL_PATH, + modelSource: MODEL_PATH, modelOpts: { resizeMode: 'stretch', interpolation: 'cubic', diff --git a/packages/react-native-executorch/__tests__/tasks/embedding.test.ts b/packages/react-native-executorch/__tests__/tasks/embedding.test.ts index ade276639a..01540eceb8 100644 --- a/packages/react-native-executorch/__tests__/tasks/embedding.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/embedding.test.ts @@ -10,7 +10,7 @@ const MODEL_PATH = '/models/embedder.pte'; const TOKENIZER_PATH = '/models/tokenizer.json'; describe('createImageEmbedder', () => { - const config = { modelPath: MODEL_PATH, modelOpts: STRETCH_PREPROCESSING }; + const config = { modelSource: MODEL_PATH, modelOpts: STRETCH_PREPROCESSING }; it('returns the raw embedding vector at the model output dimension', async () => { fakeJsi.registerModel(MODEL_PATH, { @@ -89,7 +89,7 @@ describe('createTextEmbedder', () => { fakeJsi.registerTokenizer(TOKENIZER_PATH, { tokens: VOCAB }); }; - const config = { modelPath: MODEL_PATH, tokenizerPath: TOKENIZER_PATH }; + const config = { modelSource: MODEL_PATH, tokenizerPath: TOKENIZER_PATH }; it('validates the declared equality constraint between ids and mask', async () => { register(); diff --git a/packages/react-native-executorch/__tests__/tasks/inpainter.test.ts b/packages/react-native-executorch/__tests__/tasks/inpainter.test.ts index 7098ba64cc..b258c6564b 100644 --- a/packages/react-native-executorch/__tests__/tasks/inpainter.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/inpainter.test.ts @@ -22,7 +22,7 @@ const options = { blendOpts: { shrink: 0, feather: 0 }, } as const; -const config = { modelPath: MODEL_PATH, modelOpts: options }; +const config = { modelSource: MODEL_PATH, modelOpts: options }; const BATCHED = method( 'forward', diff --git a/packages/react-native-executorch/__tests__/tasks/kokoroTextToSpeech.test.ts b/packages/react-native-executorch/__tests__/tasks/kokoroTextToSpeech.test.ts index 25241361a6..3329952b63 100644 --- a/packages/react-native-executorch/__tests__/tasks/kokoroTextToSpeech.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/kokoroTextToSpeech.test.ts @@ -121,7 +121,7 @@ const writeVoice = (rows = MAX_TOKENS) => { /* eslint-disable camelcase */ const config = { name: 'kokoro', - modelPaths: { durationPredictor: PREDICTOR_PATH, synthesizer: SYNTHESIZER_PATH }, + modelSources: { durationPredictor: PREDICTOR_PATH, synthesizer: SYNTHESIZER_PATH }, phonemizer: { lang: 'en-us' }, voices: { af_heart: VOICE_PATH }, } as const; @@ -148,6 +148,24 @@ beforeEach(() => { }); describe('createKokoroTextToSpeech — the model contract', () => { + it('synthesizes with a split-file predictor and a callback synthesizer', async () => { + const durationPredictor = { programPath: PREDICTOR_PATH, dataMapPath: '/models/durations.ptd' }; + const synthesizer = { programLoader: { size: () => 0, load: () => new ArrayBuffer(0) } }; + fakeJsi.registerModel(synthesizer, { schema: synthesizerSchema(), execute: emitsTone }); + const tts = tracked( + await createKokoroTextToSpeech({ + ...config, + modelSources: { durationPredictor, synthesizer }, + }) + ); + const chunks = await collect(tts.synthesize('hello', { voice: 'af_heart', phonemize: false })); + expect(chunks).toHaveLength(1); + expect(chunks[0]!.audio.length).toBeGreaterThan(0); + tts.dispose(); + expect(fakeJsi.liveModels()).toEqual([]); + expect(fakeJsi.liveTensors()).toBe(0); + }); + it('accepts the exported pair and exposes the synthesis API', async () => { const tts = tracked(await createKokoroTextToSpeech(config)); diff --git a/packages/react-native-executorch/__tests__/tasks/llmChatSession.test.ts b/packages/react-native-executorch/__tests__/tasks/llmChatSession.test.ts index 867ef556b6..b19582956e 100644 --- a/packages/react-native-executorch/__tests__/tasks/llmChatSession.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/llmChatSession.test.ts @@ -11,6 +11,7 @@ * is scripted (see `support/fakeLLM.ts`), so a test can drive a tool loop * without any weights and observe exactly what the model was asked to read. */ +import type { ModelSource } from '../../src/core/model'; import { createLLMChatSession } from '../../src/extensions/llm/tasks/llmChatSession'; import { createChatPreprocessor } from '../../src/extensions/llm/utils/chatPreprocessor'; import type { ToolCall, ToolParserResult } from '../../src/extensions/llm/utils/toolCalling'; @@ -37,7 +38,7 @@ const CHAT_TEMPLATE = [ ].join(''); const config = { - modelPath: MODEL_PATH, + modelSource: MODEL_PATH, tokenizerPath: TOKENIZER_PATH, tokenizerConfigPath: TOKENIZER_CONFIG_PATH, }; @@ -108,6 +109,23 @@ describe('chat preprocessor — thinking render options', () => { }); describe('createLLMChatSession — construction', () => { + it.each([ + { programPath: MODEL_PATH, dataMapPath: ['/models/weights.0.ptd', '/models/weights.1.ptd'] }, + { programLoader: { size: () => 0, load: () => new ArrayBuffer(0) } }, + ])('generates with model source %p', async (modelSource) => { + registerFakeLLM(modelSource, TOKENIZER_PATH, { + eosToken: EOS, + generations: [{ response: 'hello there ' }], + }); + const session = tracked(await createLLMChatSession({ ...config, modelSource })); + await session.sendMessage('hi'); + expect(session.getHistory()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ role: 'assistant', content: 'hello there ' }), + ]) + ); + }); + it.each([undefined, true, false])( 'passes enableThinking=%s to templates and preserves the omitted default', async (enableThinking) => { diff --git a/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts b/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts index da24ab75fb..c7060445b0 100644 --- a/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/llmRunner.test.ts @@ -3,7 +3,7 @@ import { tensor } from '../../src/core/tensor'; import { createLLMRunner } from '../../src/extensions/llm/llmRunner'; import { createLLMMultimodalRunner } from '../../src/extensions/llm/runners/llmMultimodalRunner'; import { createLLMGemmaRunner } from '../../src/extensions/llm/runners/llmGemmaRunner'; -import { loadModel } from '../../src/core/model'; +import { loadModel, type ModelSource } from '../../src/core/model'; import { loadTokenizer } from '../../src/extensions/nlp'; import { fakeJsi } from '../support/fakeJsi'; import type { FakeTensor } from '../support/fakeTensor'; @@ -98,6 +98,7 @@ const GEMMA_SPEC = exported({ function registerModelWithSchema( schema: any, options: { + modelSource?: ModelSource; maxContextLen?: number; tokenIds?: readonly number[]; onForward?: (inputs: readonly FakeTensor[]) => void; @@ -106,7 +107,7 @@ function registerModelWithSchema( ) { const maxContextLen = options.maxContextLen ?? 512; let step = 0; - fakeJsi.registerModel(MODEL_PATH, { + fakeJsi.registerModel(options.modelSource ?? MODEL_PATH, { schema, execute: (methodName, _inputs, outputs) => { switch (methodName) { @@ -400,6 +401,24 @@ describe.each([ }); describe('createLLMRunner — variant dispatch & modalities validation', () => { + it.each([ + { programPath: MODEL_PATH, dataMapPath: '/models/weights.ptd' }, + { programPath: MODEL_PATH, dataMapPath: ['/models/weights-0.ptd', '/models/weights-1.ptd'] }, + { programLoader: { size: () => 0, load: () => new ArrayBuffer(0) } }, + { + programLoader: { size: () => 0, load: () => new ArrayBuffer(0) }, + dataMapLoader: { size: () => 0, load: () => new ArrayBuffer(0) }, + }, + ])('prefills a runner loaded from an object source: %o', async (source) => { + registerModelWithSchema(TEXT_SPEC, { modelSource: source }); + const runner = tracked(await createLLMRunner(source, TOKENIZER_PATH)); + expect(runner.modelPath).toBe( + typeof source === 'string' || 'programPath' in source ? MODEL_PATH : undefined + ); + expect(runner.prefill('hello').numTokens).toBe(1); + expect(runner.getKVCacheState().pos).toBe(1); + }); + it('dispatches to text runner when schema matches LLM_TEXT_SPEC', async () => { registerModelWithSchema(TEXT_SPEC); const runner = tracked(await createLLMRunner(MODEL_PATH, TOKENIZER_PATH)); diff --git a/packages/react-native-executorch/__tests__/tasks/llmTranslation.test.ts b/packages/react-native-executorch/__tests__/tasks/llmTranslation.test.ts index b73507e3ba..f6b04dd607 100644 --- a/packages/react-native-executorch/__tests__/tasks/llmTranslation.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/llmTranslation.test.ts @@ -5,6 +5,7 @@ * is scripted (see `support/fakeLLM.ts`), so a test sees the exact prompt each * line was sent with. */ +import type { ModelSource } from '../../src/core/model'; import { createLLMTranslator } from '../../src/extensions/llm/tasks/llmTranslation'; import { fakeJsi } from '../support/fakeJsi'; import { fakeFs } from '../support/blobUtilMock'; @@ -22,7 +23,7 @@ const CHAT_TEMPLATE = "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}<|im_start|>system\n{% if messages[0]['role'] == 'system' %}{{ messages[0]['content'] }}{% endif %}<|im_end|>\n{% for message in messages %}{% if message['role'] != 'system' %}<|im_start|>{{ message['role'] }}\n{{ message['content'] }}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"; const paths = { - modelPath: MODEL_PATH, + modelSource: MODEL_PATH, tokenizerPath: TOKENIZER_PATH, tokenizerConfigPath: TOKENIZER_CONFIG_PATH, }; @@ -65,6 +66,18 @@ beforeEach(() => { }); describe('createLLMTranslator', () => { + it.each([ + { programPath: MODEL_PATH, dataMapPath: ['/models/weights.0.ptd', '/models/weights.1.ptd'] }, + { programLoader: { size: () => 0, load: () => new ArrayBuffer(0) } }, + ])('generates with model source %p', async (modelSource) => { + registerFakeLLM(modelSource, TOKENIZER_PATH, { + eosToken: EOS, + generations: [{ response: 'hello there ' }], + }); + const translator = tracked(await createLLMTranslator({ ...config, modelSource })); + expect(await translator.translate('Hello', EN_DE)).toBe('hello there '); + }); + it('sends each line with the model card prompt and keeps line breaks', async () => { script('Hallo Welt.', 'Guten Morgen.'); const translator = tracked(await createLLMTranslator(config)); diff --git a/packages/react-native-executorch/__tests__/tasks/marianTranslation.test.ts b/packages/react-native-executorch/__tests__/tasks/marianTranslation.test.ts index cf9a5889b3..f2478f85f5 100644 --- a/packages/react-native-executorch/__tests__/tasks/marianTranslation.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/marianTranslation.test.ts @@ -16,7 +16,7 @@ const NUM_CROSS_KV = 12; const config = { name: 'marian' as const, - modelPath: MODEL_PATH, + modelSource: MODEL_PATH, tokenizerPath: TOKENIZER_PATH, sourceLanguage: 'en', targetLanguage: 'de', diff --git a/packages/react-native-executorch/__tests__/tasks/objectDetection.test.ts b/packages/react-native-executorch/__tests__/tasks/objectDetection.test.ts index ba5fbfbfd4..f04cc849ba 100644 --- a/packages/react-native-executorch/__tests__/tasks/objectDetection.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/objectDetection.test.ts @@ -50,7 +50,7 @@ const register = (classes: number[] = CLASSES) => { }); }; -const config = { modelPath: MODEL_PATH, modelOpts: options }; +const config = { modelSource: MODEL_PATH, modelOpts: options }; describe('createObjectDetector — detection', () => { beforeEach(() => register()); diff --git a/packages/react-native-executorch/__tests__/tasks/paddleOcr.test.ts b/packages/react-native-executorch/__tests__/tasks/paddleOcr.test.ts index b797a111a2..8c7788bc4a 100644 --- a/packages/react-native-executorch/__tests__/tasks/paddleOcr.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/paddleOcr.test.ts @@ -110,7 +110,7 @@ const detectsAndReads = (regions: Region[], words: string[], confidence = 0.9) = }; const config = { - modelPath: MODEL_PATH, + modelSource: MODEL_PATH, charsetPath: CHARSET_PATH, modelOpts: { defaultConfidenceThreshold: 0.5 }, }; diff --git a/packages/react-native-executorch/__tests__/tasks/privacyFilter.test.ts b/packages/react-native-executorch/__tests__/tasks/privacyFilter.test.ts index 30ffbe0390..50279a3420 100644 --- a/packages/react-native-executorch/__tests__/tasks/privacyFilter.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/privacyFilter.test.ts @@ -90,7 +90,7 @@ const recordingLengths = }; const config = { - modelPath: MODEL_PATH, + modelSource: MODEL_PATH, tokenizerPath: TOKENIZER_PATH, modelOpts: { labelNames: LABELS, padTokenId: PAD_TOKEN_ID }, }; diff --git a/packages/react-native-executorch/__tests__/tasks/qwen3AsrSpeechToText.test.ts b/packages/react-native-executorch/__tests__/tasks/qwen3AsrSpeechToText.test.ts index b3f91fb02d..76b07a5dcf 100644 --- a/packages/react-native-executorch/__tests__/tasks/qwen3AsrSpeechToText.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/qwen3AsrSpeechToText.test.ts @@ -20,7 +20,7 @@ const VAD_PATH = '/models/vad.pte'; const VAD_FRAMES = RangeDim(1, 500); const VAD_SCHEMA = exported(method('forward', [f32(VAD_FRAMES, 400)], [f32(1, VAD_FRAMES, 2)])); const VAD_CONFIG = { - modelPath: VAD_PATH, + modelSource: VAD_PATH, defaultOptions: { speechThreshold: 0.5, minSpeechDurationMs: 250, @@ -32,7 +32,7 @@ const VAD_CONFIG = { const config = { name: 'qwen3-asr', - modelPath: MODEL_PATH, + modelSource: MODEL_PATH, tokenizerPath: TOKENIZER_PATH, vadModel: VAD_CONFIG, } as const; diff --git a/packages/react-native-executorch/__tests__/tasks/remainingTasks.test.ts b/packages/react-native-executorch/__tests__/tasks/remainingTasks.test.ts index 96f8008a2c..bfc2d50cad 100644 --- a/packages/react-native-executorch/__tests__/tasks/remainingTasks.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/remainingTasks.test.ts @@ -36,7 +36,7 @@ const VAD_PATH = '/models/vad.pte'; describe('createKeypointDetector', () => { const LANDMARKS = ['nose', 'left_eye', 'right_eye'] as const; const config = { - modelPath: MODEL_PATH, + modelSource: MODEL_PATH, modelOpts: { ...STRETCH_PREPROCESSING, resizeMode: 'stretch', @@ -169,7 +169,7 @@ describe('createKeypointDetector', () => { const KEYPOINTS = [1, 1, 0.9, 2, 2, 0.8, 3, 3, 0.7, 1, 1, 0.01, 2, 2, 0.01, 3, 3, 0.01]; const detectorConfig = { - modelPath: MODEL_PATH, + modelSource: MODEL_PATH, modelOpts: { ...config.modelOpts, defaultIouThreshold: 0.5 }, } as const; @@ -219,7 +219,7 @@ describe('createKeypointDetector', () => { describe('createInstanceSegmenter', () => { const config = { - modelPath: MODEL_PATH, + modelSource: MODEL_PATH, modelOpts: { ...STRETCH_PREPROCESSING, resizeMode: 'stretch', @@ -282,7 +282,7 @@ const FRAMES = RangeDim(1, 500); const VAD_SCHEMA = exported(method('forward', [f32(FRAMES, 400)], [f32(1, FRAMES, 2)])); const VAD_CONFIG = { - modelPath: VAD_PATH, + modelSource: VAD_PATH, defaultOptions: { speechThreshold: 0.5, minSpeechDurationMs: 250, @@ -363,7 +363,7 @@ describe('createWhisperSpeechToText', () => { // Whisper pads internally, so the exported encode input is a dynamic range. const AUDIO_SAMPLES = RangeDim(1, 480000); const config = { - modelPath: MODEL_PATH, + modelSource: MODEL_PATH, tokenizerPath: TOKENIZER_PATH, supportedLanguages: ['en'], vadModel: VAD_CONFIG, @@ -487,7 +487,7 @@ describe('createWhisperSpeechToText', () => { // ============================================================================ describe('createSdxsTextToImage', () => { - const config = { modelPath: MODEL_PATH, tokenizerPath: TOKENIZER_PATH }; + const config = { modelSource: MODEL_PATH, tokenizerPath: TOKENIZER_PATH }; const SDXS_SCHEMA = exported({ ...method('encode', [i64(1, 77)], [f32(1, 77, 768)]), diff --git a/packages/react-native-executorch/__tests__/tasks/semanticSegmentation.test.ts b/packages/react-native-executorch/__tests__/tasks/semanticSegmentation.test.ts index 4e907abea4..bc327817f9 100644 --- a/packages/react-native-executorch/__tests__/tasks/semanticSegmentation.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/semanticSegmentation.test.ts @@ -15,7 +15,7 @@ const options = { labels: LABELS, } as const; -const config = { modelPath: MODEL_PATH, modelOpts: options }; +const config = { modelSource: MODEL_PATH, modelOpts: options }; /** * Per-class logits for a 2x2 image in CHW order. Class 1 wins the top-left diff --git a/packages/react-native-executorch/__tests__/tasks/styleTransfer.test.ts b/packages/react-native-executorch/__tests__/tasks/styleTransfer.test.ts index 310cf1317b..2ece542d3c 100644 --- a/packages/react-native-executorch/__tests__/tasks/styleTransfer.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/styleTransfer.test.ts @@ -19,7 +19,7 @@ const options = { outInterpolation: 'nearest', } as const; -const config = { modelPath: MODEL_PATH, modelOpts: options }; +const config = { modelSource: MODEL_PATH, modelOpts: options }; /** An identity model: whatever preprocessing produced comes straight back. */ const registerIdentity = () => { diff --git a/packages/react-native-executorch/compile_flags.txt b/packages/react-native-executorch/compile_flags.txt index ca161d47eb..e4c1cff04a 100644 --- a/packages/react-native-executorch/compile_flags.txt +++ b/packages/react-native-executorch/compile_flags.txt @@ -16,3 +16,4 @@ -isystem.native-test-deps/hermes/src/API -isystem.native-test-deps/hermes/src/API/jsi -isystem.native-test-deps/hermes/src/public +-isystem.native-test-deps/executorch/third-party/flatbuffers/include diff --git a/packages/react-native-executorch/cpp/core/data_loader.cpp b/packages/react-native-executorch/cpp/core/data_loader.cpp new file mode 100644 index 0000000000..d0ec97b8a9 --- /dev/null +++ b/packages/react-native-executorch/cpp/core/data_loader.cpp @@ -0,0 +1,149 @@ +#include "data_loader.h" + +#include +#include +#include +#include +#include +#include + +#include "conversions.h" +#include "error.h" + +namespace rnexecutorch::core::data_loader { +namespace jsi = facebook::jsi; +using executorch::runtime::DataLoader; +using executorch::runtime::Error; +using executorch::runtime::FreeableBuffer; +using executorch::runtime::Result; + +namespace { +void freeBuffer(void * /*context*/, void *data, size_t /*size*/) { + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays): matching deleter for the owning runtime-sized byte buffer + std::default_delete{}(static_cast(data)); +} +} // namespace + +struct LoadSession::State { + struct Callbacks { + jsi::Runtime *rt; + std::string ctx; + jsi::Object receiver; + jsi::Function load; + std::exception_ptr failure; + + jsi::ArrayBuffer read(size_t offset, size_t length) { + auto result = load.callWithThis(*rt, receiver, static_cast(offset), static_cast(length)); + auto buffer = conversions::asType(*rt, ctx + ".load()", result); + if (buffer.size(*rt) != length) { + throw error::InvalidArgument(std::format("{}.load(): returned {} bytes, expected {}", + ctx, buffer.size(*rt), length)); + } + return buffer; + } + }; + + size_t byteLength; + std::thread::id ownerThread; + std::unique_ptr callbacks; +}; + +class LoadSession::Adapter final : public DataLoader { +public: + explicit Adapter(std::shared_ptr state) : state_(std::move(state)) {} + + [[nodiscard]] Result load(size_t offset, size_t length, const SegmentInfo & /*segmentInfo*/) const override { + if (auto status = validateRead(offset, length); status != Error::Ok) { + return status; + } + if (length == 0) { + return FreeableBuffer(nullptr, 0, nullptr); + } + auto &callbacks = *state_->callbacks; + try { + auto buffer = callbacks.read(offset, length); + // NOLINTNEXTLINE(cppcoreguidelines-avoid-c-arrays,modernize-avoid-c-arrays): owning runtime-sized byte buffer + auto copy = std::make_unique_for_overwrite(length); + std::memcpy(copy.get(), buffer.data(*callbacks.rt), length); + return FreeableBuffer(copy.release(), length, freeBuffer); + } catch (...) { + callbacks.failure = std::current_exception(); + return Error::AccessFailed; + } + } + + Error load_into(size_t offset, size_t length, const SegmentInfo & /*segmentInfo*/, void *destination) const override { + if (destination == nullptr && length != 0) { + return Error::InvalidArgument; + } + if (auto status = validateRead(offset, length); status != Error::Ok) { + return status; + } + if (length == 0) { + return Error::Ok; + } + auto &callbacks = *state_->callbacks; + try { + auto buffer = callbacks.read(offset, length); + std::memcpy(destination, buffer.data(*callbacks.rt), length); + return Error::Ok; + } catch (...) { + callbacks.failure = std::current_exception(); + return Error::AccessFailed; + } + } + + [[nodiscard]] Result size() const override { return state_->byteLength; } + +private: + [[nodiscard]] Error validateRead(size_t offset, size_t length) const { + if (offset > state_->byteLength || length > state_->byteLength - offset) { + return Error::InvalidArgument; + } + // Worker threads must never access the JS handles, including during + // initialization. Only the owner thread reads/resets callbacks. + if (state_->ownerThread != std::this_thread::get_id()) { + return Error::InvalidState; + } + if (!state_->callbacks) { + return Error::InvalidState; + } + return state_->callbacks->failure ? Error::AccessFailed : Error::Ok; + } + + std::shared_ptr state_; +}; + +LoadSession::LoadSession(jsi::Runtime &rt) : rt_(rt) {} + +LoadSession::~LoadSession() { + for (const auto &state : states_) { + state->callbacks.reset(); + } +} + +std::unique_ptr LoadSession::create(const std::string &ctx, const jsi::Value &value) { + auto receiver = conversions::asType(rt_, ctx, value); + auto size = conversions::getRequiredProperty(rt_, ctx, receiver, "size"); + auto load = conversions::getRequiredProperty(rt_, ctx, receiver, "load"); + + auto result = size.callWithThis(rt_, receiver); + const auto length = conversions::asType(rt_, ctx + ".size()", result); + + auto callbacks = std::make_unique(&rt_, ctx, std::move(receiver), std::move(load), nullptr); + auto state = std::make_shared(static_cast(length), std::this_thread::get_id(), std::move(callbacks)); + + states_.push_back(state); + + return std::make_unique(std::move(state)); +} + +void LoadSession::rethrowIfFailed() const { + for (const auto &state : states_) { + if (state->callbacks->failure) { + std::rethrow_exception(state->callbacks->failure); + } + } +} + +} // namespace rnexecutorch::core::data_loader diff --git a/packages/react-native-executorch/cpp/core/data_loader.h b/packages/react-native-executorch/cpp/core/data_loader.h new file mode 100644 index 0000000000..1c3b8b2eb3 --- /dev/null +++ b/packages/react-native-executorch/cpp/core/data_loader.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include + +#include +#include + +namespace rnexecutorch::core::data_loader { + +/** + * Owns JS callbacks only for the duration of synchronous model initialization. + * Adapters handed to Module may outlive this scope, but retain no JS handles + * afterward. Their already-loaded buffers have independent native ownership. + * Nonempty reads after initialization return InvalidState; there is no lazy + * callback or whole-source cache. + */ +class LoadSession final { +public: + explicit LoadSession(facebook::jsi::Runtime &rt); + ~LoadSession(); + + LoadSession(const LoadSession &) = delete; + LoadSession &operator=(const LoadSession &) = delete; + LoadSession(LoadSession &&) = delete; + LoadSession &operator=(LoadSession &&) = delete; + + /** + * Creates an adapter for a JS { size, load } object. Calls size once. + * @throws error::RnExecuTorchException with code InvalidArgument for invalid + * callbacks or sizes. Exceptions from the caller's size callback propagate. + */ + std::unique_ptr create(const std::string &ctx, + const facebook::jsi::Value &value); + + /// Rethrows the original callback/validation exception instead of ET's + /// transport error. Must run before this scope releases its JS handles. + void rethrowIfFailed() const; + +private: + struct State; + class Adapter; + + facebook::jsi::Runtime &rt_; + std::vector> states_; +}; + +} // namespace rnexecutorch::core::data_loader diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index 40c01d474b..36b64eed23 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -6,11 +6,14 @@ #include #include #include +#include #include #include #include #include +#include "conversions.h" +#include "data_loader.h" #include "dtype.h" #include "error.h" #include "profiler.h" @@ -24,6 +27,7 @@ #include namespace { +namespace error = rnexecutorch::core::error; using rnexecutorch::core::error::RnExecuTorchErrorCode; using rnexecutorch::core::error::RnExecuTorchException; @@ -44,9 +48,6 @@ void unwrap(RnExecuTorchErrorCode code, const std::string &ctx, executorch::runt } } -// A file can mix backends across methods: Qwen3-ASR's MLX file runs its audio -// tower on Core ML. Name the backend an app did not link instead of leaving -// load_method to fail with a generic error. void requireRegisteredBackends(const std::string &modelPath, const std::string &methodName, const std::vector &backends) { for (const auto &name : backends) { @@ -55,11 +56,11 @@ void requireRegisteredBackends(const std::string &modelPath, const std::string & } auto configName = name.substr(0, name.rfind("Backend")); std::ranges::transform(configName, configName.begin(), [](unsigned char c) { return static_cast(std::tolower(c)); }); - throw rnexecutorch::core::error::LoadFailed( - std::format("Model '{}': method '{}' runs on {}, which this app does not link. Add \"{}\" to " - "\"react-native-executorch\".backends in the app's package.json.", - modelPath, methodName, name, configName), - executorch::runtime::Error::NotFound); + + const auto errorMsg = std::format("Model '{}': method '{}' runs on {}, which this app does not link. Add \"{}\" to " + "\"react-native-executorch\".backends in the app's package.json.", + modelPath, methodName, name, configName); + throw error::LoadFailed(errorMsg, executorch::runtime::Error::NotFound); } } @@ -72,14 +73,16 @@ namespace conversions = rnexecutorch::core::conversions; using executorch::extension::Module; using rnexecutorch::core::tensor::TensorHostObject; -ModelHostObject::ModelHostObject(const std::string &modelPath, const bool eagerLoadMethods) - : modelPath_(modelPath), - etModule_(std::make_unique(modelPath, Module::LoadMode::Mmap)) { +ModelHostObject::ModelHostObject(std::optional modelPath, + std::unique_ptr module, + const bool eagerLoadMethods) + : modelPath_(std::move(modelPath)), + etModule_(std::move(module)) { auto loadError = etModule_->load(); if (!etModule_->is_loaded()) { const std::string errorMsg = executorch::runtime::to_string(loadError); - throw error::LoadFailed(std::format("Failed to load model from '{}': {}", modelPath_, errorMsg), + throw error::LoadFailed(std::format("Failed to load model from '{}': {}", modelPath_.value_or("callback source"), errorMsg), loadError); } @@ -104,7 +107,7 @@ ModelHostObject::ModelHostObject(const std::string &modelPath, const bool eagerL backends_[methodName] = schema::getUsedBackends(methodMeta); if (eagerLoadMethods && methodName != kGetModelSchemaMethod) { - requireRegisteredBackends(modelPath_, methodName, backends_[methodName]); + requireRegisteredBackends(modelPath_.value_or("callback source"), methodName, backends_[methodName]); // Every backend is registered by now, so a failure points at the export. auto loadCtx = std::format("Load method '{}' failed. The model export might be broken or " "incompatible with the linked ExecuTorch runtime version. Error", @@ -126,7 +129,7 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { auto nameStr = name.utf8(rt); if (nameStr == "path") { - return jsi::String::createFromUtf8(rt, modelPath_); + return modelPath_ ? jsi::String::createFromUtf8(rt, *modelPath_) : jsi::Value::undefined(); } if (nameStr == "schema") { @@ -212,7 +215,7 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { schema::validateRuntimeConstraints(rt, methodSpec.runtimeConstraints, inputShapes, std::format("execute: method '{}'", methodName)); if (!self->etModule_->is_method_loaded(methodName)) { - requireRegisteredBackends(self->modelPath_, methodName, self->backends_.at(methodName)); + requireRegisteredBackends(self->modelPath_.value_or("callback source"), methodName, self->backends_.at(methodName)); } auto startTime = std::chrono::high_resolution_clock::now(); @@ -222,8 +225,10 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { #ifdef EXECUTORCH_ENABLE_EXECUTION_PROFILING // Recorded in nanoseconds: the millisecond log rounds a sub-millisecond inference to 0. auto durationNs = std::chrono::duration_cast(finishTime - startTime).count(); - profiler::record(self->modelPath_, methodName, durationNs); auto durationMs = durationNs / 1'000'000; + + profiler::record(self->modelPath_.value_or("callback source"), methodName, durationNs); + auto consoleObj = conversions::asType(rt, "console", rt.global().getProperty(rt, "console")); auto logFn = conversions::asType(rt, "console.log", consoleObj.getProperty(rt, "log")); auto info = std::format("Execution of method '{}' took {} ms", methodName, durationMs); @@ -353,18 +358,75 @@ void install_loadModel(jsi::Runtime &rt, jsi::Object &module) { const auto *name = "loadModel"; auto fnBody = [](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { if (count < 1 || count > 2) { - throw error::InvalidArgument("loadModel: Usage: loadModel(path, options?)"); + throw error::InvalidArgument("loadModel: Usage: loadModel(source, options?)"); } - auto modelPath = conversions::asType(rt, "loadModel: path", args[0]); bool eagerLoadMethods = true; - if (count >= 2 && !args[1].isUndefined() && !args[1].isNull()) { auto options = conversions::asType(rt, "loadModel: options", args[1]); eagerLoadMethods = conversions::getOptionalProperty(rt, "loadModel: options", options, "eagerLoadMethods").value_or(true); } - return jsi::Object::createFromHostObject(rt, std::make_shared(modelPath, eagerLoadMethods)); + if (args[0].isString()) { + auto path = conversions::asType(rt, "loadModel: source", args[0]); + auto etModule = std::make_unique(path, Module::LoadMode::Mmap); + auto model = std::make_shared(std::move(path), std::move(etModule), eagerLoadMethods); + return jsi::Object::createFromHostObject(rt, std::move(model)); + } + + auto source = conversions::asType(rt, "loadModel: source", args[0]); + auto programPath = conversions::getOptionalProperty(rt, "loadModel: source", source, "programPath"); + auto dataMapPath = conversions::getOptionalProperty(rt, "loadModel: source", source, "dataMapPath"); + auto programLoader = conversions::getOptionalProperty(rt, "loadModel: source", source, "programLoader"); + auto dataMapLoader = conversions::getOptionalProperty(rt, "loadModel: source", source, "dataMapLoader"); + + if (programPath && dataMapPath) { + std::vector dataPaths; + + if (dataMapPath->isString()) { + auto dataPath = conversions::asType(rt, "loadModel: source.dataMapPath", *dataMapPath); + dataPaths.push_back(std::move(dataPath)); + } else { + dataPaths = conversions::asVector(rt, "loadModel: source.dataMapPath", *dataMapPath); + } + + if (dataPaths.empty() || std::ranges::any_of(dataPaths, [](const auto &path) { return path.empty(); })) { + throw error::InvalidArgument("loadModel: source.dataMapPath must contain at least one nonempty path"); + } + + auto etModule = std::make_unique(*programPath, std::move(dataPaths), Module::LoadMode::Mmap); + auto model = std::make_shared(std::move(programPath), std::move(etModule), eagerLoadMethods); + + return jsi::Object::createFromHostObject(rt, std::move(model)); + } + + if (programLoader) { + if (!eagerLoadMethods) { + throw error::InvalidArgument("loadModel: callback loaders require eagerLoadMethods: true"); + } + + data_loader::LoadSession loaders(rt); + auto program = loaders.create("loadModel: source.programLoader", *programLoader); + + std::unique_ptr data; + if (dataMapLoader) { + data = loaders.create("loadModel: source.dataMapLoader", *dataMapLoader); + } + + auto etModule = std::make_unique(std::move(program), nullptr, nullptr, nullptr, std::move(data)); + + try { + auto model = std::make_shared(std::nullopt, std::move(etModule), eagerLoadMethods); + loaders.rethrowIfFailed(); + + return jsi::Object::createFromHostObject(rt, std::move(model)); + } catch (...) { + loaders.rethrowIfFailed(); + throw; + } + } + + throw error::InvalidArgument("loadModel: source must contain either programPath/dataMapPath or programLoader/dataMapLoader"); }; auto fn = jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, name), 2, error::guarded(fnBody)); diff --git a/packages/react-native-executorch/cpp/core/model.h b/packages/react-native-executorch/cpp/core/model.h index 03eb603bdd..3b96dd9457 100644 --- a/packages/react-native-executorch/cpp/core/model.h +++ b/packages/react-native-executorch/cpp/core/model.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -34,20 +35,23 @@ class ModelHostObject final : public jsi::HostObject, public std::enable_shared_from_this { public: /** - * Loads the ExecuTorch model from the specified file path, initializes its + * Loads the supplied ExecuTorch module, initializes its * method metadata, parses schemas, and populates backend delegates. * - * @param modelPath Absolute file system path to the `.pte` model binary. + * @param modelPath Program path, absent for callback sources. + * @param module The module constructed for the requested source. * @param eagerLoadMethods Whether to eagerly load and compile all model methods. */ - explicit ModelHostObject(const std::string &modelPath, bool eagerLoadMethods); + explicit ModelHostObject(std::optional modelPath, + std::unique_ptr module, + bool eagerLoadMethods); jsi::Value get(jsi::Runtime &rt, const jsi::PropNameID &name) override; std::vector getPropertyNames(jsi::Runtime &rt) override; private: - /// File system path to the loaded ExecuTorch `.pte` model binary. - std::string modelPath_; + /// Program path, absent for a callback source. + std::optional modelPath_; /** * Parsed model schema mapping method names to their input/output parameter diff --git a/packages/react-native-executorch/cpp/tests/CMakeLists.txt b/packages/react-native-executorch/cpp/tests/CMakeLists.txt index 36cbcd6b29..bd114142e3 100644 --- a/packages/react-native-executorch/cpp/tests/CMakeLists.txt +++ b/packages/react-native-executorch/cpp/tests/CMakeLists.txt @@ -359,6 +359,8 @@ set(TEST_SUITES core/DTypeTest.cpp core/ErrorTest.cpp core/ConversionsTest.cpp + core/DataLoaderTest.cpp + core/ModelSourceTest.cpp core/TensorTest.cpp core/SchemaTest.cpp core/UtilsTest.cpp @@ -392,5 +394,11 @@ foreach(suite_path IN LISTS TEST_SUITES) # re2/abseil come in transitively via et_tokenizers, which places them after # their consumer — see the note by TOKENIZER_DEP_LIBS. target_link_libraries(${suite_name} PRIVATE rne_test_support gtest_main) + if(suite_name STREQUAL "ModelSourceTest") + # Build a tiny executable FlatBuffer in the test itself, using the same + # FlatBuffers version as the provisioned ExecuTorch host dependency. + target_include_directories(${suite_name} SYSTEM PRIVATE + "${RNE_TEST_DEPS_DIR}/executorch/third-party/flatbuffers/include") + endif() gtest_discover_tests(${suite_name} DISCOVERY_TIMEOUT 60) endforeach() diff --git a/packages/react-native-executorch/cpp/tests/core/DataLoaderTest.cpp b/packages/react-native-executorch/cpp/tests/core/DataLoaderTest.cpp new file mode 100644 index 0000000000..f4726db729 --- /dev/null +++ b/packages/react-native-executorch/cpp/tests/core/DataLoaderTest.cpp @@ -0,0 +1,183 @@ +#include +#include +#include +#include +#include +#include + +#include "core/data_loader.h" +#include "support/JsiTestEnv.h" + +namespace rnexecutorch::tests { +namespace { +using core::data_loader::LoadSession; +using core::error::RnExecuTorchErrorCode; +using executorch::runtime::DataLoader; +using executorch::runtime::Error; +using DataLoaderTest = JsiTestEnv; + +const DataLoader::SegmentInfo kSegment{DataLoader::SegmentInfo::Type::Program}; + +TEST_F(DataLoaderTest, BuffersOwnCopiesAfterSessionAndRuntimeAreDestroyed) { + auto owner = facebook::hermes::makeHermesRuntime(); + std::unique_ptr loader; + auto original = owner->evaluateJavaScript(std::make_unique(R"( + ({ size() { return 4; }, load(offset, length) { + return new Uint8Array([10, 20, 30, 40]).buffer.slice(offset, offset + length); + } }) + )"), + "loader.js"); + auto loaded = [&] { + LoadSession session(*owner); + loader = session.create("reader", original); + return loader->load(1, 2, kSegment); + }(); + original = jsi::Value::undefined(); + owner.reset(); + + ASSERT_TRUE(loaded.ok()); + EXPECT_EQ(reinterpret_cast(loaded->data()) % alignof(std::max_align_t), 0u); + const std::array expected{20, 30}; + EXPECT_EQ(std::memcmp(loaded->data(), expected.data(), expected.size()), 0); + // Even a live native adapter cannot call back into the destroyed runtime. + EXPECT_EQ(loader->load(0, 1, kSegment).error(), Error::InvalidState); + EXPECT_EQ(loader->size().get(), 4u); + + // Freeing loaded buffers and adapters needs no originating JS runtime. + std::thread cleanup([buffer = std::move(loaded.get()), adapter = std::move(loader)]() mutable { + buffer.Free(); + adapter.reset(); + }); + cleanup.join(); +} + +TEST_F(DataLoaderTest, LoadIntoCopiesDirectlyAndPreservesTheCallbackReceiver) { + LoadSession session(rt()); + auto reader = eval(R"( + return { bytes: new Uint8Array([1, 2, 3, 4]), + size() { return this.bytes.byteLength; }, + load(offset, length) { return this.bytes.buffer.slice(offset, offset + length); } + }; + )"); + auto loader = session.create("reader", reader); + std::array destination{}; + EXPECT_EQ(loader->load_into(1, 2, kSegment, destination.data()), Error::Ok); + EXPECT_EQ(destination, (std::array{2, 3})); + EXPECT_EQ(loader->load_into(0, 1, kSegment, nullptr), Error::InvalidArgument); + EXPECT_EQ(loader->load(4, 0, kSegment)->size(), 0u); + session.rethrowIfFailed(); +} + +TEST_F(DataLoaderTest, RejectsOutOfBoundsReadsBeforeCallingJs) { + LoadSession session(rt()); + auto reader = eval("return { size: () => 4, load: () => { throw new Error('must not run'); } };"); + auto loader = session.create("reader", reader); + EXPECT_EQ(loader->load(5, 0, kSegment).error(), Error::InvalidArgument); + EXPECT_EQ(loader->load(3, 2, kSegment).error(), Error::InvalidArgument); + EXPECT_EQ(loader->load(1, SIZE_MAX, kSegment).error(), Error::InvalidArgument); + EXPECT_NO_THROW(session.rethrowIfFailed()); +} + +TEST_F(DataLoaderTest, WorkerThreadsCannotAccessJsCallbacks) { + LoadSession session(rt()); + auto reader = eval("return { size: () => 4, load: () => { throw new Error('must not run'); } };"); + auto loader = session.create("reader", reader); + Error result = Error::Ok; + std::thread worker([&] { result = loader->load(0, 1, kSegment).error(); }); + worker.join(); + EXPECT_EQ(result, Error::InvalidState); + EXPECT_NO_THROW(session.rethrowIfFailed()); +} + +TEST_F(DataLoaderTest, PreservesOriginalCallbackExceptionsAndStopsFurtherReads) { + LoadSession session(rt()); + auto reader = eval("return { size: () => 4, load: () => { throw new Error('read failed'); } };"); + auto loader = session.create("reader", reader); + EXPECT_EQ(loader->load(0, 1, kSegment).error(), Error::AccessFailed); + EXPECT_EQ(loader->load(0, 1, kSegment).error(), Error::AccessFailed); + try { + session.rethrowIfFailed(); + FAIL() << "expected the original callback error"; + } catch (const jsi::JSError &e) { + EXPECT_THAT(e.getMessage(), ::testing::HasSubstr("read failed")); + } +} + +TEST_F(DataLoaderTest, RejectsWrongLengthAndNonBufferResults) { + for (const auto *body : {"new ArrayBuffer(2)", "Promise.resolve(new ArrayBuffer(1))"}) { + LoadSession session(rt()); + auto reader = eval(std::format("return {{ size: () => 4, load: () => {} }};", body)); + auto loader = session.create("reader", reader); + EXPECT_EQ(loader->load(0, 1, kSegment).error(), Error::AccessFailed); + EXPECT_TRUE(throwsCoded([&] { session.rethrowIfFailed(); }, RnExecuTorchErrorCode::InvalidArgument)); + } +} + +TEST_F(DataLoaderTest, RejectsInvalidSizesAndMalformedReaders) { + for (const auto *value : {"-1", "1.5", "NaN", "Infinity", "18446744073709551616"}) { + LoadSession session(rt()); + auto reader = eval(std::format("return {{ size: () => {}, load: () => new ArrayBuffer(0) }};", value)); + EXPECT_TRUE(throwsCoded([&] { session.create("reader", reader); }, RnExecuTorchErrorCode::InvalidArgument)); + } + LoadSession session(rt()); + auto reader = eval("return { size: () => 4 };"); + EXPECT_TRUE(throwsCoded([&] { session.create("reader", reader); }, RnExecuTorchErrorCode::InvalidArgument)); +} + +TEST_F(DataLoaderTest, ExceptionalExitAlsoDetachesCallbacks) { + std::unique_ptr loader; + EXPECT_TRUE(throwsCoded([&] { + LoadSession session(rt()); + auto reader = eval("return { size: () => 4, load: () => new ArrayBuffer(0) };"); + loader = session.create("reader", reader); + EXPECT_EQ(loader->load(0, 1, kSegment).error(), Error::AccessFailed); + session.rethrowIfFailed(); + }, + RnExecuTorchErrorCode::InvalidArgument)); + EXPECT_EQ(loader->load(0, 1, kSegment).error(), Error::InvalidState); +} + +TEST_F(DataLoaderTest, LoadModelRejectsLazyCallbackSourcesBeforeCallingThem) { + auto thrown = evalThrowing(R"( + __rnexecutorch_jsi__.loadModel({ programLoader: { + size() { throw new Error('must not run'); }, + load() { throw new Error('must not run'); } + } }, { eagerLoadMethods: false }); + )"); + EXPECT_TRUE(isCodedError(thrown, "INVALID_ARGUMENT", "require eagerLoadMethods: true")); +} + +TEST_F(DataLoaderTest, LoadModelRejectsIncompleteAndMalformedSources) { + for (const auto *source : {"{}", "new ArrayBuffer(4)", "{programPath: 'x'}", + "{programPath: 'x', dataMapPath: []}", "{programPath: 'x', dataMapPath: ['']}", + "{programPath: 'x', programLoader: {}}", "{programLoader: {}, dataMapPath: 'x'}"}) { + EXPECT_TRUE(isCodedError(evalThrowing(std::format("__rnexecutorch_jsi__.loadModel({});", source)), + "INVALID_ARGUMENT", "")); + } +} + +TEST_F(DataLoaderTest, LoadModelPrefersThePathPairOverCallbackLoaders) { + auto thrown = evalThrowing(R"( + __rnexecutorch_jsi__.loadModel({ + programPath: '/nonexistent/rne-program.pte', + dataMapPath: '/nonexistent/rne-weights.ptd', + programLoader: { + size() { throw new Error('callback must not run'); }, + load() { throw new Error('callback must not run'); } + } + }); + )"); + EXPECT_TRUE(isCodedError(thrown, "LOAD_FAILED", "rne-program.pte")); +} + +TEST_F(DataLoaderTest, LoadModelPreservesInitializationCallbackErrors) { + auto thrown = evalThrowing(R"( + __rnexecutorch_jsi__.loadModel({ programLoader: { + size() { return 64; }, load() { throw new Error('decrypt failed'); } + } }); + )"); + EXPECT_THAT(thrown.message, ::testing::HasSubstr("decrypt failed")); +} + +} // namespace +} // namespace rnexecutorch::tests diff --git a/packages/react-native-executorch/cpp/tests/core/ErrorTest.cpp b/packages/react-native-executorch/cpp/tests/core/ErrorTest.cpp index cb505ab1d8..d9d1ea9fcb 100644 --- a/packages/react-native-executorch/cpp/tests/core/ErrorTest.cpp +++ b/packages/react-native-executorch/cpp/tests/core/ErrorTest.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -52,8 +53,14 @@ TEST(Error, MapsEveryCodeToItsWireString) { // added on one side and forgotten on the other fails here rather than reaching // an app as an unmatchable string. TEST(Error, CodeListMatchesTypeScript) { - std::ifstream file(RNE_ERROR_TS_SOURCE); - ASSERT_TRUE(file.is_open()) << "cannot open " << RNE_ERROR_TS_SOURCE; +#ifdef RNE_ERROR_TS_SOURCE + const std::filesystem::path sourcePath = RNE_ERROR_TS_SOURCE; +#else + // Standalone compiler checks do not receive CMake's test definitions. + const auto sourcePath = std::filesystem::path(__FILE__).parent_path() / "../../../src/core/error.ts"; +#endif + std::ifstream file(sourcePath); + ASSERT_TRUE(file.is_open()) << "cannot open " << sourcePath; std::stringstream buffer; buffer << file.rdbuf(); @@ -125,7 +132,7 @@ TEST_F(ErrorJsTest, AttachesExecuTorchCodeWhenTheRuntimeFailed) { TEST_F(ErrorJsTest, OmitsExecuTorchCodeForOurOwnFailures) { auto thrown = evalThrowing("__rnexecutorch_jsi__.loadModel(42);"); - EXPECT_TRUE(isCodedError(thrown, "INVALID_ARGUMENT", "must be a string")); + EXPECT_TRUE(isCodedError(thrown, "INVALID_ARGUMENT", "loadModel: source")); EXPECT_FALSE(thrown.etRuntimeErrorCode.has_value()); } diff --git a/packages/react-native-executorch/cpp/tests/core/ModelSourceTest.cpp b/packages/react-native-executorch/cpp/tests/core/ModelSourceTest.cpp new file mode 100644 index 0000000000..e347a6ec63 --- /dev/null +++ b/packages/react-native-executorch/cpp/tests/core/ModelSourceTest.cpp @@ -0,0 +1,141 @@ +#include +#include +#include +#include +#include +#include + +#include "core/model.h" +#include "support/JsiTestEnv.h" + +#include + +namespace rnexecutorch::tests { +namespace { +namespace fb = executorch_flatbuffer; + +// Build an actual executable program without downloaded weights or a delegate. +// Its single method returns a constant integer, so the test exercises loading +// and inference without depending on kernel registration or device backends. +std::vector constantProgram() { + flatbuffers::FlatBufferBuilder builder; + const std::vector> values{ + fb::CreateEValue(builder, fb::KernelTypes::Int, fb::CreateInt(builder, 42).Union())}; + const std::vector inputs; + const std::vector outputs{0}; + const std::vector> instructions; + const auto chain = fb::CreateChainDirect(builder, &inputs, &outputs, &instructions); + const std::vector> chains{chain}; + const std::vector> operators; + const std::vector> delegates; + const std::vector buffers{0}; + const auto plan = fb::CreateExecutionPlanDirect(builder, "value", 0, &values, &inputs, &outputs, + &chains, &operators, &delegates, &buffers); + const std::vector> plans{plan}; + const std::vector> constants; + const std::vector constantOffsets{0}; + const auto constantSegment = fb::CreateSubsegmentOffsetsDirect(builder, 0, &constantOffsets); + fb::FinishProgramBuffer(builder, fb::CreateProgramDirect(builder, 0, &plans, &constants, nullptr, nullptr, constantSegment)); + return {builder.GetBufferPointer(), builder.GetBufferPointer() + builder.GetSize()}; +} + +class ModelSourceTest : public JsiTestEnv { +protected: + void SetUp() override { + JsiTestEnv::SetUp(); + const auto bytes = constantProgram(); + auto buffer = rt().global().getPropertyAsFunction(rt(), "ArrayBuffer").callAsConstructor(rt(), static_cast(bytes.size())).getObject(rt()); + std::memcpy(buffer.getArrayBuffer(rt()).data(rt()), bytes.data(), bytes.size()); + rt().global().setProperty(rt(), "programBytes", std::move(buffer)); + } +}; + +TEST_F(ModelSourceTest, ExecutesRepeatedlyWithoutReadingCallbacksAfterInitialization) { + EXPECT_TRUE(evalBool(R"( + let active = true; + let calls = 0; + const reader = { + size() { if (!active) throw new Error('late size'); return programBytes.byteLength; }, + load(offset, length) { + if (!active) throw new Error('late read'); + calls++; + return programBytes.slice(offset, offset + length); + } + }; + const model = __rnexecutorch_jsi__.loadModel({ programLoader: reader }); + active = false; + const before = calls; + try { + return model.path === undefined && calls > 0 && + model.execute('value', [], [])[0] === 42 && + model.execute('value', [], [])[0] === 42 && calls === before; + } finally { model.dispose(); } + )")); +} + +TEST_F(ModelSourceTest, ExecutesAndDisposesInAnotherRuntimeAfterLoadingRuntimeIsDestroyed) { + auto value = eval(R"( + return __rnexecutorch_jsi__.loadModel({ programLoader: { + size() { return programBytes.byteLength; }, + load(offset, length) { return programBytes.slice(offset, offset + length); } + } }); + )"); + auto host = value.getObject(rt()).getHostObject(rt()); + value = jsi::Value::undefined(); + // Destroy the owner runtime before accessing the model from a new runtime. + JsiTestEnv::TearDown(); + auto other = facebook::hermes::makeHermesRuntime(); + other->global().setProperty(*other, "model", jsi::Object::createFromHostObject(*other, std::move(host))); + auto result = other->evaluateJavaScript(std::make_unique(R"( + (function() { + try { return model.execute('value', [], [])[0]; } + finally { model.dispose(); } + })() + )"), + "other-runtime.js"); + EXPECT_EQ(result.asNumber(), 42); +} + +TEST_F(ModelSourceTest, PreservesExternalDataLoaderErrors) { + auto thrown = evalThrowing(R"( + __rnexecutorch_jsi__.loadModel({ + programLoader: { + size() { return programBytes.byteLength; }, + load(offset, length) { return programBytes.slice(offset, offset + length); } + }, + dataMapLoader: { + size() { return 64; }, + load() { throw new Error('weights decrypt failed'); } + } + }); + )"); + EXPECT_THAT(thrown.message, ::testing::HasSubstr("weights decrypt failed")); +} + +TEST_F(ModelSourceTest, ContinuesToSupportLazyFilesystemLoading) { + const auto bytes = constantProgram(); + // Use a unique temporary filename; remove it even if an assertion throws. + const auto path = std::filesystem::temp_directory_path() / + std::format("rne-model-source-{}.pte", reinterpret_cast(this)); + struct RemoveFile { + std::filesystem::path path; + ~RemoveFile() { + std::error_code ignored; + std::filesystem::remove(path, ignored); + } + } cleanup{path}; + { + std::ofstream output(path, std::ios::binary); + output.write(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + ASSERT_TRUE(output.good()); + } + EXPECT_EQ(evalNumber(std::format(R"( + const model = __rnexecutorch_jsi__.loadModel('{}', {{ eagerLoadMethods: false }}); + try {{ return model.execute('value', [], [])[0]; }} finally {{ model.dispose(); }} + )", + path.string())), + 42); +} + +} // namespace +} // namespace rnexecutorch::tests diff --git a/packages/react-native-executorch/cpp/tests/core/ModelTest.cpp b/packages/react-native-executorch/cpp/tests/core/ModelTest.cpp index bd163ae0d5..94e7074d2c 100644 --- a/packages/react-native-executorch/cpp/tests/core/ModelTest.cpp +++ b/packages/react-native-executorch/cpp/tests/core/ModelTest.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -28,7 +29,14 @@ using ::testing::HasSubstr; // The fixture's shape contract, from its published config.json: // forward: input [1, 3, 256, 256] float32 -> output [1, 1, 256, 256] float32 +#ifdef RNE_MODEL_FIXTURE constexpr const char *kFixture = RNE_MODEL_FIXTURE; +#else +// Standalone compiler checks do not receive CMake's test definitions. +const auto kFixture = (std::filesystem::path(__FILE__).parent_path().parent_path() / + "fixtures/selfie_segmentation_xnnpack_fp32.pte") + .string(); +#endif std::string loadFixtureJs() { return std::format("const model = __rnexecutorch_jsi__.loadModel('{}', {{ eagerLoadMethods: false }});", kFixture); @@ -199,12 +207,12 @@ TEST_F(ModelTest, ReportsAMissingFileAsLoadFailed) { TEST_F(ModelTest, RejectsWrongArgumentCount) { EXPECT_THAT(evalThrowingMessage("__rnexecutorch_jsi__.loadModel();"), - HasSubstr("Usage: loadModel(path, options?)")); + HasSubstr("Usage: loadModel(source, options?)")); } -TEST_F(ModelTest, RejectsANonStringPath) { +TEST_F(ModelTest, RejectsAnInvalidSourceType) { EXPECT_THAT(evalThrowingMessage("__rnexecutorch_jsi__.loadModel(42);"), - HasSubstr("must be a string")); + HasSubstr("must be an object")); } TEST_F(ModelTest, ExecuteRejectsWrongArgumentCount) { diff --git a/packages/react-native-executorch/src/core/lifetime.ts b/packages/react-native-executorch/src/core/lifetime.ts index b43e57eeed..2d0ab27a59 100644 --- a/packages/react-native-executorch/src/core/lifetime.ts +++ b/packages/react-native-executorch/src/core/lifetime.ts @@ -55,7 +55,7 @@ export type ResourceScope = { * const scope = createResourceScope(); * const dispose = scope.dispose; * try { - * const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + * const model = scope.track(await wrapAsync(loadModel, runtime)(modelSource)); * const { dims } = validateSpec(model.schema, { ... }); // may throw * const tensors = [scope.track(tensor('float32', shape))]; * return { run, dispose }; diff --git a/packages/react-native-executorch/src/core/model.ts b/packages/react-native-executorch/src/core/model.ts index da42f818eb..f626d6bdde 100644 --- a/packages/react-native-executorch/src/core/model.ts +++ b/packages/react-native-executorch/src/core/model.ts @@ -36,8 +36,8 @@ export type ModelOutput = Tensor | number | boolean | string | null; * @category Core / Types */ export type Model = { - /** The local filesystem path of the `.pte` model file. */ - readonly path: string; + /** The program's local filesystem path. Undefined for callback sources. */ + readonly path?: string; /** The exported schema of this model. */ readonly schema: ModelSpec; /** ExecuTorch backends used by a given method. */ @@ -90,23 +90,60 @@ export type LoadModelOptions = { * guaranteeing instantaneous first-inference latency. * * Set to `false` to lazily load and compile methods on their first execution. + * Callback sources require `true` because their callbacks are released when + * initialization finishes. * @default true */ readonly eagerLoadMethods?: boolean; }; /** - * Loads and compiles an ExecuTorch `.pte` model from the local filesystem. + * Synchronous random-access reader used only during model initialization. + * + * Reads must return exactly the requested bytes. Native code copies each range + * into independently owned memory; its release timing depends on the backend. + * The callbacks are released before {@link loadModel} returns, including on + * failure. Backends requiring later reads or reads on worker threads are not + * supported by callback sources. + * + * When passed through {@link wrapAsync}, callbacks and their captures must be + * worklet-compatible. Avoid capturing a complete weight archive when the goal + * is to reduce memory use. + * @category Core / Types + */ +export type DataLoader = { + /** Returns the source byte length as a non-negative integer. */ + size(): number; + /** Returns exactly `byteLength` bytes starting at `byteOffset`. */ + load(byteOffset: number, byteLength: number): ArrayBuffer; +}; + +/** + * A local program path, program plus external `.ptd` paths, or + * initialization-only readers. + * @category Core / Types + */ +export type ModelSource = Readonly< + | string + | { programPath: string; dataMapPath: string | readonly string[] } + | { programLoader: DataLoader; dataMapLoader?: DataLoader } +>; + +/** + * Loads and compiles an ExecuTorch `.pte` model from local files or callback readers. * * The model is loaded synchronously into native memory. Prefer calling this * inside a worklet runtime thread (via {@link wrapAsync}) to avoid blocking the * JS thread during compilation. * @category Core / Functions - * @param modelPath The absolute local path to the `.pte` model file. + * @param modelSource Local program path, program with external data paths, or + * callback readers. See {@link ModelSource}. * @param options Optional loading configuration. See {@link LoadModelOptions}. * @returns The compiled {@link Model} instance, ready for execution. * @throws {RnExecuTorchError} Thrown with code `LOAD_FAILED` if the model file * cannot be opened, has an invalid format, or fails native initialization. + * With code `INVALID_ARGUMENT` if a source is malformed or callback loaders + * are combined with lazy loading. Exceptions from user callbacks propagate. * @see {@link wrapAsync} * @example * ```typescript @@ -123,7 +160,7 @@ export type LoadModelOptions = { * } * ``` */ -export function loadModel(modelPath: string, options?: LoadModelOptions): Model { +export function loadModel(modelSource: ModelSource, options?: LoadModelOptions): Model { 'worklet'; - return rnexecutorchJsi.loadModel(modelPath, options) as Model; + return rnexecutorchJsi.loadModel(modelSource, options) as Model; } diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts b/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts index adec88a345..ec698c8123 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts @@ -6,7 +6,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; -import { loadModel } from '../../../core/model'; +import { loadModel, type ModelSource } from '../../../core/model'; import { validateSpec, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; @@ -31,8 +31,8 @@ export type ClassifierOptions = ImagePreprocessorOptions & { * @category CV / Types */ export type ClassifierModel = { - /** Local path or remote URL of the `.pte` model file. */ - readonly modelPath: string; + /** Model source: a local or remote path, program and data paths, or callback loaders. */ + readonly modelSource: ModelSource; /** * Image preprocessing and label vocabulary. The `labels` array length must * match the model's output dimension. @@ -125,8 +125,8 @@ export async function createClassifier( const dispose = scope.dispose; try { - const { modelPath, modelOpts } = config; - const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + const { modelSource, modelOpts } = config; + const model = scope.track(await wrapAsync(loadModel, runtime)(modelSource)); const { variant, dims } = validateSpec(model.schema, { batched: method( diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/depthEstimation.ts b/packages/react-native-executorch/src/extensions/cv/tasks/depthEstimation.ts index 02c2099655..684a061e3e 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/depthEstimation.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/depthEstimation.ts @@ -5,7 +5,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { createResourceScope } from '../../../core/lifetime'; -import { loadModel } from '../../../core/model'; +import { loadModel, type ModelSource } from '../../../core/model'; import { wrapAsync } from '../../../core/runtime'; import { validateSpec, method, f32 } from '../../../core/schema'; import { tensor } from '../../../core/tensor'; @@ -47,8 +47,8 @@ export type DepthEstimatorOptions = Omit = Omit< * @typeParam L The label type. */ export type InstanceSegmenterModel = { - /** Local path or remote URL of the `.pte` model file. */ - readonly modelPath: string; + /** Model source: a local or remote path, program and data paths, or callback loaders. */ + readonly modelSource: ModelSource; /** * Image preprocessing, label vocabulary, bounding box format, and default * NMS/mask/confidence thresholds. @@ -171,8 +171,8 @@ export async function createInstanceSegmenter( const dispose = scope.dispose; try { - const { modelPath, modelOpts } = config; - const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + const { modelSource, modelOpts } = config; + const model = scope.track(await wrapAsync(loadModel, runtime)(modelSource)); const { variant, dims } = validateSpec(model.schema, { batched: method( diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts b/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts index e7a7def984..077d71717a 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts @@ -6,7 +6,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor, type Tensor } from '../../../core/tensor'; -import { loadModel } from '../../../core/model'; +import { loadModel, type ModelSource } from '../../../core/model'; import { validateSpec, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import { createResourceScope } from '../../../core/lifetime'; @@ -43,8 +43,8 @@ export type KeypointDetectorOptions * @category CV / Types */ export type KeypointDetectorModel = { - /** Local path or remote URL of the `.pte` model file. */ - readonly modelPath: string; + /** Model source: a local or remote path, program and data paths, or callback loaders. */ + readonly modelSource: ModelSource; /** * Image preprocessing, landmark names, bounding box format, and default * NMS/confidence thresholds. @@ -262,9 +262,9 @@ export async function createKeypointDetector = Omit< * @category CV / Types */ export type ObjectDetectorModel = { - /** Local path or remote URL of the `.pte` model file. */ - readonly modelPath: string; + /** Model source: a local or remote path, program and data paths, or callback loaders. */ + readonly modelSource: ModelSource; /** * Image preprocessing, label vocabulary, and default NMS/confidence * thresholds. Used as fallbacks when per-call overrides are omitted. @@ -146,8 +146,8 @@ export async function createObjectDetector( const dispose = scope.dispose; try { - const { modelPath, modelOpts } = config; - const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + const { modelSource, modelOpts } = config; + const model = scope.track(await wrapAsync(loadModel, runtime)(modelSource)); const { variant, dims } = validateSpec(model.schema, { batched: method( diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/paddleOcr.ts b/packages/react-native-executorch/src/extensions/cv/tasks/paddleOcr.ts index 154b6b72f8..05ed79514b 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/paddleOcr.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/paddleOcr.ts @@ -6,7 +6,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import RNBlobUtil from 'react-native-blob-util'; -import { loadModel } from '../../../core/model'; +import { loadModel, type ModelSource } from '../../../core/model'; import { RnExecuTorchError } from '../../../core/error'; import { wrapAsync } from '../../../core/runtime'; import { tensor, type Tensor } from '../../../core/tensor'; @@ -90,11 +90,11 @@ export type RecognizeCharactersOptions = { */ export type PaddleOcrModel = { /** The fused detect/recognize PTE. Resolved to a local path by the fetcher. */ - readonly modelPath: string; + readonly modelSource: ModelSource; /** * The recognizer charset published beside the model: a JSON array of strings, * one per class, where `charset[i]` labels logit `i + 1` (logit 0 is the CTC - * blank). Resolved to a local path by the resource fetcher like `modelPath`. + * blank). Resolved to a local path by the resource fetcher like `modelSource`. */ readonly charsetPath: string; /** Run options. See {@link PaddleOcrModelOptions}. */ @@ -405,8 +405,8 @@ export async function createPaddleOcr( const dispose = scope.dispose; try { - const { modelPath, charsetPath, modelOpts } = config; - const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + const { modelSource, charsetPath, modelOpts } = config; + const model = scope.track(await wrapAsync(loadModel, runtime)(modelSource)); // Both methods are `DynamicDim` on every backend — a range on XNNPACK and // Vulkan, an enum on CoreML, which has no RangeDim — so one variant covers diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts b/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts index 5dcc136d5a..0ca225f808 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts @@ -6,7 +6,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; -import { loadModel } from '../../../core/model'; +import { loadModel, type ModelSource } from '../../../core/model'; import { validateSpec, method, i64, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import { createResourceScope } from '../../../core/lifetime'; @@ -45,7 +45,7 @@ const NOISE_COEFF = -14.579279; */ export type SdxsTextToImageModel = { /** Local path to the model `.pte`. */ - readonly modelPath: string; + readonly modelSource: ModelSource; /** Local path to the CLIP `tokenizer.json`. */ readonly tokenizerPath: string; }; @@ -99,8 +99,8 @@ export async function createSdxsTextToImage( const dispose = scope.dispose; try { - const { modelPath, tokenizerPath } = config; - const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + const { modelSource, tokenizerPath } = config; + const model = scope.track(await wrapAsync(loadModel, runtime)(modelSource)); const tokenizer = scope.track(await wrapAsync(loadTokenizer, runtime)(tokenizerPath)); validateSpec(model.schema, { diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts b/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts index 5cf5ed24e7..d27a3cb8c3 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts @@ -6,7 +6,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; -import { loadModel } from '../../../core/model'; +import { loadModel, type ModelSource } from '../../../core/model'; import { validateSpec, method, f32, i32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; @@ -43,8 +43,8 @@ export type SemanticSegmenterOptions = Omit = { - /** Local path or remote URL of the `.pte` model file. */ - readonly modelPath: string; + /** Model source: a local or remote path, program and data paths, or callback loaders. */ + readonly modelSource: ModelSource; /** * Image preprocessing, output mask interpolation, and label vocabulary. * `resizeMode` is fixed to `'stretch'`. @@ -154,8 +154,8 @@ export async function createSemanticSegmenter( const dispose = scope.dispose; try { - const { modelPath, modelOpts } = config; - const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + const { modelSource, modelOpts } = config; + const model = scope.track(await wrapAsync(loadModel, runtime)(modelSource)); const { variant, dim, dims } = validateSpec(model.schema, { batched: method( diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts b/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts index ce26eac675..a1ce6f5df9 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts @@ -6,7 +6,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; -import { loadModel } from '../../../core/model'; +import { loadModel, type ModelSource } from '../../../core/model'; import { validateSpec, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import { createResourceScope } from '../../../core/lifetime'; @@ -40,8 +40,8 @@ export type StyleTransferOptions = Omit * @category CV / Types */ export type StyleTransferModel = { - /** Local path or remote URL of the `.pte` model file. */ - readonly modelPath: string; + /** Model source: a local or remote path, program and data paths, or callback loaders. */ + readonly modelSource: ModelSource; /** * Input preprocessing and output postprocessing (normalization back to uint8, * interpolation). `resizeMode` is fixed to `'stretch'`. @@ -98,8 +98,8 @@ export async function createStyleTransfer( const dispose = scope.dispose; try { - const { modelPath, modelOpts } = config; - const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + const { modelSource, modelOpts } = config; + const model = scope.track(await wrapAsync(loadModel, runtime)(modelSource)); const { variant, dims } = validateSpec(model.schema, { batched: method( diff --git a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts index c430e54bbb..d4503f765e 100644 --- a/packages/react-native-executorch/src/extensions/llm/llmRunner.ts +++ b/packages/react-native-executorch/src/extensions/llm/llmRunner.ts @@ -10,7 +10,7 @@ import { type WorkletRuntime } from 'react-native-worklets'; import type { Tensor } from '../../core/tensor'; import { wrapAsync } from '../../core/runtime'; -import { loadModel } from '../../core/model'; +import { loadModel, type ModelSource } from '../../core/model'; import { RnExecuTorchError } from '../../core/error'; import { createResourceScope } from '../../core/lifetime'; import { validateSpec } from '../../core/schema'; @@ -113,8 +113,8 @@ export type LLMKVCacheState = { * @category LLM / Types */ export type LLMRunner = { - /** Path to the local model file. */ - readonly modelPath: string; + /** Path to the local model file. Undefined for callback sources. */ + readonly modelPath?: string; /** Path to the local tokenizer configuration file. */ readonly tokenizerPath: string; /** List of supported non-text input modalities for this runner (e.g. `['image']`). */ @@ -170,7 +170,8 @@ export type LLMRunner = { * Loads the model and its tokenizer, validates the model's exported methods * against the supported runner specs, and returns the matching runner. * @category LLM / Functions - * @param modelPath Path to the local `.pte` model file. + * @param modelSource Local program path, program with external data paths, or + * initialization-only callback readers. See {@link ModelSource}. * @param tokenizerPath Path to the local tokenizer file (e.g. `tokenizer.json`). * @param modalities List of supported input non-text modalities (e.g. * `['image']`). When omitted, defaults to text-only. @@ -179,14 +180,14 @@ export type LLMRunner = { * @returns A promise resolving to the {@link LLMRunner} instance. */ export async function createLLMRunner( - modelPath: string, + modelSource: ModelSource, tokenizerPath: string, modalities?: readonly Modality[], runtime?: WorkletRuntime ): Promise { const scope = createResourceScope(); try { - const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + const model = scope.track(await wrapAsync(loadModel, runtime)(modelSource)); const tokenizer = scope.track(await wrapAsync(loadTokenizer, runtime)(tokenizerPath)); const { variant } = validateSpec(model.schema, { diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts index ebd7293a40..4c3bd579e2 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmChatSession.ts @@ -7,6 +7,7 @@ import { scheduleOnRN, type WorkletRuntime } from 'react-native-worklets'; import RNBlobUtil from 'react-native-blob-util'; import { wrapAsync } from '../../../core/runtime'; +import type { ModelSource } from '../../../core/model'; import { RnExecuTorchError } from '../../../core/error'; import { createResourceScope } from '../../../core/lifetime'; import { @@ -32,8 +33,8 @@ import type { ToolDefinition, ToolParser } from '../utils/toolCalling'; * @category LLM / Types */ export type LLMModel = { - /** Local path or remote URL of the `.pte` model file. */ - readonly modelPath: string; + /** Local/remote program path, program with data paths, or callback readers. */ + readonly modelSource: ModelSource; /** Local path or remote URL of the `tokenizer.json` file. */ readonly tokenizerPath: string; /** Local path or remote URL of the `tokenizer_config.json` file. */ @@ -192,7 +193,7 @@ export async function createLLMChatSession( resetOnTurn = false, } = options; - const { modelPath, tokenizerPath, tokenizerConfigPath, modalities, preprocessorConfig } = + const { modelSource, tokenizerPath, tokenizerConfigPath, modalities, preprocessorConfig } = config; const { tools, parseToolCalls, maxToolTurns = DEFAULT_MAX_TURNS } = toolOpts ?? {}; @@ -214,7 +215,7 @@ export async function createLLMChatSession( // Prepare runner // prettier-ignore - const runner = scope.track(await createLLMRunner(modelPath, tokenizerPath, modalities, runtime)); + const runner = scope.track(await createLLMRunner(modelSource, tokenizerPath, modalities, runtime)); const prefill = wrapAsync(runner.prefill, runtime); const history: ChatMessage[] = []; diff --git a/packages/react-native-executorch/src/extensions/llm/tasks/llmTranslation.ts b/packages/react-native-executorch/src/extensions/llm/tasks/llmTranslation.ts index 294a8cdeda..2963aa438a 100644 --- a/packages/react-native-executorch/src/extensions/llm/tasks/llmTranslation.ts +++ b/packages/react-native-executorch/src/extensions/llm/tasks/llmTranslation.ts @@ -105,7 +105,7 @@ export async function createLLMTranslator( const dispose = scope.dispose; try { - const { modelPath, tokenizerPath, tokenizerConfigPath } = config; + const { modelSource, tokenizerPath, tokenizerConfigPath } = config; const tokenizerConfigStr = await RNBlobUtil.fs.readFile(tokenizerConfigPath, 'utf8'); const { chatTemplate, bosToken, eosToken, stopTokens } = parseTokenizerConfig( JSON.parse(tokenizerConfigStr) @@ -114,7 +114,7 @@ export async function createLLMTranslator( createChatPreprocessor({ chatTemplate, bosToken, eosToken }) ); // prettier-ignore - const runner = scope.track(await createLLMRunner(modelPath, tokenizerPath, undefined, runtime)); + const runner = scope.track(await createLLMRunner(modelSource, tokenizerPath, undefined, runtime)); const generate = wrapAsync(generateWorklet, runtime); let busy = false; diff --git a/packages/react-native-executorch/src/extensions/nlp/tasks/colbertEmbedding.ts b/packages/react-native-executorch/src/extensions/nlp/tasks/colbertEmbedding.ts index 9b526c64a6..045f339189 100644 --- a/packages/react-native-executorch/src/extensions/nlp/tasks/colbertEmbedding.ts +++ b/packages/react-native-executorch/src/extensions/nlp/tasks/colbertEmbedding.ts @@ -5,7 +5,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; -import { loadModel } from '../../../core/model'; +import { loadModel, type ModelSource } from '../../../core/model'; import { validateSpec, DynamicDim as Dyn, @@ -66,8 +66,8 @@ export type ColbertEmbedderOptions = { * @category NLP / Types */ export type ColbertEmbedderModel = { - /** Local path or remote URL of the `.pte` model file. */ - readonly modelPath: string; + /** Model source: a local or remote path, program and data paths, or callback loaders. */ + readonly modelSource: ModelSource; /** Local path or remote URL of the tokenizer file. */ readonly tokenizerPath: string; /** Tokenization settings of the model. See {@link ColbertEmbedderOptions}. */ @@ -240,8 +240,8 @@ export async function createColbertEmbedder( const dispose = scope.dispose; try { - const { modelPath, tokenizerPath, modelOpts } = config; - const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + const { modelSource, tokenizerPath, modelOpts } = config; + const model = scope.track(await wrapAsync(loadModel, runtime)(modelSource)); const tokenizer = scope.track(await wrapAsync(loadTokenizer, runtime)(tokenizerPath)); // Token ids and attention mask in, one vector per token out, all sharing diff --git a/packages/react-native-executorch/src/extensions/nlp/tasks/marianTranslation.ts b/packages/react-native-executorch/src/extensions/nlp/tasks/marianTranslation.ts index 424d7e0053..1f26cf5cdb 100644 --- a/packages/react-native-executorch/src/extensions/nlp/tasks/marianTranslation.ts +++ b/packages/react-native-executorch/src/extensions/nlp/tasks/marianTranslation.ts @@ -6,7 +6,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { scheduleOnRN, createSynchronizable } from 'react-native-worklets'; import { tensor, type Tensor } from '../../../core/tensor'; -import { loadModel } from '../../../core/model'; +import { loadModel, type ModelSource } from '../../../core/model'; import { wrapAsync } from '../../../core/runtime'; import { constraint, @@ -41,8 +41,8 @@ const NUM_CROSS_KV = 12; export type MarianTranslatorModel = { /** Identifies the pipeline this config belongs to. */ readonly name: 'marian'; - /** Local path or remote URL of the `.pte` model file. */ - readonly modelPath: string; + /** Model source: a local or remote path, program and data paths, or callback loaders. */ + readonly modelSource: ModelSource; /** Local path or remote URL of the tokenizer file. */ readonly tokenizerPath: string; /** ISO 639-1 code of the language the model translates from. */ @@ -92,8 +92,8 @@ export async function createMarianTranslator( const dispose = scope.dispose; try { - const { modelPath, tokenizerPath } = config; - const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath)); + const { modelSource, tokenizerPath } = config; + const model = scope.track(await wrapAsync(loadModel, runtime)(modelSource)); const tokenizer = scope.track(await wrapAsync(loadTokenizer, runtime)(tokenizerPath)); // The source length S must agree across the ids, the mask and every diff --git a/packages/react-native-executorch/src/extensions/nlp/tasks/privacyFilter.ts b/packages/react-native-executorch/src/extensions/nlp/tasks/privacyFilter.ts index 8c13940859..3d36d09bfc 100644 --- a/packages/react-native-executorch/src/extensions/nlp/tasks/privacyFilter.ts +++ b/packages/react-native-executorch/src/extensions/nlp/tasks/privacyFilter.ts @@ -6,7 +6,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; -import { loadModel } from '../../../core/model'; +import { loadModel, type ModelSource } from '../../../core/model'; import { validateSpec, DynamicDim as Dyn, @@ -63,7 +63,7 @@ export type PrivacyFilterOptions