Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .agents/skills/add-api-tests/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ fakeJsi.registerModel('/models/task.pte', {
execute: writesOutputs([1, 0, 2]),
});

const runner = tracked(await createMyTask({ modelPath: '/models/task.pte', modelOpts }));
const runner = tracked(await createMyTask({ modelSource: '/models/task.pte', modelOpts }));
expect(await runner.runTask(imageBuffer(8, 8))).toEqual(/* ... */);
```

Expand Down
14 changes: 9 additions & 5 deletions .agents/skills/add-task-pipeline/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ When implementing task constructors like `create<Task>` (e.g. `createClassifier`
const dispose = scope.dispose;

try {
const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath));
const model = scope.track(await wrapAsync(loadModel, runtime)(modelSource));
const { dims } = validateSpec(model.schema, { ... }); // may throw

const tensors = [scope.track(tensor('float32', shapeA)), scope.track(tensor('float32', shapeB))] as const;
Expand Down Expand Up @@ -94,6 +94,10 @@ When implementing task constructors like `create<Task>` (e.g. `createClassifier`
- **Corollary (a quick smell test):** if every variant in `models.ts` passes an _identical_ options object, those fields are not configuration — move them into the task file as constants and shrink the model type to the paths that actually differ.
- Do not keep a loop, parameter, or code path solely because it _looks_ more general. If the surrounding math is only valid for one value (e.g. coefficients pinned to a single distilled timestep), the generality is fake and the parameter is a correctness trap.

Task configs use `modelSource: ModelSource`, or `modelSources` for multiple models. Pass each source
unchanged to `loadModel`; tokenizer and other file-only resources keep their path fields. Callback
loaders passed to a worklet runtime must have worklet-compatible methods and captures.

## 🚫 Avoid / Anti-Patterns

- **Do NOT access tensors by index:** Avoid using `tensors[0]` or `tensors[1]` throughout the function body. Always destructure and name them explicitly.
Expand Down Expand Up @@ -123,7 +127,7 @@ returns scores; models returning logits may need a softmax or another decoder.
import type { WorkletRuntime } from 'react-native-worklets';

import { tensor } from '../../../core/tensor';
import { loadModel } from '../../../core/model';
import { loadModel, type ModelSource } from '../../../core/model';
import { validateSpec, method, f32 } from '../../../core/schema';
import { wrapAsync } from '../../../core/runtime';
import { createResourceScope } from '../../../core/lifetime';
Expand All @@ -135,7 +139,7 @@ export type MyTaskOptions = ImagePreprocessorOptions & {
};

export type MyTaskModel = {
readonly modelPath: string;
readonly modelSource: ModelSource;
readonly modelOpts: MyTaskOptions;
};

Expand Down Expand Up @@ -168,8 +172,8 @@ export async function createMyTask(
const dispose = scope.dispose;

try {
const { modelPath, modelOpts } = config;
const model = scope.track(await wrapAsync(loadModel, runtime)(modelPath));
const { modelSource, modelOpts } = config;
const model = scope.track(await wrapAsync(loadModel, runtime)(modelSource));

// Validate model spec
const { variant, dims } = validateSpec(model.schema, {
Expand Down
7 changes: 7 additions & 0 deletions .agents/skills/core-guidelines/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,13 @@ registry options. The [task skill](../add-task-pipeline/SKILL.md) owns that dist

## Shared conventions

- `loadModel` accepts local paths, `{ programPath, dataMapPath }` (one or multiple `.ptd` paths),
or `{ programLoader, dataMapLoader? }`. Callback sources require eager initialization. Their JS
handles are owned by the scoped `cpp/core/data_loader` session and released before loading returns;
copied ranges have native ownership. They do not support backend reads after initialization or on
worker threads. Keep model execution/disposal independent of the loading runtime; do not retain JS
callbacks in `ModelHostObject` or materialize a complete source as a fallback.

- Name options parameters `options`. Existing suffixes such as `ModelOpts` or `modelOpts` are allowed.
- Native JSI wrappers use `rnexecutorchJsi` from the bridge and start with `'worklet';`. Helpers called
from worklets must also be compatible with that runtime. See the native extension skill for bindings.
Expand Down
2 changes: 1 addition & 1 deletion apps/benchmarks/src/drivers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,7 @@ function llmDriver(): Driver<LLMBenchmarkInstance> {
})
);
const runner = scope.track(
await llmApi.createLLMRunner(config.modelPath, config.tokenizerPath, config.modalities)
await llmApi.createLLMRunner(config.modelSource, config.tokenizerPath, config.modalities)
);
return { runner, preprocessor, dispose: scope.dispose };
} catch (error) {
Expand Down
18 changes: 13 additions & 5 deletions apps/nlp/app/llm/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -210,11 +210,19 @@ function LLMContent() {
style: 'destructive',
onPress: async () => {
try {
const pathsToUnlink = [
resource.modelPath,
resource.tokenizerPath,
resource.tokenizerConfigPath,
].filter(Boolean);
const pathsToUnlink = [resource.tokenizerPath, resource.tokenizerConfigPath];
const { modelSource } = resource;

if (typeof modelSource === 'string') {
pathsToUnlink.push(modelSource);
} else if ('programPath' in modelSource) {
pathsToUnlink.push(modelSource.programPath);
if (typeof modelSource.dataMapPath === 'string') {
pathsToUnlink.push(modelSource.dataMapPath);
} else {
pathsToUnlink.push(...modelSource.dataMapPath);
}
}

for (const path of pathsToUnlink) {
if (path && (await RNBlobUtil.fs.exists(path))) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ configuration object to [`useClassifier`](../../06-api-reference/functions/useCl

```typescript
const customClassifier = await createClassifier({
modelPath: 'https://example.com/my-model.pte',
modelSource: 'https://example.com/my-model.pte',
modelOpts: {
resizeMode: 'stretch',
interpolation: 'linear',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ To use your own fine-tuned object detection `.pte` model, pass an [`ObjectDetect

```typescript
const customDetector = await createObjectDetector({
modelPath: 'https://example.com/my-detector.pte',
modelSource: 'https://example.com/my-detector.pte',
modelOpts: {
labels: ['hardhat', 'vest', 'boots'],
boxFormat: 'xyxy',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,7 @@ To use your own fine-tuned pose or landmark detection `.pte` model, pass a [`Key

```typescript
const customDetector = await createKeypointDetector({
modelPath: 'https://example.com/my-pose-model.pte',
modelSource: 'https://example.com/my-pose-model.pte',
modelOpts: {
landmarks: ['head', 'leftHand', 'rightHand'],
boxFormat: 'xyxy',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ To use your own custom-trained PaddleOCR `.pte` model and character set, pass a

```typescript
const customOcr = await createPaddleOcr({
modelPath: 'https://example.com/my-ocr.pte',
modelSource: 'https://example.com/my-ocr.pte',
charsetPath: 'https://example.com/charset.json',
modelOpts: {
defaultConfidenceThreshold: 0.5,
Expand All @@ -179,7 +179,7 @@ The pipeline automatically verifies that the model's exported input and output s
- [`PaddleOcr`](../../06-api-reference/type-aliases/PaddleOcr.md) — OCR runner interface ([`recognizeCharacters`](../../06-api-reference/type-aliases/PaddleOcr.md#recognizecharacters), [`recognizeCharactersWorklet`](../../06-api-reference/type-aliases/PaddleOcr.md#recognizecharactersworklet), `dispose`).
- [`OcrDetection`](../../06-api-reference/type-aliases/OcrDetection.md) — Single recognized text line with [`text`](../../06-api-reference/type-aliases/OcrDetection.md#text), [`confidence`](../../06-api-reference/type-aliases/OcrDetection.md#confidence), and [`quad`](../../06-api-reference/type-aliases/OcrDetection.md#quad).
- [`RecognizeCharactersOptions`](../../06-api-reference/type-aliases/RecognizeCharactersOptions.md) — Inference options (`confidenceThreshold`).
- [`PaddleOcrModel`](../../06-api-reference/type-aliases/PaddleOcrModel.md) — Model configuration spec with `modelPath` and `charsetPath`.
- [`PaddleOcrModel`](../../06-api-reference/type-aliases/PaddleOcrModel.md) — Model configuration spec with `modelSource` and `charsetPath`.
- [`PaddleOcrModelOptions`](../../06-api-reference/type-aliases/PaddleOcrModelOptions.md) — Model options (`defaultConfidenceThreshold`).
- [`Quad`](../../06-api-reference/react-native-executorch/namespaces/cv/type-aliases/Quad.md) — Oriented 4-corner polygon tuple `[Point, Point, Point, Point]` in pixel coordinates (`top-left`, `top-right`, `bottom-right`, `bottom-left`).
- [`ImageBuffer`](../../06-api-reference/react-native-executorch/namespaces/cv/type-aliases/ImageBuffer.md) — Input image buffer structure.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ To use your own fine-tuned semantic segmentation `.pte` model, pass a [`Semantic

```typescript
const customSegmenter = await createSemanticSegmenter({
modelPath: 'https://example.com/my-segmentation.pte',
modelSource: 'https://example.com/my-segmentation.pte',
modelOpts: {
labels: ['background', 'road', 'sidewalk', 'building'],
resizeMode: 'stretch',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ To use your own fine-tuned instance segmentation `.pte` model, pass an [`Instanc

```typescript
const customSegmenter = await createInstanceSegmenter({
modelPath: 'https://example.com/my-instance-seg.pte',
modelSource: 'https://example.com/my-instance-seg.pte',
modelOpts: {
labels: ['bottle', 'cup', 'can'],
boxFormat: 'xyxy',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ To use your own trained feed-forward style transfer `.pte` model, pass a [`Style

```typescript
const customStyler = await createStyleTransfer({
modelPath: 'https://example.com/my-style.pte',
modelSource: 'https://example.com/my-style.pte',
modelOpts: {
resizeMode: 'stretch',
interpolation: 'linear',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ To use your own fine-tuned vision encoder `.pte` model, pass an [`ImageEmbedderM

```typescript
const customEmbedder = await createImageEmbedder({
modelPath: 'https://example.com/my-vision-encoder.pte',
modelSource: 'https://example.com/my-vision-encoder.pte',
modelOpts: {
resizeMode: 'stretch',
interpolation: 'linear',
Expand Down
4 changes: 2 additions & 2 deletions docs/docs/02-extensions/computer-vision/10-text-to-image.md
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ To use your own fine-tuned SDXS `.pte` model and CLIP tokenizer, pass a [`SdxsTe

```typescript
const customGenerator = await createSdxsTextToImage({
modelPath: 'https://example.com/my-sdxs.pte',
modelSource: 'https://example.com/my-sdxs.pte',
tokenizerPath: 'https://example.com/tokenizer.json',
});
```
Expand All @@ -146,7 +146,7 @@ The pipeline automatically verifies that the model's exported methods (`encode`,
### Types & Options

- [`SdxsTextToImage`](../../06-api-reference/type-aliases/SdxsTextToImage.md) — Text-to-image generator runner interface ([`generate`](../../06-api-reference/type-aliases/SdxsTextToImage.md#generate), [`generateWorklet`](../../06-api-reference/type-aliases/SdxsTextToImage.md#generateworklet)).
- [`SdxsTextToImageModel`](../../06-api-reference/type-aliases/SdxsTextToImageModel.md) — Model configuration spec with `modelPath` and `tokenizerPath`.
- [`SdxsTextToImageModel`](../../06-api-reference/type-aliases/SdxsTextToImageModel.md) — Model configuration spec with `modelSource` and `tokenizerPath`.
- [`ImageBuffer`](../../06-api-reference/react-native-executorch/namespaces/cv/type-aliases/ImageBuffer.md) — Generated RGBA output image buffer structure.

### Model Presets
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ To use your own inpainting `.pte` model, pass an [`InpainterModel`](../../06-api

```typescript
const customInpainter = await createInpainter({
modelPath: 'https://example.com/my-inpainter.pte',
modelSource: 'https://example.com/my-inpainter.pte',
modelOpts: {
resizeMode: 'stretch',
interpolation: 'linear',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ Pass a [`DepthEstimatorModel`](../../06-api-reference/type-aliases/DepthEstimato
import { createDepthEstimator, download, IMAGENET_NORM } from 'react-native-executorch';

const model = await download({
modelPath: 'https://example.com/my-depth-model.pte',
modelSource: 'https://example.com/my-depth-model.pte',
modelOpts: {
resizeMode: 'stretch' as const,
interpolation: 'cubic' as const,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,7 @@ const { chatTemplate, bosToken, eosToken } = llm.parseTokenizerConfig(
JSON.parse(await RNBlobUtil.fs.readFile(model.tokenizerConfigPath, 'utf8'))
);
const preprocessor = llm.createChatPreprocessor({ chatTemplate, bosToken, eosToken });
const runner = await llm.createLLMRunner(model.modelPath, model.tokenizerPath);
const runner = await llm.createLLMRunner(model.modelSource, model.tokenizerPath);

// Runs on a background worklet thread; `generate` is synchronous.
const complete = wrapAsync((r: llm.LLMRunner, prompt: llm.Prompt) => {
Expand Down Expand Up @@ -405,7 +405,7 @@ To use your own fine-tuned LLM `.pte` model, pass an [`LLMModel`](../../06-api-r

```typescript
const customSession = await createLLMChatSession({
modelPath: 'https://example.com/my-llm.pte',
modelSource: 'https://example.com/my-llm.pte',
tokenizerPath: 'https://example.com/tokenizer.json',
tokenizerConfigPath: 'https://example.com/tokenizer_config.json',
});
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ To use your own fine-tuned sentence transformer `.pte` model, pass a [`TextEmbed

```typescript
const customEmbedder = await createTextEmbedder({
modelPath: 'https://example.com/my-sentence-transformer.pte',
modelSource: 'https://example.com/my-sentence-transformer.pte',
tokenizerPath: 'https://example.com/tokenizer.json',
defaultPrompt: 'passage: ', // Optional default prefix
});
Expand All @@ -188,7 +188,7 @@ The pipeline automatically verifies that the model's exported input and output s
### Types & Options

- [`TextEmbedder`](../../06-api-reference/type-aliases/TextEmbedder.md) — Text embedder runner interface ([`embed`](../../06-api-reference/type-aliases/TextEmbedder.md#embed), [`embedWorklet`](../../06-api-reference/type-aliases/TextEmbedder.md#embedworklet)).
- [`TextEmbedderModel`](../../06-api-reference/type-aliases/TextEmbedderModel.md) — Model configuration spec with `modelPath`, `tokenizerPath`, and `defaultPrompt`.
- [`TextEmbedderModel`](../../06-api-reference/type-aliases/TextEmbedderModel.md) — Model configuration spec with `modelSource`, `tokenizerPath`, and `defaultPrompt`.
- [`createVectorStore()`](../../06-api-reference/functions/createVectorStore.md) — In-memory vector store with nearest-neighbor search.

### Model Presets
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ To use your own fine-tuned token classification `.pte` model, pass a [`PrivacyFi

```typescript
const customFilter = await createPrivacyFilter({
modelPath: 'https://example.com/my-pii-model.pte',
modelSource: 'https://example.com/my-pii-model.pte',
tokenizerPath: 'https://example.com/tokenizer.json',
modelOpts: {
labelNames: ['O', 'B-NAME', 'I-NAME', 'E-NAME', 'S-NAME'],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ Any ColBERT model exported with a `[1, S]` token-id and attention-mask input and

```typescript
const colbert = await createColbertEmbedder({
modelPath: 'https://example.com/my-colbert.pte',
modelSource: 'https://example.com/my-colbert.pte',
tokenizerPath: 'https://example.com/tokenizer.json',
modelOpts: {
queryPrefixToken: '[Q] ',
Expand Down
8 changes: 4 additions & 4 deletions docs/docs/03-core-and-advanced/02-models-and-tensors.md
Original file line number Diff line number Diff line change
Expand Up @@ -337,8 +337,8 @@ tears it all down. The tensors are reused on every run rather than reallocated.
```typescript
import { loadModel, tensor, wrapAsync } from 'react-native-executorch';

export async function createSimpleClassifier(modelPath: string) {
const model = await wrapAsync(loadModel)(modelPath);
export async function createSimpleClassifier(modelSource: string) {
const model = await wrapAsync(loadModel)(modelSource);

const tensors = [
tensor('float32', [1, 3, 224, 224]), // tInput
Expand Down Expand Up @@ -405,12 +405,12 @@ normal disposal:
```typescript
import { createResourceScope, loadModel, tensor, wrapAsync } from 'react-native-executorch';

export async function createClassifier(modelPath: string) {
export async function createClassifier(modelSource: string) {
const scope = createResourceScope();
const dispose = scope.dispose;

try {
const model = scope.track(await wrapAsync(loadModel)(modelPath));
const model = scope.track(await wrapAsync(loadModel)(modelSource));
const tInput = scope.track(tensor('float32', [1, 3, 224, 224]));
const tOutput = scope.track(tensor('float32', [1, 1000]));

Expand Down
4 changes: 2 additions & 2 deletions docs/docs/03-core-and-advanced/06-worklets-and-threading.md
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,8 @@ High-level pipelines often need to support both async calls from React component
import { loadModel, tensor, wrapAsync } from 'react-native-executorch';
import type { WorkletRuntime } from 'react-native-worklets';

export async function createClassifier(modelPath: string, runtime?: WorkletRuntime) {
const model = await wrapAsync(loadModel, runtime)(modelPath);
export async function createClassifier(modelSource: string, runtime?: WorkletRuntime) {
const model = await wrapAsync(loadModel, runtime)(modelSource);

const tInput = tensor('float32', [1, 3, 224, 224]);
const tOutput = tensor('float32', [1, 1000]);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,20 +26,20 @@ const isObject = (value: unknown): value is Node =>
*/
const isConfig = (value: unknown): value is Node =>
isObject(value) &&
(typeof value.modelPath === 'string' ||
(isObject(value.modelPaths) &&
Object.values(value.modelPaths).every((path) => typeof path === 'string')));
(typeof value.modelSource === 'string' ||
(isObject(value.modelSources) &&
Object.values(value.modelSources).every((path) => typeof path === 'string')));

/**
* The model file(s) a config names, as one comparable identity.
*/
const modelPathsOf = (config: Node): string =>
typeof config.modelPath === 'string'
? config.modelPath
: Object.values(config.modelPaths as Node)
.map(String)
.sort()
.join('|');
const modelPathsOf = (config: Node): string => {
if (typeof config.modelSource === 'string') return config.modelSource;
return Object.values(config.modelSources as Node)
.map(String)
.sort()
.join('|');
};

/**
* A backend variant key, e.g. `XNNPACK_FP32`. Size keys (`TINY`) look the same.
Expand Down Expand Up @@ -249,7 +249,7 @@ describe('models registry — task configs', () => {
'$label bundles a tokenizer, a VAD model and its supported languages',
({ config }) => {
expect(typeof config.tokenizerPath).toBe('string');
expect(config.vadModel).toMatchObject({ modelPath: expect.any(String) });
expect(config.vadModel).toMatchObject({ modelSource: expect.any(String) });
expect(config.supportedLanguages).toEqual(expect.any(Array));
expect((config.supportedLanguages as unknown[]).length).toBeGreaterThan(0);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,15 @@ const isObject = (value: unknown): value is Node =>

const isConfig = (value: unknown): value is Node =>
isObject(value) &&
(typeof value.modelPath === 'string' ||
(isObject(value.modelPaths) &&
Object.values(value.modelPaths).every((path) => typeof path === 'string')));
(typeof value.modelSource === 'string' ||
(isObject(value.modelSources) &&
Object.values(value.modelSources).every((path) => typeof path === 'string')));

/** The `.pte` file(s) a config names, as one comparable string. */
const modelPathsOf = (config: Node): string =>
typeof config.modelPath === 'string'
? config.modelPath
: Object.values(config.modelPaths as Node)
typeof config.modelSource === 'string'
? config.modelSource
: Object.values(config.modelSources as Node)
.map(String)
.sort()
.join('|');
Expand Down
Loading
Loading