diff --git a/docs/README.md b/docs/README.md index 44d4895430..334b2d43c8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -110,3 +110,7 @@ TypeDoc runs strictly at startup. If you modify comments in the TypeScript packa 1. Stop the server. 2. Run `yarn start` again to regenerate the Markdown files. + +## Showcase recording sources + +[showcase-sources.json](./showcase-sources.json) records the devices, capture and editing details, fixture sources, and licenses for the task showcases. It is contributor metadata and is kept outside `static/` so it is not published with the website. diff --git a/docs/docs/02-extensions/computer-vision/02-image-classification.md b/docs/docs/02-extensions/computer-vision/02-image-classification.md index 35961e302f..e428358000 100644 --- a/docs/docs/02-extensions/computer-vision/02-image-classification.md +++ b/docs/docs/02-extensions/computer-vision/02-image-classification.md @@ -14,6 +14,8 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Image Classification Image classification analyzes an input image and predicts the most likely visual @@ -25,47 +27,7 @@ It is ideal for visual search, photo organization, quality inspection, and accessibility tagging. Because inference runs entirely on-device, images never leave the user's phone. - - - - - - - - - - - - - -
iOSAndroid
-
-
-
-
-
-
-
-
-
-
-
+ ## Quick Start @@ -97,7 +59,7 @@ function MyComponent() { ``` :::tip Full Interactive Example in Gallery App -See [`src/app/(screens)/image-classification.tsx`]() in the [React Native ExecuTorch Gallery](https://github.com/software-mansion-labs/react-native-executorch-gallery) for a complete, runnable screen with photo picker, result overlays, and latency tracking. +See [`src/app/(screens)/image-classification.tsx`]() in the [React Native ExecuTorch Gallery](https://github.com/software-mansion-labs/react-native-executorch-gallery) for a complete, runnable screen with a photo picker, the top prediction, and its confidence. ::: ## Output Format diff --git a/docs/docs/02-extensions/computer-vision/03-object-detection.md b/docs/docs/02-extensions/computer-vision/03-object-detection.md index 9b7d0380bc..182ee77ba7 100644 --- a/docs/docs/02-extensions/computer-vision/03-object-detection.md +++ b/docs/docs/02-extensions/computer-vision/03-object-detection.md @@ -17,53 +17,15 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Object Detection Object detection locates and classifies multiple objects within an image. For every detected item, the model predicts its category label, confidence score, and exact bounding box coordinates scaled to the original image dimensions. Unlike image classification (which predicts a single label for the entire scene), object detection tells you both **what** objects are present and **where** they are located. It is used for real-time camera tracking, retail item recognition, document scanning, robotics, and augmented reality. - - - - - - - - - - - - - -
iOSAndroid
-
-
-
-
-
-
-
-
-
-
-
+ ## Quick Start diff --git a/docs/docs/02-extensions/computer-vision/04-pose-and-keypoints.md b/docs/docs/02-extensions/computer-vision/04-pose-and-keypoints.md index e6b3ebe43e..e1bab45f69 100644 --- a/docs/docs/02-extensions/computer-vision/04-pose-and-keypoints.md +++ b/docs/docs/02-extensions/computer-vision/04-pose-and-keypoints.md @@ -19,53 +19,15 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Pose & Keypoints Pose estimation and keypoint detection locate specific anatomical landmarks on detected subjects — such as human skeletal joints (eyes, shoulders, elbows, wrists, hips, knees, ankles) or facial landmarks (eyes, nose tip, mouth, ears). Each prediction outputs a subject bounding box, detection confidence, and landmark coordinates scaled to the input image with individual landmark confidence scores. Unlike basic object detection (which only returns box boundaries), keypoint detection tracks body posture, movement, and facial alignment. Common use cases include fitness/workout tracking, gesture controls, motion analysis, face alignment, and AR filters. - - - - - - - - - - - - - -
iOSAndroid
-
-
-
-
-
-
-
-
-
-
-
+ ## Quick Start diff --git a/docs/docs/02-extensions/computer-vision/05-optical-character-recognition.md b/docs/docs/02-extensions/computer-vision/05-optical-character-recognition.md index 42cbbec875..8131467340 100644 --- a/docs/docs/02-extensions/computer-vision/05-optical-character-recognition.md +++ b/docs/docs/02-extensions/computer-vision/05-optical-character-recognition.md @@ -14,53 +14,17 @@ keywords: ] --- + + +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Optical Character Recognition (OCR) Optical Character Recognition (OCR) detects and extracts text from images. The pipeline identifies text regions with oriented quadrilateral boundaries ([`Quad`](../../06-api-reference/react-native-executorch/namespaces/cv/type-aliases/Quad.md)) and transcribes their characters in reading order (top-to-bottom, left-to-right columns). It is used for document digitizing, receipt scanning, license plate reading, sign translation, and invoice processing. Because inference runs entirely on-device with zero network latency, sensitive documents never leave the phone. - - - - - - - - - - - - - -
iOSAndroid
-
-
-
-
-
-
-
-
-
-
-
+ ## Quick Start diff --git a/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md b/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md index a768bdafec..2ff8ad84eb 100644 --- a/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md +++ b/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md @@ -17,53 +17,15 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Semantic Segmentation Semantic segmentation classifies every individual pixel of an input image into a designated category label (e.g. background, person, vehicle, dog). The pipeline produces a pixel-aligned segmentation mask matching the input dimensions. Unlike object detection (which outputs rectangular bounding boxes), semantic segmentation delivers precise pixel boundaries. It powers photo portrait effects, background blur/replacement, scene parsing, medical imaging, and autonomous navigation. - - - - - - - - - - - - - -
iOSAndroid
-
-
-
-
-
-
-
-
-
-
-
+ ## Quick Start diff --git a/docs/docs/02-extensions/computer-vision/07-instance-segmentation.md b/docs/docs/02-extensions/computer-vision/07-instance-segmentation.md index ef9d63669f..8d4150353f 100644 --- a/docs/docs/02-extensions/computer-vision/07-instance-segmentation.md +++ b/docs/docs/02-extensions/computer-vision/07-instance-segmentation.md @@ -17,53 +17,15 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Instance Segmentation Instance segmentation combines object detection and semantic segmentation. For every detected individual object in an image, the pipeline predicts its bounding box, category label, detection confidence, and a pixel-accurate binary mask cropped to the instance bounds. Unlike semantic segmentation (which groups all pixels of the same category into a single collective mask), instance segmentation distinguishes between separate instances of the same class (e.g. `person #1`, `person #2`). It powers interactive photo cutouts, object isolation, background effects, AR occlusions, and automated video editing. - - - - - - - - - - - - - -
iOSAndroid
-
-
-
-
-
-
-
-
-
-
-
+ ## Quick Start diff --git a/docs/docs/02-extensions/computer-vision/08-style-transfer.md b/docs/docs/02-extensions/computer-vision/08-style-transfer.md index a958db1505..b373a362b7 100644 --- a/docs/docs/02-extensions/computer-vision/08-style-transfer.md +++ b/docs/docs/02-extensions/computer-vision/08-style-transfer.md @@ -18,53 +18,15 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Neural Style Transfer Neural style transfer renders an input image in the artistic style of another image (such as famous paintings or pattern textures) while preserving the semantic content and structure of the original photo. Because the models run locally in real time on mobile hardware accelerators, you can apply artistic filters to live camera frames or photos without uploading user media to external servers. - - - - - - - - - - - - - -
iOSAndroid
-
-
-
-
-
-
-
-
-
-
-
+ ## Quick Start @@ -96,7 +58,7 @@ function MyComponent() { ``` :::tip Full Interactive Example in Gallery App -See [`src/app/(screens)/style-transfer.tsx`]() in the [React Native ExecuTorch Gallery](https://github.com/software-mansion-labs/react-native-executorch-gallery) for a complete, runnable screen with photo picker, side-by-side style comparisons, and latency tracking. +See [`src/app/(screens)/style-transfer.tsx`]() in the [React Native ExecuTorch Gallery](https://github.com/software-mansion-labs/react-native-executorch-gallery) for a complete, runnable screen with a photo picker, hold-to-compare interaction, and latency tracking. ::: ## Output Format diff --git a/docs/docs/02-extensions/computer-vision/09-image-embeddings.md b/docs/docs/02-extensions/computer-vision/09-image-embeddings.md index ef0bd35e3f..596b524fa6 100644 --- a/docs/docs/02-extensions/computer-vision/09-image-embeddings.md +++ b/docs/docs/02-extensions/computer-vision/09-image-embeddings.md @@ -16,53 +16,15 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Image Embeddings Image embedding models extract high-dimensional semantic feature vectors (embeddings) from raw images. When paired with multimodal models like OpenAI CLIP (Contrastive Language-Image Pretraining) and [Text Embeddings](../../02-extensions/natural-language/03-text-embeddings.md), image and text embeddings share the same joint vector space. This enables on-device cross-modal photo search (finding pictures with natural language queries), zero-shot image classification, visual similarity clustering, and vector search against local SQLite vector stores — all computed entirely on-device without network latency or cloud costs. - - - - - - - - - - - - - -
iOSAndroid
-
-
-
-
-
-
-
-
-
-
-
+ ## Quick Start @@ -94,7 +56,7 @@ function MyComponent() { ``` :::tip Full Interactive Example in Gallery App -See [`src/app/(screens)/image-embeddings.tsx`]() in the [React Native ExecuTorch Gallery](https://github.com/software-mansion-labs/react-native-executorch-gallery) for a complete, runnable screen combining image and text embeddings for real-time zero-shot photo ranking. +See [`src/app/(screens)/image-embeddings.tsx`]() in the [React Native ExecuTorch Gallery](https://github.com/software-mansion-labs/react-native-executorch-gallery) for a complete, runnable screen combining image and text embeddings for interactive on-device photo ranking. ::: ## Output Format diff --git a/docs/docs/02-extensions/computer-vision/10-text-to-image.md b/docs/docs/02-extensions/computer-vision/10-text-to-image.md index 124cbfd9dd..f19af54da9 100644 --- a/docs/docs/02-extensions/computer-vision/10-text-to-image.md +++ b/docs/docs/02-extensions/computer-vision/10-text-to-image.md @@ -16,53 +16,15 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Text to Image Text-to-image diffusion models generate photorealistic and artistic images directly from natural language descriptive prompts. The library ships with SDXS-512 (Stable Diffusion eXtreme Speed) based on DreamShaper. Through architectural distillation, SDXS collapses multi-step denoising into a fast, single-step latent diffusion pipeline capable of generating 512x512 images completely on-device without cloud GPUs. - - - - - - - - - - - - - -
iOSAndroid
-
-
-
-
-
-
-
-
-
-
-
+ ## Quick Start diff --git a/docs/docs/02-extensions/computer-vision/11-image-inpainting.md b/docs/docs/02-extensions/computer-vision/11-image-inpainting.md index 7e2cb19884..33fb7f010b 100644 --- a/docs/docs/02-extensions/computer-vision/11-image-inpainting.md +++ b/docs/docs/02-extensions/computer-vision/11-image-inpainting.md @@ -17,12 +17,16 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Image Inpainting Image inpainting fills a region of a photo with content that matches its surroundings: paint over an unwanted object, a blemish, or a passer-by, and the model reconstructs what would plausibly be behind it. The pipeline runs entirely on-device. It runs the model on the whole photo resized to the model's resolution and blends the result back into the full-resolution photo through the mask, so the rest of the image is returned pixel for pixel as it came in. + + ## Quick Start The [`useInpainter`](../../06-api-reference/functions/useInpainter.md) hook manages model downloading, initialization, and lifecycle: @@ -53,7 +57,7 @@ function MyComponent() { ``` :::tip Full Interactive Example -See [`apps/computer-vision/app/inpainting`](https://github.com/software-mansion/react-native-executorch/tree/main/apps/computer-vision/app/inpainting) in this repository for a complete screen with a photo picker, a brush to paint the mask over the photo, and before/after comparison. +See [`src/app/(screens)/inpainting.tsx`]() in the [React Native ExecuTorch Gallery](https://github.com/software-mansion-labs/react-native-executorch-gallery) for a complete, runnable screen with brush masking and hold-to-compare interaction. ::: ## The Mask diff --git a/docs/docs/02-extensions/computer-vision/12-camera-integration.md b/docs/docs/02-extensions/computer-vision/12-camera-integration.md index a3884e6e63..13879974e4 100644 --- a/docs/docs/02-extensions/computer-vision/12-camera-integration.md +++ b/docs/docs/02-extensions/computer-vision/12-camera-integration.md @@ -15,6 +15,10 @@ keywords: ] --- + + +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Real-Time Camera Processing React Native ExecuTorch models can process live camera streams directly on-device with zero cloud dependencies. By combining [VisionCamera v5](https://visioncamera.margelo.com), [VisionCamera Resizer](https://github.com/mrousavy/react-native-vision-camera/tree/main/packages/react-native-vision-camera-resizer), and [`react-native-worklets`](https://docs.swmansion.com/react-native-worklets/), live camera frames are scaled and converted on the GPU directly into an [`ImageBuffer`](../../06-api-reference/react-native-executorch/namespaces/cv/type-aliases/ImageBuffer.md), processed synchronously inside the camera frame processor worklet, and dispatched back to the React JavaScript thread via [`scheduleOnRN`](https://docs.swmansion.com/react-native-worklets/docs/threading/scheduleOnRN/) with zero thread-switching or promise overhead. @@ -22,47 +26,9 @@ React Native ExecuTorch models can process live camera streams directly on-devic import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; - - - - - - - - - - - - - -
iOSAndroid
-
-
-
-
-
-
-
-
-
-
-
+ + +The showcase runs on-device inference on prerecorded sample video of skaters and a corgi: [video by Chris Zhou on Pexels](https://www.pexels.com/video/skateboarding-with-a-dog-in-urban-park-37368575/), used under the [Pexels License](https://www.pexels.com/license/). The screen implementation below uses the live camera frame pipeline. ## Quick Start diff --git a/docs/docs/02-extensions/computer-vision/12-depth-estimation.md b/docs/docs/02-extensions/computer-vision/12-depth-estimation.md index 5c306c3df3..e0a6648768 100644 --- a/docs/docs/02-extensions/computer-vision/12-depth-estimation.md +++ b/docs/docs/02-extensions/computer-vision/12-depth-estimation.md @@ -14,10 +14,14 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Depth Estimation Depth estimation predicts a depth value for every pixel in an image. The pipeline returns a floating-point depth map matching the original image dimensions, which can be used for portrait blur, photo animation, and scene understanding. + + ## Quick Start The [`useDepthEstimator`](../../06-api-reference/functions/useDepthEstimator.md) hook manages model downloading, initialization, and lifecycle: @@ -50,7 +54,7 @@ function MyComponent() { ``` :::tip Full Interactive Example -See [`apps/computer-vision/app/depthEstimation/index.tsx`](https://github.com/software-mansion/react-native-executorch/blob/main/apps/computer-vision/app/depthEstimation/index.tsx) for a complete screen with model selection, photo capture, depth visualization, interactive photo effects, and latency tracking. +See [`src/app/(screens)/depth-estimation.tsx`]() in the [React Native ExecuTorch Gallery](https://github.com/software-mansion-labs/react-native-executorch-gallery) for a complete, runnable screen with depth-guided tap-to-focus effects. ::: ## Output Format diff --git a/docs/docs/02-extensions/natural-language/02-llm-chat-and-generation.md b/docs/docs/02-extensions/natural-language/02-llm-chat-and-generation.md index 28286e5b4c..f16d5c1a13 100644 --- a/docs/docs/02-extensions/natural-language/02-llm-chat-and-generation.md +++ b/docs/docs/02-extensions/natural-language/02-llm-chat-and-generation.md @@ -21,6 +21,8 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # LLM Chat & Text Generation The LLM extension lets you run generative Large Language Models (LLMs) and Vision-Language Models (VLMs) directly on user devices with real-time token streaming, complete privacy, and full offline support. Depending on what you are building, you can choose between two levels of control: @@ -28,47 +30,7 @@ The LLM extension lets you run generative Large Language Models (LLMs) and Visio - **Chat Sessions ([`useLLMChatSession`](#quick-start) / [`createLLMChatSession`](#imperative-session-api))**: The recommended API for conversational apps and AI assistants. It manages multi-turn conversation history, applies [Jinja2 chat templates](#chat-templates--incremental-kv-cache-diffing), supports [multimodal image inputs](#multimodal-inputs), and handles [automated tool calling](#automated-tool-calling). - **Low-Level Runner ([`LLMRunner`](#low-level-runner))**: A direct execution engine that operates on worklet threads. It processes raw text strings or media tensors without chat formatting, giving you manual control over KV cache prefilling, synchronous generation loops, and context rewinding. - - - - - - - - - - - - - -
iOSAndroid
-
-
-
-
-
-
-
-
-
-
-
+ ## Quick Start @@ -114,7 +76,7 @@ function MyChatComponent() { ``` :::tip Full Interactive Example in Gallery App -See [`src/app/(screens)/llm-chat.tsx`]() in the [React Native ExecuTorch Gallery](https://github.com/software-mansion-labs/react-native-executorch-gallery) for a complete, runnable chat UI with token streaming, token/sec benchmarking, and message history. +See [`src/app/(screens)/llm-chat.tsx`]() in the [React Native ExecuTorch Gallery](https://github.com/software-mansion-labs/react-native-executorch-gallery) for a complete, runnable chat UI with streamed responses, conversation context, and tool calls you can inspect. ::: :::tip Answering From Your Own Documents @@ -183,6 +145,10 @@ If you use a custom model whose Jinja template dynamically rewires earlier turns ## Multimodal Inputs + + +The gallery's [image descriptions screen]() sends a photo and a question in one message, then streams the model's answer. The showcase uses [`LFM3_VL_2B`](../../06-api-reference/variables/models.md#llmlfm3_vl_2b) to describe the performer in the sample photo. + Vision-Language Models (such as Liquid AI's [`LFM2_5_VL_450M`](../../06-api-reference/variables/models.md#llmlfm2_5_vl_450m) and [`LFM2_5_VL_1_6B`](../../06-api-reference/variables/models.md#llmlfm2_5_vl_1_6b)) process interleaved text and visual payloads: ```typescript @@ -217,6 +183,10 @@ The session automatically embeds the model's sentinel vision tokens, resizes and The LLM chat session supports automated, multi-turn tool calling (function calling). When tool definitions and a parser are supplied, the session automatically invokes tool callbacks, feeds their results back into the conversation, and returns the final assistant answer. +See the gallery's [local chat screen]() for a working conversation with calculator, unit conversion and device-clock tools. Expand a tool call to inspect its arguments and returned value. + + + ### 1. Define Tools with `execute` Declare tools matching standard JSON Schema specifications along with an asynchronous `execute` handler: @@ -290,6 +260,30 @@ const session = useLLMChatSession(models.llm.LFM2_5_1_2B.DEFAULT, { }); ``` +## Structured Output + +Pass `constraints` in the per-turn options to `sendMessage()` to restrict which tokens the model can generate. A grammar can use these constraints to produce JSON matching the structure your app expects. Parse and validate the completed response before using it. + +The gallery's [Tiny widget builder screen]() demonstrates this with a JSON grammar: the model streams a widget definition, and the app renders its controls. Open the output viewer to inspect the generated JSON. The screen shows the session call; [the grammar implementation](https://github.com/software-mansion-labs/react-native-executorch-gallery/blob/main/src/lib/widget-builder/app-builder-grammar.ts) defines the allowed structure. + + + +The following excerpt uses the gallery's `appBuilderConstraints` helper, not a library export. `session` is a ready chat session; `grammar` and `tokens` are the compiled widget grammar and token table from the linked implementation. + +```typescript +import { appBuilderConstraints } from '@/lib/widget-builder/app-builder-grammar'; + +// Create a fresh callback, including its grammar cursor and cache, for every sendMessage(). +const result = await session.sendMessage('Build a packing checklist.', undefined, { + enableThinking: false, + maxNewTokens: 1024, + constraints: appBuilderConstraints(grammar, tokens), +}); +console.log(result.messages); +``` + +The `constraints` callback runs before each token is sampled on the worklet thread. It must be a worklet, and any helpers it calls must also work in that runtime. Keep incremental state in worklet-compatible storage, as the gallery helper does with `createSynchronizable`; create fresh state for each `sendMessage()` so a new response starts at the beginning of the grammar. The `onToken` streaming callback (the second argument) runs on the React Native thread. + ## Imperative Session API For headless background services or non-React architectures, create a full chat session imperatively using [`createLLMChatSession`](../../06-api-reference/functions/createLLMChatSession.md): diff --git a/docs/docs/02-extensions/natural-language/03-text-embeddings.md b/docs/docs/02-extensions/natural-language/03-text-embeddings.md index da559ffb04..ae4e151956 100644 --- a/docs/docs/02-extensions/natural-language/03-text-embeddings.md +++ b/docs/docs/02-extensions/natural-language/03-text-embeddings.md @@ -18,53 +18,15 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Text Embeddings Text embedding models convert sentences, paragraphs, or documents into dense numeric vectors (embeddings). Sentences with similar semantic meaning map to nearby points in the vector space, even when using completely different vocabulary. This enables on-device semantic search, offline Retrieval-Augmented Generation (RAG) against local SQLite vector stores, intent classification, and cross-modal text-to-image queries when paired with [Image Embeddings](../computer-vision/09-image-embeddings.md) — entirely on the client without sending private text to cloud APIs. - - - - - - - - - - - - - -
iOSAndroid
-
-
-
-
-
-
-
-
-
-
-
+ ## Quick Start @@ -95,7 +57,7 @@ function MyComponent() { ``` :::tip Full Interactive Example in Gallery App -See [`src/app/(screens)/image-embeddings.tsx`]() in the [React Native ExecuTorch Gallery](https://github.com/software-mansion-labs/react-native-executorch-gallery) for a complete, runnable screen combining text and image embeddings for real-time cross-modal search. +See [`src/app/(screens)/image-embeddings.tsx`]() in the [React Native ExecuTorch Gallery](https://github.com/software-mansion-labs/react-native-executorch-gallery) for a complete, runnable screen combining text and image embeddings for interactive on-device photo search. ::: ## Output Format diff --git a/docs/docs/02-extensions/natural-language/04-privacy-filter.md b/docs/docs/02-extensions/natural-language/04-privacy-filter.md index 847dd8a9d6..0c57ac6118 100644 --- a/docs/docs/02-extensions/natural-language/04-privacy-filter.md +++ b/docs/docs/02-extensions/natural-language/04-privacy-filter.md @@ -18,53 +18,15 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Privacy Filter Privacy Filter models detect Personally Identifiable Information (PII) — such as personal names, email addresses, phone numbers, physical addresses, API keys, and credentials — in natural language text. By scanning text entirely on-device before sending prompts to cloud APIs, logging systems, or analytics backends, you can automatically redact or mask sensitive user data without exposing personal details over the network. - - - - - - - - - - - - - -
iOSAndroid
-
-
-
-
-
-
-
-
-
-
-
+ ## Quick Start diff --git a/docs/docs/02-extensions/natural-language/06-colbert-retrieval.md b/docs/docs/02-extensions/natural-language/06-colbert-retrieval.md index 49b32999b7..43ab9c2d6e 100644 --- a/docs/docs/02-extensions/natural-language/06-colbert-retrieval.md +++ b/docs/docs/02-extensions/natural-language/06-colbert-retrieval.md @@ -18,10 +18,14 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # ColBERT Retrieval A [Text Embeddings](./03-text-embeddings.md) model pools a whole text into one vector. A ColBERT model instead keeps **one vector per token** and scores a query against a document with **MaxSim**: for every query token, the similarity of the closest document token, summed over the query. Matching at the token level makes it more accurate on retrieval than a single pooled vector, at the cost of a larger index (one 128-dimensional vector per document token). + + ## Quick Start The [`useColbertEmbedder`](../../06-api-reference/functions/useColbertEmbedder.md) hook manages model downloading, tokenizer loading, and lifecycle. Embed every document once with `'document'`, embed each search with `'query'`, and rank with [`maxSim()`](../../06-api-reference/type-aliases/ColbertEmbedder.md#maxsim): @@ -50,6 +54,10 @@ function Search({ documents }: { documents: string[] }) { } ``` +:::tip Full Interactive Example in Gallery App +See [`src/app/(screens)/colbert-retrieval.tsx`]() in the [React Native ExecuTorch Gallery](https://github.com/software-mansion-labs/react-native-executorch-gallery) for a complete, runnable screen with a small city guide, MaxSim scoring, and ranked result cards. +::: + ## Output Format [`embed()`](../../06-api-reference/type-aliases/ColbertEmbedder.md#embed) returns a [`ColbertTokenEmbeddings`](../../06-api-reference/type-aliases/ColbertTokenEmbeddings.md) object: a row-major `[numTokens, dimension]` matrix of L2-normalized vectors in a single `Float32Array`. diff --git a/docs/docs/02-extensions/natural-language/07-translation.md b/docs/docs/02-extensions/natural-language/07-translation.md index 154e91c9d2..b4f39b8631 100644 --- a/docs/docs/02-extensions/natural-language/07-translation.md +++ b/docs/docs/02-extensions/natural-language/07-translation.md @@ -16,6 +16,8 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Translation Translation models turn text in one language into another without sending it off the device. Two kinds are available behind the same API: @@ -24,6 +26,8 @@ Translation models turn text in one language into another without sending it off - [EuroLLM-1.7B-Instruct](https://huggingface.co/utter-project/EuroLLM-1.7B-Instruct): one 1.7 GB model that translates between any two of its 35 languages. - [Hy-MT2-1.8B](https://huggingface.co/tencent/Hy-MT2-1.8B): one 1.2 GB model that translates between any two of its 38 languages and dialects, and is stronger than EuroLLM on Chinese. + + ## Quick Start The [`useTranslator`](../../06-api-reference/functions/useTranslator.md) hook downloads the `.pte` model and tokenizer, loads the pipeline, and manages its lifecycle. Every call names its language pair as ISO 639-1 codes: @@ -50,6 +54,10 @@ function MyComponent() { An Opus-MT model accepts only its own pair. With `models.translation.EUROLLM_1_7B.DEFAULT` or `models.translation.HY_MT2_1_8B.DEFAULT`, the pair can change on every call without reloading the model. +:::tip Full Interactive Example in Gallery App +See [`src/app/(screens)/translation.tsx`]() in the [React Native ExecuTorch Gallery](https://github.com/software-mansion-labs/react-native-executorch-gallery) for a complete, runnable screen for English-to-German translation with Opus-MT. +::: + ## Streaming Pass a callback to receive the translation as it is decoded. Each call gets the newly decoded piece of text; joined together, the pieces equal the returned translation. diff --git a/docs/docs/02-extensions/speech/02-text-to-speech.md b/docs/docs/02-extensions/speech/02-text-to-speech.md index fc4a8a2906..7f9688f396 100644 --- a/docs/docs/02-extensions/speech/02-text-to-speech.md +++ b/docs/docs/02-extensions/speech/02-text-to-speech.md @@ -18,6 +18,8 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Text-to-Speech (TTS) The Text-to-Speech extension synthesizes natural, expressive spoken audio waveforms directly on-device from input text. @@ -29,47 +31,7 @@ Speech synthesis operates through multi-stage neural pipelines that combine phon Both pipelines are wrapped uniformly by the [`useTextToSpeech`](../../06-api-reference/functions/useTextToSpeech.md) React hook. - - - - - - - - - - - - - -
iOSAndroid
-
-
-
-
-
-
-
-
-
-
-
+ ## Quick Start @@ -109,7 +71,7 @@ function SpeechComponent() { ``` :::tip Full Interactive Example in Gallery App -See [`src/app/(screens)/text-to-speech.tsx`]() in the [React Native ExecuTorch Gallery](https://github.com/software-mansion-labs/react-native-executorch-gallery) for a complete, runnable screen featuring voice selection, buffer queue streaming, Time-to-First-Audio (TTFA) benchmarking, and live waveform visualization. +See [`src/app/(screens)/text-to-speech.tsx`]() in the [React Native ExecuTorch Gallery](https://github.com/software-mansion-labs/react-native-executorch-gallery) for a complete, runnable screen featuring streamed playback and synchronized word highlighting. ::: ## Output Format diff --git a/docs/docs/02-extensions/speech/03-speech-to-text.md b/docs/docs/02-extensions/speech/03-speech-to-text.md index 33da3017b8..f6ac2ac496 100644 --- a/docs/docs/02-extensions/speech/03-speech-to-text.md +++ b/docs/docs/02-extensions/speech/03-speech-to-text.md @@ -19,6 +19,8 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Speech-to-Text (STT) The Speech-to-Text extension transcribes spoken audio into text directly on-device using OpenAI's Whisper or Alibaba's [Qwen3-ASR](https://huggingface.co/Qwen/Qwen3-ASR-0.6B) paired with an integrated FSMN Voice Activity Detector (VAD). @@ -28,47 +30,9 @@ The pipeline supports two primary workflows: - [**Live Microphone Streaming**](#output-format--live-streaming): Streams real-time audio straight from the microphone. As the user speaks, Whisper continuously returns draft transcripts and automatically commits finalized sentences upon pauses. - [**Pre-recorded Audio Transcription**](#pre-recorded-audio-transcription): Transcribes pre-recorded audio buffers or audio files in a single pass, with optional token-by-token streaming callbacks. - - - - - - - - - - - - - -
iOSAndroid
-
-
-
-
-
-
-
-
-
-
-
+ + +The showcase streams a fixed speech-and-silence sample through the on-device model for repeatable results. ## Quick Start diff --git a/docs/docs/02-extensions/speech/04-voice-activity-detection.md b/docs/docs/02-extensions/speech/04-voice-activity-detection.md index 739ca457bc..91c50f45f2 100644 --- a/docs/docs/02-extensions/speech/04-voice-activity-detection.md +++ b/docs/docs/02-extensions/speech/04-voice-activity-detection.md @@ -16,6 +16,8 @@ keywords: ] --- +import TaskShowcase from '@site/src/components/TaskShowcase'; + # Voice Activity Detection (VAD) The Voice Activity Detection extension detects speech presence and segments audio into spoken and silent intervals directly on-device using a lightweight Feedforward Sequential Memory Network (FSMN-VAD) model. @@ -25,47 +27,9 @@ The pipeline supports two primary workflows: - [**Live Microphone Event Stream**](#live-microphone-streaming): Processes incoming audio chunks from a microphone recorder in real time, firing `'speechStart'` and `'speechEnd'` transitions. - [**Batch Audio Segmentation**](#batch-audio-segmentation): Analyzes an entire recorded audio buffer and returns an array of timestamped speech segments in seconds. - - - - - - - - - - - - - -
iOSAndroid
-
-
-
-
-
-
-
-
-
-
-
+ + +The showcase streams a fixed speech-and-silence sample through the on-device model for repeatable results. ## Quick Start diff --git a/docs/showcase-sources.json b/docs/showcase-sources.json new file mode 100644 index 0000000000..90e5c6702c --- /dev/null +++ b/docs/showcase-sources.json @@ -0,0 +1,56 @@ +{ + "recordedAt": "2026-11-05", + "libraryCommit": "4b7d6fd6fea10f9b3422121d6e439ccf88df93b2", + "devices": { + "ios": { + "model": "iPhone 17", + "os": "iOS 26.6", + "captureResolution": [1206, 2622] + }, + "android": { + "model": "Pixel 10", + "os": "Android 17", + "captureResolution": [1080, 2424] + } + }, + "capture": { + "appearance": "dark", + "inference": "Actual on-device model inference. Recording-only adapters supply deterministic photo, video and PCM audio inputs; text presets are entered without focusing the keyboard. These adapters are not shipped in the gallery.", + "videoAudio": "Showcase exports are muted.", + "editing": "Idle lead-ins and excess final pauses trimmed; completed result screenshots from the same capture held for reading in OCR, translation and multimodal input. Translation loops are six seconds on both platforms. Camera clips use short continuous loops. Model processing remains at recorded speed." + }, + "fixtures": { + "camera": { + "title": "Skateboarding with a Dog in Urban Park", + "author": "Chris Zhou", + "source": "https://www.pexels.com/video/skateboarding-with-a-dog-in-urban-park-37368575/", + "license": "Pexels License", + "licenseUrl": "https://www.pexels.com/license/", + "verifiedAt": "2026-11-05", + "processing": "Muted 30 fps native video preview and synchronized 30 fps sampled inputs. Detection runs at device speed, skips stale frames, and yields between synchronous calls to keep overlays responsive. These clips are edited demonstrations, not end-to-end camera performance benchmarks." + }, + "photo": { + "origin": "AI-generated using imagegen, 2026-11-05", + "prompt": "Use case: photorealistic-natural. Asset type: a striking portrait sample photograph for on-device computer-vision demos, not a UI mockup. Create one polished, believable editorial photograph of a joyful young adult street performer in a mustard jacket and teal trousers, standing with one foot on the pedal of a stationary scarlet vintage bicycle while juggling three bright oranges in a sunlit Mediterranean courtyard. A charming corgi wearing a small cobalt-blue bandana sits prominently in the foreground beside the bicycle; a mint-green flower cart with overflowing pink and yellow flowers and one open striped caf\u00e9 umbrella sit in the middle distance. Playful and a little cinematic, tactile real-life materials, tasteful saturated colors, golden-hour light, a pale peach wall and climbing greenery. Composition: portrait 3:4, the performer's entire body and both hands and feet fully visible, clear arms and legs, bicycle fully visible, corgi fully visible with clear facial features; breathing room around all subjects. Well-separated recognizable objects with strong silhouettes for object detection and segmentation: one person, one bicycle, one dog, oranges, flowers in pots, umbrella. Several clear foreground/midground/background depth layers. Include one isolated small blue ceramic cup on the stone pavement toward the lower right, with uncluttered pavement around it, suitable for an inpainting demo. Keep the person, dog, and bicycle central enough to stay visible in a portrait phone preview. Natural realistic anatomy and plausible shadows, no illustration or miniature look, no screenshots, no overlays, no bounding boxes, no text, no logos, no watermark." + }, + "ocr": { + "origin": "Original recipe card generated for this showcase." + }, + "speech": { + "origin": "Synthetic speech generated locally with macOS Samantha voice, with leading/trailing silence.", + "text": "Welcome to the React Native ExecuTorch Gallery. Everything you see runs right on this device. Your words stay private, even when you are offline." + }, + "multimodal": { + "model": "LFM3 VL 2B", + "photo": "Generated courtyard sample documented above", + "prompt": "What is the man doing? Answer in one short sentence.", + "response": "The man is juggling oranges.", + "formats": ["mp4"] + } + }, + "frames": { + "style": "Schematic frames with shared height and original capture aspect ratios. Button positions scale with the frame, following manufacturer front-view diagrams.", + "iosReference": "https://support.apple.com/guide/iphone/iph15f87b8cf/ios", + "androidReference": "https://support.google.com/pixelphone/answer/7157629?hl=en" + } +} diff --git a/docs/src/components/TaskShowcase/index.tsx b/docs/src/components/TaskShowcase/index.tsx new file mode 100644 index 0000000000..a355ed8672 --- /dev/null +++ b/docs/src/components/TaskShowcase/index.tsx @@ -0,0 +1,111 @@ +import React, { useEffect, useRef, useState } from 'react'; +import useBaseUrl from '@docusaurus/useBaseUrl'; + +import styles from './styles.module.css'; + +function ShowcaseVideo({ src, label }: { src: string; label: string }) { + const ref = useRef(null); + const [loaded, setLoaded] = useState(false); + + useEffect(() => { + const video = ref.current; + if (!video) return; + if (typeof IntersectionObserver === 'undefined') { + setLoaded(true); + return; + } + + const observer = new IntersectionObserver( + ([entry]) => { + if (entry.isIntersecting) { + setLoaded(true); + observer.disconnect(); + } + }, + { rootMargin: '200px' } + ); + observer.observe(video); + return () => observer.disconnect(); + }, []); + + useEffect(() => { + const video = ref.current; + if (!video || !loaded) return; + // Playback can be blocked by browser autoplay settings. + const play = () => { + video.play().catch(() => {}); + }; + if (typeof IntersectionObserver === 'undefined') { + play(); + return () => video.pause(); + } + + const observer = new IntersectionObserver(([entry]) => { + if (entry.isIntersecting) play(); + else video.pause(); + }); + observer.observe(video); + return () => { + observer.disconnect(); + video.pause(); + }; + }, [loaded]); + + return ( +