diff --git a/apps/nlp/app/tokenizer/index.tsx b/apps/nlp/app/tokenizer/index.tsx index 698ba683c4..b58c78b447 100644 --- a/apps/nlp/app/tokenizer/index.tsx +++ b/apps/nlp/app/tokenizer/index.tsx @@ -35,9 +35,9 @@ function TokenizerContent() { const decoded = await decode(tokenIds, true); const vocab = getVocabSize(); - // Self-consistent inverse check on a token from the actual output - // (HFTokenizer adds special tokens per the tokenizer.json post_processor). - const sampleId = tokenIds[Math.min(1, tokenIds.length - 1)]!; + // Self-consistent inverse check on the first token of the actual output + // (encode adds no special tokens, so it is a real word piece). + const sampleId = tokenIds[0]!; const sampleToken = idToToken(sampleId); const sampleIdBack = tokenToId(sampleToken); diff --git a/docs/docs/02-extensions/natural-language/03-text-embeddings.md b/docs/docs/02-extensions/natural-language/03-text-embeddings.md index ae4e151956..cc82596ef9 100644 --- a/docs/docs/02-extensions/natural-language/03-text-embeddings.md +++ b/docs/docs/02-extensions/natural-language/03-text-embeddings.md @@ -174,7 +174,7 @@ const customEmbedder = await createTextEmbedder({ }); ``` -The pipeline automatically verifies that the model's exported input and output shapes match its requirements. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline). +The pipeline automatically verifies that the model's exported input and output shapes match its requirements. The tokenizer adds no special tokens, so a model that expects them (for example `[CLS]` and `[SEP]` for BERT-style models) must add them inside its exported graph. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline). ::: ## API Reference diff --git a/docs/docs/02-extensions/natural-language/05-tokenizers.md b/docs/docs/02-extensions/natural-language/05-tokenizers.md index 0874209fb2..308ac08aef 100644 --- a/docs/docs/02-extensions/natural-language/05-tokenizers.md +++ b/docs/docs/02-extensions/natural-language/05-tokenizers.md @@ -23,7 +23,7 @@ keywords: Tokenizers translate human-readable natural language text into numeric token ID arrays (and decode token ID sequences back into text). -The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with full support for normalizers, pre-tokenizers, truncation, padding, and post-processors (such as adding special `[CLS]` and `[SEP]` tokens automatically). +The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with support for normalizers, pre-tokenizers, truncation and padding. `encode` does not add special tokens such as `[CLS]` and `[SEP]` (see [encode](#1-encodetext)). ## Native Tokenizer (loadTokenizer) @@ -55,11 +55,11 @@ The [`Tokenizer`](../../06-api-reference/react-native-executorch/namespaces/nlp/ ### 1. encode(text) -Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens are automatically appended/prepended according to the `tokenizer.json` post-processor configuration (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece): +Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens from the `tokenizer.json` post-processor (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece) are not added. The `truncation` and `padding` settings of `tokenizer.json` still apply. If your model expects special tokens, add them yourself or inside the exported graph: ```typescript const ids: Int32Array = tokenizer.encode('ExecuTorch on React Native'); -// e.g. Int32Array([101, 10769, 2178, 2006, 2690, 3110, 102]) +// e.g. Int32Array([10769, 2178, 2006, 2690, 3110]) ``` ### 2. decode(tokens, skipSpecialTokens?) @@ -68,7 +68,8 @@ Decodes an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScrip ```typescript const cleanText = tokenizer.decode(ids); // "ExecuTorch on React Native" -const rawText = tokenizer.decode(ids, false); // "[CLS] ExecuTorch on React Native [SEP]" +const framed = new Int32Array([101, ...ids, 102]); // [CLS] ... [SEP], added by hand +const rawText = tokenizer.decode(framed, false); // "[CLS] ExecuTorch on React Native [SEP]" ``` ### 3. Vocabulary & Piece Inspection @@ -132,7 +133,7 @@ const tokenizer = useTokenizer( const nativeTokenizer = nlp.loadTokenizer('/path/to/local/tokenizer.json'); ``` -The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, and post-processor rules defined in the JSON file. +The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, truncation and padding defined in the JSON file. Special tokens from the post-processor are not added (see [encode](#1-encodetext)). ## API Reference diff --git a/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/03-text-embeddings.md b/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/03-text-embeddings.md index 27c7f62f2f..90f7d1453c 100644 --- a/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/03-text-embeddings.md +++ b/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/03-text-embeddings.md @@ -194,7 +194,7 @@ const customEmbedder = await createTextEmbedder({ }); ``` -The pipeline automatically verifies that the model's exported input and output shapes match its requirements. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline). +The pipeline automatically verifies that the model's exported input and output shapes match its requirements. The tokenizer adds no special tokens, so a model that expects them (for example `[CLS]` and `[SEP]` for BERT-style models) must add them inside its exported graph. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline). ::: ## API Reference diff --git a/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/05-tokenizers.md b/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/05-tokenizers.md index 2e4a45d59c..87b9e433c1 100644 --- a/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/05-tokenizers.md +++ b/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/05-tokenizers.md @@ -23,7 +23,7 @@ keywords: Tokenizers translate human-readable natural language text into numeric token ID arrays (and decode token ID sequences back into text). -The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with full support for normalizers, pre-tokenizers, truncation, padding, and post-processors (such as adding special `[CLS]` and `[SEP]` tokens automatically). +The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with support for normalizers, pre-tokenizers, truncation and padding. `encode` does not add special tokens such as `[CLS]` and `[SEP]` (see [encode](#1-encodetext)). ## Native Tokenizer (loadTokenizer) @@ -55,11 +55,11 @@ The [`Tokenizer`](../../06-api-reference/react-native-executorch/namespaces/nlp/ ### 1. encode(text) -Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens are automatically appended/prepended according to the `tokenizer.json` post-processor configuration (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece): +Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens from the `tokenizer.json` post-processor (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece) are not added. The `truncation` and `padding` settings of `tokenizer.json` still apply. If your model expects special tokens, add them yourself or inside the exported graph: ```typescript const ids: Int32Array = tokenizer.encode('ExecuTorch on React Native'); -// e.g. Int32Array([101, 10769, 2178, 2006, 2690, 3110, 102]) +// e.g. Int32Array([10769, 2178, 2006, 2690, 3110]) ``` ### 2. decode(tokens, skipSpecialTokens?) @@ -68,7 +68,8 @@ Decodes an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScrip ```typescript const cleanText = tokenizer.decode(ids); // "ExecuTorch on React Native" -const rawText = tokenizer.decode(ids, false); // "[CLS] ExecuTorch on React Native [SEP]" +const framed = new Int32Array([101, ...ids, 102]); // [CLS] ... [SEP], added by hand +const rawText = tokenizer.decode(framed, false); // "[CLS] ExecuTorch on React Native [SEP]" ``` ### 3. Vocabulary & Piece Inspection @@ -132,7 +133,7 @@ const tokenizer = useTokenizer( const nativeTokenizer = nlp.loadTokenizer('/path/to/local/tokenizer.json'); ``` -The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, and post-processor rules defined in the JSON file. +The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, truncation and padding defined in the JSON file. Special tokens from the post-processor are not added (see [encode](#1-encodetext)). ## API Reference diff --git a/docs/versioned_docs/version-0.10.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md b/docs/versioned_docs/version-0.10.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md index fa56c8134a..8563d7d400 100644 --- a/docs/versioned_docs/version-0.10.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md +++ b/docs/versioned_docs/version-0.10.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md @@ -81,8 +81,10 @@ has already been disposed. Defined in: [extensions/nlp/tokenizer.ts:27](https://github.com/software-mansion/react-native-executorch/blob/3f13b59a1822638b61c565b9a47fd48a19c45551/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts#L27) -Encodes a string into token ids (special tokens are added according to the -`tokenizer.json` post_processor). +Encodes a string into token ids. Special tokens from the `tokenizer.json` +post_processor (e.g. `[CLS]`/`[SEP]` or BOS/EOS) are not added. The +`truncation` and `padding` settings of `tokenizer.json` still apply. +Special tokens written in `text` are still encoded. #### Parameters diff --git a/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/03-text-embeddings.md b/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/03-text-embeddings.md index f5deb4b757..767eba59c5 100644 --- a/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/03-text-embeddings.md +++ b/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/03-text-embeddings.md @@ -212,7 +212,7 @@ const customEmbedder = await createTextEmbedder({ }); ``` -The pipeline automatically verifies that the model's exported input and output shapes match its requirements. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline). +The pipeline automatically verifies that the model's exported input and output shapes match its requirements. The tokenizer adds no special tokens, so a model that expects them (for example `[CLS]` and `[SEP]` for BERT-style models) must add them inside its exported graph. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline). ::: ## API Reference diff --git a/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/05-tokenizers.md b/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/05-tokenizers.md index aabf7f101a..4de57bbc43 100644 --- a/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/05-tokenizers.md +++ b/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/05-tokenizers.md @@ -23,7 +23,7 @@ keywords: Tokenizers translate human-readable natural language text into numeric token ID arrays (and decode token ID sequences back into text). -The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with full support for normalizers, pre-tokenizers, truncation, padding, and post-processors (such as adding special `[CLS]` and `[SEP]` tokens automatically). +The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with support for normalizers, pre-tokenizers, truncation and padding. `encode` does not add special tokens such as `[CLS]` and `[SEP]` (see [encode](#1-encodetext)). ## Native Tokenizer (loadTokenizer) @@ -55,11 +55,11 @@ The [`Tokenizer`](../../06-api-reference/react-native-executorch/namespaces/nlp/ ### 1. encode(text) -Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens are automatically appended/prepended according to the `tokenizer.json` post-processor configuration (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece): +Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens from the `tokenizer.json` post-processor (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece) are not added. The `truncation` and `padding` settings of `tokenizer.json` still apply. If your model expects special tokens, add them yourself or inside the exported graph: ```typescript const ids: Int32Array = tokenizer.encode('ExecuTorch on React Native'); -// e.g. Int32Array([101, 10769, 2178, 2006, 2690, 3110, 102]) +// e.g. Int32Array([10769, 2178, 2006, 2690, 3110]) ``` ### 2. decode(tokens, skipSpecialTokens?) @@ -68,7 +68,8 @@ Decodes an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScrip ```typescript const cleanText = tokenizer.decode(ids); // "ExecuTorch on React Native" -const rawText = tokenizer.decode(ids, false); // "[CLS] ExecuTorch on React Native [SEP]" +const framed = new Int32Array([101, ...ids, 102]); // [CLS] ... [SEP], added by hand +const rawText = tokenizer.decode(framed, false); // "[CLS] ExecuTorch on React Native [SEP]" ``` ### 3. Vocabulary & Piece Inspection @@ -132,7 +133,7 @@ const tokenizer = useTokenizer( const nativeTokenizer = nlp.loadTokenizer('/path/to/local/tokenizer.json'); ``` -The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, and post-processor rules defined in the JSON file. +The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, truncation and padding defined in the JSON file. Special tokens from the post-processor are not added (see [encode](#1-encodetext)). ## API Reference diff --git a/docs/versioned_docs/version-0.11.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md b/docs/versioned_docs/version-0.11.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md index f97ced3a8e..174da92b9e 100644 --- a/docs/versioned_docs/version-0.11.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md +++ b/docs/versioned_docs/version-0.11.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md @@ -81,8 +81,10 @@ has already been disposed. Defined in: [extensions/nlp/tokenizer.ts:27](https://github.com/software-mansion/react-native-executorch/blob/182f231651acee34a84d33d6f9965acdd10c0767/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts#L27) -Encodes a string into token ids (special tokens are added according to the -`tokenizer.json` post_processor). +Encodes a string into token ids. Special tokens from the `tokenizer.json` +post_processor (e.g. `[CLS]`/`[SEP]` or BOS/EOS) are not added. The +`truncation` and `padding` settings of `tokenizer.json` still apply. +Special tokens written in `text` are still encoded. #### Parameters diff --git a/packages/react-native-executorch/__tests__/tasks/tokenization.test.ts b/packages/react-native-executorch/__tests__/tasks/tokenization.test.ts index 57275867df..83ba1363ae 100644 --- a/packages/react-native-executorch/__tests__/tasks/tokenization.test.ts +++ b/packages/react-native-executorch/__tests__/tasks/tokenization.test.ts @@ -7,12 +7,7 @@ const TOKENIZER_PATH = '/models/tokenizer.json'; const TOKENS = ['', '', 'hello', 'world', 'react']; const register = () => - fakeJsi.registerTokenizer(TOKENIZER_PATH, { - tokens: TOKENS, - prefix: [0], - suffix: [1], - specialIds: [0, 1], - }); + fakeJsi.registerTokenizer(TOKENIZER_PATH, { tokens: TOKENS, specialIds: [0, 1] }); describe('loadTokenizer', () => { it('propagates a load failure', () => { @@ -30,13 +25,13 @@ describe('loadTokenizer', () => { describe('createTokenizer', () => { beforeEach(register); - it('encodes to an Int32Array, adding the post-processor special tokens', async () => { + it('encodes to an Int32Array without adding special tokens', async () => { const tokenizer = tracked(await createTokenizer(TOKENIZER_PATH)); const ids = await tokenizer.encode('hello world'); expect(ids).toBeInstanceOf(Int32Array); - expect([...ids]).toEqual([0, 2, 3, 1]); + expect([...ids]).toEqual([2, 3]); }); it('round-trips text through encode and decode', async () => { @@ -50,7 +45,7 @@ describe('createTokenizer', () => { it('keeps the special tokens when asked not to skip them', async () => { const tokenizer = tracked(await createTokenizer(TOKENIZER_PATH)); - const ids = await tokenizer.encode('hello'); + const ids = new Int32Array([0, 2, 1]); expect(await tokenizer.decode(ids, false)).toBe(' hello '); }); diff --git a/packages/react-native-executorch/cpp/extensions/nlp/tokenizer.cpp b/packages/react-native-executorch/cpp/extensions/nlp/tokenizer.cpp index be57baac9e..856c764364 100644 --- a/packages/react-native-executorch/cpp/extensions/nlp/tokenizer.cpp +++ b/packages/react-native-executorch/cpp/extensions/nlp/tokenizer.cpp @@ -20,9 +20,9 @@ namespace jsi = facebook::jsi; namespace conversions = rnexecutorch::core::conversions; namespace { -// Number of BOS/EOS tokens to add on top of what the tokenizer.json defines. -// Keeping these at 0 means encoding follows the tokenizer's own post_processor -// (i.e. special tokens are added exactly as configured in tokenizer.json). +// HFTokenizer::encode only checks whether either value is > 0. At 0 the +// tokenizer.json post_processor adds no special tokens: callers and exported +// graphs add them when the model needs them. constexpr uint64_t kNumAddedBosTokens = 0; constexpr uint64_t kNumAddedEosTokens = 0; diff --git a/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts b/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts index 6857c3cd3b..b0655601fe 100644 --- a/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts +++ b/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts @@ -16,8 +16,10 @@ export type Tokenizer = { readonly path: string; /** - * Encodes a string into token ids (special tokens are added according to the - * `tokenizer.json` post_processor). + * Encodes a string into token ids. Special tokens from the `tokenizer.json` + * post_processor (e.g. `[CLS]`/`[SEP]` or BOS/EOS) are not added. The + * `truncation` and `padding` settings of `tokenizer.json` still apply. + * Special tokens written in `text` are still encoded. * @param text The input text string to tokenize. * @returns The encoded token ids as an `Int32Array`. * @throws {RnExecuTorchError} With code `EXECUTION_FAILED` if tokenization