diff --git a/apps/nlp/app/tokenizer/index.tsx b/apps/nlp/app/tokenizer/index.tsx
index 698ba683c4..b58c78b447 100644
--- a/apps/nlp/app/tokenizer/index.tsx
+++ b/apps/nlp/app/tokenizer/index.tsx
@@ -35,9 +35,9 @@ function TokenizerContent() {
const decoded = await decode(tokenIds, true);
const vocab = getVocabSize();
- // Self-consistent inverse check on a token from the actual output
- // (HFTokenizer adds special tokens per the tokenizer.json post_processor).
- const sampleId = tokenIds[Math.min(1, tokenIds.length - 1)]!;
+ // Self-consistent inverse check on the first token of the actual output
+ // (encode adds no special tokens, so it is a real word piece).
+ const sampleId = tokenIds[0]!;
const sampleToken = idToToken(sampleId);
const sampleIdBack = tokenToId(sampleToken);
diff --git a/docs/docs/02-extensions/natural-language/03-text-embeddings.md b/docs/docs/02-extensions/natural-language/03-text-embeddings.md
index ae4e151956..cc82596ef9 100644
--- a/docs/docs/02-extensions/natural-language/03-text-embeddings.md
+++ b/docs/docs/02-extensions/natural-language/03-text-embeddings.md
@@ -174,7 +174,7 @@ const customEmbedder = await createTextEmbedder({
});
```
-The pipeline automatically verifies that the model's exported input and output shapes match its requirements. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline).
+The pipeline automatically verifies that the model's exported input and output shapes match its requirements. The tokenizer adds no special tokens, so a model that expects them (for example `[CLS]` and `[SEP]` for BERT-style models) must add them inside its exported graph. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline).
:::
## API Reference
diff --git a/docs/docs/02-extensions/natural-language/05-tokenizers.md b/docs/docs/02-extensions/natural-language/05-tokenizers.md
index 0874209fb2..308ac08aef 100644
--- a/docs/docs/02-extensions/natural-language/05-tokenizers.md
+++ b/docs/docs/02-extensions/natural-language/05-tokenizers.md
@@ -23,7 +23,7 @@ keywords:
Tokenizers translate human-readable natural language text into numeric token ID arrays (and decode token ID sequences back into text).
-The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with full support for normalizers, pre-tokenizers, truncation, padding, and post-processors (such as adding special `[CLS]` and `[SEP]` tokens automatically).
+The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with support for normalizers, pre-tokenizers, truncation and padding. `encode` does not add special tokens such as `[CLS]` and `[SEP]` (see [encode](#1-encodetext)).
## Native Tokenizer (loadTokenizer)
@@ -55,11 +55,11 @@ The [`Tokenizer`](../../06-api-reference/react-native-executorch/namespaces/nlp/
### 1. encode(text)
-Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens are automatically appended/prepended according to the `tokenizer.json` post-processor configuration (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece):
+Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens from the `tokenizer.json` post-processor (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece) are not added. The `truncation` and `padding` settings of `tokenizer.json` still apply. If your model expects special tokens, add them yourself or inside the exported graph:
```typescript
const ids: Int32Array = tokenizer.encode('ExecuTorch on React Native');
-// e.g. Int32Array([101, 10769, 2178, 2006, 2690, 3110, 102])
+// e.g. Int32Array([10769, 2178, 2006, 2690, 3110])
```
### 2. decode(tokens, skipSpecialTokens?)
@@ -68,7 +68,8 @@ Decodes an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScrip
```typescript
const cleanText = tokenizer.decode(ids); // "ExecuTorch on React Native"
-const rawText = tokenizer.decode(ids, false); // "[CLS] ExecuTorch on React Native [SEP]"
+const framed = new Int32Array([101, ...ids, 102]); // [CLS] ... [SEP], added by hand
+const rawText = tokenizer.decode(framed, false); // "[CLS] ExecuTorch on React Native [SEP]"
```
### 3. Vocabulary & Piece Inspection
@@ -132,7 +133,7 @@ const tokenizer = useTokenizer(
const nativeTokenizer = nlp.loadTokenizer('/path/to/local/tokenizer.json');
```
-The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, and post-processor rules defined in the JSON file.
+The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, truncation and padding defined in the JSON file. Special tokens from the post-processor are not added (see [encode](#1-encodetext)).
## API Reference
diff --git a/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/03-text-embeddings.md b/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/03-text-embeddings.md
index 27c7f62f2f..90f7d1453c 100644
--- a/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/03-text-embeddings.md
+++ b/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/03-text-embeddings.md
@@ -194,7 +194,7 @@ const customEmbedder = await createTextEmbedder({
});
```
-The pipeline automatically verifies that the model's exported input and output shapes match its requirements. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline).
+The pipeline automatically verifies that the model's exported input and output shapes match its requirements. The tokenizer adds no special tokens, so a model that expects them (for example `[CLS]` and `[SEP]` for BERT-style models) must add them inside its exported graph. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline).
:::
## API Reference
diff --git a/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/05-tokenizers.md b/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/05-tokenizers.md
index 2e4a45d59c..87b9e433c1 100644
--- a/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/05-tokenizers.md
+++ b/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/05-tokenizers.md
@@ -23,7 +23,7 @@ keywords:
Tokenizers translate human-readable natural language text into numeric token ID arrays (and decode token ID sequences back into text).
-The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with full support for normalizers, pre-tokenizers, truncation, padding, and post-processors (such as adding special `[CLS]` and `[SEP]` tokens automatically).
+The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with support for normalizers, pre-tokenizers, truncation and padding. `encode` does not add special tokens such as `[CLS]` and `[SEP]` (see [encode](#1-encodetext)).
## Native Tokenizer (loadTokenizer)
@@ -55,11 +55,11 @@ The [`Tokenizer`](../../06-api-reference/react-native-executorch/namespaces/nlp/
### 1. encode(text)
-Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens are automatically appended/prepended according to the `tokenizer.json` post-processor configuration (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece):
+Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens from the `tokenizer.json` post-processor (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece) are not added. The `truncation` and `padding` settings of `tokenizer.json` still apply. If your model expects special tokens, add them yourself or inside the exported graph:
```typescript
const ids: Int32Array = tokenizer.encode('ExecuTorch on React Native');
-// e.g. Int32Array([101, 10769, 2178, 2006, 2690, 3110, 102])
+// e.g. Int32Array([10769, 2178, 2006, 2690, 3110])
```
### 2. decode(tokens, skipSpecialTokens?)
@@ -68,7 +68,8 @@ Decodes an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScrip
```typescript
const cleanText = tokenizer.decode(ids); // "ExecuTorch on React Native"
-const rawText = tokenizer.decode(ids, false); // "[CLS] ExecuTorch on React Native [SEP]"
+const framed = new Int32Array([101, ...ids, 102]); // [CLS] ... [SEP], added by hand
+const rawText = tokenizer.decode(framed, false); // "[CLS] ExecuTorch on React Native [SEP]"
```
### 3. Vocabulary & Piece Inspection
@@ -132,7 +133,7 @@ const tokenizer = useTokenizer(
const nativeTokenizer = nlp.loadTokenizer('/path/to/local/tokenizer.json');
```
-The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, and post-processor rules defined in the JSON file.
+The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, truncation and padding defined in the JSON file. Special tokens from the post-processor are not added (see [encode](#1-encodetext)).
## API Reference
diff --git a/docs/versioned_docs/version-0.10.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md b/docs/versioned_docs/version-0.10.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md
index fa56c8134a..8563d7d400 100644
--- a/docs/versioned_docs/version-0.10.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md
+++ b/docs/versioned_docs/version-0.10.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md
@@ -81,8 +81,10 @@ has already been disposed.
Defined in: [extensions/nlp/tokenizer.ts:27](https://github.com/software-mansion/react-native-executorch/blob/3f13b59a1822638b61c565b9a47fd48a19c45551/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts#L27)
-Encodes a string into token ids (special tokens are added according to the
-`tokenizer.json` post_processor).
+Encodes a string into token ids. Special tokens from the `tokenizer.json`
+post_processor (e.g. `[CLS]`/`[SEP]` or BOS/EOS) are not added. The
+`truncation` and `padding` settings of `tokenizer.json` still apply.
+Special tokens written in `text` are still encoded.
#### Parameters
diff --git a/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/03-text-embeddings.md b/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/03-text-embeddings.md
index f5deb4b757..767eba59c5 100644
--- a/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/03-text-embeddings.md
+++ b/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/03-text-embeddings.md
@@ -212,7 +212,7 @@ const customEmbedder = await createTextEmbedder({
});
```
-The pipeline automatically verifies that the model's exported input and output shapes match its requirements. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline).
+The pipeline automatically verifies that the model's exported input and output shapes match its requirements. The tokenizer adds no special tokens, so a model that expects them (for example `[CLS]` and `[SEP]` for BERT-style models) must add them inside its exported graph. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline).
:::
## API Reference
diff --git a/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/05-tokenizers.md b/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/05-tokenizers.md
index aabf7f101a..4de57bbc43 100644
--- a/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/05-tokenizers.md
+++ b/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/05-tokenizers.md
@@ -23,7 +23,7 @@ keywords:
Tokenizers translate human-readable natural language text into numeric token ID arrays (and decode token ID sequences back into text).
-The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with full support for normalizers, pre-tokenizers, truncation, padding, and post-processors (such as adding special `[CLS]` and `[SEP]` tokens automatically).
+The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with support for normalizers, pre-tokenizers, truncation and padding. `encode` does not add special tokens such as `[CLS]` and `[SEP]` (see [encode](#1-encodetext)).
## Native Tokenizer (loadTokenizer)
@@ -55,11 +55,11 @@ The [`Tokenizer`](../../06-api-reference/react-native-executorch/namespaces/nlp/
### 1. encode(text)
-Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens are automatically appended/prepended according to the `tokenizer.json` post-processor configuration (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece):
+Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens from the `tokenizer.json` post-processor (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece) are not added. The `truncation` and `padding` settings of `tokenizer.json` still apply. If your model expects special tokens, add them yourself or inside the exported graph:
```typescript
const ids: Int32Array = tokenizer.encode('ExecuTorch on React Native');
-// e.g. Int32Array([101, 10769, 2178, 2006, 2690, 3110, 102])
+// e.g. Int32Array([10769, 2178, 2006, 2690, 3110])
```
### 2. decode(tokens, skipSpecialTokens?)
@@ -68,7 +68,8 @@ Decodes an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScrip
```typescript
const cleanText = tokenizer.decode(ids); // "ExecuTorch on React Native"
-const rawText = tokenizer.decode(ids, false); // "[CLS] ExecuTorch on React Native [SEP]"
+const framed = new Int32Array([101, ...ids, 102]); // [CLS] ... [SEP], added by hand
+const rawText = tokenizer.decode(framed, false); // "[CLS] ExecuTorch on React Native [SEP]"
```
### 3. Vocabulary & Piece Inspection
@@ -132,7 +133,7 @@ const tokenizer = useTokenizer(
const nativeTokenizer = nlp.loadTokenizer('/path/to/local/tokenizer.json');
```
-The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, and post-processor rules defined in the JSON file.
+The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, truncation and padding defined in the JSON file. Special tokens from the post-processor are not added (see [encode](#1-encodetext)).
## API Reference
diff --git a/docs/versioned_docs/version-0.11.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md b/docs/versioned_docs/version-0.11.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md
index f97ced3a8e..174da92b9e 100644
--- a/docs/versioned_docs/version-0.11.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md
+++ b/docs/versioned_docs/version-0.11.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md
@@ -81,8 +81,10 @@ has already been disposed.
Defined in: [extensions/nlp/tokenizer.ts:27](https://github.com/software-mansion/react-native-executorch/blob/182f231651acee34a84d33d6f9965acdd10c0767/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts#L27)
-Encodes a string into token ids (special tokens are added according to the
-`tokenizer.json` post_processor).
+Encodes a string into token ids. Special tokens from the `tokenizer.json`
+post_processor (e.g. `[CLS]`/`[SEP]` or BOS/EOS) are not added. The
+`truncation` and `padding` settings of `tokenizer.json` still apply.
+Special tokens written in `text` are still encoded.
#### Parameters
diff --git a/packages/react-native-executorch/__tests__/tasks/tokenization.test.ts b/packages/react-native-executorch/__tests__/tasks/tokenization.test.ts
index 57275867df..83ba1363ae 100644
--- a/packages/react-native-executorch/__tests__/tasks/tokenization.test.ts
+++ b/packages/react-native-executorch/__tests__/tasks/tokenization.test.ts
@@ -7,12 +7,7 @@ const TOKENIZER_PATH = '/models/tokenizer.json';
const TOKENS = ['', '', 'hello', 'world', 'react'];
const register = () =>
- fakeJsi.registerTokenizer(TOKENIZER_PATH, {
- tokens: TOKENS,
- prefix: [0],
- suffix: [1],
- specialIds: [0, 1],
- });
+ fakeJsi.registerTokenizer(TOKENIZER_PATH, { tokens: TOKENS, specialIds: [0, 1] });
describe('loadTokenizer', () => {
it('propagates a load failure', () => {
@@ -30,13 +25,13 @@ describe('loadTokenizer', () => {
describe('createTokenizer', () => {
beforeEach(register);
- it('encodes to an Int32Array, adding the post-processor special tokens', async () => {
+ it('encodes to an Int32Array without adding special tokens', async () => {
const tokenizer = tracked(await createTokenizer(TOKENIZER_PATH));
const ids = await tokenizer.encode('hello world');
expect(ids).toBeInstanceOf(Int32Array);
- expect([...ids]).toEqual([0, 2, 3, 1]);
+ expect([...ids]).toEqual([2, 3]);
});
it('round-trips text through encode and decode', async () => {
@@ -50,7 +45,7 @@ describe('createTokenizer', () => {
it('keeps the special tokens when asked not to skip them', async () => {
const tokenizer = tracked(await createTokenizer(TOKENIZER_PATH));
- const ids = await tokenizer.encode('hello');
+ const ids = new Int32Array([0, 2, 1]);
expect(await tokenizer.decode(ids, false)).toBe(' hello ');
});
diff --git a/packages/react-native-executorch/cpp/extensions/nlp/tokenizer.cpp b/packages/react-native-executorch/cpp/extensions/nlp/tokenizer.cpp
index be57baac9e..856c764364 100644
--- a/packages/react-native-executorch/cpp/extensions/nlp/tokenizer.cpp
+++ b/packages/react-native-executorch/cpp/extensions/nlp/tokenizer.cpp
@@ -20,9 +20,9 @@ namespace jsi = facebook::jsi;
namespace conversions = rnexecutorch::core::conversions;
namespace {
-// Number of BOS/EOS tokens to add on top of what the tokenizer.json defines.
-// Keeping these at 0 means encoding follows the tokenizer's own post_processor
-// (i.e. special tokens are added exactly as configured in tokenizer.json).
+// HFTokenizer::encode only checks whether either value is > 0. At 0 the
+// tokenizer.json post_processor adds no special tokens: callers and exported
+// graphs add them when the model needs them.
constexpr uint64_t kNumAddedBosTokens = 0;
constexpr uint64_t kNumAddedEosTokens = 0;
diff --git a/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts b/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts
index 6857c3cd3b..b0655601fe 100644
--- a/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts
+++ b/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts
@@ -16,8 +16,10 @@ export type Tokenizer = {
readonly path: string;
/**
- * Encodes a string into token ids (special tokens are added according to the
- * `tokenizer.json` post_processor).
+ * Encodes a string into token ids. Special tokens from the `tokenizer.json`
+ * post_processor (e.g. `[CLS]`/`[SEP]` or BOS/EOS) are not added. The
+ * `truncation` and `padding` settings of `tokenizer.json` still apply.
+ * Special tokens written in `text` are still encoded.
* @param text The input text string to tokenize.
* @returns The encoded token ids as an `Int32Array`.
* @throws {RnExecuTorchError} With code `EXECUTION_FAILED` if tokenization