From 8dc6fdd5e930503705c43dbea102db1884f079fc Mon Sep 17 00:00:00 2001
From: Rasaboun <40967731+Rasaboun@users.noreply.github.com>
Date: Thu, 12 Nov 2026 15:06:07 +0100
Subject: [PATCH 1/3] docs(nlp): state that Tokenizer.encode adds no special
tokens
The encode JSDoc said special tokens are added according to the tokenizer.json post_processor, but the native binding calls encode(text, 0, 0), so the post_processor template is never applied. Correct the comment and note in the text embeddings custom model tip that a model expecting special tokens must add them inside its exported graph.
---
.../02-extensions/natural-language/03-text-embeddings.md | 2 +-
.../react-native-executorch/src/extensions/nlp/tokenizer.ts | 5 +++--
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/docs/docs/02-extensions/natural-language/03-text-embeddings.md b/docs/docs/02-extensions/natural-language/03-text-embeddings.md
index ae4e151956..cc82596ef9 100644
--- a/docs/docs/02-extensions/natural-language/03-text-embeddings.md
+++ b/docs/docs/02-extensions/natural-language/03-text-embeddings.md
@@ -174,7 +174,7 @@ const customEmbedder = await createTextEmbedder({
});
```
-The pipeline automatically verifies that the model's exported input and output shapes match its requirements. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline).
+The pipeline automatically verifies that the model's exported input and output shapes match its requirements. The tokenizer adds no special tokens, so a model that expects them (for example `[CLS]` and `[SEP]` for BERT-style models) must add them inside its exported graph. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline).
:::
## API Reference
diff --git a/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts b/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts
index 6857c3cd3b..8584cb0384 100644
--- a/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts
+++ b/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts
@@ -16,8 +16,9 @@ export type Tokenizer = {
readonly path: string;
/**
- * Encodes a string into token ids (special tokens are added according to the
- * `tokenizer.json` post_processor).
+ * Encodes a string into token ids. No special tokens are added: the
+ * `tokenizer.json` post_processor template (e.g. `[CLS]`/`[SEP]` or BOS/EOS)
+ * is not applied. Special tokens written in `text` are still encoded.
* @param text The input text string to tokenize.
* @returns The encoded token ids as an `Int32Array`.
* @throws {RnExecuTorchError} With code `EXECUTION_FAILED` if tokenization
From cfe44e45f0f6c6f3c31066ce7504effda794b75a Mon Sep 17 00:00:00 2001
From: Rasaboun <40967731+Rasaboun@users.noreply.github.com>
Date: Thu, 12 Nov 2026 15:31:42 +0100
Subject: [PATCH 2/3] docs(nlp): align the Tokenizers page, native comment,
test and demo with encode
The Tokenizers page, the native comment in tokenizer.cpp, the tokenization API test and the NLP demo app all described encode as adding the tokenizer.json post_processor special tokens. The binding passes bos/eos 0, so none are added. Update them to match, and note in the encode JSDoc that truncation and padding from tokenizer.json still apply.
---
apps/nlp/app/tokenizer/index.tsx | 6 +++---
.../02-extensions/natural-language/05-tokenizers.md | 11 ++++++-----
.../__tests__/tasks/tokenization.test.ts | 13 ++++---------
.../cpp/extensions/nlp/tokenizer.cpp | 6 +++---
.../src/extensions/nlp/tokenizer.ts | 7 ++++---
5 files changed, 20 insertions(+), 23 deletions(-)
diff --git a/apps/nlp/app/tokenizer/index.tsx b/apps/nlp/app/tokenizer/index.tsx
index 698ba683c4..b58c78b447 100644
--- a/apps/nlp/app/tokenizer/index.tsx
+++ b/apps/nlp/app/tokenizer/index.tsx
@@ -35,9 +35,9 @@ function TokenizerContent() {
const decoded = await decode(tokenIds, true);
const vocab = getVocabSize();
- // Self-consistent inverse check on a token from the actual output
- // (HFTokenizer adds special tokens per the tokenizer.json post_processor).
- const sampleId = tokenIds[Math.min(1, tokenIds.length - 1)]!;
+ // Self-consistent inverse check on the first token of the actual output
+ // (encode adds no special tokens, so it is a real word piece).
+ const sampleId = tokenIds[0]!;
const sampleToken = idToToken(sampleId);
const sampleIdBack = tokenToId(sampleToken);
diff --git a/docs/docs/02-extensions/natural-language/05-tokenizers.md b/docs/docs/02-extensions/natural-language/05-tokenizers.md
index 0874209fb2..308ac08aef 100644
--- a/docs/docs/02-extensions/natural-language/05-tokenizers.md
+++ b/docs/docs/02-extensions/natural-language/05-tokenizers.md
@@ -23,7 +23,7 @@ keywords:
Tokenizers translate human-readable natural language text into numeric token ID arrays (and decode token ID sequences back into text).
-The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with full support for normalizers, pre-tokenizers, truncation, padding, and post-processors (such as adding special `[CLS]` and `[SEP]` tokens automatically).
+The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with support for normalizers, pre-tokenizers, truncation and padding. `encode` does not add special tokens such as `[CLS]` and `[SEP]` (see [encode](#1-encodetext)).
## Native Tokenizer (loadTokenizer)
@@ -55,11 +55,11 @@ The [`Tokenizer`](../../06-api-reference/react-native-executorch/namespaces/nlp/
### 1. encode(text)
-Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens are automatically appended/prepended according to the `tokenizer.json` post-processor configuration (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece):
+Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens from the `tokenizer.json` post-processor (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece) are not added. The `truncation` and `padding` settings of `tokenizer.json` still apply. If your model expects special tokens, add them yourself or inside the exported graph:
```typescript
const ids: Int32Array = tokenizer.encode('ExecuTorch on React Native');
-// e.g. Int32Array([101, 10769, 2178, 2006, 2690, 3110, 102])
+// e.g. Int32Array([10769, 2178, 2006, 2690, 3110])
```
### 2. decode(tokens, skipSpecialTokens?)
@@ -68,7 +68,8 @@ Decodes an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScrip
```typescript
const cleanText = tokenizer.decode(ids); // "ExecuTorch on React Native"
-const rawText = tokenizer.decode(ids, false); // "[CLS] ExecuTorch on React Native [SEP]"
+const framed = new Int32Array([101, ...ids, 102]); // [CLS] ... [SEP], added by hand
+const rawText = tokenizer.decode(framed, false); // "[CLS] ExecuTorch on React Native [SEP]"
```
### 3. Vocabulary & Piece Inspection
@@ -132,7 +133,7 @@ const tokenizer = useTokenizer(
const nativeTokenizer = nlp.loadTokenizer('/path/to/local/tokenizer.json');
```
-The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, and post-processor rules defined in the JSON file.
+The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, truncation and padding defined in the JSON file. Special tokens from the post-processor are not added (see [encode](#1-encodetext)).
## API Reference
diff --git a/packages/react-native-executorch/__tests__/tasks/tokenization.test.ts b/packages/react-native-executorch/__tests__/tasks/tokenization.test.ts
index 57275867df..83ba1363ae 100644
--- a/packages/react-native-executorch/__tests__/tasks/tokenization.test.ts
+++ b/packages/react-native-executorch/__tests__/tasks/tokenization.test.ts
@@ -7,12 +7,7 @@ const TOKENIZER_PATH = '/models/tokenizer.json';
const TOKENS = ['', '', 'hello', 'world', 'react'];
const register = () =>
- fakeJsi.registerTokenizer(TOKENIZER_PATH, {
- tokens: TOKENS,
- prefix: [0],
- suffix: [1],
- specialIds: [0, 1],
- });
+ fakeJsi.registerTokenizer(TOKENIZER_PATH, { tokens: TOKENS, specialIds: [0, 1] });
describe('loadTokenizer', () => {
it('propagates a load failure', () => {
@@ -30,13 +25,13 @@ describe('loadTokenizer', () => {
describe('createTokenizer', () => {
beforeEach(register);
- it('encodes to an Int32Array, adding the post-processor special tokens', async () => {
+ it('encodes to an Int32Array without adding special tokens', async () => {
const tokenizer = tracked(await createTokenizer(TOKENIZER_PATH));
const ids = await tokenizer.encode('hello world');
expect(ids).toBeInstanceOf(Int32Array);
- expect([...ids]).toEqual([0, 2, 3, 1]);
+ expect([...ids]).toEqual([2, 3]);
});
it('round-trips text through encode and decode', async () => {
@@ -50,7 +45,7 @@ describe('createTokenizer', () => {
it('keeps the special tokens when asked not to skip them', async () => {
const tokenizer = tracked(await createTokenizer(TOKENIZER_PATH));
- const ids = await tokenizer.encode('hello');
+ const ids = new Int32Array([0, 2, 1]);
expect(await tokenizer.decode(ids, false)).toBe(' hello ');
});
diff --git a/packages/react-native-executorch/cpp/extensions/nlp/tokenizer.cpp b/packages/react-native-executorch/cpp/extensions/nlp/tokenizer.cpp
index be57baac9e..856c764364 100644
--- a/packages/react-native-executorch/cpp/extensions/nlp/tokenizer.cpp
+++ b/packages/react-native-executorch/cpp/extensions/nlp/tokenizer.cpp
@@ -20,9 +20,9 @@ namespace jsi = facebook::jsi;
namespace conversions = rnexecutorch::core::conversions;
namespace {
-// Number of BOS/EOS tokens to add on top of what the tokenizer.json defines.
-// Keeping these at 0 means encoding follows the tokenizer's own post_processor
-// (i.e. special tokens are added exactly as configured in tokenizer.json).
+// HFTokenizer::encode only checks whether either value is > 0. At 0 the
+// tokenizer.json post_processor adds no special tokens: callers and exported
+// graphs add them when the model needs them.
constexpr uint64_t kNumAddedBosTokens = 0;
constexpr uint64_t kNumAddedEosTokens = 0;
diff --git a/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts b/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts
index 8584cb0384..b0655601fe 100644
--- a/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts
+++ b/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts
@@ -16,9 +16,10 @@ export type Tokenizer = {
readonly path: string;
/**
- * Encodes a string into token ids. No special tokens are added: the
- * `tokenizer.json` post_processor template (e.g. `[CLS]`/`[SEP]` or BOS/EOS)
- * is not applied. Special tokens written in `text` are still encoded.
+ * Encodes a string into token ids. Special tokens from the `tokenizer.json`
+ * post_processor (e.g. `[CLS]`/`[SEP]` or BOS/EOS) are not added. The
+ * `truncation` and `padding` settings of `tokenizer.json` still apply.
+ * Special tokens written in `text` are still encoded.
* @param text The input text string to tokenize.
* @returns The encoded token ids as an `Int32Array`.
* @throws {RnExecuTorchError} With code `EXECUTION_FAILED` if tokenization
From f7710ce8aa67bbb59dcf91b054b856043b60c091 Mon Sep 17 00:00:00 2001
From: Rasaboun <40967731+Rasaboun@users.noreply.github.com>
Date: Thu, 12 Nov 2026 16:37:04 +0100
Subject: [PATCH 3/3] docs(nlp): apply the encode special-token fix to the
0.10.x and 0.11.x docs
Both release branches call encode(text, 0, 0) like main, so the versioned Tokenizers page, text embeddings tip and generated Tokenizer API page get the same correction.
---
.../natural-language/03-text-embeddings.md | 2 +-
.../02-extensions/natural-language/05-tokenizers.md | 11 ++++++-----
.../namespaces/nlp/type-aliases/Tokenizer.md | 6 ++++--
.../natural-language/03-text-embeddings.md | 2 +-
.../02-extensions/natural-language/05-tokenizers.md | 11 ++++++-----
.../namespaces/nlp/type-aliases/Tokenizer.md | 6 ++++--
6 files changed, 22 insertions(+), 16 deletions(-)
diff --git a/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/03-text-embeddings.md b/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/03-text-embeddings.md
index 27c7f62f2f..90f7d1453c 100644
--- a/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/03-text-embeddings.md
+++ b/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/03-text-embeddings.md
@@ -194,7 +194,7 @@ const customEmbedder = await createTextEmbedder({
});
```
-The pipeline automatically verifies that the model's exported input and output shapes match its requirements. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline).
+The pipeline automatically verifies that the model's exported input and output shapes match its requirements. The tokenizer adds no special tokens, so a model that expects them (for example `[CLS]` and `[SEP]` for BERT-style models) must add them inside its exported graph. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline).
:::
## API Reference
diff --git a/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/05-tokenizers.md b/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/05-tokenizers.md
index 2e4a45d59c..87b9e433c1 100644
--- a/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/05-tokenizers.md
+++ b/docs/versioned_docs/version-0.10.x/02-extensions/natural-language/05-tokenizers.md
@@ -23,7 +23,7 @@ keywords:
Tokenizers translate human-readable natural language text into numeric token ID arrays (and decode token ID sequences back into text).
-The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with full support for normalizers, pre-tokenizers, truncation, padding, and post-processors (such as adding special `[CLS]` and `[SEP]` tokens automatically).
+The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with support for normalizers, pre-tokenizers, truncation and padding. `encode` does not add special tokens such as `[CLS]` and `[SEP]` (see [encode](#1-encodetext)).
## Native Tokenizer (loadTokenizer)
@@ -55,11 +55,11 @@ The [`Tokenizer`](../../06-api-reference/react-native-executorch/namespaces/nlp/
### 1. encode(text)
-Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens are automatically appended/prepended according to the `tokenizer.json` post-processor configuration (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece):
+Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens from the `tokenizer.json` post-processor (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece) are not added. The `truncation` and `padding` settings of `tokenizer.json` still apply. If your model expects special tokens, add them yourself or inside the exported graph:
```typescript
const ids: Int32Array = tokenizer.encode('ExecuTorch on React Native');
-// e.g. Int32Array([101, 10769, 2178, 2006, 2690, 3110, 102])
+// e.g. Int32Array([10769, 2178, 2006, 2690, 3110])
```
### 2. decode(tokens, skipSpecialTokens?)
@@ -68,7 +68,8 @@ Decodes an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScrip
```typescript
const cleanText = tokenizer.decode(ids); // "ExecuTorch on React Native"
-const rawText = tokenizer.decode(ids, false); // "[CLS] ExecuTorch on React Native [SEP]"
+const framed = new Int32Array([101, ...ids, 102]); // [CLS] ... [SEP], added by hand
+const rawText = tokenizer.decode(framed, false); // "[CLS] ExecuTorch on React Native [SEP]"
```
### 3. Vocabulary & Piece Inspection
@@ -132,7 +133,7 @@ const tokenizer = useTokenizer(
const nativeTokenizer = nlp.loadTokenizer('/path/to/local/tokenizer.json');
```
-The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, and post-processor rules defined in the JSON file.
+The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, truncation and padding defined in the JSON file. Special tokens from the post-processor are not added (see [encode](#1-encodetext)).
## API Reference
diff --git a/docs/versioned_docs/version-0.10.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md b/docs/versioned_docs/version-0.10.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md
index fa56c8134a..8563d7d400 100644
--- a/docs/versioned_docs/version-0.10.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md
+++ b/docs/versioned_docs/version-0.10.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md
@@ -81,8 +81,10 @@ has already been disposed.
Defined in: [extensions/nlp/tokenizer.ts:27](https://github.com/software-mansion/react-native-executorch/blob/3f13b59a1822638b61c565b9a47fd48a19c45551/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts#L27)
-Encodes a string into token ids (special tokens are added according to the
-`tokenizer.json` post_processor).
+Encodes a string into token ids. Special tokens from the `tokenizer.json`
+post_processor (e.g. `[CLS]`/`[SEP]` or BOS/EOS) are not added. The
+`truncation` and `padding` settings of `tokenizer.json` still apply.
+Special tokens written in `text` are still encoded.
#### Parameters
diff --git a/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/03-text-embeddings.md b/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/03-text-embeddings.md
index f5deb4b757..767eba59c5 100644
--- a/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/03-text-embeddings.md
+++ b/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/03-text-embeddings.md
@@ -212,7 +212,7 @@ const customEmbedder = await createTextEmbedder({
});
```
-The pipeline automatically verifies that the model's exported input and output shapes match its requirements. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline).
+The pipeline automatically verifies that the model's exported input and output shapes match its requirements. The tokenizer adds no special tokens, so a model that expects them (for example `[CLS]` and `[SEP]` for BERT-style models) must add them inside its exported graph. To prepare and export your own `.pte` model to match this pipeline, see [Exporting Custom Models](../../03-core-and-advanced/07-exporting-custom-models.md#using-a-built-in-pipeline).
:::
## API Reference
diff --git a/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/05-tokenizers.md b/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/05-tokenizers.md
index aabf7f101a..4de57bbc43 100644
--- a/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/05-tokenizers.md
+++ b/docs/versioned_docs/version-0.11.x/02-extensions/natural-language/05-tokenizers.md
@@ -23,7 +23,7 @@ keywords:
Tokenizers translate human-readable natural language text into numeric token ID arrays (and decode token ID sequences back into text).
-The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with full support for normalizers, pre-tokenizers, truncation, padding, and post-processors (such as adding special `[CLS]` and `[SEP]` tokens automatically).
+The library embeds PyTorch's native C++ tokenizer engine, providing high-performance Hugging Face `tokenizer.json` compatibility directly on-device without Python or Rust runtimes. It supports Byte-Pair Encoding (BPE), WordPiece, Unigram, and Byte-level tokenizers with support for normalizers, pre-tokenizers, truncation and padding. `encode` does not add special tokens such as `[CLS]` and `[SEP]` (see [encode](#1-encodetext)).
## Native Tokenizer (loadTokenizer)
@@ -55,11 +55,11 @@ The [`Tokenizer`](../../06-api-reference/react-native-executorch/namespaces/nlp/
### 1. encode(text)
-Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens are automatically appended/prepended according to the `tokenizer.json` post-processor configuration (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece):
+Converts a string into an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Int32Array) of token IDs. Special tokens from the `tokenizer.json` post-processor (e.g. `[CLS]` and `[SEP]` for BERT/WordPiece) are not added. The `truncation` and `padding` settings of `tokenizer.json` still apply. If your model expects special tokens, add them yourself or inside the exported graph:
```typescript
const ids: Int32Array = tokenizer.encode('ExecuTorch on React Native');
-// e.g. Int32Array([101, 10769, 2178, 2006, 2690, 3110, 102])
+// e.g. Int32Array([10769, 2178, 2006, 2690, 3110])
```
### 2. decode(tokens, skipSpecialTokens?)
@@ -68,7 +68,8 @@ Decodes an [`Int32Array`](https://developer.mozilla.org/en-US/docs/Web/JavaScrip
```typescript
const cleanText = tokenizer.decode(ids); // "ExecuTorch on React Native"
-const rawText = tokenizer.decode(ids, false); // "[CLS] ExecuTorch on React Native [SEP]"
+const framed = new Int32Array([101, ...ids, 102]); // [CLS] ... [SEP], added by hand
+const rawText = tokenizer.decode(framed, false); // "[CLS] ExecuTorch on React Native [SEP]"
```
### 3. Vocabulary & Piece Inspection
@@ -132,7 +133,7 @@ const tokenizer = useTokenizer(
const nativeTokenizer = nlp.loadTokenizer('/path/to/local/tokenizer.json');
```
-The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, and post-processor rules defined in the JSON file.
+The native tokenizer automatically handles the model type, vocabulary tables, regex pre-tokenizers, merges, truncation and padding defined in the JSON file. Special tokens from the post-processor are not added (see [encode](#1-encodetext)).
## API Reference
diff --git a/docs/versioned_docs/version-0.11.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md b/docs/versioned_docs/version-0.11.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md
index f97ced3a8e..174da92b9e 100644
--- a/docs/versioned_docs/version-0.11.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md
+++ b/docs/versioned_docs/version-0.11.x/06-api-reference/react-native-executorch/namespaces/nlp/type-aliases/Tokenizer.md
@@ -81,8 +81,10 @@ has already been disposed.
Defined in: [extensions/nlp/tokenizer.ts:27](https://github.com/software-mansion/react-native-executorch/blob/182f231651acee34a84d33d6f9965acdd10c0767/packages/react-native-executorch/src/extensions/nlp/tokenizer.ts#L27)
-Encodes a string into token ids (special tokens are added according to the
-`tokenizer.json` post_processor).
+Encodes a string into token ids. Special tokens from the `tokenizer.json`
+post_processor (e.g. `[CLS]`/`[SEP]` or BOS/EOS) are not added. The
+`truncation` and `padding` settings of `tokenizer.json` still apply.
+Special tokens written in `text` are still encoded.
#### Parameters