From 5550d18c973db79bbc46b5972303eb955bbb87cb Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 3 Nov 2026 11:52:01 +0100 Subject: [PATCH 01/14] docs(docs): add diagrams of the weights path, a quantized file and one quantized group --- apps/docs/src/components/QuantFile.astro | 95 +++++++++++++++++++ apps/docs/src/components/QuantGroup.astro | 101 ++++++++++++++++++++ apps/docs/src/components/WeightsPath.astro | 105 +++++++++++++++++++++ 3 files changed, 301 insertions(+) create mode 100644 apps/docs/src/components/QuantFile.astro create mode 100644 apps/docs/src/components/QuantGroup.astro create mode 100644 apps/docs/src/components/WeightsPath.astro diff --git a/apps/docs/src/components/QuantFile.astro b/apps/docs/src/components/QuantFile.astro new file mode 100644 index 00000000..96ba8b3f --- /dev/null +++ b/apps/docs/src/components/QuantFile.astro @@ -0,0 +1,95 @@ +--- +/** What a quantized safetensors file holds and how the loader reads each + * entry: the header label, an int8 weight with its F16 scales, and the float + * tensors around them. Names are all-MiniLM-L6-v2's, shortened. */ +const rows = [ + { name: 'embeddings.word_embeddings.weight', dtype: 'F32', shape: '[30522, 384]', loads: 'float' }, + { name: '…attention.self.query.weight', dtype: 'I8', shape: '[384, 384]', loads: 'int8 weight' }, + { name: '…attention.self.query.weight_scale', dtype: 'F16', shape: '[384, 12]', loads: 'its scales' }, + { name: '…attention.self.query.bias', dtype: 'F32', shape: '[384]', loads: 'float' }, + { name: '…attention.output.LayerNorm.weight', dtype: 'F32', shape: '[384]', loads: 'float' }, +]; +const rowHeight = 26; +const top = 58; +const height = top + rows.length * rowHeight + 14; +--- + + + + __metadata__ runntime.quant = + + {'{'}"version":1,"bits":8,"groupSize":32,"scheme":"symmetric"{'}'} + + + { + rows.map((row, i) => { + const y = top + i * rowHeight + 10; + const quant = row.dtype !== 'F32'; + return ( + + {quant && ( + + )} + + {row.name} + + + {row.dtype} {row.shape} + + + {row.loads} + + + ); + }) + } + + + diff --git a/apps/docs/src/components/QuantGroup.astro b/apps/docs/src/components/QuantGroup.astro new file mode 100644 index 00000000..c97a5525 --- /dev/null +++ b/apps/docs/src/components/QuantGroup.astro @@ -0,0 +1,101 @@ +--- +/** One group of 32 weights becoming 32 int8 values and one f16 scale. Six + * of the 32 are drawn; the scale is the group's largest magnitude over + * 127, and each int is its weight divided by the scale, rounded. */ +const weights = ['0.031', '-0.118', '0.054', '0.002', '0.087', '-0.070']; +const ints = ['33', '-127', '58', '2', '94', '-75']; +const cell = 50; +const left = 12; +const cells = (values: string[], y: number, strong: boolean) => + values.map((value, i) => ({ x: left + i * cell, y, value, strong })); +--- + + + 32 weights of one row, f32 + { + cells(weights, 22, false).map((c) => ( + + + + {c.value} + + + )) + } + … 26 more + + ↓ + + scale = 0.118 / 127 = 0.000929, stored as f16 + + + int = round(weight / scale), int8 + scale, f16 + { + cells(ints, 118, true).map((c) => ( + + + + {c.value} + + + )) + } + + 0.000929 + 32 bytes of ints and 2 bytes of scale, in place of 128 bytes + + + diff --git a/apps/docs/src/components/WeightsPath.astro b/apps/docs/src/components/WeightsPath.astro new file mode 100644 index 00000000..6f150b54 --- /dev/null +++ b/apps/docs/src/components/WeightsPath.astro @@ -0,0 +1,105 @@ +--- +/** The path of a weights file, from a checkpoint to the GPU: six numbered + * steps in two rows, one column at phone width. */ +const steps = [ + { title: 'Checkpoint', detail: 'PyTorch or Hugging Face' }, + { title: 'Export', detail: 'model.safetensors, F32 / F16 / BF16' }, + { title: 'Quantize', detail: 'runntime quantize, int8 or int4. Optional.' }, + { title: 'Host', detail: 'any static file server' }, + { title: 'modelPath', detail: 'create({ modelPath })' }, + { title: 'Load', detail: 'header, 8 MB chunks, cache, GPU upload' }, +]; +--- + +
    + { + steps.map((step) => ( +
  1. + {step.title} + {step.detail} +
  2. + )) + } +
+ + From fa950cf0eddc739177d7917dbb3b774f7d3ce928 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 3 Nov 2026 11:52:04 +0100 Subject: [PATCH 02/14] docs(docs): add the Exporting weights guide, from a checkpoint to a float or quantized file --- apps/docs/astro.config.mjs | 1 + .../content/docs/zoo/exporting-weights.mdx | 243 ++++++++++++++++++ 2 files changed, 244 insertions(+) create mode 100644 apps/docs/src/content/docs/zoo/exporting-weights.mdx diff --git a/apps/docs/astro.config.mjs b/apps/docs/astro.config.mjs index 8bd93b04..aeaa543e 100644 --- a/apps/docs/astro.config.mjs +++ b/apps/docs/astro.config.mjs @@ -158,6 +158,7 @@ export default defineConfig({ label: 'Guides', items: [ { label: 'Error handling', slug: 'zoo/error-handling' }, + { label: 'Exporting weights', slug: 'zoo/exporting-weights' }, { label: 'Migrating from transformers.js', slug: 'zoo/migrating-from-transformers-js', diff --git a/apps/docs/src/content/docs/zoo/exporting-weights.mdx b/apps/docs/src/content/docs/zoo/exporting-weights.mdx new file mode 100644 index 00000000..a77e8479 --- /dev/null +++ b/apps/docs/src/content/docs/zoo/exporting-weights.mdx @@ -0,0 +1,243 @@ +--- +title: Exporting weights +description: Run your own weights in ruNNtime, as float or quantized to int8 or int4. +--- + +import WeightsPath from '../../../components/WeightsPath.astro'; +import QuantFile from '../../../components/QuantFile.astro'; +import QuantGroup from '../../../components/QuantGroup.astro'; + +Every model in the zoo reads its weights from one safetensors file. The +hosted files are exports of the original checkpoints. Your own export works +the same way: a fine-tune, a smaller download, or a quantized file. This page +is the path from a checkpoint to a file a `create` call loads. + + + +## The weights file + +A weights file is one `.safetensors` file. The loader reads nothing else: no +`config.json`, no sharded index. + +- **Names** are the ones the model's loader expects. For the hosted models + these are the names of the original checkpoint, like + `encoder.layer.0.attention.self.query.weight` for all-MiniLM-L6-v2 or + `model.decoder.layers.0.self_attn.q_proj.weight` for Moonshine. The hosted + file's header lists the exact set. +- **Shapes** are the original layout. A Linear weight is `[out, in]`, as + PyTorch stores it. +- **Dtypes** are `F32`, `F16` or `BF16` for a float tensor. The file's dtype + and the math dtype are independent, see [How a file loads](#how-a-file-loads). + A quantized weight is `I8` or `U8` with an `F16` scale tensor next to it, + see [The quantized format](#the-quantized-format). + +A `create` call takes the file as `modelPath`: a URL, or a `RangeSource` for +bytes you already have. The ready-made configs in `models` point at the +hosted files. Spread one and replace `modelPath` to keep its other fields, +like the tokenizer. + +```ts twoslash +import { createEmbedder, models } from 'runntime/zoo'; + +const embedder = await createEmbedder({ + ...models.embedding.ALL_MINILM_L6_V2.DEFAULT, + modelPath: '/models/minilm.int8.safetensors', +}); +``` + +## How a file loads + +1. The loader reads the header: 8 bytes of length, then JSON with every + tensor's name, dtype, shape and byte range. It checks the names against + the model here, before any weight downloads. A file that is not + safetensors, like a dev server's `index.html` fallback for a missing URL, + throws `LOAD_FAILED` naming the URL. +2. Tensors download in file order with HTTP `Range` requests, in aligned + chunks of 8 MB, so a file costs about one request per 8 MB. A server that + ignores `Range` and answers `200` makes the loader download the whole file + once. A short answer is retried. +3. Each tensor is fetched, converted and uploaded to the GPU on its own, so + host memory holds one tensor at a time. `onProgress` fires after each + one. +4. With a `cache`, every fetched range is saved under the `cacheId`, and the + next load reads it back with no network. Cached bytes are never checked + against the server, so a file that changes content needs a new URL or a + new `cacheId`. The default `cacheId` comes from the URL. + +The file's dtype does not pick the math. The model's `dtype` does: `'f16'` +runs in half precision on a device with `shader-f16`, `'f32'` in full. An +`F32` tensor narrows to f16 when needed, an `F16` or `BF16` one widens to +f32. The vision models run in f16 only. A quantized weight stays int on the +GPU and its scales become f32, whatever the `dtype`. + +## Exporting float weights + +Save the model with Hugging Face `transformers`. `save_pretrained` writes +`model.safetensors` with the original names, and `torch_dtype` picks the +stored dtype. `F16` halves the file against `F32` and loses nothing a model +in f16 math would keep anyway. + +```bash +uv run --with transformers --with torch python -c " +from transformers import AutoModel +import torch +model = AutoModel.from_pretrained('sentence-transformers/all-MiniLM-L6-v2', torch_dtype=torch.float16) +model.save_pretrained('export', max_shard_size='100GB') +" +``` + +`max_shard_size` keeps the export in one file. A folder with shards is fine +as the input of [`runntime quantize`](#quantizing), which writes one file, +but not as a `modelPath`. + +A checkpoint from your own training code goes through +`safetensors.torch.save_file(model.state_dict(), 'model.safetensors')`. +Every tensor must be contiguous, and two names cannot share one tensor: +clone a tied `lm_head` before saving. + +### Checking a file + +Load it. The names and shapes are checked before any weight downloads, and +a mismatch throws `CHECKPOINT_MISMATCH` with the first names that are +missing or unexpected, or the tensor whose shape is wrong. See +[Error handling](/zoo/error-handling/) for how to catch it. + +```ts twoslash +import { createEmbedder, isRunntimeError, models } from 'runntime/zoo'; +// ---cut--- +try { + await createEmbedder({ + ...models.embedding.ALL_MINILM_L6_V2.DEFAULT, + modelPath: '/models/minilm.safetensors', + }); +} catch (err) { + // loadStateDict: state dict does not match model. missing: pooler.dense.weight … + if (isRunntimeError(err, 'CHECKPOINT_MISMATCH')) console.error(err.message); + throw err; +} +``` + +To see what a file holds without loading it, read its header in Python: + +```bash +uv run --with safetensors python -c " +from safetensors import safe_open +with safe_open('model.safetensors', 'pt') as f: + print(f.metadata()) + for name in f.keys(): print(name, f.get_slice(name).get_dtype(), f.get_slice(name).get_shape()) +" +``` + +## Quantizing + +`runntime quantize` turns a float file into an int8 or int4 one. It lives in +the repository under `tools/python` and runs with +[uv](https://docs.astral.sh/uv/), which installs its one dependency, NumPy, +on the first run. + +```bash +git clone https://github.com/software-mansion/runntime +cd runntime/tools/python +uv run runntime quantize model.safetensors model.int8.safetensors --bits 8 --f16 +``` + +The source is a `.safetensors` file or a Hugging Face model folder, sharded +or not. The output is one file. + +- `--bits 8` or `--bits 4`. Required. +- `--group-size N`: inputs that share one scale, a multiple of 4. Default 32. +- `--include GLOB`, repeatable: the tensors to quantize. Default `*.weight`, + which the option replaces. +- `--exclude GLOB`, repeatable: the tensors to keep as stored. Default + `*embed*`, `*pos_emb*` and `lm_head.*`, which the option replaces: the + model reads token and position embeddings as float lookup tables, and + `lm_head` often shares the embedding's weights. Passing one `--exclude` + drops all three defaults, so repeat the ones you want to keep. +- `--f16`: store every tensor that stays float as `F16`. Without it they are + copied as stored. A value beyond the f16 range stops the run. +- `--drop GLOB`, repeatable: leave tensors out of the output, like a + `pooler` the model never reads. + +A tensor is quantized when it matches `--include`, matches no `--exclude`, +is float and 2D, and fits the shape rules below. Every other tensor is +skipped and copied, so the output always loads. Each Linear is quantized or +float on its own. + +The tool prints one line per tensor: + +```text +quantized encoder.layer.0.attention.self.query.weight [384, 384]: max error 0.00302, mean 0.000425 +skipped encoder.layer.0.attention.self.query.bias [384]: not matched by --include +skipped encoder.layer.0.output.LayerNorm.weight [384]: 1D, needs [out, in] +skipped embeddings.word_embeddings.weight [30522, 384]: excluded by *embed* +``` + +The error is the absolute difference between a weight and its int times its +scale, over the whole tensor. A tensor whose error stands out from the rest +is a candidate for `--exclude`. The run fails when no tensor was quantized. + +For all-MiniLM-L6-v2 from its 91 MB `F32` export, the output is 59 MB with +`--bits 8`, 35 MB with `--bits 8 --f16` and 30 MB with `--bits 4 --f16`. The +embedding table is half of this model, so `--f16` matters more than the +bits. + +### The quantized format + + + +- A weight `W` of shape `[out, in]` is stored as `W`: `I8` of shape + `[out, in]` for int8, or `U8` of shape `[out, in / 2]` for int4, two + 4-bit two's-complement values (-8 to 7) per byte, the even index in the + low nibble. +- `W_scale` is `F16` of shape `[out, in / groupSize]`: one scale per group + of `groupSize` values along a row. A weight is its int times its group's + scale. +- The header metadata holds `runntime.quant`, one setting for the whole + file: `{"version":1,"bits":8,"groupSize":32,"scheme":"symmetric"}`. +- Every other tensor is a float tensor as before. + +The loader tells the tensors apart by dtype. `F32`, `F16` and `BF16` load as +float. `I8` and `U8` load as a quantized weight, which needs the +`runntime.quant` label and its `W_scale` of the right shape, or the load +throws `CHECKPOINT_MISMATCH`. Only a Linear weight can be quantized: an int +tensor bound to anything else, like a lookup table, throws too. The privacy +filter's hosted int8 file is a different, pre-packed layout, not this one. + + + +The scale of a group is its largest magnitude over 127 for int8, over 7 +for int4, so the range is symmetric around zero and there is no zero point. +The tool rounds the scale to f16 first and divides by that, so the ints +match the scale the file stores. + +### Shape rules + +The quantized matmul reads whole tiles, so a Linear weight `[out, in]` can +only be quantized when: + +- `in` is a multiple of 8 and of the group size. +- `out` is a multiple of 4 for int8, of 8 for int4. +- the group size is a multiple of 4. + +The tool skips a tensor that breaks one and says which. The default group +size of 32 divides every Linear of the hosted models. + +### Picking the setting + +int8 in groups of 32 kept the quality of the small models we tried, and is +the default. int4 lost some on them: try it when the download size matters +more, and check the output against the float model. A bigger group means +fewer scales and a slightly smaller file, and a bigger error on a group with +one large weight. + +## Quantization-aware training + +A QAT checkpoint saved as float, the "unquantized" form, quantizes like any +other: run `runntime quantize` with the settings the training used, and the +ints land on the grid the model was trained for. int8 in groups of 32 is the +same grid as ggml's `Q8_0`: blocks of 32, a symmetric int, one f16 scale of +the block's largest magnitude over 127. + +The format has no zero point, so an asymmetric scheme does not fit, and +activations stay float: a model trained with int8 activations runs with its +weights quantized and its activations in f16 or f32. From 3496641ca32411e48f7b7a22f5e3f1f8cbb36221 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 3 Nov 2026 12:08:39 +0100 Subject: [PATCH 03/14] docs(docs): correct the load and quantization claims on Exporting weights, with measured MiniLM figures --- .../content/docs/zoo/exporting-weights.mdx | 133 +++++++++++------- 1 file changed, 79 insertions(+), 54 deletions(-) diff --git a/apps/docs/src/content/docs/zoo/exporting-weights.mdx b/apps/docs/src/content/docs/zoo/exporting-weights.mdx index a77e8479..cbf7fb3e 100644 --- a/apps/docs/src/content/docs/zoo/exporting-weights.mdx +++ b/apps/docs/src/content/docs/zoo/exporting-weights.mdx @@ -29,7 +29,8 @@ A weights file is one `.safetensors` file. The loader reads nothing else: no - **Dtypes** are `F32`, `F16` or `BF16` for a float tensor. The file's dtype and the math dtype are independent, see [How a file loads](#how-a-file-loads). A quantized weight is `I8` or `U8` with an `F16` scale tensor next to it, - see [The quantized format](#the-quantized-format). + see [The quantized format](#the-quantized-format). Today all-MiniLM-L6-v2 + loads a quantized file; the other models load float files. A `create` call takes the file as `modelPath`: a URL, or a `RangeSource` for bytes you already have. The ready-made configs in `models` point at the @@ -56,39 +57,41 @@ const embedder = await createEmbedder({ chunks of 8 MB, so a file costs about one request per 8 MB. A server that ignores `Range` and answers `200` makes the loader download the whole file once. A short answer is retried. -3. Each tensor is fetched, converted and uploaded to the GPU on its own, so - host memory holds one tensor at a time. `onProgress` fires after each - one. +3. Each tensor is checked for shape, converted and uploaded to the GPU on + its own. `onProgress` fires after each one. 4. With a `cache`, every fetched range is saved under the `cacheId`, and the - next load reads it back with no network. Cached bytes are never checked - against the server, so a file that changes content needs a new URL or a - new `cacheId`. The default `cacheId` comes from the URL. + next load reads it back with no network. The default `cacheId` of a + `create` call holds the whole URL, so a file at a new URL is a new entry. + Cached bytes are never checked against the server, so a file that changes + content at the same URL needs a new `cacheId`. The file's dtype does not pick the math. The model's `dtype` does: `'f16'` -runs in half precision on a device with `shader-f16`, `'f32'` in full. An -`F32` tensor narrows to f16 when needed, an `F16` or `BF16` one widens to -f32. The vision models run in f16 only. A quantized weight stays int on the -GPU and its scales become f32, whatever the `dtype`. +runs in half precision on a device with `shader-f16`, `'f32'` in full. The +`DEFAULT` configs set no `dtype`, which means f16 where the device has +`shader-f16` and f32 elsewhere. An `F32` tensor narrows to f16 when needed, +an `F16` or `BF16` one widens to f32. The vision models run in f16 only. A +quantized weight stays int on the GPU and its scales become f32, whatever +the `dtype`. ## Exporting float weights Save the model with Hugging Face `transformers`. `save_pretrained` writes -`model.safetensors` with the original names, and `torch_dtype` picks the -stored dtype. `F16` halves the file against `F32` and loses nothing a model -in f16 math would keep anyway. +`model.safetensors` with the original names, and `dtype` picks the stored +dtype. `F16` halves the file against `F32` and loses nothing a model in f16 +math would keep anyway. ```bash uv run --with transformers --with torch python -c " from transformers import AutoModel import torch -model = AutoModel.from_pretrained('sentence-transformers/all-MiniLM-L6-v2', torch_dtype=torch.float16) -model.save_pretrained('export', max_shard_size='100GB') +model = AutoModel.from_pretrained('sentence-transformers/all-MiniLM-L6-v2', dtype=torch.float16) +model.save_pretrained('export', max_shard_size='100GB') # one file, not shards " ``` -`max_shard_size` keeps the export in one file. A folder with shards is fine -as the input of [`runntime quantize`](#quantizing), which writes one file, -but not as a `modelPath`. +A folder with shards is fine as the input of +[`runntime quantize`](#quantizing), which writes one file, but not as a +`modelPath`. A checkpoint from your own training code goes through `safetensors.torch.save_file(model.state_dict(), 'model.safetensors')`. @@ -97,9 +100,9 @@ clone a tied `lm_head` before saving. ### Checking a file -Load it. The names and shapes are checked before any weight downloads, and -a mismatch throws `CHECKPOINT_MISMATCH` with the first names that are -missing or unexpected, or the tensor whose shape is wrong. See +Load it. A name the model lacks or does not expect throws +`CHECKPOINT_MISMATCH` naming the tensor, before any weight downloads. A wrong +shape throws the same code when that tensor binds. See [Error handling](/zoo/error-handling/) for how to catch it. ```ts twoslash @@ -111,7 +114,7 @@ try { modelPath: '/models/minilm.safetensors', }); } catch (err) { - // loadStateDict: state dict does not match model. missing: pooler.dense.weight … + // minilm state dict: missing tensor 'encoder.layer.0.attention.self.query.weight' if (isRunntimeError(err, 'CHECKPOINT_MISMATCH')) console.error(err.message); throw err; } @@ -120,11 +123,13 @@ try { To see what a file holds without loading it, read its header in Python: ```bash -uv run --with safetensors python -c " +uv run --with safetensors --with numpy python -c " from safetensors import safe_open -with safe_open('model.safetensors', 'pt') as f: +with safe_open('path/to/model.safetensors', 'np') as f: print(f.metadata()) - for name in f.keys(): print(name, f.get_slice(name).get_dtype(), f.get_slice(name).get_shape()) + for name in f.keys(): + tensor = f.get_slice(name) + print(name, tensor.get_dtype(), tensor.get_shape()) " ``` @@ -132,36 +137,41 @@ with safe_open('model.safetensors', 'pt') as f: `runntime quantize` turns a float file into an int8 or int4 one. It lives in the repository under `tools/python` and runs with -[uv](https://docs.astral.sh/uv/), which installs its one dependency, NumPy, -on the first run. +[uv](https://docs.astral.sh/uv/), which sets up Python and NumPy on the +first run. Paths are relative to where you run it. ```bash git clone https://github.com/software-mansion/runntime cd runntime/tools/python -uv run runntime quantize model.safetensors model.int8.safetensors --bits 8 --f16 +uv run --no-dev runntime quantize path/to/model.safetensors path/to/model.int8.safetensors --bits 8 --f16 --drop 'pooler.*' ``` The source is a `.safetensors` file or a Hugging Face model folder, sharded -or not. The output is one file. +or not. The output is one file. Today all-MiniLM-L6-v2 is the model that +loads one. - `--bits 8` or `--bits 4`. Required. - `--group-size N`: inputs that share one scale, a multiple of 4. Default 32. -- `--include GLOB`, repeatable: the tensors to quantize. Default `*.weight`, - which the option replaces. +- `--include GLOB`, repeatable: the tensors to quantize. Default `*.weight`. - `--exclude GLOB`, repeatable: the tensors to keep as stored. Default - `*embed*`, `*pos_emb*` and `lm_head.*`, which the option replaces: the - model reads token and position embeddings as float lookup tables, and - `lm_head` often shares the embedding's weights. Passing one `--exclude` - drops all three defaults, so repeat the ones you want to keep. + `*embed*`, `*pos_emb*` and `lm_head.*`: the model reads token and + position embeddings as float lookup tables, and `lm_head` often shares + the embedding's weights. - `--f16`: store every tensor that stays float as `F16`. Without it they are copied as stored. A value beyond the f16 range stops the run. -- `--drop GLOB`, repeatable: leave tensors out of the output, like a - `pooler` the model never reads. +- `--drop GLOB`, repeatable: leave tensors out of the output, like the + `pooler` above, which the embedding model never reads. + +Passing `--include` or `--exclude` replaces that option's defaults, so +repeat the ones you want to keep. A tensor is quantized when it matches `--include`, matches no `--exclude`, is float and 2D, and fits the shape rules below. Every other tensor is -skipped and copied, so the output always loads. Each Linear is quantized or -float on its own. +skipped and copied, so the output always loads. A Linear is quantized or +float on its own, except weights the model fuses into one: all-MiniLM-L6-v2 +stacks query, key and value, so the three are quantized together, with the +same bits and group size, or not at all. Mixing them throws +`CHECKPOINT_MISMATCH` at load. The tool prints one line per tensor: @@ -170,11 +180,15 @@ quantized encoder.layer.0.attention.self.query.weight [384, 384]: max error 0.0 skipped encoder.layer.0.attention.self.query.bias [384]: not matched by --include skipped encoder.layer.0.output.LayerNorm.weight [384]: 1D, needs [out, in] skipped embeddings.word_embeddings.weight [30522, 384]: excluded by *embed* +dropped pooler.dense.weight [384, 384]: matched by pooler.* ``` The error is the absolute difference between a weight and its int times its scale, over the whole tensor. A tensor whose error stands out from the rest -is a candidate for `--exclude`. The run fails when no tensor was quantized. +is a candidate for `--exclude`. Read the `skipped` lines too: `*embed*` +also matches a Linear with `embed` in its name, like a projection after an +embedding, and such a tensor needs your own `--exclude` list to get +quantized. The run fails when no tensor was quantized. For all-MiniLM-L6-v2 from its 91 MB `F32` export, the output is 59 MB with `--bits 8`, 35 MB with `--bits 8 --f16` and 30 MB with `--bits 4 --f16`. The @@ -193,15 +207,22 @@ bits. of `groupSize` values along a row. A weight is its int times its group's scale. - The header metadata holds `runntime.quant`, one setting for the whole - file: `{"version":1,"bits":8,"groupSize":32,"scheme":"symmetric"}`. + file, as a JSON string: + + ```json + { "version": 1, "bits": 8, "groupSize": 32, "scheme": "symmetric" } + ``` + + `version` must be `1` and `scheme` must be `"symmetric"`, or the load + throws. + - Every other tensor is a float tensor as before. The loader tells the tensors apart by dtype. `F32`, `F16` and `BF16` load as float. `I8` and `U8` load as a quantized weight, which needs the `runntime.quant` label and its `W_scale` of the right shape, or the load throws `CHECKPOINT_MISMATCH`. Only a Linear weight can be quantized: an int -tensor bound to anything else, like a lookup table, throws too. The privacy -filter's hosted int8 file is a different, pre-packed layout, not this one. +tensor bound to anything else, like a lookup table, throws too. @@ -220,23 +241,27 @@ only be quantized when: - the group size is a multiple of 4. The tool skips a tensor that breaks one and says which. The default group -size of 32 divides every Linear of the hosted models. +size of 32 divides both Linear input sizes of all-MiniLM-L6-v2, 384 and 1536. ### Picking the setting -int8 in groups of 32 kept the quality of the small models we tried, and is -the default. int4 lost some on them: try it when the download size matters -more, and check the output against the float model. A bigger group means -fewer scales and a slightly smaller file, and a bigger error on a group with -one large weight. +Measured on all-MiniLM-L6-v2 against the float model, over f16 and f32 math +and group sizes 32 and 128: the worst cosine similarity of an embedding +is 0.999993 with int8 and 0.99782 with int4. int8 in groups of 32 is the +default. Try int4 when the download size matters more, and check the output +on your own data. A bigger group means fewer scales and a slightly smaller +file, and a bigger error on a group with one large weight. ## Quantization-aware training A QAT checkpoint saved as float, the "unquantized" form, quantizes like any -other: run `runntime quantize` with the settings the training used, and the -ints land on the grid the model was trained for. int8 in groups of 32 is the -same grid as ggml's `Q8_0`: blocks of 32, a symmetric int, one f16 scale of -the block's largest magnitude over 127. +other: run `runntime quantize` with the settings the training used. The +rule the tool applies is fixed: symmetric, groups of `--group-size` inputs +along `in`, scale = the group's largest magnitude over 127 (int8) or 7 +(int4) rounded to f16, int = weight / scale rounded to nearest. A model +trained on the same rule lands on its grid; a per-channel or asymmetric +recipe gets re-rounded to this one. int8 in groups of 32 has the same block +layout as ggml's `Q8_0`: blocks of 32 along the input, one f16 scale each. The format has no zero point, so an asymmetric scheme does not fit, and activations stay float: a model trained with int8 activations runs with its From d0454a9587b048208bd79d02ac256248eb57e9cd Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 3 Nov 2026 12:08:40 +0100 Subject: [PATCH 04/14] docs(docs): larger diagram text at phone width, F16 floats in the file diagram, no row-end arrow --- apps/docs/src/components/QuantFile.astro | 45 ++++++++---------- apps/docs/src/components/QuantGroup.astro | 54 +++++++++++----------- apps/docs/src/components/WeightsPath.astro | 16 +++---- 3 files changed, 55 insertions(+), 60 deletions(-) diff --git a/apps/docs/src/components/QuantFile.astro b/apps/docs/src/components/QuantFile.astro index 96ba8b3f..a38ba02e 100644 --- a/apps/docs/src/components/QuantFile.astro +++ b/apps/docs/src/components/QuantFile.astro @@ -1,16 +1,17 @@ --- /** What a quantized safetensors file holds and how the loader reads each * entry: the header label, an int8 weight with its F16 scales, and the float - * tensors around them. Names are all-MiniLM-L6-v2's, shortened. */ + * tensors around them. Names are all-MiniLM-L6-v2's, shortened; the floats + * are F16 as `--f16` stores them. */ const rows = [ - { name: 'embeddings.word_embeddings.weight', dtype: 'F32', shape: '[30522, 384]', loads: 'float' }, - { name: '…attention.self.query.weight', dtype: 'I8', shape: '[384, 384]', loads: 'int8 weight' }, - { name: '…attention.self.query.weight_scale', dtype: 'F16', shape: '[384, 12]', loads: 'its scales' }, - { name: '…attention.self.query.bias', dtype: 'F32', shape: '[384]', loads: 'float' }, - { name: '…attention.output.LayerNorm.weight', dtype: 'F32', shape: '[384]', loads: 'float' }, + { name: '…word_embeddings.weight', dtype: 'F16', shape: '[30522, 384]', loads: 'float' }, + { name: '…self.query.weight', dtype: 'I8', shape: '[384, 384]', loads: 'int8 weight' }, + { name: '…self.query.weight_scale', dtype: 'F16', shape: '[384, 12]', loads: 'its scales' }, + { name: '…self.query.bias', dtype: 'F16', shape: '[384]', loads: 'float' }, + { name: '…LayerNorm.weight', dtype: 'F16', shape: '[384]', loads: 'float' }, ]; -const rowHeight = 26; -const top = 58; +const rowHeight = 30; +const top = 84; const height = top + rows.length * rowHeight + 14; --- @@ -18,27 +19,26 @@ const height = top + rows.length * rowHeight + 14; class="quant-file" viewBox={`0 0 400 ${height}`} role="img" - aria-label="A quantized file: the runntime.quant header label, then tensors. Float tensors load as float, an I8 weight loads as an int8 weight and takes its F16 weight_scale with it." + aria-label="A quantized file: the runntime.quant header label, then tensors. F16 tensors load as float, an I8 weight loads as an int8 weight and takes its F16 weight_scale with it." > - __metadata__ runntime.quant = - - {'{'}"version":1,"bits":8,"groupSize":32,"scheme":"symmetric"{'}'} - - + __metadata__ runntime.quant = + {'{'}"version":1,"bits":8, + "groupSize":32,"scheme":"symmetric"{'}'} + { rows.map((row, i) => { - const y = top + i * rowHeight + 10; - const quant = row.dtype !== 'F32'; + const y = top + i * rowHeight + 12; + const quant = row.dtype === 'I8' || row.loads === 'its scales'; return ( {quant && ( - + )} {row.name} - + {row.dtype} {row.shape} @@ -55,7 +55,7 @@ const height = top + rows.length * rowHeight + 14; display: block; height: auto; margin: 1.5rem auto; - max-width: 40rem; + max-width: 30rem; width: 100%; } @@ -75,11 +75,7 @@ const height = top + rows.length * rowHeight + 14; text { fill: var(--text-muted); font-family: var(--__sl-font-mono, monospace); - font-size: 10px; - } - - .header { - font-size: 9px; + font-size: 13px; } .strong, @@ -90,6 +86,5 @@ const height = top + rows.length * rowHeight + 14; .loads { font-family: var(--__sl-font, sans-serif); - font-size: 10.5px; } diff --git a/apps/docs/src/components/QuantGroup.astro b/apps/docs/src/components/QuantGroup.astro index c97a5525..63b39993 100644 --- a/apps/docs/src/components/QuantGroup.astro +++ b/apps/docs/src/components/QuantGroup.astro @@ -1,54 +1,55 @@ --- -/** One group of 32 weights becoming 32 int8 values and one f16 scale. Six +/** One group of 32 weights becoming 32 int8 values and one f16 scale. Five * of the 32 are drawn; the scale is the group's largest magnitude over * 127, and each int is its weight divided by the scale, rounded. */ -const weights = ['0.031', '-0.118', '0.054', '0.002', '0.087', '-0.070']; -const ints = ['33', '-127', '58', '2', '94', '-75']; -const cell = 50; +const weights = ['0.031', '-0.118', '0.054', '0.002', '-0.070']; +const ints = ['33', '-127', '58', '2', '-75']; +const cell = 58; +const box = cell - 5; const left = 12; -const cells = (values: string[], y: number, strong: boolean) => - values.map((value, i) => ({ x: left + i * cell, y, value, strong })); +const cells = (values: string[], y: number) => + values.map((value, i) => ({ x: left + i * cell, y, value })); --- - 32 weights of one row, f32 + 32 weights of one row, f32 { - cells(weights, 22, false).map((c) => ( + cells(weights, 26).map((c) => ( - - + + {c.value} )) } - … 26 more + … 27 more - ↓ - + ↓ + scale = 0.118 / 127 = 0.000929, stored as f16 - int = round(weight / scale), int8 - scale, f16 + int = round(weight / scale), int8 + scale, f16 { - cells(ints, 118, true).map((c) => ( + cells(ints, 138).map((c) => ( - - + + {c.value} )) } - - 0.000929 - 32 bytes of ints and 2 bytes of scale, in place of 128 bytes + + 0.000929 + 34 bytes in place of 128 diff --git a/apps/docs/src/components/WeightsPath.astro b/apps/docs/src/components/WeightsPath.astro index 6f150b54..c19b5d62 100644 --- a/apps/docs/src/components/WeightsPath.astro +++ b/apps/docs/src/components/WeightsPath.astro @@ -58,8 +58,9 @@ const steps = [ top: 0.625rem; } - /* The arrow to the next step, between the boxes. */ - .weights-path li:not(:last-child)::after { + /* The arrow to the next step, between the boxes. The third box ends a + * row, so it gets none: the numbers carry the order to the next row. */ + .weights-path li:not(:last-child):not(:nth-child(3))::after { color: var(--text-faint); content: '→'; font-size: 1.125rem; @@ -70,11 +71,6 @@ const steps = [ transform: translateY(-50%); } - /* The third box ends a row: the path continues at the start of the next. */ - .weights-path li:nth-child(3)::after { - content: '↵'; - } - .weights-path-title { color: var(--text-primary); font-weight: 600; @@ -94,11 +90,15 @@ const steps = [ } .weights-path li:not(:last-child)::after { + bottom: -1.5rem; + color: var(--text-faint); content: '↓'; + font-size: 1.125rem; left: 50%; + line-height: 1; + position: absolute; right: auto; top: auto; - bottom: -1.5rem; transform: translateX(-50%); } } From 3f91ebed8460603897d5f4a04203d3ecc9d93acc Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 3 Nov 2026 12:08:42 +0100 Subject: [PATCH 05/14] docs(docs): point the Embedding page's quantized section at Exporting weights --- apps/docs/src/content/docs/zoo/embedding.mdx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/docs/src/content/docs/zoo/embedding.mdx b/apps/docs/src/content/docs/zoo/embedding.mdx index 3747d5de..8993126c 100644 --- a/apps/docs/src/content/docs/zoo/embedding.mdx +++ b/apps/docs/src/content/docs/zoo/embedding.mdx @@ -194,7 +194,8 @@ await createEmbedder({ ``` - **modelPath** - the weights, a `model.safetensors` file. The model is - read from the file. + read from the file. To make your own, see + [Exporting weights](/zoo/exporting-weights/). - **visionPath** - the weights for images and video. `null` skips the download, and images and video become an error. - **audioPath** - the same for audio. From 4b60dd474f9880ed0d07ffe5728658c87a8ac9c7 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 3 Nov 2026 12:18:18 +0100 Subject: [PATCH 06/14] docs(docs): say a wrong shape reports the shapes it could not bind, and name the quantized model once --- apps/docs/src/content/docs/zoo/exporting-weights.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/docs/src/content/docs/zoo/exporting-weights.mdx b/apps/docs/src/content/docs/zoo/exporting-weights.mdx index cbf7fb3e..ddd4e17a 100644 --- a/apps/docs/src/content/docs/zoo/exporting-weights.mdx +++ b/apps/docs/src/content/docs/zoo/exporting-weights.mdx @@ -102,7 +102,8 @@ clone a tied `lm_head` before saving. Load it. A name the model lacks or does not expect throws `CHECKPOINT_MISMATCH` naming the tensor, before any weight downloads. A wrong -shape throws the same code when that tensor binds. See +shape throws the same code when that tensor binds, with the shapes it could +not bind. See [Error handling](/zoo/error-handling/) for how to catch it. ```ts twoslash @@ -147,8 +148,7 @@ uv run --no-dev runntime quantize path/to/model.safetensors path/to/model.int8.s ``` The source is a `.safetensors` file or a Hugging Face model folder, sharded -or not. The output is one file. Today all-MiniLM-L6-v2 is the model that -loads one. +or not. The output is one file. - `--bits 8` or `--bits 4`. Required. - `--group-size N`: inputs that share one scale, a multiple of 4. Default 32. From e471d98f0078d3a23df47a0507acc12b7e0e476c Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 3 Nov 2026 12:18:19 +0100 Subject: [PATCH 07/14] docs(docs): key the file diagram's highlight on a quant flag and shorten its names --- apps/docs/src/components/QuantFile.astro | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/apps/docs/src/components/QuantFile.astro b/apps/docs/src/components/QuantFile.astro index a38ba02e..7ecf7d67 100644 --- a/apps/docs/src/components/QuantFile.astro +++ b/apps/docs/src/components/QuantFile.astro @@ -5,9 +5,9 @@ * are F16 as `--f16` stores them. */ const rows = [ { name: '…word_embeddings.weight', dtype: 'F16', shape: '[30522, 384]', loads: 'float' }, - { name: '…self.query.weight', dtype: 'I8', shape: '[384, 384]', loads: 'int8 weight' }, - { name: '…self.query.weight_scale', dtype: 'F16', shape: '[384, 12]', loads: 'its scales' }, - { name: '…self.query.bias', dtype: 'F16', shape: '[384]', loads: 'float' }, + { name: '…query.weight', dtype: 'I8', shape: '[384, 384]', loads: 'int8 weight', quant: true }, + { name: '…query.weight_scale', dtype: 'F16', shape: '[384, 12]', loads: 'its scales', quant: true }, + { name: '…query.bias', dtype: 'F16', shape: '[384]', loads: 'float' }, { name: '…LayerNorm.weight', dtype: 'F16', shape: '[384]', loads: 'float' }, ]; const rowHeight = 30; @@ -29,10 +29,9 @@ const height = top + rows.length * rowHeight + 14; { rows.map((row, i) => { const y = top + i * rowHeight + 12; - const quant = row.dtype === 'I8' || row.loads === 'its scales'; return ( - - {quant && ( + + {row.quant && ( )} From 43a7498796793d0857d235199ad891c4dab692ad Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 4 Nov 2026 10:34:18 +0100 Subject: [PATCH 08/14] docs(docs): quote the int8 and int4 quality measured on the real MiniLM weights, not the random-weight test --- apps/docs/src/content/docs/zoo/exporting-weights.mdx | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/apps/docs/src/content/docs/zoo/exporting-weights.mdx b/apps/docs/src/content/docs/zoo/exporting-weights.mdx index ddd4e17a..2679f2a9 100644 --- a/apps/docs/src/content/docs/zoo/exporting-weights.mdx +++ b/apps/docs/src/content/docs/zoo/exporting-weights.mdx @@ -245,11 +245,12 @@ size of 32 divides both Linear input sizes of all-MiniLM-L6-v2, 384 and 1536. ### Picking the setting -Measured on all-MiniLM-L6-v2 against the float model, over f16 and f32 math -and group sizes 32 and 128: the worst cosine similarity of an embedding -is 0.999993 with int8 and 0.99782 with int4. int8 in groups of 32 is the -default. Try int4 when the download size matters more, and check the output -on your own data. A bigger group means fewer scales and a slightly smaller +Measured on the all-MiniLM-L6-v2 weights against the float model, 20 +search queries, groups of 32: the worst cosine similarity of an embedding +is 0.9998 with int8 and 0.94 with int4, and int4 changes the top 3 results +of 8 queries. int8 in groups of 32 is the safe pick and the default. int4 +costs quality: try it only when the download size matters more, and check +the output on your own data. A bigger group means fewer scales and a slightly smaller file, and a bigger error on a group with one large weight. ## Quantization-aware training From bf2430ff0bb935f379f2c46ce2f20790190eef7e Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 4 Nov 2026 10:56:02 +0100 Subject: [PATCH 09/14] docs(docs): cut the loader internals, byte layout and rounding math from Exporting weights, keep what a reader does --- apps/docs/src/components/QuantGroup.astro | 101 ------------- .../content/docs/zoo/exporting-weights.mdx | 142 +++++------------- 2 files changed, 36 insertions(+), 207 deletions(-) delete mode 100644 apps/docs/src/components/QuantGroup.astro diff --git a/apps/docs/src/components/QuantGroup.astro b/apps/docs/src/components/QuantGroup.astro deleted file mode 100644 index 63b39993..00000000 --- a/apps/docs/src/components/QuantGroup.astro +++ /dev/null @@ -1,101 +0,0 @@ ---- -/** One group of 32 weights becoming 32 int8 values and one f16 scale. Five - * of the 32 are drawn; the scale is the group's largest magnitude over - * 127, and each int is its weight divided by the scale, rounded. */ -const weights = ['0.031', '-0.118', '0.054', '0.002', '-0.070']; -const ints = ['33', '-127', '58', '2', '-75']; -const cell = 58; -const box = cell - 5; -const left = 12; -const cells = (values: string[], y: number) => - values.map((value, i) => ({ x: left + i * cell, y, value })); ---- - - - 32 weights of one row, f32 - { - cells(weights, 26).map((c) => ( - - - - {c.value} - - - )) - } - … 27 more - - ↓ - - scale = 0.118 / 127 = 0.000929, stored as f16 - - - int = round(weight / scale), int8 - scale, f16 - { - cells(ints, 138).map((c) => ( - - - - {c.value} - - - )) - } - - 0.000929 - 34 bytes in place of 128 - - - diff --git a/apps/docs/src/content/docs/zoo/exporting-weights.mdx b/apps/docs/src/content/docs/zoo/exporting-weights.mdx index 2679f2a9..9afb15ac 100644 --- a/apps/docs/src/content/docs/zoo/exporting-weights.mdx +++ b/apps/docs/src/content/docs/zoo/exporting-weights.mdx @@ -5,7 +5,6 @@ description: Run your own weights in ruNNtime, as float or quantized to int8 or import WeightsPath from '../../../components/WeightsPath.astro'; import QuantFile from '../../../components/QuantFile.astro'; -import QuantGroup from '../../../components/QuantGroup.astro'; Every model in the zoo reads its weights from one safetensors file. The hosted files are exports of the original checkpoints. Your own export works @@ -26,11 +25,9 @@ A weights file is one `.safetensors` file. The loader reads nothing else: no file's header lists the exact set. - **Shapes** are the original layout. A Linear weight is `[out, in]`, as PyTorch stores it. -- **Dtypes** are `F32`, `F16` or `BF16` for a float tensor. The file's dtype - and the math dtype are independent, see [How a file loads](#how-a-file-loads). - A quantized weight is `I8` or `U8` with an `F16` scale tensor next to it, - see [The quantized format](#the-quantized-format). Today all-MiniLM-L6-v2 - loads a quantized file; the other models load float files. +- **Dtypes** are `F32`, `F16` or `BF16`, or int8/int4 for a file from + [`runntime quantize`](#quantizing). Today all-MiniLM-L6-v2 loads a + quantized file; the other models load float files. A `create` call takes the file as `modelPath`: a URL, or a `RangeSource` for bytes you already have. The ready-made configs in `models` point at the @@ -46,32 +43,17 @@ const embedder = await createEmbedder({ }); ``` -## How a file loads - -1. The loader reads the header: 8 bytes of length, then JSON with every - tensor's name, dtype, shape and byte range. It checks the names against - the model here, before any weight downloads. A file that is not - safetensors, like a dev server's `index.html` fallback for a missing URL, - throws `LOAD_FAILED` naming the URL. -2. Tensors download in file order with HTTP `Range` requests, in aligned - chunks of 8 MB, so a file costs about one request per 8 MB. A server that - ignores `Range` and answers `200` makes the loader download the whole file - once. A short answer is retried. -3. Each tensor is checked for shape, converted and uploaded to the GPU on - its own. `onProgress` fires after each one. -4. With a `cache`, every fetched range is saved under the `cacheId`, and the - next load reads it back with no network. The default `cacheId` of a - `create` call holds the whole URL, so a file at a new URL is a new entry. - Cached bytes are never checked against the server, so a file that changes - content at the same URL needs a new `cacheId`. - -The file's dtype does not pick the math. The model's `dtype` does: `'f16'` -runs in half precision on a device with `shader-f16`, `'f32'` in full. The -`DEFAULT` configs set no `dtype`, which means f16 where the device has -`shader-f16` and f32 elsewhere. An `F32` tensor narrows to f16 when needed, -an `F16` or `BF16` one widens to f32. The vision models run in f16 only. A -quantized weight stays int on the GPU and its scales become f32, whatever -the `dtype`. +## Caching and precision + +With a `cache`, the downloaded file is saved in the browser and the next load +needs no network. The cache key comes from the URL and the bytes are never +checked against the server: when you replace a file, give it a new URL or +pass a new `cacheId`. + +The file's dtype does not pick the math, the model's `dtype` does. The +`DEFAULT` configs run f16 where the device has `shader-f16` and f32 +elsewhere, whatever the file stores. An `F16` file is half the download of +an `F32` one. ## Exporting float weights @@ -100,11 +82,9 @@ clone a tied `lm_head` before saving. ### Checking a file -Load it. A name the model lacks or does not expect throws -`CHECKPOINT_MISMATCH` naming the tensor, before any weight downloads. A wrong -shape throws the same code when that tensor binds, with the shapes it could -not bind. See -[Error handling](/zoo/error-handling/) for how to catch it. +Load it. A missing or extra tensor throws `CHECKPOINT_MISMATCH` naming it, +before any weight downloads; a wrong shape throws the same code with the +shapes. See [Error handling](/zoo/error-handling/) for how to catch it. ```ts twoslash import { createEmbedder, isRunntimeError, models } from 'runntime/zoo'; @@ -166,12 +146,8 @@ Passing `--include` or `--exclude` replaces that option's defaults, so repeat the ones you want to keep. A tensor is quantized when it matches `--include`, matches no `--exclude`, -is float and 2D, and fits the shape rules below. Every other tensor is -skipped and copied, so the output always loads. A Linear is quantized or -float on its own, except weights the model fuses into one: all-MiniLM-L6-v2 -stacks query, key and value, so the three are quantized together, with the -same bits and group size, or not at all. Mixing them throws -`CHECKPOINT_MISMATCH` at load. +is float and 2D, and has a shape the quantized matmul can tile. Every other +tensor is copied as it is, so the output always loads. The tool prints one line per tensor: @@ -183,65 +159,26 @@ skipped embeddings.word_embeddings.weight [30522, 384]: excluded by *embed* dropped pooler.dense.weight [384, 384]: matched by pooler.* ``` -The error is the absolute difference between a weight and its int times its -scale, over the whole tensor. A tensor whose error stands out from the rest -is a candidate for `--exclude`. Read the `skipped` lines too: `*embed*` -also matches a Linear with `embed` in its name, like a projection after an -embedding, and such a tensor needs your own `--exclude` list to get -quantized. The run fails when no tensor was quantized. +The error is how far the quantized weights are from the float ones. A +tensor whose error stands out from the rest is a candidate for `--exclude`. +Read the `skipped` lines too: `*embed*` also matches a layer with `embed` in +its name, like a projection after an embedding, which then needs your own +`--exclude` list to get quantized. The run fails when no tensor was +quantized. For all-MiniLM-L6-v2 from its 91 MB `F32` export, the output is 59 MB with `--bits 8`, 35 MB with `--bits 8 --f16` and 30 MB with `--bits 4 --f16`. The embedding table is half of this model, so `--f16` matters more than the bits. -### The quantized format +### What the file holds -- A weight `W` of shape `[out, in]` is stored as `W`: `I8` of shape - `[out, in]` for int8, or `U8` of shape `[out, in / 2]` for int4, two - 4-bit two's-complement values (-8 to 7) per byte, the even index in the - low nibble. -- `W_scale` is `F16` of shape `[out, in / groupSize]`: one scale per group - of `groupSize` values along a row. A weight is its int times its group's - scale. -- The header metadata holds `runntime.quant`, one setting for the whole - file, as a JSON string: - - ```json - { "version": 1, "bits": 8, "groupSize": 32, "scheme": "symmetric" } - ``` - - `version` must be `1` and `scheme` must be `"symmetric"`, or the load - throws. - -- Every other tensor is a float tensor as before. - -The loader tells the tensors apart by dtype. `F32`, `F16` and `BF16` load as -float. `I8` and `U8` load as a quantized weight, which needs the -`runntime.quant` label and its `W_scale` of the right shape, or the load -throws `CHECKPOINT_MISMATCH`. Only a Linear weight can be quantized: an int -tensor bound to anything else, like a lookup table, throws too. - - - -The scale of a group is its largest magnitude over 127 for int8, over 7 -for int4, so the range is symmetric around zero and there is no zero point. -The tool rounds the scale to f16 first and divides by that, so the ints -match the scale the file stores. - -### Shape rules - -The quantized matmul reads whole tiles, so a Linear weight `[out, in]` can -only be quantized when: - -- `in` is a multiple of 8 and of the group size. -- `out` is a multiple of 4 for int8, of 8 for int4. -- the group size is a multiple of 4. - -The tool skips a tensor that breaks one and says which. The default group -size of 32 divides both Linear input sizes of all-MiniLM-L6-v2, 384 and 1536. +Each quantized weight is stored as int8 or int4 with one f16 scale per group +of `--group-size` inputs, and the file header records the bits and group +size. Every other tensor stays float. Only the weights of Linear layers load +quantized, which is why the default `--exclude` keeps embedding tables float. ### Picking the setting @@ -250,20 +187,13 @@ search queries, groups of 32: the worst cosine similarity of an embedding is 0.9998 with int8 and 0.94 with int4, and int4 changes the top 3 results of 8 queries. int8 in groups of 32 is the safe pick and the default. int4 costs quality: try it only when the download size matters more, and check -the output on your own data. A bigger group means fewer scales and a slightly smaller -file, and a bigger error on a group with one large weight. +the output on your own data. ## Quantization-aware training A QAT checkpoint saved as float, the "unquantized" form, quantizes like any -other: run `runntime quantize` with the settings the training used. The -rule the tool applies is fixed: symmetric, groups of `--group-size` inputs -along `in`, scale = the group's largest magnitude over 127 (int8) or 7 -(int4) rounded to f16, int = weight / scale rounded to nearest. A model -trained on the same rule lands on its grid; a per-channel or asymmetric -recipe gets re-rounded to this one. int8 in groups of 32 has the same block -layout as ggml's `Q8_0`: blocks of 32 along the input, one f16 scale each. - -The format has no zero point, so an asymmetric scheme does not fit, and -activations stay float: a model trained with int8 activations runs with its -weights quantized and its activations in f16 or f32. +other. The tool's grid is symmetric int8 or int4, one scale per group of +inputs, the scale being the group's largest magnitude over 127 or 7. A model +trained on that grid, like one trained for ggml's `Q8_0` with +`--bits 8 --group-size 32`, keeps what it learned; another recipe gets +re-rounded. Activations stay float. From b1cfbeb2ade89fba410fcd63e85b059933cade8d Mon Sep 17 00:00:00 2001 From: chmjkb Date: Thu, 12 Nov 2026 15:26:16 +0100 Subject: [PATCH 10/14] docs(docs): shorten Exporting weights to name mapping, getting a file and quantizing, with a branching diagram --- apps/docs/src/components/WeightsPath.astro | 221 ++++++++------ .../content/docs/zoo/exporting-weights.mdx | 274 ++++++++---------- 2 files changed, 252 insertions(+), 243 deletions(-) diff --git a/apps/docs/src/components/WeightsPath.astro b/apps/docs/src/components/WeightsPath.astro index c19b5d62..b6b75b9c 100644 --- a/apps/docs/src/components/WeightsPath.astro +++ b/apps/docs/src/components/WeightsPath.astro @@ -1,105 +1,160 @@ --- -/** The path of a weights file, from a checkpoint to the GPU: six numbered - * steps in two rows, one column at phone width. */ -const steps = [ - { title: 'Checkpoint', detail: 'PyTorch or Hugging Face' }, - { title: 'Export', detail: 'model.safetensors, F32 / F16 / BF16' }, - { title: 'Quantize', detail: 'runntime quantize, int8 or int4. Optional.' }, - { title: 'Host', detail: 'any static file server' }, - { title: 'modelPath', detail: 'create({ modelPath })' }, - { title: 'Load', detail: 'header, 8 MB chunks, cache, GPU upload' }, +/** The path of a weights file, from a checkpoint to the GPU, with quantizing + * as an optional branch. Laid out in a row, and in a column at phone width. */ +const BOX_HEIGHT = 56; + +const checkpoint = { title: 'Checkpoint', detail: 'PyTorch / HF' }; +const exported = { title: 'Export', detail: '.safetensors' }; +const quantize = { title: 'Quantize', detail: 'int8 / int4', optional: true }; +const modelPath = { title: 'modelPath', detail: 'create()' }; +const load = { title: 'Load', detail: 'GPU upload' }; + +/** The quantize branch leaves Export from the top and joins modelPath from + * the top; the float path runs straight between them. The column puts the + * float path on the left and the branch on the right. */ +const layouts = [ + { + name: 'row', + width: 770, + height: 120, + boxes: [ + { ...checkpoint, x: 0, y: 64, width: 130 }, + { ...exported, x: 160, y: 64, width: 130 }, + { ...quantize, x: 320, y: 0, width: 130 }, + { ...modelPath, x: 480, y: 64, width: 130 }, + { ...load, x: 640, y: 64, width: 130 }, + ], + arrows: ['M130 92 H160', 'M225 64 V28 H320', 'M450 28 H545 V64', 'M290 92 H480', 'M610 92 H640'], + skipLabel: { x: 385, y: 84, anchor: 'middle', lines: ['float file as is'] }, + }, + { + name: 'column', + width: 340, + height: 440, + boxes: [ + { ...checkpoint, x: 100, y: 0, width: 240 }, + { ...exported, x: 100, y: 96, width: 240 }, + { ...quantize, x: 190, y: 192, width: 150 }, + { ...modelPath, x: 100, y: 288, width: 240 }, + { ...load, x: 100, y: 384, width: 240 }, + ], + arrows: ['M220 56 V96', 'M265 152 V192', 'M265 248 V288', 'M130 152 V288', 'M220 344 V384'], + skipLabel: { x: 122, y: 216, anchor: 'end', lines: ['float file', 'as is'] }, + }, ]; --- -
    - { - steps.map((step) => ( -
  1. - {step.title} - {step.detail} -
  2. - )) - } -
+{ + layouts.map((layout) => ( + + + + + + + {layout.arrows.map((d) => ( + + ))} + + {layout.skipLabel.lines.map((line, i) => ( + + {line} + + ))} + + {layout.boxes.map((box) => ( + + + + {box.title} + {box.optional && optional} + + + {box.detail} + + + ))} + + )) +} diff --git a/apps/docs/src/content/docs/zoo/exporting-weights.mdx b/apps/docs/src/content/docs/zoo/exporting-weights.mdx index 9afb15ac..43440571 100644 --- a/apps/docs/src/content/docs/zoo/exporting-weights.mdx +++ b/apps/docs/src/content/docs/zoo/exporting-weights.mdx @@ -6,194 +6,148 @@ description: Run your own weights in ruNNtime, as float or quantized to int8 or import WeightsPath from '../../../components/WeightsPath.astro'; import QuantFile from '../../../components/QuantFile.astro'; -Every model in the zoo reads its weights from one safetensors file. The -hosted files are exports of the original checkpoints. Your own export works -the same way: a fine-tune, a smaller download, or a quantized file. This page -is the path from a checkpoint to a file a `create` call loads. +Every model in the zoo loads its weights from one `.safetensors` file. Your +own fine-tune or a quantized copy loads the same way as the hosted file. -## The weights file - -A weights file is one `.safetensors` file. The loader reads nothing else: no -`config.json`, no sharded index. - -- **Names** are the ones the model's loader expects. For the hosted models - these are the names of the original checkpoint, like - `encoder.layer.0.attention.self.query.weight` for all-MiniLM-L6-v2 or - `model.decoder.layers.0.self_attn.q_proj.weight` for Moonshine. The hosted - file's header lists the exact set. -- **Shapes** are the original layout. A Linear weight is `[out, in]`, as - PyTorch stores it. -- **Dtypes** are `F32`, `F16` or `BF16`, or int8/int4 for a file from - [`runntime quantize`](#quantizing). Today all-MiniLM-L6-v2 loads a - quantized file; the other models load float files. - -A `create` call takes the file as `modelPath`: a URL, or a `RangeSource` for -bytes you already have. The ready-made configs in `models` point at the -hosted files. Spread one and replace `modelPath` to keep its other fields, -like the tokenizer. - -```ts twoslash -import { createEmbedder, models } from 'runntime/zoo'; - -const embedder = await createEmbedder({ - ...models.embedding.ALL_MINILM_L6_V2.DEFAULT, - modelPath: '/models/minilm.int8.safetensors', -}); -``` +## Weights - model definition -## Caching and precision +A trained model is a list of named weights. In PyTorch, +`model.state_dict()` gives you that list, and a `.safetensors` file stores it +on disk. For all-MiniLM-L6-v2 it starts like this: -With a `cache`, the downloaded file is saved in the browser and the next load -needs no network. The cache key comes from the URL and the bytes are never -checked against the server: when you replace a file, give it a new URL or -pass a new `cacheId`. +```text +embeddings.word_embeddings.weight [30522, 384] +encoder.layer.0.attention.self.query.weight [384, 384] +encoder.layer.0.attention.self.query.bias [384] +... +``` -The file's dtype does not pick the math, the model's `dtype` does. The -`DEFAULT` configs run f16 where the device has `shader-f16` and f32 -elsewhere, whatever the file stores. An `F16` file is half the download of -an `F32` one. +ruNNtime builds these names the same way PyTorch does. A model is a tree of +modules, like `nn.Module`: each layer is a field, and a weight's name is the +path of field names from the top, joined with dots. A list of layers adds +the index, like `layer.0`. These two models give the same names: -## Exporting float weights +```python -Save the model with Hugging Face `transformers`. `save_pretrained` writes -`model.safetensors` with the original names, and `dtype` picks the stored -dtype. `F16` halves the file against `F32` and loses nothing a model in f16 -math would keep anyway. +# PyTorch +class Block(nn.Module): + def __init__(self): + super().__init__() + self.query = nn.Linear(384, 384) -```bash -uv run --with transformers --with torch python -c " -from transformers import AutoModel -import torch -model = AutoModel.from_pretrained('sentence-transformers/all-MiniLM-L6-v2', dtype=torch.float16) -model.save_pretrained('export', max_shard_size='100GB') # one file, not shards -" +class Encoder(nn.Module): + def __init__(self): + super().__init__() + self.layer = nn.ModuleList([Block() for _ in range(6)]) + +# layer.0.query.weight, layer.0.query.bias, layer.1.query.weight, ... ``` -A folder with shards is fine as the input of -[`runntime quantize`](#quantizing), which writes one file, but not as a -`modelPath`. - -A checkpoint from your own training code goes through -`safetensors.torch.save_file(model.state_dict(), 'model.safetensors')`. -Every tensor must be contiguous, and two names cannot share one tensor: -clone a tied `lm_head` before saving. - -### Checking a file - -Load it. A missing or extra tensor throws `CHECKPOINT_MISMATCH` naming it, -before any weight downloads; a wrong shape throws the same code with the -shapes. See [Error handling](/zoo/error-handling/) for how to catch it. - -```ts twoslash -import { createEmbedder, isRunntimeError, models } from 'runntime/zoo'; -// ---cut--- -try { - await createEmbedder({ - ...models.embedding.ALL_MINILM_L6_V2.DEFAULT, - modelPath: '/models/minilm.safetensors', - }); -} catch (err) { - // minilm state dict: missing tensor 'encoder.layer.0.attention.self.query.weight' - if (isRunntimeError(err, 'CHECKPOINT_MISMATCH')) console.error(err.message); - throw err; +```ts + +// ruNNtime +class Block extends Module { + query = new Linear(384, 384); } -``` -To see what a file holds without loading it, read its header in Python: +class Encoder extends Module { + layer = new ModuleList(Array.from({ length: 6 }, () => new Block())); +} -```bash -uv run --with safetensors --with numpy python -c " -from safetensors import safe_open -with safe_open('path/to/model.safetensors', 'np') as f: - print(f.metadata()) - for name in f.keys(): - tensor = f.get_slice(name) - print(name, tensor.get_dtype(), tensor.get_shape()) -" +// layer.0.query.weight, layer.0.query.bias, layer.1.query.weight, ... ``` -## Quantizing +When ruNNtime loads a file, it looks up each of its weights by that name and +uploads it to the GPU. So a file saved straight from the PyTorch model +works: no renaming, no reshaping, no conversion script. -`runntime quantize` turns a float file into an int8 or int4 one. It lives in -the repository under `tools/python` and runs with -[uv](https://docs.astral.sh/uv/), which sets up Python and NumPy on the -first run. Paths are relative to where you run it. +Weights can be stored as `F32`, `F16` or `BF16`. `F16` is half the download +of `F32` and doesn't change how the model computes. -```bash -git clone https://github.com/software-mansion/runntime -cd runntime/tools/python -uv run --no-dev runntime quantize path/to/model.safetensors path/to/model.int8.safetensors --bits 8 --f16 --drop 'pooler.*' -``` +If a weight is missing or has the wrong shape, loading stops with a +`CHECKPOINT_MISMATCH` error that names it (see +[Error handling](/zoo/error-handling/)). -The source is a `.safetensors` file or a Hugging Face model folder, sharded -or not. The output is one file. +## Getting a weights file -- `--bits 8` or `--bits 4`. Required. -- `--group-size N`: inputs that share one scale, a multiple of 4. Default 32. -- `--include GLOB`, repeatable: the tensors to quantize. Default `*.weight`. -- `--exclude GLOB`, repeatable: the tensors to keep as stored. Default - `*embed*`, `*pos_emb*` and `lm_head.*`: the model reads token and - position embeddings as float lookup tables, and `lm_head` often shares - the embedding's weights. -- `--f16`: store every tensor that stays float as `F16`. Without it they are - copied as stored. A value beyond the f16 range stops the run. -- `--drop GLOB`, repeatable: leave tensors out of the output, like the - `pooler` above, which the embedding model never reads. - -Passing `--include` or `--exclude` replaces that option's defaults, so -repeat the ones you want to keep. - -A tensor is quantized when it matches `--include`, matches no `--exclude`, -is float and 2D, and has a shape the quantized matmul can tile. Every other -tensor is copied as it is, so the output always loads. - -The tool prints one line per tensor: +Pick the case that fits: -```text -quantized encoder.layer.0.attention.self.query.weight [384, 384]: max error 0.00302, mean 0.000425 -skipped encoder.layer.0.attention.self.query.bias [384]: not matched by --include -skipped encoder.layer.0.output.LayerNorm.weight [384]: 1D, needs [out, in] -skipped embeddings.word_embeddings.weight [30522, 384]: excluded by *embed* -dropped pooler.dense.weight [384, 384]: matched by pooler.* -``` +- **The model is on Hugging Face and has one `model.safetensors`.** + Download that file and load it. If a name doesn't match, the error says + which one. +- **You fine-tuned it with `transformers`, or want a smaller `F16` copy.** + Save it with `save_pretrained`, as one file: + + ```bash + uv run --with transformers --with torch python -c " + from transformers import AutoModel + import torch + model = AutoModel.from_pretrained('path/to/your-model', dtype=torch.float16) + model.save_pretrained('export', max_shard_size='100GB') # one file, not shards + " + ``` + +- **You trained it with your own PyTorch code.** Save the state dict: + `safetensors.torch.save_file(model.state_dict(), 'model.safetensors')`. + +## Loading it into your own model + +Write the model in ruNNtime so it gives the same names, like the `Encoder` +above: + +- Name each field after the PyTorch attribute: `self.query = nn.Linear(...)` + becomes `query = new nn.Linear(...)`. +- Use `nn.ModuleList` where PyTorch does. It adds the index to the name. +- Give each layer the same sizes. -The error is how far the quantized weights are from the float ones. A -tensor whose error stands out from the rest is a candidate for `--exclude`. -Read the `skipped` lines too: `*embed*` also matches a layer with `embed` in -its name, like a projection after an embedding, which then needs your own -`--exclude` list to get quantized. The run fails when no tensor was -quantized. +Then load the file: -For all-MiniLM-L6-v2 from its 91 MB `F32` export, the output is 59 MB with -`--bits 8`, 35 MB with `--bits 8 --f16` and 30 MB with `--bits 4 --f16`. The -embedding table is half of this model, so `--f16` matters more than the -bits. +```ts +const model = new Encoder(); +await model.loadStateDict('/models/model.safetensors'); +``` + +## Quantization (optional) -### What the file holds +Full / half-precision files work as is. Optionally, you can quantize your model which can reduce the download size, memory usage, and can make the inference faster. -Each quantized weight is stored as int8 or int4 with one f16 scale per group -of `--group-size` inputs, and the file header records the bits and group -size. Every other tensor stays float. Only the weights of Linear layers load -quantized, which is why the default `--exclude` keeps embedding tables float. +**How the runtime recognizes it.** A quantized file has a `runntime.quant` +entry in its header metadata with the bits and group size. Each quantized +weight is stored as `I8` (int8) or `U8` (two int4 values per byte), with +its scales next to it under the same name plus `_scale`. Every other tensor +is a normal float tensor. An int tensor in a file without `runntime.quant` +fails to load. -### Picking the setting +**Which layers are quantized.** Only Linear weights. Embedding tables, +biases and norms stay float, because the model reads them as float. -Measured on the all-MiniLM-L6-v2 weights against the float model, 20 -search queries, groups of 32: the worst cosine similarity of an embedding -is 0.9998 with int8 and 0.94 with int4, and int4 changes the top 3 results -of 8 queries. int8 in groups of 32 is the safe pick and the default. int4 -costs quality: try it only when the download size matters more, and check -the output on your own data. +**Making a file.** `runntime quantize` lives in the repository under +`tools/python` and runs with [uv](https://docs.astral.sh/uv/): -## Quantization-aware training +```bash +git clone https://github.com/software-mansion/runntime +cd runntime/tools/python +uv run --no-dev runntime quantize model.safetensors model.int8.safetensors --bits 8 --f16 +``` -A QAT checkpoint saved as float, the "unquantized" form, quantizes like any -other. The tool's grid is symmetric int8 or int4, one scale per group of -inputs, the scale being the group's largest magnitude over 127 or 7. A model -trained on that grid, like one trained for ggml's `Q8_0` with -`--bits 8 --group-size 32`, keeps what it learned; another recipe gets -re-rounded. Activations stay float. +- `--bits 8` or `--bits 4`. Required. +- `--f16` stores the tensors that stay float as `F16`. +- `--group-size N`: how many inputs share one scale. Default 32. +- `--include` / `--exclude GLOB`: which tensors to quantize. The defaults + quantize `*.weight` and skip embeddings and `lm_head`. + +The tool prints one line per tensor with its error against the float +weights, or why it was skipped. + +**Picking a setting.** On all-MiniLM-L6-v2, int8 keeps every embedding at +cosine similarity 0.9998 or higher against the float model; int4 drops to +0.94 and changes the top 3 search results for 8 of 20 queries. Use int8. +Try int4 only when the download size matters more, and check it on your +own data. + +A quantization-aware trained checkpoint quantizes the same way if it was +trained on symmetric int8 or int4 with one scale per group of inputs. From 0a63b46f408e6aa8424cc70d32d5ab5d09d08656 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Thu, 12 Nov 2026 15:28:32 +0100 Subject: [PATCH 11/14] docs(docs): move Exporting weights to an unlisted core section until runntime core is published --- apps/docs/astro.config.mjs | 6 +++--- .../src/content/docs/{zoo => core}/exporting-weights.mdx | 3 +++ apps/docs/src/content/docs/zoo/embedding.mdx | 3 +-- 3 files changed, 7 insertions(+), 5 deletions(-) rename apps/docs/src/content/docs/{zoo => core}/exporting-weights.mdx (97%) diff --git a/apps/docs/astro.config.mjs b/apps/docs/astro.config.mjs index aeaa543e..e139fb50 100644 --- a/apps/docs/astro.config.mjs +++ b/apps/docs/astro.config.mjs @@ -158,7 +158,6 @@ export default defineConfig({ label: 'Guides', items: [ { label: 'Error handling', slug: 'zoo/error-handling' }, - { label: 'Exporting weights', slug: 'zoo/exporting-weights' }, { label: 'Migrating from transformers.js', slug: 'zoo/migrating-from-transformers-js', @@ -175,8 +174,9 @@ export default defineConfig({ }, ], { - // The splash landing page belongs to no topic. - exclude: ['/'], + // The splash landing page belongs to no topic, and the core pages stay + // unlisted until runntime core is published. + exclude: ['/', '/core/**'], // The generated index pages sit outside the sidebar groups. topics: { zoo: ['/api/**'] }, }, diff --git a/apps/docs/src/content/docs/zoo/exporting-weights.mdx b/apps/docs/src/content/docs/core/exporting-weights.mdx similarity index 97% rename from apps/docs/src/content/docs/zoo/exporting-weights.mdx rename to apps/docs/src/content/docs/core/exporting-weights.mdx index 43440571..facf0195 100644 --- a/apps/docs/src/content/docs/zoo/exporting-weights.mdx +++ b/apps/docs/src/content/docs/core/exporting-weights.mdx @@ -1,6 +1,9 @@ --- title: Exporting weights description: Run your own weights in ruNNtime, as float or quantized to int8 or int4. +# Hidden until runntime core is published: no production build, no sidebar, no search. +draft: true +pagefind: false --- import WeightsPath from '../../../components/WeightsPath.astro'; diff --git a/apps/docs/src/content/docs/zoo/embedding.mdx b/apps/docs/src/content/docs/zoo/embedding.mdx index 8993126c..3747d5de 100644 --- a/apps/docs/src/content/docs/zoo/embedding.mdx +++ b/apps/docs/src/content/docs/zoo/embedding.mdx @@ -194,8 +194,7 @@ await createEmbedder({ ``` - **modelPath** - the weights, a `model.safetensors` file. The model is - read from the file. To make your own, see - [Exporting weights](/zoo/exporting-weights/). + read from the file. - **visionPath** - the weights for images and video. `null` skips the download, and images and video become an error. - **audioPath** - the same for audio. From a4ca2780b985b9863495ab22f6cb2016e8e431f7 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Thu, 12 Nov 2026 15:33:25 +0100 Subject: [PATCH 12/14] docs(docs): say runntime.quant bits and groupSize are optional and that Conv1d and ConvTranspose1d load quantized --- .../src/content/docs/core/exporting-weights.mdx | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/docs/src/content/docs/core/exporting-weights.mdx b/apps/docs/src/content/docs/core/exporting-weights.mdx index facf0195..dbc69df2 100644 --- a/apps/docs/src/content/docs/core/exporting-weights.mdx +++ b/apps/docs/src/content/docs/core/exporting-weights.mdx @@ -119,14 +119,18 @@ Full / half-precision files work as is. Optionally, you can quantize your model **How the runtime recognizes it.** A quantized file has a `runntime.quant` -entry in its header metadata with the bits and group size. Each quantized -weight is stored as `I8` (int8) or `U8` (two int4 values per byte), with -its scales next to it under the same name plus `_scale`. Every other tensor +entry in its header metadata. Each quantized weight is stored as `I8` +(int8) or `U8` (two int4 values per byte), with its scales next to it under +the same name plus `_scale`. The entry can set the bits and group size for +the whole file; without them, each weight takes its bits from its dtype and +its group size from the shape of its scales. Every other tensor is a normal float tensor. An int tensor in a file without `runntime.quant` fails to load. -**Which layers are quantized.** Only Linear weights. Embedding tables, -biases and norms stay float, because the model reads them as float. +**Which layers are quantized.** `Linear`, `Conv1d` and `ConvTranspose1d` +weights. Embedding tables, biases and norms stay float, because the model +reads them as float. `runntime quantize` only quantizes 2D tensors, so +today it covers Linear weights and leaves conv weights float. **Making a file.** `runntime quantize` lives in the repository under `tools/python` and runs with [uv](https://docs.astral.sh/uv/): From ade49482a48a732fbf20838d9e0bad96111bcc6d Mon Sep 17 00:00:00 2001 From: chmjkb Date: Thu, 12 Nov 2026 15:39:12 +0100 Subject: [PATCH 13/14] docs(docs): drop a blank line Prettier rejects in the Exporting weights sample --- apps/docs/src/content/docs/core/exporting-weights.mdx | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/docs/src/content/docs/core/exporting-weights.mdx b/apps/docs/src/content/docs/core/exporting-weights.mdx index dbc69df2..979eeee9 100644 --- a/apps/docs/src/content/docs/core/exporting-weights.mdx +++ b/apps/docs/src/content/docs/core/exporting-weights.mdx @@ -49,7 +49,6 @@ class Encoder(nn.Module): ``` ```ts - // ruNNtime class Block extends Module { query = new Linear(384, 384); From 959b8501e47061409b6288d62aa9cf026334b1e4 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Fri, 13 Nov 2026 14:58:24 +0100 Subject: [PATCH 14/14] docs(docs): say that unread tensors only warn unless strict, and list the quantizer's --drop flag --- apps/docs/src/content/docs/core/exporting-weights.mdx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apps/docs/src/content/docs/core/exporting-weights.mdx b/apps/docs/src/content/docs/core/exporting-weights.mdx index 979eeee9..a5cab8ce 100644 --- a/apps/docs/src/content/docs/core/exporting-weights.mdx +++ b/apps/docs/src/content/docs/core/exporting-weights.mdx @@ -72,6 +72,10 @@ If a weight is missing or has the wrong shape, loading stops with a `CHECKPOINT_MISMATCH` error that names it (see [Error handling](/zoo/error-handling/)). +A tensor in the file that the model doesn't read, like the pooler in a plain +Hugging Face checkpoint, is skipped with a console warning. Pass +`{ strict: true }` to `loadStateDict` to make it the same error. + ## Getting a weights file Pick the case that fits: @@ -145,6 +149,8 @@ uv run --no-dev runntime quantize model.safetensors model.int8.safetensors --bit - `--group-size N`: how many inputs share one scale. Default 32. - `--include` / `--exclude GLOB`: which tensors to quantize. The defaults quantize `*.weight` and skip embeddings and `lm_head`. +- `--drop GLOB`: tensors to leave out of the file, like a pooler the model + never reads. The tool prints one line per tensor with its error against the float weights, or why it was skipped.