diff --git a/apps/docs/src/components/QuantFile.astro b/apps/docs/src/components/QuantFile.astro
index 7ecf7d67..3d388bbb 100644
--- a/apps/docs/src/components/QuantFile.astro
+++ b/apps/docs/src/components/QuantFile.astro
@@ -4,14 +4,13 @@
* tensors around them. Names are all-MiniLM-L6-v2's, shortened; the floats
* are F16 as `--f16` stores them. */
const rows = [
- { name: '…word_embeddings.weight', dtype: 'F16', shape: '[30522, 384]', loads: 'float' },
{ name: '…query.weight', dtype: 'I8', shape: '[384, 384]', loads: 'int8 weight', quant: true },
{ name: '…query.weight_scale', dtype: 'F16', shape: '[384, 12]', loads: 'its scales', quant: true },
{ name: '…query.bias', dtype: 'F16', shape: '[384]', loads: 'float' },
{ name: '…LayerNorm.weight', dtype: 'F16', shape: '[384]', loads: 'float' },
];
const rowHeight = 30;
-const top = 84;
+const top = 66;
const height = top + rows.length * rowHeight + 14;
---
@@ -23,9 +22,8 @@ const height = top + rows.length * rowHeight + 14;
>
-
-
-
+
+
{
rows.map((row, i) => {
const y = top + i * rowHeight + 12;
diff --git a/apps/docs/src/content/docs/core/exporting-weights.mdx b/apps/docs/src/content/docs/core/exporting-weights.mdx
index a5cab8ce..2973d89d 100644
--- a/apps/docs/src/content/docs/core/exporting-weights.mdx
+++ b/apps/docs/src/content/docs/core/exporting-weights.mdx
@@ -124,16 +124,31 @@ Full / half-precision files work as is. Optionally, you can quantize your model
**How the runtime recognizes it.** A quantized file has a `runntime.quant`
entry in its header metadata. Each quantized weight is stored as `I8`
(int8) or `U8` (two int4 values per byte), with its scales next to it under
-the same name plus `_scale`. The entry can set the bits and group size for
-the whole file; without them, each weight takes its bits from its dtype and
+the same name plus `_scale`. Each weight takes its bits from its dtype and
its group size from the shape of its scales. Every other tensor
is a normal float tensor. An int tensor in a file without `runntime.quant`
fails to load.
-**Which layers are quantized.** `Linear`, `Conv1d` and `ConvTranspose1d`
-weights. Embedding tables, biases and norms stay float, because the model
-reads them as float. `runntime quantize` only quantizes 2D tensors, so
-today it covers Linear weights and leaves conv weights float.
+**What is in the file.** `runntime quantize` quantizes every float tensor
+named `*.weight` that has 2 or more dims: Linear and conv weights. Biases
+and norms stay float, and so do embedding tables and `lm_head`, because
+quantizing them usually costs too much quality. It stores each quantized
+tensor as rows, `[first dim, everything else]`, so a `Conv1d` weight
+`[out, in, k]` becomes `[out, in·k]`, and groups of `--group-size` values
+run along that row. The group size has to divide the row: `in` for a
+Linear, `in·k` for a conv. A tensor it doesn't divide stays float. The
+runtime loads each tensor back into its layer's shape.
+
+**Which layers run quantized.** `Linear`, `Conv1d` and `ConvTranspose1d`
+run on the int weights. Any other quantized tensor, like a `Conv2d` weight
+or an embedding table you chose to quantize, is turned back into floats at
+load. It keeps the rounding of its ints, so the download is smaller but
+the model is no faster and uses no less memory than the float one. A
+table kept on the CPU takes 4 bytes per value.
+
+A `Linear` or conv weight whose shape the quantized kernels can't take
+also loads as float, and a console warning names it. To stay on the
+quantized kernels, keep `--group-size` a multiple of 4.
**Making a file.** `runntime quantize` lives in the repository under
`tools/python` and runs with [uv](https://docs.astral.sh/uv/):
@@ -146,9 +161,10 @@ uv run --no-dev runntime quantize model.safetensors model.int8.safetensors --bit
- `--bits 8` or `--bits 4`. Required.
- `--f16` stores the tensors that stay float as `F16`.
-- `--group-size N`: how many inputs share one scale. Default 32.
-- `--include` / `--exclude GLOB`: which tensors to quantize. The defaults
- quantize `*.weight` and skip embeddings and `lm_head`.
+- `--group-size N`: how many values of a row share one scale. Default 32.
+- `--include GLOB`: which tensors to quantize. Default `*.weight`.
+- `--exclude GLOB`: tensors to keep float, like a layer that loses too
+ much. Replaces the default `*embed*`, `*pos_emb*` and `lm_head.*`.
- `--drop GLOB`: tensors to leave out of the file, like a pooler the model
never reads.
@@ -156,10 +172,13 @@ The tool prints one line per tensor with its error against the float
weights, or why it was skipped.
**Picking a setting.** On all-MiniLM-L6-v2, int8 keeps every embedding at
-cosine similarity 0.9998 or higher against the float model; int4 drops to
-0.94 and changes the top 3 search results for 8 of 20 queries. Use int8.
-Try int4 only when the download size matters more, and check it on your
-own data.
+cosine similarity 0.9997 or higher against the float model. int4 drops as
+low as 0.93 and reorders the top 3 search results for 12 of 20 queries,
+where int8 reorders them for 2. Use int8. Try int4 only when the download
+size matters more, and check it on your own data.
+
+Use int8 for models built on 2D convolutions. int4 breaks the outputs of
+MobileNetV4, DepthART and YOLO26.
A quantization-aware trained checkpoint quantizes the same way if it was
trained on symmetric int8 or int4 with one scale per group of inputs.
diff --git a/packages/runntime/src/core/index.ts b/packages/runntime/src/core/index.ts
index 98cd1af9..25bb3e14 100644
--- a/packages/runntime/src/core/index.ts
+++ b/packages/runntime/src/core/index.ts
@@ -151,6 +151,7 @@ export { foldBnIntoConv } from './weights/foldBn.ts';
export {
concatRows,
dequantize,
+ inTorchShape,
permuteRows,
reshapeWeight,
scaleWeight,
diff --git a/packages/runntime/src/core/nn/layers.ts b/packages/runntime/src/core/nn/layers.ts
index 58b81b6e..a5a11bdd 100644
--- a/packages/runntime/src/core/nn/layers.ts
+++ b/packages/runntime/src/core/nn/layers.ts
@@ -20,10 +20,7 @@ import {
snake,
} from '../graph/ops.ts';
import { matrix, type Value } from '../graph/value.ts';
-import {
- MATMUL_QUANT_TILED_BK,
- matmulQuantTiledEligible,
-} from '../kernels/matmul/matmulQuantTiled.ts';
+import { matmulQuantTiledEligible } from '../kernels/matmul/matmulQuantTiled.ts';
import {
convToTapMajor,
f16ToF32,
@@ -35,7 +32,12 @@ import {
} from '../weights/convert.ts';
import { foldBnIntoConv } from '../weights/foldBn.ts';
import { packQuantW } from '../weights/quantPack.ts';
-import { derivedTensor, type LazyStateDict, type QuantTensor } from '../weights/safetensors.ts';
+import {
+ derivedTensor,
+ type LazyStateDict,
+ type LazyTensor,
+ type QuantTensor,
+} from '../weights/safetensors.ts';
import { dequantize, readFloats, transposeWeight } from '../weights/transforms.ts';
import { tensor } from '../gpu/buffers.ts';
import { CpuParameter, Module, Parameter, type AnyModule } from './module.ts';
@@ -138,6 +140,14 @@ function loadQuantWeight(
return scale;
}
+/** A quantized weight its layer's quantized kernel cannot run, as its
+ * dequantized f32 values in torch's `shape`, and noted on `sd` for
+ * loadStateDict's warning. */
+function asFloat(sd: LazyStateDict, t: QuantTensor, shape: readonly number[]): LazyTensor {
+ (sd.quantizedAsFloat ??= []).push(t.name);
+ return dequantize(t, shape);
+}
+
/** `t`'s scales in f32, as stored. */
async function scalesF32(t: QuantTensor): Promise {
const data = await t.scales();
@@ -161,7 +171,8 @@ async function scaleColumns(t: QuantTensor, groups: number, n: number): Promise<
* A quantized checkpoint weight loads as quantW words with its f32 scales in
* `weightScale` [in / groupSize, out], and forward() runs the quantized
* matmul. Only the checkpoint says which kind it is, so transformStateDict
- * retypes the slot at load.
+ * retypes the slot at load. A weight the quantized matmul cannot run loads
+ * as its dequantized floats.
*
* `addend` and `activation` fold a residual add and a following tanh or GELU
* into the matmul epilogue, as act(x·W + bias) + addend. */
@@ -202,33 +213,25 @@ export class Linear extends Module {
`expected torch layout [${this.outFeatures}, ${this.inFeatures}]`,
);
}
- if (tensor.kind === 'quant') {
- this.loadQuantized(sd, prefix, name, tensor);
+ if (tensor.kind !== 'quant') {
+ sd.tensors.set(name, transposeWeight(tensor));
return;
}
- sd.tensors.set(name, transposeWeight(tensor));
- }
- /** Loads `name` as quantW words [in, out] with its scales
+ const { bits, groupSize } = tensor;
+ // The quantized matmul reads whole tiles of 4 columns and 8 rows, groups
+ // of whole 4-row chunks, and packs int4 columns 8 to a word.
+ const fits =
+ matmulQuantTiledEligible(this.inFeatures, this.outFeatures, groupSize) &&
+ this.outFeatures % (32 / bits) === 0;
+ if (fits) this.loadQuantized(sd, prefix, tensor);
+ else sd.tensors.set(name, transposeWeight(asFloat(sd, tensor, tensor.shape)));
+ }
+ /** Loads the weight as quantW words [in, out] with its scales
* [in / groupSize, out]. */
- private loadQuantized(
- sd: LazyStateDict,
- prefix: string,
- name: string,
- tensor: QuantTensor,
- ): void {
+ private loadQuantized(sd: LazyStateDict, prefix: string, tensor: QuantTensor): void {
const { bits, groupSize } = tensor;
const k = this.inFeatures;
const n = this.outFeatures;
- // The quantized matmul reads whole tiles of 4 columns and 8 rows and
- // packs int4 columns 8 to a word.
- if (!matmulQuantTiledEligible(k, n, groupSize) || n % (32 / bits) !== 0) {
- throw new RunntimeError(
- 'CHECKPOINT_MISMATCH',
- `Linear: ${name} is int${bits} in groups of ${groupSize}, which needs in features ` +
- `a multiple of ${MATMUL_QUANT_TILED_BK} and of the group size, out features a ` +
- `multiple of ${32 / bits} and a group size a multiple of 4, got [out, in] = [${n}, ${k}]`,
- );
- }
this.quantization = { bits, groupSize };
const groups = k / groupSize;
this.weightScale = loadQuantWeight(
@@ -350,7 +353,7 @@ export class Conv1d extends Module {
this.loadQuantized(sd, prefix, name, tensor, n);
return;
}
- tensor = dequantize(tensor, [cOut, cIn, kernel]);
+ tensor = asFloat(sd, tensor, [cOut, cIn, kernel]);
}
// Without this a weight with equal in/out channels loads scrambled.
if (
@@ -528,7 +531,7 @@ export class ConvTranspose1d extends Module {
if (cOut % 4 === 0 && n % (32 / w.bits) === 0 && quantConvFits(cIn, taps, n)) {
this.loadQuantized(sd, prefix, w);
} else {
- w = dequantize(w, [cIn, cOut, k]);
+ w = asFloat(sd, w, [cIn, cOut, k]);
}
}
if (w && w.kind !== 'quant') {
diff --git a/packages/runntime/src/core/nn/loadStateDict.ts b/packages/runntime/src/core/nn/loadStateDict.ts
index 4d661883..f514af18 100644
--- a/packages/runntime/src/core/nn/loadStateDict.ts
+++ b/packages/runntime/src/core/nn/loadStateDict.ts
@@ -8,7 +8,12 @@ import { defaultRoot } from '../gpu/context.ts';
import { inGpuErrorScopes } from '../gpu/errorScopes.ts';
import { materialized, type Value, type ValueMeta } from '../graph/value.ts';
import { bf16ToF16Bits, f32ArrayToF16Bits } from '../weights/convert.ts';
-import type { DerivedTensor, FileTensor, LazyStateDict } from '../weights/safetensors.ts';
+import {
+ derivedTensor,
+ type DerivedTensor,
+ type FileTensor,
+ type LazyStateDict,
+} from '../weights/safetensors.ts';
import { CpuParameter, type AnyModule } from './module.ts';
export type UploadFn = (data: Float32Array | Uint32Array | Uint16Array, shape: ValueMeta) => Value;
@@ -120,6 +125,13 @@ async function bindStateDict(
}
};
visit(model, '');
+ if (sd.quantizedAsFloat?.length) {
+ console.warn(
+ 'loadStateDict: quantized weights loaded as float, the quantized kernel of their layer ' +
+ `cannot run their shape or group size: ${sd.quantizedAsFloat.join(', ')}`,
+ );
+ sd.quantizedAsFloat = undefined;
+ }
const params = [...model.namedParameters()];
const paramNames = new Set(params.map(([n]) => n));
@@ -143,15 +155,20 @@ async function bindStateDict(
const total = sd.totalBytes();
let done = 0;
for (const [name, p] of params) {
- const t = sd.tensors.get(name)!;
+ let t = sd.tensors.get(name)!;
// Linear's and the convs' hooks turn a quantized weight into quantW
- // words, so a QuantTensor still here is bound to something else.
+ // words, so a QuantTensor still here is bound to something no quantized
+ // kernel reads: an embedding table, a norm, a plain parameter.
if (t.kind === 'quant') {
- throw new RunntimeError(
- 'CHECKPOINT_MISMATCH',
- `loadStateDict: '${name}' is quantized (int${t.bits}), but only a Linear, Conv1d or ` +
- `ConvTranspose1d weight loads quantized and this parameter holds ${p.shape.dtype}`,
- );
+ if (p.shape.dtype === 'quantW') {
+ throw new RunntimeError(
+ 'CHECKPOINT_MISMATCH',
+ `loadStateDict: '${name}' is quantized (int${t.bits}), but this quantW parameter ` +
+ 'takes packed words, which only its layer makes',
+ );
+ }
+ const quant = t;
+ t = derivedTensor(quant.shape, () => quant.f32(), quant.byteLength);
}
const numel = t.shape.reduce((a, b) => a * b, 1);
// A quantW slot counts u32 words under logical dims. Privacy-filter's
diff --git a/packages/runntime/src/core/weights/safetensors.ts b/packages/runntime/src/core/weights/safetensors.ts
index ac9599f8..618a17a2 100644
--- a/packages/runntime/src/core/weights/safetensors.ts
+++ b/packages/runntime/src/core/weights/safetensors.ts
@@ -317,8 +317,9 @@ export interface DerivedTensor extends LazyTensorBase {
/** Symmetric int weights with one scale per `groupSize` values along the last
* dim: value = int · scale of its group. `shape` is logical, one element per
- * int even for int4, and `byteLength` covers the ints and the scales. The
- * float accessors reject with CHECKPOINT_MISMATCH. */
+ * int even for int4, and `byteLength` covers the ints and the scales. f32()
+ * gives the dequantized values, for a reader that only takes floats; the
+ * readers of stored bits reject with CHECKPOINT_MISMATCH. */
export interface QuantTensor extends LazyTensorBase {
kind: 'quant';
/** What the tensor is, for error messages: its name in the file, or the
@@ -338,6 +339,10 @@ export type LazyTensor = FileTensor | DerivedTensor | QuantTensor;
export interface LazyStateDict {
metadata: Record;
tensors: Map;
+ /** Names of the quantized weights a Linear, Conv1d or ConvTranspose1d
+ * loaded as float because its quantized kernel cannot run them. The
+ * layers' hooks fill it and loadStateDict warns about them. */
+ quantizedAsFloat?: string[];
totalBytes(): number;
}
@@ -410,6 +415,19 @@ function fileTensor(name: string, info: SafeTensorInfo, source: RangeSource): Fi
};
}
+/** The values `ints` stand for in f32: each int times the scale of its group
+ * of `groupSize`. */
+export function dequantizeInts(
+ ints: Int8Array,
+ scales: Float32Array | Uint16Array,
+ groupSize: number,
+): Float32Array {
+ const f32 = scales instanceof Float32Array ? scales : f16ToF32(scales);
+ const out = new Float32Array(ints.length);
+ for (let i = 0; i < out.length; i++) out[i] = ints[i]! * f32[Math.floor(i / groupSize)]!;
+ return out;
+}
+
/** A QuantTensor over `ints` and `scales`. Like derivedTensor it holds
* nothing between reads. */
export function quantTensor(
@@ -422,10 +440,14 @@ export function quantTensor(
Promise.reject(
new RunntimeError(
'CHECKPOINT_MISMATCH',
- `tensor ${parts.name} is quantized (int${parts.bits}, groups of ${parts.groupSize}) and has no float reading`,
+ `tensor ${parts.name} is quantized (int${parts.bits}, groups of ${parts.groupSize}) and stores no float bits`,
),
);
- return { kind: 'quant', ...parts, f32: refuse, halfWords: refuse, words: refuse, u16: refuse };
+ const f32 = async () => {
+ const [ints, scales] = await Promise.all([parts.ints(), parts.scales()]);
+ return dequantizeInts(ints, scales, parts.groupSize);
+ };
+ return { kind: 'quant', ...parts, f32, halfWords: refuse, words: refuse, u16: refuse };
}
/** Unpacks int4 bytes, two values each with the low nibble first, into one
diff --git a/packages/runntime/src/core/weights/transforms.ts b/packages/runntime/src/core/weights/transforms.ts
index ab28f7ea..060888d9 100644
--- a/packages/runntime/src/core/weights/transforms.ts
+++ b/packages/runntime/src/core/weights/transforms.ts
@@ -11,10 +11,17 @@
* A QuantTensor source gives a QuantTensor, except in `dequantize`: the
* helpers move its ints and scales together. Whatever would split a scale
* group (a column split off the group grid, a reshape of the last dim, a
- * transpose) throws CHECKPOINT_MISMATCH. */
+ * transpose) throws CHECKPOINT_MISMATCH; a hook that needs one calls
+ * `inTorchShape` first. */
import { RunntimeError } from '../error.ts';
import { f16ToF32 } from './convert.ts';
-import { derivedTensor, quantTensor, type LazyTensor, type QuantTensor } from './safetensors.ts';
+import {
+ dequantizeInts,
+ derivedTensor,
+ quantTensor,
+ type LazyTensor,
+ type QuantTensor,
+} from './safetensors.ts';
/** f32 values, or f16 halfwords. */
type Floats = Float32Array | Uint16Array;
@@ -27,12 +34,13 @@ const numel = (shape: readonly number[]): number => shape.reduce((a, b) => a * b
const mismatch = (helper: string, why: string) =>
new RunntimeError('CHECKPOINT_MISMATCH', `${helper}: ${why}`);
-/** Reads `t` as f16 halfwords when it holds F16 and as f32 otherwise. Throws
+/** Reads `t` as f16 halfwords when it holds F16 and as f32 otherwise, a
+ * QuantTensor as its dequantized values. Throws
* CHECKPOINT_MISMATCH, naming `helper`, on packed u32 words or on a derived
* tensor whose data does not fill its shape. A file tensor's length is
* checked by the header parser. */
export async function readFloats(helper: string, t: LazyTensor): Promise {
- if (t.kind === 'quant') throw mismatch(helper, `${t.name} is quantized, not float`);
+ if (t.kind === 'quant') return t.f32();
if (t.kind === 'file') return t.dtype === 'F16' ? t.u16() : t.f32();
const data = await t.data();
if (data instanceof Uint32Array) throw mismatch(helper, 'source holds packed u32 words');
@@ -395,9 +403,18 @@ export function dequantize(t: QuantTensor, shape: readonly number[] = t.shape):
[...shape],
async () => {
const [ints, scales] = await Promise.all([t.ints(), t.scales()]);
- const f32 = toF32(scales);
- return Float32Array.from(ints, (q, i) => q * f32[Math.floor(i / t.groupSize)]!);
+ return dequantizeInts(ints, scales, t.groupSize);
},
t.byteLength,
);
}
+
+/** `t` in torch's `shape`, for a hook that reshapes, transposes or splits it
+ * past dim 0. A file stores a quantized tensor as rows, [shape[0], everything
+ * else], with its scale groups along the row, so those steps would cut
+ * through the groups: it comes back as its dequantized floats in `shape`.
+ * Any other tensor comes back as it is. Throws CHECKPOINT_MISMATCH when a
+ * quantized tensor's element count is not `shape`'s. */
+export function inTorchShape(t: LazyTensor, shape: readonly number[]): LazyTensor {
+ return t.kind === 'quant' ? dequantize(t, shape) : t;
+}
diff --git a/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts b/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts
index 8e1d069d..0b7ed045 100644
--- a/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts
+++ b/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts
@@ -13,6 +13,7 @@ import {
concatRows,
derivedTensor,
f32ArrayToF16Bits,
+ inTorchShape,
permuteRows,
reshapeWeight,
scaleWeight,
@@ -175,7 +176,8 @@ export function transformEmbeddingGemma2VisionStateDict(
put('input_proj.weight', take('vision_tower.patch_embedder.input_proj.weight'));
// [2, positions, hidden]: the x table, then the y table.
- const [x, y] = splitRows(take('vision_tower.patch_embedder.position_embedding_table'), [1, 1]);
+ const table = take('vision_tower.patch_embedder.position_embedding_table');
+ const [x, y] = splitRows(inTorchShape(table, [2, positions, hidden]), [1, 1]);
put('pos_x.weight', reshapeWeight(x!, [positions, hidden]));
put('pos_y.weight', reshapeWeight(y!, [positions, hidden]));
@@ -232,7 +234,9 @@ export function transformEmbeddingGemma2AudioStateDict(
const [o, k, c] = [Math.floor(j / (9 * c0)), Math.floor(j / c0) % 9, j % c0];
return (o * c0 + c) * 9 + k;
});
- const conv1 = reshapeWeight(take(`${sub}layer1.conv.weight`), [c1 * c0 * 9]);
+ const conv1 = reshapeWeight(inTorchShape(take(`${sub}layer1.conv.weight`), [c1, c0, 3, 3]), [
+ c1 * c0 * 9,
+ ]);
put('conv1.weight', reshapeWeight(permuteRows(conv1, tapsMajor), [c1, 9 * c0]));
put('norm1.weight', take(`${sub}layer1.norm.weight`));
put('input_proj.weight', take(`${sub}input_proj_linear.weight`));
@@ -316,10 +320,8 @@ export function transformEmbeddingGemma2AudioStateDict(
put(`${L}.lconv.pre_layer_norm.scale`, take(`${C}.pre_layer_norm.weight`));
clipped(`${C}.linear_start`, `${L}.lconv.linear_start`);
// [hidden, 1, K] → [K, hidden].
- put(
- `${L}.lconv.conv`,
- transposeWeight(reshapeWeight(take(`${C}.depthwise_conv1d.weight`), [hidden, K])),
- );
+ const depthwise = inTorchShape(take(`${C}.depthwise_conv1d.weight`), [hidden, 1, K]);
+ put(`${L}.lconv.conv`, transposeWeight(reshapeWeight(depthwise, [hidden, K])));
put(`${L}.lconv.conv_norm.scale`, take(`${C}.conv_norm.weight`));
clipped(`${C}.linear_end`, `${L}.lconv.linear_end`);
for (const n of ['norm_pre_attn', 'norm_post_attn', 'norm_out']) {
diff --git a/packages/runntime/src/zoo/higgs/encoder.ts b/packages/runntime/src/zoo/higgs/encoder.ts
index 922c7498..dd30d1ab 100644
--- a/packages/runntime/src/zoo/higgs/encoder.ts
+++ b/packages/runntime/src/zoo/higgs/encoder.ts
@@ -18,6 +18,7 @@ import {
elu,
gatherRowsFrom,
gelu,
+ inTorchShape,
logitStats,
matrix,
mul,
@@ -188,10 +189,19 @@ class PositionConv extends nn.Module {
* conv checks the shape of its rows. */
private weight(sd: LazyStateDict, prefix: string): LazyTensor {
if (sd.tensors.has(at(prefix, 'conv.weight'))) return take(sd, prefix, 'conv.weight');
- const g = take(sd, prefix, 'conv.parametrizations.weight.original0');
- const v = take(sd, prefix, 'conv.parametrizations.weight.original1');
const { hidden, kernel, groups } = this;
const width = hidden / groups;
+ // The norm folds per tap, across the scale groups of a quantized tensor.
+ const g = inTorchShape(take(sd, prefix, 'conv.parametrizations.weight.original0'), [
+ 1,
+ 1,
+ kernel,
+ ]);
+ const v = inTorchShape(take(sd, prefix, 'conv.parametrizations.weight.original1'), [
+ hidden,
+ width,
+ kernel,
+ ]);
if (
v.shape.join() !== [hidden, width, kernel].join() ||
g.shape.join() !== [1, 1, kernel].join()
diff --git a/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts b/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts
index fd899902..4ec5ce73 100644
--- a/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts
+++ b/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts
@@ -30,6 +30,21 @@ function move(sd: LazyStateDict, from: string, to: string): void {
sd.tensors.set(to, take(sd, from));
}
+/** An expert tensor [E, …]. Throws CHECKPOINT_MISMATCH on a quantized one: a
+ * file stores it as rows [E, everything else], so splitting or folding it
+ * would mix the experts' values. */
+function takeExperts(sd: LazyStateDict, name: string): LazyTensor {
+ const t = take(sd, name);
+ if (t.kind === 'quant') {
+ throw checkpointMismatch(
+ 'privacy-filter state dict',
+ `'${name}' is quantized, and a quantized expert tensor cannot be split into experts: ` +
+ 'keep it float in the file',
+ );
+ }
+ return t;
+}
+
/** Folds every leading dim into rows: [E, K, H] → [E·K, H]. */
function foldRows(t: LazyTensor): LazyTensor {
const cols = t.shape[t.shape.length - 1]!;
@@ -61,13 +76,16 @@ export function transformHfPrivacyFilterStateDict(sd: LazyStateDict): void {
}
move(sd, `${src}.self_attn.sinks`, `${dst}.attn.sinks`);
- const [gluWeight, linWeight] = splitHalves(take(sd, `${src}.mlp.experts.gate_up_proj`));
+ const [gluWeight, linWeight] = splitHalves(takeExperts(sd, `${src}.mlp.experts.gate_up_proj`));
sd.tensors.set(`${dst}.mlp.gluWeight`, gluWeight);
sd.tensors.set(`${dst}.mlp.linWeight`, linWeight);
- const [gluBias, linBias] = splitHalves(take(sd, `${src}.mlp.experts.gate_up_proj_bias`));
+ const [gluBias, linBias] = splitHalves(takeExperts(sd, `${src}.mlp.experts.gate_up_proj_bias`));
sd.tensors.set(`${dst}.mlp.gluBias`, gluBias);
sd.tensors.set(`${dst}.mlp.linBias`, linBias);
- sd.tensors.set(`${dst}.mlp.outWeight`, foldRows(take(sd, `${src}.mlp.experts.down_proj`)));
+ sd.tensors.set(
+ `${dst}.mlp.outWeight`,
+ foldRows(takeExperts(sd, `${src}.mlp.experts.down_proj`)),
+ );
move(sd, `${src}.mlp.experts.down_proj_bias`, `${dst}.mlp.outBias`);
n++;
}
diff --git a/packages/runntime/test/core/nn/layers.gpu.test.ts b/packages/runntime/test/core/nn/layers.gpu.test.ts
index d9348923..78a940f7 100644
--- a/packages/runntime/test/core/nn/layers.gpu.test.ts
+++ b/packages/runntime/test/core/nn/layers.gpu.test.ts
@@ -1,8 +1,8 @@
-import { describe, it, onTestFinished } from 'vitest';
+import { describe, expect, it, onTestFinished, vi } from 'vitest';
import { toArray } from '../../../src/core/gpu/eval.ts';
import type { SlotAct } from '../../../src/core/graph/ops.ts';
import { matrix } from '../../../src/core/graph/value.ts';
-import { Conv1d, ConvTranspose1d, Linear } from '../../../src/core/nn/layers.ts';
+import { Conv1d, ConvTranspose1d, Embedding, Linear } from '../../../src/core/nn/layers.ts';
import {
bufferSource,
fromSafetensors,
@@ -26,8 +26,24 @@ interface Case {
/** Takes the tiled kernel, which stages the dequantized weight in f16
* workgroup memory for an f16 input. */
tiled?: boolean;
+ /** The bits at which the quantized matmul cannot take the weight, so it
+ * loads as its dequantized floats in the layer's dtype. */
+ float?: readonly (4 | 8)[];
}
+/** Watches console.warn for the rest of the test. Returns a reader of the
+ * loadStateDict warnings so far; TypeGPU warns there too. */
+function loadWarnings(): () => unknown[] {
+ const warn = vi.spyOn(console, 'warn');
+ onTestFinished(() => warn.mockRestore());
+ return () =>
+ warn.mock.calls
+ .map(([message]) => message as unknown)
+ .filter((message) => typeof message === 'string' && message.startsWith('loadStateDict:'));
+}
+
+const LOADED_AS_FLOAT = /^loadStateDict: quantized weights loaded as float.*: weight$/;
+
/** A Linear loaded from a quantized file matches a CPU loop of
* act(x·W + bias) + addend in f64, with W dequantized from the same ints as
* int · f16 scale. */
@@ -54,14 +70,18 @@ async function checkQuantizedLinear(c: Case, bits: 4 | 8, dtype: 'f32' | 'f16')
});
const sd = await fromSafetensors(bufferSource(file), { chunkBytes: 0 });
if (c.bias) sd.tensors.set('bias', memoryStateDict({ bias }).tensors.get('bias')!);
+ const warned = loadWarnings();
await layer.loadStateDict(sd, { root });
+ const float = c.float?.includes(bits) ?? false;
+ expect(layer.weight.shape.dtype).toBe(float ? dtype : 'quantW');
+ expect(warned()).toEqual(float ? [expect.stringMatching(LOADED_AS_FLOAT)] : []);
const x = upload(root, xs, { ...matrix(m, inF), dtype });
const call = c.addend ? { addend: upload(root, addend, { ...matrix(m, outF), dtype }) } : {};
const got = await toArray(layer.forward(x, call), ex);
const dequant = Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / groupSize)]!);
- const weight = dtype === 'f16' && c.tiled ? roundF16(dequant) : dequant;
+ const weight = dtype === 'f16' && (c.tiled || float) ? roundF16(dequant) : dequant;
const want = new Float64Array(m * outF);
for (let i = 0; i < m; i++) {
for (let o = 0; o < outF; o++) {
@@ -77,6 +97,22 @@ async function checkQuantizedLinear(c: Case, bits: 4 | 8, dtype: 'f32' | 'f16')
// output to half precision, 2^-11 relative.
const tol = dtype === 'f16' ? { atol: 1e-5 * inF, rtol: 1e-3 } : { atol: 1e-6 * inF, rtol: 1e-5 };
expectClose(got, want, tol);
+ if (!float) return;
+
+ // The fallback is the float layer: a Linear loaded from the dequantized
+ // weight takes the same route and gives the same bits.
+ const reference = new Linear(inF, outF, { bias: c.bias, activation: c.activation });
+ if (dtype === 'f16') reference.half(root);
+ onTestFinished(() => reference.dispose());
+ await reference.loadStateDict(
+ memoryStateDict({
+ weight: { data: dequant, shape: [outF, inF] },
+ ...(c.bias ? { bias } : {}),
+ }),
+ { root },
+ );
+ expect(reference.weight.shape.dtype).toBe(dtype);
+ expectClose(await toArray(reference.forward(x, call), ex), got, { atol: 0, rtol: 0 });
}
describe('Linear with a quantized weight', () => {
@@ -170,6 +206,100 @@ describe('Linear with a quantized weight', () => {
});
});
+describe('Linear with a quantized weight at the limits of the quantized matmul', () => {
+ // It takes in features a multiple of 8 and of the group size, out features
+ // a multiple of 4 (int8) or 8 (int4), and a group size a multiple of 4.
+ // Anything else runs the float matmul on the dequantized weight.
+ const base = { m: 3, bias: true, dtypes: ['f32', 'f16'] } as const;
+ const CASES: (Case & { name: string; dtypes: readonly ('f32' | 'f16')[] })[] = [
+ // quantized: in 8 % 8 == 0, out 8 % 8 == 0, groupSize 4 % 4 == 0
+ { ...base, name: 'the smallest that fits', inF: 8, outF: 8, groupSize: 4 },
+ // float: in 12 % 8 != 0
+ { ...base, name: 'in off the 8-row tile', inF: 12, outF: 8, groupSize: 4, float: [4, 8] },
+ // float: in 4 % 8 != 0
+ { ...base, name: 'in under one tile', inF: 4, outF: 8, groupSize: 4, float: [4, 8] },
+ // int8 quantized: out 4 % 4 == 0. int4 float: out 4 % 8 != 0
+ { ...base, name: 'out of one int8 word', inF: 8, outF: 4, groupSize: 4, float: [4] },
+ // int8 quantized: out 12 % 4 == 0. int4 float: out 12 % 8 != 0
+ { ...base, name: 'out off the int4 word', inF: 16, outF: 12, groupSize: 8, float: [4] },
+ // float: out 6 % 4 != 0
+ { ...base, name: 'out off the int8 word', inF: 16, outF: 6, groupSize: 8, float: [4, 8] },
+ // float: out 1
+ { ...base, name: 'one output', inF: 16, outF: 1, groupSize: 8, float: [4, 8] },
+ // quantized: groupSize 12 % 4 == 0 and 24 % 12 == 0
+ { ...base, name: 'groups of 12', inF: 24, outF: 8, groupSize: 12 },
+ // float: groupSize 6 % 4 != 0
+ { ...base, name: 'groups of 6', inF: 24, outF: 8, groupSize: 6, float: [4, 8] },
+ // float: groupSize 2 % 4 != 0
+ { ...base, name: 'groups of 2', inF: 16, outF: 8, groupSize: 2, float: [4, 8] },
+ // float: groupSize 1
+ { ...base, name: 'a scale per value', inF: 16, outF: 8, groupSize: 1, float: [4, 8] },
+ // float, with the epilogue the float matmul folds
+ {
+ ...base,
+ name: 'in off the tile with gelu and addend',
+ m: 37,
+ inF: 100,
+ outF: 30,
+ groupSize: 20,
+ addend: true,
+ activation: 'gelu',
+ float: [4, 8],
+ },
+ ];
+ it.for(
+ CASES.flatMap(({ dtypes, ...c }) =>
+ dtypes.flatMap((dtype) => ([4, 8] as const).map((bits) => ({ ...c, dtype, bits }))),
+ ),
+ )('$dtype int$bits $name: [$m, $inF] → $outF, groupSize $groupSize', async (c, ctx) => {
+ if (c.dtype === 'f16') requireFeature(ctx, root, 'shader-f16');
+ await checkQuantizedLinear(c, c.bits, c.dtype);
+ });
+});
+
+describe('Embedding with a quantized table', () => {
+ // No kernel reads a quantized lookup table, so it loads as its dequantized
+ // floats, without a warning: on the GPU in the model's dtype, on the CPU as
+ // f32.
+ const [vocab, dim, groupSize] = [10, 6, 3];
+ const ids = [9, 0, 4, 4];
+ it.for(
+ (['gpu', 'cpu'] as const).flatMap((device) =>
+ ([4, 8] as const).flatMap((bits) =>
+ (['f32', 'f16'] as const).map((dtype) => ({ device, bits, dtype })),
+ ),
+ ),
+ )('$device table, int$bits, $dtype', async ({ device, bits, dtype }, ctx) => {
+ if (dtype === 'f16') requireFeature(ctx, root, 'shader-f16');
+ const half = 2 ** (bits - 1);
+ const ints = Array.from(randomF32(vocab * dim, 1), (v) =>
+ Math.min(half - 1, Math.floor(v * half)),
+ );
+ const scales = roundF16(
+ randomF32((vocab * dim) / groupSize, 2).map((v) => (Math.abs(v) + 0.1) / half),
+ );
+ const table = new Embedding(vocab, dim, { device });
+ if (dtype === 'f16') table.half(root);
+ onTestFinished(() => table.dispose());
+ const file = quantFile(bits, groupSize, {
+ weight: { ints, scales: [...scales], shape: [vocab, dim] },
+ });
+ const warned = loadWarnings();
+ await table.loadStateDict(await fromSafetensors(bufferSource(file), { chunkBytes: 0 }), {
+ root,
+ });
+ expect(warned()).toEqual([]);
+ const got = await toArray(table.forward(ids), ex);
+
+ const dequant = Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / groupSize)]!);
+ // A GPU table is stored in the model's dtype; a CPU one stays f32 and its
+ // gathered rows round on upload.
+ const stored = dtype === 'f16' ? roundF16(dequant) : dequant;
+ const want = ids.flatMap((id) => [...stored.subarray(id * dim, (id + 1) * dim)]);
+ expectClose(got, want, { atol: 0, rtol: 0 });
+ });
+});
+
interface ConvCase {
name: string;
kind: 'conv' | 'transpose';
@@ -183,6 +313,9 @@ interface ConvCase {
groupSize: number;
bias: boolean;
activation?: 'gelu';
+ /** The quantized matmul cannot run this conv at any bits, so the weight
+ * loads as its dequantized floats. */
+ float?: boolean;
}
/** A conv loaded from a quantized file of its torch weight, [out, in·kernel]
@@ -209,10 +342,11 @@ async function checkQuantizedConv(c: ConvCase, bits: 4 | 8, dtype: 'f32' | 'f16'
c.kind === 'conv'
? new Conv1d(cIn, cOut, kernel, { stride, padding, bias: c.bias, activation: c.activation })
: new ConvTranspose1d(cIn, cOut, kernel, { stride, padding, bias: c.bias });
- const run = async (layer: Conv1d | ConvTranspose1d, sd: LazyStateDict) => {
+ const run = async (layer: Conv1d | ConvTranspose1d, sd: LazyStateDict, quantized = false) => {
if (dtype === 'f16') layer.half(root);
onTestFinished(() => layer.dispose());
await layer.loadStateDict(sd, { root });
+ if (quantized) expect(layer.weight.shape.dtype).toBe(c.float ? dtype : 'quantW');
return toArray(layer.forward(upload(root, xs, { ...matrix(tIn, cIn), dtype })), ex);
};
const file = quantFile(bits, groupSize, {
@@ -220,7 +354,9 @@ async function checkQuantizedConv(c: ConvCase, bits: 4 | 8, dtype: 'f32' | 'f16'
});
const quantSd = await fromSafetensors(bufferSource(file), { chunkBytes: 0 });
if (c.bias) quantSd.tensors.set('bias', memoryStateDict({ bias }).tensors.get('bias')!);
- const got = await run(make(), quantSd);
+ const warned = loadWarnings();
+ const got = await run(make(), quantSd, true);
+ expect(warned()).toEqual(c.float ? [expect.stringMatching(LOADED_AS_FLOAT)] : []);
const want = await run(
make(),
memoryStateDict({ weight: torchWeight, ...(c.bias ? { bias } : {}) }),
@@ -306,6 +442,91 @@ describe('Conv1d and ConvTranspose1d with a quantized weight', () => {
groupSize: 32,
bias: false,
},
+ // The quantized matmul runs a Conv1d with in a multiple of 4 and kernel·in
+ // of 8, and a ConvTranspose1d that also has out a multiple of 4 and
+ // stride·out of 4 (int8) or 8 (int4). The rest run as float convs.
+ // quantized: in 4 % 4 == 0, kernel·in 8 % 8 == 0
+ {
+ name: 'conv, the smallest that fits',
+ kind: 'conv',
+ tIn: 11,
+ cIn: 4,
+ cOut: 5,
+ kernel: 2,
+ stride: 1,
+ padding: 0,
+ groupSize: 8,
+ bias: true,
+ },
+ // float: kernel·in 12 % 8 != 0
+ {
+ name: 'conv, kernel·in off the 8-row tile',
+ kind: 'conv',
+ tIn: 11,
+ cIn: 4,
+ cOut: 5,
+ kernel: 3,
+ stride: 1,
+ padding: 1,
+ groupSize: 4,
+ bias: true,
+ float: true,
+ },
+ // float: in 6 % 4 != 0, though kernel·in 24 % 8 == 0
+ {
+ name: 'conv, in off 4',
+ kind: 'conv',
+ tIn: 11,
+ cIn: 6,
+ cOut: 8,
+ kernel: 4,
+ stride: 2,
+ padding: 1,
+ groupSize: 6,
+ bias: false,
+ float: true,
+ },
+ // quantized: out 4 % 4 == 0, stride·out 8 % 8 == 0, taps·in 8 % 8 == 0
+ {
+ name: 'transposed, the smallest that fits',
+ kind: 'transpose',
+ tIn: 7,
+ cIn: 4,
+ cOut: 4,
+ kernel: 4,
+ stride: 2,
+ padding: 1,
+ groupSize: 16,
+ bias: true,
+ },
+ // float: out 6 % 4 != 0
+ {
+ name: 'transposed, out off 4',
+ kind: 'transpose',
+ tIn: 7,
+ cIn: 8,
+ cOut: 6,
+ kernel: 4,
+ stride: 2,
+ padding: 1,
+ groupSize: 3,
+ bias: true,
+ float: true,
+ },
+ // float: taps·in 12 % 8 != 0
+ {
+ name: 'transposed, taps·in off the 8-row tile',
+ kind: 'transpose',
+ tIn: 7,
+ cIn: 4,
+ cOut: 4,
+ kernel: 6,
+ stride: 2,
+ padding: 2,
+ groupSize: 8,
+ bias: false,
+ float: true,
+ },
];
it.for(
(['f32', 'f16'] as const).flatMap((dtype) =>
diff --git a/packages/runntime/test/core/nn/layers.test.ts b/packages/runntime/test/core/nn/layers.test.ts
index a9de2e0e..025d9f98 100644
--- a/packages/runntime/test/core/nn/layers.test.ts
+++ b/packages/runntime/test/core/nn/layers.test.ts
@@ -1,3 +1,4 @@
+import { readFileSync } from 'node:fs';
import { describe, expect, it, vi } from 'vitest';
import { ACT_CODE } from '../../../src/core/graph/ops/shared.ts';
import { pending, type ValueMeta } from '../../../src/core/graph/value.ts';
@@ -362,11 +363,18 @@ describe('Linear with a quantized weight', () => {
});
});
+ // The quantized matmul takes in features a multiple of 8 and of the group
+ // size, out features a multiple of 4 (int8) or 8 (int4) and a group size a
+ // multiple of 4. A case names the one limit it sits on.
it.each([
- { name: 'in features off the 8-row tiles', inF: 12, outF: 4, bits: 8 as const, groupSize: 4 },
- { name: 'int4 out features off 8 per word', inF: 8, outF: 4, bits: 4 as const, groupSize: 4 },
- { name: 'a group size off 4', inF: 16, outF: 4, bits: 8 as const, groupSize: 2 },
- ])('refuses $name with CHECKPOINT_MISMATCH', async ({ inF, outF, bits, groupSize }) => {
+ // quantized: in 8 % 8 == 0, out 4 % 4 == 0, groups of 4
+ { name: 'int8 [4, 8] in groups of 4', inF: 8, outF: 4, bits: 8 as const, groupSize: 4 },
+ // quantized: int4 out 8 % 8 == 0
+ { name: 'int4 [8, 8] in groups of 4', inF: 8, outF: 8, bits: 4 as const, groupSize: 4 },
+ // quantized: one group over the whole row, 16 % 4 == 0
+ { name: 'int8 [4, 16] in groups of 16', inF: 16, outF: 4, bits: 8 as const, groupSize: 16 },
+ ])('loads $name as quantW', async ({ inF, outF, bits, groupSize }) => {
+ const layer = new Linear(inF, outF);
const sd = await quantCheckpoint(bits, groupSize, {
weight: {
ints: new Array(inF * outF).fill(0),
@@ -374,11 +382,46 @@ describe('Linear with a quantized weight', () => {
shape: [outF, inF],
},
});
+ layer.transformStateDict(sd, '');
+ expect(shapes(layer)).toEqual({
+ weight: { elems: (inF * outF * bits) / 32, dtype: 'quantW', dims: [inF, outF] },
+ bias: meta([outF]),
+ weightScale: meta([inF / groupSize, outF]),
+ });
+ expect(sd.quantizedAsFloat).toBeUndefined();
+ });
+
+ it.each([
+ // float: in 12 % 8 != 0
+ { name: 'in features off the 8-row tiles', inF: 12, outF: 4, bits: 8 as const, groupSize: 4 },
+ // float: in 4 % 8 != 0
+ { name: 'in features under one tile', inF: 4, outF: 4, bits: 8 as const, groupSize: 4 },
+ // float: int8 out 6 % 4 != 0
+ { name: 'int8 out features off 4 per word', inF: 8, outF: 6, bits: 8 as const, groupSize: 4 },
+ // float: int4 out 4 % 8 != 0
+ { name: 'int4 out features off 8 per word', inF: 8, outF: 4, bits: 4 as const, groupSize: 4 },
+ // float: group size 2 % 4 != 0
+ { name: 'a group size off 4', inF: 16, outF: 4, bits: 8 as const, groupSize: 2 },
+ // float: group size 1
+ { name: 'a scale per value', inF: 8, outF: 4, bits: 8 as const, groupSize: 1 },
+ ])('loads $name as its dequantized floats', async ({ inF, outF, bits, groupSize }) => {
+ // ints −7 … 7 and one scale per group, (g % 5 + 1) / 4, f16-exact
+ const ints = Array.from({ length: inF * outF }, (_, i) => (i % 15) - 7);
+ const scales = Array.from({ length: (inF * outF) / groupSize }, (_, g) => ((g % 5) + 1) / 4);
+ const sd = await quantCheckpoint(bits, groupSize, {
+ weight: { ints, scales, shape: [outF, inF] },
+ });
const layer = new Linear(inF, outF);
- expect(() => layer.transformStateDict(sd, '')).toThrow(
- expect.objectContaining({ code: 'CHECKPOINT_MISMATCH' }),
- );
- expect(layer.weight.shape.dtype).toBe('f32');
+ layer.transformStateDict(sd, '');
+ expect(shapes(layer)).toEqual({ weight: meta([inF, outF]), bias: meta([outF]) });
+ expect(layer.weightScale).toBeUndefined();
+ expect(sd.quantizedAsFloat).toEqual(['weight']);
+ // stored [in, out]: row i, column o is torch value o·in + i
+ const want = Array.from({ length: inF * outF }, (_, j) => {
+ const at = (j % outF) * inF + Math.floor(j / outF);
+ return ints[at]! * scales[Math.floor(at / groupSize)]!;
+ });
+ expect([...(await sd.tensors.get('weight')!.f32())]).toEqual(want);
});
it.each([
@@ -484,6 +527,25 @@ describe('Conv1d and ConvTranspose1d with a quantized weight', () => {
]);
});
+ it.for([8, 4] as const)(
+ 'takes the int%i Conv1d weight the Python quantizer writes',
+ async (bits) => {
+ // tools/python's golden file holds torch [out 6, in 16, kernel 4] as
+ // [6, 64] in groups of 32. 6 outputs pad to the packing width, 8.
+ const golden = new URL(
+ `../../../../../tools/python/tests/golden/tiny.int${bits}.safetensors`,
+ import.meta.url,
+ );
+ const sd = await fromSafetensors(bufferSource(readFileSync(golden)));
+ const layer = new Conv1d(16, 6, 4, { bias: false });
+ layer.transformStateDict(sd, 'conv');
+ expect(shapes(layer)).toEqual({
+ weight: { elems: (64 * 8 * bits) / 32, dtype: 'quantW', dims: [64, 8] },
+ weightScale: meta([2, 8]),
+ });
+ },
+ );
+
it('keeps a Conv1d group that ends inside a channel', async () => {
const layer = new Conv1d(8, 4, 3);
// groups of 4 values: a channel's 3 taps and the next channel's first
diff --git a/packages/runntime/test/core/nn/loadStateDict.test.ts b/packages/runntime/test/core/nn/loadStateDict.test.ts
index db8f7ed1..5ac03e13 100644
--- a/packages/runntime/test/core/nn/loadStateDict.test.ts
+++ b/packages/runntime/test/core/nn/loadStateDict.test.ts
@@ -1,12 +1,13 @@
import type { TgpuRoot } from 'typegpu';
import { describe, expect, it, vi } from 'vitest';
import { pending, type ValueMeta } from '../../../src/core/graph/value.ts';
-import { Linear } from '../../../src/core/nn/layers.ts';
+import { Conv1d, ConvTranspose1d, Linear } from '../../../src/core/nn/layers.ts';
import { gpuUpload, loadLazyStateDict } from '../../../src/core/nn/loadStateDict.ts';
import {
Buffer,
CpuParameter,
Module,
+ ModuleList,
Parameter,
type AnyModule,
} from '../../../src/core/nn/module.ts';
@@ -21,6 +22,7 @@ import {
type MemoryStateDict,
} from '../../../src/core/weights/safetensors.ts';
import { f32ToF16Bits } from '../../../src/core/weights/convert.ts';
+import { concatRows } from '../../../src/core/weights/transforms.ts';
import { meta } from '../graph/values.ts';
import { safetensorsFile } from '../weights/checkpoint.ts';
@@ -244,10 +246,9 @@ describe('binding', () => {
});
describe('a quantized tensor', () => {
- // A [2, 2] int8 weight in groups of 2; the loader must refuse it before
- // reading either part.
+ // A [2, 2] int8 weight in groups of 2: ints 1 2 | 3 4, scales 0.5 and 2.
const ints = vi.fn(async () => new Int8Array([1, 2, 3, 4]));
- const scales = vi.fn(async () => new Float32Array([1, 2]));
+ const scales = vi.fn(async () => new Float32Array([0.5, 2]));
const quant = quantTensor({
name: 'w',
shape: [2, 2],
@@ -257,30 +258,172 @@ describe('a quantized tensor', () => {
ints,
scales,
});
+ const DEQUANTIZED = [0.5, 1, 6, 8];
it.each([
- { name: 'an f32 parameter', param: () => new Parameter(meta([2, 2])), dtype: 'f32' },
- { name: 'an f16 parameter', param: () => new Parameter(meta([2, 2], 'f16')), dtype: 'f16' },
- { name: 'a CpuParameter', param: () => new CpuParameter(meta([2, 2])), dtype: 'f32' },
- {
- name: 'a quantW parameter',
- param: () => new Parameter({ elems: 1, dtype: 'quantW', dims: [2, 2] }),
- dtype: 'quantW',
- },
- ])('is refused by $name with CHECKPOINT_MISMATCH', async ({ param, dtype }) => {
- class NotLinear extends Module {
- weight = param();
+ { name: 'f32', dtype: 'f32' as const, want: new Float32Array(DEQUANTIZED) },
+ { name: 'f16', dtype: 'f16' as const, want: Uint16Array.from(DEQUANTIZED, f32ToF16Bits) },
+ ])('loads into an $name parameter as its dequantized values', async ({ dtype, want }) => {
+ class Table extends Module {
+ weight = new Parameter(meta([2, 2], dtype));
+ }
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ const upload = recorder();
+ await load(new Table(), { weight: quant }, upload);
+ expect(upload.mock.calls).toEqual([[want, meta([2, 2], dtype)]]);
+ // No quantized kernel reads a plain parameter, so there is nothing to say.
+ expect(warn).not.toHaveBeenCalled();
+ warn.mockRestore();
+ });
+
+ it('loads into a CpuParameter as its dequantized f32 values, off the GPU', async () => {
+ class Table extends Module {
+ weight = new CpuParameter(meta([2, 2]));
+ }
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ const upload = recorder();
+ const table = new Table();
+ await load(table, { weight: quant }, upload);
+ expect(table.weight.data).toEqual(new Float32Array(DEQUANTIZED));
+ expect(upload).not.toHaveBeenCalled();
+ expect(warn).not.toHaveBeenCalled();
+ warn.mockRestore();
+ });
+
+ it('is refused by a quantW parameter with CHECKPOINT_MISMATCH, before any read', async () => {
+ class Packed extends Module {
+ weight = new Parameter({ elems: 1, dtype: 'quantW', dims: [2, 2] });
}
+ ints.mockClear();
const upload = recorder();
- await expect(load(new NotLinear(), { weight: quant }, upload)).rejects.toMatchObject({
+ await expect(load(new Packed(), { weight: quant }, upload)).rejects.toMatchObject({
code: 'CHECKPOINT_MISMATCH',
message:
- "loadStateDict: 'weight' is quantized (int8), but only a Linear, Conv1d or " +
- `ConvTranspose1d weight loads quantized and this parameter holds ${dtype}`,
+ "loadStateDict: 'weight' is quantized (int8), but this quantW parameter takes packed " +
+ 'words, which only its layer makes',
});
expect(upload).not.toHaveBeenCalled();
expect(ints).not.toHaveBeenCalled();
- expect(scales).not.toHaveBeenCalled();
+ });
+});
+
+describe('the warning about quantized weights loaded as float', () => {
+ // int8 in groups of 4, all zeros.
+ const named = (prefix: string, rows: number, cols: number) => ({
+ [`${prefix}.weight`]: { data: new Int8Array(rows * cols), shape: [rows, cols] },
+ [`${prefix}.weight_scale`]: {
+ data: new Uint16Array((rows * cols) / 4),
+ shape: [rows, cols / 4],
+ },
+ });
+ const setting = { 'runntime.quant': JSON.stringify({ version: 1, scheme: 'symmetric' }) };
+ const open = (tensors: Parameters[0]) =>
+ fromSafetensors(bufferSource(safetensorsFile(tensors, setting)), { chunkBytes: 0 });
+ const WARNING =
+ 'loadStateDict: quantized weights loaded as float, the quantized kernel of their layer ' +
+ 'cannot run their shape or group size: ';
+
+ class Table extends Module {
+ weight = new Parameter(meta([4, 4]));
+ }
+ class Net extends Module {
+ // quantized: [out 4, in 8], in 8 % 8 == 0
+ fc = new Linear(8, 4, { bias: false });
+ // float, warned: [out 4, in 12], in 12 % 8 != 0
+ odd = new Linear(12, 4, { bias: false });
+ // float, silent: no quantized kernel reads a plain parameter
+ table = new Table();
+ // float, warned: Conv1d [out 4, in 3, kernel 4] as [4, 12], in 3 % 4 != 0
+ conv = new Conv1d(3, 4, 4, { bias: false });
+ // float, warned: ConvTranspose1d [in 4, out 2, kernel 2] as [4, 4], out 2 % 4 != 0
+ up = new ConvTranspose1d(4, 2, 2, { stride: 2, bias: false });
+ }
+ class Holder extends Module {
+ table = new Table();
+ }
+ class Fits extends Module {
+ fc = new Linear(8, 4, { bias: false });
+ }
+
+ it('names every weight whose layer has a quantized kernel that cannot run it', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ const sd = await open({
+ ...named('fc', 4, 8),
+ ...named('odd', 4, 12),
+ ...named('table', 4, 4),
+ ...named('conv', 4, 12),
+ ...named('up', 4, 4),
+ });
+ const upload = recorder();
+ await loadLazyStateDict(new Net(), sd, { upload });
+ expect(warn.mock.calls).toEqual([[`${WARNING}odd.weight, conv.weight, up.weight`]]);
+ expect(upload.mock.calls.map(([, shape]) => shape.dtype)).toEqual([
+ 'quantW',
+ 'f32',
+ 'f32',
+ 'f32',
+ 'f32',
+ 'f32',
+ ]);
+ warn.mockRestore();
+ });
+
+ it('lists every name, however many', async () => {
+ class Many extends Module {
+ layers = new ModuleList(Array.from({ length: 12 }, () => new Linear(12, 4, { bias: false })));
+ }
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ const tensors = Object.assign(
+ {},
+ ...Array.from({ length: 12 }, (_, i) => named(`layers.${i}`, 4, 12)),
+ ) as Parameters[0];
+ await loadLazyStateDict(new Many(), await open(tensors), { upload: recorder() });
+ const names = Array.from({ length: 12 }, (_, i) => `layers.${i}.weight`);
+ expect(warn.mock.calls).toEqual([[WARNING + names.join(', ')]]);
+ warn.mockRestore();
+ });
+
+ it('names the sources of a weight a hook stacked', async () => {
+ // q and k stack into one [8, 12] weight: in 12 % 8 != 0.
+ class Fused extends Module {
+ qk = new Linear(12, 8, { bias: false });
+ override transformStateDict(sd: LazyStateDict): void {
+ const take = (name: string) => {
+ const t = sd.tensors.get(name)!;
+ sd.tensors.delete(name);
+ return t;
+ };
+ sd.tensors.set('qk.weight', concatRows([take('q.weight'), take('k.weight')]));
+ }
+ }
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ const sd = await open({ ...named('q', 4, 12), ...named('k', 4, 12) });
+ await loadLazyStateDict(new Fused(), sd, { upload: recorder() });
+ expect(warn.mock.calls).toEqual([[`${WARNING}concatRows(q.weight, k.weight)`]]);
+ warn.mockRestore();
+ });
+
+ it('stays silent about a tensor no quantized kernel reads', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ await loadLazyStateDict(new Holder(), await open(named('table', 4, 4)), {
+ upload: recorder(),
+ });
+ expect(warn).not.toHaveBeenCalled();
+ warn.mockRestore();
+ });
+
+ it('stays silent when every quantized weight loads quantized', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ await loadLazyStateDict(new Fits(), await open(named('fc', 4, 8)), { upload: recorder() });
+ expect(warn).not.toHaveBeenCalled();
+ warn.mockRestore();
+ });
+
+ it('stays silent on a float checkpoint', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ await load(new Fits(), { 'fc.weight': { data: new Float32Array(32), shape: [4, 8] } });
+ expect(warn).not.toHaveBeenCalled();
+ warn.mockRestore();
});
});
diff --git a/packages/runntime/test/core/weights/checkpoint.ts b/packages/runntime/test/core/weights/checkpoint.ts
index 17d5c95b..6b7d4a33 100644
--- a/packages/runntime/test/core/weights/checkpoint.ts
+++ b/packages/runntime/test/core/weights/checkpoint.ts
@@ -209,3 +209,20 @@ export async function quantContents(
scales: [...(scales instanceof Uint16Array ? f16ToF32(scales) : scales)],
};
}
+
+/** Replaces `name` in `sd` with the tensor `runntime quantize` stores for
+ * it: int8 rows [dim 0, everything else] of `values`, whole numbers, one
+ * scale of 1 per row, so it dequantizes to `values` exactly. */
+export async function storeAsQuantizedRows(
+ sd: LazyStateDict,
+ name: string,
+ values: number[],
+ rows: number,
+): Promise {
+ const entry: QuantEntry = {
+ ints: values,
+ scales: new Array(rows).fill(1),
+ shape: [rows, values.length / rows],
+ };
+ sd.tensors.set(name, await quantSource(8, values.length / rows, name, entry));
+}
diff --git a/packages/runntime/test/core/weights/safetensors.test.ts b/packages/runntime/test/core/weights/safetensors.test.ts
index 0f277d95..d4b45a46 100644
--- a/packages/runntime/test/core/weights/safetensors.test.ts
+++ b/packages/runntime/test/core/weights/safetensors.test.ts
@@ -534,7 +534,26 @@ describe('fromSafetensors', () => {
expect(sd.tensors.get('fc.weight')!.kind).toBe('quant');
});
- it.each(['f32', 'u16', 'halfWords', 'words'] as const)(
+ it('reads f32() of a quantized tensor as each int times the scale of its group', async () => {
+ // ints 1 2 | 3 4 and −1 −2 | −3 −4, scales 0.5 2 and 1 0.25
+ const sd = await open({
+ 'fc.weight': {
+ dtype: 'I8',
+ shape: [2, 4],
+ bytes: Uint8Array.from([1, 2, 3, 4, 255, 254, 253, 252]),
+ },
+ 'fc.weight_scale': {
+ dtype: 'F16',
+ shape: [2, 2],
+ bytes: u16Bytes([0x3800, 0x4000, 0x3c00, 0x3400]),
+ },
+ });
+ expect(await sd.tensors.get('fc.weight')!.f32()).toEqual(
+ new Float32Array([0.5, 1, 6, 8, -1, -2, -0.75, -1]),
+ );
+ });
+
+ it.each(['u16', 'halfWords', 'words'] as const)(
'rejects %s() on a quantized tensor with CHECKPOINT_MISMATCH',
async (call) => {
const sd = await open({
@@ -615,21 +634,30 @@ describe('fromSafetensors', () => {
it.for([4, 8])('reads int%i weights within half a step of the float source', async (bits) => {
const source = await golden('tiny.safetensors');
const sd = await golden(`tiny.int${bits}.safetensors`);
- const w = sd.tensors.get('layer.weight')!;
- if (w.kind !== 'quant') throw new Error('expected a quantized tensor');
- expect(w).toMatchObject({ shape: [8, 64], bits, groupSize: 32 });
- const ints = await w.ints();
- const halves = await w.scales();
- const scales = halves instanceof Float32Array ? halves : f16ToF32(halves);
- const want = await source.tensors.get('layer.weight')!.f32();
- want.forEach((v, i) => {
- const scale = scales[Math.floor(i / 32)]!;
- expect(Math.abs(ints[i]! * scale - v)).toBeLessThanOrEqual(scale / 2 + 1e-6);
- });
- // The tensors it leaves alone keep their bytes.
- for (const name of ['embed.weight', 'layer.bias', 'norm.weight']) {
+ // The tool stores a weight as rows of everything after its first dim:
+ // conv.weight is torch [6, 16, 4], and no file setting names the bits
+ // or the group size.
+ const quantized = { 'layer.weight': [8, 64], 'conv.weight': [6, 64] };
+ for (const [name, shape] of Object.entries(quantized)) {
+ const w = sd.tensors.get(name)!;
+ if (w.kind !== 'quant') throw new Error('expected a quantized tensor');
+ expect(w).toMatchObject({ shape, bits, groupSize: 32 });
+ const ints = await w.ints();
+ const halves = await w.scales();
+ const scales = halves instanceof Float32Array ? halves : f16ToF32(halves);
+ const want = await source.tensors.get(name)!.f32();
+ expect(ints.length).toBe(want.length);
+ want.forEach((v, i) => {
+ const scale = scales[Math.floor(i / 32)]!;
+ expect(Math.abs(ints[i]! * scale - v)).toBeLessThanOrEqual(scale / 2 + 1e-6);
+ });
+ }
+ // The tensors it leaves alone keep their bytes: an embedding table, a
+ // bias, a 1D norm and a row of 40 values that groups of 32 do not divide.
+ for (const name of ['embed.weight', 'layer.bias', 'norm.weight', 'odd.weight']) {
expect(await sd.tensors.get(name)!.f32()).toEqual(await source.tensors.get(name)!.f32());
}
+ expect(sd.tensors.size).toBe(6);
});
});
diff --git a/packages/runntime/test/core/weights/transforms.test.ts b/packages/runntime/test/core/weights/transforms.test.ts
index f184f568..1627d9f9 100644
--- a/packages/runntime/test/core/weights/transforms.test.ts
+++ b/packages/runntime/test/core/weights/transforms.test.ts
@@ -468,10 +468,18 @@ describe.each([8, 4] as const)('int%i quantized sources', (bits) => {
expect(() => transposeWeight(t)).toThrow(mismatch(/w\.weight is quantized/));
});
- it('readFloats and the float accessors of an output reject it', async () => {
+ it('readFloats and f32() of an output give the dequantized values', async () => {
const t = await quantSource(bits, 2, 'w.weight', w);
- await expect(readFloats('test', t)).rejects.toThrow(mismatch(/^test: w\.weight is quantized/));
- await expect(splitRows(t, [1, 1])[0]!.f32()).rejects.toThrow(
+ // int · scale of its group of 2
+ expect(await readFloats('test', t)).toEqual(
+ new Float32Array([0.5, -1, 3, -4, 10, -12, 28, -32]),
+ );
+ expect(await splitRows(t, [1, 1])[1]!.f32()).toEqual(new Float32Array([10, -12, 28, -32]));
+ });
+
+ it('the readers of stored bits of an output reject it', async () => {
+ const t = await quantSource(bits, 2, 'w.weight', w);
+ await expect(splitRows(t, [1, 1])[0]!.u16()).rejects.toThrow(
mismatch(/splitRows\(w\.weight\) is quantized/),
);
});
diff --git a/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts b/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts
index 7a1f1f85..8654fd69 100644
--- a/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts
+++ b/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts
@@ -13,7 +13,12 @@ import {
transformEmbeddingGemma2StateDict,
transformEmbeddingGemma2VisionStateDict,
} from '../../../src/zoo/embeddinggemma2/stateDictHooks.ts';
-import { checkpoint, contents, type Entry } from '../../core/weights/checkpoint.ts';
+import {
+ checkpoint,
+ contents,
+ storeAsQuantizedRows,
+ type Entry,
+} from '../../core/weights/checkpoint.ts';
const DTYPES = [
{ dtype: 'F32', type: 'f32', elemBytes: 4 },
@@ -277,6 +282,20 @@ describe('transformEmbeddingGemma2VisionStateDict', () => {
expect(sd.totalBytes()).toBe(total);
});
+
+ it('splits a position table quantized as rows [2, positions·hidden] into x and y', async () => {
+ const name = 'vision_tower.patch_embedder.position_embedding_table';
+ const sd = await checkpoint('F32', CHECKPOINT);
+ await storeAsQuantizedRows(sd, name, CHECKPOINT[name]!.values, 2);
+ transformEmbeddingGemma2VisionStateDict(sd, '', cfg);
+ for (const [table, values] of [
+ ['pos_x.weight', [1, 2, 3, 4]],
+ ['pos_y.weight', [5, 6, 7, 8]],
+ ] as const) {
+ expect(sd.tensors.get(table)!.shape).toEqual([2, 2]);
+ expect(await contents(sd.tensors.get(table)!)).toEqual({ type: 'f32', values });
+ }
+ });
});
describe('transformEmbeddingGemma2AudioStateDict', () => {
@@ -466,4 +485,25 @@ describe('transformEmbeddingGemma2AudioStateDict', () => {
// 10 clipped linears' worth of bounds, 4 scalars each.
expect(sd.totalBytes()).toBe(total - 40 * elemBytes);
});
+
+ it('reorders the convs quantized as rows [out, everything else] by their values', async () => {
+ const conv1 = `${sub}layer1.conv.weight`;
+ const depthwise = `${T}.lconv1d.depthwise_conv1d.weight`;
+ const sd = await checkpoint('F32', CHECKPOINT);
+ await storeAsQuantizedRows(sd, conv1, CHECKPOINT[conv1]!.values, 1);
+ await storeAsQuantizedRows(sd, depthwise, CHECKPOINT[depthwise]!.values, 4);
+ transformEmbeddingGemma2AudioStateDict(sd, '', cfg);
+ // The same values as from the float checkpoint: tap k of channel c at
+ // k·2 + c, and tap 0 of every channel, then tap 1.
+ expect(sd.tensors.get('conv1.weight')!.shape).toEqual([1, 18]);
+ expect(await contents(sd.tensors.get('conv1.weight')!)).toEqual({
+ type: 'f32',
+ values: [1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17, 9, 18],
+ });
+ expect(sd.tensors.get('layers.0.lconv.conv')!.shape).toEqual([2, 4]);
+ expect(await contents(sd.tensors.get('layers.0.lconv.conv')!)).toEqual({
+ type: 'f32',
+ values: [1, 3, 5, 7, 2, 4, 6, 8],
+ });
+ });
});
diff --git a/packages/runntime/test/zoo/higgs/encoder.test.ts b/packages/runntime/test/zoo/higgs/encoder.test.ts
index c6d6e0e5..dd8e9b8a 100644
--- a/packages/runntime/test/zoo/higgs/encoder.test.ts
+++ b/packages/runntime/test/zoo/higgs/encoder.test.ts
@@ -10,6 +10,7 @@ import { HIGGS_ENCODER } from '../../../src/zoo/higgs/config.ts';
import { HiggsEncoder } from '../../../src/zoo/higgs/encoder.ts';
import { resample } from '../../../src/zoo/higgs/resample.ts';
import { expectClose } from '../../gpu.ts';
+import { dequantizeWeight, quantizeWeight, quantSource } from '../../core/weights/checkpoint.ts';
import { CFG, checkpoint, QUANT_CFG, quantize, type Sd } from './encoderCheckpoint.ts';
const POS = 'semantic_model.encoder.pos_conv_embed';
@@ -64,6 +65,31 @@ describe('HiggsEncoder', () => {
expect([...sd.tensors.keys()].filter((n) => n.startsWith(`${POS}.conv.`))).toEqual([]);
});
+ it('folds a position conv weight norm quantized as rows [out, in·kernel]', async () => {
+ const raw = checkpoint();
+ const sd = memoryStateDict(raw);
+ const { hidden, posGroups, posKernel: k } = CFG.hubert;
+ const width = hidden / posGroups;
+ const gName = `${POS}.conv.parametrizations.weight.original0`;
+ const vName = `${POS}.conv.parametrizations.weight.original1`;
+ // g [1, 1, k] as one row of k, v [hidden, width, k] as rows of width·k.
+ const gQuant = quantizeWeight(raw[gName]!.data, [1, k], 8, k);
+ const vQuant = quantizeWeight(raw[vName]!.data, [hidden, width * k], 8, width);
+ sd.tensors.set(gName, await quantSource(8, k, gName, gQuant));
+ sd.tensors.set(vName, await quantSource(8, width, vName, vQuant));
+ new HiggsEncoder(CFG).semantic_model.encoder.pos_conv_embed.transformStateDict(sd, POS);
+ const g = dequantizeWeight(gQuant);
+ const v = dequantizeWeight(vQuant);
+ const norm = Array.from({ length: k }, (_, j) =>
+ Math.sqrt(v.reduce((s, x, i) => (i % k === j ? s + x * x : s), 0)),
+ );
+ const want = Array.from(v.subarray(width * width * k, 2 * width * width * k), (x, i) => {
+ return (x * g[i % k]!) / norm[i % k]!;
+ });
+ expectClose(await read(sd, `${POS}.convs.1.weight`), want, { atol: 1e-6, rtol: 1e-6 });
+ expect(sd.tensors.get(`${POS}.convs.1.weight`)!.shape).toEqual([width, width, k]);
+ });
+
it("takes the checkpoint's folded position conv weight and splits it into groups", async () => {
const raw = checkpoint();
const { hidden, posGroups, posKernel: k } = CFG.hubert;
diff --git a/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts b/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts
index f8c0f73d..1d24bcb2 100644
--- a/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts
+++ b/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts
@@ -1,6 +1,11 @@
import { describe, expect, it } from 'vitest';
import { transformHfPrivacyFilterStateDict } from '../../../src/zoo/privacy-filter/stateDictHooks.ts';
-import { checkpoint, contents, type Entry } from '../../core/weights/checkpoint.ts';
+import {
+ checkpoint,
+ contents,
+ storeAsQuantizedRows,
+ type Entry,
+} from '../../core/weights/checkpoint.ts';
const ZERO = { values: [0, 0], shape: [2] };
@@ -108,4 +113,23 @@ describe('transformHfPrivacyFilterStateDict', () => {
// The dropped score.bias (2 elems) is the only byte loss.
expect(sd.totalBytes()).toBe(total - 2 * elemBytes);
});
+
+ // A file stores a quantized [E, K, 2H] as rows [E, K·2H], which the hook
+ // cannot split into glu and lin halves per expert.
+ it.for([
+ 'model.layers.0.mlp.experts.gate_up_proj',
+ 'model.layers.0.mlp.experts.gate_up_proj_bias',
+ 'model.layers.0.mlp.experts.down_proj',
+ ])('refuses a quantized %s with CHECKPOINT_MISMATCH', async (name) => {
+ const sd = await checkpoint('F32', CHECKPOINT);
+ await storeAsQuantizedRows(sd, name, CHECKPOINT[name]!.values, 2);
+ expect(() => transformHfPrivacyFilterStateDict(sd)).toThrow(
+ expect.objectContaining({
+ code: 'CHECKPOINT_MISMATCH',
+ message:
+ `privacy-filter state dict: '${name}' is quantized, and a quantized expert ` +
+ 'tensor cannot be split into experts: keep it float in the file',
+ }),
+ );
+ });
});
diff --git a/tools/python/README.md b/tools/python/README.md
index 083750df..e640d156 100644
--- a/tools/python/README.md
+++ b/tools/python/README.md
@@ -3,22 +3,40 @@
Quantizes a safetensors checkpoint to runntime's int8/int4 weight format, so
a model downloads smaller and runs on the quantized matmul.
+The tool knows the file format, not the model: it never asks which layer
+reads a tensor or what a kernel can run. runntime fits each weight to its
+layer at load, and loads a weight no quantized kernel runs as float.
+
```bash
uv run runntime quantize model.safetensors model.int8.safetensors --bits 8
```
- The source is a `.safetensors` file or a Hugging Face model folder, sharded
or not. The output is one `.safetensors` file.
-- 2D tensors named `*.weight` are quantized in groups of `--group-size`
- inputs (default 32), each group with one f16 scale. Embeddings, position
- embeddings and `lm_head` stay as stored. `--include` and `--exclude` take
- glob patterns and replace those defaults.
+- Float tensors named `*.weight` with 2 or more dims are quantized, under
+ their own names. `--include GLOB` replaces `*.weight`.
+- Tensors matching `*embed*`, `*pos_emb*` or `lm_head.*` stay as stored. This
+ is a quality default, not a format rule: quantized embedding tables only
+ rarely hold up. `--exclude GLOB` replaces it, so `--exclude none` quantizes
+ them too.
+- A quantized tensor is stored as rows, `[first dim, everything else]`: a
+ Conv1d weight `[out, in, k]` becomes `[out, in·k]`. Each row is cut into
+ groups of `--group-size` values (default 32, any positive integer), each
+ group with one f16 scale, stored as `_scale`.
+- A selected tensor stays as stored only when the format cannot hold it: it
+ is not float, it has fewer than 2 dims, the group size does not divide its
+ row, or, for int4, its row is odd (two values pack into a byte).
+- The header's `runntime.quant` entry is `{"version":1,"scheme":"symmetric"}`.
+ The reader takes each tensor's bits from its dtype (`I8` int8, `U8` int4)
+ and its group size from the shape of its scales.
- Every other tensor is copied as stored, or as F16 with `--f16`, and the
header metadata is kept.
- `--drop` takes glob patterns of tensors to leave out, like a pooler the
model never reads.
- It prints the max and mean absolute error of every quantized tensor and
why each other tensor was skipped. Exclude the layers that lose too much.
+- Use `--bits 8` for models built on 2D convolutions: int4 breaks the outputs
+ of MobileNetV4, DepthART and YOLO26.
## Development
diff --git a/tools/python/src/runntime_tools/cli.py b/tools/python/src/runntime_tools/cli.py
index 711904b3..1c8f8c7d 100644
--- a/tools/python/src/runntime_tools/cli.py
+++ b/tools/python/src/runntime_tools/cli.py
@@ -1,8 +1,11 @@
"""`runntime quantize`: writes a safetensors checkpoint in runntime's int8/int4
-format v1 and prints the error of every quantized tensor."""
+format v1 and prints the error of every quantized tensor. It knows the file
+format only, not which layer reads a tensor: runntime fits each weight to its
+layer at load."""
import argparse
import json
+import math
import sys
from collections.abc import Sequence
from dataclasses import dataclass
@@ -24,19 +27,12 @@
DEFAULT_INCLUDE = ("*.weight",)
DEFAULT_EXCLUDE = ("*embed*", "*pos_emb*", "lm_head.*")
-"""runntime reads token and position embeddings as float lookup tables, and
-lm_head often shares the embedding's weights."""
-
-IN_FEATURES_MULTIPLE = 8
-"""runntime's quantized matmul reads 8 input rows per tile."""
+"""A quality choice, not a format rule: quantized embedding tables only rarely
+hold up, and lm_head often shares the embedding's weights."""
F16_MAX = float(np.finfo(np.float16).max)
"""The largest finite f16. A tensor with a bigger value cannot be stored as F16."""
-PACKED_WORD_BITS = 32
-"""runntime packs a weight 32 bits to a word, so out features must fill whole
-words: a multiple of 4 for int8, 8 for int4."""
-
@dataclass(frozen=True)
class Options:
@@ -58,16 +54,13 @@ def skip_reason(name: str, tensor: Tensor, options: Options) -> str | None:
return f"excluded by {excluded_by}"
if tensor.dtype not in FLOAT_DTYPES:
return f"dtype {tensor.dtype}"
- if len(tensor.shape) != 2:
- return f"{len(tensor.shape)}D, needs [out, in]"
- out_features, in_features = tensor.shape
- if in_features % IN_FEATURES_MULTIPLE:
- return f"in features {in_features} not a multiple of {IN_FEATURES_MULTIPLE}"
- if in_features % options.group_size:
- return f"in features {in_features} not a multiple of the group size {options.group_size}"
- out_multiple = PACKED_WORD_BITS // options.bits
- if out_features % out_multiple:
- return f"out features {out_features} not a multiple of {out_multiple}"
+ if len(tensor.shape) < 2:
+ return f"{len(tensor.shape)}D, needs 2 or more dims"
+ row = math.prod(tensor.shape[1:])
+ if row % options.group_size:
+ return f"row of {row} values not a multiple of the group size {options.group_size}"
+ if options.bits == 4 and row % 2:
+ return f"row of {row} values is odd, int4 packs two to a byte"
return None
@@ -102,7 +95,8 @@ def quantize_checkpoint(source: Path, destination: Path, options: Options) -> li
tensors[name] = to_float16(name, tensor) if options.float16 else tensor
report.append(f"skipped {name} {shape}: {reason}")
continue
- weight = to_float32(tensor)
+ # The format stores a weight as rows: dim 0 by everything else.
+ weight = to_float32(tensor).reshape(tensor.shape[0], -1)
quantized = quantize(weight, options.bits, options.group_size)
if options.bits == 8:
tensors[name] = from_array("I8", quantized.ints)
@@ -111,20 +105,23 @@ def quantize_checkpoint(source: Path, destination: Path, options: Options) -> li
tensors[f"{name}_scale"] = from_array("F16", quantized.scales)
quantized_count += 1
error = np.abs(dequantize(quantized) - weight)
- report.append(f"quantized {name} {shape}: max error {error.max():.3g}, mean {error.mean():.3g}")
+ stored = "" if len(tensor.shape) == 2 else f" as [{weight.shape[0]}, {weight.shape[1]}]"
+ report.append(f"quantized {name} {shape}{stored}: max error {error.max():.3g}, mean {error.mean():.3g}")
if quantized_count == 0:
raise ValueError("no tensor was quantized:\n" + "\n".join(report))
- setting = {"version": FORMAT_VERSION, "bits": options.bits, "groupSize": options.group_size, "scheme": "symmetric"}
+ # No bits or groupSize: the reader takes them from each tensor's dtype and
+ # its scales' shape, which holds for any file, one setting or mixed.
+ setting = {"version": FORMAT_VERSION, "scheme": "symmetric"}
metadata = checkpoint.metadata | {QUANT_METADATA_KEY: json.dumps(setting, separators=(",", ":"))}
write_safetensors(destination, tensors, metadata)
return report
def group_size(value: str) -> int:
- """argparse type: a positive multiple of 4."""
+ """argparse type: a positive integer."""
size = int(value)
- if size <= 0 or size % 4:
- raise argparse.ArgumentTypeError(f"{value} is not a positive multiple of 4")
+ if size <= 0:
+ raise argparse.ArgumentTypeError(f"{value} is not a positive integer")
return size
@@ -134,12 +131,14 @@ def parse_args(argv: Sequence[str] | None) -> argparse.Namespace:
command = commands.add_parser(
"quantize",
help="quantize a safetensors checkpoint to int8 or int4",
- description="Quantizes the 2D weights of a safetensors checkpoint to runntime's int8/int4 format.",
+ description="Quantizes the weights of a safetensors checkpoint to runntime's int8/int4 format.",
)
command.add_argument("source", type=Path, help="a .safetensors file or a Hugging Face model folder")
command.add_argument("destination", type=Path, help="the .safetensors file to write")
command.add_argument("--bits", type=int, choices=(4, 8), required=True)
- command.add_argument("--group-size", type=group_size, default=32, help="inputs that share one scale (default: 32)")
+ command.add_argument(
+ "--group-size", type=group_size, default=32, help="values of a row that share one scale (default: 32)"
+ )
command.add_argument(
"--include",
action="append",
diff --git a/tools/python/src/runntime_tools/quantize.py b/tools/python/src/runntime_tools/quantize.py
index 3a517168..85a51b9f 100644
--- a/tools/python/src/runntime_tools/quantize.py
+++ b/tools/python/src/runntime_tools/quantize.py
@@ -1,4 +1,4 @@
-"""Symmetric group-wise int8/int4 quantization of [out, in] weights, in the
+"""Symmetric group-wise int8/int4 quantization of [rows, cols] weights, in the
runntime format v1 layout."""
from dataclasses import dataclass
@@ -12,22 +12,22 @@
@dataclass(frozen=True)
class QuantizedWeight:
- """`ints` [out, in], one signed value per element, and `scales`
- [out, in / group_size], one f16 scale per group of inputs."""
+ """`ints` [rows, cols], one signed value per element, and `scales`
+ [rows, cols / group_size], one f16 scale per group of a row."""
ints: NDArray[np.int8]
scales: NDArray[np.float16]
def quantize(weight: NDArray[np.float32], bits: Bits, group_size: int) -> QuantizedWeight:
- """Quantizes each group of `group_size` inputs of each output row to
+ """Quantizes each group of `group_size` values of each row to
`int = round(w / scale)` with `scale = absmax / (2^(bits-1) - 1)`, so the
range is symmetric and has no zero point. The ints divide by the scale
rounded to f16, the one the file stores. An all-zero group has scale 0
and ints 0."""
- out_features, in_features = weight.shape
+ rows, cols = weight.shape
largest = 2 ** (bits - 1) - 1
- groups = weight.reshape(out_features, in_features // group_size, group_size)
+ groups = weight.reshape(rows, cols // group_size, group_size)
scales = (np.abs(groups).max(axis=-1, keepdims=True) / largest).astype(np.float16)
divisor = scales.astype(np.float32)
ratio = np.divide(groups, divisor, out=np.zeros_like(groups), where=divisor != 0)
@@ -37,11 +37,11 @@ def quantize(weight: NDArray[np.float32], bits: Bits, group_size: int) -> Quanti
def dequantize(weight: QuantizedWeight) -> NDArray[np.float32]:
"""The float values the ints stand for: int * scale."""
- out_features, in_features = weight.ints.shape
+ rows, cols = weight.ints.shape
group_count = weight.scales.shape[1]
- groups = weight.ints.reshape(out_features, group_count, in_features // group_count)
+ groups = weight.ints.reshape(rows, group_count, cols // group_count)
values = groups * weight.scales.astype(np.float32)[..., np.newaxis]
- return values.reshape(out_features, in_features)
+ return values.reshape(rows, cols)
def pack_int4(ints: NDArray[np.int8]) -> NDArray[np.uint8]:
diff --git a/tools/python/tests/golden/tiny.int4.safetensors b/tools/python/tests/golden/tiny.int4.safetensors
index cff48760..af44a054 100644
Binary files a/tools/python/tests/golden/tiny.int4.safetensors and b/tools/python/tests/golden/tiny.int4.safetensors differ
diff --git a/tools/python/tests/golden/tiny.int8.safetensors b/tools/python/tests/golden/tiny.int8.safetensors
index 293573d0..99212431 100644
Binary files a/tools/python/tests/golden/tiny.int8.safetensors and b/tools/python/tests/golden/tiny.int8.safetensors differ
diff --git a/tools/python/tests/golden/tiny.safetensors b/tools/python/tests/golden/tiny.safetensors
index 3be7e539..365b1867 100644
Binary files a/tools/python/tests/golden/tiny.safetensors and b/tools/python/tests/golden/tiny.safetensors differ
diff --git a/tools/python/tests/test_cli.py b/tools/python/tests/test_cli.py
index 4763f465..8e549029 100644
--- a/tools/python/tests/test_cli.py
+++ b/tools/python/tests/test_cli.py
@@ -1,10 +1,12 @@
+import argparse
from pathlib import Path
import numpy as np
import pytest
+from numpy.typing import NDArray
from safetensors import safe_open
-from runntime_tools.cli import Options, main, skip_reason
+from runntime_tools.cli import Options, group_size, main, skip_reason
from runntime_tools.quantize import Bits
from runntime_tools.safetensors_io import Tensor, from_array, write_safetensors
@@ -36,40 +38,122 @@ def test_golden_file_reads_back_with_the_safetensors_library() -> None:
):
metadata = quantized.metadata()
assert metadata["format"] == "pt"
- assert metadata["runntime.quant"] == '{"version":1,"bits":4,"groupSize":32,"scheme":"symmetric"}'
+ assert metadata["runntime.quant"] == '{"version":1,"scheme":"symmetric"}'
weight = quantized.get_tensor("layer.weight")
assert (weight.dtype, weight.shape) == (np.uint8, (8, 32))
scale = quantized.get_tensor("layer.weight_scale")
assert (scale.dtype, scale.shape) == (np.float16, (8, 2))
- for name in ("embed.weight", "layer.bias", "norm.weight"):
+ for name in ("embed.weight", "layer.bias", "norm.weight", "odd.weight"):
np.testing.assert_array_equal(quantized.get_tensor(name), source.get_tensor(name))
+@pytest.mark.parametrize(("bits", "dtype", "stored_cols"), [(8, np.int8, 64), (4, np.uint8, 32)])
+def test_golden_file_stores_a_conv_weight_as_rows(bits: Bits, dtype: type, stored_cols: int) -> None:
+ # conv.weight is [out 6, in 16, kernel 4]: rows of 16 * 4 = 64 values, two groups of 32.
+ with (
+ safe_open(GOLDEN / f"tiny.int{bits}.safetensors", "numpy") as quantized,
+ safe_open(GOLDEN / "tiny.safetensors", "numpy") as source,
+ ):
+ stored = quantized.get_tensor("conv.weight")
+ assert (stored.dtype, stored.shape) == (dtype, (6, stored_cols))
+ scales = quantized.get_tensor("conv.weight_scale")
+ assert (scales.dtype, scales.shape) == (np.float16, (6, 2))
+ ints = stored if bits == 8 else unpack_int4(stored)
+ # Row-major, so value (o, c, k) of the conv is value c * 4 + k of row o.
+ values = (ints.reshape(6, 2, 32) * scales.astype(np.float32)[..., np.newaxis]).reshape(6, 16, 4)
+ step = np.repeat(scales.astype(np.float32), 32, axis=1).reshape(6, 16, 4)
+ assert np.all(np.abs(values - source.get_tensor("conv.weight")) <= step / 2 + 1e-6)
+
+
+def unpack_int4(packed: NDArray[np.uint8]) -> NDArray[np.int8]:
+ """Two signed nibbles per byte, the low one first, as one value each."""
+ nibbles = np.stack([packed & 0x0F, packed >> 4], axis=-1).reshape(packed.shape[0], -1).astype(np.int8)
+ return np.where(nibbles > 7, nibbles - 16, nibbles).astype(np.int8)
+
+
def tensor(shape: tuple[int, ...], dtype: str = "F32") -> Tensor:
return Tensor(dtype, shape, np.zeros(0, dtype=np.uint8))
@pytest.mark.parametrize(
- ("name", "stored", "bits", "reason"),
+ ("name", "stored", "bits", "group_size", "reason"),
[
- ("fc.weight", tensor((8, 64)), 4, None),
- ("fc.weight", tensor((4, 64), "BF16"), 8, None),
- ("fc.bias", tensor((8,)), 8, "not matched by --include"),
- ("embed.weight", tensor((8, 64)), 8, "excluded by *embed*"),
- ("fc.weight", tensor((8, 64), "I8"), 8, "dtype I8"),
- ("conv.weight", tensor((8, 64, 3)), 8, "3D, needs [out, in]"),
- ("conv.weight", tensor((8, 64, 3, 3)), 8, "4D, needs [out, in]"),
- ("fc.weight", tensor((8, 36)), 8, "in features 36 not a multiple of 8"),
- ("fc.weight", tensor((8, 48)), 8, "in features 48 not a multiple of the group size 32"),
- ("fc.weight", tensor((4, 64)), 4, "out features 4 not a multiple of 8"),
- ("fc.weight", tensor((6, 64)), 8, "out features 6 not a multiple of 4"),
+ ("fc.weight", tensor((8, 64)), 4, 32, None),
+ ("fc.weight", tensor((4, 64), "BF16"), 8, 32, None),
+ ("fc.weight", tensor((8, 64), "F16"), 8, 32, None),
+ # Shapes no layer rule looks at: any out, any row the group size divides.
+ ("fc.weight", tensor((1, 32)), 4, 32, None),
+ ("fc.weight", tensor((5, 96)), 8, 32, None),
+ ("fc.weight", tensor((7, 36)), 8, 12, None),
+ ("fc.weight", tensor((3, 5)), 8, 1, None),
+ ("fc.weight", tensor((3, 9)), 8, 3, None),
+ # 3 or more dims quantize as rows of everything after dim 0.
+ ("conv.weight", tensor((8, 64, 3)), 8, 32, None),
+ ("conv.weight", tensor((8, 16, 3, 3)), 4, 16, None),
+ ("conv.weight", tensor((5, 3, 7)), 8, 7, None),
+ ("fc.bias", tensor((8,)), 8, 32, "not matched by --include"),
+ ("skip.weight", tensor((8, 64)), 8, 32, "excluded by skip.*"),
+ ("fc.weight", tensor((8, 64), "I8"), 8, 32, "dtype I8"),
+ ("fc.weight", tensor((8, 64), "F64"), 8, 32, "dtype F64"),
+ ("norm.weight", tensor((64,)), 8, 32, "1D, needs 2 or more dims"),
+ ("scale.weight", tensor(()), 8, 1, "0D, needs 2 or more dims"),
+ ("fc.weight", tensor((8, 48)), 8, 32, "row of 48 values not a multiple of the group size 32"),
+ ("conv.weight", tensor((8, 16, 3)), 8, 32, "row of 48 values not a multiple of the group size 32"),
+ ("fc.weight", tensor((8, 9)), 4, 3, "row of 9 values is odd, int4 packs two to a byte"),
],
)
-def test_skip_reason(name: str, stored: Tensor, bits: Bits, reason: str | None) -> None:
- options = Options(bits=bits, group_size=32, include=("*.weight",), exclude=("*embed*",))
+def test_skip_reason(name: str, stored: Tensor, bits: Bits, group_size: int, reason: str | None) -> None:
+ options = Options(bits=bits, group_size=group_size, include=("*.weight",), exclude=("skip.*",))
assert skip_reason(name, stored, options) == reason
+EMBEDDING_NAMES = ("embed_tokens.weight", "decoder.pos_emb.weight", "lm_head.weight")
+
+
+def embedding_checkpoint(folder: Path) -> Path:
+ """A checkpoint of EMBEDDING_NAMES and one Linear weight, each [4, 32] of
+ 127: the top int8 at scale 1, so it quantizes without error."""
+ source = folder / "model.safetensors"
+ weight = from_array("F32", np.full((4, 32), 127, dtype=np.float32))
+ write_safetensors(source, dict.fromkeys((*EMBEDDING_NAMES, "fc.weight"), weight), {})
+ return source
+
+
+def test_keeps_embeddings_and_lm_head_float_by_default(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ source = embedding_checkpoint(tmp_path)
+ assert main(["quantize", str(source), str(tmp_path / "out.safetensors"), "--bits", "8"]) == 0
+ assert capsys.readouterr().out.splitlines() == [
+ "skipped embed_tokens.weight [4, 32]: excluded by *embed*",
+ "skipped decoder.pos_emb.weight [4, 32]: excluded by *pos_emb*",
+ "skipped lm_head.weight [4, 32]: excluded by lm_head.*",
+ "quantized fc.weight [4, 32]: max error 0, mean 0",
+ ]
+
+
+def test_exclude_replaces_the_default(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ source = embedding_checkpoint(tmp_path)
+ argv = ["quantize", str(source), str(tmp_path / "out.safetensors"), "--bits", "8", "--exclude", "fc.*"]
+ assert main(argv) == 0
+ assert [line.split()[0] for line in capsys.readouterr().out.splitlines()] == [*["quantized"] * 3, "skipped"]
+
+
+@pytest.mark.parametrize("size", ["1", "3", "6", "640"])
+def test_group_size_takes_any_positive_integer(size: str) -> None:
+ assert group_size(size) == int(size)
+
+
+@pytest.mark.parametrize("size", ["0", "-4"])
+def test_group_size_refuses_zero_and_negatives(size: str) -> None:
+ with pytest.raises(argparse.ArgumentTypeError):
+ group_size(size)
+
+
+def test_report_names_the_stored_shape_of_a_conv_weight(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
+ argv = ["quantize", str(GOLDEN / "tiny.safetensors"), str(tmp_path / "out.safetensors"), "--bits", "8"]
+ assert main(argv) == 0
+ assert "quantized conv.weight [6, 16, 4] as [6, 64]: max error" in capsys.readouterr().out
+
+
def test_quantize_fails_when_no_tensor_is_selected(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None:
destination = tmp_path / "out.safetensors"
argv = ["quantize", str(GOLDEN / "tiny.safetensors"), str(destination), "--bits", "8", "--include", "*.bias"]
@@ -95,7 +179,7 @@ def test_f16_stores_the_float_tensors_that_stay_as_f16(tmp_path: Path) -> None:
safe_open(GOLDEN / "tiny.safetensors", "numpy") as source,
):
assert quantized.get_tensor("layer.weight").dtype == np.int8
- for name in ("embed.weight", "layer.bias", "norm.weight"):
+ for name in ("embed.weight", "layer.bias", "norm.weight", "odd.weight"):
stored = quantized.get_tensor(name)
assert stored.dtype == np.float16
np.testing.assert_array_equal(stored, source.get_tensor(name).astype(np.float16))
@@ -115,4 +199,7 @@ def test_drop_leaves_matching_tensors_out(tmp_path: Path) -> None:
argv = ["quantize", str(GOLDEN / "tiny.safetensors"), str(destination), "--bits", "8", "--drop", "norm.*"]
assert main(argv) == 0
with safe_open(destination, "numpy") as quantized:
- assert set(quantized.keys()) == {"embed.weight", "layer.bias", "layer.weight", "layer.weight_scale"}
+ dropped = {"norm.weight"}
+ scales = {"conv.weight_scale", "layer.weight_scale"}
+ with safe_open(GOLDEN / "tiny.safetensors", "numpy") as source:
+ assert set(quantized.keys()) == set(source.keys()) - dropped | scales