diff --git a/apps/docs/src/components/QuantFile.astro b/apps/docs/src/components/QuantFile.astro index 7ecf7d67..3d388bbb 100644 --- a/apps/docs/src/components/QuantFile.astro +++ b/apps/docs/src/components/QuantFile.astro @@ -4,14 +4,13 @@ * tensors around them. Names are all-MiniLM-L6-v2's, shortened; the floats * are F16 as `--f16` stores them. */ const rows = [ - { name: '…word_embeddings.weight', dtype: 'F16', shape: '[30522, 384]', loads: 'float' }, { name: '…query.weight', dtype: 'I8', shape: '[384, 384]', loads: 'int8 weight', quant: true }, { name: '…query.weight_scale', dtype: 'F16', shape: '[384, 12]', loads: 'its scales', quant: true }, { name: '…query.bias', dtype: 'F16', shape: '[384]', loads: 'float' }, { name: '…LayerNorm.weight', dtype: 'F16', shape: '[384]', loads: 'float' }, ]; const rowHeight = 30; -const top = 84; +const top = 66; const height = top + rows.length * rowHeight + 14; --- @@ -23,9 +22,8 @@ const height = top + rows.length * rowHeight + 14; > __metadata__ runntime.quant = - {'{'}"version":1,"bits":8, - "groupSize":32,"scheme":"symmetric"{'}'} - + {'{'}"version":1,"scheme":"symmetric"{'}'} + { rows.map((row, i) => { const y = top + i * rowHeight + 12; diff --git a/apps/docs/src/content/docs/core/exporting-weights.mdx b/apps/docs/src/content/docs/core/exporting-weights.mdx index a5cab8ce..2973d89d 100644 --- a/apps/docs/src/content/docs/core/exporting-weights.mdx +++ b/apps/docs/src/content/docs/core/exporting-weights.mdx @@ -124,16 +124,31 @@ Full / half-precision files work as is. Optionally, you can quantize your model **How the runtime recognizes it.** A quantized file has a `runntime.quant` entry in its header metadata. Each quantized weight is stored as `I8` (int8) or `U8` (two int4 values per byte), with its scales next to it under -the same name plus `_scale`. The entry can set the bits and group size for -the whole file; without them, each weight takes its bits from its dtype and +the same name plus `_scale`. Each weight takes its bits from its dtype and its group size from the shape of its scales. Every other tensor is a normal float tensor. An int tensor in a file without `runntime.quant` fails to load. -**Which layers are quantized.** `Linear`, `Conv1d` and `ConvTranspose1d` -weights. Embedding tables, biases and norms stay float, because the model -reads them as float. `runntime quantize` only quantizes 2D tensors, so -today it covers Linear weights and leaves conv weights float. +**What is in the file.** `runntime quantize` quantizes every float tensor +named `*.weight` that has 2 or more dims: Linear and conv weights. Biases +and norms stay float, and so do embedding tables and `lm_head`, because +quantizing them usually costs too much quality. It stores each quantized +tensor as rows, `[first dim, everything else]`, so a `Conv1d` weight +`[out, in, k]` becomes `[out, in·k]`, and groups of `--group-size` values +run along that row. The group size has to divide the row: `in` for a +Linear, `in·k` for a conv. A tensor it doesn't divide stays float. The +runtime loads each tensor back into its layer's shape. + +**Which layers run quantized.** `Linear`, `Conv1d` and `ConvTranspose1d` +run on the int weights. Any other quantized tensor, like a `Conv2d` weight +or an embedding table you chose to quantize, is turned back into floats at +load. It keeps the rounding of its ints, so the download is smaller but +the model is no faster and uses no less memory than the float one. A +table kept on the CPU takes 4 bytes per value. + +A `Linear` or conv weight whose shape the quantized kernels can't take +also loads as float, and a console warning names it. To stay on the +quantized kernels, keep `--group-size` a multiple of 4. **Making a file.** `runntime quantize` lives in the repository under `tools/python` and runs with [uv](https://docs.astral.sh/uv/): @@ -146,9 +161,10 @@ uv run --no-dev runntime quantize model.safetensors model.int8.safetensors --bit - `--bits 8` or `--bits 4`. Required. - `--f16` stores the tensors that stay float as `F16`. -- `--group-size N`: how many inputs share one scale. Default 32. -- `--include` / `--exclude GLOB`: which tensors to quantize. The defaults - quantize `*.weight` and skip embeddings and `lm_head`. +- `--group-size N`: how many values of a row share one scale. Default 32. +- `--include GLOB`: which tensors to quantize. Default `*.weight`. +- `--exclude GLOB`: tensors to keep float, like a layer that loses too + much. Replaces the default `*embed*`, `*pos_emb*` and `lm_head.*`. - `--drop GLOB`: tensors to leave out of the file, like a pooler the model never reads. @@ -156,10 +172,13 @@ The tool prints one line per tensor with its error against the float weights, or why it was skipped. **Picking a setting.** On all-MiniLM-L6-v2, int8 keeps every embedding at -cosine similarity 0.9998 or higher against the float model; int4 drops to -0.94 and changes the top 3 search results for 8 of 20 queries. Use int8. -Try int4 only when the download size matters more, and check it on your -own data. +cosine similarity 0.9997 or higher against the float model. int4 drops as +low as 0.93 and reorders the top 3 search results for 12 of 20 queries, +where int8 reorders them for 2. Use int8. Try int4 only when the download +size matters more, and check it on your own data. + +Use int8 for models built on 2D convolutions. int4 breaks the outputs of +MobileNetV4, DepthART and YOLO26. A quantization-aware trained checkpoint quantizes the same way if it was trained on symmetric int8 or int4 with one scale per group of inputs. diff --git a/packages/runntime/src/core/index.ts b/packages/runntime/src/core/index.ts index 98cd1af9..25bb3e14 100644 --- a/packages/runntime/src/core/index.ts +++ b/packages/runntime/src/core/index.ts @@ -151,6 +151,7 @@ export { foldBnIntoConv } from './weights/foldBn.ts'; export { concatRows, dequantize, + inTorchShape, permuteRows, reshapeWeight, scaleWeight, diff --git a/packages/runntime/src/core/nn/layers.ts b/packages/runntime/src/core/nn/layers.ts index 58b81b6e..a5a11bdd 100644 --- a/packages/runntime/src/core/nn/layers.ts +++ b/packages/runntime/src/core/nn/layers.ts @@ -20,10 +20,7 @@ import { snake, } from '../graph/ops.ts'; import { matrix, type Value } from '../graph/value.ts'; -import { - MATMUL_QUANT_TILED_BK, - matmulQuantTiledEligible, -} from '../kernels/matmul/matmulQuantTiled.ts'; +import { matmulQuantTiledEligible } from '../kernels/matmul/matmulQuantTiled.ts'; import { convToTapMajor, f16ToF32, @@ -35,7 +32,12 @@ import { } from '../weights/convert.ts'; import { foldBnIntoConv } from '../weights/foldBn.ts'; import { packQuantW } from '../weights/quantPack.ts'; -import { derivedTensor, type LazyStateDict, type QuantTensor } from '../weights/safetensors.ts'; +import { + derivedTensor, + type LazyStateDict, + type LazyTensor, + type QuantTensor, +} from '../weights/safetensors.ts'; import { dequantize, readFloats, transposeWeight } from '../weights/transforms.ts'; import { tensor } from '../gpu/buffers.ts'; import { CpuParameter, Module, Parameter, type AnyModule } from './module.ts'; @@ -138,6 +140,14 @@ function loadQuantWeight( return scale; } +/** A quantized weight its layer's quantized kernel cannot run, as its + * dequantized f32 values in torch's `shape`, and noted on `sd` for + * loadStateDict's warning. */ +function asFloat(sd: LazyStateDict, t: QuantTensor, shape: readonly number[]): LazyTensor { + (sd.quantizedAsFloat ??= []).push(t.name); + return dequantize(t, shape); +} + /** `t`'s scales in f32, as stored. */ async function scalesF32(t: QuantTensor): Promise { const data = await t.scales(); @@ -161,7 +171,8 @@ async function scaleColumns(t: QuantTensor, groups: number, n: number): Promise< * A quantized checkpoint weight loads as quantW words with its f32 scales in * `weightScale` [in / groupSize, out], and forward() runs the quantized * matmul. Only the checkpoint says which kind it is, so transformStateDict - * retypes the slot at load. + * retypes the slot at load. A weight the quantized matmul cannot run loads + * as its dequantized floats. * * `addend` and `activation` fold a residual add and a following tanh or GELU * into the matmul epilogue, as act(x·W + bias) + addend. */ @@ -202,33 +213,25 @@ export class Linear extends Module { `expected torch layout [${this.outFeatures}, ${this.inFeatures}]`, ); } - if (tensor.kind === 'quant') { - this.loadQuantized(sd, prefix, name, tensor); + if (tensor.kind !== 'quant') { + sd.tensors.set(name, transposeWeight(tensor)); return; } - sd.tensors.set(name, transposeWeight(tensor)); - } - /** Loads `name` as quantW words [in, out] with its scales + const { bits, groupSize } = tensor; + // The quantized matmul reads whole tiles of 4 columns and 8 rows, groups + // of whole 4-row chunks, and packs int4 columns 8 to a word. + const fits = + matmulQuantTiledEligible(this.inFeatures, this.outFeatures, groupSize) && + this.outFeatures % (32 / bits) === 0; + if (fits) this.loadQuantized(sd, prefix, tensor); + else sd.tensors.set(name, transposeWeight(asFloat(sd, tensor, tensor.shape))); + } + /** Loads the weight as quantW words [in, out] with its scales * [in / groupSize, out]. */ - private loadQuantized( - sd: LazyStateDict, - prefix: string, - name: string, - tensor: QuantTensor, - ): void { + private loadQuantized(sd: LazyStateDict, prefix: string, tensor: QuantTensor): void { const { bits, groupSize } = tensor; const k = this.inFeatures; const n = this.outFeatures; - // The quantized matmul reads whole tiles of 4 columns and 8 rows and - // packs int4 columns 8 to a word. - if (!matmulQuantTiledEligible(k, n, groupSize) || n % (32 / bits) !== 0) { - throw new RunntimeError( - 'CHECKPOINT_MISMATCH', - `Linear: ${name} is int${bits} in groups of ${groupSize}, which needs in features ` + - `a multiple of ${MATMUL_QUANT_TILED_BK} and of the group size, out features a ` + - `multiple of ${32 / bits} and a group size a multiple of 4, got [out, in] = [${n}, ${k}]`, - ); - } this.quantization = { bits, groupSize }; const groups = k / groupSize; this.weightScale = loadQuantWeight( @@ -350,7 +353,7 @@ export class Conv1d extends Module { this.loadQuantized(sd, prefix, name, tensor, n); return; } - tensor = dequantize(tensor, [cOut, cIn, kernel]); + tensor = asFloat(sd, tensor, [cOut, cIn, kernel]); } // Without this a weight with equal in/out channels loads scrambled. if ( @@ -528,7 +531,7 @@ export class ConvTranspose1d extends Module { if (cOut % 4 === 0 && n % (32 / w.bits) === 0 && quantConvFits(cIn, taps, n)) { this.loadQuantized(sd, prefix, w); } else { - w = dequantize(w, [cIn, cOut, k]); + w = asFloat(sd, w, [cIn, cOut, k]); } } if (w && w.kind !== 'quant') { diff --git a/packages/runntime/src/core/nn/loadStateDict.ts b/packages/runntime/src/core/nn/loadStateDict.ts index 4d661883..f514af18 100644 --- a/packages/runntime/src/core/nn/loadStateDict.ts +++ b/packages/runntime/src/core/nn/loadStateDict.ts @@ -8,7 +8,12 @@ import { defaultRoot } from '../gpu/context.ts'; import { inGpuErrorScopes } from '../gpu/errorScopes.ts'; import { materialized, type Value, type ValueMeta } from '../graph/value.ts'; import { bf16ToF16Bits, f32ArrayToF16Bits } from '../weights/convert.ts'; -import type { DerivedTensor, FileTensor, LazyStateDict } from '../weights/safetensors.ts'; +import { + derivedTensor, + type DerivedTensor, + type FileTensor, + type LazyStateDict, +} from '../weights/safetensors.ts'; import { CpuParameter, type AnyModule } from './module.ts'; export type UploadFn = (data: Float32Array | Uint32Array | Uint16Array, shape: ValueMeta) => Value; @@ -120,6 +125,13 @@ async function bindStateDict( } }; visit(model, ''); + if (sd.quantizedAsFloat?.length) { + console.warn( + 'loadStateDict: quantized weights loaded as float, the quantized kernel of their layer ' + + `cannot run their shape or group size: ${sd.quantizedAsFloat.join(', ')}`, + ); + sd.quantizedAsFloat = undefined; + } const params = [...model.namedParameters()]; const paramNames = new Set(params.map(([n]) => n)); @@ -143,15 +155,20 @@ async function bindStateDict( const total = sd.totalBytes(); let done = 0; for (const [name, p] of params) { - const t = sd.tensors.get(name)!; + let t = sd.tensors.get(name)!; // Linear's and the convs' hooks turn a quantized weight into quantW - // words, so a QuantTensor still here is bound to something else. + // words, so a QuantTensor still here is bound to something no quantized + // kernel reads: an embedding table, a norm, a plain parameter. if (t.kind === 'quant') { - throw new RunntimeError( - 'CHECKPOINT_MISMATCH', - `loadStateDict: '${name}' is quantized (int${t.bits}), but only a Linear, Conv1d or ` + - `ConvTranspose1d weight loads quantized and this parameter holds ${p.shape.dtype}`, - ); + if (p.shape.dtype === 'quantW') { + throw new RunntimeError( + 'CHECKPOINT_MISMATCH', + `loadStateDict: '${name}' is quantized (int${t.bits}), but this quantW parameter ` + + 'takes packed words, which only its layer makes', + ); + } + const quant = t; + t = derivedTensor(quant.shape, () => quant.f32(), quant.byteLength); } const numel = t.shape.reduce((a, b) => a * b, 1); // A quantW slot counts u32 words under logical dims. Privacy-filter's diff --git a/packages/runntime/src/core/weights/safetensors.ts b/packages/runntime/src/core/weights/safetensors.ts index ac9599f8..618a17a2 100644 --- a/packages/runntime/src/core/weights/safetensors.ts +++ b/packages/runntime/src/core/weights/safetensors.ts @@ -317,8 +317,9 @@ export interface DerivedTensor extends LazyTensorBase { /** Symmetric int weights with one scale per `groupSize` values along the last * dim: value = int · scale of its group. `shape` is logical, one element per - * int even for int4, and `byteLength` covers the ints and the scales. The - * float accessors reject with CHECKPOINT_MISMATCH. */ + * int even for int4, and `byteLength` covers the ints and the scales. f32() + * gives the dequantized values, for a reader that only takes floats; the + * readers of stored bits reject with CHECKPOINT_MISMATCH. */ export interface QuantTensor extends LazyTensorBase { kind: 'quant'; /** What the tensor is, for error messages: its name in the file, or the @@ -338,6 +339,10 @@ export type LazyTensor = FileTensor | DerivedTensor | QuantTensor; export interface LazyStateDict { metadata: Record; tensors: Map; + /** Names of the quantized weights a Linear, Conv1d or ConvTranspose1d + * loaded as float because its quantized kernel cannot run them. The + * layers' hooks fill it and loadStateDict warns about them. */ + quantizedAsFloat?: string[]; totalBytes(): number; } @@ -410,6 +415,19 @@ function fileTensor(name: string, info: SafeTensorInfo, source: RangeSource): Fi }; } +/** The values `ints` stand for in f32: each int times the scale of its group + * of `groupSize`. */ +export function dequantizeInts( + ints: Int8Array, + scales: Float32Array | Uint16Array, + groupSize: number, +): Float32Array { + const f32 = scales instanceof Float32Array ? scales : f16ToF32(scales); + const out = new Float32Array(ints.length); + for (let i = 0; i < out.length; i++) out[i] = ints[i]! * f32[Math.floor(i / groupSize)]!; + return out; +} + /** A QuantTensor over `ints` and `scales`. Like derivedTensor it holds * nothing between reads. */ export function quantTensor( @@ -422,10 +440,14 @@ export function quantTensor( Promise.reject( new RunntimeError( 'CHECKPOINT_MISMATCH', - `tensor ${parts.name} is quantized (int${parts.bits}, groups of ${parts.groupSize}) and has no float reading`, + `tensor ${parts.name} is quantized (int${parts.bits}, groups of ${parts.groupSize}) and stores no float bits`, ), ); - return { kind: 'quant', ...parts, f32: refuse, halfWords: refuse, words: refuse, u16: refuse }; + const f32 = async () => { + const [ints, scales] = await Promise.all([parts.ints(), parts.scales()]); + return dequantizeInts(ints, scales, parts.groupSize); + }; + return { kind: 'quant', ...parts, f32, halfWords: refuse, words: refuse, u16: refuse }; } /** Unpacks int4 bytes, two values each with the low nibble first, into one diff --git a/packages/runntime/src/core/weights/transforms.ts b/packages/runntime/src/core/weights/transforms.ts index ab28f7ea..060888d9 100644 --- a/packages/runntime/src/core/weights/transforms.ts +++ b/packages/runntime/src/core/weights/transforms.ts @@ -11,10 +11,17 @@ * A QuantTensor source gives a QuantTensor, except in `dequantize`: the * helpers move its ints and scales together. Whatever would split a scale * group (a column split off the group grid, a reshape of the last dim, a - * transpose) throws CHECKPOINT_MISMATCH. */ + * transpose) throws CHECKPOINT_MISMATCH; a hook that needs one calls + * `inTorchShape` first. */ import { RunntimeError } from '../error.ts'; import { f16ToF32 } from './convert.ts'; -import { derivedTensor, quantTensor, type LazyTensor, type QuantTensor } from './safetensors.ts'; +import { + dequantizeInts, + derivedTensor, + quantTensor, + type LazyTensor, + type QuantTensor, +} from './safetensors.ts'; /** f32 values, or f16 halfwords. */ type Floats = Float32Array | Uint16Array; @@ -27,12 +34,13 @@ const numel = (shape: readonly number[]): number => shape.reduce((a, b) => a * b const mismatch = (helper: string, why: string) => new RunntimeError('CHECKPOINT_MISMATCH', `${helper}: ${why}`); -/** Reads `t` as f16 halfwords when it holds F16 and as f32 otherwise. Throws +/** Reads `t` as f16 halfwords when it holds F16 and as f32 otherwise, a + * QuantTensor as its dequantized values. Throws * CHECKPOINT_MISMATCH, naming `helper`, on packed u32 words or on a derived * tensor whose data does not fill its shape. A file tensor's length is * checked by the header parser. */ export async function readFloats(helper: string, t: LazyTensor): Promise { - if (t.kind === 'quant') throw mismatch(helper, `${t.name} is quantized, not float`); + if (t.kind === 'quant') return t.f32(); if (t.kind === 'file') return t.dtype === 'F16' ? t.u16() : t.f32(); const data = await t.data(); if (data instanceof Uint32Array) throw mismatch(helper, 'source holds packed u32 words'); @@ -395,9 +403,18 @@ export function dequantize(t: QuantTensor, shape: readonly number[] = t.shape): [...shape], async () => { const [ints, scales] = await Promise.all([t.ints(), t.scales()]); - const f32 = toF32(scales); - return Float32Array.from(ints, (q, i) => q * f32[Math.floor(i / t.groupSize)]!); + return dequantizeInts(ints, scales, t.groupSize); }, t.byteLength, ); } + +/** `t` in torch's `shape`, for a hook that reshapes, transposes or splits it + * past dim 0. A file stores a quantized tensor as rows, [shape[0], everything + * else], with its scale groups along the row, so those steps would cut + * through the groups: it comes back as its dequantized floats in `shape`. + * Any other tensor comes back as it is. Throws CHECKPOINT_MISMATCH when a + * quantized tensor's element count is not `shape`'s. */ +export function inTorchShape(t: LazyTensor, shape: readonly number[]): LazyTensor { + return t.kind === 'quant' ? dequantize(t, shape) : t; +} diff --git a/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts b/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts index 8e1d069d..0b7ed045 100644 --- a/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts +++ b/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts @@ -13,6 +13,7 @@ import { concatRows, derivedTensor, f32ArrayToF16Bits, + inTorchShape, permuteRows, reshapeWeight, scaleWeight, @@ -175,7 +176,8 @@ export function transformEmbeddingGemma2VisionStateDict( put('input_proj.weight', take('vision_tower.patch_embedder.input_proj.weight')); // [2, positions, hidden]: the x table, then the y table. - const [x, y] = splitRows(take('vision_tower.patch_embedder.position_embedding_table'), [1, 1]); + const table = take('vision_tower.patch_embedder.position_embedding_table'); + const [x, y] = splitRows(inTorchShape(table, [2, positions, hidden]), [1, 1]); put('pos_x.weight', reshapeWeight(x!, [positions, hidden])); put('pos_y.weight', reshapeWeight(y!, [positions, hidden])); @@ -232,7 +234,9 @@ export function transformEmbeddingGemma2AudioStateDict( const [o, k, c] = [Math.floor(j / (9 * c0)), Math.floor(j / c0) % 9, j % c0]; return (o * c0 + c) * 9 + k; }); - const conv1 = reshapeWeight(take(`${sub}layer1.conv.weight`), [c1 * c0 * 9]); + const conv1 = reshapeWeight(inTorchShape(take(`${sub}layer1.conv.weight`), [c1, c0, 3, 3]), [ + c1 * c0 * 9, + ]); put('conv1.weight', reshapeWeight(permuteRows(conv1, tapsMajor), [c1, 9 * c0])); put('norm1.weight', take(`${sub}layer1.norm.weight`)); put('input_proj.weight', take(`${sub}input_proj_linear.weight`)); @@ -316,10 +320,8 @@ export function transformEmbeddingGemma2AudioStateDict( put(`${L}.lconv.pre_layer_norm.scale`, take(`${C}.pre_layer_norm.weight`)); clipped(`${C}.linear_start`, `${L}.lconv.linear_start`); // [hidden, 1, K] → [K, hidden]. - put( - `${L}.lconv.conv`, - transposeWeight(reshapeWeight(take(`${C}.depthwise_conv1d.weight`), [hidden, K])), - ); + const depthwise = inTorchShape(take(`${C}.depthwise_conv1d.weight`), [hidden, 1, K]); + put(`${L}.lconv.conv`, transposeWeight(reshapeWeight(depthwise, [hidden, K]))); put(`${L}.lconv.conv_norm.scale`, take(`${C}.conv_norm.weight`)); clipped(`${C}.linear_end`, `${L}.lconv.linear_end`); for (const n of ['norm_pre_attn', 'norm_post_attn', 'norm_out']) { diff --git a/packages/runntime/src/zoo/higgs/encoder.ts b/packages/runntime/src/zoo/higgs/encoder.ts index 922c7498..dd30d1ab 100644 --- a/packages/runntime/src/zoo/higgs/encoder.ts +++ b/packages/runntime/src/zoo/higgs/encoder.ts @@ -18,6 +18,7 @@ import { elu, gatherRowsFrom, gelu, + inTorchShape, logitStats, matrix, mul, @@ -188,10 +189,19 @@ class PositionConv extends nn.Module { * conv checks the shape of its rows. */ private weight(sd: LazyStateDict, prefix: string): LazyTensor { if (sd.tensors.has(at(prefix, 'conv.weight'))) return take(sd, prefix, 'conv.weight'); - const g = take(sd, prefix, 'conv.parametrizations.weight.original0'); - const v = take(sd, prefix, 'conv.parametrizations.weight.original1'); const { hidden, kernel, groups } = this; const width = hidden / groups; + // The norm folds per tap, across the scale groups of a quantized tensor. + const g = inTorchShape(take(sd, prefix, 'conv.parametrizations.weight.original0'), [ + 1, + 1, + kernel, + ]); + const v = inTorchShape(take(sd, prefix, 'conv.parametrizations.weight.original1'), [ + hidden, + width, + kernel, + ]); if ( v.shape.join() !== [hidden, width, kernel].join() || g.shape.join() !== [1, 1, kernel].join() diff --git a/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts b/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts index fd899902..4ec5ce73 100644 --- a/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts +++ b/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts @@ -30,6 +30,21 @@ function move(sd: LazyStateDict, from: string, to: string): void { sd.tensors.set(to, take(sd, from)); } +/** An expert tensor [E, …]. Throws CHECKPOINT_MISMATCH on a quantized one: a + * file stores it as rows [E, everything else], so splitting or folding it + * would mix the experts' values. */ +function takeExperts(sd: LazyStateDict, name: string): LazyTensor { + const t = take(sd, name); + if (t.kind === 'quant') { + throw checkpointMismatch( + 'privacy-filter state dict', + `'${name}' is quantized, and a quantized expert tensor cannot be split into experts: ` + + 'keep it float in the file', + ); + } + return t; +} + /** Folds every leading dim into rows: [E, K, H] → [E·K, H]. */ function foldRows(t: LazyTensor): LazyTensor { const cols = t.shape[t.shape.length - 1]!; @@ -61,13 +76,16 @@ export function transformHfPrivacyFilterStateDict(sd: LazyStateDict): void { } move(sd, `${src}.self_attn.sinks`, `${dst}.attn.sinks`); - const [gluWeight, linWeight] = splitHalves(take(sd, `${src}.mlp.experts.gate_up_proj`)); + const [gluWeight, linWeight] = splitHalves(takeExperts(sd, `${src}.mlp.experts.gate_up_proj`)); sd.tensors.set(`${dst}.mlp.gluWeight`, gluWeight); sd.tensors.set(`${dst}.mlp.linWeight`, linWeight); - const [gluBias, linBias] = splitHalves(take(sd, `${src}.mlp.experts.gate_up_proj_bias`)); + const [gluBias, linBias] = splitHalves(takeExperts(sd, `${src}.mlp.experts.gate_up_proj_bias`)); sd.tensors.set(`${dst}.mlp.gluBias`, gluBias); sd.tensors.set(`${dst}.mlp.linBias`, linBias); - sd.tensors.set(`${dst}.mlp.outWeight`, foldRows(take(sd, `${src}.mlp.experts.down_proj`))); + sd.tensors.set( + `${dst}.mlp.outWeight`, + foldRows(takeExperts(sd, `${src}.mlp.experts.down_proj`)), + ); move(sd, `${src}.mlp.experts.down_proj_bias`, `${dst}.mlp.outBias`); n++; } diff --git a/packages/runntime/test/core/nn/layers.gpu.test.ts b/packages/runntime/test/core/nn/layers.gpu.test.ts index d9348923..78a940f7 100644 --- a/packages/runntime/test/core/nn/layers.gpu.test.ts +++ b/packages/runntime/test/core/nn/layers.gpu.test.ts @@ -1,8 +1,8 @@ -import { describe, it, onTestFinished } from 'vitest'; +import { describe, expect, it, onTestFinished, vi } from 'vitest'; import { toArray } from '../../../src/core/gpu/eval.ts'; import type { SlotAct } from '../../../src/core/graph/ops.ts'; import { matrix } from '../../../src/core/graph/value.ts'; -import { Conv1d, ConvTranspose1d, Linear } from '../../../src/core/nn/layers.ts'; +import { Conv1d, ConvTranspose1d, Embedding, Linear } from '../../../src/core/nn/layers.ts'; import { bufferSource, fromSafetensors, @@ -26,8 +26,24 @@ interface Case { /** Takes the tiled kernel, which stages the dequantized weight in f16 * workgroup memory for an f16 input. */ tiled?: boolean; + /** The bits at which the quantized matmul cannot take the weight, so it + * loads as its dequantized floats in the layer's dtype. */ + float?: readonly (4 | 8)[]; } +/** Watches console.warn for the rest of the test. Returns a reader of the + * loadStateDict warnings so far; TypeGPU warns there too. */ +function loadWarnings(): () => unknown[] { + const warn = vi.spyOn(console, 'warn'); + onTestFinished(() => warn.mockRestore()); + return () => + warn.mock.calls + .map(([message]) => message as unknown) + .filter((message) => typeof message === 'string' && message.startsWith('loadStateDict:')); +} + +const LOADED_AS_FLOAT = /^loadStateDict: quantized weights loaded as float.*: weight$/; + /** A Linear loaded from a quantized file matches a CPU loop of * act(x·W + bias) + addend in f64, with W dequantized from the same ints as * int · f16 scale. */ @@ -54,14 +70,18 @@ async function checkQuantizedLinear(c: Case, bits: 4 | 8, dtype: 'f32' | 'f16') }); const sd = await fromSafetensors(bufferSource(file), { chunkBytes: 0 }); if (c.bias) sd.tensors.set('bias', memoryStateDict({ bias }).tensors.get('bias')!); + const warned = loadWarnings(); await layer.loadStateDict(sd, { root }); + const float = c.float?.includes(bits) ?? false; + expect(layer.weight.shape.dtype).toBe(float ? dtype : 'quantW'); + expect(warned()).toEqual(float ? [expect.stringMatching(LOADED_AS_FLOAT)] : []); const x = upload(root, xs, { ...matrix(m, inF), dtype }); const call = c.addend ? { addend: upload(root, addend, { ...matrix(m, outF), dtype }) } : {}; const got = await toArray(layer.forward(x, call), ex); const dequant = Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / groupSize)]!); - const weight = dtype === 'f16' && c.tiled ? roundF16(dequant) : dequant; + const weight = dtype === 'f16' && (c.tiled || float) ? roundF16(dequant) : dequant; const want = new Float64Array(m * outF); for (let i = 0; i < m; i++) { for (let o = 0; o < outF; o++) { @@ -77,6 +97,22 @@ async function checkQuantizedLinear(c: Case, bits: 4 | 8, dtype: 'f32' | 'f16') // output to half precision, 2^-11 relative. const tol = dtype === 'f16' ? { atol: 1e-5 * inF, rtol: 1e-3 } : { atol: 1e-6 * inF, rtol: 1e-5 }; expectClose(got, want, tol); + if (!float) return; + + // The fallback is the float layer: a Linear loaded from the dequantized + // weight takes the same route and gives the same bits. + const reference = new Linear(inF, outF, { bias: c.bias, activation: c.activation }); + if (dtype === 'f16') reference.half(root); + onTestFinished(() => reference.dispose()); + await reference.loadStateDict( + memoryStateDict({ + weight: { data: dequant, shape: [outF, inF] }, + ...(c.bias ? { bias } : {}), + }), + { root }, + ); + expect(reference.weight.shape.dtype).toBe(dtype); + expectClose(await toArray(reference.forward(x, call), ex), got, { atol: 0, rtol: 0 }); } describe('Linear with a quantized weight', () => { @@ -170,6 +206,100 @@ describe('Linear with a quantized weight', () => { }); }); +describe('Linear with a quantized weight at the limits of the quantized matmul', () => { + // It takes in features a multiple of 8 and of the group size, out features + // a multiple of 4 (int8) or 8 (int4), and a group size a multiple of 4. + // Anything else runs the float matmul on the dequantized weight. + const base = { m: 3, bias: true, dtypes: ['f32', 'f16'] } as const; + const CASES: (Case & { name: string; dtypes: readonly ('f32' | 'f16')[] })[] = [ + // quantized: in 8 % 8 == 0, out 8 % 8 == 0, groupSize 4 % 4 == 0 + { ...base, name: 'the smallest that fits', inF: 8, outF: 8, groupSize: 4 }, + // float: in 12 % 8 != 0 + { ...base, name: 'in off the 8-row tile', inF: 12, outF: 8, groupSize: 4, float: [4, 8] }, + // float: in 4 % 8 != 0 + { ...base, name: 'in under one tile', inF: 4, outF: 8, groupSize: 4, float: [4, 8] }, + // int8 quantized: out 4 % 4 == 0. int4 float: out 4 % 8 != 0 + { ...base, name: 'out of one int8 word', inF: 8, outF: 4, groupSize: 4, float: [4] }, + // int8 quantized: out 12 % 4 == 0. int4 float: out 12 % 8 != 0 + { ...base, name: 'out off the int4 word', inF: 16, outF: 12, groupSize: 8, float: [4] }, + // float: out 6 % 4 != 0 + { ...base, name: 'out off the int8 word', inF: 16, outF: 6, groupSize: 8, float: [4, 8] }, + // float: out 1 + { ...base, name: 'one output', inF: 16, outF: 1, groupSize: 8, float: [4, 8] }, + // quantized: groupSize 12 % 4 == 0 and 24 % 12 == 0 + { ...base, name: 'groups of 12', inF: 24, outF: 8, groupSize: 12 }, + // float: groupSize 6 % 4 != 0 + { ...base, name: 'groups of 6', inF: 24, outF: 8, groupSize: 6, float: [4, 8] }, + // float: groupSize 2 % 4 != 0 + { ...base, name: 'groups of 2', inF: 16, outF: 8, groupSize: 2, float: [4, 8] }, + // float: groupSize 1 + { ...base, name: 'a scale per value', inF: 16, outF: 8, groupSize: 1, float: [4, 8] }, + // float, with the epilogue the float matmul folds + { + ...base, + name: 'in off the tile with gelu and addend', + m: 37, + inF: 100, + outF: 30, + groupSize: 20, + addend: true, + activation: 'gelu', + float: [4, 8], + }, + ]; + it.for( + CASES.flatMap(({ dtypes, ...c }) => + dtypes.flatMap((dtype) => ([4, 8] as const).map((bits) => ({ ...c, dtype, bits }))), + ), + )('$dtype int$bits $name: [$m, $inF] → $outF, groupSize $groupSize', async (c, ctx) => { + if (c.dtype === 'f16') requireFeature(ctx, root, 'shader-f16'); + await checkQuantizedLinear(c, c.bits, c.dtype); + }); +}); + +describe('Embedding with a quantized table', () => { + // No kernel reads a quantized lookup table, so it loads as its dequantized + // floats, without a warning: on the GPU in the model's dtype, on the CPU as + // f32. + const [vocab, dim, groupSize] = [10, 6, 3]; + const ids = [9, 0, 4, 4]; + it.for( + (['gpu', 'cpu'] as const).flatMap((device) => + ([4, 8] as const).flatMap((bits) => + (['f32', 'f16'] as const).map((dtype) => ({ device, bits, dtype })), + ), + ), + )('$device table, int$bits, $dtype', async ({ device, bits, dtype }, ctx) => { + if (dtype === 'f16') requireFeature(ctx, root, 'shader-f16'); + const half = 2 ** (bits - 1); + const ints = Array.from(randomF32(vocab * dim, 1), (v) => + Math.min(half - 1, Math.floor(v * half)), + ); + const scales = roundF16( + randomF32((vocab * dim) / groupSize, 2).map((v) => (Math.abs(v) + 0.1) / half), + ); + const table = new Embedding(vocab, dim, { device }); + if (dtype === 'f16') table.half(root); + onTestFinished(() => table.dispose()); + const file = quantFile(bits, groupSize, { + weight: { ints, scales: [...scales], shape: [vocab, dim] }, + }); + const warned = loadWarnings(); + await table.loadStateDict(await fromSafetensors(bufferSource(file), { chunkBytes: 0 }), { + root, + }); + expect(warned()).toEqual([]); + const got = await toArray(table.forward(ids), ex); + + const dequant = Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / groupSize)]!); + // A GPU table is stored in the model's dtype; a CPU one stays f32 and its + // gathered rows round on upload. + const stored = dtype === 'f16' ? roundF16(dequant) : dequant; + const want = ids.flatMap((id) => [...stored.subarray(id * dim, (id + 1) * dim)]); + expectClose(got, want, { atol: 0, rtol: 0 }); + }); +}); + interface ConvCase { name: string; kind: 'conv' | 'transpose'; @@ -183,6 +313,9 @@ interface ConvCase { groupSize: number; bias: boolean; activation?: 'gelu'; + /** The quantized matmul cannot run this conv at any bits, so the weight + * loads as its dequantized floats. */ + float?: boolean; } /** A conv loaded from a quantized file of its torch weight, [out, in·kernel] @@ -209,10 +342,11 @@ async function checkQuantizedConv(c: ConvCase, bits: 4 | 8, dtype: 'f32' | 'f16' c.kind === 'conv' ? new Conv1d(cIn, cOut, kernel, { stride, padding, bias: c.bias, activation: c.activation }) : new ConvTranspose1d(cIn, cOut, kernel, { stride, padding, bias: c.bias }); - const run = async (layer: Conv1d | ConvTranspose1d, sd: LazyStateDict) => { + const run = async (layer: Conv1d | ConvTranspose1d, sd: LazyStateDict, quantized = false) => { if (dtype === 'f16') layer.half(root); onTestFinished(() => layer.dispose()); await layer.loadStateDict(sd, { root }); + if (quantized) expect(layer.weight.shape.dtype).toBe(c.float ? dtype : 'quantW'); return toArray(layer.forward(upload(root, xs, { ...matrix(tIn, cIn), dtype })), ex); }; const file = quantFile(bits, groupSize, { @@ -220,7 +354,9 @@ async function checkQuantizedConv(c: ConvCase, bits: 4 | 8, dtype: 'f32' | 'f16' }); const quantSd = await fromSafetensors(bufferSource(file), { chunkBytes: 0 }); if (c.bias) quantSd.tensors.set('bias', memoryStateDict({ bias }).tensors.get('bias')!); - const got = await run(make(), quantSd); + const warned = loadWarnings(); + const got = await run(make(), quantSd, true); + expect(warned()).toEqual(c.float ? [expect.stringMatching(LOADED_AS_FLOAT)] : []); const want = await run( make(), memoryStateDict({ weight: torchWeight, ...(c.bias ? { bias } : {}) }), @@ -306,6 +442,91 @@ describe('Conv1d and ConvTranspose1d with a quantized weight', () => { groupSize: 32, bias: false, }, + // The quantized matmul runs a Conv1d with in a multiple of 4 and kernel·in + // of 8, and a ConvTranspose1d that also has out a multiple of 4 and + // stride·out of 4 (int8) or 8 (int4). The rest run as float convs. + // quantized: in 4 % 4 == 0, kernel·in 8 % 8 == 0 + { + name: 'conv, the smallest that fits', + kind: 'conv', + tIn: 11, + cIn: 4, + cOut: 5, + kernel: 2, + stride: 1, + padding: 0, + groupSize: 8, + bias: true, + }, + // float: kernel·in 12 % 8 != 0 + { + name: 'conv, kernel·in off the 8-row tile', + kind: 'conv', + tIn: 11, + cIn: 4, + cOut: 5, + kernel: 3, + stride: 1, + padding: 1, + groupSize: 4, + bias: true, + float: true, + }, + // float: in 6 % 4 != 0, though kernel·in 24 % 8 == 0 + { + name: 'conv, in off 4', + kind: 'conv', + tIn: 11, + cIn: 6, + cOut: 8, + kernel: 4, + stride: 2, + padding: 1, + groupSize: 6, + bias: false, + float: true, + }, + // quantized: out 4 % 4 == 0, stride·out 8 % 8 == 0, taps·in 8 % 8 == 0 + { + name: 'transposed, the smallest that fits', + kind: 'transpose', + tIn: 7, + cIn: 4, + cOut: 4, + kernel: 4, + stride: 2, + padding: 1, + groupSize: 16, + bias: true, + }, + // float: out 6 % 4 != 0 + { + name: 'transposed, out off 4', + kind: 'transpose', + tIn: 7, + cIn: 8, + cOut: 6, + kernel: 4, + stride: 2, + padding: 1, + groupSize: 3, + bias: true, + float: true, + }, + // float: taps·in 12 % 8 != 0 + { + name: 'transposed, taps·in off the 8-row tile', + kind: 'transpose', + tIn: 7, + cIn: 4, + cOut: 4, + kernel: 6, + stride: 2, + padding: 2, + groupSize: 8, + bias: false, + float: true, + }, ]; it.for( (['f32', 'f16'] as const).flatMap((dtype) => diff --git a/packages/runntime/test/core/nn/layers.test.ts b/packages/runntime/test/core/nn/layers.test.ts index a9de2e0e..025d9f98 100644 --- a/packages/runntime/test/core/nn/layers.test.ts +++ b/packages/runntime/test/core/nn/layers.test.ts @@ -1,3 +1,4 @@ +import { readFileSync } from 'node:fs'; import { describe, expect, it, vi } from 'vitest'; import { ACT_CODE } from '../../../src/core/graph/ops/shared.ts'; import { pending, type ValueMeta } from '../../../src/core/graph/value.ts'; @@ -362,11 +363,18 @@ describe('Linear with a quantized weight', () => { }); }); + // The quantized matmul takes in features a multiple of 8 and of the group + // size, out features a multiple of 4 (int8) or 8 (int4) and a group size a + // multiple of 4. A case names the one limit it sits on. it.each([ - { name: 'in features off the 8-row tiles', inF: 12, outF: 4, bits: 8 as const, groupSize: 4 }, - { name: 'int4 out features off 8 per word', inF: 8, outF: 4, bits: 4 as const, groupSize: 4 }, - { name: 'a group size off 4', inF: 16, outF: 4, bits: 8 as const, groupSize: 2 }, - ])('refuses $name with CHECKPOINT_MISMATCH', async ({ inF, outF, bits, groupSize }) => { + // quantized: in 8 % 8 == 0, out 4 % 4 == 0, groups of 4 + { name: 'int8 [4, 8] in groups of 4', inF: 8, outF: 4, bits: 8 as const, groupSize: 4 }, + // quantized: int4 out 8 % 8 == 0 + { name: 'int4 [8, 8] in groups of 4', inF: 8, outF: 8, bits: 4 as const, groupSize: 4 }, + // quantized: one group over the whole row, 16 % 4 == 0 + { name: 'int8 [4, 16] in groups of 16', inF: 16, outF: 4, bits: 8 as const, groupSize: 16 }, + ])('loads $name as quantW', async ({ inF, outF, bits, groupSize }) => { + const layer = new Linear(inF, outF); const sd = await quantCheckpoint(bits, groupSize, { weight: { ints: new Array(inF * outF).fill(0), @@ -374,11 +382,46 @@ describe('Linear with a quantized weight', () => { shape: [outF, inF], }, }); + layer.transformStateDict(sd, ''); + expect(shapes(layer)).toEqual({ + weight: { elems: (inF * outF * bits) / 32, dtype: 'quantW', dims: [inF, outF] }, + bias: meta([outF]), + weightScale: meta([inF / groupSize, outF]), + }); + expect(sd.quantizedAsFloat).toBeUndefined(); + }); + + it.each([ + // float: in 12 % 8 != 0 + { name: 'in features off the 8-row tiles', inF: 12, outF: 4, bits: 8 as const, groupSize: 4 }, + // float: in 4 % 8 != 0 + { name: 'in features under one tile', inF: 4, outF: 4, bits: 8 as const, groupSize: 4 }, + // float: int8 out 6 % 4 != 0 + { name: 'int8 out features off 4 per word', inF: 8, outF: 6, bits: 8 as const, groupSize: 4 }, + // float: int4 out 4 % 8 != 0 + { name: 'int4 out features off 8 per word', inF: 8, outF: 4, bits: 4 as const, groupSize: 4 }, + // float: group size 2 % 4 != 0 + { name: 'a group size off 4', inF: 16, outF: 4, bits: 8 as const, groupSize: 2 }, + // float: group size 1 + { name: 'a scale per value', inF: 8, outF: 4, bits: 8 as const, groupSize: 1 }, + ])('loads $name as its dequantized floats', async ({ inF, outF, bits, groupSize }) => { + // ints −7 … 7 and one scale per group, (g % 5 + 1) / 4, f16-exact + const ints = Array.from({ length: inF * outF }, (_, i) => (i % 15) - 7); + const scales = Array.from({ length: (inF * outF) / groupSize }, (_, g) => ((g % 5) + 1) / 4); + const sd = await quantCheckpoint(bits, groupSize, { + weight: { ints, scales, shape: [outF, inF] }, + }); const layer = new Linear(inF, outF); - expect(() => layer.transformStateDict(sd, '')).toThrow( - expect.objectContaining({ code: 'CHECKPOINT_MISMATCH' }), - ); - expect(layer.weight.shape.dtype).toBe('f32'); + layer.transformStateDict(sd, ''); + expect(shapes(layer)).toEqual({ weight: meta([inF, outF]), bias: meta([outF]) }); + expect(layer.weightScale).toBeUndefined(); + expect(sd.quantizedAsFloat).toEqual(['weight']); + // stored [in, out]: row i, column o is torch value o·in + i + const want = Array.from({ length: inF * outF }, (_, j) => { + const at = (j % outF) * inF + Math.floor(j / outF); + return ints[at]! * scales[Math.floor(at / groupSize)]!; + }); + expect([...(await sd.tensors.get('weight')!.f32())]).toEqual(want); }); it.each([ @@ -484,6 +527,25 @@ describe('Conv1d and ConvTranspose1d with a quantized weight', () => { ]); }); + it.for([8, 4] as const)( + 'takes the int%i Conv1d weight the Python quantizer writes', + async (bits) => { + // tools/python's golden file holds torch [out 6, in 16, kernel 4] as + // [6, 64] in groups of 32. 6 outputs pad to the packing width, 8. + const golden = new URL( + `../../../../../tools/python/tests/golden/tiny.int${bits}.safetensors`, + import.meta.url, + ); + const sd = await fromSafetensors(bufferSource(readFileSync(golden))); + const layer = new Conv1d(16, 6, 4, { bias: false }); + layer.transformStateDict(sd, 'conv'); + expect(shapes(layer)).toEqual({ + weight: { elems: (64 * 8 * bits) / 32, dtype: 'quantW', dims: [64, 8] }, + weightScale: meta([2, 8]), + }); + }, + ); + it('keeps a Conv1d group that ends inside a channel', async () => { const layer = new Conv1d(8, 4, 3); // groups of 4 values: a channel's 3 taps and the next channel's first diff --git a/packages/runntime/test/core/nn/loadStateDict.test.ts b/packages/runntime/test/core/nn/loadStateDict.test.ts index db8f7ed1..5ac03e13 100644 --- a/packages/runntime/test/core/nn/loadStateDict.test.ts +++ b/packages/runntime/test/core/nn/loadStateDict.test.ts @@ -1,12 +1,13 @@ import type { TgpuRoot } from 'typegpu'; import { describe, expect, it, vi } from 'vitest'; import { pending, type ValueMeta } from '../../../src/core/graph/value.ts'; -import { Linear } from '../../../src/core/nn/layers.ts'; +import { Conv1d, ConvTranspose1d, Linear } from '../../../src/core/nn/layers.ts'; import { gpuUpload, loadLazyStateDict } from '../../../src/core/nn/loadStateDict.ts'; import { Buffer, CpuParameter, Module, + ModuleList, Parameter, type AnyModule, } from '../../../src/core/nn/module.ts'; @@ -21,6 +22,7 @@ import { type MemoryStateDict, } from '../../../src/core/weights/safetensors.ts'; import { f32ToF16Bits } from '../../../src/core/weights/convert.ts'; +import { concatRows } from '../../../src/core/weights/transforms.ts'; import { meta } from '../graph/values.ts'; import { safetensorsFile } from '../weights/checkpoint.ts'; @@ -244,10 +246,9 @@ describe('binding', () => { }); describe('a quantized tensor', () => { - // A [2, 2] int8 weight in groups of 2; the loader must refuse it before - // reading either part. + // A [2, 2] int8 weight in groups of 2: ints 1 2 | 3 4, scales 0.5 and 2. const ints = vi.fn(async () => new Int8Array([1, 2, 3, 4])); - const scales = vi.fn(async () => new Float32Array([1, 2])); + const scales = vi.fn(async () => new Float32Array([0.5, 2])); const quant = quantTensor({ name: 'w', shape: [2, 2], @@ -257,30 +258,172 @@ describe('a quantized tensor', () => { ints, scales, }); + const DEQUANTIZED = [0.5, 1, 6, 8]; it.each([ - { name: 'an f32 parameter', param: () => new Parameter(meta([2, 2])), dtype: 'f32' }, - { name: 'an f16 parameter', param: () => new Parameter(meta([2, 2], 'f16')), dtype: 'f16' }, - { name: 'a CpuParameter', param: () => new CpuParameter(meta([2, 2])), dtype: 'f32' }, - { - name: 'a quantW parameter', - param: () => new Parameter({ elems: 1, dtype: 'quantW', dims: [2, 2] }), - dtype: 'quantW', - }, - ])('is refused by $name with CHECKPOINT_MISMATCH', async ({ param, dtype }) => { - class NotLinear extends Module { - weight = param(); + { name: 'f32', dtype: 'f32' as const, want: new Float32Array(DEQUANTIZED) }, + { name: 'f16', dtype: 'f16' as const, want: Uint16Array.from(DEQUANTIZED, f32ToF16Bits) }, + ])('loads into an $name parameter as its dequantized values', async ({ dtype, want }) => { + class Table extends Module { + weight = new Parameter(meta([2, 2], dtype)); + } + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const upload = recorder(); + await load(new Table(), { weight: quant }, upload); + expect(upload.mock.calls).toEqual([[want, meta([2, 2], dtype)]]); + // No quantized kernel reads a plain parameter, so there is nothing to say. + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it('loads into a CpuParameter as its dequantized f32 values, off the GPU', async () => { + class Table extends Module { + weight = new CpuParameter(meta([2, 2])); + } + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const upload = recorder(); + const table = new Table(); + await load(table, { weight: quant }, upload); + expect(table.weight.data).toEqual(new Float32Array(DEQUANTIZED)); + expect(upload).not.toHaveBeenCalled(); + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it('is refused by a quantW parameter with CHECKPOINT_MISMATCH, before any read', async () => { + class Packed extends Module { + weight = new Parameter({ elems: 1, dtype: 'quantW', dims: [2, 2] }); } + ints.mockClear(); const upload = recorder(); - await expect(load(new NotLinear(), { weight: quant }, upload)).rejects.toMatchObject({ + await expect(load(new Packed(), { weight: quant }, upload)).rejects.toMatchObject({ code: 'CHECKPOINT_MISMATCH', message: - "loadStateDict: 'weight' is quantized (int8), but only a Linear, Conv1d or " + - `ConvTranspose1d weight loads quantized and this parameter holds ${dtype}`, + "loadStateDict: 'weight' is quantized (int8), but this quantW parameter takes packed " + + 'words, which only its layer makes', }); expect(upload).not.toHaveBeenCalled(); expect(ints).not.toHaveBeenCalled(); - expect(scales).not.toHaveBeenCalled(); + }); +}); + +describe('the warning about quantized weights loaded as float', () => { + // int8 in groups of 4, all zeros. + const named = (prefix: string, rows: number, cols: number) => ({ + [`${prefix}.weight`]: { data: new Int8Array(rows * cols), shape: [rows, cols] }, + [`${prefix}.weight_scale`]: { + data: new Uint16Array((rows * cols) / 4), + shape: [rows, cols / 4], + }, + }); + const setting = { 'runntime.quant': JSON.stringify({ version: 1, scheme: 'symmetric' }) }; + const open = (tensors: Parameters[0]) => + fromSafetensors(bufferSource(safetensorsFile(tensors, setting)), { chunkBytes: 0 }); + const WARNING = + 'loadStateDict: quantized weights loaded as float, the quantized kernel of their layer ' + + 'cannot run their shape or group size: '; + + class Table extends Module { + weight = new Parameter(meta([4, 4])); + } + class Net extends Module { + // quantized: [out 4, in 8], in 8 % 8 == 0 + fc = new Linear(8, 4, { bias: false }); + // float, warned: [out 4, in 12], in 12 % 8 != 0 + odd = new Linear(12, 4, { bias: false }); + // float, silent: no quantized kernel reads a plain parameter + table = new Table(); + // float, warned: Conv1d [out 4, in 3, kernel 4] as [4, 12], in 3 % 4 != 0 + conv = new Conv1d(3, 4, 4, { bias: false }); + // float, warned: ConvTranspose1d [in 4, out 2, kernel 2] as [4, 4], out 2 % 4 != 0 + up = new ConvTranspose1d(4, 2, 2, { stride: 2, bias: false }); + } + class Holder extends Module { + table = new Table(); + } + class Fits extends Module { + fc = new Linear(8, 4, { bias: false }); + } + + it('names every weight whose layer has a quantized kernel that cannot run it', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const sd = await open({ + ...named('fc', 4, 8), + ...named('odd', 4, 12), + ...named('table', 4, 4), + ...named('conv', 4, 12), + ...named('up', 4, 4), + }); + const upload = recorder(); + await loadLazyStateDict(new Net(), sd, { upload }); + expect(warn.mock.calls).toEqual([[`${WARNING}odd.weight, conv.weight, up.weight`]]); + expect(upload.mock.calls.map(([, shape]) => shape.dtype)).toEqual([ + 'quantW', + 'f32', + 'f32', + 'f32', + 'f32', + 'f32', + ]); + warn.mockRestore(); + }); + + it('lists every name, however many', async () => { + class Many extends Module { + layers = new ModuleList(Array.from({ length: 12 }, () => new Linear(12, 4, { bias: false }))); + } + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const tensors = Object.assign( + {}, + ...Array.from({ length: 12 }, (_, i) => named(`layers.${i}`, 4, 12)), + ) as Parameters[0]; + await loadLazyStateDict(new Many(), await open(tensors), { upload: recorder() }); + const names = Array.from({ length: 12 }, (_, i) => `layers.${i}.weight`); + expect(warn.mock.calls).toEqual([[WARNING + names.join(', ')]]); + warn.mockRestore(); + }); + + it('names the sources of a weight a hook stacked', async () => { + // q and k stack into one [8, 12] weight: in 12 % 8 != 0. + class Fused extends Module { + qk = new Linear(12, 8, { bias: false }); + override transformStateDict(sd: LazyStateDict): void { + const take = (name: string) => { + const t = sd.tensors.get(name)!; + sd.tensors.delete(name); + return t; + }; + sd.tensors.set('qk.weight', concatRows([take('q.weight'), take('k.weight')])); + } + } + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const sd = await open({ ...named('q', 4, 12), ...named('k', 4, 12) }); + await loadLazyStateDict(new Fused(), sd, { upload: recorder() }); + expect(warn.mock.calls).toEqual([[`${WARNING}concatRows(q.weight, k.weight)`]]); + warn.mockRestore(); + }); + + it('stays silent about a tensor no quantized kernel reads', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + await loadLazyStateDict(new Holder(), await open(named('table', 4, 4)), { + upload: recorder(), + }); + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it('stays silent when every quantized weight loads quantized', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + await loadLazyStateDict(new Fits(), await open(named('fc', 4, 8)), { upload: recorder() }); + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it('stays silent on a float checkpoint', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + await load(new Fits(), { 'fc.weight': { data: new Float32Array(32), shape: [4, 8] } }); + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); }); }); diff --git a/packages/runntime/test/core/weights/checkpoint.ts b/packages/runntime/test/core/weights/checkpoint.ts index 17d5c95b..6b7d4a33 100644 --- a/packages/runntime/test/core/weights/checkpoint.ts +++ b/packages/runntime/test/core/weights/checkpoint.ts @@ -209,3 +209,20 @@ export async function quantContents( scales: [...(scales instanceof Uint16Array ? f16ToF32(scales) : scales)], }; } + +/** Replaces `name` in `sd` with the tensor `runntime quantize` stores for + * it: int8 rows [dim 0, everything else] of `values`, whole numbers, one + * scale of 1 per row, so it dequantizes to `values` exactly. */ +export async function storeAsQuantizedRows( + sd: LazyStateDict, + name: string, + values: number[], + rows: number, +): Promise { + const entry: QuantEntry = { + ints: values, + scales: new Array(rows).fill(1), + shape: [rows, values.length / rows], + }; + sd.tensors.set(name, await quantSource(8, values.length / rows, name, entry)); +} diff --git a/packages/runntime/test/core/weights/safetensors.test.ts b/packages/runntime/test/core/weights/safetensors.test.ts index 0f277d95..d4b45a46 100644 --- a/packages/runntime/test/core/weights/safetensors.test.ts +++ b/packages/runntime/test/core/weights/safetensors.test.ts @@ -534,7 +534,26 @@ describe('fromSafetensors', () => { expect(sd.tensors.get('fc.weight')!.kind).toBe('quant'); }); - it.each(['f32', 'u16', 'halfWords', 'words'] as const)( + it('reads f32() of a quantized tensor as each int times the scale of its group', async () => { + // ints 1 2 | 3 4 and −1 −2 | −3 −4, scales 0.5 2 and 1 0.25 + const sd = await open({ + 'fc.weight': { + dtype: 'I8', + shape: [2, 4], + bytes: Uint8Array.from([1, 2, 3, 4, 255, 254, 253, 252]), + }, + 'fc.weight_scale': { + dtype: 'F16', + shape: [2, 2], + bytes: u16Bytes([0x3800, 0x4000, 0x3c00, 0x3400]), + }, + }); + expect(await sd.tensors.get('fc.weight')!.f32()).toEqual( + new Float32Array([0.5, 1, 6, 8, -1, -2, -0.75, -1]), + ); + }); + + it.each(['u16', 'halfWords', 'words'] as const)( 'rejects %s() on a quantized tensor with CHECKPOINT_MISMATCH', async (call) => { const sd = await open({ @@ -615,21 +634,30 @@ describe('fromSafetensors', () => { it.for([4, 8])('reads int%i weights within half a step of the float source', async (bits) => { const source = await golden('tiny.safetensors'); const sd = await golden(`tiny.int${bits}.safetensors`); - const w = sd.tensors.get('layer.weight')!; - if (w.kind !== 'quant') throw new Error('expected a quantized tensor'); - expect(w).toMatchObject({ shape: [8, 64], bits, groupSize: 32 }); - const ints = await w.ints(); - const halves = await w.scales(); - const scales = halves instanceof Float32Array ? halves : f16ToF32(halves); - const want = await source.tensors.get('layer.weight')!.f32(); - want.forEach((v, i) => { - const scale = scales[Math.floor(i / 32)]!; - expect(Math.abs(ints[i]! * scale - v)).toBeLessThanOrEqual(scale / 2 + 1e-6); - }); - // The tensors it leaves alone keep their bytes. - for (const name of ['embed.weight', 'layer.bias', 'norm.weight']) { + // The tool stores a weight as rows of everything after its first dim: + // conv.weight is torch [6, 16, 4], and no file setting names the bits + // or the group size. + const quantized = { 'layer.weight': [8, 64], 'conv.weight': [6, 64] }; + for (const [name, shape] of Object.entries(quantized)) { + const w = sd.tensors.get(name)!; + if (w.kind !== 'quant') throw new Error('expected a quantized tensor'); + expect(w).toMatchObject({ shape, bits, groupSize: 32 }); + const ints = await w.ints(); + const halves = await w.scales(); + const scales = halves instanceof Float32Array ? halves : f16ToF32(halves); + const want = await source.tensors.get(name)!.f32(); + expect(ints.length).toBe(want.length); + want.forEach((v, i) => { + const scale = scales[Math.floor(i / 32)]!; + expect(Math.abs(ints[i]! * scale - v)).toBeLessThanOrEqual(scale / 2 + 1e-6); + }); + } + // The tensors it leaves alone keep their bytes: an embedding table, a + // bias, a 1D norm and a row of 40 values that groups of 32 do not divide. + for (const name of ['embed.weight', 'layer.bias', 'norm.weight', 'odd.weight']) { expect(await sd.tensors.get(name)!.f32()).toEqual(await source.tensors.get(name)!.f32()); } + expect(sd.tensors.size).toBe(6); }); }); diff --git a/packages/runntime/test/core/weights/transforms.test.ts b/packages/runntime/test/core/weights/transforms.test.ts index f184f568..1627d9f9 100644 --- a/packages/runntime/test/core/weights/transforms.test.ts +++ b/packages/runntime/test/core/weights/transforms.test.ts @@ -468,10 +468,18 @@ describe.each([8, 4] as const)('int%i quantized sources', (bits) => { expect(() => transposeWeight(t)).toThrow(mismatch(/w\.weight is quantized/)); }); - it('readFloats and the float accessors of an output reject it', async () => { + it('readFloats and f32() of an output give the dequantized values', async () => { const t = await quantSource(bits, 2, 'w.weight', w); - await expect(readFloats('test', t)).rejects.toThrow(mismatch(/^test: w\.weight is quantized/)); - await expect(splitRows(t, [1, 1])[0]!.f32()).rejects.toThrow( + // int · scale of its group of 2 + expect(await readFloats('test', t)).toEqual( + new Float32Array([0.5, -1, 3, -4, 10, -12, 28, -32]), + ); + expect(await splitRows(t, [1, 1])[1]!.f32()).toEqual(new Float32Array([10, -12, 28, -32])); + }); + + it('the readers of stored bits of an output reject it', async () => { + const t = await quantSource(bits, 2, 'w.weight', w); + await expect(splitRows(t, [1, 1])[0]!.u16()).rejects.toThrow( mismatch(/splitRows\(w\.weight\) is quantized/), ); }); diff --git a/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts b/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts index 7a1f1f85..8654fd69 100644 --- a/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts +++ b/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts @@ -13,7 +13,12 @@ import { transformEmbeddingGemma2StateDict, transformEmbeddingGemma2VisionStateDict, } from '../../../src/zoo/embeddinggemma2/stateDictHooks.ts'; -import { checkpoint, contents, type Entry } from '../../core/weights/checkpoint.ts'; +import { + checkpoint, + contents, + storeAsQuantizedRows, + type Entry, +} from '../../core/weights/checkpoint.ts'; const DTYPES = [ { dtype: 'F32', type: 'f32', elemBytes: 4 }, @@ -277,6 +282,20 @@ describe('transformEmbeddingGemma2VisionStateDict', () => { expect(sd.totalBytes()).toBe(total); }); + + it('splits a position table quantized as rows [2, positions·hidden] into x and y', async () => { + const name = 'vision_tower.patch_embedder.position_embedding_table'; + const sd = await checkpoint('F32', CHECKPOINT); + await storeAsQuantizedRows(sd, name, CHECKPOINT[name]!.values, 2); + transformEmbeddingGemma2VisionStateDict(sd, '', cfg); + for (const [table, values] of [ + ['pos_x.weight', [1, 2, 3, 4]], + ['pos_y.weight', [5, 6, 7, 8]], + ] as const) { + expect(sd.tensors.get(table)!.shape).toEqual([2, 2]); + expect(await contents(sd.tensors.get(table)!)).toEqual({ type: 'f32', values }); + } + }); }); describe('transformEmbeddingGemma2AudioStateDict', () => { @@ -466,4 +485,25 @@ describe('transformEmbeddingGemma2AudioStateDict', () => { // 10 clipped linears' worth of bounds, 4 scalars each. expect(sd.totalBytes()).toBe(total - 40 * elemBytes); }); + + it('reorders the convs quantized as rows [out, everything else] by their values', async () => { + const conv1 = `${sub}layer1.conv.weight`; + const depthwise = `${T}.lconv1d.depthwise_conv1d.weight`; + const sd = await checkpoint('F32', CHECKPOINT); + await storeAsQuantizedRows(sd, conv1, CHECKPOINT[conv1]!.values, 1); + await storeAsQuantizedRows(sd, depthwise, CHECKPOINT[depthwise]!.values, 4); + transformEmbeddingGemma2AudioStateDict(sd, '', cfg); + // The same values as from the float checkpoint: tap k of channel c at + // k·2 + c, and tap 0 of every channel, then tap 1. + expect(sd.tensors.get('conv1.weight')!.shape).toEqual([1, 18]); + expect(await contents(sd.tensors.get('conv1.weight')!)).toEqual({ + type: 'f32', + values: [1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17, 9, 18], + }); + expect(sd.tensors.get('layers.0.lconv.conv')!.shape).toEqual([2, 4]); + expect(await contents(sd.tensors.get('layers.0.lconv.conv')!)).toEqual({ + type: 'f32', + values: [1, 3, 5, 7, 2, 4, 6, 8], + }); + }); }); diff --git a/packages/runntime/test/zoo/higgs/encoder.test.ts b/packages/runntime/test/zoo/higgs/encoder.test.ts index c6d6e0e5..dd8e9b8a 100644 --- a/packages/runntime/test/zoo/higgs/encoder.test.ts +++ b/packages/runntime/test/zoo/higgs/encoder.test.ts @@ -10,6 +10,7 @@ import { HIGGS_ENCODER } from '../../../src/zoo/higgs/config.ts'; import { HiggsEncoder } from '../../../src/zoo/higgs/encoder.ts'; import { resample } from '../../../src/zoo/higgs/resample.ts'; import { expectClose } from '../../gpu.ts'; +import { dequantizeWeight, quantizeWeight, quantSource } from '../../core/weights/checkpoint.ts'; import { CFG, checkpoint, QUANT_CFG, quantize, type Sd } from './encoderCheckpoint.ts'; const POS = 'semantic_model.encoder.pos_conv_embed'; @@ -64,6 +65,31 @@ describe('HiggsEncoder', () => { expect([...sd.tensors.keys()].filter((n) => n.startsWith(`${POS}.conv.`))).toEqual([]); }); + it('folds a position conv weight norm quantized as rows [out, in·kernel]', async () => { + const raw = checkpoint(); + const sd = memoryStateDict(raw); + const { hidden, posGroups, posKernel: k } = CFG.hubert; + const width = hidden / posGroups; + const gName = `${POS}.conv.parametrizations.weight.original0`; + const vName = `${POS}.conv.parametrizations.weight.original1`; + // g [1, 1, k] as one row of k, v [hidden, width, k] as rows of width·k. + const gQuant = quantizeWeight(raw[gName]!.data, [1, k], 8, k); + const vQuant = quantizeWeight(raw[vName]!.data, [hidden, width * k], 8, width); + sd.tensors.set(gName, await quantSource(8, k, gName, gQuant)); + sd.tensors.set(vName, await quantSource(8, width, vName, vQuant)); + new HiggsEncoder(CFG).semantic_model.encoder.pos_conv_embed.transformStateDict(sd, POS); + const g = dequantizeWeight(gQuant); + const v = dequantizeWeight(vQuant); + const norm = Array.from({ length: k }, (_, j) => + Math.sqrt(v.reduce((s, x, i) => (i % k === j ? s + x * x : s), 0)), + ); + const want = Array.from(v.subarray(width * width * k, 2 * width * width * k), (x, i) => { + return (x * g[i % k]!) / norm[i % k]!; + }); + expectClose(await read(sd, `${POS}.convs.1.weight`), want, { atol: 1e-6, rtol: 1e-6 }); + expect(sd.tensors.get(`${POS}.convs.1.weight`)!.shape).toEqual([width, width, k]); + }); + it("takes the checkpoint's folded position conv weight and splits it into groups", async () => { const raw = checkpoint(); const { hidden, posGroups, posKernel: k } = CFG.hubert; diff --git a/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts b/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts index f8c0f73d..1d24bcb2 100644 --- a/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts +++ b/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it } from 'vitest'; import { transformHfPrivacyFilterStateDict } from '../../../src/zoo/privacy-filter/stateDictHooks.ts'; -import { checkpoint, contents, type Entry } from '../../core/weights/checkpoint.ts'; +import { + checkpoint, + contents, + storeAsQuantizedRows, + type Entry, +} from '../../core/weights/checkpoint.ts'; const ZERO = { values: [0, 0], shape: [2] }; @@ -108,4 +113,23 @@ describe('transformHfPrivacyFilterStateDict', () => { // The dropped score.bias (2 elems) is the only byte loss. expect(sd.totalBytes()).toBe(total - 2 * elemBytes); }); + + // A file stores a quantized [E, K, 2H] as rows [E, K·2H], which the hook + // cannot split into glu and lin halves per expert. + it.for([ + 'model.layers.0.mlp.experts.gate_up_proj', + 'model.layers.0.mlp.experts.gate_up_proj_bias', + 'model.layers.0.mlp.experts.down_proj', + ])('refuses a quantized %s with CHECKPOINT_MISMATCH', async (name) => { + const sd = await checkpoint('F32', CHECKPOINT); + await storeAsQuantizedRows(sd, name, CHECKPOINT[name]!.values, 2); + expect(() => transformHfPrivacyFilterStateDict(sd)).toThrow( + expect.objectContaining({ + code: 'CHECKPOINT_MISMATCH', + message: + `privacy-filter state dict: '${name}' is quantized, and a quantized expert ` + + 'tensor cannot be split into experts: keep it float in the file', + }), + ); + }); }); diff --git a/tools/python/README.md b/tools/python/README.md index 083750df..e640d156 100644 --- a/tools/python/README.md +++ b/tools/python/README.md @@ -3,22 +3,40 @@ Quantizes a safetensors checkpoint to runntime's int8/int4 weight format, so a model downloads smaller and runs on the quantized matmul. +The tool knows the file format, not the model: it never asks which layer +reads a tensor or what a kernel can run. runntime fits each weight to its +layer at load, and loads a weight no quantized kernel runs as float. + ```bash uv run runntime quantize model.safetensors model.int8.safetensors --bits 8 ``` - The source is a `.safetensors` file or a Hugging Face model folder, sharded or not. The output is one `.safetensors` file. -- 2D tensors named `*.weight` are quantized in groups of `--group-size` - inputs (default 32), each group with one f16 scale. Embeddings, position - embeddings and `lm_head` stay as stored. `--include` and `--exclude` take - glob patterns and replace those defaults. +- Float tensors named `*.weight` with 2 or more dims are quantized, under + their own names. `--include GLOB` replaces `*.weight`. +- Tensors matching `*embed*`, `*pos_emb*` or `lm_head.*` stay as stored. This + is a quality default, not a format rule: quantized embedding tables only + rarely hold up. `--exclude GLOB` replaces it, so `--exclude none` quantizes + them too. +- A quantized tensor is stored as rows, `[first dim, everything else]`: a + Conv1d weight `[out, in, k]` becomes `[out, in·k]`. Each row is cut into + groups of `--group-size` values (default 32, any positive integer), each + group with one f16 scale, stored as `_scale`. +- A selected tensor stays as stored only when the format cannot hold it: it + is not float, it has fewer than 2 dims, the group size does not divide its + row, or, for int4, its row is odd (two values pack into a byte). +- The header's `runntime.quant` entry is `{"version":1,"scheme":"symmetric"}`. + The reader takes each tensor's bits from its dtype (`I8` int8, `U8` int4) + and its group size from the shape of its scales. - Every other tensor is copied as stored, or as F16 with `--f16`, and the header metadata is kept. - `--drop` takes glob patterns of tensors to leave out, like a pooler the model never reads. - It prints the max and mean absolute error of every quantized tensor and why each other tensor was skipped. Exclude the layers that lose too much. +- Use `--bits 8` for models built on 2D convolutions: int4 breaks the outputs + of MobileNetV4, DepthART and YOLO26. ## Development diff --git a/tools/python/src/runntime_tools/cli.py b/tools/python/src/runntime_tools/cli.py index 711904b3..1c8f8c7d 100644 --- a/tools/python/src/runntime_tools/cli.py +++ b/tools/python/src/runntime_tools/cli.py @@ -1,8 +1,11 @@ """`runntime quantize`: writes a safetensors checkpoint in runntime's int8/int4 -format v1 and prints the error of every quantized tensor.""" +format v1 and prints the error of every quantized tensor. It knows the file +format only, not which layer reads a tensor: runntime fits each weight to its +layer at load.""" import argparse import json +import math import sys from collections.abc import Sequence from dataclasses import dataclass @@ -24,19 +27,12 @@ DEFAULT_INCLUDE = ("*.weight",) DEFAULT_EXCLUDE = ("*embed*", "*pos_emb*", "lm_head.*") -"""runntime reads token and position embeddings as float lookup tables, and -lm_head often shares the embedding's weights.""" - -IN_FEATURES_MULTIPLE = 8 -"""runntime's quantized matmul reads 8 input rows per tile.""" +"""A quality choice, not a format rule: quantized embedding tables only rarely +hold up, and lm_head often shares the embedding's weights.""" F16_MAX = float(np.finfo(np.float16).max) """The largest finite f16. A tensor with a bigger value cannot be stored as F16.""" -PACKED_WORD_BITS = 32 -"""runntime packs a weight 32 bits to a word, so out features must fill whole -words: a multiple of 4 for int8, 8 for int4.""" - @dataclass(frozen=True) class Options: @@ -58,16 +54,13 @@ def skip_reason(name: str, tensor: Tensor, options: Options) -> str | None: return f"excluded by {excluded_by}" if tensor.dtype not in FLOAT_DTYPES: return f"dtype {tensor.dtype}" - if len(tensor.shape) != 2: - return f"{len(tensor.shape)}D, needs [out, in]" - out_features, in_features = tensor.shape - if in_features % IN_FEATURES_MULTIPLE: - return f"in features {in_features} not a multiple of {IN_FEATURES_MULTIPLE}" - if in_features % options.group_size: - return f"in features {in_features} not a multiple of the group size {options.group_size}" - out_multiple = PACKED_WORD_BITS // options.bits - if out_features % out_multiple: - return f"out features {out_features} not a multiple of {out_multiple}" + if len(tensor.shape) < 2: + return f"{len(tensor.shape)}D, needs 2 or more dims" + row = math.prod(tensor.shape[1:]) + if row % options.group_size: + return f"row of {row} values not a multiple of the group size {options.group_size}" + if options.bits == 4 and row % 2: + return f"row of {row} values is odd, int4 packs two to a byte" return None @@ -102,7 +95,8 @@ def quantize_checkpoint(source: Path, destination: Path, options: Options) -> li tensors[name] = to_float16(name, tensor) if options.float16 else tensor report.append(f"skipped {name} {shape}: {reason}") continue - weight = to_float32(tensor) + # The format stores a weight as rows: dim 0 by everything else. + weight = to_float32(tensor).reshape(tensor.shape[0], -1) quantized = quantize(weight, options.bits, options.group_size) if options.bits == 8: tensors[name] = from_array("I8", quantized.ints) @@ -111,20 +105,23 @@ def quantize_checkpoint(source: Path, destination: Path, options: Options) -> li tensors[f"{name}_scale"] = from_array("F16", quantized.scales) quantized_count += 1 error = np.abs(dequantize(quantized) - weight) - report.append(f"quantized {name} {shape}: max error {error.max():.3g}, mean {error.mean():.3g}") + stored = "" if len(tensor.shape) == 2 else f" as [{weight.shape[0]}, {weight.shape[1]}]" + report.append(f"quantized {name} {shape}{stored}: max error {error.max():.3g}, mean {error.mean():.3g}") if quantized_count == 0: raise ValueError("no tensor was quantized:\n" + "\n".join(report)) - setting = {"version": FORMAT_VERSION, "bits": options.bits, "groupSize": options.group_size, "scheme": "symmetric"} + # No bits or groupSize: the reader takes them from each tensor's dtype and + # its scales' shape, which holds for any file, one setting or mixed. + setting = {"version": FORMAT_VERSION, "scheme": "symmetric"} metadata = checkpoint.metadata | {QUANT_METADATA_KEY: json.dumps(setting, separators=(",", ":"))} write_safetensors(destination, tensors, metadata) return report def group_size(value: str) -> int: - """argparse type: a positive multiple of 4.""" + """argparse type: a positive integer.""" size = int(value) - if size <= 0 or size % 4: - raise argparse.ArgumentTypeError(f"{value} is not a positive multiple of 4") + if size <= 0: + raise argparse.ArgumentTypeError(f"{value} is not a positive integer") return size @@ -134,12 +131,14 @@ def parse_args(argv: Sequence[str] | None) -> argparse.Namespace: command = commands.add_parser( "quantize", help="quantize a safetensors checkpoint to int8 or int4", - description="Quantizes the 2D weights of a safetensors checkpoint to runntime's int8/int4 format.", + description="Quantizes the weights of a safetensors checkpoint to runntime's int8/int4 format.", ) command.add_argument("source", type=Path, help="a .safetensors file or a Hugging Face model folder") command.add_argument("destination", type=Path, help="the .safetensors file to write") command.add_argument("--bits", type=int, choices=(4, 8), required=True) - command.add_argument("--group-size", type=group_size, default=32, help="inputs that share one scale (default: 32)") + command.add_argument( + "--group-size", type=group_size, default=32, help="values of a row that share one scale (default: 32)" + ) command.add_argument( "--include", action="append", diff --git a/tools/python/src/runntime_tools/quantize.py b/tools/python/src/runntime_tools/quantize.py index 3a517168..85a51b9f 100644 --- a/tools/python/src/runntime_tools/quantize.py +++ b/tools/python/src/runntime_tools/quantize.py @@ -1,4 +1,4 @@ -"""Symmetric group-wise int8/int4 quantization of [out, in] weights, in the +"""Symmetric group-wise int8/int4 quantization of [rows, cols] weights, in the runntime format v1 layout.""" from dataclasses import dataclass @@ -12,22 +12,22 @@ @dataclass(frozen=True) class QuantizedWeight: - """`ints` [out, in], one signed value per element, and `scales` - [out, in / group_size], one f16 scale per group of inputs.""" + """`ints` [rows, cols], one signed value per element, and `scales` + [rows, cols / group_size], one f16 scale per group of a row.""" ints: NDArray[np.int8] scales: NDArray[np.float16] def quantize(weight: NDArray[np.float32], bits: Bits, group_size: int) -> QuantizedWeight: - """Quantizes each group of `group_size` inputs of each output row to + """Quantizes each group of `group_size` values of each row to `int = round(w / scale)` with `scale = absmax / (2^(bits-1) - 1)`, so the range is symmetric and has no zero point. The ints divide by the scale rounded to f16, the one the file stores. An all-zero group has scale 0 and ints 0.""" - out_features, in_features = weight.shape + rows, cols = weight.shape largest = 2 ** (bits - 1) - 1 - groups = weight.reshape(out_features, in_features // group_size, group_size) + groups = weight.reshape(rows, cols // group_size, group_size) scales = (np.abs(groups).max(axis=-1, keepdims=True) / largest).astype(np.float16) divisor = scales.astype(np.float32) ratio = np.divide(groups, divisor, out=np.zeros_like(groups), where=divisor != 0) @@ -37,11 +37,11 @@ def quantize(weight: NDArray[np.float32], bits: Bits, group_size: int) -> Quanti def dequantize(weight: QuantizedWeight) -> NDArray[np.float32]: """The float values the ints stand for: int * scale.""" - out_features, in_features = weight.ints.shape + rows, cols = weight.ints.shape group_count = weight.scales.shape[1] - groups = weight.ints.reshape(out_features, group_count, in_features // group_count) + groups = weight.ints.reshape(rows, group_count, cols // group_count) values = groups * weight.scales.astype(np.float32)[..., np.newaxis] - return values.reshape(out_features, in_features) + return values.reshape(rows, cols) def pack_int4(ints: NDArray[np.int8]) -> NDArray[np.uint8]: diff --git a/tools/python/tests/golden/tiny.int4.safetensors b/tools/python/tests/golden/tiny.int4.safetensors index cff48760..af44a054 100644 Binary files a/tools/python/tests/golden/tiny.int4.safetensors and b/tools/python/tests/golden/tiny.int4.safetensors differ diff --git a/tools/python/tests/golden/tiny.int8.safetensors b/tools/python/tests/golden/tiny.int8.safetensors index 293573d0..99212431 100644 Binary files a/tools/python/tests/golden/tiny.int8.safetensors and b/tools/python/tests/golden/tiny.int8.safetensors differ diff --git a/tools/python/tests/golden/tiny.safetensors b/tools/python/tests/golden/tiny.safetensors index 3be7e539..365b1867 100644 Binary files a/tools/python/tests/golden/tiny.safetensors and b/tools/python/tests/golden/tiny.safetensors differ diff --git a/tools/python/tests/test_cli.py b/tools/python/tests/test_cli.py index 4763f465..8e549029 100644 --- a/tools/python/tests/test_cli.py +++ b/tools/python/tests/test_cli.py @@ -1,10 +1,12 @@ +import argparse from pathlib import Path import numpy as np import pytest +from numpy.typing import NDArray from safetensors import safe_open -from runntime_tools.cli import Options, main, skip_reason +from runntime_tools.cli import Options, group_size, main, skip_reason from runntime_tools.quantize import Bits from runntime_tools.safetensors_io import Tensor, from_array, write_safetensors @@ -36,40 +38,122 @@ def test_golden_file_reads_back_with_the_safetensors_library() -> None: ): metadata = quantized.metadata() assert metadata["format"] == "pt" - assert metadata["runntime.quant"] == '{"version":1,"bits":4,"groupSize":32,"scheme":"symmetric"}' + assert metadata["runntime.quant"] == '{"version":1,"scheme":"symmetric"}' weight = quantized.get_tensor("layer.weight") assert (weight.dtype, weight.shape) == (np.uint8, (8, 32)) scale = quantized.get_tensor("layer.weight_scale") assert (scale.dtype, scale.shape) == (np.float16, (8, 2)) - for name in ("embed.weight", "layer.bias", "norm.weight"): + for name in ("embed.weight", "layer.bias", "norm.weight", "odd.weight"): np.testing.assert_array_equal(quantized.get_tensor(name), source.get_tensor(name)) +@pytest.mark.parametrize(("bits", "dtype", "stored_cols"), [(8, np.int8, 64), (4, np.uint8, 32)]) +def test_golden_file_stores_a_conv_weight_as_rows(bits: Bits, dtype: type, stored_cols: int) -> None: + # conv.weight is [out 6, in 16, kernel 4]: rows of 16 * 4 = 64 values, two groups of 32. + with ( + safe_open(GOLDEN / f"tiny.int{bits}.safetensors", "numpy") as quantized, + safe_open(GOLDEN / "tiny.safetensors", "numpy") as source, + ): + stored = quantized.get_tensor("conv.weight") + assert (stored.dtype, stored.shape) == (dtype, (6, stored_cols)) + scales = quantized.get_tensor("conv.weight_scale") + assert (scales.dtype, scales.shape) == (np.float16, (6, 2)) + ints = stored if bits == 8 else unpack_int4(stored) + # Row-major, so value (o, c, k) of the conv is value c * 4 + k of row o. + values = (ints.reshape(6, 2, 32) * scales.astype(np.float32)[..., np.newaxis]).reshape(6, 16, 4) + step = np.repeat(scales.astype(np.float32), 32, axis=1).reshape(6, 16, 4) + assert np.all(np.abs(values - source.get_tensor("conv.weight")) <= step / 2 + 1e-6) + + +def unpack_int4(packed: NDArray[np.uint8]) -> NDArray[np.int8]: + """Two signed nibbles per byte, the low one first, as one value each.""" + nibbles = np.stack([packed & 0x0F, packed >> 4], axis=-1).reshape(packed.shape[0], -1).astype(np.int8) + return np.where(nibbles > 7, nibbles - 16, nibbles).astype(np.int8) + + def tensor(shape: tuple[int, ...], dtype: str = "F32") -> Tensor: return Tensor(dtype, shape, np.zeros(0, dtype=np.uint8)) @pytest.mark.parametrize( - ("name", "stored", "bits", "reason"), + ("name", "stored", "bits", "group_size", "reason"), [ - ("fc.weight", tensor((8, 64)), 4, None), - ("fc.weight", tensor((4, 64), "BF16"), 8, None), - ("fc.bias", tensor((8,)), 8, "not matched by --include"), - ("embed.weight", tensor((8, 64)), 8, "excluded by *embed*"), - ("fc.weight", tensor((8, 64), "I8"), 8, "dtype I8"), - ("conv.weight", tensor((8, 64, 3)), 8, "3D, needs [out, in]"), - ("conv.weight", tensor((8, 64, 3, 3)), 8, "4D, needs [out, in]"), - ("fc.weight", tensor((8, 36)), 8, "in features 36 not a multiple of 8"), - ("fc.weight", tensor((8, 48)), 8, "in features 48 not a multiple of the group size 32"), - ("fc.weight", tensor((4, 64)), 4, "out features 4 not a multiple of 8"), - ("fc.weight", tensor((6, 64)), 8, "out features 6 not a multiple of 4"), + ("fc.weight", tensor((8, 64)), 4, 32, None), + ("fc.weight", tensor((4, 64), "BF16"), 8, 32, None), + ("fc.weight", tensor((8, 64), "F16"), 8, 32, None), + # Shapes no layer rule looks at: any out, any row the group size divides. + ("fc.weight", tensor((1, 32)), 4, 32, None), + ("fc.weight", tensor((5, 96)), 8, 32, None), + ("fc.weight", tensor((7, 36)), 8, 12, None), + ("fc.weight", tensor((3, 5)), 8, 1, None), + ("fc.weight", tensor((3, 9)), 8, 3, None), + # 3 or more dims quantize as rows of everything after dim 0. + ("conv.weight", tensor((8, 64, 3)), 8, 32, None), + ("conv.weight", tensor((8, 16, 3, 3)), 4, 16, None), + ("conv.weight", tensor((5, 3, 7)), 8, 7, None), + ("fc.bias", tensor((8,)), 8, 32, "not matched by --include"), + ("skip.weight", tensor((8, 64)), 8, 32, "excluded by skip.*"), + ("fc.weight", tensor((8, 64), "I8"), 8, 32, "dtype I8"), + ("fc.weight", tensor((8, 64), "F64"), 8, 32, "dtype F64"), + ("norm.weight", tensor((64,)), 8, 32, "1D, needs 2 or more dims"), + ("scale.weight", tensor(()), 8, 1, "0D, needs 2 or more dims"), + ("fc.weight", tensor((8, 48)), 8, 32, "row of 48 values not a multiple of the group size 32"), + ("conv.weight", tensor((8, 16, 3)), 8, 32, "row of 48 values not a multiple of the group size 32"), + ("fc.weight", tensor((8, 9)), 4, 3, "row of 9 values is odd, int4 packs two to a byte"), ], ) -def test_skip_reason(name: str, stored: Tensor, bits: Bits, reason: str | None) -> None: - options = Options(bits=bits, group_size=32, include=("*.weight",), exclude=("*embed*",)) +def test_skip_reason(name: str, stored: Tensor, bits: Bits, group_size: int, reason: str | None) -> None: + options = Options(bits=bits, group_size=group_size, include=("*.weight",), exclude=("skip.*",)) assert skip_reason(name, stored, options) == reason +EMBEDDING_NAMES = ("embed_tokens.weight", "decoder.pos_emb.weight", "lm_head.weight") + + +def embedding_checkpoint(folder: Path) -> Path: + """A checkpoint of EMBEDDING_NAMES and one Linear weight, each [4, 32] of + 127: the top int8 at scale 1, so it quantizes without error.""" + source = folder / "model.safetensors" + weight = from_array("F32", np.full((4, 32), 127, dtype=np.float32)) + write_safetensors(source, dict.fromkeys((*EMBEDDING_NAMES, "fc.weight"), weight), {}) + return source + + +def test_keeps_embeddings_and_lm_head_float_by_default(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + source = embedding_checkpoint(tmp_path) + assert main(["quantize", str(source), str(tmp_path / "out.safetensors"), "--bits", "8"]) == 0 + assert capsys.readouterr().out.splitlines() == [ + "skipped embed_tokens.weight [4, 32]: excluded by *embed*", + "skipped decoder.pos_emb.weight [4, 32]: excluded by *pos_emb*", + "skipped lm_head.weight [4, 32]: excluded by lm_head.*", + "quantized fc.weight [4, 32]: max error 0, mean 0", + ] + + +def test_exclude_replaces_the_default(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + source = embedding_checkpoint(tmp_path) + argv = ["quantize", str(source), str(tmp_path / "out.safetensors"), "--bits", "8", "--exclude", "fc.*"] + assert main(argv) == 0 + assert [line.split()[0] for line in capsys.readouterr().out.splitlines()] == [*["quantized"] * 3, "skipped"] + + +@pytest.mark.parametrize("size", ["1", "3", "6", "640"]) +def test_group_size_takes_any_positive_integer(size: str) -> None: + assert group_size(size) == int(size) + + +@pytest.mark.parametrize("size", ["0", "-4"]) +def test_group_size_refuses_zero_and_negatives(size: str) -> None: + with pytest.raises(argparse.ArgumentTypeError): + group_size(size) + + +def test_report_names_the_stored_shape_of_a_conv_weight(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + argv = ["quantize", str(GOLDEN / "tiny.safetensors"), str(tmp_path / "out.safetensors"), "--bits", "8"] + assert main(argv) == 0 + assert "quantized conv.weight [6, 16, 4] as [6, 64]: max error" in capsys.readouterr().out + + def test_quantize_fails_when_no_tensor_is_selected(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: destination = tmp_path / "out.safetensors" argv = ["quantize", str(GOLDEN / "tiny.safetensors"), str(destination), "--bits", "8", "--include", "*.bias"] @@ -95,7 +179,7 @@ def test_f16_stores_the_float_tensors_that_stay_as_f16(tmp_path: Path) -> None: safe_open(GOLDEN / "tiny.safetensors", "numpy") as source, ): assert quantized.get_tensor("layer.weight").dtype == np.int8 - for name in ("embed.weight", "layer.bias", "norm.weight"): + for name in ("embed.weight", "layer.bias", "norm.weight", "odd.weight"): stored = quantized.get_tensor(name) assert stored.dtype == np.float16 np.testing.assert_array_equal(stored, source.get_tensor(name).astype(np.float16)) @@ -115,4 +199,7 @@ def test_drop_leaves_matching_tensors_out(tmp_path: Path) -> None: argv = ["quantize", str(GOLDEN / "tiny.safetensors"), str(destination), "--bits", "8", "--drop", "norm.*"] assert main(argv) == 0 with safe_open(destination, "numpy") as quantized: - assert set(quantized.keys()) == {"embed.weight", "layer.bias", "layer.weight", "layer.weight_scale"} + dropped = {"norm.weight"} + scales = {"conv.weight_scale", "layer.weight_scale"} + with safe_open(GOLDEN / "tiny.safetensors", "numpy") as source: + assert set(quantized.keys()) == set(source.keys()) - dropped | scales