From c74ae1e69b3ecf01e5d1db506bbd5b049b1ba33b Mon Sep 17 00:00:00 2001 From: chmjkb Date: Mon, 16 Nov 2026 15:45:20 +0100 Subject: [PATCH 1/7] feat(core): load a quantized tensor no quantized kernel runs as its dequantized floats, with one warning per load A quantized Linear weight off the quantized matmul's limits, and a quantized tensor bound to an embedding table, a CpuParameter, a norm or a plain parameter, used to throw CHECKPOINT_MISMATCH. They load as float now, in the model's dtype. QuantTensor.f32() gives the dequantized values, so a hook that reads a tensor as floats works on a quantized one. loadStateDict names every tensor read that way in one console.warn, Conv1d and ConvTranspose1d's float fallback included. A hook's own dequantize() is left out. --- packages/runntime/src/core/nn/layers.ts | 61 ++--- .../runntime/src/core/nn/loadStateDict.ts | 41 +++- .../runntime/src/core/weights/safetensors.ts | 32 ++- .../runntime/src/core/weights/transforms.ts | 25 +- .../runntime/test/core/nn/layers.gpu.test.ts | 214 +++++++++++++++++- packages/runntime/test/core/nn/layers.test.ts | 53 ++++- .../test/core/nn/loadStateDict.test.ts | 176 ++++++++++++-- .../test/core/weights/safetensors.test.ts | 21 +- .../test/core/weights/transforms.test.ts | 14 +- 9 files changed, 550 insertions(+), 87 deletions(-) diff --git a/packages/runntime/src/core/nn/layers.ts b/packages/runntime/src/core/nn/layers.ts index 58b81b6e..0e45408c 100644 --- a/packages/runntime/src/core/nn/layers.ts +++ b/packages/runntime/src/core/nn/layers.ts @@ -20,10 +20,7 @@ import { snake, } from '../graph/ops.ts'; import { matrix, type Value } from '../graph/value.ts'; -import { - MATMUL_QUANT_TILED_BK, - matmulQuantTiledEligible, -} from '../kernels/matmul/matmulQuantTiled.ts'; +import { matmulQuantTiledEligible } from '../kernels/matmul/matmulQuantTiled.ts'; import { convToTapMajor, f16ToF32, @@ -35,8 +32,13 @@ import { } from '../weights/convert.ts'; import { foldBnIntoConv } from '../weights/foldBn.ts'; import { packQuantW } from '../weights/quantPack.ts'; -import { derivedTensor, type LazyStateDict, type QuantTensor } from '../weights/safetensors.ts'; -import { dequantize, readFloats, transposeWeight } from '../weights/transforms.ts'; +import { + derivedTensor, + type LazyStateDict, + type LazyTensor, + type QuantTensor, +} from '../weights/safetensors.ts'; +import { readFloats, transposeWeight } from '../weights/transforms.ts'; import { tensor } from '../gpu/buffers.ts'; import { CpuParameter, Module, Parameter, type AnyModule } from './module.ts'; @@ -138,6 +140,12 @@ function loadQuantWeight( return scale; } +/** A quantized weight its layer's quantized kernel cannot run, as its + * dequantized f32 values in torch's `shape`. Reading it puts the tensor in + * loadStateDict's warning. */ +const asFloat = (t: QuantTensor, shape: readonly number[]): LazyTensor => + derivedTensor([...shape], () => t.f32(), t.byteLength); + /** `t`'s scales in f32, as stored. */ async function scalesF32(t: QuantTensor): Promise { const data = await t.scales(); @@ -161,7 +169,8 @@ async function scaleColumns(t: QuantTensor, groups: number, n: number): Promise< * A quantized checkpoint weight loads as quantW words with its f32 scales in * `weightScale` [in / groupSize, out], and forward() runs the quantized * matmul. Only the checkpoint says which kind it is, so transformStateDict - * retypes the slot at load. + * retypes the slot at load. A weight the quantized matmul cannot run loads + * as its dequantized floats. * * `addend` and `activation` fold a residual add and a following tanh or GELU * into the matmul epilogue, as act(x·W + bias) + addend. */ @@ -202,33 +211,25 @@ export class Linear extends Module { `expected torch layout [${this.outFeatures}, ${this.inFeatures}]`, ); } - if (tensor.kind === 'quant') { - this.loadQuantized(sd, prefix, name, tensor); + if (tensor.kind !== 'quant') { + sd.tensors.set(name, transposeWeight(tensor)); return; } - sd.tensors.set(name, transposeWeight(tensor)); - } - /** Loads `name` as quantW words [in, out] with its scales + const { bits, groupSize } = tensor; + // The quantized matmul reads whole tiles of 4 columns and 8 rows, groups + // of whole 4-row chunks, and packs int4 columns 8 to a word. + const fits = + matmulQuantTiledEligible(this.inFeatures, this.outFeatures, groupSize) && + this.outFeatures % (32 / bits) === 0; + if (fits) this.loadQuantized(sd, prefix, tensor); + else sd.tensors.set(name, transposeWeight(asFloat(tensor, tensor.shape))); + } + /** Loads the weight as quantW words [in, out] with its scales * [in / groupSize, out]. */ - private loadQuantized( - sd: LazyStateDict, - prefix: string, - name: string, - tensor: QuantTensor, - ): void { + private loadQuantized(sd: LazyStateDict, prefix: string, tensor: QuantTensor): void { const { bits, groupSize } = tensor; const k = this.inFeatures; const n = this.outFeatures; - // The quantized matmul reads whole tiles of 4 columns and 8 rows and - // packs int4 columns 8 to a word. - if (!matmulQuantTiledEligible(k, n, groupSize) || n % (32 / bits) !== 0) { - throw new RunntimeError( - 'CHECKPOINT_MISMATCH', - `Linear: ${name} is int${bits} in groups of ${groupSize}, which needs in features ` + - `a multiple of ${MATMUL_QUANT_TILED_BK} and of the group size, out features a ` + - `multiple of ${32 / bits} and a group size a multiple of 4, got [out, in] = [${n}, ${k}]`, - ); - } this.quantization = { bits, groupSize }; const groups = k / groupSize; this.weightScale = loadQuantWeight( @@ -350,7 +351,7 @@ export class Conv1d extends Module { this.loadQuantized(sd, prefix, name, tensor, n); return; } - tensor = dequantize(tensor, [cOut, cIn, kernel]); + tensor = asFloat(tensor, [cOut, cIn, kernel]); } // Without this a weight with equal in/out channels loads scrambled. if ( @@ -528,7 +529,7 @@ export class ConvTranspose1d extends Module { if (cOut % 4 === 0 && n % (32 / w.bits) === 0 && quantConvFits(cIn, taps, n)) { this.loadQuantized(sd, prefix, w); } else { - w = dequantize(w, [cIn, cOut, k]); + w = asFloat(w, [cIn, cOut, k]); } } if (w && w.kind !== 'quant') { diff --git a/packages/runntime/src/core/nn/loadStateDict.ts b/packages/runntime/src/core/nn/loadStateDict.ts index 4d661883..0b8415c1 100644 --- a/packages/runntime/src/core/nn/loadStateDict.ts +++ b/packages/runntime/src/core/nn/loadStateDict.ts @@ -8,7 +8,13 @@ import { defaultRoot } from '../gpu/context.ts'; import { inGpuErrorScopes } from '../gpu/errorScopes.ts'; import { materialized, type Value, type ValueMeta } from '../graph/value.ts'; import { bf16ToF16Bits, f32ArrayToF16Bits } from '../weights/convert.ts'; -import type { DerivedTensor, FileTensor, LazyStateDict } from '../weights/safetensors.ts'; +import { + derivedTensor, + quantTensor, + type DerivedTensor, + type FileTensor, + type LazyStateDict, +} from '../weights/safetensors.ts'; import { CpuParameter, type AnyModule } from './module.ts'; export type UploadFn = (data: Float32Array | Uint32Array | Uint16Array, shape: ValueMeta) => Value; @@ -111,6 +117,14 @@ async function bindStateDict( upload: UploadFn, opts: LoadStateDictOpts, ): Promise { + // Whatever reads a quantized tensor as floats gets it dequantized. A weight + // its layer runs quantized is read as ints and never lands here. + const loadedAsFloat = new Set(); + for (const [name, t] of sd.tensors) { + if (t.kind !== 'quant') continue; + sd.tensors.set(name, quantTensor({ ...t, onFloatRead: () => loadedAsFloat.add(name) })); + } + model.transformStateDict?.(sd, ''); const visit = (parent: AnyModule, parentPrefix: string): void => { for (const [name, child] of parent.namedChildren()) { @@ -143,15 +157,20 @@ async function bindStateDict( const total = sd.totalBytes(); let done = 0; for (const [name, p] of params) { - const t = sd.tensors.get(name)!; + let t = sd.tensors.get(name)!; // Linear's and the convs' hooks turn a quantized weight into quantW - // words, so a QuantTensor still here is bound to something else. + // words, so a QuantTensor still here is bound to something that reads + // floats: an embedding table, a norm, a plain parameter. if (t.kind === 'quant') { - throw new RunntimeError( - 'CHECKPOINT_MISMATCH', - `loadStateDict: '${name}' is quantized (int${t.bits}), but only a Linear, Conv1d or ` + - `ConvTranspose1d weight loads quantized and this parameter holds ${p.shape.dtype}`, - ); + if (p.shape.dtype === 'quantW') { + throw new RunntimeError( + 'CHECKPOINT_MISMATCH', + `loadStateDict: '${name}' is quantized (int${t.bits}), but this quantW parameter ` + + 'takes packed words, which only its layer makes', + ); + } + const quant = t; + t = derivedTensor(quant.shape, () => quant.f32(), quant.byteLength); } const numel = t.shape.reduce((a, b) => a * b, 1); // A quantW slot counts u32 words under logical dims. Privacy-filter's @@ -178,6 +197,12 @@ async function bindStateDict( done += t.byteLength; opts.onProgress?.(name, done, total); } + if (loadedAsFloat.size > 0) { + console.warn( + 'loadStateDict: quantized tensors loaded as float, no quantized kernel runs them: ' + + list([...loadedAsFloat]), + ); + } for (const [, b] of model.namedBuffers()) { // A Buffer's data is computed CPU-side and always f32, and half() only // retyped the slot, so the narrowing happens here. diff --git a/packages/runntime/src/core/weights/safetensors.ts b/packages/runntime/src/core/weights/safetensors.ts index ac9599f8..3a38d12c 100644 --- a/packages/runntime/src/core/weights/safetensors.ts +++ b/packages/runntime/src/core/weights/safetensors.ts @@ -317,8 +317,9 @@ export interface DerivedTensor extends LazyTensorBase { /** Symmetric int weights with one scale per `groupSize` values along the last * dim: value = int · scale of its group. `shape` is logical, one element per - * int even for int4, and `byteLength` covers the ints and the scales. The - * float accessors reject with CHECKPOINT_MISMATCH. */ + * int even for int4, and `byteLength` covers the ints and the scales. f32() + * gives the dequantized values, for a reader that only takes floats; the + * readers of stored bits reject with CHECKPOINT_MISMATCH. */ export interface QuantTensor extends LazyTensorBase { kind: 'quant'; /** What the tensor is, for error messages: its name in the file, or the @@ -331,6 +332,9 @@ export interface QuantTensor extends LazyTensorBase { /** Row-major, shape `[...lead, last / groupSize]`. F16 halfwords as stored * in the file; f32 once a helper changed them. */ scales(): Promise; + /** Runs when f32() is called. loadStateDict sets it to learn which tensors + * loaded as float; a helper passes its sources' on to its outputs. */ + onFloatRead?: () => void; } export type LazyTensor = FileTensor | DerivedTensor | QuantTensor; @@ -410,22 +414,40 @@ function fileTensor(name: string, info: SafeTensorInfo, source: RangeSource): Fi }; } +/** The values `ints` stand for in f32: each int times the scale of its group + * of `groupSize`. */ +export function dequantizeInts( + ints: Int8Array, + scales: Float32Array | Uint16Array, + groupSize: number, +): Float32Array { + const f32 = scales instanceof Float32Array ? scales : f16ToF32(scales); + const out = new Float32Array(ints.length); + for (let i = 0; i < out.length; i++) out[i] = ints[i]! * f32[Math.floor(i / groupSize)]!; + return out; +} + /** A QuantTensor over `ints` and `scales`. Like derivedTensor it holds * nothing between reads. */ export function quantTensor( parts: Pick< QuantTensor, - 'name' | 'shape' | 'bits' | 'groupSize' | 'byteLength' | 'ints' | 'scales' + 'name' | 'shape' | 'bits' | 'groupSize' | 'byteLength' | 'ints' | 'scales' | 'onFloatRead' >, ): QuantTensor { const refuse = () => Promise.reject( new RunntimeError( 'CHECKPOINT_MISMATCH', - `tensor ${parts.name} is quantized (int${parts.bits}, groups of ${parts.groupSize}) and has no float reading`, + `tensor ${parts.name} is quantized (int${parts.bits}, groups of ${parts.groupSize}) and stores no float bits`, ), ); - return { kind: 'quant', ...parts, f32: refuse, halfWords: refuse, words: refuse, u16: refuse }; + const f32 = async () => { + parts.onFloatRead?.(); + const [ints, scales] = await Promise.all([parts.ints(), parts.scales()]); + return dequantizeInts(ints, scales, parts.groupSize); + }; + return { kind: 'quant', ...parts, f32, halfWords: refuse, words: refuse, u16: refuse }; } /** Unpacks int4 bytes, two values each with the low nibble first, into one diff --git a/packages/runntime/src/core/weights/transforms.ts b/packages/runntime/src/core/weights/transforms.ts index ab28f7ea..c94256cc 100644 --- a/packages/runntime/src/core/weights/transforms.ts +++ b/packages/runntime/src/core/weights/transforms.ts @@ -14,7 +14,13 @@ * transpose) throws CHECKPOINT_MISMATCH. */ import { RunntimeError } from '../error.ts'; import { f16ToF32 } from './convert.ts'; -import { derivedTensor, quantTensor, type LazyTensor, type QuantTensor } from './safetensors.ts'; +import { + dequantizeInts, + derivedTensor, + quantTensor, + type LazyTensor, + type QuantTensor, +} from './safetensors.ts'; /** f32 values, or f16 halfwords. */ type Floats = Float32Array | Uint16Array; @@ -27,12 +33,13 @@ const numel = (shape: readonly number[]): number => shape.reduce((a, b) => a * b const mismatch = (helper: string, why: string) => new RunntimeError('CHECKPOINT_MISMATCH', `${helper}: ${why}`); -/** Reads `t` as f16 halfwords when it holds F16 and as f32 otherwise. Throws +/** Reads `t` as f16 halfwords when it holds F16 and as f32 otherwise, a + * QuantTensor as its dequantized values. Throws * CHECKPOINT_MISMATCH, naming `helper`, on packed u32 words or on a derived * tensor whose data does not fill its shape. A file tensor's length is * checked by the header parser. */ export async function readFloats(helper: string, t: LazyTensor): Promise { - if (t.kind === 'quant') throw mismatch(helper, `${t.name} is quantized, not float`); + if (t.kind === 'quant') return t.f32(); if (t.kind === 'file') return t.dtype === 'F16' ? t.u16() : t.f32(); const data = await t.data(); if (data instanceof Uint32Array) throw mismatch(helper, 'source holds packed u32 words'); @@ -152,6 +159,7 @@ export function concatRows(parts: readonly LazyTensor[]): LazyTensor { bits: quant[0]!.bits, groupSize: quant[0]!.groupSize, byteLength, + onFloatRead: () => quant.forEach((p) => p.onFloatRead?.()), async ints() { const out = new Int8Array(numel(shape)); let base = 0; @@ -193,6 +201,7 @@ export function splitRows(t: LazyTensor, sizes: readonly number[]): LazyTensor[] bits: t.bits, groupSize: t.groupSize, byteLength: share(rows), + onFloatRead: t.onFloatRead, ints: async () => (await ints()).slice(first * rowElems, (first + rows) * rowElems), scales: async () => (await scales()).slice(first * scaleRowElems, (first + rows) * scaleRowElems), @@ -242,6 +251,7 @@ export function splitCols(t: LazyTensor, sizes: readonly number[]): LazyTensor[] bits: t.bits, groupSize, byteLength: share(cols), + onFloatRead: t.onFloatRead, ints: async () => copyCols(await ints(), new Int8Array(rows * cols), width, from, cols), async scales() { const data = await scales(); @@ -385,8 +395,10 @@ export function reshapeWeight(t: LazyTensor, shape: readonly number[]): LazyTens return { ...t, shape: [...shape] }; } -/** The values of `t` as f32 in `shape`, each int times the scale of its group. - * Throws CHECKPOINT_MISMATCH when the element counts differ. */ +/** The values of `t` as f32 in `shape`, each int times the scale of its group, + * for a hook that means to load `t` as floats: loadStateDict's warning about + * quantized tensors loaded as float leaves it out. Throws CHECKPOINT_MISMATCH + * when the element counts differ. */ export function dequantize(t: QuantTensor, shape: readonly number[] = t.shape): LazyTensor { if (numel(shape) !== numel(t.shape)) { throw mismatch('dequantize', `cannot give [${t.shape.join(', ')}] shape [${shape.join(', ')}]`); @@ -395,8 +407,7 @@ export function dequantize(t: QuantTensor, shape: readonly number[] = t.shape): [...shape], async () => { const [ints, scales] = await Promise.all([t.ints(), t.scales()]); - const f32 = toF32(scales); - return Float32Array.from(ints, (q, i) => q * f32[Math.floor(i / t.groupSize)]!); + return dequantizeInts(ints, scales, t.groupSize); }, t.byteLength, ); diff --git a/packages/runntime/test/core/nn/layers.gpu.test.ts b/packages/runntime/test/core/nn/layers.gpu.test.ts index d9348923..f1297fee 100644 --- a/packages/runntime/test/core/nn/layers.gpu.test.ts +++ b/packages/runntime/test/core/nn/layers.gpu.test.ts @@ -1,8 +1,8 @@ -import { describe, it, onTestFinished } from 'vitest'; +import { describe, expect, it, onTestFinished, vi } from 'vitest'; import { toArray } from '../../../src/core/gpu/eval.ts'; import type { SlotAct } from '../../../src/core/graph/ops.ts'; import { matrix } from '../../../src/core/graph/value.ts'; -import { Conv1d, ConvTranspose1d, Linear } from '../../../src/core/nn/layers.ts'; +import { Conv1d, ConvTranspose1d, Embedding, Linear } from '../../../src/core/nn/layers.ts'; import { bufferSource, fromSafetensors, @@ -26,8 +26,24 @@ interface Case { /** Takes the tiled kernel, which stages the dequantized weight in f16 * workgroup memory for an f16 input. */ tiled?: boolean; + /** The bits at which the quantized matmul cannot take the weight, so it + * loads as its dequantized floats in the layer's dtype. */ + float?: readonly (4 | 8)[]; } +/** Watches console.warn for the rest of the test. Returns a reader of the + * loadStateDict warnings so far; TypeGPU warns there too. */ +function loadWarnings(): () => unknown[] { + const warn = vi.spyOn(console, 'warn'); + onTestFinished(() => warn.mockRestore()); + return () => + warn.mock.calls + .map(([message]) => message as unknown) + .filter((message) => typeof message === 'string' && message.startsWith('loadStateDict:')); +} + +const LOADED_AS_FLOAT = /^loadStateDict: quantized tensors loaded as float.*: weight$/; + /** A Linear loaded from a quantized file matches a CPU loop of * act(x·W + bias) + addend in f64, with W dequantized from the same ints as * int · f16 scale. */ @@ -54,14 +70,18 @@ async function checkQuantizedLinear(c: Case, bits: 4 | 8, dtype: 'f32' | 'f16') }); const sd = await fromSafetensors(bufferSource(file), { chunkBytes: 0 }); if (c.bias) sd.tensors.set('bias', memoryStateDict({ bias }).tensors.get('bias')!); + const warned = loadWarnings(); await layer.loadStateDict(sd, { root }); + const float = c.float?.includes(bits) ?? false; + expect(layer.weight.shape.dtype).toBe(float ? dtype : 'quantW'); + expect(warned()).toEqual(float ? [expect.stringMatching(LOADED_AS_FLOAT)] : []); const x = upload(root, xs, { ...matrix(m, inF), dtype }); const call = c.addend ? { addend: upload(root, addend, { ...matrix(m, outF), dtype }) } : {}; const got = await toArray(layer.forward(x, call), ex); const dequant = Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / groupSize)]!); - const weight = dtype === 'f16' && c.tiled ? roundF16(dequant) : dequant; + const weight = dtype === 'f16' && (c.tiled || float) ? roundF16(dequant) : dequant; const want = new Float64Array(m * outF); for (let i = 0; i < m; i++) { for (let o = 0; o < outF; o++) { @@ -170,6 +190,99 @@ describe('Linear with a quantized weight', () => { }); }); +describe('Linear with a quantized weight at the limits of the quantized matmul', () => { + // It takes in features a multiple of 8 and of the group size, out features + // a multiple of 4 (int8) or 8 (int4), and a group size a multiple of 4. + // Anything else runs the float matmul on the dequantized weight. + const base = { m: 3, bias: true, dtypes: ['f32', 'f16'] } as const; + const CASES: (Case & { name: string; dtypes: readonly ('f32' | 'f16')[] })[] = [ + // quantized: in 8 % 8 == 0, out 8 % 8 == 0, groupSize 4 % 4 == 0 + { ...base, name: 'the smallest that fits', inF: 8, outF: 8, groupSize: 4 }, + // float: in 12 % 8 != 0 + { ...base, name: 'in off the 8-row tile', inF: 12, outF: 8, groupSize: 4, float: [4, 8] }, + // float: in 4 % 8 != 0 + { ...base, name: 'in under one tile', inF: 4, outF: 8, groupSize: 4, float: [4, 8] }, + // int8 quantized: out 4 % 4 == 0. int4 float: out 4 % 8 != 0 + { ...base, name: 'out of one int8 word', inF: 8, outF: 4, groupSize: 4, float: [4] }, + // int8 quantized: out 12 % 4 == 0. int4 float: out 12 % 8 != 0 + { ...base, name: 'out off the int4 word', inF: 16, outF: 12, groupSize: 8, float: [4] }, + // float: out 6 % 4 != 0 + { ...base, name: 'out off the int8 word', inF: 16, outF: 6, groupSize: 8, float: [4, 8] }, + // float: out 1 + { ...base, name: 'one output', inF: 16, outF: 1, groupSize: 8, float: [4, 8] }, + // quantized: groupSize 12 % 4 == 0 and 24 % 12 == 0 + { ...base, name: 'groups of 12', inF: 24, outF: 8, groupSize: 12 }, + // float: groupSize 6 % 4 != 0 + { ...base, name: 'groups of 6', inF: 24, outF: 8, groupSize: 6, float: [4, 8] }, + // float: groupSize 2 % 4 != 0 + { ...base, name: 'groups of 2', inF: 16, outF: 8, groupSize: 2, float: [4, 8] }, + // float: groupSize 1 + { ...base, name: 'a scale per value', inF: 16, outF: 8, groupSize: 1, float: [4, 8] }, + // float, with the epilogue the float matmul folds + { + ...base, + name: 'in off the tile with gelu and addend', + m: 37, + inF: 100, + outF: 30, + groupSize: 20, + addend: true, + activation: 'gelu', + float: [4, 8], + }, + ]; + it.for( + CASES.flatMap(({ dtypes, ...c }) => + dtypes.flatMap((dtype) => ([4, 8] as const).map((bits) => ({ ...c, dtype, bits }))), + ), + )('$dtype int$bits $name: [$m, $inF] → $outF, groupSize $groupSize', async (c, ctx) => { + if (c.dtype === 'f16') requireFeature(ctx, root, 'shader-f16'); + await checkQuantizedLinear(c, c.bits, c.dtype); + }); +}); + +describe('Embedding with a quantized table', () => { + // No kernel reads a quantized lookup table, so it loads as its dequantized + // floats: on the GPU in the model's dtype, on the CPU as f32. + const [vocab, dim, groupSize] = [10, 6, 3]; + const ids = [9, 0, 4, 4]; + it.for( + (['gpu', 'cpu'] as const).flatMap((device) => + ([4, 8] as const).flatMap((bits) => + (['f32', 'f16'] as const).map((dtype) => ({ device, bits, dtype })), + ), + ), + )('$device table, int$bits, $dtype', async ({ device, bits, dtype }, ctx) => { + if (dtype === 'f16') requireFeature(ctx, root, 'shader-f16'); + const half = 2 ** (bits - 1); + const ints = Array.from(randomF32(vocab * dim, 1), (v) => + Math.min(half - 1, Math.floor(v * half)), + ); + const scales = roundF16( + randomF32((vocab * dim) / groupSize, 2).map((v) => (Math.abs(v) + 0.1) / half), + ); + const table = new Embedding(vocab, dim, { device }); + if (dtype === 'f16') table.half(root); + onTestFinished(() => table.dispose()); + const file = quantFile(bits, groupSize, { + weight: { ints, scales: [...scales], shape: [vocab, dim] }, + }); + const warned = loadWarnings(); + await table.loadStateDict(await fromSafetensors(bufferSource(file), { chunkBytes: 0 }), { + root, + }); + expect(warned()).toEqual([expect.stringMatching(LOADED_AS_FLOAT)]); + const got = await toArray(table.forward(ids), ex); + + const dequant = Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / groupSize)]!); + // A GPU table is stored in the model's dtype; a CPU one stays f32 and its + // gathered rows round on upload. + const stored = dtype === 'f16' ? roundF16(dequant) : dequant; + const want = ids.flatMap((id) => [...stored.subarray(id * dim, (id + 1) * dim)]); + expectClose(got, want, { atol: 0, rtol: 0 }); + }); +}); + interface ConvCase { name: string; kind: 'conv' | 'transpose'; @@ -183,6 +296,9 @@ interface ConvCase { groupSize: number; bias: boolean; activation?: 'gelu'; + /** The quantized matmul cannot run this conv at any bits, so the weight + * loads as its dequantized floats. */ + float?: boolean; } /** A conv loaded from a quantized file of its torch weight, [out, in·kernel] @@ -209,10 +325,11 @@ async function checkQuantizedConv(c: ConvCase, bits: 4 | 8, dtype: 'f32' | 'f16' c.kind === 'conv' ? new Conv1d(cIn, cOut, kernel, { stride, padding, bias: c.bias, activation: c.activation }) : new ConvTranspose1d(cIn, cOut, kernel, { stride, padding, bias: c.bias }); - const run = async (layer: Conv1d | ConvTranspose1d, sd: LazyStateDict) => { + const run = async (layer: Conv1d | ConvTranspose1d, sd: LazyStateDict, quantized = false) => { if (dtype === 'f16') layer.half(root); onTestFinished(() => layer.dispose()); await layer.loadStateDict(sd, { root }); + if (quantized) expect(layer.weight.shape.dtype).toBe(c.float ? dtype : 'quantW'); return toArray(layer.forward(upload(root, xs, { ...matrix(tIn, cIn), dtype })), ex); }; const file = quantFile(bits, groupSize, { @@ -220,7 +337,9 @@ async function checkQuantizedConv(c: ConvCase, bits: 4 | 8, dtype: 'f32' | 'f16' }); const quantSd = await fromSafetensors(bufferSource(file), { chunkBytes: 0 }); if (c.bias) quantSd.tensors.set('bias', memoryStateDict({ bias }).tensors.get('bias')!); - const got = await run(make(), quantSd); + const warned = loadWarnings(); + const got = await run(make(), quantSd, true); + expect(warned()).toEqual(c.float ? [expect.stringMatching(LOADED_AS_FLOAT)] : []); const want = await run( make(), memoryStateDict({ weight: torchWeight, ...(c.bias ? { bias } : {}) }), @@ -306,6 +425,91 @@ describe('Conv1d and ConvTranspose1d with a quantized weight', () => { groupSize: 32, bias: false, }, + // The quantized matmul runs a Conv1d with in a multiple of 4 and kernel·in + // of 8, and a ConvTranspose1d that also has out a multiple of 4 and + // stride·out of 4 (int8) or 8 (int4). The rest run as float convs. + // quantized: in 4 % 4 == 0, kernel·in 8 % 8 == 0 + { + name: 'conv, the smallest that fits', + kind: 'conv', + tIn: 11, + cIn: 4, + cOut: 5, + kernel: 2, + stride: 1, + padding: 0, + groupSize: 8, + bias: true, + }, + // float: kernel·in 12 % 8 != 0 + { + name: 'conv, kernel·in off the 8-row tile', + kind: 'conv', + tIn: 11, + cIn: 4, + cOut: 5, + kernel: 3, + stride: 1, + padding: 1, + groupSize: 4, + bias: true, + float: true, + }, + // float: in 6 % 4 != 0, though kernel·in 24 % 8 == 0 + { + name: 'conv, in off 4', + kind: 'conv', + tIn: 11, + cIn: 6, + cOut: 8, + kernel: 4, + stride: 2, + padding: 1, + groupSize: 6, + bias: false, + float: true, + }, + // quantized: out 4 % 4 == 0, stride·out 8 % 8 == 0, taps·in 8 % 8 == 0 + { + name: 'transposed, the smallest that fits', + kind: 'transpose', + tIn: 7, + cIn: 4, + cOut: 4, + kernel: 4, + stride: 2, + padding: 1, + groupSize: 16, + bias: true, + }, + // float: out 6 % 4 != 0 + { + name: 'transposed, out off 4', + kind: 'transpose', + tIn: 7, + cIn: 8, + cOut: 6, + kernel: 4, + stride: 2, + padding: 1, + groupSize: 3, + bias: true, + float: true, + }, + // float: taps·in 12 % 8 != 0 + { + name: 'transposed, taps·in off the 8-row tile', + kind: 'transpose', + tIn: 7, + cIn: 4, + cOut: 4, + kernel: 6, + stride: 2, + padding: 2, + groupSize: 8, + bias: false, + float: true, + }, ]; it.for( (['f32', 'f16'] as const).flatMap((dtype) => diff --git a/packages/runntime/test/core/nn/layers.test.ts b/packages/runntime/test/core/nn/layers.test.ts index a9de2e0e..a853175c 100644 --- a/packages/runntime/test/core/nn/layers.test.ts +++ b/packages/runntime/test/core/nn/layers.test.ts @@ -362,11 +362,18 @@ describe('Linear with a quantized weight', () => { }); }); + // The quantized matmul takes in features a multiple of 8 and of the group + // size, out features a multiple of 4 (int8) or 8 (int4) and a group size a + // multiple of 4. A case names the one limit it sits on. it.each([ - { name: 'in features off the 8-row tiles', inF: 12, outF: 4, bits: 8 as const, groupSize: 4 }, - { name: 'int4 out features off 8 per word', inF: 8, outF: 4, bits: 4 as const, groupSize: 4 }, - { name: 'a group size off 4', inF: 16, outF: 4, bits: 8 as const, groupSize: 2 }, - ])('refuses $name with CHECKPOINT_MISMATCH', async ({ inF, outF, bits, groupSize }) => { + // quantized: in 8 % 8 == 0, out 4 % 4 == 0, groups of 4 + { name: 'int8 [4, 8] in groups of 4', inF: 8, outF: 4, bits: 8 as const, groupSize: 4 }, + // quantized: int4 out 8 % 8 == 0 + { name: 'int4 [8, 8] in groups of 4', inF: 8, outF: 8, bits: 4 as const, groupSize: 4 }, + // quantized: one group over the whole row, 16 % 4 == 0 + { name: 'int8 [4, 16] in groups of 16', inF: 16, outF: 4, bits: 8 as const, groupSize: 16 }, + ])('loads $name as quantW', async ({ inF, outF, bits, groupSize }) => { + const layer = new Linear(inF, outF); const sd = await quantCheckpoint(bits, groupSize, { weight: { ints: new Array(inF * outF).fill(0), @@ -374,11 +381,41 @@ describe('Linear with a quantized weight', () => { shape: [outF, inF], }, }); + layer.transformStateDict(sd, ''); + expect(layer.weight.shape.dtype).toBe('quantW'); + expect(layer.weightScale).toBeDefined(); + }); + + it.each([ + // float: in 12 % 8 != 0 + { name: 'in features off the 8-row tiles', inF: 12, outF: 4, bits: 8 as const, groupSize: 4 }, + // float: in 4 % 8 != 0 + { name: 'in features under one tile', inF: 4, outF: 4, bits: 8 as const, groupSize: 4 }, + // float: int8 out 6 % 4 != 0 + { name: 'int8 out features off 4 per word', inF: 8, outF: 6, bits: 8 as const, groupSize: 4 }, + // float: int4 out 4 % 8 != 0 + { name: 'int4 out features off 8 per word', inF: 8, outF: 4, bits: 4 as const, groupSize: 4 }, + // float: group size 2 % 4 != 0 + { name: 'a group size off 4', inF: 16, outF: 4, bits: 8 as const, groupSize: 2 }, + // float: group size 1 + { name: 'a scale per value', inF: 8, outF: 4, bits: 8 as const, groupSize: 1 }, + ])('loads $name as its dequantized floats', async ({ inF, outF, bits, groupSize }) => { + // ints −7 … 7 and one scale per group, (g % 5 + 1) / 4, f16-exact + const ints = Array.from({ length: inF * outF }, (_, i) => (i % 15) - 7); + const scales = Array.from({ length: (inF * outF) / groupSize }, (_, g) => ((g % 5) + 1) / 4); + const sd = await quantCheckpoint(bits, groupSize, { + weight: { ints, scales, shape: [outF, inF] }, + }); const layer = new Linear(inF, outF); - expect(() => layer.transformStateDict(sd, '')).toThrow( - expect.objectContaining({ code: 'CHECKPOINT_MISMATCH' }), - ); - expect(layer.weight.shape.dtype).toBe('f32'); + layer.transformStateDict(sd, ''); + expect(shapes(layer)).toEqual({ weight: meta([inF, outF]), bias: meta([outF]) }); + expect(layer.weightScale).toBeUndefined(); + // stored [in, out]: row i, column o is torch value o·in + i + const want = Array.from({ length: inF * outF }, (_, j) => { + const at = (j % outF) * inF + Math.floor(j / outF); + return ints[at]! * scales[Math.floor(at / groupSize)]!; + }); + expect([...(await sd.tensors.get('weight')!.f32())]).toEqual(want); }); it.each([ diff --git a/packages/runntime/test/core/nn/loadStateDict.test.ts b/packages/runntime/test/core/nn/loadStateDict.test.ts index db8f7ed1..86e3d2e5 100644 --- a/packages/runntime/test/core/nn/loadStateDict.test.ts +++ b/packages/runntime/test/core/nn/loadStateDict.test.ts @@ -1,7 +1,7 @@ import type { TgpuRoot } from 'typegpu'; import { describe, expect, it, vi } from 'vitest'; import { pending, type ValueMeta } from '../../../src/core/graph/value.ts'; -import { Linear } from '../../../src/core/nn/layers.ts'; +import { Conv1d, Linear } from '../../../src/core/nn/layers.ts'; import { gpuUpload, loadLazyStateDict } from '../../../src/core/nn/loadStateDict.ts'; import { Buffer, @@ -21,6 +21,7 @@ import { type MemoryStateDict, } from '../../../src/core/weights/safetensors.ts'; import { f32ToF16Bits } from '../../../src/core/weights/convert.ts'; +import { concatRows, dequantize } from '../../../src/core/weights/transforms.ts'; import { meta } from '../graph/values.ts'; import { safetensorsFile } from '../weights/checkpoint.ts'; @@ -244,10 +245,9 @@ describe('binding', () => { }); describe('a quantized tensor', () => { - // A [2, 2] int8 weight in groups of 2; the loader must refuse it before - // reading either part. + // A [2, 2] int8 weight in groups of 2: ints 1 2 | 3 4, scales 0.5 and 2. const ints = vi.fn(async () => new Int8Array([1, 2, 3, 4])); - const scales = vi.fn(async () => new Float32Array([1, 2])); + const scales = vi.fn(async () => new Float32Array([0.5, 2])); const quant = quantTensor({ name: 'w', shape: [2, 2], @@ -257,30 +257,166 @@ describe('a quantized tensor', () => { ints, scales, }); + const DEQUANTIZED = [0.5, 1, 6, 8]; it.each([ - { name: 'an f32 parameter', param: () => new Parameter(meta([2, 2])), dtype: 'f32' }, - { name: 'an f16 parameter', param: () => new Parameter(meta([2, 2], 'f16')), dtype: 'f16' }, - { name: 'a CpuParameter', param: () => new CpuParameter(meta([2, 2])), dtype: 'f32' }, - { - name: 'a quantW parameter', - param: () => new Parameter({ elems: 1, dtype: 'quantW', dims: [2, 2] }), - dtype: 'quantW', - }, - ])('is refused by $name with CHECKPOINT_MISMATCH', async ({ param, dtype }) => { - class NotLinear extends Module { - weight = param(); + { name: 'f32', dtype: 'f32' as const, want: new Float32Array(DEQUANTIZED) }, + { name: 'f16', dtype: 'f16' as const, want: Uint16Array.from(DEQUANTIZED, f32ToF16Bits) }, + ])('loads into an $name parameter as its dequantized values', async ({ dtype, want }) => { + class Table extends Module { + weight = new Parameter(meta([2, 2], dtype)); } + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); const upload = recorder(); - await expect(load(new NotLinear(), { weight: quant }, upload)).rejects.toMatchObject({ + await load(new Table(), { weight: quant }, upload); + expect(upload.mock.calls).toEqual([[want, meta([2, 2], dtype)]]); + warn.mockRestore(); + }); + + it('loads into a CpuParameter as its dequantized f32 values, off the GPU', async () => { + class Table extends Module { + weight = new CpuParameter(meta([2, 2])); + } + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const upload = recorder(); + const table = new Table(); + await load(table, { weight: quant }, upload); + expect(table.weight.data).toEqual(new Float32Array(DEQUANTIZED)); + expect(upload).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it('is refused by a quantW parameter with CHECKPOINT_MISMATCH, before any read', async () => { + class Packed extends Module { + weight = new Parameter({ elems: 1, dtype: 'quantW', dims: [2, 2] }); + } + ints.mockClear(); + const upload = recorder(); + await expect(load(new Packed(), { weight: quant }, upload)).rejects.toMatchObject({ code: 'CHECKPOINT_MISMATCH', - message: - "loadStateDict: 'weight' is quantized (int8), but only a Linear, Conv1d or " + - `ConvTranspose1d weight loads quantized and this parameter holds ${dtype}`, + message: expect.stringMatching(/'weight' is quantized \(int8\), but this quantW parameter/), }); expect(upload).not.toHaveBeenCalled(); expect(ints).not.toHaveBeenCalled(); - expect(scales).not.toHaveBeenCalled(); + }); +}); + +describe('the warning about quantized tensors loaded as float', () => { + // int8 in groups of 4. fc is [out 4, in 8], which the quantized matmul + // takes. odd is [out 4, in 12]: 12 % 8 != 0, so it loads as float. + const named = (prefix: string, rows: number, cols: number) => ({ + [`${prefix}.weight`]: { data: new Int8Array(rows * cols), shape: [rows, cols] }, + [`${prefix}.weight_scale`]: { + data: new Uint16Array((rows * cols) / 4), + shape: [rows, cols / 4], + }, + }); + const setting = { 'runntime.quant': JSON.stringify({ version: 1, scheme: 'symmetric' }) }; + const open = (tensors: Parameters[0]) => + fromSafetensors(bufferSource(safetensorsFile(tensors, setting)), { chunkBytes: 0 }); + + class Net extends Module { + fc = new Linear(8, 4, { bias: false }); + odd = new Linear(12, 4, { bias: false }); + table = new Table(); + conv = new Conv1d(3, 4, 4, { bias: false }); + } + class Table extends Module { + weight = new Parameter(meta([4, 4])); + } + class Fits extends Module { + fc = new Linear(8, 4, { bias: false }); + } + + it('names every tensor that fell back, once, and not the one that loaded quantized', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const sd = await open({ + ...named('fc', 4, 8), + ...named('odd', 4, 12), + ...named('table', 4, 4), + // Conv1d [out 4, in 3, kernel 4] as [4, 12]: in 3 % 4 != 0 + ...named('conv', 4, 12), + }); + const upload = recorder(); + await loadLazyStateDict(new Net(), sd, { upload }); + expect(warn.mock.calls).toEqual([ + [ + 'loadStateDict: quantized tensors loaded as float, no quantized kernel runs them: ' + + 'odd.weight, table.weight, conv.weight', + ], + ]); + expect(upload.mock.calls.map(([, shape]) => shape.dtype)).toEqual([ + 'quantW', + 'f32', + 'f32', + 'f32', + 'f32', + ]); + warn.mockRestore(); + }); + + it('names the file tensors behind a weight a hook renamed, stacked and read as floats', async () => { + // q and k stack into one [8, 12] weight: in 12 % 8 != 0, so it loads as + // float. The hook reads pos as floats itself. + class Fused extends Module { + qk = new Linear(12, 8, { bias: false }); + pos = new Parameter(meta([4, 4])); + override transformStateDict(sd: LazyStateDict): void { + const take = (name: string) => { + const t = sd.tensors.get(name)!; + sd.tensors.delete(name); + return t; + }; + sd.tensors.set('qk.weight', concatRows([take('q.weight'), take('k.weight')])); + const table = take('table.weight'); + sd.tensors.set( + 'pos', + derivedTensor([4, 4], () => table.f32()), + ); + } + } + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const sd = await open({ ...named('q', 4, 12), ...named('k', 4, 12), ...named('table', 4, 4) }); + await loadLazyStateDict(new Fused(), sd, { upload: recorder() }); + expect(warn.mock.calls).toEqual([ + [ + 'loadStateDict: quantized tensors loaded as float, no quantized kernel runs them: ' + + 'q.weight, k.weight, table.weight', + ], + ]); + warn.mockRestore(); + }); + + it('leaves out a tensor a hook dequantized on purpose', async () => { + class Folded extends Module { + table = new Parameter(meta([4, 4])); + override transformStateDict(sd: LazyStateDict): void { + const t = sd.tensors.get('table.weight')!; + sd.tensors.delete('table.weight'); + if (t.kind !== 'quant') throw new Error('expected a quantized tensor'); + sd.tensors.set('table', dequantize(t)); + } + } + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + await loadLazyStateDict(new Folded(), await open(named('table', 4, 4)), { + upload: recorder(), + }); + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it('stays silent when every quantized tensor loads quantized', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + await loadLazyStateDict(new Fits(), await open(named('fc', 4, 8)), { upload: recorder() }); + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + + it('stays silent on a float checkpoint', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + await load(new Fits(), { 'fc.weight': { data: new Float32Array(32), shape: [4, 8] } }); + expect(warn).not.toHaveBeenCalled(); + warn.mockRestore(); }); }); diff --git a/packages/runntime/test/core/weights/safetensors.test.ts b/packages/runntime/test/core/weights/safetensors.test.ts index 0f277d95..4d446b76 100644 --- a/packages/runntime/test/core/weights/safetensors.test.ts +++ b/packages/runntime/test/core/weights/safetensors.test.ts @@ -534,7 +534,26 @@ describe('fromSafetensors', () => { expect(sd.tensors.get('fc.weight')!.kind).toBe('quant'); }); - it.each(['f32', 'u16', 'halfWords', 'words'] as const)( + it('reads f32() of a quantized tensor as each int times the scale of its group', async () => { + // ints 1 2 | 3 4 and −1 −2 | −3 −4, scales 0.5 2 and 1 0.25 + const sd = await open({ + 'fc.weight': { + dtype: 'I8', + shape: [2, 4], + bytes: Uint8Array.from([1, 2, 3, 4, 255, 254, 253, 252]), + }, + 'fc.weight_scale': { + dtype: 'F16', + shape: [2, 2], + bytes: u16Bytes([0x3800, 0x4000, 0x3c00, 0x3400]), + }, + }); + expect(await sd.tensors.get('fc.weight')!.f32()).toEqual( + new Float32Array([0.5, 1, 6, 8, -1, -2, -0.75, -1]), + ); + }); + + it.each(['u16', 'halfWords', 'words'] as const)( 'rejects %s() on a quantized tensor with CHECKPOINT_MISMATCH', async (call) => { const sd = await open({ diff --git a/packages/runntime/test/core/weights/transforms.test.ts b/packages/runntime/test/core/weights/transforms.test.ts index f184f568..1627d9f9 100644 --- a/packages/runntime/test/core/weights/transforms.test.ts +++ b/packages/runntime/test/core/weights/transforms.test.ts @@ -468,10 +468,18 @@ describe.each([8, 4] as const)('int%i quantized sources', (bits) => { expect(() => transposeWeight(t)).toThrow(mismatch(/w\.weight is quantized/)); }); - it('readFloats and the float accessors of an output reject it', async () => { + it('readFloats and f32() of an output give the dequantized values', async () => { const t = await quantSource(bits, 2, 'w.weight', w); - await expect(readFloats('test', t)).rejects.toThrow(mismatch(/^test: w\.weight is quantized/)); - await expect(splitRows(t, [1, 1])[0]!.f32()).rejects.toThrow( + // int · scale of its group of 2 + expect(await readFloats('test', t)).toEqual( + new Float32Array([0.5, -1, 3, -4, 10, -12, 28, -32]), + ); + expect(await splitRows(t, [1, 1])[1]!.f32()).toEqual(new Float32Array([10, -12, 28, -32])); + }); + + it('the readers of stored bits of an output reject it', async () => { + const t = await quantSource(bits, 2, 'w.weight', w); + await expect(splitRows(t, [1, 1])[0]!.u16()).rejects.toThrow( mismatch(/splitRows\(w\.weight\) is quantized/), ); }); From 1884bfe2bc394e46e562850ddf4e87322575d459 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Mon, 16 Nov 2026 15:45:40 +0100 Subject: [PATCH 2/7] feat(tools): quantize any float tensor with 2 or more dims as rows, with no kernel or layer rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit runntime quantize knew the quantized matmul: it took 2D tensors only, wanted in features a multiple of 8, out features of 4 or 8 and a group size of 4, and kept embeddings and lm_head float because only Linear loaded quantized. It knows the file format only now. A selected float tensor with 2 or more dims is stored as [first dim, everything else] in groups of --group-size along that row, so a Conv1d [out, in, k] is [out, in·k], the layout the conv loaders read. It stays as stored only when it is not float, has fewer than 2 dims, the group size does not divide the row, or an int4 row is odd. The header leaves out bits and groupSize: the reader takes them from each tensor. No default excludes: int8 embedding tables move an embedding's cosine by under 1e-4 on all-MiniLM-L6-v2 and EmbeddingGemma 2. --- packages/runntime/test/core/nn/layers.test.ts | 20 ++++ .../test/core/weights/safetensors.test.ts | 39 ++++--- tools/python/README.md | 23 +++- tools/python/src/runntime_tools/cli.py | 60 +++++----- tools/python/src/runntime_tools/quantize.py | 18 +-- .../python/tests/golden/tiny.int4.safetensors | Bin 5032 -> 2472 bytes .../python/tests/golden/tiny.int8.safetensors | Bin 5288 -> 3432 bytes tools/python/tests/golden/tiny.safetensors | Bin 6632 -> 8792 bytes tools/python/tests/test_cli.py | 106 ++++++++++++++---- 9 files changed, 188 insertions(+), 78 deletions(-) diff --git a/packages/runntime/test/core/nn/layers.test.ts b/packages/runntime/test/core/nn/layers.test.ts index a853175c..fb7db3bd 100644 --- a/packages/runntime/test/core/nn/layers.test.ts +++ b/packages/runntime/test/core/nn/layers.test.ts @@ -1,3 +1,4 @@ +import { readFileSync } from 'node:fs'; import { describe, expect, it, vi } from 'vitest'; import { ACT_CODE } from '../../../src/core/graph/ops/shared.ts'; import { pending, type ValueMeta } from '../../../src/core/graph/value.ts'; @@ -521,6 +522,25 @@ describe('Conv1d and ConvTranspose1d with a quantized weight', () => { ]); }); + it.for([8, 4] as const)( + 'takes the int%i Conv1d weight the Python quantizer writes', + async (bits) => { + // tools/python's golden file holds torch [out 6, in 16, kernel 4] as + // [6, 64] in groups of 32. 6 outputs pad to the packing width, 8. + const golden = new URL( + `../../../../../tools/python/tests/golden/tiny.int${bits}.safetensors`, + import.meta.url, + ); + const sd = await fromSafetensors(bufferSource(readFileSync(golden))); + const layer = new Conv1d(16, 6, 4, { bias: false }); + layer.transformStateDict(sd, 'conv'); + expect(shapes(layer)).toEqual({ + weight: { elems: (64 * 8 * bits) / 32, dtype: 'quantW', dims: [64, 8] }, + weightScale: meta([2, 8]), + }); + }, + ); + it('keeps a Conv1d group that ends inside a channel', async () => { const layer = new Conv1d(8, 4, 3); // groups of 4 values: a channel's 3 taps and the next channel's first diff --git a/packages/runntime/test/core/weights/safetensors.test.ts b/packages/runntime/test/core/weights/safetensors.test.ts index 4d446b76..92372828 100644 --- a/packages/runntime/test/core/weights/safetensors.test.ts +++ b/packages/runntime/test/core/weights/safetensors.test.ts @@ -634,21 +634,34 @@ describe('fromSafetensors', () => { it.for([4, 8])('reads int%i weights within half a step of the float source', async (bits) => { const source = await golden('tiny.safetensors'); const sd = await golden(`tiny.int${bits}.safetensors`); - const w = sd.tensors.get('layer.weight')!; - if (w.kind !== 'quant') throw new Error('expected a quantized tensor'); - expect(w).toMatchObject({ shape: [8, 64], bits, groupSize: 32 }); - const ints = await w.ints(); - const halves = await w.scales(); - const scales = halves instanceof Float32Array ? halves : f16ToF32(halves); - const want = await source.tensors.get('layer.weight')!.f32(); - want.forEach((v, i) => { - const scale = scales[Math.floor(i / 32)]!; - expect(Math.abs(ints[i]! * scale - v)).toBeLessThanOrEqual(scale / 2 + 1e-6); - }); - // The tensors it leaves alone keep their bytes. - for (const name of ['embed.weight', 'layer.bias', 'norm.weight']) { + // The tool stores a weight as rows of everything after its first dim: + // conv.weight is torch [6, 16, 4], and no file setting names the bits + // or the group size. + const quantized = { + 'layer.weight': [8, 64], + 'embed.weight': [16, 64], + 'conv.weight': [6, 64], + }; + for (const [name, shape] of Object.entries(quantized)) { + const w = sd.tensors.get(name)!; + if (w.kind !== 'quant') throw new Error('expected a quantized tensor'); + expect(w).toMatchObject({ shape, bits, groupSize: 32 }); + const ints = await w.ints(); + const halves = await w.scales(); + const scales = halves instanceof Float32Array ? halves : f16ToF32(halves); + const want = await source.tensors.get(name)!.f32(); + expect(ints.length).toBe(want.length); + want.forEach((v, i) => { + const scale = scales[Math.floor(i / 32)]!; + expect(Math.abs(ints[i]! * scale - v)).toBeLessThanOrEqual(scale / 2 + 1e-6); + }); + } + // The tensors it leaves alone keep their bytes: a bias, a 1D norm and a + // row of 40 values that groups of 32 do not divide. + for (const name of ['layer.bias', 'norm.weight', 'odd.weight']) { expect(await sd.tensors.get(name)!.f32()).toEqual(await source.tensors.get(name)!.f32()); } + expect(sd.tensors.size).toBe(6); }); }); diff --git a/tools/python/README.md b/tools/python/README.md index 083750df..c7a6cebc 100644 --- a/tools/python/README.md +++ b/tools/python/README.md @@ -3,16 +3,31 @@ Quantizes a safetensors checkpoint to runntime's int8/int4 weight format, so a model downloads smaller and runs on the quantized matmul. +The tool knows the file format, not the model: it never asks which layer +reads a tensor or what a kernel can run. runntime fits each weight to its +layer at load, and loads a weight no quantized kernel runs as float. + ```bash uv run runntime quantize model.safetensors model.int8.safetensors --bits 8 ``` - The source is a `.safetensors` file or a Hugging Face model folder, sharded or not. The output is one `.safetensors` file. -- 2D tensors named `*.weight` are quantized in groups of `--group-size` - inputs (default 32), each group with one f16 scale. Embeddings, position - embeddings and `lm_head` stay as stored. `--include` and `--exclude` take - glob patterns and replace those defaults. +- Float tensors named `*.weight` with 2 or more dims are quantized, under + their own names. `--include GLOB` replaces `*.weight`, and `--exclude GLOB` + keeps matching tensors as stored. Embedding tables are quantized too: + int8 changes an embedding's cosine by under 0.0001 on all-MiniLM-L6-v2 and + EmbeddingGemma 2. +- A quantized tensor is stored as rows, `[first dim, everything else]`: a + Conv1d weight `[out, in, k]` becomes `[out, in·k]`. Each row is cut into + groups of `--group-size` values (default 32, any positive integer), each + group with one f16 scale, stored as `_scale`. +- A selected tensor stays as stored only when the format cannot hold it: it + is not float, it has fewer than 2 dims, the group size does not divide its + row, or, for int4, its row is odd (two values pack into a byte). +- The header's `runntime.quant` entry is `{"version":1,"scheme":"symmetric"}`. + The reader takes each tensor's bits from its dtype (`I8` int8, `U8` int4) + and its group size from the shape of its scales. - Every other tensor is copied as stored, or as F16 with `--f16`, and the header metadata is kept. - `--drop` takes glob patterns of tensors to leave out, like a pooler the diff --git a/tools/python/src/runntime_tools/cli.py b/tools/python/src/runntime_tools/cli.py index 711904b3..3f70e4cd 100644 --- a/tools/python/src/runntime_tools/cli.py +++ b/tools/python/src/runntime_tools/cli.py @@ -1,8 +1,11 @@ """`runntime quantize`: writes a safetensors checkpoint in runntime's int8/int4 -format v1 and prints the error of every quantized tensor.""" +format v1 and prints the error of every quantized tensor. It knows the file +format only, not which layer reads a tensor: runntime fits each weight to its +layer at load.""" import argparse import json +import math import sys from collections.abc import Sequence from dataclasses import dataclass @@ -23,20 +26,9 @@ DEFAULT_INCLUDE = ("*.weight",) -DEFAULT_EXCLUDE = ("*embed*", "*pos_emb*", "lm_head.*") -"""runntime reads token and position embeddings as float lookup tables, and -lm_head often shares the embedding's weights.""" - -IN_FEATURES_MULTIPLE = 8 -"""runntime's quantized matmul reads 8 input rows per tile.""" - F16_MAX = float(np.finfo(np.float16).max) """The largest finite f16. A tensor with a bigger value cannot be stored as F16.""" -PACKED_WORD_BITS = 32 -"""runntime packs a weight 32 bits to a word, so out features must fill whole -words: a multiple of 4 for int8, 8 for int4.""" - @dataclass(frozen=True) class Options: @@ -58,16 +50,13 @@ def skip_reason(name: str, tensor: Tensor, options: Options) -> str | None: return f"excluded by {excluded_by}" if tensor.dtype not in FLOAT_DTYPES: return f"dtype {tensor.dtype}" - if len(tensor.shape) != 2: - return f"{len(tensor.shape)}D, needs [out, in]" - out_features, in_features = tensor.shape - if in_features % IN_FEATURES_MULTIPLE: - return f"in features {in_features} not a multiple of {IN_FEATURES_MULTIPLE}" - if in_features % options.group_size: - return f"in features {in_features} not a multiple of the group size {options.group_size}" - out_multiple = PACKED_WORD_BITS // options.bits - if out_features % out_multiple: - return f"out features {out_features} not a multiple of {out_multiple}" + if len(tensor.shape) < 2: + return f"{len(tensor.shape)}D, needs 2 or more dims" + row = math.prod(tensor.shape[1:]) + if row % options.group_size: + return f"row of {row} values not a multiple of the group size {options.group_size}" + if options.bits == 4 and row % 2: + return f"row of {row} values is odd, int4 packs two to a byte" return None @@ -102,7 +91,8 @@ def quantize_checkpoint(source: Path, destination: Path, options: Options) -> li tensors[name] = to_float16(name, tensor) if options.float16 else tensor report.append(f"skipped {name} {shape}: {reason}") continue - weight = to_float32(tensor) + # The format stores a weight as rows: dim 0 by everything else. + weight = to_float32(tensor).reshape(tensor.shape[0], -1) quantized = quantize(weight, options.bits, options.group_size) if options.bits == 8: tensors[name] = from_array("I8", quantized.ints) @@ -111,20 +101,23 @@ def quantize_checkpoint(source: Path, destination: Path, options: Options) -> li tensors[f"{name}_scale"] = from_array("F16", quantized.scales) quantized_count += 1 error = np.abs(dequantize(quantized) - weight) - report.append(f"quantized {name} {shape}: max error {error.max():.3g}, mean {error.mean():.3g}") + stored = "" if len(tensor.shape) == 2 else f" as [{weight.shape[0]}, {weight.shape[1]}]" + report.append(f"quantized {name} {shape}{stored}: max error {error.max():.3g}, mean {error.mean():.3g}") if quantized_count == 0: raise ValueError("no tensor was quantized:\n" + "\n".join(report)) - setting = {"version": FORMAT_VERSION, "bits": options.bits, "groupSize": options.group_size, "scheme": "symmetric"} + # No bits or groupSize: the reader takes them from each tensor's dtype and + # its scales' shape, which holds for any file, one setting or mixed. + setting = {"version": FORMAT_VERSION, "scheme": "symmetric"} metadata = checkpoint.metadata | {QUANT_METADATA_KEY: json.dumps(setting, separators=(",", ":"))} write_safetensors(destination, tensors, metadata) return report def group_size(value: str) -> int: - """argparse type: a positive multiple of 4.""" + """argparse type: a positive integer.""" size = int(value) - if size <= 0 or size % 4: - raise argparse.ArgumentTypeError(f"{value} is not a positive multiple of 4") + if size <= 0: + raise argparse.ArgumentTypeError(f"{value} is not a positive integer") return size @@ -134,12 +127,14 @@ def parse_args(argv: Sequence[str] | None) -> argparse.Namespace: command = commands.add_parser( "quantize", help="quantize a safetensors checkpoint to int8 or int4", - description="Quantizes the 2D weights of a safetensors checkpoint to runntime's int8/int4 format.", + description="Quantizes the weights of a safetensors checkpoint to runntime's int8/int4 format.", ) command.add_argument("source", type=Path, help="a .safetensors file or a Hugging Face model folder") command.add_argument("destination", type=Path, help="the .safetensors file to write") command.add_argument("--bits", type=int, choices=(4, 8), required=True) - command.add_argument("--group-size", type=group_size, default=32, help="inputs that share one scale (default: 32)") + command.add_argument( + "--group-size", type=group_size, default=32, help="values of a row that share one scale (default: 32)" + ) command.add_argument( "--include", action="append", @@ -149,8 +144,9 @@ def parse_args(argv: Sequence[str] | None) -> argparse.Namespace: command.add_argument( "--exclude", action="append", + default=[], metavar="GLOB", - help=f"tensor names to keep as stored, repeatable, replaces the default {' '.join(DEFAULT_EXCLUDE)}", + help="tensor names to keep as stored, repeatable", ) command.add_argument( "--drop", @@ -169,7 +165,7 @@ def main(argv: Sequence[str] | None = None) -> int: bits=args.bits, group_size=args.group_size, include=args.include or DEFAULT_INCLUDE, - exclude=args.exclude or DEFAULT_EXCLUDE, + exclude=args.exclude, drop=args.drop, float16=args.f16, ) diff --git a/tools/python/src/runntime_tools/quantize.py b/tools/python/src/runntime_tools/quantize.py index 3a517168..85a51b9f 100644 --- a/tools/python/src/runntime_tools/quantize.py +++ b/tools/python/src/runntime_tools/quantize.py @@ -1,4 +1,4 @@ -"""Symmetric group-wise int8/int4 quantization of [out, in] weights, in the +"""Symmetric group-wise int8/int4 quantization of [rows, cols] weights, in the runntime format v1 layout.""" from dataclasses import dataclass @@ -12,22 +12,22 @@ @dataclass(frozen=True) class QuantizedWeight: - """`ints` [out, in], one signed value per element, and `scales` - [out, in / group_size], one f16 scale per group of inputs.""" + """`ints` [rows, cols], one signed value per element, and `scales` + [rows, cols / group_size], one f16 scale per group of a row.""" ints: NDArray[np.int8] scales: NDArray[np.float16] def quantize(weight: NDArray[np.float32], bits: Bits, group_size: int) -> QuantizedWeight: - """Quantizes each group of `group_size` inputs of each output row to + """Quantizes each group of `group_size` values of each row to `int = round(w / scale)` with `scale = absmax / (2^(bits-1) - 1)`, so the range is symmetric and has no zero point. The ints divide by the scale rounded to f16, the one the file stores. An all-zero group has scale 0 and ints 0.""" - out_features, in_features = weight.shape + rows, cols = weight.shape largest = 2 ** (bits - 1) - 1 - groups = weight.reshape(out_features, in_features // group_size, group_size) + groups = weight.reshape(rows, cols // group_size, group_size) scales = (np.abs(groups).max(axis=-1, keepdims=True) / largest).astype(np.float16) divisor = scales.astype(np.float32) ratio = np.divide(groups, divisor, out=np.zeros_like(groups), where=divisor != 0) @@ -37,11 +37,11 @@ def quantize(weight: NDArray[np.float32], bits: Bits, group_size: int) -> Quanti def dequantize(weight: QuantizedWeight) -> NDArray[np.float32]: """The float values the ints stand for: int * scale.""" - out_features, in_features = weight.ints.shape + rows, cols = weight.ints.shape group_count = weight.scales.shape[1] - groups = weight.ints.reshape(out_features, group_count, in_features // group_count) + groups = weight.ints.reshape(rows, group_count, cols // group_count) values = groups * weight.scales.astype(np.float32)[..., np.newaxis] - return values.reshape(out_features, in_features) + return values.reshape(rows, cols) def pack_int4(ints: NDArray[np.int8]) -> NDArray[np.uint8]: diff --git a/tools/python/tests/golden/tiny.int4.safetensors b/tools/python/tests/golden/tiny.int4.safetensors index cff48760fddff03e94f02ef4ba4319fcc6089dfe..f6f867c606cf87bed179f8ac01b17eb63a442647 100644 GIT binary patch delta 1574 zcmZvWc~BE~9L0B&C}Lrg5NK6Gj+LXwl4G+Qf;RgL0G8moO zT4;3&X{Xa+svSOk+k(Et0buEsG3Sl5A+Z+_ki7KJh7&Nv*G+*?< z5L&JF;g-tHo1>E@O`l+L9YJaSyZS+q(9+qY?Eq@DJ}`tfi`1Cw>lF`Ap9`4Ti*>;qN@BP%B0~4lehXW-K+bfkS)Usfv{f>2X)pgffE1JNqQ=L(=SQ z<7XCs?`^03o?r&!ZVpP1xU!K z#@1NZ(ShwRbSjeZ@t)rT`OS)3KQS9UE@H|04L+^s3h6qhnvaP_Qgca%TRNPQqoJ0G z#eunukaM7)2Dz9vy+9Hy2=xY4>peg4R?f&dBIpE#GLe60n;P(Z!?{D_HztFEN*6MM zV-?(tLO78zjJb`0NbH1sSZtNaRS8XqU*sR*MPwc+Kj3c<20FixCP86(ookj{S2sBn zZLb;E<#fbzoGj=^2gGu7rctOlhyi}d)L-F&>kC(wxR=!AVW^lFQHh3C3Bt&@MkAbG zja{9YmD2a2IyvoTlA>RZb-#YUZ|2rxg$dTl1+DU=Q;3y+OPPFdYG#uW2TQzu7Ms_z(03evY66%7eS5|ylZ z3OEu&3XdoVpHyM^KLp@Pr6JwfH^{AwlR|I@EZ$adlN_T_%JmrS|liZfaj zE+kVjIMSDNre_#&=fsPU_)Nq|#t6hp;p(q3G|wG5;f;)_S0_|MgdM;5sEyWP`c;adzTu9eZ zv2+?;M^{t4tOZ$`Z}o+SNrg8mw{-LQ&!nJT7ex`BS3cFm~Z-)Tz5-6KZ|8Xg}}?-UZ4Vjez|&$R7NT4;C3?*o_J2*e?wwA9;qDB^Ik_Ut(*_}#L2Z&^B|(s#hyJBmT~5dkc^0KvY)&%%#ybu;b`YdLF+$$_q_ zM}gy%jOD$u0N79G*ki(t@SE5^d$4D%alpJC0JI#wI8SMRXUnh9bRiwYRy4v(`a7_A zU$yvNur<^GXXDLm=T!%6&fEbr%i4kM IamlWK01vdZQvd(} delta 4289 zcmWlbc{G*#7l%XU$k1S_Ol3|uN%4O7L5f6FN`o?lif)8T#Zi$d86%=hiA07>b>8nj z>QY>l(u6{q@YA)j#Zq@YP zw>KZgNJPb8=c!hdx-|OcW;u{*GxopcKL(Zb5 z=IO_UEEWC(Lyn25g-3*;W}sKN@4j>|Va3!VGU@*c&650YVaPGF;G|Czwv+r13^^PV z^Yj8?hW8&Bau#zq9uX1hfjEWmR)|yflNR(pddw7PNa9b;CKCRx1m1+~gI_uvbek-| z;ZyVQFFQTP`OXZ%h?FN4IdmQe!Yna)y)-I)DxggB12Q8x2C~kH&=o_KV75vhH70Ax z0|$SIi5D;vZUjxg1fbLw-Y|W1$cq?Q9>sa48|a|47VQm{f}DF>$gZ(E;^L))K}}QX zxf**A=pszlD?sbp&2Yi)GKfvJhiiVZB=TB54a~m=`0g7_o_zyr-AW<#TRkzpl!M>K zQm<24T@Zg)j6qKAL}~Uj(rauAMyXl2(N_b z?T{r@27X$a;4CI1a8q+*ri&Rv=g}z8HJ*#*OWwhtNEyD}q=!M<_d?`*d&W#>3fD;5 zng}avqL527t+{$Qmzmp2A`6bZUO04S1@V&uO8(6|%Q@ zq*N>#YWlT_#p{_ktD+pI`qpCCl_~JZBLbdS>C*P-c&z4ahq0GMSdeIr;@{_ji?E!) z>Ol+H{bmMA-tA}nCC{>oU!93{oe9Kf+aNnw2lg+P!t-An!B*uoc%OHGgqiNp)tSS@ z=GXJk)CG5B>)Jn1QFAPy_aVDmR0zR;E!*d0*| z1s3U`?KN9)%q|Zu4_1)miKiq+o)Vn6No*5}$dQ{3v~A}YUXkr*w>LZCKQ;-#=}xB_ zN!?7Vy z`QgY0`&Kjl?i7WeJ%Gr8-LN(_4h}E3WnPx;20Fr$FOzBz{#2E2E#+fW^>Mgy`74fB zhf(*xTxsx;7%UoB14}I@y11gBZkizq2Ew1IGOxWF{x@1nT6;gyisus;w&Nn&%}IpC zE7q_r<0({O!vp9sD#r5nGg0L0C^b`=1;Nkt@Uz4mj1Rkm#r4Zz!FFf3_;!qD+UStK z!Y*KB>L{)8tb*S^q{#1QirfP+4j7X(11e4JX;8Nu*g1_{ue>xJHPyBu@6*!d+*$tT z*aE&Jx`;S1`xt=#3OPfQ(`GpJULS+>bco*N_1LH}iyLXfLC?vz)N4;1 z-ZQVJR}ux+A0{n_9(y&6wIuY!nz}zXvkrB(FXD!+jDelf_V~!S96K-^C913W*N7|= zaEf4fWidv{)q|_;TjF&KsmM6bi>^r)M}r|tf!e?blPw;PUp7V|Go*#;2QA@q*J+H; zY(V!rTCjGmHB=r6L&wSK(Cu=XPT1s-`Thl%X5j`qrsjaz7ERuxf!?j&jEqnhA=6{&1B>@GeP39cf^OMWk35M!TRP`a zA^+5lr;u$s4_upWqM%-g%O~x0g|-*|Y~@2uha+pM#;1EFJxSpzcd&PighH_@BzJ#9 zjPE`uUT}j}r@Nw={6TzN@q_VwH6L%^wZ(0fYap%k8X=$LSGfJx!uBaK;n~5b>#J++;dnkHtxRpWQ^BtbIxH5*ulx_e?mh(TE3g#JO{B zmt(}i02qr(MlF|gP!VdOc}sbkaN0PVn*A%ry?effYA2k{&}^PVhny2&=Op6r-xbgrc#M^NE`|MV04=;rvEU+a2lT^luyjj+PY%U!f+K}m zdOb}3JUK9MPvw8un?wBdd&qWmcUY^`$qp*jQr@)nm}b8N2Ng0QN4bJs6y^Y;awYg` z<^-O%KY^G3PJ^5EaqxA@B_g)P2{lGVsa)t8x+%YkhWMxp@?`T@dEy~n53_}1kB+)4rV21A~|J_B}G;AS7Zgm zf0VPF@3jg8?d6I1Kif#+93wpJmIeWJqZnMinYLK#!wuJR!!yJuR)a77`USa^X#w9n7USA08FYN30_o&R(oJ4 zktLRiWwdh95F~W10Xat(TK0n1hDXOJQS?j0xy8NsY2+0Bu9iaFb&DZ3FNLh$_~#+L z%SHdwjr^nDzZiY~Rxqk&(G9{p%*BS>ZlVdc(#}3HBJ?xqj@lYHZ zh03WbvBS3zn09+w-PS-?^_^pejx9pjh)a<2>O6XwtI&?3YT`chh;8t8!e-}fxcW{v zX?t@V^@8Sut-;~EAr8JcmdS+W%1?CMdQD_Z=jX$ZJdyBWq87h8R&Vn z5+hp!Y1$2WZj16MQsQh2PabR{F~Rn*ez+Xv1E0}b_hN`cYZ|{ldpYghuaE7kqUl4E z73lI?MR4);8N3}gM%}p8@kb7{HN8Ormf*Dvud%JO}zpO=O^66tvZM0{{MVxG1Yi96a-2LBMgg@xC-JSyVuhW~pL! zqXNFlP({lj1%aB}Q)Wxc9~ZpR%VwNuf|Z`4IHy?(PiIs@O^-BG{tO5HP#yI?27sl; z7$R$g2a1&7$3kU%;k6vL$}4lbyLeCVKnfSfJd=73$7T zCsJ``#6T|>H$=qa9+g^9Q_m!-9oDsUy8g(Taib4$O0_PHR z{aFA4m(QeLT0;=JIiK3kRU|+Dgv8uYC6lO$GZf#m@*gwFqb2dQWf!Fm%kyDy{XQHR zsi7`YgayJ8QlKrl9tw9&gUpfL;8Og87;U>kOv*lyi_hYyhUyuRI#@$DExeArfW%Gk z@Z|^`=m^8_TAi#$xG~=Ay+MEcW5r~Qox}6Gr|5&?>$vTJJZjv}LZ{?dlvwZ#I~8-G z##I>h?;XeLHYI#sV1NxT7U5yDCJdZs4W*UsbV>CGa-drb&d62)-)RV=6Z#-wXCZ`M zRe;u$O5As^U3nyU^;)QvU@+e_71DYPNS(-A`r)uR_o{3zRWX*gx*@WPmV{X2K_Lm) zCBcAeP#tMdoGq9pb`*lw*3u(YXYqWH3cQ&m2SLYU(8p>nSNc&7N_0KPSJ%_AHvEOG)Vcn zUYHorzSx%QXp%4dA0+&2sS`odxU z9S8RC&W}`4x0>?y)YFq9{xolO8^+0u(1G}F`fQgq%{4nl%OngT_)j^TqyZYmmjcf( zL3B-t2g%%I06x>6@`&TiDDq9mf##(w1dp3$M1lxm(;gLi>3?T2PIo_Qh>YTo=?hV3 z>=MZAm&Fyi4lp*$`?|qKGo-7_Ay|4QOtBom^3^fWk`|AO#^&^}qycO`kxi4ACWG;y z4NQO1hOY)k=|n&ZnPaAiSqD7mRk|AVOn8N)sv-bII!wt`Rt*Ca&*J3^Dfsw*X{5pV z4d3NwwdJ?^G!%EYqC?HmIA3=U{+y*pKPptCvYP?S)|7r#BiR^1iC4=p+D!1{7#GR7A|rDeF! z+$@17)m)1{()t4Lux3_;lg2D>%)pM7i9~0^a=fO0m$+Ukp-ZCvA)*@ncs+7}Cg~~B z)!y~E|Kmqc50b(&!_}}(K_8PN0;qCA7MLWTC9lht6IL&d*!ET8sgPUre0M0`U9gJm zx0S{X2RLks<2iQD1l8v+_2fv^gBiJ&M}$7-#`m{zKIq^D}BwzYPgx9_xd zn^9{y9j8?*Rg2bz%3|F}{(vBh23dpv3A>Smumlo90$GOkoPL^5_j&H~JNG^}q6|y1 z2JAWP+OUJW_U7-)-@#4Y!9Do-zJ2+JJ}%g`gL{;FRLI?RVE?B@eqPIXiNZvQLde~n zcPQ_@1AF!y%s+IHn<^Ixct{2ZiNz2vf|u7g0Sn7&t2sDEjpoC z5+C;moWJk={M|*1G*7-t z?%vp`^>+_yqsL~kTgAbei&?4m@YuK{z5JdeG@&r}@sHvsp7={B9XE}A{)_GDrBFA$ z=XFmXd5G%!w`QKp(=Zu=xNx)4o}2m#)Fi>SYm>-W7giADqYfNMsC8hf-^w=f87Ret ze{PverDJk>^yvP~j94;lat8l_z^Qrp5szRfUr5)Rr^)BcbCSo;z3y%(+hDZ(`afFpE>3!UfMXgoc!&B@O$9f%8OwhE9QkrIMJZeCf zJwC7W{cZ7n+DGL7%hd&=nZH7IwYZ~8MA@$!dhYjA0tNLz8{Kh2h^NXrO!NFFeR-imfx zvz{5oIr>pmX!tm(c|3)6Ra{I&gw?z8Crr(GWGV55{}wi!`Ed*@hSPb)ickZJSj3Xbw; zqTZeb6P7w>j?(godfQL{yYDeG&9T&gpJeZqrxwos4 zzn`58XnA24_aA0Dc1Cf=4kudv8+h;RBy%e>vD8gYB`5ZCg6~jFXojI-)H{E6*$yas zFy6UV`F!i_IKDUMk)1i;(aLFnZVh1{K_2AktzU7V(t^-dP=t?+RFAE1MuXM_Zhnst zE7D!ZUYlC>J66UF63NLKt`Wb%;lX1><@|6OlUIvcV{%q&H;8gqsrVABs%njQx^u zsBXT~fGeLWAFgSo=W%JMGBw5- zENfQ`oTGtv|dXP>7pWxdx4f>HiFkSf-+N!+bL>I@r$= zL+Q7b0<=A^I|ZZ8TaKN2MWD1BevDD`(^?pQjy9d#ievO26vGka-= zZ`>QhS}ba`jz6C55AU|x0*{%Ysrz)grKxYa>zdYOa3c5>1w19XfNS*CGR@38!?t2X z_il%I6U&!z?KSY#x)$8-mk&GE21Y9Ga_<}OLZiGw=p@tyX?QUanGq)b1pyl(K@0p0 zNGG_&tK;3_ori{@c8JO6KrN6HGH{PV=XrGAC|}M$&%^BIDyNs1!HVNLr7e;HD(rqD zEOH|E3_OKD0f+aTf>poIs4m~f!Kvva)lNo(>dR|)R18H4INtgIB)zj9PU^^19r;Ni zr6hj|uhaB_>Z-M1{#3lQq`FZ$N~=-vtGtOKB7h{~2zYg5J6vCa2f2*^K7BPCoFA!|4pt$1RYfEj7${`J)fJ&C z?1xKXWu;6>a-IUE-fha6mbcPhDn5Z>uf1R!CtP`Kj}BxsM!??tPl4vdQLz2=$4d6` z(=aRi38>Ec7s$+(g508S!FyS?(ks7h1mb(i0Nvsb?=6sl?ceuFD{dbKa=``QWgY|i zm6Nc0m;I{hR@3LwQ*w6|Ud@yGe!V{ZlME~T?}e{{X?p|EA4dVu+ysVX7~Y8A3^#B- z1!_kbFkHW<%B7dX9S0@VxHcl3TYeH$FaHKEaGV50?suvhbv5`d`Ur^LRR$MDvcCKu D#TQ2K delta 4247 zcmWMoc`(-NAGKtO5>eSITiKE|DZbC=MT$fu(O!g#ZiGt3t0G%=*&=JQM3!uMzt87Y zm*T3FHWbn(SHF_0X#MrqnKN_d%$%7sXU>RXskYFNV6u>{g2#cNz#VF<&6e#@^Eq-j z*x%*A39mog*eH43qT~tr)DuEdsq%sgW*96pF-VONWW-DiR+%m{F-*N9Xenl5Xk@m` z#57e}Xq~u;smULW>B`g+A;SHGM$1gh3=DRLhN}3X0O76`qwHsO7<~MMDNvn>Kh>H^ z$omp_8+ZhMX_}zxbODYgEXKcVv>C^{a{MugJyiHa9u5VYW8!8>l>b~nnbwCyE+7&z zE(z1MBUNC$UI$gD>&QbpABc?MGgGbvjlTGz#CGl|eSBgs(KSDfi;cF>VM%q`A0z?U z_qUS+6ZOPtuO|98&!CrTZGo?aFh{!pE$_6#Wt*!YI>Q!jct?@28~M~P{|4auZ!mrF zEo^cvh2(DyM87;6zfB~!P$?}CdtZ$H4xL1P!E@5DZv=YD8R+JvjRD-CK{}Ur0WTH} zfbhsqTK`um$@7e$%(YI)5G(_4bv1Aloy~Vuc4g*>>O=SGaM03UgcU2_!?17}zT2ve z{<{xD*aut2SW|$bCuvE96x2}Asg+ii$DwFk5tewUg6(ZNP`rK_p6OZQRg-6MQX`d` zp6dXvQu+&8XlsG&Z7wMljey!g4Py3Y9?q|fI3L1MC@Xy%f!qwqQk~sB@M9NTtQ@4n9Y!NwitC4o>pTKKU zgY52B2mHq>7EF3msp@1obDO&rHJgM`chZ8_uEM40J&{EYSoE?LZ5dc{N`dd;CJY~U zWYWSaF^qf|K<&>$uFmVB93hLC4U zbVn%{r$d>x|N`sBVSW8v; z98^=@h1}1p)^O(gTwp)vlK z$iq_YY?znx52$tIqo-vCX6+hAO+1RL?ycn;nWmtNU$X zI-e6}WrBOA-_gB?qVc|I4ZRk}Z+R5I2KsE3G0L3Ka~tdbbjAfWcN=hm*G0m9Nn3oZ zUx8g1g5otbyc7?ULygJ0aj zkr`1(mE-2{U(W@MNpD2=yXvrMktI}}3P$_sIne8LflgUvlO;X{m}2G%`(|W=`2uy= z^-&5Y{Q~&b0g3Kx9?WdP9zy0s(T8RqXzKo4s5ljmD%|I1VYAc< z^6jkyo;jXI>mqzn?7=cvu2M^kVx=&2UOYyh$R?{61cSKxd)C3|2`Myi=V73rn7X}X_@=2= zIB&59nHBkxexGXt7GF+7nDYW!ca@^cvKrtP#KHyrENc8eQO>=*TB;GBjCQ$;Y4DdA znD}uRY3nr5GhQg-8U(NQN`hMJLOS9Y3;U-LNB^#bcE2<1%oh?k*a6VSEsp{x85^J< ze}lPeEPS>rhI1wosIJ||!S^+Kef-`*3(} zI%F$UvIfC+AR=9Yujfr+p6xlj`gaQ4YKVrf0_8+>y926@i%{vHOu99{ng)8R@TCj3 z~h+iaS?>ESacb4@TK zF&#;QDV7vf(_dkg5c5gee97MR=w~ZK#Qxbu;uh-RN!JwctslpLify#bQU`82S0Lw0 z6NwDH4B6{Oh_>4^{CRLSF5=~K!5cG)XOt>W^36+9o^A%;cCNrp*V5>u+g#Gkk;HRX z#A%sI2}s^LhQm2uV0*$R_~G1-uwRpeGd9uXyTpr_{He(c7R#U2WHVT2QZNp!~vT|vg^Pja%CzHz4^u@ zQM8z9Hcz0&wqS_0X{HvY+ptW(mF|t0O_mF{g4d`v^%@)}fe)@hTe>(jaNDCnae9J= zdfEUveu&+8E(VH&!cif49d>yY0@GsO`T5>=)d^-Lg`cUoXc*tj$OL(WTHZkinC_6oLOccoQwW z?% zO%Eh-r9lCSpRa^jO>^;eni86i%;hUfKV!DH{rQ5|`q{M1W>~jJ1Q)hS;Dxj*sO^)4 zs-Gdi8>y!rX8^EN9|NWI@K}*N{8+AlFZZs29Wn}>-X88#JeI`4i9Lz1J5i2iUP^-w zyVLmjav3?St4Q71IYc75jOc3T;Fi!BJfv6$$|~tZsjHW%jWS``^9_VInTnQtE~WWl?Qf9=#faV_m`cUA>!C4bjK@{Ws~4e=L}^iA$KLl|UaBx8Sa0GN}3> z1052hP<-ig?3T@eT4y0RdUz6RI^^+1fi5<_G{BR_&FHt-5=yH&>B^dq&S1V#GNknBl6v8H^y5h}&UL9As;Dnx zaZ`9bEeW*5FJhj*BD|e14gO~%(bHlPNAhtt ziub(0*Da~o_PicCWKHPGFiPdK!r1B-c~q=7z`CUxm}@xF}+RNFkDjaL37@z`eE$xDD9VnU8z%mm*#g($|Y`auo3 z`_Nra90wc?QD@)_J8T!v!$D29+f)iY$AE0n+5^$wy)pOm{?#XHGuY)_AIbVVXV?&z za9&VTC~~@LEb;{W&`4wk1n%o$FIcHFvzP3{Ikz0(l(`{1KEDIP;-xrAi`-bHnb~kS z>nD4^t%IDI-9or2$D_!Jvsw_Jdy(dpYBEw1GoiA15UPvaaJ&9HlN$_8@X7%1fe=`7*N#28{}Ywfs-fIN4fMRQ56#`sfzh+a=uk{AeSW}_<`|!$W#Y>q z;7>TK_#qn3lYl*6{OQJ$og{s$E_lv*#wGUi!pS#HJDQuc9CqF^CgMa8n-3|{@_#R4 zwAN8n6&}YQbC#p#L^;epDuruv>|kQPM~kkTG13hc5Fj}Z1k8u9VnZafrNp4Dz9~I9 zQx~?K%c6;^5JUmgn@d#{64D3W~W~(2>>%T%vUdf6mvYpXSz}f~zhp&|b_L9vvlp zp^^CLSuD+vzl|Tav{IKrK^S{EAC^i-;3e%WvRot$o|X)Oy^k@PTI@uE3c5%9#=_vXe>1dO2bz<4AvCq(=oL@=yPWhzkghVE8d-AtM3Wo$F^*C zXmbTK9q9!@lCwF_UCn_j(OQR|k~(~k;8s@AB!yYil!jgF;)v#!HF!hk9&s)&p)14x zAtI`S*b+8G_)B2sXbrf`)xpG2U#bwB0fvbe$(yn@gw>8F)&o_T z5O|yB^#aFHO gXJIvDe*JSc&I^H9P7D_7RnWRS;vAvWky+vY2V1HRR{#J2 diff --git a/tools/python/tests/golden/tiny.safetensors b/tools/python/tests/golden/tiny.safetensors index 3be7e5396ad6e32e4aa969b9ecc377938d524afe..365b1867de876a3483c482466df42dab00ff78bb 100644 GIT binary patch delta 2215 zcmZXVYc$nq8^&$KY$>J)^{O07X*ZHn;s4y^kQj$_-pJ$>Q;{JlX;aQo4kLuhp~#_B zDD3}pZ&8#~qNp^b6lE|O6;jPu>wQ1H_m|(guXSDP`gry0by;#Aq;QrK0)5WLF&dXD|osFdWLS{Jaz9q9^PL6wWl~Lai?^i zfuY{Yl-(xWDgOHy>Zeqg$xo4IWTd-tDi+}Nzt93bp8kseZ;Tr)?WGka`MC6T0&zLIkhK2_ zG-H;UsHsoD#ld20_SnpRCj40gD&&k(jo4Vb0R1*O6K3}$X(`aB zJ6|}{+zve`Rh|anj#1dQG@tztC(rQ5V=#{!O}GUv==LNOblk0o^V@1B-(Cr;-i-)Y zt?KXisVx~bZq<-L;~zwu>$ZYY#0$)=F^9vyZN*Q!*1~LMDI$KY5$E-uf$sS0P}3O0 z){{i4U8zCS3l8An(~0nTLI!2Z*Rux{?Qo;@LKGwx;Q79@pnhAG7FzEIKS#c)f1@ti zS`<oQOzYXgzZ8{y82U2x6%9nF7V4{ph4@Ku&8l=M!L$aC(n_2^&VC{AHXdMbJ? z?!v+a>tS#p634+#xMOTHjP87h16*lQ=Zz@xrPY9Gy&6XI9;-3}k*zE0)YfBakv8qC z55+9EhqykN18U)6>=DfeBw$TACWI`(b@c-LG~s~~DmFwa&<33ys!8~0Ibm`*X78-8 zV5+?*?6g^@8_7duHWg0<}_gXE(vqmWksdSbTO7Q zDZmsZCsARQG%8&wfmShdW9-sBa)BW*=^uX*)kTNbM<5Jaiwp9&Kt2q<>Je#Qhv3?Bc zYKB99guwCYUt!2}8KkH8FzOdviMVtm%{$~o_wU)xKuIw*jZ*`R%<%W?W6Z6<3Y z3)74DK`$>5Uybu|`;Q!4T9{6jGS0Lu%Z02S${X+kpX$>eoEN>>Q zv0ar6N36yCj&z*g;LTcd4d@rIG%T_kV$|2C;Z+;XLS;eRw&O4?83$@l5`b2Z&@ZE8MuYabIQheYk%bHIl%g`L9$ojZ+skC2KLG-P%=^qfx(URo=*lvon!&dbW=C8 zF43d5`YrIWauH76)FM4Yk5F5>k-Bw;f#&M7SWn_1prnlGoQ=kdzG7G`GeGZj*s^tI zT-3b10S891>4LzcXz=eORc^FmWhFfz_(lUtzVkMT^<0eliw|P&K|b6(FNxcl5t0hl zV#e#6_?-Ktd>xj_+p{mhY{ z{+wJb3q=>LY~=j)9^aS@!8gyb0$O_e07-E9CX{T|ATy%un0;oWkoYAE2l!3+CN>hW zvjqEB0XRrHLz8j{wvJ!M{-P?nP5uhB2kDC>+N5CHm3S0sM}yyZJW8pj()v~rrY=8- M%RMhcny*U4KZ7zk+5i9m delta 77 zcmccN^1_(WVIrf(!~$D31053sOS6gFtT{onj)|d>#l&9@oFH1q%+TCyau=hm5Qw2; XW^7;*TU)CD2AglND@aVfq1XTbTRRoW diff --git a/tools/python/tests/test_cli.py b/tools/python/tests/test_cli.py index 4763f465..b6bbf3e4 100644 --- a/tools/python/tests/test_cli.py +++ b/tools/python/tests/test_cli.py @@ -1,10 +1,12 @@ +import argparse from pathlib import Path import numpy as np import pytest +from numpy.typing import NDArray from safetensors import safe_open -from runntime_tools.cli import Options, main, skip_reason +from runntime_tools.cli import Options, group_size, main, skip_reason from runntime_tools.quantize import Bits from runntime_tools.safetensors_io import Tensor, from_array, write_safetensors @@ -36,40 +38,101 @@ def test_golden_file_reads_back_with_the_safetensors_library() -> None: ): metadata = quantized.metadata() assert metadata["format"] == "pt" - assert metadata["runntime.quant"] == '{"version":1,"bits":4,"groupSize":32,"scheme":"symmetric"}' + assert metadata["runntime.quant"] == '{"version":1,"scheme":"symmetric"}' weight = quantized.get_tensor("layer.weight") assert (weight.dtype, weight.shape) == (np.uint8, (8, 32)) scale = quantized.get_tensor("layer.weight_scale") assert (scale.dtype, scale.shape) == (np.float16, (8, 2)) - for name in ("embed.weight", "layer.bias", "norm.weight"): + for name in ("layer.bias", "norm.weight", "odd.weight"): np.testing.assert_array_equal(quantized.get_tensor(name), source.get_tensor(name)) +@pytest.mark.parametrize(("bits", "dtype", "stored_cols"), [(8, np.int8, 64), (4, np.uint8, 32)]) +def test_golden_file_stores_a_conv_weight_as_rows(bits: Bits, dtype: type, stored_cols: int) -> None: + # conv.weight is [out 6, in 16, kernel 4]: rows of 16 * 4 = 64 values, two groups of 32. + with ( + safe_open(GOLDEN / f"tiny.int{bits}.safetensors", "numpy") as quantized, + safe_open(GOLDEN / "tiny.safetensors", "numpy") as source, + ): + stored = quantized.get_tensor("conv.weight") + assert (stored.dtype, stored.shape) == (dtype, (6, stored_cols)) + scales = quantized.get_tensor("conv.weight_scale") + assert (scales.dtype, scales.shape) == (np.float16, (6, 2)) + ints = stored if bits == 8 else unpack_int4(stored) + # Row-major, so value (o, c, k) of the conv is value c * 4 + k of row o. + values = (ints.reshape(6, 2, 32) * scales.astype(np.float32)[..., np.newaxis]).reshape(6, 16, 4) + step = np.repeat(scales.astype(np.float32), 32, axis=1).reshape(6, 16, 4) + assert np.all(np.abs(values - source.get_tensor("conv.weight")) <= step / 2 + 1e-6) + + +def unpack_int4(packed: NDArray[np.uint8]) -> NDArray[np.int8]: + """Two signed nibbles per byte, the low one first, as one value each.""" + nibbles = np.stack([packed & 0x0F, packed >> 4], axis=-1).reshape(packed.shape[0], -1).astype(np.int8) + return np.where(nibbles > 7, nibbles - 16, nibbles).astype(np.int8) + + def tensor(shape: tuple[int, ...], dtype: str = "F32") -> Tensor: return Tensor(dtype, shape, np.zeros(0, dtype=np.uint8)) @pytest.mark.parametrize( - ("name", "stored", "bits", "reason"), + ("name", "stored", "bits", "group_size", "reason"), [ - ("fc.weight", tensor((8, 64)), 4, None), - ("fc.weight", tensor((4, 64), "BF16"), 8, None), - ("fc.bias", tensor((8,)), 8, "not matched by --include"), - ("embed.weight", tensor((8, 64)), 8, "excluded by *embed*"), - ("fc.weight", tensor((8, 64), "I8"), 8, "dtype I8"), - ("conv.weight", tensor((8, 64, 3)), 8, "3D, needs [out, in]"), - ("conv.weight", tensor((8, 64, 3, 3)), 8, "4D, needs [out, in]"), - ("fc.weight", tensor((8, 36)), 8, "in features 36 not a multiple of 8"), - ("fc.weight", tensor((8, 48)), 8, "in features 48 not a multiple of the group size 32"), - ("fc.weight", tensor((4, 64)), 4, "out features 4 not a multiple of 8"), - ("fc.weight", tensor((6, 64)), 8, "out features 6 not a multiple of 4"), + ("fc.weight", tensor((8, 64)), 4, 32, None), + ("fc.weight", tensor((4, 64), "BF16"), 8, 32, None), + ("fc.weight", tensor((8, 64), "F16"), 8, 32, None), + # Shapes no layer rule looks at: any out, any row the group size divides. + ("fc.weight", tensor((1, 32)), 4, 32, None), + ("fc.weight", tensor((5, 96)), 8, 32, None), + ("fc.weight", tensor((7, 36)), 8, 12, None), + ("fc.weight", tensor((3, 5)), 8, 1, None), + ("fc.weight", tensor((3, 9)), 8, 3, None), + # 3 or more dims quantize as rows of everything after dim 0. + ("conv.weight", tensor((8, 64, 3)), 8, 32, None), + ("conv.weight", tensor((8, 16, 3, 3)), 4, 16, None), + ("conv.weight", tensor((5, 3, 7)), 8, 7, None), + ("fc.bias", tensor((8,)), 8, 32, "not matched by --include"), + ("skip.weight", tensor((8, 64)), 8, 32, "excluded by skip.*"), + ("fc.weight", tensor((8, 64), "I8"), 8, 32, "dtype I8"), + ("fc.weight", tensor((8, 64), "F64"), 8, 32, "dtype F64"), + ("norm.weight", tensor((64,)), 8, 32, "1D, needs 2 or more dims"), + ("scale.weight", tensor(()), 8, 1, "0D, needs 2 or more dims"), + ("fc.weight", tensor((8, 48)), 8, 32, "row of 48 values not a multiple of the group size 32"), + ("conv.weight", tensor((8, 16, 3)), 8, 32, "row of 48 values not a multiple of the group size 32"), + ("fc.weight", tensor((8, 9)), 4, 3, "row of 9 values is odd, int4 packs two to a byte"), ], ) -def test_skip_reason(name: str, stored: Tensor, bits: Bits, reason: str | None) -> None: - options = Options(bits=bits, group_size=32, include=("*.weight",), exclude=("*embed*",)) +def test_skip_reason(name: str, stored: Tensor, bits: Bits, group_size: int, reason: str | None) -> None: + options = Options(bits=bits, group_size=group_size, include=("*.weight",), exclude=("skip.*",)) assert skip_reason(name, stored, options) == reason +def test_quantizes_embeddings_and_lm_head_by_default(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + source = tmp_path / "model.safetensors" + weight = from_array("F32", np.ones((4, 32), dtype=np.float32)) + names = ("embed_tokens.weight", "pos_emb.weight", "lm_head.weight") + write_safetensors(source, dict.fromkeys(names, weight), {}) + assert main(["quantize", str(source), str(tmp_path / "out.safetensors"), "--bits", "8"]) == 0 + assert [line.split()[0] for line in capsys.readouterr().out.splitlines()] == ["quantized"] * 3 + + +@pytest.mark.parametrize("size", ["1", "3", "6", "640"]) +def test_group_size_takes_any_positive_integer(size: str) -> None: + assert group_size(size) == int(size) + + +@pytest.mark.parametrize("size", ["0", "-4"]) +def test_group_size_refuses_zero_and_negatives(size: str) -> None: + with pytest.raises(argparse.ArgumentTypeError): + group_size(size) + + +def test_report_names_the_stored_shape_of_a_conv_weight(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + argv = ["quantize", str(GOLDEN / "tiny.safetensors"), str(tmp_path / "out.safetensors"), "--bits", "8"] + assert main(argv) == 0 + assert "quantized conv.weight [6, 16, 4] as [6, 64]: max error" in capsys.readouterr().out + + def test_quantize_fails_when_no_tensor_is_selected(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: destination = tmp_path / "out.safetensors" argv = ["quantize", str(GOLDEN / "tiny.safetensors"), str(destination), "--bits", "8", "--include", "*.bias"] @@ -80,7 +143,7 @@ def test_quantize_fails_when_no_tensor_is_selected(tmp_path: Path, capsys: pytes def test_quantize_fails_on_a_quantized_source(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: destination = tmp_path / "out.safetensors" - argv = ["quantize", str(GOLDEN / "tiny.int8.safetensors"), str(destination), "--bits", "4", "--exclude", "none"] + argv = ["quantize", str(GOLDEN / "tiny.int8.safetensors"), str(destination), "--bits", "4"] assert main(argv) == 1 assert "is already quantized" in capsys.readouterr().err assert not destination.exists() @@ -95,7 +158,7 @@ def test_f16_stores_the_float_tensors_that_stay_as_f16(tmp_path: Path) -> None: safe_open(GOLDEN / "tiny.safetensors", "numpy") as source, ): assert quantized.get_tensor("layer.weight").dtype == np.int8 - for name in ("embed.weight", "layer.bias", "norm.weight"): + for name in ("layer.bias", "norm.weight", "odd.weight"): stored = quantized.get_tensor(name) assert stored.dtype == np.float16 np.testing.assert_array_equal(stored, source.get_tensor(name).astype(np.float16)) @@ -115,4 +178,7 @@ def test_drop_leaves_matching_tensors_out(tmp_path: Path) -> None: argv = ["quantize", str(GOLDEN / "tiny.safetensors"), str(destination), "--bits", "8", "--drop", "norm.*"] assert main(argv) == 0 with safe_open(destination, "numpy") as quantized: - assert set(quantized.keys()) == {"embed.weight", "layer.bias", "layer.weight", "layer.weight_scale"} + dropped = {"norm.weight"} + scales = {"conv.weight_scale", "embed.weight_scale", "layer.weight_scale"} + with safe_open(GOLDEN / "tiny.safetensors", "numpy") as source: + assert set(quantized.keys()) == set(source.keys()) - dropped | scales From 94077de0b2ef4d0cd3caca9b38f0d5c63365c99a Mon Sep 17 00:00:00 2001 From: chmjkb Date: Mon, 16 Nov 2026 15:47:49 +0100 Subject: [PATCH 3/7] docs(docs): describe what runntime quantize stores now: any weight as rows, embedding tables too, float fallback at load --- apps/docs/src/components/QuantFile.astro | 8 ++--- .../content/docs/core/exporting-weights.mdx | 29 +++++++++++++------ 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/apps/docs/src/components/QuantFile.astro b/apps/docs/src/components/QuantFile.astro index 7ecf7d67..3d388bbb 100644 --- a/apps/docs/src/components/QuantFile.astro +++ b/apps/docs/src/components/QuantFile.astro @@ -4,14 +4,13 @@ * tensors around them. Names are all-MiniLM-L6-v2's, shortened; the floats * are F16 as `--f16` stores them. */ const rows = [ - { name: '…word_embeddings.weight', dtype: 'F16', shape: '[30522, 384]', loads: 'float' }, { name: '…query.weight', dtype: 'I8', shape: '[384, 384]', loads: 'int8 weight', quant: true }, { name: '…query.weight_scale', dtype: 'F16', shape: '[384, 12]', loads: 'its scales', quant: true }, { name: '…query.bias', dtype: 'F16', shape: '[384]', loads: 'float' }, { name: '…LayerNorm.weight', dtype: 'F16', shape: '[384]', loads: 'float' }, ]; const rowHeight = 30; -const top = 84; +const top = 66; const height = top + rows.length * rowHeight + 14; --- @@ -23,9 +22,8 @@ const height = top + rows.length * rowHeight + 14; > __metadata__ runntime.quant = - {'{'}"version":1,"bits":8, - "groupSize":32,"scheme":"symmetric"{'}'} - + {'{'}"version":1,"scheme":"symmetric"{'}'} + { rows.map((row, i) => { const y = top + i * rowHeight + 12; diff --git a/apps/docs/src/content/docs/core/exporting-weights.mdx b/apps/docs/src/content/docs/core/exporting-weights.mdx index a5cab8ce..9f35cc1d 100644 --- a/apps/docs/src/content/docs/core/exporting-weights.mdx +++ b/apps/docs/src/content/docs/core/exporting-weights.mdx @@ -124,16 +124,26 @@ Full / half-precision files work as is. Optionally, you can quantize your model **How the runtime recognizes it.** A quantized file has a `runntime.quant` entry in its header metadata. Each quantized weight is stored as `I8` (int8) or `U8` (two int4 values per byte), with its scales next to it under -the same name plus `_scale`. The entry can set the bits and group size for -the whole file; without them, each weight takes its bits from its dtype and +the same name plus `_scale`. Each weight takes its bits from its dtype and its group size from the shape of its scales. Every other tensor is a normal float tensor. An int tensor in a file without `runntime.quant` fails to load. -**Which layers are quantized.** `Linear`, `Conv1d` and `ConvTranspose1d` -weights. Embedding tables, biases and norms stay float, because the model -reads them as float. `runntime quantize` only quantizes 2D tensors, so -today it covers Linear weights and leaves conv weights float. +**What is in the file.** `runntime quantize` quantizes every float tensor +named `*.weight` that has 2 or more dims: Linear and conv weights and +embedding tables. Biases and norms stay float. It stores each quantized +tensor as rows, `[first dim, everything else]`, so a `Conv1d` weight +`[out, in, k]` becomes `[out, in·k]`, and groups of `--group-size` values +run along that row. The group size has to divide the row: `in` for a +Linear, `in·k` for a conv. A tensor it doesn't divide stays float. The +runtime loads each tensor back into its layer's shape. + +**Which layers run quantized.** `Linear`, `Conv1d` and `ConvTranspose1d` +run on the int weights. Everything else, like an embedding table, loads +as float: the download is still smaller, the memory and speed are those of +the float model. The same happens to a weight whose shape the quantized +kernels can't take, such as a Linear with a group size that isn't a +multiple of 4. A console warning lists the tensors that loaded as float. **Making a file.** `runntime quantize` lives in the repository under `tools/python` and runs with [uv](https://docs.astral.sh/uv/): @@ -146,9 +156,10 @@ uv run --no-dev runntime quantize model.safetensors model.int8.safetensors --bit - `--bits 8` or `--bits 4`. Required. - `--f16` stores the tensors that stay float as `F16`. -- `--group-size N`: how many inputs share one scale. Default 32. -- `--include` / `--exclude GLOB`: which tensors to quantize. The defaults - quantize `*.weight` and skip embeddings and `lm_head`. +- `--group-size N`: how many values of a row share one scale. Default 32. +- `--include GLOB`: which tensors to quantize. Default `*.weight`. +- `--exclude GLOB`: tensors to keep float, like a layer that loses too + much. - `--drop GLOB`: tensors to leave out of the file, like a pooler the model never reads. From 17daa76e482c11570ba42ecc94e56938549ea25f Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 17 Nov 2026 10:57:50 +0100 Subject: [PATCH 4/7] fix(core): warn about a quantized weight loaded as float only where its layer has a quantized kernel The warning listed every quantized tensor read as floats, so a file with a quantized embedding table or Conv2d weight warned on every load, about tensors no quantized kernel could ever run. Linear, Conv1d and ConvTranspose1d now note a weight their quantized kernel cannot run on the state dict, and loadStateDict warns about those only, with every name. The onFloatRead callback on QuantTensor is gone. Claude-Session: https://claude.ai/code/session_019uvkwmTDn1DksCJwKRLHTK --- packages/runntime/src/core/nn/layers.ts | 16 ++-- .../runntime/src/core/nn/loadStateDict.ts | 26 ++--- .../runntime/src/core/weights/safetensors.ts | 10 +- .../runntime/src/core/weights/transforms.ts | 9 +- .../runntime/test/core/nn/layers.gpu.test.ts | 23 ++++- packages/runntime/test/core/nn/layers.test.ts | 9 +- .../test/core/nn/loadStateDict.test.ts | 95 ++++++++++--------- 7 files changed, 103 insertions(+), 85 deletions(-) diff --git a/packages/runntime/src/core/nn/layers.ts b/packages/runntime/src/core/nn/layers.ts index 0e45408c..a5a11bdd 100644 --- a/packages/runntime/src/core/nn/layers.ts +++ b/packages/runntime/src/core/nn/layers.ts @@ -38,7 +38,7 @@ import { type LazyTensor, type QuantTensor, } from '../weights/safetensors.ts'; -import { readFloats, transposeWeight } from '../weights/transforms.ts'; +import { dequantize, readFloats, transposeWeight } from '../weights/transforms.ts'; import { tensor } from '../gpu/buffers.ts'; import { CpuParameter, Module, Parameter, type AnyModule } from './module.ts'; @@ -141,10 +141,12 @@ function loadQuantWeight( } /** A quantized weight its layer's quantized kernel cannot run, as its - * dequantized f32 values in torch's `shape`. Reading it puts the tensor in + * dequantized f32 values in torch's `shape`, and noted on `sd` for * loadStateDict's warning. */ -const asFloat = (t: QuantTensor, shape: readonly number[]): LazyTensor => - derivedTensor([...shape], () => t.f32(), t.byteLength); +function asFloat(sd: LazyStateDict, t: QuantTensor, shape: readonly number[]): LazyTensor { + (sd.quantizedAsFloat ??= []).push(t.name); + return dequantize(t, shape); +} /** `t`'s scales in f32, as stored. */ async function scalesF32(t: QuantTensor): Promise { @@ -222,7 +224,7 @@ export class Linear extends Module { matmulQuantTiledEligible(this.inFeatures, this.outFeatures, groupSize) && this.outFeatures % (32 / bits) === 0; if (fits) this.loadQuantized(sd, prefix, tensor); - else sd.tensors.set(name, transposeWeight(asFloat(tensor, tensor.shape))); + else sd.tensors.set(name, transposeWeight(asFloat(sd, tensor, tensor.shape))); } /** Loads the weight as quantW words [in, out] with its scales * [in / groupSize, out]. */ @@ -351,7 +353,7 @@ export class Conv1d extends Module { this.loadQuantized(sd, prefix, name, tensor, n); return; } - tensor = asFloat(tensor, [cOut, cIn, kernel]); + tensor = asFloat(sd, tensor, [cOut, cIn, kernel]); } // Without this a weight with equal in/out channels loads scrambled. if ( @@ -529,7 +531,7 @@ export class ConvTranspose1d extends Module { if (cOut % 4 === 0 && n % (32 / w.bits) === 0 && quantConvFits(cIn, taps, n)) { this.loadQuantized(sd, prefix, w); } else { - w = asFloat(w, [cIn, cOut, k]); + w = asFloat(sd, w, [cIn, cOut, k]); } } if (w && w.kind !== 'quant') { diff --git a/packages/runntime/src/core/nn/loadStateDict.ts b/packages/runntime/src/core/nn/loadStateDict.ts index 0b8415c1..f514af18 100644 --- a/packages/runntime/src/core/nn/loadStateDict.ts +++ b/packages/runntime/src/core/nn/loadStateDict.ts @@ -10,7 +10,6 @@ import { materialized, type Value, type ValueMeta } from '../graph/value.ts'; import { bf16ToF16Bits, f32ArrayToF16Bits } from '../weights/convert.ts'; import { derivedTensor, - quantTensor, type DerivedTensor, type FileTensor, type LazyStateDict, @@ -117,14 +116,6 @@ async function bindStateDict( upload: UploadFn, opts: LoadStateDictOpts, ): Promise { - // Whatever reads a quantized tensor as floats gets it dequantized. A weight - // its layer runs quantized is read as ints and never lands here. - const loadedAsFloat = new Set(); - for (const [name, t] of sd.tensors) { - if (t.kind !== 'quant') continue; - sd.tensors.set(name, quantTensor({ ...t, onFloatRead: () => loadedAsFloat.add(name) })); - } - model.transformStateDict?.(sd, ''); const visit = (parent: AnyModule, parentPrefix: string): void => { for (const [name, child] of parent.namedChildren()) { @@ -134,6 +125,13 @@ async function bindStateDict( } }; visit(model, ''); + if (sd.quantizedAsFloat?.length) { + console.warn( + 'loadStateDict: quantized weights loaded as float, the quantized kernel of their layer ' + + `cannot run their shape or group size: ${sd.quantizedAsFloat.join(', ')}`, + ); + sd.quantizedAsFloat = undefined; + } const params = [...model.namedParameters()]; const paramNames = new Set(params.map(([n]) => n)); @@ -159,8 +157,8 @@ async function bindStateDict( for (const [name, p] of params) { let t = sd.tensors.get(name)!; // Linear's and the convs' hooks turn a quantized weight into quantW - // words, so a QuantTensor still here is bound to something that reads - // floats: an embedding table, a norm, a plain parameter. + // words, so a QuantTensor still here is bound to something no quantized + // kernel reads: an embedding table, a norm, a plain parameter. if (t.kind === 'quant') { if (p.shape.dtype === 'quantW') { throw new RunntimeError( @@ -197,12 +195,6 @@ async function bindStateDict( done += t.byteLength; opts.onProgress?.(name, done, total); } - if (loadedAsFloat.size > 0) { - console.warn( - 'loadStateDict: quantized tensors loaded as float, no quantized kernel runs them: ' + - list([...loadedAsFloat]), - ); - } for (const [, b] of model.namedBuffers()) { // A Buffer's data is computed CPU-side and always f32, and half() only // retyped the slot, so the narrowing happens here. diff --git a/packages/runntime/src/core/weights/safetensors.ts b/packages/runntime/src/core/weights/safetensors.ts index 3a38d12c..618a17a2 100644 --- a/packages/runntime/src/core/weights/safetensors.ts +++ b/packages/runntime/src/core/weights/safetensors.ts @@ -332,9 +332,6 @@ export interface QuantTensor extends LazyTensorBase { /** Row-major, shape `[...lead, last / groupSize]`. F16 halfwords as stored * in the file; f32 once a helper changed them. */ scales(): Promise; - /** Runs when f32() is called. loadStateDict sets it to learn which tensors - * loaded as float; a helper passes its sources' on to its outputs. */ - onFloatRead?: () => void; } export type LazyTensor = FileTensor | DerivedTensor | QuantTensor; @@ -342,6 +339,10 @@ export type LazyTensor = FileTensor | DerivedTensor | QuantTensor; export interface LazyStateDict { metadata: Record; tensors: Map; + /** Names of the quantized weights a Linear, Conv1d or ConvTranspose1d + * loaded as float because its quantized kernel cannot run them. The + * layers' hooks fill it and loadStateDict warns about them. */ + quantizedAsFloat?: string[]; totalBytes(): number; } @@ -432,7 +433,7 @@ export function dequantizeInts( export function quantTensor( parts: Pick< QuantTensor, - 'name' | 'shape' | 'bits' | 'groupSize' | 'byteLength' | 'ints' | 'scales' | 'onFloatRead' + 'name' | 'shape' | 'bits' | 'groupSize' | 'byteLength' | 'ints' | 'scales' >, ): QuantTensor { const refuse = () => @@ -443,7 +444,6 @@ export function quantTensor( ), ); const f32 = async () => { - parts.onFloatRead?.(); const [ints, scales] = await Promise.all([parts.ints(), parts.scales()]); return dequantizeInts(ints, scales, parts.groupSize); }; diff --git a/packages/runntime/src/core/weights/transforms.ts b/packages/runntime/src/core/weights/transforms.ts index c94256cc..e4a26eaf 100644 --- a/packages/runntime/src/core/weights/transforms.ts +++ b/packages/runntime/src/core/weights/transforms.ts @@ -159,7 +159,6 @@ export function concatRows(parts: readonly LazyTensor[]): LazyTensor { bits: quant[0]!.bits, groupSize: quant[0]!.groupSize, byteLength, - onFloatRead: () => quant.forEach((p) => p.onFloatRead?.()), async ints() { const out = new Int8Array(numel(shape)); let base = 0; @@ -201,7 +200,6 @@ export function splitRows(t: LazyTensor, sizes: readonly number[]): LazyTensor[] bits: t.bits, groupSize: t.groupSize, byteLength: share(rows), - onFloatRead: t.onFloatRead, ints: async () => (await ints()).slice(first * rowElems, (first + rows) * rowElems), scales: async () => (await scales()).slice(first * scaleRowElems, (first + rows) * scaleRowElems), @@ -251,7 +249,6 @@ export function splitCols(t: LazyTensor, sizes: readonly number[]): LazyTensor[] bits: t.bits, groupSize, byteLength: share(cols), - onFloatRead: t.onFloatRead, ints: async () => copyCols(await ints(), new Int8Array(rows * cols), width, from, cols), async scales() { const data = await scales(); @@ -395,10 +392,8 @@ export function reshapeWeight(t: LazyTensor, shape: readonly number[]): LazyTens return { ...t, shape: [...shape] }; } -/** The values of `t` as f32 in `shape`, each int times the scale of its group, - * for a hook that means to load `t` as floats: loadStateDict's warning about - * quantized tensors loaded as float leaves it out. Throws CHECKPOINT_MISMATCH - * when the element counts differ. */ +/** The values of `t` as f32 in `shape`, each int times the scale of its group. + * Throws CHECKPOINT_MISMATCH when the element counts differ. */ export function dequantize(t: QuantTensor, shape: readonly number[] = t.shape): LazyTensor { if (numel(shape) !== numel(t.shape)) { throw mismatch('dequantize', `cannot give [${t.shape.join(', ')}] shape [${shape.join(', ')}]`); diff --git a/packages/runntime/test/core/nn/layers.gpu.test.ts b/packages/runntime/test/core/nn/layers.gpu.test.ts index f1297fee..78a940f7 100644 --- a/packages/runntime/test/core/nn/layers.gpu.test.ts +++ b/packages/runntime/test/core/nn/layers.gpu.test.ts @@ -42,7 +42,7 @@ function loadWarnings(): () => unknown[] { .filter((message) => typeof message === 'string' && message.startsWith('loadStateDict:')); } -const LOADED_AS_FLOAT = /^loadStateDict: quantized tensors loaded as float.*: weight$/; +const LOADED_AS_FLOAT = /^loadStateDict: quantized weights loaded as float.*: weight$/; /** A Linear loaded from a quantized file matches a CPU loop of * act(x·W + bias) + addend in f64, with W dequantized from the same ints as @@ -97,6 +97,22 @@ async function checkQuantizedLinear(c: Case, bits: 4 | 8, dtype: 'f32' | 'f16') // output to half precision, 2^-11 relative. const tol = dtype === 'f16' ? { atol: 1e-5 * inF, rtol: 1e-3 } : { atol: 1e-6 * inF, rtol: 1e-5 }; expectClose(got, want, tol); + if (!float) return; + + // The fallback is the float layer: a Linear loaded from the dequantized + // weight takes the same route and gives the same bits. + const reference = new Linear(inF, outF, { bias: c.bias, activation: c.activation }); + if (dtype === 'f16') reference.half(root); + onTestFinished(() => reference.dispose()); + await reference.loadStateDict( + memoryStateDict({ + weight: { data: dequant, shape: [outF, inF] }, + ...(c.bias ? { bias } : {}), + }), + { root }, + ); + expect(reference.weight.shape.dtype).toBe(dtype); + expectClose(await toArray(reference.forward(x, call), ex), got, { atol: 0, rtol: 0 }); } describe('Linear with a quantized weight', () => { @@ -243,7 +259,8 @@ describe('Linear with a quantized weight at the limits of the quantized matmul', describe('Embedding with a quantized table', () => { // No kernel reads a quantized lookup table, so it loads as its dequantized - // floats: on the GPU in the model's dtype, on the CPU as f32. + // floats, without a warning: on the GPU in the model's dtype, on the CPU as + // f32. const [vocab, dim, groupSize] = [10, 6, 3]; const ids = [9, 0, 4, 4]; it.for( @@ -271,7 +288,7 @@ describe('Embedding with a quantized table', () => { await table.loadStateDict(await fromSafetensors(bufferSource(file), { chunkBytes: 0 }), { root, }); - expect(warned()).toEqual([expect.stringMatching(LOADED_AS_FLOAT)]); + expect(warned()).toEqual([]); const got = await toArray(table.forward(ids), ex); const dequant = Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / groupSize)]!); diff --git a/packages/runntime/test/core/nn/layers.test.ts b/packages/runntime/test/core/nn/layers.test.ts index fb7db3bd..025d9f98 100644 --- a/packages/runntime/test/core/nn/layers.test.ts +++ b/packages/runntime/test/core/nn/layers.test.ts @@ -383,8 +383,12 @@ describe('Linear with a quantized weight', () => { }, }); layer.transformStateDict(sd, ''); - expect(layer.weight.shape.dtype).toBe('quantW'); - expect(layer.weightScale).toBeDefined(); + expect(shapes(layer)).toEqual({ + weight: { elems: (inF * outF * bits) / 32, dtype: 'quantW', dims: [inF, outF] }, + bias: meta([outF]), + weightScale: meta([inF / groupSize, outF]), + }); + expect(sd.quantizedAsFloat).toBeUndefined(); }); it.each([ @@ -411,6 +415,7 @@ describe('Linear with a quantized weight', () => { layer.transformStateDict(sd, ''); expect(shapes(layer)).toEqual({ weight: meta([inF, outF]), bias: meta([outF]) }); expect(layer.weightScale).toBeUndefined(); + expect(sd.quantizedAsFloat).toEqual(['weight']); // stored [in, out]: row i, column o is torch value o·in + i const want = Array.from({ length: inF * outF }, (_, j) => { const at = (j % outF) * inF + Math.floor(j / outF); diff --git a/packages/runntime/test/core/nn/loadStateDict.test.ts b/packages/runntime/test/core/nn/loadStateDict.test.ts index 86e3d2e5..5ac03e13 100644 --- a/packages/runntime/test/core/nn/loadStateDict.test.ts +++ b/packages/runntime/test/core/nn/loadStateDict.test.ts @@ -1,12 +1,13 @@ import type { TgpuRoot } from 'typegpu'; import { describe, expect, it, vi } from 'vitest'; import { pending, type ValueMeta } from '../../../src/core/graph/value.ts'; -import { Conv1d, Linear } from '../../../src/core/nn/layers.ts'; +import { Conv1d, ConvTranspose1d, Linear } from '../../../src/core/nn/layers.ts'; import { gpuUpload, loadLazyStateDict } from '../../../src/core/nn/loadStateDict.ts'; import { Buffer, CpuParameter, Module, + ModuleList, Parameter, type AnyModule, } from '../../../src/core/nn/module.ts'; @@ -21,7 +22,7 @@ import { type MemoryStateDict, } from '../../../src/core/weights/safetensors.ts'; import { f32ToF16Bits } from '../../../src/core/weights/convert.ts'; -import { concatRows, dequantize } from '../../../src/core/weights/transforms.ts'; +import { concatRows } from '../../../src/core/weights/transforms.ts'; import { meta } from '../graph/values.ts'; import { safetensorsFile } from '../weights/checkpoint.ts'; @@ -270,6 +271,8 @@ describe('a quantized tensor', () => { const upload = recorder(); await load(new Table(), { weight: quant }, upload); expect(upload.mock.calls).toEqual([[want, meta([2, 2], dtype)]]); + // No quantized kernel reads a plain parameter, so there is nothing to say. + expect(warn).not.toHaveBeenCalled(); warn.mockRestore(); }); @@ -283,6 +286,7 @@ describe('a quantized tensor', () => { await load(table, { weight: quant }, upload); expect(table.weight.data).toEqual(new Float32Array(DEQUANTIZED)); expect(upload).not.toHaveBeenCalled(); + expect(warn).not.toHaveBeenCalled(); warn.mockRestore(); }); @@ -294,16 +298,17 @@ describe('a quantized tensor', () => { const upload = recorder(); await expect(load(new Packed(), { weight: quant }, upload)).rejects.toMatchObject({ code: 'CHECKPOINT_MISMATCH', - message: expect.stringMatching(/'weight' is quantized \(int8\), but this quantW parameter/), + message: + "loadStateDict: 'weight' is quantized (int8), but this quantW parameter takes packed " + + 'words, which only its layer makes', }); expect(upload).not.toHaveBeenCalled(); expect(ints).not.toHaveBeenCalled(); }); }); -describe('the warning about quantized tensors loaded as float', () => { - // int8 in groups of 4. fc is [out 4, in 8], which the quantized matmul - // takes. odd is [out 4, in 12]: 12 % 8 != 0, so it loads as float. +describe('the warning about quantized weights loaded as float', () => { + // int8 in groups of 4, all zeros. const named = (prefix: string, rows: number, cols: number) => ({ [`${prefix}.weight`]: { data: new Int8Array(rows * cols), shape: [rows, cols] }, [`${prefix}.weight_scale`]: { @@ -314,53 +319,74 @@ describe('the warning about quantized tensors loaded as float', () => { const setting = { 'runntime.quant': JSON.stringify({ version: 1, scheme: 'symmetric' }) }; const open = (tensors: Parameters[0]) => fromSafetensors(bufferSource(safetensorsFile(tensors, setting)), { chunkBytes: 0 }); + const WARNING = + 'loadStateDict: quantized weights loaded as float, the quantized kernel of their layer ' + + 'cannot run their shape or group size: '; + class Table extends Module { + weight = new Parameter(meta([4, 4])); + } class Net extends Module { + // quantized: [out 4, in 8], in 8 % 8 == 0 fc = new Linear(8, 4, { bias: false }); + // float, warned: [out 4, in 12], in 12 % 8 != 0 odd = new Linear(12, 4, { bias: false }); + // float, silent: no quantized kernel reads a plain parameter table = new Table(); + // float, warned: Conv1d [out 4, in 3, kernel 4] as [4, 12], in 3 % 4 != 0 conv = new Conv1d(3, 4, 4, { bias: false }); + // float, warned: ConvTranspose1d [in 4, out 2, kernel 2] as [4, 4], out 2 % 4 != 0 + up = new ConvTranspose1d(4, 2, 2, { stride: 2, bias: false }); } - class Table extends Module { - weight = new Parameter(meta([4, 4])); + class Holder extends Module { + table = new Table(); } class Fits extends Module { fc = new Linear(8, 4, { bias: false }); } - it('names every tensor that fell back, once, and not the one that loaded quantized', async () => { + it('names every weight whose layer has a quantized kernel that cannot run it', async () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); const sd = await open({ ...named('fc', 4, 8), ...named('odd', 4, 12), ...named('table', 4, 4), - // Conv1d [out 4, in 3, kernel 4] as [4, 12]: in 3 % 4 != 0 ...named('conv', 4, 12), + ...named('up', 4, 4), }); const upload = recorder(); await loadLazyStateDict(new Net(), sd, { upload }); - expect(warn.mock.calls).toEqual([ - [ - 'loadStateDict: quantized tensors loaded as float, no quantized kernel runs them: ' + - 'odd.weight, table.weight, conv.weight', - ], - ]); + expect(warn.mock.calls).toEqual([[`${WARNING}odd.weight, conv.weight, up.weight`]]); expect(upload.mock.calls.map(([, shape]) => shape.dtype)).toEqual([ 'quantW', 'f32', 'f32', 'f32', 'f32', + 'f32', ]); warn.mockRestore(); }); - it('names the file tensors behind a weight a hook renamed, stacked and read as floats', async () => { - // q and k stack into one [8, 12] weight: in 12 % 8 != 0, so it loads as - // float. The hook reads pos as floats itself. + it('lists every name, however many', async () => { + class Many extends Module { + layers = new ModuleList(Array.from({ length: 12 }, () => new Linear(12, 4, { bias: false }))); + } + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const tensors = Object.assign( + {}, + ...Array.from({ length: 12 }, (_, i) => named(`layers.${i}`, 4, 12)), + ) as Parameters[0]; + await loadLazyStateDict(new Many(), await open(tensors), { upload: recorder() }); + const names = Array.from({ length: 12 }, (_, i) => `layers.${i}.weight`); + expect(warn.mock.calls).toEqual([[WARNING + names.join(', ')]]); + warn.mockRestore(); + }); + + it('names the sources of a weight a hook stacked', async () => { + // q and k stack into one [8, 12] weight: in 12 % 8 != 0. class Fused extends Module { qk = new Linear(12, 8, { bias: false }); - pos = new Parameter(meta([4, 4])); override transformStateDict(sd: LazyStateDict): void { const take = (name: string) => { const t = sd.tensors.get(name)!; @@ -368,44 +394,25 @@ describe('the warning about quantized tensors loaded as float', () => { return t; }; sd.tensors.set('qk.weight', concatRows([take('q.weight'), take('k.weight')])); - const table = take('table.weight'); - sd.tensors.set( - 'pos', - derivedTensor([4, 4], () => table.f32()), - ); } } const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); - const sd = await open({ ...named('q', 4, 12), ...named('k', 4, 12), ...named('table', 4, 4) }); + const sd = await open({ ...named('q', 4, 12), ...named('k', 4, 12) }); await loadLazyStateDict(new Fused(), sd, { upload: recorder() }); - expect(warn.mock.calls).toEqual([ - [ - 'loadStateDict: quantized tensors loaded as float, no quantized kernel runs them: ' + - 'q.weight, k.weight, table.weight', - ], - ]); + expect(warn.mock.calls).toEqual([[`${WARNING}concatRows(q.weight, k.weight)`]]); warn.mockRestore(); }); - it('leaves out a tensor a hook dequantized on purpose', async () => { - class Folded extends Module { - table = new Parameter(meta([4, 4])); - override transformStateDict(sd: LazyStateDict): void { - const t = sd.tensors.get('table.weight')!; - sd.tensors.delete('table.weight'); - if (t.kind !== 'quant') throw new Error('expected a quantized tensor'); - sd.tensors.set('table', dequantize(t)); - } - } + it('stays silent about a tensor no quantized kernel reads', async () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); - await loadLazyStateDict(new Folded(), await open(named('table', 4, 4)), { + await loadLazyStateDict(new Holder(), await open(named('table', 4, 4)), { upload: recorder(), }); expect(warn).not.toHaveBeenCalled(); warn.mockRestore(); }); - it('stays silent when every quantized tensor loads quantized', async () => { + it('stays silent when every quantized weight loads quantized', async () => { const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); await loadLazyStateDict(new Fits(), await open(named('fc', 4, 8)), { upload: recorder() }); expect(warn).not.toHaveBeenCalled(); From f4114d349626f9b531c2852eaa8cc9670e8f2ed8 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 17 Nov 2026 10:58:09 +0100 Subject: [PATCH 5/7] fix(zoo): dequantize a tensor stored as rows before a hook reshapes, transposes or splits it A file stores a quantized tensor as rows [dim 0, everything else], so a hook that works past dim 0 cut through its scale groups: EmbeddingGemma 2's audio layer1 conv threw at load, its depthwise conv and vision position table and Higgs's weight norm refused the shape. inTorchShape gives such a hook the dequantized floats in torch's shape. The privacy-filter experts cannot be split from rows at all and would mix the experts' values, so the hook throws CHECKPOINT_MISMATCH on a quantized one. Claude-Session: https://claude.ai/code/session_019uvkwmTDn1DksCJwKRLHTK --- packages/runntime/src/core/index.ts | 1 + .../runntime/src/core/weights/transforms.ts | 13 +++++- .../src/zoo/embeddinggemma2/stateDictHooks.ts | 14 ++++--- packages/runntime/src/zoo/higgs/encoder.ts | 14 ++++++- .../src/zoo/privacy-filter/stateDictHooks.ts | 24 +++++++++-- .../runntime/test/core/weights/checkpoint.ts | 17 ++++++++ .../embeddinggemma2/stateDictHooks.test.ts | 42 ++++++++++++++++++- .../runntime/test/zoo/higgs/encoder.test.ts | 26 ++++++++++++ .../zoo/privacy-filter/stateDictHooks.test.ts | 26 +++++++++++- 9 files changed, 163 insertions(+), 14 deletions(-) diff --git a/packages/runntime/src/core/index.ts b/packages/runntime/src/core/index.ts index 98cd1af9..25bb3e14 100644 --- a/packages/runntime/src/core/index.ts +++ b/packages/runntime/src/core/index.ts @@ -151,6 +151,7 @@ export { foldBnIntoConv } from './weights/foldBn.ts'; export { concatRows, dequantize, + inTorchShape, permuteRows, reshapeWeight, scaleWeight, diff --git a/packages/runntime/src/core/weights/transforms.ts b/packages/runntime/src/core/weights/transforms.ts index e4a26eaf..060888d9 100644 --- a/packages/runntime/src/core/weights/transforms.ts +++ b/packages/runntime/src/core/weights/transforms.ts @@ -11,7 +11,8 @@ * A QuantTensor source gives a QuantTensor, except in `dequantize`: the * helpers move its ints and scales together. Whatever would split a scale * group (a column split off the group grid, a reshape of the last dim, a - * transpose) throws CHECKPOINT_MISMATCH. */ + * transpose) throws CHECKPOINT_MISMATCH; a hook that needs one calls + * `inTorchShape` first. */ import { RunntimeError } from '../error.ts'; import { f16ToF32 } from './convert.ts'; import { @@ -407,3 +408,13 @@ export function dequantize(t: QuantTensor, shape: readonly number[] = t.shape): t.byteLength, ); } + +/** `t` in torch's `shape`, for a hook that reshapes, transposes or splits it + * past dim 0. A file stores a quantized tensor as rows, [shape[0], everything + * else], with its scale groups along the row, so those steps would cut + * through the groups: it comes back as its dequantized floats in `shape`. + * Any other tensor comes back as it is. Throws CHECKPOINT_MISMATCH when a + * quantized tensor's element count is not `shape`'s. */ +export function inTorchShape(t: LazyTensor, shape: readonly number[]): LazyTensor { + return t.kind === 'quant' ? dequantize(t, shape) : t; +} diff --git a/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts b/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts index 8e1d069d..0b7ed045 100644 --- a/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts +++ b/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts @@ -13,6 +13,7 @@ import { concatRows, derivedTensor, f32ArrayToF16Bits, + inTorchShape, permuteRows, reshapeWeight, scaleWeight, @@ -175,7 +176,8 @@ export function transformEmbeddingGemma2VisionStateDict( put('input_proj.weight', take('vision_tower.patch_embedder.input_proj.weight')); // [2, positions, hidden]: the x table, then the y table. - const [x, y] = splitRows(take('vision_tower.patch_embedder.position_embedding_table'), [1, 1]); + const table = take('vision_tower.patch_embedder.position_embedding_table'); + const [x, y] = splitRows(inTorchShape(table, [2, positions, hidden]), [1, 1]); put('pos_x.weight', reshapeWeight(x!, [positions, hidden])); put('pos_y.weight', reshapeWeight(y!, [positions, hidden])); @@ -232,7 +234,9 @@ export function transformEmbeddingGemma2AudioStateDict( const [o, k, c] = [Math.floor(j / (9 * c0)), Math.floor(j / c0) % 9, j % c0]; return (o * c0 + c) * 9 + k; }); - const conv1 = reshapeWeight(take(`${sub}layer1.conv.weight`), [c1 * c0 * 9]); + const conv1 = reshapeWeight(inTorchShape(take(`${sub}layer1.conv.weight`), [c1, c0, 3, 3]), [ + c1 * c0 * 9, + ]); put('conv1.weight', reshapeWeight(permuteRows(conv1, tapsMajor), [c1, 9 * c0])); put('norm1.weight', take(`${sub}layer1.norm.weight`)); put('input_proj.weight', take(`${sub}input_proj_linear.weight`)); @@ -316,10 +320,8 @@ export function transformEmbeddingGemma2AudioStateDict( put(`${L}.lconv.pre_layer_norm.scale`, take(`${C}.pre_layer_norm.weight`)); clipped(`${C}.linear_start`, `${L}.lconv.linear_start`); // [hidden, 1, K] → [K, hidden]. - put( - `${L}.lconv.conv`, - transposeWeight(reshapeWeight(take(`${C}.depthwise_conv1d.weight`), [hidden, K])), - ); + const depthwise = inTorchShape(take(`${C}.depthwise_conv1d.weight`), [hidden, 1, K]); + put(`${L}.lconv.conv`, transposeWeight(reshapeWeight(depthwise, [hidden, K]))); put(`${L}.lconv.conv_norm.scale`, take(`${C}.conv_norm.weight`)); clipped(`${C}.linear_end`, `${L}.lconv.linear_end`); for (const n of ['norm_pre_attn', 'norm_post_attn', 'norm_out']) { diff --git a/packages/runntime/src/zoo/higgs/encoder.ts b/packages/runntime/src/zoo/higgs/encoder.ts index 922c7498..dd30d1ab 100644 --- a/packages/runntime/src/zoo/higgs/encoder.ts +++ b/packages/runntime/src/zoo/higgs/encoder.ts @@ -18,6 +18,7 @@ import { elu, gatherRowsFrom, gelu, + inTorchShape, logitStats, matrix, mul, @@ -188,10 +189,19 @@ class PositionConv extends nn.Module { * conv checks the shape of its rows. */ private weight(sd: LazyStateDict, prefix: string): LazyTensor { if (sd.tensors.has(at(prefix, 'conv.weight'))) return take(sd, prefix, 'conv.weight'); - const g = take(sd, prefix, 'conv.parametrizations.weight.original0'); - const v = take(sd, prefix, 'conv.parametrizations.weight.original1'); const { hidden, kernel, groups } = this; const width = hidden / groups; + // The norm folds per tap, across the scale groups of a quantized tensor. + const g = inTorchShape(take(sd, prefix, 'conv.parametrizations.weight.original0'), [ + 1, + 1, + kernel, + ]); + const v = inTorchShape(take(sd, prefix, 'conv.parametrizations.weight.original1'), [ + hidden, + width, + kernel, + ]); if ( v.shape.join() !== [hidden, width, kernel].join() || g.shape.join() !== [1, 1, kernel].join() diff --git a/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts b/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts index fd899902..4ec5ce73 100644 --- a/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts +++ b/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts @@ -30,6 +30,21 @@ function move(sd: LazyStateDict, from: string, to: string): void { sd.tensors.set(to, take(sd, from)); } +/** An expert tensor [E, …]. Throws CHECKPOINT_MISMATCH on a quantized one: a + * file stores it as rows [E, everything else], so splitting or folding it + * would mix the experts' values. */ +function takeExperts(sd: LazyStateDict, name: string): LazyTensor { + const t = take(sd, name); + if (t.kind === 'quant') { + throw checkpointMismatch( + 'privacy-filter state dict', + `'${name}' is quantized, and a quantized expert tensor cannot be split into experts: ` + + 'keep it float in the file', + ); + } + return t; +} + /** Folds every leading dim into rows: [E, K, H] → [E·K, H]. */ function foldRows(t: LazyTensor): LazyTensor { const cols = t.shape[t.shape.length - 1]!; @@ -61,13 +76,16 @@ export function transformHfPrivacyFilterStateDict(sd: LazyStateDict): void { } move(sd, `${src}.self_attn.sinks`, `${dst}.attn.sinks`); - const [gluWeight, linWeight] = splitHalves(take(sd, `${src}.mlp.experts.gate_up_proj`)); + const [gluWeight, linWeight] = splitHalves(takeExperts(sd, `${src}.mlp.experts.gate_up_proj`)); sd.tensors.set(`${dst}.mlp.gluWeight`, gluWeight); sd.tensors.set(`${dst}.mlp.linWeight`, linWeight); - const [gluBias, linBias] = splitHalves(take(sd, `${src}.mlp.experts.gate_up_proj_bias`)); + const [gluBias, linBias] = splitHalves(takeExperts(sd, `${src}.mlp.experts.gate_up_proj_bias`)); sd.tensors.set(`${dst}.mlp.gluBias`, gluBias); sd.tensors.set(`${dst}.mlp.linBias`, linBias); - sd.tensors.set(`${dst}.mlp.outWeight`, foldRows(take(sd, `${src}.mlp.experts.down_proj`))); + sd.tensors.set( + `${dst}.mlp.outWeight`, + foldRows(takeExperts(sd, `${src}.mlp.experts.down_proj`)), + ); move(sd, `${src}.mlp.experts.down_proj_bias`, `${dst}.mlp.outBias`); n++; } diff --git a/packages/runntime/test/core/weights/checkpoint.ts b/packages/runntime/test/core/weights/checkpoint.ts index 17d5c95b..6b7d4a33 100644 --- a/packages/runntime/test/core/weights/checkpoint.ts +++ b/packages/runntime/test/core/weights/checkpoint.ts @@ -209,3 +209,20 @@ export async function quantContents( scales: [...(scales instanceof Uint16Array ? f16ToF32(scales) : scales)], }; } + +/** Replaces `name` in `sd` with the tensor `runntime quantize` stores for + * it: int8 rows [dim 0, everything else] of `values`, whole numbers, one + * scale of 1 per row, so it dequantizes to `values` exactly. */ +export async function storeAsQuantizedRows( + sd: LazyStateDict, + name: string, + values: number[], + rows: number, +): Promise { + const entry: QuantEntry = { + ints: values, + scales: new Array(rows).fill(1), + shape: [rows, values.length / rows], + }; + sd.tensors.set(name, await quantSource(8, values.length / rows, name, entry)); +} diff --git a/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts b/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts index 7a1f1f85..8654fd69 100644 --- a/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts +++ b/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts @@ -13,7 +13,12 @@ import { transformEmbeddingGemma2StateDict, transformEmbeddingGemma2VisionStateDict, } from '../../../src/zoo/embeddinggemma2/stateDictHooks.ts'; -import { checkpoint, contents, type Entry } from '../../core/weights/checkpoint.ts'; +import { + checkpoint, + contents, + storeAsQuantizedRows, + type Entry, +} from '../../core/weights/checkpoint.ts'; const DTYPES = [ { dtype: 'F32', type: 'f32', elemBytes: 4 }, @@ -277,6 +282,20 @@ describe('transformEmbeddingGemma2VisionStateDict', () => { expect(sd.totalBytes()).toBe(total); }); + + it('splits a position table quantized as rows [2, positions·hidden] into x and y', async () => { + const name = 'vision_tower.patch_embedder.position_embedding_table'; + const sd = await checkpoint('F32', CHECKPOINT); + await storeAsQuantizedRows(sd, name, CHECKPOINT[name]!.values, 2); + transformEmbeddingGemma2VisionStateDict(sd, '', cfg); + for (const [table, values] of [ + ['pos_x.weight', [1, 2, 3, 4]], + ['pos_y.weight', [5, 6, 7, 8]], + ] as const) { + expect(sd.tensors.get(table)!.shape).toEqual([2, 2]); + expect(await contents(sd.tensors.get(table)!)).toEqual({ type: 'f32', values }); + } + }); }); describe('transformEmbeddingGemma2AudioStateDict', () => { @@ -466,4 +485,25 @@ describe('transformEmbeddingGemma2AudioStateDict', () => { // 10 clipped linears' worth of bounds, 4 scalars each. expect(sd.totalBytes()).toBe(total - 40 * elemBytes); }); + + it('reorders the convs quantized as rows [out, everything else] by their values', async () => { + const conv1 = `${sub}layer1.conv.weight`; + const depthwise = `${T}.lconv1d.depthwise_conv1d.weight`; + const sd = await checkpoint('F32', CHECKPOINT); + await storeAsQuantizedRows(sd, conv1, CHECKPOINT[conv1]!.values, 1); + await storeAsQuantizedRows(sd, depthwise, CHECKPOINT[depthwise]!.values, 4); + transformEmbeddingGemma2AudioStateDict(sd, '', cfg); + // The same values as from the float checkpoint: tap k of channel c at + // k·2 + c, and tap 0 of every channel, then tap 1. + expect(sd.tensors.get('conv1.weight')!.shape).toEqual([1, 18]); + expect(await contents(sd.tensors.get('conv1.weight')!)).toEqual({ + type: 'f32', + values: [1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17, 9, 18], + }); + expect(sd.tensors.get('layers.0.lconv.conv')!.shape).toEqual([2, 4]); + expect(await contents(sd.tensors.get('layers.0.lconv.conv')!)).toEqual({ + type: 'f32', + values: [1, 3, 5, 7, 2, 4, 6, 8], + }); + }); }); diff --git a/packages/runntime/test/zoo/higgs/encoder.test.ts b/packages/runntime/test/zoo/higgs/encoder.test.ts index c6d6e0e5..dd8e9b8a 100644 --- a/packages/runntime/test/zoo/higgs/encoder.test.ts +++ b/packages/runntime/test/zoo/higgs/encoder.test.ts @@ -10,6 +10,7 @@ import { HIGGS_ENCODER } from '../../../src/zoo/higgs/config.ts'; import { HiggsEncoder } from '../../../src/zoo/higgs/encoder.ts'; import { resample } from '../../../src/zoo/higgs/resample.ts'; import { expectClose } from '../../gpu.ts'; +import { dequantizeWeight, quantizeWeight, quantSource } from '../../core/weights/checkpoint.ts'; import { CFG, checkpoint, QUANT_CFG, quantize, type Sd } from './encoderCheckpoint.ts'; const POS = 'semantic_model.encoder.pos_conv_embed'; @@ -64,6 +65,31 @@ describe('HiggsEncoder', () => { expect([...sd.tensors.keys()].filter((n) => n.startsWith(`${POS}.conv.`))).toEqual([]); }); + it('folds a position conv weight norm quantized as rows [out, in·kernel]', async () => { + const raw = checkpoint(); + const sd = memoryStateDict(raw); + const { hidden, posGroups, posKernel: k } = CFG.hubert; + const width = hidden / posGroups; + const gName = `${POS}.conv.parametrizations.weight.original0`; + const vName = `${POS}.conv.parametrizations.weight.original1`; + // g [1, 1, k] as one row of k, v [hidden, width, k] as rows of width·k. + const gQuant = quantizeWeight(raw[gName]!.data, [1, k], 8, k); + const vQuant = quantizeWeight(raw[vName]!.data, [hidden, width * k], 8, width); + sd.tensors.set(gName, await quantSource(8, k, gName, gQuant)); + sd.tensors.set(vName, await quantSource(8, width, vName, vQuant)); + new HiggsEncoder(CFG).semantic_model.encoder.pos_conv_embed.transformStateDict(sd, POS); + const g = dequantizeWeight(gQuant); + const v = dequantizeWeight(vQuant); + const norm = Array.from({ length: k }, (_, j) => + Math.sqrt(v.reduce((s, x, i) => (i % k === j ? s + x * x : s), 0)), + ); + const want = Array.from(v.subarray(width * width * k, 2 * width * width * k), (x, i) => { + return (x * g[i % k]!) / norm[i % k]!; + }); + expectClose(await read(sd, `${POS}.convs.1.weight`), want, { atol: 1e-6, rtol: 1e-6 }); + expect(sd.tensors.get(`${POS}.convs.1.weight`)!.shape).toEqual([width, width, k]); + }); + it("takes the checkpoint's folded position conv weight and splits it into groups", async () => { const raw = checkpoint(); const { hidden, posGroups, posKernel: k } = CFG.hubert; diff --git a/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts b/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts index f8c0f73d..1d24bcb2 100644 --- a/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts +++ b/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it } from 'vitest'; import { transformHfPrivacyFilterStateDict } from '../../../src/zoo/privacy-filter/stateDictHooks.ts'; -import { checkpoint, contents, type Entry } from '../../core/weights/checkpoint.ts'; +import { + checkpoint, + contents, + storeAsQuantizedRows, + type Entry, +} from '../../core/weights/checkpoint.ts'; const ZERO = { values: [0, 0], shape: [2] }; @@ -108,4 +113,23 @@ describe('transformHfPrivacyFilterStateDict', () => { // The dropped score.bias (2 elems) is the only byte loss. expect(sd.totalBytes()).toBe(total - 2 * elemBytes); }); + + // A file stores a quantized [E, K, 2H] as rows [E, K·2H], which the hook + // cannot split into glu and lin halves per expert. + it.for([ + 'model.layers.0.mlp.experts.gate_up_proj', + 'model.layers.0.mlp.experts.gate_up_proj_bias', + 'model.layers.0.mlp.experts.down_proj', + ])('refuses a quantized %s with CHECKPOINT_MISMATCH', async (name) => { + const sd = await checkpoint('F32', CHECKPOINT); + await storeAsQuantizedRows(sd, name, CHECKPOINT[name]!.values, 2); + expect(() => transformHfPrivacyFilterStateDict(sd)).toThrow( + expect.objectContaining({ + code: 'CHECKPOINT_MISMATCH', + message: + `privacy-filter state dict: '${name}' is quantized, and a quantized expert ` + + 'tensor cannot be split into experts: keep it float in the file', + }), + ); + }); }); From e851295a0542e6b896219c2aa602fb73f057b483 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 17 Nov 2026 10:59:09 +0100 Subject: [PATCH 6/7] feat(tools): keep embeddings and lm_head float by default again, as a quality choice The default exclude went away with the kernel rules. It is back as *embed*, *pos_emb* and lm_head.*: not because the runtime cannot load a quantized table, it can, but because quantized embedding tables only rarely hold up. --exclude replaces it. Claude-Session: https://claude.ai/code/session_019uvkwmTDn1DksCJwKRLHTK --- .../test/core/weights/safetensors.test.ts | 12 ++--- tools/python/README.md | 9 ++-- tools/python/src/runntime_tools/cli.py | 9 ++-- .../python/tests/golden/tiny.int4.safetensors | Bin 2472 -> 5920 bytes .../python/tests/golden/tiny.int8.safetensors | Bin 3432 -> 6368 bytes tools/python/tests/test_cli.py | 41 +++++++++++++----- 6 files changed, 46 insertions(+), 25 deletions(-) diff --git a/packages/runntime/test/core/weights/safetensors.test.ts b/packages/runntime/test/core/weights/safetensors.test.ts index 92372828..d4b45a46 100644 --- a/packages/runntime/test/core/weights/safetensors.test.ts +++ b/packages/runntime/test/core/weights/safetensors.test.ts @@ -637,11 +637,7 @@ describe('fromSafetensors', () => { // The tool stores a weight as rows of everything after its first dim: // conv.weight is torch [6, 16, 4], and no file setting names the bits // or the group size. - const quantized = { - 'layer.weight': [8, 64], - 'embed.weight': [16, 64], - 'conv.weight': [6, 64], - }; + const quantized = { 'layer.weight': [8, 64], 'conv.weight': [6, 64] }; for (const [name, shape] of Object.entries(quantized)) { const w = sd.tensors.get(name)!; if (w.kind !== 'quant') throw new Error('expected a quantized tensor'); @@ -656,9 +652,9 @@ describe('fromSafetensors', () => { expect(Math.abs(ints[i]! * scale - v)).toBeLessThanOrEqual(scale / 2 + 1e-6); }); } - // The tensors it leaves alone keep their bytes: a bias, a 1D norm and a - // row of 40 values that groups of 32 do not divide. - for (const name of ['layer.bias', 'norm.weight', 'odd.weight']) { + // The tensors it leaves alone keep their bytes: an embedding table, a + // bias, a 1D norm and a row of 40 values that groups of 32 do not divide. + for (const name of ['embed.weight', 'layer.bias', 'norm.weight', 'odd.weight']) { expect(await sd.tensors.get(name)!.f32()).toEqual(await source.tensors.get(name)!.f32()); } expect(sd.tensors.size).toBe(6); diff --git a/tools/python/README.md b/tools/python/README.md index c7a6cebc..b4411b4b 100644 --- a/tools/python/README.md +++ b/tools/python/README.md @@ -14,10 +14,11 @@ uv run runntime quantize model.safetensors model.int8.safetensors --bits 8 - The source is a `.safetensors` file or a Hugging Face model folder, sharded or not. The output is one `.safetensors` file. - Float tensors named `*.weight` with 2 or more dims are quantized, under - their own names. `--include GLOB` replaces `*.weight`, and `--exclude GLOB` - keeps matching tensors as stored. Embedding tables are quantized too: - int8 changes an embedding's cosine by under 0.0001 on all-MiniLM-L6-v2 and - EmbeddingGemma 2. + their own names. `--include GLOB` replaces `*.weight`. +- Tensors matching `*embed*`, `*pos_emb*` or `lm_head.*` stay as stored. This + is a quality default, not a format rule: quantized embedding tables only + rarely hold up. `--exclude GLOB` replaces it, so `--exclude none` quantizes + them too. - A quantized tensor is stored as rows, `[first dim, everything else]`: a Conv1d weight `[out, in, k]` becomes `[out, in·k]`. Each row is cut into groups of `--group-size` values (default 32, any positive integer), each diff --git a/tools/python/src/runntime_tools/cli.py b/tools/python/src/runntime_tools/cli.py index 3f70e4cd..1c8f8c7d 100644 --- a/tools/python/src/runntime_tools/cli.py +++ b/tools/python/src/runntime_tools/cli.py @@ -26,6 +26,10 @@ DEFAULT_INCLUDE = ("*.weight",) +DEFAULT_EXCLUDE = ("*embed*", "*pos_emb*", "lm_head.*") +"""A quality choice, not a format rule: quantized embedding tables only rarely +hold up, and lm_head often shares the embedding's weights.""" + F16_MAX = float(np.finfo(np.float16).max) """The largest finite f16. A tensor with a bigger value cannot be stored as F16.""" @@ -144,9 +148,8 @@ def parse_args(argv: Sequence[str] | None) -> argparse.Namespace: command.add_argument( "--exclude", action="append", - default=[], metavar="GLOB", - help="tensor names to keep as stored, repeatable", + help=f"tensor names to keep as stored, repeatable, replaces the default {' '.join(DEFAULT_EXCLUDE)}", ) command.add_argument( "--drop", @@ -165,7 +168,7 @@ def main(argv: Sequence[str] | None = None) -> int: bits=args.bits, group_size=args.group_size, include=args.include or DEFAULT_INCLUDE, - exclude=args.exclude, + exclude=args.exclude or DEFAULT_EXCLUDE, drop=args.drop, float16=args.f16, ) diff --git a/tools/python/tests/golden/tiny.int4.safetensors b/tools/python/tests/golden/tiny.int4.safetensors index f6f867c606cf87bed179f8ac01b17eb63a442647..af44a054a2e655c342c0030337595c1a677139e3 100644 GIT binary patch delta 4265 zcmWlbc`(-d7soAGBC;i+vXw2#k`}(7^B`RkQHd612^HN4m0LU#*%BdJlr32zOSU}U z&w13PxGJR$g|x}lZIPn&=l9oZUgyl5nKN_d%)3-Rf2m-8ji7{JTArZv0wc2}W~S~- zGl^G760EU$yYY10)Wj$)O3*@Xn#R*&YU&=oL?gg6%;#X5r;z-aV^V4V z2`Y;Jzp=#BY+0J5&|2|nE}IsUw5>ugEhffGmKzzR!H18S0`-~rQ=^%Ly)A*)!3W`&wkf(y72xRUh4`n9F5`Gxfj=hUL52UifJ33? zn7mmMl|B_vru9Bi2#SHs^TKrPNEMi@(?j*CI&$C64`Sl^%%lrJ<1hXwv3-<2{L7Q* zn;*x8##^{_SW=Vrhe$xq-R)%Wcs+6Q)W*Q(8T5RuE%0>^X6qK9<*in@Xmc4vXV}6u z-&hiHEuRMDUjuym4W`b$hK(+zaOPVBF)Yu)Z{ufLsI(4T=0&j%#}{a6f6T@O$~4qmEyaox-hdv4WavZ zBF=Z-1>pKh*~0zS|T{LQlzJ zVfP7~eLM!g?vE#ZMUBL6Z5eF)=FJn_9s~Qr>!4s|8fbaWK3qmg#(8OJNqgY2$W2mHq>0Ze<-sQN@XbCbIjwVQ-cf5L*-u13*!Je%yb z=w&O~GPziCY#!gsRT$py$fAW+Vi6SEcr5}3Sm!F=#EkzM%J8w>zBXcL`^7l`_q{Q9gD%D2~{xHbfC*C2kBM? z5zrU8zVaEu+s zBq>0Zu`Lbkl>r-vv6iax*{Gqq6F;q5!%_4*&rZq+@~gQLi%~e;0Fnd*P+7AIG8`kJ z?3W5%*Zz@i%!@=JBNP0b%)?UM9GH{)4`_7cqqk)yX73zEZ9Igl?yTh-o2B9=!4mlS zy_$-*Zv*XZ-{>j#=WzSCJ-KuC1|8j|$uUqbp|hNGk=yD>!etk-o5JJ3S`0+%bT%DU6B)EWr`kCZ>Z<~IJ|3CL$4(ATOK5>fj(PRj5R0pJ-lD7EJumZa<48?0|c-M$D$yJzHp`CLud`Cb|(0)qI~K{2HEU^j1Gnj%f!uZ~{rUW}Ar4HU@O zXxg4Us5q8{YEMtVX6fbR+iM4mI>Jq-bSy9<7`aG~>k^Ucy_v=R zJ?`i``HgYwvd7YpDVm!dh6f)kz=7OIjML(g@8R|!{BwdHV>Z(ys(=at;vnqZz_>_t z!Tyz}@LhH@dA#uj$xCda5nglPgnAPm$r0nszg22K&`R`P?(GUk;1t_4PPT)l$Ovg{$6?L(;RL z|N3+Id*u{;)x&3w=9yw`{GO@BpHLi`6A^F^NPFu+!pi2bvZB+fU$qb{l7 zU;hz@{E=6 z&3!p;ypm2QT;)hNM-oq75~pQqB_Mg@Fb?N_f$gV1zz^qsggr9UCiDskzGa0WjSMQ! z52o#_8EAhkgv$5cLfGMS@=NYM>w3HyzHU+j9eI|RCzjEwWh0Q#vjJr6ooLzf4m>_i ziM($rE-3Ej;-|6G^t);banmV=*t`_7-gWvRz0F0xvrW9?UcVSU-VRt<^Nv~Y!~sm@ z1#!Tpk?h?2fLxlqfWCYak}O(GwVTILYg;HJ*fdiMvu#*r*h)R4rN~m@R`40sr9Oin zN$|ZZ(3T+%4efEDJT*?ky={OT;qGTQoQ#L!kVu?&<`3-hDFmj|mezDM(scvpnUSbv zC>>r7DK9UeyO}cWDykuFBM;d|F9&RO+==UN_mYm+Cr~$V5!lbYfxG0TF~44ciC&wJ zhNFw2VIY$yb3PjX+jku;d~V^SxHBUppN<|^t1zNHfTnV<%W~T0ohBuY*6{fLRuU6r z3!6tPP&VKxy?Hl=*tMtf2D8`D{zH1$xh|SMFj|XFzm@ryUS;8}xN+j*coTdyN8sk7 zcBb>{ZXiK>;Pnb4RNB&uequi@ZYMv-?%PE)rBVeR{aufGwlkQ~rt_eu)J%pNM1ZTM zwg-6kp1~z)6=LU+2aEkruubQE|!r4`pVReolPX-%80&hE^Y~r$NkE6psJQZRJwYZ z+E`PTJ=H*X6KQD4=c3xT5_tJWnK)N&g|1mDn6R=MTy9m9KA$fnHf$DjiLb|xcmG3; zw0NAl+6~>uN}%#W4c!=;LnE(5Ls2ky30>@1g3dn+fbaB~G)SuRL$>8p+XeFE$Mlex z+bU!VHBdqR9V`1GgFIXjPuuoVYPTjIhBqI?p|M)(Bp}2W3YP$_nVX?-?<~j|+Xqg? z&xyg#E5xYm6S?#>j;gCS2a>xBoi*@s963clH}w5cG0s)#T&ip+YjIt89W4pA#3O>@uveS`=fHZ>C_k4!OY}GdZLFims?Xtt zKxKHXC-TvLo2)79~`!UxsyywJ^guOM4;N{9&NM=B*}+2qdV_3 z^bk{Wx2CL^_IH_pD{0LRRY z;Nhtq5RoL!Nm<~^s?5xR1KB^>yKNmLN~(pV9*HG?ozQ`#ymK^{TdK`SOU#7I=0T_~ zcE#<6Z&14LEA?H{#26cxkgBIVrotx^ds1U9nDKw z3S9RaCPbVFV)K4wTK?}jjMF)U>cSuK$Lyu3Jzfq{hoo_Bt{sdkdbQ}gnjl?Y0YQ>; zK)`$mE7r$ATWUPY8=BFhGxcHH$!wauDj5uitzh=!4tzQMkxu%jkohL^n0eTpUZv|n z*Qk(GSNfxHmod4@s&X+P@f=>hn1YY~O(l(vuX#>CYs|kjq@tLc1s!RP#zi{&@u#9L z{UBF^^IY^{uI@t4@aQP%3y;AEPZDUR(oKB7rIl_P6oj$oim+HF8qe!ylcgf*@VI0M z?EOs8%)%WFUhHHZ_?Yvbd=x`-Hh~T<{g>|GaXXP8bd>hoU5k51EkRG+9}|*O=$rUI z>5lcXp!`9DzuUKpY96uz6T@bD`QQy4T5*Z2_PR_m!V~DSkaT>J%V14$J{{BWK)+iP z`2GDFT>j=5TYX0m-?!zkLz^p@sTdy!k(A;*bukBt);jc-)Z=@FwzA4x(^O_nQ#yA2 zkw~<+tifw~cZhR&30)ES4-rux#FmI5nxv~l*LyYKp${KGEl>ipMr&Y`oE|2J`_p*| znP8NBj=U;cLs;E7Vm(lWr-N_O3%wzDXYo36$XXJ&95!WB?9a3FrxaL$P0}n!NE=h$ so0x!(?+J!UVb za0li0f|VH#ipEKc!#@w6%S>y*Ns{yfx3$N~@FEqOCkkLyq_~C(u)3hW z%0iz<5I)_Y4xLcEE7ZeZHY)7{a1bJ@= zi1VuX4G;8a`$lF+bD#&c5wKlezN_g;C*8&Y6a@4xv1nvp27DOgFg=aTTAWayVbsf!e?0CG$7kC7L6aKI?N@zvIURCE~r?2*N7bvEqXWLA-HV^uSsJsM?lCw8mM_p(~(G7{V%P zW1OPY!r_@sR+)@$*WZsV3&xjqeMYB``@<3H@{nnGrh#K)lzu&wQ(*`JWt#7iwY*o$ zL4d8wRROtWKn0Z{M~uqm=P+2yN-ttkguqO^M*On%Q|1|6a5X+eMU8H@2snzZc7|85 zFu4mSq$D9?2|>s2$M-m(^F!8Zn4c28ZK4)A(!l^5t<^T;NjKxAvzQXb%kyX(+qQm8Dav=W4fPt!hEylGZ$!(p0a6d4fOPF&4lbfDL5_= diff --git a/tools/python/tests/golden/tiny.int8.safetensors b/tools/python/tests/golden/tiny.int8.safetensors index 7b4ef3a4fecb591427d029eeb2f66e64ecbfb0d6..9921243195e6afc0251be36192bd12be2d04ff17 100644 GIT binary patch delta 4254 zcmW-jc{JAh6UQytOSVd=Y-LM?r1*a3LAoTO677~y(Tz~4cvNJ|60$|s6e3HuJm1ee z>QY>l(uP9X4=W8W70j;|loU)oA|#!vD5yNmWSOB! z>M22UiT_}+$ueWp)DA(A`wtfXR?N1CEmrsU2=zXY$`RB`wGx^o@t^v;WM-J^E<}E3 zW`>KGEi*Mty(q-U{RfMenVS7>3se1n)P4}k6y*A&0O76{qwGfw7mc z^lb^e4mtoowM@`ussKk47vP^ZI*j9O1^%d{I~6{Xhl3&Jn6yy}6+aeGrsY0S2#kV^ z^TKrXa3vV8(M7eXT5{jc7ouYM%%lrJqtAXQxpjm-JmNw0%#Y&&qfJ~oB&9+7f+ZpQ z?pCsAypA||XkkFpGUC6u(=GP(`@0IPc#X?mQVfjuK~XO3RCA^ z!+MuXaOP`0F(}K%uj6N$skAnTy)DK7hYq4R=PBtkFarHE8R+V*gMq;VRE~ER&lUB9 z@G$oWt^4y5$@7Y&%#{wv5G(~B4Rvr7oxyieabaeP8bH_a2+%f|kLAnX!jNz&zS*pU z0Xz3Y_&Zz1SWAGTFJ(!D=BlHhQwyyqi$~G;A}sM#1KXPlpmg;DJkhtr%O+3YsAeiP zJ=G4%>Cb4Ptp&0-x#SX8G!kkCG>O@(Svb3*9H)8LV)qpRc(^?b9$RSBj>uT7=I(^? z7e!bQZ;E2y=7W=v4Bz5@E7|*60VVDXFuoG!SotrG#IkNFL}^+fJ5&d@&sV^+pRQo7 zbQV1G>>zHI8+3PNGtv3=Xl*A+-~L>MzAE*|eYYW!gq)Ve!mg7z^LP|~*%wQCiyDaC z>QdP9)r%*%H465I)k1+;DrkDl;h(U{h08+~Bx&*qiISxRCvOnzxFT}wMgwi%J&sqT z2iTo04*0iK9GLW^QniUP<|cPDYBdU>-h>6OO_ie0cox}X(ZiOvW^l3O*j&D+t1!IZ zmPrdM#W3oAAhkO&3`*ba*+B1V#@CIa(9`=6KDZawpNWB^tE`z9rF(&nvgGrW3WPpU zrrR#@FrxY-T)+GUC#plJ+n>%f@K_WUO{jpmh67zzF+evfh=88ZCpx#I8vYrpC2f5l zX~i?{B!=v|h&J=$VcF_+Z0kfamD_Y5di9I3{M{@R{xU|5m1aZWGadXSJ`ZCG4GBU>8KBLg-Lqs^6N zGf`b-2Yy_!iZk2yJUb~P$gko`E<)ineMk@xKqZYzNOz2Y(x1w7P1_hgvl1}oNH$q9Cj`Vb-m(rxk4V8OK5g=dV7W+T*3*DVmcN ziU%Id$NroMjM3zgZ(;Tz{9}S1V>Z$Rs(=at;vno*&$!I!gneeG@m*FEdA$Aw$&GKM z;hwYLq*@~$&KBd$yH$>1hy7qYA_+B|Qb9?umFBKcC*Q)iFnupiLw`Xrm%6@Y_@=2= zIBS6gnI83=ev`8Si_ga)+<6YIy-ab#;%eX)#KBpEENc9(DCbUI4b@CIgLb(KXvpVS z82`Q>Y0DJQ6W%D|5)3aqq(Hr89vya!gWXezBY#yuoBs(`;+Z54v;(x3MT3*94bb<$ zz}$r!2OsT<;gpFaYUuPb`3qz~&+QEF{r+s?tJ_OX^mIdVQX zlBlo?kp0In(Q$o(KlZM~`Mg~4!A#;6t;Umj^_+0a(#_!O_GP&KN*bMTl_OmoDLi#i zoR+GVfYgmcIF$1lwkCdn@6LS)yJe_N$Q2TF%L;=V7*v`UMB7#}(DqsgmF~TT&_ikD zr`&zk^>`C}*`NyA@+>irFQt`+!w}cK4rJ_|XzBBIJU&i|yiW?wFYe>w$I(RkO(mJQ zX%|CuZZcWx`uich%|YL@jlAQYKN(%#HZZGx$IO4?04DN+*l*K7cI7m558s$1 zi563>rg7BV5(05HP1M443zizRP>;wNWQlMKc#r5%?}0HAbngnZri(*;TMQ^ojngnM z8z6_d``C4-Vxc%V0_UFj13SG7f$6ZN)$I**P5*gj_=F)!hm}F{%RJm}szf`Bs)^h1 zL$<-w0b3k*;M&_gr2X|t)CpJ!_H%CFPI+m}uTx+mSLdU_$Reok&)~_NkHmlXUPlY> zTR181%m~S+q5IWJ3~%$NDctL_oYuLCq{Pu09^cA1QJNZ@XGy?iMuZt6i_ zu^$$hBs z;N=@7;#{#AI;SgRoLLpP+^QnI-k(Wy=yd25UyEaR|3&rGSe&}r1zpEVpdzoDt`Etk z5mzFiD2Th5E^;hE=N|>Yclt!?rPTPrTk@&xe0lQy_mHUD%47=FQ9=G4EBhgxJX{`2 zTlY|Ew<;foHXgvi(HiO`AjB66lLSqPjZnB}I;4;81*hWYM1RK>vb6Lgx%f1OswroJ zhbdw*`|nejf9*6Y2fpX54W|7S--$ zphHqLiZ6PKUGh0l<17RR_fKGTyCOa-(8Gr3hIrJt3H=vX!llX%y1e>5In*NxnbK9j za~Q_RxPFM+T?oNf<)H1fBIoTZXA-z}J=BUbm~V6jxGBAQq)zw^eScJpb5%NrDjCRH zTo+zLOM)!%u%I~X5of?TppG=i&*4uOJr05EYw5A7bC?&P1g~eyK)}f;^s<=Gk$RYo z;@!{iWpgUFKCOdxc@w%koKnTCaJH&h5tZr;v38Lrrdlk6wdYU3>KW0fKN-v=<2UV~ z=U^O~ToORLfk^b{T)?BbUCiPj3#>S!21Im>_sY7PD3_e2hDqx9sHTZct1oK)SYv{X zdwa=aRe=42YNUKaA50Fa;awXMsIs|78>|9I(xHvGotFsR#Dp9^mkE9|3sJ1{J6+1% zg>L%d*zd?)in{%u*&(|G9u8=+U8d6LH40>twmZap^TFJYyH_5m$zYdsz9(yLonS*Z zMDT(e!;sTiZILJ7k47TXAZS-Nd)7*WnXzye&b;9O$IO?)!_(U!JVBb1Jl~a7mdJ+v zSwGmjt?lH*jAoK@IGP+esSOFa=V%W1k`^N^DFGEt15j1$idzldpmgsS>a)0!G150C zl}~w0xpxNm^oPR2+ji{H-5;pDb~WYhtEZ=heQEC6c8r-ZN(W96;BVY$xfP^}uWT6JkFrf_&Aoqq)gTfV=&MF%c(%*tAcHmi==M zW3&&Vn(!EYpSc9J#>-&FL1|o_V+Z52J)8AhjghV`hd`-WAYeX-u*3*F2{m)#hL8Q&7y!f)2Mt;zI3x_+z#X{UBG3b6xacj?Mzk(8vhs z4U57DPvU5X;!S+NsfBJB5QNd^vtf};B%asFB1=Tl;BmJky(rs&HLFt1!f0s`s)i`Jc z#s*FF@_`#Txcnkn>3Nx?hs9CD;52-Z!(erBJ{?teN8ei$`0f2FT=wP|TXjbe-?wJ7 zgB#14sVHvS7L(Ew$(+rOWpWX6=8K^h;@G@ zCI;Q4c|E~+XVDsR&{_&N9Wr5)?a#CGrW9C#4bm(}NDGtSofXpEGHI=^~-7PE%E$m{{zFG4G91M delta 1258 zcmVG3Xir=&=C`0g-$SlNkXU4KXw`GF^QvB4ur2Wn`1h0Vf|ZHY_q-EFxrK zbYWj_W@d9`baNs)TQM{p~58}zm4iiTIH%dK&K@t`e6cO#4 z`qeq{1k>gJu@L&ffg_`akX!Fhp z{w&&mAtU1l+-2hI((wQx=J>5}_#a7V8wfOO?M5ToLf6^#5unpnvDc zUj69~Hb1F(=Gf5F+dM=NAM-(GU~y8Nyq~ zC(tqQ2Ldnhf!_$_E%8|t$acy(A@@P1QvBa7KrVPeP2UC*{|XKp{q^PJ zCjA=L!QLe!3+DXx^{(XlF$FI3_<;{4ci#rlvC!#T0Luyw(@lZ$Ev6V(ACAvc6#pK7 zFwpD@-1k<%e|(Sk7tYV{G2VC+`Cus+>UcKum>IQ??h{oLHm?2%ItVu27#yt(>w!S% z$L9%U^&ng%N8mI3NWIKe_3Q-NO1k(H`qR)D$0V!a0=gvs#;d6VzqIpz4=W5Ty6F!8 z_TBQ=fx#J4Uj;8P+@}Jh?;y4UK@%H)Fk1*f{o@hf;^2k`z4}e$<47Ge?I_&bAmAx~ zrr109KFSjK2NA-`M-K)562{I#2rJ=55f~iUqRLW2?(sDEzCaj(@ChCY*!dFg=}yJ! zfg&^_IUE%p?@H>WHFU)K)#wAV@3P{-$fQ9Vn&8~qvncBs97&&DL?3?(EZIDN3FvX~ z0QK1$CK87<6+rHV&_y7IZ4nfTT{2ToL#{tV7`U1@^CozBhMfnyL4<_#dA7s#(FDv6;p6k~E!^TI@DD8<5%Vs84Bqyxzk$+W*f{{rVJd-3vhy(NQ8$j>6xQ%M7rhPC zsIC?M0Il=W*!jdAY!uV(q(+x6A;@bj&>GbTo_1+=a^!6Es@FefI5D@dp-SO6> z+x_VL1S@GHy@C4!$PL>Q@Dr`#_Mm^E+#`J>m{2f79 None: assert (weight.dtype, weight.shape) == (np.uint8, (8, 32)) scale = quantized.get_tensor("layer.weight_scale") assert (scale.dtype, scale.shape) == (np.float16, (8, 2)) - for name in ("layer.bias", "norm.weight", "odd.weight"): + for name in ("embed.weight", "layer.bias", "norm.weight", "odd.weight"): np.testing.assert_array_equal(quantized.get_tensor(name), source.get_tensor(name)) @@ -107,13 +107,34 @@ def test_skip_reason(name: str, stored: Tensor, bits: Bits, group_size: int, rea assert skip_reason(name, stored, options) == reason -def test_quantizes_embeddings_and_lm_head_by_default(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: - source = tmp_path / "model.safetensors" - weight = from_array("F32", np.ones((4, 32), dtype=np.float32)) - names = ("embed_tokens.weight", "pos_emb.weight", "lm_head.weight") - write_safetensors(source, dict.fromkeys(names, weight), {}) +EMBEDDING_NAMES = ("embed_tokens.weight", "decoder.pos_emb.weight", "lm_head.weight") + + +def embedding_checkpoint(folder: Path) -> Path: + """A checkpoint of EMBEDDING_NAMES and one Linear weight, each [4, 32] of + 127: the top int8 at scale 1, so it quantizes without error.""" + source = folder / "model.safetensors" + weight = from_array("F32", np.full((4, 32), 127, dtype=np.float32)) + write_safetensors(source, dict.fromkeys((*EMBEDDING_NAMES, "fc.weight"), weight), {}) + return source + + +def test_keeps_embeddings_and_lm_head_float_by_default(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + source = embedding_checkpoint(tmp_path) assert main(["quantize", str(source), str(tmp_path / "out.safetensors"), "--bits", "8"]) == 0 - assert [line.split()[0] for line in capsys.readouterr().out.splitlines()] == ["quantized"] * 3 + assert capsys.readouterr().out.splitlines() == [ + "skipped embed_tokens.weight [4, 32]: excluded by *embed*", + "skipped decoder.pos_emb.weight [4, 32]: excluded by *pos_emb*", + "skipped lm_head.weight [4, 32]: excluded by lm_head.*", + "quantized fc.weight [4, 32]: max error 0, mean 0", + ] + + +def test_exclude_replaces_the_default(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + source = embedding_checkpoint(tmp_path) + argv = ["quantize", str(source), str(tmp_path / "out.safetensors"), "--bits", "8", "--exclude", "fc.*"] + assert main(argv) == 0 + assert [line.split()[0] for line in capsys.readouterr().out.splitlines()] == [*["quantized"] * 3, "skipped"] @pytest.mark.parametrize("size", ["1", "3", "6", "640"]) @@ -143,7 +164,7 @@ def test_quantize_fails_when_no_tensor_is_selected(tmp_path: Path, capsys: pytes def test_quantize_fails_on_a_quantized_source(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: destination = tmp_path / "out.safetensors" - argv = ["quantize", str(GOLDEN / "tiny.int8.safetensors"), str(destination), "--bits", "4"] + argv = ["quantize", str(GOLDEN / "tiny.int8.safetensors"), str(destination), "--bits", "4", "--exclude", "none"] assert main(argv) == 1 assert "is already quantized" in capsys.readouterr().err assert not destination.exists() @@ -158,7 +179,7 @@ def test_f16_stores_the_float_tensors_that_stay_as_f16(tmp_path: Path) -> None: safe_open(GOLDEN / "tiny.safetensors", "numpy") as source, ): assert quantized.get_tensor("layer.weight").dtype == np.int8 - for name in ("layer.bias", "norm.weight", "odd.weight"): + for name in ("embed.weight", "layer.bias", "norm.weight", "odd.weight"): stored = quantized.get_tensor(name) assert stored.dtype == np.float16 np.testing.assert_array_equal(stored, source.get_tensor(name).astype(np.float16)) @@ -179,6 +200,6 @@ def test_drop_leaves_matching_tensors_out(tmp_path: Path) -> None: assert main(argv) == 0 with safe_open(destination, "numpy") as quantized: dropped = {"norm.weight"} - scales = {"conv.weight_scale", "embed.weight_scale", "layer.weight_scale"} + scales = {"conv.weight_scale", "layer.weight_scale"} with safe_open(GOLDEN / "tiny.safetensors", "numpy") as source: assert set(quantized.keys()) == set(source.keys()) - dropped | scales From 09f16236f7fca6fd37f5d671edc96225ab7bfc28 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 17 Nov 2026 11:00:58 +0100 Subject: [PATCH 7/7] docs(docs): re-measure the int8 and int4 numbers, warn off int4 for conv models, say what loading as float costs - Embedding tables and lm_head stay float by default, as a quality choice. - A quantized tensor no quantized kernel runs is turned back into floats: a smaller download only, and 4 bytes per value for a table kept on the CPU. - MiniLM against the float model, 20 queries over 20 documents: int8 keeps cosine 0.9997 or higher, int4 falls as low as 0.93 and reorders the top 3 results for 12 queries, int8 for 2. Claude-Session: https://claude.ai/code/session_019uvkwmTDn1DksCJwKRLHTK --- .../content/docs/core/exporting-weights.mdx | 32 ++++++++++++------- tools/python/README.md | 2 ++ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/apps/docs/src/content/docs/core/exporting-weights.mdx b/apps/docs/src/content/docs/core/exporting-weights.mdx index 9f35cc1d..2973d89d 100644 --- a/apps/docs/src/content/docs/core/exporting-weights.mdx +++ b/apps/docs/src/content/docs/core/exporting-weights.mdx @@ -130,8 +130,9 @@ is a normal float tensor. An int tensor in a file without `runntime.quant` fails to load. **What is in the file.** `runntime quantize` quantizes every float tensor -named `*.weight` that has 2 or more dims: Linear and conv weights and -embedding tables. Biases and norms stay float. It stores each quantized +named `*.weight` that has 2 or more dims: Linear and conv weights. Biases +and norms stay float, and so do embedding tables and `lm_head`, because +quantizing them usually costs too much quality. It stores each quantized tensor as rows, `[first dim, everything else]`, so a `Conv1d` weight `[out, in, k]` becomes `[out, in·k]`, and groups of `--group-size` values run along that row. The group size has to divide the row: `in` for a @@ -139,11 +140,15 @@ Linear, `in·k` for a conv. A tensor it doesn't divide stays float. The runtime loads each tensor back into its layer's shape. **Which layers run quantized.** `Linear`, `Conv1d` and `ConvTranspose1d` -run on the int weights. Everything else, like an embedding table, loads -as float: the download is still smaller, the memory and speed are those of -the float model. The same happens to a weight whose shape the quantized -kernels can't take, such as a Linear with a group size that isn't a -multiple of 4. A console warning lists the tensors that loaded as float. +run on the int weights. Any other quantized tensor, like a `Conv2d` weight +or an embedding table you chose to quantize, is turned back into floats at +load. It keeps the rounding of its ints, so the download is smaller but +the model is no faster and uses no less memory than the float one. A +table kept on the CPU takes 4 bytes per value. + +A `Linear` or conv weight whose shape the quantized kernels can't take +also loads as float, and a console warning names it. To stay on the +quantized kernels, keep `--group-size` a multiple of 4. **Making a file.** `runntime quantize` lives in the repository under `tools/python` and runs with [uv](https://docs.astral.sh/uv/): @@ -159,7 +164,7 @@ uv run --no-dev runntime quantize model.safetensors model.int8.safetensors --bit - `--group-size N`: how many values of a row share one scale. Default 32. - `--include GLOB`: which tensors to quantize. Default `*.weight`. - `--exclude GLOB`: tensors to keep float, like a layer that loses too - much. + much. Replaces the default `*embed*`, `*pos_emb*` and `lm_head.*`. - `--drop GLOB`: tensors to leave out of the file, like a pooler the model never reads. @@ -167,10 +172,13 @@ The tool prints one line per tensor with its error against the float weights, or why it was skipped. **Picking a setting.** On all-MiniLM-L6-v2, int8 keeps every embedding at -cosine similarity 0.9998 or higher against the float model; int4 drops to -0.94 and changes the top 3 search results for 8 of 20 queries. Use int8. -Try int4 only when the download size matters more, and check it on your -own data. +cosine similarity 0.9997 or higher against the float model. int4 drops as +low as 0.93 and reorders the top 3 search results for 12 of 20 queries, +where int8 reorders them for 2. Use int8. Try int4 only when the download +size matters more, and check it on your own data. + +Use int8 for models built on 2D convolutions. int4 breaks the outputs of +MobileNetV4, DepthART and YOLO26. A quantization-aware trained checkpoint quantizes the same way if it was trained on symmetric int8 or int4 with one scale per group of inputs. diff --git a/tools/python/README.md b/tools/python/README.md index b4411b4b..e640d156 100644 --- a/tools/python/README.md +++ b/tools/python/README.md @@ -35,6 +35,8 @@ uv run runntime quantize model.safetensors model.int8.safetensors --bits 8 model never reads. - It prints the max and mean absolute error of every quantized tensor and why each other tensor was skipped. Exclude the layers that lose too much. +- Use `--bits 8` for models built on 2D convolutions: int4 breaks the outputs + of MobileNetV4, DepthART and YOLO26. ## Development