diff --git a/packages/runntime/bench/kernels/inputs.ts b/packages/runntime/bench/kernels/inputs.ts index d8dca166..4aaa64f9 100644 --- a/packages/runntime/bench/kernels/inputs.ts +++ b/packages/runntime/bench/kernels/inputs.ts @@ -1,7 +1,7 @@ /** Input builders shared by the kernel A/B cases. Random data comes from * `Version.rand`, so A and B get the same numbers. */ import type { Version } from '../ab.ts'; -import type { EagerDtype, Value, ValueMeta } from '../../src/core/index.ts'; +import { packQuantW, type EagerDtype, type Value, type ValueMeta } from '../../src/core/index.ts'; type Float = 'f32' | 'f16'; @@ -55,7 +55,9 @@ export function convWeight( ); } -/** A quantized [E·K, O] weight and its f32 scales [E·K/groupSize, O]. */ +/** A quantized [E·K, O] weight and its f32 scales [E·K/groupSize, O]. Both + * versions get words from this checkout's packer: the layout is the + * kernels' contract, so A and B read the same words. */ export function quantWeight( version: Version, e: number, @@ -64,17 +66,22 @@ export function quantWeight( bits: 8 | 4, groupSize: number, ): { w: Value; scales: Value } { - const perByte = bits === 8 ? 1 : 2; - const qbytes = new Uint8Array((e * k * o) / perByte).map((_, i) => (i * 37 + 11) & 0xff); + const half = 2 ** (bits - 1); + const ints = Int8Array.from({ length: e * k * o }, (_, i) => ((i * 37 + 11) % (2 * half)) - half); + // Each expert is its own run of tiles, so the experts pack one by one. + const words = new Uint32Array((e * k * o * bits) / 32); + for (let x = 0; x < e; x++) { + const expert = packQuantW(ints.subarray(x * k * o, (x + 1) * k * o), k, o, bits); + words.set(expert, x * expert.length); + } const raw = version.rand((e * k * o) / groupSize).map((x) => Math.abs(x) * 0.01 + 1e-3); - const packed = version.lib.packQuantColSlice(qbytes, raw, bits, groupSize, e, k, o, 0, o); const w = version.freeAfterCase( version.lib.materialized( - { elems: (e * k * o) / (32 / bits), dtype: 'quantW', dims: [e * k, o] }, - version.lib.uploadU32(version.root, packed.qdata), + { elems: words.length, dtype: 'quantW', dims: [e * k, o] }, + version.lib.uploadU32(version.root, words), ), ); - const scales = version.upload(packed.scales, mat((e * k) / groupSize, o)); + const scales = version.upload(raw, mat((e * k) / groupSize, o)); return { w, scales }; } diff --git a/packages/runntime/src/core/index.ts b/packages/runntime/src/core/index.ts index 1c3872fb..769041e9 100644 --- a/packages/runntime/src/core/index.ts +++ b/packages/runntime/src/core/index.ts @@ -123,7 +123,7 @@ export { } from './error.ts'; export { createResourceScope, type Disposable, type ResourceScope } from './lifetime.ts'; export { BufferPool } from './gpu/bufferPool.ts'; -export { packQuantColSlice } from './weights/quantPack.ts'; +export { packQuantW } from './weights/quantPack.ts'; export type { WeightCache } from './weights/cache.ts'; export { bufferSource, diff --git a/packages/runntime/src/core/nn/layers.ts b/packages/runntime/src/core/nn/layers.ts index 36b9a1cc..1f0b0659 100644 --- a/packages/runntime/src/core/nn/layers.ts +++ b/packages/runntime/src/core/nn/layers.ts @@ -11,6 +11,7 @@ import { gatherRows, gatherRowsFrom, layerNorm, + matmul, reshape, rmsNorm, sdpa, @@ -19,8 +20,13 @@ import { snake, } from '../graph/ops.ts'; import { matrix, type Value } from '../graph/value.ts'; +import { + MATMUL_QUANT_TILED_BK, + matmulQuantTiledEligible, +} from '../kernels/matmul/matmulQuantTiled.ts'; import { convToTapMajor, + f16ToF32, packConv1x1Hwc4F16, packConvHwc4F16, packConvWinogradF16, @@ -28,7 +34,8 @@ import { permuteConvTransposeF16, } from '../weights/convert.ts'; import { foldBnIntoConv } from '../weights/foldBn.ts'; -import { derivedTensor, type LazyStateDict } from '../weights/safetensors.ts'; +import { packQuantW } from '../weights/quantPack.ts'; +import { derivedTensor, type LazyStateDict, type QuantTensor } from '../weights/safetensors.ts'; import { readFloats, transposeWeight } from '../weights/transforms.ts'; import { tensor } from '../gpu/buffers.ts'; import { CpuParameter, Module, Parameter, type AnyModule } from './module.ts'; @@ -100,12 +107,21 @@ export class Embedding extends Module<[readonly number[] | Value], Value> { /** y = x @ weight + bias. weight is stored [in, out]; torch checkpoints * transpose once at load. * + * A quantized checkpoint weight loads as quantW words with its f32 scales in + * `weightScale` [in / groupSize, out], and forward() runs the quantized + * matmul. Only the checkpoint says which kind it is, so transformStateDict + * retypes the slot at load. + * * `addend` and `activation` fold a residual add and a following tanh or GELU * into the matmul epilogue, as act(x·W + bias) + addend. */ export class Linear extends Module { readonly weight: Parameter; readonly bias?: Parameter; readonly activation?: SlotAct; + /** Set when a quantized weight loads. */ + weightScale?: Parameter; + /** How the loaded weight is quantized, or undefined for a float weight. */ + private quantization?: { bits: 4 | 8; groupSize: number }; constructor( readonly inFeatures: number, readonly outFeatures: number, @@ -135,10 +151,76 @@ export class Linear extends Module { `expected torch layout [${this.outFeatures}, ${this.inFeatures}]`, ); } + if (tensor.kind === 'quant') { + this.loadQuantized(sd, prefix, name, tensor); + return; + } sd.tensors.set(name, transposeWeight(tensor)); } + /** Replaces `name` with quantW words [in, out] and adds its scales + * [in / groupSize, out] right after it, so they bind in file order. The + * words carry the whole byteLength of the ints and scales. */ + private loadQuantized( + sd: LazyStateDict, + prefix: string, + name: string, + tensor: QuantTensor, + ): void { + const { bits, groupSize } = tensor; + const k = this.inFeatures; + const n = this.outFeatures; + // The quantized matmul reads whole tiles of 4 columns and 8 rows and + // packs int4 columns 8 to a word. + if (!matmulQuantTiledEligible(k, n, groupSize) || n % (32 / bits) !== 0) { + throw new RunntimeError( + 'CHECKPOINT_MISMATCH', + `Linear: ${name} is int${bits} in groups of ${groupSize}, which needs in features ` + + `a multiple of ${MATMUL_QUANT_TILED_BK} and of the group size, out features a ` + + `multiple of ${32 / bits} and a group size a multiple of 4, got [out, in] = [${n}, ${k}]`, + ); + } + this.quantization = { bits, groupSize }; + this.weight.retype('quantW', (k * n * bits) / 32); + this.weightScale = new Parameter( + { elems: (k / groupSize) * n, dtype: 'f32', dims: [k / groupSize, n] }, + undefined, + true, + ); + const words = derivedTensor( + [k, n], + async () => packQuantW(await tensor.ints(), k, n, bits), + tensor.byteLength, + ); + const groups = k / groupSize; + const scales = derivedTensor([groups, n], async () => { + const data = await tensor.scales(); + const f32 = data instanceof Float32Array ? data : f16ToF32(data); + // torch's [out, in / groupSize] to the kernels' [in / groupSize, out] + const out = new Float32Array(f32.length); + for (let o = 0; o < n; o++) { + for (let g = 0; g < groups; g++) out[g * n + o] = f32[o * groups + g]!; + } + return out; + }); + const scaleName = prefix ? `${prefix}.weightScale` : 'weightScale'; + const entries = [...sd.tensors]; + sd.tensors.clear(); + for (const [key, t] of entries) { + sd.tensors.set(key, key === name ? words : t); + if (key === name) sd.tensors.set(scaleName, scales); + } + } forward(x: Value, call: { addend?: Value } = {}): Value { - return linear(x, this.weight.value, this.bias?.value, call.addend, this.activation); + if (!this.weightScale || !this.quantization) { + return linear(x, this.weight.value, this.bias?.value, call.addend, this.activation); + } + return matmul(x, this.weight.value, { + scales: this.weightScale.value, + ...this.quantization, + bias: this.bias?.value, + addend: call.addend, + activation: this.activation, + }); } } diff --git a/packages/runntime/src/core/nn/loadStateDict.ts b/packages/runntime/src/core/nn/loadStateDict.ts index ea0ba469..05f4580b 100644 --- a/packages/runntime/src/core/nn/loadStateDict.ts +++ b/packages/runntime/src/core/nn/loadStateDict.ts @@ -144,18 +144,28 @@ async function bindStateDict( let done = 0; for (const [name, p] of params) { const t = sd.tensors.get(name)!; + // Linear's hook turns its quantized weight into quantW words, so a + // QuantTensor still here is bound to something else. if (t.kind === 'quant') { throw new RunntimeError( 'CHECKPOINT_MISMATCH', - `loadStateDict: '${name}' is quantized (int${t.bits}), but the parameter holds ${p.shape.dtype}`, + `loadStateDict: '${name}' is quantized (int${t.bits}), but only a Linear weight loads ` + + `quantized and this parameter holds ${p.shape.dtype}`, ); } const numel = t.shape.reduce((a, b) => a * b, 1); - const expected = numel; - if (expected !== p.shape.elems) { + // A quantW slot counts u32 words under logical dims. Privacy-filter's + // pre-packed U32 file tensors count words too; anything else, like + // Linear's packed weight, counts logical values. + const words = p.shape.dtype === 'quantW' && t.kind === 'file' && t.dtype === 'U32'; + const expected = + p.shape.dtype === 'quantW' && !words + ? p.shape.dims!.reduce((a, b) => a * b, 1) + : p.shape.elems; + if (numel !== expected) { throw new RunntimeError( 'CHECKPOINT_MISMATCH', - `loadStateDict: '${name}' checkpoint shape [${t.shape}] (${numel} elems) != model ${JSON.stringify(p.shape.dims)} (${p.shape.elems} ${p.shape.dtype === 'quantW' ? 'words' : 'elems'})`, + `loadStateDict: '${name}' checkpoint shape [${t.shape}] (${numel} elems) != model ${JSON.stringify(p.shape.dims)} (${expected} ${words ? 'words' : 'elems'})`, ); } if (p instanceof CpuParameter) { diff --git a/packages/runntime/src/core/nn/module.ts b/packages/runntime/src/core/nn/module.ts index 42e262fb..9dcee4b0 100644 --- a/packages/runntime/src/core/nn/module.ts +++ b/packages/runntime/src/core/nn/module.ts @@ -44,9 +44,10 @@ export class Parameter { if (!this._value) throw new Error(`Parameter ${this.qualifiedName || '(unnamed)'} not loaded`); return this._value; } - /** @internal Swaps the storage dtype, keeping elems and dims. */ - retype(dtype: EagerDtype): void { - this._shape = { ...this._shape, dtype }; + /** @internal Swaps the storage dtype, keeping dims. quantW counts u32 + * words, so a float slot turned quantW passes its word count as `elems`. */ + retype(dtype: EagerDtype, elems = this._shape.elems): void { + this._shape = { ...this._shape, dtype, elems }; } /** @internal */ bind(v: Value): void { diff --git a/packages/runntime/src/core/weights/quantPack.ts b/packages/runntime/src/core/weights/quantPack.ts index 4121ef2a..e1267b30 100644 --- a/packages/runntime/src/core/weights/quantPack.ts +++ b/packages/runntime/src/core/weights/quantPack.ts @@ -1,70 +1,28 @@ -/** Pack a column slice [oStart,oEnd) of a raw quantized expert tensor into - * eager's blocked-unit quantW layout + the matching scale slice. - * Input qbytes: canonical [E, K, O] int stream (int8 = 1 signed byte; int4 = - * 2/byte, even scalar index = low nibble). Input scales: f32 [E, KG, O], - * KG = K/groupSize. Output qdata: the layout kernels/quantCommon.ts documents — - * 16-value tiles per (expert, 4-col unit, 4-row chunk), col-major within the - * tile, packed at `bits` stride (tile = bits/2 words), tiles k4-major per - * (e, unit): word base ((e·W/4 + n)·K/4 + k4)·(bits/2) for width W. - * Output scales: f32 [E, KG, width] (unchanged layout — a unit's 4 scales - * stay adjacent). */ -export function packQuantColSlice( - qbytes: Uint8Array, - scales: Float32Array, - bits: 8 | 4, - groupSize: number, - E: number, - K: number, - O: number, - oStart: number, - oEnd: number, -): { qdata: Uint32Array; scales: Float32Array } { - const width = oEnd - oStart; - // Tiles are 4 cols × 4 rows: both the slice width and K must tile evenly, - // or the packing here would silently disagree with the kernels (the real - // model's width=640/K=640 always do). - if (width <= 0 || width % 4 !== 0) { - throw new Error(`packQuantColSlice: slice width ${width} must be a positive multiple of 4`); - } - if (K % 4 !== 0) - throw new Error(`packQuantColSlice: K ${K} must be a multiple of 4 (4-row tiles)`); - if (K % groupSize !== 0) - throw new Error(`packQuantColSlice: groupSize ${groupSize} must divide K ${K}`); - const KG = K / groupSize; - const expectQ = bits === 8 ? E * K * O : (E * K * O) / 2; - if (qbytes.length !== expectQ) - throw new Error(`packQuantColSlice: qbytes ${qbytes.length} != ${expectQ}`); - if (scales.length !== E * KG * O) - throw new Error(`packQuantColSlice: scales ${scales.length} != ${E * KG * O}`); - const mask = bits === 8 ? 0xff : 0xf; - const perU32 = bits === 8 ? 4 : 8; - const qdata = new Uint32Array((E * K * width) / perU32); - const outScales = new Float32Array(E * KG * width); - const readQ = (s: number): number => { - if (bits === 8) return (qbytes[s]! << 24) >> 24; - const nib = s & 1 ? qbytes[s >> 1]! >> 4 : qbytes[s >> 1]! & 0xf; - return nib >= 8 ? nib - 16 : nib; - }; - const n4 = width / 4; - const k4count = K / 4; - for (let e = 0; e < E; e++) { - for (let n = 0; n < n4; n++) { - for (let k4 = 0; k4 < k4count; k4++) { - const tileBit = ((e * n4 + n) * k4count + k4) * 16 * bits; - for (let j = 0; j < 4; j++) { - for (let kk = 0; kk < 4; kk++) { - const v = readQ((e * K + k4 * 4 + kk) * O + (oStart + n * 4 + j)); - const bit = tileBit + (j * 4 + kk) * bits; - qdata[bit >> 5] = (qdata[bit >> 5]! | ((v & mask) << (bit & 31))) >>> 0; - } +/** Packs int weights into the quantW layout that kernels/quantCommon.ts + * documents. */ + +/** Packs `ints`, a row-major output-first [n, k] matrix of signed int`bits` + * values (torch's Linear layout, as the checkpoint stores it), into quantW + * words for one expert's logical [k, n] weight: 16-value tiles per (4-column + * unit, 4-row chunk), column-major in the tile at `bits` per value, lowest + * bits first, tiles k4-major per unit. k and n must be multiples of 4. + * + * The f32 scales [k / groupSize, n] need no packing: the kernels read them + * row-major as they are, so the group size does not change the words. */ +export function packQuantW(ints: Int8Array, k: number, n: number, bits: 4 | 8): Uint32Array { + const words = new Uint32Array((k * n * bits) / 32); + const mask = (1 << bits) - 1; + for (let unit = 0; unit < n / 4; unit++) { + for (let k4 = 0; k4 < k / 4; k4++) { + const tileBit = (unit * (k / 4) + k4) * 16 * bits; + for (let c = 0; c < 4; c++) { + for (let r = 0; r < 4; r++) { + const v = ints[(unit * 4 + c) * k + k4 * 4 + r]!; + const bit = tileBit + (c * 4 + r) * bits; + words[bit >> 5] = (words[bit >> 5]! | ((v & mask) << (bit & 31))) >>> 0; } } } - for (let g = 0; g < KG; g++) { - for (let oo = 0; oo < width; oo++) { - outScales[(e * KG + g) * width + oo] = scales[(e * KG + g) * O + (oStart + oo)]!; - } - } } - return { qdata, scales: outScales }; + return words; } diff --git a/packages/runntime/test/core/nn/layers.gpu.test.ts b/packages/runntime/test/core/nn/layers.gpu.test.ts new file mode 100644 index 00000000..76171bbc --- /dev/null +++ b/packages/runntime/test/core/nn/layers.gpu.test.ts @@ -0,0 +1,175 @@ +import { describe, it, onTestFinished } from 'vitest'; +import { toArray } from '../../../src/core/gpu/eval.ts'; +import type { SlotAct } from '../../../src/core/graph/ops.ts'; +import { matrix } from '../../../src/core/graph/value.ts'; +import { Linear } from '../../../src/core/nn/layers.ts'; +import { + bufferSource, + fromSafetensors, + memoryStateDict, +} from '../../../src/core/weights/safetensors.ts'; +import { expectClose, gpu, randomF32, requireFeature, roundF16, upload } from '../../gpu.ts'; +import { ACTIVATION_REF } from '../kernels/matmul/checkMatmul.ts'; +import { quantFile } from '../weights/checkpoint.ts'; + +const { root, ex } = await gpu(); + +interface Case { + m: number; + inF: number; + outF: number; + groupSize: number; + bias: boolean; + addend?: boolean; + activation?: SlotAct; +} + +/** A Linear loaded from a quantized file matches a CPU loop of + * act(x·W + bias) + addend in f64, with W dequantized from the same ints as + * int · f16 scale. */ +async function checkQuantizedLinear(c: Case, bits: 4 | 8, dtype: 'f32' | 'f16') { + const { m, inF, outF, groupSize } = c; + const half = 2 ** (bits - 1); + // torch [out, in] ints over the full signed range, positive scales + const ints = Array.from(randomF32(outF * inF, 1), (v) => + Math.min(half - 1, Math.floor(v * half)), + ); + const scales = roundF16( + randomF32((outF * inF) / groupSize, 2).map((v) => (Math.abs(v) + 0.1) / half), + ); + // Rounded to f16 for both dtypes, so the f16 upload stores them exactly. + const bias = roundF16(randomF32(outF, 3)); + const xs = roundF16(randomF32(m * inF, 4)); + const addend = roundF16(randomF32(m * outF, 5)); + + const layer = new Linear(inF, outF, { bias: c.bias, activation: c.activation }); + if (dtype === 'f16') layer.half(root); + onTestFinished(() => layer.dispose()); + const file = quantFile(bits, groupSize, { + weight: { ints, scales: [...scales], shape: [outF, inF] }, + }); + const sd = await fromSafetensors(bufferSource(file), { chunkBytes: 0 }); + if (c.bias) sd.tensors.set('bias', memoryStateDict({ bias }).tensors.get('bias')!); + await layer.loadStateDict(sd, { root }); + + const x = upload(root, xs, { ...matrix(m, inF), dtype }); + const call = c.addend ? { addend: upload(root, addend, { ...matrix(m, outF), dtype }) } : {}; + const got = await toArray(layer.forward(x, call), ex); + + // The f16 kernel stages the dequantized weight in f16 workgroup memory. + const dequant = Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / groupSize)]!); + const weight = dtype === 'f16' ? roundF16(dequant) : dequant; + const want = new Float64Array(m * outF); + for (let i = 0; i < m; i++) { + for (let o = 0; o < outF; o++) { + let sum = 0; + for (let p = 0; p < inF; p++) sum += xs[i * inF + p]! * weight[o * inF + p]!; + if (c.bias) sum += bias[o]!; + if (c.activation) sum = ACTIVATION_REF[c.activation](sum); + if (c.addend) sum += addend[i * outF + o]!; + want[i * outF + o] = sum; + } + } + // The kernel sums in f32, so its error grows with inF; f16 also rounds the + // output to half precision, 2^-11 relative. + const tol = dtype === 'f16' ? { atol: 1e-5 * inF, rtol: 1e-3 } : { atol: 1e-6 * inF, rtol: 1e-5 }; + expectClose(got, want, tol); +} + +describe('Linear with a quantized weight', () => { + // A quantized Linear takes the plain quantW kernel only for an f32 input + // under 8 rows with no epilogue; anything else takes the tiled kernel. + const CASES: (Case & { name: string; dtypes: ('f32' | 'f16')[] })[] = [ + // plain: f32, m < 8, no bias + { name: 'plain, 1 row', m: 1, inF: 64, outF: 48, groupSize: 16, bias: false, dtypes: ['f32'] }, + { name: 'plain, 7 rows', m: 7, inF: 64, outF: 48, groupSize: 16, bias: false, dtypes: ['f32'] }, + // tiled: m >= 8 + { name: 'tiled, 8 rows', m: 8, inF: 64, outF: 48, groupSize: 16, bias: false, dtypes: ['f32'] }, + // tiled: a bias, an activation or an addend is an epilogue, whatever m + { + name: 'tiled by its bias, 7 rows', + m: 7, + inF: 64, + outF: 48, + groupSize: 16, + bias: true, + dtypes: ['f32'], + }, + { + name: 'tiled by its activation, 3 rows', + m: 3, + inF: 64, + outF: 48, + groupSize: 16, + bias: false, + activation: 'gelu', + dtypes: ['f32', 'f16'], + }, + { + name: 'tiled by its addend, 2 rows', + m: 2, + inF: 64, + outF: 48, + groupSize: 16, + bias: false, + addend: true, + dtypes: ['f32', 'f16'], + }, + { + name: 'bias, gelu and addend, 37 rows', + m: 37, + inF: 256, + outF: 136, + groupSize: 32, + bias: true, + addend: true, + activation: 'gelu', + dtypes: ['f32', 'f16'], + }, + // tiled: an f16 input, whatever m + { + name: 'f16 input, 1 row', + m: 1, + inF: 64, + outF: 48, + groupSize: 16, + bias: false, + dtypes: ['f16'], + }, + { + name: 'f16 input with bias, 5 rows', + m: 5, + inF: 64, + outF: 48, + groupSize: 16, + bias: true, + dtypes: ['f16'], + }, + { + name: 'partial tiles with bias, 37 rows', + m: 37, + inF: 256, + outF: 136, + groupSize: 32, + bias: true, + dtypes: ['f32', 'f16'], + }, + { + name: 'one group per row, 37 rows', + m: 37, + inF: 128, + outF: 64, + groupSize: 128, + bias: false, + dtypes: ['f32', 'f16'], + }, + ]; + it.for( + CASES.flatMap(({ dtypes, ...c }) => + dtypes.flatMap((dtype) => ([4, 8] as const).map((bits) => ({ ...c, dtype, bits }))), + ), + )('$dtype int$bits $name: [$m, $inF] → $outF, groupSize $groupSize', async (c, ctx) => { + if (c.dtype === 'f16') requireFeature(ctx, root, 'shader-f16'); + await checkQuantizedLinear(c, c.bits, c.dtype); + }); +}); diff --git a/packages/runntime/test/core/nn/layers.test.ts b/packages/runntime/test/core/nn/layers.test.ts index 071a6f29..303e3945 100644 --- a/packages/runntime/test/core/nn/layers.test.ts +++ b/packages/runntime/test/core/nn/layers.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from 'vitest'; +import { ACT_CODE } from '../../../src/core/graph/ops/shared.ts'; import { pending, type ValueMeta } from '../../../src/core/graph/value.ts'; import { BatchNorm2d, @@ -24,8 +25,9 @@ import { memoryStateDict, type LazyStateDict, } from '../../../src/core/weights/safetensors.ts'; +import { scaleWeight } from '../../../src/core/weights/transforms.ts'; import { hwc4Input, input, meta } from '../graph/values.ts'; -import { safetensorsFile } from '../weights/checkpoint.ts'; +import { contents, quantFile, safetensorsFile, type QuantEntry } from '../weights/checkpoint.ts'; /** Binds every parameter to a stand-in of its declared shape, so forward() * can build its graph without real weights. */ @@ -242,6 +244,178 @@ describe('Linear', () => { ); }); +/** A quantized file of `entries` as a lazy state dict. */ +const quantCheckpoint = (bits: 4 | 8, groupSize: number, entries: Record) => + fromSafetensors(bufferSource(quantFile(bits, groupSize, entries)), { chunkBytes: 0 }); + +describe('Linear with a quantized weight', () => { + // torch [out 4, in 8], value o·8 + i, so a kernel column (one torch row) + // holds o·8 + 0..7 and its 4-row chunks are 4 consecutive bytes. + const numbered = Array.from({ length: 32 }, (_, i) => i); + // f16-exact scales [out 4, in/4 = 2]: o + g/2 + const scales = [0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5]; + const entry: QuantEntry = { ints: numbered, scales, shape: [4, 8] }; + + // int4 packs out features 8 to a word, so it takes [out 8, in 8]. + const int4Entry: QuantEntry = { + ints: Array.from({ length: 64 }, (_, i) => (i % 16) - 8), + scales: Array.from({ length: 16 }, (_, i) => i / 2), + shape: [8, 8], + }; + + it.each([ + { name: 'int8', bits: 8 as const, quant: entry, words: 8 }, + { name: 'int4', bits: 4 as const, quant: int4Entry, words: 8 }, + ])('retypes the weight to $name quantW and adds f32 scales', async ({ bits, quant, words }) => { + const out = quant.shape[0]; + const layer = new Linear(8, out); + const sd = await quantCheckpoint(bits, 4, { 'fc.weight': quant }); + layer.transformStateDict(sd, 'fc'); + expect(shapes(layer)).toEqual({ + weight: { elems: words, dtype: 'quantW', dims: [8, out] }, + bias: meta([out]), + weightScale: meta([2, out]), + }); + expect(layer.weightScale!.pinned).toBe(true); + expect([...sd.tensors.keys()]).toEqual(['fc.weight', 'fc.weightScale']); + expect(sd.tensors.get('fc.weight')!.shape).toEqual([8, out]); + }); + + it('packs the torch [out, in] int8 ints as kernel [in, out] tiles', async () => { + const sd = await quantCheckpoint(8, 4, { weight: entry }); + new Linear(8, 4).transformStateDict(sd, ''); + // Tile (unit 0, rows 0..3) then (unit 0, rows 4..7); column c of a tile + // is torch row c, so its word holds bytes c·8 + 4·k4 + 0..3. + expect([...(await sd.tensors.get('weight')!.words())]).toEqual([ + 0x03020100, 0x0b0a0908, 0x13121110, 0x1b1a1918, 0x07060504, 0x0f0e0d0c, 0x17161514, + 0x1f1e1d1c, + ]); + }); + + it('packs int4 ints two columns to a word', async () => { + const sd = await quantCheckpoint(4, 4, { weight: int4Entry }); + new Linear(8, 8).transformStateDict(sd, ''); + // Column c is torch row c: i - 8 for an even row, i for an odd one. A + // word holds an even column's 4 nibbles, then an odd one's. + expect([...(await sd.tensors.get('weight')!.words())]).toEqual([ + 0x3210ba98, + 0x3210ba98, + 0x7654fedc, + 0x7654fedc, // columns 0..3: rows 0..3, rows 4..7 + 0x3210ba98, + 0x3210ba98, + 0x7654fedc, + 0x7654fedc, // columns 4..7 + ]); + }); + + it('transposes the scales to [in / groupSize, out]', async () => { + const sd = await quantCheckpoint(8, 4, { weight: entry }); + new Linear(8, 4).transformStateDict(sd, ''); + const scale = sd.tensors.get('weightScale')!; + expect(scale.shape).toEqual([2, 4]); + expect(await contents(scale)).toEqual({ + type: 'f32', + values: [0, 1, 2, 3, 0.5, 1.5, 2.5, 3.5], + }); + }); + + it('takes f32 scales from scaleWeight as they are', async () => { + const sd = await quantCheckpoint(8, 4, { weight: entry }); + sd.tensors.set('weight', scaleWeight(sd.tensors.get('weight')!, [1, 2, 3, 4])); + new Linear(8, 4).transformStateDict(sd, ''); + // torch row o times o + 1, then transposed + expect(await contents(sd.tensors.get('weightScale')!)).toEqual({ + type: 'f32', + values: [0, 2, 6, 12, 0.5, 3, 7.5, 14], + }); + }); + + it('puts the scales right after the weight, so binding keeps file order', async () => { + const sd = await quantCheckpoint(8, 4, { 'fc.weight': entry }); + const float = derivedTensor([4], () => Promise.resolve(new Float32Array(4))); + sd.tensors = new Map([ + ['first', float], + ['fc.weight', sd.tensors.get('fc.weight')!], + ['fc.bias', float], + ]); + new Linear(8, 4).transformStateDict(sd, 'fc'); + expect([...sd.tensors.keys()]).toEqual(['first', 'fc.weight', 'fc.weightScale', 'fc.bias']); + }); + + it('gives the words the bytes of the ints and scales, and the scales none', async () => { + const sd = await quantCheckpoint(4, 4, { weight: int4Entry }); + const total = sd.totalBytes(); + new Linear(8, 8).transformStateDict(sd, ''); + expect(sd.tensors.get('weight')!.byteLength).toBe(32 + 32); // 64 nibbles, 16 f16 scales + expect(sd.tensors.get('weightScale')!.byteLength).toBe(0); + expect(sd.totalBytes()).toBe(total); + }); + + it('keeps the bias of a half() layer f16 and the scales f32', async () => { + const layer = new Linear(8, 4).half(); + layer.transformStateDict(await quantCheckpoint(8, 4, { weight: entry }), ''); + expect(shapes(layer)).toEqual({ + weight: { elems: 8, dtype: 'quantW', dims: [8, 4] }, + bias: meta([4], 'f16'), + weightScale: meta([2, 4]), + }); + }); + + it.each([ + { name: 'in features off the 8-row tiles', inF: 12, outF: 4, bits: 8 as const, groupSize: 4 }, + { name: 'int4 out features off 8 per word', inF: 8, outF: 4, bits: 4 as const, groupSize: 4 }, + { name: 'a group size off 4', inF: 16, outF: 4, bits: 8 as const, groupSize: 2 }, + ])('refuses $name with CHECKPOINT_MISMATCH', async ({ inF, outF, bits, groupSize }) => { + const sd = await quantCheckpoint(bits, groupSize, { + weight: { + ints: new Array(inF * outF).fill(0), + scales: new Array((inF * outF) / groupSize).fill(1), + shape: [outF, inF], + }, + }); + const layer = new Linear(inF, outF); + expect(() => layer.transformStateDict(sd, '')).toThrow( + expect.objectContaining({ code: 'CHECKPOINT_MISMATCH' }), + ); + expect(layer.weight.shape.dtype).toBe('f32'); + }); + + it.each([ + { name: 'f32', half: false }, + { name: 'f16', half: true }, + ])( + 'runs the $name forward as a quantized matmul with bias and activation in the epilogue', + async ({ half }) => { + const layer = new Linear(8, 4, { activation: 'gelu' }); + if (half) layer.half(); + layer.transformStateDict(await quantCheckpoint(8, 4, { weight: entry }), ''); + fakeLoad(layer); + const x = input([5, 8], half ? 'f16' : 'f32'); + const addend = input([5, 4], half ? 'f16' : 'f32'); + const y = layer.forward(x, { addend }); + expect(y.shape).toEqual(meta([5, 4], half ? 'f16' : 'f32')); + expect(y.pending?.op).toBe('matmulQuantW'); + expect(y.pending?.inputs).toEqual([ + x, + layer.weight.value, + layer.weightScale!.value, + layer.bias!.value, + addend, + ]); + // baseRow, scaleBase, bits, groupSize, bias, addend, activation + expect(y.pending?.attrs).toEqual([0, 0, 8, 4, 1, 1, ACT_CODE.gelu]); + }, + ); + + it('refuses a weight in [in, out] layout', async () => { + const sd = await quantCheckpoint(8, 4, { weight: { ints: numbered, scales, shape: [8, 4] } }); + expect(() => new Linear(8, 4).transformStateDict(sd, '')).toThrow( + expect.objectContaining({ code: 'CHECKPOINT_MISMATCH' }), + ); + }); +}); + describe('Conv1d', () => { it('stores the weight tap-major as [kernelSize·in, out]', () => { expect(shapes(new Conv1d(4, 6, 3))).toEqual({ weight: meta([12, 6]), bias: meta([6]) }); diff --git a/packages/runntime/test/core/nn/loadStateDict.test.ts b/packages/runntime/test/core/nn/loadStateDict.test.ts index f2c72749..03e4d534 100644 --- a/packages/runntime/test/core/nn/loadStateDict.test.ts +++ b/packages/runntime/test/core/nn/loadStateDict.test.ts @@ -1,6 +1,7 @@ import type { TgpuRoot } from 'typegpu'; import { describe, expect, it, vi } from 'vitest'; import { pending, type ValueMeta } from '../../../src/core/graph/value.ts'; +import { Linear } from '../../../src/core/nn/layers.ts'; import { gpuUpload, loadLazyStateDict } from '../../../src/core/nn/loadStateDict.ts'; import { Buffer, @@ -19,6 +20,7 @@ import { type LazyTensor, type MemoryStateDict, } from '../../../src/core/weights/safetensors.ts'; +import { f32ToF16Bits } from '../../../src/core/weights/convert.ts'; import { meta } from '../graph/values.ts'; import { safetensorsFile } from '../weights/checkpoint.ts'; @@ -260,14 +262,21 @@ describe('a quantized tensor', () => { { name: 'an f32 parameter', param: () => new Parameter(meta([2, 2])), dtype: 'f32' }, { name: 'an f16 parameter', param: () => new Parameter(meta([2, 2], 'f16')), dtype: 'f16' }, { name: 'a CpuParameter', param: () => new CpuParameter(meta([2, 2])), dtype: 'f32' }, + { + name: 'a quantW parameter', + param: () => new Parameter({ elems: 1, dtype: 'quantW', dims: [2, 2] }), + dtype: 'quantW', + }, ])('is refused by $name with CHECKPOINT_MISMATCH', async ({ param, dtype }) => { - class Linear extends Module { + class NotLinear extends Module { weight = param(); } const upload = recorder(); - await expect(load(new Linear(), { weight: quant }, upload)).rejects.toMatchObject({ + await expect(load(new NotLinear(), { weight: quant }, upload)).rejects.toMatchObject({ code: 'CHECKPOINT_MISMATCH', - message: `loadStateDict: 'weight' is quantized (int8), but the parameter holds ${dtype}`, + message: + "loadStateDict: 'weight' is quantized (int8), but only a Linear weight loads quantized " + + `and this parameter holds ${dtype}`, }); expect(upload).not.toHaveBeenCalled(); expect(ints).not.toHaveBeenCalled(); @@ -275,6 +284,109 @@ describe('a quantized tensor', () => { }); }); +describe('a quantized Linear', () => { + // torch [out 4, in 8] int8 in groups of 4: ints o·8 + i, f16-exact scales + // o + g/2, and an f32 bias. + const file = safetensorsFile( + { + 'fc.weight': { data: Int8Array.from({ length: 32 }, (_, i) => i), shape: [4, 8] }, + 'fc.weight_scale': { + data: Uint16Array.from([0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5], f32ToF16Bits), + shape: [4, 2], + }, + 'fc.bias': { data: new Float32Array([1, 2, 3, 4]), shape: [4] }, + }, + { + 'runntime.quant': JSON.stringify({ version: 1, bits: 8, groupSize: 4, scheme: 'symmetric' }), + }, + ); + class Net extends Module { + fc = new Linear(8, 4); + } + + it('uploads the words, the scales and the bias in file order', async () => { + const sd = await fromSafetensors(bufferSource(file), { chunkBytes: 0 }); + const weight = sd.tensors.get('fc.weight')!; + if (weight.kind !== 'quant') throw new Error('expected a quantized tensor'); + const ints = vi.spyOn(weight, 'ints'); + const upload = recorder(); + const progress: [string, number, number][] = []; + const net = new Net(); + await loadLazyStateDict(net, sd, { + upload, + onProgress: (name, done, total) => progress.push([name, done, total]), + }); + expect(upload.mock.calls.map(([data, shape]) => [data.constructor.name, shape])).toEqual([ + ['Uint32Array', { elems: 8, dtype: 'quantW', dims: [8, 4] }], + ['Float32Array', meta([2, 4])], + ['Float32Array', meta([4])], + ]); + expect(upload.mock.calls[1]![0]).toEqual(new Float32Array([0, 1, 2, 3, 0.5, 1.5, 2.5, 3.5])); + expect(ints).toHaveBeenCalledTimes(1); + // 32 int8 bytes + 16 scale bytes, then the 16-byte bias: the payload size + expect(progress).toEqual([ + ['fc.weight', 48, 64], + ['fc.weightScale', 48, 64], + ['fc.bias', 64, 64], + ]); + expect(net.fc.weightScale!.loaded).toBe(true); + }); + + it('loads a float checkpoint into the same layer as before', async () => { + const upload = recorder(); + const net = new Net(); + await load( + net, + { + 'fc.weight': { data: new Float32Array(32), shape: [4, 8] }, + 'fc.bias': new Float32Array(4), + }, + upload, + ); + expect(upload.mock.calls.map(([, shape]) => shape)).toEqual([meta([8, 4]), meta([4])]); + expect(net.fc.weightScale).toBeUndefined(); + }); +}); + +describe('a quantW slot', () => { + // int8 [4, 4]: 16 values in 4 words. + class Packed extends Module { + w = new Parameter({ elems: 4, dtype: 'quantW', dims: [4, 4] }); + } + + /** A file holding `words` as one U32 tensor `w` of `shape`. */ + const wordFile = (words: Uint32Array, shape: number[]) => + fromSafetensors(bufferSource(safetensorsFile({ w: { data: words, shape } })), { + chunkBytes: 0, + }); + + it('takes pre-packed U32 words from a file, counted in words', async () => { + const words = new Uint32Array([1, 2, 3, 4]); + const upload = recorder(); + await loadLazyStateDict(new Packed(), await wordFile(words, [4]), { upload }); + expect(upload).toHaveBeenCalledWith(words, { elems: 4, dtype: 'quantW', dims: [4, 4] }); + }); + + it('takes derived words counted in logical values', async () => { + const words = new Uint32Array([1, 2, 3, 4]); + const upload = recorder(); + await load(new Packed(), { w: derivedTensor([4, 4], () => Promise.resolve(words)) }, upload); + expect(upload).toHaveBeenCalledWith(words, { elems: 4, dtype: 'quantW', dims: [4, 4] }); + }); + + it.each([ + { name: 'U32 words of another count', sd: () => wordFile(new Uint32Array(5), [5]) }, + { + name: 'derived words of another logical size', + sd: async () => dict({ w: derivedTensor([4], () => Promise.resolve(new Uint32Array(4))) }), + }, + ])('refuses $name with CHECKPOINT_MISMATCH', async ({ sd }) => { + await expect( + loadLazyStateDict(new Packed(), await sd(), { upload: recorder() }), + ).rejects.toMatchObject({ code: 'CHECKPOINT_MISMATCH' }); + }); +}); + describe('materializing an f16 slot', () => { class Half extends Module { w = new Parameter(meta([2], 'f16')); diff --git a/packages/runntime/test/core/weights/checkpoint.ts b/packages/runntime/test/core/weights/checkpoint.ts index 1adece61..a9cca4e8 100644 --- a/packages/runntime/test/core/weights/checkpoint.ts +++ b/packages/runntime/test/core/weights/checkpoint.ts @@ -11,22 +11,23 @@ import { export type Entry = { values: number[]; shape: number[] }; +/** The array types an entry's data can come in. */ +type EntryData = Float32Array | Uint16Array | Uint32Array | Int8Array | Uint8Array; + /** Safetensors dtype by the array type that holds an entry's data. */ -const dtypeOf = (data: Float32Array | Uint16Array | Int8Array | Uint8Array) => { +const dtypeOf = (data: EntryData) => { if (data instanceof Float32Array) return 'F32'; if (data instanceof Uint16Array) return 'F16'; + if (data instanceof Uint32Array) return 'U32'; return data instanceof Int8Array ? 'I8' : 'U8'; }; /** A safetensors file built from the format spec: an 8-byte little-endian * header length, the JSON header, then the payload. An entry's dtype follows - * its data: F32 for a Float32Array, F16 for a Uint16Array of halfwords, I8 - * for an Int8Array, U8 for a Uint8Array. `metadata` goes in `__metadata__`. */ + * its data: F32 for a Float32Array, F16 for a Uint16Array of halfwords, U32 + * for a Uint32Array, I8 for an Int8Array, U8 for a Uint8Array. `metadata` goes in `__metadata__`. */ export function safetensorsFile( - entries: Record< - string, - { data: Float32Array | Uint16Array | Int8Array | Uint8Array; shape: number[] } - >, + entries: Record, metadata?: Record, ): Uint8Array { const header: Record = metadata ? { __metadata__: metadata } : {}; @@ -92,15 +93,41 @@ export async function contents(t: LazyTensor): Promise<{ type: 'f16' | 'f32'; va * `[out, in / groupSize]`. */ export type QuantEntry = { ints: number[]; scales: number[]; shape: [number, number] }; +/** `values`, a torch-layout `[out, in]` weight, quantized the way the format + * spec's symmetric scheme reads it: per group of `groupSize` along `in`, the + * scale is the group's largest |value| over the top int (127 for int8, 7 for + * int4) rounded to f16, and each int is value / scale rounded to nearest. */ +export function quantizeWeight( + values: Float32Array, + shape: [number, number], + bits: 4 | 8, + groupSize: number, +): QuantEntry { + const top = 2 ** (bits - 1) - 1; + const ints = new Array(values.length); + const scales = new Array(values.length / groupSize); + for (let g = 0; g < scales.length; g++) { + const group = values.subarray(g * groupSize, (g + 1) * groupSize); + const largest = group.reduce((most, v) => Math.max(most, Math.abs(v)), 0); + const scale = f16ToF32(Uint16Array.of(f32ToF16Bits(largest / top)))[0]!; + scales[g] = scale; + // An all-zero group has scale 0 and ints 0. + group.forEach((v, i) => (ints[g * groupSize + i] = scale === 0 ? 0 : Math.round(v / scale))); + } + return { ints, scales, shape }; +} + /** A quantized safetensors file in the format spec: each entry's ints as * `name` (I8, or U8 with two int4 per byte, low nibble first), its scales - * rounded to f16 as `name_scale`, and the setting in `runntime.quant`. */ + * rounded to f16 as `name_scale`, and the setting in `runntime.quant`. + * `floats` go in as F32 tensors next to them. */ export function quantFile( bits: 4 | 8, groupSize: number, entries: Record, + floats: Record = {}, ): Uint8Array { - const tensors: Parameters[0] = {}; + const tensors: Parameters[0] = { ...floats }; for (const [name, { ints, scales, shape }] of Object.entries(entries)) { const [out, cols] = shape; tensors[name] = diff --git a/packages/runntime/test/core/weights/quantPack.test.ts b/packages/runntime/test/core/weights/quantPack.test.ts index 165e22dc..294208c8 100644 --- a/packages/runntime/test/core/weights/quantPack.test.ts +++ b/packages/runntime/test/core/weights/quantPack.test.ts @@ -1,34 +1,24 @@ import { describe, expect, it } from 'vitest'; -import { packQuantColSlice } from '../../../src/core/weights/quantPack.ts'; +import { columnWord } from '../../../src/core/kernels/quantCommon.ts'; +import { packQuantW } from '../../../src/core/weights/quantPack.ts'; -// The layout (kernels/quantCommon.ts): 16-value tiles, one per (expert, 4-column -// unit, 4-row chunk). Inside a tile, column by column, each column's 4 rows in -// a row. Values sit back to back at `bits` width, lowest bits first. +// The layout (kernels/quantCommon.ts): 16-value tiles, one per (4-column unit, +// 4-row chunk). Inside a tile, column by column, each column's 4 rows in a +// row. Values sit back to back at `bits` width, lowest bits first. // // Expected words below are worked out by hand from that description. -/** Signed int8 values as the raw bytes a checkpoint stores. */ -const int8 = (values: number[]) => new Uint8Array(new Int8Array(values).buffer); +/** Output-first [n, k] ints where row r, column c of the logical [k, n] weight + * holds r·n + c, so each value names its spot. */ +const numbered = (k: number, n: number) => + Int8Array.from({ length: k * n }, (_, i) => (i % k) * n + Math.floor(i / k)); -/** Signed int4 values, two per byte, the even index in the low nibble. */ -function int4(values: number[]): Uint8Array { - const out = new Uint8Array(values.length / 2); - values.forEach((v, i) => (out[i >> 1]! |= (v & 0xf) << (i & 1 ? 4 : 0))); - return out; -} - -/** Row-major [K, O] values numbered 0, 1, 2, ... so each value names its spot. */ -const numbered = (k: number, o: number) => Array.from({ length: k * o }, (_, i) => i); - -const noScales = (E: number, KG: number, O: number) => new Float32Array(E * KG * O); - -describe('packQuantColSlice', () => { +describe('packQuantW', () => { it('packs one int8 tile column by column', () => { - // [K=4, O=4]; value r*4 + c sits at row r, column c - const q = int8(numbered(4, 4)); - const { qdata } = packQuantColSlice(q, noScales(1, 1, 4), 8, 4, 1, 4, 4, 0, 4); + // [K=4, N=4]; value r*4 + c sits at row r, column c + const words = packQuantW(numbered(4, 4), 4, 4, 8); // one word per column, its 4 rows from the lowest byte up - expect([...qdata]).toEqual([ + expect([...words]).toEqual([ 0x0c080400, // column 0: rows hold 0, 4, 8, 12 0x0d090501, // column 1: 1, 5, 9, 13 0x0e0a0602, // column 2 @@ -37,108 +27,68 @@ describe('packQuantColSlice', () => { }); it('packs one int4 tile, two columns per word', () => { - const q = int4(numbered(4, 4).map((v) => (v >= 8 ? v - 16 : v))); // keep in int4 range - const { qdata } = packQuantColSlice(q, noScales(1, 1, 4), 4, 4, 1, 4, 4, 0, 4); - expect([...qdata]).toEqual([ + const ints = numbered(4, 4).map((v) => (v >= 8 ? v - 16 : v)); // keep in int4 range + expect([...packQuantW(ints, 4, 4, 4)]).toEqual([ 0xd951c840, // columns 0 and 1: nibbles 0, 4, 8, 12, 1, 5, 9, 13 from the bottom 0xfb73ea62, // columns 2 and 3 ]); }); it.each([ - { bits: 8 as const, pack: int8, mask: 0xff }, - { bits: 4 as const, pack: int4, mask: 0xf }, - ])('keeps negative int$bits values as two’s complement bits', ({ bits, pack, mask }) => { - const values = Array.from({ length: 16 }, (_, i) => (i % 2 ? -1 : -(bits === 8 ? 128 : 8))); - const { qdata } = packQuantColSlice(pack(values), noScales(1, 1, 4), bits, 4, 1, 4, 4, 0, 4); + { bits: 8 as const, mask: 0xff }, + { bits: 4 as const, mask: 0xf }, + ])('keeps negative int$bits values as two’s complement bits', ({ bits, mask }) => { + const ints = Int8Array.from({ length: 16 }, (_, i) => (i % 2 ? -1 : -(2 ** (bits - 1)))); + const words = packQuantW(ints, 4, 4, bits); const unpacked = Array.from( { length: 16 }, - (_, i) => (qdata[(i * bits) >> 5]! >>> ((i * bits) & 31)) & mask, + (_, i) => (words[(i * bits) >> 5]! >>> ((i * bits) & 31)) & mask, ); - // column-major in, so value (row r, column c) comes out at c*4 + r - const want = Array.from({ length: 16 }, (_, i) => values[(i % 4) * 4 + (i >> 2)]! & mask); + // a tile is column-major like the output-first input, so values keep their order + const want = Array.from(ints, (v) => v & mask); expect(unpacked).toEqual(want); }); - it('takes only the requested columns', () => { - // [K=4, O=8], slice columns 4..7: the same tile as columns 0..3 shifted by 4 - const q = int8(numbered(4, 8)); - const { qdata } = packQuantColSlice(q, noScales(1, 1, 8), 8, 4, 1, 4, 8, 4, 8); - expect([...qdata]).toEqual([ - 0x1c140c04, // column 4: rows hold 4, 12, 20, 28 - 0x1d150d05, - 0x1e160e06, - 0x1f170f07, - ]); - }); - - it('orders tiles by expert, then column unit, then row chunk', () => { - // E=2, K=8, O=8: per expert 2 column units × 2 row chunks = 4 tiles of 4 words - const E = 2; - const K = 8; - const O = 8; - const q = int8(Array.from({ length: E * K * O }, (_, i) => i - 128)); - const { qdata } = packQuantColSlice(q, noScales(E, 2, O), 8, 4, E, K, O, 0, O); - expect(qdata).toHaveLength(E * 4 * 4); - // first word of each tile = rows 0..3 of the chunk's first column - const firstByte = (tile: number) => ((qdata[tile * 4]! << 24) >> 24) + 128; - const index = (e: number, row: number, col: number) => (e * K + row) * O + col; - expect(firstByte(0)).toBe(index(0, 0, 0)); // expert 0, columns 0..3, rows 0..3 - expect(firstByte(1)).toBe(index(0, 4, 0)); // expert 0, columns 0..3, rows 4..7 - expect(firstByte(2)).toBe(index(0, 0, 4)); // expert 0, columns 4..7, rows 0..3 - expect(firstByte(3)).toBe(index(0, 4, 4)); - expect(firstByte(4)).toBe(index(1, 0, 0)); // expert 1 starts here - expect(firstByte(7)).toBe(index(1, 4, 4)); - }); - - it('puts every value of the slice in exactly once', () => { - const E = 2; - const K = 8; - const O = 12; - const values = Array.from({ length: E * K * O }, (_, i) => (i % 251) - 125); - const { qdata } = packQuantColSlice(int8(values), noScales(E, 2, O), 8, 4, E, K, O, 4, 12); - const got = [...new Int8Array(qdata.buffer)].sort((a, b) => a - b); - const want = values - .filter((_, i) => i % O >= 4) // columns 4..11 of every row - .sort((a, b) => a - b); - expect(got).toEqual(want); - }); - - it('slices the scales to the same columns', () => { - // scales [E=1, KG=2, O=8]: value g*10 + column - const scales = Float32Array.from({ length: 16 }, (_, i) => Math.floor(i / 8) * 10 + (i % 8)); - const q = int8(new Array(8 * 8).fill(0)); - const out = packQuantColSlice(q, scales, 8, 4, 1, 8, 8, 4, 8); - expect([...out.scales]).toEqual([4, 5, 6, 7, 14, 15, 16, 17]); + it('orders tiles by column unit, then row chunk', () => { + // K=8, N=8: 2 column units × 2 row chunks = 4 tiles of 4 words + const k = 8; + const n = 8; + const words = packQuantW(numbered(k, n), k, n, 8); + expect(words).toHaveLength(16); + // first byte of each tile = row 0 of the chunk's first column + const firstByte = (tile: number) => words[tile * 4]! & 0xff; + expect(firstByte(0)).toBe(0 * n + 0); // columns 0..3, rows 0..3 + expect(firstByte(1)).toBe(4 * n + 0); // columns 0..3, rows 4..7 + expect(firstByte(2)).toBe(0 * n + 4); // columns 4..7, rows 0..3 + expect(firstByte(3)).toBe(4 * n + 4); }); + // What the kernels read: a tile starts at word (unit·K/4 + k4)·bits/2, column + // c sits in word columnWord(c) at bit (c·4·bits) mod 32, and row r of it + // r·bits higher, sign-extended from `bits`. it.each([ - { name: 'an empty slice', args: [8, 4, 1, 4, 4, 2, 2], error: /width 0/ }, - { name: 'a slice width not a multiple of 4', args: [8, 4, 1, 4, 8, 0, 6], error: /width 6/ }, - { name: 'K not a multiple of 4', args: [8, 2, 1, 6, 4, 0, 4], error: /K 6/ }, - { - name: 'a group size that does not divide K', - args: [8, 3, 1, 4, 4, 0, 4], - error: /groupSize 3/, - }, - ] as const)('rejects $name', ({ args, error }) => { - const [bits, groupSize, E, K, O, oStart, oEnd] = args; - const q = new Uint8Array(E * K * O); - expect(() => - packQuantColSlice(q, noScales(E, 1, O), bits, groupSize, E, K, O, oStart, oEnd), - ).toThrow(error); - }); - - it('rejects quantized data or scales of the wrong length', () => { - expect(() => - packQuantColSlice(new Uint8Array(15), noScales(1, 1, 4), 8, 4, 1, 4, 4, 0, 4), - ).toThrow(/qbytes 15 != 16/); - // int4 holds two values per byte - expect(() => - packQuantColSlice(new Uint8Array(16), noScales(1, 1, 4), 4, 4, 1, 4, 4, 0, 4), - ).toThrow(/qbytes 16 != 8/); - expect(() => - packQuantColSlice(new Uint8Array(16), new Float32Array(3), 8, 4, 1, 4, 4, 0, 4), - ).toThrow(/scales 3 != 4/); + { bits: 8 as const, k: 8, n: 12 }, + { bits: 4 as const, k: 12, n: 8 }, + { bits: 4 as const, k: 16, n: 16 }, + ])('reads back as the kernels read it: int$bits [$k, $n]', ({ bits, k, n }) => { + const half = 2 ** (bits - 1); + const ints = Int8Array.from({ length: k * n }, (_, i) => ((i * 37 + 5) % (2 * half)) - half); + const words = packQuantW(ints, k, n, bits); + expect(words).toHaveLength((k * n * bits) / 32); + const read = new Int8Array(k * n); + for (let unit = 0; unit < n / 4; unit++) { + for (let k4 = 0; k4 < k / 4; k4++) { + const tile = (unit * (k / 4) + k4) * (bits / 2); + for (let c = 0; c < 4; c++) { + const word = words[tile + columnWord(c, bits)]!; + for (let r = 0; r < 4; r++) { + const off = ((c * 4 * bits) % 32) + r * bits; + const raw = (word >>> off) & ((1 << bits) - 1); + read[(unit * 4 + c) * k + k4 * 4 + r] = raw >= half ? raw - 2 * half : raw; + } + } + } + } + expect([...read]).toEqual([...ints]); }); }); diff --git a/packages/runntime/test/zoo/minilm/model.gpu.test.ts b/packages/runntime/test/zoo/minilm/model.gpu.test.ts new file mode 100644 index 00000000..fa5f9b8f --- /dev/null +++ b/packages/runntime/test/zoo/minilm/model.gpu.test.ts @@ -0,0 +1,157 @@ +/** MiniLM loaded from a quantized checkpoint, through the same state dict + * hooks and embed core createEmbedder uses. The checkpoint has MiniLM's real + * widths, two layers, a small vocab and seeded random weights in HF names. */ +import { describe, expect, it, onTestFinished } from 'vitest'; +import { bufferSource, fromSafetensors } from '../../../src/core/weights/safetensors.ts'; +import { MINILM_L6 } from '../../../src/zoo/minilm/config.ts'; +import { createMinilmEmbedCore } from '../../../src/zoo/minilm/embedCore.ts'; +import { MinilmModel } from '../../../src/zoo/minilm/model.ts'; +import { expectClose, gpu, randomF32, requireFeature } from '../../gpu.ts'; +import { quantFile, quantizeWeight, safetensorsFile } from '../../core/weights/checkpoint.ts'; + +const { root } = await gpu(); + +/** Real widths; the vocab only sizes the float word embedding table. */ +const CFG = { ...MINILM_L6, layers: 2, vocab: 64 }; + +/** Half-width of the uniform random weights, a standard deviation of 0.029 + * near BERT's 0.02 init, so attention does not saturate and a small weight + * change moves the output smoothly. */ +const WEIGHT_SPREAD = 0.05; + +/** One text for the replay path, two for the packed eager path. */ +const TEXT = [2, 7, 1, 8, 28, 18, 28, 45, 9, 4, 52, 3]; +const BATCH = [ + [3, 14, 15, 9, 26], + [5, 35, 8, 9, 7, 9, 32, 38, 46], +]; + +type Tensors = Record; +/** A torch-layout Linear weight, [out, in]. */ +type LinearWeight = { data: Float32Array; shape: [number, number] }; + +/** A BertModel checkpoint in HF names, split into the Linear weights the + * quantized file quantizes and every other tensor. */ +function bertCheckpoint(): { linear: Record; other: Tensors } { + const { hidden, ffn, layers, vocab, maxPositions } = CFG; + let seed = 1; + const random = (n: number, center = 0) => + randomF32(n, seed++).map((v) => center + WEIGHT_SPREAD * v); + const other: Tensors = { + 'embeddings.word_embeddings.weight': { data: random(vocab * hidden), shape: [vocab, hidden] }, + 'embeddings.position_embeddings.weight': { + data: random(maxPositions * hidden), + shape: [maxPositions, hidden], + }, + 'embeddings.token_type_embeddings.weight': { data: random(2 * hidden), shape: [2, hidden] }, + 'embeddings.LayerNorm.weight': { data: random(hidden, 1), shape: [hidden] }, + 'embeddings.LayerNorm.bias': { data: random(hidden), shape: [hidden] }, + }; + const linear: Record = {}; + for (let i = 0; i < layers; i++) { + const hf = (name: string) => `encoder.layer.${i}.${name}`; + const dense = (name: string, out: number, inF: number) => { + linear[hf(`${name}.weight`)] = { data: random(out * inF), shape: [out, inF] }; + other[hf(`${name}.bias`)] = { data: random(out), shape: [out] }; + }; + const norm = (name: string) => { + other[hf(`${name}.weight`)] = { data: random(hidden, 1), shape: [hidden] }; + other[hf(`${name}.bias`)] = { data: random(hidden), shape: [hidden] }; + }; + for (const p of ['query', 'key', 'value']) dense(`attention.self.${p}`, hidden, hidden); + dense('attention.output.dense', hidden, hidden); + norm('attention.output.LayerNorm'); + dense('intermediate.dense', ffn, hidden); + dense('output.dense', hidden, ffn); + norm('output.LayerNorm'); + } + return { linear, other }; +} + +/** The model loaded from `file`, and its embeddings of TEXT and BATCH. */ +async function embed(file: Uint8Array, dtype: 'f32' | 'f16') { + const model = new MinilmModel(CFG); + onTestFinished(() => model.dispose()); + if (dtype === 'f16') model.half(root); + await model.loadStateDict(await fromSafetensors(bufferSource(file), { chunkBytes: 0 }), { + root, + }); + const core = createMinilmEmbedCore(model); + onTestFinished(() => core.dispose()); + return { model, embeddings: [await core.embedPooled(TEXT), await core.embedPooledBatch(BATCH)] }; +} + +const cosine = (a: Float32Array, b: Float32Array) => { + let dot = 0; + let aa = 0; + let bb = 0; + for (let i = 0; i < a.length; i++) { + dot += a[i]! * b[i]!; + aa += a[i]! ** 2; + bb += b[i]! ** 2; + } + return dot / Math.sqrt(aa * bb); +}; + +/** Tolerance against the same ints dequantized on the CPU and loaded as + * floats, which isolates packing and kernel error from quantization error. + * Measured worst case: 6e-7 in f32, where only the summation order differs, + * and one f16 step (2^-10) in f16, on outputs up to 1.93. f16 allows two + * steps at that size. */ +const KERNEL_TOLERANCE = { + f32: { atol: 5e-6, rtol: 0 }, + f16: { atol: 2 ** -9, rtol: 0 }, +}; + +/** Least cosine similarity to the model with the original float weights, + * set from the measured worst case across dtypes and group sizes: int8 + * 0.999993 (1 - cos = 7e-6), int4 0.99782 (2.2e-3). Each bound allows + * about 5 to 15 times the measured 1 - cos. */ +const LEAST_COSINE = { 8: 0.9999, 4: 0.99 }; + +describe('MiniLM from a quantized checkpoint', () => { + it.for( + (['f32', 'f16'] as const).flatMap((dtype) => + ([8, 4] as const).flatMap((bits) => + [32, 128].map((groupSize) => ({ dtype, bits, groupSize })), + ), + ), + )('$dtype int$bits, groupSize $groupSize', async ({ dtype, bits, groupSize }, ctx) => { + if (dtype === 'f16') requireFeature(ctx, root, 'shader-f16'); + const { linear, other } = bertCheckpoint(); + const quantized = Object.fromEntries( + Object.entries(linear).map(([name, { data, shape }]) => [ + name, + quantizeWeight(data, shape, bits, groupSize), + ]), + ); + // The same ints times the same f16 scales, as float weights. + const dequantized: Tensors = Object.fromEntries( + Object.entries(quantized).map(([name, { ints, scales, shape }]) => [ + name, + { data: Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / groupSize)]!), shape }, + ]), + ); + + const { model, embeddings: got } = await embed( + quantFile(bits, groupSize, quantized, other), + dtype, + ); + // Every Linear loaded quantized, so no layer took the float matmul. + const scaleNames = [...model.namedParameters()] + .map(([name]) => name) + .filter((name) => name.endsWith('weightScale')); + expect(scaleNames).toEqual( + Array.from({ length: CFG.layers }, (_, i) => + ['qkv', 'out', 'up', 'down'].map((m) => `layers.${i}.${m}.weightScale`), + ).flat(), + ); + + const reference = await embed(safetensorsFile({ ...other, ...dequantized }), dtype); + const float = await embed(safetensorsFile({ ...other, ...linear }), dtype); + for (let i = 0; i < got.length; i++) { + expectClose(got[i]!, reference.embeddings[i]!, KERNEL_TOLERANCE[dtype]); + expect(cosine(got[i]!, float.embeddings[i]!)).toBeGreaterThan(LEAST_COSINE[bits]); + } + }); +});