From 8d3c63969c97148fb768954b7bb312f9c91fa91e Mon Sep 17 00:00:00 2001 From: Kamil Kaczmarczyk <143009372+xxpopielxx@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:02:36 +0100 Subject: [PATCH 1/4] feat(core): take bits and groupSize from each tensor in runntime.quant version 2 A version 2 file sets bits by dtype and groupSize by scale shape per weight, so one file can mix them. Version 1 loads as before. --- .../runntime/src/core/weights/safetensors.ts | 38 +++++++++++---- .../test/core/weights/safetensors.test.ts | 46 ++++++++++++++++++- 2 files changed, 73 insertions(+), 11 deletions(-) diff --git a/packages/runntime/src/core/weights/safetensors.ts b/packages/runntime/src/core/weights/safetensors.ts index d23ecb45..3142c34f 100644 --- a/packages/runntime/src/core/weights/safetensors.ts +++ b/packages/runntime/src/core/weights/safetensors.ts @@ -443,8 +443,9 @@ function unpackInt4(bytes: Uint8Array): Int8Array { /** The int weight `name` paired with its F16 `scale`, checked against the * file's quantization setting. bits 8 needs I8; bits 4 needs U8, where U8 - * means packed bytes of two signed nibbles, not unsigned values. Any other - * pairing throws CHECKPOINT_MISMATCH. */ + * means packed bytes of two signed nibbles, not unsigned values. A setting + * without bits or groupSize takes them from the dtype and the scale's shape. + * Any other pairing throws CHECKPOINT_MISMATCH. */ function fileQuantTensor( name: string, info: SafeTensorInfo, @@ -455,14 +456,22 @@ function fileQuantTensor( const bad = (why: string): never => { throw new RunntimeError('CHECKPOINT_MISMATCH', `safetensors: quantized tensor ${name} ${why}`); }; - const { bits, groupSize } = quant; + const bits = quant.bits ?? (info.dtype === 'I8' ? 8 : 4); const dtype = bits === 8 ? 'I8' : 'U8'; if (info.dtype !== dtype) bad(`is ${info.dtype}, but int${bits} is stored as ${dtype}`); if (info.shape.length !== 2) bad(`needs a 2D shape, got [${info.shape.join(', ')}]`); const out = info.shape[0]!; const cols = bits === 8 ? info.shape[1]! : info.shape[1]! * 2; - if (cols % groupSize !== 0) bad(`has ${cols} columns, not a multiple of groupSize ${groupSize}`); if (!scale) return bad(`has no ${name}_scale`); + const groups = scale.shape.length === 2 ? scale.shape[1]! : 0; + const groupSize = quant.groupSize ?? cols / groups; + if (!Number.isInteger(groupSize) || groupSize <= 0) { + bad( + `needs an F16 ${name}_scale [${out}, groups] with groups dividing ${cols}, ` + + `got ${scale.dtype} [${scale.shape.join(', ')}]`, + ); + } + if (cols % groupSize !== 0) bad(`has ${cols} columns, not a multiple of groupSize ${groupSize}`); const scaleShape = [out, cols / groupSize]; if (scale.dtype !== 'F16' || scale.shape.join() !== scaleShape.join()) { bad( @@ -577,14 +586,17 @@ function bf16Tensor(bits: Uint16Array, shape: number[]): FileTensor { } /** Metadata key of a quantized file: a JSON string - * `{"version":1,"bits":8|4,"groupSize":N,"scheme":"symmetric"}`. */ + * `{"version":1,"bits":8|4,"groupSize":N,"scheme":"symmetric"}`, or + * `{"version":2,"scheme":"symmetric"}`, where each int weight's bits come + * from its dtype and its groupSize from its scale's shape. */ const QUANT_METADATA_KEY = 'runntime.quant'; /** How a quantized file stores its int weights: `bits` per value, one f16 - * scale per `groupSize` values along each row. */ + * scale per `groupSize` values along each row. Empty in version 2, where + * each tensor sets both. */ interface QuantConfig { - bits: 4 | 8; - groupSize: number; + bits?: 4 | 8; + groupSize?: number; } /** The file's quantization setting, or undefined for a float file. Throws @@ -609,12 +621,18 @@ function quantConfigOf(metadata: Record): QuantConfig | undefine return bad('is not a JSON object'); } const { version, bits, groupSize, scheme } = json as Record; - if (version !== 1) return bad('has a version other than 1'); + if (version !== 1 && version !== 2) return bad('has a version other than 1 or 2'); + if (scheme !== 'symmetric') return bad("needs scheme 'symmetric'"); + if (version === 2) { + if (bits !== undefined || groupSize !== undefined) { + return bad('sets bits or groupSize, which version 2 takes from each tensor'); + } + return {}; + } if (bits !== 4 && bits !== 8) return bad('needs bits 4 or 8'); if (typeof groupSize !== 'number' || !Number.isSafeInteger(groupSize) || groupSize <= 0) { return bad('needs a positive integer groupSize'); } - if (scheme !== 'symmetric') return bad("needs scheme 'symmetric'"); return { bits, groupSize }; } diff --git a/packages/runntime/test/core/weights/safetensors.test.ts b/packages/runntime/test/core/weights/safetensors.test.ts index d716c15a..71060f11 100644 --- a/packages/runntime/test/core/weights/safetensors.test.ts +++ b/packages/runntime/test/core/weights/safetensors.test.ts @@ -375,7 +375,12 @@ describe('fromSafetensors', () => { it.each([ { name: 'text that is not JSON', metadata: { 'runntime.quant': '{bits' }, error: /JSON/ }, { name: 'a JSON array', metadata: quant([1, 8]), error: /not a JSON object/ }, - { name: 'version 2', metadata: quant({ ...valid, version: 2 }), error: /version/ }, + { name: 'version 3', metadata: quant({ ...valid, version: 3 }), error: /version/ }, + { + name: 'version 2 with bits and groupSize', + metadata: quant({ ...valid, version: 2 }), + error: /version 2 takes from each tensor/, + }, { name: 'no version', metadata: quant({ ...valid, version: undefined }), error: /version/ }, { name: 'bits 2', metadata: quant({ ...valid, bits: 2 }), error: /bits 4 or 8/ }, { name: "bits '8'", metadata: quant({ ...valid, bits: '8' }), error: /bits 4 or 8/ }, @@ -415,6 +420,7 @@ describe('fromSafetensors', () => { const int4Setting = { 'runntime.quant': '{"version":1,"bits":4,"groupSize":2,"scheme":"symmetric"}', }; + const v2Setting = { 'runntime.quant': '{"version":2,"scheme":"symmetric"}' }; const i8Bytes = (values: number[]) => new Uint8Array(new Int8Array(values).buffer); // f16 bits of 1, 0.5, 2 and -1: one scale per 2 columns of a [2, 4] weight. const scale: TensorSpec = { @@ -471,6 +477,44 @@ describe('fromSafetensors', () => { expect(sd.totalBytes()).toBe(4 + 8); }); + it('takes bits and groupSize from each tensor in a version 2 file', async () => { + const sd = await open( + { + 'a.weight': { dtype: 'I8', shape: [2, 4], bytes: new Uint8Array(8) }, + 'a.weight_scale': scale, + 'b.weight': { dtype: 'U8', shape: [2, 4], bytes: new Uint8Array(8) }, + 'b.weight_scale': { dtype: 'F16', shape: [2, 2], bytes: new Uint8Array(8) }, + 'c.weight': { dtype: 'I8', shape: [2, 4], bytes: new Uint8Array(8) }, + 'c.weight_scale': { dtype: 'F16', shape: [2, 1], bytes: new Uint8Array(4) }, + }, + v2Setting, + ); + const meta = (name: string) => { + const t = sd.tensors.get(name)!; + if (t.kind !== 'quant') throw new Error('expected a quantized tensor'); + return { shape: t.shape, bits: t.bits, groupSize: t.groupSize }; + }; + expect(meta('a.weight')).toEqual({ shape: [2, 4], bits: 8, groupSize: 2 }); + expect(meta('b.weight')).toEqual({ shape: [2, 8], bits: 4, groupSize: 4 }); + expect(meta('c.weight')).toEqual({ shape: [2, 4], bits: 8, groupSize: 4 }); + }); + + it('rejects version 2 scales that do not split the columns evenly', async () => { + const opening = open( + { + w: { dtype: 'I8', shape: [2, 4], bytes: new Uint8Array(8) }, + w_scale: { dtype: 'F16', shape: [2, 3], bytes: new Uint8Array(12) }, + }, + v2Setting, + ); + await expect(opening).rejects.toMatchObject({ + code: 'CHECKPOINT_MISMATCH', + message: expect.stringMatching( + /needs an F16 w_scale \[2, groups\] with groups dividing 4, got F16 \[2, 3\]/, + ), + }); + }); + it('pairs a weight whose scales come first in the file', async () => { const sd = await open({ 'fc.weight_scale': scale, From 61187aa889578f89c6a2c50dc7fb790f839d0ea1 Mon Sep 17 00:00:00 2001 From: Kamil Kaczmarczyk <143009372+xxpopielxx@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:03:34 +0100 Subject: [PATCH 2/4] feat(core): run Conv1d and ConvTranspose1d on quantized weights conv1d takes a quantW weight and runs on matmulQuantTiled, which gains the loader slot and a scale period so a group of input channels shares a scale in every tap. Linear, Conv1d and ConvTranspose1d share the loader. --- packages/runntime/bench/kernels/conv.ab.ts | 93 +++++- .../runntime/src/core/gpu/dispatch/conv.ts | 79 +++-- .../src/core/graph/ops/conv/conv1d.ts | 51 +++- .../runntime/src/core/graph/ops/matmul.ts | 18 +- .../runntime/src/core/graph/ops/shared.ts | 19 ++ .../runntime/src/core/kernels/conv/conv1d.ts | 10 +- .../core/kernels/matmul/matmulQuantTiled.ts | 60 +++- .../core/kernels/matmul/matmulTiledVec4.ts | 15 +- packages/runntime/src/core/nn/layers.ts | 269 +++++++++++++++--- .../runntime/src/core/nn/loadStateDict.ts | 8 +- packages/runntime/src/core/nn/module.ts | 9 +- .../test/core/graph/ops/conv/conv1d.test.ts | 58 +++- .../test/core/kernels/conv/conv1d.gpu.test.ts | 154 ++++++++++ .../test/core/kernels/matmul/quantW.ts | 18 +- .../runntime/test/core/nn/layers.gpu.test.ts | 163 ++++++++++- packages/runntime/test/core/nn/layers.test.ts | 99 +++++++ .../test/core/nn/loadStateDict.test.ts | 4 +- 17 files changed, 1007 insertions(+), 120 deletions(-) diff --git a/packages/runntime/bench/kernels/conv.ab.ts b/packages/runntime/bench/kernels/conv.ab.ts index 0edf311e..116fe558 100644 --- a/packages/runntime/bench/kernels/conv.ab.ts +++ b/packages/runntime/bench/kernels/conv.ab.ts @@ -4,7 +4,7 @@ * DepthART and Moonshine. */ import { abSuite, type AbCase, type Version } from '../ab.ts'; import type { Value } from '../../src/core/index.ts'; -import { chw, convWeight, randHwc4, randMat } from './inputs.ts'; +import { chw, convWeight, quantWeight, randHwc4, randMat } from './inputs.ts'; interface Conv { cIn: number; @@ -91,6 +91,43 @@ function conv1d( }; } +/** conv1d with a quantW weight [k·cIn, cOut] and f32 scales [cIn / group, + * cOut], one per group of input channels in every tap. */ +function conv1dQuant( + name: string, + tIn: number, + cIn: number, + cOut: number, + k: number, + bits: 8 | 4, + group: number, + geometry: { padLeft?: number; padRight?: number; dilation?: number } = {}, + dtype: 'f32' | 'f16' = 'f32', +): AbCase { + return { + name, + needs: dtype === 'f16' ? ['shader-f16'] : [], + build: (version) => { + const x = randMat(version, tIn, cIn, dtype); + // Groups of k·group rows give as many scales as the conv reads; their + // values do not matter for timing. + const { w, scales } = quantWeight(version, 1, k * cIn, cOut, bits, k * group); + const call = () => + version.lib.conv1d(x, w, { + kernelSize: k, + stride: 1, + ...geometry, + scales, + bits, + groupSize: group, + }); + // Built once here, so a base without quantW conv1d skips the case. + call(); + return call; + }, + }; +} + /** GLU then a causal depthwise conv over time: x [T, 2C], taps [K, C]. */ function gluConv1d(name: string, t: number, c: number, k: number, dtype: 'f32' | 'f16'): AbCase { return { @@ -294,6 +331,60 @@ abSuite('conv', [ padRight: 1, }), + // conv1d with a quantW weight: matmulQuantTiled, groups of 8 channels. + conv1dQuant('conv1d int8 same, Higgs conv_in 256→1024 k7, 10 s', 250, 256, 1024, 7, 8, 8, { + padLeft: 3, + padRight: 3, + }), + conv1dQuant('conv1d int8 same, Higgs residual 64ch k7, 10 s', 80000, 64, 64, 7, 8, 8, { + padLeft: 3, + padRight: 3, + }), + conv1dQuant( + 'conv1d int8 same f16, Higgs residual 64ch k7, 10 s', + 80000, + 64, + 64, + 7, + 8, + 8, + { padLeft: 3, padRight: 3 }, + 'f16', + ), + conv1dQuant('conv1d int4 same, Higgs residual 64ch k7, 10 s', 80000, 64, 64, 7, 4, 8, { + padLeft: 3, + padRight: 3, + }), + conv1dQuant('conv1d int8 dilated, Higgs residual 64ch k7 d9, 10 s', 80000, 64, 64, 7, 8, 8, { + padLeft: 27, + padRight: 27, + dilation: 9, + }), + // Unpadded: the strided loader. + conv1dQuant('conv1d int8 1×1, Higgs residual 64ch, 10 s', 80000, 64, 64, 1, 8, 8), + // One output channel pads to 4 columns; the float conv takes the direct kernel. + conv1d('conv1d same, Higgs output 32→1 k7, 10 s', 240000, 32, 1, 7, 1, undefined, { + padLeft: 3, + padRight: 3, + }), + conv1dQuant('conv1d int8 same, Higgs output 32→1 k7 as 4 columns, 10 s', 240000, 32, 4, 7, 8, 8, { + padLeft: 3, + padRight: 3, + }), + conv1dQuant( + 'conv1d int8, Higgs upsampling 1024→8·512 as 2 taps, 10 s', + 250, + 1024, + 4096, + 2, + 8, + 8, + { + padLeft: 1, + padRight: 1, + }, + ), + // gluConv1d: its own kernel. gluConv1d('gluConv1d, EmbeddingGemma 2 audio 30 s, 1024 channels k5', 750, 1024, 5, 'f32'), gluConv1d('gluConv1d f16, EmbeddingGemma 2 audio 30 s', 750, 1024, 5, 'f16'), diff --git a/packages/runntime/src/core/gpu/dispatch/conv.ts b/packages/runntime/src/core/gpu/dispatch/conv.ts index facfacb0..77a9d1e5 100644 --- a/packages/runntime/src/core/gpu/dispatch/conv.ts +++ b/packages/runntime/src/core/gpu/dispatch/conv.ts @@ -3,6 +3,11 @@ import { elemFor } from '../../kernels/elem.ts'; import { conv1dHandle, conv1dLoadA, createConv1dPipeline } from '../../kernels/conv/conv1d.ts'; import { createGluConv1dPipeline, gluConv1dHandle } from '../../kernels/conv/gluConv1d.ts'; +import { + createMatmulQuantTiledPipeline, + matmulQuantTiledHandle, + matmulQuantTiledLayout, +} from '../../kernels/matmul/matmulQuantTiled.ts'; import { createMatmulTiledVec4Pipeline, matmulTiledVec4Eligible, @@ -44,24 +49,19 @@ import { defineSpec, floatDtype, narrow, narrowFloat } from './spec.ts'; export const conv1dSpec = defineSpec({ attrs: (p) => { - const [kernel, stride, padLeft, padRight, hasBias, act, dilation] = p.attrs as [ - number, - number, - number, - number, - number, - number, - number, - ]; - return { kernel, stride, padLeft, padRight, hasBias, act, dilation }; + const [kernel, stride, padLeft, padRight, hasBias, act, dilation, bits, groupSize] = + p.attrs as [number, number, number, number, number, number, number, number, number]; + return { kernel, stride, padLeft, padRight, hasBias, act, dilation, bits, groupSize }; }, cfg: (node, attrs) => { const cIn = node.pending!.inputs[0]!.shape.dims![1]!; const cOut = node.shape.dims![1]!; - // An output frame reads kernel·cIn inputs, so the conv runs as the tiled - // matmul. Unpadded and undilated they are contiguous; otherwise - // conv1dLoadA maps each tap to its input frame. - const gemm = cIn % 4 === 0 && matmulTiledVec4Eligible(attrs.kernel * cIn, cOut); + // An output frame reads kernel·cIn inputs, so the conv runs as a tiled + // matmul: matmulQuantTiled for a quantW weight, matmulTiledVec4 + // otherwise. Unpadded and undilated, those inputs are contiguous; + // otherwise conv1dLoadA finds each tap's input frame. + const gemm = + attrs.bits > 0 || (cIn % 4 === 0 && matmulTiledVec4Eligible(attrs.kernel * cIn, cOut)); const mapped = attrs.padLeft > 0 || attrs.padRight > 0 || attrs.dilation > 1; return { kernel: attrs.kernel, @@ -73,29 +73,70 @@ export const conv1dSpec = defineSpec({ hasBias: attrs.hasBias, act: attrs.act, gemm: gemm ? (mapped ? 2 : 1) : 0, + bits: attrs.bits, + groupSize: attrs.groupSize, }; }, pipeline: (root, dtype, cfg) => - cfg.gemm - ? createMatmulTiledVec4Pipeline( + cfg.bits > 0 + ? createMatmulQuantTiledPipeline( root, { k: cfg.kernel * cfg.cIn, n: cfg.cOut, + bits: cfg.bits, + groupSize: cfg.groupSize, hasBias: cfg.hasBias, hasAdd: 0, act: cfg.act, aStride: cfg.stride * cfg.cIn, + period: cfg.cIn, }, elemFor(dtype), - cfg.gemm === 2 ? conv1dLoadA(elemFor(dtype), cfg) : undefined, + cfg.gemm === 2 + ? conv1dLoadA(elemFor(dtype), cfg, matmulQuantTiledLayout(elemFor(dtype))) + : undefined, ) - : createConv1dPipeline(root, cfg, elemFor(dtype)), + : cfg.gemm + ? createMatmulTiledVec4Pipeline( + root, + { + k: cfg.kernel * cfg.cIn, + n: cfg.cOut, + hasBias: cfg.hasBias, + hasAdd: 0, + act: cfg.act, + aStride: cfg.stride * cfg.cIn, + }, + elemFor(dtype), + cfg.gemm === 2 ? conv1dLoadA(elemFor(dtype), cfg) : undefined, + ) + : createConv1dPipeline(root, cfg, elemFor(dtype)), encode: ({ node, attrs, cfg, pipeline, inputs, out, ctx }) => { const dtype = node.shape.dtype; const tIn = node.pending!.inputs[0]!.shape.dims![0]!; const [tOut, cOut] = node.shape.dims as [number, number]; - const bias = attrs.hasBias === 1 ? narrowFloat(inputs[2]!, dtype) : undefined; + const quant = cfg.bits > 0; + const bias = attrs.hasBias === 1 ? narrowFloat(inputs[quant ? 3 : 2]!, dtype) : undefined; + if (quant) { + return [ + matmulQuantTiledHandle( + ctx.root, + pipeline, + tOut, + cOut, + { + a: narrowFloat(inputs[0]!, dtype), + w: narrow(inputs[1]!, 'u32'), + scales: narrow(inputs[2]!, 'f32'), + bias, + out: narrowFloat(out, dtype), + }, + elemFor(dtype), + tIn, + ), + ]; + } if (cfg.gemm) { return [ matmulTiledVec4Handle( diff --git a/packages/runntime/src/core/graph/ops/conv/conv1d.ts b/packages/runntime/src/core/graph/ops/conv/conv1d.ts index 8244fe4e..29a719c8 100644 --- a/packages/runntime/src/core/graph/ops/conv/conv1d.ts +++ b/packages/runntime/src/core/graph/ops/conv/conv1d.ts @@ -1,5 +1,9 @@ import { pending, Value } from '../../value.ts'; -import { ACT_CODE, FLOAT_DTYPES, want2d, type SlotAct } from '../shared.ts'; +import { + MATMUL_QUANT_TILED_BK, + matmulQuantTiledEligible, +} from '../../../kernels/matmul/matmulQuantTiled.ts'; +import { ACT_CODE, FLOAT_DTYPES, want2d, wantQuantW, type SlotAct } from '../shared.ts'; /** Strided, dilated 1-D convolution over time-major [T_in, C_in], with w * tap-major [kernelSize·C_in, C_out]. Tap j reads frame j·dilation of the @@ -10,7 +14,11 @@ import { ACT_CODE, FLOAT_DTYPES, want2d, type SlotAct } from '../shared.ts'; * Passing both is an error, and every pad must stay below the span — an * output frame computed from padding alone is a model bug. * - * `bias` and `activation` fold into the epilogue as act(conv + bias). */ + * `bias` and `activation` fold into the epilogue as act(conv + bias). + * + * A quantW w takes f32 `scales` [C_in / groupSize, C_out], `bits` and + * `groupSize`: each group of groupSize input channels has one scale per + * output channel, shared by every tap. */ export function conv1d( x: Value, w: Value, @@ -23,17 +31,44 @@ export function conv1d( padRight?: number; bias?: Value; activation?: SlotAct; + scales?: Value; + bits?: 8 | 4; + groupSize?: number; }, ): Value { - const { kernelSize, stride, padding, bias, activation } = opts; + const { kernelSize, stride, padding, bias, activation, scales, bits, groupSize } = opts; const dilation = opts.dilation ?? 1; if (padding !== undefined && (opts.padLeft !== undefined || opts.padRight !== undefined)) { throw new Error('conv1d: padding excludes padLeft/padRight — give one form or the other'); } const [tIn, cIn, dtype] = want2d('conv1d', x, FLOAT_DTYPES); - const [wRows, cOut, wDtype] = want2d('conv1d.w', w, FLOAT_DTYPES); - if (wDtype !== dtype) { - throw new Error(`conv1d: weight dtype ${wDtype} must match the input's ${dtype}`); + const quant = w.shape.dtype === 'quantW'; + let wRows: number; + let cOut: number; + if (quant) { + if (!scales || !bits || !groupSize) { + throw new Error('conv1d: quantW weights need scales, bits and groupSize'); + } + [wRows, cOut] = wantQuantW('conv1d', w, scales, bits); + if (!matmulQuantTiledEligible(kernelSize * cIn, cOut, groupSize, cIn)) { + throw new Error( + `conv1d: a quantW weight needs groupSize a multiple of 4, C_in a multiple of groupSize, kernelSize·C_in a multiple of ${MATMUL_QUANT_TILED_BK} and C_out of 4, got C_in ${cIn}, C_out ${cOut}, groupSize ${groupSize}`, + ); + } + if (scales.shape.elems !== (cIn / groupSize) * cOut) { + throw new Error( + `conv1d: scales must be f32 [C_in/groupSize, C_out] = [${cIn / groupSize}, ${cOut}]`, + ); + } + } else { + if (scales || bits || groupSize) { + throw new Error('conv1d: scales/bits/groupSize apply to quantW weights only'); + } + let wDtype: string; + [wRows, cOut, wDtype] = want2d('conv1d.w', w, FLOAT_DTYPES); + if (wDtype !== dtype) { + throw new Error(`conv1d: weight dtype ${wDtype} must match the input's ${dtype}`); + } } if (!Number.isInteger(kernelSize) || kernelSize < 1 || !Number.isInteger(stride) || stride < 1) { throw new Error( @@ -86,9 +121,9 @@ export function conv1d( return pending( { elems: tOut * cOut, dtype, dims: [tOut, cOut] }, 'conv1d', - [x, w, ...(bias ? [bias] : [])], + [x, w, ...(scales ? [scales] : []), ...(bias ? [bias] : [])], undefined, - [kernelSize, stride, padLeft, padRight, bias ? 1 : 0, act, dilation], + [kernelSize, stride, padLeft, padRight, bias ? 1 : 0, act, dilation, bits ?? 0, groupSize ?? 0], ); } diff --git a/packages/runntime/src/core/graph/ops/matmul.ts b/packages/runntime/src/core/graph/ops/matmul.ts index 69e408b4..977df8ce 100644 --- a/packages/runntime/src/core/graph/ops/matmul.ts +++ b/packages/runntime/src/core/graph/ops/matmul.ts @@ -8,7 +8,7 @@ import { MATMUL_QUANT_TILED_BK, matmulQuantTiledEligible, } from '../../kernels/matmul/matmulQuantTiled.ts'; -import { ACT_CODE, FLOAT_DTYPES, want2d, type SlotAct } from './shared.ts'; +import { ACT_CODE, FLOAT_DTYPES, want2d, wantQuantW, type SlotAct } from './shared.ts'; /** Options for matmul. `bias` and `addend` are fused epilogue adds; on a * quantW weight they need the whole weight, no baseRow or scaleBase view. @@ -114,22 +114,8 @@ function matmulQuantW( }, ): Value { const [m, k, dtype] = want2d('matmul', a, FLOAT_DTYPES); - const wd = w.shape.dims; - if (!wd || wd.length !== 2) { - throw new Error('matmul: quantW weight must be 2D'); - } - if (scales.shape.dtype !== 'f32') throw new Error('matmul: scales must be f32'); - const [r, n] = [wd[0]!, wd[1]!]; const { baseRow, scaleBase, bits, groupSize } = opts; - if (bits !== 8 && bits !== 4) throw new Error(`matmul: bits must be 8 or 4, got ${bits}`); - const perU32 = bits === 8 ? 4 : 8; - if (n % perU32 !== 0) - throw new Error(`matmul: cols ${n} must be divisible by ${perU32} for int${bits} packing`); - if (w.shape.elems !== (r * n) / perU32) { - throw new Error( - `matmul: quantW weight elems ${w.shape.elems} != rows*cols/${perU32} = ${(r * n) / perU32} words`, - ); - } + const [r, n] = wantQuantW('matmul', w, scales, bits); if (groupSize <= 0 || k % groupSize !== 0) throw new Error(`matmul: groupSize ${groupSize} must divide K ${k}`); if (k % 4 !== 0 || groupSize % 4 !== 0) { diff --git a/packages/runntime/src/core/graph/ops/shared.ts b/packages/runntime/src/core/graph/ops/shared.ts index 0ccef1a0..682a3503 100644 --- a/packages/runntime/src/core/graph/ops/shared.ts +++ b/packages/runntime/src/core/graph/ops/shared.ts @@ -69,6 +69,25 @@ export function want2d( return [dims[0]!, dims[1]!, dtype]; } +/** The [rows, cols] of a 2D quantW weight with its f32 `scales`, checked + * against `bits`: int8 packs 4 columns to a word, int4 8. */ +export function wantQuantW(name: string, w: Value, scales: Value, bits: number): [number, number] { + const wd = w.shape.dims; + if (!wd || wd.length !== 2) throw new Error(`${name}: quantW weight must be 2D`); + if (scales.shape.dtype !== 'f32') throw new Error(`${name}: scales must be f32`); + const [r, n] = [wd[0]!, wd[1]!]; + if (bits !== 8 && bits !== 4) throw new Error(`${name}: bits must be 8 or 4, got ${bits}`); + const perU32 = bits === 8 ? 4 : 8; + if (n % perU32 !== 0) + throw new Error(`${name}: cols ${n} must be divisible by ${perU32} for int${bits} packing`); + if (w.shape.elems !== (r * n) / perU32) { + throw new Error( + `${name}: quantW weight elems ${w.shape.elems} != rows*cols/${perU32} = ${(r * n) / perU32} words`, + ); + } + return [r, n]; +} + /** wantND flattened to [c, h, w, dtype]. */ export function want3d( name: string, diff --git a/packages/runntime/src/core/kernels/conv/conv1d.ts b/packages/runntime/src/core/kernels/conv/conv1d.ts index 7576c982..b6573458 100644 --- a/packages/runntime/src/core/kernels/conv/conv1d.ts +++ b/packages/runntime/src/core/kernels/conv/conv1d.ts @@ -10,7 +10,7 @@ import { } from '../common.ts'; import { activationFor, actSlot } from '../activations.ts'; import { type Elem, F32_ELEM } from '../elem.ts'; -import { type LoadA, matmulTiledVec4Layout } from '../matmul/matmulTiledVec4.ts'; +import { type ASource, type LoadA, matmulTiledVec4Layout } from '../matmul/matmulTiledVec4.ts'; /** Direct strided, dilated 1-D convolution, time-major. * @@ -219,14 +219,14 @@ export function conv1dHandle( ); } -/** Runs a padded or dilated conv1d on the tiled matmul: row i of a is output +/** Runs a padded or dilated conv1d on a tiled matmul: row i of a is output * frame i's window over the [aRows, cIn] input, and tap j of it reads frame * i·stride + j·dilation − padLeft, zero outside the input. */ export function conv1dLoadA( elem: Elem, geom: { cIn: number; stride: number; dilation: number; padLeft: number }, + source: ASource = matmulTiledVec4Layout(elem), ): LoadA { - const layout = matmulTiledVec4Layout(elem); const storeVec4 = elem.vec4; const cIn4 = geom.cIn / 4; const { stride, dilation, padLeft } = geom; @@ -235,8 +235,8 @@ export function conv1dLoadA( const tap = std.intdiv(k4, cIn4); const pos = d.i32(row * stride + tap * dilation) - padLeft; let v = storeVec4(); - if (pos >= 0 && pos < d.i32(layout.$.dims.aRows)) { - v = storeVec4(layout.$.a[d.u32(pos) * cIn4 + (k4 % cIn4)]!); + if (pos >= 0 && pos < d.i32(source.$.dims.aRows)) { + v = storeVec4(source.$.a[d.u32(pos) * cIn4 + (k4 % cIn4)]!); } return v; }; diff --git a/packages/runntime/src/core/kernels/matmul/matmulQuantTiled.ts b/packages/runntime/src/core/kernels/matmul/matmulQuantTiled.ts index ff463f46..6c058a69 100644 --- a/packages/runntime/src/core/kernels/matmul/matmulQuantTiled.ts +++ b/packages/runntime/src/core/kernels/matmul/matmulQuantTiled.ts @@ -3,7 +3,13 @@ * f32 group scales. a, bias, addend and out are f16 or f32; sums are f32. * * Each step unpacks a slice of W into workgroup memory as floats, so the - * inner loop is a plain float matmul with matmulTiledVec4's tiling. */ + * inner loop is a plain float matmul with matmulTiledVec4's tiling. A + * loader reads a, so conv1d runs on this kernel too. + * + * Row k of W uses scale row (k mod period) / groupSize. For a Linear, period + * is K. For a conv, K holds the input channels of each tap in turn and + * period is the input channel count, so a group of channels has one scale + * for all taps. */ import tgpu, { d, std } from 'typegpu'; import type { TgpuRoot } from 'typegpu'; @@ -11,6 +17,7 @@ import { type KernelHandle, makeHandle } from '../common.ts'; import { cachedBindGroup, cachedUniform } from '../../gpu/dispatchCache.ts'; import { activationFor, actSlot } from '../activations.ts'; import { type Elem, F32_ELEM } from '../elem.ts'; +import { type LoadA, stridedLoadA } from './matmulTiledVec4.ts'; import { columnWord, deqRow, @@ -18,7 +25,7 @@ import { type MatmulQuantCfg, } from '../quantCommon.ts'; -const Dims = d.struct({ m: d.u32 }); +const Dims = d.struct({ m: d.u32, aRows: d.u32 }); function makeLayout(elem: Elem) { return tgpu.bindGroupLayout({ @@ -42,6 +49,21 @@ function layoutFor(elem: Elem): ReturnType { return l; } +const loadASlot = tgpu.slot(); + +/** The layout every pipeline of this kernel binds, so a loader can read a and + * aRows. */ +export function matmulQuantTiledLayout(elem: Elem): ReturnType { + return layoutFor(elem); +} + +export interface MatmulQuantTiledCfg extends MatmulQuantCfg { + /** Distance between rows of a, in values. K when omitted. */ + aStride?: number; + /** Rows of W after which the scale rows start again. K when omitted. */ + period?: number; +} + // Each workgroup computes a BM×BN tile of out, each thread TM rows × TN // groups of 4 columns. const BM = 32; @@ -53,13 +75,20 @@ const TN = 1; export const MATMUL_QUANT_TILED_BK = 8; const BK = MATMUL_QUANT_TILED_BK; -/** Whether a [M,K]·[K,N] quantW matmul can use this kernel. */ -export function matmulQuantTiledEligible(k: number, n: number, groupSize: number): boolean { - return k % BK === 0 && n % 4 === 0 && groupSize % 4 === 0 && k % groupSize === 0; +/** Whether a [M,K]·[K,N] quantW matmul with scale period `period` can use + * this kernel. */ +export function matmulQuantTiledEligible( + k: number, + n: number, + groupSize: number, + period = k, +): boolean { + return k % BK === 0 && n % 4 === 0 && groupSize % 4 === 0 && period % groupSize === 0; } -function makeMatmulQuantTiledKernel(cfg: MatmulQuantCfg, elem: Elem) { +function makeMatmulQuantTiledKernel(cfg: MatmulQuantTiledCfg, elem: Elem) { const { k, n, bits, groupSize, hasBias, hasAdd } = cfg; + const period = cfg.period ?? k; const layout = layoutFor(elem); const storeVec4 = elem.vec4; const bk4 = BK / 4; @@ -99,7 +128,7 @@ function makeMatmulQuantTiledKernel(cfg: MatmulQuantCfg, elem: Elem) { const am = mBase + std.intdiv(e, bk4); let v = storeVec4(); if (am < m) { - v = storeVec4(layout.$.a[am * k4 + t * bk4 + (e % bk4)]!); + v = storeVec4(loadASlot.$(am, t * bk4 + (e % bk4))); } tileA.$[e] = storeVec4(v); } @@ -120,7 +149,8 @@ function makeMatmulQuantTiledKernel(cfg: MatmulQuantCfg, elem: Elem) { bk % 4, bits, ); - wv = storeVec4(q * layout.$.scales[std.intdiv(bk, groupSize) * n4 + bc4]!); + const row = std.intdiv(bk % period, groupSize); + wv = storeVec4(q * layout.$.scales[row * n4 + bc4]!); } tileB.$[e] = storeVec4(wv); } @@ -170,18 +200,23 @@ function makeMatmulQuantTiledKernel(cfg: MatmulQuantCfg, elem: Elem) { }); } -/** One pipeline per dtype, K, N, bits, group size, epilogue and activation. */ +/** One pipeline per dtype, K, N, bits, group size, scale period, loader, + * epilogue and activation. `loadA` reads a; by default it reads rows + * aStride apart. */ export function createMatmulQuantTiledPipeline( root: TgpuRoot, - cfg: MatmulQuantCfg, + cfg: MatmulQuantTiledCfg, elem: Elem = F32_ELEM, + loadA: LoadA = stridedLoadA(elem, cfg.aStride ?? cfg.k, layoutFor(elem)), ) { return root .with(actSlot, activationFor(cfg.act)) + .with(loadASlot, loadA) .createComputePipeline({ compute: makeMatmulQuantTiledKernel(cfg, elem) }); } -/** One workgroup per 32×32 output tile. */ +/** One workgroup per 32×32 output tile. aRows is how many rows a's buffer + * holds, for loaders that bound-check it. */ export function matmulQuantTiledHandle( root: TgpuRoot, pipeline: ReturnType, @@ -189,8 +224,9 @@ export function matmulQuantTiledHandle( n: number, buffers: MatmulQuantBuffers, elem: Elem = F32_ELEM, + aRows = m, ): KernelHandle { - const dims = cachedUniform(root, Dims, { m }); + const dims = cachedUniform(root, Dims, { m, aRows }); // Absent epilogue operands alias `a`; those pipelines never read them. const bindGroup = cachedBindGroup(root, layoutFor(elem), { ...buffers, diff --git a/packages/runntime/src/core/kernels/matmul/matmulTiledVec4.ts b/packages/runntime/src/core/kernels/matmul/matmulTiledVec4.ts index 92b1739d..7c49402e 100644 --- a/packages/runntime/src/core/kernels/matmul/matmulTiledVec4.ts +++ b/packages/runntime/src/core/kernels/matmul/matmulTiledVec4.ts @@ -71,6 +71,12 @@ export interface MatmulTiledVec4Cfg { aStride?: number; } +/** What a loader reads: a, and how many rows a's buffer holds. Every kernel + * that takes a loader binds both. */ +export interface ASource { + readonly $: { readonly a: d.v4f[]; readonly dims: { readonly aRows: number } }; +} + /** The layout every pipeline of this kernel binds, so a loader can read a and * aRows. */ export function matmulTiledVec4Layout(elem: Elem): ReturnType { @@ -78,13 +84,16 @@ export function matmulTiledVec4Layout(elem: Elem): ReturnType } /** Reads row `row` of a at aStride. */ -function stridedLoadA(elem: Elem, aStride: number): LoadA { - const layout = layoutFor(elem); +export function stridedLoadA( + elem: Elem, + aStride: number, + source: ASource = layoutFor(elem), +): LoadA { const storeVec4 = elem.vec4; const aStride4 = aStride / 4; return (row: number, k4: number) => { 'use gpu'; - return storeVec4(layout.$.a[row * aStride4 + k4]!); + return storeVec4(source.$.a[row * aStride4 + k4]!); }; } diff --git a/packages/runntime/src/core/nn/layers.ts b/packages/runntime/src/core/nn/layers.ts index 1f0b0659..32b894a9 100644 --- a/packages/runntime/src/core/nn/layers.ts +++ b/packages/runntime/src/core/nn/layers.ts @@ -104,6 +104,53 @@ export class Embedding extends Module<[readonly number[] | Value], Value> { } } +/** Turns `weight` into quantW words [k, n], packed from `ints` with one row + * of k per output, and puts its f32 scales [groups, n] right after it as + * `.weightScale`, so both bind in file order. The words carry t's + * whole download size. Returns the scale parameter. */ +function loadQuantWeight( + sd: LazyStateDict, + prefix: string, + weight: Parameter, + t: QuantTensor, + shape: { k: number; n: number; groups: number }, + ints: () => Promise, + scales: () => Promise, +): Parameter { + const { k, n, groups } = shape; + const at = (name: string) => (prefix ? `${prefix}.${name}` : name); + weight.retype('quantW', (k * n * t.bits) / 32, [k, n]); + const scale = new Parameter( + { elems: groups * n, dtype: 'f32', dims: [groups, n] }, + undefined, + true, + ); + const words = derivedTensor( + [k, n], + async () => packQuantW(await ints(), k, n, t.bits), + t.byteLength, + ); + const entries = [...sd.tensors]; + sd.tensors.clear(); + for (const [key, tensor] of entries) { + sd.tensors.set(key, key === at('weight') ? words : tensor); + if (key === at('weight')) sd.tensors.set(at('weightScale'), derivedTensor([groups, n], scales)); + } + return scale; +} + +/** `t`'s scales, torch's [out, groups], as the kernels' [groups, n] in f32. + * Columns past out stay 0. */ +async function scaleColumns(t: QuantTensor, groups: number, n: number): Promise { + const data = await t.scales(); + const f32 = data instanceof Float32Array ? data : f16ToF32(data); + const out = new Float32Array(groups * n); + for (let o = 0; o < t.shape[0]!; o++) { + for (let g = 0; g < groups; g++) out[g * n + o] = f32[o * groups + g]!; + } + return out; +} + /** y = x @ weight + bias. weight is stored [in, out]; torch checkpoints * transpose once at load. * @@ -157,9 +204,8 @@ export class Linear extends Module { } sd.tensors.set(name, transposeWeight(tensor)); } - /** Replaces `name` with quantW words [in, out] and adds its scales - * [in / groupSize, out] right after it, so they bind in file order. The - * words carry the whole byteLength of the ints and scales. */ + /** Loads `name` as quantW words [in, out] with its scales + * [in / groupSize, out]. */ private loadQuantized( sd: LazyStateDict, prefix: string, @@ -180,35 +226,16 @@ export class Linear extends Module { ); } this.quantization = { bits, groupSize }; - this.weight.retype('quantW', (k * n * bits) / 32); - this.weightScale = new Parameter( - { elems: (k / groupSize) * n, dtype: 'f32', dims: [k / groupSize, n] }, - undefined, - true, - ); - const words = derivedTensor( - [k, n], - async () => packQuantW(await tensor.ints(), k, n, bits), - tensor.byteLength, - ); const groups = k / groupSize; - const scales = derivedTensor([groups, n], async () => { - const data = await tensor.scales(); - const f32 = data instanceof Float32Array ? data : f16ToF32(data); - // torch's [out, in / groupSize] to the kernels' [in / groupSize, out] - const out = new Float32Array(f32.length); - for (let o = 0; o < n; o++) { - for (let g = 0; g < groups; g++) out[g * n + o] = f32[o * groups + g]!; - } - return out; - }); - const scaleName = prefix ? `${prefix}.weightScale` : 'weightScale'; - const entries = [...sd.tensors]; - sd.tensors.clear(); - for (const [key, t] of entries) { - sd.tensors.set(key, key === name ? words : t); - if (key === name) sd.tensors.set(scaleName, scales); - } + this.weightScale = loadQuantWeight( + sd, + prefix, + this.weight, + tensor, + { k, n, groups }, + () => tensor.ints(), + () => scaleColumns(tensor, groups, n), + ); } forward(x: Value, call: { addend?: Value } = {}): Value { if (!this.weightScale || !this.quantization) { @@ -224,6 +251,39 @@ export class Linear extends Module { } } +/** The number of input channels in each scale group of a quantized conv + * weight [out, in·kernel]. A group holds whole channels with all their + * kernel values. Throws CHECKPOINT_MISMATCH when the weight does not fit + * the quantized conv on n output columns. */ +function quantConvGroup( + layer: string, + name: string, + t: QuantTensor, + shape: { cIn: number; cOut: number; kernel: number; taps: number; n: number }, +): number { + const { cIn, cOut, kernel, taps, n } = shape; + const bad = (why: string): never => { + throw new RunntimeError('CHECKPOINT_MISMATCH', `${layer}: ${name} ${why}`); + }; + if (t.shape[0] !== cOut || t.shape[1] !== cIn * kernel) { + bad( + `is quantized as [${t.shape.join(', ')}], expected [out, in·kernel] = [${cOut}, ${cIn * kernel}]`, + ); + } + const group = t.groupSize / kernel; + if (!Number.isInteger(group)) { + bad(`has groups of ${t.groupSize} values, not whole channels of ${kernel} values`); + } + if (!matmulQuantTiledEligible(taps * cIn, n, group, cIn)) { + bad( + `is int${t.bits} in groups of ${group} channels, which needs a group a multiple of 4, ` + + `in channels a multiple of the group and ${taps}·in a multiple of ` + + `${MATMUL_QUANT_TILED_BK}, got in ${cIn}`, + ); + } + return group; +} + /** Strided, dilated 1D convolution over time-major [T, inChannels]. No batch * dim, no groups. * @@ -231,10 +291,19 @@ export class Linear extends Module { * each side independently instead. span − 1 on the left is causal, with * span = dilation·(kernelSize − 1) + 1. * weight is stored tap-major and permutes from torch layout at load. bias and - * `activation` fold into the conv epilogue. */ + * `activation` fold into the conv epilogue. + * + * A quantized checkpoint weight loads as quantW words with its f32 scales in + * `weightScale` [in / group, out], one per group of input channels, the same + * for every tap. Output channels pad up to the packing width and are cut + * after the conv. */ export class Conv1d extends Module { readonly weight: Parameter; readonly bias?: Parameter; + /** Set when a quantized weight loads. */ + weightScale?: Parameter; + /** How the loaded weight is quantized, or undefined for a float weight. */ + private quantization?: { bits: 4 | 8; groupSize: number }; readonly stride: number; readonly dilation: number; readonly padding?: number | 'valid' | 'same'; @@ -273,6 +342,10 @@ export class Conv1d extends Module { const name = prefix ? `${prefix}.weight` : 'weight'; const tensor = sd.tensors.get(name); if (!tensor) return; // a missing key is reported by the strict key check + if (tensor.kind === 'quant') { + this.loadQuantized(sd, prefix, name, tensor); + return; + } // Without this a weight with equal in/out channels loads scrambled. if ( tensor.shape.length !== 3 || @@ -301,8 +374,65 @@ export class Conv1d extends Module { ), ); } + /** Loads `name` as quantW words [kernel·in, n] with its scales + * [in / group, n], where n is outChannels padded to the packing width. The + * bias pads to n too. */ + private loadQuantized(sd: LazyStateDict, prefix: string, name: string, t: QuantTensor): void { + const { inChannels: cIn, outChannels: cOut, kernelSize: kernel } = this; + const width = 32 / t.bits; + const n = Math.ceil(cOut / width) * width; + const group = quantConvGroup('Conv1d', name, t, { cIn, cOut, kernel, taps: kernel, n }); + const k = kernel * cIn; + const groups = cIn / group; + this.quantization = { bits: t.bits, groupSize: group }; + this.weightScale = loadQuantWeight( + sd, + prefix, + this.weight, + t, + { k, n, groups }, + async () => { + // torch's [out, in, kernel] to one tap-major row per output + const src = await t.ints(); + const out = new Int8Array(n * k); + for (let o = 0; o < cOut; o++) { + for (let c = 0; c < cIn; c++) { + for (let tap = 0; tap < kernel; tap++) { + out[o * k + tap * cIn + c] = src[(o * cIn + c) * kernel + tap]!; + } + } + } + return out; + }, + () => scaleColumns(t, groups, n), + ); + const biasName = prefix ? `${prefix}.bias` : 'bias'; + const b = sd.tensors.get(biasName); + if (n > cOut && b && this.bias) { + if (b.shape.join() !== `${cOut}`) { + throw new RunntimeError( + 'CHECKPOINT_MISMATCH', + `Conv1d: ${biasName} has shape ${JSON.stringify(b.shape)}, expected [${cOut}]`, + ); + } + this.bias.retype(this.bias.shape.dtype, n, [n]); + sd.tensors.set( + biasName, + derivedTensor( + [n], + async () => { + const src = await readFloats('Conv1d', b); + const out = src instanceof Uint16Array ? new Uint16Array(n) : new Float32Array(n); + out.set(src); + return out; + }, + b.byteLength, + ), + ); + } + } forward(x: Value): Value { - return conv1d(x, this.weight.value, { + const y = conv1d(x, this.weight.value, { kernelSize: this.kernelSize, stride: this.stride, dilation: this.dilation, @@ -311,7 +441,12 @@ export class Conv1d extends Module { padRight: this.padRight, bias: this.bias?.value, activation: this.activation, + ...(this.weightScale && this.quantization + ? { scales: this.weightScale.value, ...this.quantization } + : {}), }); + const cols = this.weight.shape.dims![1]!; + return cols > this.outChannels ? slice(y, 1, 0, this.outChannels) : y; } } @@ -321,10 +456,18 @@ export class Conv1d extends Module { * Runs as a plain conv: kernelSize / stride taps to stride·outChannels * channels. Each output row holds `stride` frames side by side, so a reshape * unfolds them and a slice cuts the padding. The weight permutes from torch - * layout [in, out, kernel] at load, and the bias repeats per frame. */ + * layout [in, out, kernel] at load, and the bias repeats per frame. + * + * A quantized checkpoint weight is the equivalent Conv1d weight: [out, + * in·kernel] with the kernel reversed. stride·outChannels must be a + * multiple of the packing width. */ export class ConvTranspose1d extends Module { readonly weight: Parameter; readonly bias?: Parameter; + /** Set when a quantized weight loads. */ + weightScale?: Parameter; + /** How the loaded weight is quantized, or undefined for a float weight. */ + private quantization?: { bits: 4 | 8; groupSize: number }; readonly taps: number; readonly stride: number; readonly padding: number; @@ -365,7 +508,9 @@ export class ConvTranspose1d extends Module { const at = (n: string) => (prefix ? `${prefix}.${n}` : n); const { inChannels: cIn, outChannels: cOut, kernelSize: k, taps, stride: s } = this; const w = sd.tensors.get(at('weight')); - if (w) { + if (w?.kind === 'quant') { + this.loadQuantized(sd, prefix, at('weight'), w); + } else if (w) { if (w.shape.length !== 3 || w.shape[0] !== cIn || w.shape[1] !== cOut || w.shape[2] !== k) { throw new RunntimeError( 'CHECKPOINT_MISMATCH', @@ -416,6 +561,57 @@ export class ConvTranspose1d extends Module { ); } } + /** Loads `name` as quantW words [taps·in, stride·out] with its scales + * [in / group, stride·out]; each output's scales repeat for every frame. */ + private loadQuantized(sd: LazyStateDict, prefix: string, name: string, t: QuantTensor): void { + const { inChannels: cIn, outChannels: cOut, kernelSize: kernel, taps, stride: s } = this; + const n = s * cOut; + if (n % (32 / t.bits) !== 0) { + throw new RunntimeError( + 'CHECKPOINT_MISMATCH', + `ConvTranspose1d: ${name} is int${t.bits}, which needs stride·out a multiple of ` + + `${32 / t.bits}, got ${n}`, + ); + } + const group = quantConvGroup('ConvTranspose1d', name, t, { cIn, cOut, kernel, taps, n }); + const k = taps * cIn; + const groups = cIn / group; + this.quantization = { bits: t.bits, groupSize: group }; + this.weightScale = loadQuantWeight( + sd, + prefix, + this.weight, + t, + { k, n, groups }, + async () => { + // Column r·out + o, row i·in + c takes kernel position + // (taps − 1 − i)·s + r. The file holds the kernel reversed. + const src = await t.ints(); + const out = new Int8Array(n * k); + for (let r = 0; r < s; r++) { + for (let o = 0; o < cOut; o++) { + for (let i = 0; i < taps; i++) { + const kk = (taps - 1 - i) * s + r; + for (let c = 0; c < cIn; c++) { + out[(r * cOut + o) * k + i * cIn + c] = + src[(o * cIn + c) * kernel + kernel - 1 - kk]!; + } + } + } + } + return out; + }, + async () => { + const base = await scaleColumns(t, groups, cOut); + const out = new Float32Array(groups * n); + for (let g = 0; g < groups; g++) { + for (let r = 0; r < s; r++) + out.set(base.subarray(g * cOut, (g + 1) * cOut), g * n + r * cOut); + } + return out; + }, + ); + } forward(x: Value): Value { const { taps, stride: s, padding: p, outChannels } = this; const t = x.shape.dims![0]!; @@ -425,6 +621,9 @@ export class ConvTranspose1d extends Module { padLeft: taps - 1, padRight: taps - 1, bias: this.bias?.value, + ...(this.weightScale && this.quantization + ? { scales: this.weightScale.value, ...this.quantization } + : {}), }); const rows = (t + taps - 1) * s; const frames = (t - 1) * s - 2 * p + this.kernelSize + this.outputPadding; diff --git a/packages/runntime/src/core/nn/loadStateDict.ts b/packages/runntime/src/core/nn/loadStateDict.ts index 05f4580b..4d661883 100644 --- a/packages/runntime/src/core/nn/loadStateDict.ts +++ b/packages/runntime/src/core/nn/loadStateDict.ts @@ -144,13 +144,13 @@ async function bindStateDict( let done = 0; for (const [name, p] of params) { const t = sd.tensors.get(name)!; - // Linear's hook turns its quantized weight into quantW words, so a - // QuantTensor still here is bound to something else. + // Linear's and the convs' hooks turn a quantized weight into quantW + // words, so a QuantTensor still here is bound to something else. if (t.kind === 'quant') { throw new RunntimeError( 'CHECKPOINT_MISMATCH', - `loadStateDict: '${name}' is quantized (int${t.bits}), but only a Linear weight loads ` + - `quantized and this parameter holds ${p.shape.dtype}`, + `loadStateDict: '${name}' is quantized (int${t.bits}), but only a Linear, Conv1d or ` + + `ConvTranspose1d weight loads quantized and this parameter holds ${p.shape.dtype}`, ); } const numel = t.shape.reduce((a, b) => a * b, 1); diff --git a/packages/runntime/src/core/nn/module.ts b/packages/runntime/src/core/nn/module.ts index 9dcee4b0..e3512cdd 100644 --- a/packages/runntime/src/core/nn/module.ts +++ b/packages/runntime/src/core/nn/module.ts @@ -44,10 +44,11 @@ export class Parameter { if (!this._value) throw new Error(`Parameter ${this.qualifiedName || '(unnamed)'} not loaded`); return this._value; } - /** @internal Swaps the storage dtype, keeping dims. quantW counts u32 - * words, so a float slot turned quantW passes its word count as `elems`. */ - retype(dtype: EagerDtype, elems = this._shape.elems): void { - this._shape = { ...this._shape, dtype, elems }; + /** @internal Sets the storage dtype, and the dims when given. quantW + * counts u32 words, so a slot set to quantW passes its word count as + * `elems`. */ + retype(dtype: EagerDtype, elems = this._shape.elems, dims = this._shape.dims): void { + this._shape = { ...this._shape, dtype, elems, dims }; } /** @internal */ bind(v: Value): void { diff --git a/packages/runntime/test/core/graph/ops/conv/conv1d.test.ts b/packages/runntime/test/core/graph/ops/conv/conv1d.test.ts index 0aecf229..3d24ed2a 100644 --- a/packages/runntime/test/core/graph/ops/conv/conv1d.test.ts +++ b/packages/runntime/test/core/graph/ops/conv/conv1d.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import { ACT_CODE, conv1d, gluConv1d } from '../../../../../src/core/graph/ops.ts'; -import { input, meta } from '../../values.ts'; +import { input, meta, quantWeight } from '../../values.ts'; // x is time-major [T, C_in]; w is tap-major [kernelSize·C_in, C_out] const T = 10; @@ -34,8 +34,60 @@ describe('conv1d', () => { const bias = input([COUT]); const c = conv1d(x, w, { kernelSize: 3, stride: 1, bias, activation: 'gelu' }); expect(c.pending?.inputs).toEqual([x, w, bias]); - // attrs: kernel, stride, padLeft, padRight, hasBias, act, dilation - expect(c.pending?.attrs?.slice(4)).toEqual([1, ACT_CODE.gelu, 1]); + // attrs: kernel, stride, padLeft, padRight, hasBias, act, dilation, bits, + // groupSize + expect(c.pending?.attrs?.slice(4)).toEqual([1, ACT_CODE.gelu, 1, 0, 0]); + }); + + it('takes a quantW weight with its scales before the bias', () => { + const w = quantWeight(3 * 8, 4, 8); + const scales = input([2, 4]); + const bias = input([4]); + const c = conv1d(input([T, 8]), w, { + kernelSize: 3, + stride: 1, + bias, + scales, + bits: 8, + groupSize: 4, + }); + expect(c.shape).toEqual(meta([8, 4])); + expect(c.pending?.inputs).toEqual([expect.anything(), w, scales, bias]); + expect(c.pending?.attrs?.slice(7)).toEqual([8, 4]); + }); + + it.each([ + { + name: 'a quantW weight without scales', + opts: { bits: 8 as const, groupSize: 4 }, + err: /quantW weights need scales, bits and groupSize/, + }, + { + name: 'C_in not a multiple of groupSize', + opts: { scales: input([2, 4]), bits: 8 as const, groupSize: 12 }, + err: /C_in a multiple of groupSize/, + }, + { + name: 'kernelSize·C_in not a multiple of 8', + x: input([T, 4]), + w: quantWeight(3 * 4, 4, 8), + opts: { scales: input([1, 4]), bits: 8 as const, groupSize: 4 }, + err: /kernelSize·C_in a multiple of 8/, + }, + { + name: 'scales of the wrong size', + opts: { scales: input([1, 4]), bits: 8 as const, groupSize: 4 }, + err: /scales must be f32 \[C_in\/groupSize, C_out\] = \[2, 4\]/, + }, + ])('refuses $name', ({ x = input([T, 8]), w = quantWeight(3 * 8, 4, 8), opts, err }) => { + expect(() => conv1d(x, w, { kernelSize: 3, stride: 1, ...opts })).toThrow(err); + }); + + it('refuses scales on a float weight', () => { + const w = weight(3); + expect(() => + conv1d(x, w, { kernelSize: 3, stride: 1, scales: input([1, COUT]), bits: 8, groupSize: 4 }), + ).toThrow(/scales\/bits\/groupSize apply to quantW weights only/); }); it('widens the window by the dilation', () => { diff --git a/packages/runntime/test/core/kernels/conv/conv1d.gpu.test.ts b/packages/runntime/test/core/kernels/conv/conv1d.gpu.test.ts index 1d5d142f..336afda9 100644 --- a/packages/runntime/test/core/kernels/conv/conv1d.gpu.test.ts +++ b/packages/runntime/test/core/kernels/conv/conv1d.gpu.test.ts @@ -4,6 +4,7 @@ import { conv1d, type SlotAct } from '../../../../src/core/graph/ops.ts'; import { matrix } from '../../../../src/core/graph/value.ts'; import { expectClose, gpu, randomF32, requireFeature, roundF16, upload } from '../../../gpu.ts'; import { ACTIVATION_REF } from '../matmul/checkMatmul.ts'; +import { randomQuantWeight } from '../matmul/quantW.ts'; const { root, ex } = await gpu(); @@ -69,6 +70,45 @@ async function checkConv1d(c: Case, dtype: 'f32' | 'f16') { expectClose(got, conv1dRef(x, w, c, bias), tol); } +/** conv1d with a quantW weight against conv1dRef on its dequantized values. + * The weight's rows hold each tap's input channels in turn, so a group of + * groupSize channels has one scale in every tap. */ +async function checkQuantConv1d( + c: Case & { bits: 4 | 8; groupSize: number }, + dtype: 'f32' | 'f16', +) { + const { cIn, cOut, kernelSize, bits, groupSize } = c; + const round = dtype === 'f16' ? roundF16 : (v: Float32Array) => v; + const meta = (rows: number, cols: number) => ({ ...matrix(rows, cols), dtype }); + const k = kernelSize * cIn; + const weight = randomQuantWeight( + root, + { experts: 1, k, n: cOut, bits, groupSize, period: cIn }, + 2, + ); + const x = round(randomF32(c.tIn * cIn, 1)); + const bias = c.bias ? round(randomF32(cOut, 3)) : undefined; + const got = await toArray( + conv1d(upload(root, x, meta(c.tIn, cIn)), weight.w, { + kernelSize, + stride: c.stride, + dilation: c.dilation, + padLeft: c.padLeft, + padRight: c.padRight, + bias: bias && upload(root, bias, meta(1, cOut)), + activation: c.activation, + scales: upload(root, weight.scales, [cIn / groupSize, cOut]), + bits, + groupSize, + }), + ex, + ); + // The f16 route stores dequantized weights in f16 before the f32 sums. + const tol = + dtype === 'f16' ? { atol: 2e-3 * Math.sqrt(k), rtol: 5e-3 } : { atol: 1e-6 * k, rtol: 1e-5 }; + expectClose(got, conv1dRef(x, Float32Array.from(weight.dequant), c, bias), tol); +} + describe('conv1d kernel', () => { // Routes are picked in gpu/dispatch/conv.ts. const SHAPES: { name: string; c: Case }[] = [ @@ -264,6 +304,120 @@ describe('conv1d kernel', () => { await checkConv1d({ ...c, bias, activation }, dtype); }); + // A quantW weight always takes matmulQuantTiled: gemm when unpadded and + // undilated, mapped gemm otherwise, with no minimum K. + const QUANT_SHAPES: { name: string; c: Case & { bits: 4 | 8; groupSize: number } }[] = [ + { + name: 'int8, stride 2 (gemm)', + c: { tIn: 70, cIn: 16, cOut: 36, kernelSize: 4, stride: 2, bits: 8, groupSize: 8 }, + }, + { + name: 'int8, all channels in one group (gemm)', + c: { tIn: 40, cIn: 32, cOut: 12, kernelSize: 3, stride: 1, bits: 8, groupSize: 32 }, + }, + { + name: 'int8, 1×1 under the float tiled minimum K (gemm)', + c: { tIn: 50, cIn: 32, cOut: 32, kernelSize: 1, stride: 1, bits: 8, groupSize: 8 }, + }, + { + name: 'int8, same padding (mapped gemm)', + c: { + tIn: 45, + cIn: 16, + cOut: 20, + kernelSize: 7, + stride: 1, + padLeft: 3, + padRight: 3, + bits: 8, + groupSize: 8, + }, + }, + { + name: 'int8, dilation 3 (mapped gemm)', + c: { + tIn: 50, + cIn: 16, + cOut: 16, + kernelSize: 7, + stride: 1, + dilation: 3, + padLeft: 9, + padRight: 9, + bits: 8, + groupSize: 4, + }, + }, + { + name: 'int8, C_in 4, two taps per K step (mapped gemm)', + c: { + tIn: 30, + cIn: 4, + cOut: 8, + kernelSize: 16, + stride: 1, + padLeft: 5, + padRight: 7, + bits: 8, + groupSize: 4, + }, + }, + { + name: 'int8, stride 2 with padding (mapped gemm)', + c: { + tIn: 41, + cIn: 16, + cOut: 12, + kernelSize: 5, + stride: 2, + padLeft: 2, + padRight: 2, + bits: 8, + groupSize: 8, + }, + }, + { + name: 'int4, stride 2 (gemm)', + c: { tIn: 70, cIn: 16, cOut: 40, kernelSize: 4, stride: 2, bits: 4, groupSize: 8 }, + }, + { + name: 'int4, same padding (mapped gemm)', + c: { + tIn: 45, + cIn: 32, + cOut: 16, + kernelSize: 5, + stride: 1, + padLeft: 2, + padRight: 2, + bits: 4, + groupSize: 16, + }, + }, + ]; + it.for((['f32', 'f16'] as const).flatMap((dtype) => QUANT_SHAPES.map((s) => ({ dtype, ...s }))))( + '$dtype quantW $name', + async ({ dtype, c }, ctx) => { + if (dtype === 'f16') requireFeature(ctx, root, 'shader-f16'); + await checkQuantConv1d(c, dtype); + }, + ); + + it.for( + (['f32', 'f16'] as const).flatMap((dtype) => + [ + { route: 'gemm', c: { tIn: 40, cIn: 16, cOut: 12, kernelSize: 4, stride: 2 } }, + { + route: 'mapped gemm', + c: { tIn: 40, cIn: 16, cOut: 12, kernelSize: 4, stride: 1, padLeft: 3, padRight: 3 }, + }, + ].flatMap((r) => EPILOGUES.map((e) => ({ dtype, ...r, ...e }))), + ), + )('$dtype quantW $route with $name', async ({ dtype, c, bias, activation }, ctx) => { + if (dtype === 'f16') requireFeature(ctx, root, 'shader-f16'); + await checkQuantConv1d({ ...c, bias, activation, bits: 8, groupSize: 8 }, dtype); + }); + // padding 'same' with an even kernel pads one more on the right, as torch does. it("padding 'same' with an even kernel", async () => { const x = randomF32(9 * 2, 1); diff --git a/packages/runntime/test/core/kernels/matmul/quantW.ts b/packages/runntime/test/core/kernels/matmul/quantW.ts index 757f302f..638eadcb 100644 --- a/packages/runntime/test/core/kernels/matmul/quantW.ts +++ b/packages/runntime/test/core/kernels/matmul/quantW.ts @@ -10,7 +10,8 @@ import { randomF32 } from '../../../gpu.ts'; * dequantizes to. */ export interface QuantWeight { w: Value; - /** f32 [E·K / groupSize, N]: one scale per group of rows in each column. */ + /** f32 [period / groupSize, N]: one scale per group of rows in each + * column, the same in every period of rows. */ scales: Float32Array; /** q · scale for row r and column c at [r·N + c], in f64. */ dequant: Float64Array; @@ -42,7 +43,8 @@ function packQuantW(q: Int8Array, experts: number, k: number, n: number, bits: 4 } /** Random ints over the full signed range of `bits` and positive scales, - * packed and uploaded. The buffer is destroyed when the test ends. */ + * packed and uploaded. Row r takes scale row (r mod period) / groupSize, + * period E·K when omitted. The buffer is destroyed when the test ends. */ export function randomQuantWeight( root: TgpuRoot, { @@ -51,7 +53,8 @@ export function randomQuantWeight( n, bits, groupSize, - }: { experts: number; k: number; n: number; bits: 4 | 8; groupSize: number }, + period = experts * k, + }: { experts: number; k: number; n: number; bits: 4 | 8; groupSize: number; period?: number }, seed: number, ): QuantWeight { const rows = experts * k; @@ -59,12 +62,13 @@ export function randomQuantWeight( const q = Int8Array.from(randomF32(rows * n, seed), (v) => Math.min(half - 1, Math.floor(v * half)), ); - const scales = randomF32((rows / groupSize) * n, seed + 1).map((v) => (Math.abs(v) + 0.1) / half); + const scales = randomF32((period / groupSize) * n, seed + 1).map( + (v) => (Math.abs(v) + 0.1) / half, + ); const dequant = new Float64Array(rows * n); for (let r = 0; r < rows; r++) { - for (let c = 0; c < n; c++) { - dequant[r * n + c] = q[r * n + c]! * scales[Math.floor(r / groupSize) * n + c]!; - } + const row = Math.floor((r % period) / groupSize); + for (let c = 0; c < n; c++) dequant[r * n + c] = q[r * n + c]! * scales[row * n + c]!; } const words = packQuantW(q, experts, k, n, bits); const buffer = uploadU32(root, words); diff --git a/packages/runntime/test/core/nn/layers.gpu.test.ts b/packages/runntime/test/core/nn/layers.gpu.test.ts index 6518236b..5344f528 100644 --- a/packages/runntime/test/core/nn/layers.gpu.test.ts +++ b/packages/runntime/test/core/nn/layers.gpu.test.ts @@ -2,11 +2,12 @@ import { describe, it, onTestFinished } from 'vitest'; import { toArray } from '../../../src/core/gpu/eval.ts'; import type { SlotAct } from '../../../src/core/graph/ops.ts'; import { matrix } from '../../../src/core/graph/value.ts'; -import { Linear } from '../../../src/core/nn/layers.ts'; +import { Conv1d, ConvTranspose1d, Linear } from '../../../src/core/nn/layers.ts'; import { bufferSource, fromSafetensors, memoryStateDict, + type LazyStateDict, } from '../../../src/core/weights/safetensors.ts'; import { expectClose, gpu, randomF32, requireFeature, roundF16, upload } from '../../gpu.ts'; import { ACTIVATION_REF } from '../kernels/matmul/checkMatmul.ts'; @@ -168,3 +169,163 @@ describe('Linear with a quantized weight', () => { await checkQuantizedLinear(c, c.bits, c.dtype); }); }); + +interface ConvCase { + name: string; + kind: 'conv' | 'transpose'; + tIn: number; + cIn: number; + cOut: number; + kernel: number; + stride: number; + padding: number; + /** Input channels per scale group, each group spanning every tap. */ + group: number; + bias: boolean; + activation?: 'gelu'; +} + +/** A conv loaded from a quantized file [out, in·kernel] matches the same conv + * loaded with the dequantized floats, int · f16 scale, in torch's layout. A + * transposed conv's file holds the kernel reversed. */ +async function checkQuantizedConv(c: ConvCase, bits: 4 | 8, dtype: 'f32' | 'f16') { + const { tIn, cIn, cOut, kernel, stride, padding, group } = c; + const half = 2 ** (bits - 1); + const cols = cIn * kernel; + const groupSize = group * kernel; + const ints = Array.from(randomF32(cOut * cols, 1), (v) => + Math.min(half - 1, Math.floor(v * half)), + ); + const scales = roundF16( + randomF32((cOut * cols) / groupSize, 2).map((v) => (Math.abs(v) + 0.1) / half), + ); + const bias = roundF16(randomF32(cOut, 3)); + const xs = roundF16(randomF32(tIn * cIn, 4)); + // [out, in, kernel] dequantized; a transposed conv's torch weight is [in, + // out, kernel] with the kernel the other way round. + const dequant = Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / groupSize)]!); + const torchWeight = + c.kind === 'conv' + ? { data: dequant, shape: [cOut, cIn, kernel] } + : { + data: Float32Array.from({ length: dequant.length }, (_, i) => { + const [ci, o, t] = [ + Math.floor(i / (cOut * kernel)), + Math.floor(i / kernel) % cOut, + i % kernel, + ]; + return dequant[(o * cIn + ci) * kernel + kernel - 1 - t]!; + }), + shape: [cIn, cOut, kernel], + }; + const make = () => + c.kind === 'conv' + ? new Conv1d(cIn, cOut, kernel, { stride, padding, bias: c.bias, activation: c.activation }) + : new ConvTranspose1d(cIn, cOut, kernel, { stride, padding, bias: c.bias }); + const run = async (layer: Conv1d | ConvTranspose1d, sd: LazyStateDict) => { + if (dtype === 'f16') layer.half(root); + onTestFinished(() => layer.dispose()); + await layer.loadStateDict(sd, { root }); + return toArray(layer.forward(upload(root, xs, { ...matrix(tIn, cIn), dtype })), ex); + }; + const file = quantFile(bits, groupSize, { + weight: { ints, scales: [...scales], shape: [cOut, cols] }, + }); + const quantSd = await fromSafetensors(bufferSource(file), { chunkBytes: 0 }); + if (c.bias) quantSd.tensors.set('bias', memoryStateDict({ bias }).tensors.get('bias')!); + const got = await run(make(), quantSd); + const want = await run( + make(), + memoryStateDict({ weight: torchWeight, ...(c.bias ? { bias } : {}) }), + ); + // Both sum in f32 in another order; f16 rounds the output to half precision. + const tol = dtype === 'f16' ? { atol: 2e-3, rtol: 2e-3 } : { atol: 1e-5, rtol: 1e-5 }; + expectClose(got, want, tol); +} + +describe('Conv1d and ConvTranspose1d with a quantized weight', () => { + const CASES: ConvCase[] = [ + { + name: 'conv, padded, bias and gelu', + kind: 'conv', + tIn: 40, + cIn: 16, + cOut: 24, + kernel: 7, + stride: 1, + padding: 3, + group: 8, + bias: true, + activation: 'gelu', + }, + { + name: 'conv 1×1, stride 2', + kind: 'conv', + tIn: 33, + cIn: 32, + cOut: 16, + kernel: 1, + stride: 2, + padding: 0, + group: 32, + bias: false, + }, + // 1 and 3 output channels pad up to the packing width and are cut after. + { + name: 'conv to 1 channel, bias', + kind: 'conv', + tIn: 30, + cIn: 32, + cOut: 1, + kernel: 7, + stride: 1, + padding: 3, + group: 8, + bias: true, + }, + { + name: 'conv to 3 channels', + kind: 'conv', + tIn: 20, + cIn: 8, + cOut: 3, + kernel: 3, + stride: 1, + padding: 1, + group: 4, + bias: true, + }, + { + name: 'transposed, stride 4, bias', + kind: 'transpose', + tIn: 12, + cIn: 16, + cOut: 8, + kernel: 8, + stride: 4, + padding: 2, + group: 8, + bias: true, + }, + { + name: 'transposed, stride 2', + kind: 'transpose', + tIn: 9, + cIn: 32, + cOut: 8, + kernel: 4, + stride: 2, + padding: 1, + group: 16, + bias: false, + }, + ]; + it.for( + (['f32', 'f16'] as const).flatMap((dtype) => + CASES.flatMap((c) => ([8, 4] as const).map((bits) => ({ ...c, dtype, bits }))), + ), + )('$dtype int$bits $name', async (c, ctx) => { + if (c.dtype === 'f16') requireFeature(ctx, root, 'shader-f16'); + await checkQuantizedConv(c, c.bits, c.dtype); + }); +}); diff --git a/packages/runntime/test/core/nn/layers.test.ts b/packages/runntime/test/core/nn/layers.test.ts index 303e3945..6c13621f 100644 --- a/packages/runntime/test/core/nn/layers.test.ts +++ b/packages/runntime/test/core/nn/layers.test.ts @@ -416,6 +416,105 @@ describe('Linear with a quantized weight', () => { }); }); +describe('Conv1d and ConvTranspose1d with a quantized weight', () => { + /** A [out, in·kernel] int8 entry of zeros, with `groups` scales per row. */ + const zeros = (out: number, cols: number, groups: number): QuantEntry => ({ + ints: Array(out * cols).fill(0), + scales: Array(out * groups).fill(1), + shape: [out, cols], + }); + + it('retypes a Conv1d weight to quantW [kernel·in, out] with a scale per channel group', async () => { + const layer = new Conv1d(8, 4, 3); + // groups of 12 values: 4 input channels with their 3 taps + const sd = await quantCheckpoint(8, 12, { weight: zeros(4, 24, 2) }); + layer.transformStateDict(sd, ''); + expect(shapes(layer)).toEqual({ + weight: { elems: 24, dtype: 'quantW', dims: [24, 4] }, + bias: meta([4]), + weightScale: meta([2, 4]), + }); + expect([...sd.tensors.keys()]).toEqual(['weight', 'weightScale']); + }); + + it('pads a Conv1d to 1 output channel up to 4 columns, bias included', async () => { + const layer = new Conv1d(8, 1, 3); + const sd = await quantCheckpoint(8, 12, { weight: zeros(1, 24, 2) }); + sd.tensors.set('bias', memoryStateDict({ bias: Float32Array.of(2.5) }).tensors.get('bias')!); + layer.transformStateDict(sd, ''); + expect(shapes(layer)).toEqual({ + weight: { elems: 24, dtype: 'quantW', dims: [24, 4] }, + bias: meta([4]), + weightScale: meta([2, 4]), + }); + expect([...(await sd.tensors.get('bias')!.f32())]).toEqual([2.5, 0, 0, 0]); + }); + + it('refuses a padded Conv1d bias of another length with CHECKPOINT_MISMATCH', async () => { + const layer = new Conv1d(8, 1, 3); + const sd = await quantCheckpoint(8, 12, { weight: zeros(1, 24, 2) }); + sd.tensors.set('bias', memoryStateDict({ bias: Float32Array.of(1, 2) }).tensors.get('bias')!); + expect(() => layer.transformStateDict(sd, '')).toThrow( + expect.objectContaining({ + code: 'CHECKPOINT_MISMATCH', + message: expect.stringMatching(/bias has shape \[2\], expected \[1\]/), + }), + ); + }); + + it('retypes a ConvTranspose1d weight to quantW [taps·in, stride·out]', async () => { + const layer = new ConvTranspose1d(8, 2, 4, { stride: 2 }); + const sd = await quantCheckpoint(8, 16, { weight: zeros(2, 32, 2) }); + layer.transformStateDict(sd, ''); + expect(shapes(layer)).toEqual({ + weight: { elems: 16, dtype: 'quantW', dims: [16, 4] }, + bias: meta([4]), + weightScale: meta([2, 4]), + }); + }); + + it.each([ + { + name: 'groups that split a channel', + layer: () => new Conv1d(8, 4, 3), + file: () => quantCheckpoint(8, 4, { weight: zeros(4, 24, 6) }), + error: /groups of 4 values, not whole channels of 3 values/, + }, + { + name: 'a group of 2 channels', + layer: () => new Conv1d(8, 4, 2), + file: () => quantCheckpoint(8, 4, { weight: zeros(4, 16, 4) }), + error: /groups of 2 channels, which needs a group a multiple of 4/, + }, + { + name: 'kernel·in that is not a multiple of 8', + layer: () => new Conv1d(4, 4, 3), + file: () => quantCheckpoint(8, 12, { weight: zeros(4, 12, 1) }), + error: /3·in a multiple of 8, got in 4/, + }, + { + name: 'a weight of another shape', + layer: () => new Conv1d(8, 4, 3), + file: () => quantCheckpoint(8, 8, { weight: zeros(4, 16, 2) }), + error: /expected \[out, in·kernel\] = \[4, 24\]/, + }, + { + name: 'stride·out that is not a multiple of 4', + layer: () => new ConvTranspose1d(8, 3, 2, { stride: 2 }), + file: () => quantCheckpoint(8, 8, { weight: zeros(3, 16, 2) }), + error: /needs stride·out a multiple of 4, got 6/, + }, + ])('refuses $name with CHECKPOINT_MISMATCH', async ({ layer, file, error }) => { + const sd = await file(); + expect(() => layer().transformStateDict(sd, '')).toThrow( + expect.objectContaining({ + code: 'CHECKPOINT_MISMATCH', + message: expect.stringMatching(error), + }), + ); + }); +}); + describe('Conv1d', () => { it('stores the weight tap-major as [kernelSize·in, out]', () => { expect(shapes(new Conv1d(4, 6, 3))).toEqual({ weight: meta([12, 6]), bias: meta([6]) }); diff --git a/packages/runntime/test/core/nn/loadStateDict.test.ts b/packages/runntime/test/core/nn/loadStateDict.test.ts index 03e4d534..db8f7ed1 100644 --- a/packages/runntime/test/core/nn/loadStateDict.test.ts +++ b/packages/runntime/test/core/nn/loadStateDict.test.ts @@ -275,8 +275,8 @@ describe('a quantized tensor', () => { await expect(load(new NotLinear(), { weight: quant }, upload)).rejects.toMatchObject({ code: 'CHECKPOINT_MISMATCH', message: - "loadStateDict: 'weight' is quantized (int8), but only a Linear weight loads quantized " + - `and this parameter holds ${dtype}`, + "loadStateDict: 'weight' is quantized (int8), but only a Linear, Conv1d or " + + `ConvTranspose1d weight loads quantized and this parameter holds ${dtype}`, }); expect(upload).not.toHaveBeenCalled(); expect(ints).not.toHaveBeenCalled(); From d2991e5d9840aeff8061895768b0dad6912a08ca Mon Sep 17 00:00:00 2001 From: Kamil Kaczmarczyk <143009372+xxpopielxx@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:59:03 +0100 Subject: [PATCH 3/4] refactor(core): fold runntime.quant version 2 into version 1 with optional bits and groupSize A file without bits or groupSize takes them from each tensor's dtype and scale shape. A file that sets them is checked against every tensor, as before. --- .../runntime/src/core/weights/safetensors.ts | 27 ++++++++--------- .../test/core/weights/safetensors.test.ts | 30 ++++++++++++------- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/packages/runntime/src/core/weights/safetensors.ts b/packages/runntime/src/core/weights/safetensors.ts index 3142c34f..ac9599f8 100644 --- a/packages/runntime/src/core/weights/safetensors.ts +++ b/packages/runntime/src/core/weights/safetensors.ts @@ -586,14 +586,14 @@ function bf16Tensor(bits: Uint16Array, shape: number[]): FileTensor { } /** Metadata key of a quantized file: a JSON string - * `{"version":1,"bits":8|4,"groupSize":N,"scheme":"symmetric"}`, or - * `{"version":2,"scheme":"symmetric"}`, where each int weight's bits come - * from its dtype and its groupSize from its scale's shape. */ + * `{"version":1,"bits":8|4,"groupSize":N,"scheme":"symmetric"}`. bits and + * groupSize are optional: without them, each int weight's bits come from its + * dtype and its groupSize from its scale's shape. */ const QUANT_METADATA_KEY = 'runntime.quant'; /** How a quantized file stores its int weights: `bits` per value, one f16 - * scale per `groupSize` values along each row. Empty in version 2, where - * each tensor sets both. */ + * scale per `groupSize` values along each row. A field the file leaves out + * comes from each tensor. */ interface QuantConfig { bits?: 4 | 8; groupSize?: number; @@ -621,18 +621,15 @@ function quantConfigOf(metadata: Record): QuantConfig | undefine return bad('is not a JSON object'); } const { version, bits, groupSize, scheme } = json as Record; - if (version !== 1 && version !== 2) return bad('has a version other than 1 or 2'); - if (scheme !== 'symmetric') return bad("needs scheme 'symmetric'"); - if (version === 2) { - if (bits !== undefined || groupSize !== undefined) { - return bad('sets bits or groupSize, which version 2 takes from each tensor'); - } - return {}; - } - if (bits !== 4 && bits !== 8) return bad('needs bits 4 or 8'); - if (typeof groupSize !== 'number' || !Number.isSafeInteger(groupSize) || groupSize <= 0) { + if (version !== 1) return bad('has a version other than 1'); + if (bits !== undefined && bits !== 4 && bits !== 8) return bad('needs bits 4 or 8'); + if ( + groupSize !== undefined && + (typeof groupSize !== 'number' || !Number.isSafeInteger(groupSize) || groupSize <= 0) + ) { return bad('needs a positive integer groupSize'); } + if (scheme !== 'symmetric') return bad("needs scheme 'symmetric'"); return { bits, groupSize }; } diff --git a/packages/runntime/test/core/weights/safetensors.test.ts b/packages/runntime/test/core/weights/safetensors.test.ts index 71060f11..33319bbf 100644 --- a/packages/runntime/test/core/weights/safetensors.test.ts +++ b/packages/runntime/test/core/weights/safetensors.test.ts @@ -375,12 +375,7 @@ describe('fromSafetensors', () => { it.each([ { name: 'text that is not JSON', metadata: { 'runntime.quant': '{bits' }, error: /JSON/ }, { name: 'a JSON array', metadata: quant([1, 8]), error: /not a JSON object/ }, - { name: 'version 3', metadata: quant({ ...valid, version: 3 }), error: /version/ }, - { - name: 'version 2 with bits and groupSize', - metadata: quant({ ...valid, version: 2 }), - error: /version 2 takes from each tensor/, - }, + { name: 'version 2', metadata: quant({ ...valid, version: 2 }), error: /version/ }, { name: 'no version', metadata: quant({ ...valid, version: undefined }), error: /version/ }, { name: 'bits 2', metadata: quant({ ...valid, bits: 2 }), error: /bits 4 or 8/ }, { name: "bits '8'", metadata: quant({ ...valid, bits: '8' }), error: /bits 4 or 8/ }, @@ -420,7 +415,7 @@ describe('fromSafetensors', () => { const int4Setting = { 'runntime.quant': '{"version":1,"bits":4,"groupSize":2,"scheme":"symmetric"}', }; - const v2Setting = { 'runntime.quant': '{"version":2,"scheme":"symmetric"}' }; + const perTensorSetting = { 'runntime.quant': '{"version":1,"scheme":"symmetric"}' }; const i8Bytes = (values: number[]) => new Uint8Array(new Int8Array(values).buffer); // f16 bits of 1, 0.5, 2 and -1: one scale per 2 columns of a [2, 4] weight. const scale: TensorSpec = { @@ -477,7 +472,7 @@ describe('fromSafetensors', () => { expect(sd.totalBytes()).toBe(4 + 8); }); - it('takes bits and groupSize from each tensor in a version 2 file', async () => { + it('takes bits and groupSize from each tensor when the file sets neither', async () => { const sd = await open( { 'a.weight': { dtype: 'I8', shape: [2, 4], bytes: new Uint8Array(8) }, @@ -487,7 +482,7 @@ describe('fromSafetensors', () => { 'c.weight': { dtype: 'I8', shape: [2, 4], bytes: new Uint8Array(8) }, 'c.weight_scale': { dtype: 'F16', shape: [2, 1], bytes: new Uint8Array(4) }, }, - v2Setting, + perTensorSetting, ); const meta = (name: string) => { const t = sd.tensors.get(name)!; @@ -499,13 +494,26 @@ describe('fromSafetensors', () => { expect(meta('c.weight')).toEqual({ shape: [2, 4], bits: 8, groupSize: 4 }); }); - it('rejects version 2 scales that do not split the columns evenly', async () => { + it('takes groupSize from the scale when the file sets only bits', async () => { + const sd = await open( + { + 'a.weight': { dtype: 'I8', shape: [2, 4], bytes: new Uint8Array(8) }, + 'a.weight_scale': scale, + }, + { 'runntime.quant': '{"version":1,"bits":8,"scheme":"symmetric"}' }, + ); + const t = sd.tensors.get('a.weight')!; + if (t.kind !== 'quant') throw new Error('expected a quantized tensor'); + expect({ bits: t.bits, groupSize: t.groupSize }).toEqual({ bits: 8, groupSize: 2 }); + }); + + it('rejects scales that do not split the columns evenly', async () => { const opening = open( { w: { dtype: 'I8', shape: [2, 4], bytes: new Uint8Array(8) }, w_scale: { dtype: 'F16', shape: [2, 3], bytes: new Uint8Array(12) }, }, - v2Setting, + perTensorSetting, ); await expect(opening).rejects.toMatchObject({ code: 'CHECKPOINT_MISMATCH', From b9e91adda66b8ffd12f9e20ea60e5b62dbe871ef Mon Sep 17 00:00:00 2001 From: Kamil Kaczmarczyk <143009372+xxpopielxx@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:01:02 +0100 Subject: [PATCH 4/4] refactor(core): read quantized conv scales where the torch layout puts them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit matmulQuantTiled takes a scale loader next to loadA. Conv1d and ConvTranspose1d load quantized weights as torch's [out, in·kernel] and [in, out·kernel], one scale per group of values along each row, so a quantizer needs no layer knowledge. --- packages/runntime/bench/kernels/conv.ab.ts | 110 ++++++++++--- .../runntime/src/core/gpu/dispatch/conv.ts | 57 ++++++- .../src/core/graph/ops/conv/conv1d.ts | 70 ++++++-- .../runntime/src/core/kernels/conv/conv1d.ts | 52 ++++++ .../core/kernels/matmul/matmulQuantTiled.ts | 64 ++++---- packages/runntime/src/core/nn/layers.ts | 150 +++++++++--------- .../test/core/graph/ops/conv/conv1d.test.ts | 45 ++++-- .../test/core/kernels/conv/conv1d.gpu.test.ts | 126 ++++++++++++--- .../test/core/kernels/matmul/quantW.ts | 36 +++-- .../runntime/test/core/nn/layers.gpu.test.ts | 55 +++---- packages/runntime/test/core/nn/layers.test.ts | 105 ++++++++---- 11 files changed, 620 insertions(+), 250 deletions(-) diff --git a/packages/runntime/bench/kernels/conv.ab.ts b/packages/runntime/bench/kernels/conv.ab.ts index 116fe558..049b2caa 100644 --- a/packages/runntime/bench/kernels/conv.ab.ts +++ b/packages/runntime/bench/kernels/conv.ab.ts @@ -91,8 +91,8 @@ function conv1d( }; } -/** conv1d with a quantW weight [k·cIn, cOut] and f32 scales [cIn / group, - * cOut], one per group of input channels in every tap. */ +/** conv1d with a quantW weight [k·cIn, cOut] and f32 scales [k·cIn / + * groupSize, cOut], one per groupSize values of a torch row [cOut, cIn·k]. */ function conv1dQuant( name: string, tIn: number, @@ -100,7 +100,7 @@ function conv1dQuant( cOut: number, k: number, bits: 8 | 4, - group: number, + groupSize: number, geometry: { padLeft?: number; padRight?: number; dilation?: number } = {}, dtype: 'f32' | 'f16' = 'f32', ): AbCase { @@ -109,9 +109,8 @@ function conv1dQuant( needs: dtype === 'f16' ? ['shader-f16'] : [], build: (version) => { const x = randMat(version, tIn, cIn, dtype); - // Groups of k·group rows give as many scales as the conv reads; their - // values do not matter for timing. - const { w, scales } = quantWeight(version, 1, k * cIn, cOut, bits, k * group); + // The scales' values do not matter for timing. + const { w, scales } = quantWeight(version, 1, k * cIn, cOut, bits, groupSize); const call = () => version.lib.conv1d(x, w, { kernelSize: k, @@ -119,7 +118,7 @@ function conv1dQuant( ...geometry, scales, bits, - groupSize: group, + groupSize, }); // Built once here, so a base without quantW conv1d skips the case. call(); @@ -128,6 +127,46 @@ function conv1dQuant( }; } +/** A quantized ConvTranspose1d cIn→out with kernel 2·stride, as the layer + * runs it: conv1d over 2 taps to stride·out channels, its scales [cIn, + * out·kernel / groupSize] in torch layout. */ +function convTransposeQuant( + name: string, + tIn: number, + cIn: number, + out: number, + stride: number, + bits: 8 | 4, + groupSize: number, + dtype: 'f32' | 'f16' = 'f32', +): AbCase { + const kernel = 2 * stride; + return { + name, + needs: dtype === 'f16' ? ['shader-f16'] : [], + build: (version) => { + const x = randMat(version, tIn, cIn, dtype); + // As many scales as torch [cIn, out·kernel] has; their values do not + // matter for timing. + const { w, scales } = quantWeight(version, 1, 2 * cIn, stride * out, bits, groupSize); + const call = () => + version.lib.conv1d(x, w, { + kernelSize: 2, + stride: 1, + padLeft: 1, + padRight: 1, + scales, + bits, + groupSize, + transposedKernel: kernel, + }); + // Built once here, so a base without it skips the case. + call(); + return call; + }, + }; +} + /** GLU then a causal depthwise conv over time: x [T, 2C], taps [K, C]. */ function gluConv1d(name: string, t: number, c: number, k: number, dtype: 'f32' | 'f16'): AbCase { return { @@ -331,31 +370,45 @@ abSuite('conv', [ padRight: 1, }), - // conv1d with a quantW weight: matmulQuantTiled, groups of 8 channels. - conv1dQuant('conv1d int8 same, Higgs conv_in 256→1024 k7, 10 s', 250, 256, 1024, 7, 8, 8, { + // conv1d with a quantW weight: matmulQuantTiled. Groups of 56 values are 8 + // channels of a k7 conv. + conv1dQuant('conv1d int8 same, Higgs conv_in 256→1024 k7, 10 s', 250, 256, 1024, 7, 8, 56, { padLeft: 3, padRight: 3, }), - conv1dQuant('conv1d int8 same, Higgs residual 64ch k7, 10 s', 80000, 64, 64, 7, 8, 8, { + conv1dQuant('conv1d int8 same, Higgs residual 64ch k7, 10 s', 80000, 64, 64, 7, 8, 56, { padLeft: 3, padRight: 3, }), conv1dQuant( - 'conv1d int8 same f16, Higgs residual 64ch k7, 10 s', + 'conv1d int8 same, groups of 32, Higgs residual 64ch k7, 10 s', 80000, 64, 64, 7, 8, + 32, + { + padLeft: 3, + padRight: 3, + }, + ), + conv1dQuant( + 'conv1d int8 same f16, Higgs residual 64ch k7, 10 s', + 80000, + 64, + 64, + 7, 8, + 56, { padLeft: 3, padRight: 3 }, 'f16', ), - conv1dQuant('conv1d int4 same, Higgs residual 64ch k7, 10 s', 80000, 64, 64, 7, 4, 8, { + conv1dQuant('conv1d int4 same, Higgs residual 64ch k7, 10 s', 80000, 64, 64, 7, 4, 56, { padLeft: 3, padRight: 3, }), - conv1dQuant('conv1d int8 dilated, Higgs residual 64ch k7 d9, 10 s', 80000, 64, 64, 7, 8, 8, { + conv1dQuant('conv1d int8 dilated, Higgs residual 64ch k7 d9, 10 s', 80000, 64, 64, 7, 8, 56, { padLeft: 27, padRight: 27, dilation: 9, @@ -367,23 +420,32 @@ abSuite('conv', [ padLeft: 3, padRight: 3, }), - conv1dQuant('conv1d int8 same, Higgs output 32→1 k7 as 4 columns, 10 s', 240000, 32, 4, 7, 8, 8, { - padLeft: 3, - padRight: 3, - }), conv1dQuant( - 'conv1d int8, Higgs upsampling 1024→8·512 as 2 taps, 10 s', + 'conv1d int8 same, Higgs output 32→1 k7 as 4 columns, 10 s', + 240000, + 32, + 4, + 7, + 8, + 56, + { + padLeft: 3, + padRight: 3, + }, + ), + // ConvTranspose1d with a quantW weight: each column reads its own scale. + convTransposeQuant('convT int8, Higgs upsampling 1024→512 s8, 10 s', 250, 1024, 512, 8, 8, 32), + convTransposeQuant( + 'convT int8 f16, Higgs upsampling 1024→512 s8, 10 s', 250, 1024, - 4096, - 2, + 512, 8, 8, - { - padLeft: 1, - padRight: 1, - }, + 32, + 'f16', ), + convTransposeQuant('convT int8, Higgs upsampling 64→32 s3, 10 s', 80000, 64, 32, 3, 8, 32), // gluConv1d: its own kernel. gluConv1d('gluConv1d, EmbeddingGemma 2 audio 30 s, 1024 channels k5', 750, 1024, 5, 'f32'), diff --git a/packages/runntime/src/core/gpu/dispatch/conv.ts b/packages/runntime/src/core/gpu/dispatch/conv.ts index 77a9d1e5..39cec55c 100644 --- a/packages/runntime/src/core/gpu/dispatch/conv.ts +++ b/packages/runntime/src/core/gpu/dispatch/conv.ts @@ -1,7 +1,13 @@ /** Dispatch specs for the convolution kernels (kernels/conv). */ -import { elemFor } from '../../kernels/elem.ts'; -import { conv1dHandle, conv1dLoadA, createConv1dPipeline } from '../../kernels/conv/conv1d.ts'; +import { type Elem, elemFor } from '../../kernels/elem.ts'; +import { + conv1dHandle, + conv1dLoadA, + conv1dScales, + convTransposeScales, + createConv1dPipeline, +} from '../../kernels/conv/conv1d.ts'; import { createGluConv1dPipeline, gluConv1dHandle } from '../../kernels/conv/gluConv1d.ts'; import { createMatmulQuantTiledPipeline, @@ -47,11 +53,49 @@ import { } from '../../kernels/conv/hwc4.ts'; import { defineSpec, floatDtype, narrow, narrowFloat } from './spec.ts'; +/** The scale reader of a quantW conv1d: where the torch weight of a Conv1d, + * or of a ConvTranspose1d when transposedKernel is set, has its scales. */ +function quantConvScales( + elem: Elem, + cfg: { cIn: number; cOut: number; kernel: number; groupSize: number; transposedKernel: number }, +) { + const layout = matmulQuantTiledLayout(elem); + const { cIn, cOut, kernel, groupSize, transposedKernel } = cfg; + if (transposedKernel === 0) return conv1dScales({ cIn, kernel, groupSize, n: cOut }, layout); + // The ConvTranspose1d's stride, not this conv's. + const stride = transposedKernel / kernel; + return convTransposeScales( + { cIn, out: cOut / stride, kernel: transposedKernel, stride, groupSize }, + layout, + ); +} + export const conv1dSpec = defineSpec({ attrs: (p) => { - const [kernel, stride, padLeft, padRight, hasBias, act, dilation, bits, groupSize] = - p.attrs as [number, number, number, number, number, number, number, number, number]; - return { kernel, stride, padLeft, padRight, hasBias, act, dilation, bits, groupSize }; + const [ + kernel, + stride, + padLeft, + padRight, + hasBias, + act, + dilation, + bits, + groupSize, + transposedKernel, + ] = p.attrs as [number, number, number, number, number, number, number, number, number, number]; + return { + kernel, + stride, + padLeft, + padRight, + hasBias, + act, + dilation, + bits, + groupSize, + transposedKernel, + }; }, cfg: (node, attrs) => { const cIn = node.pending!.inputs[0]!.shape.dims![1]!; @@ -75,6 +119,7 @@ export const conv1dSpec = defineSpec({ gemm: gemm ? (mapped ? 2 : 1) : 0, bits: attrs.bits, groupSize: attrs.groupSize, + transposedKernel: attrs.transposedKernel, }; }, pipeline: (root, dtype, cfg) => @@ -90,12 +135,12 @@ export const conv1dSpec = defineSpec({ hasAdd: 0, act: cfg.act, aStride: cfg.stride * cfg.cIn, - period: cfg.cIn, }, elemFor(dtype), cfg.gemm === 2 ? conv1dLoadA(elemFor(dtype), cfg, matmulQuantTiledLayout(elemFor(dtype))) : undefined, + quantConvScales(elemFor(dtype), cfg), ) : cfg.gemm ? createMatmulTiledVec4Pipeline( diff --git a/packages/runntime/src/core/graph/ops/conv/conv1d.ts b/packages/runntime/src/core/graph/ops/conv/conv1d.ts index 29a719c8..42360c5d 100644 --- a/packages/runntime/src/core/graph/ops/conv/conv1d.ts +++ b/packages/runntime/src/core/graph/ops/conv/conv1d.ts @@ -16,9 +16,13 @@ import { ACT_CODE, FLOAT_DTYPES, want2d, wantQuantW, type SlotAct } from '../sha * * `bias` and `activation` fold into the epilogue as act(conv + bias). * - * A quantW w takes f32 `scales` [C_in / groupSize, C_out], `bits` and - * `groupSize`: each group of groupSize input channels has one scale per - * output channel, shared by every tap. */ + * A quantW w takes f32 `scales`, `bits` and `groupSize`, with the scales of + * the torch weight: one per groupSize values of a torch row. For a Conv1d, + * torch rows are [C_out, C_in·kernelSize] and the scales are stored + * [C_in·kernelSize / groupSize, C_out]. `transposedKernel` marks the weight + * of a ConvTranspose1d with that kernel, run as this conv: torch rows are + * [C_in, out·transposedKernel] with out = C_out·kernelSize / + * transposedKernel a multiple of 4, and the scales are stored as is. */ export function conv1d( x: Value, w: Value, @@ -34,9 +38,11 @@ export function conv1d( scales?: Value; bits?: 8 | 4; groupSize?: number; + transposedKernel?: number; }, ): Value { const { kernelSize, stride, padding, bias, activation, scales, bits, groupSize } = opts; + const transposedKernel = opts.transposedKernel ?? 0; const dilation = opts.dilation ?? 1; if (padding !== undefined && (opts.padLeft !== undefined || opts.padRight !== undefined)) { throw new Error('conv1d: padding excludes padLeft/padRight — give one form or the other'); @@ -50,20 +56,18 @@ export function conv1d( throw new Error('conv1d: quantW weights need scales, bits and groupSize'); } [wRows, cOut] = wantQuantW('conv1d', w, scales, bits); - if (!matmulQuantTiledEligible(kernelSize * cIn, cOut, groupSize, cIn)) { + if (cIn % 4 !== 0 || !matmulQuantTiledEligible(kernelSize * cIn, cOut)) { throw new Error( - `conv1d: a quantW weight needs groupSize a multiple of 4, C_in a multiple of groupSize, kernelSize·C_in a multiple of ${MATMUL_QUANT_TILED_BK} and C_out of 4, got C_in ${cIn}, C_out ${cOut}, groupSize ${groupSize}`, + `conv1d: a quantW weight needs C_in and C_out multiples of 4 and kernelSize·C_in of ${MATMUL_QUANT_TILED_BK}, got kernelSize ${kernelSize}, C_in ${cIn}, C_out ${cOut}`, ); } - if (scales.shape.elems !== (cIn / groupSize) * cOut) { + checkQuantScales(scales, groupSize, { cIn, cOut, kernelSize, transposedKernel }); + } else { + if (scales || bits || groupSize || transposedKernel) { throw new Error( - `conv1d: scales must be f32 [C_in/groupSize, C_out] = [${cIn / groupSize}, ${cOut}]`, + 'conv1d: scales/bits/groupSize/transposedKernel apply to quantW weights only', ); } - } else { - if (scales || bits || groupSize) { - throw new Error('conv1d: scales/bits/groupSize apply to quantW weights only'); - } let wDtype: string; [wRows, cOut, wDtype] = want2d('conv1d.w', w, FLOAT_DTYPES); if (wDtype !== dtype) { @@ -123,10 +127,52 @@ export function conv1d( 'conv1d', [x, w, ...(scales ? [scales] : []), ...(bias ? [bias] : [])], undefined, - [kernelSize, stride, padLeft, padRight, bias ? 1 : 0, act, dilation, bits ?? 0, groupSize ?? 0], + [ + kernelSize, + stride, + padLeft, + padRight, + bias ? 1 : 0, + act, + dilation, + bits ?? 0, + groupSize ?? 0, + transposedKernel, + ], ); } +/** Throws unless `scales` hold one scale per groupSize values of each torch + * row of a quantW conv1d weight, as conv1d describes. */ +function checkQuantScales( + scales: Value, + groupSize: number, + geom: { cIn: number; cOut: number; kernelSize: number; transposedKernel: number }, +): void { + const { cIn, cOut, kernelSize, transposedKernel } = geom; + let rows = cOut; + let cols = cIn * kernelSize; + if (transposedKernel > 0) { + const stride = transposedKernel / kernelSize; + if (!Number.isInteger(stride) || cOut % (4 * stride) !== 0) { + throw new Error( + `conv1d: transposedKernel ${transposedKernel} must be kernelSize ${kernelSize} times a stride s with C_out ${cOut} a multiple of 4·s`, + ); + } + rows = cIn; + cols = (cOut / stride) * transposedKernel; + } + if (!Number.isInteger(groupSize) || groupSize < 1 || cols % groupSize !== 0) { + throw new Error(`conv1d: groupSize ${groupSize} must divide the torch row of ${cols} values`); + } + const elems = (rows * cols) / groupSize; + if (scales.shape.elems !== elems) { + throw new Error( + `conv1d: scales must be f32 with ${elems} elems, one per ${groupSize} values of ${rows} torch rows, got ${scales.shape.elems}`, + ); + } +} + /** GLU then a causal depthwise conv over time, in one dispatch. x [T, 2C] is * clamped to [lo, hi] and gated, first half by σ of the second; w tap-major * [kernelSize, C] then mixes each channel's last kernelSize frames. diff --git a/packages/runntime/src/core/kernels/conv/conv1d.ts b/packages/runntime/src/core/kernels/conv/conv1d.ts index b6573458..26139e6e 100644 --- a/packages/runntime/src/core/kernels/conv/conv1d.ts +++ b/packages/runntime/src/core/kernels/conv/conv1d.ts @@ -10,6 +10,7 @@ import { } from '../common.ts'; import { activationFor, actSlot } from '../activations.ts'; import { type Elem, F32_ELEM } from '../elem.ts'; +import type { ScaleAt, ScaleSource } from '../matmul/matmulQuantTiled.ts'; import { type ASource, type LoadA, matmulTiledVec4Layout } from '../matmul/matmulTiledVec4.ts'; /** Direct strided, dilated 1-D convolution, time-major. @@ -241,3 +242,54 @@ export function conv1dLoadA( return v; }; } + +/** Reads the scales of a Conv1d weight quantized in torch layout [out, + * in·kernel], one per groupSize values of a row, stored as [in·kernel / + * groupSize, n]. Row tap·cIn + c of W is value c·kernel + tap of the torch + * row, so a group can end inside a channel. */ +export function conv1dScales( + geom: { cIn: number; kernel: number; groupSize: number; n: number }, + source: ScaleSource, +): ScaleAt { + const { cIn, kernel, groupSize } = geom; + const n4 = geom.n / 4; + return (row: number, col4: number) => { + 'use gpu'; + const g = std.intdiv((row % cIn) * kernel + std.intdiv(row, cIn), groupSize); + return d.vec4f(source.$.scales[g * n4 + col4]!); + }; +} + +/** Reads the scales of a ConvTranspose1d weight quantized in torch layout + * [in, out·kernel], one per groupSize values of a row, stored as is. The + * layer runs as a conv over kernel / stride taps: row i·cIn + c, column + * r·out + o of W is kernel position (taps − 1 − i)·stride + r of input c and + * output o. A group can span several outputs, so each column reads its own + * scale. out is a multiple of 4. */ +export function convTransposeScales( + geom: { cIn: number; out: number; kernel: number; stride: number; groupSize: number }, + source: ScaleSource, +): ScaleAt { + const { cIn, out, kernel, stride, groupSize } = geom; + const last = kernel / stride - 1; + const groups = (out * kernel) / groupSize; + const scale = (i: number) => { + 'use gpu'; + return source.$.scales[std.intdiv(i, 4)]![i % 4]!; + }; + return (row: number, col4: number) => { + 'use gpu'; + const c = row % cIn; + const col = col4 * 4; + // The 4 columns share a frame r and hold outputs o to o + 3, kernel + // values apart in the torch row. + const at = (col % out) * kernel + (last - std.intdiv(row, cIn)) * stride + std.intdiv(col, out); + const base = c * groups; + return d.vec4f( + scale(base + std.intdiv(at, groupSize)), + scale(base + std.intdiv(at + kernel, groupSize)), + scale(base + std.intdiv(at + 2 * kernel, groupSize)), + scale(base + std.intdiv(at + 3 * kernel, groupSize)), + ); + }; +} diff --git a/packages/runntime/src/core/kernels/matmul/matmulQuantTiled.ts b/packages/runntime/src/core/kernels/matmul/matmulQuantTiled.ts index 6c058a69..bc755f0f 100644 --- a/packages/runntime/src/core/kernels/matmul/matmulQuantTiled.ts +++ b/packages/runntime/src/core/kernels/matmul/matmulQuantTiled.ts @@ -4,12 +4,9 @@ * * Each step unpacks a slice of W into workgroup memory as floats, so the * inner loop is a plain float matmul with matmulTiledVec4's tiling. A - * loader reads a, so conv1d runs on this kernel too. - * - * Row k of W uses scale row (k mod period) / groupSize. For a Linear, period - * is K. For a conv, K holds the input channels of each tap in turn and - * period is the input channel count, so a group of channels has one scale - * for all taps. */ + * loader reads a and another reads the scales, so conv1d runs on this kernel + * too, its scales where its torch weight has them. By default row k of W + * uses scale row k / groupSize. */ import tgpu, { d, std } from 'typegpu'; import type { TgpuRoot } from 'typegpu'; @@ -31,7 +28,7 @@ function makeLayout(elem: Elem) { return tgpu.bindGroupLayout({ a: { storage: d.arrayOf(elem.vec4), access: 'readonly' }, w: { storage: d.arrayOf(d.u32), access: 'readonly' }, - scales: { storage: d.arrayOf(d.vec4f), access: 'readonly' }, // [K/group, N/4] + scales: { storage: d.arrayOf(d.vec4f), access: 'readonly' }, // read through scaleAt bias: { storage: d.arrayOf(elem.vec4), access: 'readonly' }, // [N/4]; dead when hasBias=0 addend: { storage: d.arrayOf(elem.vec4), access: 'readonly' }, // [M,N/4]; dead when hasAdd=0 out: { storage: d.arrayOf(elem.vec4), access: 'mutable' }, @@ -51,8 +48,17 @@ function layoutFor(elem: Elem): ReturnType { const loadASlot = tgpu.slot(); -/** The layout every pipeline of this kernel binds, so a loader can read a and - * aRows. */ +/** The scales of row `row` of W in columns 4·col4 to 4·col4 + 3. */ +export type ScaleAt = (row: number, col4: number) => d.v4f; +const scaleAtSlot = tgpu.slot(); + +/** What a scale loader reads: the scales, bound as vec4s. */ +export interface ScaleSource { + readonly $: { readonly scales: d.v4f[] }; +} + +/** The layout every pipeline of this kernel binds, so a loader can read a, + * aRows and the scales. */ export function matmulQuantTiledLayout(elem: Elem): ReturnType { return layoutFor(elem); } @@ -60,8 +66,16 @@ export function matmulQuantTiledLayout(elem: Elem): ReturnType { + 'use gpu'; + return d.vec4f(layout.$.scales[std.intdiv(row, groupSize) * n4 + col4]!); + }; } // Each workgroup computes a BM×BN tile of out, each thread TM rows × TN @@ -75,20 +89,15 @@ const TN = 1; export const MATMUL_QUANT_TILED_BK = 8; const BK = MATMUL_QUANT_TILED_BK; -/** Whether a [M,K]·[K,N] quantW matmul with scale period `period` can use - * this kernel. */ -export function matmulQuantTiledEligible( - k: number, - n: number, - groupSize: number, - period = k, -): boolean { - return k % BK === 0 && n % 4 === 0 && groupSize % 4 === 0 && period % groupSize === 0; +/** Whether a [M,K]·[K,N] quantW matmul can use this kernel. groupSize also + * checks the default scale groups. */ +export function matmulQuantTiledEligible(k: number, n: number, groupSize?: number): boolean { + const grouped = groupSize === undefined || (groupSize % 4 === 0 && k % groupSize === 0); + return k % BK === 0 && n % 4 === 0 && grouped; } function makeMatmulQuantTiledKernel(cfg: MatmulQuantTiledCfg, elem: Elem) { - const { k, n, bits, groupSize, hasBias, hasAdd } = cfg; - const period = cfg.period ?? k; + const { k, n, bits, hasBias, hasAdd } = cfg; const layout = layoutFor(elem); const storeVec4 = elem.vec4; const bk4 = BK / 4; @@ -149,8 +158,7 @@ function makeMatmulQuantTiledKernel(cfg: MatmulQuantTiledCfg, elem: Elem) { bk % 4, bits, ); - const row = std.intdiv(bk % period, groupSize); - wv = storeVec4(q * layout.$.scales[row * n4 + bc4]!); + wv = storeVec4(q * scaleAtSlot.$(bk, bc4)); } tileB.$[e] = storeVec4(wv); } @@ -200,18 +208,20 @@ function makeMatmulQuantTiledKernel(cfg: MatmulQuantTiledCfg, elem: Elem) { }); } -/** One pipeline per dtype, K, N, bits, group size, scale period, loader, - * epilogue and activation. `loadA` reads a; by default it reads rows - * aStride apart. */ +/** One pipeline per dtype, K, N, bits, group size, loaders, epilogue and + * activation. `loadA` reads a; by default it reads rows aStride apart. + * `scaleAt` reads the scales; by default [K / groupSize, N]. */ export function createMatmulQuantTiledPipeline( root: TgpuRoot, cfg: MatmulQuantTiledCfg, elem: Elem = F32_ELEM, loadA: LoadA = stridedLoadA(elem, cfg.aStride ?? cfg.k, layoutFor(elem)), + scaleAt: ScaleAt = groupedScales(elem, cfg.n, cfg.groupSize), ) { return root .with(actSlot, activationFor(cfg.act)) .with(loadASlot, loadA) + .with(scaleAtSlot, scaleAt) .createComputePipeline({ compute: makeMatmulQuantTiledKernel(cfg, elem) }); } diff --git a/packages/runntime/src/core/nn/layers.ts b/packages/runntime/src/core/nn/layers.ts index 32b894a9..7be374c9 100644 --- a/packages/runntime/src/core/nn/layers.ts +++ b/packages/runntime/src/core/nn/layers.ts @@ -105,7 +105,7 @@ export class Embedding extends Module<[readonly number[] | Value], Value> { } /** Turns `weight` into quantW words [k, n], packed from `ints` with one row - * of k per output, and puts its f32 scales [groups, n] right after it as + * of k per output, and puts its f32 scales right after it as * `.weightScale`, so both bind in file order. The words carry t's * whole download size. Returns the scale parameter. */ function loadQuantWeight( @@ -113,18 +113,15 @@ function loadQuantWeight( prefix: string, weight: Parameter, t: QuantTensor, - shape: { k: number; n: number; groups: number }, + shape: { k: number; n: number }, ints: () => Promise, - scales: () => Promise, + scales: { dims: number[]; data: () => Promise }, ): Parameter { - const { k, n, groups } = shape; + const { k, n } = shape; const at = (name: string) => (prefix ? `${prefix}.${name}` : name); weight.retype('quantW', (k * n * t.bits) / 32, [k, n]); - const scale = new Parameter( - { elems: groups * n, dtype: 'f32', dims: [groups, n] }, - undefined, - true, - ); + const elems = scales.dims.reduce((a, b) => a * b, 1); + const scale = new Parameter({ elems, dtype: 'f32', dims: scales.dims }, undefined, true); const words = derivedTensor( [k, n], async () => packQuantW(await ints(), k, n, t.bits), @@ -134,16 +131,23 @@ function loadQuantWeight( sd.tensors.clear(); for (const [key, tensor] of entries) { sd.tensors.set(key, key === at('weight') ? words : tensor); - if (key === at('weight')) sd.tensors.set(at('weightScale'), derivedTensor([groups, n], scales)); + if (key === at('weight')) { + sd.tensors.set(at('weightScale'), derivedTensor(scales.dims, scales.data)); + } } return scale; } +/** `t`'s scales in f32, as stored. */ +async function scalesF32(t: QuantTensor): Promise { + const data = await t.scales(); + return data instanceof Float32Array ? data : f16ToF32(data); +} + /** `t`'s scales, torch's [out, groups], as the kernels' [groups, n] in f32. * Columns past out stay 0. */ async function scaleColumns(t: QuantTensor, groups: number, n: number): Promise { - const data = await t.scales(); - const f32 = data instanceof Float32Array ? data : f16ToF32(data); + const f32 = await scalesF32(t); const out = new Float32Array(groups * n); for (let o = 0; o < t.shape[0]!; o++) { for (let g = 0; g < groups; g++) out[g * n + o] = f32[o * groups + g]!; @@ -232,9 +236,9 @@ export class Linear extends Module { prefix, this.weight, tensor, - { k, n, groups }, + { k, n }, () => tensor.ints(), - () => scaleColumns(tensor, groups, n), + { dims: [groups, n], data: () => scaleColumns(tensor, groups, n) }, ); } forward(x: Value, call: { addend?: Value } = {}): Value { @@ -251,37 +255,25 @@ export class Linear extends Module { } } -/** The number of input channels in each scale group of a quantized conv - * weight [out, in·kernel]. A group holds whole channels with all their - * kernel values. Throws CHECKPOINT_MISMATCH when the weight does not fit - * the quantized conv on n output columns. */ -function quantConvGroup( +/** Throws CHECKPOINT_MISMATCH unless the quantized conv weight `t` is its + * torch weight with the dims after the first flattened, [rows, cols], and + * the conv runs on the quantized matmul over `taps` taps to n columns. */ +function checkQuantConv( layer: string, name: string, t: QuantTensor, - shape: { cIn: number; cOut: number; kernel: number; taps: number; n: number }, -): number { - const { cIn, cOut, kernel, taps, n } = shape; + shape: { rows: number; cols: number; layout: string; cIn: number; taps: number; n: number }, +): void { + const { rows, cols, layout, cIn, taps, n } = shape; const bad = (why: string): never => { throw new RunntimeError('CHECKPOINT_MISMATCH', `${layer}: ${name} ${why}`); }; - if (t.shape[0] !== cOut || t.shape[1] !== cIn * kernel) { - bad( - `is quantized as [${t.shape.join(', ')}], expected [out, in·kernel] = [${cOut}, ${cIn * kernel}]`, - ); + if (t.shape[0] !== rows || t.shape[1] !== cols) { + bad(`is quantized as [${t.shape.join(', ')}], expected ${layout} = [${rows}, ${cols}]`); } - const group = t.groupSize / kernel; - if (!Number.isInteger(group)) { - bad(`has groups of ${t.groupSize} values, not whole channels of ${kernel} values`); - } - if (!matmulQuantTiledEligible(taps * cIn, n, group, cIn)) { - bad( - `is int${t.bits} in groups of ${group} channels, which needs a group a multiple of 4, ` + - `in channels a multiple of the group and ${taps}·in a multiple of ` + - `${MATMUL_QUANT_TILED_BK}, got in ${cIn}`, - ); + if (cIn % 4 !== 0 || !matmulQuantTiledEligible(taps * cIn, n)) { + bad(`needs in a multiple of 4 and ${taps}·in of ${MATMUL_QUANT_TILED_BK}, got in ${cIn}`); } - return group; } /** Strided, dilated 1D convolution over time-major [T, inChannels]. No batch @@ -293,10 +285,11 @@ function quantConvGroup( * weight is stored tap-major and permutes from torch layout at load. bias and * `activation` fold into the conv epilogue. * - * A quantized checkpoint weight loads as quantW words with its f32 scales in - * `weightScale` [in / group, out], one per group of input channels, the same - * for every tap. Output channels pad up to the packing width and are cut - * after the conv. */ + * A quantized checkpoint weight is torch's [out, in·kernel], one scale per + * group of values along each row. It loads as quantW words with its f32 + * scales in `weightScale` [in·kernel / groupSize, out]. Output channels pad + * up to the packing width, scales and bias with them, and are cut after the + * conv. */ export class Conv1d extends Module { readonly weight: Parameter; readonly bias?: Parameter; @@ -375,22 +368,29 @@ export class Conv1d extends Module { ); } /** Loads `name` as quantW words [kernel·in, n] with its scales - * [in / group, n], where n is outChannels padded to the packing width. The - * bias pads to n too. */ + * [in·kernel / groupSize, n], where n is outChannels padded to the packing + * width. The bias pads to n too. */ private loadQuantized(sd: LazyStateDict, prefix: string, name: string, t: QuantTensor): void { const { inChannels: cIn, outChannels: cOut, kernelSize: kernel } = this; const width = 32 / t.bits; const n = Math.ceil(cOut / width) * width; - const group = quantConvGroup('Conv1d', name, t, { cIn, cOut, kernel, taps: kernel, n }); const k = kernel * cIn; - const groups = cIn / group; - this.quantization = { bits: t.bits, groupSize: group }; + checkQuantConv('Conv1d', name, t, { + rows: cOut, + cols: k, + layout: '[out, in·kernel]', + cIn, + taps: kernel, + n, + }); + const groups = k / t.groupSize; + this.quantization = { bits: t.bits, groupSize: t.groupSize }; this.weightScale = loadQuantWeight( sd, prefix, this.weight, t, - { k, n, groups }, + { k, n }, async () => { // torch's [out, in, kernel] to one tap-major row per output const src = await t.ints(); @@ -404,7 +404,7 @@ export class Conv1d extends Module { } return out; }, - () => scaleColumns(t, groups, n), + { dims: [groups, n], data: () => scaleColumns(t, groups, n) }, ); const biasName = prefix ? `${prefix}.bias` : 'bias'; const b = sd.tensors.get(biasName); @@ -458,9 +458,10 @@ export class Conv1d extends Module { * unfolds them and a slice cuts the padding. The weight permutes from torch * layout [in, out, kernel] at load, and the bias repeats per frame. * - * A quantized checkpoint weight is the equivalent Conv1d weight: [out, - * in·kernel] with the kernel reversed. stride·outChannels must be a - * multiple of the packing width. */ + * A quantized checkpoint weight is torch's [in, out·kernel], one scale per + * group of values along each row. It loads as quantW words with its f32 + * scales as stored in `weightScale`. outChannels must be a multiple of 4 + * and stride·outChannels of the packing width. */ export class ConvTranspose1d extends Module { readonly weight: Parameter; readonly bias?: Parameter; @@ -562,30 +563,38 @@ export class ConvTranspose1d extends Module { } } /** Loads `name` as quantW words [taps·in, stride·out] with its scales - * [in / group, stride·out]; each output's scales repeat for every frame. */ + * [in, out·kernel / groupSize] as stored. */ private loadQuantized(sd: LazyStateDict, prefix: string, name: string, t: QuantTensor): void { const { inChannels: cIn, outChannels: cOut, kernelSize: kernel, taps, stride: s } = this; const n = s * cOut; - if (n % (32 / t.bits) !== 0) { + if (cOut % 4 !== 0 || n % (32 / t.bits) !== 0) { throw new RunntimeError( 'CHECKPOINT_MISMATCH', - `ConvTranspose1d: ${name} is int${t.bits}, which needs stride·out a multiple of ` + - `${32 / t.bits}, got ${n}`, + `ConvTranspose1d: ${name} is int${t.bits}, which needs out a multiple of 4 and ` + + `stride·out of ${32 / t.bits}, got out ${cOut}, stride ${s}`, ); } - const group = quantConvGroup('ConvTranspose1d', name, t, { cIn, cOut, kernel, taps, n }); const k = taps * cIn; - const groups = cIn / group; - this.quantization = { bits: t.bits, groupSize: group }; + const cols = cOut * kernel; + checkQuantConv('ConvTranspose1d', name, t, { + rows: cIn, + cols, + layout: '[in, out·kernel]', + cIn, + taps, + n, + }); + this.quantization = { bits: t.bits, groupSize: t.groupSize }; + const groups = cols / t.groupSize; this.weightScale = loadQuantWeight( sd, prefix, this.weight, t, - { k, n, groups }, + { k, n }, async () => { - // Column r·out + o, row i·in + c takes kernel position - // (taps − 1 − i)·s + r. The file holds the kernel reversed. + // Row i·in + c, column r·out + o takes kernel position + // (taps − 1 − i)·s + r. const src = await t.ints(); const out = new Int8Array(n * k); for (let r = 0; r < s; r++) { @@ -593,23 +602,14 @@ export class ConvTranspose1d extends Module { for (let i = 0; i < taps; i++) { const kk = (taps - 1 - i) * s + r; for (let c = 0; c < cIn; c++) { - out[(r * cOut + o) * k + i * cIn + c] = - src[(o * cIn + c) * kernel + kernel - 1 - kk]!; + out[(r * cOut + o) * k + i * cIn + c] = src[c * cols + o * kernel + kk]!; } } } } return out; }, - async () => { - const base = await scaleColumns(t, groups, cOut); - const out = new Float32Array(groups * n); - for (let g = 0; g < groups; g++) { - for (let r = 0; r < s; r++) - out.set(base.subarray(g * cOut, (g + 1) * cOut), g * n + r * cOut); - } - return out; - }, + { dims: [cIn, groups], data: () => scalesF32(t) }, ); } forward(x: Value): Value { @@ -622,7 +622,11 @@ export class ConvTranspose1d extends Module { padRight: taps - 1, bias: this.bias?.value, ...(this.weightScale && this.quantization - ? { scales: this.weightScale.value, ...this.quantization } + ? { + scales: this.weightScale.value, + ...this.quantization, + transposedKernel: this.kernelSize, + } : {}), }); const rows = (t + taps - 1) * s; diff --git a/packages/runntime/test/core/graph/ops/conv/conv1d.test.ts b/packages/runntime/test/core/graph/ops/conv/conv1d.test.ts index 3d24ed2a..bcf51254 100644 --- a/packages/runntime/test/core/graph/ops/conv/conv1d.test.ts +++ b/packages/runntime/test/core/graph/ops/conv/conv1d.test.ts @@ -35,13 +35,14 @@ describe('conv1d', () => { const c = conv1d(x, w, { kernelSize: 3, stride: 1, bias, activation: 'gelu' }); expect(c.pending?.inputs).toEqual([x, w, bias]); // attrs: kernel, stride, padLeft, padRight, hasBias, act, dilation, bits, - // groupSize - expect(c.pending?.attrs?.slice(4)).toEqual([1, ACT_CODE.gelu, 1, 0, 0]); + // groupSize, transposedKernel + expect(c.pending?.attrs?.slice(4)).toEqual([1, ACT_CODE.gelu, 1, 0, 0, 0]); }); it('takes a quantW weight with its scales before the bias', () => { const w = quantWeight(3 * 8, 4, 8); - const scales = input([2, 4]); + // groups of 4 values of each torch row [4, 8·3] + const scales = input([6, 4]); const bias = input([4]); const c = conv1d(input([T, 8]), w, { kernelSize: 3, @@ -53,7 +54,22 @@ describe('conv1d', () => { }); expect(c.shape).toEqual(meta([8, 4])); expect(c.pending?.inputs).toEqual([expect.anything(), w, scales, bias]); - expect(c.pending?.attrs?.slice(7)).toEqual([8, 4]); + expect(c.pending?.attrs?.slice(7)).toEqual([8, 4, 0]); + }); + + it('takes the scales of a transposed conv weight in its torch layout', () => { + // A ConvTranspose1d 8→4 with kernel 4 and stride 2 runs over 2 taps to + // 8 channels. Its torch rows are [8, 4·4]; groups of 4 give 4 per row. + const w = quantWeight(2 * 8, 8, 8); + const c = conv1d(input([T, 8]), w, { + kernelSize: 2, + stride: 1, + scales: input([8, 4]), + bits: 8, + groupSize: 4, + transposedKernel: 4, + }); + expect(c.pending?.attrs?.slice(7)).toEqual([8, 4, 4]); }); it.each([ @@ -63,21 +79,26 @@ describe('conv1d', () => { err: /quantW weights need scales, bits and groupSize/, }, { - name: 'C_in not a multiple of groupSize', - opts: { scales: input([2, 4]), bits: 8 as const, groupSize: 12 }, - err: /C_in a multiple of groupSize/, + name: 'a groupSize that does not divide the torch row', + opts: { scales: input([2, 4]), bits: 8 as const, groupSize: 10 }, + err: /groupSize 10 must divide the torch row of 24 values/, }, { name: 'kernelSize·C_in not a multiple of 8', x: input([T, 4]), w: quantWeight(3 * 4, 4, 8), - opts: { scales: input([1, 4]), bits: 8 as const, groupSize: 4 }, - err: /kernelSize·C_in a multiple of 8/, + opts: { scales: input([3, 4]), bits: 8 as const, groupSize: 4 }, + err: /kernelSize·C_in of 8/, }, { name: 'scales of the wrong size', - opts: { scales: input([1, 4]), bits: 8 as const, groupSize: 4 }, - err: /scales must be f32 \[C_in\/groupSize, C_out\] = \[2, 4\]/, + opts: { scales: input([2, 4]), bits: 8 as const, groupSize: 4 }, + err: /scales must be f32 with 24 elems, one per 4 values of 4 torch rows, got 8/, + }, + { + name: 'a transposedKernel that is not a multiple of kernelSize', + opts: { scales: input([8, 2]), bits: 8 as const, groupSize: 4, transposedKernel: 4 }, + err: /transposedKernel 4 must be kernelSize 3 times a stride s with C_out 4 a multiple of 4·s/, }, ])('refuses $name', ({ x = input([T, 8]), w = quantWeight(3 * 8, 4, 8), opts, err }) => { expect(() => conv1d(x, w, { kernelSize: 3, stride: 1, ...opts })).toThrow(err); @@ -87,7 +108,7 @@ describe('conv1d', () => { const w = weight(3); expect(() => conv1d(x, w, { kernelSize: 3, stride: 1, scales: input([1, COUT]), bits: 8, groupSize: 4 }), - ).toThrow(/scales\/bits\/groupSize apply to quantW weights only/); + ).toThrow(/scales\/bits\/groupSize\/transposedKernel apply to quantW weights only/); }); it('widens the window by the dilation', () => { diff --git a/packages/runntime/test/core/kernels/conv/conv1d.gpu.test.ts b/packages/runntime/test/core/kernels/conv/conv1d.gpu.test.ts index 336afda9..ce7c3770 100644 --- a/packages/runntime/test/core/kernels/conv/conv1d.gpu.test.ts +++ b/packages/runntime/test/core/kernels/conv/conv1d.gpu.test.ts @@ -4,7 +4,7 @@ import { conv1d, type SlotAct } from '../../../../src/core/graph/ops.ts'; import { matrix } from '../../../../src/core/graph/value.ts'; import { expectClose, gpu, randomF32, requireFeature, roundF16, upload } from '../../../gpu.ts'; import { ACTIVATION_REF } from '../matmul/checkMatmul.ts'; -import { randomQuantWeight } from '../matmul/quantW.ts'; +import { randomQuantWeight, type ScaleIndex } from '../matmul/quantW.ts'; const { root, ex } = await gpu(); @@ -70,20 +70,52 @@ async function checkConv1d(c: Case, dtype: 'f32' | 'f16') { expectClose(got, conv1dRef(x, w, c, bias), tol); } -/** conv1d with a quantW weight against conv1dRef on its dequantized values. - * The weight's rows hold each tap's input channels in turn, so a group of - * groupSize channels has one scale in every tap. */ -async function checkQuantConv1d( - c: Case & { bits: 4 | 8; groupSize: number }, - dtype: 'f32' | 'f16', -) { +type QuantCase = Case & { + bits: 4 | 8; + /** Values of a torch row per scale. */ + groupSize: number; + /** A ConvTranspose1d's kernel, run as this conv over kernelSize taps. */ + transposedKernel?: number; +}; + +/** Where the scales of a quantW conv1d weight are, in the torch layout of a + * Conv1d [C_out, C_in·kernelSize] or a ConvTranspose1d [C_in, out·kernel]. + * Row r, column col of the tap-major weight is one value of a torch row. */ +function convScaleIndex(c: QuantCase): ScaleIndex { + const { cIn, cOut, kernelSize, groupSize, transposedKernel: kernel } = c; + if (!kernel) { + const groups = (cIn * kernelSize) / groupSize; + return { + count: groups * cOut, + at: (r, col) => { + const [tap, ch] = [Math.floor(r / cIn), r % cIn]; + return Math.floor((ch * kernelSize + tap) / groupSize) * cOut + col; + }, + }; + } + const stride = kernel / kernelSize; + const out = cOut / stride; + const groups = (out * kernel) / groupSize; + return { + count: cIn * groups, + at: (r, col) => { + const [i, ch] = [Math.floor(r / cIn), r % cIn]; + const kk = (kernelSize - 1 - i) * stride + Math.floor(col / out); + return ch * groups + Math.floor(((col % out) * kernel + kk) / groupSize); + }, + }; +} + +/** conv1d with a quantW weight against conv1dRef on its dequantized values. */ +async function checkQuantConv1d(c: QuantCase, dtype: 'f32' | 'f16') { const { cIn, cOut, kernelSize, bits, groupSize } = c; const round = dtype === 'f16' ? roundF16 : (v: Float32Array) => v; const meta = (rows: number, cols: number) => ({ ...matrix(rows, cols), dtype }); const k = kernelSize * cIn; + const scaleIndex = convScaleIndex(c); const weight = randomQuantWeight( root, - { experts: 1, k, n: cOut, bits, groupSize, period: cIn }, + { experts: 1, k, n: cOut, bits, groupSize, scaleIndex }, 2, ); const x = round(randomF32(c.tIn * cIn, 1)); @@ -97,9 +129,10 @@ async function checkQuantConv1d( padRight: c.padRight, bias: bias && upload(root, bias, meta(1, cOut)), activation: c.activation, - scales: upload(root, weight.scales, [cIn / groupSize, cOut]), + scales: upload(root, weight.scales, [1, scaleIndex.count]), bits, groupSize, + transposedKernel: c.transposedKernel, }), ex, ); @@ -306,21 +339,21 @@ describe('conv1d kernel', () => { // A quantW weight always takes matmulQuantTiled: gemm when unpadded and // undilated, mapped gemm otherwise, with no minimum K. - const QUANT_SHAPES: { name: string; c: Case & { bits: 4 | 8; groupSize: number } }[] = [ + const QUANT_SHAPES: { name: string; c: QuantCase }[] = [ { name: 'int8, stride 2 (gemm)', - c: { tIn: 70, cIn: 16, cOut: 36, kernelSize: 4, stride: 2, bits: 8, groupSize: 8 }, + c: { tIn: 70, cIn: 16, cOut: 36, kernelSize: 4, stride: 2, bits: 8, groupSize: 32 }, }, { - name: 'int8, all channels in one group (gemm)', - c: { tIn: 40, cIn: 32, cOut: 12, kernelSize: 3, stride: 1, bits: 8, groupSize: 32 }, + name: 'int8, one group per torch row (gemm)', + c: { tIn: 40, cIn: 32, cOut: 12, kernelSize: 3, stride: 1, bits: 8, groupSize: 96 }, }, { name: 'int8, 1×1 under the float tiled minimum K (gemm)', c: { tIn: 50, cIn: 32, cOut: 32, kernelSize: 1, stride: 1, bits: 8, groupSize: 8 }, }, { - name: 'int8, same padding (mapped gemm)', + name: 'int8, same padding, groups that end inside a channel (mapped gemm)', c: { tIn: 45, cIn: 16, @@ -330,7 +363,7 @@ describe('conv1d kernel', () => { padLeft: 3, padRight: 3, bits: 8, - groupSize: 8, + groupSize: 16, }, }, { @@ -345,7 +378,7 @@ describe('conv1d kernel', () => { padLeft: 9, padRight: 9, bits: 8, - groupSize: 4, + groupSize: 28, }, }, { @@ -359,11 +392,11 @@ describe('conv1d kernel', () => { padLeft: 5, padRight: 7, bits: 8, - groupSize: 4, + groupSize: 32, }, }, { - name: 'int8, stride 2 with padding (mapped gemm)', + name: 'int8, stride 2 with padding, groups of 2 values (mapped gemm)', c: { tIn: 41, cIn: 16, @@ -373,12 +406,12 @@ describe('conv1d kernel', () => { padLeft: 2, padRight: 2, bits: 8, - groupSize: 8, + groupSize: 2, }, }, { name: 'int4, stride 2 (gemm)', - c: { tIn: 70, cIn: 16, cOut: 40, kernelSize: 4, stride: 2, bits: 4, groupSize: 8 }, + c: { tIn: 70, cIn: 16, cOut: 40, kernelSize: 4, stride: 2, bits: 4, groupSize: 32 }, }, { name: 'int4, same padding (mapped gemm)', @@ -391,7 +424,54 @@ describe('conv1d kernel', () => { padLeft: 2, padRight: 2, bits: 4, - groupSize: 16, + groupSize: 32, + }, + }, + // A ConvTranspose1d [C_in, out, kernel] runs over kernel / stride taps to + // stride·out channels, padded taps − 1 on each side. + { + name: 'int8, transposed kernel 8 stride 4 (mapped gemm)', + c: { + tIn: 12, + cIn: 16, + cOut: 4 * 8, + kernelSize: 2, + stride: 1, + padLeft: 1, + padRight: 1, + bits: 8, + groupSize: 32, + transposedKernel: 8, + }, + }, + { + name: 'int8, transposed kernel 16 stride 8, groups across outputs (mapped gemm)', + c: { + tIn: 9, + cIn: 32, + cOut: 8 * 12, + kernelSize: 2, + stride: 1, + padLeft: 1, + padRight: 1, + bits: 8, + groupSize: 24, + transposedKernel: 16, + }, + }, + { + name: 'int4, transposed kernel 4 stride 2 (mapped gemm)', + c: { + tIn: 10, + cIn: 8, + cOut: 2 * 8, + kernelSize: 2, + stride: 1, + padLeft: 1, + padRight: 1, + bits: 4, + groupSize: 32, + transposedKernel: 4, }, }, ]; @@ -415,7 +495,7 @@ describe('conv1d kernel', () => { ), )('$dtype quantW $route with $name', async ({ dtype, c, bias, activation }, ctx) => { if (dtype === 'f16') requireFeature(ctx, root, 'shader-f16'); - await checkQuantConv1d({ ...c, bias, activation, bits: 8, groupSize: 8 }, dtype); + await checkQuantConv1d({ ...c, bias, activation, bits: 8, groupSize: 32 }, dtype); }); // padding 'same' with an even kernel pads one more on the right, as torch does. diff --git a/packages/runntime/test/core/kernels/matmul/quantW.ts b/packages/runntime/test/core/kernels/matmul/quantW.ts index 638eadcb..4e172546 100644 --- a/packages/runntime/test/core/kernels/matmul/quantW.ts +++ b/packages/runntime/test/core/kernels/matmul/quantW.ts @@ -10,8 +10,8 @@ import { randomF32 } from '../../../gpu.ts'; * dequantizes to. */ export interface QuantWeight { w: Value; - /** f32 [period / groupSize, N]: one scale per group of rows in each - * column, the same in every period of rows. */ + /** f32 [E·K / groupSize, N] by default: one scale per group of rows in + * each column. */ scales: Float32Array; /** q · scale for row r and column c at [r·N + c], in f64. */ dequant: Float64Array; @@ -42,9 +42,16 @@ function packQuantW(q: Int8Array, experts: number, k: number, n: number, bits: 4 return words; } +/** Where a weight's scales are: `count` of them, the one of row r and + * column c at `at(r, c)`. */ +export interface ScaleIndex { + count: number; + at: (row: number, col: number) => number; +} + /** Random ints over the full signed range of `bits` and positive scales, - * packed and uploaded. Row r takes scale row (r mod period) / groupSize, - * period E·K when omitted. The buffer is destroyed when the test ends. */ + * packed and uploaded. Row r takes scale row r / groupSize unless `scaleIndex` + * says otherwise. The buffer is destroyed when the test ends. */ export function randomQuantWeight( root: TgpuRoot, { @@ -53,8 +60,18 @@ export function randomQuantWeight( n, bits, groupSize, - period = experts * k, - }: { experts: number; k: number; n: number; bits: 4 | 8; groupSize: number; period?: number }, + scaleIndex = { + count: ((experts * k) / groupSize) * n, + at: (r, c) => Math.floor(r / groupSize) * n + c, + }, + }: { + experts: number; + k: number; + n: number; + bits: 4 | 8; + groupSize: number; + scaleIndex?: ScaleIndex; + }, seed: number, ): QuantWeight { const rows = experts * k; @@ -62,13 +79,10 @@ export function randomQuantWeight( const q = Int8Array.from(randomF32(rows * n, seed), (v) => Math.min(half - 1, Math.floor(v * half)), ); - const scales = randomF32((period / groupSize) * n, seed + 1).map( - (v) => (Math.abs(v) + 0.1) / half, - ); + const scales = randomF32(scaleIndex.count, seed + 1).map((v) => (Math.abs(v) + 0.1) / half); const dequant = new Float64Array(rows * n); for (let r = 0; r < rows; r++) { - const row = Math.floor((r % period) / groupSize); - for (let c = 0; c < n; c++) dequant[r * n + c] = q[r * n + c]! * scales[row * n + c]!; + for (let c = 0; c < n; c++) dequant[r * n + c] = q[r * n + c]! * scales[scaleIndex.at(r, c)]!; } const words = packQuantW(q, experts, k, n, bits); const buffer = uploadU32(root, words); diff --git a/packages/runntime/test/core/nn/layers.gpu.test.ts b/packages/runntime/test/core/nn/layers.gpu.test.ts index 5344f528..d9348923 100644 --- a/packages/runntime/test/core/nn/layers.gpu.test.ts +++ b/packages/runntime/test/core/nn/layers.gpu.test.ts @@ -179,45 +179,32 @@ interface ConvCase { kernel: number; stride: number; padding: number; - /** Input channels per scale group, each group spanning every tap. */ - group: number; + /** Values of a torch row per scale. */ + groupSize: number; bias: boolean; activation?: 'gelu'; } -/** A conv loaded from a quantized file [out, in·kernel] matches the same conv - * loaded with the dequantized floats, int · f16 scale, in torch's layout. A - * transposed conv's file holds the kernel reversed. */ +/** A conv loaded from a quantized file of its torch weight, [out, in·kernel] + * or a transposed conv's [in, out·kernel], matches the same conv loaded + * with the dequantized floats, int · f16 scale. */ async function checkQuantizedConv(c: ConvCase, bits: 4 | 8, dtype: 'f32' | 'f16') { - const { tIn, cIn, cOut, kernel, stride, padding, group } = c; + const { tIn, cIn, cOut, kernel, stride, padding, groupSize } = c; const half = 2 ** (bits - 1); - const cols = cIn * kernel; - const groupSize = group * kernel; - const ints = Array.from(randomF32(cOut * cols, 1), (v) => + const [rows, cols] = c.kind === 'conv' ? [cOut, cIn * kernel] : [cIn, cOut * kernel]; + const ints = Array.from(randomF32(rows * cols, 1), (v) => Math.min(half - 1, Math.floor(v * half)), ); const scales = roundF16( - randomF32((cOut * cols) / groupSize, 2).map((v) => (Math.abs(v) + 0.1) / half), + randomF32((rows * cols) / groupSize, 2).map((v) => (Math.abs(v) + 0.1) / half), ); const bias = roundF16(randomF32(cOut, 3)); const xs = roundF16(randomF32(tIn * cIn, 4)); - // [out, in, kernel] dequantized; a transposed conv's torch weight is [in, - // out, kernel] with the kernel the other way round. const dequant = Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / groupSize)]!); - const torchWeight = - c.kind === 'conv' - ? { data: dequant, shape: [cOut, cIn, kernel] } - : { - data: Float32Array.from({ length: dequant.length }, (_, i) => { - const [ci, o, t] = [ - Math.floor(i / (cOut * kernel)), - Math.floor(i / kernel) % cOut, - i % kernel, - ]; - return dequant[(o * cIn + ci) * kernel + kernel - 1 - t]!; - }), - shape: [cIn, cOut, kernel], - }; + const torchWeight = { + data: dequant, + shape: c.kind === 'conv' ? [cOut, cIn, kernel] : [cIn, cOut, kernel], + }; const make = () => c.kind === 'conv' ? new Conv1d(cIn, cOut, kernel, { stride, padding, bias: c.bias, activation: c.activation }) @@ -229,7 +216,7 @@ async function checkQuantizedConv(c: ConvCase, bits: 4 | 8, dtype: 'f32' | 'f16' return toArray(layer.forward(upload(root, xs, { ...matrix(tIn, cIn), dtype })), ex); }; const file = quantFile(bits, groupSize, { - weight: { ints, scales: [...scales], shape: [cOut, cols] }, + weight: { ints, scales: [...scales], shape: [rows, cols] }, }); const quantSd = await fromSafetensors(bufferSource(file), { chunkBytes: 0 }); if (c.bias) quantSd.tensors.set('bias', memoryStateDict({ bias }).tensors.get('bias')!); @@ -246,7 +233,7 @@ async function checkQuantizedConv(c: ConvCase, bits: 4 | 8, dtype: 'f32' | 'f16' describe('Conv1d and ConvTranspose1d with a quantized weight', () => { const CASES: ConvCase[] = [ { - name: 'conv, padded, bias and gelu', + name: 'conv, padded, groups that end inside a channel, bias and gelu', kind: 'conv', tIn: 40, cIn: 16, @@ -254,7 +241,7 @@ describe('Conv1d and ConvTranspose1d with a quantized weight', () => { kernel: 7, stride: 1, padding: 3, - group: 8, + groupSize: 16, bias: true, activation: 'gelu', }, @@ -267,7 +254,7 @@ describe('Conv1d and ConvTranspose1d with a quantized weight', () => { kernel: 1, stride: 2, padding: 0, - group: 32, + groupSize: 32, bias: false, }, // 1 and 3 output channels pad up to the packing width and are cut after. @@ -280,7 +267,7 @@ describe('Conv1d and ConvTranspose1d with a quantized weight', () => { kernel: 7, stride: 1, padding: 3, - group: 8, + groupSize: 56, bias: true, }, { @@ -292,7 +279,7 @@ describe('Conv1d and ConvTranspose1d with a quantized weight', () => { kernel: 3, stride: 1, padding: 1, - group: 4, + groupSize: 6, bias: true, }, { @@ -304,7 +291,7 @@ describe('Conv1d and ConvTranspose1d with a quantized weight', () => { kernel: 8, stride: 4, padding: 2, - group: 8, + groupSize: 16, bias: true, }, { @@ -316,7 +303,7 @@ describe('Conv1d and ConvTranspose1d with a quantized weight', () => { kernel: 4, stride: 2, padding: 1, - group: 16, + groupSize: 32, bias: false, }, ]; diff --git a/packages/runntime/test/core/nn/layers.test.ts b/packages/runntime/test/core/nn/layers.test.ts index 6c13621f..6c370bb1 100644 --- a/packages/runntime/test/core/nn/layers.test.ts +++ b/packages/runntime/test/core/nn/layers.test.ts @@ -417,14 +417,14 @@ describe('Linear with a quantized weight', () => { }); describe('Conv1d and ConvTranspose1d with a quantized weight', () => { - /** A [out, in·kernel] int8 entry of zeros, with `groups` scales per row. */ - const zeros = (out: number, cols: number, groups: number): QuantEntry => ({ - ints: Array(out * cols).fill(0), - scales: Array(out * groups).fill(1), - shape: [out, cols], + /** A [rows, cols] int8 entry of zeros, with `groups` scales per row. */ + const zeros = (rows: number, cols: number, groups: number): QuantEntry => ({ + ints: Array(rows * cols).fill(0), + scales: Array(rows * groups).fill(1), + shape: [rows, cols], }); - it('retypes a Conv1d weight to quantW [kernel·in, out] with a scale per channel group', async () => { + it('retypes a Conv1d weight to quantW [kernel·in, out] with its scales', async () => { const layer = new Conv1d(8, 4, 3); // groups of 12 values: 4 input channels with their 3 taps const sd = await quantCheckpoint(8, 12, { weight: zeros(4, 24, 2) }); @@ -462,35 +462,84 @@ describe('Conv1d and ConvTranspose1d with a quantized weight', () => { ); }); - it('retypes a ConvTranspose1d weight to quantW [taps·in, stride·out]', async () => { - const layer = new ConvTranspose1d(8, 2, 4, { stride: 2 }); - const sd = await quantCheckpoint(8, 16, { weight: zeros(2, 32, 2) }); + it('packs torch [out, in, kernel] ints tap-major and transposes the scales', async () => { + // torch [out 4, in 4, kernel 2], value o·8 + c·2 + tap; scales [4, 2] + // are o + g/2, f16-exact. + const sd = await quantCheckpoint(8, 4, { + weight: { + ints: Array.from({ length: 32 }, (_, i) => i), + scales: [0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5], + shape: [4, 8], + }, + }); + new Conv1d(4, 4, 2).transformStateDict(sd, ''); + // Row tap·4 + c of column o holds o·8 + c·2 + tap: tile rows 0..3 are + // tap 0, rows 4..7 tap 1. + expect([...(await sd.tensors.get('weight')!.words())]).toEqual([ + 0x06040200, 0x0e0c0a08, 0x16141210, 0x1e1c1a18, 0x07050301, 0x0f0d0b09, 0x17151311, + 0x1f1d1b19, + ]); + expect([...(await sd.tensors.get('weightScale')!.f32())]).toEqual([ + 0, 1, 2, 3, 0.5, 1.5, 2.5, 3.5, + ]); + }); + + it('keeps a Conv1d group that ends inside a channel', async () => { + const layer = new Conv1d(8, 4, 3); + // groups of 4 values: a channel's 3 taps and the next channel's first + const sd = await quantCheckpoint(8, 4, { weight: zeros(4, 24, 6) }); + layer.transformStateDict(sd, ''); + expect(shapes(layer).weightScale).toEqual(meta([6, 4])); + }); + + it('retypes a ConvTranspose1d weight [in, out·kernel] to quantW [taps·in, stride·out]', async () => { + const layer = new ConvTranspose1d(8, 4, 4, { stride: 2 }); + // groups of 4 values: one output's 4 kernel positions + const sd = await quantCheckpoint(8, 4, { weight: zeros(8, 16, 4) }); layer.transformStateDict(sd, ''); expect(shapes(layer)).toEqual({ - weight: { elems: 16, dtype: 'quantW', dims: [16, 4] }, - bias: meta([4]), - weightScale: meta([2, 4]), + weight: { elems: 32, dtype: 'quantW', dims: [16, 8] }, + bias: meta([8]), + weightScale: meta([8, 4]), }); }); + it('packs torch [in, out, kernel] ints by frame and tap and keeps the scales', async () => { + // torch [in 4, out 4, kernel 4] with stride 2, so 2 taps; value + // c·16 + o·4 + kk; scales [4, 2] are c + g/2, f16-exact. + const scales = [0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5]; + const sd = await quantCheckpoint(8, 8, { + weight: { ints: Array.from({ length: 64 }, (_, i) => i), scales, shape: [4, 16] }, + }); + new ConvTranspose1d(4, 4, 4, { stride: 2 }).transformStateDict(sd, ''); + // Row i·4 + c, column r·4 + o holds kernel position (1 − i)·2 + r. Tiles + // run frame 0 taps 0 and 1, then frame 1. + expect([...(await sd.tensors.get('weight')!.words())]).toEqual([ + 0x32221202, 0x36261606, 0x3a2a1a0a, 0x3e2e1e0e, 0x30201000, 0x34241404, 0x38281808, + 0x3c2c1c0c, 0x33231303, 0x37271707, 0x3b2b1b0b, 0x3f2f1f0f, 0x31211101, 0x35251505, + 0x39291909, 0x3d2d1d0d, + ]); + expect([...(await sd.tensors.get('weightScale')!.f32())]).toEqual(scales); + }); + it.each([ { - name: 'groups that split a channel', - layer: () => new Conv1d(8, 4, 3), - file: () => quantCheckpoint(8, 4, { weight: zeros(4, 24, 6) }), - error: /groups of 4 values, not whole channels of 3 values/, + name: 'kernel·in that is not a multiple of 8', + layer: () => new Conv1d(4, 4, 3), + file: () => quantCheckpoint(8, 12, { weight: zeros(4, 12, 1) }), + error: /needs in a multiple of 4 and 3·in of 8, got in 4/, }, { - name: 'a group of 2 channels', - layer: () => new Conv1d(8, 4, 2), - file: () => quantCheckpoint(8, 4, { weight: zeros(4, 16, 4) }), - error: /groups of 2 channels, which needs a group a multiple of 4/, + name: 'in that is not a multiple of 4', + layer: () => new Conv1d(2, 4, 4), + file: () => quantCheckpoint(8, 8, { weight: zeros(4, 8, 1) }), + error: /needs in a multiple of 4 and 4·in of 8, got in 2/, }, { - name: 'kernel·in that is not a multiple of 8', - layer: () => new Conv1d(4, 4, 3), - file: () => quantCheckpoint(8, 12, { weight: zeros(4, 12, 1) }), - error: /3·in a multiple of 8, got in 4/, + name: 'a ConvTranspose1d weight in Conv1d layout', + layer: () => new ConvTranspose1d(16, 8, 4, { stride: 2 }), + file: () => quantCheckpoint(8, 8, { weight: zeros(8, 64, 8) }), + error: /expected \[in, out·kernel\] = \[16, 32\]/, }, { name: 'a weight of another shape', @@ -499,10 +548,10 @@ describe('Conv1d and ConvTranspose1d with a quantized weight', () => { error: /expected \[out, in·kernel\] = \[4, 24\]/, }, { - name: 'stride·out that is not a multiple of 4', - layer: () => new ConvTranspose1d(8, 3, 2, { stride: 2 }), - file: () => quantCheckpoint(8, 8, { weight: zeros(3, 16, 2) }), - error: /needs stride·out a multiple of 4, got 6/, + name: 'out that is not a multiple of 4', + layer: () => new ConvTranspose1d(8, 2, 4, { stride: 2 }), + file: () => quantCheckpoint(8, 8, { weight: zeros(8, 8, 1) }), + error: /needs out a multiple of 4 and stride·out of 4, got out 2, stride 2/, }, ])('refuses $name with CHECKPOINT_MISMATCH', async ({ layer, file, error }) => { const sd = await file();