From 0f5f1f9d91eddee3a9c09f34a6b00a51a4563f64 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Mon, 2 Nov 2026 11:13:10 +0100 Subject: [PATCH 1/9] =?UTF-8?q?perf(core):=20split=20K=20across=20lanes=20?= =?UTF-8?q?in=20matmulQuantW=20and=20route=20quantW=20matmuls=20below=2048?= =?UTF-8?q?=C2=B71024=20outputs=20to=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../runntime/src/core/gpu/dispatch/matmul.ts | 53 ++-- .../runntime/src/core/graph/ops/matmul.ts | 4 +- .../src/core/kernels/matmul/matmulQuantW.ts | 277 ++++++++++-------- .../runntime/test/core/nn/layers.gpu.test.ts | 87 +++--- 4 files changed, 224 insertions(+), 197 deletions(-) diff --git a/packages/runntime/src/core/gpu/dispatch/matmul.ts b/packages/runntime/src/core/gpu/dispatch/matmul.ts index 083a7914..96bc5ccf 100644 --- a/packages/runntime/src/core/gpu/dispatch/matmul.ts +++ b/packages/runntime/src/core/gpu/dispatch/matmul.ts @@ -147,6 +147,12 @@ export const matmulSpec = defineSpec({ }, }); +/** From this many output values (M·N) a whole quantW weight takes + * matmulQuantTiled. Below it the tiled kernel has too few 32×32 tiles to fill + * the GPU, and matmulQuantW, which splits K across lanes, is faster. Measured + * on an Apple M4 Pro for N from 384 to 2048. */ +const QUANT_TILED_MIN_OUTPUTS = 48 * 1024; + export const matmulQuantWSpec = defineSpec({ attrs: (p) => { const [baseRow, scaleBase, bits, groupSize, hasBias, hasAdd, act] = p.attrs as [ @@ -164,16 +170,13 @@ export const matmulQuantWSpec = defineSpec({ const [m, k] = node.pending!.inputs[0]!.shape.dims as [number, number]; const n = node.shape.dims![1]!; const rows = node.pending!.inputs[1]!.shape.dims![0]!; - const epilogue = attrs.hasBias + attrs.hasAdd + attrs.act > 0; - // A whole weight goes to the tiled kernel from 8 rows, and always for an - // f16 input or an epilogue, which the plain kernel lacks. Expert views - // stay on the plain kernel. + // Expert views stay on matmulQuantW whatever m. const tiled = attrs.baseRow === 0 && attrs.scaleBase === 0 && rows === k && - matmulQuantTiledEligible(k, n, attrs.groupSize) && - (node.shape.dtype === 'f16' || epilogue || m >= MATMUL_TILED_MIN_M); + m * n >= QUANT_TILED_MIN_OUTPUTS && + matmulQuantTiledEligible(k, n, attrs.groupSize); return { k, n, @@ -188,41 +191,29 @@ export const matmulQuantWSpec = defineSpec({ pipeline: (root, dtype, cfg) => cfg.tiled ? createMatmulQuantTiledPipeline(root, cfg, elemFor(dtype)) - : createMatmulQuantWPipeline(root, cfg), + : createMatmulQuantWPipeline(root, cfg, elemFor(dtype)), encode: ({ node, attrs, cfg, pipeline, inputs, out, ctx }) => { const m = node.pending!.inputs[0]!.shape.dims![0]!; const dtype = node.shape.dtype; + const addendAt = attrs.hasBias === 1 ? 4 : 3; + const buffers = { + a: narrowFloat(inputs[0]!, dtype), + w: narrow(inputs[1]!, 'u32'), + scales: narrow(inputs[2]!, 'f32'), + bias: attrs.hasBias === 1 ? narrowFloat(inputs[3]!, dtype) : undefined, + addend: attrs.hasAdd === 1 ? narrowFloat(inputs[addendAt]!, dtype) : undefined, + out: narrowFloat(out, dtype), + }; if (cfg.tiled) { - const addendAt = attrs.hasBias === 1 ? 4 : 3; - return [ - matmulQuantTiledHandle( - ctx.root, - pipeline, - m, - cfg.n, - { - a: narrowFloat(inputs[0]!, dtype), - w: narrow(inputs[1]!, 'u32'), - scales: narrow(inputs[2]!, 'f32'), - bias: attrs.hasBias === 1 ? narrowFloat(inputs[3]!, dtype) : undefined, - addend: attrs.hasAdd === 1 ? narrowFloat(inputs[addendAt]!, dtype) : undefined, - out: narrowFloat(out, dtype), - }, - elemFor(dtype), - ), - ]; + return [matmulQuantTiledHandle(ctx.root, pipeline, m, cfg.n, buffers, elemFor(dtype))]; } return [ matmulQuantWHandle( ctx.root, pipeline, { m, n: cfg.n, baseRow: attrs.baseRow, scaleBase: attrs.scaleBase }, - { - a: narrow(inputs[0]!, 'f32'), - w: narrow(inputs[1]!, 'u32'), - scales: narrow(inputs[2]!, 'f32'), - out: narrow(out, 'f32'), - }, + buffers, + elemFor(dtype), ), ]; }, diff --git a/packages/runntime/src/core/graph/ops/matmul.ts b/packages/runntime/src/core/graph/ops/matmul.ts index 67079a9b..eab75568 100644 --- a/packages/runntime/src/core/graph/ops/matmul.ts +++ b/packages/runntime/src/core/graph/ops/matmul.ts @@ -97,8 +97,8 @@ export function matmul(a: Value, b: Value, opts: MatmulOpts = {}): Value { } /** matmul's quantW path. The result takes a's dtype. An f16 a, a bias, an - * addend or an activation need the tiled kernel: the whole weight (baseRow - * and scaleBase 0) and a shape it supports. */ + * addend or an activation need the whole weight (baseRow and scaleBase 0) + * and a shape the tiled kernel supports, since large inputs take it. */ function matmulQuantW( a: Value, w: Value, diff --git a/packages/runntime/src/core/kernels/matmul/matmulQuantW.ts b/packages/runntime/src/core/kernels/matmul/matmulQuantW.ts index c1b6b4dd..e489ac41 100644 --- a/packages/runntime/src/core/kernels/matmul/matmulQuantW.ts +++ b/packages/runntime/src/core/kernels/matmul/matmulQuantW.ts @@ -1,161 +1,202 @@ +/** Matmul with a quantW weight for a few rows: out = act(a·W + bias) + addend. + * W holds int8 or int4 values in the blocked-unit layout, times f32 group + * scales. a, bias, addend and out are f16 or f32; sums are f32. + * + * A workgroup serves ROWS rows × UNITS 4-column units, with K split across + * SLICES lanes per unit; partials add up in a fixed tree in workgroup memory, + * so every run gives the same result. W can be a view into one expert of a + * fused [E·K, N] weight: baseRow must be a K-aligned expert boundary, which + * the op enforces, and scales are indexed [scaleBase + kGroup·N + col]. */ + import tgpu, { d, std } from 'typegpu'; import type { TgpuRoot } from 'typegpu'; import { type F32Buffer, + flatWorkgroupId, + type FloatBuffer, type KernelHandle, - flatIndex, makeHandle, type U32Buffer, - WORKGROUP_SIZE, } from '../common.ts'; import { cachedBindGroup, cachedUniform } from '../../gpu/dispatchCache.ts'; -import { deqVec4 } from '../quantCommon.ts'; +import { activationFor, actSlot } from '../activations.ts'; +import { type Elem, F32_ELEM } from '../elem.ts'; +import { columnWord, deqVec4 } from '../quantCommon.ts'; -/** a[M,K] · dequant(view(w)[baseRow..+K, N]) giving out[M,N], where w is a - * quantW weight in the blocked-unit layout. - * - * Tiles are expert-granular, so baseRow must be a K-aligned expert boundary, - * which the op enforces. scales are indexed [scaleBase + kGroup*N + col]. - * - * One thread per 4-column unit streams its K-contiguous tile run and dots each - * x vec4 against 4 words. */ const Dims = d.struct({ m: d.u32, - baseRow: d.u32, // per-expert view offset — varies per CALL, so stays a uniform + baseRow: d.u32, // per-expert view offset, varies per call scaleBase: d.u32, }); -const Config = d.struct({ - k: d.u32, - n: d.u32, - n4: d.u32, - k4count: d.u32, - tileWords: d.u32, - g4: d.u32, - bits: d.u32, -}); -const config = tgpu.accessor(Config, { - k: 0, - n: 0, - n4: 0, - k4count: 0, - tileWords: 0, - g4: 1, - bits: 0, -}); - -export const matmulQuantWLayout = tgpu.bindGroupLayout({ - a: { storage: d.arrayOf(d.f32), access: 'readonly' }, - w: { storage: d.arrayOf(d.u32), access: 'readonly' }, - scales: { storage: d.arrayOf(d.f32), access: 'readonly' }, - out: { storage: d.arrayOf(d.f32), access: 'mutable' }, - dims: { uniform: Dims }, -}); +function makeLayout(elem: Elem) { + return tgpu.bindGroupLayout({ + a: { storage: d.arrayOf(elem.vec4), access: 'readonly' }, // [M, K/4] + w: { storage: d.arrayOf(d.u32), access: 'readonly' }, + scales: { storage: d.arrayOf(d.vec4f), access: 'readonly' }, // [.., N/4] + bias: { storage: d.arrayOf(elem.vec4), access: 'readonly' }, // [N/4]; dead when hasBias=0 + addend: { storage: d.arrayOf(elem.vec4), access: 'readonly' }, // [M, N/4]; dead when hasAdd=0 + out: { storage: d.arrayOf(elem.vec4), access: 'mutable' }, + dims: { uniform: Dims }, + }); +} -export const matmulQuantWKernel = tgpu.computeFn({ - in: { gid: d.builtin.globalInvocationId }, - workgroupSize: [WORKGROUP_SIZE], -})(({ gid }) => { - 'use gpu'; - const D = matmulQuantWLayout.$.dims; - const C = config.$; - const idx = flatIndex(gid); - if (idx >= D.m * C.n4) { - return; +const layouts = new Map>(); +function layoutFor(elem: Elem): ReturnType { + let l = layouts.get(elem.key); + if (!l) { + l = makeLayout(elem); + layouts.set(elem.key, l); } - const row = std.intdiv(idx, C.n4); // u32 division: f32 / rounds past 2^23 units - const unit = idx % C.n4; // this thread's 4-col unit - const col0 = unit * 4; + return l; +} - const expert = d.u32(D.baseRow / C.k); // blocked tiles are expert-granular - const wBase = (expert * C.n4 + unit) * C.k4count * C.tileWords; - const b4 = C.bits * 4; // tile bits per column +const ROWS = 4; +const UNITS = 8; +const SLICES = 16; +const LANES = UNITS * SLICES; - let sv = d.vec4f( - matmulQuantWLayout.$.scales[D.scaleBase + col0]!, - matmulQuantWLayout.$.scales[D.scaleBase + col0 + 1]!, - matmulQuantWLayout.$.scales[D.scaleBase + col0 + 2]!, - matmulQuantWLayout.$.scales[D.scaleBase + col0 + 3]!, - ); - let acc = d.vec4f(); // scaled total - let raw = d.vec4f(); // current group's unscaled dots - let curG = d.u32(0); - for (let k4 = d.u32(0); k4 < C.k4count; k4++) { - const g = d.u32(k4 / C.g4); - if (g !== curG) { - acc += raw * sv; - raw = d.vec4f(); - curG = g; - const s = D.scaleBase + g * C.n + col0; - sv = d.vec4f( - matmulQuantWLayout.$.scales[s]!, - matmulQuantWLayout.$.scales[s + 1]!, - matmulQuantWLayout.$.scales[s + 2]!, - matmulQuantWLayout.$.scales[s + 3]!, - ); +const SLICE_STRIDES = Array.from( + { length: Math.log2(SLICES) }, + (_, level) => SLICES >> (level + 1), +); + +const partial = tgpu.workgroupVar(d.arrayOf(d.vec4f, ROWS * LANES)); + +export interface MatmulQuantWCfg { + k: number; + n: number; + bits: number; + groupSize: number; + hasBias: number; + hasAdd: number; + act?: number; +} + +function makeMatmulQuantWKernel(cfg: MatmulQuantWCfg, elem: Elem) { + const { k, n, bits, groupSize, hasBias, hasAdd } = cfg; + const layout = layoutFor(elem); + const storeVec4 = elem.vec4; + const k4 = k / 4; + const n4 = n / 4; + const unitGroups = Math.ceil(n4 / UNITS); + const tileWords = bits / 2; + const cw1 = columnWord(1, bits); + const cw2 = columnWord(2, bits); + const cw3 = columnWord(3, bits); + return tgpu.computeFn({ + in: { wid: d.builtin.workgroupId, lid: d.builtin.localInvocationIndex }, + workgroupSize: [LANES], + })(({ wid, lid }) => { + 'use gpu'; + const D = layout.$.dims; + const group = flatWorkgroupId(wid); + const row0 = std.intdiv(group, unitGroups) * ROWS; + if (row0 >= D.m) { + return; } - const xb = row * C.k + k4 * 4; - const x = d.vec4f( - matmulQuantWLayout.$.a[xb]!, - matmulQuantWLayout.$.a[xb + 1]!, - matmulQuantWLayout.$.a[xb + 2]!, - matmulQuantWLayout.$.a[xb + 3]!, - ); - const tb = wBase + k4 * C.tileWords; - raw += d.vec4f( - std.dot(x, deqVec4(matmulQuantWLayout.$.w[tb]!, 0, C.bits)), - std.dot(x, deqVec4(matmulQuantWLayout.$.w[tb + d.u32(b4 / 32)]!, b4 % 32, C.bits)), - std.dot( - x, - deqVec4(matmulQuantWLayout.$.w[tb + d.u32((2 * b4) / 32)]!, (2 * b4) % 32, C.bits), - ), - std.dot( - x, - deqVec4(matmulQuantWLayout.$.w[tb + d.u32((3 * b4) / 32)]!, (3 * b4) % 32, C.bits), - ), - ); - } - acc += raw * sv; - const oBase = row * C.n + col0; - matmulQuantWLayout.$.out[oBase] = acc.x; - matmulQuantWLayout.$.out[oBase + 1] = acc.y; - matmulQuantWLayout.$.out[oBase + 2] = acc.z; - matmulQuantWLayout.$.out[oBase + 3] = acc.w; -}); + const lane = lid % UNITS; + const slice = std.intdiv(lid, UNITS); + const unit = (group % unitGroups) * UNITS + lane; + const expert = std.intdiv(D.baseRow, k); // blocked tiles are expert-granular + const wBase = (expert * n4 + unit) * k4 * tileWords; + const sBase = std.intdiv(D.scaleBase, 4) + unit; + // Indexed with a comptime `r`, so they stay in registers. + const accs = d.arrayOf(d.vec4f, ROWS)(); + if (unit < n4) { + for (let i = slice; i < k4; i += SLICES) { + const tb = wBase + i * tileWords; + const c0 = deqVec4(layout.$.w[tb]!, 0, bits); + const c1 = deqVec4(layout.$.w[tb + cw1]!, (4 * bits) % 32, bits); + const c2 = deqVec4(layout.$.w[tb + cw2]!, (8 * bits) % 32, bits); + const c3 = deqVec4(layout.$.w[tb + cw3]!, (12 * bits) % 32, bits); + const s = layout.$.scales[sBase + std.intdiv(i * 4, groupSize) * n4]!; + for (const r of tgpu.unroll(std.range(ROWS))) { + if (row0 + r < D.m) { + const x = d.vec4f(layout.$.a[(row0 + r) * k4 + i]!); + accs[r] = + accs[r]! + + d.vec4f(std.dot(x, c0), std.dot(x, c1), std.dot(x, c2), std.dot(x, c3)) * s; + } + } + } + } + for (const r of tgpu.unroll(std.range(ROWS))) { + partial.$[r * LANES + lid] = d.vec4f(accs[r]!); + } + std.workgroupBarrier(); + for (const stride of tgpu.unroll(SLICE_STRIDES)) { + if (slice < stride) { + for (const r of tgpu.unroll(std.range(ROWS))) { + partial.$[r * LANES + lid] = + partial.$[r * LANES + lid]! + partial.$[r * LANES + lid + stride * UNITS]!; + } + } + std.workgroupBarrier(); + } + + if (slice === 0 && unit < n4) { + let bias = d.vec4f(); + if (hasBias > 0) { + bias = d.vec4f(layout.$.bias[unit]!); + } + for (const r of tgpu.unroll(std.range(ROWS))) { + const row = row0 + r; + if (row < D.m) { + let v = actSlot.$(partial.$[r * LANES + lane]! + bias); + if (hasAdd > 0) { + v += d.vec4f(layout.$.addend[row * n4 + unit]!); + } + layout.$.out[row * n4 + unit] = storeVec4(v); + } + } + } + }); +} + +/** One pipeline per dtype, K, N, bits, group size, epilogue and activation. */ export function createMatmulQuantWPipeline( root: TgpuRoot, - cfg: { k: number; n: number; bits: number; groupSize: number }, + cfg: MatmulQuantWCfg, + elem: Elem = F32_ELEM, ) { return root - .with(config, { - k: cfg.k, - n: cfg.n, - n4: cfg.n / 4, - k4count: cfg.k / 4, - tileWords: cfg.bits / 2, - g4: cfg.groupSize / 4, - bits: cfg.bits, - }) - .createComputePipeline({ compute: matmulQuantWKernel }); + .with(actSlot, activationFor(cfg.act)) + .createComputePipeline({ compute: makeMatmulQuantWKernel(cfg, elem) }); } +/** One workgroup per ROWS rows × UNITS 4-column units. */ export function matmulQuantWHandle( root: TgpuRoot, pipeline: ReturnType, args: { m: number; n: number; baseRow: number; scaleBase: number }, - buffers: { a: F32Buffer; w: U32Buffer; scales: F32Buffer; out: F32Buffer }, + buffers: { + a: FloatBuffer; + w: U32Buffer; + scales: F32Buffer; + out: FloatBuffer; + bias?: FloatBuffer; + addend?: FloatBuffer; + }, + elem: Elem = F32_ELEM, ): KernelHandle { const dims = cachedUniform(root, Dims, { m: args.m, baseRow: args.baseRow, scaleBase: args.scaleBase, }); - const bindGroup = cachedBindGroup(root, matmulQuantWLayout, { ...buffers, dims }); + // Absent epilogue operands alias `a`; those pipelines never read them. + const bindGroup = cachedBindGroup(root, layoutFor(elem), { + ...buffers, + bias: buffers.bias ?? buffers.a, + addend: buffers.addend ?? buffers.a, + dims, + }); return makeHandle( pipeline, 'matmulQuantW', bindGroup, - Math.ceil((args.m * (args.n / 4)) / WORKGROUP_SIZE), + Math.ceil(args.n / 4 / UNITS) * Math.ceil(args.m / ROWS), ); } diff --git a/packages/runntime/test/core/nn/layers.gpu.test.ts b/packages/runntime/test/core/nn/layers.gpu.test.ts index 76171bbc..6518236b 100644 --- a/packages/runntime/test/core/nn/layers.gpu.test.ts +++ b/packages/runntime/test/core/nn/layers.gpu.test.ts @@ -22,6 +22,9 @@ interface Case { bias: boolean; addend?: boolean; activation?: SlotAct; + /** Takes the tiled kernel, which stages the dequantized weight in f16 + * workgroup memory for an f16 input. */ + tiled?: boolean; } /** A Linear loaded from a quantized file matches a CPU loop of @@ -56,9 +59,8 @@ async function checkQuantizedLinear(c: Case, bits: 4 | 8, dtype: 'f32' | 'f16') const call = c.addend ? { addend: upload(root, addend, { ...matrix(m, outF), dtype }) } : {}; const got = await toArray(layer.forward(x, call), ex); - // The f16 kernel stages the dequantized weight in f16 workgroup memory. const dequant = Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / groupSize)]!); - const weight = dtype === 'f16' ? roundF16(dequant) : dequant; + const weight = dtype === 'f16' && c.tiled ? roundF16(dequant) : dequant; const want = new Float64Array(m * outF); for (let i = 0; i < m; i++) { for (let o = 0; o < outF; o++) { @@ -77,26 +79,15 @@ async function checkQuantizedLinear(c: Case, bits: 4 | 8, dtype: 'f32' | 'f16') } describe('Linear with a quantized weight', () => { - // A quantized Linear takes the plain quantW kernel only for an f32 input - // under 8 rows with no epilogue; anything else takes the tiled kernel. + // A quantized Linear takes matmulQuantW below 48·1024 output values (m · + // outF) and the tiled kernel from there, for either dtype and any epilogue. const CASES: (Case & { name: string; dtypes: ('f32' | 'f16')[] })[] = [ - // plain: f32, m < 8, no bias - { name: 'plain, 1 row', m: 1, inF: 64, outF: 48, groupSize: 16, bias: false, dtypes: ['f32'] }, - { name: 'plain, 7 rows', m: 7, inF: 64, outF: 48, groupSize: 16, bias: false, dtypes: ['f32'] }, - // tiled: m >= 8 - { name: 'tiled, 8 rows', m: 8, inF: 64, outF: 48, groupSize: 16, bias: false, dtypes: ['f32'] }, - // tiled: a bias, an activation or an addend is an epilogue, whatever m + // matmulQuantW: m · outF < 48·1024 + { name: '1 row', m: 1, inF: 64, outF: 48, groupSize: 16, bias: false, dtypes: ['f32', 'f16'] }, + { name: '8 rows', m: 8, inF: 64, outF: 48, groupSize: 16, bias: false, dtypes: ['f32'] }, + { name: 'bias, 7 rows', m: 7, inF: 64, outF: 48, groupSize: 16, bias: true, dtypes: ['f32'] }, { - name: 'tiled by its bias, 7 rows', - m: 7, - inF: 64, - outF: 48, - groupSize: 16, - bias: true, - dtypes: ['f32'], - }, - { - name: 'tiled by its activation, 3 rows', + name: 'activation, 3 rows', m: 3, inF: 64, outF: 48, @@ -106,7 +97,7 @@ describe('Linear with a quantized weight', () => { dtypes: ['f32', 'f16'], }, { - name: 'tiled by its addend, 2 rows', + name: 'addend, 2 rows', m: 2, inF: 64, outF: 48, @@ -116,28 +107,7 @@ describe('Linear with a quantized weight', () => { dtypes: ['f32', 'f16'], }, { - name: 'bias, gelu and addend, 37 rows', - m: 37, - inF: 256, - outF: 136, - groupSize: 32, - bias: true, - addend: true, - activation: 'gelu', - dtypes: ['f32', 'f16'], - }, - // tiled: an f16 input, whatever m - { - name: 'f16 input, 1 row', - m: 1, - inF: 64, - outF: 48, - groupSize: 16, - bias: false, - dtypes: ['f16'], - }, - { - name: 'f16 input with bias, 5 rows', + name: 'bias, 5 rows', m: 5, inF: 64, outF: 48, @@ -146,21 +116,46 @@ describe('Linear with a quantized weight', () => { dtypes: ['f16'], }, { - name: 'partial tiles with bias, 37 rows', + name: 'bias, gelu and addend, 37 rows', m: 37, inF: 256, outF: 136, groupSize: 32, bias: true, + addend: true, + activation: 'gelu', dtypes: ['f32', 'f16'], }, { - name: 'one group per row, 37 rows', - m: 37, + name: 'one group per row, 767 rows', + m: 767, + inF: 128, + outF: 64, + groupSize: 128, + bias: false, + dtypes: ['f32', 'f16'], + }, + // tiled: m · outF >= 48·1024 + { + name: 'one group per row, 768 rows', + m: 768, inF: 128, outF: 64, groupSize: 128, bias: false, + tiled: true, + dtypes: ['f32', 'f16'], + }, + { + name: 'partial tiles with bias, gelu and addend, 400 rows', + m: 400, + inF: 256, + outF: 136, + groupSize: 32, + bias: true, + addend: true, + activation: 'gelu', + tiled: true, dtypes: ['f32', 'f16'], }, ]; From 5cc4ef4c42b3d6f41125ca4ba32dec3e2b55759e Mon Sep 17 00:00:00 2001 From: chmjkb Date: Mon, 2 Nov 2026 11:15:13 +0100 Subject: [PATCH 2/9] =?UTF-8?q?test(core):=20cover=20matmulQuantW's=20f16?= =?UTF-8?q?=20and=20epilogue=20cases=20and=20both=20sides=20of=20the=2048?= =?UTF-8?q?=C2=B71024-output=20tiled=20threshold?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../matmul/matmulQuantTiled.gpu.test.ts | 20 +-- .../kernels/matmul/matmulQuantW.gpu.test.ts | 131 ++++++++++++------ 2 files changed, 100 insertions(+), 51 deletions(-) diff --git a/packages/runntime/test/core/kernels/matmul/matmulQuantTiled.gpu.test.ts b/packages/runntime/test/core/kernels/matmul/matmulQuantTiled.gpu.test.ts index a19238e8..56ebddb3 100644 --- a/packages/runntime/test/core/kernels/matmul/matmulQuantTiled.gpu.test.ts +++ b/packages/runntime/test/core/kernels/matmul/matmulQuantTiled.gpu.test.ts @@ -20,7 +20,7 @@ interface Case { } /** act(a · dequant(W) + bias) + addend on the whole quantW weight, through the - * tiled route: an f16 input, an epilogue, or at least 8 rows. */ + * tiled route: m · n >= 48·1024 and K % 8 == 0. */ async function checkQuantTiled(c: Case, dtype: 'f32' | 'f16') { const { m, k, n, bits, groupSize } = c; const round = dtype === 'f16' ? roundF16 : (v: Float32Array) => v; @@ -57,13 +57,15 @@ async function checkQuantTiled(c: Case, dtype: 'f32' | 'f16') { } describe('matmulQuantTiled kernel', () => { - // tiled: a whole weight from 8 rows, K % 8 == 0 + // tiled: a whole weight from 48·1024 output values (m · n), K % 8 == 0. + // matmulQuantW's suite has the case one output below. const SHAPES = [ - { name: 'one tile', m: 8, k: 8, n: 8, groupSize: 4 }, - { name: 'partial row and column tiles', m: 37, k: 48, n: 40, groupSize: 8 }, - { name: 'one group per column', m: 9, k: 64, n: 16, groupSize: 64 }, + { name: 'the first output on the tiled route', m: 1536, k: 32, n: 32, groupSize: 16 }, + { name: 'one step of the K loop', m: 1536, k: 8, n: 32, groupSize: 4 }, + { name: 'partial row and column tiles', m: 1229, k: 48, n: 40, groupSize: 8 }, + { name: 'one group per column', m: 3072, k: 64, n: 16, groupSize: 64 }, { name: 'a fused q|k projection', m: 97, k: 1024, n: 2304, groupSize: 32 }, - { name: 'a per-channel down projection', m: 40, k: 2048, n: 1024, groupSize: 2048 }, + { name: 'a per-channel down projection', m: 48, k: 2048, n: 1024, groupSize: 2048 }, ]; it.for( (['f32', 'f16'] as const).flatMap((dtype) => @@ -74,7 +76,7 @@ describe('matmulQuantTiled kernel', () => { await checkQuantTiled(c, c.dtype); }); - // tiled: an f16 input or an epilogue on a whole weight, whatever m + // tiled: every epilogue, m · n >= 48·1024 it.for( (['f32', 'f16'] as const).flatMap((dtype) => ([4, 8] as const).flatMap((bits) => @@ -86,8 +88,8 @@ describe('matmulQuantTiled kernel', () => { ].map((e) => ({ dtype, bits, ...e })), ), ), - )('$dtype int$bits with $name on 3 rows', async ({ dtype, ...e }, ctx) => { + )('$dtype int$bits with $name on 2048 rows', async ({ dtype, ...e }, ctx) => { if (dtype === 'f16') requireFeature(ctx, root, 'shader-f16'); - await checkQuantTiled({ m: 3, k: 64, n: 24, groupSize: 16, ...e }, dtype); + await checkQuantTiled({ m: 2048, k: 64, n: 24, groupSize: 16, ...e }, dtype); }); }); diff --git a/packages/runntime/test/core/kernels/matmul/matmulQuantW.gpu.test.ts b/packages/runntime/test/core/kernels/matmul/matmulQuantW.gpu.test.ts index c007bad0..640798ab 100644 --- a/packages/runntime/test/core/kernels/matmul/matmulQuantW.gpu.test.ts +++ b/packages/runntime/test/core/kernels/matmul/matmulQuantW.gpu.test.ts @@ -2,82 +2,129 @@ import { describe, expect, it } from 'vitest'; import { toArray } from '../../../../src/core/gpu/eval.ts'; import { matmul, type MatmulOpts } from '../../../../src/core/graph/ops.ts'; import { matrix } from '../../../../src/core/graph/value.ts'; -import { GRID_Y_STRIDE } from '../../../../src/core/kernels/common.ts'; -import { expectClose, gpu, randomF32, requireFeature, upload } from '../../../gpu.ts'; +import { MAX_WORKGROUPS_PER_DIM } from '../../../../src/core/kernels/common.ts'; +import { expectClose, gpu, randomF32, requireFeature, roundF16, upload } from '../../../gpu.ts'; +import { ACTIVATION_REF, EPILOGUES, type Epilogue } from './checkMatmul.ts'; import { randomQuantWeight } from './quantW.ts'; const { root, ex } = await gpu(); -interface Case { +interface Case extends Epilogue { m: number; k: number; n: number; bits: 4 | 8; groupSize: number; + dtype?: 'f32' | 'f16'; experts?: number; /** The expert the call reads, through baseRow and scaleBase. */ expert?: number; } -async function checkQuantMatmul({ m, k, n, bits, groupSize, experts = 1, expert = 0 }: Case) { +/** act(a · dequant(W[expert]) + bias) + addend against a plain loop in f64. */ +async function checkQuantMatmul(c: Case) { + const { m, k, n, bits, groupSize, dtype = 'f32', experts = 1, expert = 0 } = c; + const round = dtype === 'f16' ? roundF16 : (v: Float32Array) => v; + const meta = (rows: number, cols: number) => ({ ...matrix(rows, cols), dtype }); const weight = randomQuantWeight(root, { experts, k, n, bits, groupSize }, 2); - const a = randomF32(m * k, 1); + const a = round(randomF32(m * k, 1)); + const bias = c.bias ? round(randomF32(n, 3)) : undefined; + const addend = c.addend ? round(randomF32(m * n, 4)) : undefined; const got = await toArray( - matmul(upload(root, a, [m, k]), weight.w, { + matmul(upload(root, a, meta(m, k)), weight.w, { scales: upload(root, weight.scales, [weight.scales.length / n, n]), bits, groupSize, baseRow: expert * k, scaleBase: ((expert * k) / groupSize) * n, + bias: bias && upload(root, bias, meta(1, n)), + addend: addend && upload(root, addend, meta(m, n)), + activation: c.activation, }), ex, ); - // a · W[expert] from the dequantized weights, in f64. const want = new Float64Array(m * n); for (let i = 0; i < m; i++) { - for (let c = 0; c < n; c++) { - let sum = 0; - for (let j = 0; j < k; j++) sum += a[i * k + j]! * weight.dequant[(expert * k + j) * n + c]!; - want[i * n + c] = sum; + for (let col = 0; col < n; col++) { + let sum = bias ? bias[col]! : 0; + for (let j = 0; j < k; j++) + sum += a[i * k + j]! * weight.dequant[(expert * k + j) * n + col]!; + if (c.activation) sum = ACTIVATION_REF[c.activation](sum); + if (addend) sum += addend[i * n + col]!; + want[i * n + col] = sum; } } - expectClose(got, want, { atol: 1e-6 * k, rtol: 1e-5 }); + // Sums are f32 for both dtypes; f16 rounds the output to half precision. + const tol = dtype === 'f16' ? { atol: 1e-5 * k, rtol: 1e-3 } : { atol: 1e-6 * k, rtol: 1e-5 }; + expectClose(got, want, tol); } describe('matmulQuantW kernel', () => { - // Routes are picked in gpu/dispatch/matmul.ts. This kernel takes an f32 - // input with no epilogue on one expert, below 8 rows, or when K is not a - // multiple of 8. Other calls on a whole weight take matmulQuantTiled. K and - // groupSize are multiples of 4, N of 4 for int8 and of 8 for int4. + // Routes are picked in gpu/dispatch/matmul.ts. This kernel takes a whole + // weight below 48·1024 output values (m · n), one expert of a fused weight + // whatever m, and any call with K not a multiple of 8. Other calls take + // matmulQuantTiled. K and groupSize are multiples of 4, N of 4 for int8 and + // of 8 for int4. A workgroup serves 4 rows × 8 four-column units, with K + // split across 16 lanes. + // f16 needs a whole weight with K % 8 == 0, which the op checks. + const both = ['f32', 'f16'] as const; + const f32 = ['f32'] as const; it.for( - ([8, 4] as const).flatMap((bits) => - [ - // plain: m < 8 - { name: 'one tile', m: 1, k: 4, n: 8, groupSize: 4 }, - { name: 'one group per column', m: 3, k: 12, n: 8, groupSize: 12 }, - { name: 'a group every 4 rows', m: 3, k: 12, n: 16, groupSize: 4 }, - { name: 'groups of 8', m: 5, k: 64, n: 24, groupSize: 8 }, - { name: 'a wide row', m: 2, k: 640, n: 640, groupSize: 32 }, - { name: 'one row short of the tiled route', m: 7, k: 32, n: 8, groupSize: 16 }, - // tiled: a whole weight, m >= 8 and K % 8 == 0 - { name: 'the first m on the tiled route', m: 8, k: 32, n: 8, groupSize: 16 }, - // plain: one expert of a fused weight, whatever m - { - name: 'rows past a workgroup of units on one expert', - m: 70, - k: 32, - n: 8, - groupSize: 16, - experts: 2, - expert: 1, - }, - // plain: K % 8 != 0. Past GRID_Y_STRIDE units the dispatch goes 2-D. - { name: 'units past GRID_Y_STRIDE', m: GRID_Y_STRIDE / 2 + 5, k: 4, n: 8, groupSize: 4 }, - ].map((c) => ({ bits, ...c })), + [ + // matmulQuantW: m · n < 48·1024 + { name: 'one tile', m: 1, k: 4, n: 8, groupSize: 4, dtypes: f32 }, + { name: 'one group per column', m: 3, k: 12, n: 8, groupSize: 12, dtypes: f32 }, + { name: 'a group every 4 rows', m: 3, k: 12, n: 16, groupSize: 4, dtypes: f32 }, + { name: 'one 8-row tile', m: 1, k: 8, n: 8, groupSize: 4, dtypes: both }, + { name: 'groups of 8, a partial row block', m: 5, k: 64, n: 24, groupSize: 8, dtypes: both }, + { name: 'a partial block of units', m: 8, k: 96, n: 40, groupSize: 32, dtypes: both }, + { name: 'a wide row', m: 2, k: 640, n: 640, groupSize: 32, dtypes: both }, + { name: 'an LLM decode row', m: 1, k: 2048, n: 2048, groupSize: 64, dtypes: both }, + // matmulQuantW: one below 48·1024 outputs; the tiled suite has the first above + { + name: 'one output short of the tiled route', + m: 1535, + k: 32, + n: 32, + groupSize: 16, + dtypes: both, + }, + // matmulQuantW: one expert of a fused weight, whatever m + { + name: 'rows past a workgroup on one expert', + m: 70, + k: 32, + n: 8, + groupSize: 16, + experts: 2, + expert: 1, + dtypes: f32, + }, + ].flatMap(({ dtypes, ...c }) => + dtypes.flatMap((dtype) => ([8, 4] as const).map((bits) => ({ dtype, bits, ...c }))), ), - )('int$bits $name: [$m, $k] × [$k, $n], groupSize $groupSize', (c) => checkQuantMatmul(c)); + )('$dtype int$bits $name: [$m, $k] × [$k, $n], groupSize $groupSize', async (c, ctx) => { + if (c.dtype === 'f16') requireFeature(ctx, root, 'shader-f16'); + await checkQuantMatmul(c); + }); + + // matmulQuantW: K % 8 != 0. Past MAX_WORKGROUPS_PER_DIM workgroups the + // dispatch goes 2-D. + it.for([8, 4] as const)('int%i workgroups past MAX_WORKGROUPS_PER_DIM', (bits) => + checkQuantMatmul({ m: 4 * MAX_WORKGROUPS_PER_DIM + 5, k: 4, n: 8, bits, groupSize: 4 }), + ); + + // matmulQuantW: every epilogue on a few rows, m · n < 48·1024 + it.for( + (['f32', 'f16'] as const).flatMap((dtype) => + ([8, 4] as const).flatMap((bits) => EPILOGUES.map((e) => ({ dtype, bits, ...e }))), + ), + )('$dtype int$bits with $name on 5 rows', async ({ dtype, bits, epilogue }, ctx) => { + if (dtype === 'f16') requireFeature(ctx, root, 'shader-f16'); + await checkQuantMatmul({ m: 5, k: 64, n: 40, bits, groupSize: 16, dtype, ...epilogue }); + }); - // plain: baseRow and scaleBase read one expert of a fused [E·K, N] weight. + // matmulQuantW: baseRow and scaleBase read one expert of a fused [E·K, N] weight. it.for(([8, 4] as const).flatMap((bits) => [0, 1, 2].map((expert) => ({ bits, expert }))))( 'int$bits expert $expert of 3', ({ bits, expert }) => From 3a1564c87bd29b2988d27a599cb969e2970a9536 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Mon, 2 Nov 2026 11:15:40 +0100 Subject: [PATCH 3/9] test(core): add f16 and epilogue quantW cases at MiniLM and decode shapes to the matmul A/B bench --- packages/runntime/bench/kernels/matmul.ab.ts | 87 ++++++++++++++++++-- 1 file changed, 82 insertions(+), 5 deletions(-) diff --git a/packages/runntime/bench/kernels/matmul.ab.ts b/packages/runntime/bench/kernels/matmul.ab.ts index e1c42c01..73de525a 100644 --- a/packages/runntime/bench/kernels/matmul.ab.ts +++ b/packages/runntime/bench/kernels/matmul.ab.ts @@ -49,7 +49,8 @@ function quant(name: string, t: number, e: number, k: number, o: number, bits: 8 }; } -/** [m, k] × a whole quantW weight [k, n]. */ +/** [m, k] × a whole quantW weight [k, n], with an optional bias [n], addend + * [m, n] and activation. */ function quantWhole( name: string, m: number, @@ -57,13 +58,26 @@ function quantWhole( n: number, bits: 8 | 4, groupSize: number, + opts: { dtype?: Float; bias?: boolean; addend?: boolean; act?: Act } = {}, ): AbCase { + const { dtype = 'f32' } = opts; return { name, + needs: dtype === 'f16' ? ['shader-f16'] : [], build: (version) => { - const a = randMat(version, m, k); + const a = randMat(version, m, k, dtype); const { w, scales } = quantWeight(version, 1, k, n, bits, groupSize); - return () => version.lib.matmul(a, w, { scales, bits, groupSize }); + const bias = opts.bias ? randMat(version, 1, n, dtype) : undefined; + const addend = opts.addend ? randMat(version, m, n, dtype) : undefined; + return () => + version.lib.matmul(a, w, { + scales, + bits, + groupSize, + bias, + addend, + activation: opts.act, + }); }, }; } @@ -172,15 +186,78 @@ abSuite('matmul', [ noSg(mm('tiledVec4, MiniLM qkv bucket 16 [16,384]×[384,1152]', 16, 384, 1152, { bias: true })), noSg(mm('plain, m = 4 [4,384]×[384,1152]', 4, 384, 1152)), - // Quantized weights, privacy-filter MoE expert. + // Quantized weights, privacy-filter MoE expert: matmulQuantW whatever m. quant('quantW int8, privacy-filter expert [32,640]×[640,640]', 32, 16, 640, 640, 8), quant('quantW int4, privacy-filter expert [32,640]×[640,640]', 32, 16, 640, 640, 4), quant('quantW int8, decode [1,640]×[640,640]', 1, 16, 640, 640, 8), - // Quantized weights, whole: the tiled route from 8 rows, plain below. + // Quantized weights, whole: the tiled route from 48·1024 outputs (m · n), + // matmulQuantW below. quantWhole('quantW int4 whole, q|k projection [256,1024]×[1024,2304]', 256, 1024, 2304, 4, 32), quantWhole('quantW int8 whole, per channel [256,2048]×[2048,1024]', 256, 2048, 1024, 8, 2048), + quantWhole( + 'quantW int8 whole f16 + bias, MiniLM qkv 256 tokens [256,384]×[384,1152]', + 256, + 384, + 1152, + 8, + 32, + { + dtype: 'f16', + bias: true, + }, + ), quantWhole('quantW int4 whole, decode [1,1024]×[1024,2304]', 1, 1024, 2304, 4, 32), + quantWhole( + 'quantW int8 whole f16 + bias, MiniLM qkv 16 tokens [16,384]×[384,1152]', + 16, + 384, + 1152, + 8, + 32, + { + dtype: 'f16', + bias: true, + }, + ), + quantWhole( + 'quantW int4 whole f16 + gelu, MiniLM fc1 16 tokens [16,384]×[384,1536]', + 16, + 384, + 1536, + 4, + 32, + { + dtype: 'f16', + bias: true, + act: 'gelu', + }, + ), + quantWhole( + 'quantW int8 whole f16 + addend, MiniLM fc2 16 tokens [16,1536]×[1536,384]', + 16, + 1536, + 384, + 8, + 32, + { + dtype: 'f16', + bias: true, + addend: true, + }, + ), + quantWhole( + 'quantW int8 whole f16 + bias, LLM decode [1,2048]×[2048,2048]', + 1, + 2048, + 2048, + 8, + 64, + { + dtype: 'f16', + bias: true, + }, + ), // matmulGather: one expert per row. gather('matmulGather f32, privacy-filter [64,640], 16 experts', 64, 16, 640, 640), From c817a9411de9718160da29692e8fe118d939acd5 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Mon, 2 Nov 2026 11:42:59 +0100 Subject: [PATCH 4/9] fix(core): refuse a quantW scaleBase that is not a multiple of 4, since matmulQuantW reads scales as vec4 --- packages/runntime/src/core/graph/ops/matmul.ts | 5 +++++ packages/runntime/src/core/kernels/matmul/matmulQuantW.ts | 3 ++- .../test/core/kernels/matmul/matmulQuantW.gpu.test.ts | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/runntime/src/core/graph/ops/matmul.ts b/packages/runntime/src/core/graph/ops/matmul.ts index eab75568..69e408b4 100644 --- a/packages/runntime/src/core/graph/ops/matmul.ts +++ b/packages/runntime/src/core/graph/ops/matmul.ts @@ -147,6 +147,11 @@ function matmulQuantW( } if (!Number.isInteger(scaleBase) || scaleBase < 0) throw new Error(`matmul: bad scaleBase ${scaleBase}`); + if (scaleBase % 4 !== 0) { + throw new Error( + `matmul: scaleBase ${scaleBase} must be a multiple of 4 — scales are read as vec4`, + ); + } const { bias, addend, activation } = opts; const whole = baseRow === 0 && scaleBase === 0 && r === k; if (whole && scales.shape.elems !== (k / groupSize) * n) { diff --git a/packages/runntime/src/core/kernels/matmul/matmulQuantW.ts b/packages/runntime/src/core/kernels/matmul/matmulQuantW.ts index e489ac41..5574bfd5 100644 --- a/packages/runntime/src/core/kernels/matmul/matmulQuantW.ts +++ b/packages/runntime/src/core/kernels/matmul/matmulQuantW.ts @@ -6,7 +6,8 @@ * SLICES lanes per unit; partials add up in a fixed tree in workgroup memory, * so every run gives the same result. W can be a view into one expert of a * fused [E·K, N] weight: baseRow must be a K-aligned expert boundary, which - * the op enforces, and scales are indexed [scaleBase + kGroup·N + col]. */ + * the op enforces, and scales are indexed [scaleBase + kGroup·N + col], with + * scaleBase a multiple of 4 since scales are read as vec4. */ import tgpu, { d, std } from 'typegpu'; import type { TgpuRoot } from 'typegpu'; diff --git a/packages/runntime/test/core/kernels/matmul/matmulQuantW.gpu.test.ts b/packages/runntime/test/core/kernels/matmul/matmulQuantW.gpu.test.ts index 640798ab..b2237cf9 100644 --- a/packages/runntime/test/core/kernels/matmul/matmulQuantW.gpu.test.ts +++ b/packages/runntime/test/core/kernels/matmul/matmulQuantW.gpu.test.ts @@ -160,6 +160,11 @@ describe('matmulQuantW kernel', () => { err: /baseRow 4 must be K-aligned \(8\)/, }, { name: 'a baseRow past the last expert', opts: { baseRow: 16 }, err: /bad baseRow 16/ }, + { + name: 'a scaleBase that is not a multiple of 4', + opts: { baseRow: 8, scaleBase: 2 }, + err: /scaleBase 2 must be a multiple of 4/, + }, { name: 'bits other than 4 or 8', opts: { bits: 2 }, err: /bits must be 8 or 4, got 2/ }, ])('refuses $name', ({ opts, err }) => { const weight = randomQuantWeight(root, fused, 2); @@ -170,6 +175,7 @@ describe('matmulQuantW kernel', () => { bits: (opts.bits ?? 8) as 8 | 4, groupSize: opts.groupSize ?? 4, baseRow: opts.baseRow, + scaleBase: opts.scaleBase, bias: opts.bias ? upload(root, new Float32Array(8), [1, 8]) : undefined, }; expect(() => matmul(a, weight.w, call)).toThrow(err); From f5ef0084c8688d1fab4e67585bdcfa0ad65fcc5d Mon Sep 17 00:00:00 2001 From: chmjkb Date: Mon, 2 Nov 2026 11:43:27 +0100 Subject: [PATCH 5/9] refactor(core): share one config and one buffers type between matmulQuantW and matmulQuantTiled --- .../core/kernels/matmul/matmulQuantTiled.ts | 38 +++++------------- .../src/core/kernels/matmul/matmulQuantW.ts | 39 +++++-------------- .../runntime/src/core/kernels/quantCommon.ts | 22 +++++++++++ 3 files changed, 42 insertions(+), 57 deletions(-) diff --git a/packages/runntime/src/core/kernels/matmul/matmulQuantTiled.ts b/packages/runntime/src/core/kernels/matmul/matmulQuantTiled.ts index d4b54090..ff463f46 100644 --- a/packages/runntime/src/core/kernels/matmul/matmulQuantTiled.ts +++ b/packages/runntime/src/core/kernels/matmul/matmulQuantTiled.ts @@ -7,17 +7,16 @@ import tgpu, { d, std } from 'typegpu'; import type { TgpuRoot } from 'typegpu'; -import { - type F32Buffer, - type FloatBuffer, - type KernelHandle, - makeHandle, - type U32Buffer, -} from '../common.ts'; +import { type KernelHandle, makeHandle } from '../common.ts'; import { cachedBindGroup, cachedUniform } from '../../gpu/dispatchCache.ts'; import { activationFor, actSlot } from '../activations.ts'; import { type Elem, F32_ELEM } from '../elem.ts'; -import { columnWord, deqRow } from '../quantCommon.ts'; +import { + columnWord, + deqRow, + type MatmulQuantBuffers, + type MatmulQuantCfg, +} from '../quantCommon.ts'; const Dims = d.struct({ m: d.u32 }); @@ -59,17 +58,7 @@ export function matmulQuantTiledEligible(k: number, n: number, groupSize: number return k % BK === 0 && n % 4 === 0 && groupSize % 4 === 0 && k % groupSize === 0; } -export interface MatmulQuantTiledCfg { - k: number; - n: number; - bits: number; - groupSize: number; - hasBias: number; - hasAdd: number; - act?: number; -} - -function makeMatmulQuantTiledKernel(cfg: MatmulQuantTiledCfg, elem: Elem) { +function makeMatmulQuantTiledKernel(cfg: MatmulQuantCfg, elem: Elem) { const { k, n, bits, groupSize, hasBias, hasAdd } = cfg; const layout = layoutFor(elem); const storeVec4 = elem.vec4; @@ -184,7 +173,7 @@ function makeMatmulQuantTiledKernel(cfg: MatmulQuantTiledCfg, elem: Elem) { /** One pipeline per dtype, K, N, bits, group size, epilogue and activation. */ export function createMatmulQuantTiledPipeline( root: TgpuRoot, - cfg: MatmulQuantTiledCfg, + cfg: MatmulQuantCfg, elem: Elem = F32_ELEM, ) { return root @@ -198,14 +187,7 @@ export function matmulQuantTiledHandle( pipeline: ReturnType, m: number, n: number, - buffers: { - a: FloatBuffer; - w: U32Buffer; - scales: F32Buffer; - out: FloatBuffer; - bias?: FloatBuffer; - addend?: FloatBuffer; - }, + buffers: MatmulQuantBuffers, elem: Elem = F32_ELEM, ): KernelHandle { const dims = cachedUniform(root, Dims, { m }); diff --git a/packages/runntime/src/core/kernels/matmul/matmulQuantW.ts b/packages/runntime/src/core/kernels/matmul/matmulQuantW.ts index 5574bfd5..73bd44ee 100644 --- a/packages/runntime/src/core/kernels/matmul/matmulQuantW.ts +++ b/packages/runntime/src/core/kernels/matmul/matmulQuantW.ts @@ -11,18 +11,16 @@ import tgpu, { d, std } from 'typegpu'; import type { TgpuRoot } from 'typegpu'; -import { - type F32Buffer, - flatWorkgroupId, - type FloatBuffer, - type KernelHandle, - makeHandle, - type U32Buffer, -} from '../common.ts'; +import { flatWorkgroupId, type KernelHandle, makeHandle } from '../common.ts'; import { cachedBindGroup, cachedUniform } from '../../gpu/dispatchCache.ts'; import { activationFor, actSlot } from '../activations.ts'; import { type Elem, F32_ELEM } from '../elem.ts'; -import { columnWord, deqVec4 } from '../quantCommon.ts'; +import { + columnWord, + deqVec4, + type MatmulQuantBuffers, + type MatmulQuantCfg, +} from '../quantCommon.ts'; const Dims = d.struct({ m: d.u32, @@ -64,17 +62,7 @@ const SLICE_STRIDES = Array.from( const partial = tgpu.workgroupVar(d.arrayOf(d.vec4f, ROWS * LANES)); -export interface MatmulQuantWCfg { - k: number; - n: number; - bits: number; - groupSize: number; - hasBias: number; - hasAdd: number; - act?: number; -} - -function makeMatmulQuantWKernel(cfg: MatmulQuantWCfg, elem: Elem) { +function makeMatmulQuantWKernel(cfg: MatmulQuantCfg, elem: Elem) { const { k, n, bits, groupSize, hasBias, hasAdd } = cfg; const layout = layoutFor(elem); const storeVec4 = elem.vec4; @@ -159,7 +147,7 @@ function makeMatmulQuantWKernel(cfg: MatmulQuantWCfg, elem: Elem) { /** One pipeline per dtype, K, N, bits, group size, epilogue and activation. */ export function createMatmulQuantWPipeline( root: TgpuRoot, - cfg: MatmulQuantWCfg, + cfg: MatmulQuantCfg, elem: Elem = F32_ELEM, ) { return root @@ -172,14 +160,7 @@ export function matmulQuantWHandle( root: TgpuRoot, pipeline: ReturnType, args: { m: number; n: number; baseRow: number; scaleBase: number }, - buffers: { - a: FloatBuffer; - w: U32Buffer; - scales: F32Buffer; - out: FloatBuffer; - bias?: FloatBuffer; - addend?: FloatBuffer; - }, + buffers: MatmulQuantBuffers, elem: Elem = F32_ELEM, ): KernelHandle { const dims = cachedUniform(root, Dims, { diff --git a/packages/runntime/src/core/kernels/quantCommon.ts b/packages/runntime/src/core/kernels/quantCommon.ts index 739716c5..a3bf3e2e 100644 --- a/packages/runntime/src/core/kernels/quantCommon.ts +++ b/packages/runntime/src/core/kernels/quantCommon.ts @@ -1,4 +1,5 @@ import { d, std } from 'typegpu'; +import type { F32Buffer, FloatBuffer, U32Buffer } from './common.ts'; /** The physical layout contract for quantW weight buffers, shared by the * packer and the matmulQuantW, moeQuantW and matmulQuantTiled kernels. @@ -41,3 +42,24 @@ export const deqRow = (c0: number, c1: number, c2: number, c3: number, r: number d.f32(std.extractBits(std.bitcast(d.u32, d.i32)(c3), (12 * bits + off) % 32, bits)), ); }; + +/** Pipeline config of the matmulQuantW and matmulQuantTiled kernels. */ +export interface MatmulQuantCfg { + k: number; + n: number; + bits: number; + groupSize: number; + hasBias: number; + hasAdd: number; + act?: number; +} + +/** Buffers of a matmulQuantW or matmulQuantTiled dispatch. */ +export interface MatmulQuantBuffers { + a: FloatBuffer; + w: U32Buffer; + scales: F32Buffer; + out: FloatBuffer; + bias?: FloatBuffer; + addend?: FloatBuffer; +} From cfc6d6f841bfca5a12996d3b4eeb55ffb358ff9b Mon Sep 17 00:00:00 2001 From: chmjkb Date: Mon, 2 Nov 2026 11:43:39 +0100 Subject: [PATCH 6/9] refactor(core): add doc comments to matmulQuantW's workgroup shape constants --- packages/runntime/src/core/kernels/matmul/matmulQuantW.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/runntime/src/core/kernels/matmul/matmulQuantW.ts b/packages/runntime/src/core/kernels/matmul/matmulQuantW.ts index 73bd44ee..d8557b96 100644 --- a/packages/runntime/src/core/kernels/matmul/matmulQuantW.ts +++ b/packages/runntime/src/core/kernels/matmul/matmulQuantW.ts @@ -50,11 +50,16 @@ function layoutFor(elem: Elem): ReturnType { return l; } +/** Rows of a per workgroup. Each lane keeps one f32 vec4 accumulator per row. */ const ROWS = 4; +/** Units per workgroup. A unit computes 4 output columns. */ const UNITS = 8; +/** Lanes that split one unit's K between them. */ const SLICES = 16; +/** Threads per workgroup. */ const LANES = UNITS * SLICES; +/** Steps of the tree that adds the slices: SLICES/2 down to 1. */ const SLICE_STRIDES = Array.from( { length: Math.log2(SLICES) }, (_, level) => SLICES >> (level + 1), From d9a137efb96fa9ecf09eef51ae7bfbba87689b7b Mon Sep 17 00:00:00 2001 From: chmjkb Date: Mon, 2 Nov 2026 11:43:51 +0100 Subject: [PATCH 7/9] refactor(core): look up the element type once in matmulQuantWSpec's encode --- packages/runntime/src/core/gpu/dispatch/matmul.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/runntime/src/core/gpu/dispatch/matmul.ts b/packages/runntime/src/core/gpu/dispatch/matmul.ts index 96bc5ccf..4a99de18 100644 --- a/packages/runntime/src/core/gpu/dispatch/matmul.ts +++ b/packages/runntime/src/core/gpu/dispatch/matmul.ts @@ -195,6 +195,7 @@ export const matmulQuantWSpec = defineSpec({ encode: ({ node, attrs, cfg, pipeline, inputs, out, ctx }) => { const m = node.pending!.inputs[0]!.shape.dims![0]!; const dtype = node.shape.dtype; + const elem = elemFor(dtype); const addendAt = attrs.hasBias === 1 ? 4 : 3; const buffers = { a: narrowFloat(inputs[0]!, dtype), @@ -205,7 +206,7 @@ export const matmulQuantWSpec = defineSpec({ out: narrowFloat(out, dtype), }; if (cfg.tiled) { - return [matmulQuantTiledHandle(ctx.root, pipeline, m, cfg.n, buffers, elemFor(dtype))]; + return [matmulQuantTiledHandle(ctx.root, pipeline, m, cfg.n, buffers, elem)]; } return [ matmulQuantWHandle( @@ -213,7 +214,7 @@ export const matmulQuantWSpec = defineSpec({ pipeline, { m, n: cfg.n, baseRow: attrs.baseRow, scaleBase: attrs.scaleBase }, buffers, - elemFor(dtype), + elem, ), ]; }, From 63e1363be65e7425f1b87bba8fe1954fbcf9faeb Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 4 Nov 2026 16:02:44 +0100 Subject: [PATCH 8/9] fix(core): keep quantW matmuls under 32 rows on matmulQuantW, since a wide N alone passes the tiled threshold --- packages/runntime/src/core/gpu/dispatch/matmul.ts | 6 ++++++ .../test/core/kernels/matmul/matmulQuantTiled.gpu.test.ts | 7 ++++--- .../test/core/kernels/matmul/matmulQuantW.gpu.test.ts | 3 +++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/runntime/src/core/gpu/dispatch/matmul.ts b/packages/runntime/src/core/gpu/dispatch/matmul.ts index 4a99de18..63435cf5 100644 --- a/packages/runntime/src/core/gpu/dispatch/matmul.ts +++ b/packages/runntime/src/core/gpu/dispatch/matmul.ts @@ -153,6 +153,11 @@ export const matmulSpec = defineSpec({ * on an Apple M4 Pro for N from 384 to 2048. */ const QUANT_TILED_MIN_OUTPUTS = 48 * 1024; +/** And from this many rows, one full row tile of matmulQuantTiled. With fewer + * a wide N alone passes QUANT_TILED_MIN_OUTPUTS, and each tile computes + * mostly empty rows: [1,2048]×[2048,131072] is 4.8× slower tiled. */ +const QUANT_TILED_MIN_ROWS = 32; + export const matmulQuantWSpec = defineSpec({ attrs: (p) => { const [baseRow, scaleBase, bits, groupSize, hasBias, hasAdd, act] = p.attrs as [ @@ -175,6 +180,7 @@ export const matmulQuantWSpec = defineSpec({ attrs.baseRow === 0 && attrs.scaleBase === 0 && rows === k && + m >= QUANT_TILED_MIN_ROWS && m * n >= QUANT_TILED_MIN_OUTPUTS && matmulQuantTiledEligible(k, n, attrs.groupSize); return { diff --git a/packages/runntime/test/core/kernels/matmul/matmulQuantTiled.gpu.test.ts b/packages/runntime/test/core/kernels/matmul/matmulQuantTiled.gpu.test.ts index 56ebddb3..50ec82a6 100644 --- a/packages/runntime/test/core/kernels/matmul/matmulQuantTiled.gpu.test.ts +++ b/packages/runntime/test/core/kernels/matmul/matmulQuantTiled.gpu.test.ts @@ -20,7 +20,7 @@ interface Case { } /** act(a · dequant(W) + bias) + addend on the whole quantW weight, through the - * tiled route: m · n >= 48·1024 and K % 8 == 0. */ + * tiled route: m >= 32, m · n >= 48·1024 and K % 8 == 0. */ async function checkQuantTiled(c: Case, dtype: 'f32' | 'f16') { const { m, k, n, bits, groupSize } = c; const round = dtype === 'f16' ? roundF16 : (v: Float32Array) => v; @@ -57,10 +57,11 @@ async function checkQuantTiled(c: Case, dtype: 'f32' | 'f16') { } describe('matmulQuantTiled kernel', () => { - // tiled: a whole weight from 48·1024 output values (m · n), K % 8 == 0. - // matmulQuantW's suite has the case one output below. + // tiled: a whole weight from 32 rows and 48·1024 output values (m · n), + // K % 8 == 0. matmulQuantW's suite has the cases one row and one output below. const SHAPES = [ { name: 'the first output on the tiled route', m: 1536, k: 32, n: 32, groupSize: 16 }, + { name: 'the first row tile on the tiled route', m: 32, k: 32, n: 2048, groupSize: 16 }, { name: 'one step of the K loop', m: 1536, k: 8, n: 32, groupSize: 4 }, { name: 'partial row and column tiles', m: 1229, k: 48, n: 40, groupSize: 8 }, { name: 'one group per column', m: 3072, k: 64, n: 16, groupSize: 64 }, diff --git a/packages/runntime/test/core/kernels/matmul/matmulQuantW.gpu.test.ts b/packages/runntime/test/core/kernels/matmul/matmulQuantW.gpu.test.ts index b2237cf9..809524e3 100644 --- a/packages/runntime/test/core/kernels/matmul/matmulQuantW.gpu.test.ts +++ b/packages/runntime/test/core/kernels/matmul/matmulQuantW.gpu.test.ts @@ -89,6 +89,9 @@ describe('matmulQuantW kernel', () => { groupSize: 16, dtypes: both, }, + // matmulQuantW: m < 32, though m · n >= 48·1024; the tiled suite has m = 32 + { name: 'a wide decode row', m: 1, k: 64, n: 49152, groupSize: 32, dtypes: both }, + { name: 'one row short of a row tile', m: 31, k: 32, n: 2048, groupSize: 16, dtypes: both }, // matmulQuantW: one expert of a fused weight, whatever m { name: 'rows past a workgroup on one expert', From fcbcf19b6dddf617e71a425822291d5fdc21f54e Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 4 Nov 2026 16:02:49 +0100 Subject: [PATCH 9/9] test(core): add wide-N quantW cases under 32 rows to the matmul A/B bench --- packages/runntime/bench/kernels/matmul.ab.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/runntime/bench/kernels/matmul.ab.ts b/packages/runntime/bench/kernels/matmul.ab.ts index 73de525a..91be274d 100644 --- a/packages/runntime/bench/kernels/matmul.ab.ts +++ b/packages/runntime/bench/kernels/matmul.ab.ts @@ -258,6 +258,9 @@ abSuite('matmul', [ bias: true, }, ), + // matmulQuantW: fewer than 32 rows on a wide N, past 48·1024 outputs. + quantWhole('quantW int8 whole, wide decode [1,2048]×[2048,65536]', 1, 2048, 65536, 8, 32), + quantWhole('quantW int8 whole, LLM MLP 4 rows [4,4096]×[4096,14336]', 4, 4096, 14336, 8, 32), // matmulGather: one expert per row. gather('matmulGather f32, privacy-filter [64,640], 16 experts', 64, 16, 640, 640),