diff --git a/packages/runntime/bench/kernels/attention.ab.ts b/packages/runntime/bench/kernels/attention.ab.ts index 93a974da..b08f3249 100644 --- a/packages/runntime/bench/kernels/attention.ab.ts +++ b/packages/runntime/bench/kernels/attention.ab.ts @@ -203,6 +203,30 @@ abSuite('attention', [ dtype: 'f16', keyBias: { start: 55, end: 305, value: 1 }, }), + sdpa('flash, 8 lanes, GQA 16/4 heads hd 64, causal', { + qLen: 256, + kvLen: 256, + qHeads: 16, + kvHeads: 4, + headDim: 64, + causal: true, + }), + sdpa('flash f16, EmbeddingGemma 2 sliding layer 4/2 heads hd 256, window 512', { + qLen: 512, + kvLen: 512, + qHeads: 4, + kvHeads: 2, + headDim: 256, + dtype: 'f16', + window: 512, + }), + sdpa('flash, EmbeddingGemma 2 full layer 4/1 heads hd 512', { + qLen: 512, + kvLen: 512, + qHeads: 4, + kvHeads: 1, + headDim: 512, + }), packed('flash packed + segments, MiniLM bucket 128, hd 32', 128, 12, 32, 'f32'), packed('flash packed f16 + segments, MiniLM bucket 512, hd 32', 512, 12, 32, 'f16'), diff --git a/packages/runntime/src/core/gpu/dispatch/attention.ts b/packages/runntime/src/core/gpu/dispatch/attention.ts index ab9354be..59aa22dc 100644 --- a/packages/runntime/src/core/gpu/dispatch/attention.ts +++ b/packages/runntime/src/core/gpu/dispatch/attention.ts @@ -153,6 +153,8 @@ export const attnSpec = defineSpec({ kvLen, qPosOffset: attrs.qPosOffset, qHeads: attrs.qHeads, + kvHeads: attrs.kvHeads, + headDim: attrs.headDim, keyBias: attrs.keyBias, }; const bufs = { q, k, v, out: narrowFloat(out, dtype), sinks, segs, dummyF32: ctx.dummy('f32') }; diff --git a/packages/runntime/src/core/kernels/attention/attnFlash.ts b/packages/runntime/src/core/kernels/attention/attnFlash.ts index 30897ee7..ccb4402c 100644 --- a/packages/runntime/src/core/kernels/attention/attnFlash.ts +++ b/packages/runntime/src/core/kernels/attention/attnFlash.ts @@ -1,8 +1,9 @@ /** Prefill attention (more than one query row) for head sizes that are a * multiple of 16. Needs subgroups. * - * A workgroup handles 16 rows of one head. It loads a block of keys and - * values into workgroup memory once, and all 16 rows use it. Each row is + * A workgroup handles 16 rows of one head, or of two heads that share a kv + * head. It loads a block of keys and values into workgroup memory once, and + * all its rows use it. Each row is * split across 4 or 8 lanes, which add up q·k with subgroup shuffles. * Windows, segments, sinks and the key bias work as in the rows kernel. * @@ -64,16 +65,31 @@ export function attnFlashLanes(headDim: number): number { return headDim % 32 === 0 ? 8 : headDim % 16 === 0 ? 4 : 0; } -/** Bytes of one key or value tile for heads past 128. */ +/** Widest head whose q, output and scores stay in registers. */ +const REGISTER_HEAD_DIM = 128; + +/** Bytes of one key or value tile for heads past REGISTER_HEAD_DIM. */ const TILE_BYTES = 8192; -/** Keys per block, so the key and value tiles fit in 16 KiB of workgroup - * memory: 32, 16 for f32 heads wider than 32, fewer for heads past 128. */ +/** Keys per block: 8 with 8 lanes, which keeps the scores in registers, and + * 32 with 4 lanes, 16 for f32 heads wider than 32. Wider heads take fewer, + * so the key and value tiles fit in workgroup memory. */ function tileKeys(headDim: number, f16: boolean): number { - if (headDim > 128) return Math.max(1, Math.floor(TILE_BYTES / (headDim * (f16 ? 2 : 4)))); + if (headDim > REGISTER_HEAD_DIM) { + return Math.max(1, Math.floor(TILE_BYTES / (headDim * (f16 ? 2 : 4)))); + } + if (attnFlashLanes(headDim) === 8) return 8; return !f16 && headDim > 32 ? 16 : 32; } +/** q heads per workgroup: 2 when they come in pairs that share a kv head, so + * every key and value read from workgroup memory feeds both. Only with 8 + * lanes and heads up to REGISTER_HEAD_DIM, where both fit in registers. */ +function headsPerWorkgroup(qHeads: number, kvHeads: number, headDim: number): number { + const pairs = Math.floor(qHeads / kvHeads) % 2 === 0; + return pairs && attnFlashLanes(headDim) === 8 && headDim <= REGISTER_HEAD_DIM ? 2 : 1; +} + function makeAttnFlashKernel(cfg: AttnGeoCfg, elem: Elem) { const { qHeads, kvHeads, headDim: hd, windowLeft, windowRight, hasSinks, hasSegs } = cfg; const hasKeyBias = cfg.hasKeyBias ?? 0; @@ -85,6 +101,7 @@ function makeAttnFlashKernel(cfg: AttnGeoCfg, elem: Elem) { const tileK = tileKeys(hd, f16); const lanes = TILE_Q * lpq; const group = Math.floor(qHeads / kvHeads); + const hp = headsPerWorkgroup(qHeads, kvHeads, hd); const packed = (cfg.qkvPacked ?? 0) > 0; const qStride4 = packed ? (qHeads + 2 * kvHeads) * hd4 : qHeads * hd4; const kvStride4 = packed ? qStride4 : kvHeads * hd4; @@ -111,9 +128,9 @@ function makeAttnFlashKernel(cfg: AttnGeoCfg, elem: Elem) { })(({ lid, wid }) => { 'use gpu'; const D = L.$.dims; - const h = wid.y; - const kvh = d.u32(h / group); - const qSub = d.u32(lid / lpq); + const h0 = wid.y * hp; + const kvh = std.intdiv(h0, group); + const qSub = std.intdiv(lid, lpq); const lane = lid % lpq; const row = wid.x * TILE_Q + qSub; const rowValid = row < D.qLen; @@ -138,25 +155,28 @@ function makeAttnFlashKernel(cfg: AttnGeoCfg, elem: Elem) { } } - const qBase4 = rowC * qStride4 + h * hd4 + lane * slice; - const qr = d.arrayOf(d.vec4f, slice)(); - const o = d.arrayOf(d.vec4f, slice)(); - for (const c of tgpu.unroll(std.range(slice))) { - qr[c] = d.vec4f(L.$.q[qBase4 + c]!); - } - - let m = d.f32(NEG_SEED); - let l = d.f32(0); - if (hasSinks > 0) { - m = L.$.sinks[h]! * LN2; - l = d.f32(1); + // Head j of the workgroup holds q, out and scores at j·slice and j·tileK. + const qBase4 = rowC * qStride4 + h0 * hd4 + lane * slice; + const qr = d.arrayOf(d.vec4f, hp * slice)(); + const o = d.arrayOf(d.vec4f, hp * slice)(); + const m = d.arrayOf(d.f32, hp)(); + const l = d.arrayOf(d.f32, hp)(); + for (const j of tgpu.unroll(std.range(hp))) { + for (const c of tgpu.unroll(std.range(slice))) { + qr[j * slice + c] = d.vec4f(L.$.q[qBase4 + j * hd4 + c]!); + } + m[j] = d.f32(NEG_SEED); + if (hasSinks > 0) { + m[j] = L.$.sinks[h0 + j]! * LN2; + l[j] = d.f32(1); + } } - const s = d.arrayOf(d.f32, tileK)(); + const s = d.arrayOf(d.f32, hp * tileK)(); for (let t0 = kStart; t0 < kStop; t0 += tileK) { std.workgroupBarrier(); for (let i = lid; i < tileK * hd4; i += lanes) { - const kj = t0 + d.u32(i / hd4); + const kj = t0 + std.intdiv(i, hd4); const col = i % hd4; let kv = d.vec4f(); let vv = d.vec4f(); @@ -176,51 +196,79 @@ function makeAttnFlashKernel(cfg: AttnGeoCfg, elem: Elem) { } std.workgroupBarrier(); - let tileMax = d.f32(MASKED); + // Each key and value read from workgroup memory feeds every head. + const part = d.arrayOf(d.f32, hp)(); + const tileMax = d.arrayOf(d.f32, hp)(); + for (const j of tgpu.unroll(std.range(hp))) { + tileMax[j] = d.f32(MASKED); + } for (const kk of tgpu.unroll(std.range(tileK))) { - let part = d.f32(0); - for (const c of tgpu.unroll(std.range(slice))) { - part += std.dot(qr[c]!, d.vec4f(kTile.$[kk * hd4 + lane * slice + c]!)); + for (const j of tgpu.unroll(std.range(hp))) { + part[j] = d.f32(0); } - part += std.subgroupShuffleXor(part, 1); - part += std.subgroupShuffleXor(part, 2); - if (lpq > 4) { - part += std.subgroupShuffleXor(part, 4); + for (const c of tgpu.unroll(std.range(slice))) { + const kv = d.vec4f(kTile.$[kk * hd4 + lane * slice + c]!); + for (const j of tgpu.unroll(std.range(hp))) { + part[j] = part[j]! + std.dot(qr[j * slice + c]!, kv); + } } const kj = t0 + kk; - if (hasKeyBias > 0) { - part += keyBiasAt(kj, D.biasStart, D.biasEnd, D.bias); + const seen = kj >= band.x && kj < band.y; + for (const j of tgpu.unroll(std.range(hp))) { + let x = part[j]!; + x += std.subgroupShuffleXor(x, 1); + x += std.subgroupShuffleXor(x, 2); + if (lpq > 4) { + x += std.subgroupShuffleXor(x, 4); + } + if (hasKeyBias > 0) { + x += keyBiasAt(kj, D.biasStart, D.biasEnd, D.bias); + } + s[j * tileK + kk] = std.select(d.f32(MASKED), x, seen); + tileMax[j] = std.max(tileMax[j]!, s[j * tileK + kk]!); } - s[kk] = std.select(d.f32(MASKED), part, kj >= band.x && kj < band.y); - tileMax = std.max(tileMax, s[kk]!); } - const mNew = std.max(m, tileMax); - const corr = std.exp(m - mNew); - let pSum = d.f32(0); - for (const kk of tgpu.unroll(std.range(tileK))) { - const p = std.select(d.f32(0), std.exp(s[kk]! - mNew), s[kk]! > d.f32(MASKED)); - s[kk] = p; - pSum += p; + const corr = d.arrayOf(d.f32, hp)(); + for (const j of tgpu.unroll(std.range(hp))) { + const mNew = std.max(m[j]!, tileMax[j]!); + corr[j] = std.exp(m[j]! - mNew); + let pSum = d.f32(0); + for (const kk of tgpu.unroll(std.range(tileK))) { + const sk = s[j * tileK + kk]!; + const p = std.select(d.f32(0), std.exp(sk - mNew), sk > d.f32(MASKED)); + s[j * tileK + kk] = p; + pSum += p; + } + l[j] = l[j]! * corr[j]! + pSum; + m[j] = mNew; } - l = l * corr + pSum; - m = mNew; for (const c of tgpu.unroll(std.range(slice))) { - let acc = o[c]! * corr; + const acc = d.arrayOf(d.vec4f, hp)(); + for (const j of tgpu.unroll(std.range(hp))) { + acc[j] = o[j * slice + c]! * corr[j]!; + } for (const kk of tgpu.unroll(std.range(tileK))) { - acc += s[kk]! * d.vec4f(vTile.$[kk * hd4 + lane * slice + c]!); + const vv = d.vec4f(vTile.$[kk * hd4 + lane * slice + c]!); + for (const j of tgpu.unroll(std.range(hp))) { + acc[j] = acc[j]! + s[j * tileK + kk]! * vv; + } + } + for (const j of tgpu.unroll(std.range(hp))) { + o[j * slice + c] = d.vec4f(acc[j]!); } - o[c] = d.vec4f(acc); } } if (rowValid) { - const invL = std.select(d.f32(0), 1 / l, l > 0); - const oBase4 = row * qHeads * hd4 + h * hd4 + lane * slice; - for (const c of tgpu.unroll(std.range(slice))) { - if (f16) { - L.$.out[oBase4 + c] = d.vec4h(o[c]! * invL) as unknown as d.v4f; - } else { - L.$.out[oBase4 + c] = d.vec4f(o[c]! * invL); + const oBase4 = row * qHeads * hd4 + h0 * hd4 + lane * slice; + for (const j of tgpu.unroll(std.range(hp))) { + const invL = std.select(d.f32(0), 1 / l[j]!, l[j]! > 0); + for (const c of tgpu.unroll(std.range(slice))) { + if (f16) { + L.$.out[oBase4 + j * hd4 + c] = d.vec4h(o[j * slice + c]! * invL) as unknown as d.v4f; + } else { + L.$.out[oBase4 + j * hd4 + c] = d.vec4f(o[j * slice + c]! * invL); + } } } } @@ -232,11 +280,19 @@ export function createAttnFlashPipeline(root: TgpuRoot, cfg: AttnGeoCfg, elem: E return root.createComputePipeline({ compute: makeAttnFlashKernel(cfg, elem) }); } -/** One workgroup per 16 rows of one head. */ +/** One workgroup per 16 rows of one head, or of two heads that share a kv head. */ export function attnFlashHandle( root: TgpuRoot, pipeline: ReturnType, - args: { qLen: number; kvLen: number; qPosOffset: number; qHeads: number; keyBias?: KeyBias }, + args: { + qLen: number; + kvLen: number; + qPosOffset: number; + qHeads: number; + kvHeads: number; + headDim: number; + keyBias?: KeyBias; + }, buffers: { q: FloatBuffer; k: FloatBuffer; @@ -262,5 +318,8 @@ export function attnFlashHandle( segs: buffers.segs ?? filler, dims, }); - return makeHandle(pipeline, 'attnFlash', bindGroup, [Math.ceil(args.qLen / TILE_Q), args.qHeads]); + return makeHandle(pipeline, 'attnFlash', bindGroup, [ + Math.ceil(args.qLen / TILE_Q), + args.qHeads / headsPerWorkgroup(args.qHeads, args.kvHeads, args.headDim), + ]); } diff --git a/packages/runntime/src/core/kernels/matmul/matmulQuantTiled.ts b/packages/runntime/src/core/kernels/matmul/matmulQuantTiled.ts index bc755f0f..f8851c15 100644 --- a/packages/runntime/src/core/kernels/matmul/matmulQuantTiled.ts +++ b/packages/runntime/src/core/kernels/matmul/matmulQuantTiled.ts @@ -107,9 +107,10 @@ function makeMatmulQuantTiledKernel(cfg: MatmulQuantTiledCfg, elem: Elem) { const lanes = colThreads * (BM / TM); const k4 = k / 4; const n4 = n / 4; - const aVecs = Math.ceil((BM * bk4) / lanes); - const bUnits = Math.ceil((BK * bn4) / lanes); const tiles = k / BK; + if (BM * bk4 !== lanes || BK * bn4 !== lanes) { + throw new Error('matmulQuantTiled: each tile needs one vec4 per thread'); + } // Words per packed 4×4 block of W. const tileWords = bits / 2; const cw1 = columnWord(1, bits); @@ -130,40 +131,49 @@ function makeMatmulQuantTiledKernel(cfg: MatmulQuantTiledCfg, elem: Elem) { const accs = d.arrayOf(d.vec4f, TM * TN)(); const bv = d.arrayOf(d.vec4f, 4 * TN)(); + // Thread lid fills slot lid of each tile. + const am = mBase + std.intdiv(lid, bk4); + const aCol = lid % bk4; + const bRow = std.intdiv(lid, bn4); + const bc4 = nBase4 + (lid % bn4); + let ra = storeVec4(); + let w0 = d.u32(0); + let w1 = d.u32(0); + let w2 = d.u32(0); + let w3 = d.u32(0); + let scale = d.vec4f(); + if (am < m) { + ra = storeVec4(loadASlot.$(am, aCol)); + } + if (bc4 < n4) { + const wb = (bc4 * k4 + std.intdiv(bRow, 4)) * tileWords; + w0 = layout.$.w[wb]!; + w1 = layout.$.w[wb + cw1]!; + w2 = layout.$.w[wb + cw2]!; + w3 = layout.$.w[wb + cw3]!; + scale = scaleAtSlot.$(bRow, bc4); + } for (const t of std.range(tiles)) { - for (const i of tgpu.unroll(std.range(aVecs))) { - const e = lid + i * lanes; - if (e < BM * bk4) { - const am = mBase + std.intdiv(e, bk4); - let v = storeVec4(); - if (am < m) { - v = storeVec4(loadASlot.$(am, t * bk4 + (e % bk4))); - } - tileA.$[e] = storeVec4(v); + tileA.$[lid] = storeVec4(ra); + tileB.$[lid] = storeVec4(deqRow(w0, w1, w2, w3, bRow % 4, bits) * scale); + std.workgroupBarrier(); + // The next tile's loads are in flight while this one computes. + const tn = t + 1; + if (tn < tiles) { + ra = storeVec4(); + if (am < m) { + ra = storeVec4(loadASlot.$(am, tn * bk4 + aCol)); } - } - for (const i of tgpu.unroll(std.range(bUnits))) { - const e = lid + i * lanes; - if (e < BK * bn4) { - const bk = t * BK + std.intdiv(e, bn4); - const bc4 = nBase4 + (e % bn4); - let wv = storeVec4(); - if (bc4 < n4) { - const wb = (bc4 * k4 + std.intdiv(bk, 4)) * tileWords; - const q = deqRow( - layout.$.w[wb]!, - layout.$.w[wb + cw1]!, - layout.$.w[wb + cw2]!, - layout.$.w[wb + cw3]!, - bk % 4, - bits, - ); - wv = storeVec4(q * scaleAtSlot.$(bk, bc4)); - } - tileB.$[e] = storeVec4(wv); + if (bc4 < n4) { + const bk = tn * BK + bRow; + const wb = (bc4 * k4 + std.intdiv(bk, 4)) * tileWords; + w0 = layout.$.w[wb]!; + w1 = layout.$.w[wb + cw1]!; + w2 = layout.$.w[wb + cw2]!; + w3 = layout.$.w[wb + cw3]!; + scale = scaleAtSlot.$(bk, bc4); } } - std.workgroupBarrier(); for (const kv of tgpu.unroll(std.range(bk4))) { for (const j of tgpu.unroll(std.range(TN))) { diff --git a/packages/runntime/src/core/kernels/matmul/matmulTiledVec4.ts b/packages/runntime/src/core/kernels/matmul/matmulTiledVec4.ts index 7c49402e..a27cdf8b 100644 --- a/packages/runntime/src/core/kernels/matmul/matmulTiledVec4.ts +++ b/packages/runntime/src/core/kernels/matmul/matmulTiledVec4.ts @@ -107,9 +107,10 @@ function makeMatmulTiledVec4Kernel(cfg: MatmulTiledVec4Cfg, elem: Elem) { const bn4 = BN / 4; const k4 = k / 4; const n4 = n / 4; - const aVecs = (BM * bk4) / lanes; - const bVecs = (BK * bn4) / lanes; const tiles = Math.ceil(k / BK); + if (BM * bk4 !== lanes || BK * bn4 !== lanes) { + throw new Error('matmulTiledVec4: each tile needs one vec4 per thread'); + } const tileA = tgpu.workgroupVar(d.arrayOf(elem.vec4, BM * bk4)); const tileB = tgpu.workgroupVar(d.arrayOf(elem.vec4, BK * bn4)); return tgpu.computeFn({ @@ -124,30 +125,37 @@ function makeMatmulTiledVec4Kernel(cfg: MatmulTiledVec4Cfg, elem: Elem) { const nBase4 = wid.x * bn4; const accs = d.arrayOf(d.vec4f, tm)(); + // Thread lid fills slot lid of each tile. + const am = mBase + std.intdiv(lid, bk4); + const aCol = lid % bk4; + const bRow = std.intdiv(lid, bn4); + const bc4 = nBase4 + (lid % bn4); + let ra = storeVec4(); + let rb = storeVec4(); + if (am < m && aCol < k4) { + ra = storeVec4(loadASlot.$(am, aCol)); + } + if (bRow < k && bc4 < n4) { + rb = storeVec4(layout.$.b[bRow * n4 + bc4]!); + } for (const t of std.range(tiles)) { - // Neighbouring threads load neighbouring vec4s. - for (const i of tgpu.unroll(std.range(aVecs))) { - const e = lid + i * lanes; - const am = mBase + d.u32(e / bk4); - const ak4 = t * bk4 + (e % bk4); - let v = storeVec4(); + tileA.$[lid] = storeVec4(ra); + tileB.$[lid] = storeVec4(rb); + std.workgroupBarrier(); + // The next tile's loads are in flight while this one computes. + const tn = t + 1; + if (tn < tiles) { + const ak4 = tn * bk4 + aCol; + ra = storeVec4(); if (am < m && ak4 < k4) { - v = storeVec4(loadASlot.$(am, ak4)); + ra = storeVec4(loadASlot.$(am, ak4)); } - tileA.$[e] = storeVec4(v); - } - for (const i of tgpu.unroll(std.range(bVecs))) { - const e = lid + i * lanes; - const bk = t * BK + d.u32(e / bn4); - const bc4 = nBase4 + (e % bn4); - let v = storeVec4(); + const bk = tn * BK + bRow; + rb = storeVec4(); if (bk < k && bc4 < n4) { - v = storeVec4(layout.$.b[bk * n4 + bc4]!); + rb = storeVec4(layout.$.b[bk * n4 + bc4]!); } - tileB.$[e] = storeVec4(v); } - std.workgroupBarrier(); - for (const kv of tgpu.unroll(std.range(bk4))) { const b0 = d.vec4f(tileB.$[kv * 4 * bn4 + tx]!); const b1 = d.vec4f(tileB.$[(kv * 4 + 1) * bn4 + tx]!); diff --git a/packages/runntime/test/core/kernels/attention/attnFlash.gpu.test.ts b/packages/runntime/test/core/kernels/attention/attnFlash.gpu.test.ts index a5213cb2..ed6d5ab7 100644 --- a/packages/runntime/test/core/kernels/attention/attnFlash.gpu.test.ts +++ b/packages/runntime/test/core/kernels/attention/attnFlash.gpu.test.ts @@ -14,8 +14,9 @@ describe('attnFlash kernel', () => { // flash: qLen > 1, subgroups, headDim % 16 == 0. Picked in // gpu/dispatch/attention.ts before every other prefill route. - // 8 lanes per row when headDim % 32 == 0, 4 otherwise. Keys per block: 32, - // or 16 for f32 heads wider than 32, fewer past 128. + // 8 lanes per row when headDim % 32 == 0, 4 otherwise. Keys per block: 8 + // with 8 lanes, 32 with 4 (16 for f32 heads wider than 32), fewer past 128. + // Two query heads on one kv head share a workgroup with 8 lanes up to 128. const HEAD_DIMS = [ { headDim: 16, lanes: 4 }, { headDim: 32, lanes: 8 }, @@ -126,7 +127,7 @@ describe('attnFlash kernel', () => { { name: 'no subgroups (rows)', c: { qLen: 37, kvLen: 37, qHeads: 2, kvHeads: 1, headDim: 64 } }, ])('$name: headDim 64', ({ c }) => checkSdpa(withoutSubgroups, c)); - // Known bug: flash dispatches [ceil(qLen / 16), qHeads] workgroups with no + // Known bug: flash dispatches ceil(qLen / 16) workgroups of rows with no // 2-D split, so past 65535 × 16 rows its grid passes the per-dimension // limit: the dispatch fails validation and the output comes back all 0, // with no error. The batch fallback only checks row-heads past diff --git a/packages/runntime/test/core/kernels/attention/checkSdpa.ts b/packages/runntime/test/core/kernels/attention/checkSdpa.ts index 01fc6a34..1d8e60bf 100644 --- a/packages/runntime/test/core/kernels/attention/checkSdpa.ts +++ b/packages/runntime/test/core/kernels/attention/checkSdpa.ts @@ -44,6 +44,10 @@ export const SDPA_OPTIONS: readonly { name: string; opts: Partial }[] { name: 'a key bias', opts: { keyBias: true, windowRight: 0 } }, { name: 'a key bias, sinks and segments', opts: { keyBias: true, sinks: true, segments: true } }, { name: 'every query head on one kv head', opts: { kvHeads: 1, windowRight: 0 } }, + { + name: 'three query heads per kv head', + opts: { qHeads: 6, kvHeads: 2, sinks: true, windowRight: 0 }, + }, { name: 'as many kv heads as query heads', opts: { kvHeads: 4 } }, ];