Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions packages/runntime/bench/kernels/attention.ab.ts
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,30 @@ abSuite('attention', [
dtype: 'f16',
keyBias: { start: 55, end: 305, value: 1 },
}),
sdpa('flash, 8 lanes, GQA 16/4 heads hd 64, causal', {
qLen: 256,
kvLen: 256,
qHeads: 16,
kvHeads: 4,
headDim: 64,
causal: true,
}),
sdpa('flash f16, EmbeddingGemma 2 sliding layer 4/2 heads hd 256, window 512', {
qLen: 512,
kvLen: 512,
qHeads: 4,
kvHeads: 2,
headDim: 256,
dtype: 'f16',
window: 512,
}),
sdpa('flash, EmbeddingGemma 2 full layer 4/1 heads hd 512', {
qLen: 512,
kvLen: 512,
qHeads: 4,
kvHeads: 1,
headDim: 512,
}),
packed('flash packed + segments, MiniLM bucket 128, hd 32', 128, 12, 32, 'f32'),
packed('flash packed f16 + segments, MiniLM bucket 512, hd 32', 512, 12, 32, 'f16'),

Expand Down
2 changes: 2 additions & 0 deletions packages/runntime/src/core/gpu/dispatch/attention.ts
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,8 @@ export const attnSpec = defineSpec({
kvLen,
qPosOffset: attrs.qPosOffset,
qHeads: attrs.qHeads,
kvHeads: attrs.kvHeads,
headDim: attrs.headDim,
keyBias: attrs.keyBias,
};
const bufs = { q, k, v, out: narrowFloat(out, dtype), sinks, segs, dummyF32: ctx.dummy('f32') };
Expand Down
173 changes: 116 additions & 57 deletions packages/runntime/src/core/kernels/attention/attnFlash.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
/** Prefill attention (more than one query row) for head sizes that are a
* multiple of 16. Needs subgroups.
*
* A workgroup handles 16 rows of one head. It loads a block of keys and
* values into workgroup memory once, and all 16 rows use it. Each row is
* A workgroup handles 16 rows of one head, or of two heads that share a kv
* head. It loads a block of keys and values into workgroup memory once, and
* all its rows use it. Each row is
* split across 4 or 8 lanes, which add up q·k with subgroup shuffles.
* Windows, segments, sinks and the key bias work as in the rows kernel.
*
Expand Down Expand Up @@ -64,16 +65,31 @@ export function attnFlashLanes(headDim: number): number {
return headDim % 32 === 0 ? 8 : headDim % 16 === 0 ? 4 : 0;
}

/** Bytes of one key or value tile for heads past 128. */
/** Widest head whose q, output and scores stay in registers. */
const REGISTER_HEAD_DIM = 128;

/** Bytes of one key or value tile for heads past REGISTER_HEAD_DIM. */
const TILE_BYTES = 8192;

/** Keys per block, so the key and value tiles fit in 16 KiB of workgroup
* memory: 32, 16 for f32 heads wider than 32, fewer for heads past 128. */
/** Keys per block: 8 with 8 lanes, which keeps the scores in registers, and
* 32 with 4 lanes, 16 for f32 heads wider than 32. Wider heads take fewer,
* so the key and value tiles fit in workgroup memory. */
function tileKeys(headDim: number, f16: boolean): number {
if (headDim > 128) return Math.max(1, Math.floor(TILE_BYTES / (headDim * (f16 ? 2 : 4))));
if (headDim > REGISTER_HEAD_DIM) {
return Math.max(1, Math.floor(TILE_BYTES / (headDim * (f16 ? 2 : 4))));
}
if (attnFlashLanes(headDim) === 8) return 8;
return !f16 && headDim > 32 ? 16 : 32;
}

/** q heads per workgroup: 2 when they come in pairs that share a kv head, so
* every key and value read from workgroup memory feeds both. Only with 8
* lanes and heads up to REGISTER_HEAD_DIM, where both fit in registers. */
function headsPerWorkgroup(qHeads: number, kvHeads: number, headDim: number): number {
const pairs = Math.floor(qHeads / kvHeads) % 2 === 0;
return pairs && attnFlashLanes(headDim) === 8 && headDim <= REGISTER_HEAD_DIM ? 2 : 1;
}

function makeAttnFlashKernel(cfg: AttnGeoCfg, elem: Elem) {
const { qHeads, kvHeads, headDim: hd, windowLeft, windowRight, hasSinks, hasSegs } = cfg;
const hasKeyBias = cfg.hasKeyBias ?? 0;
Expand All @@ -85,6 +101,7 @@ function makeAttnFlashKernel(cfg: AttnGeoCfg, elem: Elem) {
const tileK = tileKeys(hd, f16);
const lanes = TILE_Q * lpq;
const group = Math.floor(qHeads / kvHeads);
const hp = headsPerWorkgroup(qHeads, kvHeads, hd);
const packed = (cfg.qkvPacked ?? 0) > 0;
const qStride4 = packed ? (qHeads + 2 * kvHeads) * hd4 : qHeads * hd4;
const kvStride4 = packed ? qStride4 : kvHeads * hd4;
Expand All @@ -111,9 +128,9 @@ function makeAttnFlashKernel(cfg: AttnGeoCfg, elem: Elem) {
})(({ lid, wid }) => {
'use gpu';
const D = L.$.dims;
const h = wid.y;
const kvh = d.u32(h / group);
const qSub = d.u32(lid / lpq);
const h0 = wid.y * hp;
const kvh = std.intdiv(h0, group);
const qSub = std.intdiv(lid, lpq);
const lane = lid % lpq;
const row = wid.x * TILE_Q + qSub;
const rowValid = row < D.qLen;
Expand All @@ -138,25 +155,28 @@ function makeAttnFlashKernel(cfg: AttnGeoCfg, elem: Elem) {
}
}

const qBase4 = rowC * qStride4 + h * hd4 + lane * slice;
const qr = d.arrayOf(d.vec4f, slice)();
const o = d.arrayOf(d.vec4f, slice)();
for (const c of tgpu.unroll(std.range(slice))) {
qr[c] = d.vec4f(L.$.q[qBase4 + c]!);
}

let m = d.f32(NEG_SEED);
let l = d.f32(0);
if (hasSinks > 0) {
m = L.$.sinks[h]! * LN2;
l = d.f32(1);
// Head j of the workgroup holds q, out and scores at j·slice and j·tileK.
const qBase4 = rowC * qStride4 + h0 * hd4 + lane * slice;
const qr = d.arrayOf(d.vec4f, hp * slice)();
const o = d.arrayOf(d.vec4f, hp * slice)();
const m = d.arrayOf(d.f32, hp)();
const l = d.arrayOf(d.f32, hp)();
for (const j of tgpu.unroll(std.range(hp))) {
for (const c of tgpu.unroll(std.range(slice))) {
qr[j * slice + c] = d.vec4f(L.$.q[qBase4 + j * hd4 + c]!);
}
m[j] = d.f32(NEG_SEED);
if (hasSinks > 0) {
m[j] = L.$.sinks[h0 + j]! * LN2;
l[j] = d.f32(1);
}
}

const s = d.arrayOf(d.f32, tileK)();
const s = d.arrayOf(d.f32, hp * tileK)();
for (let t0 = kStart; t0 < kStop; t0 += tileK) {
std.workgroupBarrier();
for (let i = lid; i < tileK * hd4; i += lanes) {
const kj = t0 + d.u32(i / hd4);
const kj = t0 + std.intdiv(i, hd4);
const col = i % hd4;
let kv = d.vec4f();
let vv = d.vec4f();
Expand All @@ -176,51 +196,79 @@ function makeAttnFlashKernel(cfg: AttnGeoCfg, elem: Elem) {
}
std.workgroupBarrier();

let tileMax = d.f32(MASKED);
// Each key and value read from workgroup memory feeds every head.
const part = d.arrayOf(d.f32, hp)();
const tileMax = d.arrayOf(d.f32, hp)();
for (const j of tgpu.unroll(std.range(hp))) {
tileMax[j] = d.f32(MASKED);
}
for (const kk of tgpu.unroll(std.range(tileK))) {
let part = d.f32(0);
for (const c of tgpu.unroll(std.range(slice))) {
part += std.dot(qr[c]!, d.vec4f(kTile.$[kk * hd4 + lane * slice + c]!));
for (const j of tgpu.unroll(std.range(hp))) {
part[j] = d.f32(0);
}
part += std.subgroupShuffleXor(part, 1);
part += std.subgroupShuffleXor(part, 2);
if (lpq > 4) {
part += std.subgroupShuffleXor(part, 4);
for (const c of tgpu.unroll(std.range(slice))) {
const kv = d.vec4f(kTile.$[kk * hd4 + lane * slice + c]!);
for (const j of tgpu.unroll(std.range(hp))) {
part[j] = part[j]! + std.dot(qr[j * slice + c]!, kv);
}
}
const kj = t0 + kk;
if (hasKeyBias > 0) {
part += keyBiasAt(kj, D.biasStart, D.biasEnd, D.bias);
const seen = kj >= band.x && kj < band.y;
for (const j of tgpu.unroll(std.range(hp))) {
let x = part[j]!;
x += std.subgroupShuffleXor(x, 1);
x += std.subgroupShuffleXor(x, 2);
if (lpq > 4) {
x += std.subgroupShuffleXor(x, 4);
}
if (hasKeyBias > 0) {
x += keyBiasAt(kj, D.biasStart, D.biasEnd, D.bias);
}
s[j * tileK + kk] = std.select(d.f32(MASKED), x, seen);
tileMax[j] = std.max(tileMax[j]!, s[j * tileK + kk]!);
}
s[kk] = std.select(d.f32(MASKED), part, kj >= band.x && kj < band.y);
tileMax = std.max(tileMax, s[kk]!);
}
const mNew = std.max(m, tileMax);
const corr = std.exp(m - mNew);
let pSum = d.f32(0);
for (const kk of tgpu.unroll(std.range(tileK))) {
const p = std.select(d.f32(0), std.exp(s[kk]! - mNew), s[kk]! > d.f32(MASKED));
s[kk] = p;
pSum += p;
const corr = d.arrayOf(d.f32, hp)();
for (const j of tgpu.unroll(std.range(hp))) {
const mNew = std.max(m[j]!, tileMax[j]!);
corr[j] = std.exp(m[j]! - mNew);
let pSum = d.f32(0);
for (const kk of tgpu.unroll(std.range(tileK))) {
const sk = s[j * tileK + kk]!;
const p = std.select(d.f32(0), std.exp(sk - mNew), sk > d.f32(MASKED));
s[j * tileK + kk] = p;
pSum += p;
}
l[j] = l[j]! * corr[j]! + pSum;
m[j] = mNew;
}
l = l * corr + pSum;
m = mNew;
for (const c of tgpu.unroll(std.range(slice))) {
let acc = o[c]! * corr;
const acc = d.arrayOf(d.vec4f, hp)();
for (const j of tgpu.unroll(std.range(hp))) {
acc[j] = o[j * slice + c]! * corr[j]!;
}
for (const kk of tgpu.unroll(std.range(tileK))) {
acc += s[kk]! * d.vec4f(vTile.$[kk * hd4 + lane * slice + c]!);
const vv = d.vec4f(vTile.$[kk * hd4 + lane * slice + c]!);
for (const j of tgpu.unroll(std.range(hp))) {
acc[j] = acc[j]! + s[j * tileK + kk]! * vv;
}
}
for (const j of tgpu.unroll(std.range(hp))) {
o[j * slice + c] = d.vec4f(acc[j]!);
}
o[c] = d.vec4f(acc);
}
}

if (rowValid) {
const invL = std.select(d.f32(0), 1 / l, l > 0);
const oBase4 = row * qHeads * hd4 + h * hd4 + lane * slice;
for (const c of tgpu.unroll(std.range(slice))) {
if (f16) {
L.$.out[oBase4 + c] = d.vec4h(o[c]! * invL) as unknown as d.v4f;
} else {
L.$.out[oBase4 + c] = d.vec4f(o[c]! * invL);
const oBase4 = row * qHeads * hd4 + h0 * hd4 + lane * slice;
for (const j of tgpu.unroll(std.range(hp))) {
const invL = std.select(d.f32(0), 1 / l[j]!, l[j]! > 0);
for (const c of tgpu.unroll(std.range(slice))) {
if (f16) {
L.$.out[oBase4 + j * hd4 + c] = d.vec4h(o[j * slice + c]! * invL) as unknown as d.v4f;
} else {
L.$.out[oBase4 + j * hd4 + c] = d.vec4f(o[j * slice + c]! * invL);
}
}
}
}
Expand All @@ -232,11 +280,19 @@ export function createAttnFlashPipeline(root: TgpuRoot, cfg: AttnGeoCfg, elem: E
return root.createComputePipeline({ compute: makeAttnFlashKernel(cfg, elem) });
}

/** One workgroup per 16 rows of one head. */
/** One workgroup per 16 rows of one head, or of two heads that share a kv head. */
export function attnFlashHandle(
root: TgpuRoot,
pipeline: ReturnType<typeof createAttnFlashPipeline>,
args: { qLen: number; kvLen: number; qPosOffset: number; qHeads: number; keyBias?: KeyBias },
args: {
qLen: number;
kvLen: number;
qPosOffset: number;
qHeads: number;
kvHeads: number;
headDim: number;
keyBias?: KeyBias;
},
buffers: {
q: FloatBuffer;
k: FloatBuffer;
Expand All @@ -262,5 +318,8 @@ export function attnFlashHandle(
segs: buffers.segs ?? filler,
dims,
});
return makeHandle(pipeline, 'attnFlash', bindGroup, [Math.ceil(args.qLen / TILE_Q), args.qHeads]);
return makeHandle(pipeline, 'attnFlash', bindGroup, [
Math.ceil(args.qLen / TILE_Q),
args.qHeads / headsPerWorkgroup(args.qHeads, args.kvHeads, args.headDim),
]);
}
74 changes: 42 additions & 32 deletions packages/runntime/src/core/kernels/matmul/matmulQuantTiled.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,10 @@ function makeMatmulQuantTiledKernel(cfg: MatmulQuantTiledCfg, elem: Elem) {
const lanes = colThreads * (BM / TM);
const k4 = k / 4;
const n4 = n / 4;
const aVecs = Math.ceil((BM * bk4) / lanes);
const bUnits = Math.ceil((BK * bn4) / lanes);
const tiles = k / BK;
if (BM * bk4 !== lanes || BK * bn4 !== lanes) {
throw new Error('matmulQuantTiled: each tile needs one vec4 per thread');
}
// Words per packed 4×4 block of W.
const tileWords = bits / 2;
const cw1 = columnWord(1, bits);
Expand All @@ -130,40 +131,49 @@ function makeMatmulQuantTiledKernel(cfg: MatmulQuantTiledCfg, elem: Elem) {

const accs = d.arrayOf(d.vec4f, TM * TN)();
const bv = d.arrayOf(d.vec4f, 4 * TN)();
// Thread lid fills slot lid of each tile.
const am = mBase + std.intdiv(lid, bk4);
const aCol = lid % bk4;
const bRow = std.intdiv(lid, bn4);
const bc4 = nBase4 + (lid % bn4);
let ra = storeVec4();
let w0 = d.u32(0);
let w1 = d.u32(0);
let w2 = d.u32(0);
let w3 = d.u32(0);
let scale = d.vec4f();
if (am < m) {
ra = storeVec4(loadASlot.$(am, aCol));
}
if (bc4 < n4) {
const wb = (bc4 * k4 + std.intdiv(bRow, 4)) * tileWords;
w0 = layout.$.w[wb]!;
w1 = layout.$.w[wb + cw1]!;
w2 = layout.$.w[wb + cw2]!;
w3 = layout.$.w[wb + cw3]!;
scale = scaleAtSlot.$(bRow, bc4);
}
for (const t of std.range(tiles)) {
for (const i of tgpu.unroll(std.range(aVecs))) {
const e = lid + i * lanes;
if (e < BM * bk4) {
const am = mBase + std.intdiv(e, bk4);
let v = storeVec4();
if (am < m) {
v = storeVec4(loadASlot.$(am, t * bk4 + (e % bk4)));
}
tileA.$[e] = storeVec4(v);
tileA.$[lid] = storeVec4(ra);
tileB.$[lid] = storeVec4(deqRow(w0, w1, w2, w3, bRow % 4, bits) * scale);
std.workgroupBarrier();
// The next tile's loads are in flight while this one computes.
const tn = t + 1;
if (tn < tiles) {
ra = storeVec4();
if (am < m) {
ra = storeVec4(loadASlot.$(am, tn * bk4 + aCol));
}
}
for (const i of tgpu.unroll(std.range(bUnits))) {
const e = lid + i * lanes;
if (e < BK * bn4) {
const bk = t * BK + std.intdiv(e, bn4);
const bc4 = nBase4 + (e % bn4);
let wv = storeVec4();
if (bc4 < n4) {
const wb = (bc4 * k4 + std.intdiv(bk, 4)) * tileWords;
const q = deqRow(
layout.$.w[wb]!,
layout.$.w[wb + cw1]!,
layout.$.w[wb + cw2]!,
layout.$.w[wb + cw3]!,
bk % 4,
bits,
);
wv = storeVec4(q * scaleAtSlot.$(bk, bc4));
}
tileB.$[e] = storeVec4(wv);
if (bc4 < n4) {
const bk = tn * BK + bRow;
const wb = (bc4 * k4 + std.intdiv(bk, 4)) * tileWords;
w0 = layout.$.w[wb]!;
w1 = layout.$.w[wb + cw1]!;
w2 = layout.$.w[wb + cw2]!;
w3 = layout.$.w[wb + cw3]!;
scale = scaleAtSlot.$(bk, bc4);
}
}
std.workgroupBarrier();

for (const kv of tgpu.unroll(std.range(bk4))) {
for (const j of tgpu.unroll(std.range(TN))) {
Expand Down
Loading
Loading