Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 154 additions & 1 deletion packages/runntime/bench/kernels/conv.ab.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
* DepthART and Moonshine. */
import { abSuite, type AbCase, type Version } from '../ab.ts';
import type { Value } from '../../src/core/index.ts';
import { chw, convWeight, randHwc4, randMat } from './inputs.ts';
import { chw, convWeight, quantWeight, randHwc4, randMat } from './inputs.ts';

interface Conv {
cIn: number;
Expand Down Expand Up @@ -91,6 +91,82 @@ function conv1d(
};
}

/** conv1d with a quantW weight [k·cIn, cOut] and f32 scales [k·cIn /
* groupSize, cOut], one per groupSize values of a torch row [cOut, cIn·k]. */
function conv1dQuant(
name: string,
tIn: number,
cIn: number,
cOut: number,
k: number,
bits: 8 | 4,
groupSize: number,
geometry: { padLeft?: number; padRight?: number; dilation?: number } = {},
dtype: 'f32' | 'f16' = 'f32',
): AbCase {
return {
name,
needs: dtype === 'f16' ? ['shader-f16'] : [],
build: (version) => {
const x = randMat(version, tIn, cIn, dtype);
// The scales' values do not matter for timing.
const { w, scales } = quantWeight(version, 1, k * cIn, cOut, bits, groupSize);
const call = () =>
version.lib.conv1d(x, w, {
kernelSize: k,
stride: 1,
...geometry,
scales,
bits,
groupSize,
});
// Built once here, so a base without quantW conv1d skips the case.
call();
return call;
},
};
}

/** A quantized ConvTranspose1d cIn→out with kernel 2·stride, as the layer
* runs it: conv1d over 2 taps to stride·out channels, its scales [cIn,
* out·kernel / groupSize] in torch layout. */
function convTransposeQuant(
name: string,
tIn: number,
cIn: number,
out: number,
stride: number,
bits: 8 | 4,
groupSize: number,
dtype: 'f32' | 'f16' = 'f32',
): AbCase {
const kernel = 2 * stride;
return {
name,
needs: dtype === 'f16' ? ['shader-f16'] : [],
build: (version) => {
const x = randMat(version, tIn, cIn, dtype);
// As many scales as torch [cIn, out·kernel] has; their values do not
// matter for timing.
const { w, scales } = quantWeight(version, 1, 2 * cIn, stride * out, bits, groupSize);
const call = () =>
version.lib.conv1d(x, w, {
kernelSize: 2,
stride: 1,
padLeft: 1,
padRight: 1,
scales,
bits,
groupSize,
transposedKernel: kernel,
});
// Built once here, so a base without it skips the case.
call();
return call;
},
};
}

/** GLU then a causal depthwise conv over time: x [T, 2C], taps [K, C]. */
function gluConv1d(name: string, t: number, c: number, k: number, dtype: 'f32' | 'f16'): AbCase {
return {
Expand Down Expand Up @@ -294,6 +370,83 @@ abSuite('conv', [
padRight: 1,
}),

// conv1d with a quantW weight: matmulQuantTiled. Groups of 56 values are 8
// channels of a k7 conv.
conv1dQuant('conv1d int8 same, Higgs conv_in 256→1024 k7, 10 s', 250, 256, 1024, 7, 8, 56, {
padLeft: 3,
padRight: 3,
}),
conv1dQuant('conv1d int8 same, Higgs residual 64ch k7, 10 s', 80000, 64, 64, 7, 8, 56, {
padLeft: 3,
padRight: 3,
}),
conv1dQuant(
'conv1d int8 same, groups of 32, Higgs residual 64ch k7, 10 s',
80000,
64,
64,
7,
8,
32,
{
padLeft: 3,
padRight: 3,
},
),
conv1dQuant(
'conv1d int8 same f16, Higgs residual 64ch k7, 10 s',
80000,
64,
64,
7,
8,
56,
{ padLeft: 3, padRight: 3 },
'f16',
),
conv1dQuant('conv1d int4 same, Higgs residual 64ch k7, 10 s', 80000, 64, 64, 7, 4, 56, {
padLeft: 3,
padRight: 3,
}),
conv1dQuant('conv1d int8 dilated, Higgs residual 64ch k7 d9, 10 s', 80000, 64, 64, 7, 8, 56, {
padLeft: 27,
padRight: 27,
dilation: 9,
}),
// Unpadded: the strided loader.
conv1dQuant('conv1d int8 1×1, Higgs residual 64ch, 10 s', 80000, 64, 64, 1, 8, 8),
// One output channel pads to 4 columns; the float conv takes the direct kernel.
conv1d('conv1d same, Higgs output 32→1 k7, 10 s', 240000, 32, 1, 7, 1, undefined, {
padLeft: 3,
padRight: 3,
}),
conv1dQuant(
'conv1d int8 same, Higgs output 32→1 k7 as 4 columns, 10 s',
240000,
32,
4,
7,
8,
56,
{
padLeft: 3,
padRight: 3,
},
),
// ConvTranspose1d with a quantW weight: each column reads its own scale.
convTransposeQuant('convT int8, Higgs upsampling 1024→512 s8, 10 s', 250, 1024, 512, 8, 8, 32),
convTransposeQuant(
'convT int8 f16, Higgs upsampling 1024→512 s8, 10 s',
250,
1024,
512,
8,
8,
32,
'f16',
),
convTransposeQuant('convT int8, Higgs upsampling 64→32 s3, 10 s', 80000, 64, 32, 3, 8, 32),

// gluConv1d: its own kernel.
gluConv1d('gluConv1d, EmbeddingGemma 2 audio 30 s, 1024 channels k5', 750, 1024, 5, 'f32'),
gluConv1d('gluConv1d f16, EmbeddingGemma 2 audio 30 s', 750, 1024, 5, 'f16'),
Expand Down
128 changes: 107 additions & 21 deletions packages/runntime/src/core/gpu/dispatch/conv.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,19 @@
/** Dispatch specs for the convolution kernels (kernels/conv). */

import { elemFor } from '../../kernels/elem.ts';
import { conv1dHandle, conv1dLoadA, createConv1dPipeline } from '../../kernels/conv/conv1d.ts';
import { type Elem, elemFor } from '../../kernels/elem.ts';
import {
conv1dHandle,
conv1dLoadA,
conv1dScales,
convTransposeScales,
createConv1dPipeline,
} from '../../kernels/conv/conv1d.ts';
import { createGluConv1dPipeline, gluConv1dHandle } from '../../kernels/conv/gluConv1d.ts';
import {
createMatmulQuantTiledPipeline,
matmulQuantTiledHandle,
matmulQuantTiledLayout,
} from '../../kernels/matmul/matmulQuantTiled.ts';
import {
createMatmulTiledVec4Pipeline,
matmulTiledVec4Eligible,
Expand Down Expand Up @@ -42,26 +53,59 @@ import {
} from '../../kernels/conv/hwc4.ts';
import { defineSpec, floatDtype, narrow, narrowFloat } from './spec.ts';

/** The scale reader of a quantW conv1d: where the torch weight of a Conv1d,
* or of a ConvTranspose1d when transposedKernel is set, has its scales. */
function quantConvScales(
elem: Elem,
cfg: { cIn: number; cOut: number; kernel: number; groupSize: number; transposedKernel: number },
) {
const layout = matmulQuantTiledLayout(elem);
const { cIn, cOut, kernel, groupSize, transposedKernel } = cfg;
if (transposedKernel === 0) return conv1dScales({ cIn, kernel, groupSize, n: cOut }, layout);
// The ConvTranspose1d's stride, not this conv's.
const stride = transposedKernel / kernel;
return convTransposeScales(
{ cIn, out: cOut / stride, kernel: transposedKernel, stride, groupSize },
layout,
);
}

export const conv1dSpec = defineSpec({
attrs: (p) => {
const [kernel, stride, padLeft, padRight, hasBias, act, dilation] = p.attrs as [
number,
number,
number,
number,
number,
number,
number,
];
return { kernel, stride, padLeft, padRight, hasBias, act, dilation };
const [
kernel,
stride,
padLeft,
padRight,
hasBias,
act,
dilation,
bits,
groupSize,
transposedKernel,
] = p.attrs as [number, number, number, number, number, number, number, number, number, number];
return {
kernel,
stride,
padLeft,
padRight,
hasBias,
act,
dilation,
bits,
groupSize,
transposedKernel,
};
},
cfg: (node, attrs) => {
const cIn = node.pending!.inputs[0]!.shape.dims![1]!;
const cOut = node.shape.dims![1]!;
// An output frame reads kernel·cIn inputs, so the conv runs as the tiled
// matmul. Unpadded and undilated they are contiguous; otherwise
// conv1dLoadA maps each tap to its input frame.
const gemm = cIn % 4 === 0 && matmulTiledVec4Eligible(attrs.kernel * cIn, cOut);
// An output frame reads kernel·cIn inputs, so the conv runs as a tiled
// matmul: matmulQuantTiled for a quantW weight, matmulTiledVec4
// otherwise. Unpadded and undilated, those inputs are contiguous;
// otherwise conv1dLoadA finds each tap's input frame.
const gemm =
attrs.bits > 0 || (cIn % 4 === 0 && matmulTiledVec4Eligible(attrs.kernel * cIn, cOut));
const mapped = attrs.padLeft > 0 || attrs.padRight > 0 || attrs.dilation > 1;
return {
kernel: attrs.kernel,
Expand All @@ -73,29 +117,71 @@ export const conv1dSpec = defineSpec({
hasBias: attrs.hasBias,
act: attrs.act,
gemm: gemm ? (mapped ? 2 : 1) : 0,
bits: attrs.bits,
groupSize: attrs.groupSize,
transposedKernel: attrs.transposedKernel,
};
},
pipeline: (root, dtype, cfg) =>
cfg.gemm
? createMatmulTiledVec4Pipeline(
cfg.bits > 0
? createMatmulQuantTiledPipeline(
root,
{
k: cfg.kernel * cfg.cIn,
n: cfg.cOut,
bits: cfg.bits,
groupSize: cfg.groupSize,
hasBias: cfg.hasBias,
hasAdd: 0,
act: cfg.act,
aStride: cfg.stride * cfg.cIn,
},
elemFor(dtype),
cfg.gemm === 2 ? conv1dLoadA(elemFor(dtype), cfg) : undefined,
cfg.gemm === 2
? conv1dLoadA(elemFor(dtype), cfg, matmulQuantTiledLayout(elemFor(dtype)))
: undefined,
quantConvScales(elemFor(dtype), cfg),
)
: createConv1dPipeline(root, cfg, elemFor(dtype)),
: cfg.gemm
? createMatmulTiledVec4Pipeline(
root,
{
k: cfg.kernel * cfg.cIn,
n: cfg.cOut,
hasBias: cfg.hasBias,
hasAdd: 0,
act: cfg.act,
aStride: cfg.stride * cfg.cIn,
},
elemFor(dtype),
cfg.gemm === 2 ? conv1dLoadA(elemFor(dtype), cfg) : undefined,
)
: createConv1dPipeline(root, cfg, elemFor(dtype)),
encode: ({ node, attrs, cfg, pipeline, inputs, out, ctx }) => {
const dtype = node.shape.dtype;
const tIn = node.pending!.inputs[0]!.shape.dims![0]!;
const [tOut, cOut] = node.shape.dims as [number, number];
const bias = attrs.hasBias === 1 ? narrowFloat(inputs[2]!, dtype) : undefined;
const quant = cfg.bits > 0;
const bias = attrs.hasBias === 1 ? narrowFloat(inputs[quant ? 3 : 2]!, dtype) : undefined;
if (quant) {
return [
matmulQuantTiledHandle(
ctx.root,
pipeline,
tOut,
cOut,
{
a: narrowFloat(inputs[0]!, dtype),
w: narrow(inputs[1]!, 'u32'),
scales: narrow(inputs[2]!, 'f32'),
bias,
out: narrowFloat(out, dtype),
},
elemFor(dtype),
tIn,
),
];
}
if (cfg.gemm) {
return [
matmulTiledVec4Handle(
Expand Down
Loading
Loading