Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/runntime/src/core/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ export { createOpfsCache } from './weights/opfsCache.ts';
export { foldBnIntoConv } from './weights/foldBn.ts';
export {
concatRows,
dequantize,
permuteRows,
reshapeWeight,
scaleWeight,
Expand Down
105 changes: 55 additions & 50 deletions packages/runntime/src/core/nn/layers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ import {
import { foldBnIntoConv } from '../weights/foldBn.ts';
import { packQuantW } from '../weights/quantPack.ts';
import { derivedTensor, type LazyStateDict, type QuantTensor } from '../weights/safetensors.ts';
import { readFloats, transposeWeight } from '../weights/transforms.ts';
import { dequantize, readFloats, transposeWeight } from '../weights/transforms.ts';
import { tensor } from '../gpu/buffers.ts';
import { CpuParameter, Module, Parameter, type AnyModule } from './module.ts';

Expand Down Expand Up @@ -256,26 +256,27 @@ export class Linear extends Module {
}

/** Throws CHECKPOINT_MISMATCH unless the quantized conv weight `t` is its
* torch weight with the dims after the first flattened, [rows, cols], and
* the conv runs on the quantized matmul over `taps` taps to n columns. */
function checkQuantConv(
* torch weight with the dims after the first flattened, [rows, cols]. */
function checkQuantConvShape(
layer: string,
name: string,
t: QuantTensor,
shape: { rows: number; cols: number; layout: string; cIn: number; taps: number; n: number },
{ rows, cols, layout }: { rows: number; cols: number; layout: string },
): void {
const { rows, cols, layout, cIn, taps, n } = shape;
const bad = (why: string): never => {
throw new RunntimeError('CHECKPOINT_MISMATCH', `${layer}: ${name} ${why}`);
};
if (t.shape[0] !== rows || t.shape[1] !== cols) {
bad(`is quantized as [${t.shape.join(', ')}], expected ${layout} = [${rows}, ${cols}]`);
}
if (cIn % 4 !== 0 || !matmulQuantTiledEligible(taps * cIn, n)) {
bad(`needs in a multiple of 4 and ${taps}·in of ${MATMUL_QUANT_TILED_BK}, got in ${cIn}`);
throw new RunntimeError(
'CHECKPOINT_MISMATCH',
`${layer}: ${name} is quantized as [${t.shape.join(', ')}], expected ${layout} = [${rows}, ${cols}]`,
);
}
}

/** Whether a conv over `taps` taps of cIn channels to n columns runs on the
* quantized matmul. */
function quantConvFits(cIn: number, taps: number, n: number): boolean {
return cIn % 4 === 0 && matmulQuantTiledEligible(taps * cIn, n);
}

/** Strided, dilated 1D convolution over time-major [T, inChannels]. No batch
* dim, no groups.
*
Expand All @@ -289,7 +290,8 @@ function checkQuantConv(
* group of values along each row. It loads as quantW words with its f32
* scales in `weightScale` [in·kernel / groupSize, out]. Output channels pad
* up to the packing width, scales and bias with them, and are cut after the
* conv. */
* conv. A conv the quantized matmul cannot run loads a quantized weight as
* its dequantized floats. */
export class Conv1d extends Module {
readonly weight: Parameter;
readonly bias?: Parameter;
Expand Down Expand Up @@ -333,11 +335,22 @@ export class Conv1d extends Module {
}
override transformStateDict(sd: LazyStateDict, prefix: string): void {
const name = prefix ? `${prefix}.weight` : 'weight';
const tensor = sd.tensors.get(name);
let tensor = sd.tensors.get(name);
if (!tensor) return; // a missing key is reported by the strict key check
if (tensor.kind === 'quant') {
this.loadQuantized(sd, prefix, name, tensor);
return;
const { inChannels: cIn, outChannels: cOut, kernelSize: kernel } = this;
checkQuantConvShape('Conv1d', name, tensor, {
rows: cOut,
cols: cIn * kernel,
layout: '[out, in·kernel]',
});
const width = 32 / tensor.bits;
const n = Math.ceil(cOut / width) * width;
if (quantConvFits(cIn, kernel, n)) {
this.loadQuantized(sd, prefix, name, tensor, n);
return;
}
tensor = dequantize(tensor, [cOut, cIn, kernel]);
}
// Without this a weight with equal in/out channels loads scrambled.
if (
Expand Down Expand Up @@ -370,19 +383,15 @@ export class Conv1d extends Module {
/** Loads `name` as quantW words [kernel·in, n] with its scales
* [in·kernel / groupSize, n], where n is outChannels padded to the packing
* width. The bias pads to n too. */
private loadQuantized(sd: LazyStateDict, prefix: string, name: string, t: QuantTensor): void {
private loadQuantized(
sd: LazyStateDict,
prefix: string,
name: string,
t: QuantTensor,
n: number,
): void {
const { inChannels: cIn, outChannels: cOut, kernelSize: kernel } = this;
const width = 32 / t.bits;
const n = Math.ceil(cOut / width) * width;
const k = kernel * cIn;
checkQuantConv('Conv1d', name, t, {
rows: cOut,
cols: k,
layout: '[out, in·kernel]',
cIn,
taps: kernel,
n,
});
const groups = k / t.groupSize;
this.quantization = { bits: t.bits, groupSize: t.groupSize };
this.weightScale = loadQuantWeight(
Expand Down Expand Up @@ -460,8 +469,8 @@ export class Conv1d extends Module {
*
* A quantized checkpoint weight is torch's [in, out·kernel], one scale per
* group of values along each row. It loads as quantW words with its f32
* scales as stored in `weightScale`. outChannels must be a multiple of 4
* and stride·outChannels of the packing width. */
* scales as stored in `weightScale`. A transposed conv the quantized matmul
* cannot run loads a quantized weight as its dequantized floats. */
export class ConvTranspose1d extends Module {
readonly weight: Parameter;
readonly bias?: Parameter;
Expand Down Expand Up @@ -508,10 +517,21 @@ export class ConvTranspose1d extends Module {
override transformStateDict(sd: LazyStateDict, prefix: string): void {
const at = (n: string) => (prefix ? `${prefix}.${n}` : n);
const { inChannels: cIn, outChannels: cOut, kernelSize: k, taps, stride: s } = this;
const w = sd.tensors.get(at('weight'));
let w = sd.tensors.get(at('weight'));
if (w?.kind === 'quant') {
this.loadQuantized(sd, prefix, at('weight'), w);
} else if (w) {
checkQuantConvShape('ConvTranspose1d', at('weight'), w, {
rows: cIn,
cols: cOut * k,
layout: '[in, out·kernel]',
});
const n = s * cOut;
if (cOut % 4 === 0 && n % (32 / w.bits) === 0 && quantConvFits(cIn, taps, n)) {
this.loadQuantized(sd, prefix, w);
} else {
w = dequantize(w, [cIn, cOut, k]);
}
}
if (w && w.kind !== 'quant') {
if (w.shape.length !== 3 || w.shape[0] !== cIn || w.shape[1] !== cOut || w.shape[2] !== k) {
throw new RunntimeError(
'CHECKPOINT_MISMATCH',
Expand Down Expand Up @@ -562,28 +582,13 @@ export class ConvTranspose1d extends Module {
);
}
}
/** Loads `name` as quantW words [taps·in, stride·out] with its scales
/** Loads `t` as quantW words [taps·in, stride·out] with its scales
* [in, out·kernel / groupSize] as stored. */
private loadQuantized(sd: LazyStateDict, prefix: string, name: string, t: QuantTensor): void {
private loadQuantized(sd: LazyStateDict, prefix: string, t: QuantTensor): void {
const { inChannels: cIn, outChannels: cOut, kernelSize: kernel, taps, stride: s } = this;
const n = s * cOut;
if (cOut % 4 !== 0 || n % (32 / t.bits) !== 0) {
throw new RunntimeError(
'CHECKPOINT_MISMATCH',
`ConvTranspose1d: ${name} is int${t.bits}, which needs out a multiple of 4 and ` +
`stride·out of ${32 / t.bits}, got out ${cOut}, stride ${s}`,
);
}
const k = taps * cIn;
const cols = cOut * kernel;
checkQuantConv('ConvTranspose1d', name, t, {
rows: cIn,
cols,
layout: '[in, out·kernel]',
cIn,
taps,
n,
});
this.quantization = { bits: t.bits, groupSize: t.groupSize };
const groups = cols / t.groupSize;
this.weightScale = loadQuantWeight(
Expand Down
29 changes: 23 additions & 6 deletions packages/runntime/src/core/weights/transforms.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
/** Lazy reshaping helpers for `transformStateDict` hooks: concatenate, split,
* permute, scale, transpose and reshape checkpoint tensors without fetching
* them.
* permute, scale, transpose, reshape and dequantize checkpoint tensors
* without fetching them.
*
* F16 sources give f16 halfwords and F32 sources give f32, except
* `scaleWeight`, which always gives f32. BF16 decodes to
* `scaleWeight` and `dequantize`, which always give f32. BF16 decodes to
* f32, which is exact; a derived tensor's Uint16Array means f16, so bf16 bits
* have no derived form. Each output's byteLength is its share of the source
* bytes, so load progress still sums to the checkpoint size.
*
* A QuantTensor source gives a QuantTensor: the helpers move its ints and
* scales together and never dequantize. Whatever would split a scale group
* (a column split off the group grid, a reshape of the last dim, a
* A QuantTensor source gives a QuantTensor, except in `dequantize`: the
* helpers move its ints and scales together. Whatever would split a scale
* group (a column split off the group grid, a reshape of the last dim, a
* transpose) throws CHECKPOINT_MISMATCH. */
import { RunntimeError } from '../error.ts';
import { f16ToF32 } from './convert.ts';
Expand Down Expand Up @@ -384,3 +384,20 @@ export function reshapeWeight(t: LazyTensor, shape: readonly number[]): LazyTens
// source, so a shallow copy shares the data and reads it the same way.
return { ...t, shape: [...shape] };
}

/** The values of `t` as f32 in `shape`, each int times the scale of its group.
* Throws CHECKPOINT_MISMATCH when the element counts differ. */
export function dequantize(t: QuantTensor, shape: readonly number[] = t.shape): LazyTensor {
if (numel(shape) !== numel(t.shape)) {
throw mismatch('dequantize', `cannot give [${t.shape.join(', ')}] shape [${shape.join(', ')}]`);
}
return derivedTensor(
[...shape],
async () => {
const [ints, scales] = await Promise.all([t.ints(), t.scales()]);
const f32 = toF32(scales);
return Float32Array.from(ints, (q, i) => q * f32[Math.floor(i / t.groupSize)]!);
},
t.byteLength,
);
}
9 changes: 3 additions & 6 deletions packages/runntime/src/zoo/higgs/decoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@

import {
add,
dequantize,
derivedTensor,
f16ToF32,
nn,
RunntimeError,
type LazyStateDict,
Expand Down Expand Up @@ -133,11 +133,8 @@ interface Codebook {

/** A weight's values: as stored, or a quantized weight's ints times the
* scale of their group. */
async function weightValues(t: LazyTensor): Promise<Float32Array> {
if (t.kind !== 'quant') return t.f32();
const [ints, stored] = await Promise.all([t.ints(), t.scales()]);
const scales = stored instanceof Uint16Array ? f16ToF32(stored) : stored;
return Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / t.groupSize)]!);
function weightValues(t: LazyTensor): Promise<Float32Array> {
return (t.kind === 'quant' ? dequantize(t) : t).f32();
}

/** The decoder input for code i of codebook k is embed_k[i]·project_out_kᵀ·
Expand Down
29 changes: 19 additions & 10 deletions packages/runntime/src/zoo/higgs/encoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -147,8 +147,9 @@ class FeatureProjection extends nn.Module {

/** HuBERT's conv position embedding: a grouped conv with weight norm, as one
* conv per group of channels, then GELU. An even kernel pads one frame less
* on the right, the frame torch cuts. The load folds the weight norm and
* splits the weight and bias into the groups. */
* on the right, the frame torch cuts. The load folds the weight norm, or
* takes the checkpoint's folded weight, and splits the weight and bias into
* the groups. */
class PositionConv extends nn.Module {
readonly convs: nn.ModuleList<nn.Conv1d>;
constructor(
Expand All @@ -172,9 +173,23 @@ class PositionConv extends nn.Module {
);
}
override transformStateDict(sd: LazyStateDict, prefix: string): void {
const bias = take(sd, prefix, 'conv.bias');
const { hidden, groups } = this;
const width = hidden / groups;
const sizes = Array<number>(groups).fill(width);
const weights = splitRows(this.weight(sd, prefix), sizes);
const biases = splitRows(bias, sizes);
for (let i = 0; i < groups; i++) {
sd.tensors.set(at(prefix, `convs.${i}.weight`), weights[i]!);
sd.tensors.set(at(prefix, `convs.${i}.bias`), biases[i]!);
}
}
/** The checkpoint's folded weight, or its weight norm folded. Each group's
* conv checks the shape of its rows. */
private weight(sd: LazyStateDict, prefix: string): LazyTensor {
if (sd.tensors.has(at(prefix, 'conv.weight'))) return take(sd, prefix, 'conv.weight');
const g = take(sd, prefix, 'conv.parametrizations.weight.original0');
const v = take(sd, prefix, 'conv.parametrizations.weight.original1');
const bias = take(sd, prefix, 'conv.bias');
const { hidden, kernel, groups } = this;
const width = hidden / groups;
if (
Expand All @@ -187,13 +202,7 @@ class PositionConv extends nn.Module {
`expected [1,1,${kernel}] and [${hidden},${width},${kernel}]`,
);
}
const sizes = Array<number>(groups).fill(width);
const weights = splitRows(foldWeightNorm(g, v), sizes);
const biases = splitRows(bias, sizes);
for (let i = 0; i < groups; i++) {
sd.tensors.set(at(prefix, `convs.${i}.weight`), weights[i]!);
sd.tensors.set(at(prefix, `convs.${i}.bias`), biases[i]!);
}
return foldWeightNorm(g, v);
}
override forward(x: Value): Value {
const width = this.hidden / this.groups;
Expand Down
71 changes: 54 additions & 17 deletions packages/runntime/test/core/nn/layers.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -523,36 +523,73 @@ describe('Conv1d and ConvTranspose1d with a quantized weight', () => {
});

it.each([
{
name: 'kernel·in that is not a multiple of 8',
layer: () => new Conv1d(4, 4, 3),
file: () => quantCheckpoint(8, 12, { weight: zeros(4, 12, 1) }),
error: /needs in a multiple of 4 and 3·in of 8, got in 4/,
},
{
name: 'in that is not a multiple of 4',
layer: () => new Conv1d(2, 4, 4),
file: () => quantCheckpoint(8, 8, { weight: zeros(4, 8, 1) }),
error: /needs in a multiple of 4 and 4·in of 8, got in 2/,
{ name: 'one input channel', cIn: 1, kernel: 3 },
{ name: 'kernel·in that is not a multiple of 8', cIn: 4, kernel: 3 },
{ name: 'in that is not a multiple of 4', cIn: 2, kernel: 4 },
])('loads a Conv1d weight on $name as its dequantized floats', async ({ cIn, kernel }) => {
// One scale per output row, 0.5 and 0.25; ints 1, 2, … and −1, −2, …
const cols = cIn * kernel;
const ints = Array.from({ length: 2 * cols }, (_, i) => (i < cols ? i + 1 : cols - i - 1));
const layer = new Conv1d(cIn, 2, kernel, { bias: false });
const sd = await quantCheckpoint(8, cols, {
weight: { ints, scales: [0.5, 0.25], shape: [2, cols] },
});
layer.transformStateDict(sd, '');
expect(shapes(layer)).toEqual({ weight: meta([cols, 2]) });
// tap-major [kernel·in, out]: row tap·in + c holds both outputs' (c, tap)
const want = Array.from({ length: 2 * cols }, (_, i) => {
const [row, o] = [Math.floor(i / 2), i % 2];
const [tap, c] = [Math.floor(row / cIn), row % cIn];
return ints[o * cols + c * kernel + tap]! * (o === 0 ? 0.5 : 0.25);
});
expect([...(await sd.tensors.get('weight')!.f32())]).toEqual(want);
});

it.each([
{ name: 'taps·in that is not a multiple of 8', cIn: 4, cOut: 4, kernel: 6 },
{ name: 'out that is not a multiple of 4', cIn: 8, cOut: 2, kernel: 4 },
])(
'loads a ConvTranspose1d weight on $name as its dequantized floats',
async ({ cIn, cOut, kernel }) => {
// stride 2; one scale per torch row c, (c + 1) / 4; ints −25 … 24
const [s, taps, cols] = [2, kernel / 2, cOut * kernel];
const ints = Array.from({ length: cIn * cols }, (_, i) => (i % 50) - 25);
const scales = Array.from({ length: cIn }, (_, c) => (c + 1) / 4);
const layer = new ConvTranspose1d(cIn, cOut, kernel, { stride: s, bias: false });
const sd = await quantCheckpoint(8, cols, {
weight: { ints, scales, shape: [cIn, cols] },
});
layer.transformStateDict(sd, '');
expect(shapes(layer)).toEqual({ weight: meta([taps * cIn, s * cOut]) });
// row i·in + c, column r·out + o holds kernel position (taps − 1 − i)·s + r
const want = Array.from({ length: taps * cIn * s * cOut }, (_, j) => {
const [row, col] = [Math.floor(j / (s * cOut)), j % (s * cOut)];
const [i, c, r, o] = [Math.floor(row / cIn), row % cIn, Math.floor(col / cOut), col % cOut];
return ints[c * cols + o * kernel + (taps - 1 - i) * s + r]! * scales[c]!;
});
expect([...(await sd.tensors.get('weight')!.f32())]).toEqual(want);
},
);

it.each([
{
name: 'a ConvTranspose1d weight in Conv1d layout',
layer: () => new ConvTranspose1d(16, 8, 4, { stride: 2 }),
file: () => quantCheckpoint(8, 8, { weight: zeros(8, 64, 8) }),
error: /expected \[in, out·kernel\] = \[16, 32\]/,
},
{
name: 'a weight of another shape on one input channel',
layer: () => new Conv1d(1, 4, 3),
file: () => quantCheckpoint(8, 4, { weight: zeros(3, 4, 1) }),
error: /expected \[out, in·kernel\] = \[4, 3\]/,
},
{
name: 'a weight of another shape',
layer: () => new Conv1d(8, 4, 3),
file: () => quantCheckpoint(8, 8, { weight: zeros(4, 16, 2) }),
error: /expected \[out, in·kernel\] = \[4, 24\]/,
},
{
name: 'out that is not a multiple of 4',
layer: () => new ConvTranspose1d(8, 2, 4, { stride: 2 }),
file: () => quantCheckpoint(8, 8, { weight: zeros(8, 8, 1) }),
error: /needs out a multiple of 4 and stride·out of 4, got out 2, stride 2/,
},
])('refuses $name with CHECKPOINT_MISMATCH', async ({ layer, file, error }) => {
const sd = await file();
expect(() => layer().transformStateDict(sd, '')).toThrow(
Expand Down
Loading
Loading