diff --git a/packages/runntime/src/core/index.ts b/packages/runntime/src/core/index.ts index 0cffc663..98cd1af9 100644 --- a/packages/runntime/src/core/index.ts +++ b/packages/runntime/src/core/index.ts @@ -150,6 +150,7 @@ export { createOpfsCache } from './weights/opfsCache.ts'; export { foldBnIntoConv } from './weights/foldBn.ts'; export { concatRows, + dequantize, permuteRows, reshapeWeight, scaleWeight, diff --git a/packages/runntime/src/core/nn/layers.ts b/packages/runntime/src/core/nn/layers.ts index 7be374c9..58b81b6e 100644 --- a/packages/runntime/src/core/nn/layers.ts +++ b/packages/runntime/src/core/nn/layers.ts @@ -36,7 +36,7 @@ import { import { foldBnIntoConv } from '../weights/foldBn.ts'; import { packQuantW } from '../weights/quantPack.ts'; import { derivedTensor, type LazyStateDict, type QuantTensor } from '../weights/safetensors.ts'; -import { readFloats, transposeWeight } from '../weights/transforms.ts'; +import { dequantize, readFloats, transposeWeight } from '../weights/transforms.ts'; import { tensor } from '../gpu/buffers.ts'; import { CpuParameter, Module, Parameter, type AnyModule } from './module.ts'; @@ -256,26 +256,27 @@ export class Linear extends Module { } /** Throws CHECKPOINT_MISMATCH unless the quantized conv weight `t` is its - * torch weight with the dims after the first flattened, [rows, cols], and - * the conv runs on the quantized matmul over `taps` taps to n columns. */ -function checkQuantConv( + * torch weight with the dims after the first flattened, [rows, cols]. */ +function checkQuantConvShape( layer: string, name: string, t: QuantTensor, - shape: { rows: number; cols: number; layout: string; cIn: number; taps: number; n: number }, + { rows, cols, layout }: { rows: number; cols: number; layout: string }, ): void { - const { rows, cols, layout, cIn, taps, n } = shape; - const bad = (why: string): never => { - throw new RunntimeError('CHECKPOINT_MISMATCH', `${layer}: ${name} ${why}`); - }; if (t.shape[0] !== rows || t.shape[1] !== cols) { - bad(`is quantized as [${t.shape.join(', ')}], expected ${layout} = [${rows}, ${cols}]`); - } - if (cIn % 4 !== 0 || !matmulQuantTiledEligible(taps * cIn, n)) { - bad(`needs in a multiple of 4 and ${taps}·in of ${MATMUL_QUANT_TILED_BK}, got in ${cIn}`); + throw new RunntimeError( + 'CHECKPOINT_MISMATCH', + `${layer}: ${name} is quantized as [${t.shape.join(', ')}], expected ${layout} = [${rows}, ${cols}]`, + ); } } +/** Whether a conv over `taps` taps of cIn channels to n columns runs on the + * quantized matmul. */ +function quantConvFits(cIn: number, taps: number, n: number): boolean { + return cIn % 4 === 0 && matmulQuantTiledEligible(taps * cIn, n); +} + /** Strided, dilated 1D convolution over time-major [T, inChannels]. No batch * dim, no groups. * @@ -289,7 +290,8 @@ function checkQuantConv( * group of values along each row. It loads as quantW words with its f32 * scales in `weightScale` [in·kernel / groupSize, out]. Output channels pad * up to the packing width, scales and bias with them, and are cut after the - * conv. */ + * conv. A conv the quantized matmul cannot run loads a quantized weight as + * its dequantized floats. */ export class Conv1d extends Module { readonly weight: Parameter; readonly bias?: Parameter; @@ -333,11 +335,22 @@ export class Conv1d extends Module { } override transformStateDict(sd: LazyStateDict, prefix: string): void { const name = prefix ? `${prefix}.weight` : 'weight'; - const tensor = sd.tensors.get(name); + let tensor = sd.tensors.get(name); if (!tensor) return; // a missing key is reported by the strict key check if (tensor.kind === 'quant') { - this.loadQuantized(sd, prefix, name, tensor); - return; + const { inChannels: cIn, outChannels: cOut, kernelSize: kernel } = this; + checkQuantConvShape('Conv1d', name, tensor, { + rows: cOut, + cols: cIn * kernel, + layout: '[out, in·kernel]', + }); + const width = 32 / tensor.bits; + const n = Math.ceil(cOut / width) * width; + if (quantConvFits(cIn, kernel, n)) { + this.loadQuantized(sd, prefix, name, tensor, n); + return; + } + tensor = dequantize(tensor, [cOut, cIn, kernel]); } // Without this a weight with equal in/out channels loads scrambled. if ( @@ -370,19 +383,15 @@ export class Conv1d extends Module { /** Loads `name` as quantW words [kernel·in, n] with its scales * [in·kernel / groupSize, n], where n is outChannels padded to the packing * width. The bias pads to n too. */ - private loadQuantized(sd: LazyStateDict, prefix: string, name: string, t: QuantTensor): void { + private loadQuantized( + sd: LazyStateDict, + prefix: string, + name: string, + t: QuantTensor, + n: number, + ): void { const { inChannels: cIn, outChannels: cOut, kernelSize: kernel } = this; - const width = 32 / t.bits; - const n = Math.ceil(cOut / width) * width; const k = kernel * cIn; - checkQuantConv('Conv1d', name, t, { - rows: cOut, - cols: k, - layout: '[out, in·kernel]', - cIn, - taps: kernel, - n, - }); const groups = k / t.groupSize; this.quantization = { bits: t.bits, groupSize: t.groupSize }; this.weightScale = loadQuantWeight( @@ -460,8 +469,8 @@ export class Conv1d extends Module { * * A quantized checkpoint weight is torch's [in, out·kernel], one scale per * group of values along each row. It loads as quantW words with its f32 - * scales as stored in `weightScale`. outChannels must be a multiple of 4 - * and stride·outChannels of the packing width. */ + * scales as stored in `weightScale`. A transposed conv the quantized matmul + * cannot run loads a quantized weight as its dequantized floats. */ export class ConvTranspose1d extends Module { readonly weight: Parameter; readonly bias?: Parameter; @@ -508,10 +517,21 @@ export class ConvTranspose1d extends Module { override transformStateDict(sd: LazyStateDict, prefix: string): void { const at = (n: string) => (prefix ? `${prefix}.${n}` : n); const { inChannels: cIn, outChannels: cOut, kernelSize: k, taps, stride: s } = this; - const w = sd.tensors.get(at('weight')); + let w = sd.tensors.get(at('weight')); if (w?.kind === 'quant') { - this.loadQuantized(sd, prefix, at('weight'), w); - } else if (w) { + checkQuantConvShape('ConvTranspose1d', at('weight'), w, { + rows: cIn, + cols: cOut * k, + layout: '[in, out·kernel]', + }); + const n = s * cOut; + if (cOut % 4 === 0 && n % (32 / w.bits) === 0 && quantConvFits(cIn, taps, n)) { + this.loadQuantized(sd, prefix, w); + } else { + w = dequantize(w, [cIn, cOut, k]); + } + } + if (w && w.kind !== 'quant') { if (w.shape.length !== 3 || w.shape[0] !== cIn || w.shape[1] !== cOut || w.shape[2] !== k) { throw new RunntimeError( 'CHECKPOINT_MISMATCH', @@ -562,28 +582,13 @@ export class ConvTranspose1d extends Module { ); } } - /** Loads `name` as quantW words [taps·in, stride·out] with its scales + /** Loads `t` as quantW words [taps·in, stride·out] with its scales * [in, out·kernel / groupSize] as stored. */ - private loadQuantized(sd: LazyStateDict, prefix: string, name: string, t: QuantTensor): void { + private loadQuantized(sd: LazyStateDict, prefix: string, t: QuantTensor): void { const { inChannels: cIn, outChannels: cOut, kernelSize: kernel, taps, stride: s } = this; const n = s * cOut; - if (cOut % 4 !== 0 || n % (32 / t.bits) !== 0) { - throw new RunntimeError( - 'CHECKPOINT_MISMATCH', - `ConvTranspose1d: ${name} is int${t.bits}, which needs out a multiple of 4 and ` + - `stride·out of ${32 / t.bits}, got out ${cOut}, stride ${s}`, - ); - } const k = taps * cIn; const cols = cOut * kernel; - checkQuantConv('ConvTranspose1d', name, t, { - rows: cIn, - cols, - layout: '[in, out·kernel]', - cIn, - taps, - n, - }); this.quantization = { bits: t.bits, groupSize: t.groupSize }; const groups = cols / t.groupSize; this.weightScale = loadQuantWeight( diff --git a/packages/runntime/src/core/weights/transforms.ts b/packages/runntime/src/core/weights/transforms.ts index 469cb109..ab28f7ea 100644 --- a/packages/runntime/src/core/weights/transforms.ts +++ b/packages/runntime/src/core/weights/transforms.ts @@ -1,16 +1,16 @@ /** Lazy reshaping helpers for `transformStateDict` hooks: concatenate, split, - * permute, scale, transpose and reshape checkpoint tensors without fetching - * them. + * permute, scale, transpose, reshape and dequantize checkpoint tensors + * without fetching them. * * F16 sources give f16 halfwords and F32 sources give f32, except - * `scaleWeight`, which always gives f32. BF16 decodes to + * `scaleWeight` and `dequantize`, which always give f32. BF16 decodes to * f32, which is exact; a derived tensor's Uint16Array means f16, so bf16 bits * have no derived form. Each output's byteLength is its share of the source * bytes, so load progress still sums to the checkpoint size. * - * A QuantTensor source gives a QuantTensor: the helpers move its ints and - * scales together and never dequantize. Whatever would split a scale group - * (a column split off the group grid, a reshape of the last dim, a + * A QuantTensor source gives a QuantTensor, except in `dequantize`: the + * helpers move its ints and scales together. Whatever would split a scale + * group (a column split off the group grid, a reshape of the last dim, a * transpose) throws CHECKPOINT_MISMATCH. */ import { RunntimeError } from '../error.ts'; import { f16ToF32 } from './convert.ts'; @@ -384,3 +384,20 @@ export function reshapeWeight(t: LazyTensor, shape: readonly number[]): LazyTens // source, so a shallow copy shares the data and reads it the same way. return { ...t, shape: [...shape] }; } + +/** The values of `t` as f32 in `shape`, each int times the scale of its group. + * Throws CHECKPOINT_MISMATCH when the element counts differ. */ +export function dequantize(t: QuantTensor, shape: readonly number[] = t.shape): LazyTensor { + if (numel(shape) !== numel(t.shape)) { + throw mismatch('dequantize', `cannot give [${t.shape.join(', ')}] shape [${shape.join(', ')}]`); + } + return derivedTensor( + [...shape], + async () => { + const [ints, scales] = await Promise.all([t.ints(), t.scales()]); + const f32 = toF32(scales); + return Float32Array.from(ints, (q, i) => q * f32[Math.floor(i / t.groupSize)]!); + }, + t.byteLength, + ); +} diff --git a/packages/runntime/src/zoo/higgs/decoder.ts b/packages/runntime/src/zoo/higgs/decoder.ts index 59c4793d..e1d6d3f1 100644 --- a/packages/runntime/src/zoo/higgs/decoder.ts +++ b/packages/runntime/src/zoo/higgs/decoder.ts @@ -11,8 +11,8 @@ import { add, + dequantize, derivedTensor, - f16ToF32, nn, RunntimeError, type LazyStateDict, @@ -133,11 +133,8 @@ interface Codebook { /** A weight's values: as stored, or a quantized weight's ints times the * scale of their group. */ -async function weightValues(t: LazyTensor): Promise { - if (t.kind !== 'quant') return t.f32(); - const [ints, stored] = await Promise.all([t.ints(), t.scales()]); - const scales = stored instanceof Uint16Array ? f16ToF32(stored) : stored; - return Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / t.groupSize)]!); +function weightValues(t: LazyTensor): Promise { + return (t.kind === 'quant' ? dequantize(t) : t).f32(); } /** The decoder input for code i of codebook k is embed_k[i]·project_out_kᵀ· diff --git a/packages/runntime/src/zoo/higgs/encoder.ts b/packages/runntime/src/zoo/higgs/encoder.ts index b078ac2e..922c7498 100644 --- a/packages/runntime/src/zoo/higgs/encoder.ts +++ b/packages/runntime/src/zoo/higgs/encoder.ts @@ -147,8 +147,9 @@ class FeatureProjection extends nn.Module { /** HuBERT's conv position embedding: a grouped conv with weight norm, as one * conv per group of channels, then GELU. An even kernel pads one frame less - * on the right, the frame torch cuts. The load folds the weight norm and - * splits the weight and bias into the groups. */ + * on the right, the frame torch cuts. The load folds the weight norm, or + * takes the checkpoint's folded weight, and splits the weight and bias into + * the groups. */ class PositionConv extends nn.Module { readonly convs: nn.ModuleList; constructor( @@ -172,9 +173,23 @@ class PositionConv extends nn.Module { ); } override transformStateDict(sd: LazyStateDict, prefix: string): void { + const bias = take(sd, prefix, 'conv.bias'); + const { hidden, groups } = this; + const width = hidden / groups; + const sizes = Array(groups).fill(width); + const weights = splitRows(this.weight(sd, prefix), sizes); + const biases = splitRows(bias, sizes); + for (let i = 0; i < groups; i++) { + sd.tensors.set(at(prefix, `convs.${i}.weight`), weights[i]!); + sd.tensors.set(at(prefix, `convs.${i}.bias`), biases[i]!); + } + } + /** The checkpoint's folded weight, or its weight norm folded. Each group's + * conv checks the shape of its rows. */ + private weight(sd: LazyStateDict, prefix: string): LazyTensor { + if (sd.tensors.has(at(prefix, 'conv.weight'))) return take(sd, prefix, 'conv.weight'); const g = take(sd, prefix, 'conv.parametrizations.weight.original0'); const v = take(sd, prefix, 'conv.parametrizations.weight.original1'); - const bias = take(sd, prefix, 'conv.bias'); const { hidden, kernel, groups } = this; const width = hidden / groups; if ( @@ -187,13 +202,7 @@ class PositionConv extends nn.Module { `expected [1,1,${kernel}] and [${hidden},${width},${kernel}]`, ); } - const sizes = Array(groups).fill(width); - const weights = splitRows(foldWeightNorm(g, v), sizes); - const biases = splitRows(bias, sizes); - for (let i = 0; i < groups; i++) { - sd.tensors.set(at(prefix, `convs.${i}.weight`), weights[i]!); - sd.tensors.set(at(prefix, `convs.${i}.bias`), biases[i]!); - } + return foldWeightNorm(g, v); } override forward(x: Value): Value { const width = this.hidden / this.groups; diff --git a/packages/runntime/test/core/nn/layers.test.ts b/packages/runntime/test/core/nn/layers.test.ts index 6c370bb1..a9de2e0e 100644 --- a/packages/runntime/test/core/nn/layers.test.ts +++ b/packages/runntime/test/core/nn/layers.test.ts @@ -523,36 +523,73 @@ describe('Conv1d and ConvTranspose1d with a quantized weight', () => { }); it.each([ - { - name: 'kernel·in that is not a multiple of 8', - layer: () => new Conv1d(4, 4, 3), - file: () => quantCheckpoint(8, 12, { weight: zeros(4, 12, 1) }), - error: /needs in a multiple of 4 and 3·in of 8, got in 4/, - }, - { - name: 'in that is not a multiple of 4', - layer: () => new Conv1d(2, 4, 4), - file: () => quantCheckpoint(8, 8, { weight: zeros(4, 8, 1) }), - error: /needs in a multiple of 4 and 4·in of 8, got in 2/, + { name: 'one input channel', cIn: 1, kernel: 3 }, + { name: 'kernel·in that is not a multiple of 8', cIn: 4, kernel: 3 }, + { name: 'in that is not a multiple of 4', cIn: 2, kernel: 4 }, + ])('loads a Conv1d weight on $name as its dequantized floats', async ({ cIn, kernel }) => { + // One scale per output row, 0.5 and 0.25; ints 1, 2, … and −1, −2, … + const cols = cIn * kernel; + const ints = Array.from({ length: 2 * cols }, (_, i) => (i < cols ? i + 1 : cols - i - 1)); + const layer = new Conv1d(cIn, 2, kernel, { bias: false }); + const sd = await quantCheckpoint(8, cols, { + weight: { ints, scales: [0.5, 0.25], shape: [2, cols] }, + }); + layer.transformStateDict(sd, ''); + expect(shapes(layer)).toEqual({ weight: meta([cols, 2]) }); + // tap-major [kernel·in, out]: row tap·in + c holds both outputs' (c, tap) + const want = Array.from({ length: 2 * cols }, (_, i) => { + const [row, o] = [Math.floor(i / 2), i % 2]; + const [tap, c] = [Math.floor(row / cIn), row % cIn]; + return ints[o * cols + c * kernel + tap]! * (o === 0 ? 0.5 : 0.25); + }); + expect([...(await sd.tensors.get('weight')!.f32())]).toEqual(want); + }); + + it.each([ + { name: 'taps·in that is not a multiple of 8', cIn: 4, cOut: 4, kernel: 6 }, + { name: 'out that is not a multiple of 4', cIn: 8, cOut: 2, kernel: 4 }, + ])( + 'loads a ConvTranspose1d weight on $name as its dequantized floats', + async ({ cIn, cOut, kernel }) => { + // stride 2; one scale per torch row c, (c + 1) / 4; ints −25 … 24 + const [s, taps, cols] = [2, kernel / 2, cOut * kernel]; + const ints = Array.from({ length: cIn * cols }, (_, i) => (i % 50) - 25); + const scales = Array.from({ length: cIn }, (_, c) => (c + 1) / 4); + const layer = new ConvTranspose1d(cIn, cOut, kernel, { stride: s, bias: false }); + const sd = await quantCheckpoint(8, cols, { + weight: { ints, scales, shape: [cIn, cols] }, + }); + layer.transformStateDict(sd, ''); + expect(shapes(layer)).toEqual({ weight: meta([taps * cIn, s * cOut]) }); + // row i·in + c, column r·out + o holds kernel position (taps − 1 − i)·s + r + const want = Array.from({ length: taps * cIn * s * cOut }, (_, j) => { + const [row, col] = [Math.floor(j / (s * cOut)), j % (s * cOut)]; + const [i, c, r, o] = [Math.floor(row / cIn), row % cIn, Math.floor(col / cOut), col % cOut]; + return ints[c * cols + o * kernel + (taps - 1 - i) * s + r]! * scales[c]!; + }); + expect([...(await sd.tensors.get('weight')!.f32())]).toEqual(want); }, + ); + + it.each([ { name: 'a ConvTranspose1d weight in Conv1d layout', layer: () => new ConvTranspose1d(16, 8, 4, { stride: 2 }), file: () => quantCheckpoint(8, 8, { weight: zeros(8, 64, 8) }), error: /expected \[in, out·kernel\] = \[16, 32\]/, }, + { + name: 'a weight of another shape on one input channel', + layer: () => new Conv1d(1, 4, 3), + file: () => quantCheckpoint(8, 4, { weight: zeros(3, 4, 1) }), + error: /expected \[out, in·kernel\] = \[4, 3\]/, + }, { name: 'a weight of another shape', layer: () => new Conv1d(8, 4, 3), file: () => quantCheckpoint(8, 8, { weight: zeros(4, 16, 2) }), error: /expected \[out, in·kernel\] = \[4, 24\]/, }, - { - name: 'out that is not a multiple of 4', - layer: () => new ConvTranspose1d(8, 2, 4, { stride: 2 }), - file: () => quantCheckpoint(8, 8, { weight: zeros(8, 8, 1) }), - error: /needs out a multiple of 4 and stride·out of 4, got out 2, stride 2/, - }, ])('refuses $name with CHECKPOINT_MISMATCH', async ({ layer, file, error }) => { const sd = await file(); expect(() => layer().transformStateDict(sd, '')).toThrow( diff --git a/packages/runntime/test/core/weights/transforms.test.ts b/packages/runntime/test/core/weights/transforms.test.ts index 73dc0ca3..f184f568 100644 --- a/packages/runntime/test/core/weights/transforms.test.ts +++ b/packages/runntime/test/core/weights/transforms.test.ts @@ -9,6 +9,7 @@ import { } from '../../../src/core/weights/safetensors.ts'; import { concatRows, + dequantize, permuteRows, readFloats, reshapeWeight, @@ -504,6 +505,29 @@ describe('quantized helpers', () => { expect(scales).toHaveBeenCalledOnce(); }); + it('dequantize gives each int times the f16 scale of its group, in the shape asked', async () => { + // int4, groups of 2: scales 0.5 and 0.25 + const t = await quantSource(4, 2, 'w', { + ints: [1, -2, 7, -8], + scales: [0.5, 0.25], + shape: [1, 4], + }); + const d = dequantize(t, [2, 2]); + expect(d.shape).toEqual([2, 2]); + expect([...(await d.f32())]).toEqual([0.5, -1, 1.75, -2]); + expect(d.byteLength).toBe(t.byteLength); + }); + + it('dequantize refuses a shape of another size', () => { + const { t } = counted(); + expect(() => dequantize(t, [3])).toThrow( + expect.objectContaining({ + code: 'CHECKPOINT_MISMATCH', + message: expect.stringMatching(/cannot give \[2, 2\]/), + }), + ); + }); + it('splits read the source once for all their outputs', async () => { const { t, ints, scales } = counted(); await Promise.all(splitRows(t, [1, 1]).map(quantContents)); diff --git a/packages/runntime/test/zoo/higgs/encoder.gpu.test.ts b/packages/runntime/test/zoo/higgs/encoder.gpu.test.ts index cd363e0c..5cf4daed 100644 --- a/packages/runntime/test/zoo/higgs/encoder.gpu.test.ts +++ b/packages/runntime/test/zoo/higgs/encoder.gpu.test.ts @@ -1,10 +1,15 @@ -import { afterAll, describe, expect, it } from 'vitest'; +import { afterAll, describe, expect, it, onTestFinished } from 'vitest'; import { toArray } from '../../../src/core/gpu/eval.ts'; import { matrix } from '../../../src/core/graph/value.ts'; +import { + bufferSource, + fromSafetensors, + type LazyStateDict, +} from '../../../src/core/weights/safetensors.ts'; import { HiggsEncoder } from '../../../src/zoo/higgs/encoder.ts'; import { expectClose, gpu, randomF32, upload } from '../../gpu.ts'; import { ACTIVATION_REF } from '../../core/kernels/matmul/checkMatmul.ts'; -import { CFG, checkpoint } from './encoderCheckpoint.ts'; +import { CFG, checkpoint, QUANT_CFG, quantize, type Sd } from './encoderCheckpoint.ts'; const { root, ex } = await gpu(); @@ -95,3 +100,39 @@ describe('HiggsEncoder on the GPU', () => { expect(Int32Array.from(await toArray(y, ex))).toEqual(want); }); }); + +describe('HiggsEncoder from a quantized checkpoint', () => { + it.for([8, 4] as const)( + 'with int%i Linears matches the encoder loaded with the dequantized floats', + async (bits) => { + const { file, dequantized } = quantize(checkpoint(QUANT_CFG), bits); + const hidden = QUANT_CFG.acoustic.latent + QUANT_CFG.hubert.hidden; + // 20 samples give HuBERT 4 frames, two whole strides. + const [audio, semantic, rows] = [ + randomF32(30, 21), + randomF32(20, 22), + randomF32(5 * hidden, 23), + ]; + const run = async (sd: LazyStateDict | Sd) => { + const e = new HiggsEncoder(QUANT_CFG); + onTestFinished(() => e.dispose()); + await e.loadStateDict(sd, { root }); + const a = e.acoustic_encoder.forward(upload(root, audio, [audio.length, 1])); + const s = e.encoder_semantic.forward( + e.semantic_model.forward(upload(root, semantic, [semantic.length, 1])), + ); + const fc = e.fc.forward(upload(root, rows, [5, hidden])); + const out: Float32Array[] = []; + for (const v of [a, s, fc]) out.push(await toArray(v, ex)); + return out; + }; + const got = await run(await fromSafetensors(bufferSource(file), { chunkBytes: 0 })); + const want = await run(dequantized); + // Both sum in f32 in another order. + got.forEach((g, i) => { + const peak = want[i]!.reduce((m, v) => Math.max(m, Math.abs(v)), 0); + expectClose(g, want[i]!, { atol: 1e-5 * peak, rtol: 1e-4 }); + }); + }, + ); +}); diff --git a/packages/runntime/test/zoo/higgs/encoder.test.ts b/packages/runntime/test/zoo/higgs/encoder.test.ts index 64cd0b10..c6d6e0e5 100644 --- a/packages/runntime/test/zoo/higgs/encoder.test.ts +++ b/packages/runntime/test/zoo/higgs/encoder.test.ts @@ -1,11 +1,16 @@ import { describe, expect, it } from 'vitest'; import { matrix, pending, type ValueMeta } from '../../../src/core/graph/value.ts'; -import { memoryStateDict } from '../../../src/core/weights/safetensors.ts'; +import { Conv1d, Linear } from '../../../src/core/nn/layers.ts'; +import { + bufferSource, + fromSafetensors, + memoryStateDict, +} from '../../../src/core/weights/safetensors.ts'; import { HIGGS_ENCODER } from '../../../src/zoo/higgs/config.ts'; import { HiggsEncoder } from '../../../src/zoo/higgs/encoder.ts'; import { resample } from '../../../src/zoo/higgs/resample.ts'; import { expectClose } from '../../gpu.ts'; -import { CFG, checkpoint, type Sd } from './encoderCheckpoint.ts'; +import { CFG, checkpoint, QUANT_CFG, quantize, type Sd } from './encoderCheckpoint.ts'; const POS = 'semantic_model.encoder.pos_conv_embed'; const ATTN = 'semantic_model.encoder.layers.0.attention'; @@ -59,6 +64,51 @@ describe('HiggsEncoder', () => { expect([...sd.tensors.keys()].filter((n) => n.startsWith(`${POS}.conv.`))).toEqual([]); }); + it("takes the checkpoint's folded position conv weight and splits it into groups", async () => { + const raw = checkpoint(); + const { hidden, posGroups, posKernel: k } = CFG.hubert; + const width = hidden / posGroups; + const weight = { + data: Float32Array.from({ length: hidden * width * k }, (_, i) => i), + shape: [hidden, width, k], + }; + delete raw[`${POS}.conv.parametrizations.weight.original0`]; + delete raw[`${POS}.conv.parametrizations.weight.original1`]; + const sd = memoryStateDict({ ...raw, [`${POS}.conv.weight`]: weight }); + new HiggsEncoder(CFG).semantic_model.encoder.pos_conv_embed.transformStateDict(sd, POS); + expect(await read(sd, `${POS}.convs.1.weight`)).toEqual( + weight.data.subarray(width * width * k, 2 * width * width * k), + ); + expect([...sd.tensors.keys()].filter((n) => n.startsWith(`${POS}.conv.`))).toEqual([]); + }); + + it.for([8, 4] as const)( + 'loads int%i Linears and int8 convs quantized, the one-input-channel convs as floats', + async (bits) => { + const { file } = quantize(checkpoint(QUANT_CFG), bits); + const enc = new HiggsEncoder(QUANT_CFG); + await enc.loadStateDict(await fromSafetensors(bufferSource(file), { chunkBytes: 0 }), { + upload: (_data, shape: ValueMeta) => pending(shape, 'reshape', []), + }); + const floats = [ + 'acoustic_encoder.conv1', + 'semantic_model.feature_extractor.conv_layers.0.conv', + ]; + const layers = [...enc.namedModules()] + .filter( + ([name, m]) => + (m instanceof Conv1d || m instanceof Linear) && !name.startsWith('quantizer.'), + ) + .map(([name]) => name); + const quantized = [...enc.namedParameters()] + .map(([name]) => name) + .filter((name) => name.endsWith('.weightScale')) + .map((name) => name.slice(0, -'.weightScale'.length)); + expect(layers).toEqual(expect.arrayContaining(floats)); + expect(quantized).toEqual(layers.filter((name) => !floats.includes(name))); + }, + ); + it('stacks q, k and v into one projection, q scaled by 1/√headDim', async () => { const raw = checkpoint(); const sd = memoryStateDict(raw); diff --git a/packages/runntime/test/zoo/higgs/encoderCheckpoint.ts b/packages/runntime/test/zoo/higgs/encoderCheckpoint.ts index 86698b1f..f414e2b6 100644 --- a/packages/runntime/test/zoo/higgs/encoderCheckpoint.ts +++ b/packages/runntime/test/zoo/higgs/encoderCheckpoint.ts @@ -1,5 +1,11 @@ import type { HiggsEncoderConfig } from '../../../src/zoo/higgs/config.ts'; import { randomF32 } from '../../gpu.ts'; +import { + dequantizeWeight, + mixedQuantFile, + quantizeWeight, + type BitsQuantEntry, +} from '../../core/weights/checkpoint.ts'; /** The real encoder's layout, small. */ export const CFG: HiggsEncoderConfig = { @@ -133,3 +139,64 @@ export function checkpoint(cfg: HiggsEncoderConfig = CFG): Sd { conv('decoder_semantic.conv1', hu.hidden, hu.hidden, k, false); return sd; } + +/** CFG with every conv but the two on one input channel, and every Linear, + * loadable quantized. */ +export const QUANT_CFG: HiggsEncoderConfig = { + ...CFG, + acoustic: { ...CFG.acoustic, channels: 8, latent: 8 }, + hubert: { ...CFG.hubert, ffn: 16 }, +}; + +/** Input channels per scale group of an acoustic conv. */ +const CONV_GROUP = 4; +/** Values per scale group of a Linear. */ +const LINEAR_GROUP = 4; + +const POS = 'semantic_model.encoder.pos_conv_embed.conv'; + +/** The position conv's weight norm folded, [hidden, width, kernel], in f64. */ +function foldedPositionWeight(sd: Sd): { data: Float32Array; shape: number[] } { + const g = sd[`${POS}.parametrizations.weight.original0`]!.data; + const { data: v, shape } = sd[`${POS}.parametrizations.weight.original1`]!; + const kernel = shape[2]!; + const norm = new Float64Array(kernel); + v.forEach((x, i) => (norm[i % kernel]! += x * x)); + return { data: v.map((x, i) => (x * g[i % kernel]!) / Math.sqrt(norm[i % kernel]!)), shape }; +} + +/** `sd` as a quantized file the way the codec's int files hold it: the + * position conv as its folded weight, the convs in int8, the acoustic ones + * in groups of input channels and HuBERT's and the semantic ones with one + * scale per output channel, and every Linear but the quantizer's in `bits`, + * in small groups. Also `sd` with each of those replaced by its dequantized + * values. The quantizer, norms, biases and Snake alphas stay float. */ +export function quantize(sd: Sd, bits: 4 | 8): { file: Uint8Array; dequantized: Sd } { + const source: Sd = { ...sd, [`${POS}.weight`]: foldedPositionWeight(sd) }; + delete source[`${POS}.parametrizations.weight.original0`]; + delete source[`${POS}.parametrizations.weight.original1`]; + const entries: Record = {}; + const floats: Sd = {}; + const dequantized: Sd = { ...source }; + for (const [name, { data, shape }] of Object.entries(source)) { + const encoder = /^(acoustic_encoder|semantic_model|encoder_semantic|fc)\./.test(name); + if (!encoder || !name.endsWith('.weight') || shape.length === 1) { + floats[name] = { data, shape }; + continue; + } + // torch [out, in, kernel] is [out, in·kernel] in the file, the same order. + const [cOut, cIn, kernel = 1] = shape as [number, number, number?]; + const cols = cIn * kernel; + const groupSize = + shape.length === 2 + ? LINEAR_GROUP + : name.startsWith('acoustic_encoder.') && cIn % CONV_GROUP === 0 + ? CONV_GROUP * kernel + : cols; + const entryBits = shape.length === 2 ? bits : 8; + const entry = quantizeWeight(data, [cOut, cols], entryBits, groupSize); + entries[name] = { ...entry, bits: entryBits }; + dequantized[name] = { data: dequantizeWeight(entry), shape }; + } + return { file: mixedQuantFile(entries, floats), dequantized }; +}