diff --git a/packages/runntime/src/zoo/higgs/decoder.ts b/packages/runntime/src/zoo/higgs/decoder.ts index 516efea0..59c4793d 100644 --- a/packages/runntime/src/zoo/higgs/decoder.ts +++ b/packages/runntime/src/zoo/higgs/decoder.ts @@ -12,6 +12,7 @@ import { add, derivedTensor, + f16ToF32, nn, RunntimeError, type LazyStateDict, @@ -65,6 +66,35 @@ class Block extends nn.Module { this.res_unit2 = new ResidualUnit(out, kernel, 3); this.res_unit3 = new ResidualUnit(out, kernel, 9); } + /** A quantized conv_t1 stored as its equivalent conv, [out, in·kernel] + * with the kernel reversed, loads as its dequantized floats in torch's + * [in, out, kernel]. Every block halves the channels, so the shape tells + * it from torch's [in, out·kernel]. */ + override transformStateDict(sd: LazyStateDict, prefix: string): void { + const name = `${prefix}.conv_t1.weight`; + const t = sd.tensors.get(name); + const { inChannels: cIn, outChannels: out, kernelSize: k } = this.conv_t1; + if (t?.kind !== 'quant' || t.shape.join() !== `${out},${cIn * k}`) return; + sd.tensors.set( + name, + derivedTensor( + [cIn, out, k], + async () => { + const src = await weightValues(t); + const dst = new Float32Array(src.length); + for (let o = 0; o < out; o++) { + for (let c = 0; c < cIn; c++) { + for (let kk = 0; kk < k; kk++) { + dst[(c * out + o) * k + kk] = src[(o * cIn + c) * k + k - 1 - kk]!; + } + } + } + return dst; + }, + t.byteLength, + ), + ); + } override forward(x: Value): Value { const h = this.conv_t1.forward(this.snake1.forward(x)); return this.res_unit3.forward(this.res_unit2.forward(this.res_unit1.forward(h))); @@ -101,9 +131,19 @@ interface Codebook { bias: LazyTensor; } +/** A weight's values: as stored, or a quantized weight's ints times the + * scale of their group. */ +async function weightValues(t: LazyTensor): Promise { + if (t.kind !== 'quant') return t.f32(); + const [ints, stored] = await Promise.all([t.ints(), t.scales()]); + const scales = stored instanceof Uint16Array ? f16ToF32(stored) : stored; + return Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / t.groupSize)]!); +} + /** The decoder input for code i of codebook k is embed_k[i]·project_out_kᵀ· * fc2ᵀ plus a bias shared by every frame: fc2's bias and each project_out - * bias through fc2. Both come out as derived tensors, computed in f64. */ + * bias through fc2. Both come out as derived tensors, computed in f64. A + * quantized project_out or fc2 weight folds as its dequantized values. */ function foldCodebooks( books: Codebook[], fc2W: LazyTensor, @@ -128,7 +168,7 @@ function foldCodebooks( // tensors share it. let transposed: Promise | undefined; const fc2T = () => - (transposed ??= fc2W.f32().then((w) => { + (transposed ??= weightValues(fc2W).then((w) => { const t = new Float64Array(hidden * latent); for (let o = 0; o < latent; o++) { for (let j = 0; j < hidden; j++) t[j * latent + o] = w[o * hidden + j]!; @@ -144,7 +184,7 @@ function foldCodebooks( const m = new Float64Array(dim * latent); const row = new Float64Array(latent); for (const [k, b] of books.entries()) { - const w = await b.weight.f32(); + const w = await weightValues(b.weight); const e = await b.embed.f32(); // m = project_outᵀ·fc2ᵀ, [dim, latent]. m.fill(0); diff --git a/packages/runntime/test/core/weights/checkpoint.ts b/packages/runntime/test/core/weights/checkpoint.ts index a9cca4e8..17d5c95b 100644 --- a/packages/runntime/test/core/weights/checkpoint.ts +++ b/packages/runntime/test/core/weights/checkpoint.ts @@ -93,6 +93,9 @@ export async function contents(t: LazyTensor): Promise<{ type: 'f16' | 'f32'; va * `[out, in / groupSize]`. */ export type QuantEntry = { ints: number[]; scales: number[]; shape: [number, number] }; +/** A QuantEntry with its own bits, for a mixedQuantFile. */ +export type BitsQuantEntry = QuantEntry & { bits: 4 | 8 }; + /** `values`, a torch-layout `[out, in]` weight, quantized the way the format * spec's symmetric scheme reads it: per group of `groupSize` along `in`, the * scale is the group's largest |value| over the top int (127 for int8, 7 for @@ -117,18 +120,19 @@ export function quantizeWeight( return { ints, scales, shape }; } -/** A quantized safetensors file in the format spec: each entry's ints as - * `name` (I8, or U8 with two int4 per byte, low nibble first), its scales - * rounded to f16 as `name_scale`, and the setting in `runntime.quant`. - * `floats` go in as F32 tensors next to them. */ -export function quantFile( - bits: 4 | 8, - groupSize: number, - entries: Record, - floats: Record = {}, -): Uint8Array { - const tensors: Parameters[0] = { ...floats }; - for (const [name, { ints, scales, shape }] of Object.entries(entries)) { +/** The values `entry` stands for: each int times the scale of its group. */ +export function dequantizeWeight({ ints, scales, shape }: QuantEntry): Float32Array { + const groupSize = (shape[0] * shape[1]) / scales.length; + return Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / groupSize)]!); +} + +/** Each entry's ints as `name` (I8, or U8 with two int4 per byte, low nibble + * first) and its scales rounded to f16 as `name_scale`. */ +function quantTensors( + entries: Record, +): Parameters[0] { + const tensors: Parameters[0] = {}; + for (const [name, { bits, ints, scales, shape }] of Object.entries(entries)) { const [out, cols] = shape; tensors[name] = bits === 8 @@ -141,11 +145,42 @@ export function quantFile( }; tensors[`${name}_scale`] = { data: Uint16Array.from(scales, f32ToF16Bits), - shape: [out, cols / groupSize], + shape: [out, scales.length / out], }; } + return tensors; +} + +/** A quantized safetensors file in the format spec: the entries' ints and + * scales, and the setting in `runntime.quant`. `floats` go in as F32 tensors + * next to them. */ +export function quantFile( + bits: 4 | 8, + groupSize: number, + entries: Record, + floats: Record = {}, +): Uint8Array { + const quant = Object.fromEntries( + Object.entries(entries).map(([name, entry]) => [name, { ...entry, bits }]), + ); const setting = { version: 1, bits, groupSize, scheme: 'symmetric' }; - return safetensorsFile(tensors, { 'runntime.quant': JSON.stringify(setting) }); + return safetensorsFile( + { ...floats, ...quantTensors(quant) }, + { 'runntime.quant': JSON.stringify(setting) }, + ); +} + +/** quantFile for entries that differ in bits and group size: the setting + * sets neither, so each tensor gives its own. */ +export function mixedQuantFile( + entries: Record, + floats: Record = {}, +): Uint8Array { + const setting = { version: 1, scheme: 'symmetric' }; + return safetensorsFile( + { ...floats, ...quantTensors(entries) }, + { 'runntime.quant': JSON.stringify(setting) }, + ); } /** The quantized tensor `name` of `entries`, read back from a quantFile. */ diff --git a/packages/runntime/test/zoo/higgs/decoder.gpu.test.ts b/packages/runntime/test/zoo/higgs/decoder.gpu.test.ts new file mode 100644 index 00000000..d2d61b80 --- /dev/null +++ b/packages/runntime/test/zoo/higgs/decoder.gpu.test.ts @@ -0,0 +1,52 @@ +/** The codec decoder loaded from a quantized checkpoint matches the same + * decoder loaded with the dequantized floats in torch's layout. */ +import { describe, it, onTestFinished } from 'vitest'; +import { toArray } from '../../../src/core/gpu/eval.ts'; +import { matrix } from '../../../src/core/graph/value.ts'; +import { + bufferSource, + fromSafetensors, + type LazyStateDict, +} from '../../../src/core/weights/safetensors.ts'; +import { HiggsDecoder } from '../../../src/zoo/higgs/decoder.ts'; +import { expectClose, gpu, requireFeature, upload } from '../../gpu.ts'; +import { checkpoint, QUANT_CFG, quantize, type Sd } from './decoderCheckpoint.ts'; + +const { root, ex } = await gpu(); + +/** Weights in ±0.3 keep every layer's output below 1, so f16 rounding stays + * small. */ +const SPREAD = 0.3; + +describe('HiggsDecoder from a quantized checkpoint', () => { + it.for( + (['f32', 'f16'] as const).flatMap((dtype) => + ([8, 4] as const).map((bits) => ({ dtype, bits })), + ), + )('$dtype int$bits matches the decoder loaded with the dequantized floats', async (c, ctx) => { + if (c.dtype === 'f16') requireFeature(ctx, root, 'shader-f16'); + const { file, dequantized } = quantize(checkpoint(QUANT_CFG, SPREAD).sd, c.bits); + const run = async (sd: LazyStateDict | Sd) => { + const dec = new HiggsDecoder(QUANT_CFG); + if (c.dtype === 'f16') dec.half(root); + onTestFinished(() => dec.dispose()); + await dec.loadStateDict(sd, { root }); + const frames = 16; + const codes = Int32Array.from( + { length: QUANT_CFG.codebooks * frames }, + (_, i) => (i * 3) % QUANT_CFG.codebookSize, + ); + const x = upload(root, dec.latents(codes, frames), { + ...matrix(frames, QUANT_CFG.latent), + dtype: c.dtype, + }); + return toArray(dec.forward(x), ex); + }; + const got = await run(await fromSafetensors(bufferSource(file), { chunkBytes: 0 })); + const want = await run(dequantized); + // Both sum in f32 in another order; f16 rounds every layer's output. + const peak = want.reduce((m, v) => Math.max(m, Math.abs(v)), 0); + const tol = c.dtype === 'f16' ? 2e-3 : 1e-5; + expectClose(got, want, { atol: tol * peak, rtol: tol }); + }); +}); diff --git a/packages/runntime/test/zoo/higgs/decoder.test.ts b/packages/runntime/test/zoo/higgs/decoder.test.ts index aa0af3ff..2dee078d 100644 --- a/packages/runntime/test/zoo/higgs/decoder.test.ts +++ b/packages/runntime/test/zoo/higgs/decoder.test.ts @@ -1,103 +1,19 @@ import { describe, expect, it } from 'vitest'; import { pending, type ValueMeta } from '../../../src/core/graph/value.ts'; -import { memoryStateDict } from '../../../src/core/weights/safetensors.ts'; +import { + bufferSource, + fromSafetensors, + memoryStateDict, + type LazyStateDict, +} from '../../../src/core/weights/safetensors.ts'; import type { HiggsDecoderConfig } from '../../../src/zoo/higgs/config.ts'; import { HiggsDecoder } from '../../../src/zoo/higgs/decoder.ts'; -import { expectClose, randomF32 } from '../../gpu.ts'; - -// The real decoder's layout, small. -const CFG: HiggsDecoderConfig = { - codebooks: 3, - codebookSize: 5, - latent: 8, - channels: 8, - strides: [2], - kernel: 3, -}; -/** The quantizer's codebook width and its project_out width, which only the - * load reads. */ -const DIM = 4; -const HIDDEN = 6; - -type Sd = Record; - -/** Random weights under the codec checkpoint's names, and the decoder input - * they give for codes [codebooks, frames]: each codebook's row through its - * project_out, summed, then fc2, in f64. */ -function checkpoint() { - const sd: Sd = {}; - let seed = 1; - const param = (name: string, shape: number[]) => { - const data = randomF32( - shape.reduce((a, b) => a * b, 1), - seed++, - ); - sd[name] = { data, shape }; - return data; - }; - const { codebooks, codebookSize, latent, channels, kernel: k } = CFG; - const books = Array.from({ length: codebooks }, (_, i) => { - const q = `quantizer.quantizers.${i}`; - param(`${q}.codebook.embed_avg`, [codebookSize, DIM]); - param(`${q}.project_in.weight`, [DIM, HIDDEN]); - return { - embed: param(`${q}.codebook.embed`, [codebookSize, DIM]), - w: param(`${q}.project_out.weight`, [HIDDEN, DIM]), - b: param(`${q}.project_out.bias`, [HIDDEN]), - }; - }); - const fc2 = { w: param('fc2.weight', [latent, HIDDEN]), b: param('fc2.bias', [latent]) }; - param('fc1.weight', [3, HIDDEN]); - param('acoustic_encoder.conv1.weight', [4, 1, 7]); - param('semantic_model.feature_projection.weight', [3, 3]); - - const D = 'acoustic_decoder'; - const conv = (name: string, cIn: number, cOut: number, kk: number) => { - param(`${name}.weight`, [cOut, cIn, kk]); - param(`${name}.bias`, [cOut]); - }; - const snake = (name: string, c: number) => param(`${name}.alpha`, [1, c, 1]); - conv(`${D}.conv1`, latent, channels, k); - const out = channels / 2; - snake(`${D}.block.0.snake1`, channels); - param(`${D}.block.0.conv_t1.weight`, [channels, out, 4]); - param(`${D}.block.0.conv_t1.bias`, [out]); - for (const j of [1, 2, 3]) { - const R = `${D}.block.0.res_unit${j}`; - snake(`${R}.snake1`, out); - conv(`${R}.conv1`, out, out, k); - snake(`${R}.snake2`, out); - conv(`${R}.conv2`, out, out, 1); - } - snake(`${D}.snake1`, out); - conv(`${D}.conv2`, out, 1, k); - - const latents = (codes: Int32Array, frames: number): Float64Array => { - const rows = new Float64Array(frames * latent); - for (let t = 0; t < frames; t++) { - const h = new Float64Array(HIDDEN); - books.forEach((book, c) => { - const code = codes[c * frames + t]!; - for (let j = 0; j < HIDDEN; j++) { - let v = book.b[j]!; - for (let a = 0; a < DIM; a++) v += book.embed[code * DIM + a]! * book.w[j * DIM + a]!; - h[j]! += v; - } - }); - for (let o = 0; o < latent; o++) { - let v = fc2.b[o]!; - for (let j = 0; j < HIDDEN; j++) v += fc2.w[o * HIDDEN + j]! * h[j]!; - rows[t * latent + o] = v; - } - } - return rows; - }; - return { sd, latents }; -} +import { expectClose } from '../../gpu.ts'; +import { CFG, checkpoint, DIM, QUANT_CFG, quantize, type Sd } from './decoderCheckpoint.ts'; /** Loads with stand-ins for the GPU uploads; the CPU table is real. */ -async function load(sd: Sd): Promise { - const dec = new HiggsDecoder(CFG); +async function load(sd: LazyStateDict | Sd, cfg: HiggsDecoderConfig = CFG): Promise { + const dec = new HiggsDecoder(cfg); await dec.loadStateDict(sd, { upload: (_data, shape: ValueMeta) => pending(shape, 'reshape', []), }); @@ -116,6 +32,41 @@ describe('HiggsDecoder', () => { expectClose(dec.latents(codes, frames), latents(codes, frames), { atol: 1e-6, rtol: 1e-5 }); }); + it.for([8, 4] as const)( + 'folds an int%i project_out and fc2 as their dequantized values, with every conv quantized', + async (bits) => { + const { file, dequantized } = quantize(checkpoint(QUANT_CFG).sd, bits); + const sd = await fromSafetensors(bufferSource(file), { chunkBytes: 0 }); + const quant = await load(sd, QUANT_CFG); + const float = await load(dequantized, QUANT_CFG); + const frames = 4; + const codes = Int32Array.from( + { length: QUANT_CFG.codebooks * frames }, + (_, i) => (i * 3) % QUANT_CFG.codebookSize, + ); + // Both fold the same values in f64. + expect(quant.latents(codes, frames)).toEqual(float.latents(codes, frames)); + const scales = [...quant.namedParameters()] + .map(([name]) => name) + .filter((name) => name.endsWith('weightScale')); + const convs = Object.keys(dequantized) + .filter((name) => name.startsWith('acoustic_decoder.') && name.endsWith('.weight')) + .map((name) => name.replace(/weight$/, 'weightScale')); + expect(scales.sort()).toEqual(convs.sort()); + }, + ); + + it('loads a conv_t1 stored as its equivalent conv as its dequantized floats', async () => { + const { file, dequantized } = quantize(checkpoint(QUANT_CFG).sd, 8, { equivalentConvT: true }); + const quantSd = await fromSafetensors(bufferSource(file), { chunkBytes: 0 }); + const floatSd = memoryStateDict(dequantized); + const dec = await load(quantSd, QUANT_CFG); + await load(floatSd, QUANT_CFG); + const name = 'acoustic_decoder.block.0.conv_t1.weight'; + expect(dec.acoustic_decoder.block.items[0]!.conv_t1.weightScale).toBeUndefined(); + expect(await quantSd.tensors.get(name)!.f32()).toEqual(await floatSd.tensors.get(name)!.f32()); + }); + it('keeps only the decoder of the codec checkpoint', () => { const sd = memoryStateDict(checkpoint().sd); new HiggsDecoder(CFG).transformStateDict(sd, ''); diff --git a/packages/runntime/test/zoo/higgs/decoderCheckpoint.ts b/packages/runntime/test/zoo/higgs/decoderCheckpoint.ts new file mode 100644 index 00000000..e1eca281 --- /dev/null +++ b/packages/runntime/test/zoo/higgs/decoderCheckpoint.ts @@ -0,0 +1,171 @@ +import type { HiggsDecoderConfig } from '../../../src/zoo/higgs/config.ts'; +import { randomF32 } from '../../gpu.ts'; +import { + dequantizeWeight, + mixedQuantFile, + quantizeWeight, + type BitsQuantEntry, +} from '../../core/weights/checkpoint.ts'; + +/** The real decoder's layout, small. */ +export const CFG: HiggsDecoderConfig = { + codebooks: 3, + codebookSize: 5, + latent: 8, + channels: 8, + strides: [2], + kernel: 3, +}; +/** The quantizer's codebook width and its project_out width, which only the + * load reads. */ +export const DIM = 4; +const HIDDEN = 6; + +export type Sd = Record; + +/** Random weights in [-spread, spread) under the codec checkpoint's names, + * and the decoder input they give for codes [codebooks, frames]: each + * codebook's row through its project_out, summed, then fc2, in f64. */ +export function checkpoint(cfg: HiggsDecoderConfig = CFG, spread = 1) { + const sd: Sd = {}; + let seed = 1; + const param = (name: string, shape: number[]) => { + const data = randomF32( + shape.reduce((a, b) => a * b, 1), + seed++, + ).map((v) => v * spread); + sd[name] = { data, shape }; + return data; + }; + const { codebooks, codebookSize, latent, channels, strides, kernel: k } = cfg; + const books = Array.from({ length: codebooks }, (_, i) => { + const q = `quantizer.quantizers.${i}`; + param(`${q}.codebook.embed_avg`, [codebookSize, DIM]); + param(`${q}.project_in.weight`, [DIM, HIDDEN]); + return { + embed: param(`${q}.codebook.embed`, [codebookSize, DIM]), + w: param(`${q}.project_out.weight`, [HIDDEN, DIM]), + b: param(`${q}.project_out.bias`, [HIDDEN]), + }; + }); + const fc2 = { w: param('fc2.weight', [latent, HIDDEN]), b: param('fc2.bias', [latent]) }; + param('fc1.weight', [3, HIDDEN]); + param('acoustic_encoder.conv1.weight', [4, 1, 7]); + param('semantic_model.feature_projection.weight', [3, 3]); + + const D = 'acoustic_decoder'; + const conv = (name: string, cIn: number, cOut: number, kk: number) => { + param(`${name}.weight`, [cOut, cIn, kk]); + param(`${name}.bias`, [cOut]); + }; + const snake = (name: string, c: number) => param(`${name}.alpha`, [1, c, 1]); + conv(`${D}.conv1`, latent, channels, k); + strides.forEach((stride, i) => { + const B = `${D}.block.${i}`; + const cIn = channels / 2 ** i; + const out = cIn / 2; + snake(`${B}.snake1`, cIn); + param(`${B}.conv_t1.weight`, [cIn, out, 2 * stride]); + param(`${B}.conv_t1.bias`, [out]); + for (const j of [1, 2, 3]) { + const R = `${B}.res_unit${j}`; + snake(`${R}.snake1`, out); + conv(`${R}.conv1`, out, out, k); + snake(`${R}.snake2`, out); + conv(`${R}.conv2`, out, out, 1); + } + }); + const out = channels / 2 ** strides.length; + snake(`${D}.snake1`, out); + conv(`${D}.conv2`, out, 1, k); + + const latents = (codes: Int32Array, frames: number): Float64Array => { + const rows = new Float64Array(frames * latent); + for (let t = 0; t < frames; t++) { + const h = new Float64Array(HIDDEN); + books.forEach((book, c) => { + const code = codes[c * frames + t]!; + for (let j = 0; j < HIDDEN; j++) { + let v = book.b[j]!; + for (let a = 0; a < DIM; a++) v += book.embed[code * DIM + a]! * book.w[j * DIM + a]!; + h[j]! += v; + } + }); + for (let o = 0; o < latent; o++) { + let v = fc2.b[o]!; + for (let j = 0; j < HIDDEN; j++) v += fc2.w[o * HIDDEN + j]! * h[j]!; + rows[t * latent + o] = v; + } + } + return rows; + }; + return { sd, latents }; +} + +/** CFG with every conv's shape fit for the quantized matmul, so each conv + * loads quantized. */ +export const QUANT_CFG: HiggsDecoderConfig = { ...CFG, channels: 16 }; + +/** Values per scale group of a conv's torch row. */ +const CONV_GROUP = 8; +/** Values per scale group of project_out and fc2, which the load folds. */ +const FOLD_GROUP = 2; + +/** A transposed conv's weight [in, out, kernel] as its equivalent conv + * [out, in·kernel] with the kernel reversed: for each file value, its index + * in torch's layout. */ +function equivalentConv(shape: number[]): Int32Array { + const [cIn, out, kernel] = shape as [number, number, number]; + const index = new Int32Array(cIn * out * kernel); + for (let o = 0; o < out; o++) { + for (let c = 0; c < cIn; c++) { + for (let t = 0; t < kernel; t++) { + index[(o * cIn + c) * kernel + t] = (c * out + o) * kernel + kernel - 1 - t; + } + } + } + return index; +} + +/** `sd` as a quantized file with every conv weight, project_out and fc2 + * quantized, and `sd` with each of those replaced by its dequantized values. + * A conv is quantized as its torch weight with the dims after the first + * flattened. With `equivalentConvT`, a transposed conv is quantized as its + * equivalent conv in groups of CONV_GROUP input channels, as the codec's QAT + * checkpoint stores it. Codebook tables, biases and Snake alphas stay + * float. */ +export function quantize( + sd: Sd, + bits: 4 | 8, + { equivalentConvT = false } = {}, +): { file: Uint8Array; dequantized: Sd } { + const entries: Record = {}; + const floats: Sd = {}; + const dequantized: Sd = { ...sd }; + for (const [name, { data, shape }] of Object.entries(sd)) { + if (name === 'fc2.weight' || name.endsWith('.project_out.weight')) { + const entry = quantizeWeight(data, shape as [number, number], bits, FOLD_GROUP); + entries[name] = { ...entry, bits }; + dequantized[name] = { data: dequantizeWeight(entry), shape }; + } else if (equivalentConvT && /\.conv_t\d+\.weight$/.test(name)) { + const index = equivalentConv(shape); + const rows = Float32Array.from(index, (i) => data[i]!); + const [, out, kernel] = shape as [number, number, number]; + const group = CONV_GROUP * kernel; + const entry = quantizeWeight(rows, [out, rows.length / out], bits, group); + entries[name] = { ...entry, bits }; + const values = dequantizeWeight(entry); + const back = new Float32Array(data.length); + index.forEach((i, r) => (back[i] = values[r]!)); + dequantized[name] = { data: back, shape }; + } else if (name.startsWith('acoustic_decoder.') && name.endsWith('.weight')) { + const rows = shape[0]!; + const entry = quantizeWeight(data, [rows, data.length / rows], bits, CONV_GROUP); + entries[name] = { ...entry, bits }; + dequantized[name] = { data: dequantizeWeight(entry), shape }; + } else { + floats[name] = { data, shape }; + } + } + return { file: mixedQuantFile(entries, floats), dequantized }; +}