Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 43 additions & 3 deletions packages/runntime/src/zoo/higgs/decoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import {
add,
derivedTensor,
f16ToF32,
nn,
RunntimeError,
type LazyStateDict,
Expand Down Expand Up @@ -65,6 +66,35 @@ class Block extends nn.Module {
this.res_unit2 = new ResidualUnit(out, kernel, 3);
this.res_unit3 = new ResidualUnit(out, kernel, 9);
}
/** A quantized conv_t1 stored as its equivalent conv, [out, in·kernel]
* with the kernel reversed, loads as its dequantized floats in torch's
* [in, out, kernel]. Every block halves the channels, so the shape tells
* it from torch's [in, out·kernel]. */
override transformStateDict(sd: LazyStateDict, prefix: string): void {
const name = `${prefix}.conv_t1.weight`;
const t = sd.tensors.get(name);
const { inChannels: cIn, outChannels: out, kernelSize: k } = this.conv_t1;
if (t?.kind !== 'quant' || t.shape.join() !== `${out},${cIn * k}`) return;
sd.tensors.set(
name,
derivedTensor(
[cIn, out, k],
async () => {
const src = await weightValues(t);
const dst = new Float32Array(src.length);
for (let o = 0; o < out; o++) {
for (let c = 0; c < cIn; c++) {
for (let kk = 0; kk < k; kk++) {
dst[(c * out + o) * k + kk] = src[(o * cIn + c) * k + k - 1 - kk]!;
}
}
}
return dst;
},
t.byteLength,
),
);
}
override forward(x: Value): Value {
const h = this.conv_t1.forward(this.snake1.forward(x));
return this.res_unit3.forward(this.res_unit2.forward(this.res_unit1.forward(h)));
Expand Down Expand Up @@ -101,9 +131,19 @@ interface Codebook {
bias: LazyTensor;
}

/** A weight's values: as stored, or a quantized weight's ints times the
* scale of their group. */
async function weightValues(t: LazyTensor): Promise<Float32Array> {
if (t.kind !== 'quant') return t.f32();
const [ints, stored] = await Promise.all([t.ints(), t.scales()]);
const scales = stored instanceof Uint16Array ? f16ToF32(stored) : stored;
return Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / t.groupSize)]!);
}

/** The decoder input for code i of codebook k is embed_k[i]·project_out_kᵀ·
* fc2ᵀ plus a bias shared by every frame: fc2's bias and each project_out
* bias through fc2. Both come out as derived tensors, computed in f64. */
* bias through fc2. Both come out as derived tensors, computed in f64. A
* quantized project_out or fc2 weight folds as its dequantized values. */
function foldCodebooks(
books: Codebook[],
fc2W: LazyTensor,
Expand All @@ -128,7 +168,7 @@ function foldCodebooks(
// tensors share it.
let transposed: Promise<Float64Array> | undefined;
const fc2T = () =>
(transposed ??= fc2W.f32().then((w) => {
(transposed ??= weightValues(fc2W).then((w) => {
const t = new Float64Array(hidden * latent);
for (let o = 0; o < latent; o++) {
for (let j = 0; j < hidden; j++) t[j * latent + o] = w[o * hidden + j]!;
Expand All @@ -144,7 +184,7 @@ function foldCodebooks(
const m = new Float64Array(dim * latent);
const row = new Float64Array(latent);
for (const [k, b] of books.entries()) {
const w = await b.weight.f32();
const w = await weightValues(b.weight);
const e = await b.embed.f32();
// m = project_outᵀ·fc2ᵀ, [dim, latent].
m.fill(0);
Expand Down
63 changes: 49 additions & 14 deletions packages/runntime/test/core/weights/checkpoint.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,9 @@ export async function contents(t: LazyTensor): Promise<{ type: 'f16' | 'f32'; va
* `[out, in / groupSize]`. */
export type QuantEntry = { ints: number[]; scales: number[]; shape: [number, number] };

/** A QuantEntry with its own bits, for a mixedQuantFile. */
export type BitsQuantEntry = QuantEntry & { bits: 4 | 8 };

/** `values`, a torch-layout `[out, in]` weight, quantized the way the format
* spec's symmetric scheme reads it: per group of `groupSize` along `in`, the
* scale is the group's largest |value| over the top int (127 for int8, 7 for
Expand All @@ -117,18 +120,19 @@ export function quantizeWeight(
return { ints, scales, shape };
}

/** A quantized safetensors file in the format spec: each entry's ints as
* `name` (I8, or U8 with two int4 per byte, low nibble first), its scales
* rounded to f16 as `name_scale`, and the setting in `runntime.quant`.
* `floats` go in as F32 tensors next to them. */
export function quantFile(
bits: 4 | 8,
groupSize: number,
entries: Record<string, QuantEntry>,
floats: Record<string, { data: Float32Array; shape: number[] }> = {},
): Uint8Array {
const tensors: Parameters<typeof safetensorsFile>[0] = { ...floats };
for (const [name, { ints, scales, shape }] of Object.entries(entries)) {
/** The values `entry` stands for: each int times the scale of its group. */
export function dequantizeWeight({ ints, scales, shape }: QuantEntry): Float32Array {
const groupSize = (shape[0] * shape[1]) / scales.length;
return Float32Array.from(ints, (q, i) => q * scales[Math.floor(i / groupSize)]!);
}

/** Each entry's ints as `name` (I8, or U8 with two int4 per byte, low nibble
* first) and its scales rounded to f16 as `name_scale`. */
function quantTensors(
entries: Record<string, BitsQuantEntry>,
): Parameters<typeof safetensorsFile>[0] {
const tensors: Parameters<typeof safetensorsFile>[0] = {};
for (const [name, { bits, ints, scales, shape }] of Object.entries(entries)) {
const [out, cols] = shape;
tensors[name] =
bits === 8
Expand All @@ -141,11 +145,42 @@ export function quantFile(
};
tensors[`${name}_scale`] = {
data: Uint16Array.from(scales, f32ToF16Bits),
shape: [out, cols / groupSize],
shape: [out, scales.length / out],
};
}
return tensors;
}

/** A quantized safetensors file in the format spec: the entries' ints and
* scales, and the setting in `runntime.quant`. `floats` go in as F32 tensors
* next to them. */
export function quantFile(
bits: 4 | 8,
groupSize: number,
entries: Record<string, QuantEntry>,
floats: Record<string, { data: Float32Array; shape: number[] }> = {},
): Uint8Array {
const quant = Object.fromEntries(
Object.entries(entries).map(([name, entry]) => [name, { ...entry, bits }]),
);
const setting = { version: 1, bits, groupSize, scheme: 'symmetric' };
return safetensorsFile(tensors, { 'runntime.quant': JSON.stringify(setting) });
return safetensorsFile(
{ ...floats, ...quantTensors(quant) },
{ 'runntime.quant': JSON.stringify(setting) },
);
}

/** quantFile for entries that differ in bits and group size: the setting
* sets neither, so each tensor gives its own. */
export function mixedQuantFile(
entries: Record<string, BitsQuantEntry>,
floats: Record<string, { data: Float32Array; shape: number[] }> = {},
): Uint8Array {
const setting = { version: 1, scheme: 'symmetric' };
return safetensorsFile(
{ ...floats, ...quantTensors(entries) },
{ 'runntime.quant': JSON.stringify(setting) },
);
Comment thread
xxpopielxx marked this conversation as resolved.
Resolved
}

/** The quantized tensor `name` of `entries`, read back from a quantFile. */
Expand Down
52 changes: 52 additions & 0 deletions packages/runntime/test/zoo/higgs/decoder.gpu.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/** The codec decoder loaded from a quantized checkpoint matches the same
* decoder loaded with the dequantized floats in torch's layout. */
import { describe, it, onTestFinished } from 'vitest';
import { toArray } from '../../../src/core/gpu/eval.ts';
import { matrix } from '../../../src/core/graph/value.ts';
import {
bufferSource,
fromSafetensors,
type LazyStateDict,
} from '../../../src/core/weights/safetensors.ts';
import { HiggsDecoder } from '../../../src/zoo/higgs/decoder.ts';
import { expectClose, gpu, requireFeature, upload } from '../../gpu.ts';
import { checkpoint, QUANT_CFG, quantize, type Sd } from './decoderCheckpoint.ts';

const { root, ex } = await gpu();

/** Weights in ±0.3 keep every layer's output below 1, so f16 rounding stays
* small. */
const SPREAD = 0.3;

describe('HiggsDecoder from a quantized checkpoint', () => {
it.for(
(['f32', 'f16'] as const).flatMap((dtype) =>
([8, 4] as const).map((bits) => ({ dtype, bits })),
),
)('$dtype int$bits matches the decoder loaded with the dequantized floats', async (c, ctx) => {
if (c.dtype === 'f16') requireFeature(ctx, root, 'shader-f16');
const { file, dequantized } = quantize(checkpoint(QUANT_CFG, SPREAD).sd, c.bits);
const run = async (sd: LazyStateDict | Sd) => {
const dec = new HiggsDecoder(QUANT_CFG);
if (c.dtype === 'f16') dec.half(root);
onTestFinished(() => dec.dispose());
await dec.loadStateDict(sd, { root });
const frames = 16;
const codes = Int32Array.from(
{ length: QUANT_CFG.codebooks * frames },
(_, i) => (i * 3) % QUANT_CFG.codebookSize,
);
const x = upload(root, dec.latents(codes, frames), {
...matrix(frames, QUANT_CFG.latent),
dtype: c.dtype,
});
return toArray(dec.forward(x), ex);
};
const got = await run(await fromSafetensors(bufferSource(file), { chunkBytes: 0 }));
const want = await run(dequantized);
// Both sum in f32 in another order; f16 rounds every layer's output.
const peak = want.reduce((m, v) => Math.max(m, Math.abs(v)), 0);
const tol = c.dtype === 'f16' ? 2e-3 : 1e-5;
expectClose(got, want, { atol: tol * peak, rtol: tol });
});
});
139 changes: 45 additions & 94 deletions packages/runntime/test/zoo/higgs/decoder.test.ts
Original file line number Diff line number Diff line change
@@ -1,103 +1,19 @@
import { describe, expect, it } from 'vitest';
import { pending, type ValueMeta } from '../../../src/core/graph/value.ts';
import { memoryStateDict } from '../../../src/core/weights/safetensors.ts';
import {
bufferSource,
fromSafetensors,
memoryStateDict,
type LazyStateDict,
} from '../../../src/core/weights/safetensors.ts';
import type { HiggsDecoderConfig } from '../../../src/zoo/higgs/config.ts';
import { HiggsDecoder } from '../../../src/zoo/higgs/decoder.ts';
import { expectClose, randomF32 } from '../../gpu.ts';

// The real decoder's layout, small.
const CFG: HiggsDecoderConfig = {
codebooks: 3,
codebookSize: 5,
latent: 8,
channels: 8,
strides: [2],
kernel: 3,
};
/** The quantizer's codebook width and its project_out width, which only the
* load reads. */
const DIM = 4;
const HIDDEN = 6;

type Sd = Record<string, { data: Float32Array; shape: number[] }>;

/** Random weights under the codec checkpoint's names, and the decoder input
* they give for codes [codebooks, frames]: each codebook's row through its
* project_out, summed, then fc2, in f64. */
function checkpoint() {
const sd: Sd = {};
let seed = 1;
const param = (name: string, shape: number[]) => {
const data = randomF32(
shape.reduce((a, b) => a * b, 1),
seed++,
);
sd[name] = { data, shape };
return data;
};
const { codebooks, codebookSize, latent, channels, kernel: k } = CFG;
const books = Array.from({ length: codebooks }, (_, i) => {
const q = `quantizer.quantizers.${i}`;
param(`${q}.codebook.embed_avg`, [codebookSize, DIM]);
param(`${q}.project_in.weight`, [DIM, HIDDEN]);
return {
embed: param(`${q}.codebook.embed`, [codebookSize, DIM]),
w: param(`${q}.project_out.weight`, [HIDDEN, DIM]),
b: param(`${q}.project_out.bias`, [HIDDEN]),
};
});
const fc2 = { w: param('fc2.weight', [latent, HIDDEN]), b: param('fc2.bias', [latent]) };
param('fc1.weight', [3, HIDDEN]);
param('acoustic_encoder.conv1.weight', [4, 1, 7]);
param('semantic_model.feature_projection.weight', [3, 3]);

const D = 'acoustic_decoder';
const conv = (name: string, cIn: number, cOut: number, kk: number) => {
param(`${name}.weight`, [cOut, cIn, kk]);
param(`${name}.bias`, [cOut]);
};
const snake = (name: string, c: number) => param(`${name}.alpha`, [1, c, 1]);
conv(`${D}.conv1`, latent, channels, k);
const out = channels / 2;
snake(`${D}.block.0.snake1`, channels);
param(`${D}.block.0.conv_t1.weight`, [channels, out, 4]);
param(`${D}.block.0.conv_t1.bias`, [out]);
for (const j of [1, 2, 3]) {
const R = `${D}.block.0.res_unit${j}`;
snake(`${R}.snake1`, out);
conv(`${R}.conv1`, out, out, k);
snake(`${R}.snake2`, out);
conv(`${R}.conv2`, out, out, 1);
}
snake(`${D}.snake1`, out);
conv(`${D}.conv2`, out, 1, k);

const latents = (codes: Int32Array, frames: number): Float64Array => {
const rows = new Float64Array(frames * latent);
for (let t = 0; t < frames; t++) {
const h = new Float64Array(HIDDEN);
books.forEach((book, c) => {
const code = codes[c * frames + t]!;
for (let j = 0; j < HIDDEN; j++) {
let v = book.b[j]!;
for (let a = 0; a < DIM; a++) v += book.embed[code * DIM + a]! * book.w[j * DIM + a]!;
h[j]! += v;
}
});
for (let o = 0; o < latent; o++) {
let v = fc2.b[o]!;
for (let j = 0; j < HIDDEN; j++) v += fc2.w[o * HIDDEN + j]! * h[j]!;
rows[t * latent + o] = v;
}
}
return rows;
};
return { sd, latents };
}
import { expectClose } from '../../gpu.ts';
import { CFG, checkpoint, DIM, QUANT_CFG, quantize, type Sd } from './decoderCheckpoint.ts';

/** Loads with stand-ins for the GPU uploads; the CPU table is real. */
async function load(sd: Sd): Promise<HiggsDecoder> {
const dec = new HiggsDecoder(CFG);
async function load(sd: LazyStateDict | Sd, cfg: HiggsDecoderConfig = CFG): Promise<HiggsDecoder> {
const dec = new HiggsDecoder(cfg);
await dec.loadStateDict(sd, {
upload: (_data, shape: ValueMeta) => pending(shape, 'reshape', []),
});
Expand All @@ -116,6 +32,41 @@ describe('HiggsDecoder', () => {
expectClose(dec.latents(codes, frames), latents(codes, frames), { atol: 1e-6, rtol: 1e-5 });
});

it.for([8, 4] as const)(
'folds an int%i project_out and fc2 as their dequantized values, with every conv quantized',
async (bits) => {
const { file, dequantized } = quantize(checkpoint(QUANT_CFG).sd, bits);
const sd = await fromSafetensors(bufferSource(file), { chunkBytes: 0 });
const quant = await load(sd, QUANT_CFG);
const float = await load(dequantized, QUANT_CFG);
const frames = 4;
const codes = Int32Array.from(
{ length: QUANT_CFG.codebooks * frames },
(_, i) => (i * 3) % QUANT_CFG.codebookSize,
);
// Both fold the same values in f64.
expect(quant.latents(codes, frames)).toEqual(float.latents(codes, frames));
const scales = [...quant.namedParameters()]
.map(([name]) => name)
.filter((name) => name.endsWith('weightScale'));
const convs = Object.keys(dequantized)
.filter((name) => name.startsWith('acoustic_decoder.') && name.endsWith('.weight'))
.map((name) => name.replace(/weight$/, 'weightScale'));
expect(scales.sort()).toEqual(convs.sort());
},
);

it('loads a conv_t1 stored as its equivalent conv as its dequantized floats', async () => {
const { file, dequantized } = quantize(checkpoint(QUANT_CFG).sd, 8, { equivalentConvT: true });
const quantSd = await fromSafetensors(bufferSource(file), { chunkBytes: 0 });
const floatSd = memoryStateDict(dequantized);
const dec = await load(quantSd, QUANT_CFG);
await load(floatSd, QUANT_CFG);
const name = 'acoustic_decoder.block.0.conv_t1.weight';
expect(dec.acoustic_decoder.block.items[0]!.conv_t1.weightScale).toBeUndefined();
expect(await quantSd.tensors.get(name)!.f32()).toEqual(await floatSd.tensors.get(name)!.f32());
});

it('keeps only the decoder of the codec checkpoint', () => {
const sd = memoryStateDict(checkpoint().sd);
new HiggsDecoder(CFG).transformStateDict(sd, '');
Expand Down
Loading
Loading