From 6a53fc1e6f5faa9f0e6ee30047603a871c85495e Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 27 Oct 2026 18:13:06 +0100 Subject: [PATCH 01/25] perf(core): keep F16 checkpoints in f16 through the Linear weight transpose --- packages/runntime/src/core/index.ts | 1 - packages/runntime/src/core/nn/layers.ts | 11 +- packages/runntime/src/core/weights/convert.ts | 12 -- packages/runntime/test/core/nn/layers.test.ts | 135 +++++++++++++++++- .../test/core/weights/convert.test.ts | 18 --- 5 files changed, 136 insertions(+), 41 deletions(-) diff --git a/packages/runntime/src/core/index.ts b/packages/runntime/src/core/index.ts index 12f0f3c9..e60b504a 100644 --- a/packages/runntime/src/core/index.ts +++ b/packages/runntime/src/core/index.ts @@ -10,7 +10,6 @@ export { packConvHwc4F16, packDwHwc4F16, permuteConvTransposeF16, - transposeF32, } from './weights/convert.ts'; export { createStorageFor, diff --git a/packages/runntime/src/core/nn/layers.ts b/packages/runntime/src/core/nn/layers.ts index 4573d3db..3bf145f7 100644 --- a/packages/runntime/src/core/nn/layers.ts +++ b/packages/runntime/src/core/nn/layers.ts @@ -26,10 +26,10 @@ import { packConvWinogradF16, packDwHwc4F16, permuteConvTransposeF16, - transposeF32, } from '../weights/convert.ts'; import { foldBnIntoConv } from '../weights/foldBn.ts'; import { derivedTensor, type LazyStateDict } from '../weights/safetensors.ts'; +import { transposeWeight } from '../weights/transforms.ts'; import { tensor } from '../gpu/buffers.ts'; import { CpuParameter, Module, Parameter, type AnyModule } from './module.ts'; @@ -135,14 +135,7 @@ export class Linear extends Module { `expected torch layout [${this.outFeatures}, ${this.inFeatures}]`, ); } - sd.tensors.set( - name, - derivedTensor( - [this.inFeatures, this.outFeatures], - async () => transposeF32(await tensor.f32(), this.outFeatures, this.inFeatures), - tensor.byteLength, - ), - ); + sd.tensors.set(name, transposeWeight(tensor)); } forward(x: Value, call: { addend?: Value } = {}): Value { return linear(x, this.weight.value, this.bias?.value, call.addend, this.activation); diff --git a/packages/runntime/src/core/weights/convert.ts b/packages/runntime/src/core/weights/convert.ts index 5c5d6106..1293458c 100644 --- a/packages/runntime/src/core/weights/convert.ts +++ b/packages/runntime/src/core/weights/convert.ts @@ -1,15 +1,3 @@ -/** Checkpoints store dense weights [out, in]; nn.Linear wants [in, out]. */ -export function transposeF32(src: Float32Array, rows: number, cols: number): Float32Array { - if (src.length !== rows * cols) { - throw new Error(`transposeF32: length ${src.length} != ${rows}×${cols}`); - } - const out = new Float32Array(src.length); - for (let r = 0; r < rows; r++) { - for (let c = 0; c < cols; c++) out[c * rows + r] = src[r * cols + c]!; - } - return out; -} - export function convToTapMajor( src: Float32Array, cOut: number, diff --git a/packages/runntime/test/core/nn/layers.test.ts b/packages/runntime/test/core/nn/layers.test.ts index 483c1441..458e52b9 100644 --- a/packages/runntime/test/core/nn/layers.test.ts +++ b/packages/runntime/test/core/nn/layers.test.ts @@ -14,8 +14,15 @@ import { RMSNorm, Snake, } from '../../../src/core/nn/layers.ts'; +import { loadLazyStateDict } from '../../../src/core/nn/loadStateDict.ts'; import { CpuParameter, Module, type AnyModule } from '../../../src/core/nn/module.ts'; -import { memoryStateDict } from '../../../src/core/weights/safetensors.ts'; +import { f16ToF32, f32ArrayToF16Bits } from '../../../src/core/weights/convert.ts'; +import { + bufferSource, + fromSafetensors, + memoryStateDict, + type LazyStateDict, +} from '../../../src/core/weights/safetensors.ts'; import { hwc4Input, input, meta } from '../graph/values.ts'; /** Binds every parameter to a stand-in of its declared shape, so forward() @@ -29,6 +36,53 @@ function fakeLoad(m: M): M { const shapes = (m: AnyModule) => Object.fromEntries([...m.namedParameters()].map(([n, p]) => [n, p.shape])); +/** A state dict with one F16 tensor named `weight`, read from a real + * safetensors file built from the format spec: an 8-byte little-endian + * header length, the JSON header, then the payload. memoryStateDict reads a + * Uint16Array as BF16, so F16 needs the file. */ +function f16Checkpoint(bits: Uint16Array, shape: number[]): Promise { + const header = new TextEncoder().encode( + JSON.stringify({ weight: { dtype: 'F16', shape, data_offsets: [0, bits.byteLength] } }), + ); + const file = new Uint8Array(8 + header.length + bits.byteLength); + new DataView(file.buffer).setBigUint64(0, BigInt(header.length), true); + file.set(header, 8); + file.set(new Uint8Array(bits.buffer, bits.byteOffset, bits.byteLength), 8 + header.length); + return fromSafetensors(bufferSource(file), { chunkBytes: 0 }); +} + +/** bf16 bits: the top half of the f32 pattern. */ +const bf16 = (values: number[]) => + Uint16Array.from(values, (v) => new Uint32Array(new Float32Array([v]).buffer)[0]! >>> 16); + +/** Loads `sd` into a layer with only a weight and returns what reached its + * upload. */ +async function uploadedWeight(layer: AnyModule, sd: LazyStateDict) { + const upload = vi.fn((_data: Float32Array | Uint32Array | Uint16Array, shape: ValueMeta) => + pending(shape, 'reshape', []), + ); + await loadLazyStateDict(layer, sd, { upload }); + return upload.mock.calls[0]![0]; +} + +/** The array type and raw bit patterns, so -0, subnormals and infinities + * compare exactly. */ +function bitsOf(data: Float32Array | Uint32Array | Uint16Array) { + if (data instanceof Float32Array) return { type: 'f32', bits: [...new Uint32Array(data.buffer)] }; + if (data instanceof Uint16Array) return { type: 'f16', bits: [...data] }; + return { type: 'u32', bits: [...data] }; +} + +/** f16 bits a widening to f32 could disturb: 1, −0, the smallest and largest + * subnormals, the largest finite value, −inf. */ +const EDGE_BITS = [0x3c00, 0x8000, 0x0001, 0x03ff, 0x7bff, 0xfc00]; + +/** Every f16 bit pattern but NaN. f16 → f32 → f16 maps each of these back to + * itself; NaN payloads collapse to one quiet NaN on that trip. */ +const ALL_F16 = Uint16Array.from({ length: 1 << 16 }, (_, i) => i).filter( + (b) => (b & 0x7c00) !== 0x7c00 || (b & 0x3ff) === 0, +); + describe('Linear', () => { it('stores the weight as [in, out] and the bias as [out]', () => { expect(shapes(new Linear(3, 4))).toEqual({ weight: meta([3, 4]), bias: meta([4]) }); @@ -71,6 +125,85 @@ describe('Linear', () => { new Linear(3, 2).transformStateDict(sd, ''); expect(sd.tensors.get('weight')!.shape).toEqual([3, 2]); }); + + // torch [2, 3] rows [a b c], [d e f] load as [3, 2]: a d, b e, c f. + const [a, b, c, d, e, f] = EDGE_BITS as [number, number, number, number, number, number]; + it.each([ + { + name: 'F16 into an f16 layer uploads the halfwords transposed', + checkpoint: () => f16Checkpoint(new Uint16Array(EDGE_BITS), [2, 3]), + half: true, + want: { type: 'f16', bits: [a, d, b, e, c, f] }, + }, + { + name: 'F16 into an f32 layer decodes the transposed halfwords', + checkpoint: () => f16Checkpoint(new Uint16Array(EDGE_BITS), [2, 3]), + half: false, + // a d b e c f = 1, (1023/1024)·2^-14, -0, 65504, 2^-24, -inf + want: bitsOf(new Float32Array([1, (1023 / 1024) * 2 ** -14, -0, 65504, 2 ** -24, -Infinity])), + }, + { + name: 'F32 into an f32 layer uploads f32 transposed', + checkpoint: async () => + memoryStateDict({ weight: { data: new Float32Array([1, 2, 3, 4, 5, 6]), shape: [2, 3] } }), + half: false, + want: bitsOf(new Float32Array([1, 4, 2, 5, 3, 6])), + }, + { + name: 'F32 into an f16 layer narrows the transposed f32', + checkpoint: async () => + memoryStateDict({ weight: { data: new Float32Array([1, 2, 3, 4, 5, 6]), shape: [2, 3] } }), + half: true, + // f16 bits of 1, 4, 2, 5, 3, 6 + want: { type: 'f16', bits: [0x3c00, 0x4400, 0x4000, 0x4500, 0x4200, 0x4600] }, + }, + { + name: 'BF16 into an f32 layer decodes, then transposes', + checkpoint: async () => + memoryStateDict({ weight: { data: bf16([1, 2, 3, 4, 5, 6]), shape: [2, 3] } }), + half: false, + want: bitsOf(new Float32Array([1, 4, 2, 5, 3, 6])), + }, + ])('$name', async ({ checkpoint, half, want }) => { + const layer = new Linear(3, 2, { bias: false }); + if (half) layer.half(); + expect(bitsOf(await uploadedWeight(layer, await checkpoint()))).toEqual(want); + }); + + it('transposes F16 halfwords without decoding them to f32', async () => { + const sd = await f16Checkpoint(new Uint16Array(EDGE_BITS), [2, 3]); + const f32 = vi.spyOn(sd.tensors.get('weight')!, 'f32'); + new Linear(3, 2, { bias: false }).transformStateDict(sd, ''); + const w = sd.tensors.get('weight')!; + if (w.kind !== 'derived') throw new Error('expected a derived tensor'); + expect(await w.data()).toBeInstanceOf(Uint16Array); + expect(w.byteLength).toBe(12); + expect(f32).not.toHaveBeenCalled(); + }); + + // The reference is the f32 round trip: decode, transpose in f32, then + // narrow for an f16 layer. + it.each([ + { name: 'an f16 layer', half: true }, + { name: 'an f32 layer', half: false }, + ])( + 'loads every F16 value into $name with the same bits as the f32 round trip', + async ({ half }) => { + // 63490 values as 70 × 907. + const rows = 70; + const cols = ALL_F16.length / rows; + const values = f16ToF32(ALL_F16); + const transposed = new Float32Array(values.length); + for (let r = 0; r < rows; r++) { + for (let col = 0; col < cols; col++) transposed[col * rows + r] = values[r * cols + col]!; + } + const want = half ? f32ArrayToF16Bits(transposed).data : transposed; + const layer = new Linear(cols, rows, { bias: false }); + if (half) layer.half(); + const got = await uploadedWeight(layer, await f16Checkpoint(ALL_F16, [rows, cols])); + expect(bitsOf(got)).toEqual(bitsOf(want)); + }, + ); }); describe('Conv1d', () => { diff --git a/packages/runntime/test/core/weights/convert.test.ts b/packages/runntime/test/core/weights/convert.test.ts index 6ad5042f..92dcfc1a 100644 --- a/packages/runntime/test/core/weights/convert.test.ts +++ b/packages/runntime/test/core/weights/convert.test.ts @@ -12,7 +12,6 @@ import { packConvWinogradF16, packDwHwc4F16, permuteConvTransposeF16, - transposeF32, } from '../../../src/core/weights/convert.ts'; // f16 is 1 sign bit, 5 exponent bits, 10 mantissa bits. Every value here is @@ -185,23 +184,6 @@ describe('array conversion edge cases', () => { // Weight values here are just labels (1, 2, 3...) so you can follow where each // one lands. The layout comments use [outer, ..., inner] order. -describe('transposeF32', () => { - it.each([ - { name: '2x3', rows: 2, cols: 3, src: [1, 2, 3, 4, 5, 6], want: [1, 4, 2, 5, 3, 6] }, - { name: '1x1', rows: 1, cols: 1, src: [7], want: [7] }, - { name: 'a single row', rows: 1, cols: 3, src: [1, 2, 3], want: [1, 2, 3] }, - { name: 'a single column', rows: 3, cols: 1, src: [1, 2, 3], want: [1, 2, 3] }, - { name: 'empty', rows: 0, cols: 3, src: [], want: [] }, - ])('$name', ({ rows, cols, src, want }) => { - expect(transposeF32(new Float32Array(src), rows, cols)).toEqual(new Float32Array(want)); - }); - - it('rejects a wrong length', () => { - expect(() => transposeF32(new Float32Array(5), 2, 3)).toThrow(/length 5/); - expect(() => transposeF32(new Float32Array(1), 0, 0)).toThrow(/length 1/); - }); -}); - describe('convToTapMajor', () => { // [cOut, cIn, k] -> [k, cIn, cOut] it.each([ From b62a3591a1bbe970ec9431a741affe5e9a6f20c3 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 27 Oct 2026 18:14:35 +0100 Subject: [PATCH 02/25] perf(core): keep F16 checkpoints in f16 through the Conv1d tap-major reorder --- packages/runntime/src/core/nn/layers.ts | 9 ++- packages/runntime/src/core/weights/convert.ts | 19 ++++- .../runntime/src/core/weights/transforms.ts | 4 +- packages/runntime/test/core/nn/layers.test.ts | 78 +++++++++++++++++++ .../test/core/weights/convert.test.ts | 8 ++ 5 files changed, 113 insertions(+), 5 deletions(-) diff --git a/packages/runntime/src/core/nn/layers.ts b/packages/runntime/src/core/nn/layers.ts index 3bf145f7..56ddb956 100644 --- a/packages/runntime/src/core/nn/layers.ts +++ b/packages/runntime/src/core/nn/layers.ts @@ -29,7 +29,7 @@ import { } from '../weights/convert.ts'; import { foldBnIntoConv } from '../weights/foldBn.ts'; import { derivedTensor, type LazyStateDict } from '../weights/safetensors.ts'; -import { transposeWeight } from '../weights/transforms.ts'; +import { readFloats, transposeWeight } from '../weights/transforms.ts'; import { tensor } from '../gpu/buffers.ts'; import { CpuParameter, Module, Parameter, type AnyModule } from './module.ts'; @@ -209,7 +209,12 @@ export class Conv1d extends Module { derivedTensor( [this.kernelSize * this.inChannels, this.outChannels], async () => - convToTapMajor(await tensor.f32(), this.outChannels, this.inChannels, this.kernelSize), + convToTapMajor( + await readFloats('Conv1d', tensor), + this.outChannels, + this.inChannels, + this.kernelSize, + ), tensor.byteLength, ), ); diff --git a/packages/runntime/src/core/weights/convert.ts b/packages/runntime/src/core/weights/convert.ts index 1293458c..69577770 100644 --- a/packages/runntime/src/core/weights/convert.ts +++ b/packages/runntime/src/core/weights/convert.ts @@ -1,13 +1,28 @@ +/** A torch [cOut, cIn, k] conv1d weight as tap-major [k·cIn, cOut]. A pure + * permutation, so f16 halfwords move as they are. */ export function convToTapMajor( src: Float32Array, cOut: number, cIn: number, k: number, -): Float32Array { +): Float32Array; +export function convToTapMajor( + src: Float32Array | Uint16Array, + cOut: number, + cIn: number, + k: number, +): Float32Array | Uint16Array; +export function convToTapMajor( + src: Float32Array | Uint16Array, + cOut: number, + cIn: number, + k: number, +): Float32Array | Uint16Array { if (src.length !== cOut * cIn * k) { throw new Error(`convToTapMajor: length ${src.length} != ${cOut}×${cIn}×${k}`); } - const out = new Float32Array(src.length); + const out = + src instanceof Uint16Array ? new Uint16Array(src.length) : new Float32Array(src.length); for (let cout = 0; cout < cOut; cout++) { for (let cin = 0; cin < cIn; cin++) { for (let tap = 0; tap < k; tap++) { diff --git a/packages/runntime/src/core/weights/transforms.ts b/packages/runntime/src/core/weights/transforms.ts index 0dde58f8..92283643 100644 --- a/packages/runntime/src/core/weights/transforms.ts +++ b/packages/runntime/src/core/weights/transforms.ts @@ -18,7 +18,9 @@ const numel = (shape: readonly number[]): number => shape.reduce((a, b) => a * b const mismatch = (helper: string, why: string) => new RunntimeError('CHECKPOINT_MISMATCH', `${helper}: ${why}`); -async function readFloats(helper: string, t: LazyTensor): Promise { +/** Reads `t` as f16 halfwords when it holds F16 and as f32 otherwise. Throws + * CHECKPOINT_MISMATCH, naming `helper`, on packed u32 words. */ +export async function readFloats(helper: string, t: LazyTensor): Promise { if (t.kind === 'file') return t.dtype === 'F16' ? t.u16() : t.f32(); const data = await t.data(); if (data instanceof Uint32Array) throw mismatch(helper, 'source holds packed u32 words'); diff --git a/packages/runntime/test/core/nn/layers.test.ts b/packages/runntime/test/core/nn/layers.test.ts index 458e52b9..239d5c17 100644 --- a/packages/runntime/test/core/nn/layers.test.ts +++ b/packages/runntime/test/core/nn/layers.test.ts @@ -237,6 +237,84 @@ describe('Conv1d', () => { ); }); + // torch [out 2, in 3, k 2] holding elements 0..11 loads tap-major as + // 0 6 2 8 4 10 1 7 3 9 5 11 (row tap·in + i, column out), as in the f32 + // case above. + const TAP_MAJOR = [0, 6, 2, 8, 4, 10, 1, 7, 3, 9, 5, 11]; + // EDGE_BITS, then each with its sign flipped. + const halves = [...EDGE_BITS, ...EDGE_BITS.map((b) => b ^ 0x8000)]; + const reordered = new Uint16Array(TAP_MAJOR.map((i) => halves[i]!)); + const f32Values = Array.from({ length: 12 }, (_, i) => i + 1); + it.each([ + { + name: 'F16 into an f16 layer uploads the halfwords reordered', + checkpoint: () => f16Checkpoint(new Uint16Array(halves), [2, 3, 2]), + half: true, + want: bitsOf(reordered), + }, + { + name: 'F16 into an f32 layer decodes the reordered halfwords', + checkpoint: () => f16Checkpoint(new Uint16Array(halves), [2, 3, 2]), + half: false, + want: bitsOf(f16ToF32(reordered)), + }, + { + name: 'F32 into an f32 layer uploads f32 reordered', + checkpoint: async () => + memoryStateDict({ weight: { data: new Float32Array(f32Values), shape: [2, 3, 2] } }), + half: false, + want: bitsOf(new Float32Array(TAP_MAJOR.map((i) => i + 1))), + }, + { + name: 'BF16 into an f32 layer decodes, then reorders', + checkpoint: async () => + memoryStateDict({ weight: { data: bf16(f32Values), shape: [2, 3, 2] } }), + half: false, + want: bitsOf(new Float32Array(TAP_MAJOR.map((i) => i + 1))), + }, + ])('$name', async ({ checkpoint, half, want }) => { + const layer = new Conv1d(3, 2, 2, { bias: false }); + if (half) layer.half(); + expect(bitsOf(await uploadedWeight(layer, await checkpoint()))).toEqual(want); + }); + + it('reorders F16 halfwords without decoding them to f32', async () => { + const sd = await f16Checkpoint(new Uint16Array(halves), [2, 3, 2]); + const f32 = vi.spyOn(sd.tensors.get('weight')!, 'f32'); + new Conv1d(3, 2, 2, { bias: false }).transformStateDict(sd, ''); + const w = sd.tensors.get('weight')!; + if (w.kind !== 'derived') throw new Error('expected a derived tensor'); + expect(await w.data()).toBeInstanceOf(Uint16Array); + expect(w.byteLength).toBe(24); + expect(f32).not.toHaveBeenCalled(); + }); + + // The reference is the f32 round trip: decode, reorder in f32, then narrow + // for an f16 layer. + it.each([ + { name: 'an f16 layer', half: true }, + { name: 'an f32 layer', half: false }, + ])( + 'loads every F16 value into $name with the same bits as the f32 round trip', + async ({ half }) => { + // 63490 values as out 10, in 907, k 7. + const [cOut, cIn, k] = [10, 907, 7]; + const values = f16ToF32(ALL_F16); + const tapMajor = new Float32Array(values.length); + for (let o = 0; o < cOut; o++) { + for (let i = 0; i < cIn; i++) { + for (let t = 0; t < k; t++) + tapMajor[(t * cIn + i) * cOut + o] = values[(o * cIn + i) * k + t]!; + } + } + const want = half ? f32ArrayToF16Bits(tapMajor).data : tapMajor; + const layer = new Conv1d(cIn, cOut, k, { bias: false }); + if (half) layer.half(); + const got = await uploadedWeight(layer, await f16Checkpoint(ALL_F16, [cOut, cIn, k])); + expect(bitsOf(got)).toEqual(bitsOf(want)); + }, + ); + it('widens the window by the dilation', () => { const conv = (padding?: 'same') => fakeLoad(new Conv1d(4, 6, 3, { dilation: 2, padding })).forward(input([10, 4])).shape; diff --git a/packages/runntime/test/core/weights/convert.test.ts b/packages/runntime/test/core/weights/convert.test.ts index 92dcfc1a..e33f4e05 100644 --- a/packages/runntime/test/core/weights/convert.test.ts +++ b/packages/runntime/test/core/weights/convert.test.ts @@ -202,6 +202,14 @@ describe('convToTapMajor', () => { expect(convToTapMajor(new Float32Array(src), cOut, cIn, k)).toEqual(new Float32Array(want)); }); + it('moves f16 halfwords as they are', () => { + // 2 out, 1 in, 2 taps, as above: 1 2 3 4 -> 1 3 2 4. + const [one, two, three, four] = [0x3c00, 0x4000, 0x4200, 0x4400]; + expect(convToTapMajor(new Uint16Array([one, two, three, four]), 2, 1, 2)).toEqual( + new Uint16Array([one, three, two, four]), + ); + }); + it('rejects a wrong length', () => { expect(() => convToTapMajor(new Float32Array(3), 2, 1, 2)).toThrow(/length 3/); }); From cc68ce1e8a0b0de4b4e4b9f50173562c7f8e74e7 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 28 Oct 2026 11:42:06 +0100 Subject: [PATCH 03/25] test(core): declare EDGE_BITS as const instead of asserting its tuple type --- packages/runntime/test/core/nn/layers.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/runntime/test/core/nn/layers.test.ts b/packages/runntime/test/core/nn/layers.test.ts index 239d5c17..7c21cec2 100644 --- a/packages/runntime/test/core/nn/layers.test.ts +++ b/packages/runntime/test/core/nn/layers.test.ts @@ -75,7 +75,7 @@ function bitsOf(data: Float32Array | Uint32Array | Uint16Array) { /** f16 bits a widening to f32 could disturb: 1, −0, the smallest and largest * subnormals, the largest finite value, −inf. */ -const EDGE_BITS = [0x3c00, 0x8000, 0x0001, 0x03ff, 0x7bff, 0xfc00]; +const EDGE_BITS = [0x3c00, 0x8000, 0x0001, 0x03ff, 0x7bff, 0xfc00] as const; /** Every f16 bit pattern but NaN. f16 → f32 → f16 maps each of these back to * itself; NaN payloads collapse to one quiet NaN on that trip. */ @@ -127,7 +127,7 @@ describe('Linear', () => { }); // torch [2, 3] rows [a b c], [d e f] load as [3, 2]: a d, b e, c f. - const [a, b, c, d, e, f] = EDGE_BITS as [number, number, number, number, number, number]; + const [a, b, c, d, e, f] = EDGE_BITS; it.each([ { name: 'F16 into an f16 layer uploads the halfwords transposed', From 0c1f53fa759b62956c94ebedc31ffa7712f33efd Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 28 Oct 2026 11:42:57 +0100 Subject: [PATCH 04/25] test(core): cover BF16 into f16 and derived halfword and packed-word weights in Linear and Conv1d --- packages/runntime/test/core/nn/layers.test.ts | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/packages/runntime/test/core/nn/layers.test.ts b/packages/runntime/test/core/nn/layers.test.ts index 7c21cec2..db8a4f11 100644 --- a/packages/runntime/test/core/nn/layers.test.ts +++ b/packages/runntime/test/core/nn/layers.test.ts @@ -19,6 +19,7 @@ import { CpuParameter, Module, type AnyModule } from '../../../src/core/nn/modul import { f16ToF32, f32ArrayToF16Bits } from '../../../src/core/weights/convert.ts'; import { bufferSource, + derivedTensor, fromSafetensors, memoryStateDict, type LazyStateDict, @@ -51,6 +52,18 @@ function f16Checkpoint(bits: Uint16Array, shape: number[]): Promise Promise.resolve(data), data.byteLength); + return { + metadata: {}, + tensors: new Map([['weight', tensor]]), + totalBytes: () => data.byteLength, + }; +} + /** bf16 bits: the top half of the f32 pattern. */ const bf16 = (values: number[]) => Uint16Array.from(values, (v) => new Uint32Array(new Float32Array([v]).buffer)[0]! >>> 16); @@ -164,12 +177,42 @@ describe('Linear', () => { half: false, want: bitsOf(new Float32Array([1, 4, 2, 5, 3, 6])), }, + { + name: 'BF16 into an f16 layer decodes, transposes, then narrows', + checkpoint: async () => + memoryStateDict({ weight: { data: bf16([1, 2, 3, 4, 5, 6]), shape: [2, 3] } }), + half: true, + // f16 bits of 1, 4, 2, 5, 3, 6 + want: { type: 'f16', bits: [0x3c00, 0x4400, 0x4000, 0x4500, 0x4200, 0x4600] }, + }, + { + name: 'derived halfwords into an f16 layer upload transposed', + checkpoint: async () => derivedCheckpoint(new Uint16Array(EDGE_BITS), [2, 3]), + half: true, + want: { type: 'f16', bits: [a, d, b, e, c, f] }, + }, + { + name: 'derived halfwords into an f32 layer decode transposed', + checkpoint: async () => derivedCheckpoint(new Uint16Array(EDGE_BITS), [2, 3]), + half: false, + want: bitsOf(new Float32Array([1, (1023 / 1024) * 2 ** -14, -0, 65504, 2 ** -24, -Infinity])), + }, ])('$name', async ({ checkpoint, half, want }) => { const layer = new Linear(3, 2, { bias: false }); if (half) layer.half(); expect(bitsOf(await uploadedWeight(layer, await checkpoint()))).toEqual(want); }); + it('rejects derived packed words', async () => { + const sd = derivedCheckpoint(new Uint32Array(6), [2, 3]); + await expect(uploadedWeight(new Linear(3, 2, { bias: false }), sd)).rejects.toThrow( + expect.objectContaining({ + code: 'CHECKPOINT_MISMATCH', + message: 'transposeWeight: source holds packed u32 words', + }), + ); + }); + it('transposes F16 halfwords without decoding them to f32', async () => { const sd = await f16Checkpoint(new Uint16Array(EDGE_BITS), [2, 3]); const f32 = vi.spyOn(sd.tensors.get('weight')!, 'f32'); @@ -272,12 +315,41 @@ describe('Conv1d', () => { half: false, want: bitsOf(new Float32Array(TAP_MAJOR.map((i) => i + 1))), }, + { + name: 'BF16 into an f16 layer decodes, reorders, then narrows', + checkpoint: async () => + memoryStateDict({ weight: { data: bf16(f32Values), shape: [2, 3, 2] } }), + half: true, + want: bitsOf(f32ArrayToF16Bits(new Float32Array(TAP_MAJOR.map((i) => i + 1))).data), + }, + { + name: 'derived halfwords into an f16 layer upload reordered', + checkpoint: async () => derivedCheckpoint(new Uint16Array(halves), [2, 3, 2]), + half: true, + want: bitsOf(reordered), + }, + { + name: 'derived halfwords into an f32 layer decode reordered', + checkpoint: async () => derivedCheckpoint(new Uint16Array(halves), [2, 3, 2]), + half: false, + want: bitsOf(f16ToF32(reordered)), + }, ])('$name', async ({ checkpoint, half, want }) => { const layer = new Conv1d(3, 2, 2, { bias: false }); if (half) layer.half(); expect(bitsOf(await uploadedWeight(layer, await checkpoint()))).toEqual(want); }); + it('rejects derived packed words', async () => { + const sd = derivedCheckpoint(new Uint32Array(12), [2, 3, 2]); + await expect(uploadedWeight(new Conv1d(3, 2, 2, { bias: false }), sd)).rejects.toThrow( + expect.objectContaining({ + code: 'CHECKPOINT_MISMATCH', + message: 'Conv1d: source holds packed u32 words', + }), + ); + }); + it('reorders F16 halfwords without decoding them to f32', async () => { const sd = await f16Checkpoint(new Uint16Array(halves), [2, 3, 2]); const f32 = vi.spyOn(sd.tensors.get('weight')!, 'f32'); From c97813b43e8d76ba6ac04a1f1b596d841a472ddd Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 28 Oct 2026 11:43:39 +0100 Subject: [PATCH 05/25] refactor(core): give convToTapMajor one signature and upload random conv1d bench taps unreordered --- packages/runntime/bench/kernels/conv.ab.ts | 15 +++++++-------- packages/runntime/src/core/weights/convert.ts | 12 ------------ 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/packages/runntime/bench/kernels/conv.ab.ts b/packages/runntime/bench/kernels/conv.ab.ts index 93a5139b..0edf311e 100644 --- a/packages/runntime/bench/kernels/conv.ab.ts +++ b/packages/runntime/bench/kernels/conv.ab.ts @@ -75,14 +75,13 @@ function conv1d( name, build: (version) => { const x = randMat(version, tIn, cIn); - const w = version.upload( - version.lib.convToTapMajor(version.rand(cOut * cIn * k), cOut, cIn, k), - { - elems: k * cIn * cOut, - dtype: 'f32', - dims: [k * cIn, cOut], - }, - ); + // Random taps need no tap-major reorder: a permutation of them is as + // random, and A and B draw the same numbers either way. + const w = version.upload(version.rand(cOut * cIn * k), { + elems: k * cIn * cOut, + dtype: 'f32', + dims: [k * cIn, cOut], + }); const call = () => version.lib.conv1d(x, w, { kernelSize: k, stride, activation: act, ...geometry }); // Built once here, so a base that refuses the geometry skips the case. diff --git a/packages/runntime/src/core/weights/convert.ts b/packages/runntime/src/core/weights/convert.ts index 69577770..c220bb54 100644 --- a/packages/runntime/src/core/weights/convert.ts +++ b/packages/runntime/src/core/weights/convert.ts @@ -1,17 +1,5 @@ /** A torch [cOut, cIn, k] conv1d weight as tap-major [k·cIn, cOut]. A pure * permutation, so f16 halfwords move as they are. */ -export function convToTapMajor( - src: Float32Array, - cOut: number, - cIn: number, - k: number, -): Float32Array; -export function convToTapMajor( - src: Float32Array | Uint16Array, - cOut: number, - cIn: number, - k: number, -): Float32Array | Uint16Array; export function convToTapMajor( src: Float32Array | Uint16Array, cOut: number, From 75b704f644c24b25db8728410677e516098aa659 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Fri, 30 Oct 2026 12:10:48 +0100 Subject: [PATCH 06/25] fix(core): reject a derived tensor whose data does not fill its shape in readFloats MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit transposeF32 checked the data length against rows × cols and threw on a mismatch. transposeWeight dropped that check, so a hook bug that derived a [2, 3] tensor holding 5 values uploaded 5 values with no error. readFloats now compares the length to the shape for every helper. File tensors are already covered by the safetensors header check. --- packages/runntime/src/core/weights/transforms.ts | 7 ++++++- .../test/core/weights/transforms.test.ts | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/runntime/src/core/weights/transforms.ts b/packages/runntime/src/core/weights/transforms.ts index 92283643..0ecff59b 100644 --- a/packages/runntime/src/core/weights/transforms.ts +++ b/packages/runntime/src/core/weights/transforms.ts @@ -19,11 +19,16 @@ const mismatch = (helper: string, why: string) => new RunntimeError('CHECKPOINT_MISMATCH', `${helper}: ${why}`); /** Reads `t` as f16 halfwords when it holds F16 and as f32 otherwise. Throws - * CHECKPOINT_MISMATCH, naming `helper`, on packed u32 words. */ + * CHECKPOINT_MISMATCH, naming `helper`, on packed u32 words or on a derived + * tensor whose data does not fill its shape. A file tensor's length is + * checked by the header parser. */ export async function readFloats(helper: string, t: LazyTensor): Promise { if (t.kind === 'file') return t.dtype === 'F16' ? t.u16() : t.f32(); const data = await t.data(); if (data instanceof Uint32Array) throw mismatch(helper, 'source holds packed u32 words'); + if (data.length !== numel(t.shape)) { + throw mismatch(helper, `source holds ${data.length} values for shape [${t.shape.join(', ')}]`); + } return data; } diff --git a/packages/runntime/test/core/weights/transforms.test.ts b/packages/runntime/test/core/weights/transforms.test.ts index 733848b6..1b1dac38 100644 --- a/packages/runntime/test/core/weights/transforms.test.ts +++ b/packages/runntime/test/core/weights/transforms.test.ts @@ -270,4 +270,20 @@ describe('weight helpers', () => { message: `${name}: source holds packed u32 words`, }); }); + + it.for([ + { name: 'concatRows', run: (t: LazyTensor) => concatRows([t]) }, + { name: 'splitRows', run: (t: LazyTensor) => splitRows(t, [2])[0]! }, + { name: 'splitCols', run: (t: LazyTensor) => splitCols(t, [3])[0]! }, + { name: 'scaleWeight', run: (t: LazyTensor) => scaleWeight(t, 2) }, + { name: 'transposeWeight', run: (t: LazyTensor) => transposeWeight(t) }, + ])('$name rejects a derived source whose data does not fill its shape', async ({ name, run }) => { + // A hook bug: shape [2, 3] says 6 values, data holds 5. A file tensor + // cannot get here, its header check rejects the length mismatch. + const short = derivedTensor([2, 3], async () => new Float32Array([1, 2, 3, 4, 5])); + await expect(contents(run(short))).rejects.toMatchObject({ + code: 'CHECKPOINT_MISMATCH', + message: `${name}: source holds 5 values for shape [2, 3]`, + }); + }); }); From 0f59f08e64b7c7aee0bb950652ebaa21ee0a54e9 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 27 Oct 2026 18:22:16 +0100 Subject: [PATCH 07/25] feat(core): add a reshapeWeight helper for state dict hooks --- packages/runntime/src/core/index.ts | 1 + .../runntime/src/core/weights/transforms.ts | 16 +++++++++- .../test/core/weights/transforms.test.ts | 32 +++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/packages/runntime/src/core/index.ts b/packages/runntime/src/core/index.ts index e60b504a..0e477ea8 100644 --- a/packages/runntime/src/core/index.ts +++ b/packages/runntime/src/core/index.ts @@ -139,6 +139,7 @@ export { createOpfsCache } from './weights/opfsCache.ts'; export { foldBnIntoConv } from './weights/foldBn.ts'; export { concatRows, + reshapeWeight, scaleWeight, splitCols, splitRows, diff --git a/packages/runntime/src/core/weights/transforms.ts b/packages/runntime/src/core/weights/transforms.ts index 0ecff59b..e7064ffb 100644 --- a/packages/runntime/src/core/weights/transforms.ts +++ b/packages/runntime/src/core/weights/transforms.ts @@ -1,5 +1,5 @@ /** Lazy reshaping helpers for `transformStateDict` hooks: concatenate, split, - * scale and transpose checkpoint tensors without fetching them. + * scale, transpose and reshape checkpoint tensors without fetching them. * * F16 sources give f16 halfwords and F32 sources give f32, except * `scaleWeight`, which always gives f32. BF16 decodes to @@ -187,3 +187,17 @@ export function transposeWeight(t: LazyTensor): LazyTensor { t.byteLength, ); } + +/** Gives `t` a new shape with the same element count, like torch `reshape`. + * The data is untouched: a file tensor stays a file tensor of its dtype, and + * byteLength stays the same. Throws CHECKPOINT_MISMATCH when the element + * counts differ. */ +export function reshapeWeight(t: LazyTensor, shape: readonly number[]): LazyTensor { + if (numel(shape) !== numel(t.shape)) { + throw mismatch( + 'reshapeWeight', + `cannot reshape [${t.shape.join(', ')}] to [${shape.join(', ')}]`, + ); + } + return { ...t, shape: [...shape] }; +} diff --git a/packages/runntime/test/core/weights/transforms.test.ts b/packages/runntime/test/core/weights/transforms.test.ts index 1b1dac38..a2f48d2d 100644 --- a/packages/runntime/test/core/weights/transforms.test.ts +++ b/packages/runntime/test/core/weights/transforms.test.ts @@ -9,6 +9,7 @@ import { } from '../../../src/core/weights/safetensors.ts'; import { concatRows, + reshapeWeight, scaleWeight, splitCols, splitRows, @@ -226,6 +227,36 @@ describe('transposeWeight', () => { }); }); +describe('reshapeWeight', () => { + // [2, 1, 3] → [2, 3]: an expert tensor with its expert axis folded into rows. + it.for(DTYPES)('reshapes $dtype, keeping the source dtype', async ({ dtype }) => { + const t = await source(dtype, [1, 2, 3, 4, 5, 6], [2, 1, 3]); + const out = reshapeWeight(t, [2, 3]); + expect(out.shape).toEqual([2, 3]); + expect(out.byteLength).toBe(t.byteLength); + expect([out.kind, out.dtype]).toEqual([t.kind, t.dtype]); + expect([...(await out.f32())]).toEqual([1, 2, 3, 4, 5, 6]); + if (dtype !== 'F32') expect(await out.u16()).toEqual(await t.u16()); + }); + + it('keeps the f16 halfwords of a derived source', async () => { + const [left] = splitCols(await source('F16', [1, 2, 3, 4, 5, 6, 7, 8], [2, 1, 4]), [2, 2]); + const out = reshapeWeight(left!, [2, 2]); + expect(out.shape).toEqual([2, 2]); + expect(await contents(out)).toEqual({ type: 'f16', values: [1, 2, 5, 6] }); + }); + + it('throws CHECKPOINT_MISMATCH when the element counts differ', async () => { + const t = await source('F32', [1, 2, 3, 4, 5, 6], [2, 3]); + expect(() => reshapeWeight(t, [4, 2])).toThrow( + expect.objectContaining({ + code: 'CHECKPOINT_MISMATCH', + message: 'reshapeWeight: cannot reshape [2, 3] to [4, 2]', + }), + ); + }); +}); + describe('weight helpers', () => { it('fetch nothing until an output is read, and split sources read once', async () => { const compute = vi.fn(async () => new Float32Array([1, 2, 3, 4])); @@ -235,6 +266,7 @@ describe('weight helpers', () => { concatRows([t, t]); scaleWeight(t, 2); transposeWeight(t); + reshapeWeight(t, [4]); expect(compute).not.toHaveBeenCalled(); await Promise.all([top!, bottom!].map(contents)); expect(compute).toHaveBeenCalledOnce(); From 9e5efc555e15b8caa0955ef0019bcbf8e02fc1b9 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 27 Oct 2026 18:24:40 +0100 Subject: [PATCH 08/25] test(core): share the synthetic F16 checkpoint builder between weight tests --- .../runntime/test/core/weights/checkpoint.ts | 56 +++++++++++++++++++ .../test/core/weights/transforms.test.ts | 27 +-------- .../test/zoo/minilm/stateDictHooks.test.ts | 55 +----------------- 3 files changed, 60 insertions(+), 78 deletions(-) create mode 100644 packages/runntime/test/core/weights/checkpoint.ts diff --git a/packages/runntime/test/core/weights/checkpoint.ts b/packages/runntime/test/core/weights/checkpoint.ts new file mode 100644 index 00000000..1d5f77d7 --- /dev/null +++ b/packages/runntime/test/core/weights/checkpoint.ts @@ -0,0 +1,56 @@ +/** Synthetic checkpoints for the weight helper and state dict hook tests. */ +import { f16ToF32, f32ToF16Bits } from '../../../src/core/weights/convert.ts'; +import { + bufferSource, + fromSafetensors, + memoryStateDict, + type LazyStateDict, + type LazyTensor, +} from '../../../src/core/weights/safetensors.ts'; + +export type Entry = { values: number[]; shape: number[] }; + +/** An F16 safetensors file, built from the format spec: an 8-byte + * little-endian header length, the JSON header, then the payload. */ +export function f16File(entries: Record): Uint8Array { + const header: Record = {}; + const halves: number[] = []; + for (const [name, { values, shape }] of Object.entries(entries)) { + const begin = halves.length * 2; + halves.push(...values.map(f32ToF16Bits)); + header[name] = { dtype: 'F16', shape, data_offsets: [begin, halves.length * 2] }; + } + const json = new TextEncoder().encode(JSON.stringify(header)); + const payload = new Uint8Array(new Uint16Array(halves).buffer); + const out = new Uint8Array(8 + json.length + payload.length); + new DataView(out.buffer).setBigUint64(0, BigInt(json.length), true); + out.set(json, 8); + out.set(payload, 8 + json.length); + return out; +} + +/** `entries` as an F16 safetensors file, or as f32 in memory. memoryStateDict + * reads a Uint16Array as BF16, so F16 has to go through a real file. */ +export async function checkpoint( + dtype: 'F32' | 'F16', + entries: Record, +): Promise { + if (dtype === 'F16') return fromSafetensors(bufferSource(f16File(entries)), { chunkBytes: 0 }); + return memoryStateDict( + Object.fromEntries( + Object.entries(entries).map(([name, { values, shape }]) => [ + name, + { data: new Float32Array(values), shape }, + ]), + ), + ); +} + +/** What a derived tensor holds: f16 halfwords or f32, decoded to numbers. */ +export async function contents(t: LazyTensor): Promise<{ type: 'f16' | 'f32'; values: number[] }> { + if (t.kind !== 'derived') throw new Error('expected a derived tensor'); + const data = await t.data(); + if (data instanceof Uint16Array) return { type: 'f16', values: [...f16ToF32(data)] }; + if (data instanceof Float32Array) return { type: 'f32', values: [...data] }; + throw new Error('expected floats'); +} diff --git a/packages/runntime/test/core/weights/transforms.test.ts b/packages/runntime/test/core/weights/transforms.test.ts index a2f48d2d..7af5ffcf 100644 --- a/packages/runntime/test/core/weights/transforms.test.ts +++ b/packages/runntime/test/core/weights/transforms.test.ts @@ -1,5 +1,4 @@ import { describe, expect, it, vi } from 'vitest'; -import { f16ToF32, f32ToF16Bits } from '../../../src/core/weights/convert.ts'; import { bufferSource, derivedTensor, @@ -15,23 +14,10 @@ import { splitRows, transposeWeight, } from '../../../src/core/weights/transforms.ts'; +import { contents, f16File } from './checkpoint.ts'; type Dtype = 'F32' | 'F16' | 'BF16'; -/** A one-tensor safetensors file, built from the format spec: an 8-byte - * little-endian header length, the JSON header, then the payload. */ -function f16File(values: number[], shape: number[]): Uint8Array { - const payload = new Uint8Array(Uint16Array.from(values, f32ToF16Bits).buffer); - const header = new TextEncoder().encode( - JSON.stringify({ x: { dtype: 'F16', shape, data_offsets: [0, payload.length] } }), - ); - const out = new Uint8Array(8 + header.length + payload.length); - new DataView(out.buffer).setBigUint64(0, BigInt(header.length), true); - out.set(header, 8); - out.set(payload, 8 + header.length); - return out; -} - /** Values as bf16 bits. Every value used here is exact in bf16 and f16. */ const bf16 = (values: number[]) => Uint16Array.from(values, (v) => new Uint32Array(new Float32Array([v]).buffer)[0]! >>> 16); @@ -41,22 +27,13 @@ const bf16 = (values: number[]) => async function source(dtype: Dtype, values: number[], shape: number[]): Promise { const sd = dtype === 'F16' - ? await fromSafetensors(bufferSource(f16File(values, shape)), { chunkBytes: 0 }) + ? await fromSafetensors(bufferSource(f16File({ x: { values, shape } })), { chunkBytes: 0 }) : memoryStateDict({ x: { data: dtype === 'F32' ? new Float32Array(values) : bf16(values), shape }, }); return sd.tensors.get('x')!; } -/** What a helper's output holds: f16 halfwords or f32, decoded to numbers. */ -async function contents(t: LazyTensor): Promise<{ type: 'f16' | 'f32'; values: number[] }> { - if (t.kind !== 'derived') throw new Error('expected a derived tensor'); - const data = await t.data(); - if (data instanceof Uint16Array) return { type: 'f16', values: [...f16ToF32(data)] }; - if (data instanceof Float32Array) return { type: 'f32', values: [...data] }; - throw new Error('expected floats'); -} - /** F16 stays f16; F32 stays f32; BF16 decodes to f32, which is exact. */ const DTYPES: { dtype: Dtype; type: 'f16' | 'f32' }[] = [ { dtype: 'F32', type: 'f32' }, diff --git a/packages/runntime/test/zoo/minilm/stateDictHooks.test.ts b/packages/runntime/test/zoo/minilm/stateDictHooks.test.ts index 7d2e9b9a..e67a1d03 100644 --- a/packages/runntime/test/zoo/minilm/stateDictHooks.test.ts +++ b/packages/runntime/test/zoo/minilm/stateDictHooks.test.ts @@ -1,21 +1,12 @@ import { describe, expect, it } from 'vitest'; -import { f16ToF32, f32ToF16Bits } from '../../../src/core/weights/convert.ts'; -import { - bufferSource, - fromSafetensors, - memoryStateDict, - type LazyStateDict, - type LazyTensor, -} from '../../../src/core/weights/safetensors.ts'; import { MINILM_L6 } from '../../../src/zoo/minilm/config.ts'; import { transformMinilmStateDict } from '../../../src/zoo/minilm/stateDictHooks.ts'; +import { checkpoint, contents, type Entry } from '../../core/weights/checkpoint.ts'; // The hook reads only hidden, headDim and layers. headDim 16 makes the q/k // scale 16^-0.25 = 0.5, exact in f16. const CFG = { ...MINILM_L6, hidden: 2, layers: 1, headDim: 16 }; -type Entry = { values: number[]; shape: number[] }; - /** A one-layer BertModel checkpoint with HF names. Every value is exact in f16. */ const CHECKPOINT: Record = { 'embeddings.word_embeddings.weight': { values: [0, 0, 0, 0, 0, 0], shape: [3, 2] }, @@ -44,54 +35,12 @@ const CHECKPOINT: Record = { 'pooler.dense.bias': { values: [0, 0], shape: [2] }, }; -/** An F16 safetensors file, built from the format spec: an 8-byte - * little-endian header length, the JSON header, then the payload. */ -function f16File(entries: Record): Uint8Array { - const header: Record = {}; - const halves: number[] = []; - for (const [name, { values, shape }] of Object.entries(entries)) { - const begin = halves.length * 2; - halves.push(...values.map(f32ToF16Bits)); - header[name] = { dtype: 'F16', shape, data_offsets: [begin, halves.length * 2] }; - } - const json = new TextEncoder().encode(JSON.stringify(header)); - const payload = new Uint8Array(new Uint16Array(halves).buffer); - const out = new Uint8Array(8 + json.length + payload.length); - new DataView(out.buffer).setBigUint64(0, BigInt(json.length), true); - out.set(json, 8); - out.set(payload, 8 + json.length); - return out; -} - -async function checkpoint(dtype: 'F32' | 'F16'): Promise { - if (dtype === 'F16') { - return fromSafetensors(bufferSource(f16File(CHECKPOINT)), { chunkBytes: 0 }); - } - return memoryStateDict( - Object.fromEntries( - Object.entries(CHECKPOINT).map(([name, { values, shape }]) => [ - name, - { data: new Float32Array(values), shape }, - ]), - ), - ); -} - -/** What a derived tensor holds: f16 halfwords or f32, decoded to numbers. */ -async function contents(t: LazyTensor): Promise<{ type: 'f16' | 'f32'; values: number[] }> { - if (t.kind !== 'derived') throw new Error('expected a derived tensor'); - const data = await t.data(); - if (data instanceof Uint16Array) return { type: 'f16', values: [...f16ToF32(data)] }; - if (data instanceof Float32Array) return { type: 'f32', values: [...data] }; - throw new Error('expected floats'); -} - describe('transformMinilmStateDict', () => { it.for([ { dtype: 'F32', elemBytes: 4 }, { dtype: 'F16', elemBytes: 2 }, ] as const)('maps a $dtype checkpoint', async ({ dtype, elemBytes }) => { - const sd = await checkpoint(dtype); + const sd = await checkpoint(dtype, CHECKPOINT); const total = sd.totalBytes(); transformMinilmStateDict(sd, '', CFG); const get = (name: string) => sd.tensors.get(name)!; From b9b85355c24bcea700b8e9cf336cbb324f8007fc Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 27 Oct 2026 18:25:12 +0100 Subject: [PATCH 09/25] refactor(zoo): fuse Moonshine q/k/v with the shared weight helpers and keep f16 checkpoints in f16 --- .../zoo/moonshine-streaming/stateDictHooks.ts | 8 +- .../src/zoo/moonshine/stateDictHooks.ts | 77 ++++--------------- 2 files changed, 18 insertions(+), 67 deletions(-) diff --git a/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts b/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts index a92b8b45..463d8019 100644 --- a/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts +++ b/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts @@ -72,7 +72,7 @@ export function transformStreamingStateDict( const hf = q(`model.encoder.layers.${i}`); const en = q(`encoder.layers.${i}`); gammaNorm(`${hf}.input_layernorm.gamma`, `${en}.norm1.weight`); - fuseQkv(sd, `${hf}.self_attn`, `${en}.attn.qkv.weight`, eh, encQScale); + fuseQkv(sd, `${hf}.self_attn`, `${en}.attn.qkv.weight`, encQScale); move(sd, `${hf}.self_attn.o_proj.weight`, `${en}.attn.out.weight`); gammaNorm(`${hf}.post_attention_layernorm.gamma`, `${en}.norm2.weight`); move(sd, `${hf}.mlp.fc1.weight`, `${en}.mlp.fc1.weight`); @@ -85,9 +85,5 @@ export function transformStreamingStateDict( move(sd, q('model.decoder.pos_emb.weight'), q('adapter.posEmb')); if (eh !== dh) move(sd, q('model.decoder.proj.weight'), q('adapter.proj.weight')); - transformDecoder(sd, prefix, { - layers: cfg.dec.layers, - hidden: dh, - headDim: cfg.dec.headDim, - }); + transformDecoder(sd, prefix, { layers: cfg.dec.layers, headDim: cfg.dec.headDim }); } diff --git a/packages/runntime/src/zoo/moonshine/stateDictHooks.ts b/packages/runntime/src/zoo/moonshine/stateDictHooks.ts index 884757d5..25df3db4 100644 --- a/packages/runntime/src/zoo/moonshine/stateDictHooks.ts +++ b/packages/runntime/src/zoo/moonshine/stateDictHooks.ts @@ -7,10 +7,11 @@ * Everything else renames. * * Weights stay in torch layout for the core Linear and Conv1d hooks to - * transpose. Derived byteLengths carry the source file bytes, so load progress - * still sums to the checkpoint size. */ + * transpose. An F16 checkpoint gives f16 derived tensors. Derived byteLengths + * carry the source file bytes, so load progress still sums to the checkpoint + * size. */ -import { derivedTensor, type LazyStateDict, type LazyTensor } from '../../core/index.ts'; +import { concatRows, scaleWeight, type LazyStateDict, type LazyTensor } from '../../core/index.ts'; import { checkpointMismatch } from '../errors.ts'; import type { MoonshineConfig } from './config.ts'; @@ -28,53 +29,14 @@ export function move(sd: LazyStateDict, from: string, to: string): void { sd.tensors.set(to, take(sd, from)); } -export function stackRows( - parts: readonly { t: LazyTensor; scale?: number }[], - cols: number, -): LazyTensor { - let rows = 0; - for (const { t } of parts) { - if (t.shape.length !== 2 || t.shape[1] !== cols) { - throw checkpointMismatch( - 'moonshine state dict', - `stackRows part shape [${t.shape}] != [_, ${cols}]`, - ); - } - rows += t.shape[0]!; - } - return derivedTensor( - [rows, cols], - async () => { - const out = new Float32Array(rows * cols); - let base = 0; - for (const { t, scale } of parts) { - const data = await t.f32(); - if (scale === undefined) out.set(data, base); - else for (let i = 0; i < data.length; i++) out[base + i] = data[i]! * scale; - base += data.length; - } - return out; - }, - parts.reduce((sum, p) => sum + p.t.byteLength, 0), - ); -} - -export function fuseQkv( - sd: LazyStateDict, - hfPrefix: string, - ours: string, - cols: number, - qScale?: number, -): void { +/** Stacks q|k|v into one torch-layout [3·hidden, hidden] weight, with q times + * `qScale` when given. */ +export function fuseQkv(sd: LazyStateDict, hfPrefix: string, ours: string, qScale?: number): void { + const part = (p: string) => take(sd, `${hfPrefix}.${p}_proj.weight`); + const query = part('q'); sd.tensors.set( ours, - stackRows( - ['q', 'k', 'v'].map((p) => ({ - t: take(sd, `${hfPrefix}.${p}_proj.weight`), - scale: p === 'q' ? qScale : undefined, - })), - cols, - ), + concatRows([qScale === undefined ? query : scaleWeight(query, qScale), part('k'), part('v')]), ); } @@ -90,7 +52,7 @@ export function moveTiedEmbedding(sd: LazyStateDict, prefix: string): void { export function transformDecoder( sd: LazyStateDict, prefix: string, - dec: { layers: number; hidden: number; headDim: number }, + dec: { layers: number; headDim: number }, ): void { const q = (name: string) => qualify(prefix, name); const crossQScale = 1 / Math.sqrt(dec.headDim); @@ -98,22 +60,16 @@ export function transformDecoder( const hf = q(`model.decoder.layers.${i}`); const en = q(`decoder.layers.${i}`); move(sd, `${hf}.input_layernorm.weight`, `${en}.norm1.weight`); - fuseQkv(sd, `${hf}.self_attn`, `${en}.selfAttn.qkv.weight`, dec.hidden); + fuseQkv(sd, `${hf}.self_attn`, `${en}.selfAttn.qkv.weight`); move(sd, `${hf}.self_attn.o_proj.weight`, `${en}.selfAttn.out.weight`); move(sd, `${hf}.post_attention_layernorm.weight`, `${en}.norm2.weight`); sd.tensors.set( `${en}.crossAttn.q.weight`, - stackRows( - [{ t: take(sd, `${hf}.encoder_attn.q_proj.weight`), scale: crossQScale }], - dec.hidden, - ), + scaleWeight(take(sd, `${hf}.encoder_attn.q_proj.weight`), crossQScale), ); sd.tensors.set( `${en}.crossAttn.kv.weight`, - stackRows( - ['k', 'v'].map((p) => ({ t: take(sd, `${hf}.encoder_attn.${p}_proj.weight`) })), - dec.hidden, - ), + concatRows(['k', 'v'].map((p) => take(sd, `${hf}.encoder_attn.${p}_proj.weight`))), ); move(sd, `${hf}.encoder_attn.o_proj.weight`, `${en}.crossAttn.out.weight`); move(sd, `${hf}.final_layernorm.weight`, `${en}.norm3.weight`); @@ -133,7 +89,6 @@ export function transformMoonshineStateDict( cfg: MoonshineConfig, ): void { const q = (name: string) => qualify(prefix, name); - const d = cfg.dModel; moveTiedEmbedding(sd, prefix); @@ -149,7 +104,7 @@ export function transformMoonshineStateDict( const hf = q(`model.encoder.layers.${i}`); const en = q(`encoder.layers.${i}`); move(sd, `${hf}.input_layernorm.weight`, `${en}.norm1.weight`); - fuseQkv(sd, `${hf}.self_attn`, `${en}.attn.qkv.weight`, d); + fuseQkv(sd, `${hf}.self_attn`, `${en}.attn.qkv.weight`); move(sd, `${hf}.self_attn.o_proj.weight`, `${en}.attn.out.weight`); move(sd, `${hf}.post_attention_layernorm.weight`, `${en}.norm2.weight`); move(sd, `${hf}.mlp.fc1.weight`, `${en}.mlp.fc1.weight`); @@ -159,5 +114,5 @@ export function transformMoonshineStateDict( } move(sd, q('model.encoder.layer_norm.weight'), q('encoder.norm.weight')); - transformDecoder(sd, prefix, { layers: cfg.decLayers, hidden: d, headDim: cfg.headDim }); + transformDecoder(sd, prefix, { layers: cfg.decLayers, headDim: cfg.headDim }); } From f0d18d720441a5c08aaaab5cda462205d8f2c1e6 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 27 Oct 2026 18:26:03 +0100 Subject: [PATCH 10/25] test(zoo): cover the Moonshine state dict hook on F32 and F16 checkpoints --- .../test/zoo/moonshine/stateDictHooks.test.ts | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 packages/runntime/test/zoo/moonshine/stateDictHooks.test.ts diff --git a/packages/runntime/test/zoo/moonshine/stateDictHooks.test.ts b/packages/runntime/test/zoo/moonshine/stateDictHooks.test.ts new file mode 100644 index 00000000..714483cc --- /dev/null +++ b/packages/runntime/test/zoo/moonshine/stateDictHooks.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from 'vitest'; +import { MOONSHINE_TINY } from '../../../src/zoo/moonshine/config.ts'; +import { transformMoonshineStateDict } from '../../../src/zoo/moonshine/stateDictHooks.ts'; +import { checkpoint, contents, type Entry } from '../../core/weights/checkpoint.ts'; + +// The hook reads only encLayers, decLayers and headDim. headDim 4 makes the +// cross-attention q scale 1/√4 = 0.5, exact in f16. +const CFG = { ...MOONSHINE_TINY, encLayers: 1, decLayers: 1, headDim: 4 }; + +const square = (values: number[]): Entry => ({ values, shape: [2, 2] }); +const ZERO = square([0, 0, 0, 0]); + +/** Tensors the hook only renames, HF name → ours. */ +const RENAMED: Record = { + 'model.decoder.embed_tokens.weight': 'embedding.weight', + 'model.encoder.conv1.weight': 'stem.conv1.weight', + 'model.encoder.groupnorm.weight': 'stem.groupnorm.weight', + 'model.encoder.groupnorm.bias': 'stem.groupnorm.bias', + 'model.encoder.conv2.weight': 'stem.conv2.weight', + 'model.encoder.conv2.bias': 'stem.conv2.bias', + 'model.encoder.conv3.weight': 'stem.conv3.weight', + 'model.encoder.conv3.bias': 'stem.conv3.bias', + 'model.encoder.layers.0.input_layernorm.weight': 'encoder.layers.0.norm1.weight', + 'model.encoder.layers.0.self_attn.o_proj.weight': 'encoder.layers.0.attn.out.weight', + 'model.encoder.layers.0.post_attention_layernorm.weight': 'encoder.layers.0.norm2.weight', + 'model.encoder.layers.0.mlp.fc1.weight': 'encoder.layers.0.mlp.fc1.weight', + 'model.encoder.layers.0.mlp.fc1.bias': 'encoder.layers.0.mlp.fc1.bias', + 'model.encoder.layers.0.mlp.fc2.weight': 'encoder.layers.0.mlp.fc2.weight', + 'model.encoder.layers.0.mlp.fc2.bias': 'encoder.layers.0.mlp.fc2.bias', + 'model.encoder.layer_norm.weight': 'encoder.norm.weight', + 'model.decoder.layers.0.input_layernorm.weight': 'decoder.layers.0.norm1.weight', + 'model.decoder.layers.0.self_attn.o_proj.weight': 'decoder.layers.0.selfAttn.out.weight', + 'model.decoder.layers.0.post_attention_layernorm.weight': 'decoder.layers.0.norm2.weight', + 'model.decoder.layers.0.encoder_attn.o_proj.weight': 'decoder.layers.0.crossAttn.out.weight', + 'model.decoder.layers.0.final_layernorm.weight': 'decoder.layers.0.norm3.weight', + 'model.decoder.layers.0.mlp.fc1.weight': 'decoder.layers.0.mlp.fc1.weight', + 'model.decoder.layers.0.mlp.fc1.bias': 'decoder.layers.0.mlp.fc1.bias', + 'model.decoder.layers.0.mlp.fc2.weight': 'decoder.layers.0.mlp.fc2.weight', + 'model.decoder.layers.0.mlp.fc2.bias': 'decoder.layers.0.mlp.fc2.bias', + 'model.decoder.norm.weight': 'decoder.norm.weight', +}; + +/** A one-layer Moonshine checkpoint with HF names. Every value is exact in f16. */ +const CHECKPOINT: Record = { + ...Object.fromEntries(Object.keys(RENAMED).map((name) => [name, ZERO])), + 'proj_out.weight': ZERO, + 'model.encoder.layers.0.self_attn.q_proj.weight': square([1, 2, 3, 4]), + 'model.encoder.layers.0.self_attn.k_proj.weight': square([5, 6, 7, 8]), + 'model.encoder.layers.0.self_attn.v_proj.weight': square([9, 10, 11, 12]), + 'model.decoder.layers.0.self_attn.q_proj.weight': square([-1, -2, -3, -4]), + 'model.decoder.layers.0.self_attn.k_proj.weight': square([-5, -6, -7, -8]), + 'model.decoder.layers.0.self_attn.v_proj.weight': square([-9, -10, -11, -12]), + 'model.decoder.layers.0.encoder_attn.q_proj.weight': square([1, 3, 5, 7]), + 'model.decoder.layers.0.encoder_attn.k_proj.weight': square([2, 4, 6, 8]), + 'model.decoder.layers.0.encoder_attn.v_proj.weight': square([10, 20, 30, 40]), +}; + +describe('transformMoonshineStateDict', () => { + it.for([ + { dtype: 'F32', type: 'f32', elemBytes: 4 }, + { dtype: 'F16', type: 'f16', elemBytes: 2 }, + ] as const)('maps a $dtype checkpoint', async ({ dtype, type, elemBytes }) => { + const sd = await checkpoint(dtype, CHECKPOINT); + const total = sd.totalBytes(); + const before = new Map(sd.tensors); + transformMoonshineStateDict(sd, '', CFG); + const get = (name: string) => sd.tensors.get(name)!; + + const fused = [ + 'encoder.layers.0.attn.qkv.weight', + 'decoder.layers.0.selfAttn.qkv.weight', + 'decoder.layers.0.crossAttn.q.weight', + 'decoder.layers.0.crossAttn.kv.weight', + ]; + // proj_out is tied to the embedding and dropped. + expect([...sd.tensors.keys()].sort()).toEqual([...Object.values(RENAMED), ...fused].sort()); + for (const [hf, ours] of Object.entries(RENAMED)) expect(get(ours)).toBe(before.get(hf)); + + // Self-attention q|k|v rows in that order, unscaled: rope carries the scale. + const encQkv = get('encoder.layers.0.attn.qkv.weight'); + expect(encQkv.shape).toEqual([6, 2]); + expect(encQkv.byteLength).toBe(12 * elemBytes); + expect(await contents(encQkv)).toEqual({ + type, + values: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], + }); + const decQkv = get('decoder.layers.0.selfAttn.qkv.weight'); + expect(decQkv.shape).toEqual([6, 2]); + expect(decQkv.byteLength).toBe(12 * elemBytes); + expect(await contents(decQkv)).toEqual({ + type, + values: [-1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12], + }); + + // Cross-attention q times 1/√headDim = 0.5; k|v stacked as is. + const crossQ = get('decoder.layers.0.crossAttn.q.weight'); + expect(crossQ.shape).toEqual([2, 2]); + expect(crossQ.byteLength).toBe(4 * elemBytes); + expect(await contents(crossQ)).toEqual({ type, values: [0.5, 1.5, 2.5, 3.5] }); + const crossKv = get('decoder.layers.0.crossAttn.kv.weight'); + expect(crossKv.shape).toEqual([4, 2]); + expect(crossKv.byteLength).toBe(8 * elemBytes); + expect(await contents(crossKv)).toEqual({ type, values: [2, 4, 6, 8, 10, 20, 30, 40] }); + + // The dropped proj_out (4 elems) is the only byte loss. + expect(sd.totalBytes()).toBe(total - 4 * elemBytes); + }); +}); From 4edf30964bd7f26c40b2a6ff787a7fb8ae8d30fc Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 27 Oct 2026 18:26:54 +0100 Subject: [PATCH 11/25] refactor(zoo): keep the Moonshine v2 norm gammas in f32 on purpose and say so in the hook --- .../runntime/src/zoo/moonshine-streaming/stateDictHooks.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts b/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts index 463d8019..ce40ce53 100644 --- a/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts +++ b/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts @@ -7,8 +7,9 @@ * rope tables carry the scale. Everything else renames. * * Weights stay in torch layout for the core Linear and Conv1d hooks to - * transpose. Derived byteLengths carry the source file bytes, so load progress - * still sums to the checkpoint size. */ + * transpose. An F16 checkpoint gives f16 fused q/k/v; the norm gammas and + * frontend constants stay f32. Derived byteLengths carry the source file + * bytes, so load progress still sums to the checkpoint size. */ import { derivedTensor, type LazyStateDict, type LazyTensor } from '../../core/index.ts'; import { @@ -21,6 +22,8 @@ import { } from '../moonshine/stateDictHooks.ts'; import type { MoonshineStreamingConfig } from './config.ts'; +/** Stays f32 for any checkpoint: rounding gamma + 1 to f16 would drop the low + * bits of gamma, which an f32 run keeps. */ function gammaPlusOne(t: LazyTensor): LazyTensor { return derivedTensor( t.shape, From bcef5138e1b96b3355d88b67d51da2dec03c6efd Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 27 Oct 2026 18:26:58 +0100 Subject: [PATCH 12/25] test(zoo): cover the Moonshine v2 state dict hook on F32 and F16 checkpoints --- .../stateDictHooks.test.ts | 148 ++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 packages/runntime/test/zoo/moonshine-streaming/stateDictHooks.test.ts diff --git a/packages/runntime/test/zoo/moonshine-streaming/stateDictHooks.test.ts b/packages/runntime/test/zoo/moonshine-streaming/stateDictHooks.test.ts new file mode 100644 index 00000000..3235fc33 --- /dev/null +++ b/packages/runntime/test/zoo/moonshine-streaming/stateDictHooks.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, it } from 'vitest'; +import { MOONSHINE_STREAMING_SMALL } from '../../../src/zoo/moonshine-streaming/config.ts'; +import { transformStreamingStateDict } from '../../../src/zoo/moonshine-streaming/stateDictHooks.ts'; +import { checkpoint, contents, type Entry } from '../../core/weights/checkpoint.ts'; + +// The hook reads only frameLen and the layer counts, head dims and hidden +// sizes. Encoder headDim 4 makes the q scale 1/√4 = 0.5, exact in f16. The +// hidden sizes differ, as in the small model, so the adapter projection maps. +const SMALL = MOONSHINE_STREAMING_SMALL; +const CFG = { + ...SMALL, + enc: { ...SMALL.enc, hidden: 2, layers: 1, headDim: 4 }, + dec: { ...SMALL.dec, hidden: 3, layers: 1, headDim: 4 }, + frameLen: 3, +}; + +const square = (values: number[]): Entry => ({ values, shape: [2, 2] }); +const ZERO = square([0, 0, 0, 0]); + +/** Tensors the hook only renames, HF name → ours. */ +const RENAMED: Record = { + 'model.decoder.embed_tokens.weight': 'embedding.weight', + 'model.encoder.embedder.linear.weight': 'frontend.linear.weight', + 'model.encoder.embedder.conv1.weight': 'frontend.conv1.weight', + 'model.encoder.embedder.conv1.bias': 'frontend.conv1.bias', + 'model.encoder.embedder.conv2.weight': 'frontend.conv2.weight', + 'model.encoder.embedder.conv2.bias': 'frontend.conv2.bias', + 'model.encoder.layers.0.self_attn.o_proj.weight': 'encoder.layers.0.attn.out.weight', + 'model.encoder.layers.0.mlp.fc1.weight': 'encoder.layers.0.mlp.fc1.weight', + 'model.encoder.layers.0.mlp.fc1.bias': 'encoder.layers.0.mlp.fc1.bias', + 'model.encoder.layers.0.mlp.fc2.weight': 'encoder.layers.0.mlp.fc2.weight', + 'model.encoder.layers.0.mlp.fc2.bias': 'encoder.layers.0.mlp.fc2.bias', + 'model.decoder.pos_emb.weight': 'adapter.posEmb', + 'model.decoder.proj.weight': 'adapter.proj.weight', + 'model.decoder.layers.0.input_layernorm.weight': 'decoder.layers.0.norm1.weight', + 'model.decoder.layers.0.self_attn.o_proj.weight': 'decoder.layers.0.selfAttn.out.weight', + 'model.decoder.layers.0.post_attention_layernorm.weight': 'decoder.layers.0.norm2.weight', + 'model.decoder.layers.0.encoder_attn.o_proj.weight': 'decoder.layers.0.crossAttn.out.weight', + 'model.decoder.layers.0.final_layernorm.weight': 'decoder.layers.0.norm3.weight', + 'model.decoder.layers.0.mlp.fc1.weight': 'decoder.layers.0.mlp.fc1.weight', + 'model.decoder.layers.0.mlp.fc1.bias': 'decoder.layers.0.mlp.fc1.bias', + 'model.decoder.layers.0.mlp.fc2.weight': 'decoder.layers.0.mlp.fc2.weight', + 'model.decoder.layers.0.mlp.fc2.bias': 'decoder.layers.0.mlp.fc2.bias', + 'model.decoder.norm.weight': 'decoder.norm.weight', +}; + +/** A one-layer Moonshine v2 checkpoint with HF names. Every value is exact in + * f16. */ +const CHECKPOINT: Record = { + ...Object.fromEntries(Object.keys(RENAMED).map((name) => [name, ZERO])), + 'proj_out.weight': ZERO, + 'model.encoder.embedder.comp.log_k': { values: [1], shape: [1] }, + 'model.encoder.layers.0.input_layernorm.gamma': { values: [0.5, -0.25], shape: [2] }, + 'model.encoder.layers.0.post_attention_layernorm.gamma': { values: [0, 2], shape: [2] }, + 'model.encoder.final_norm.gamma': { values: [-1, 0.125], shape: [2] }, + 'model.encoder.layers.0.self_attn.q_proj.weight': square([1, 2, 3, 4]), + 'model.encoder.layers.0.self_attn.k_proj.weight': square([5, 6, 7, 8]), + 'model.encoder.layers.0.self_attn.v_proj.weight': square([9, 10, 11, 12]), + 'model.decoder.layers.0.self_attn.q_proj.weight': square([-1, -2, -3, -4]), + 'model.decoder.layers.0.self_attn.k_proj.weight': square([-5, -6, -7, -8]), + 'model.decoder.layers.0.self_attn.v_proj.weight': square([-9, -10, -11, -12]), + 'model.decoder.layers.0.encoder_attn.q_proj.weight': square([1, 3, 5, 7]), + 'model.decoder.layers.0.encoder_attn.k_proj.weight': square([2, 4, 6, 8]), + 'model.decoder.layers.0.encoder_attn.v_proj.weight': square([10, 20, 30, 40]), +}; + +describe('transformStreamingStateDict', () => { + it.for([ + { dtype: 'F32', type: 'f32', elemBytes: 4 }, + { dtype: 'F16', type: 'f16', elemBytes: 2 }, + ] as const)('maps a $dtype checkpoint', async ({ dtype, type, elemBytes }) => { + const sd = await checkpoint(dtype, CHECKPOINT); + const total = sd.totalBytes(); + const before = new Map(sd.tensors); + transformStreamingStateDict(sd, '', CFG); + const get = (name: string) => sd.tensors.get(name)!; + + const derived = [ + 'frontend.cmvn.weight', + 'frontend.k', + 'encoder.layers.0.norm1.weight', + 'encoder.layers.0.norm2.weight', + 'encoder.norm.weight', + 'encoder.layers.0.attn.qkv.weight', + 'decoder.layers.0.selfAttn.qkv.weight', + 'decoder.layers.0.crossAttn.q.weight', + 'decoder.layers.0.crossAttn.kv.weight', + ]; + // proj_out is tied to the embedding and dropped. + expect([...sd.tensors.keys()].sort()).toEqual([...Object.values(RENAMED), ...derived].sort()); + for (const [hf, ours] of Object.entries(RENAMED)) expect(get(ours)).toBe(before.get(hf)); + + // CMVN has no learned scale: ones, with no file bytes behind them. + const cmvn = get('frontend.cmvn.weight'); + expect(cmvn.shape).toEqual([3]); + expect(cmvn.byteLength).toBe(0); + expect(await contents(cmvn)).toEqual({ type: 'f32', values: [1, 1, 1] }); + + // k = exp(log_k) = e, in f32 for any checkpoint. + const k = get('frontend.k'); + expect(k.shape).toEqual([1]); + expect(k.byteLength).toBe(elemBytes); + expect(await contents(k)).toEqual({ type: 'f32', values: [Math.fround(Math.E)] }); + + // Norm gammas gain back their −1 offset, in f32 for any checkpoint. + for (const [name, values] of [ + ['encoder.layers.0.norm1.weight', [1.5, 0.75]], + ['encoder.layers.0.norm2.weight', [1, 3]], + ['encoder.norm.weight', [0, 1.125]], + ] as const) { + const gamma = get(name); + expect(gamma.shape).toEqual([2]); + expect(gamma.byteLength).toBe(2 * elemBytes); + expect(await contents(gamma)).toEqual({ type: 'f32', values }); + } + + // Encoder q|k|v with q times 1/√headDim = 0.5: the encoder has no rope. + const encQkv = get('encoder.layers.0.attn.qkv.weight'); + expect(encQkv.shape).toEqual([6, 2]); + expect(encQkv.byteLength).toBe(12 * elemBytes); + expect(await contents(encQkv)).toEqual({ + type, + values: [0.5, 1, 1.5, 2, 5, 6, 7, 8, 9, 10, 11, 12], + }); + + // Decoder q|k|v unscaled: its rope tables carry the scale. + const decQkv = get('decoder.layers.0.selfAttn.qkv.weight'); + expect(decQkv.shape).toEqual([6, 2]); + expect(decQkv.byteLength).toBe(12 * elemBytes); + expect(await contents(decQkv)).toEqual({ + type, + values: [-1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12], + }); + + // Cross-attention q times 1/√headDim = 0.5; k|v stacked as is. + const crossQ = get('decoder.layers.0.crossAttn.q.weight'); + expect(crossQ.shape).toEqual([2, 2]); + expect(crossQ.byteLength).toBe(4 * elemBytes); + expect(await contents(crossQ)).toEqual({ type, values: [0.5, 1.5, 2.5, 3.5] }); + const crossKv = get('decoder.layers.0.crossAttn.kv.weight'); + expect(crossKv.shape).toEqual([4, 2]); + expect(crossKv.byteLength).toBe(8 * elemBytes); + expect(await contents(crossKv)).toEqual({ type, values: [2, 4, 6, 8, 10, 20, 30, 40] }); + + // The dropped proj_out (4 elems) is the only byte loss. + expect(sd.totalBytes()).toBe(total - 4 * elemBytes); + }); +}); From c8dd0920271647943746edd9acf0da83fae2d870 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 27 Oct 2026 18:27:58 +0100 Subject: [PATCH 13/25] refactor(zoo): map the privacy-filter float checkpoint with the shared weight helpers and keep f16 in f16 --- .../src/zoo/privacy-filter/stateDictHooks.ts | 85 ++++--------------- 1 file changed, 17 insertions(+), 68 deletions(-) diff --git a/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts b/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts index 4fbdac95..fd899902 100644 --- a/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts +++ b/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts @@ -6,10 +6,17 @@ * expert axis folded into rows: [E, K, 2H] → two [E·K, H]; * - the rest renames. score.bias is all zero and the head has no bias, so * it is dropped. + * An F16 checkpoint gives f16 derived tensors. * Files from tools/export_pf_safetensors.py use these names as is and * bypass this mapping. */ -import { type LazyStateDict, type LazyTensor } from '../../core/index.ts'; +import { + concatRows, + reshapeWeight, + splitCols, + type LazyStateDict, + type LazyTensor, +} from '../../core/index.ts'; import { checkpointMismatch } from '../errors.ts'; function take(sd: LazyStateDict, name: string): LazyTensor { @@ -23,76 +30,18 @@ function move(sd: LazyStateDict, from: string, to: string): void { sd.tensors.set(to, take(sd, from)); } -function lazyF32( - shape: number[], - byteLength: number, - compute: () => Promise, -): LazyTensor { - const packed = async (): Promise => { - throw new Error('privacy-filter state dict: derived tensor holds f32'); - }; - return { - kind: 'derived', - dtype: 'F32', - shape, - byteLength, - data: compute, - f32: compute, - halfWords: packed, - words: packed, - u16: packed, - }; -} - -const numel = (shape: readonly number[]): number => shape.reduce((a, b) => a * b, 1); - -function concatRows(parts: readonly LazyTensor[]): LazyTensor { - const rest = parts[0]!.shape.slice(1); - for (const p of parts) { - if (p.shape.slice(1).join(',') !== rest.join(',')) { - throw checkpointMismatch( - 'privacy-filter state dict', - `cannot stack shapes [${parts.map((q) => q.shape.join('x')).join('], [')}]`, - ); - } - } - const rows = parts.reduce((sum, p) => sum + p.shape[0]!, 0); - const bytes = parts.reduce((sum, p) => sum + p.byteLength, 0); - return lazyF32([rows, ...rest], bytes, async () => { - const out = new Float32Array(rows * numel(rest)); - let base = 0; - for (const p of parts) { - const data = await p.f32(); - out.set(data, base); - base += data.length; - } - return out; - }); +/** Folds every leading dim into rows: [E, K, H] → [E·K, H]. */ +function foldRows(t: LazyTensor): LazyTensor { + const cols = t.shape[t.shape.length - 1]!; + return reshapeWeight(t, [t.shape.reduce((a, b) => a * b, 1) / cols, cols]); } +/** Splits the last dim into glu and lin halves, each folded to 2D: + * [E, K, 2H] → two [E·K, H], and a [E, 2H] bias → two [E, H]. */ function splitHalves(t: LazyTensor): [LazyTensor, LazyTensor] { - const width = t.shape[t.shape.length - 1]!; - const half = width / 2; - const rows = numel(t.shape.slice(0, -1)); - let shared: Promise | undefined; - let reads = 0; - const source = () => (shared ??= t.f32()); - const columns = (start: number) => - lazyF32([rows, half], t.byteLength / 2, async () => { - const data = await source(); - if (++reads === 2) shared = undefined; - const out = new Float32Array(rows * half); - for (let r = 0; r < rows; r++) { - out.set(data.subarray(r * width + start, r * width + start + half), r * half); - } - return out; - }); - return [columns(0), columns(half)]; -} - -function foldRows(t: LazyTensor): LazyTensor { - const cols = t.shape[t.shape.length - 1]!; - return lazyF32([numel(t.shape) / cols, cols], t.byteLength, () => t.f32()); + const half = t.shape[t.shape.length - 1]! / 2; + const [glu, lin] = splitCols(t, [half, half]); + return [foldRows(glu!), foldRows(lin!)]; } export function transformHfPrivacyFilterStateDict(sd: LazyStateDict): void { From 435ca96b874bcbabc40b318e394961e50a6a0b0c Mon Sep 17 00:00:00 2001 From: chmjkb Date: Tue, 27 Oct 2026 18:28:41 +0100 Subject: [PATCH 14/25] test(zoo): cover the privacy-filter state dict hook on F32 and F16 checkpoints --- .../zoo/privacy-filter/stateDictHooks.test.ts | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts diff --git a/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts b/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts new file mode 100644 index 00000000..9607e88d --- /dev/null +++ b/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts @@ -0,0 +1,108 @@ +import { describe, expect, it } from 'vitest'; +import { transformHfPrivacyFilterStateDict } from '../../../src/zoo/privacy-filter/stateDictHooks.ts'; +import { checkpoint, contents, type Entry } from '../../core/weights/checkpoint.ts'; + +const ZERO = { values: [0, 0], shape: [2] }; + +/** Tensors the hook only renames, HF name → ours. */ +const RENAMED: Record = { + 'model.layers.0.input_layernorm.weight': 'block.0.attn.norm.scale', + 'model.layers.0.post_attention_layernorm.weight': 'block.0.mlp.norm.scale', + 'model.layers.0.self_attn.o_proj.weight': 'block.0.attn.out.weight', + 'model.layers.0.self_attn.o_proj.bias': 'block.0.attn.out.bias', + 'model.layers.0.mlp.router.weight': 'block.0.mlp.gate.weight', + 'model.layers.0.mlp.router.bias': 'block.0.mlp.gate.bias', + 'model.layers.0.self_attn.sinks': 'block.0.attn.sinks', + 'model.layers.0.mlp.experts.down_proj_bias': 'block.0.mlp.outBias', + 'model.embed_tokens.weight': 'embedding.weight', + 'model.norm.weight': 'norm.scale', + 'score.weight': 'unembedding.weight', +}; + +/** A one-layer checkpoint in the transformers layout, with 2 experts, hidden + * 1 and expert width 2. Every value is exact in f16. */ +const CHECKPOINT: Record = { + ...Object.fromEntries(Object.keys(RENAMED).map((name) => [name, ZERO])), + 'score.bias': ZERO, + // Grouped-query shapes: one q row pair, one k row, one v row. + 'model.layers.0.self_attn.q_proj.weight': { values: [1, 2, 3, 4], shape: [2, 2] }, + 'model.layers.0.self_attn.k_proj.weight': { values: [5, 6], shape: [1, 2] }, + 'model.layers.0.self_attn.v_proj.weight': { values: [7, 8], shape: [1, 2] }, + 'model.layers.0.self_attn.q_proj.bias': { values: [1, 2], shape: [2] }, + 'model.layers.0.self_attn.k_proj.bias': { values: [3], shape: [1] }, + 'model.layers.0.self_attn.v_proj.bias': { values: [4], shape: [1] }, + // [E, K, 2H] = [2, 1, 4]: expert 0 is 1 2 | 3 4, expert 1 is 5 6 | 7 8. + 'model.layers.0.mlp.experts.gate_up_proj': { + values: [1, 2, 3, 4, 5, 6, 7, 8], + shape: [2, 1, 4], + }, + // [E, 2H] = [2, 4]: expert 0 is 10 11 | 12 13, expert 1 is 14 15 | 16 17. + 'model.layers.0.mlp.experts.gate_up_proj_bias': { + values: [10, 11, 12, 13, 14, 15, 16, 17], + shape: [2, 4], + }, + // [E, H, K] = [2, 2, 1]. + 'model.layers.0.mlp.experts.down_proj': { values: [1, 2, 3, 4], shape: [2, 2, 1] }, +}; + +describe('transformHfPrivacyFilterStateDict', () => { + it.for([ + { dtype: 'F32', type: 'f32', elemBytes: 4 }, + { dtype: 'F16', type: 'f16', elemBytes: 2 }, + ] as const)('maps a $dtype checkpoint', async ({ dtype, type, elemBytes }) => { + const sd = await checkpoint(dtype, CHECKPOINT); + const total = sd.totalBytes(); + const before = new Map(sd.tensors); + transformHfPrivacyFilterStateDict(sd); + const get = (name: string) => sd.tensors.get(name)!; + + const mapped = [ + 'block.0.attn.qkv.weight', + 'block.0.attn.qkv.bias', + 'block.0.mlp.gluWeight', + 'block.0.mlp.linWeight', + 'block.0.mlp.gluBias', + 'block.0.mlp.linBias', + 'block.0.mlp.outWeight', + ]; + // score.bias is dropped. + expect([...sd.tensors.keys()].sort()).toEqual([...Object.values(RENAMED), ...mapped].sort()); + for (const [hf, ours] of Object.entries(RENAMED)) expect(get(ours)).toBe(before.get(hf)); + expect(sd.metadata['model']).toBe('privacy-filter'); + + // q|k|v rows in that order. + const qkvWeight = get('block.0.attn.qkv.weight'); + expect(qkvWeight.shape).toEqual([4, 2]); + expect(qkvWeight.byteLength).toBe(8 * elemBytes); + expect(await contents(qkvWeight)).toEqual({ type, values: [1, 2, 3, 4, 5, 6, 7, 8] }); + const qkvBias = get('block.0.attn.qkv.bias'); + expect(qkvBias.shape).toEqual([4]); + expect(qkvBias.byteLength).toBe(4 * elemBytes); + expect(await contents(qkvBias)).toEqual({ type, values: [1, 2, 3, 4] }); + + // Each expert's first half of columns is glu, the second lin, with the + // expert axis folded into rows. + for (const [name, shape, values] of [ + ['block.0.mlp.gluWeight', [2, 2], [1, 2, 5, 6]], + ['block.0.mlp.linWeight', [2, 2], [3, 4, 7, 8]], + ['block.0.mlp.gluBias', [2, 2], [10, 11, 14, 15]], + ['block.0.mlp.linBias', [2, 2], [12, 13, 16, 17]], + ] as const) { + const half = get(name); + expect(half.shape).toEqual(shape); + expect(half.byteLength).toBe(4 * elemBytes); + expect(await contents(half)).toEqual({ type, values }); + } + + // down_proj only folds [2, 2, 1] → [4, 1]: the source tensor, reshaped. + const source = before.get('model.layers.0.mlp.experts.down_proj')!; + const outWeight = get('block.0.mlp.outWeight'); + expect(outWeight.shape).toEqual([4, 1]); + expect(outWeight.byteLength).toBe(4 * elemBytes); + expect([outWeight.kind, outWeight.dtype]).toEqual([source.kind, source.dtype]); + expect([...(await outWeight.f32())]).toEqual([1, 2, 3, 4]); + + // The dropped score.bias (2 elems) is the only byte loss. + expect(sd.totalBytes()).toBe(total - 2 * elemBytes); + }); +}); From 0610147f33e753990fd107b242687ca2a2877795 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 28 Oct 2026 10:15:34 +0100 Subject: [PATCH 15/25] test(zoo): expect f32 for the scaled Moonshine q now that scaleWeight keeps products in f32 --- .../src/zoo/moonshine-streaming/stateDictHooks.ts | 7 ++++--- packages/runntime/src/zoo/moonshine/stateDictHooks.ts | 7 ++++--- .../test/zoo/moonshine-streaming/stateDictHooks.test.ts | 8 +++++--- .../runntime/test/zoo/moonshine/stateDictHooks.test.ts | 5 +++-- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts b/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts index ce40ce53..b7d65e30 100644 --- a/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts +++ b/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts @@ -7,9 +7,10 @@ * rope tables carry the scale. Everything else renames. * * Weights stay in torch layout for the core Linear and Conv1d hooks to - * transpose. An F16 checkpoint gives f16 fused q/k/v; the norm gammas and - * frontend constants stay f32. Derived byteLengths carry the source file - * bytes, so load progress still sums to the checkpoint size. */ + * transpose. An F16 checkpoint gives f16 decoder q/k/v; the scaled encoder + * q/k/v and cross-attention q, the norm gammas and the frontend constants + * stay f32. Derived byteLengths carry the source file bytes, so load progress + * still sums to the checkpoint size. */ import { derivedTensor, type LazyStateDict, type LazyTensor } from '../../core/index.ts'; import { diff --git a/packages/runntime/src/zoo/moonshine/stateDictHooks.ts b/packages/runntime/src/zoo/moonshine/stateDictHooks.ts index 25df3db4..f5d9984f 100644 --- a/packages/runntime/src/zoo/moonshine/stateDictHooks.ts +++ b/packages/runntime/src/zoo/moonshine/stateDictHooks.ts @@ -7,9 +7,10 @@ * Everything else renames. * * Weights stay in torch layout for the core Linear and Conv1d hooks to - * transpose. An F16 checkpoint gives f16 derived tensors. Derived byteLengths - * carry the source file bytes, so load progress still sums to the checkpoint - * size. */ + * transpose. An F16 checkpoint gives f16 derived tensors, except the scaled + * cross-attention q, which stays f32 so an f32 model loads exact values. + * Derived byteLengths carry the source file bytes, so load progress still + * sums to the checkpoint size. */ import { concatRows, scaleWeight, type LazyStateDict, type LazyTensor } from '../../core/index.ts'; import { checkpointMismatch } from '../errors.ts'; diff --git a/packages/runntime/test/zoo/moonshine-streaming/stateDictHooks.test.ts b/packages/runntime/test/zoo/moonshine-streaming/stateDictHooks.test.ts index 3235fc33..8230dc04 100644 --- a/packages/runntime/test/zoo/moonshine-streaming/stateDictHooks.test.ts +++ b/packages/runntime/test/zoo/moonshine-streaming/stateDictHooks.test.ts @@ -115,11 +115,12 @@ describe('transformStreamingStateDict', () => { } // Encoder q|k|v with q times 1/√headDim = 0.5: the encoder has no rope. + // The scaled part makes it f32 on every checkpoint. const encQkv = get('encoder.layers.0.attn.qkv.weight'); expect(encQkv.shape).toEqual([6, 2]); expect(encQkv.byteLength).toBe(12 * elemBytes); expect(await contents(encQkv)).toEqual({ - type, + type: 'f32', values: [0.5, 1, 1.5, 2, 5, 6, 7, 8, 9, 10, 11, 12], }); @@ -132,11 +133,12 @@ describe('transformStreamingStateDict', () => { values: [-1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12], }); - // Cross-attention q times 1/√headDim = 0.5; k|v stacked as is. + // Cross-attention q times 1/√headDim = 0.5, f32 on every checkpoint; + // k|v stacked as is. const crossQ = get('decoder.layers.0.crossAttn.q.weight'); expect(crossQ.shape).toEqual([2, 2]); expect(crossQ.byteLength).toBe(4 * elemBytes); - expect(await contents(crossQ)).toEqual({ type, values: [0.5, 1.5, 2.5, 3.5] }); + expect(await contents(crossQ)).toEqual({ type: 'f32', values: [0.5, 1.5, 2.5, 3.5] }); const crossKv = get('decoder.layers.0.crossAttn.kv.weight'); expect(crossKv.shape).toEqual([4, 2]); expect(crossKv.byteLength).toBe(8 * elemBytes); diff --git a/packages/runntime/test/zoo/moonshine/stateDictHooks.test.ts b/packages/runntime/test/zoo/moonshine/stateDictHooks.test.ts index 714483cc..4e3bd167 100644 --- a/packages/runntime/test/zoo/moonshine/stateDictHooks.test.ts +++ b/packages/runntime/test/zoo/moonshine/stateDictHooks.test.ts @@ -92,11 +92,12 @@ describe('transformMoonshineStateDict', () => { values: [-1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12], }); - // Cross-attention q times 1/√headDim = 0.5; k|v stacked as is. + // Cross-attention q times 1/√headDim = 0.5, f32 on every checkpoint; + // k|v stacked as is. const crossQ = get('decoder.layers.0.crossAttn.q.weight'); expect(crossQ.shape).toEqual([2, 2]); expect(crossQ.byteLength).toBe(4 * elemBytes); - expect(await contents(crossQ)).toEqual({ type, values: [0.5, 1.5, 2.5, 3.5] }); + expect(await contents(crossQ)).toEqual({ type: 'f32', values: [0.5, 1.5, 2.5, 3.5] }); const crossKv = get('decoder.layers.0.crossAttn.kv.weight'); expect(crossKv.shape).toEqual([4, 2]); expect(crossKv.byteLength).toBe(8 * elemBytes); From 03b68cf2a66f9f02ce658c4c035117c897875a0e Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 28 Oct 2026 11:42:57 +0100 Subject: [PATCH 16/25] fix(zoo): check Moonshine q/k/v projection widths against the hidden size again --- .../zoo/moonshine-streaming/stateDictHooks.ts | 4 +- .../src/zoo/moonshine/stateDictHooks.ts | 38 +++++++++++++++---- .../stateDictHooks.test.ts | 35 +++++++++-------- .../test/zoo/moonshine/stateDictHooks.test.ts | 20 ++++++++-- 4 files changed, 69 insertions(+), 28 deletions(-) diff --git a/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts b/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts index b7d65e30..84e60747 100644 --- a/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts +++ b/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts @@ -76,7 +76,7 @@ export function transformStreamingStateDict( const hf = q(`model.encoder.layers.${i}`); const en = q(`encoder.layers.${i}`); gammaNorm(`${hf}.input_layernorm.gamma`, `${en}.norm1.weight`); - fuseQkv(sd, `${hf}.self_attn`, `${en}.attn.qkv.weight`, encQScale); + fuseQkv(sd, `${hf}.self_attn`, `${en}.attn.qkv.weight`, eh, encQScale); move(sd, `${hf}.self_attn.o_proj.weight`, `${en}.attn.out.weight`); gammaNorm(`${hf}.post_attention_layernorm.gamma`, `${en}.norm2.weight`); move(sd, `${hf}.mlp.fc1.weight`, `${en}.mlp.fc1.weight`); @@ -89,5 +89,5 @@ export function transformStreamingStateDict( move(sd, q('model.decoder.pos_emb.weight'), q('adapter.posEmb')); if (eh !== dh) move(sd, q('model.decoder.proj.weight'), q('adapter.proj.weight')); - transformDecoder(sd, prefix, { layers: cfg.dec.layers, headDim: cfg.dec.headDim }); + transformDecoder(sd, prefix, { layers: cfg.dec.layers, hidden: dh, headDim: cfg.dec.headDim }); } diff --git a/packages/runntime/src/zoo/moonshine/stateDictHooks.ts b/packages/runntime/src/zoo/moonshine/stateDictHooks.ts index f5d9984f..e8f1e99a 100644 --- a/packages/runntime/src/zoo/moonshine/stateDictHooks.ts +++ b/packages/runntime/src/zoo/moonshine/stateDictHooks.ts @@ -30,10 +30,28 @@ export function move(sd: LazyStateDict, from: string, to: string): void { sd.tensors.set(to, take(sd, from)); } +/** Takes an attention projection weight, which must be [_, hidden]. */ +function takeProjection(sd: LazyStateDict, name: string, hidden: number): LazyTensor { + const t = take(sd, name); + if (t.shape.length !== 2 || t.shape[1] !== hidden) { + throw checkpointMismatch( + 'moonshine state dict', + `'${name}' has shape [${t.shape.join(', ')}], expected [_, ${hidden}]`, + ); + } + return t; +} + /** Stacks q|k|v into one torch-layout [3·hidden, hidden] weight, with q times * `qScale` when given. */ -export function fuseQkv(sd: LazyStateDict, hfPrefix: string, ours: string, qScale?: number): void { - const part = (p: string) => take(sd, `${hfPrefix}.${p}_proj.weight`); +export function fuseQkv( + sd: LazyStateDict, + hfPrefix: string, + ours: string, + hidden: number, + qScale?: number, +): void { + const part = (p: string) => takeProjection(sd, `${hfPrefix}.${p}_proj.weight`, hidden); const query = part('q'); sd.tensors.set( ours, @@ -53,7 +71,7 @@ export function moveTiedEmbedding(sd: LazyStateDict, prefix: string): void { export function transformDecoder( sd: LazyStateDict, prefix: string, - dec: { layers: number; headDim: number }, + dec: { layers: number; hidden: number; headDim: number }, ): void { const q = (name: string) => qualify(prefix, name); const crossQScale = 1 / Math.sqrt(dec.headDim); @@ -61,16 +79,20 @@ export function transformDecoder( const hf = q(`model.decoder.layers.${i}`); const en = q(`decoder.layers.${i}`); move(sd, `${hf}.input_layernorm.weight`, `${en}.norm1.weight`); - fuseQkv(sd, `${hf}.self_attn`, `${en}.selfAttn.qkv.weight`); + fuseQkv(sd, `${hf}.self_attn`, `${en}.selfAttn.qkv.weight`, dec.hidden); move(sd, `${hf}.self_attn.o_proj.weight`, `${en}.selfAttn.out.weight`); move(sd, `${hf}.post_attention_layernorm.weight`, `${en}.norm2.weight`); sd.tensors.set( `${en}.crossAttn.q.weight`, - scaleWeight(take(sd, `${hf}.encoder_attn.q_proj.weight`), crossQScale), + scaleWeight(takeProjection(sd, `${hf}.encoder_attn.q_proj.weight`, dec.hidden), crossQScale), ); sd.tensors.set( `${en}.crossAttn.kv.weight`, - concatRows(['k', 'v'].map((p) => take(sd, `${hf}.encoder_attn.${p}_proj.weight`))), + concatRows( + ['k', 'v'].map((p) => + takeProjection(sd, `${hf}.encoder_attn.${p}_proj.weight`, dec.hidden), + ), + ), ); move(sd, `${hf}.encoder_attn.o_proj.weight`, `${en}.crossAttn.out.weight`); move(sd, `${hf}.final_layernorm.weight`, `${en}.norm3.weight`); @@ -105,7 +127,7 @@ export function transformMoonshineStateDict( const hf = q(`model.encoder.layers.${i}`); const en = q(`encoder.layers.${i}`); move(sd, `${hf}.input_layernorm.weight`, `${en}.norm1.weight`); - fuseQkv(sd, `${hf}.self_attn`, `${en}.attn.qkv.weight`); + fuseQkv(sd, `${hf}.self_attn`, `${en}.attn.qkv.weight`, cfg.dModel); move(sd, `${hf}.self_attn.o_proj.weight`, `${en}.attn.out.weight`); move(sd, `${hf}.post_attention_layernorm.weight`, `${en}.norm2.weight`); move(sd, `${hf}.mlp.fc1.weight`, `${en}.mlp.fc1.weight`); @@ -115,5 +137,5 @@ export function transformMoonshineStateDict( } move(sd, q('model.encoder.layer_norm.weight'), q('encoder.norm.weight')); - transformDecoder(sd, prefix, { layers: cfg.decLayers, headDim: cfg.headDim }); + transformDecoder(sd, prefix, { layers: cfg.decLayers, hidden: cfg.dModel, headDim: cfg.headDim }); } diff --git a/packages/runntime/test/zoo/moonshine-streaming/stateDictHooks.test.ts b/packages/runntime/test/zoo/moonshine-streaming/stateDictHooks.test.ts index 8230dc04..f41b5153 100644 --- a/packages/runntime/test/zoo/moonshine-streaming/stateDictHooks.test.ts +++ b/packages/runntime/test/zoo/moonshine-streaming/stateDictHooks.test.ts @@ -15,6 +15,8 @@ const CFG = { }; const square = (values: number[]): Entry => ({ values, shape: [2, 2] }); +/** A decoder projection: 2 rows of the decoder's hidden size 3. */ +const wide = (values: number[]): Entry => ({ values, shape: [2, 3] }); const ZERO = square([0, 0, 0, 0]); /** Tensors the hook only renames, HF name → ours. */ @@ -56,12 +58,12 @@ const CHECKPOINT: Record = { 'model.encoder.layers.0.self_attn.q_proj.weight': square([1, 2, 3, 4]), 'model.encoder.layers.0.self_attn.k_proj.weight': square([5, 6, 7, 8]), 'model.encoder.layers.0.self_attn.v_proj.weight': square([9, 10, 11, 12]), - 'model.decoder.layers.0.self_attn.q_proj.weight': square([-1, -2, -3, -4]), - 'model.decoder.layers.0.self_attn.k_proj.weight': square([-5, -6, -7, -8]), - 'model.decoder.layers.0.self_attn.v_proj.weight': square([-9, -10, -11, -12]), - 'model.decoder.layers.0.encoder_attn.q_proj.weight': square([1, 3, 5, 7]), - 'model.decoder.layers.0.encoder_attn.k_proj.weight': square([2, 4, 6, 8]), - 'model.decoder.layers.0.encoder_attn.v_proj.weight': square([10, 20, 30, 40]), + 'model.decoder.layers.0.self_attn.q_proj.weight': wide([-1, -2, -3, -4, -5, -6]), + 'model.decoder.layers.0.self_attn.k_proj.weight': wide([-7, -8, -9, -10, -11, -12]), + 'model.decoder.layers.0.self_attn.v_proj.weight': wide([-13, -14, -15, -16, -17, -18]), + 'model.decoder.layers.0.encoder_attn.q_proj.weight': wide([1, 3, 5, 7, 9, 11]), + 'model.decoder.layers.0.encoder_attn.k_proj.weight': wide([2, 4, 6, 8, 10, 12]), + 'model.decoder.layers.0.encoder_attn.v_proj.weight': wide([10, 20, 30, 40, 50, 60]), }; describe('transformStreamingStateDict', () => { @@ -126,23 +128,26 @@ describe('transformStreamingStateDict', () => { // Decoder q|k|v unscaled: its rope tables carry the scale. const decQkv = get('decoder.layers.0.selfAttn.qkv.weight'); - expect(decQkv.shape).toEqual([6, 2]); - expect(decQkv.byteLength).toBe(12 * elemBytes); + expect(decQkv.shape).toEqual([6, 3]); + expect(decQkv.byteLength).toBe(18 * elemBytes); expect(await contents(decQkv)).toEqual({ type, - values: [-1, -2, -3, -4, -5, -6, -7, -8, -9, -10, -11, -12], + values: Array.from({ length: 18 }, (_, i) => -(i + 1)), }); // Cross-attention q times 1/√headDim = 0.5, f32 on every checkpoint; // k|v stacked as is. const crossQ = get('decoder.layers.0.crossAttn.q.weight'); - expect(crossQ.shape).toEqual([2, 2]); - expect(crossQ.byteLength).toBe(4 * elemBytes); - expect(await contents(crossQ)).toEqual({ type: 'f32', values: [0.5, 1.5, 2.5, 3.5] }); + expect(crossQ.shape).toEqual([2, 3]); + expect(crossQ.byteLength).toBe(6 * elemBytes); + expect(await contents(crossQ)).toEqual({ type: 'f32', values: [0.5, 1.5, 2.5, 3.5, 4.5, 5.5] }); const crossKv = get('decoder.layers.0.crossAttn.kv.weight'); - expect(crossKv.shape).toEqual([4, 2]); - expect(crossKv.byteLength).toBe(8 * elemBytes); - expect(await contents(crossKv)).toEqual({ type, values: [2, 4, 6, 8, 10, 20, 30, 40] }); + expect(crossKv.shape).toEqual([4, 3]); + expect(crossKv.byteLength).toBe(12 * elemBytes); + expect(await contents(crossKv)).toEqual({ + type, + values: [2, 4, 6, 8, 10, 12, 10, 20, 30, 40, 50, 60], + }); // The dropped proj_out (4 elems) is the only byte loss. expect(sd.totalBytes()).toBe(total - 4 * elemBytes); diff --git a/packages/runntime/test/zoo/moonshine/stateDictHooks.test.ts b/packages/runntime/test/zoo/moonshine/stateDictHooks.test.ts index 4e3bd167..0673cb12 100644 --- a/packages/runntime/test/zoo/moonshine/stateDictHooks.test.ts +++ b/packages/runntime/test/zoo/moonshine/stateDictHooks.test.ts @@ -3,9 +3,9 @@ import { MOONSHINE_TINY } from '../../../src/zoo/moonshine/config.ts'; import { transformMoonshineStateDict } from '../../../src/zoo/moonshine/stateDictHooks.ts'; import { checkpoint, contents, type Entry } from '../../core/weights/checkpoint.ts'; -// The hook reads only encLayers, decLayers and headDim. headDim 4 makes the -// cross-attention q scale 1/√4 = 0.5, exact in f16. -const CFG = { ...MOONSHINE_TINY, encLayers: 1, decLayers: 1, headDim: 4 }; +// The hook reads only dModel, encLayers, decLayers and headDim. headDim 4 +// makes the cross-attention q scale 1/√4 = 0.5, exact in f16. +const CFG = { ...MOONSHINE_TINY, dModel: 2, encLayers: 1, decLayers: 1, headDim: 4 }; const square = (values: number[]): Entry => ({ values, shape: [2, 2] }); const ZERO = square([0, 0, 0, 0]); @@ -106,4 +106,18 @@ describe('transformMoonshineStateDict', () => { // The dropped proj_out (4 elems) is the only byte loss. expect(sd.totalBytes()).toBe(total - 4 * elemBytes); }); + + it('rejects a projection whose width is not dModel', async () => { + const name = 'model.encoder.layers.0.self_attn.k_proj.weight'; + const sd = await checkpoint('F32', { + ...CHECKPOINT, + [name]: { values: [1, 2, 3], shape: [1, 3] }, + }); + expect(() => transformMoonshineStateDict(sd, '', CFG)).toThrow( + expect.objectContaining({ + code: 'CHECKPOINT_MISMATCH', + message: expect.stringContaining(name), + }), + ); + }); }); From 97c04ff169544de2332a7977ea07d52c8238b236 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 28 Oct 2026 11:43:17 +0100 Subject: [PATCH 17/25] docs(core): say why reshapeWeight can shallow-copy a LazyTensor --- packages/runntime/src/core/weights/transforms.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/runntime/src/core/weights/transforms.ts b/packages/runntime/src/core/weights/transforms.ts index e7064ffb..007f94cc 100644 --- a/packages/runntime/src/core/weights/transforms.ts +++ b/packages/runntime/src/core/weights/transforms.ts @@ -199,5 +199,7 @@ export function reshapeWeight(t: LazyTensor, shape: readonly number[]): LazyTens `cannot reshape [${t.shape.join(', ')}] to [${shape.join(', ')}]`, ); } + // Every LazyTensor is a plain object whose readers close over their own + // source, so a shallow copy shares the data and reads it the same way. return { ...t, shape: [...shape] }; } From 52294eb50949cc19ceb309832346463fc72e6329 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 28 Oct 2026 11:44:13 +0100 Subject: [PATCH 18/25] test(core): build every synthetic safetensors file through one shared test builder --- packages/runntime/test/core/nn/layers.test.ts | 21 ++++------ .../test/core/nn/loadStateDict.test.ts | 10 +---- .../runntime/test/core/weights/checkpoint.ts | 42 ++++++++++++++----- 3 files changed, 40 insertions(+), 33 deletions(-) diff --git a/packages/runntime/test/core/nn/layers.test.ts b/packages/runntime/test/core/nn/layers.test.ts index db8a4f11..f1a31d74 100644 --- a/packages/runntime/test/core/nn/layers.test.ts +++ b/packages/runntime/test/core/nn/layers.test.ts @@ -25,6 +25,7 @@ import { type LazyStateDict, } from '../../../src/core/weights/safetensors.ts'; import { hwc4Input, input, meta } from '../graph/values.ts'; +import { safetensorsFile } from '../weights/checkpoint.ts'; /** Binds every parameter to a stand-in of its declared shape, so forward() * can build its graph without real weights. */ @@ -37,20 +38,12 @@ function fakeLoad(m: M): M { const shapes = (m: AnyModule) => Object.fromEntries([...m.namedParameters()].map(([n, p]) => [n, p.shape])); -/** A state dict with one F16 tensor named `weight`, read from a real - * safetensors file built from the format spec: an 8-byte little-endian - * header length, the JSON header, then the payload. memoryStateDict reads a - * Uint16Array as BF16, so F16 needs the file. */ -function f16Checkpoint(bits: Uint16Array, shape: number[]): Promise { - const header = new TextEncoder().encode( - JSON.stringify({ weight: { dtype: 'F16', shape, data_offsets: [0, bits.byteLength] } }), - ); - const file = new Uint8Array(8 + header.length + bits.byteLength); - new DataView(file.buffer).setBigUint64(0, BigInt(header.length), true); - file.set(header, 8); - file.set(new Uint8Array(bits.buffer, bits.byteOffset, bits.byteLength), 8 + header.length); - return fromSafetensors(bufferSource(file), { chunkBytes: 0 }); -} +/** A state dict with one F16 tensor named `weight`, read from a real file: + * memoryStateDict reads a Uint16Array as BF16. */ +const f16Checkpoint = (bits: Uint16Array, shape: number[]): Promise => + fromSafetensors(bufferSource(safetensorsFile({ weight: { data: bits, shape } })), { + chunkBytes: 0, + }); /** A state dict whose `weight` is a derived tensor holding `data`, as a zoo * hook hands one over: a Uint16Array means f16 halfwords, a Uint32Array diff --git a/packages/runntime/test/core/nn/loadStateDict.test.ts b/packages/runntime/test/core/nn/loadStateDict.test.ts index 7a85119b..13d03b54 100644 --- a/packages/runntime/test/core/nn/loadStateDict.test.ts +++ b/packages/runntime/test/core/nn/loadStateDict.test.ts @@ -19,6 +19,7 @@ import { type MemoryStateDict, } from '../../../src/core/weights/safetensors.ts'; import { meta } from '../graph/values.ts'; +import { safetensorsFile } from '../weights/checkpoint.ts'; /** Records every upload and hands back a stand-in of the slot's shape. */ const recorder = () => @@ -250,14 +251,7 @@ describe('materializing an f16 slot', () => { }; it('narrows f32 from a file', async () => { - // A safetensors file built from the spec: 8-byte header length, JSON, payload. - const header = new TextEncoder().encode( - JSON.stringify({ w: { dtype: 'F32', shape: [2], data_offsets: [0, 8] } }), - ); - const file = new Uint8Array(8 + header.length + 8); - new DataView(file.buffer).setBigUint64(0, BigInt(header.length), true); - file.set(header, 8); - file.set(new Uint8Array(new Float32Array([1, 2]).buffer), 8 + header.length); + const file = safetensorsFile({ w: { data: new Float32Array([1, 2]), shape: [2] } }); expect(await bits(await fromSafetensors(bufferSource(file)))).toEqual( new Uint16Array([F16[1], F16[2]]), ); diff --git a/packages/runntime/test/core/weights/checkpoint.ts b/packages/runntime/test/core/weights/checkpoint.ts index 1d5f77d7..7acd60e3 100644 --- a/packages/runntime/test/core/weights/checkpoint.ts +++ b/packages/runntime/test/core/weights/checkpoint.ts @@ -10,25 +10,45 @@ import { export type Entry = { values: number[]; shape: number[] }; -/** An F16 safetensors file, built from the format spec: an 8-byte - * little-endian header length, the JSON header, then the payload. */ -export function f16File(entries: Record): Uint8Array { +/** A safetensors file built from the format spec: an 8-byte little-endian + * header length, the JSON header, then the payload. An entry's dtype follows + * its data: F32 for a Float32Array, F16 for a Uint16Array of halfwords. */ +export function safetensorsFile( + entries: Record, +): Uint8Array { const header: Record = {}; - const halves: number[] = []; - for (const [name, { values, shape }] of Object.entries(entries)) { - const begin = halves.length * 2; - halves.push(...values.map(f32ToF16Bits)); - header[name] = { dtype: 'F16', shape, data_offsets: [begin, halves.length * 2] }; + const chunks: Uint8Array[] = []; + let offset = 0; + for (const [name, { data, shape }] of Object.entries(entries)) { + const bytes = new Uint8Array(data.buffer, data.byteOffset, data.byteLength); + const dtype = data instanceof Uint16Array ? 'F16' : 'F32'; + header[name] = { dtype, shape, data_offsets: [offset, offset + bytes.length] }; + chunks.push(bytes); + offset += bytes.length; } const json = new TextEncoder().encode(JSON.stringify(header)); - const payload = new Uint8Array(new Uint16Array(halves).buffer); - const out = new Uint8Array(8 + json.length + payload.length); + const out = new Uint8Array(8 + json.length + offset); new DataView(out.buffer).setBigUint64(0, BigInt(json.length), true); out.set(json, 8); - out.set(payload, 8 + json.length); + let at = 8 + json.length; + for (const bytes of chunks) { + out.set(bytes, at); + at += bytes.length; + } return out; } +/** An F16 safetensors file of `entries`, each value rounded to f16. */ +export const f16File = (entries: Record): Uint8Array => + safetensorsFile( + Object.fromEntries( + Object.entries(entries).map(([name, { values, shape }]) => [ + name, + { data: Uint16Array.from(values, f32ToF16Bits), shape }, + ]), + ), + ); + /** `entries` as an F16 safetensors file, or as f32 in memory. memoryStateDict * reads a Uint16Array as BF16, so F16 has to go through a real file. */ export async function checkpoint( From efd206588c3091ff63049991f521d8de598e0af7 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 28 Oct 2026 07:59:23 +0100 Subject: [PATCH 19/25] feat(core): add permuteRows to the weight helpers to reorder dim-0 rows lazily --- packages/runntime/src/core/index.ts | 1 + .../runntime/src/core/weights/transforms.ts | 31 +++++++++++- .../test/core/weights/transforms.test.ts | 49 +++++++++++++++++++ 3 files changed, 80 insertions(+), 1 deletion(-) diff --git a/packages/runntime/src/core/index.ts b/packages/runntime/src/core/index.ts index 0e477ea8..3e6059d9 100644 --- a/packages/runntime/src/core/index.ts +++ b/packages/runntime/src/core/index.ts @@ -139,6 +139,7 @@ export { createOpfsCache } from './weights/opfsCache.ts'; export { foldBnIntoConv } from './weights/foldBn.ts'; export { concatRows, + permuteRows, reshapeWeight, scaleWeight, splitCols, diff --git a/packages/runntime/src/core/weights/transforms.ts b/packages/runntime/src/core/weights/transforms.ts index 007f94cc..b2c956c9 100644 --- a/packages/runntime/src/core/weights/transforms.ts +++ b/packages/runntime/src/core/weights/transforms.ts @@ -1,5 +1,6 @@ /** Lazy reshaping helpers for `transformStateDict` hooks: concatenate, split, - * scale, transpose and reshape checkpoint tensors without fetching them. + * permute, scale, transpose and reshape checkpoint tensors without fetching + * them. * * F16 sources give f16 halfwords and F32 sources give f32, except * `scaleWeight`, which always gives f32. BF16 decodes to @@ -141,6 +142,34 @@ export function splitCols(t: LazyTensor, sizes: readonly number[]): LazyTensor[] }); } +/** Reorders the rows of dim 0: row i of the result is row `order[i]` of `t`, + * like torch `t[order]`. Throws CHECKPOINT_MISMATCH when `order` is not a + * permutation of 0 … rows − 1. */ +export function permuteRows(t: LazyTensor, order: readonly number[]): LazyTensor { + if (t.shape.length === 0) throw mismatch('permuteRows', 'cannot permute a scalar'); + const rows = t.shape[0]!; + const inRange = order.every((r) => Number.isInteger(r) && r >= 0 && r < rows); + if (order.length !== rows || new Set(order).size !== rows || !inRange) { + throw mismatch( + 'permuteRows', + `order is not a permutation of the ${rows} rows of shape [${t.shape.join(', ')}]`, + ); + } + const rowElems = numel(t.shape) / rows; + return derivedTensor( + t.shape, + async () => { + const data = await readFloats('permuteRows', t); + const out = allocLike(data, data.length); + for (const [i, r] of order.entries()) { + out.set(data.subarray(r * rowElems, (r + 1) * rowElems), i * rowElems); + } + return out; + }, + t.byteLength, + ); +} + /** Multiplies `t` by `scale`: one number for every element, or one per row * of dim 0. The result is f32 even for an F16 source: rounding the product * back to f16 would change what an f32 model loads. */ diff --git a/packages/runntime/test/core/weights/transforms.test.ts b/packages/runntime/test/core/weights/transforms.test.ts index 7af5ffcf..b3fc4dff 100644 --- a/packages/runntime/test/core/weights/transforms.test.ts +++ b/packages/runntime/test/core/weights/transforms.test.ts @@ -8,6 +8,7 @@ import { } from '../../../src/core/weights/safetensors.ts'; import { concatRows, + permuteRows, reshapeWeight, scaleWeight, splitCols, @@ -150,6 +151,52 @@ describe('splitCols', () => { }); }); +describe('permuteRows', () => { + // [[1, 2], [3, 4], [5, 6]] with order [2, 0, 1] → [[5, 6], [1, 2], [3, 4]] + it.for(DTYPES)('reorders the rows of $dtype', async ({ dtype, type }) => { + const t = await source(dtype, [1, 2, 3, 4, 5, 6], [3, 2]); + const out = permuteRows(t, [2, 0, 1]); + expect(out.shape).toEqual([3, 2]); + expect(out.byteLength).toBe(t.byteLength); + expect(await contents(out)).toEqual({ type, values: [5, 6, 1, 2, 3, 4] }); + }); + + it('moves whole blocks of a 3D tensor and single elements of a 1D one', async () => { + const blocks = permuteRows(await source('F32', [1, 2, 3, 4], [2, 1, 2]), [1, 0]); + expect(blocks.shape).toEqual([2, 1, 2]); + expect(await contents(blocks)).toEqual({ type: 'f32', values: [3, 4, 1, 2] }); + const elems = permuteRows(await source('F16', [1, 2, 3, 4], [4]), [0, 2, 1, 3]); + expect(await contents(elems)).toEqual({ type: 'f16', values: [1, 3, 2, 4] }); + }); + + it.for([ + { name: 'too few rows', order: [0, 1] }, + { name: 'too many rows', order: [0, 1, 2, 0] }, + { name: 'a repeated row', order: [0, 0, 1] }, + { name: 'a row past the end', order: [0, 1, 3] }, + { name: 'a negative row', order: [0, 1, -1] }, + { name: 'a fractional row', order: [0, 1, 1.5] }, + ])('throws CHECKPOINT_MISMATCH for $name', async ({ order }) => { + const t = await source('F32', [1, 2, 3, 4, 5, 6], [3, 2]); + expect(() => permuteRows(t, order)).toThrow( + expect.objectContaining({ + code: 'CHECKPOINT_MISMATCH', + message: 'permuteRows: order is not a permutation of the 3 rows of shape [3, 2]', + }), + ); + }); + + it('throws CHECKPOINT_MISMATCH for a scalar', async () => { + const t = await source('F32', [1], []); + expect(() => permuteRows(t, [])).toThrow( + expect.objectContaining({ + code: 'CHECKPOINT_MISMATCH', + message: 'permuteRows: cannot permute a scalar', + }), + ); + }); +}); + describe('scaleWeight', () => { // 3 · 0.1 = 0.3, which f16 would round to 0.300048828125. The product is // f32 for every source, so an f32 model loads what it did before the helper. @@ -241,6 +288,7 @@ describe('weight helpers', () => { const [top, bottom] = splitRows(t, [1, 1]); const [left, right] = splitCols(t, [1, 1]); concatRows([t, t]); + permuteRows(t, [1, 0]); scaleWeight(t, 2); transposeWeight(t); reshapeWeight(t, [4]); @@ -271,6 +319,7 @@ describe('weight helpers', () => { it.for([ { name: 'concatRows', run: (t: LazyTensor) => concatRows([t]) }, { name: 'splitRows', run: (t: LazyTensor) => splitRows(t, [1])[0]! }, + { name: 'permuteRows', run: (t: LazyTensor) => permuteRows(t, [0]) }, { name: 'scaleWeight', run: (t: LazyTensor) => scaleWeight(t, 2) }, ])('$name rejects packed u32 words with CHECKPOINT_MISMATCH', async ({ name, run }) => { const words = derivedTensor([1], async () => new Uint32Array([1])); From 019b4412b1f5c66d70be884fc098896e445aada9 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 28 Oct 2026 08:03:53 +0100 Subject: [PATCH 20/25] refactor(zoo): map the EmbeddingGemma 2 checkpoint with the shared weight helpers and keep f16 in f16 --- .../src/zoo/embeddinggemma2/stateDictHooks.ts | 193 +++++++----------- 1 file changed, 71 insertions(+), 122 deletions(-) diff --git a/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts b/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts index 1e008616..5b52d057 100644 --- a/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts +++ b/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts @@ -1,9 +1,24 @@ /** Maps the HF EmbeddingGemma2Model checkpoint onto the text model and the * vision and audio towers as lazy derived tensors; nothing is fetched here. * Each keeps only its own tensors, so one checkpoint with every tower loads - * into any of them. */ + * into any of them. + * + * An F16 checkpoint gives f16 derived tensors, except the text layer + * scalars and the audio tower's clip bounds, distance bias rows and q + * scales, which the CPU code and the attention kernel read as f32. */ -import { derivedTensor, type LazyStateDict, type LazyTensor } from '../../core/index.ts'; +import { + concatRows, + derivedTensor, + f32ArrayToF16Bits, + permuteRows, + reshapeWeight, + scaleWeight, + splitRows, + transposeWeight, + type LazyStateDict, + type LazyTensor, +} from '../../core/index.ts'; import { checkpointMismatch } from '../errors.ts'; import type { EmbeddingGemma2AudioConfig, @@ -15,16 +30,14 @@ const SOURCE = 'language_model.'; const VISION_SOURCES = ['vision_tower.', 'embed_vision.'] as const; const AUDIO_SOURCES = ['audio_tower.', 'embed_audio.'] as const; -/** Row r of each d-row head moves so (p, p + d/2) become (2p, 2p + 1). */ -function interleaveHeads(src: Float32Array, rows: number, cols: number, d: number): Float32Array { - const out = new Float32Array(src.length); - for (let r = 0; r < rows; r++) { - const head = r - (r % d); - const i = r % d; - const to = head + (i < d / 2 ? 2 * i : 2 * (i - d / 2) + 1); - out.set(src.subarray(r * cols, (r + 1) * cols), to * cols); - } - return out; +/** Row order for permuteRows that takes each `block` rows from the + * checkpoint's half-split rope layout to the kernel's interleaved pairs: + * rows (p, p + block/2) become (2p, 2p + 1). */ +function ropeOrder(rows: number, block: number): number[] { + return Array.from({ length: rows }, (_, r) => { + const i = r % block; + return r - i + (i % 2 ? (i - 1) / 2 + block / 2 : i / 2); + }); } /** Each layer's RMSNorms, renamed `weight` → `scale`. */ @@ -60,34 +73,17 @@ function remap(sd: LazyStateDict, prefix: string, sources: readonly string[], wh }; } -/** A weightless norm's ones. */ +/** A weightless norm's ones, as f16 halfwords: exact, and stacked with an F16 + * checkpoint's norms they keep the result f16. */ const ones = (n: number) => - derivedTensor([n], () => Promise.resolve(new Float32Array(n).fill(1)), 0); + derivedTensor([n], () => Promise.resolve(f32ArrayToF16Bits(new Float32Array(n).fill(1)).data)); /** q, k and v in one weight. q and k rows go from the checkpoint's * half-split rope layout to the kernel's interleaved pairs, per `block` * rows. */ -function fusedQkv( - wq: LazyTensor, - wk: LazyTensor, - wv: LazyTensor, - qRows: number, - kvRows: number, - hidden: number, - block: number, -): LazyTensor { - const width = qRows + 2 * kvRows; - return derivedTensor( - [width, hidden], - async () => { - const out = new Float32Array(width * hidden); - out.set(interleaveHeads(await wq.f32(), qRows, hidden, block), 0); - out.set(interleaveHeads(await wk.f32(), kvRows, hidden, block), qRows * hidden); - out.set(await wv.f32(), (qRows + kvRows) * hidden); - return out; - }, - wq.byteLength + wk.byteLength + wv.byteLength, - ); +function fusedQkv(wq: LazyTensor, wk: LazyTensor, wv: LazyTensor, block: number): LazyTensor { + const rope = (w: LazyTensor) => permuteRows(w, ropeOrder(w.shape[0]!, block)); + return concatRows([rope(wq), rope(wk), wv]); } /** q_norm and k_norm tiled over their heads and interleaved like @@ -97,22 +93,19 @@ function fusedQkvNorm( nk: LazyTensor, heads: number, kvHeads: number, - d: number, block: number, ): LazyTensor { - const width = (heads + 2 * kvHeads) * d; - return derivedTensor( - [width], - async () => { - const out = new Float32Array(width).fill(1); - const qn = interleaveHeads(await nq.f32(), d, 1, block); - const kn = interleaveHeads(await nk.f32(), d, 1, block); - for (let h = 0; h < heads; h++) out.set(qn, h * d); - for (let h = 0; h < kvHeads; h++) out.set(kn, (heads + h) * d); - return out; - }, - nq.byteLength + nk.byteLength, - ); + const d = nq.shape[0]!; + const order = ropeOrder(d, block); + const qn = permuteRows(nq, order); + const kn = permuteRows(nk, order); + const tiled = concatRows([ + ...Array(heads).fill(qn), + ...Array(kvHeads).fill(kn), + ones(kvHeads * d), + ]); + // Each norm is downloaded once, however many heads it is tiled over. + return { ...tiled, byteLength: nq.byteLength + nk.byteLength }; } /** The text model's mapping. PLE's hidden^-½ folds into its projection. */ @@ -129,19 +122,8 @@ export function transformEmbeddingGemma2StateDict( const L = `layers.${i}`; const { headDim: d, kvHeads } = cfg.attention[type]; const attn = (n: string) => take(`${L}.self_attn.${n}.weight`); - put( - `${L}.qkv.weight`, - fusedQkv( - attn('q_proj'), - attn('k_proj'), - attn('v_proj'), - cfg.heads * d, - kvHeads * d, - cfg.hidden, - d, - ), - ); - put(`${L}.qkv_norm`, fusedQkvNorm(attn('q_norm'), attn('k_norm'), cfg.heads, kvHeads, d, d)); + put(`${L}.qkv.weight`, fusedQkv(attn('q_proj'), attn('k_proj'), attn('v_proj'), d)); + put(`${L}.qkv_norm`, fusedQkvNorm(attn('q_norm'), attn('k_norm'), cfg.heads, kvHeads, d)); put(`${L}.o_proj.weight`, attn('o_proj')); for (const n of ['gate_proj', 'up_proj', 'down_proj']) { put(`${L}.${n}.weight`, take(`${L}.mlp.${n}.weight`)); @@ -161,11 +143,9 @@ export function transformEmbeddingGemma2StateDict( derivedTensor([1], () => s.f32(), s.byteLength), ); } - const p = take('ple.per_layer_model_projection.weight'); - const scale = cfg.hidden ** -0.5; put( 'ple.per_layer_model_projection.weight', - derivedTensor(p.shape, async () => (await p.f32()).map((v) => v * scale), p.byteLength), + scaleWeight(take('ple.per_layer_model_projection.weight'), cfg.hidden ** -0.5), ); put('ple.per_layer_projection_norm.scale', take('ple.per_layer_projection_norm.weight')); put('embed_tokens.weight', take('embed_tokens.weight')); @@ -191,30 +171,18 @@ export function transformEmbeddingGemma2VisionStateDict( const { hidden, heads, headDim: d, positions } = cfg; put('input_proj.weight', take('vision_tower.patch_embedder.input_proj.weight')); - const table = take('vision_tower.patch_embedder.position_embedding_table'); - // [2, positions, hidden], read once for both axes. - let both: Promise | undefined; - const axis = (i: number) => - derivedTensor( - [positions, hidden], - async () => - (await (both ??= table.f32())).slice(i * positions * hidden, (i + 1) * positions * hidden), - table.byteLength / 2, - ); - put('pos_x.weight', axis(0)); - put('pos_y.weight', axis(1)); + // [2, positions, hidden]: the x table, then the y table. + const [x, y] = splitRows(take('vision_tower.patch_embedder.position_embedding_table'), [1, 1]); + put('pos_x.weight', reshapeWeight(x!, [positions, hidden])); + put('pos_y.weight', reshapeWeight(y!, [positions, hidden])); for (let i = 0; i < cfg.layers; i++) { const S = `vision_tower.encoder.layers.${i}`; const L = `layers.${i}`; const attn = (n: string) => take(`${S}.self_attn.${n}.weight`); const proj = (n: string) => take(`${S}.self_attn.${n}.linear.weight`); - const rows = heads * d; - put( - `${L}.qkv.weight`, - fusedQkv(proj('q_proj'), proj('k_proj'), proj('v_proj'), rows, rows, hidden, d / 2), - ); - put(`${L}.qkv_norm`, fusedQkvNorm(attn('q_norm'), attn('k_norm'), heads, heads, d, d / 2)); + put(`${L}.qkv.weight`, fusedQkv(proj('q_proj'), proj('k_proj'), proj('v_proj'), d / 2)); + put(`${L}.qkv_norm`, fusedQkvNorm(attn('q_norm'), attn('k_norm'), heads, heads, d / 2)); put(`${L}.o_proj.weight`, proj('o_proj')); for (const n of ['gate_proj', 'up_proj', 'down_proj']) { put(`${L}.${n}.weight`, take(`${S}.mlp.${n}.linear.weight`)); @@ -250,26 +218,19 @@ export function transformEmbeddingGemma2AudioStateDict( put(`${to}.weight`, take(`${from}.linear.weight`)); put(`${to}.clip`, clip(bounds(from))); }; - const reshaped = (t: LazyTensor, shape: number[], map?: (src: Float32Array) => Float32Array) => - derivedTensor(shape, async () => (map ? map(await t.f32()) : t.f32()), t.byteLength); // The first conv has one input channel: [c0, 9] as a linear over 3×3 - // taps. The second reads its taps (row, column, channel) major. + // taps. The second reads its taps (row, column, channel) major, so each + // output's [c0, 9] block becomes [9, c0], one element at a time. const sub = `${S}subsample_conv_projection.`; - put('conv0.weight', reshaped(take(`${sub}layer0.conv.weight`), [c0, 9])); + put('conv0.weight', reshapeWeight(take(`${sub}layer0.conv.weight`), [c0, 9])); put('norm0.weight', take(`${sub}layer0.norm.weight`)); - put( - 'conv1.weight', - reshaped(take(`${sub}layer1.conv.weight`), [c1, 9 * c0], (w) => { - const out = new Float32Array(w.length); - for (let o = 0; o < c1; o++) { - for (let c = 0; c < c0; c++) { - for (let k = 0; k < 9; k++) out[(o * 9 + k) * c0 + c] = w[(o * c0 + c) * 9 + k]!; - } - } - return out; - }), - ); + const tapsMajor = Array.from({ length: c1 * 9 * c0 }, (_, j) => { + const [o, k, c] = [Math.floor(j / (9 * c0)), Math.floor(j / c0) % 9, j % c0]; + return (o * c0 + c) * 9 + k; + }); + const conv1 = reshapeWeight(take(`${sub}layer1.conv.weight`), [c1 * c0 * 9]); + put('conv1.weight', reshapeWeight(permuteRows(conv1, tapsMajor), [c1, 9 * c0])); put('norm1.weight', take(`${sub}layer1.norm.weight`)); put('input_proj.weight', take(`${sub}input_proj_linear.weight`)); @@ -295,9 +256,7 @@ export function transformEmbeddingGemma2AudioStateDict( clipped(`${T}.${ff}.ffw_layer_2`, `${L}.${ff}.ffw_layer_2`); put( `${L}.${ff}.post_layer_norm.scale`, - reshaped(take(`${T}.${ff}.post_layer_norm.weight`), [hidden], (w) => - w.map((v) => v * cfg.residualWeight), - ), + scaleWeight(take(`${T}.${ff}.post_layer_norm.weight`), cfg.residualWeight), ); } @@ -306,18 +265,7 @@ export function transformEmbeddingGemma2AudioStateDict( w: take(`${A}.${n}.linear.weight`), b: bounds(`${A}.${n}`), })); - put( - `${L}.qkv.weight`, - derivedTensor( - [3 * hidden, hidden], - async () => { - const out = new Float32Array(3 * hidden * hidden); - for (const [n, t] of qkv.entries()) out.set(await t.w.f32(), n * hidden * hidden); - return out; - }, - qkv.reduce((s, t) => s + t.w.byteLength, 0), - ), - ); + put(`${L}.qkv.weight`, concatRows(qkv.map((t) => t.w))); put( `${L}.qkv.clip`, derivedTensor([4], async () => { @@ -329,6 +277,8 @@ export function transformEmbeddingGemma2AudioStateDict( }), ); clipped(`${A}.post`, `${L}.post`); + // rel and q_scale stay f32 on purpose: the attention kernel reads them as + // f32, so rounding them to f16 would only lose precision. const rk = take(`${A}.relative_k_proj.weight`); put( `${L}.rel`, @@ -349,24 +299,23 @@ export function transformEmbeddingGemma2AudioStateDict( rk.byteLength, ), ); + const dimScale = take(`${A}.per_dim_scale`); put( `${L}.q_scale`, - reshaped(take(`${A}.per_dim_scale`), [headDim], (s) => - s.map((x) => qScale * Math.log1p(Math.exp(x))), + derivedTensor( + [headDim], + async () => (await dimScale.f32()).map((x) => qScale * Math.log1p(Math.exp(x))), + dimScale.byteLength, ), ); const C = `${T}.lconv1d`; put(`${L}.lconv.pre_layer_norm.scale`, take(`${C}.pre_layer_norm.weight`)); clipped(`${C}.linear_start`, `${L}.lconv.linear_start`); + // [hidden, 1, K] → [K, hidden]. put( `${L}.lconv.conv`, - reshaped(take(`${C}.depthwise_conv1d.weight`), [K, hidden], (w) => { - const out = new Float32Array(w.length); - for (let c = 0; c < hidden; c++) - for (let t = 0; t < K; t++) out[t * hidden + c] = w[c * K + t]!; - return out; - }), + transposeWeight(reshapeWeight(take(`${C}.depthwise_conv1d.weight`), [hidden, K])), ); put(`${L}.lconv.conv_norm.scale`, take(`${C}.conv_norm.weight`)); clipped(`${C}.linear_end`, `${L}.lconv.linear_end`); From 0445c1a523ef5d33acde05edc98d0c050837fbf1 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 28 Oct 2026 08:03:59 +0100 Subject: [PATCH 21/25] test(zoo): cover the EmbeddingGemma 2 state dict hooks on F32 and F16 checkpoints --- .../embeddinggemma2/stateDictHooks.test.ts | 432 ++++++++++++++++++ 1 file changed, 432 insertions(+) create mode 100644 packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts diff --git a/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts b/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts new file mode 100644 index 00000000..43336b7b --- /dev/null +++ b/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts @@ -0,0 +1,432 @@ +import { describe, expect, it } from 'vitest'; +import type { LazyStateDict } from '../../../src/core/weights/safetensors.ts'; +import { + EMBEDDINGGEMMA2_AUDIO, + EMBEDDINGGEMMA2_TEXT, + EMBEDDINGGEMMA2_VISION, + type EmbeddingGemma2AudioConfig, + type EmbeddingGemma2Config, + type EmbeddingGemma2VisionConfig, +} from '../../../src/zoo/embeddinggemma2/config.ts'; +import { + transformEmbeddingGemma2AudioStateDict, + transformEmbeddingGemma2StateDict, + transformEmbeddingGemma2VisionStateDict, +} from '../../../src/zoo/embeddinggemma2/stateDictHooks.ts'; +import { checkpoint, contents, type Entry } from '../../core/weights/checkpoint.ts'; + +const DTYPES = [ + { dtype: 'F32', type: 'f32', elemBytes: 4 }, + { dtype: 'F16', type: 'f16', elemBytes: 2 }, +] as const; + +const ZERO: Entry = { values: [0], shape: [1] }; + +/** `rows` rows of `cols`: row r holds base + 10r + c. Exact in f16. */ +const grid = (base: number, rows: number, cols: number): Entry => ({ + values: Array.from( + { length: rows * cols }, + (_, i) => base + 10 * Math.floor(i / cols) + (i % cols), + ), + shape: [rows, cols], +}); + +/** The rows of `grid(base, …, cols)` listed in `order`. */ +const gridRows = (base: number, order: number[], cols: number) => + order.flatMap((r) => Array.from({ length: cols }, (_, c) => base + 10 * r + c)); + +const ones = (n: number) => new Array(n).fill(1); + +/** Maps `sd` and returns a getter over the result, after checking that the + * result holds exactly `renamed`'s targets plus `mapped`, with each renamed + * tensor passed through untouched. */ +function mapAndCheck( + sd: LazyStateDict, + run: (sd: LazyStateDict) => void, + renamed: Record, + mapped: string[], +) { + const before = new Map(sd.tensors); + run(sd); + expect([...sd.tensors.keys()].sort()).toEqual([...Object.values(renamed), ...mapped].sort()); + for (const [hf, ours] of Object.entries(renamed)) { + expect(sd.tensors.get(ours)).toBe(before.get(hf)); + } + return (name: string) => sd.tensors.get(name)!; +} + +describe('transformEmbeddingGemma2StateDict', () => { + // One sliding layer: hidden 4, 2 q heads and 1 kv head of size 4. + // hidden^-½ = 0.5, so the PLE scale is exact in f16. + const cfg: EmbeddingGemma2Config = { + ...EMBEDDINGGEMMA2_TEXT, + hidden: 4, + heads: 2, + layerTypes: ['sliding'], + attention: { + sliding: { headDim: 4, kvHeads: 1, theta: 10000 }, + full: { headDim: 8, kvHeads: 1, theta: 10000 }, + }, + }; + + const S = 'language_model.'; + const L = 'language_model.layers.0'; + const RENAMED: Record = { + [`${L}.self_attn.o_proj.weight`]: 'layers.0.o_proj.weight', + [`${L}.mlp.gate_proj.weight`]: 'layers.0.gate_proj.weight', + [`${L}.mlp.up_proj.weight`]: 'layers.0.up_proj.weight', + [`${L}.mlp.down_proj.weight`]: 'layers.0.down_proj.weight', + [`${L}.ple_block.per_layer_input_gate.weight`]: 'layers.0.per_layer_input_gate.weight', + [`${L}.ple_block.per_layer_projection.weight`]: 'layers.0.per_layer_projection.weight', + [`${L}.ple_block.post_per_layer_input_norm.weight`]: 'layers.0.post_per_layer_input_norm.scale', + [`${L}.input_layernorm.weight`]: 'layers.0.input_layernorm.scale', + [`${L}.post_attention_layernorm.weight`]: 'layers.0.post_attention_layernorm.scale', + [`${L}.pre_feedforward_layernorm.weight`]: 'layers.0.pre_feedforward_layernorm.scale', + [`${L}.post_feedforward_layernorm.weight`]: 'layers.0.post_feedforward_layernorm.scale', + [`${S}ple.per_layer_projection_norm.weight`]: 'ple.per_layer_projection_norm.scale', + [`${S}embed_tokens.weight`]: 'embed_tokens.weight', + [`${S}norm.weight`]: 'norm.scale', + [`${S}embedding_projection.weight`]: 'embedding_projection.weight', + }; + const CHECKPOINT: Record = { + ...Object.fromEntries(Object.keys(RENAMED).map((name) => [name, ZERO])), + [`${L}.self_attn.q_proj.weight`]: grid(0, 8, 4), + [`${L}.self_attn.k_proj.weight`]: grid(100, 4, 4), + [`${L}.self_attn.v_proj.weight`]: grid(200, 4, 4), + [`${L}.self_attn.q_norm.weight`]: { values: [1, 2, 3, 4], shape: [4] }, + [`${L}.self_attn.k_norm.weight`]: { values: [5, 6, 7, 8], shape: [4] }, + [`${L}.layer_scalar`]: { values: [0.25], shape: [1] }, + [`${S}ple.per_layer_model_projection.weight`]: grid(0, 2, 4), + }; + + it.for(DTYPES)('maps a $dtype checkpoint', async ({ dtype, type, elemBytes }) => { + const sd = await checkpoint(dtype, CHECKPOINT); + const total = sd.totalBytes(); + const get = mapAndCheck(sd, (s) => transformEmbeddingGemma2StateDict(s, '', cfg), RENAMED, [ + 'layers.0.qkv.weight', + 'layers.0.qkv_norm', + 'layers.0.layer_scalar', + 'ple.per_layer_model_projection.weight', + ]); + + // Rope pairs (p, p + 2) of each 4-row head become rows (2p, 2p + 1): + // rows 0 2 1 3 of each head. v keeps its order. + const qkv = get('layers.0.qkv.weight'); + expect(qkv.shape).toEqual([16, 4]); + expect(qkv.byteLength).toBe(64 * elemBytes); + expect(await contents(qkv)).toEqual({ + type, + values: [ + ...gridRows(0, [0, 2, 1, 3, 4, 6, 5, 7], 4), + ...gridRows(100, [0, 2, 1, 3], 4), + ...gridRows(200, [0, 1, 2, 3], 4), + ], + }); + + // q_norm reordered like q, once per q head; k_norm once; ones for v. + // Each norm's bytes count once. + const norm = get('layers.0.qkv_norm'); + expect(norm.shape).toEqual([16]); + expect(norm.byteLength).toBe(8 * elemBytes); + expect(await contents(norm)).toEqual({ + type, + values: [1, 3, 2, 4, 1, 3, 2, 4, 5, 7, 6, 8, ...ones(4)], + }); + + // The CPU parameter reads f32 whatever the checkpoint holds. + const scalar = get('layers.0.layer_scalar'); + expect(scalar.byteLength).toBe(elemBytes); + expect(await contents(scalar)).toEqual({ type: 'f32', values: [0.25] }); + + const ple = get('ple.per_layer_model_projection.weight'); + expect(ple.shape).toEqual([2, 4]); + expect(ple.byteLength).toBe(8 * elemBytes); + expect(await contents(ple)).toEqual({ type, values: [0, 0.5, 1, 1.5, 5, 5.5, 6, 6.5] }); + + expect(sd.totalBytes()).toBe(total); + }); +}); + +describe('transformEmbeddingGemma2VisionStateDict', () => { + // One layer with one head of size 8: each half head of 4 rows has its own + // rope, so q and k reorder per 4 rows. + const cfg: EmbeddingGemma2VisionConfig = { + ...EMBEDDINGGEMMA2_VISION, + hidden: 2, + heads: 1, + headDim: 8, + positions: 2, + layers: 1, + }; + + const T = 'vision_tower.encoder.layers.0'; + const RENAMED: Record = { + 'vision_tower.patch_embedder.input_proj.weight': 'input_proj.weight', + [`${T}.self_attn.o_proj.linear.weight`]: 'layers.0.o_proj.weight', + [`${T}.mlp.gate_proj.linear.weight`]: 'layers.0.gate_proj.weight', + [`${T}.mlp.up_proj.linear.weight`]: 'layers.0.up_proj.weight', + [`${T}.mlp.down_proj.linear.weight`]: 'layers.0.down_proj.weight', + [`${T}.input_layernorm.weight`]: 'layers.0.input_layernorm.scale', + [`${T}.post_attention_layernorm.weight`]: 'layers.0.post_attention_layernorm.scale', + [`${T}.pre_feedforward_layernorm.weight`]: 'layers.0.pre_feedforward_layernorm.scale', + [`${T}.post_feedforward_layernorm.weight`]: 'layers.0.post_feedforward_layernorm.scale', + 'embed_vision.embedding_projection.weight': 'embedding_projection.weight', + }; + const CHECKPOINT: Record = { + ...Object.fromEntries(Object.keys(RENAMED).map((name) => [name, ZERO])), + // [2, positions, hidden]: x rows 1 2 | 3 4, y rows 5 6 | 7 8. + 'vision_tower.patch_embedder.position_embedding_table': { + values: [1, 2, 3, 4, 5, 6, 7, 8], + shape: [2, 2, 2], + }, + [`${T}.self_attn.q_proj.linear.weight`]: grid(0, 8, 2), + [`${T}.self_attn.k_proj.linear.weight`]: grid(100, 8, 2), + [`${T}.self_attn.v_proj.linear.weight`]: grid(200, 8, 2), + [`${T}.self_attn.q_norm.weight`]: { values: [1, 2, 3, 4, 5, 6, 7, 8], shape: [8] }, + [`${T}.self_attn.k_norm.weight`]: { values: [11, 12, 13, 14, 15, 16, 17, 18], shape: [8] }, + }; + + it.for(DTYPES)('maps a $dtype checkpoint', async ({ dtype, type, elemBytes }) => { + const sd = await checkpoint(dtype, CHECKPOINT); + const total = sd.totalBytes(); + const get = mapAndCheck( + sd, + (s) => transformEmbeddingGemma2VisionStateDict(s, '', cfg), + RENAMED, + [ + 'pos_x.weight', + 'pos_y.weight', + 'layers.0.qkv.weight', + 'layers.0.qkv_norm', + 'embedding_pre_projection_norm.scale', + ], + ); + + for (const [name, values] of [ + ['pos_x.weight', [1, 2, 3, 4]], + ['pos_y.weight', [5, 6, 7, 8]], + ] as const) { + const table = get(name); + expect(table.shape).toEqual([2, 2]); + expect(table.byteLength).toBe(4 * elemBytes); + expect(await contents(table)).toEqual({ type, values }); + } + + // Rows 0 2 1 3 of each half head. + const halfHeads = [0, 2, 1, 3, 4, 6, 5, 7]; + const qkv = get('layers.0.qkv.weight'); + expect(qkv.shape).toEqual([24, 2]); + expect(qkv.byteLength).toBe(48 * elemBytes); + expect(await contents(qkv)).toEqual({ + type, + values: [ + ...gridRows(0, halfHeads, 2), + ...gridRows(100, halfHeads, 2), + ...gridRows(200, [0, 1, 2, 3, 4, 5, 6, 7], 2), + ], + }); + + const norm = get('layers.0.qkv_norm'); + expect(norm.shape).toEqual([24]); + expect(norm.byteLength).toBe(16 * elemBytes); + expect(await contents(norm)).toEqual({ + type, + values: [1, 3, 2, 4, 5, 7, 6, 8, 11, 13, 12, 14, 15, 17, 16, 18, ...ones(8)], + }); + + // Ones download nothing and are exact in f16. + const pre = get('embedding_pre_projection_norm.scale'); + expect(pre.shape).toEqual([2]); + expect(pre.byteLength).toBe(0); + expect(await contents(pre)).toEqual({ type: 'f16', values: [1, 1] }); + + expect(sd.totalBytes()).toBe(total); + }); +}); + +describe('transformEmbeddingGemma2AudioStateDict', () => { + // One layer, hidden 4, one 4-wide head (scale 4^-½ = 0.5), conv channels + // 2 then 1, 2 conv taps, window 2. + const cfg: EmbeddingGemma2AudioConfig = { + ...EMBEDDINGGEMMA2_AUDIO, + hidden: 4, + headDim: 4, + convKernel: 2, + window: 2, + convChannels: [2, 1], + layers: 1, + residualWeight: 0.5, + outputDim: 2, + }; + + const A = 'audio_tower.'; + const sub = `${A}subsample_conv_projection.`; + const T = `${A}layers.0`; + /** Every clipped linear, HF prefix → ours. */ + const CLIPPED: Record = { + [`${T}.feed_forward1.ffw_layer_1`]: 'layers.0.feed_forward1.ffw_layer_1', + [`${T}.feed_forward1.ffw_layer_2`]: 'layers.0.feed_forward1.ffw_layer_2', + [`${T}.feed_forward2.ffw_layer_1`]: 'layers.0.feed_forward2.ffw_layer_1', + [`${T}.feed_forward2.ffw_layer_2`]: 'layers.0.feed_forward2.ffw_layer_2', + [`${T}.self_attn.post`]: 'layers.0.post', + [`${T}.lconv1d.linear_start`]: 'layers.0.lconv.linear_start', + [`${T}.lconv1d.linear_end`]: 'layers.0.lconv.linear_end', + }; + const BOUNDS = ['input_min', 'input_max', 'output_min', 'output_max']; + const RENAMED: Record = { + ...Object.fromEntries( + Object.entries(CLIPPED).map(([hf, ours]) => [`${hf}.linear.weight`, `${ours}.weight`]), + ), + [`${sub}layer0.norm.weight`]: 'norm0.weight', + [`${sub}layer1.norm.weight`]: 'norm1.weight', + [`${sub}input_proj_linear.weight`]: 'input_proj.weight', + [`${T}.feed_forward1.pre_layer_norm.weight`]: 'layers.0.feed_forward1.pre_layer_norm.scale', + [`${T}.feed_forward2.pre_layer_norm.weight`]: 'layers.0.feed_forward2.pre_layer_norm.scale', + [`${T}.lconv1d.pre_layer_norm.weight`]: 'layers.0.lconv.pre_layer_norm.scale', + [`${T}.lconv1d.conv_norm.weight`]: 'layers.0.lconv.conv_norm.scale', + [`${T}.norm_pre_attn.weight`]: 'layers.0.norm_pre_attn.scale', + [`${T}.norm_post_attn.weight`]: 'layers.0.norm_post_attn.scale', + [`${T}.norm_out.weight`]: 'layers.0.norm_out.scale', + [`${A}output_proj.weight`]: 'output_proj.weight', + [`${A}output_proj.bias`]: 'output_proj.bias', + 'embed_audio.embedding_projection.weight': 'embedding_projection.weight', + }; + const bound = (prefix: string, values: number[]) => + Object.fromEntries( + BOUNDS.map((b, i) => [`${prefix}.${b}`, { values: [values[i]!], shape: [1] }]), + ); + const CHECKPOINT: Record = { + ...Object.fromEntries(Object.keys(RENAMED).map((name) => [name, ZERO])), + ...Object.assign({}, ...Object.keys(CLIPPED).map((p) => bound(p, [-1, 1, -2, 2]))), + ...Object.assign( + {}, + ...['q_proj', 'k_proj', 'v_proj'].map((p) => bound(`${T}.self_attn.${p}`, [-3, 3, -4, 4])), + ), + // [c0, 1, 3, 3] and [c1, c0, 3, 3], values 1 … 18. + [`${sub}layer0.conv.weight`]: { + values: Array.from({ length: 18 }, (_, i) => i + 1), + shape: [2, 1, 3, 3], + }, + [`${sub}layer1.conv.weight`]: { + values: Array.from({ length: 18 }, (_, i) => i + 1), + shape: [1, 2, 3, 3], + }, + [`${T}.feed_forward1.post_layer_norm.weight`]: { values: [2, 4, 6, 8], shape: [4] }, + [`${T}.feed_forward2.post_layer_norm.weight`]: { values: [1, 3, 5, 7], shape: [4] }, + [`${T}.self_attn.q_proj.linear.weight`]: grid(0, 4, 4), + [`${T}.self_attn.k_proj.linear.weight`]: grid(100, 4, 4), + [`${T}.self_attn.v_proj.linear.weight`]: grid(200, 4, 4), + // Identity, so each bias row is the sinusoid row itself. + [`${T}.self_attn.relative_k_proj.weight`]: { + values: [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1], + shape: [4, 4], + }, + // softplus(0) = ln 2, so each q scale is 4^-½ = 0.5. + [`${T}.self_attn.per_dim_scale`]: { values: [0, 0, 0, 0], shape: [4] }, + // [hidden, 1, K]: channel c holds taps 2c + 1, 2c + 2. + [`${T}.lconv1d.depthwise_conv1d.weight`]: { + values: [1, 2, 3, 4, 5, 6, 7, 8], + shape: [4, 1, 2], + }, + }; + const clipNames = Object.values(CLIPPED).map((ours) => `${ours}.clip`); + + it.for(DTYPES)('maps a $dtype checkpoint', async ({ dtype, type, elemBytes }) => { + const sd = await checkpoint(dtype, CHECKPOINT); + const total = sd.totalBytes(); + const before = new Map(sd.tensors); + const get = mapAndCheck( + sd, + (s) => transformEmbeddingGemma2AudioStateDict(s, '', cfg), + RENAMED, + [ + ...clipNames, + 'conv0.weight', + 'conv1.weight', + 'layers.0.feed_forward1.post_layer_norm.scale', + 'layers.0.feed_forward2.post_layer_norm.scale', + 'layers.0.qkv.weight', + 'layers.0.qkv.clip', + 'layers.0.rel', + 'layers.0.q_scale', + 'layers.0.lconv.conv', + 'embedding_pre_projection_norm.scale', + ], + ); + + // conv0 only folds its taps: the source tensor, reshaped. + const source = before.get(`${sub}layer0.conv.weight`)!; + const conv0 = get('conv0.weight'); + expect(conv0.shape).toEqual([2, 9]); + expect(conv0.byteLength).toBe(18 * elemBytes); + expect([conv0.kind, conv0.dtype]).toEqual([source.kind, source.dtype]); + + // Tap k of channel c moves from c·9 + k to k·2 + c. + const conv1 = get('conv1.weight'); + expect(conv1.shape).toEqual([1, 18]); + expect(conv1.byteLength).toBe(18 * elemBytes); + expect(await contents(conv1)).toEqual({ + type, + values: [1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17, 9, 18], + }); + + // Times the residual weight 0.5. + for (const [name, values] of [ + ['layers.0.feed_forward1.post_layer_norm.scale', [1, 2, 3, 4]], + ['layers.0.feed_forward2.post_layer_norm.scale', [0.5, 1.5, 2.5, 3.5]], + ] as const) { + const norm = get(name); + expect(norm.shape).toEqual([4]); + expect(norm.byteLength).toBe(4 * elemBytes); + expect(await contents(norm)).toEqual({ type, values }); + } + + const qkv = get('layers.0.qkv.weight'); + expect(qkv.shape).toEqual([12, 4]); + expect(qkv.byteLength).toBe(48 * elemBytes); + expect(await contents(qkv)).toEqual({ + type, + values: [0, 100, 200].flatMap((base) => gridRows(base, [0, 1, 2, 3], 4)), + }); + + // Bounds are CPU data: f32, and their bytes are not counted. + for (const name of clipNames) { + expect(await contents(get(name))).toEqual({ type: 'f32', values: [-1, 1, -2, 2] }); + } + expect(await contents(get('layers.0.qkv.clip'))).toEqual({ + type: 'f32', + values: [-3, 3, -4, 4], + }); + + // Sinusoid rows for distances 0 and 1 at hidden 4: angle j·10000^-i for + // i = 0, 1, sines then cosines. + const rel = get('layers.0.rel'); + expect(rel.shape).toEqual([2, 4]); + expect(rel.byteLength).toBe(16 * elemBytes); + const relValues = await contents(rel); + expect(relValues.type).toBe('f32'); + const want = [0, 0, 1, 1, Math.sin(1), Math.sin(1e-4), Math.cos(1), Math.cos(1e-4)]; + relValues.values.forEach((v, i) => expect(v).toBeCloseTo(want[i]!, 6)); + + const qScale = get('layers.0.q_scale'); + expect(qScale.shape).toEqual([4]); + expect(qScale.byteLength).toBe(4 * elemBytes); + const qScaleValues = await contents(qScale); + expect(qScaleValues.type).toBe('f32'); + qScaleValues.values.forEach((v) => expect(v).toBeCloseTo(0.5, 6)); + + // [hidden, 1, K] → [K, hidden]: tap 0 of every channel, then tap 1. + const lconv = get('layers.0.lconv.conv'); + expect(lconv.shape).toEqual([2, 4]); + expect(lconv.byteLength).toBe(8 * elemBytes); + expect(await contents(lconv)).toEqual({ type, values: [1, 3, 5, 7, 2, 4, 6, 8] }); + + expect(await contents(get('embedding_pre_projection_norm.scale'))).toEqual({ + type: 'f16', + values: [1, 1], + }); + + // 10 clipped linears' worth of bounds, 4 scalars each. + expect(sd.totalBytes()).toBe(total - 40 * elemBytes); + }); +}); From 3cd19c261aa3d25bcef0282d7994eafed171dce6 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 28 Oct 2026 08:05:15 +0100 Subject: [PATCH 22/25] docs(zoo): say why the DepthART weight transforms stay in f32 --- packages/runntime/src/zoo/depthart/blocks.ts | 2 ++ packages/runntime/src/zoo/depthart/layers.ts | 3 +++ packages/runntime/src/zoo/depthart/ss2d.ts | 1 + 3 files changed, 6 insertions(+) diff --git a/packages/runntime/src/zoo/depthart/blocks.ts b/packages/runntime/src/zoo/depthart/blocks.ts index 6bb3785a..f2ff8e3e 100644 --- a/packages/runntime/src/zoo/depthart/blocks.ts +++ b/packages/runntime/src/zoo/depthart/blocks.ts @@ -50,6 +50,8 @@ export class ConvBlock extends Module { const key = (n: string) => (prefix ? `${prefix}.${n}` : n); const at = (n: string) => requireTensor(sd, key(n), `ConvBlock at '${prefix}'`); const scale = at('layer_scale'); + // f32 on purpose: the BatchNorm fold reads it next, and an f16 product + // would round once more before the fold. for (const n of ['mlp.fc2.bn.weight', 'mlp.fc2.bn.bias']) { const src = at(n); sd.tensors.set( diff --git a/packages/runntime/src/zoo/depthart/layers.ts b/packages/runntime/src/zoo/depthart/layers.ts index 472f93cd..8b5936e3 100644 --- a/packages/runntime/src/zoo/depthart/layers.ts +++ b/packages/runntime/src/zoo/depthart/layers.ts @@ -66,6 +66,8 @@ export class BatchNormC extends Module { const [g, b, m, v] = [at('weight'), at('bias'), at('running_mean'), at('running_var')]; const c = this.channels; const eps = this.eps; + // f32 on purpose: channelAffine reads f32, and shift = beta − mean · scale + // would lose its low bits in f16. let memo: Promise<{ scale: Float32Array; shift: Float32Array }> | undefined; const affine = () => (memo ??= (async () => { @@ -157,6 +159,7 @@ export class RepDW extends Module { const center = centerTap(this.kH, this.kW); const eps = this.eps; + // f32: the loader rounds the merged taps to f16 once, for the pinned conv. let memo: Promise<{ w: Float32Array; b: Float32Array }> | undefined; const merged = () => (memo ??= (async () => { diff --git a/packages/runntime/src/zoo/depthart/ss2d.ts b/packages/runntime/src/zoo/depthart/ss2d.ts index eb18de88..a7ef476a 100644 --- a/packages/runntime/src/zoo/depthart/ss2d.ts +++ b/packages/runntime/src/zoo/depthart/ss2d.ts @@ -97,6 +97,7 @@ export class SS2D extends Module { // prefix is empty when this module is the root. const key = prefix ? `${prefix}.A_logs` : 'A_logs'; const src = requireTensor(sd, key, `SS2D at '${prefix}'`); + // f32 on purpose: A is an f32 parameter of the scan. sd.tensors.set( key, derivedTensor( From 604a8383e8bef2730e6e1adcde3a10b4051068db Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 28 Oct 2026 11:47:42 +0100 Subject: [PATCH 23/25] docs(zoo): say the scaled EmbeddingGemma 2 PLE projection and audio post norms stay f32 --- packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts b/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts index 5b52d057..44d51821 100644 --- a/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts +++ b/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts @@ -5,7 +5,9 @@ * * An F16 checkpoint gives f16 derived tensors, except the text layer * scalars and the audio tower's clip bounds, distance bias rows and q - * scales, which the CPU code and the attention kernel read as f32. */ + * scales, which the CPU code and the attention kernel read as f32, and the + * scaled PLE projection and audio post norms, which stay f32 so an f32 + * model loads exact values. */ import { concatRows, From 38aebd9fedbada6a8e552ae688ad15741632eade Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 28 Oct 2026 11:47:48 +0100 Subject: [PATCH 24/25] test(zoo): expect f32 for the scaled EmbeddingGemma 2 PLE projection and audio post norms --- .../test/zoo/embeddinggemma2/stateDictHooks.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts b/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts index 43336b7b..0389cb72 100644 --- a/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts +++ b/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts @@ -138,10 +138,11 @@ describe('transformEmbeddingGemma2StateDict', () => { expect(scalar.byteLength).toBe(elemBytes); expect(await contents(scalar)).toEqual({ type: 'f32', values: [0.25] }); + // Scaled, so f32 on every checkpoint. const ple = get('ple.per_layer_model_projection.weight'); expect(ple.shape).toEqual([2, 4]); expect(ple.byteLength).toBe(8 * elemBytes); - expect(await contents(ple)).toEqual({ type, values: [0, 0.5, 1, 1.5, 5, 5.5, 6, 6.5] }); + expect(await contents(ple)).toEqual({ type: 'f32', values: [0, 0.5, 1, 1.5, 5, 5.5, 6, 6.5] }); expect(sd.totalBytes()).toBe(total); }); @@ -370,7 +371,7 @@ describe('transformEmbeddingGemma2AudioStateDict', () => { values: [1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17, 9, 18], }); - // Times the residual weight 0.5. + // Times the residual weight 0.5, so f32 on every checkpoint. for (const [name, values] of [ ['layers.0.feed_forward1.post_layer_norm.scale', [1, 2, 3, 4]], ['layers.0.feed_forward2.post_layer_norm.scale', [0.5, 1.5, 2.5, 3.5]], @@ -378,7 +379,7 @@ describe('transformEmbeddingGemma2AudioStateDict', () => { const norm = get(name); expect(norm.shape).toEqual([4]); expect(norm.byteLength).toBe(4 * elemBytes); - expect(await contents(norm)).toEqual({ type, values }); + expect(await contents(norm)).toEqual({ type: 'f32', values }); } const qkv = get('layers.0.qkv.weight'); From 2bd5993d10d8b29496b01277e0fb49b0f89cbd98 Mon Sep 17 00:00:00 2001 From: chmjkb Date: Wed, 28 Oct 2026 11:47:10 +0100 Subject: [PATCH 25/25] docs(zoo): say concatRows reads each tiled EmbeddingGemma 2 qkv norm once --- packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts b/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts index 44d51821..8e1d069d 100644 --- a/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts +++ b/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts @@ -106,7 +106,8 @@ function fusedQkvNorm( ...Array(kvHeads).fill(kn), ones(kvHeads * d), ]); - // Each norm is downloaded once, however many heads it is tiled over. + // concatRows reads a part listed several times once, so each norm is + // downloaded once however many heads it is tiled over. return { ...tiled, byteLength: nq.byteLength + nk.byteLength }; }