Skip to content
Closed

wip #94

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
6a53fc1
perf(core): keep F16 checkpoints in f16 through the Linear weight tra…
chmjkb Oct 27, 2026
b62a359
perf(core): keep F16 checkpoints in f16 through the Conv1d tap-major …
chmjkb Oct 27, 2026
cc68ce1
test(core): declare EDGE_BITS as const instead of asserting its tuple…
chmjkb Oct 28, 2026
0c1f53f
test(core): cover BF16 into f16 and derived halfword and packed-word …
chmjkb Oct 28, 2026
c97813b
refactor(core): give convToTapMajor one signature and upload random c…
chmjkb Oct 28, 2026
75b704f
fix(core): reject a derived tensor whose data does not fill its shape…
chmjkb Oct 30, 2026
0f59f08
feat(core): add a reshapeWeight helper for state dict hooks
chmjkb Oct 27, 2026
9e5efc5
test(core): share the synthetic F16 checkpoint builder between weight…
chmjkb Oct 27, 2026
b9b8535
refactor(zoo): fuse Moonshine q/k/v with the shared weight helpers an…
chmjkb Oct 27, 2026
f0d18d7
test(zoo): cover the Moonshine state dict hook on F32 and F16 checkpo…
chmjkb Oct 27, 2026
4edf309
refactor(zoo): keep the Moonshine v2 norm gammas in f32 on purpose an…
chmjkb Oct 27, 2026
bcef513
test(zoo): cover the Moonshine v2 state dict hook on F32 and F16 chec…
chmjkb Oct 27, 2026
c8dd092
refactor(zoo): map the privacy-filter float checkpoint with the share…
chmjkb Oct 27, 2026
435ca96
test(zoo): cover the privacy-filter state dict hook on F32 and F16 ch…
chmjkb Oct 27, 2026
0610147
test(zoo): expect f32 for the scaled Moonshine q now that scaleWeight…
chmjkb Oct 28, 2026
03b68cf
fix(zoo): check Moonshine q/k/v projection widths against the hidden …
chmjkb Oct 28, 2026
97c04ff
docs(core): say why reshapeWeight can shallow-copy a LazyTensor
chmjkb Oct 28, 2026
52294eb
test(core): build every synthetic safetensors file through one shared…
chmjkb Oct 28, 2026
efd2065
feat(core): add permuteRows to the weight helpers to reorder dim-0 ro…
chmjkb Oct 28, 2026
019b441
refactor(zoo): map the EmbeddingGemma 2 checkpoint with the shared we…
chmjkb Oct 28, 2026
0445c1a
test(zoo): cover the EmbeddingGemma 2 state dict hooks on F32 and F16…
chmjkb Oct 28, 2026
3cd19c2
docs(zoo): say why the DepthART weight transforms stay in f32
chmjkb Oct 28, 2026
604a838
docs(zoo): say the scaled EmbeddingGemma 2 PLE projection and audio p…
chmjkb Oct 28, 2026
38aebd9
test(zoo): expect f32 for the scaled EmbeddingGemma 2 PLE projection …
chmjkb Oct 28, 2026
2bd5993
docs(zoo): say concatRows reads each tiled EmbeddingGemma 2 qkv norm …
chmjkb Oct 28, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 7 additions & 8 deletions packages/runntime/bench/kernels/conv.ab.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,14 +75,13 @@ function conv1d(
name,
build: (version) => {
const x = randMat(version, tIn, cIn);
const w = version.upload(
version.lib.convToTapMajor(version.rand(cOut * cIn * k), cOut, cIn, k),
{
elems: k * cIn * cOut,
dtype: 'f32',
dims: [k * cIn, cOut],
},
);
// Random taps need no tap-major reorder: a permutation of them is as
// random, and A and B draw the same numbers either way.
const w = version.upload(version.rand(cOut * cIn * k), {
elems: k * cIn * cOut,
dtype: 'f32',
dims: [k * cIn, cOut],
});
const call = () =>
version.lib.conv1d(x, w, { kernelSize: k, stride, activation: act, ...geometry });
// Built once here, so a base that refuses the geometry skips the case.
Expand Down
3 changes: 2 additions & 1 deletion packages/runntime/src/core/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ export {
packConvHwc4F16,
packDwHwc4F16,
permuteConvTransposeF16,
transposeF32,
} from './weights/convert.ts';
export {
createStorageFor,
Expand Down Expand Up @@ -140,6 +139,8 @@ export { createOpfsCache } from './weights/opfsCache.ts';
export { foldBnIntoConv } from './weights/foldBn.ts';
export {
concatRows,
permuteRows,
reshapeWeight,
scaleWeight,
splitCols,
splitRows,
Expand Down
18 changes: 8 additions & 10 deletions packages/runntime/src/core/nn/layers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,10 @@ import {
packConvWinogradF16,
packDwHwc4F16,
permuteConvTransposeF16,
transposeF32,
} from '../weights/convert.ts';
import { foldBnIntoConv } from '../weights/foldBn.ts';
import { derivedTensor, type LazyStateDict } from '../weights/safetensors.ts';
import { readFloats, transposeWeight } from '../weights/transforms.ts';
import { tensor } from '../gpu/buffers.ts';
import { CpuParameter, Module, Parameter, type AnyModule } from './module.ts';

Expand Down Expand Up @@ -135,14 +135,7 @@ export class Linear extends Module {
`expected torch layout [${this.outFeatures}, ${this.inFeatures}]`,
);
}
sd.tensors.set(
name,
derivedTensor(
[this.inFeatures, this.outFeatures],
async () => transposeF32(await tensor.f32(), this.outFeatures, this.inFeatures),
tensor.byteLength,
),
);
sd.tensors.set(name, transposeWeight(tensor));
}
forward(x: Value, call: { addend?: Value } = {}): Value {
return linear(x, this.weight.value, this.bias?.value, call.addend, this.activation);
Expand Down Expand Up @@ -216,7 +209,12 @@ export class Conv1d extends Module {
derivedTensor(
[this.kernelSize * this.inChannels, this.outChannels],
async () =>
convToTapMajor(await tensor.f32(), this.outChannels, this.inChannels, this.kernelSize),
convToTapMajor(
await readFloats('Conv1d', tensor),
this.outChannels,
this.inChannels,
this.kernelSize,
),
tensor.byteLength,
),
);
Expand Down
21 changes: 6 additions & 15 deletions packages/runntime/src/core/weights/convert.ts
Original file line number Diff line number Diff line change
@@ -1,25 +1,16 @@
/** Checkpoints store dense weights [out, in]; nn.Linear wants [in, out]. */
export function transposeF32(src: Float32Array, rows: number, cols: number): Float32Array {
if (src.length !== rows * cols) {
throw new Error(`transposeF32: length ${src.length} != ${rows}×${cols}`);
}
const out = new Float32Array(src.length);
for (let r = 0; r < rows; r++) {
for (let c = 0; c < cols; c++) out[c * rows + r] = src[r * cols + c]!;
}
return out;
}

/** A torch [cOut, cIn, k] conv1d weight as tap-major [k·cIn, cOut]. A pure
* permutation, so f16 halfwords move as they are. */
export function convToTapMajor(
src: Float32Array,
src: Float32Array | Uint16Array,
cOut: number,
cIn: number,
k: number,
): Float32Array {
): Float32Array | Uint16Array {
if (src.length !== cOut * cIn * k) {
throw new Error(`convToTapMajor: length ${src.length} != ${cOut}×${cIn}×${k}`);
}
const out = new Float32Array(src.length);
const out =
src instanceof Uint16Array ? new Uint16Array(src.length) : new Float32Array(src.length);
for (let cout = 0; cout < cOut; cout++) {
for (let cin = 0; cin < cIn; cin++) {
for (let tap = 0; tap < k; tap++) {
Expand Down
56 changes: 54 additions & 2 deletions packages/runntime/src/core/weights/transforms.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
/** Lazy reshaping helpers for `transformStateDict` hooks: concatenate, split,
* scale and transpose checkpoint tensors without fetching them.
* permute, scale, transpose and reshape checkpoint tensors without fetching
* them.
*
* F16 sources give f16 halfwords and F32 sources give f32, except
* `scaleWeight`, which always gives f32. BF16 decodes to
Expand All @@ -18,10 +19,17 @@ const numel = (shape: readonly number[]): number => shape.reduce((a, b) => a * b
const mismatch = (helper: string, why: string) =>
new RunntimeError('CHECKPOINT_MISMATCH', `${helper}: ${why}`);

async function readFloats(helper: string, t: LazyTensor): Promise<Floats> {
/** Reads `t` as f16 halfwords when it holds F16 and as f32 otherwise. Throws
* CHECKPOINT_MISMATCH, naming `helper`, on packed u32 words or on a derived
* tensor whose data does not fill its shape. A file tensor's length is
* checked by the header parser. */
export async function readFloats(helper: string, t: LazyTensor): Promise<Floats> {
if (t.kind === 'file') return t.dtype === 'F16' ? t.u16() : t.f32();
const data = await t.data();
if (data instanceof Uint32Array) throw mismatch(helper, 'source holds packed u32 words');
if (data.length !== numel(t.shape)) {
throw mismatch(helper, `source holds ${data.length} values for shape [${t.shape.join(', ')}]`);
}
return data;
}

Expand Down Expand Up @@ -134,6 +142,34 @@ export function splitCols(t: LazyTensor, sizes: readonly number[]): LazyTensor[]
});
}

/** Reorders the rows of dim 0: row i of the result is row `order[i]` of `t`,
* like torch `t[order]`. Throws CHECKPOINT_MISMATCH when `order` is not a
* permutation of 0 … rows − 1. */
export function permuteRows(t: LazyTensor, order: readonly number[]): LazyTensor {
if (t.shape.length === 0) throw mismatch('permuteRows', 'cannot permute a scalar');
const rows = t.shape[0]!;
const inRange = order.every((r) => Number.isInteger(r) && r >= 0 && r < rows);
if (order.length !== rows || new Set(order).size !== rows || !inRange) {
throw mismatch(
'permuteRows',
`order is not a permutation of the ${rows} rows of shape [${t.shape.join(', ')}]`,
);
}
const rowElems = numel(t.shape) / rows;
return derivedTensor(
t.shape,
async () => {
const data = await readFloats('permuteRows', t);
const out = allocLike(data, data.length);
for (const [i, r] of order.entries()) {
out.set(data.subarray(r * rowElems, (r + 1) * rowElems), i * rowElems);
}
return out;
},
t.byteLength,
);
}

/** Multiplies `t` by `scale`: one number for every element, or one per row
* of dim 0. The result is f32 even for an F16 source: rounding the product
* back to f16 would change what an f32 model loads. */
Expand Down Expand Up @@ -180,3 +216,19 @@ export function transposeWeight(t: LazyTensor): LazyTensor {
t.byteLength,
);
}

/** Gives `t` a new shape with the same element count, like torch `reshape`.
* The data is untouched: a file tensor stays a file tensor of its dtype, and
* byteLength stays the same. Throws CHECKPOINT_MISMATCH when the element
* counts differ. */
export function reshapeWeight(t: LazyTensor, shape: readonly number[]): LazyTensor {
if (numel(shape) !== numel(t.shape)) {
throw mismatch(
'reshapeWeight',
`cannot reshape [${t.shape.join(', ')}] to [${shape.join(', ')}]`,
);
}
// Every LazyTensor is a plain object whose readers close over their own
// source, so a shallow copy shares the data and reads it the same way.
return { ...t, shape: [...shape] };
}
2 changes: 2 additions & 0 deletions packages/runntime/src/zoo/depthart/blocks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,8 @@ export class ConvBlock extends Module {
const key = (n: string) => (prefix ? `${prefix}.${n}` : n);
const at = (n: string) => requireTensor(sd, key(n), `ConvBlock at '${prefix}'`);
const scale = at('layer_scale');
// f32 on purpose: the BatchNorm fold reads it next, and an f16 product
// would round once more before the fold.
for (const n of ['mlp.fc2.bn.weight', 'mlp.fc2.bn.bias']) {
const src = at(n);
sd.tensors.set(
Expand Down
3 changes: 3 additions & 0 deletions packages/runntime/src/zoo/depthart/layers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ export class BatchNormC extends Module {
const [g, b, m, v] = [at('weight'), at('bias'), at('running_mean'), at('running_var')];
const c = this.channels;
const eps = this.eps;
// f32 on purpose: channelAffine reads f32, and shift = beta − mean · scale
// would lose its low bits in f16.
let memo: Promise<{ scale: Float32Array; shift: Float32Array }> | undefined;
const affine = () =>
(memo ??= (async () => {
Expand Down Expand Up @@ -157,6 +159,7 @@ export class RepDW extends Module {
const center = centerTap(this.kH, this.kW);
const eps = this.eps;

// f32: the loader rounds the merged taps to f16 once, for the pinned conv.
let memo: Promise<{ w: Float32Array; b: Float32Array }> | undefined;
const merged = () =>
(memo ??= (async () => {
Expand Down
1 change: 1 addition & 0 deletions packages/runntime/src/zoo/depthart/ss2d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ export class SS2D extends Module {
// prefix is empty when this module is the root.
const key = prefix ? `${prefix}.A_logs` : 'A_logs';
const src = requireTensor(sd, key, `SS2D at '${prefix}'`);
// f32 on purpose: A is an f32 parameter of the scan.
sd.tensors.set(
key,
derivedTensor(
Expand Down
Loading