Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .claude/skills/code-style/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,13 @@ Short bad/good pairs are in [examples.md](examples.md).
- Match the file: naming, error messages, how options and defaults are
destructured. If you see a better way, do not mix two styles: change
every place in its own PR, or suggest it and leave it.
- Structure a model so its module tree matches the checkpoint: field
names and nesting follow the state-dict keys, in their case
(`self_attn`, `LayerNorm`, `model.decoder.embed_tokens`), so loading
needs no renames. A state-dict hook only builds derived tensors (a fused
`qkv`, a folded scale, a table computed on the CPU), under the
checkpoint-named parent. Moonshine, MiniLM and EmbeddingGemma 2 are
the examples.
- Keep a concept in one place. Two copies of the same logic drift apart.

## Types
Expand Down
15 changes: 15 additions & 0 deletions .claude/skills/code-style/examples.md
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,21 @@ A caught error is `unknown`: anything can be thrown, not only an `Error`.
}
```

## Hook that renames instead of a field named like the checkpoint

```ts
// Bad: the module invents a name and the hook maps every key to it
class DecoderLayer extends nn.Module {
readonly crossAttn: DecoderCrossAttn;
}
move(sd, `${L}.encoder_attn.o_proj.weight`, `${L}.crossAttn.out.weight`);

// Good: the field is the checkpoint's key, so the tensor binds as is
class DecoderLayer extends nn.Module {
readonly encoder_attn: DecoderCrossAttn;
}
```

## Magic number

```ts
Expand Down
10 changes: 5 additions & 5 deletions packages/runntime/src/core/nn/layers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ export class Linear extends Module {
override transformStateDict(sd: LazyStateDict, prefix: string): void {
const name = prefix ? `${prefix}.weight` : 'weight';
const tensor = sd.tensors.get(name);
if (!tensor) return; // missing keys are the strict check's job, not ours
if (!tensor) return; // missing keys are the key check's job, not ours
// Without this a square wrong-layout weight loads silently transposed.
if (
tensor.shape.length !== 2 ||
Expand Down Expand Up @@ -336,7 +336,7 @@ export class Conv1d extends Module {
override transformStateDict(sd: LazyStateDict, prefix: string): void {
const name = prefix ? `${prefix}.weight` : 'weight';
let tensor = sd.tensors.get(name);
if (!tensor) return; // a missing key is reported by the strict key check
if (!tensor) return; // a missing key is reported by the key check
if (tensor.kind === 'quant') {
const { inChannels: cIn, outChannels: cOut, kernelSize: kernel } = this;
checkQuantConvShape('Conv1d', name, tensor, {
Expand Down Expand Up @@ -881,16 +881,16 @@ export class BatchNorm2d extends Module {

/** RMSNorm over the last dim of width `n`. */
export class RMSNorm extends Module {
readonly scale: Parameter;
readonly weight: Parameter;
constructor(
readonly n: number,
readonly eps = 1e-6,
) {
super();
this.scale = new Parameter({ elems: n, dtype: 'f32', dims: [n] });
this.weight = new Parameter({ elems: n, dtype: 'f32', dims: [n] });
}
forward(x: Value): Value {
return rmsNorm(x, this.scale.value, this.eps);
return rmsNorm(x, this.weight.value, this.eps);
}
}

Expand Down
17 changes: 14 additions & 3 deletions packages/runntime/src/core/nn/loadStateDict.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/** Streaming state-dict loader: transform hooks, then a strict name check over
/** Streaming state-dict loader: transform hooks, then a name check over
* headers alone, then per-tensor fetch, adapt, preprocess and upload. Peak
* host memory is one tensor's working set. */
import type { TgpuRoot } from 'typegpu';
Expand All @@ -20,6 +20,11 @@ export interface LoadStateDictOpts {
upload?: UploadFn;
/** After each bound tensor: cumulative file bytes done / total. */
onProgress?: (name: string, doneBytes: number, totalBytes: number) => void;
/** Whether a tensor in the file that no parameter reads is an error
* (CHECKPOINT_MISMATCH) rather than a warning. Default false, like
* PyTorch's `strict=False`: the tensor is left unbound. A parameter the
* file lacks is always an error, as it has no value to fall back to. */
strict?: boolean;
}

/** The default GPU upload.
Expand Down Expand Up @@ -125,22 +130,28 @@ async function bindStateDict(
const paramNames = new Set(params.map(([n]) => n));
const missing = params.filter(([n]) => !sd.tensors.has(n)).map(([n]) => n);
const unexpected = [...sd.tensors.keys()].filter((n) => !paramNames.has(n));
if (missing.length || unexpected.length) {
if (missing.length || (unexpected.length && opts.strict)) {
throw new RunntimeError(
'CHECKPOINT_MISMATCH',
'loadStateDict: state dict does not match model.' +
(missing.length ? ` missing: ${list(missing)}.` : '') +
(unexpected.length ? ` unexpected: ${list(unexpected)}.` : ''),
);
}
if (unexpected.length) {
console.warn(
`loadStateDict: state dict has tensors the model does not read: ${list(unexpected)}.`,
);
}

// Bind in state-dict (file) order, so each chunk is read once. Module order
// jumps around the payload and refetches evicted chunks. Keys a transform
// added sort last; they read nothing new.
const position = new Map([...sd.tensors.keys()].map((name, i) => [name, i]));
params.sort(([a], [b]) => (position.get(a) ?? Infinity) - (position.get(b) ?? Infinity));

const total = sd.totalBytes();
// Unbound tensors are never read, so they do not count towards the total.
const total = params.reduce((sum, [name]) => sum + sd.tensors.get(name)!.byteLength, 0);
let done = 0;
for (const [name, p] of params) {
const t = sd.tensors.get(name)!;
Expand Down
2 changes: 1 addition & 1 deletion packages/runntime/src/core/weights/transforms.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
* `scaleWeight` and `dequantize`, which always give f32. BF16 decodes to
* f32, which is exact; a derived tensor's Uint16Array means f16, so bf16 bits
* have no derived form. Each output's byteLength is its share of the source
* bytes, so load progress still sums to the checkpoint size.
* bytes, so load progress counts every byte the model reads.
*
* A QuantTensor source gives a QuantTensor, except in `dequantize`: the
* helpers move its ints and scales together. Whatever would split a scale
Expand Down
5 changes: 3 additions & 2 deletions packages/runntime/src/zoo/embeddinggemma2/audioEncoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -195,8 +195,9 @@ export async function encodeAudio(
const { mels } = model.cfg;
const tokens = audioTokens(samples.length);
const frames = frameCount(samples.length);
// A norm scale, not a Linear weight: a quantized weight is quantW either way.
const act = model.embedding_pre_projection_norm.scale.shape.dtype === 'f16' ? 'f16' : 'f32';
// The pre-projection norm's weight, not a Linear's: a quantized Linear
// weight is quantW either way.
const act = model.embed_audio.embedding_pre_projection_norm.shape.dtype === 'f16' ? 'f16' : 'f32';
const patches = convPatches(logMel(samples, frames, mels), frames, mels);
const taps = convTaps(Math.ceil(frames / 2), mels / 2);
const inputs: EmbeddingGemma2AudioInputs = {
Expand Down
Loading
Loading