diff --git a/.claude/skills/code-style/SKILL.md b/.claude/skills/code-style/SKILL.md index 39ae7c1d..1c9dea11 100644 --- a/.claude/skills/code-style/SKILL.md +++ b/.claude/skills/code-style/SKILL.md @@ -44,6 +44,13 @@ Short bad/good pairs are in [examples.md](examples.md). - Match the file: naming, error messages, how options and defaults are destructured. If you see a better way, do not mix two styles: change every place in its own PR, or suggest it and leave it. +- Structure a model so its module tree matches the checkpoint: field + names and nesting follow the state-dict keys, in their case + (`self_attn`, `LayerNorm`, `model.decoder.embed_tokens`), so loading + needs no renames. A state-dict hook only builds derived tensors (a fused + `qkv`, a folded scale, a table computed on the CPU), under the + checkpoint-named parent. Moonshine, MiniLM and EmbeddingGemma 2 are + the examples. - Keep a concept in one place. Two copies of the same logic drift apart. ## Types diff --git a/.claude/skills/code-style/examples.md b/.claude/skills/code-style/examples.md index 380f4ad0..eed24065 100644 --- a/.claude/skills/code-style/examples.md +++ b/.claude/skills/code-style/examples.md @@ -99,6 +99,21 @@ A caught error is `unknown`: anything can be thrown, not only an `Error`. } ``` +## Hook that renames instead of a field named like the checkpoint + +```ts +// Bad: the module invents a name and the hook maps every key to it +class DecoderLayer extends nn.Module { + readonly crossAttn: DecoderCrossAttn; +} +move(sd, `${L}.encoder_attn.o_proj.weight`, `${L}.crossAttn.out.weight`); + +// Good: the field is the checkpoint's key, so the tensor binds as is +class DecoderLayer extends nn.Module { + readonly encoder_attn: DecoderCrossAttn; +} +``` + ## Magic number ```ts diff --git a/packages/runntime/src/core/nn/layers.ts b/packages/runntime/src/core/nn/layers.ts index 58b81b6e..4603b10a 100644 --- a/packages/runntime/src/core/nn/layers.ts +++ b/packages/runntime/src/core/nn/layers.ts @@ -189,7 +189,7 @@ export class Linear extends Module { override transformStateDict(sd: LazyStateDict, prefix: string): void { const name = prefix ? `${prefix}.weight` : 'weight'; const tensor = sd.tensors.get(name); - if (!tensor) return; // missing keys are the strict check's job, not ours + if (!tensor) return; // missing keys are the key check's job, not ours // Without this a square wrong-layout weight loads silently transposed. if ( tensor.shape.length !== 2 || @@ -336,7 +336,7 @@ export class Conv1d extends Module { override transformStateDict(sd: LazyStateDict, prefix: string): void { const name = prefix ? `${prefix}.weight` : 'weight'; let tensor = sd.tensors.get(name); - if (!tensor) return; // a missing key is reported by the strict key check + if (!tensor) return; // a missing key is reported by the key check if (tensor.kind === 'quant') { const { inChannels: cIn, outChannels: cOut, kernelSize: kernel } = this; checkQuantConvShape('Conv1d', name, tensor, { @@ -881,16 +881,16 @@ export class BatchNorm2d extends Module { /** RMSNorm over the last dim of width `n`. */ export class RMSNorm extends Module { - readonly scale: Parameter; + readonly weight: Parameter; constructor( readonly n: number, readonly eps = 1e-6, ) { super(); - this.scale = new Parameter({ elems: n, dtype: 'f32', dims: [n] }); + this.weight = new Parameter({ elems: n, dtype: 'f32', dims: [n] }); } forward(x: Value): Value { - return rmsNorm(x, this.scale.value, this.eps); + return rmsNorm(x, this.weight.value, this.eps); } } diff --git a/packages/runntime/src/core/nn/loadStateDict.ts b/packages/runntime/src/core/nn/loadStateDict.ts index 4d661883..09dd36c3 100644 --- a/packages/runntime/src/core/nn/loadStateDict.ts +++ b/packages/runntime/src/core/nn/loadStateDict.ts @@ -1,4 +1,4 @@ -/** Streaming state-dict loader: transform hooks, then a strict name check over +/** Streaming state-dict loader: transform hooks, then a name check over * headers alone, then per-tensor fetch, adapt, preprocess and upload. Peak * host memory is one tensor's working set. */ import type { TgpuRoot } from 'typegpu'; @@ -20,6 +20,11 @@ export interface LoadStateDictOpts { upload?: UploadFn; /** After each bound tensor: cumulative file bytes done / total. */ onProgress?: (name: string, doneBytes: number, totalBytes: number) => void; + /** Whether a tensor in the file that no parameter reads is an error + * (CHECKPOINT_MISMATCH) rather than a warning. Default false, like + * PyTorch's `strict=False`: the tensor is left unbound. A parameter the + * file lacks is always an error, as it has no value to fall back to. */ + strict?: boolean; } /** The default GPU upload. @@ -125,7 +130,7 @@ async function bindStateDict( const paramNames = new Set(params.map(([n]) => n)); const missing = params.filter(([n]) => !sd.tensors.has(n)).map(([n]) => n); const unexpected = [...sd.tensors.keys()].filter((n) => !paramNames.has(n)); - if (missing.length || unexpected.length) { + if (missing.length || (unexpected.length && opts.strict)) { throw new RunntimeError( 'CHECKPOINT_MISMATCH', 'loadStateDict: state dict does not match model.' + @@ -133,6 +138,11 @@ async function bindStateDict( (unexpected.length ? ` unexpected: ${list(unexpected)}.` : ''), ); } + if (unexpected.length) { + console.warn( + `loadStateDict: state dict has tensors the model does not read: ${list(unexpected)}.`, + ); + } // Bind in state-dict (file) order, so each chunk is read once. Module order // jumps around the payload and refetches evicted chunks. Keys a transform @@ -140,7 +150,8 @@ async function bindStateDict( const position = new Map([...sd.tensors.keys()].map((name, i) => [name, i])); params.sort(([a], [b]) => (position.get(a) ?? Infinity) - (position.get(b) ?? Infinity)); - const total = sd.totalBytes(); + // Unbound tensors are never read, so they do not count towards the total. + const total = params.reduce((sum, [name]) => sum + sd.tensors.get(name)!.byteLength, 0); let done = 0; for (const [name, p] of params) { const t = sd.tensors.get(name)!; diff --git a/packages/runntime/src/core/weights/transforms.ts b/packages/runntime/src/core/weights/transforms.ts index ab28f7ea..90461622 100644 --- a/packages/runntime/src/core/weights/transforms.ts +++ b/packages/runntime/src/core/weights/transforms.ts @@ -6,7 +6,7 @@ * `scaleWeight` and `dequantize`, which always give f32. BF16 decodes to * f32, which is exact; a derived tensor's Uint16Array means f16, so bf16 bits * have no derived form. Each output's byteLength is its share of the source - * bytes, so load progress still sums to the checkpoint size. + * bytes, so load progress counts every byte the model reads. * * A QuantTensor source gives a QuantTensor, except in `dequantize`: the * helpers move its ints and scales together. Whatever would split a scale diff --git a/packages/runntime/src/zoo/embeddinggemma2/audioEncoder.ts b/packages/runntime/src/zoo/embeddinggemma2/audioEncoder.ts index 6c7a3342..fbcf2769 100644 --- a/packages/runntime/src/zoo/embeddinggemma2/audioEncoder.ts +++ b/packages/runntime/src/zoo/embeddinggemma2/audioEncoder.ts @@ -195,8 +195,9 @@ export async function encodeAudio( const { mels } = model.cfg; const tokens = audioTokens(samples.length); const frames = frameCount(samples.length); - // A norm scale, not a Linear weight: a quantized weight is quantW either way. - const act = model.embedding_pre_projection_norm.scale.shape.dtype === 'f16' ? 'f16' : 'f32'; + // The pre-projection norm's weight, not a Linear's: a quantized Linear + // weight is quantW either way. + const act = model.embed_audio.embedding_pre_projection_norm.shape.dtype === 'f16' ? 'f16' : 'f32'; const patches = convPatches(logMel(samples, frames, mels), frames, mels); const taps = convTaps(Math.ceil(frames / 2), mels / 2); const inputs: EmbeddingGemma2AudioInputs = { diff --git a/packages/runntime/src/zoo/embeddinggemma2/audioModel.ts b/packages/runntime/src/zoo/embeddinggemma2/audioModel.ts index 7c9b623b..d80d47db 100644 --- a/packages/runntime/src/zoo/embeddinggemma2/audioModel.ts +++ b/packages/runntime/src/zoo/embeddinggemma2/audioModel.ts @@ -5,7 +5,10 @@ * half-weighted feed-forward, local attention over this token and the 11 * before it, a gated causal depthwise conv, a second feed-forward and a * norm. Most linears clamp their input and output to bounds from the - * checkpoint. */ + * checkpoint. + * + * Module fields are named like the checkpoint: the tower under + * `audio_tower`, its projection to the text model under `embed_audio`. */ import { add, @@ -42,28 +45,33 @@ export interface EmbeddingGemma2AudioInputs { tokens: number; } +/** A bound of a clipped linear, read on the CPU. */ +const bound = () => new nn.CpuParameter({ elems: 1, dtype: 'f32', dims: [1] }); + /** A linear that clamps its input and output to the checkpoint's bounds. */ -class ClippedLinear extends nn.Linear { - /** [inMin, inMax, outMin, outMax], read on the CPU. */ - readonly clip = new nn.CpuParameter({ elems: 4, dtype: 'f32', dims: [4] }); +class ClippedLinear extends nn.Module { + readonly linear: nn.Linear; + readonly input_min = bound(); + readonly input_max = bound(); + readonly output_min = bound(); + readonly output_max = bound(); constructor(inFeatures: number, outFeatures: number) { - super(inFeatures, outFeatures, { bias: false }); + super(); + this.linear = new nn.Linear(inFeatures, outFeatures, { bias: false }); } /** Bounds of the output. */ get outBounds(): { lo: number; hi: number } { - const [, , lo = -Infinity, hi = Infinity] = this.clip.data; - return { lo, hi }; + return { lo: this.output_min.data[0]!, hi: this.output_max.data[0]! }; } /** The matmul on the clamped input, its output left unclamped. */ unclamped(x: Value): Value { - const [lo = -Infinity, hi = Infinity] = this.clip.data; - return super.forward(clamp(x, lo, hi)); + return this.linear.forward(clamp(x, this.input_min.data[0]!, this.input_max.data[0]!)); } - override forward(x: Value): Value { + forward(x: Value): Value { const { lo, hi } = this.outBounds; return clamp(this.unclamped(x), lo, hi); } @@ -73,7 +81,7 @@ class FeedForward extends nn.Module<[Value], Value> { readonly pre_layer_norm: nn.RMSNorm; readonly ffw_layer_1: ClippedLinear; readonly ffw_layer_2: ClippedLinear; - /** Its scale carries the residual weight. */ + /** Its weight carries the residual weight, folded in at load. */ readonly post_layer_norm: nn.RMSNorm; constructor({ hidden, ffn, eps }: EmbeddingGemma2AudioConfig) { @@ -87,7 +95,21 @@ class FeedForward extends nn.Module<[Value], Value> { override forward(x: Value): Value { const h = silu(this.ffw_layer_1.forward(this.pre_layer_norm.forward(x))); const post = this.post_layer_norm; - return rmsNorm(this.ffw_layer_2.forward(h), post.scale.value, post.eps, { residual: x }); + return rmsNorm(this.ffw_layer_2.forward(h), post.weight.value, post.eps, { residual: x }); + } +} + +/** The depthwise conv's taps [convKernel, hidden], oldest first, permuted + * from the checkpoint's [hidden, 1, convKernel] at load. */ +class DepthwiseConv extends nn.Module { + readonly weight: nn.Parameter; + constructor({ hidden, convKernel }: EmbeddingGemma2AudioConfig) { + super(); + this.weight = new nn.Parameter({ + elems: convKernel * hidden, + dtype: 'f32', + dims: [convKernel, hidden], + }); } } @@ -95,49 +117,41 @@ class LightConv extends nn.Module<[Value], Value> { readonly pre_layer_norm: nn.RMSNorm; /** The two GLU halves, value then gate. */ readonly linear_start: ClippedLinear; - /** Depthwise taps [convKernel, hidden], oldest first. */ - readonly conv: nn.Parameter; + readonly depthwise_conv1d: DepthwiseConv; readonly conv_norm: nn.RMSNorm; readonly linear_end: ClippedLinear; - constructor({ hidden, eps, convKernel }: EmbeddingGemma2AudioConfig) { + constructor(cfg: EmbeddingGemma2AudioConfig) { super(); + const { hidden, eps } = cfg; this.pre_layer_norm = new nn.RMSNorm(hidden, eps); this.linear_start = new ClippedLinear(hidden, 2 * hidden); - this.conv = new nn.Parameter({ - elems: convKernel * hidden, - dtype: 'f32', - dims: [convKernel, hidden], - }); + this.depthwise_conv1d = new DepthwiseConv(cfg); this.conv_norm = new nn.RMSNorm(hidden, eps); this.linear_end = new ClippedLinear(hidden, hidden); } override forward(x: Value): Value { const z = this.linear_start.unclamped(this.pre_layer_norm.forward(x)); - const g = gluConv1d(z, this.conv.value, this.linear_start.outBounds); + const g = gluConv1d(z, this.depthwise_conv1d.weight.value, this.linear_start.outBounds); return add(this.linear_end.forward(silu(this.conv_norm.forward(g))), x); } } -class AudioLayer extends nn.Module<[Value], Value> { - readonly feed_forward1: FeedForward; - readonly norm_pre_attn: nn.RMSNorm; - /** q | k | v rows. */ +/** Local attention: the projections fused at load, the distance bias and q + * scale computed from the checkpoint at load. */ +class AudioAttention extends nn.Module { + /** q | k | v rows, with the bounds of all three. */ readonly qkv: ClippedLinear; - /** One bias row per key distance [window, hidden], f32. */ + /** One bias row per key distance [window, hidden], f32, from + * `relative_k_proj`. */ readonly rel: nn.Parameter; - /** Scale of each column of q [headDim], f32. */ + /** Scale of each column of q [headDim], f32, from `per_dim_scale`. */ readonly q_scale: nn.Parameter; readonly post: ClippedLinear; - readonly norm_post_attn: nn.RMSNorm; - readonly lconv: LightConv; - readonly feed_forward2: FeedForward; - readonly norm_out: nn.RMSNorm; - constructor(private readonly cfg: EmbeddingGemma2AudioConfig) { + constructor({ hidden, headDim, window }: EmbeddingGemma2AudioConfig) { super(); - const { hidden, headDim, window, eps } = cfg; // Pinned: the attention kernel reads them as f32. const f32 = (...dims: number[]) => new nn.Parameter( @@ -145,23 +159,40 @@ class AudioLayer extends nn.Module<[Value], Value> { undefined, true, ); - this.feed_forward1 = new FeedForward(cfg); - this.norm_pre_attn = new nn.RMSNorm(hidden, eps); this.qkv = new ClippedLinear(hidden, 3 * hidden); this.rel = f32(window, hidden); this.q_scale = f32(headDim); this.post = new ClippedLinear(hidden, hidden); + } +} + +class AudioLayer extends nn.Module<[Value], Value> { + readonly feed_forward1: FeedForward; + readonly norm_pre_attn: nn.RMSNorm; + readonly self_attn: AudioAttention; + readonly norm_post_attn: nn.RMSNorm; + readonly lconv1d: LightConv; + readonly feed_forward2: FeedForward; + readonly norm_out: nn.RMSNorm; + + constructor(private readonly cfg: EmbeddingGemma2AudioConfig) { + super(); + const { hidden, eps } = cfg; + this.feed_forward1 = new FeedForward(cfg); + this.norm_pre_attn = new nn.RMSNorm(hidden, eps); + this.self_attn = new AudioAttention(cfg); this.norm_post_attn = new nn.RMSNorm(hidden, eps); - this.lconv = new LightConv(cfg); + this.lconv1d = new LightConv(cfg); this.feed_forward2 = new FeedForward(cfg); this.norm_out = new nn.RMSNorm(hidden, eps); } override forward(x: Value): Value { const { heads, headDim, window, logitCap } = this.cfg; + const { self_attn } = this; x = this.feed_forward1.forward(x); - const qkv = this.qkv.forward(this.norm_pre_attn.forward(x)); - const a = localAttention(qkv, this.rel.value, this.q_scale.value, { + const qkv = self_attn.qkv.forward(this.norm_pre_attn.forward(x)); + const a = localAttention(qkv, self_attn.rel.value, self_attn.q_scale.value, { heads, headDim, window, @@ -169,55 +200,101 @@ class AudioLayer extends nn.Module<[Value], Value> { softcap: logitCap, }); const norm = this.norm_post_attn; - x = rmsNorm(this.post.forward(a), norm.scale.value, norm.eps, { residual: x }); - x = this.lconv.forward(x); + x = rmsNorm(self_attn.post.forward(a), norm.weight.value, norm.eps, { residual: x }); + x = this.lconv1d.forward(x); return this.norm_out.forward(this.feed_forward2.forward(x)); } } -export class EmbeddingGemma2AudioModel extends nn.Module<[EmbeddingGemma2AudioInputs], Value> { - readonly conv0: nn.Linear; - readonly norm0: nn.LayerNorm; - readonly conv1: nn.Linear; - readonly norm1: nn.LayerNorm; - readonly input_proj: nn.Linear; +/** A stride-2 conv as a matmul over its taps, LayerNorm over channels, ReLU. */ +class ConvLayer extends nn.Module { + readonly conv: nn.Linear; + readonly norm: nn.LayerNorm; + constructor(taps: number, channels: number, eps: number) { + super(); + this.conv = new nn.Linear(taps, channels, { bias: false }); + this.norm = new nn.LayerNorm(channels, eps); + } + forward(x: Value): Value { + return clamp(this.norm.forward(this.conv.forward(x)), 0, Infinity); + } +} + +/** The two convs over the mel frames and the projection to the layers' + * width. layer0 reads [P + 1, 9] taps of the one-channel input, layer1 the + * [P', 9·c0] taps of layer0's output; both weights permute at load. */ +class SubsampleConvProjection extends nn.Module { + readonly layer0: ConvLayer; + readonly layer1: ConvLayer; + readonly input_proj_linear: nn.Linear; + constructor({ hidden, mels, eps, convChannels: [c0, c1] }: EmbeddingGemma2AudioConfig) { + super(); + this.layer0 = new ConvLayer(9, c0, eps); + this.layer1 = new ConvLayer(9 * c0, c1, eps); + this.input_proj_linear = new nn.Linear((mels / 4) * c1, hidden, { bias: false }); + } +} + +class AudioTower extends nn.Module { + readonly subsample_conv_projection: SubsampleConvProjection; readonly layers: nn.ModuleList; readonly output_proj: nn.Linear; - readonly embedding_pre_projection_norm: nn.RMSNorm; - readonly embedding_projection: nn.Linear; - - constructor(readonly cfg: EmbeddingGemma2AudioConfig = EMBEDDINGGEMMA2_AUDIO) { + constructor(cfg: EmbeddingGemma2AudioConfig) { super(); - const { hidden, mels, eps, outputDim } = cfg; - const [c0, c1] = cfg.convChannels; - this.conv0 = new nn.Linear(9, c0, { bias: false }); - this.norm0 = new nn.LayerNorm(c0, eps); - this.conv1 = new nn.Linear(9 * c0, c1, { bias: false }); - this.norm1 = new nn.LayerNorm(c1, eps); - this.input_proj = new nn.Linear((mels / 4) * c1, hidden, { bias: false }); + this.subsample_conv_projection = new SubsampleConvProjection(cfg); this.layers = new nn.ModuleList(Array.from({ length: cfg.layers }, () => new AudioLayer(cfg))); - this.output_proj = new nn.Linear(hidden, outputDim); - this.embedding_pre_projection_norm = new nn.RMSNorm(outputDim, eps); + this.output_proj = new nn.Linear(cfg.hidden, cfg.outputDim); + } +} + +/** The tower output's projection to the text model's width. */ +class EmbedAudio extends nn.Module { + private readonly eps: number; + /** A weightless norm: its weight is ones, a Buffer since the checkpoint + * has none. */ + readonly embedding_pre_projection_norm: nn.Buffer; + readonly embedding_projection: nn.Linear; + constructor({ outputDim, eps }: EmbeddingGemma2AudioConfig) { + super(); + this.eps = eps; + this.embedding_pre_projection_norm = new nn.Buffer(new Float32Array(outputDim).fill(1), { + elems: outputDim, + dtype: 'f32', + dims: [outputDim], + }); this.embedding_projection = new nn.Linear(outputDim, EMBEDDINGGEMMA2_TEXT.hidden, { bias: false, }); } + forward(x: Value): Value { + return this.embedding_projection.forward( + rmsNorm(x, this.embedding_pre_projection_norm.value, this.eps), + ); + } +} + +export class EmbeddingGemma2AudioModel extends nn.Module<[EmbeddingGemma2AudioInputs], Value> { + readonly audio_tower: AudioTower; + readonly embed_audio: EmbedAudio; + + constructor(readonly cfg: EmbeddingGemma2AudioConfig = EMBEDDINGGEMMA2_AUDIO) { + super(); + this.audio_tower = new AudioTower(cfg); + this.embed_audio = new EmbedAudio(cfg); + } /** One clip to its tokens [tokens, text hidden], ready to stand in for * the placeholder rows of the text model's input. */ override forward(inp: EmbeddingGemma2AudioInputs): Value { const [c0, c1] = this.cfg.convChannels; - const relu = (v: Value) => clamp(v, 0, Infinity); - // Each conv is a matmul over its taps, LayerNorm over channels, ReLU. - const y0 = relu(this.norm0.forward(this.conv0.forward(inp.patches))); + const { subsample_conv_projection: sub, layers, output_proj } = this.audio_tower; + const y0 = sub.layer0.forward(inp.patches); const pixels = inp.taps.shape.elems / 9; - const y1 = reshape(gatherRowsFrom(y0, inp.taps), [pixels, 9 * c0]); - const y = relu(this.norm1.forward(this.conv1.forward(y1))); + const y = sub.layer1.forward(reshape(gatherRowsFrom(y0, inp.taps), [pixels, 9 * c0])); // Rows are (token, frequency); each token's frequencies become one row. - let x = this.input_proj.forward(reshape(y, [inp.tokens, (pixels / inp.tokens) * c1])); - for (const layer of this.layers.items) x = layer.forward(x); - const out = this.embedding_pre_projection_norm.forward(this.output_proj.forward(x)); - return this.embedding_projection.forward(out); + let x = sub.input_proj_linear.forward(reshape(y, [inp.tokens, (pixels / inp.tokens) * c1])); + for (const layer of layers.items) x = layer.forward(x); + return this.embed_audio.forward(output_proj.forward(x)); } override transformStateDict(sd: LazyStateDict, prefix: string): void { diff --git a/packages/runntime/src/zoo/embeddinggemma2/embedCore.ts b/packages/runntime/src/zoo/embeddinggemma2/embedCore.ts index b09871cd..945a5c4f 100644 --- a/packages/runntime/src/zoo/embeddinggemma2/embedCore.ts +++ b/packages/runntime/src/zoo/embeddinggemma2/embedCore.ts @@ -88,7 +88,7 @@ export function createEmbeddingGemma2EmbedCore( const { hidden, embeddingDim, attention } = model.cfg; // x0 and the pool weights meet the hidden states, so they follow the // model's dtype; the index planes stay f32. - const act = model.norm.scale.shape.dtype === 'f16' ? 'f16' : 'f32'; + const act = model.language_model.norm.weight.shape.dtype === 'f16' ? 'f16' : 'f32'; const maxT = EMBEDDINGGEMMA2_MAX_TOKENS; // Rope rows for every position, gathered per forward by the positions plane. const table = (type: LayerType) => { diff --git a/packages/runntime/src/zoo/embeddinggemma2/model.ts b/packages/runntime/src/zoo/embeddinggemma2/model.ts index 2a7be926..92a255f4 100644 --- a/packages/runntime/src/zoo/embeddinggemma2/model.ts +++ b/packages/runntime/src/zoo/embeddinggemma2/model.ts @@ -9,7 +9,10 @@ * Built for few dispatches: q/k/v are one matmul, their per-head norms one * grouped RMSNorm, and every post-norm also adds the residual. GELU rides * the gate matmul's epilogue. Long inputs run attention per head as two - * matmuls around a masked softmax, short ones as one sdpa. */ + * matmuls around a masked softmax, short ones as one sdpa. + * + * Module fields are named like the checkpoint: everything sits under + * `language_model`. */ import { cat, @@ -48,22 +51,62 @@ const MATMUL_ATTENTION_MIN_SEGMENT = 256; * outside its window. */ const MATMUL_ATTENTION_MAX_SLIDING_ROWS = 1024; -class Layer extends nn.Module<[Value, Value, EmbeddingGemma2Inputs], Value> { - readonly input_layernorm: nn.RMSNorm; +/** The attention projections and per-head norms, fused at load. */ +class Attention extends nn.Module { /** q | k | v rows, with q and k permuted for the interleaved rope. */ readonly qkv: nn.Linear; /** q_norm tiled per q head, k_norm per kv head, ones for v. */ readonly qkv_norm: nn.Parameter; readonly o_proj: nn.Linear; - readonly post_attention_layernorm: nn.RMSNorm; - readonly pre_feedforward_layernorm: nn.RMSNorm; + constructor(hidden: number, heads: number, kvHeads: number, headDim: number) { + super(); + const width = (heads + 2 * kvHeads) * headDim; + this.qkv = new nn.Linear(hidden, width, { bias: false }); + this.qkv_norm = new nn.Parameter({ elems: width, dtype: 'f32', dims: [width] }); + this.o_proj = new nn.Linear(heads * headDim, hidden, { bias: false }); + } +} + +/** GeGLU MLP, GELU in the gate matmul's epilogue. */ +class Mlp extends nn.Module { readonly gate_proj: nn.Linear; readonly up_proj: nn.Linear; readonly down_proj: nn.Linear; - readonly post_feedforward_layernorm: nn.RMSNorm; + constructor(hidden: number, ffn: number) { + super(); + this.gate_proj = new nn.Linear(hidden, ffn, { bias: false, activation: 'geluTanh' }); + this.up_proj = new nn.Linear(hidden, ffn, { bias: false }); + this.down_proj = new nn.Linear(ffn, hidden, { bias: false }); + } + forward(x: Value): Value { + return this.down_proj.forward(mul(this.gate_proj.forward(x), this.up_proj.forward(x))); + } +} + +/** The PLE gate and projection, between the layer's output and its norm. */ +class PleBlock extends nn.Module { readonly per_layer_input_gate: nn.Linear; readonly per_layer_projection: nn.Linear; readonly post_per_layer_input_norm: nn.RMSNorm; + constructor({ hidden, pleDim, eps }: EmbeddingGemma2Config) { + super(); + this.per_layer_input_gate = new nn.Linear(hidden, pleDim, { + bias: false, + activation: 'geluTanh', + }); + this.per_layer_projection = new nn.Linear(pleDim, hidden, { bias: false }); + this.post_per_layer_input_norm = new nn.RMSNorm(hidden, eps); + } +} + +class Layer extends nn.Module<[Value, Value, EmbeddingGemma2Inputs], Value> { + readonly input_layernorm: nn.RMSNorm; + readonly self_attn: Attention; + readonly post_attention_layernorm: nn.RMSNorm; + readonly pre_feedforward_layernorm: nn.RMSNorm; + readonly mlp: Mlp; + readonly post_feedforward_layernorm: nn.RMSNorm; + readonly ple_block: PleBlock; /** One number that scales the layer's output; read on the CPU. */ readonly layer_scalar: nn.CpuParameter; @@ -72,26 +115,16 @@ class Layer extends nn.Module<[Value, Value, EmbeddingGemma2Inputs], Value> { readonly type: LayerType, ) { super(); - const { hidden, heads, ffn, pleDim, eps } = cfg; + const { hidden, heads, ffn, eps } = cfg; const { headDim, kvHeads } = cfg.attention[type]; - const width = (heads + 2 * kvHeads) * headDim; const norm = () => new nn.RMSNorm(hidden, eps); this.input_layernorm = norm(); - this.qkv = new nn.Linear(hidden, width, { bias: false }); - this.qkv_norm = new nn.Parameter({ elems: width, dtype: 'f32', dims: [width] }); - this.o_proj = new nn.Linear(heads * headDim, hidden, { bias: false }); + this.self_attn = new Attention(hidden, heads, kvHeads, headDim); this.post_attention_layernorm = norm(); this.pre_feedforward_layernorm = norm(); - this.gate_proj = new nn.Linear(hidden, ffn, { bias: false, activation: 'geluTanh' }); - this.up_proj = new nn.Linear(hidden, ffn, { bias: false }); - this.down_proj = new nn.Linear(ffn, hidden, { bias: false }); + this.mlp = new Mlp(hidden, ffn); this.post_feedforward_layernorm = norm(); - this.per_layer_input_gate = new nn.Linear(hidden, pleDim, { - bias: false, - activation: 'geluTanh', - }); - this.per_layer_projection = new nn.Linear(pleDim, hidden, { bias: false }); - this.post_per_layer_input_norm = norm(); + this.ple_block = new PleBlock(cfg); this.layer_scalar = new nn.CpuParameter({ elems: 1, dtype: 'f32', dims: [1] }); } @@ -101,12 +134,13 @@ class Layer extends nn.Module<[Value, Value, EmbeddingGemma2Inputs], Value> { const { cos, sin } = inp.rope[this.type]; const window = this.type === 'sliding' ? slidingWindow : Infinity; const postNorm = (norm: nn.RMSNorm, y: Value, residual: Value, scale?: number) => - rmsNorm(y, norm.scale.value, norm.eps, { residual, scale }); + rmsNorm(y, norm.weight.value, norm.eps, { residual, scale }); // Attention: qkv, one grouped norm for all heads, rope q and k. Columns // of n: q heads, then k heads, then v heads. - const qkv = this.qkv.forward(this.input_layernorm.forward(x)); - const n = rmsNorm(qkv, this.qkv_norm.value, eps, { group: d }); + const { self_attn, ple_block } = this; + const qkv = self_attn.qkv.forward(this.input_layernorm.forward(x)); + const n = rmsNorm(qkv, self_attn.qkv_norm.value, eps, { group: d }); const roped = (start: number, width: number) => rope(n, cos, sin, { headDim: d, srcStart: start, width }); const vStart = (heads + kvHeads) * d; @@ -144,17 +178,15 @@ class Layer extends nn.Module<[Value, Value, EmbeddingGemma2Inputs], Value> { }, ); } - x = postNorm(this.post_attention_layernorm, this.o_proj.forward(a), x); + x = postNorm(this.post_attention_layernorm, self_attn.o_proj.forward(a), x); - // GeGLU MLP, GELU in the gate matmul's epilogue. - const f = this.pre_feedforward_layernorm.forward(x); - const m = this.down_proj.forward(mul(this.gate_proj.forward(f), this.up_proj.forward(f))); + const m = this.mlp.forward(this.pre_feedforward_layernorm.forward(x)); x = postNorm(this.post_feedforward_layernorm, m, x); // PLE: gate against this layer's slice, project back, then scale the layer. - const g = mul(this.per_layer_input_gate.forward(x), perLayer); - const p = this.per_layer_projection.forward(g); - return postNorm(this.post_per_layer_input_norm, p, x, this.layer_scalar.data[0]!); + const g = mul(ple_block.per_layer_input_gate.forward(x), perLayer); + const p = ple_block.per_layer_projection.forward(g); + return postNorm(ple_block.post_per_layer_input_norm, p, x, this.layer_scalar.data[0]!); } } @@ -171,19 +203,20 @@ class Ple extends nn.Module<[Value], Value> { * hidden^-½ scale is folded into the projection at load. */ override forward(x: Value): Value { const norm = this.per_layer_projection_norm; - return rmsNorm(this.per_layer_model_projection.forward(x), norm.scale.value, norm.eps, { + return rmsNorm(this.per_layer_model_projection.forward(x), norm.weight.value, norm.eps, { group: this.cfg.pleDim, }); } } -export class EmbeddingGemma2Model extends nn.Module<[EmbeddingGemma2Inputs], Value> { +/** The checkpoint's `language_model`: the whole text model. */ +class LanguageModel extends nn.Module { readonly embed_tokens: nn.Embedding; readonly ple: Ple; readonly layers: nn.ModuleList; readonly norm: nn.RMSNorm; readonly embedding_projection: nn.Linear; - constructor(readonly cfg: EmbeddingGemma2Config = EMBEDDINGGEMMA2_TEXT) { + constructor(cfg: EmbeddingGemma2Config) { super(); // 262k rows of 512, and a call reads a handful: keep it on the CPU. this.embed_tokens = new nn.Embedding(cfg.vocab, cfg.hidden, { device: 'cpu' }); @@ -192,11 +225,19 @@ export class EmbeddingGemma2Model extends nn.Module<[EmbeddingGemma2Inputs], Val this.norm = new nn.RMSNorm(cfg.hidden, cfg.eps); this.embedding_projection = new nn.Linear(cfg.hidden, cfg.embeddingDim, { bias: false }); } +} + +export class EmbeddingGemma2Model extends nn.Module<[EmbeddingGemma2Inputs], Value> { + readonly language_model: LanguageModel; + constructor(readonly cfg: EmbeddingGemma2Config = EMBEDDINGGEMMA2_TEXT) { + super(); + this.language_model = new LanguageModel(cfg); + } /** Token ids to input rows [ids.length · hidden], scaled by √hidden, * written into `out` at row `offset`. */ embedRows(ids: readonly number[], out: Float32Array, offset: number): void { - const rows = this.embed_tokens.gather(ids); + const rows = this.language_model.embed_tokens.gather(ids); const s = Math.sqrt(this.cfg.hidden); const base = offset * this.cfg.hidden; for (let i = 0; i < rows.length; i++) out[base + i] = rows[i]! * s; @@ -205,15 +246,16 @@ export class EmbeddingGemma2Model extends nn.Module<[EmbeddingGemma2Inputs], Val /** Packed inputs to their pooled, projected embeddings [B, embeddingDim], * not yet unit length. */ override forward(inp: EmbeddingGemma2Inputs): Value { - const perLayer = this.ple.forward(inp.x0); + const { ple, layers, norm, embedding_projection } = this.language_model; + const perLayer = ple.forward(inp.x0); const pd = this.cfg.pleDim; let x = inp.x0; - this.layers.items.forEach((layer, i) => { + layers.items.forEach((layer, i) => { x = layer.forward(x, slice(perLayer, 1, i * pd, (i + 1) * pd), inp); }); // Pooling before the projection: both are linear, and this projects B // rows instead of T. - return this.embedding_projection.forward(matmul(inp.selector, this.norm.forward(x))); + return embedding_projection.forward(matmul(inp.selector, norm.forward(x))); } override transformStateDict(sd: LazyStateDict, prefix: string): void { diff --git a/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts b/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts index 8e1d069d..9bd2ccee 100644 --- a/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts +++ b/packages/runntime/src/zoo/embeddinggemma2/stateDictHooks.ts @@ -1,7 +1,7 @@ -/** Maps the HF EmbeddingGemma2Model checkpoint onto the text model and the - * vision and audio towers as lazy derived tensors; nothing is fetched here. - * Each keeps only its own tensors, so one checkpoint with every tower loads - * into any of them. +/** Derives what the HF EmbeddingGemma2Model checkpoint stores differently + * from the text model's and the vision and audio towers' parameters, as + * lazy tensors; nothing is fetched here. Every other tensor binds under its + * own name, since the modules are named like the checkpoint. * * An F16 checkpoint gives f16 derived tensors, except the text layer * scalars and the audio tower's clip bounds, distance bias rows and q @@ -28,9 +28,15 @@ import type { EmbeddingGemma2VisionConfig, } from './config.ts'; -const SOURCE = 'language_model.'; -const VISION_SOURCES = ['vision_tower.', 'embed_vision.'] as const; -const AUDIO_SOURCES = ['audio_tower.', 'embed_audio.'] as const; +const qualify = (prefix: string, name: string): string => (prefix ? `${prefix}.${name}` : name); + +/** Takes a tensor out of `sd`, for a derived one to replace. */ +function take(sd: LazyStateDict, what: string, name: string): LazyTensor { + const t = sd.tensors.get(name); + if (!t) throw checkpointMismatch(what, `missing tensor '${name}'`); + sd.tensors.delete(name); + return t; +} /** Row order for permuteRows that takes each `block` rows from the * checkpoint's half-split rope layout to the kernel's interleaved pairs: @@ -42,39 +48,6 @@ function ropeOrder(rows: number, block: number): number[] { }); } -/** Each layer's RMSNorms, renamed `weight` → `scale`. */ -const LAYER_NORMS = [ - 'input_layernorm', - 'post_attention_layernorm', - 'pre_feedforward_layernorm', - 'post_feedforward_layernorm', -] as const; - -/** Moves the tensors under `sources` out of `sd`. take() throws for a - * missing one, done() for any left unused. */ -function remap(sd: LazyStateDict, prefix: string, sources: readonly string[], what: string) { - const src = new Map(); - for (const [key, t] of sd.tensors) { - if (sources.some((s) => key.startsWith(s))) src.set(key, t); - } - sd.tensors.clear(); - return { - take(name: string): LazyTensor { - const t = src.get(name); - if (!t) throw checkpointMismatch(what, `missing tensor '${name}'`); - src.delete(name); - return t; - }, - put(name: string, t: LazyTensor): void { - sd.tensors.set(prefix ? `${prefix}.${name}` : name, t); - }, - done(): void { - if (src.size) - throw checkpointMismatch(what, `unused tensors: ${[...src.keys()].slice(0, 5)}`); - }, - }; -} - /** A weightless norm's ones, as f16 halfwords: exact, and stacked with an F16 * checkpoint's norms they keep the result f16. */ const ones = (n: number) => @@ -117,125 +90,99 @@ export function transformEmbeddingGemma2StateDict( prefix: string, cfg: EmbeddingGemma2Config, ): void { - const io = remap(sd, prefix, [SOURCE], 'embeddinggemma2 state dict'); - const take = (name: string) => io.take(SOURCE + name); - const { put } = io; + const what = 'embeddinggemma2 state dict'; + const q = (name: string) => qualify(prefix, `language_model.${name}`); + const put = (name: string, t: LazyTensor) => sd.tensors.set(q(name), t); for (const [i, type] of cfg.layerTypes.entries()) { const L = `layers.${i}`; const { headDim: d, kvHeads } = cfg.attention[type]; - const attn = (n: string) => take(`${L}.self_attn.${n}.weight`); - put(`${L}.qkv.weight`, fusedQkv(attn('q_proj'), attn('k_proj'), attn('v_proj'), d)); - put(`${L}.qkv_norm`, fusedQkvNorm(attn('q_norm'), attn('k_norm'), cfg.heads, kvHeads, d)); - put(`${L}.o_proj.weight`, attn('o_proj')); - for (const n of ['gate_proj', 'up_proj', 'down_proj']) { - put(`${L}.${n}.weight`, take(`${L}.mlp.${n}.weight`)); - } - for (const n of ['per_layer_input_gate', 'per_layer_projection']) { - put(`${L}.${n}.weight`, take(`${L}.ple_block.${n}.weight`)); - } + const attn = (n: string) => take(sd, what, q(`${L}.self_attn.${n}.weight`)); + put(`${L}.self_attn.qkv.weight`, fusedQkv(attn('q_proj'), attn('k_proj'), attn('v_proj'), d)); put( - `${L}.post_per_layer_input_norm.scale`, - take(`${L}.ple_block.post_per_layer_input_norm.weight`), + `${L}.self_attn.qkv_norm`, + fusedQkvNorm(attn('q_norm'), attn('k_norm'), cfg.heads, kvHeads, d), ); - for (const n of LAYER_NORMS) put(`${L}.${n}.scale`, take(`${L}.${n}.weight`)); - const s = take(`${L}.layer_scalar`); + const s = take(sd, what, q(`${L}.layer_scalar`)); // Derived, so the CPU parameter binds f32 rather than raw bf16 bits. put( `${L}.layer_scalar`, derivedTensor([1], () => s.f32(), s.byteLength), ); } - put( - 'ple.per_layer_model_projection.weight', - scaleWeight(take('ple.per_layer_model_projection.weight'), cfg.hidden ** -0.5), - ); - put('ple.per_layer_projection_norm.scale', take('ple.per_layer_projection_norm.weight')); - put('embed_tokens.weight', take('embed_tokens.weight')); - put('norm.scale', take('norm.weight')); - put('embedding_projection.weight', take('embedding_projection.weight')); - io.done(); + const ple = 'ple.per_layer_model_projection.weight'; + put(ple, scaleWeight(take(sd, what, q(ple)), cfg.hidden ** -0.5)); } /** The vision tower's mapping. Its 2D rope turns each half head on its * own, so q and k interleave per half head. The position table splits - * into x and y, and the weightless norm gets ones. */ + * into x and y. */ export function transformEmbeddingGemma2VisionStateDict( sd: LazyStateDict, prefix: string, cfg: EmbeddingGemma2VisionConfig, ): void { - const { take, put, done } = remap( - sd, - prefix, - VISION_SOURCES, - 'embeddinggemma2 vision state dict', - ); + const what = 'embeddinggemma2 vision state dict'; + const q = (name: string) => qualify(prefix, `vision_tower.${name}`); + const put = (name: string, t: LazyTensor) => sd.tensors.set(q(name), t); const { hidden, heads, headDim: d, positions } = cfg; - put('input_proj.weight', take('vision_tower.patch_embedder.input_proj.weight')); // [2, positions, hidden]: the x table, then the y table. - const [x, y] = splitRows(take('vision_tower.patch_embedder.position_embedding_table'), [1, 1]); - put('pos_x.weight', reshapeWeight(x!, [positions, hidden])); - put('pos_y.weight', reshapeWeight(y!, [positions, hidden])); + const [x, y] = splitRows(take(sd, what, q('patch_embedder.position_embedding_table')), [1, 1]); + put('patch_embedder.pos_x.weight', reshapeWeight(x!, [positions, hidden])); + put('patch_embedder.pos_y.weight', reshapeWeight(y!, [positions, hidden])); for (let i = 0; i < cfg.layers; i++) { - const S = `vision_tower.encoder.layers.${i}`; - const L = `layers.${i}`; - const attn = (n: string) => take(`${S}.self_attn.${n}.weight`); - const proj = (n: string) => take(`${S}.self_attn.${n}.linear.weight`); - put(`${L}.qkv.weight`, fusedQkv(proj('q_proj'), proj('k_proj'), proj('v_proj'), d / 2)); - put(`${L}.qkv_norm`, fusedQkvNorm(attn('q_norm'), attn('k_norm'), heads, heads, d / 2)); - put(`${L}.o_proj.weight`, proj('o_proj')); - for (const n of ['gate_proj', 'up_proj', 'down_proj']) { - put(`${L}.${n}.weight`, take(`${S}.mlp.${n}.linear.weight`)); - } - for (const n of LAYER_NORMS) put(`${L}.${n}.scale`, take(`${S}.${n}.weight`)); + const A = `encoder.layers.${i}.self_attn`; + const attn = (n: string) => take(sd, what, q(`${A}.${n}.weight`)); + const proj = (n: string) => take(sd, what, q(`${A}.${n}.linear.weight`)); + put(`${A}.qkv.weight`, fusedQkv(proj('q_proj'), proj('k_proj'), proj('v_proj'), d / 2)); + put(`${A}.qkv_norm`, fusedQkvNorm(attn('q_norm'), attn('k_norm'), heads, heads, d / 2)); } - put('embedding_pre_projection_norm.scale', ones(hidden)); - put('embedding_projection.weight', take('embed_vision.embedding_projection.weight')); - done(); } -/** The audio tower's mapping. Each clipped linear's four bounds become one - * CPU tensor [inMin, inMax, outMin, outMax]. q, k and v fuse into one - * weight; they share their bounds. Each layer's distance bias rows and q - * column scales are computed here, and the feed-forward residual weight - * folds into its post norm. */ +/** The audio tower's mapping. The two conv weights take the taps-major + * layout the convs read as matmuls. q, k and v fuse into one weight and + * share their bounds. Each layer's distance bias rows and q column scales + * are computed here, and the feed-forward residual weight folds into its + * post norm. */ export function transformEmbeddingGemma2AudioStateDict( sd: LazyStateDict, prefix: string, cfg: EmbeddingGemma2AudioConfig, ): void { const what = 'embeddinggemma2 audio state dict'; - const { take, put, done } = remap(sd, prefix, AUDIO_SOURCES, what); + const q = (name: string) => qualify(prefix, `audio_tower.${name}`); + const put = (name: string, t: LazyTensor) => sd.tensors.set(q(name), t); const { hidden, headDim, convKernel: K, window } = cfg; const [c0, c1] = cfg.convChannels; - const S = 'audio_tower.'; const scalar = async (t: LazyTensor) => (await t.f32())[0]!; - const bounds = (name: string) => - ['input_min', 'input_max', 'output_min', 'output_max'].map((b) => take(`${name}.${b}`)); - const clip = (b: LazyTensor[]) => - derivedTensor([4], async () => Float32Array.from(await Promise.all(b.map(scalar))), 0); - const clipped = (from: string, to: string) => { - put(`${to}.weight`, take(`${from}.linear.weight`)); - put(`${to}.clip`, clip(bounds(from))); + const BOUNDS = ['input_min', 'input_max', 'output_min', 'output_max']; + /** Rebuilds a clipped linear's bounds as f32: the CPU parameters they bind + * to would hold a BF16 bound as raw bits. */ + const f32Bounds = (linear: string) => { + for (const b of BOUNDS) { + const t = take(sd, what, q(`${linear}.${b}`)); + put( + `${linear}.${b}`, + derivedTensor([1], () => t.f32(), t.byteLength), + ); + } }; // The first conv has one input channel: [c0, 9] as a linear over 3×3 // taps. The second reads its taps (row, column, channel) major, so each // output's [c0, 9] block becomes [9, c0], one element at a time. - const sub = `${S}subsample_conv_projection.`; - put('conv0.weight', reshapeWeight(take(`${sub}layer0.conv.weight`), [c0, 9])); - put('norm0.weight', take(`${sub}layer0.norm.weight`)); + const sub = 'subsample_conv_projection.'; + const conv0 = `${sub}layer0.conv.weight`; + put(conv0, reshapeWeight(take(sd, what, q(conv0)), [c0, 9])); const tapsMajor = Array.from({ length: c1 * 9 * c0 }, (_, j) => { const [o, k, c] = [Math.floor(j / (9 * c0)), Math.floor(j / c0) % 9, j % c0]; return (o * c0 + c) * 9 + k; }); - const conv1 = reshapeWeight(take(`${sub}layer1.conv.weight`), [c1 * c0 * 9]); - put('conv1.weight', reshapeWeight(permuteRows(conv1, tapsMajor), [c1, 9 * c0])); - put('norm1.weight', take(`${sub}layer1.norm.weight`)); - put('input_proj.weight', take(`${sub}input_proj_linear.weight`)); + const conv1 = `${sub}layer1.conv.weight`; + const taps = reshapeWeight(take(sd, what, q(conv1)), [c1 * c0 * 9]); + put(conv1, reshapeWeight(permuteRows(taps, tapsMajor), [c1, 9 * c0])); // Sinusoidal rows for distances 0 … window − 1: sines, then cosines. const half = hidden / 2; @@ -252,39 +199,43 @@ export function transformEmbeddingGemma2AudioStateDict( for (let i = 0; i < cfg.layers; i++) { const L = `layers.${i}`; - const T = `${S}layers.${i}`; for (const ff of ['feed_forward1', 'feed_forward2']) { - put(`${L}.${ff}.pre_layer_norm.scale`, take(`${T}.${ff}.pre_layer_norm.weight`)); - clipped(`${T}.${ff}.ffw_layer_1`, `${L}.${ff}.ffw_layer_1`); - clipped(`${T}.${ff}.ffw_layer_2`, `${L}.${ff}.ffw_layer_2`); - put( - `${L}.${ff}.post_layer_norm.scale`, - scaleWeight(take(`${T}.${ff}.post_layer_norm.weight`), cfg.residualWeight), - ); + const post = `${L}.${ff}.post_layer_norm.weight`; + put(post, scaleWeight(take(sd, what, q(post)), cfg.residualWeight)); + f32Bounds(`${L}.${ff}.ffw_layer_1`); + f32Bounds(`${L}.${ff}.ffw_layer_2`); } - const A = `${T}.self_attn`; - const qkv = ['q_proj', 'k_proj', 'v_proj'].map((n) => ({ - w: take(`${A}.${n}.linear.weight`), - b: bounds(`${A}.${n}`), - })); - put(`${L}.qkv.weight`, concatRows(qkv.map((t) => t.w))); + const A = `${L}.self_attn`; + const parts = ['q_proj', 'k_proj', 'v_proj']; put( - `${L}.qkv.clip`, - derivedTensor([4], async () => { - const all = await Promise.all(qkv.map((t) => Promise.all(t.b.map(scalar)))); - if (new Set(all.map(String)).size > 1) { - throw checkpointMismatch(what, `${A}: q, k and v have different bounds`); - } - return Float32Array.from(all.flat().slice(0, 4)); - }), + `${A}.qkv.linear.weight`, + concatRows(parts.map((n) => take(sd, what, q(`${A}.${n}.linear.weight`)))), ); - clipped(`${A}.post`, `${L}.post`); + // One set of bounds for the fused projection, checked to agree. + for (const b of BOUNDS) { + const bounds = parts.map((n) => take(sd, what, q(`${A}.${n}.${b}`))); + put( + `${A}.qkv.${b}`, + derivedTensor( + [1], + async () => { + const values = await Promise.all(bounds.map(scalar)); + if (values.some((v) => v !== values[0])) { + throw checkpointMismatch(what, `${q(A)}: q, k and v have different ${b}`); + } + return Float32Array.of(values[0]!); + }, + bounds[0]!.byteLength, + ), + ); + } + f32Bounds(`${A}.post`); // rel and q_scale stay f32 on purpose: the attention kernel reads them as // f32, so rounding them to f16 would only lose precision. - const rk = take(`${A}.relative_k_proj.weight`); + const rk = take(sd, what, q(`${A}.relative_k_proj.weight`)); put( - `${L}.rel`, + `${A}.rel`, derivedTensor( [window, hidden], async () => { @@ -302,9 +253,9 @@ export function transformEmbeddingGemma2AudioStateDict( rk.byteLength, ), ); - const dimScale = take(`${A}.per_dim_scale`); + const dimScale = take(sd, what, q(`${A}.per_dim_scale`)); put( - `${L}.q_scale`, + `${A}.q_scale`, derivedTensor( [headDim], async () => (await dimScale.f32()).map((x) => qScale * Math.log1p(Math.exp(x))), @@ -312,23 +263,10 @@ export function transformEmbeddingGemma2AudioStateDict( ), ); - const C = `${T}.lconv1d`; - put(`${L}.lconv.pre_layer_norm.scale`, take(`${C}.pre_layer_norm.weight`)); - clipped(`${C}.linear_start`, `${L}.lconv.linear_start`); // [hidden, 1, K] → [K, hidden]. - put( - `${L}.lconv.conv`, - transposeWeight(reshapeWeight(take(`${C}.depthwise_conv1d.weight`), [hidden, K])), - ); - put(`${L}.lconv.conv_norm.scale`, take(`${C}.conv_norm.weight`)); - clipped(`${C}.linear_end`, `${L}.lconv.linear_end`); - for (const n of ['norm_pre_attn', 'norm_post_attn', 'norm_out']) { - put(`${L}.${n}.scale`, take(`${T}.${n}.weight`)); - } + const conv = `${L}.lconv1d.depthwise_conv1d.weight`; + put(conv, transposeWeight(reshapeWeight(take(sd, what, q(conv)), [hidden, K]))); + f32Bounds(`${L}.lconv1d.linear_start`); + f32Bounds(`${L}.lconv1d.linear_end`); } - put('output_proj.weight', take(`${S}output_proj.weight`)); - put('output_proj.bias', take(`${S}output_proj.bias`)); - put('embedding_pre_projection_norm.scale', ones(cfg.outputDim)); - put('embedding_projection.weight', take('embed_audio.embedding_projection.weight')); - done(); } diff --git a/packages/runntime/src/zoo/embeddinggemma2/visionEncoder.ts b/packages/runntime/src/zoo/embeddinggemma2/visionEncoder.ts index 681f051d..fb64d284 100644 --- a/packages/runntime/src/zoo/embeddinggemma2/visionEncoder.ts +++ b/packages/runntime/src/zoo/embeddinggemma2/visionEncoder.ts @@ -87,8 +87,10 @@ export async function encodeImage( const { patch, pool, headDim, ropeTheta } = model.cfg; const [cols, rows] = patchGrid(image.width, image.height, maxTokens, patch, pool); const p = cols * rows; - // A norm scale, not a Linear weight: a quantized weight is quantW either way. - const act = model.embedding_pre_projection_norm.scale.shape.dtype === 'f16' ? 'f16' : 'f32'; + // The pre-projection norm's weight, not a Linear's: a quantized Linear + // weight is quantW either way. + const act = + model.embed_vision.embedding_pre_projection_norm.shape.dtype === 'f16' ? 'f16' : 'f32'; const upload = (data: Float32Array, r: number, dtype: 'f32' | 'f16') => tensor(data, { elems: data.length, dtype, dims: [r, data.length / r] }); const xs = new Float32Array(p); diff --git a/packages/runntime/src/zoo/embeddinggemma2/visionModel.ts b/packages/runntime/src/zoo/embeddinggemma2/visionModel.ts index 9106454e..210b755d 100644 --- a/packages/runntime/src/zoo/embeddinggemma2/visionModel.ts +++ b/packages/runntime/src/zoo/embeddinggemma2/visionModel.ts @@ -4,7 +4,10 @@ * * Layers are the text model's without PLE. Positions enter twice: x and y * embeddings on the patches and a 2D rope on q and k. q, k and v are - * RMS-normed per head, so scores carry no 1/√d scale. */ + * RMS-normed per head, so scores carry no 1/√d scale. + * + * Module fields are named like the checkpoint: the tower under + * `vision_tower`, its projection to the text model under `embed_vision`. */ import { add, @@ -51,34 +54,66 @@ export interface EmbeddingGemma2VisionInputs { pool: Value; } -class VisionLayer extends nn.Module<[Value, EmbeddingGemma2VisionInputs], Value> { - readonly input_layernorm: nn.RMSNorm; +/** A Linear stored under a `linear` child, as the tower's projections are. */ +class WrappedLinear extends nn.Module { + readonly linear: nn.Linear; + constructor(inFeatures: number, outFeatures: number, opts?: { activation?: 'geluTanh' }) { + super(); + this.linear = new nn.Linear(inFeatures, outFeatures, { bias: false, ...opts }); + } + forward(x: Value): Value { + return this.linear.forward(x); + } +} + +/** The attention projections and per-head norms, fused at load. */ +class Attention extends nn.Module { /** q | k | v rows, with q and k permuted for the interleaved rope. */ readonly qkv: nn.Linear; /** q_norm and k_norm tiled per head, ones for v. */ readonly qkv_norm: nn.Parameter; - readonly o_proj: nn.Linear; + readonly o_proj: WrappedLinear; + constructor({ hidden, heads, headDim }: EmbeddingGemma2VisionConfig) { + super(); + const width = 3 * heads * headDim; + this.qkv = new nn.Linear(hidden, width, { bias: false }); + this.qkv_norm = new nn.Parameter({ elems: width, dtype: 'f32', dims: [width] }); + this.o_proj = new WrappedLinear(heads * headDim, hidden); + } +} + +/** GeGLU MLP, GELU in the gate matmul's epilogue. */ +class Mlp extends nn.Module { + readonly gate_proj: WrappedLinear; + readonly up_proj: WrappedLinear; + readonly down_proj: WrappedLinear; + constructor({ hidden, ffn }: EmbeddingGemma2VisionConfig) { + super(); + this.gate_proj = new WrappedLinear(hidden, ffn, { activation: 'geluTanh' }); + this.up_proj = new WrappedLinear(hidden, ffn); + this.down_proj = new WrappedLinear(ffn, hidden); + } + forward(x: Value): Value { + return this.down_proj.forward(mul(this.gate_proj.forward(x), this.up_proj.forward(x))); + } +} + +class VisionLayer extends nn.Module<[Value, EmbeddingGemma2VisionInputs], Value> { + readonly input_layernorm: nn.RMSNorm; + readonly self_attn: Attention; readonly post_attention_layernorm: nn.RMSNorm; readonly pre_feedforward_layernorm: nn.RMSNorm; - readonly gate_proj: nn.Linear; - readonly up_proj: nn.Linear; - readonly down_proj: nn.Linear; + readonly mlp: Mlp; readonly post_feedforward_layernorm: nn.RMSNorm; constructor(private readonly cfg: EmbeddingGemma2VisionConfig) { super(); - const { hidden, heads, headDim, ffn, eps } = cfg; - const width = 3 * heads * headDim; - const norm = () => new nn.RMSNorm(hidden, eps); + const norm = () => new nn.RMSNorm(cfg.hidden, cfg.eps); this.input_layernorm = norm(); - this.qkv = new nn.Linear(hidden, width, { bias: false }); - this.qkv_norm = new nn.Parameter({ elems: width, dtype: 'f32', dims: [width] }); - this.o_proj = new nn.Linear(heads * headDim, hidden, { bias: false }); + this.self_attn = new Attention(cfg); this.post_attention_layernorm = norm(); this.pre_feedforward_layernorm = norm(); - this.gate_proj = new nn.Linear(hidden, ffn, { bias: false, activation: 'geluTanh' }); - this.up_proj = new nn.Linear(hidden, ffn, { bias: false }); - this.down_proj = new nn.Linear(ffn, hidden, { bias: false }); + this.mlp = new Mlp(cfg); this.post_feedforward_layernorm = norm(); } @@ -86,11 +121,12 @@ class VisionLayer extends nn.Module<[Value, EmbeddingGemma2VisionInputs], Value> const { hidden, heads, headDim: d, eps } = this.cfg; const w = heads * d; const postNorm = (norm: nn.RMSNorm, y: Value, residual: Value) => - rmsNorm(y, norm.scale.value, norm.eps, { residual }); + rmsNorm(y, norm.weight.value, norm.eps, { residual }); // One grouped norm for every head, then q and k roped in one dispatch. - const qkv = this.qkv.forward(this.input_layernorm.forward(x)); - const n = rmsNorm(qkv, this.qkv_norm.value, eps, { group: d }); + const { self_attn } = this; + const qkv = self_attn.qkv.forward(this.input_layernorm.forward(x)); + const n = rmsNorm(qkv, self_attn.qkv_norm.value, eps, { group: d }); const qk = rope(n, inp.cos, inp.sin, { headDim: d, width: 2 * w }); const head = (v: Value, start: number) => slice(v, 1, start, start + d); const p = x.shape.elems / hidden; @@ -119,44 +155,91 @@ class VisionLayer extends nn.Module<[Value, EmbeddingGemma2VisionInputs], Value> windowRight: Infinity, }); } - x = postNorm(this.post_attention_layernorm, this.o_proj.forward(a), x); + x = postNorm(this.post_attention_layernorm, self_attn.o_proj.forward(a), x); - // GeGLU MLP, GELU in the gate matmul's epilogue. - const f = this.pre_feedforward_layernorm.forward(x); - const m = this.down_proj.forward(mul(this.gate_proj.forward(f), this.up_proj.forward(f))); + const m = this.mlp.forward(this.pre_feedforward_layernorm.forward(x)); return postNorm(this.post_feedforward_layernorm, m, x); } } -export class EmbeddingGemma2VisionModel extends nn.Module<[EmbeddingGemma2VisionInputs], Value> { +/** Patches to the tower's width, plus their x and y position embeddings. */ +class PatchEmbedder extends nn.Module<[Value, Value, Value], Value> { readonly input_proj: nn.Linear; + /** The checkpoint's `position_embedding_table`, split at load. */ readonly pos_x: nn.Embedding; readonly pos_y: nn.Embedding; - readonly layers: nn.ModuleList; - /** The pooled tokens enter this weightless norm scaled by √hidden. The - * norm cancels the scale, so eps divided by hidden stands in for it. */ - readonly embedding_pre_projection_norm: nn.RMSNorm; - readonly embedding_projection: nn.Linear; - - constructor(readonly cfg: EmbeddingGemma2VisionConfig = EMBEDDINGGEMMA2_VISION) { + constructor({ hidden, patch, positions }: EmbeddingGemma2VisionConfig) { super(); - const { hidden, patch, positions, eps } = cfg; this.input_proj = new nn.Linear(3 * patch * patch, hidden, { bias: false }); this.pos_x = new nn.Embedding(positions, hidden); this.pos_y = new nn.Embedding(positions, hidden); + } + forward(patches: Value, xs: Value, ys: Value): Value { + const addend = add(this.pos_x.forward(xs), this.pos_y.forward(ys)); + return this.input_proj.forward(patches, { addend }); + } +} + +class VisionEncoder extends nn.Module { + readonly layers: nn.ModuleList; + constructor(cfg: EmbeddingGemma2VisionConfig) { + super(); this.layers = new nn.ModuleList(Array.from({ length: cfg.layers }, () => new VisionLayer(cfg))); - this.embedding_pre_projection_norm = new nn.RMSNorm(hidden, eps / hidden); + } +} + +class VisionTower extends nn.Module { + readonly patch_embedder: PatchEmbedder; + readonly encoder: VisionEncoder; + constructor(cfg: EmbeddingGemma2VisionConfig) { + super(); + this.patch_embedder = new PatchEmbedder(cfg); + this.encoder = new VisionEncoder(cfg); + } +} + +/** The pooled tokens' projection to the text model's width. */ +class EmbedVision extends nn.Module { + /** The pooled tokens enter this weightless norm scaled by √hidden. The + * norm cancels the scale, so eps divided by hidden stands in for it. Its + * weight is ones, a Buffer since the checkpoint has none. */ + readonly embedding_pre_projection_norm: nn.Buffer; + readonly embedding_projection: nn.Linear; + private readonly eps: number; + constructor({ hidden, eps }: EmbeddingGemma2VisionConfig) { + super(); + this.eps = eps / hidden; + this.embedding_pre_projection_norm = new nn.Buffer(new Float32Array(hidden).fill(1), { + elems: hidden, + dtype: 'f32', + dims: [hidden], + }); this.embedding_projection = new nn.Linear(hidden, EMBEDDINGGEMMA2_TEXT.hidden, { bias: false }); } + forward(pooled: Value): Value { + return this.embedding_projection.forward( + rmsNorm(pooled, this.embedding_pre_projection_norm.value, this.eps), + ); + } +} + +export class EmbeddingGemma2VisionModel extends nn.Module<[EmbeddingGemma2VisionInputs], Value> { + readonly vision_tower: VisionTower; + readonly embed_vision: EmbedVision; + + constructor(readonly cfg: EmbeddingGemma2VisionConfig = EMBEDDINGGEMMA2_VISION) { + super(); + this.vision_tower = new VisionTower(cfg); + this.embed_vision = new EmbedVision(cfg); + } /** One image to its tokens [N, text hidden], ready to stand in for N * placeholder rows of the text model's input. */ override forward(inp: EmbeddingGemma2VisionInputs): Value { - const addend = add(this.pos_x.forward(inp.xs), this.pos_y.forward(inp.ys)); - let x = this.input_proj.forward(inp.patches, { addend }); - for (const layer of this.layers.items) x = layer.forward(x, inp); - const pooled = matmul(inp.pool, x); - return this.embedding_projection.forward(this.embedding_pre_projection_norm.forward(pooled)); + const { patch_embedder, encoder } = this.vision_tower; + let x = patch_embedder.forward(inp.patches, inp.xs, inp.ys); + for (const layer of encoder.layers.items) x = layer.forward(x, inp); + return this.embed_vision.forward(matmul(inp.pool, x)); } override transformStateDict(sd: LazyStateDict, prefix: string): void { diff --git a/packages/runntime/src/zoo/minilm/embedCore.ts b/packages/runntime/src/zoo/minilm/embedCore.ts index 81c95ebf..1d96c6c6 100644 --- a/packages/runntime/src/zoo/minilm/embedCore.ts +++ b/packages/runntime/src/zoo/minilm/embedCore.ts @@ -56,7 +56,7 @@ export interface MinilmEmbedCore { export function createMinilmEmbedCore(model: MinilmModel): MinilmEmbedCore { // The selector multiplies the hidden states, so it follows the model's // dtype; the index planes stay f32. - const actDtype = model.embeddings.norm.weight.shape.dtype === 'f16' ? 'f16' : 'f32'; + const actDtype = model.embeddings.LayerNorm.weight.shape.dtype === 'f16' ? 'f16' : 'f32'; const pooler = (data: Float32Array, dims: [number, number]): Value => tensor(data, { elems: data.length, dtype: actDtype, dims }); // The replay mechanics live in core; this is the MiniLM-specific config. diff --git a/packages/runntime/src/zoo/minilm/model.ts b/packages/runntime/src/zoo/minilm/model.ts index f2449d0f..3ec4d026 100644 --- a/packages/runntime/src/zoo/minilm/model.ts +++ b/packages/runntime/src/zoo/minilm/model.ts @@ -4,7 +4,8 @@ * Attention is one fused qkv matmul sliced back into q/k/v, with an unbounded * window both ways. The score scale is folded into the q and k projections at * load, so the kernel applies none. Positions are a learned table with the - * constant token-type row folded in. */ + * constant token-type row folded in. Module fields are named like the + * checkpoint (BertModel), so `LayerNorm` is capitalized. */ import { add, @@ -23,40 +24,43 @@ class MinilmEmbeddings extends nn.Module< [readonly number[] | Value, (readonly number[] | Value)?], Value > { - readonly word: nn.Embedding; - readonly position: nn.Embedding; - readonly norm: nn.LayerNorm; + readonly word_embeddings: nn.Embedding; + /** The checkpoint's table plus its token-type row 0, summed at load. */ + readonly position_embeddings: nn.Embedding; + readonly LayerNorm: nn.LayerNorm; constructor(cfg: MinilmConfig) { super(); - this.word = new nn.Embedding(cfg.vocab, cfg.hidden); - this.position = new nn.Embedding(cfg.maxPositions, cfg.hidden); - this.norm = new nn.LayerNorm(cfg.hidden, cfg.eps, { bias: true }); + this.word_embeddings = new nn.Embedding(cfg.vocab, cfg.hidden); + this.position_embeddings = new nn.Embedding(cfg.maxPositions, cfg.hidden); + this.LayerNorm = new nn.LayerNorm(cfg.hidden, cfg.eps, { bias: true }); } forward(ids: readonly number[], lengths?: readonly number[]): Value; forward(ids: Value, positions: Value): Value; forward(ids: readonly number[] | Value, second?: readonly number[] | Value): Value { if (!Array.isArray(ids)) { - return this.norm.forward( - add(this.word.forward(ids as Value), this.position.forward(second as Value)), + return this.LayerNorm.forward( + add( + this.word_embeddings.forward(ids as Value), + this.position_embeddings.forward(second as Value), + ), ); } const lengths = second as readonly number[] | undefined; const positions = lengths ? lengths.flatMap((len) => Array.from({ length: len }, (_, i) => i)) : Array.from({ length: ids.length }, (_, i) => i); - return this.norm.forward(add(this.word.forward(ids), this.position.forward(positions))); + return this.LayerNorm.forward( + add(this.word_embeddings.forward(ids), this.position_embeddings.forward(positions)), + ); } } -class MinilmLayer extends nn.Module<[Value, Value?, number?], Value> { +/** BertSelfAttention: q, k and v as one projection, scores through `mha`. */ +class SelfAttention extends nn.Module { + /** q | k | v rows, fused at load with the score scale split over q and k. */ readonly qkv: nn.Linear; readonly mha: nn.MultiHeadAttention; - readonly out: nn.Linear; - readonly attnNorm: nn.LayerNorm; - readonly up: nn.Linear; - readonly down: nn.Linear; - readonly ffnNorm: nn.LayerNorm; - constructor(private readonly cfg: MinilmConfig) { + constructor(cfg: MinilmConfig) { super(); const qw = cfg.heads * cfg.headDim; this.qkv = new nn.Linear(cfg.hidden, 3 * qw); @@ -67,17 +71,64 @@ class MinilmLayer extends nn.Module<[Value, Value?, number?], Value> { windowLeft: Infinity, windowRight: Infinity, }); - this.out = new nn.Linear(qw, cfg.hidden); - this.attnNorm = new nn.LayerNorm(cfg.hidden, cfg.eps, { bias: true }); - this.up = new nn.Linear(cfg.hidden, cfg.ffn, { activation: 'gelu' }); - this.down = new nn.Linear(cfg.ffn, cfg.hidden); - this.ffnNorm = new nn.LayerNorm(cfg.hidden, cfg.eps, { bias: true }); + } +} + +/** A projection and the post-LN after its residual add. */ +class Output extends nn.Module<[Value, Value], Value> { + readonly dense: nn.Linear; + readonly LayerNorm: nn.LayerNorm; + constructor(cfg: MinilmConfig, inFeatures: number) { + super(); + this.dense = new nn.Linear(inFeatures, cfg.hidden); + this.LayerNorm = new nn.LayerNorm(cfg.hidden, cfg.eps, { bias: true }); + } + forward(x: Value, residual: Value): Value { + return this.LayerNorm.forward(this.dense.forward(x, { addend: residual })); + } +} + +class Attention extends nn.Module { + readonly self: SelfAttention; + readonly output: Output; + constructor(cfg: MinilmConfig) { + super(); + this.self = new SelfAttention(cfg); + this.output = new Output(cfg, cfg.heads * cfg.headDim); + } +} + +class Intermediate extends nn.Module { + readonly dense: nn.Linear; + constructor(cfg: MinilmConfig) { + super(); + this.dense = new nn.Linear(cfg.hidden, cfg.ffn, { activation: 'gelu' }); + } +} + +class MinilmLayer extends nn.Module<[Value, Value?, number?], Value> { + readonly attention: Attention; + readonly intermediate: Intermediate; + readonly output: Output; + constructor(cfg: MinilmConfig) { + super(); + this.attention = new Attention(cfg); + this.intermediate = new Intermediate(cfg); + this.output = new Output(cfg, cfg.ffn); } override forward(x: Value, segments?: Value, maxSegment?: number): Value { - const qkv = this.qkv.forward(x); // [T, 3·qw], columns q|k|v read in place - const attn = this.mha.forwardPacked(qkv, { segments, maxSegment }); - const h = this.attnNorm.forward(this.out.forward(attn, { addend: x })); - return this.ffnNorm.forward(this.down.forward(this.up.forward(h), { addend: h })); + const { self, output } = this.attention; + const qkv = self.qkv.forward(x); // [T, 3·qw], columns q|k|v read in place + const h = output.forward(self.mha.forwardPacked(qkv, { segments, maxSegment }), x); + return this.output.forward(this.intermediate.dense.forward(h), h); + } +} + +class MinilmEncoder extends nn.Module { + readonly layer: nn.ModuleList; + constructor(cfg: MinilmConfig) { + super(); + this.layer = new nn.ModuleList(Array.from({ length: cfg.layers }, () => new MinilmLayer(cfg))); } } @@ -92,11 +143,11 @@ export class MinilmModel extends nn.Module< Value > { readonly embeddings: MinilmEmbeddings; - readonly layers: nn.ModuleList; + readonly encoder: MinilmEncoder; constructor(readonly cfg: MinilmConfig = MINILM_L6) { super(); this.embeddings = new MinilmEmbeddings(cfg); - this.layers = new nn.ModuleList(Array.from({ length: cfg.layers }, () => new MinilmLayer(cfg))); + this.encoder = new MinilmEncoder(cfg); } override forward(ids: readonly number[], lengths?: readonly number[]): Value; override forward(ids: Value, replay: MinilmReplayInputs): Value; @@ -107,7 +158,7 @@ export class MinilmModel extends nn.Module< if (!Array.isArray(ids)) { const { positions, segments, maxSegment } = second as MinilmReplayInputs; let h: Value = this.embeddings.forward(ids as Value, positions); - for (const layer of this.layers.items) h = layer.forward(h, segments, maxSegment); + for (const layer of this.encoder.layer.items) h = layer.forward(h, segments, maxSegment); return h; } const lengths = second as readonly number[] | undefined; @@ -133,7 +184,7 @@ export class MinilmModel extends nn.Module< maxSegment = Math.max(...lengths); } let h: Value = this.embeddings.forward(ids, lengths); - for (const layer of this.layers.items) h = layer.forward(h, segments, maxSegment); + for (const layer of this.encoder.layer.items) h = layer.forward(h, segments, maxSegment); return h; } override transformStateDict(sd: LazyStateDict, prefix: string): void { diff --git a/packages/runntime/src/zoo/minilm/stateDictHooks.ts b/packages/runntime/src/zoo/minilm/stateDictHooks.ts index 308fa781..44295e79 100644 --- a/packages/runntime/src/zoo/minilm/stateDictHooks.ts +++ b/packages/runntime/src/zoo/minilm/stateDictHooks.ts @@ -1,16 +1,17 @@ -/** Maps the HF BertModel checkpoint onto the model's parameter tree as lazy - * derived tensors; nothing is fetched here. +/** Derives what the HF BertModel checkpoint stores differently from the + * model's parameters, as lazy tensors; nothing is fetched here. Every other + * tensor binds under its own name, since the modules are named like the + * checkpoint; the pooler and `position_ids` are left unread. * - * Drops the unused pooler keys, folds the constant token-type row into the - * position table, and fuses q/k/v into one weight with the score scale split - * across the q and k parts, so the scale-free kernel still computes - * q·kᵀ/√headDim. + * The constant token-type row folds into the position table, and q/k/v fuse + * into one weight with the score scale split across the q and k parts, so + * the scale-free kernel still computes q·kᵀ/√headDim. * * Weights stay in torch layout for the core Linear hook to transpose. The * fused q/k/v and the position sum are f32 even from an F16 checkpoint, so * an f32 model loads exact values and an f16 model rounds once in the * loader. Derived byteLengths carry the source file bytes, so load progress - * still sums to the checkpoint size. */ + * counts every byte the model reads. */ import { concatRows, @@ -31,10 +32,6 @@ function take(sd: LazyStateDict, name: string): LazyTensor { return t; } -function move(sd: LazyStateDict, from: string, to: string): void { - sd.tensors.set(to, take(sd, from)); -} - export function transformMinilmStateDict( sd: LazyStateDict, prefix: string, @@ -44,15 +41,12 @@ export function transformMinilmStateDict( // qkScale on q and k each — their product is the 1/√headDim score scale. const qkScale = cfg.headDim ** -0.25; - sd.tensors.delete(q('pooler.dense.weight')); - sd.tensors.delete(q('pooler.dense.bias')); - sd.tensors.delete(q('embeddings.position_ids')); - // position' = position + tokenType[0] (every real input is token type 0). - const position = take(sd, q('embeddings.position_embeddings.weight')); + const positions = q('embeddings.position_embeddings.weight'); + const position = take(sd, positions); const tokenType = take(sd, q('embeddings.token_type_embeddings.weight')); sd.tensors.set( - q('embeddings.position.weight'), + positions, derivedTensor( position.shape, async () => { @@ -65,20 +59,15 @@ export function transformMinilmStateDict( position.byteLength + tokenType.byteLength, ), ); - move(sd, q('embeddings.word_embeddings.weight'), q('embeddings.word.weight')); - move(sd, q('embeddings.LayerNorm.weight'), q('embeddings.norm.weight')); - move(sd, q('embeddings.LayerNorm.bias'), q('embeddings.norm.bias')); for (let i = 0; i < cfg.layers; i++) { - const hf = (name: string) => q(`encoder.layer.${i}.${name}`); - const ours = (name: string) => q(`layers.${i}.${name}`); - - // q|k|v → one torch-layout [3·hidden, hidden] weight and [3·hidden] bias: - // the three parts stacked row-wise, q/k pre-scaled. + const self = q(`encoder.layer.${i}.attention.self`); + // query|key|value → one torch-layout [3·hidden, hidden] weight and + // [3·hidden] bias: the three parts stacked row-wise, q/k pre-scaled. for (const suffix of ['weight', 'bias']) { - const part = (p: string) => take(sd, hf(`attention.self.${p}.${suffix}`)); + const part = (p: string) => take(sd, `${self}.${p}.${suffix}`); sd.tensors.set( - ours(`qkv.${suffix}`), + `${self}.qkv.${suffix}`, concatRows([ scaleWeight(part('query'), qkScale), scaleWeight(part('key'), qkScale), @@ -86,17 +75,5 @@ export function transformMinilmStateDict( ]), ); } - - move(sd, hf('attention.output.dense.weight'), ours('out.weight')); - move(sd, hf('attention.output.dense.bias'), ours('out.bias')); - move(sd, hf('attention.output.LayerNorm.weight'), ours('attnNorm.weight')); - move(sd, hf('attention.output.LayerNorm.bias'), ours('attnNorm.bias')); - - move(sd, hf('intermediate.dense.weight'), ours('up.weight')); - move(sd, hf('intermediate.dense.bias'), ours('up.bias')); - move(sd, hf('output.dense.weight'), ours('down.weight')); - move(sd, hf('output.dense.bias'), ours('down.bias')); - move(sd, hf('output.LayerNorm.weight'), ours('ffnNorm.weight')); - move(sd, hf('output.LayerNorm.bias'), ours('ffnNorm.bias')); } } diff --git a/packages/runntime/src/zoo/moonshine-streaming/decoder.ts b/packages/runntime/src/zoo/moonshine-streaming/decoder.ts new file mode 100644 index 00000000..2b00754f --- /dev/null +++ b/packages/runntime/src/zoo/moonshine-streaming/decoder.ts @@ -0,0 +1,44 @@ +/** Moonshine v2 decoder: the v1 decoder (same architecture, different dims) + * plus the position embedding and, where the widths differ, the projection + * that bring the encoder output to its cross attention. Positions enter + * here, as the encoder carries none. */ + +import { add, nn, slice, type Value } from '../../core/index.ts'; +import { MoonshineDecoder } from '../moonshine/decoder.ts'; +import type { MoonshineConfig } from '../moonshine/config.ts'; +import type { MoonshineStreamingConfig } from './config.ts'; + +export function decoderView(cfg: MoonshineStreamingConfig): MoonshineConfig { + return { + dModel: cfg.dec.hidden, + ffn: cfg.dec.ffn, + encLayers: cfg.enc.layers, + decLayers: cfg.dec.layers, + heads: cfg.dec.heads, + headDim: cfg.dec.headDim, + rotaryDim: cfg.dec.rotaryDim, + ropeTheta: cfg.dec.ropeTheta, + vocab: cfg.vocab, + maxPositions: cfg.maxPositions, + bos: cfg.bos, + eos: cfg.eos, + }; +} + +export class StreamingDecoder extends MoonshineDecoder { + readonly pos_emb: nn.Embedding; + readonly proj?: nn.Linear; + constructor(cfg: MoonshineStreamingConfig) { + super(decoderView(cfg)); + const [eh, dh] = [cfg.enc.hidden, cfg.dec.hidden]; + this.pos_emb = new nn.Embedding(cfg.maxPositions, eh); + this.proj = eh === dh ? undefined : new nn.Linear(eh, dh, { bias: false }); + } + /** Encoder frames [T, enc.hidden] to what cross attention reads + * [T, dec.hidden]: positions added, the width projected. */ + adapt(encOut: Value): Value { + const t = encOut.shape.dims![0]!; + const positioned = add(encOut, slice(this.pos_emb.weight.value, 0, 0, t)); + return this.proj ? this.proj.forward(positioned) : positioned; + } +} diff --git a/packages/runntime/src/zoo/moonshine-streaming/encoder.ts b/packages/runntime/src/zoo/moonshine-streaming/encoder.ts index 8cbd7d54..8e067bbb 100644 --- a/packages/runntime/src/zoo/moonshine-streaming/encoder.ts +++ b/packages/runntime/src/zoo/moonshine-streaming/encoder.ts @@ -1,21 +1,22 @@ -/** Moonshine v2 encoder: pre-norm transformer layers carrying no positional - * information. Locality comes entirely from per-layer asymmetric - * sliding-window attention, with windows converted from HF's strict bounds to - * the kernel's inclusive form. +/** Moonshine v2 encoder: the embedder, then pre-norm transformer layers + * carrying no positional information. Locality comes entirely from + * per-layer asymmetric sliding-window attention, with windows converted from + * HF's strict bounds to the kernel's inclusive form. * * There is no rope, so the score scale folds into the q columns at load. The * loader also adds 1 back to the checkpoint's offset norm gamma, so plain - * weight-only LayerNorms serve here. - * - * The adapter bridges encoder to decoder, and is where positions enter. */ + * weight-only LayerNorms serve here. Module fields are named like the + * checkpoint. */ -import { add, chunk, gelu, nn, slice, type Value } from '../../core/index.ts'; +import { chunk, gelu, nn, type Value } from '../../core/index.ts'; import type { MoonshineStreamingConfig } from './config.ts'; +import { StreamingEmbedder } from './frontend.ts'; class EncoderAttn extends nn.Module { + /** q | k | v rows, fused at load. */ readonly qkv: nn.Linear; readonly mha: nn.MultiHeadAttention; - readonly out: nn.Linear; + readonly o_proj: nn.Linear; constructor( private readonly cfg: MoonshineStreamingConfig, window: readonly [number, number], @@ -31,12 +32,12 @@ class EncoderAttn extends nn.Module { windowLeft: window[0] - 1, windowRight: Math.max(0, window[1] - 1), }); - this.out = new nn.Linear(qw, hidden, { bias: false }); + this.o_proj = new nn.Linear(qw, hidden, { bias: false }); } forward(x: Value, residual?: Value): Value { const qkv = this.qkv.forward(x); // [T, 3·qw], columns q|k|v const [q, k, v] = chunk(qkv, 3, 1) as [Value, Value, Value]; - return this.out.forward(this.mha.forward(q, k, v), { addend: residual }); + return this.o_proj.forward(this.mha.forward(q, k, v), { addend: residual }); } } @@ -54,54 +55,37 @@ class EncoderMlp extends nn.Module { } export class EncoderLayer extends nn.Module { - readonly norm1: nn.LayerNorm; - readonly attn: EncoderAttn; - readonly norm2: nn.LayerNorm; + readonly input_layernorm: nn.LayerNorm; + readonly self_attn: EncoderAttn; + readonly post_attention_layernorm: nn.LayerNorm; readonly mlp: EncoderMlp; constructor(cfg: MoonshineStreamingConfig, window: readonly [number, number]) { super(); - this.norm1 = new nn.LayerNorm(cfg.enc.hidden); - this.attn = new EncoderAttn(cfg, window); - this.norm2 = new nn.LayerNorm(cfg.enc.hidden); + this.input_layernorm = new nn.LayerNorm(cfg.enc.hidden); + this.self_attn = new EncoderAttn(cfg, window); + this.post_attention_layernorm = new nn.LayerNorm(cfg.enc.hidden); this.mlp = new EncoderMlp(cfg); } forward(x: Value): Value { - const h = this.attn.forward(this.norm1.forward(x), x); - return this.mlp.forward(this.norm2.forward(h), h); + const h = this.self_attn.forward(this.input_layernorm.forward(x), x); + return this.mlp.forward(this.post_attention_layernorm.forward(h), h); } } export class StreamingEncoder extends nn.Module { + readonly embedder: StreamingEmbedder; readonly layers: nn.ModuleList; - readonly norm: nn.LayerNorm; + readonly final_norm: nn.LayerNorm; constructor(cfg: MoonshineStreamingConfig) { super(); + this.embedder = new StreamingEmbedder(cfg); this.layers = new nn.ModuleList(cfg.enc.windows.map((w) => new EncoderLayer(cfg, w))); - this.norm = new nn.LayerNorm(cfg.enc.hidden); + this.final_norm = new nn.LayerNorm(cfg.enc.hidden); } + /** Audio frames [T, frameLen] to encoder frames [T/4, enc.hidden]. */ forward(x: Value): Value { - let h = x; + let h = this.embedder.forward(x); for (const layer of this.layers.items) h = layer.forward(h); - return this.norm.forward(h); - } -} - -export class StreamingAdapter extends nn.Module { - readonly posEmb: nn.Parameter; - readonly proj?: nn.Linear; - constructor(cfg: MoonshineStreamingConfig) { - super(); - const [eh, dh] = [cfg.enc.hidden, cfg.dec.hidden]; - this.posEmb = new nn.Parameter({ - elems: cfg.maxPositions * eh, - dtype: 'f32', - dims: [cfg.maxPositions, eh], - }); - this.proj = eh === dh ? undefined : new nn.Linear(eh, dh, { bias: false }); - } - forward(x: Value): Value { - const t = x.shape.dims![0]!; - const positioned = add(x, slice(this.posEmb.value, 0, 0, t)); - return this.proj ? this.proj.forward(positioned) : positioned; + return this.final_norm.forward(h); } } diff --git a/packages/runntime/src/zoo/moonshine-streaming/frontend.ts b/packages/runntime/src/zoo/moonshine-streaming/frontend.ts index 008829fb..816ea5c9 100644 --- a/packages/runntime/src/zoo/moonshine-streaming/frontend.ts +++ b/packages/runntime/src/zoo/moonshine-streaming/frontend.ts @@ -1,15 +1,16 @@ -/** Moonshine v2 frontend: raw 16 kHz mono audio, reshaped on the CPU into - * non-overlapping 80-sample frames [T, 80] (5 ms, 200 Hz), → encoder frames - * [T/4, enc.hidden] (~50 Hz): +/** Moonshine v2 embedder (the checkpoint's `model.encoder.embedder`): raw + * 16 kHz mono audio, reshaped on the CPU into non-overlapping 80-sample + * frames [T, 80] (5 ms, 200 Hz), → encoder frames [T/4, enc.hidden] (~50 Hz): * per-frame CMVN → asinh(k·x) → linear [80 → d] (no bias) → silu * → causal conv1 k=5 s=2 (d → 2d) → silu → causal conv2 k=5 s=2 (2d → d) * CMVN (mean-center, divide by RMS, eps 1e-6) is exactly a bias-free - * LayerNorm whose weight the loader supplies as ones. k is the checkpoint's - * learnable asinh compression scale — a [1] Parameter holding exp(log_k), - * broadcast-multiplied before the asinh. Both convs left-pad kernel−1 zeros - * (causal), so each keeps ceil(T/2) frames; no activation after conv2. */ + * LayerNorm with a weight of ones, a Buffer since the checkpoint has none. + * k is the checkpoint's learnable asinh compression scale — a [1] Parameter + * holding exp(log_k), broadcast-multiplied before the asinh. Both convs + * left-pad kernel−1 zeros (causal), so each keeps ceil(T/2) frames; no + * activation after conv2. */ -import { asinh, mul, nn, silu, type Value } from '../../core/index.ts'; +import { asinh, layerNorm, mul, nn, silu, type Value } from '../../core/index.ts'; import type { MoonshineStreamingConfig } from './config.ts'; const K = 5; @@ -30,23 +31,37 @@ export function frontendFrames( return { input, conv1, conv2: Math.floor((conv1 - 1) / S) + 1 }; } -export class StreamingFrontend extends nn.Module { - readonly cmvn: nn.LayerNorm; - readonly k: nn.Parameter; +/** CMVN's eps. */ +const CMVN_EPS = 1e-6; + +/** The asinh compression: `k` is exp of the checkpoint's `log_k`. */ +class Compression extends nn.Module { + readonly k = new nn.Parameter({ elems: 1, dtype: 'f32', dims: [1] }); + forward(x: Value): Value { + return asinh(mul(x, this.k.value)); + } +} + +export class StreamingEmbedder extends nn.Module { + readonly cmvn: nn.Buffer; + readonly comp = new Compression(); readonly linear: nn.Linear; readonly conv1: nn.Conv1d; readonly conv2: nn.Conv1d; constructor(cfg: MoonshineStreamingConfig) { super(); const d = cfg.enc.hidden; - this.cmvn = new nn.LayerNorm(cfg.frameLen, 1e-6); - this.k = new nn.Parameter({ elems: 1, dtype: 'f32', dims: [1] }); + this.cmvn = new nn.Buffer(new Float32Array(cfg.frameLen).fill(1), { + elems: cfg.frameLen, + dtype: 'f32', + dims: [cfg.frameLen], + }); this.linear = new nn.Linear(cfg.frameLen, d, { bias: false }); this.conv1 = new nn.Conv1d(d, 2 * d, K, { stride: S, padLeft: PAD }); this.conv2 = new nn.Conv1d(2 * d, d, K, { stride: S, padLeft: PAD }); } forward(x: Value): Value { - const compressed = asinh(mul(this.cmvn.forward(x), this.k.value)); + const compressed = this.comp.forward(layerNorm(x, this.cmvn.value, CMVN_EPS)); const h = silu(this.conv1.forward(silu(this.linear.forward(compressed)))); return this.conv2.forward(h); } diff --git a/packages/runntime/src/zoo/moonshine-streaming/index.ts b/packages/runntime/src/zoo/moonshine-streaming/index.ts index b338417e..97f41d86 100644 --- a/packages/runntime/src/zoo/moonshine-streaming/index.ts +++ b/packages/runntime/src/zoo/moonshine-streaming/index.ts @@ -10,9 +10,10 @@ export { type StreamingEncoderConfig, type StreamingDecoderConfig, } from './config.ts'; -export { StreamingFrontend } from './frontend.ts'; -export { StreamingEncoder, StreamingAdapter, EncoderLayer } from './encoder.ts'; -export { MoonshineStreamingModel } from './model.ts'; +export { StreamingEmbedder } from './frontend.ts'; +export { StreamingEncoder, EncoderLayer } from './encoder.ts'; +export { StreamingDecoder } from './decoder.ts'; +export { MoonshineStreamingModel, StreamingEncoderDecoder } from './model.ts'; export { createTranscriber, type TranscribeResult, type Transcriber } from './transcriber.ts'; export type { BurstSample, TranscriberPerf } from '../moonshine/transcriber.ts'; export { createMoonshineTokenizer, type MoonshineTokenizer } from '../moonshine/tokenizer.ts'; diff --git a/packages/runntime/src/zoo/moonshine-streaming/model.ts b/packages/runntime/src/zoo/moonshine-streaming/model.ts index 4724ec09..f1a410f8 100644 --- a/packages/runntime/src/zoo/moonshine-streaming/model.ts +++ b/packages/runntime/src/zoo/moonshine-streaming/model.ts @@ -1,12 +1,9 @@ -/** The full Moonshine v2 module tree. No constructor takes a sequence length, - * and one loadStateDict binds the whole checkpoint. +/** The full Moonshine v2 module tree, named like the HF checkpoint: + * `model.encoder` and `model.decoder`. No constructor takes a sequence + * length, and one loadStateDict binds the whole checkpoint. * - * The decoder is the v1 class: same architecture, different dims, so - * `decoderView` bridges the two config shapes. Cross K and V read the adapter - * output rather than the encoder's. - * - * The embedding weight serves as both token lookup and the tied LM head. No - * transposed copy exists, so tying is exact by construction. */ + * The decoder's embedding weight serves as both token lookup and the tied LM + * head, so the checkpoint's `proj_out.weight` is left unread. */ import { argmaxDot, @@ -18,43 +15,27 @@ import { type LazyStateDict, type Value, } from '../../core/index.ts'; -import type { MoonshineConfig } from '../moonshine/config.ts'; -import { MoonshineDecoder } from '../moonshine/decoder.ts'; import type { MoonshineStreamingConfig } from './config.ts'; -import { StreamingFrontend } from './frontend.ts'; -import { StreamingAdapter, StreamingEncoder } from './encoder.ts'; +import { StreamingDecoder } from './decoder.ts'; +import { StreamingEncoder } from './encoder.ts'; import { transformStreamingStateDict } from './stateDictHooks.ts'; -export function decoderView(cfg: MoonshineStreamingConfig): MoonshineConfig { - return { - dModel: cfg.dec.hidden, - ffn: cfg.dec.ffn, - encLayers: cfg.enc.layers, - decLayers: cfg.dec.layers, - heads: cfg.dec.heads, - headDim: cfg.dec.headDim, - rotaryDim: cfg.dec.rotaryDim, - ropeTheta: cfg.dec.ropeTheta, - vocab: cfg.vocab, - maxPositions: cfg.maxPositions, - bos: cfg.bos, - eos: cfg.eos, - }; +/** HF's MoonshineStreamingModel: the encoder and decoder pair. */ +export class StreamingEncoderDecoder extends nn.Module { + readonly encoder: StreamingEncoder; + readonly decoder: StreamingDecoder; + constructor(cfg: MoonshineStreamingConfig) { + super(); + this.encoder = new StreamingEncoder(cfg); + this.decoder = new StreamingDecoder(cfg); + } } export class MoonshineStreamingModel extends nn.Module { - readonly embedding: nn.Embedding; - readonly frontend: StreamingFrontend; - readonly encoder: StreamingEncoder; - readonly adapter: StreamingAdapter; - readonly decoder: MoonshineDecoder; + readonly model: StreamingEncoderDecoder; constructor(readonly cfg: MoonshineStreamingConfig) { super(); - this.embedding = new nn.Embedding(cfg.vocab, cfg.dec.hidden); - this.frontend = new StreamingFrontend(cfg); - this.encoder = new StreamingEncoder(cfg); - this.adapter = new StreamingAdapter(cfg); - this.decoder = new MoonshineDecoder(decoderView(cfg)); + this.model = new StreamingEncoderDecoder(cfg); } override transformStateDict(sd: LazyStateDict, prefix: string): void { @@ -62,15 +43,15 @@ export class MoonshineStreamingModel extends nn.Module { } embed(ids: readonly number[]): Value { - return this.embedding.forward(ids); + return this.model.decoder.embed_tokens.forward(ids); } embedFrom(ids: Value): Value { - return gatherRowsFrom(this.embedding.weight.value, ids); + return gatherRowsFrom(this.model.decoder.embed_tokens.weight.value, ids); } logitsLast(h: Value): Value { - const w = this.embedding.weight.value; + const w = this.model.decoder.embed_tokens.weight.value; if (w.shape.dtype !== h.shape.dtype) { throw new Error( `MoonshineStreamingModel.logitsLast: embedding is ${w.shape.dtype} but h is ${h.shape.dtype}`, @@ -81,12 +62,12 @@ export class MoonshineStreamingModel extends nn.Module { } nextTokenId(h1: Value): Value { - return argmaxDot(this.embedding.weight.value, h1); + return argmaxDot(this.model.decoder.embed_tokens.weight.value, h1); } forward(): Value { throw new Error( - 'MoonshineStreamingModel: use frontend/encoder/adapter/decoder forwards, embed() and logitsLast()', + 'MoonshineStreamingModel: use model.encoder/model.decoder forwards, embed() and logitsLast()', ); } } diff --git a/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts b/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts index 84e60747..ac66e598 100644 --- a/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts +++ b/packages/runntime/src/zoo/moonshine-streaming/stateDictHooks.ts @@ -1,26 +1,20 @@ -/** Maps the HF Moonshine v2 checkpoint onto the model's parameter tree as lazy - * derived tensors; nothing is fetched here. +/** Derives what the HF Moonshine v2 checkpoint stores differently from the + * model's parameters, as lazy tensors; nothing is fetched here. Every other + * tensor binds under its own name, since the modules are named like the + * checkpoint. * - * The notable transforms: encoder norm gamma gains back its −1 offset, CMVN's - * missing weight is synthesized as ones, and q/k/v stack into fused weights — - * scaled on the encoder, which has no rope, and unscaled on the decoder, whose - * rope tables carry the scale. Everything else renames. + * The encoder norm gammas gain back their −1 offset, `log_k` becomes k, and + * q/k/v stack into fused weights — scaled on the encoder, which has no + * rope, and unscaled on the decoder, whose rope tables carry the scale. * - * Weights stay in torch layout for the core Linear and Conv1d hooks to - * transpose. An F16 checkpoint gives f16 decoder q/k/v; the scaled encoder - * q/k/v and cross-attention q, the norm gammas and the frontend constants - * stay f32. Derived byteLengths carry the source file bytes, so load progress - * still sums to the checkpoint size. */ + * Weights stay in torch layout for the core Linear hook to transpose. An F16 + * checkpoint gives f16 decoder q/k/v; the scaled encoder q/k/v and + * cross-attention q, the norm gammas and k stay f32. Derived byteLengths + * carry the source file bytes, so load progress counts every byte the model + * reads. */ import { derivedTensor, type LazyStateDict, type LazyTensor } from '../../core/index.ts'; -import { - fuseQkv, - move, - moveTiedEmbedding, - qualify, - take, - transformDecoder, -} from '../moonshine/stateDictHooks.ts'; +import { fuseQkv, qualify, take, transformDecoder } from '../moonshine/stateDictHooks.ts'; import type { MoonshineStreamingConfig } from './config.ts'; /** Stays f32 for any checkpoint: rounding gamma + 1 to f16 would drop the low @@ -44,50 +38,34 @@ export function transformStreamingStateDict( cfg: MoonshineStreamingConfig, ): void { const q = (name: string) => qualify(prefix, name); - const [eh, dh] = [cfg.enc.hidden, cfg.dec.hidden]; - const gammaNorm = (from: string, to: string) => sd.tensors.set(to, gammaPlusOne(take(sd, from))); + /** The norm at `norm` reads its `gamma` as a weight. */ + const gammaNorm = (norm: string) => + sd.tensors.set(`${norm}.weight`, gammaPlusOne(take(sd, `${norm}.gamma`))); - moveTiedEmbedding(sd, prefix); - - // CMVN is a LayerNorm with no learned scale, so its weight is all ones and - // is not in the file. - sd.tensors.set( - q('frontend.cmvn.weight'), - derivedTensor([cfg.frameLen], async () => new Float32Array(cfg.frameLen).fill(1), 0), - ); - const logK = take(sd, q('model.encoder.embedder.comp.log_k')); + const comp = q('model.encoder.embedder.comp'); + const logK = take(sd, `${comp}.log_k`); sd.tensors.set( - q('frontend.k'), + `${comp}.k`, derivedTensor( [1], async () => Float32Array.from([Math.exp((await logK.f32())[0]!)]), logK.byteLength, ), ); - move(sd, q('model.encoder.embedder.linear.weight'), q('frontend.linear.weight')); - move(sd, q('model.encoder.embedder.conv1.weight'), q('frontend.conv1.weight')); - move(sd, q('model.encoder.embedder.conv1.bias'), q('frontend.conv1.bias')); - move(sd, q('model.encoder.embedder.conv2.weight'), q('frontend.conv2.weight')); - move(sd, q('model.encoder.embedder.conv2.bias'), q('frontend.conv2.bias')); // Encoder: no rope, so the full score scale folds into the q rows. const encQScale = 1 / Math.sqrt(cfg.enc.headDim); for (let i = 0; i < cfg.enc.layers; i++) { - const hf = q(`model.encoder.layers.${i}`); - const en = q(`encoder.layers.${i}`); - gammaNorm(`${hf}.input_layernorm.gamma`, `${en}.norm1.weight`); - fuseQkv(sd, `${hf}.self_attn`, `${en}.attn.qkv.weight`, eh, encQScale); - move(sd, `${hf}.self_attn.o_proj.weight`, `${en}.attn.out.weight`); - gammaNorm(`${hf}.post_attention_layernorm.gamma`, `${en}.norm2.weight`); - move(sd, `${hf}.mlp.fc1.weight`, `${en}.mlp.fc1.weight`); - move(sd, `${hf}.mlp.fc1.bias`, `${en}.mlp.fc1.bias`); - move(sd, `${hf}.mlp.fc2.weight`, `${en}.mlp.fc2.weight`); - move(sd, `${hf}.mlp.fc2.bias`, `${en}.mlp.fc2.bias`); + const L = q(`model.encoder.layers.${i}`); + gammaNorm(`${L}.input_layernorm`); + fuseQkv(sd, `${L}.self_attn`, cfg.enc.hidden, encQScale); + gammaNorm(`${L}.post_attention_layernorm`); } - gammaNorm(q('model.encoder.final_norm.gamma'), q('encoder.norm.weight')); - - move(sd, q('model.decoder.pos_emb.weight'), q('adapter.posEmb')); - if (eh !== dh) move(sd, q('model.decoder.proj.weight'), q('adapter.proj.weight')); + gammaNorm(q('model.encoder.final_norm')); - transformDecoder(sd, prefix, { layers: cfg.dec.layers, hidden: dh, headDim: cfg.dec.headDim }); + transformDecoder(sd, prefix, { + layers: cfg.dec.layers, + hidden: cfg.dec.hidden, + headDim: cfg.dec.headDim, + }); } diff --git a/packages/runntime/src/zoo/moonshine-streaming/transcriber.ts b/packages/runntime/src/zoo/moonshine-streaming/transcriber.ts index 39170be6..e739de58 100644 --- a/packages/runntime/src/zoo/moonshine-streaming/transcriber.ts +++ b/packages/runntime/src/zoo/moonshine-streaming/transcriber.ts @@ -29,7 +29,8 @@ import type { SelfKV } from '../moonshine/decoder.ts'; import type { TranscribeResult, Transcriber, TranscriberPerf } from '../moonshine/transcriber.ts'; import { presetFromStateDict, type MoonshineStreamingConfig } from './config.ts'; import { frontendFrames } from './frontend.ts'; -import { MoonshineStreamingModel, decoderView } from './model.ts'; +import { decoderView } from './decoder.ts'; +import { MoonshineStreamingModel } from './model.ts'; import type { LoadOptions } from '../load.ts'; export type { TranscribeResult, Transcriber }; @@ -112,8 +113,8 @@ export async function createTranscriber( const frames = frontendFrames(audio.length, cfg.frameLen); const x = actTensor(audio, [frames.input, cfg.frameLen]); - const adapterOut = model.adapter.forward(model.encoder.forward(model.frontend.forward(x))); - const cross = model.decoder.precomputeCrossKV(adapterOut); + const adapterOut = model.model.decoder.adapt(model.model.encoder.forward(x)); + const cross = model.model.decoder.precomputeCrossKV(adapterOut); // The K/V co-targets pin every buffer the decode loop reads. evalValues([adapterOut, ...cross.flatMap((c) => [c.k, c.v])], ex); // Keeps the encoder's GPU tail out of the first burst's readback wait. @@ -149,7 +150,7 @@ export async function createTranscriber( let x1 = model.embed([ids[base]!]); // burst opener: CPU-known id for (let j = 0; j < burst; j++) { const t = base + j; - const h = model.decoder.forwardStep( + const h = model.model.decoder.forwardStep( x1, slice(decCos, 0, t, t + 1), slice(decSin, 0, t, t + 1), diff --git a/packages/runntime/src/zoo/moonshine/convStem.ts b/packages/runntime/src/zoo/moonshine/convStem.ts index 1a5a90be..bee7ac8a 100644 --- a/packages/runntime/src/zoo/moonshine/convStem.ts +++ b/packages/runntime/src/zoo/moonshine/convStem.ts @@ -1,20 +1,14 @@ -/** Moonshine conv stem: raw 16 kHz mono audio [L, 1] → encoder frames - * [T, d] (384× downsample, ~41.7 frames/s): - * conv1 k=127 s=64 (no bias) → tanh → nn.GroupNorm(1, d) - * → conv2 k=7 s=3 (+bias) → gelu → conv3 k=3 s=2 (+bias) → gelu - * Valid padding throughout; conv weights arrive tap-major [k·C_in, C_out] - * from the loader. Each stage's bias and activation ride the conv - * epilogue — one dispatch per conv. */ +/** Frame counts of the Moonshine conv stem (MoonshineEncoder's three convs): + * raw 16 kHz mono audio [L, 1] → encoder frames [T, d] (384× downsample, + * ~41.7 frames/s): + * conv1 k=127 s=64 → conv2 k=7 s=3 → conv3 k=3 s=2, valid padding. */ -import { nn, type Value } from '../../core/index.ts'; -import type { MoonshineConfig } from './config.ts'; - -const K1 = 127; -const S1 = 64; -const K2 = 7; -const S2 = 3; -const K3 = 3; -const S3 = 2; +export const K1 = 127; +export const S1 = 64; +export const K2 = 7; +export const S2 = 3; +export const K3 = 3; +export const S3 = 2; const convOut = (t: number, k: number, s: number) => Math.floor((t - k) / s) + 1; @@ -30,21 +24,3 @@ export function stemFrames(samples: number): { conv1: number; conv2: number; con const t2 = convOut(t1, K2, S2); return { conv1: t1, conv2: t2, conv3: convOut(t2, K3, S3) }; } - -export class ConvStem extends nn.Module { - readonly conv1: nn.Conv1d; - readonly groupnorm: nn.GroupNorm; - readonly conv2: nn.Conv1d; - readonly conv3: nn.Conv1d; - constructor(cfg: MoonshineConfig) { - super(); - const d = cfg.dModel; - this.conv1 = new nn.Conv1d(1, d, K1, { stride: S1, bias: false, activation: 'tanh' }); - this.groupnorm = new nn.GroupNorm(1, d); - this.conv2 = new nn.Conv1d(d, 2 * d, K2, { stride: S2, activation: 'gelu' }); - this.conv3 = new nn.Conv1d(2 * d, d, K3, { stride: S3, activation: 'gelu' }); - } - forward(x: Value): Value { - return this.conv3.forward(this.conv2.forward(this.groupnorm.forward(this.conv1.forward(x)))); - } -} diff --git a/packages/runntime/src/zoo/moonshine/decoder.ts b/packages/runntime/src/zoo/moonshine/decoder.ts index 9a7d5787..f7130cf0 100644 --- a/packages/runntime/src/zoo/moonshine/decoder.ts +++ b/packages/runntime/src/zoo/moonshine/decoder.ts @@ -12,7 +12,8 @@ * * Score scales are folded into the weights and rope tables, so none appears * here. Cross K and V are computed once per utterance and reused by every - * decode step. */ + * decode step. Module fields are named like the checkpoint; the token + * embedding lives here, as in HF, and doubles as the tied LM head. */ import { chunk, nn, rope, slice, swigluChunk, writeRows, type Value } from '../../core/index.ts'; import type { MoonshineConfig } from './config.ts'; @@ -28,9 +29,10 @@ export interface SelfKV { } class DecoderSelfAttn extends nn.Module<[Value, Value, Value, Value?], Value> { + /** q | k | v rows, fused at load. */ readonly qkv: nn.Linear; readonly mha: nn.MultiHeadAttention; - readonly out: nn.Linear; + readonly o_proj: nn.Linear; constructor(private readonly cfg: MoonshineConfig) { super(); const qw = cfg.heads * cfg.headDim; @@ -42,7 +44,7 @@ class DecoderSelfAttn extends nn.Module<[Value, Value, Value, Value?], Value> { windowLeft: Infinity, windowRight: 0, // causal }); - this.out = new nn.Linear(qw, cfg.dModel, { bias: false }); + this.o_proj = new nn.Linear(qw, cfg.dModel, { bias: false }); } forward(x: Value, cos: Value, sin: Value, residual?: Value): Value { const { heads, headDim } = this.cfg; @@ -51,7 +53,7 @@ class DecoderSelfAttn extends nn.Module<[Value, Value, Value, Value?], Value> { const q = rope(qkv, cos, sin, { headDim, srcStart: 0, width: qw }); const k = rope(qkv, cos, sin, { headDim, srcStart: qw, width: qw }); const v = slice(qkv, 1, 2 * qw, 3 * qw); - return this.out.forward(this.mha.forward(q, k, v), { addend: residual }); + return this.o_proj.forward(this.mha.forward(q, k, v), { addend: residual }); } forwardStep( x1: Value, @@ -70,19 +72,21 @@ class DecoderSelfAttn extends nn.Module<[Value, Value, Value, Value?], Value> { const k = writeRows(cache.k, kNew, t); // [t+1, qw] view of the cache buffer const v = writeRows(cache.v, vNew, t); const ctx = this.mha.forward(q, k, v, { qPosOffset: t }); - return this.out.forward(ctx, { addend: residual }); + return this.o_proj.forward(ctx, { addend: residual }); } } class DecoderCrossAttn extends nn.Module<[Value, CrossKV, Value?], Value> { - readonly q: nn.Linear; + /** Carries the 1/√headDim score scale, folded in at load. */ + readonly q_proj: nn.Linear; + /** k | v rows, fused at load. */ readonly kv: nn.Linear; readonly mha: nn.MultiHeadAttention; - readonly out: nn.Linear; + readonly o_proj: nn.Linear; constructor(cfg: MoonshineConfig) { super(); const qw = cfg.heads * cfg.headDim; - this.q = new nn.Linear(cfg.dModel, qw, { bias: false }); + this.q_proj = new nn.Linear(cfg.dModel, qw, { bias: false }); this.kv = new nn.Linear(cfg.dModel, 2 * qw, { bias: false }); this.mha = new nn.MultiHeadAttention({ qHeads: cfg.heads, @@ -91,7 +95,7 @@ class DecoderCrossAttn extends nn.Module<[Value, CrossKV, Value?], Value> { windowLeft: Infinity, windowRight: Infinity, }); - this.out = new nn.Linear(qw, cfg.dModel, { bias: false }); + this.o_proj = new nn.Linear(qw, cfg.dModel, { bias: false }); } precompute(encOut: Value): CrossKV { const fused = this.kv.forward(encOut); // [kvLen, 2·qw], columns k|v @@ -99,7 +103,7 @@ class DecoderCrossAttn extends nn.Module<[Value, CrossKV, Value?], Value> { return { k: k!, v: v! }; } forward(x: Value, cross: CrossKV, residual?: Value): Value { - return this.out.forward(this.mha.forward(this.q.forward(x), cross.k, cross.v), { + return this.o_proj.forward(this.mha.forward(this.q_proj.forward(x), cross.k, cross.v), { addend: residual, }); } @@ -119,25 +123,25 @@ class DecoderMlp extends nn.Module { } export class DecoderLayer extends nn.Module<[Value, Value, Value, CrossKV], Value> { - readonly norm1: nn.LayerNorm; - readonly selfAttn: DecoderSelfAttn; - readonly norm2: nn.LayerNorm; - readonly crossAttn: DecoderCrossAttn; - readonly norm3: nn.LayerNorm; + readonly input_layernorm: nn.LayerNorm; + readonly self_attn: DecoderSelfAttn; + readonly post_attention_layernorm: nn.LayerNorm; + readonly encoder_attn: DecoderCrossAttn; + readonly final_layernorm: nn.LayerNorm; readonly mlp: DecoderMlp; constructor(cfg: MoonshineConfig) { super(); - this.norm1 = new nn.LayerNorm(cfg.dModel); - this.selfAttn = new DecoderSelfAttn(cfg); - this.norm2 = new nn.LayerNorm(cfg.dModel); - this.crossAttn = new DecoderCrossAttn(cfg); - this.norm3 = new nn.LayerNorm(cfg.dModel); + this.input_layernorm = new nn.LayerNorm(cfg.dModel); + this.self_attn = new DecoderSelfAttn(cfg); + this.post_attention_layernorm = new nn.LayerNorm(cfg.dModel); + this.encoder_attn = new DecoderCrossAttn(cfg); + this.final_layernorm = new nn.LayerNorm(cfg.dModel); this.mlp = new DecoderMlp(cfg.ffn, cfg.dModel); } forward(x: Value, cos: Value, sin: Value, cross: CrossKV): Value { - const h1 = this.selfAttn.forward(this.norm1.forward(x), cos, sin, x); - const h2 = this.crossAttn.forward(this.norm2.forward(h1), cross, h1); - return this.mlp.forward(this.norm3.forward(h2), h2); + const h1 = this.self_attn.forward(this.input_layernorm.forward(x), cos, sin, x); + const h2 = this.encoder_attn.forward(this.post_attention_layernorm.forward(h1), cross, h1); + return this.mlp.forward(this.final_layernorm.forward(h2), h2); } forwardStep( x1: Value, @@ -147,24 +151,35 @@ export class DecoderLayer extends nn.Module<[Value, Value, Value, CrossKV], Valu cache: SelfKV, t: number, ): Value { - const h1 = this.selfAttn.forwardStep(this.norm1.forward(x1), cosRow, sinRow, cache, t, x1); - const h2 = this.crossAttn.forward(this.norm2.forward(h1), cross, h1); - return this.mlp.forward(this.norm3.forward(h2), h2); + const h1 = this.self_attn.forwardStep( + this.input_layernorm.forward(x1), + cosRow, + sinRow, + cache, + t, + x1, + ); + const h2 = this.encoder_attn.forward(this.post_attention_layernorm.forward(h1), cross, h1); + return this.mlp.forward(this.final_layernorm.forward(h2), h2); } } export class MoonshineDecoder extends nn.Module<[Value, Value, Value, readonly CrossKV[]], Value> { + /** Token lookup and the tied LM head: no transposed copy exists, so tying + * is exact by construction. */ + readonly embed_tokens: nn.Embedding; readonly layers: nn.ModuleList; readonly norm: nn.LayerNorm; constructor(cfg: MoonshineConfig) { super(); + this.embed_tokens = new nn.Embedding(cfg.vocab, cfg.dModel); this.layers = new nn.ModuleList( Array.from({ length: cfg.decLayers }, () => new DecoderLayer(cfg)), ); this.norm = new nn.LayerNorm(cfg.dModel); } precomputeCrossKV(encOut: Value): CrossKV[] { - return this.layers.items.map((layer) => layer.crossAttn.precompute(encOut)); + return this.layers.items.map((layer) => layer.encoder_attn.precompute(encOut)); } forward(x: Value, cos: Value, sin: Value, cross: readonly CrossKV[]): Value { if (cross.length !== this.layers.items.length) { diff --git a/packages/runntime/src/zoo/moonshine/encoder.ts b/packages/runntime/src/zoo/moonshine/encoder.ts index ab16fc2b..7fc248bd 100644 --- a/packages/runntime/src/zoo/moonshine/encoder.ts +++ b/packages/runntime/src/zoo/moonshine/encoder.ts @@ -1,18 +1,24 @@ -/** Moonshine encoder: pre-norm transformer layers over the conv-stem frames, - * each running LN, bidirectional MHA with partial rope, and an MLP, with a - * final weight-only LN. +/** Moonshine encoder: the conv stem over raw audio, then pre-norm + * transformer layers, each running LN, bidirectional MHA with partial rope, + * and an MLP, with a final weight-only LN. + * + * The stem is conv1 k=127 s=64 (no bias) → tanh → nn.GroupNorm(1, d) → conv2 + * k=7 s=3 (+bias) → gelu → conv3 k=3 s=2 (+bias) → gelu, valid padding + * throughout. Each stage's bias and activation ride the conv epilogue. * * Attention is one fused qkv matmul with q and k roped in one dispatch each, * then a fully unbounded window. The rope tables carry the score scale, so the - * kernel needs none. */ + * kernel needs none. Module fields are named like the checkpoint. */ import { gelu, nn, rope, slice, type Value } from '../../core/index.ts'; import type { MoonshineConfig } from './config.ts'; +import { K1, K2, K3, S1, S2, S3 } from './convStem.ts'; class EncoderAttn extends nn.Module<[Value, Value, Value, Value?], Value> { + /** q | k | v rows, fused at load. */ readonly qkv: nn.Linear; readonly mha: nn.MultiHeadAttention; - readonly out: nn.Linear; + readonly o_proj: nn.Linear; constructor(private readonly cfg: MoonshineConfig) { super(); const qw = cfg.heads * cfg.headDim; @@ -24,7 +30,7 @@ class EncoderAttn extends nn.Module<[Value, Value, Value, Value?], Value> { windowLeft: Infinity, windowRight: Infinity, }); - this.out = new nn.Linear(qw, cfg.dModel, { bias: false }); + this.o_proj = new nn.Linear(qw, cfg.dModel, { bias: false }); } forward(x: Value, cos: Value, sin: Value, residual?: Value): Value { const { heads, headDim } = this.cfg; @@ -33,7 +39,7 @@ class EncoderAttn extends nn.Module<[Value, Value, Value, Value?], Value> { const q = rope(qkv, cos, sin, { headDim, srcStart: 0, width: qw }); const k = rope(qkv, cos, sin, { headDim, srcStart: qw, width: qw }); const v = slice(qkv, 1, 2 * qw, 3 * qw); - return this.out.forward(this.mha.forward(q, k, v), { addend: residual }); + return this.o_proj.forward(this.mha.forward(q, k, v), { addend: residual }); } } @@ -51,36 +57,46 @@ class EncoderMlp extends nn.Module { } export class EncoderLayer extends nn.Module<[Value, Value, Value], Value> { - readonly norm1: nn.LayerNorm; - readonly attn: EncoderAttn; - readonly norm2: nn.LayerNorm; + readonly input_layernorm: nn.LayerNorm; + readonly self_attn: EncoderAttn; + readonly post_attention_layernorm: nn.LayerNorm; readonly mlp: EncoderMlp; constructor(cfg: MoonshineConfig) { super(); - this.norm1 = new nn.LayerNorm(cfg.dModel); - this.attn = new EncoderAttn(cfg); - this.norm2 = new nn.LayerNorm(cfg.dModel); + this.input_layernorm = new nn.LayerNorm(cfg.dModel); + this.self_attn = new EncoderAttn(cfg); + this.post_attention_layernorm = new nn.LayerNorm(cfg.dModel); this.mlp = new EncoderMlp(cfg); } forward(x: Value, cos: Value, sin: Value): Value { - const h = this.attn.forward(this.norm1.forward(x), cos, sin, x); - return this.mlp.forward(this.norm2.forward(h), h); + const h = this.self_attn.forward(this.input_layernorm.forward(x), cos, sin, x); + return this.mlp.forward(this.post_attention_layernorm.forward(h), h); } } export class MoonshineEncoder extends nn.Module<[Value, Value, Value], Value> { + readonly conv1: nn.Conv1d; + readonly groupnorm: nn.GroupNorm; + readonly conv2: nn.Conv1d; + readonly conv3: nn.Conv1d; readonly layers: nn.ModuleList; - readonly norm: nn.LayerNorm; + readonly layer_norm: nn.LayerNorm; constructor(cfg: MoonshineConfig) { super(); + const d = cfg.dModel; + this.conv1 = new nn.Conv1d(1, d, K1, { stride: S1, bias: false, activation: 'tanh' }); + this.groupnorm = new nn.GroupNorm(1, d); + this.conv2 = new nn.Conv1d(d, 2 * d, K2, { stride: S2, activation: 'gelu' }); + this.conv3 = new nn.Conv1d(2 * d, d, K3, { stride: S3, activation: 'gelu' }); this.layers = new nn.ModuleList( Array.from({ length: cfg.encLayers }, () => new EncoderLayer(cfg)), ); - this.norm = new nn.LayerNorm(cfg.dModel); + this.layer_norm = new nn.LayerNorm(cfg.dModel); } + /** Raw audio [L, 1] to encoder frames [T, d]; the rope tables cover T. */ forward(x: Value, cos: Value, sin: Value): Value { - let h = x; + let h = this.conv3.forward(this.conv2.forward(this.groupnorm.forward(this.conv1.forward(x)))); for (const layer of this.layers.items) h = layer.forward(h, cos, sin); - return this.norm.forward(h); + return this.layer_norm.forward(h); } } diff --git a/packages/runntime/src/zoo/moonshine/index.ts b/packages/runntime/src/zoo/moonshine/index.ts index 82b6d1c7..d01c590b 100644 --- a/packages/runntime/src/zoo/moonshine/index.ts +++ b/packages/runntime/src/zoo/moonshine/index.ts @@ -1,8 +1,8 @@ export { MOONSHINE_BASE, MOONSHINE_TINY, type MoonshineConfig } from './config.ts'; -export { ConvStem, MIN_SAMPLES } from './convStem.ts'; +export { MIN_SAMPLES } from './convStem.ts'; export { EncoderLayer, MoonshineEncoder } from './encoder.ts'; export { DecoderLayer, MoonshineDecoder, type CrossKV, type SelfKV } from './decoder.ts'; -export { MoonshineModel } from './model.ts'; +export { MoonshineEncoderDecoder, MoonshineModel } from './model.ts'; export { createMoonshineTokenizer, type MoonshineTokenizer } from './tokenizer.ts'; export { analyzeSegment, type SegmentAnalysis, type SegmenterOpts } from './segmenter.ts'; export { diff --git a/packages/runntime/src/zoo/moonshine/model.ts b/packages/runntime/src/zoo/moonshine/model.ts index cb392851..d82587a0 100644 --- a/packages/runntime/src/zoo/moonshine/model.ts +++ b/packages/runntime/src/zoo/moonshine/model.ts @@ -1,9 +1,11 @@ -/** The full Moonshine module tree. No constructor takes a sequence length — - * rope tables and cross K/V arrive as forward arguments — and one - * loadStateDict binds the whole checkpoint. +/** The full Moonshine module tree, named like the HF checkpoint: + * `model.encoder` and `model.decoder` under MoonshineForConditionalGeneration. + * No constructor takes a sequence length — rope tables and cross K/V arrive + * as forward arguments — and one loadStateDict binds the whole checkpoint. * - * The embedding weight serves as both token lookup and the tied LM head. No - * transposed copy exists, so tying is exact by construction. */ + * The decoder's embedding weight serves as both token lookup and the tied LM + * head, so a checkpoint that also stores the tied `proj_out.weight` gets it + * reported as unread. */ import { argmaxDot, @@ -16,38 +18,42 @@ import { type Value, } from '../../core/index.ts'; import type { MoonshineConfig } from './config.ts'; -import { ConvStem } from './convStem.ts'; import { MoonshineEncoder } from './encoder.ts'; import { MoonshineDecoder } from './decoder.ts'; import { transformMoonshineStateDict } from './stateDictHooks.ts'; -export class MoonshineModel extends nn.Module { - readonly embedding: nn.Embedding; - readonly stem: ConvStem; +/** HF's MoonshineModel: the encoder and decoder pair. */ +export class MoonshineEncoderDecoder extends nn.Module { readonly encoder: MoonshineEncoder; readonly decoder: MoonshineDecoder; - constructor(readonly cfg: MoonshineConfig) { + constructor(cfg: MoonshineConfig) { super(); - this.embedding = new nn.Embedding(cfg.vocab, cfg.dModel); - this.stem = new ConvStem(cfg); this.encoder = new MoonshineEncoder(cfg); this.decoder = new MoonshineDecoder(cfg); } +} + +export class MoonshineModel extends nn.Module { + readonly model: MoonshineEncoderDecoder; + constructor(readonly cfg: MoonshineConfig) { + super(); + this.model = new MoonshineEncoderDecoder(cfg); + } override transformStateDict(sd: LazyStateDict, prefix: string): void { transformMoonshineStateDict(sd, prefix, this.cfg); } embed(ids: readonly number[]): Value { - return this.embedding.forward(ids); + return this.model.decoder.embed_tokens.forward(ids); } embedFrom(ids: Value): Value { - return gatherRowsFrom(this.embedding.weight.value, ids); + return gatherRowsFrom(this.model.decoder.embed_tokens.weight.value, ids); } logitsLast(h: Value): Value { - const w = this.embedding.weight.value; + const w = this.model.decoder.embed_tokens.weight.value; if (w.shape.dtype !== h.shape.dtype) { throw new Error( `MoonshineModel.logitsLast: embedding is ${w.shape.dtype} but h is ${h.shape.dtype}`, @@ -58,10 +64,12 @@ export class MoonshineModel extends nn.Module { } nextTokenId(h1: Value): Value { - return argmaxDot(this.embedding.weight.value, h1); + return argmaxDot(this.model.decoder.embed_tokens.weight.value, h1); } forward(): Value { - throw new Error('MoonshineModel: use stem/encoder/decoder forwards, embed() and logitsLast()'); + throw new Error( + 'MoonshineModel: use model.encoder/model.decoder forwards, embed() and logitsLast()', + ); } } diff --git a/packages/runntime/src/zoo/moonshine/stateDictHooks.ts b/packages/runntime/src/zoo/moonshine/stateDictHooks.ts index e8f1e99a..77e5a388 100644 --- a/packages/runntime/src/zoo/moonshine/stateDictHooks.ts +++ b/packages/runntime/src/zoo/moonshine/stateDictHooks.ts @@ -1,16 +1,16 @@ -/** Maps the HF Moonshine checkpoint onto the model's parameter tree as lazy - * derived tensors; nothing is fetched here. +/** Fuses the HF Moonshine checkpoint's attention projections as lazy derived + * tensors; nothing is fetched here. Every other tensor binds under its own + * name, since the modules are named like the checkpoint. * * q/k/v row-stack into one fused weight, so the hot path is a single matmul. * No rotary permutation is needed, since HF's convention already matches the * kernel's. Cross-attention q absorbs the full score scale, having no rope. - * Everything else renames. * - * Weights stay in torch layout for the core Linear and Conv1d hooks to - * transpose. An F16 checkpoint gives f16 derived tensors, except the scaled - * cross-attention q, which stays f32 so an f32 model loads exact values. - * Derived byteLengths carry the source file bytes, so load progress still - * sums to the checkpoint size. */ + * Weights stay in torch layout for the core Linear hook to transpose. An F16 + * checkpoint gives f16 derived tensors, except the scaled cross-attention q, + * which stays f32 so an f32 model loads exact values. Derived byteLengths + * carry the source file bytes, so load progress counts every byte the model + * reads. */ import { concatRows, scaleWeight, type LazyStateDict, type LazyTensor } from '../../core/index.ts'; import { checkpointMismatch } from '../errors.ts'; @@ -26,10 +26,6 @@ export function take(sd: LazyStateDict, name: string): LazyTensor { return t; } -export function move(sd: LazyStateDict, from: string, to: string): void { - sd.tensors.set(to, take(sd, from)); -} - /** Takes an attention projection weight, which must be [_, hidden]. */ function takeProjection(sd: LazyStateDict, name: string, hidden: number): LazyTensor { const t = take(sd, name); @@ -42,68 +38,38 @@ function takeProjection(sd: LazyStateDict, name: string, hidden: number): LazyTe return t; } -/** Stacks q|k|v into one torch-layout [3·hidden, hidden] weight, with q times - * `qScale` when given. */ -export function fuseQkv( - sd: LazyStateDict, - hfPrefix: string, - ours: string, - hidden: number, - qScale?: number, -): void { - const part = (p: string) => takeProjection(sd, `${hfPrefix}.${p}_proj.weight`, hidden); +/** Stacks `attn`'s q|k|v projections into its one torch-layout + * [3·hidden, hidden] `qkv.weight`, with q times `qScale` when given. */ +export function fuseQkv(sd: LazyStateDict, attn: string, hidden: number, qScale?: number): void { + const part = (p: string) => takeProjection(sd, `${attn}.${p}_proj.weight`, hidden); const query = part('q'); sd.tensors.set( - ours, + `${attn}.qkv.weight`, concatRows([qScale === undefined ? query : scaleWeight(query, qScale), part('k'), part('v')]), ); } -export function moveTiedEmbedding(sd: LazyStateDict, prefix: string): void { - sd.tensors.delete(qualify(prefix, 'proj_out.weight')); - move( - sd, - qualify(prefix, 'model.decoder.embed_tokens.weight'), - qualify(prefix, 'embedding.weight'), - ); -} - export function transformDecoder( sd: LazyStateDict, prefix: string, dec: { layers: number; hidden: number; headDim: number }, ): void { - const q = (name: string) => qualify(prefix, name); const crossQScale = 1 / Math.sqrt(dec.headDim); for (let i = 0; i < dec.layers; i++) { - const hf = q(`model.decoder.layers.${i}`); - const en = q(`decoder.layers.${i}`); - move(sd, `${hf}.input_layernorm.weight`, `${en}.norm1.weight`); - fuseQkv(sd, `${hf}.self_attn`, `${en}.selfAttn.qkv.weight`, dec.hidden); - move(sd, `${hf}.self_attn.o_proj.weight`, `${en}.selfAttn.out.weight`); - move(sd, `${hf}.post_attention_layernorm.weight`, `${en}.norm2.weight`); + const L = qualify(prefix, `model.decoder.layers.${i}`); + fuseQkv(sd, `${L}.self_attn`, dec.hidden); + const cross = `${L}.encoder_attn`; sd.tensors.set( - `${en}.crossAttn.q.weight`, - scaleWeight(takeProjection(sd, `${hf}.encoder_attn.q_proj.weight`, dec.hidden), crossQScale), + `${cross}.q_proj.weight`, + scaleWeight(takeProjection(sd, `${cross}.q_proj.weight`, dec.hidden), crossQScale), ); sd.tensors.set( - `${en}.crossAttn.kv.weight`, + `${cross}.kv.weight`, concatRows( - ['k', 'v'].map((p) => - takeProjection(sd, `${hf}.encoder_attn.${p}_proj.weight`, dec.hidden), - ), + ['k', 'v'].map((p) => takeProjection(sd, `${cross}.${p}_proj.weight`, dec.hidden)), ), ); - move(sd, `${hf}.encoder_attn.o_proj.weight`, `${en}.crossAttn.out.weight`); - move(sd, `${hf}.final_layernorm.weight`, `${en}.norm3.weight`); - // fc1 is one fused [2·ffn, d] projection; the swigluChunk kernel - // splits it (hidden = first half, gate = second half). - move(sd, `${hf}.mlp.fc1.weight`, `${en}.mlp.fc1.weight`); - move(sd, `${hf}.mlp.fc1.bias`, `${en}.mlp.fc1.bias`); - move(sd, `${hf}.mlp.fc2.weight`, `${en}.mlp.fc2.weight`); - move(sd, `${hf}.mlp.fc2.bias`, `${en}.mlp.fc2.bias`); } - move(sd, q('model.decoder.norm.weight'), q('decoder.norm.weight')); } export function transformMoonshineStateDict( @@ -111,31 +77,8 @@ export function transformMoonshineStateDict( prefix: string, cfg: MoonshineConfig, ): void { - const q = (name: string) => qualify(prefix, name); - - moveTiedEmbedding(sd, prefix); - - move(sd, q('model.encoder.conv1.weight'), q('stem.conv1.weight')); - move(sd, q('model.encoder.groupnorm.weight'), q('stem.groupnorm.weight')); - move(sd, q('model.encoder.groupnorm.bias'), q('stem.groupnorm.bias')); - move(sd, q('model.encoder.conv2.weight'), q('stem.conv2.weight')); - move(sd, q('model.encoder.conv2.bias'), q('stem.conv2.bias')); - move(sd, q('model.encoder.conv3.weight'), q('stem.conv3.weight')); - move(sd, q('model.encoder.conv3.bias'), q('stem.conv3.bias')); - for (let i = 0; i < cfg.encLayers; i++) { - const hf = q(`model.encoder.layers.${i}`); - const en = q(`encoder.layers.${i}`); - move(sd, `${hf}.input_layernorm.weight`, `${en}.norm1.weight`); - fuseQkv(sd, `${hf}.self_attn`, `${en}.attn.qkv.weight`, cfg.dModel); - move(sd, `${hf}.self_attn.o_proj.weight`, `${en}.attn.out.weight`); - move(sd, `${hf}.post_attention_layernorm.weight`, `${en}.norm2.weight`); - move(sd, `${hf}.mlp.fc1.weight`, `${en}.mlp.fc1.weight`); - move(sd, `${hf}.mlp.fc1.bias`, `${en}.mlp.fc1.bias`); - move(sd, `${hf}.mlp.fc2.weight`, `${en}.mlp.fc2.weight`); - move(sd, `${hf}.mlp.fc2.bias`, `${en}.mlp.fc2.bias`); + fuseQkv(sd, qualify(prefix, `model.encoder.layers.${i}.self_attn`), cfg.dModel); } - move(sd, q('model.encoder.layer_norm.weight'), q('encoder.norm.weight')); - transformDecoder(sd, prefix, { layers: cfg.decLayers, hidden: cfg.dModel, headDim: cfg.headDim }); } diff --git a/packages/runntime/src/zoo/moonshine/transcriber.ts b/packages/runntime/src/zoo/moonshine/transcriber.ts index 3c29b917..c3d98d62 100644 --- a/packages/runntime/src/zoo/moonshine/transcriber.ts +++ b/packages/runntime/src/zoo/moonshine/transcriber.ts @@ -131,8 +131,8 @@ export async function createTranscriber( const encCos = tensor(root, enc.cos, [frames.conv3, cfg.headDim]); const encSin = tensor(root, enc.sin, [frames.conv3, cfg.headDim]); - const encOut = model.encoder.forward(model.stem.forward(x), encCos, encSin); - const cross = model.decoder.precomputeCrossKV(encOut); + const encOut = model.model.encoder.forward(x, encCos, encSin); + const cross = model.model.decoder.precomputeCrossKV(encOut); // The K/V co-targets pin every buffer the decode loop reads. evalValues([encOut, ...cross.flatMap((c) => [c.k, c.v])], ex); // Keeps the encoder's GPU tail out of the first burst's readback wait. @@ -168,7 +168,7 @@ export async function createTranscriber( let x = model.embed([ids[base]!]); // burst opener: CPU-known id for (let j = 0; j < burst; j++) { const t = base + j; - const h = model.decoder.forwardStep( + const h = model.model.decoder.forwardStep( x, slice(decCos, 0, t, t + 1), slice(decSin, 0, t, t + 1), diff --git a/packages/runntime/src/zoo/privacy-filter/detector.ts b/packages/runntime/src/zoo/privacy-filter/detector.ts index 4f1adc19..08eaaf4c 100644 --- a/packages/runntime/src/zoo/privacy-filter/detector.ts +++ b/packages/runntime/src/zoo/privacy-filter/detector.ts @@ -17,7 +17,7 @@ import { createTokenizer } from './tokenizer.ts'; import { ViterbiDecoder } from './viterbi.ts'; import { assembleDetect, type Detector } from './pipeline.ts'; import { EagerModel } from './model.ts'; -import { transformHfPrivacyFilterStateDict } from './stateDictHooks.ts'; +import { renameExportedNormScales, transformHfPrivacyFilterStateDict } from './stateDictHooks.ts'; import type { ModelLoadOptions } from '../load.ts'; const BLOCKS = 8; @@ -45,6 +45,8 @@ export async function loadPrivacyFilterModel( if (isRunntimeError(err)) throw err; throw mismatch(messageOf(err)); } + } else { + renameExportedNormScales(sd); } if (sd.metadata['model'] !== 'privacy-filter') { throw mismatch(`file is for model '${sd.metadata['model']}', expected 'privacy-filter'`); diff --git a/packages/runntime/src/zoo/privacy-filter/moeMlp.ts b/packages/runntime/src/zoo/privacy-filter/moeMlp.ts index fd7919f6..998ef4f8 100644 --- a/packages/runntime/src/zoo/privacy-filter/moeMlp.ts +++ b/packages/runntime/src/zoo/privacy-filter/moeMlp.ts @@ -73,7 +73,7 @@ export class EagerMoeMlp extends nn.Module { } private get actDtype(): 'f32' | 'f16' { - return this.norm.scale.shape.dtype === 'f16' ? 'f16' : 'f32'; + return this.norm.weight.shape.dtype === 'f16' ? 'f16' : 'f32'; } override forward(x: Value): Value { diff --git a/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts b/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts index fd899902..94be74bc 100644 --- a/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts +++ b/packages/runntime/src/zoo/privacy-filter/stateDictHooks.ts @@ -7,8 +7,8 @@ * - the rest renames. score.bias is all zero and the head has no bias, so * it is dropped. * An F16 checkpoint gives f16 derived tensors. - * Files from tools/export_pf_safetensors.py use these names as is and - * bypass this mapping. */ + * Files from tools/export_pf_safetensors.py use these names as is, with the + * RMSNorm scales as `scale`, and bypass this mapping. */ import { concatRows, @@ -49,8 +49,8 @@ export function transformHfPrivacyFilterStateDict(sd: LazyStateDict): void { while (sd.tensors.has(`model.layers.${n}.input_layernorm.weight`)) { const src = `model.layers.${n}`; const dst = `block.${n}`; - move(sd, `${src}.input_layernorm.weight`, `${dst}.attn.norm.scale`); - move(sd, `${src}.post_attention_layernorm.weight`, `${dst}.mlp.norm.scale`); + move(sd, `${src}.input_layernorm.weight`, `${dst}.attn.norm.weight`); + move(sd, `${src}.post_attention_layernorm.weight`, `${dst}.mlp.norm.weight`); for (const suffix of ['weight', 'bias']) { sd.tensors.set( `${dst}.attn.qkv.${suffix}`, @@ -72,8 +72,16 @@ export function transformHfPrivacyFilterStateDict(sd: LazyStateDict): void { n++; } move(sd, 'model.embed_tokens.weight', 'embedding.weight'); - move(sd, 'model.norm.weight', 'norm.scale'); + move(sd, 'model.norm.weight', 'norm.weight'); move(sd, 'score.weight', 'unembedding.weight'); sd.tensors.delete('score.bias'); sd.metadata['model'] = 'privacy-filter'; } + +/** Renames the RMSNorm scales of an exported file, written as `scale`, to + * the core layer's `weight`. */ +export function renameExportedNormScales(sd: LazyStateDict): void { + for (const name of [...sd.tensors.keys()]) { + if (name.endsWith('.scale')) move(sd, name, `${name.slice(0, -'scale'.length)}weight`); + } +} diff --git a/packages/runntime/test/core/nn/layers.test.ts b/packages/runntime/test/core/nn/layers.test.ts index a9de2e0e..96f92339 100644 --- a/packages/runntime/test/core/nn/layers.test.ts +++ b/packages/runntime/test/core/nn/layers.test.ts @@ -1019,8 +1019,8 @@ describe('normalization layers', () => { expect(() => new GroupNorm(3, 8)).toThrow(/numGroups 3 must divide the 8 channels/); }); - it('RMSNorm has one scale per feature', () => { - expect(shapes(new RMSNorm(8))).toEqual({ scale: meta([8]) }); + it('RMSNorm has one weight per feature', () => { + expect(shapes(new RMSNorm(8))).toEqual({ weight: meta([8]) }); }); it.each([ diff --git a/packages/runntime/test/core/nn/loadStateDict.test.ts b/packages/runntime/test/core/nn/loadStateDict.test.ts index db8f7ed1..d9b53d66 100644 --- a/packages/runntime/test/core/nn/loadStateDict.test.ts +++ b/packages/runntime/test/core/nn/loadStateDict.test.ts @@ -118,34 +118,61 @@ describe('transform hooks', () => { }); }); -describe('strict key check', () => { - it.each<{ name: string; sd: Entries; message: RegExp }>([ +describe('key check', () => { + const unexpected: Entries = { + a: new Float32Array(2), + b: new Float32Array(2), + c: new Float32Array(2), + }; + + it.each<{ name: string; sd: Entries; strict?: boolean; message: RegExp }>([ { name: 'missing', sd: { a: new Float32Array(2) }, message: /missing: b\./ }, - { - name: 'unexpected', - sd: { a: new Float32Array(2), b: new Float32Array(2), c: new Float32Array(2) }, - message: /unexpected: c\./, - }, + { name: 'unexpected, when strict', sd: unexpected, strict: true, message: /unexpected: c\./ }, { name: 'both', sd: { a: new Float32Array(2), c: new Float32Array(2) }, message: /missing: b\. unexpected: c\./, }, - ])('reports $name keys before uploading anything', async ({ sd, message }) => { + ])('reports $name keys before uploading anything', async ({ sd, strict, message }) => { const upload = recorder(); - await expect(load(new Pair(), sd, upload)).rejects.toMatchObject({ - code: 'CHECKPOINT_MISMATCH', - message, - }); + await expect(loadLazyStateDict(new Pair(), dict(sd), { upload, strict })).rejects.toMatchObject( + { code: 'CHECKPOINT_MISMATCH', message }, + ); expect(upload).not.toHaveBeenCalled(); }); + it('warns about unexpected keys and binds the rest when not strict', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const m = new Pair(); + const upload = recorder(); + const progress: [number, number][] = []; + await loadLazyStateDict(m, dict(unexpected), { + upload, + onProgress: (_, done, total) => progress.push([done, total]), + }); + expect(warn).toHaveBeenCalledWith( + 'loadStateDict: state dict has tensors the model does not read: c.', + ); + expect(upload).toHaveBeenCalledTimes(2); + expect(m.a.loaded && m.b.loaded).toBe(true); + // The unbound tensor's bytes are never read, so they are not in the total. + expect(progress).toEqual([ + [8, 16], + [16, 16], + ]); + warn.mockRestore(); + }); + it('lists eight names and counts the rest', async () => { const extra = Object.fromEntries( Array.from({ length: 10 }, (_, i) => [`x${i}`, new Float32Array(1)]), ); await expect( - load(new Pair(), { a: new Float32Array(2), b: new Float32Array(2), ...extra }), + loadLazyStateDict( + new Pair(), + dict({ a: new Float32Array(2), b: new Float32Array(2), ...extra }), + { upload: recorder(), strict: true }, + ), ).rejects.toThrow(/unexpected: x0, x1, x2, x3, x4, x5, x6, x7 … \(\+2\)\./); }); diff --git a/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts b/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts index 7a1f1f85..ed577c85 100644 --- a/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts +++ b/packages/runntime/test/zoo/embeddinggemma2/stateDictHooks.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import type { LazyStateDict } from '../../../src/core/weights/safetensors.ts'; +import { memoryStateDict, type LazyStateDict } from '../../../src/core/weights/safetensors.ts'; import { EMBEDDINGGEMMA2_AUDIO, EMBEDDINGGEMMA2_TEXT, @@ -38,20 +38,18 @@ const gridRows = (base: number, order: number[], cols: number) => const ones = (n: number) => new Array(n).fill(1); /** Maps `sd` and returns a getter over the result, after checking that the - * result holds exactly `renamed`'s targets plus `mapped`, with each renamed - * tensor passed through untouched. */ + * result holds exactly `untouched` plus `mapped`, with each untouched tensor + * passed through as it was. */ function mapAndCheck( sd: LazyStateDict, run: (sd: LazyStateDict) => void, - renamed: Record, + untouched: string[], mapped: string[], ) { const before = new Map(sd.tensors); run(sd); - expect([...sd.tensors.keys()].sort()).toEqual([...Object.values(renamed), ...mapped].sort()); - for (const [hf, ours] of Object.entries(renamed)) { - expect(sd.tensors.get(ours)).toBe(before.get(hf)); - } + expect([...sd.tensors.keys()].sort()).toEqual([...untouched, ...mapped].sort()); + for (const name of untouched) expect(sd.tensors.get(name)).toBe(before.get(name)); return (name: string) => sd.tensors.get(name)!; } @@ -72,27 +70,29 @@ describe('transformEmbeddingGemma2StateDict', () => { const S = 'language_model.'; const L = (i: number) => `language_model.layers.${i}`; - const layerRenamed = (i: number): Record => ({ - [`${L(i)}.self_attn.o_proj.weight`]: `layers.${i}.o_proj.weight`, - [`${L(i)}.mlp.gate_proj.weight`]: `layers.${i}.gate_proj.weight`, - [`${L(i)}.mlp.up_proj.weight`]: `layers.${i}.up_proj.weight`, - [`${L(i)}.mlp.down_proj.weight`]: `layers.${i}.down_proj.weight`, - [`${L(i)}.ple_block.per_layer_input_gate.weight`]: `layers.${i}.per_layer_input_gate.weight`, - [`${L(i)}.ple_block.per_layer_projection.weight`]: `layers.${i}.per_layer_projection.weight`, - [`${L(i)}.ple_block.post_per_layer_input_norm.weight`]: `layers.${i}.post_per_layer_input_norm.scale`, - [`${L(i)}.input_layernorm.weight`]: `layers.${i}.input_layernorm.scale`, - [`${L(i)}.post_attention_layernorm.weight`]: `layers.${i}.post_attention_layernorm.scale`, - [`${L(i)}.pre_feedforward_layernorm.weight`]: `layers.${i}.pre_feedforward_layernorm.scale`, - [`${L(i)}.post_feedforward_layernorm.weight`]: `layers.${i}.post_feedforward_layernorm.scale`, - }); - const RENAMED: Record = { - ...layerRenamed(0), - ...layerRenamed(1), - [`${S}ple.per_layer_projection_norm.weight`]: 'ple.per_layer_projection_norm.scale', - [`${S}embed_tokens.weight`]: 'embed_tokens.weight', - [`${S}norm.weight`]: 'norm.scale', - [`${S}embedding_projection.weight`]: 'embedding_projection.weight', - }; + /** Tensors the hook leaves as they are, under their own names. */ + const layerUntouched = (i: number) => + [ + 'self_attn.o_proj.weight', + 'mlp.gate_proj.weight', + 'mlp.up_proj.weight', + 'mlp.down_proj.weight', + 'ple_block.per_layer_input_gate.weight', + 'ple_block.per_layer_projection.weight', + 'ple_block.post_per_layer_input_norm.weight', + 'input_layernorm.weight', + 'post_attention_layernorm.weight', + 'pre_feedforward_layernorm.weight', + 'post_feedforward_layernorm.weight', + ].map((n) => `${L(i)}.${n}`); + const UNTOUCHED = [ + ...layerUntouched(0), + ...layerUntouched(1), + `${S}ple.per_layer_projection_norm.weight`, + `${S}embed_tokens.weight`, + `${S}norm.weight`, + `${S}embedding_projection.weight`, + ]; /** Layer `i`'s attention: q over `heads` heads of `d` rows, k and v over * one, norms 1 … d and d + 1 … 2d. */ const attention = (i: number, d: number): Record => ({ @@ -110,7 +110,7 @@ describe('transformEmbeddingGemma2StateDict', () => { [`${L(i)}.layer_scalar`]: { values: [0.25], shape: [1] }, }); const CHECKPOINT: Record = { - ...Object.fromEntries(Object.keys(RENAMED).map((name) => [name, ZERO])), + ...Object.fromEntries(UNTOUCHED.map((name) => [name, ZERO])), ...attention(0, 4), ...attention(1, 8), [`${S}ple.per_layer_model_projection.weight`]: grid(0, 2, 4), @@ -119,14 +119,14 @@ describe('transformEmbeddingGemma2StateDict', () => { it.for(DTYPES)('maps a $dtype checkpoint', async ({ dtype, type, elemBytes }) => { const sd = await checkpoint(dtype, CHECKPOINT); const total = sd.totalBytes(); - const get = mapAndCheck(sd, (s) => transformEmbeddingGemma2StateDict(s, '', cfg), RENAMED, [ - 'layers.0.qkv.weight', - 'layers.0.qkv_norm', - 'layers.0.layer_scalar', - 'layers.1.qkv.weight', - 'layers.1.qkv_norm', - 'layers.1.layer_scalar', - 'ple.per_layer_model_projection.weight', + const get = mapAndCheck(sd, (s) => transformEmbeddingGemma2StateDict(s, '', cfg), UNTOUCHED, [ + `${L(0)}.self_attn.qkv.weight`, + `${L(0)}.self_attn.qkv_norm`, + `${L(0)}.layer_scalar`, + `${L(1)}.self_attn.qkv.weight`, + `${L(1)}.self_attn.qkv_norm`, + `${L(1)}.layer_scalar`, + `${S}ple.per_layer_model_projection.weight`, ]); // Rope pairs (p, p + d/2) of each d-row head become rows (2p, 2p + 1): @@ -138,7 +138,7 @@ describe('transformEmbeddingGemma2StateDict', () => { [1, 8, [0, 4, 1, 5, 2, 6, 3, 7]], ]; for (const [i, d, head] of layers) { - const qkv = get(`layers.${i}.qkv.weight`); + const qkv = get(`${L(i)}.self_attn.qkv.weight`); expect(qkv.shape).toEqual([4 * d, 4]); expect(qkv.byteLength).toBe(16 * d * elemBytes); expect(await contents(qkv)).toEqual({ @@ -156,7 +156,7 @@ describe('transformEmbeddingGemma2StateDict', () => { // q_norm reordered like q, once per q head; k_norm once; ones for v. // Each norm's bytes count once. - const norm = get(`layers.${i}.qkv_norm`); + const norm = get(`${L(i)}.self_attn.qkv_norm`); expect(norm.shape).toEqual([4 * d]); expect(norm.byteLength).toBe(2 * d * elemBytes); const qn = head.map((r) => r + 1); @@ -167,13 +167,13 @@ describe('transformEmbeddingGemma2StateDict', () => { }); // The CPU parameter reads f32 whatever the checkpoint holds. - const scalar = get(`layers.${i}.layer_scalar`); + const scalar = get(`${L(i)}.layer_scalar`); expect(scalar.byteLength).toBe(elemBytes); expect(await contents(scalar)).toEqual({ type: 'f32', values: [0.25] }); } // Scaled, so f32 on every checkpoint. - const ple = get('ple.per_layer_model_projection.weight'); + const ple = get(`${S}ple.per_layer_model_projection.weight`); expect(ple.shape).toEqual([2, 4]); expect(ple.byteLength).toBe(8 * elemBytes); expect(await contents(ple)).toEqual({ type: 'f32', values: [0, 0.5, 1, 1.5, 5, 5.5, 6, 6.5] }); @@ -194,21 +194,23 @@ describe('transformEmbeddingGemma2VisionStateDict', () => { layers: 1, }; + const P = 'vision_tower.patch_embedder'; const T = 'vision_tower.encoder.layers.0'; - const RENAMED: Record = { - 'vision_tower.patch_embedder.input_proj.weight': 'input_proj.weight', - [`${T}.self_attn.o_proj.linear.weight`]: 'layers.0.o_proj.weight', - [`${T}.mlp.gate_proj.linear.weight`]: 'layers.0.gate_proj.weight', - [`${T}.mlp.up_proj.linear.weight`]: 'layers.0.up_proj.weight', - [`${T}.mlp.down_proj.linear.weight`]: 'layers.0.down_proj.weight', - [`${T}.input_layernorm.weight`]: 'layers.0.input_layernorm.scale', - [`${T}.post_attention_layernorm.weight`]: 'layers.0.post_attention_layernorm.scale', - [`${T}.pre_feedforward_layernorm.weight`]: 'layers.0.pre_feedforward_layernorm.scale', - [`${T}.post_feedforward_layernorm.weight`]: 'layers.0.post_feedforward_layernorm.scale', - 'embed_vision.embedding_projection.weight': 'embedding_projection.weight', - }; + /** Tensors the hook leaves as they are, under their own names. */ + const UNTOUCHED = [ + `${P}.input_proj.weight`, + `${T}.self_attn.o_proj.linear.weight`, + `${T}.mlp.gate_proj.linear.weight`, + `${T}.mlp.up_proj.linear.weight`, + `${T}.mlp.down_proj.linear.weight`, + `${T}.input_layernorm.weight`, + `${T}.post_attention_layernorm.weight`, + `${T}.pre_feedforward_layernorm.weight`, + `${T}.post_feedforward_layernorm.weight`, + 'embed_vision.embedding_projection.weight', + ]; const CHECKPOINT: Record = { - ...Object.fromEntries(Object.keys(RENAMED).map((name) => [name, ZERO])), + ...Object.fromEntries(UNTOUCHED.map((name) => [name, ZERO])), // [2, positions, hidden]: x rows 1 2 | 3 4, y rows 5 6 | 7 8. 'vision_tower.patch_embedder.position_embedding_table': { values: [1, 2, 3, 4, 5, 6, 7, 8], @@ -227,19 +229,18 @@ describe('transformEmbeddingGemma2VisionStateDict', () => { const get = mapAndCheck( sd, (s) => transformEmbeddingGemma2VisionStateDict(s, '', cfg), - RENAMED, + UNTOUCHED, [ - 'pos_x.weight', - 'pos_y.weight', - 'layers.0.qkv.weight', - 'layers.0.qkv_norm', - 'embedding_pre_projection_norm.scale', + `${P}.pos_x.weight`, + `${P}.pos_y.weight`, + `${T}.self_attn.qkv.weight`, + `${T}.self_attn.qkv_norm`, ], ); for (const [name, values] of [ - ['pos_x.weight', [1, 2, 3, 4]], - ['pos_y.weight', [5, 6, 7, 8]], + [`${P}.pos_x.weight`, [1, 2, 3, 4]], + [`${P}.pos_y.weight`, [5, 6, 7, 8]], ] as const) { const table = get(name); expect(table.shape).toEqual([2, 2]); @@ -249,7 +250,7 @@ describe('transformEmbeddingGemma2VisionStateDict', () => { // Rows 0 2 1 3 of each half head. const halfHeads = [0, 2, 1, 3, 4, 6, 5, 7]; - const qkv = get('layers.0.qkv.weight'); + const qkv = get(`${T}.self_attn.qkv.weight`); expect(qkv.shape).toEqual([24, 2]); expect(qkv.byteLength).toBe(48 * elemBytes); expect(await contents(qkv)).toEqual({ @@ -261,7 +262,7 @@ describe('transformEmbeddingGemma2VisionStateDict', () => { ], }); - const norm = get('layers.0.qkv_norm'); + const norm = get(`${T}.self_attn.qkv_norm`); expect(norm.shape).toEqual([24]); expect(norm.byteLength).toBe(16 * elemBytes); expect(await contents(norm)).toEqual({ @@ -269,12 +270,6 @@ describe('transformEmbeddingGemma2VisionStateDict', () => { values: [1, 3, 2, 4, 5, 7, 6, 8, 11, 13, 12, 14, 15, 17, 16, 18, ...ones(8)], }); - // Ones download nothing and are exact in f16. - const pre = get('embedding_pre_projection_norm.scale'); - expect(pre.shape).toEqual([2]); - expect(pre.byteLength).toBe(0); - expect(await contents(pre)).toEqual({ type: 'f16', values: [1, 1] }); - expect(sd.totalBytes()).toBe(total); }); }); @@ -297,42 +292,41 @@ describe('transformEmbeddingGemma2AudioStateDict', () => { const A = 'audio_tower.'; const sub = `${A}subsample_conv_projection.`; const T = `${A}layers.0`; - /** Every clipped linear, HF prefix → ours. */ - const CLIPPED: Record = { - [`${T}.feed_forward1.ffw_layer_1`]: 'layers.0.feed_forward1.ffw_layer_1', - [`${T}.feed_forward1.ffw_layer_2`]: 'layers.0.feed_forward1.ffw_layer_2', - [`${T}.feed_forward2.ffw_layer_1`]: 'layers.0.feed_forward2.ffw_layer_1', - [`${T}.feed_forward2.ffw_layer_2`]: 'layers.0.feed_forward2.ffw_layer_2', - [`${T}.self_attn.post`]: 'layers.0.post', - [`${T}.lconv1d.linear_start`]: 'layers.0.lconv.linear_start', - [`${T}.lconv1d.linear_end`]: 'layers.0.lconv.linear_end', - }; + /** Every clipped linear but q, k and v. */ + const CLIPPED = [ + `${T}.feed_forward1.ffw_layer_1`, + `${T}.feed_forward1.ffw_layer_2`, + `${T}.feed_forward2.ffw_layer_1`, + `${T}.feed_forward2.ffw_layer_2`, + `${T}.self_attn.post`, + `${T}.lconv1d.linear_start`, + `${T}.lconv1d.linear_end`, + ]; const BOUNDS = ['input_min', 'input_max', 'output_min', 'output_max']; - const RENAMED: Record = { - ...Object.fromEntries( - Object.entries(CLIPPED).map(([hf, ours]) => [`${hf}.linear.weight`, `${ours}.weight`]), - ), - [`${sub}layer0.norm.weight`]: 'norm0.weight', - [`${sub}layer1.norm.weight`]: 'norm1.weight', - [`${sub}input_proj_linear.weight`]: 'input_proj.weight', - [`${T}.feed_forward1.pre_layer_norm.weight`]: 'layers.0.feed_forward1.pre_layer_norm.scale', - [`${T}.feed_forward2.pre_layer_norm.weight`]: 'layers.0.feed_forward2.pre_layer_norm.scale', - [`${T}.lconv1d.pre_layer_norm.weight`]: 'layers.0.lconv.pre_layer_norm.scale', - [`${T}.lconv1d.conv_norm.weight`]: 'layers.0.lconv.conv_norm.scale', - [`${T}.norm_pre_attn.weight`]: 'layers.0.norm_pre_attn.scale', - [`${T}.norm_post_attn.weight`]: 'layers.0.norm_post_attn.scale', - [`${T}.norm_out.weight`]: 'layers.0.norm_out.scale', - [`${A}output_proj.weight`]: 'output_proj.weight', - [`${A}output_proj.bias`]: 'output_proj.bias', - 'embed_audio.embedding_projection.weight': 'embedding_projection.weight', - }; + /** Tensors the hook leaves as they are, under their own names. */ + const UNTOUCHED = [ + ...CLIPPED.map((c) => `${c}.linear.weight`), + `${sub}layer0.norm.weight`, + `${sub}layer1.norm.weight`, + `${sub}input_proj_linear.weight`, + `${T}.feed_forward1.pre_layer_norm.weight`, + `${T}.feed_forward2.pre_layer_norm.weight`, + `${T}.lconv1d.pre_layer_norm.weight`, + `${T}.lconv1d.conv_norm.weight`, + `${T}.norm_pre_attn.weight`, + `${T}.norm_post_attn.weight`, + `${T}.norm_out.weight`, + `${A}output_proj.weight`, + `${A}output_proj.bias`, + 'embed_audio.embedding_projection.weight', + ]; const bound = (prefix: string, values: number[]) => Object.fromEntries( BOUNDS.map((b, i) => [`${prefix}.${b}`, { values: [values[i]!], shape: [1] }]), ); const CHECKPOINT: Record = { - ...Object.fromEntries(Object.keys(RENAMED).map((name) => [name, ZERO])), - ...Object.assign({}, ...Object.keys(CLIPPED).map((p) => bound(p, [-1, 1, -2, 2]))), + ...Object.fromEntries(UNTOUCHED.map((name) => [name, ZERO])), + ...Object.assign({}, ...CLIPPED.map((p) => bound(p, [-1, 1, -2, 2]))), ...Object.assign( {}, ...['q_proj', 'k_proj', 'v_proj'].map((p) => bound(`${T}.self_attn.${p}`, [-3, 3, -4, 4])), @@ -364,8 +358,6 @@ describe('transformEmbeddingGemma2AudioStateDict', () => { shape: [4, 1, 2], }, }; - const clipNames = Object.values(CLIPPED).map((ours) => `${ours}.clip`); - it.for(DTYPES)('maps a $dtype checkpoint', async ({ dtype, type, elemBytes }) => { const sd = await checkpoint(dtype, CHECKPOINT); const total = sd.totalBytes(); @@ -373,25 +365,24 @@ describe('transformEmbeddingGemma2AudioStateDict', () => { const get = mapAndCheck( sd, (s) => transformEmbeddingGemma2AudioStateDict(s, '', cfg), - RENAMED, + UNTOUCHED, [ - ...clipNames, - 'conv0.weight', - 'conv1.weight', - 'layers.0.feed_forward1.post_layer_norm.scale', - 'layers.0.feed_forward2.post_layer_norm.scale', - 'layers.0.qkv.weight', - 'layers.0.qkv.clip', - 'layers.0.rel', - 'layers.0.q_scale', - 'layers.0.lconv.conv', - 'embedding_pre_projection_norm.scale', + `${sub}layer0.conv.weight`, + `${sub}layer1.conv.weight`, + `${T}.feed_forward1.post_layer_norm.weight`, + `${T}.feed_forward2.post_layer_norm.weight`, + `${T}.self_attn.qkv.linear.weight`, + ...BOUNDS.map((b) => `${T}.self_attn.qkv.${b}`), + ...CLIPPED.flatMap((c) => BOUNDS.map((b) => `${c}.${b}`)), + `${T}.self_attn.rel`, + `${T}.self_attn.q_scale`, + `${T}.lconv1d.depthwise_conv1d.weight`, ], ); // conv0 only folds its taps: the source tensor, reshaped. const source = before.get(`${sub}layer0.conv.weight`)!; - const conv0 = get('conv0.weight'); + const conv0 = get(`${sub}layer0.conv.weight`); expect(conv0.shape).toEqual([2, 9]); expect(conv0.byteLength).toBe(18 * elemBytes); if (source.kind === 'quant' || conv0.kind === 'quant') @@ -399,7 +390,7 @@ describe('transformEmbeddingGemma2AudioStateDict', () => { expect([conv0.kind, conv0.dtype]).toEqual([source.kind, source.dtype]); // Tap k of channel c moves from c·9 + k to k·2 + c. - const conv1 = get('conv1.weight'); + const conv1 = get(`${sub}layer1.conv.weight`); expect(conv1.shape).toEqual([1, 18]); expect(conv1.byteLength).toBe(18 * elemBytes); expect(await contents(conv1)).toEqual({ @@ -409,8 +400,8 @@ describe('transformEmbeddingGemma2AudioStateDict', () => { // Times the residual weight 0.5, so f32 on every checkpoint. for (const [name, values] of [ - ['layers.0.feed_forward1.post_layer_norm.scale', [1, 2, 3, 4]], - ['layers.0.feed_forward2.post_layer_norm.scale', [0.5, 1.5, 2.5, 3.5]], + [`${T}.feed_forward1.post_layer_norm.weight`, [1, 2, 3, 4]], + [`${T}.feed_forward2.post_layer_norm.weight`, [0.5, 1.5, 2.5, 3.5]], ] as const) { const norm = get(name); expect(norm.shape).toEqual([4]); @@ -418,7 +409,7 @@ describe('transformEmbeddingGemma2AudioStateDict', () => { expect(await contents(norm)).toEqual({ type: 'f32', values }); } - const qkv = get('layers.0.qkv.weight'); + const qkv = get(`${T}.self_attn.qkv.linear.weight`); expect(qkv.shape).toEqual([12, 4]); expect(qkv.byteLength).toBe(48 * elemBytes); expect(await contents(qkv)).toEqual({ @@ -426,18 +417,26 @@ describe('transformEmbeddingGemma2AudioStateDict', () => { values: [0, 100, 200].flatMap((base) => gridRows(base, [0, 1, 2, 3], 4)), }); - // Bounds are CPU data: f32, and their bytes are not counted. - for (const name of clipNames) { - expect(await contents(get(name))).toEqual({ type: 'f32', values: [-1, 1, -2, 2] }); + // The fused projection's bounds are q's, f32, each carrying one + // scalar's bytes. + for (const [i, b] of BOUNDS.entries()) { + const value = get(`${T}.self_attn.qkv.${b}`); + expect(value.byteLength).toBe(elemBytes); + expect(await contents(value)).toEqual({ type: 'f32', values: [[-3, 3, -4, 4][i]!] }); + } + + // Every other clipped linear keeps its own bounds, f32. + for (const c of CLIPPED) { + for (const [i, b] of BOUNDS.entries()) { + const value = get(`${c}.${b}`); + expect(value.byteLength).toBe(elemBytes); + expect(await contents(value)).toEqual({ type: 'f32', values: [[-1, 1, -2, 2][i]!] }); + } } - expect(await contents(get('layers.0.qkv.clip'))).toEqual({ - type: 'f32', - values: [-3, 3, -4, 4], - }); // Sinusoid rows for distances 0 and 1 at hidden 4: angle j·10000^-i for // i = 0, 1, sines then cosines. - const rel = get('layers.0.rel'); + const rel = get(`${T}.self_attn.rel`); expect(rel.shape).toEqual([2, 4]); expect(rel.byteLength).toBe(16 * elemBytes); const relValues = await contents(rel); @@ -445,7 +444,7 @@ describe('transformEmbeddingGemma2AudioStateDict', () => { const want = [0, 0, 1, 1, Math.sin(1), Math.sin(1e-4), Math.cos(1), Math.cos(1e-4)]; relValues.values.forEach((v, i) => expect(v).toBeCloseTo(want[i]!, 6)); - const qScale = get('layers.0.q_scale'); + const qScale = get(`${T}.self_attn.q_scale`); expect(qScale.shape).toEqual([4]); expect(qScale.byteLength).toBe(4 * elemBytes); const qScaleValues = await contents(qScale); @@ -453,17 +452,40 @@ describe('transformEmbeddingGemma2AudioStateDict', () => { qScaleValues.values.forEach((v) => expect(v).toBeCloseTo(0.5, 6)); // [hidden, 1, K] → [K, hidden]: tap 0 of every channel, then tap 1. - const lconv = get('layers.0.lconv.conv'); + const lconv = get(`${T}.lconv1d.depthwise_conv1d.weight`); expect(lconv.shape).toEqual([2, 4]); expect(lconv.byteLength).toBe(8 * elemBytes); expect(await contents(lconv)).toEqual({ type, values: [1, 3, 5, 7, 2, 4, 6, 8] }); - expect(await contents(get('embedding_pre_projection_norm.scale'))).toEqual({ - type: 'f16', - values: [1, 1], - }); + // k's and v's bounds, 4 scalars each, are the only bytes lost. + expect(sd.totalBytes()).toBe(total - 8 * elemBytes); + }); - // 10 clipped linears' worth of bounds, 4 scalars each. - expect(sd.totalBytes()).toBe(total - 40 * elemBytes); + // The bounds bind to CPU parameters, which hold a BF16 file tensor as raw + // bits: 2 would read as 16384. + it('decodes the bounds of a BF16 checkpoint to f32', async () => { + /** bf16 bits: the top half of the f32 pattern. */ + const bf16 = (values: number[]) => + Uint16Array.from(new Uint32Array(new Float32Array(values).buffer), (bits) => bits >>> 16); + const sd = memoryStateDict( + Object.fromEntries( + Object.entries(CHECKPOINT).map(([name, { values, shape }]) => [ + name, + { data: bf16(values), shape }, + ]), + ), + ); + transformEmbeddingGemma2AudioStateDict(sd, '', cfg); + for (const [prefix, want] of [ + ...CLIPPED.map((c) => [c, [-1, 1, -2, 2]] as const), + [`${T}.self_attn.qkv`, [-3, 3, -4, 4]] as const, + ]) { + for (const [i, b] of BOUNDS.entries()) { + expect(await contents(sd.tensors.get(`${prefix}.${b}`)!)).toEqual({ + type: 'f32', + values: [want[i]!], + }); + } + } }); }); diff --git a/packages/runntime/test/zoo/minilm/model.gpu.test.ts b/packages/runntime/test/zoo/minilm/model.gpu.test.ts index fa5f9b8f..8e304824 100644 --- a/packages/runntime/test/zoo/minilm/model.gpu.test.ts +++ b/packages/runntime/test/zoo/minilm/model.gpu.test.ts @@ -143,7 +143,9 @@ describe('MiniLM from a quantized checkpoint', () => { .filter((name) => name.endsWith('weightScale')); expect(scaleNames).toEqual( Array.from({ length: CFG.layers }, (_, i) => - ['qkv', 'out', 'up', 'down'].map((m) => `layers.${i}.${m}.weightScale`), + ['attention.self.qkv', 'attention.output.dense', 'intermediate.dense', 'output.dense'].map( + (m) => `encoder.layer.${i}.${m}.weightScale`, + ), ).flat(), ); diff --git a/packages/runntime/test/zoo/minilm/stateDictHooks.test.ts b/packages/runntime/test/zoo/minilm/stateDictHooks.test.ts index e67a1d03..9d716916 100644 --- a/packages/runntime/test/zoo/minilm/stateDictHooks.test.ts +++ b/packages/runntime/test/zoo/minilm/stateDictHooks.test.ts @@ -35,6 +35,15 @@ const CHECKPOINT: Record = { 'pooler.dense.bias': { values: [0, 0], shape: [2] }, }; +/** The checkpoint's tensors the hook consumes into derived ones. */ +const CONSUMED = [ + 'embeddings.token_type_embeddings.weight', + ...['query', 'key', 'value'].flatMap((p) => [ + `encoder.layer.0.attention.self.${p}.weight`, + `encoder.layer.0.attention.self.${p}.bias`, + ]), +]; + describe('transformMinilmStateDict', () => { it.for([ { dtype: 'F32', elemBytes: 4 }, @@ -42,44 +51,45 @@ describe('transformMinilmStateDict', () => { ] as const)('maps a $dtype checkpoint', async ({ dtype, elemBytes }) => { const sd = await checkpoint(dtype, CHECKPOINT); const total = sd.totalBytes(); + const before = new Map(sd.tensors); transformMinilmStateDict(sd, '', CFG); const get = (name: string) => sd.tensors.get(name)!; - // Renamed keys only: pooler and position_ids are gone. + // Every other tensor stays as it is, the pooler and position_ids too. + const untouched = Object.keys(CHECKPOINT).filter( + (name) => !CONSUMED.includes(name) && name !== 'embeddings.position_embeddings.weight', + ); expect([...sd.tensors.keys()].sort()).toEqual( [ - 'embeddings.word.weight', - 'embeddings.position.weight', - 'embeddings.norm.weight', - 'embeddings.norm.bias', - ...['qkv', 'out', 'attnNorm', 'up', 'down', 'ffnNorm'].flatMap((m) => [ - `layers.0.${m}.weight`, - `layers.0.${m}.bias`, - ]), + ...untouched, + 'embeddings.position_embeddings.weight', + 'encoder.layer.0.attention.self.qkv.weight', + 'encoder.layer.0.attention.self.qkv.bias', ].sort(), ); + for (const name of untouched) expect(get(name)).toBe(before.get(name)); // q|k|v rows in that order, q and k times 0.5, v as is. Derived // tensors are f32 on every checkpoint. - const qkvWeight = get('layers.0.qkv.weight'); + const qkvWeight = get('encoder.layer.0.attention.self.qkv.weight'); expect(qkvWeight.shape).toEqual([6, 2]); expect(qkvWeight.byteLength).toBe(12 * elemBytes); expect(await contents(qkvWeight)).toEqual({ type: 'f32', values: [0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4, 9, 10, 11, 12], }); - const qkvBias = get('layers.0.qkv.bias'); + const qkvBias = get('encoder.layer.0.attention.self.qkv.bias'); expect(qkvBias.shape).toEqual([6]); expect(qkvBias.byteLength).toBe(6 * elemBytes); expect(await contents(qkvBias)).toEqual({ type: 'f32', values: [0.5, 1, 1.5, 2, 5, 6] }); // position + tokenType row 0: [1 + 10, 2 + 20, 3 + 10, 4 + 20]. - const position = get('embeddings.position.weight'); + const position = get('embeddings.position_embeddings.weight'); expect(position.shape).toEqual([2, 2]); expect(position.byteLength).toBe(8 * elemBytes); expect(await contents(position)).toEqual({ type: 'f32', values: [11, 22, 13, 24] }); - // The dropped pooler (6 elems) and position_ids (2) are the only bytes lost. - expect(sd.totalBytes()).toBe(total - 8 * elemBytes); + // Nothing is dropped: every derived tensor carries its sources' bytes. + expect(sd.totalBytes()).toBe(total); }); }); diff --git a/packages/runntime/test/zoo/moonshine-streaming/stateDictHooks.test.ts b/packages/runntime/test/zoo/moonshine-streaming/stateDictHooks.test.ts index f41b5153..d9f3f2bb 100644 --- a/packages/runntime/test/zoo/moonshine-streaming/stateDictHooks.test.ts +++ b/packages/runntime/test/zoo/moonshine-streaming/stateDictHooks.test.ts @@ -3,9 +3,9 @@ import { MOONSHINE_STREAMING_SMALL } from '../../../src/zoo/moonshine-streaming/ import { transformStreamingStateDict } from '../../../src/zoo/moonshine-streaming/stateDictHooks.ts'; import { checkpoint, contents, type Entry } from '../../core/weights/checkpoint.ts'; -// The hook reads only frameLen and the layer counts, head dims and hidden -// sizes. Encoder headDim 4 makes the q scale 1/√4 = 0.5, exact in f16. The -// hidden sizes differ, as in the small model, so the adapter projection maps. +// The hook reads only the layer counts, head dims and hidden sizes. Encoder +// headDim 4 makes the q scale 1/√4 = 0.5, exact in f16. The hidden sizes +// differ, as in the small model, so the decoder projection is there. const SMALL = MOONSHINE_STREAMING_SMALL; const CFG = { ...SMALL, @@ -19,38 +19,38 @@ const square = (values: number[]): Entry => ({ values, shape: [2, 2] }); const wide = (values: number[]): Entry => ({ values, shape: [2, 3] }); const ZERO = square([0, 0, 0, 0]); -/** Tensors the hook only renames, HF name → ours. */ -const RENAMED: Record = { - 'model.decoder.embed_tokens.weight': 'embedding.weight', - 'model.encoder.embedder.linear.weight': 'frontend.linear.weight', - 'model.encoder.embedder.conv1.weight': 'frontend.conv1.weight', - 'model.encoder.embedder.conv1.bias': 'frontend.conv1.bias', - 'model.encoder.embedder.conv2.weight': 'frontend.conv2.weight', - 'model.encoder.embedder.conv2.bias': 'frontend.conv2.bias', - 'model.encoder.layers.0.self_attn.o_proj.weight': 'encoder.layers.0.attn.out.weight', - 'model.encoder.layers.0.mlp.fc1.weight': 'encoder.layers.0.mlp.fc1.weight', - 'model.encoder.layers.0.mlp.fc1.bias': 'encoder.layers.0.mlp.fc1.bias', - 'model.encoder.layers.0.mlp.fc2.weight': 'encoder.layers.0.mlp.fc2.weight', - 'model.encoder.layers.0.mlp.fc2.bias': 'encoder.layers.0.mlp.fc2.bias', - 'model.decoder.pos_emb.weight': 'adapter.posEmb', - 'model.decoder.proj.weight': 'adapter.proj.weight', - 'model.decoder.layers.0.input_layernorm.weight': 'decoder.layers.0.norm1.weight', - 'model.decoder.layers.0.self_attn.o_proj.weight': 'decoder.layers.0.selfAttn.out.weight', - 'model.decoder.layers.0.post_attention_layernorm.weight': 'decoder.layers.0.norm2.weight', - 'model.decoder.layers.0.encoder_attn.o_proj.weight': 'decoder.layers.0.crossAttn.out.weight', - 'model.decoder.layers.0.final_layernorm.weight': 'decoder.layers.0.norm3.weight', - 'model.decoder.layers.0.mlp.fc1.weight': 'decoder.layers.0.mlp.fc1.weight', - 'model.decoder.layers.0.mlp.fc1.bias': 'decoder.layers.0.mlp.fc1.bias', - 'model.decoder.layers.0.mlp.fc2.weight': 'decoder.layers.0.mlp.fc2.weight', - 'model.decoder.layers.0.mlp.fc2.bias': 'decoder.layers.0.mlp.fc2.bias', - 'model.decoder.norm.weight': 'decoder.norm.weight', -}; +/** Tensors the hook leaves as they are, under their own names. */ +const UNTOUCHED = [ + 'model.decoder.embed_tokens.weight', + 'model.encoder.embedder.linear.weight', + 'model.encoder.embedder.conv1.weight', + 'model.encoder.embedder.conv1.bias', + 'model.encoder.embedder.conv2.weight', + 'model.encoder.embedder.conv2.bias', + 'model.encoder.layers.0.self_attn.o_proj.weight', + 'model.encoder.layers.0.mlp.fc1.weight', + 'model.encoder.layers.0.mlp.fc1.bias', + 'model.encoder.layers.0.mlp.fc2.weight', + 'model.encoder.layers.0.mlp.fc2.bias', + 'model.decoder.pos_emb.weight', + 'model.decoder.proj.weight', + 'model.decoder.layers.0.input_layernorm.weight', + 'model.decoder.layers.0.self_attn.o_proj.weight', + 'model.decoder.layers.0.post_attention_layernorm.weight', + 'model.decoder.layers.0.encoder_attn.o_proj.weight', + 'model.decoder.layers.0.final_layernorm.weight', + 'model.decoder.layers.0.mlp.fc1.weight', + 'model.decoder.layers.0.mlp.fc1.bias', + 'model.decoder.layers.0.mlp.fc2.weight', + 'model.decoder.layers.0.mlp.fc2.bias', + 'model.decoder.norm.weight', + 'proj_out.weight', +]; /** A one-layer Moonshine v2 checkpoint with HF names. Every value is exact in * f16. */ const CHECKPOINT: Record = { - ...Object.fromEntries(Object.keys(RENAMED).map((name) => [name, ZERO])), - 'proj_out.weight': ZERO, + ...Object.fromEntries(UNTOUCHED.map((name) => [name, ZERO])), 'model.encoder.embedder.comp.log_k': { values: [1], shape: [1] }, 'model.encoder.layers.0.input_layernorm.gamma': { values: [0.5, -0.25], shape: [2] }, 'model.encoder.layers.0.post_attention_layernorm.gamma': { values: [0, 2], shape: [2] }, @@ -78,37 +78,29 @@ describe('transformStreamingStateDict', () => { const get = (name: string) => sd.tensors.get(name)!; const derived = [ - 'frontend.cmvn.weight', - 'frontend.k', - 'encoder.layers.0.norm1.weight', - 'encoder.layers.0.norm2.weight', - 'encoder.norm.weight', - 'encoder.layers.0.attn.qkv.weight', - 'decoder.layers.0.selfAttn.qkv.weight', - 'decoder.layers.0.crossAttn.q.weight', - 'decoder.layers.0.crossAttn.kv.weight', + 'model.encoder.embedder.comp.k', + 'model.encoder.layers.0.input_layernorm.weight', + 'model.encoder.layers.0.post_attention_layernorm.weight', + 'model.encoder.final_norm.weight', + 'model.encoder.layers.0.self_attn.qkv.weight', + 'model.decoder.layers.0.self_attn.qkv.weight', + 'model.decoder.layers.0.encoder_attn.q_proj.weight', + 'model.decoder.layers.0.encoder_attn.kv.weight', ]; - // proj_out is tied to the embedding and dropped. - expect([...sd.tensors.keys()].sort()).toEqual([...Object.values(RENAMED), ...derived].sort()); - for (const [hf, ours] of Object.entries(RENAMED)) expect(get(ours)).toBe(before.get(hf)); - - // CMVN has no learned scale: ones, with no file bytes behind them. - const cmvn = get('frontend.cmvn.weight'); - expect(cmvn.shape).toEqual([3]); - expect(cmvn.byteLength).toBe(0); - expect(await contents(cmvn)).toEqual({ type: 'f32', values: [1, 1, 1] }); + expect([...sd.tensors.keys()].sort()).toEqual([...UNTOUCHED, ...derived].sort()); + for (const name of UNTOUCHED) expect(get(name)).toBe(before.get(name)); // k = exp(log_k) = e, in f32 for any checkpoint. - const k = get('frontend.k'); + const k = get('model.encoder.embedder.comp.k'); expect(k.shape).toEqual([1]); expect(k.byteLength).toBe(elemBytes); expect(await contents(k)).toEqual({ type: 'f32', values: [Math.fround(Math.E)] }); // Norm gammas gain back their −1 offset, in f32 for any checkpoint. for (const [name, values] of [ - ['encoder.layers.0.norm1.weight', [1.5, 0.75]], - ['encoder.layers.0.norm2.weight', [1, 3]], - ['encoder.norm.weight', [0, 1.125]], + ['model.encoder.layers.0.input_layernorm.weight', [1.5, 0.75]], + ['model.encoder.layers.0.post_attention_layernorm.weight', [1, 3]], + ['model.encoder.final_norm.weight', [0, 1.125]], ] as const) { const gamma = get(name); expect(gamma.shape).toEqual([2]); @@ -118,7 +110,7 @@ describe('transformStreamingStateDict', () => { // Encoder q|k|v with q times 1/√headDim = 0.5: the encoder has no rope. // The scaled part makes it f32 on every checkpoint. - const encQkv = get('encoder.layers.0.attn.qkv.weight'); + const encQkv = get('model.encoder.layers.0.self_attn.qkv.weight'); expect(encQkv.shape).toEqual([6, 2]); expect(encQkv.byteLength).toBe(12 * elemBytes); expect(await contents(encQkv)).toEqual({ @@ -127,7 +119,7 @@ describe('transformStreamingStateDict', () => { }); // Decoder q|k|v unscaled: its rope tables carry the scale. - const decQkv = get('decoder.layers.0.selfAttn.qkv.weight'); + const decQkv = get('model.decoder.layers.0.self_attn.qkv.weight'); expect(decQkv.shape).toEqual([6, 3]); expect(decQkv.byteLength).toBe(18 * elemBytes); expect(await contents(decQkv)).toEqual({ @@ -137,11 +129,11 @@ describe('transformStreamingStateDict', () => { // Cross-attention q times 1/√headDim = 0.5, f32 on every checkpoint; // k|v stacked as is. - const crossQ = get('decoder.layers.0.crossAttn.q.weight'); + const crossQ = get('model.decoder.layers.0.encoder_attn.q_proj.weight'); expect(crossQ.shape).toEqual([2, 3]); expect(crossQ.byteLength).toBe(6 * elemBytes); expect(await contents(crossQ)).toEqual({ type: 'f32', values: [0.5, 1.5, 2.5, 3.5, 4.5, 5.5] }); - const crossKv = get('decoder.layers.0.crossAttn.kv.weight'); + const crossKv = get('model.decoder.layers.0.encoder_attn.kv.weight'); expect(crossKv.shape).toEqual([4, 3]); expect(crossKv.byteLength).toBe(12 * elemBytes); expect(await contents(crossKv)).toEqual({ @@ -149,7 +141,7 @@ describe('transformStreamingStateDict', () => { values: [2, 4, 6, 8, 10, 12, 10, 20, 30, 40, 50, 60], }); - // The dropped proj_out (4 elems) is the only byte loss. - expect(sd.totalBytes()).toBe(total - 4 * elemBytes); + // Nothing is dropped: every derived tensor carries its source's bytes. + expect(sd.totalBytes()).toBe(total); }); }); diff --git a/packages/runntime/test/zoo/moonshine/stateDictHooks.test.ts b/packages/runntime/test/zoo/moonshine/stateDictHooks.test.ts index 0673cb12..aeb96a1e 100644 --- a/packages/runntime/test/zoo/moonshine/stateDictHooks.test.ts +++ b/packages/runntime/test/zoo/moonshine/stateDictHooks.test.ts @@ -10,40 +10,40 @@ const CFG = { ...MOONSHINE_TINY, dModel: 2, encLayers: 1, decLayers: 1, headDim: const square = (values: number[]): Entry => ({ values, shape: [2, 2] }); const ZERO = square([0, 0, 0, 0]); -/** Tensors the hook only renames, HF name → ours. */ -const RENAMED: Record = { - 'model.decoder.embed_tokens.weight': 'embedding.weight', - 'model.encoder.conv1.weight': 'stem.conv1.weight', - 'model.encoder.groupnorm.weight': 'stem.groupnorm.weight', - 'model.encoder.groupnorm.bias': 'stem.groupnorm.bias', - 'model.encoder.conv2.weight': 'stem.conv2.weight', - 'model.encoder.conv2.bias': 'stem.conv2.bias', - 'model.encoder.conv3.weight': 'stem.conv3.weight', - 'model.encoder.conv3.bias': 'stem.conv3.bias', - 'model.encoder.layers.0.input_layernorm.weight': 'encoder.layers.0.norm1.weight', - 'model.encoder.layers.0.self_attn.o_proj.weight': 'encoder.layers.0.attn.out.weight', - 'model.encoder.layers.0.post_attention_layernorm.weight': 'encoder.layers.0.norm2.weight', - 'model.encoder.layers.0.mlp.fc1.weight': 'encoder.layers.0.mlp.fc1.weight', - 'model.encoder.layers.0.mlp.fc1.bias': 'encoder.layers.0.mlp.fc1.bias', - 'model.encoder.layers.0.mlp.fc2.weight': 'encoder.layers.0.mlp.fc2.weight', - 'model.encoder.layers.0.mlp.fc2.bias': 'encoder.layers.0.mlp.fc2.bias', - 'model.encoder.layer_norm.weight': 'encoder.norm.weight', - 'model.decoder.layers.0.input_layernorm.weight': 'decoder.layers.0.norm1.weight', - 'model.decoder.layers.0.self_attn.o_proj.weight': 'decoder.layers.0.selfAttn.out.weight', - 'model.decoder.layers.0.post_attention_layernorm.weight': 'decoder.layers.0.norm2.weight', - 'model.decoder.layers.0.encoder_attn.o_proj.weight': 'decoder.layers.0.crossAttn.out.weight', - 'model.decoder.layers.0.final_layernorm.weight': 'decoder.layers.0.norm3.weight', - 'model.decoder.layers.0.mlp.fc1.weight': 'decoder.layers.0.mlp.fc1.weight', - 'model.decoder.layers.0.mlp.fc1.bias': 'decoder.layers.0.mlp.fc1.bias', - 'model.decoder.layers.0.mlp.fc2.weight': 'decoder.layers.0.mlp.fc2.weight', - 'model.decoder.layers.0.mlp.fc2.bias': 'decoder.layers.0.mlp.fc2.bias', - 'model.decoder.norm.weight': 'decoder.norm.weight', -}; +/** Tensors the hook leaves as they are, under their own names. */ +const UNTOUCHED = [ + 'model.decoder.embed_tokens.weight', + 'model.encoder.conv1.weight', + 'model.encoder.groupnorm.weight', + 'model.encoder.groupnorm.bias', + 'model.encoder.conv2.weight', + 'model.encoder.conv2.bias', + 'model.encoder.conv3.weight', + 'model.encoder.conv3.bias', + 'model.encoder.layers.0.input_layernorm.weight', + 'model.encoder.layers.0.self_attn.o_proj.weight', + 'model.encoder.layers.0.post_attention_layernorm.weight', + 'model.encoder.layers.0.mlp.fc1.weight', + 'model.encoder.layers.0.mlp.fc1.bias', + 'model.encoder.layers.0.mlp.fc2.weight', + 'model.encoder.layers.0.mlp.fc2.bias', + 'model.encoder.layer_norm.weight', + 'model.decoder.layers.0.input_layernorm.weight', + 'model.decoder.layers.0.self_attn.o_proj.weight', + 'model.decoder.layers.0.post_attention_layernorm.weight', + 'model.decoder.layers.0.encoder_attn.o_proj.weight', + 'model.decoder.layers.0.final_layernorm.weight', + 'model.decoder.layers.0.mlp.fc1.weight', + 'model.decoder.layers.0.mlp.fc1.bias', + 'model.decoder.layers.0.mlp.fc2.weight', + 'model.decoder.layers.0.mlp.fc2.bias', + 'model.decoder.norm.weight', + 'proj_out.weight', +]; /** A one-layer Moonshine checkpoint with HF names. Every value is exact in f16. */ const CHECKPOINT: Record = { - ...Object.fromEntries(Object.keys(RENAMED).map((name) => [name, ZERO])), - 'proj_out.weight': ZERO, + ...Object.fromEntries(UNTOUCHED.map((name) => [name, ZERO])), 'model.encoder.layers.0.self_attn.q_proj.weight': square([1, 2, 3, 4]), 'model.encoder.layers.0.self_attn.k_proj.weight': square([5, 6, 7, 8]), 'model.encoder.layers.0.self_attn.v_proj.weight': square([9, 10, 11, 12]), @@ -67,24 +67,23 @@ describe('transformMoonshineStateDict', () => { const get = (name: string) => sd.tensors.get(name)!; const fused = [ - 'encoder.layers.0.attn.qkv.weight', - 'decoder.layers.0.selfAttn.qkv.weight', - 'decoder.layers.0.crossAttn.q.weight', - 'decoder.layers.0.crossAttn.kv.weight', + 'model.encoder.layers.0.self_attn.qkv.weight', + 'model.decoder.layers.0.self_attn.qkv.weight', + 'model.decoder.layers.0.encoder_attn.q_proj.weight', + 'model.decoder.layers.0.encoder_attn.kv.weight', ]; - // proj_out is tied to the embedding and dropped. - expect([...sd.tensors.keys()].sort()).toEqual([...Object.values(RENAMED), ...fused].sort()); - for (const [hf, ours] of Object.entries(RENAMED)) expect(get(ours)).toBe(before.get(hf)); + expect([...sd.tensors.keys()].sort()).toEqual([...UNTOUCHED, ...fused].sort()); + for (const name of UNTOUCHED) expect(get(name)).toBe(before.get(name)); // Self-attention q|k|v rows in that order, unscaled: rope carries the scale. - const encQkv = get('encoder.layers.0.attn.qkv.weight'); + const encQkv = get('model.encoder.layers.0.self_attn.qkv.weight'); expect(encQkv.shape).toEqual([6, 2]); expect(encQkv.byteLength).toBe(12 * elemBytes); expect(await contents(encQkv)).toEqual({ type, values: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], }); - const decQkv = get('decoder.layers.0.selfAttn.qkv.weight'); + const decQkv = get('model.decoder.layers.0.self_attn.qkv.weight'); expect(decQkv.shape).toEqual([6, 2]); expect(decQkv.byteLength).toBe(12 * elemBytes); expect(await contents(decQkv)).toEqual({ @@ -94,17 +93,17 @@ describe('transformMoonshineStateDict', () => { // Cross-attention q times 1/√headDim = 0.5, f32 on every checkpoint; // k|v stacked as is. - const crossQ = get('decoder.layers.0.crossAttn.q.weight'); + const crossQ = get('model.decoder.layers.0.encoder_attn.q_proj.weight'); expect(crossQ.shape).toEqual([2, 2]); expect(crossQ.byteLength).toBe(4 * elemBytes); expect(await contents(crossQ)).toEqual({ type: 'f32', values: [0.5, 1.5, 2.5, 3.5] }); - const crossKv = get('decoder.layers.0.crossAttn.kv.weight'); + const crossKv = get('model.decoder.layers.0.encoder_attn.kv.weight'); expect(crossKv.shape).toEqual([4, 2]); expect(crossKv.byteLength).toBe(8 * elemBytes); expect(await contents(crossKv)).toEqual({ type, values: [2, 4, 6, 8, 10, 20, 30, 40] }); - // The dropped proj_out (4 elems) is the only byte loss. - expect(sd.totalBytes()).toBe(total - 4 * elemBytes); + // Nothing is dropped: the fused tensors carry their parts' bytes. + expect(sd.totalBytes()).toBe(total); }); it('rejects a projection whose width is not dModel', async () => { diff --git a/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts b/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts index f8c0f73d..58aa7966 100644 --- a/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts +++ b/packages/runntime/test/zoo/privacy-filter/stateDictHooks.test.ts @@ -1,13 +1,16 @@ import { describe, expect, it } from 'vitest'; -import { transformHfPrivacyFilterStateDict } from '../../../src/zoo/privacy-filter/stateDictHooks.ts'; +import { + renameExportedNormScales, + transformHfPrivacyFilterStateDict, +} from '../../../src/zoo/privacy-filter/stateDictHooks.ts'; import { checkpoint, contents, type Entry } from '../../core/weights/checkpoint.ts'; const ZERO = { values: [0, 0], shape: [2] }; /** Tensors the hook only renames, HF name → ours. */ const RENAMED: Record = { - 'model.layers.0.input_layernorm.weight': 'block.0.attn.norm.scale', - 'model.layers.0.post_attention_layernorm.weight': 'block.0.mlp.norm.scale', + 'model.layers.0.input_layernorm.weight': 'block.0.attn.norm.weight', + 'model.layers.0.post_attention_layernorm.weight': 'block.0.mlp.norm.weight', 'model.layers.0.self_attn.o_proj.weight': 'block.0.attn.out.weight', 'model.layers.0.self_attn.o_proj.bias': 'block.0.attn.out.bias', 'model.layers.0.mlp.router.weight': 'block.0.mlp.gate.weight', @@ -15,7 +18,7 @@ const RENAMED: Record = { 'model.layers.0.self_attn.sinks': 'block.0.attn.sinks', 'model.layers.0.mlp.experts.down_proj_bias': 'block.0.mlp.outBias', 'model.embed_tokens.weight': 'embedding.weight', - 'model.norm.weight': 'norm.scale', + 'model.norm.weight': 'norm.weight', 'score.weight': 'unembedding.weight', }; @@ -109,3 +112,23 @@ describe('transformHfPrivacyFilterStateDict', () => { expect(sd.totalBytes()).toBe(total - 2 * elemBytes); }); }); + +describe('renameExportedNormScales', () => { + it('renames every RMSNorm scale of an exported file to weight', async () => { + const sd = await checkpoint('F32', { + 'block.0.attn.norm.scale': ZERO, + 'block.0.mlp.norm.scale': ZERO, + 'norm.scale': ZERO, + 'block.0.attn.qkv.weight': ZERO, + }); + const before = new Map(sd.tensors); + renameExportedNormScales(sd); + expect([...sd.tensors.keys()].sort()).toEqual([ + 'block.0.attn.norm.weight', + 'block.0.attn.qkv.weight', + 'block.0.mlp.norm.weight', + 'norm.weight', + ]); + expect(sd.tensors.get('norm.weight')).toBe(before.get('norm.scale')); + }); +});