Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
0380194
feat(core): pack an input-first int matrix into quantW words with pac…
chmjkb Oct 28, 2026
44f7a33
feat(core): load a quantized checkpoint weight into Linear as quantW …
chmjkb Oct 28, 2026
0d5ca7f
feat(core): run a quantized Linear through the quantW matmul with its…
chmjkb Oct 28, 2026
8d465c6
feat(core): count a quantW slot's logical values in the loader and re…
chmjkb Oct 28, 2026
25ec658
test(core): check a quantized Linear against its dequantized float we…
chmjkb Oct 28, 2026
0cc9b05
docs(core): say the checkpoint decides whether a Linear weight loads …
chmjkb Oct 28, 2026
b4077b6
perf(core): pack a quantized Linear's ints straight from torch's [out…
chmjkb Oct 28, 2026
a8a599f
test(core): check a quantized Linear against a CPU loop of its dequan…
chmjkb Oct 28, 2026
7a8b900
test(core): add a symmetric per-group quantizer and float entries to …
chmjkb Oct 28, 2026
a2d529d
test(zoo): load MiniLM from an int8 and int4 checkpoint and check it …
chmjkb Oct 28, 2026
73b2aa9
docs(docs): describe the quantized safetensors layout all-MiniLM-L6-v…
chmjkb Oct 28, 2026
76ae400
docs(docs): say MiniLM's query, key and value weights are quantized t…
chmjkb Oct 28, 2026
5924041
docs(docs): say int4 values are two's-complement nibbles in the quant…
chmjkb Oct 28, 2026
0b626c2
docs(docs): say the quantized MiniLM file needs version 1 and the sym…
chmjkb Oct 28, 2026
506da9c
docs(docs): state the group size rule for quantized MiniLM, with 32, …
chmjkb Oct 28, 2026
0f0998d
docs(docs): drop the quantized weights section from the Embedding page
chmjkb Nov 4, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 15 additions & 8 deletions packages/runntime/bench/kernels/inputs.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/** Input builders shared by the kernel A/B cases. Random data comes from
* `Version.rand`, so A and B get the same numbers. */
import type { Version } from '../ab.ts';
import type { EagerDtype, Value, ValueMeta } from '../../src/core/index.ts';
import { packQuantW, type EagerDtype, type Value, type ValueMeta } from '../../src/core/index.ts';

type Float = 'f32' | 'f16';

Expand Down Expand Up @@ -55,7 +55,9 @@ export function convWeight(
);
}

/** A quantized [E·K, O] weight and its f32 scales [E·K/groupSize, O]. */
/** A quantized [E·K, O] weight and its f32 scales [E·K/groupSize, O]. Both
* versions get words from this checkout's packer: the layout is the
* kernels' contract, so A and B read the same words. */
export function quantWeight(
version: Version,
e: number,
Expand All @@ -64,17 +66,22 @@ export function quantWeight(
bits: 8 | 4,
groupSize: number,
): { w: Value; scales: Value } {
const perByte = bits === 8 ? 1 : 2;
const qbytes = new Uint8Array((e * k * o) / perByte).map((_, i) => (i * 37 + 11) & 0xff);
const half = 2 ** (bits - 1);
const ints = Int8Array.from({ length: e * k * o }, (_, i) => ((i * 37 + 11) % (2 * half)) - half);
// Each expert is its own run of tiles, so the experts pack one by one.
const words = new Uint32Array((e * k * o * bits) / 32);
for (let x = 0; x < e; x++) {
const expert = packQuantW(ints.subarray(x * k * o, (x + 1) * k * o), k, o, bits);
words.set(expert, x * expert.length);
}
const raw = version.rand((e * k * o) / groupSize).map((x) => Math.abs(x) * 0.01 + 1e-3);
const packed = version.lib.packQuantColSlice(qbytes, raw, bits, groupSize, e, k, o, 0, o);
const w = version.freeAfterCase(
version.lib.materialized(
{ elems: (e * k * o) / (32 / bits), dtype: 'quantW', dims: [e * k, o] },
version.lib.uploadU32(version.root, packed.qdata),
{ elems: words.length, dtype: 'quantW', dims: [e * k, o] },
version.lib.uploadU32(version.root, words),
),
);
const scales = version.upload(packed.scales, mat((e * k) / groupSize, o));
const scales = version.upload(raw, mat((e * k) / groupSize, o));
return { w, scales };
}

Expand Down
2 changes: 1 addition & 1 deletion packages/runntime/src/core/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ export {
} from './error.ts';
export { createResourceScope, type Disposable, type ResourceScope } from './lifetime.ts';
export { BufferPool } from './gpu/bufferPool.ts';
export { packQuantColSlice } from './weights/quantPack.ts';
export { packQuantW } from './weights/quantPack.ts';
export type { WeightCache } from './weights/cache.ts';
export {
bufferSource,
Expand Down
86 changes: 84 additions & 2 deletions packages/runntime/src/core/nn/layers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
gatherRows,
gatherRowsFrom,
layerNorm,
matmul,
reshape,
rmsNorm,
sdpa,
Expand All @@ -19,16 +20,22 @@ import {
snake,
} from '../graph/ops.ts';
import { matrix, type Value } from '../graph/value.ts';
import {
MATMUL_QUANT_TILED_BK,
matmulQuantTiledEligible,
} from '../kernels/matmul/matmulQuantTiled.ts';
import {
convToTapMajor,
f16ToF32,
packConv1x1Hwc4F16,
packConvHwc4F16,
packConvWinogradF16,
packDwHwc4F16,
permuteConvTransposeF16,
} from '../weights/convert.ts';
import { foldBnIntoConv } from '../weights/foldBn.ts';
import { derivedTensor, type LazyStateDict } from '../weights/safetensors.ts';
import { packQuantW } from '../weights/quantPack.ts';
import { derivedTensor, type LazyStateDict, type QuantTensor } from '../weights/safetensors.ts';
import { readFloats, transposeWeight } from '../weights/transforms.ts';
import { tensor } from '../gpu/buffers.ts';
import { CpuParameter, Module, Parameter, type AnyModule } from './module.ts';
Expand Down Expand Up @@ -100,12 +107,21 @@ export class Embedding extends Module<[readonly number[] | Value], Value> {
/** y = x @ weight + bias. weight is stored [in, out]; torch checkpoints
* transpose once at load.
*
* A quantized checkpoint weight loads as quantW words with its f32 scales in
* `weightScale` [in / groupSize, out], and forward() runs the quantized
* matmul. Only the checkpoint says which kind it is, so transformStateDict
* retypes the slot at load.
*
* `addend` and `activation` fold a residual add and a following tanh or GELU
* into the matmul epilogue, as act(x·W + bias) + addend. */
export class Linear extends Module {
readonly weight: Parameter;
readonly bias?: Parameter;
readonly activation?: SlotAct;
/** Set when a quantized weight loads. */
weightScale?: Parameter;
/** How the loaded weight is quantized, or undefined for a float weight. */
private quantization?: { bits: 4 | 8; groupSize: number };
constructor(
readonly inFeatures: number,
readonly outFeatures: number,
Expand Down Expand Up @@ -135,10 +151,76 @@ export class Linear extends Module {
`expected torch layout [${this.outFeatures}, ${this.inFeatures}]`,
);
}
if (tensor.kind === 'quant') {
this.loadQuantized(sd, prefix, name, tensor);
return;
}
sd.tensors.set(name, transposeWeight(tensor));
}
/** Replaces `name` with quantW words [in, out] and adds its scales
* [in / groupSize, out] right after it, so they bind in file order. The
* words carry the whole byteLength of the ints and scales. */
private loadQuantized(
sd: LazyStateDict,
prefix: string,
name: string,
tensor: QuantTensor,
): void {
const { bits, groupSize } = tensor;
const k = this.inFeatures;
const n = this.outFeatures;
// The quantized matmul reads whole tiles of 4 columns and 8 rows and
// packs int4 columns 8 to a word.
if (!matmulQuantTiledEligible(k, n, groupSize) || n % (32 / bits) !== 0) {
throw new RunntimeError(
'CHECKPOINT_MISMATCH',
`Linear: ${name} is int${bits} in groups of ${groupSize}, which needs in features ` +
`a multiple of ${MATMUL_QUANT_TILED_BK} and of the group size, out features a ` +
`multiple of ${32 / bits} and a group size a multiple of 4, got [out, in] = [${n}, ${k}]`,
);
}
this.quantization = { bits, groupSize };
this.weight.retype('quantW', (k * n * bits) / 32);
this.weightScale = new Parameter(
{ elems: (k / groupSize) * n, dtype: 'f32', dims: [k / groupSize, n] },
undefined,
true,
);
const words = derivedTensor(
[k, n],
async () => packQuantW(await tensor.ints(), k, n, bits),
tensor.byteLength,
);
const groups = k / groupSize;
const scales = derivedTensor([groups, n], async () => {
const data = await tensor.scales();
const f32 = data instanceof Float32Array ? data : f16ToF32(data);
// torch's [out, in / groupSize] to the kernels' [in / groupSize, out]
const out = new Float32Array(f32.length);
for (let o = 0; o < n; o++) {
for (let g = 0; g < groups; g++) out[g * n + o] = f32[o * groups + g]!;
}
return out;
});
const scaleName = prefix ? `${prefix}.weightScale` : 'weightScale';
const entries = [...sd.tensors];
sd.tensors.clear();
for (const [key, t] of entries) {
sd.tensors.set(key, key === name ? words : t);
if (key === name) sd.tensors.set(scaleName, scales);
}
}
forward(x: Value, call: { addend?: Value } = {}): Value {
return linear(x, this.weight.value, this.bias?.value, call.addend, this.activation);
if (!this.weightScale || !this.quantization) {
return linear(x, this.weight.value, this.bias?.value, call.addend, this.activation);
}
return matmul(x, this.weight.value, {
scales: this.weightScale.value,
...this.quantization,
bias: this.bias?.value,
addend: call.addend,
activation: this.activation,
});
}
}

Expand Down
18 changes: 14 additions & 4 deletions packages/runntime/src/core/nn/loadStateDict.ts
Original file line number Diff line number Diff line change
Expand Up @@ -144,18 +144,28 @@ async function bindStateDict(
let done = 0;
for (const [name, p] of params) {
const t = sd.tensors.get(name)!;
// Linear's hook turns its quantized weight into quantW words, so a
// QuantTensor still here is bound to something else.
if (t.kind === 'quant') {
throw new RunntimeError(
'CHECKPOINT_MISMATCH',
`loadStateDict: '${name}' is quantized (int${t.bits}), but the parameter holds ${p.shape.dtype}`,
`loadStateDict: '${name}' is quantized (int${t.bits}), but only a Linear weight loads ` +
`quantized and this parameter holds ${p.shape.dtype}`,
);
}
const numel = t.shape.reduce((a, b) => a * b, 1);
const expected = numel;
if (expected !== p.shape.elems) {
// A quantW slot counts u32 words under logical dims. Privacy-filter's
// pre-packed U32 file tensors count words too; anything else, like
// Linear's packed weight, counts logical values.
const words = p.shape.dtype === 'quantW' && t.kind === 'file' && t.dtype === 'U32';
const expected =
p.shape.dtype === 'quantW' && !words
? p.shape.dims!.reduce((a, b) => a * b, 1)
: p.shape.elems;
if (numel !== expected) {
throw new RunntimeError(
'CHECKPOINT_MISMATCH',
`loadStateDict: '${name}' checkpoint shape [${t.shape}] (${numel} elems) != model ${JSON.stringify(p.shape.dims)} (${p.shape.elems} ${p.shape.dtype === 'quantW' ? 'words' : 'elems'})`,
`loadStateDict: '${name}' checkpoint shape [${t.shape}] (${numel} elems) != model ${JSON.stringify(p.shape.dims)} (${expected} ${words ? 'words' : 'elems'})`,
);
}
if (p instanceof CpuParameter) {
Expand Down
7 changes: 4 additions & 3 deletions packages/runntime/src/core/nn/module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,10 @@ export class Parameter {
if (!this._value) throw new Error(`Parameter ${this.qualifiedName || '(unnamed)'} not loaded`);
return this._value;
}
/** @internal Swaps the storage dtype, keeping elems and dims. */
retype(dtype: EagerDtype): void {
this._shape = { ...this._shape, dtype };
/** @internal Swaps the storage dtype, keeping dims. quantW counts u32
* words, so a float slot turned quantW passes its word count as `elems`. */
retype(dtype: EagerDtype, elems = this._shape.elems): void {
this._shape = { ...this._shape, dtype, elems };
}
/** @internal */
bind(v: Value): void {
Expand Down
88 changes: 23 additions & 65 deletions packages/runntime/src/core/weights/quantPack.ts
Original file line number Diff line number Diff line change
@@ -1,70 +1,28 @@
/** Pack a column slice [oStart,oEnd) of a raw quantized expert tensor into
* eager's blocked-unit quantW layout + the matching scale slice.
* Input qbytes: canonical [E, K, O] int stream (int8 = 1 signed byte; int4 =
* 2/byte, even scalar index = low nibble). Input scales: f32 [E, KG, O],
* KG = K/groupSize. Output qdata: the layout kernels/quantCommon.ts documents —
* 16-value tiles per (expert, 4-col unit, 4-row chunk), col-major within the
* tile, packed at `bits` stride (tile = bits/2 words), tiles k4-major per
* (e, unit): word base ((e·W/4 + n)·K/4 + k4)·(bits/2) for width W.
* Output scales: f32 [E, KG, width] (unchanged layout — a unit's 4 scales
* stay adjacent). */
export function packQuantColSlice(
qbytes: Uint8Array,
scales: Float32Array,
bits: 8 | 4,
groupSize: number,
E: number,
K: number,
O: number,
oStart: number,
oEnd: number,
): { qdata: Uint32Array; scales: Float32Array } {
const width = oEnd - oStart;
// Tiles are 4 cols × 4 rows: both the slice width and K must tile evenly,
// or the packing here would silently disagree with the kernels (the real
// model's width=640/K=640 always do).
if (width <= 0 || width % 4 !== 0) {
throw new Error(`packQuantColSlice: slice width ${width} must be a positive multiple of 4`);
}
if (K % 4 !== 0)
throw new Error(`packQuantColSlice: K ${K} must be a multiple of 4 (4-row tiles)`);
if (K % groupSize !== 0)
throw new Error(`packQuantColSlice: groupSize ${groupSize} must divide K ${K}`);
const KG = K / groupSize;
const expectQ = bits === 8 ? E * K * O : (E * K * O) / 2;
if (qbytes.length !== expectQ)
throw new Error(`packQuantColSlice: qbytes ${qbytes.length} != ${expectQ}`);
if (scales.length !== E * KG * O)
throw new Error(`packQuantColSlice: scales ${scales.length} != ${E * KG * O}`);
const mask = bits === 8 ? 0xff : 0xf;
const perU32 = bits === 8 ? 4 : 8;
const qdata = new Uint32Array((E * K * width) / perU32);
const outScales = new Float32Array(E * KG * width);
const readQ = (s: number): number => {
if (bits === 8) return (qbytes[s]! << 24) >> 24;
const nib = s & 1 ? qbytes[s >> 1]! >> 4 : qbytes[s >> 1]! & 0xf;
return nib >= 8 ? nib - 16 : nib;
};
const n4 = width / 4;
const k4count = K / 4;
for (let e = 0; e < E; e++) {
for (let n = 0; n < n4; n++) {
for (let k4 = 0; k4 < k4count; k4++) {
const tileBit = ((e * n4 + n) * k4count + k4) * 16 * bits;
for (let j = 0; j < 4; j++) {
for (let kk = 0; kk < 4; kk++) {
const v = readQ((e * K + k4 * 4 + kk) * O + (oStart + n * 4 + j));
const bit = tileBit + (j * 4 + kk) * bits;
qdata[bit >> 5] = (qdata[bit >> 5]! | ((v & mask) << (bit & 31))) >>> 0;
}
/** Packs int weights into the quantW layout that kernels/quantCommon.ts
* documents. */

/** Packs `ints`, a row-major output-first [n, k] matrix of signed int`bits`
* values (torch's Linear layout, as the checkpoint stores it), into quantW
* words for one expert's logical [k, n] weight: 16-value tiles per (4-column
* unit, 4-row chunk), column-major in the tile at `bits` per value, lowest
* bits first, tiles k4-major per unit. k and n must be multiples of 4.
*
* The f32 scales [k / groupSize, n] need no packing: the kernels read them
* row-major as they are, so the group size does not change the words. */
export function packQuantW(ints: Int8Array, k: number, n: number, bits: 4 | 8): Uint32Array {
const words = new Uint32Array((k * n * bits) / 32);
const mask = (1 << bits) - 1;
for (let unit = 0; unit < n / 4; unit++) {
for (let k4 = 0; k4 < k / 4; k4++) {
const tileBit = (unit * (k / 4) + k4) * 16 * bits;
for (let c = 0; c < 4; c++) {
for (let r = 0; r < 4; r++) {
const v = ints[(unit * 4 + c) * k + k4 * 4 + r]!;
const bit = tileBit + (c * 4 + r) * bits;
words[bit >> 5] = (words[bit >> 5]! | ((v & mask) << (bit & 31))) >>> 0;
}
}
}
for (let g = 0; g < KG; g++) {
for (let oo = 0; oo < width; oo++) {
outScales[(e * KG + g) * width + oo] = scales[(e * KG + g) * O + (oStart + oo)]!;
}
}
}
return { qdata, scales: outScales };
return words;
}
Loading
Loading