Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion clean/reference/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@
"""NumPy reference implementation of spec/02-swin-block.md (the clean team's oracle)."""
from .fp8 import q8, f16, fp8_encode, fp8_decode, is_fp8_exact # noqa: F401
from .swin import (STAGES, run_stage, swin_block, patch_merge, expand, window_offset, # noqa: F401
random_params, random_inputs)
random_params, random_inputs, MODEL_STAGES, WINDOW_OFFSETS, stage_table)
44 changes: 33 additions & 11 deletions clean/reference/swin.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,33 @@
}


def window_offset(stage, parity):
"""(ox, oy) from the table in spec 02 "Window partition"."""
if stage in ("b1", "b3", "b5", "dec2", "dec1"):
return (0, 0) if parity == 0 else (2, 0)
return (2, 2) if parity == 0 else (0, 2)
# Model-keyed tables (#89): every function below takes `model` (default "M"); a model with a Swin stage table of its own
# adds its rows here. STAGES stays Model M's table (the same object as MODEL_STAGES["M"]).
MODEL_STAGES = {"M": STAGES}

# The window offsets of spec 02 "Window partition" per model: (ox, oy) at parity 0 and parity 1 for the stages listed,
# and the model's default pair for every other stage (also for a name the model does not have, as before).
_M_SHIFTED = {s: ((0, 0), (2, 0)) for s in ("b1", "b3", "b5", "dec2", "dec1")}
WINDOW_OFFSETS = {"M": {"stages": _M_SHIFTED, "default": ((2, 2), (0, 2))}}


def _model_row(table, model, what):
try:
return table[model]
except KeyError:
raise ValueError(f"no {what} for model {model!r}; known: {', '.join(sorted(table))}") from None


def stage_table(model="M"):
"""The Swin stage table of `model` (Model M: STAGES)."""
return _model_row(MODEL_STAGES, model, "reference stage table")


def window_offset(stage, parity, model="M"):
"""(ox, oy) from the table in spec 02 "Window partition" (per model, WINDOW_OFFSETS)."""
row = _model_row(WINDOW_OFFSETS, model, "window offset table")
pair = row["stages"].get(stage, row["default"])
return pair[0] if parity == 0 else pair[1]


def mirror(p, n):
Expand Down Expand Up @@ -200,9 +222,9 @@ def expand(y, skip, We, be):
return x + skip


def run_stage(stage, p, x, offset, skip=None, round_attn_f16=True, gelu="exact"):
def run_stage(stage, p, x, offset, skip=None, round_attn_f16=True, gelu="exact", model="M"):
"""Reference for one stage with the inputs/outputs of spec/90-test-interface.md (float32 FP8-exact arrays)."""
cfg = STAGES[stage]
cfg = stage_table(model)[stage]
if cfg["kind"] == "dec":
xin = expand(x, skip, p["We"], p["be"])
else:
Expand All @@ -214,9 +236,9 @@ def run_stage(stage, p, x, offset, skip=None, round_attn_f16=True, gelu="exact")
return {"out": out.astype(np.float32)}


def random_params(stage, rng, weight_scale=1.0):
def random_params(stage, rng, weight_scale=1.0, model="M"):
"""Synthetic parameters with the shapes of spec/90 (matrices FP8-exact). Never model data."""
cfg = STAGES[stage]
cfg = stage_table(model)[stage]
C, heads, mlp = cfg["C"], cfg["heads"], cfg["mlp"]
D = heads * HEAD_DIM

Expand Down Expand Up @@ -245,9 +267,9 @@ def vec(n, scale=0.1):
return p


def random_inputs(stage, H, W, rng, scale=1.0):
def random_inputs(stage, H, W, rng, scale=1.0, model="M"):
"""Synthetic FP8-exact inputs for a stage at stage resolution H x W: returns (x, skip)."""
cfg = STAGES[stage]
cfg = stage_table(model)[stage]
C = cfg["C"]
if cfg["kind"] == "dec":
x = q8(rng.standard_normal((H // 2, W // 2, cfg["C_low"])) * scale).astype(np.float32)
Expand Down
96 changes: 63 additions & 33 deletions clean/runner/packing.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import numpy as np

from clean.reference.fp8 import fp8_encode
from clean.reference.swin import STAGES, HEAD_DIM
from clean.reference.swin import STAGES, HEAD_DIM, stage_table # noqa: F401 (STAGES: Model M's table)

D_ROW_LAYOUTS = {
"gfx12": np.array([[v + 8 * h for v in range(8)] for h in range(2)]), # row = v + 8h
Expand Down Expand Up @@ -208,15 +208,15 @@ def pack_expand(We, be, C, C_low, drow):
return pack_tile_vector(be, RE, drow) + pack_frags(We, RE, kord_layout_l(C_low))


def stage_sections(stage):
def stage_sections(stage, model="M"):
"""(block size, offset of the merge/expand section) inside a packed stage blob."""
cfg = STAGES[stage]
cfg = stage_table(model)[stage]
size = block_layout(cfg["C"], cfg["heads"], cfg["mlp"])["SIZE"]
return size, size + (-size) % ALIGN


def check_tensors(stage, t):
cfg = STAGES[stage]
def check_tensors(stage, t, model="M"):
cfg = stage_table(model)[stage]
C, heads, mlp, D = cfg["C"], cfg["heads"], cfg["mlp"], cfg["heads"] * HEAD_DIM
shapes = {
"gamma1": (C,), "gamma2": (C,), "Wqk": (heads, HEAD_DIM, C), "Wv": (heads, HEAD_DIM, C),
Expand Down Expand Up @@ -247,11 +247,11 @@ def resolve_d_row(d_row):
return t


def pack_stage(stage, tensors, d_row="gfx12"):
def pack_stage(stage, tensors, d_row="gfx12", model="M"):
"""Logical tensors of spec/90 -> one byte blob: [block weights][pad to 256][merge or expand weights]."""
check_tensors(stage, tensors)
check_tensors(stage, tensors, model)
drow = resolve_d_row(d_row)
cfg = STAGES[stage]
cfg = stage_table(model)[stage]
blob = _pad(pack_block(tensors, cfg["C"], cfg["heads"], cfg["mlp"], drow))
if cfg["kind"] == "enc":
blob += pack_merge(tensors["Wm"], tensors["bm"], cfg["C"], cfg["C_out"], drow)
Expand All @@ -263,68 +263,98 @@ def pack_stage(stage, tensors, d_row="gfx12"):
# Pseudo-stage "bneck": the six bottleneck blocks b0..b5 as one unit (megakernel swin_bneck6, FORMATS.md). Its blob is
# the six b-stage blobs back to back (equal sizes), so block k's section starts at k * len(blob) / 6.
BNECK_STAGES = tuple(f"b{k}" for k in range(6))
# Model-keyed (#89): the bneck and mid stage lists per model (Model M: BNECK_STAGES, MID_STAGES below); a model adds
# its rows next to its stage table (clean/reference/swin.py MODEL_STAGES).
MODEL_BNECK_STAGES = {"M": BNECK_STAGES}
_COUNT_WORDS = {6: "six", 8: "eight"}


def pack_bneck(tensors, d_row="gfx12"):
def _count(n):
return _COUNT_WORDS.get(n, str(n))


def _model_list(table, model, what):
try:
return table[model]
except KeyError:
raise ValueError(f"no {what} for model {model!r}; known: {', '.join(sorted(table))}") from None


def bneck_stages(model="M"):
"""The stages of the pseudo-stage \"bneck\" of `model` (Model M: b0..b5)."""
return _model_list(MODEL_BNECK_STAGES, model, "bneck stage list")


def pack_bneck(tensors, d_row="gfx12", model="M"):
"""tensors: {"b0": tensors of b0, ..., "b5": ...} or a sequence of the six -> the six packed b-stage blobs, joined."""
seq = [tensors[s] for s in BNECK_STAGES] if isinstance(tensors, dict) else list(tensors)
if len(seq) != 6:
raise ValueError("bneck needs the tensors of the six blocks b0..b5")
return b"".join(pack_stage(s, t, d_row) for s, t in zip(BNECK_STAGES, seq))
names = bneck_stages(model)
seq = [tensors[s] for s in names] if isinstance(tensors, dict) else list(tensors)
if len(seq) != len(names):
raise ValueError(f"bneck needs the tensors of the {_count(len(names))} blocks {names[0]}..{names[-1]}")
return b"".join(pack_stage(s, t, d_row, model) for s, t in zip(names, seq))


def split_bneck(blob):
def split_bneck(blob, model="M"):
"""The six b-stage blobs of a bneck blob (or a sequence of six blobs, returned as a list)."""
k_n = len(bneck_stages(model))
if not isinstance(blob, (bytes, bytearray, memoryview)):
seq = [bytes(b) for b in blob]
else:
blob = bytes(blob)
n = len(blob) // 6
if n * 6 != len(blob) or n == 0:
raise ValueError("a bneck blob holds six equal b-stage blobs")
seq = [blob[k * n:(k + 1) * n] for k in range(6)]
if len(seq) != 6:
raise ValueError("bneck needs six blobs")
n = len(blob) // k_n
if n * k_n != len(blob) or n == 0:
raise ValueError(f"a bneck blob holds {_count(k_n)} equal b-stage blobs")
seq = [blob[k * n:(k + 1) * n] for k in range(k_n)]
if len(seq) != k_n:
raise ValueError(f"bneck needs {_count(k_n)} blobs")
return seq


# Pseudo-stage "mid": enc2 (with its merge), b0..b5, dec2 (with its expand) as one unit (persistent kernel swin_mid,
# FORMATS.md). Its blob is the eight stage blobs back to back, in this order.
MID_STAGES = ("enc2",) + BNECK_STAGES + ("dec2",)
MODEL_MID_STAGES = {"M": MID_STAGES}


def mid_stages(model="M"):
"""The stages of the pseudo-stage \"mid\" of `model`, in blob order (Model M: enc2, b0..b5, dec2)."""
return _model_list(MODEL_MID_STAGES, model, "mid stage list")


def stage_blob_size(stage):
def stage_blob_size(stage, model="M"):
"""Length of pack_stage(stage, ...): the padded block section plus the merge or expand section."""
cfg = STAGES[stage]
cfg = stage_table(model)[stage]
C = cfg["C"]
tail = 0
if cfg["kind"] == "enc":
tail = 4 * cfg["C_out"] + cfg["C_out"] * 4 * C # bias, then (C_out / 16) x (4 C / 16) fragments
elif cfg["kind"] == "dec":
tail = 4 * (4 * C) + (4 * C) * cfg["C_low"] # bias, then (4 C / 16) x (C_low / 16) fragments
return stage_sections(stage)[1] + tail
return stage_sections(stage, model)[1] + tail


def pack_mid(tensors, d_row="gfx12"):
def pack_mid(tensors, d_row="gfx12", model="M"):
"""tensors: {"enc2": ..., "b0": ..., ..., "b5": ..., "dec2": ...} or the eight in that order -> blobs joined."""
seq = [tensors[s] for s in MID_STAGES] if isinstance(tensors, dict) else list(tensors)
if len(seq) != len(MID_STAGES):
raise ValueError("mid needs the tensors of enc2, b0..b5 and dec2")
return b"".join(pack_stage(s, t, d_row) for s, t in zip(MID_STAGES, seq))
names = mid_stages(model)
seq = [tensors[s] for s in names] if isinstance(tensors, dict) else list(tensors)
if len(seq) != len(names):
raise ValueError(f"mid needs the tensors of {names[0]}, {names[1]}..{names[-2]} and {names[-1]}")
return b"".join(pack_stage(s, t, d_row, model) for s, t in zip(names, seq))


def split_mid(blob):
def split_mid(blob, model="M"):
"""The eight stage blobs of a mid blob (or a sequence of eight blobs, returned as a list)."""
names = mid_stages(model)
if not isinstance(blob, (bytes, bytearray, memoryview)):
seq = [bytes(b) for b in blob]
else:
blob, seq, o = bytes(blob), [], 0
for s in MID_STAGES:
n = stage_blob_size(s)
for s in names:
n = stage_blob_size(s, model)
seq.append(blob[o:o + n])
o += n
if o != len(blob):
raise ValueError(f"a mid blob holds {o} bytes, got {len(blob)}")
if len(seq) != len(MID_STAGES):
raise ValueError("mid needs eight blobs")
if len(seq) != len(names):
raise ValueError(f"mid needs {_count(len(names))} blobs")
return seq
14 changes: 14 additions & 0 deletions clean/runner/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,20 @@
DEFAULT_MAP = os.path.join(_ROOT, "data", "preset-M.v1.map.json")
MODEL_FORMATS = {"auto": 0, "raw": 1, "wgcm": 2}
SWIN_ORDER = ["enc1", "enc2", "b0", "b1", "b2", "b3", "b4", "b5", "dec2", "dec1"]
# Model-keyed (#89): the Swin stage order per model (Model M: SWIN_ORDER); a model adds its row next to its stage
# table (clean/reference/swin.py MODEL_STAGES).
MODEL_SWIN_ORDER = {"M": SWIN_ORDER}


def swin_order(model="M"):
"""The Swin stages of `model` in frame order (Model M: SWIN_ORDER)."""
try:
return MODEL_SWIN_ORDER[model]
except KeyError:
known = ", ".join(sorted(MODEL_SWIN_ORDER))
raise ValueError(f"no Swin stage order for model {model!r}; known: {known}") from None


SWIN_CHANNELS = {"enc1": (64, 96), "enc2": (96, 128), "dec2": (96, None), "dec1": (64, None)}


Expand Down
22 changes: 12 additions & 10 deletions clean/runner/stage_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
import numpy as np # noqa: E402

from clean.reference.fp8 import fp8_decode, fp8_encode # noqa: E402
from clean.reference.swin import STAGES, window_offset # noqa: E402
from clean.reference.swin import MODEL_STAGES, STAGES, stage_table, window_offset # noqa: E402,F401
from clean.runner import dec0, dec1dec0, enc0, exposure, post # noqa: E402
from clean.runner import perf_variants as PV # noqa: E402
from clean.runner import variants as V # noqa: E402
Expand Down Expand Up @@ -73,7 +73,8 @@ def _module_stage(m):
# Model K (spec 40/41, clean/k): stages "k.<stage>" (or plain stage names with Runner(..., model="K")).
from clean.k.runner import k_stages # noqa: E402
KERNEL_CODE_OBJECTS.update(k_stages.KERNEL_CODE_OBJECTS)
MODELS = ("M", "K")
# Models: the ones with a reference stage table (clean/reference/swin.py MODEL_STAGES: "M") and model K (clean/k)
MODELS = tuple(MODEL_STAGES) + ("K",)
EMU_LIBRARY = "wg_emu.so"


Expand Down Expand Up @@ -129,15 +130,15 @@ def pack_weights(self, stage: str, tensors: dict) -> bytes:
if self._k_stage(stage) is not None:
return k_stages.pack(self._k_stage(stage), tensors, self.d_row)
if stage == "bneck": # {"b0": ..., "b5": ...} (or the six in order) -> the six b-stage blobs joined
return pack_bneck(tensors, self.d_row)
return pack_bneck(tensors, self.d_row, self.model)
if stage == "mid": # {"enc2": ..., "b0": ..., ..., "b5": ..., "dec2": ...} -> the eight stage blobs joined
return pack_mid(tensors, self.d_row)
return pack_mid(tensors, self.d_row, self.model)
if stage in EXTRA_STAGES:
packer = EXTRA_STAGES[stage][0]
if packer is None:
raise ValueError(f"stage {stage!r} has no weights")
return packer(tensors, self.d_row)
return pack_stage(stage, tensors, self.d_row)
return pack_stage(stage, tensors, self.d_row, self.model)

def run(self, stage: str, weights: bytes, x, offset=None, skip=None) -> dict:
return self._run_plan(self._prepare(stage, weights, x, offset, skip))
Expand Down Expand Up @@ -440,20 +441,21 @@ def _prepare(self, stage, weights, x, offset, skip):
if self._k_stage(stage) is not None:
return k_stages.prepare(self.dev, self._k_stage(stage), weights, x, offset, skip)
if stage == "bneck": # b0..b5; offset = b0's window offset (it gives the parity)
parity = [p for p in (0, 1) if tuple(window_offset("b0", p)) == tuple(int(v) for v in offset)]
parity = [p for p in (0, 1) if tuple(window_offset("b0", p, self.model)) == tuple(int(v) for v in offset)]
if not parity:
raise ValueError(f"bneck offset {offset} is not a b0 window offset")
return self._prepare_bneck(weights, x, parity[0], "mega" in self.geo, None)
if stage == "mid": # enc2 (+ merge), b0..b5, dec2 (+ expand); offset = enc2's window offset
parity = [p for p in (0, 1) if tuple(window_offset("enc2", p)) == tuple(int(v) for v in offset)]
parity = [p for p in (0, 1) if tuple(window_offset("enc2", p, self.model)) == tuple(int(v) for v in offset)]
if not parity:
raise ValueError(f"mid offset {offset} is not an enc2 window offset")
return self._prepare_mid(weights, x, parity[0], "mid" in self.geo, None)
if stage in EXTRA_STAGES:
return EXTRA_STAGES[stage][1](self.dev, weights, x)
if stage not in STAGES:
stages = stage_table(self.model)
if stage not in stages:
raise ValueError(f"unknown stage {stage!r}")
cfg = STAGES[stage]
cfg = stages[stage]
C = cfg["C"]
ox, oy = (int(v) for v in offset)
if not (0 <= ox < 4 and 0 <= oy < 4):
Expand All @@ -475,7 +477,7 @@ def _prepare(self, stage, weights, x, offset, skip):
raise ValueError(f"H and W must be multiples of 4, got {H}x{W}")

weights = bytes(weights)
block_size, tail_off = stage_sections(stage)
block_size, tail_off = stage_sections(stage, self.model)
bufs, sizes = {}, {}

def alloc(name, nbytes, data=None):
Expand Down
Loading
Loading