From 2335640eb571fda9b0255d1bf47353d8fc41a958 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:37:21 -0500 Subject: [PATCH] Model L: structure table and NumPy oracle (CPU reference, step 1) clean/l/reference/table.py holds Model L's structure as data: the input encoder's op list, the decoder's channel segments and block, window offsets, block constants and map names, plus parameter shapes and the matrix work per frame derived from it. Entries listed in PROVISIONAL are pending confirmation and change only there. clean/l/reference/oracle.py interprets the table: encode, decode and the whole network chain in float64 with only the prescribed FP8 / FP16 roundings. The shared Swin stages call Model M's oracle. The output reconstruction is a stub that refuses until it is written. Tests (synthetic data) reduce Model L to Model M through the table and require Model M's oracle results byte for byte: the block with M's constants, the encoder with M's op list, the decoder without a block, and the whole chain. Refs #89 Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/l/README.md | 16 ++ clean/l/__init__.py | 1 + clean/l/reference/__init__.py | 1 + clean/l/reference/oracle.py | 315 +++++++++++++++++++++++++++++ clean/l/reference/table.py | 230 ++++++++++++++++++++++ clean/l/tests/__init__.py | 1 + clean/l/tests/test_l_reference.py | 316 ++++++++++++++++++++++++++++++ 7 files changed, 880 insertions(+) create mode 100644 clean/l/README.md create mode 100644 clean/l/__init__.py create mode 100644 clean/l/reference/__init__.py create mode 100644 clean/l/reference/oracle.py create mode 100644 clean/l/reference/table.py create mode 100644 clean/l/tests/__init__.py create mode 100644 clean/l/tests/test_l_reference.py diff --git a/clean/l/README.md b/clean/l/README.md new file mode 100644 index 00000000..b40b284e --- /dev/null +++ b/clean/l/README.md @@ -0,0 +1,16 @@ +# Model L (issue #89) + +Work in progress for a new model choice, milestone 0.7.0. Nothing here is used by the runtime, the front end or the +launcher yet. + +| Path | Content | +|---|---| +| `reference/table.py` | Model L's structure table: the encoder's op list, the decoder's channel segments and block, window offsets, block constants, map names; parameter shapes from the table; matrix work per frame (computed, not measured). Entries in `PROVISIONAL` are pending confirmation and are changed here only | +| `reference/oracle.py` | NumPy oracle (float64, only the prescribed FP8 / FP16 roundings): `encode`, `decode`, `network`, synthetic `random_params`. The Swin stages it shares with Model M call Model M's oracle (`clean/reference/swin.py`); the output reconstruction is not written yet | +| `tests/test_l_reference.py` | self-checks on synthetic data; the main ones reduce Model L to Model M through the table and require Model M's oracle results byte for byte | + +``` +python -m unittest discover -s clean/l/tests -t . +``` + +Test data is synthetic. Model data never goes into this repository. diff --git a/clean/l/__init__.py b/clean/l/__init__.py new file mode 100644 index 00000000..548d2d44 --- /dev/null +++ b/clean/l/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: MIT diff --git a/clean/l/reference/__init__.py b/clean/l/reference/__init__.py new file mode 100644 index 00000000..548d2d44 --- /dev/null +++ b/clean/l/reference/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: MIT diff --git a/clean/l/reference/oracle.py b/clean/l/reference/oracle.py new file mode 100644 index 00000000..1d5fd933 --- /dev/null +++ b/clean/l/reference/oracle.py @@ -0,0 +1,315 @@ +# SPDX-License-Identifier: MIT +"""Model L NumPy oracle (issue #89): float64 arithmetic with only the prescribed roundings (q8, f16), as Model M's +oracle in clean/reference. Model L's structure comes from clean/l/reference/table.py; nothing here hard-codes it. + + enc = encode(x, params["enc0"], parity) # x: Model M's enc0 input dict (clean/reference/encoder.py) + out = network(x_enc, x_dec, params, parity) # encoder -> enc1 ... dec1 (Model M's stages) -> decoder + dec = decode(x, params["dec0"], parity) # x: Model M's dec0 input dict (features = dec1's output) + +The Swin stages shared with Model M call clean/reference/swin.py, so for them this oracle is Model M's oracle. +""" +import math + +import numpy as np + +from clean.reference import decoder as _dec +from clean.reference import encoder as _enc +from clean.reference import swin as _swin +from clean.reference.fp8 import f16, q8 + +from . import table as _table + +_erf = np.frompyfunc(math.erf, 1, 1) + + +def _round(x, mode): + if mode == "q8": + return q8(x) + if mode == "f16": + return f16(x) + if mode in (None, "none"): + return np.asarray(x, dtype=np.float64) + raise ValueError(f"unknown rounding {mode!r}") + + +# ---- Swin block with explicit constants ----------------------------------------------------------------------------- + +def gelu(x, kind): + if kind == "exact": + return 0.5 * x * (1.0 + _erf(x / math.sqrt(2.0)).astype(np.float64)) + return _swin.GELU_FUNCTIONS[kind](x) + + +def rmsnorm(v, gamma, C, eps_num): + eps = eps_num / C + return gamma * v / np.sqrt(np.mean(v * v, axis=-1, keepdims=True) + eps) + + +def soft_cap(S, cap): + Sc = np.clip(S, -cap, cap) + return Sc - Sc ** 3 / (3.0 * cap * cap) + + +def _block_tokens(xw, p, heads, k): + """One block on gathered windows xw [..., T, C] (T = window^2), constants k. Same step order as Model M's.""" + C = xw.shape[-1] + T = xw.shape[-2] + lead = xw.shape[:-2] + a = q8(rmsnorm(xw, p["gamma1"], C, k["eps_num"])) + u = q8(np.einsum("hdc,...tc->...htd", p["Wqk"], a)) + v = f16(np.einsum("hdc,...tc->...htd", p["Wv"], a)) + S = np.einsum("...htd,...hsd->...hts", u, u) + p["B"] + z = k["tau"] * soft_cap(S, k["cap"]) + z = z - z.max(axis=-1, keepdims=True) + A = np.exp(z) + A = f16(A / A.sum(axis=-1, keepdims=True)) + o = np.einsum("...hts,...hsd->...htd", A, v) + o = np.moveaxis(o, -3, -2).reshape(lead + (T, heads * k["head_dim"])) + x1 = xw + q8(o) @ p["Wp"].T + p["bp"] + m = q8(rmsnorm(x1, p["gamma2"], C, k["eps_num"])) + h = q8(gelu(m @ p["W1"].T + p["b1"], k["gelu"])) + return x1 + h @ p["W2"].T + p["b2"] + + +def window_grid(H, W, offset, win): + ox, oy = offset + nwx = (W + ox + win - 1) // win + nwy = (H + oy + win - 1) // win + cols = np.arange(nwx)[:, None] * win - ox + np.arange(win)[None, :] + rows = np.arange(nwy)[:, None] * win - oy + np.arange(win)[None, :] + return rows, cols + + +def swin_block(x, p, offset, heads, consts, max_tokens=1 << 16): + """One block on a whole map x [H, W, C] with mirrored borders (as clean/reference/swin.py swin_block); returns the + block output in float64, not yet rounded.""" + x = np.asarray(x, dtype=np.float64) + H, W, C = x.shape + win = int(consts["window"]) + p = {k: np.asarray(v, dtype=np.float64) for k, v in p.items()} + rows, cols = window_grid(H, W, offset, win) + out = np.full_like(x, np.nan) + rows_per_chunk = max(1, max_tokens // (win * win * cols.shape[0])) + for l0 in range(0, rows.shape[0], rows_per_chunk): + r = rows[l0:l0 + rows_per_chunk] + rr = _swin.mirror(r, H)[:, None, :, None] + cc = _swin.mirror(cols, W)[None, :, None, :] + xw = x[rr, cc] + nl, nk = xw.shape[:2] + x2 = _block_tokens(xw.reshape(nl, nk, win * win, C), p, heads, consts).reshape(nl, nk, win, win, C) + rv = (r >= 0) & (r < H) + cv = (cols >= 0) & (cols < W) + sel = rv[:, None, :, None] & cv[None, :, None, :] + R = np.broadcast_to(r[:, None, :, None], sel.shape) + K = np.broadcast_to(cols[None, :, None, :], sel.shape) + out[R[sel], K[sel]] = x2[sel] + if np.isnan(out).any(): + raise AssertionError("window grid does not cover the map") + return out + + +def _block_params(params, prefix): + n = len(prefix) + return {k[n:]: v for k, v in params.items() if k.startswith(prefix)} + + +# ---- encoder -------------------------------------------------------------------------------------------------------- + +def space_to_depth(v): + """[fh, fw, n] -> [fh/2, fw/2, 4n], sub-pixel p = dx + 2 dy, channel n * p + i.""" + fh, fw, n = v.shape + a = v.reshape(fh // 2, 2, fw // 2, 2, n) # [r, dy, c, dx, i] + return np.concatenate([a[:, p // 2, :, p % 2, :] for p in range(4)], axis=-1) + + +def project_groups(z, W, b, groups, clamp=_enc.FEATURE_CLAMP): + """Running sum s = b; for each channel group g: s = FP16(s + W[:, g] z[g]) (exact group sums); then clamp and + round to FP8. With groups (20, 20, 20, 20) this is clean/reference/encoder.py project.""" + W = np.asarray(W, dtype=np.float64) + z = np.asarray(z, dtype=np.float64) + s = np.broadcast_to(np.asarray(b, dtype=np.float64), z.shape[:-1] + (W.shape[0],)) + lo = 0 + for g in groups: + s = f16(s + z[..., lo:lo + g] @ W[:, lo:lo + g].T) + lo += g + return q8(np.clip(s, -clamp, clamp)) + + +def encode(x, params, parity=None, t=_table.TABLE, keep=False): + """Model L's input encoder. x: Model M's enc0 input dict; params: logical names of table.encoder_shapes(t). + Returns Model M's enc0 outputs ("features" [fh/2, fw/2, 64] FP8-exact, motion surfaces) and, with keep=True, every + named map ("maps").""" + _table.encoder_plan(t) + if parity is None: + parity = int(x["frame"]) & 1 + rw, rh, fw, fh, dw, dh = _enc.check_inputs(x) + pv_params = {"sigma1": params.get("sigma1", 0.0), "sigma2": params.get("sigma2", 0.0)} + maps = {"values": _enc.pixel_values(x, pv_params)} + feats = None + for op in t["enc_ops"]: + k = op["op"] + if k == "quant": + v = maps[op["src"]] + if op["mode"] == "dither": + d = _enc.dither_table(params["T"], x["frame"], fw, fh) + out = _enc.dither_fp8(v, d[..., None]) + elif op["mode"] == "q8": + out = q8(v) + else: + raise ValueError(f"unknown quant mode {op['mode']!r}") + elif k == "space_to_depth": + out = space_to_depth(maps[op["src"]]) + elif k == "select": + out = maps[op["src"]][..., op["lo"]:op["hi"]] + elif k == "linear": + W = np.asarray(params[op["w"]], dtype=np.float64) + b = np.asarray(params[op["b"]], dtype=np.float64) + out = _round(maps[op["src"]] @ W.T + b, op.get("out_round", "q8")) + elif k == "swin": + consts = _table.block_consts(t, op.get("name", "enc0." + op["prefix"])) + off = _table.block_offset(t, op["offsets"], parity) + y = swin_block(maps[op["src"]], _block_params(params, op["prefix"]), off, op["heads"], consts) + out = _round(y, op.get("out_round", "q8")) + elif k == "concat": + parts = [maps[s] for s in op["srcs"]] + c = sum(p.shape[-1] for p in parts) + pad = op.get("pad_to", c) + if pad > c: + parts.append(np.zeros(parts[0].shape[:-1] + (pad - c,))) + out = np.concatenate(parts, axis=-1) + elif k == "project": + out = project_groups(maps[op["src"]], params[op["w"]], params[op["b"]], op["groups"]) + feats = out + maps[op.get("dst", "features")] = out + m_near, m_far, m_center = _enc.motion_surfaces(x) + res = {"features": feats.astype(np.float32), + "m_near": f16(m_near).astype(np.float32), + "m_far": f16(m_far).astype(np.float32), + "m_center": f16(m_center).astype(np.float32)} + if keep: + res["maps"] = maps + return res + + +# ---- decoder -------------------------------------------------------------------------------------------------------- + +def decoder_front(features, params, parity, t=_table.TABLE): + """dec1's FP8 map [fh/2, fw/2, 64] -> the 64-channel map Model M's decoder projects (table entry dec_connect).""" + Z = np.asarray(features, dtype=np.float64) + segs = t["dec_connect"] + blk = [s for s in segs if s[0] == "block"] + if not blk: + return Z + if t["dec_block_level"] != "token": + raise NotImplementedError("decoder block level other than token: pending") + b = t["dec_block"] + lo = min(s[1] for s in blk) + hi = max(s[2] for s in blk) + if sum(s[2] - s[1] for s in blk) != hi - lo or hi - lo != b["C"]: + raise ValueError("block segments must be one contiguous range of the block's width") + consts = _table.block_consts(t, b.get("name", "dec0." + b["prefix"])) + off = _table.block_offset(t, b["offsets"], parity) + y = _round(swin_block(Z[..., lo:hi], _block_params(params, b["prefix"]), off, b["heads"], consts), + b.get("out_round", "q8")) + out = Z.copy() + for kind, s0, s1 in segs: + if kind == "block": + out[..., s0:s1] = y[..., s0 - lo:s1 - lo] + elif kind != "pass": + raise ValueError(f"unknown decoder segment {kind!r}") + return out + + +def decode(x, params, parity, t=_table.TABLE): + """Model L's coefficient decoder: decoder_front, then Model M's decoder (table entry dec_pixel = "m").""" + if t["dec_pixel"] != "m": + raise NotImplementedError("decoder per-pixel variant other than Model M's: pending") + xx = dict(x) + xx["features"] = decoder_front(x["features"], params, parity, t).astype(np.float32) + out = _dec.decode(xx, {k: params[k] for k in ("W", "b", "S", "T")}) + out["front"] = xx["features"] + return out + + +def reconstruct(*args, **kwargs): + raise NotImplementedError("Model L output reconstruction (table entry post_variant): pending") + + +# ---- whole network -------------------------------------------------------------------------------------------------- + +def network(x_enc, x_dec, params, parity, t=_table.TABLE, keep=False): + """Encoder -> the ten shared Swin stages (Model M's oracle, L's weights) -> decoder. + + x_enc: Model M's enc0 input dict. x_dec: Model M's dec0 input dict without "features" (motion, latent_prev, + jitter, jitter_prev, reset, feature_size, recon_size). params: {"enc0": ..., "enc1": ..., ..., "dec0": ...}. + Returns {"enc0": ..., stage outputs ..., "dec0": ...}.""" + res = {"enc0": encode(x_enc, params["enc0"], parity, t, keep)} + x = res["enc0"]["features"] + skips = {} + for s in _table.SHARED_STAGES: + cfg = _swin.STAGES[s] + off = _swin.window_offset(s, parity) + if cfg["kind"] == "enc": + o = _swin.run_stage(s, params[s], x, off) + skips[s] = o["skip"] + x = o["merged"] + elif cfg["kind"] == "dec": + skip = skips["enc2" if s == "dec2" else "enc1"] + o = _swin.run_stage(s, params[s], x, off, skip) + x = o["out"] + else: + o = _swin.run_stage(s, params[s], x, off) + x = o["out"] + res[s] = o + xd = dict(x_dec) + xd["features"] = x + res["dec0"] = decode(xd, params["dec0"], parity, t) + return res + + +# ---- synthetic parameters (tests only; never model data) ------------------------------------------------------------ + +def random_block_params(rng, C, heads, mlp, prefix="", weight_scale=1.0): + s = weight_scale + p = { + "gamma1": f16(rng.uniform(0.5, 1.5, C)), + "gamma2": f16(rng.uniform(0.5, 1.5, C)), + "Wqk": q8(rng.normal(0, s * 0.6 / math.sqrt(C), (heads, 32, C))), + "Wv": q8(rng.normal(0, s * 1.0 / math.sqrt(C), (heads, 32, C))), + "B": f16(rng.normal(0, 0.5, (heads, 16, 16))), + "Wp": q8(rng.normal(0, s * 0.5 / math.sqrt(32 * heads), (C, 32 * heads))), + "bp": f16(rng.normal(0, 0.1, C)), + "W1": q8(rng.normal(0, s * 1.0 / math.sqrt(C), (mlp, C))), + "b1": f16(rng.normal(0, 0.1, mlp)), + "W2": q8(rng.normal(0, s * 0.5 / math.sqrt(mlp), (C, mlp))), + "b2": f16(rng.normal(0, 0.1, C)), + } + return {prefix + k: v.astype(np.float32) for k, v in p.items()} + + +def random_params(rng, t=_table.TABLE): + """Synthetic parameters for every stage of Model L (shapes from the table and Model M's stages).""" + enc = {} + for name, shape in _table.encoder_shapes(t).items(): + if name == "T": + enc[name] = rng.integers(0, 128, size=shape).astype(np.float32) + elif name in ("sigma1", "sigma2"): + enc[name] = np.array(rng.uniform(0.5, 1.5), dtype=np.float32) + for op in t["enc_ops"]: + if op["op"] == "linear": + enc[op["w"]] = q8(rng.normal(0, 1.0 / math.sqrt(op["n_in"]), (op["n_out"], op["n_in"]))).astype(np.float32) + enc[op["b"]] = f16(rng.normal(0, 0.1, op["n_out"])).astype(np.float32) + elif op["op"] == "swin": + enc.update(random_block_params(rng, op["C"], op["heads"], op["mlp"], op["prefix"])) + elif op["op"] == "project": + n = sum(op["groups"]) + enc[op["w"]] = q8(rng.normal(0, 0.6 / math.sqrt(n), (op["n_out"], n))).astype(np.float32) + enc[op["b"]] = f16(rng.normal(0, 0.3, op["n_out"])).astype(np.float32) + dec = _dec.random_params(rng) + if any(s[0] == "block" for s in t["dec_connect"]): + b = t["dec_block"] + dec.update(random_block_params(rng, b["C"], b["heads"], b["mlp"], b["prefix"])) + p = {"enc0": enc, "dec0": dec} + for s in _table.SHARED_STAGES: + p[s] = {k: np.asarray(v, dtype=np.float32) for k, v in _swin.random_params(s, rng).items()} + return p diff --git a/clean/l/reference/table.py b/clean/l/reference/table.py new file mode 100644 index 00000000..2c3d5ce6 --- /dev/null +++ b/clean/l/reference/table.py @@ -0,0 +1,230 @@ +# SPDX-License-Identifier: MIT +"""Model L structure table (issue #89): the single source of Model L's shape for the oracle, the packers, the +runtime and the tests. + +Every consumer interprets this table instead of hard-coding L's structure, so a provisional entry can be changed +without touching code. Entries listed in PROVISIONAL are pending confirmation; each carries a short key (I1 ...). + +SHARED_STAGES run with Model M's code and stage table (clean/reference/swin.py STAGES) and L's own weights. +""" +import copy + +from clean.reference import swin as _swin + +# ---- stages shared with Model M ------------------------------------------------------------------------------------- + +SHARED_STAGES = ("enc1", "enc2", "b0", "b1", "b2", "b3", "b4", "b5", "dec2", "dec1") + +# ---- block constants ------------------------------------------------------------------------------------------------ + +# Model M's block constants (clean/reference/swin.py), the default for every L block. +M_BLOCK_CONSTS = {"tau": _swin.TAU, "cap": _swin.CAP, "eps_num": 2.0 ** -13, "gelu": "exact", + "window": _swin.WIN, "head_dim": _swin.HEAD_DIM} + +# ---- the table ------------------------------------------------------------------------------------------------------ + +C32_BLOCK = {"C": 32, "heads": 2, "mlp": 128} + +TABLE = { + # Input encoder: ops applied in order. Each op reads and writes named maps; "values" is the per-pixel input + # (clean/reference/encoder.py pixel_values: [fh, fw, 20] on the padded grid F). A map's level follows from its + # size: the pixel grid F, or the token grid F/2 after space_to_depth (2 x 2 pixels, sub-pixel p = dx + 2 dy, + # channel = n * p + i). The last op must be "project" (the 64 FP8 features enc1 reads). + "enc_ops": ( + {"op": "quant", "mode": "dither", "src": "values", "dst": "q"}, # I5 + {"op": "space_to_depth", "src": "q", "dst": "raw"}, # I2 + {"op": "select", "src": "raw", "dst": "x", "lo": 0, "hi": 20}, # I2 + {"op": "linear", "src": "x", "dst": "x", "w": "Wa", "b": "ba", "n_in": 20, "n_out": 32, + "out_round": "q8"}, # I1 + {"op": "linear", "src": "x", "dst": "x", "w": "Wb", "b": "bb", "n_in": 32, "n_out": 32, + "out_round": "q8"}, # I1 + {"op": "swin", "src": "x", "dst": "x", "prefix": "blk.", **C32_BLOCK, "offsets": "first", + "out_round": "q8"}, # I1, I6, I10 + {"op": "concat", "srcs": ("raw", "x"), "dst": "z", "pad_to": 128}, # I2 + {"op": "project", "src": "z", "w": "W", "b": "b", "groups": (32, 32, 32, 32), "n_out": 64}, # I2 + ), + # Coefficient decoder: dec1's 64 channels split into segments; "block" segments go through one block, "pass" + # segments are kept; the result (same channel order) feeds Model M's decoder (clean/reference/decoder.py). + "dec_connect": (("block", 0, 32), ("pass", 32, 64)), # I4 + "dec_block": {"prefix": "blk.", **C32_BLOCK, "offsets": "second", "out_round": "q8"}, # I6, I10 + "dec_block_level": "token", # I3 + "dec_pixel": "m", # I8 + "post_variant": "unfolded", # I8 + "recon_ratio": (3, 2), # I7 + # Window offsets (ox, oy) of an L block for frame parity 0 and 1. + "window_offsets": {"first": ((2, 2), (0, 2)), "second": ((0, 0), (2, 0))}, # I10 + # Per-block constants; a block's own entry wins over "default". + "consts": {"default": dict(M_BLOCK_CONSTS)}, # I6 + # Logical tensor name -> converter map name, per stage ("enc0", "dec0"); identity when absent. + "map_names": {}, # I9 +} + +PROVISIONAL = { + "I1": "enc_ops: order and kind of the ops", + "I2": "enc_ops: levels, select / concat segments, project groups", + "I3": "dec_block_level", + "I4": "dec_connect", + "I5": "enc_ops: quant mode", + "I6": "consts", + "I7": "recon_ratio", + "I8": "dec_pixel, post_variant", + "I9": "map_names", + "I10": "window_offsets", +} + + +def table(**overrides): + """A deep copy of TABLE with top-level entries replaced (tests and experiments).""" + t = copy.deepcopy(TABLE) + for k, v in overrides.items(): + if k not in t: + raise KeyError(f"unknown table entry {k!r}") + t[k] = v + return t + + +def block_consts(t, name): + c = dict(t["consts"]["default"]) + c.update(t["consts"].get(name, {})) + return c + + +def block_offset(t, kind, parity): + return tuple(t["window_offsets"][kind][int(parity) & 1]) + + +def map_name(t, stage, logical): + return t["map_names"].get(stage, {}).get(logical, logical) + + +# ---- parameter shapes ----------------------------------------------------------------------------------------------- + +def block_shapes(C, heads, mlp, head_dim=32, window=4): + t = window * window + return {"gamma1": (C,), "gamma2": (C,), "Wqk": (heads, head_dim, C), "Wv": (heads, head_dim, C), + "B": (heads, t, t), "Wp": (C, heads * head_dim), "bp": (C,), "W1": (mlp, C), "b1": (mlp,), + "W2": (C, mlp), "b2": (C,)} + + +def encoder_shapes(t=TABLE): + """Tensor shapes of the L encoder stage, logical names.""" + shapes = {} + for op in t["enc_ops"]: + if op["op"] == "quant": + shapes.update({"sigma1": (), "sigma2": ()}) + if op["mode"] == "dither": + shapes["T"] = (8, 16, 16) + elif op["op"] == "linear": + shapes.update({op["w"]: (op["n_out"], op["n_in"]), op["b"]: (op["n_out"],)}) + elif op["op"] == "swin": + for k, s in block_shapes(op["C"], op["heads"], op["mlp"]).items(): + shapes[op["prefix"] + k] = s + elif op["op"] == "project": + shapes.update({op["w"]: (op["n_out"], sum(op["groups"])), op["b"]: (op["n_out"],)}) + return shapes + + +def encoder_plan(t=TABLE): + """(level, channels) of every named map after each op, from the op list alone: [(op, level, channels), ...].""" + maps = {"values": ("pixel", 20)} + plan = [] + for op in t["enc_ops"]: + k = op["op"] + if k == "quant": + out = maps[op["src"]] + elif k == "space_to_depth": + lvl, c = maps[op["src"]] + if lvl != "pixel": + raise ValueError("space_to_depth needs a pixel-level map") + out = ("token", 4 * c) + elif k == "select": + lvl, c = maps[op["src"]] + if not 0 <= op["lo"] < op["hi"] <= c: + raise ValueError(f"select {op['lo']}..{op['hi']} outside {c} channels") + out = (lvl, op["hi"] - op["lo"]) + elif k == "linear": + lvl, c = maps[op["src"]] + if c != op["n_in"]: + raise ValueError(f"linear {op['w']}: input has {c} channels, op wants {op['n_in']}") + out = (lvl, op["n_out"]) + elif k == "swin": + lvl, c = maps[op["src"]] + if c != op["C"]: + raise ValueError(f"swin: input has {c} channels, block width {op['C']}") + out = (lvl, c) + elif k == "concat": + lv = {maps[s][0] for s in op["srcs"]} + if len(lv) != 1: + raise ValueError("concat of maps at different levels") + c = sum(maps[s][1] for s in op["srcs"]) + pad = op.get("pad_to", c) + if pad < c: + raise ValueError("concat: pad_to below the channel count") + out = (lv.pop(), pad) + elif k == "project": + lvl, c = maps[op["src"]] + if lvl != "token" or c != sum(op["groups"]): + raise ValueError(f"project: needs a token-level map of {sum(op['groups'])} channels, got {lvl} {c}") + out = ("token", op["n_out"]) + else: + raise ValueError(f"unknown encoder op {k!r}") + maps[op.get("dst", "features")] = out + plan.append((k, out[0], out[1])) + if not plan or plan[-1][0] != "project": + raise ValueError("the encoder's last op must be project") + return plan + + +def decoder_shapes(t=TABLE): + shapes = {"W": (68, 64), "b": (68,), "S": (), "T": ()} + if any(seg[0] == "block" for seg in t["dec_connect"]): + b = t["dec_block"] + for k, s in block_shapes(b["C"], b["heads"], b["mlp"]).items(): + shapes[b["prefix"] + k] = s + return shapes + + +# ---- work per frame (computed, not measured) ------------------------------------------------------------------------ + +def block_macs_per_token(C, heads, mlp, head_dim=32, window=4): + """Multiply-adds of the matrix parts of one block per token: QKV, scores + weighted sum, output projection, MLP.""" + t = window * window + return 2 * head_dim * heads * C + 2 * t * head_dim * heads + head_dim * heads * C + 2 * mlp * C + + +def frame_macs(t, render_w, render_h): + """Matrix multiply-adds of one frame for Model L and for Model M at the same render size (no window padding). + Returns {"M": ..., "L": ..., "parts": {...}}.""" + fw, fh = -(-render_w // 32) * 32, -(-render_h // 32) * 32 + n = {"pixel": fw * fh, "token": (fw // 2) * (fh // 2)} + n2, n3 = n["token"] // 4, n["token"] // 16 + S = _swin.STAGES + parts = {} + for s in SHARED_STAGES: + cfg = S[s] + tokens = {64: n["token"], 96: n2, 128: n3}[cfg["C"]] + m = block_macs_per_token(cfg["C"], cfg["heads"], cfg["mlp"]) * tokens + if cfg["kind"] == "enc": + m += 4 * cfg["C"] * cfg["C_out"] * (tokens // 4) + if cfg["kind"] == "dec": + m += cfg["C_low"] * 4 * cfg["C"] * (tokens // 4) + parts[s] = m + m_enc0 = 80 * 64 * n["token"] + m_dec0 = 64 * 68 * n["token"] + M = sum(parts.values()) + m_enc0 + m_dec0 + enc = 0 + for op, (_, lvl, _c) in zip(t["enc_ops"], encoder_plan(t)): + if op["op"] == "linear": + enc += op["n_in"] * op["n_out"] * n[lvl] + elif op["op"] == "swin": + enc += block_macs_per_token(op["C"], op["heads"], op["mlp"]) * n[lvl] + elif op["op"] == "project": + enc += sum(op["groups"]) * op["n_out"] * n["token"] + dec = m_dec0 + if any(seg[0] == "block" for seg in t["dec_connect"]): + b = t["dec_block"] + lvl = n["token"] if t["dec_block_level"] == "token" else render_w * render_h + dec += block_macs_per_token(b["C"], b["heads"], b["mlp"]) * lvl + L = sum(parts.values()) + enc + dec + parts.update({"enc0.M": m_enc0, "dec0.M": m_dec0, "enc0.L": enc, "dec0.L": dec}) + return {"M": M, "L": L, "parts": parts} diff --git a/clean/l/tests/__init__.py b/clean/l/tests/__init__.py new file mode 100644 index 00000000..548d2d44 --- /dev/null +++ b/clean/l/tests/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: MIT diff --git a/clean/l/tests/test_l_reference.py b/clean/l/tests/test_l_reference.py new file mode 100644 index 00000000..81103461 --- /dev/null +++ b/clean/l/tests/test_l_reference.py @@ -0,0 +1,316 @@ +# SPDX-License-Identifier: MIT +"""Model L oracle self-checks (no GPU, no compiler; synthetic data only). + +Run: python -m unittest discover -s clean/l/tests -t . + +The strongest checks reduce Model L to Model M through the table and require the L oracle to give Model M's oracle +results byte for byte: the block with M's constants, the encoder with M's op list, the decoder without a block, and +the whole network chain. +""" +import unittest + +import numpy as np + +from clean.l.reference import oracle as L +from clean.l.reference import table as T +from clean.reference import decoder as MD +from clean.reference import encoder as ME +from clean.reference import swin as MS +from clean.reference.fp8 import is_fp8_exact, q8 + +# Model M's input encoder written as an L op list. +M_ENC_OPS = ( + {"op": "quant", "mode": "dither", "src": "values", "dst": "q"}, + {"op": "space_to_depth", "src": "q", "dst": "z"}, + {"op": "project", "src": "z", "w": "W", "b": "b", "groups": (20, 20, 20, 20), "n_out": 64}, +) + + +def _same(a, b): + a = np.asarray(a) + b = np.asarray(b) + return a.shape == b.shape and a.dtype == b.dtype and a.tobytes() == b.tobytes() + + +class BlockTest(unittest.TestCase): + def test_block_equals_model_m_oracle(self): + """With Model M's constants the L block is clean/reference/swin.py swin_block, byte for byte.""" + rng = np.random.default_rng(1) + for C, heads, mlp in ((32, 2, 128), (64, 2, 256), (96, 4, 384)): + p = L.random_block_params(rng, C, heads, mlp) + for H, W in ((9, 13), (16, 8), (5, 7)): + x = q8(rng.normal(0, 1.5, (H, W, C))) + for off in ((0, 0), (2, 0), (2, 2), (0, 2), (1, 3)): + want = MS.swin_block(x, p, off, heads) + got = L.swin_block(x, p, off, heads, T.M_BLOCK_CONSTS) + self.assertTrue(_same(got, want), (C, H, W, off)) + + def test_constants_are_used(self): + rng = np.random.default_rng(2) + p = L.random_block_params(rng, 32, 2, 128) + x = q8(rng.normal(0, 1.5, (8, 8, 32))) + base = q8(L.swin_block(x, p, (2, 2), 2, T.M_BLOCK_CONSTS)) + for key, val in (("tau", 0.5), ("cap", 8.0), ("eps_num", 1e-2), ("gelu", "sigmoid")): + k = dict(T.M_BLOCK_CONSTS) + k[key] = val + self.assertFalse(_same(q8(L.swin_block(x, p, (2, 2), 2, k)), base), key) + + def test_block_slow_loop(self): + """Vectorised block vs a per-window, per-token loop (float64; outputs agree to rounding noise).""" + rng = np.random.default_rng(3) + C, heads = 32, 2 + p = {k: np.asarray(v, np.float64) for k, v in L.random_block_params(rng, C, heads, 128).items()} + H, W, off = 6, 7, (2, 2) + x = q8(rng.normal(0, 1.5, (H, W, C))) + got = L.swin_block(x, p, off, heads, T.M_BLOCK_CONSTS) + k = T.M_BLOCK_CONSTS + want = np.full_like(got, np.nan) + rows, cols = L.window_grid(H, W, off, 4) + for r in rows: + for c in cols: + toks = [(MS.mirror(rr, H), MS.mirror(cc, W)) for rr in r for cc in c] + xw = np.stack([x[a, b] for a, b in toks]) + a = q8(np.stack([L.rmsnorm(xw[i], p["gamma1"], C, k["eps_num"]) for i in range(16)])) + o = np.zeros((16, 32 * heads)) + for h in range(heads): + u = q8(np.array([[np.dot(p["Wqk"][h, d], a[i]) for d in range(32)] for i in range(16)])) + v = L.f16(np.array([[np.dot(p["Wv"][h, d], a[i]) for d in range(32)] for i in range(16)])) + for i in range(16): + S = np.array([np.dot(u[i], u[j]) for j in range(16)]) + p["B"][h, i] + z = k["tau"] * L.soft_cap(S, k["cap"]) + e = np.exp(z - z.max()) + A = L.f16(e / e.sum()) + o[i, 32 * h:32 * h + 32] = A @ v + x1 = xw + q8(o) @ p["Wp"].T + p["bp"] + m = q8(np.stack([L.rmsnorm(x1[i], p["gamma2"], C, k["eps_num"]) for i in range(16)])) + hdn = q8(L.gelu(m @ p["W1"].T + p["b1"], "exact")) + y = x1 + hdn @ p["W2"].T + p["b2"] + for i, (rr, cc) in enumerate((rr, cc) for rr in r for cc in c): + if 0 <= rr < H and 0 <= cc < W: + want[rr, cc] = y[i] + self.assertFalse(np.isnan(want).any()) + np.testing.assert_allclose(got, want, rtol=0, atol=1e-9) + + +class EncoderTest(unittest.TestCase): + def test_model_m_op_list_equals_model_m_encoder(self): + rng = np.random.default_rng(4) + t = T.table(enc_ops=M_ENC_OPS) + for reset in (False, True): + x = ME.random_inputs(rng, rw=37, rh=21, reset=reset) + p = ME.random_params(rng) + want = ME.encode(x, p) + got = L.encode(x, p, t=t) + for k in ("features", "m_near", "m_far", "m_center"): + self.assertTrue(_same(got[k], want[k]), (k, reset)) + + def test_default_encoder(self): + rng = np.random.default_rng(5) + x = ME.random_inputs(rng, rw=40, rh=24) + p = L.random_params(rng)["enc0"] + self.assertEqual(set(p), set(T.encoder_shapes())) + for name, shape in T.encoder_shapes().items(): + self.assertEqual(np.shape(p[name]), shape, name) + out = L.encode(x, p, keep=True) + fw, fh = x["feature_size"] + self.assertEqual(out["features"].shape, (fh // 2, fw // 2, 64)) + self.assertTrue(np.isfinite(out["features"]).all()) + self.assertTrue(is_fp8_exact(out["features"])) + maps = out["maps"] + self.assertEqual(maps["raw"].shape, (fh // 2, fw // 2, 80)) + self.assertEqual(maps["x"].shape, (fh // 2, fw // 2, 32)) + self.assertEqual(maps["z"].shape, (fh // 2, fw // 2, 128)) + self.assertTrue(np.all(maps["z"][..., 112:] == 0)) + again = L.encode(x, p) + self.assertTrue(_same(again["features"], out["features"])) + + def test_parts_by_hand(self): + """linear, select, concat and project of the default op list recomputed directly.""" + rng = np.random.default_rng(6) + x = ME.random_inputs(rng, rw=33, rh=17) + p = L.random_params(rng)["enc0"] + out = L.encode(x, p, parity=1, keep=True) + m = out["maps"] + fw, fh = x["feature_size"] + q = ME.dither_fp8(ME.pixel_values(x, p), ME.dither_table(p["T"], x["frame"], fw, fh)[..., None]) + self.assertTrue(_same(m["q"], q)) + raw = np.zeros((fh // 2, fw // 2, 80)) + for r in range(fh // 2): + for c in range(fw // 2): + for pp in range(4): + raw[r, c, 20 * pp:20 * pp + 20] = q[2 * r + pp // 2, 2 * c + pp % 2] + self.assertTrue(_same(m["raw"], raw)) + h = q8(raw[..., :20] @ np.asarray(p["Wa"], np.float64).T + p["ba"]) + h = q8(h @ np.asarray(p["Wb"], np.float64).T + p["bb"]) + bp = {k[4:]: v for k, v in p.items() if k.startswith("blk.")} + h = q8(L.swin_block(h, bp, T.block_offset(T.TABLE, "first", 1), 2, T.M_BLOCK_CONSTS)) + self.assertTrue(_same(m["x"], h)) + z = np.concatenate([raw, h, np.zeros(h.shape[:-1] + (16,))], axis=-1) + W = np.asarray(p["W"], np.float64) + s = np.broadcast_to(np.asarray(p["b"], np.float64), z.shape[:-1] + (64,)) + for g in range(4): + s = L.f16(s + z[..., 32 * g:32 * g + 32] @ W[:, 32 * g:32 * g + 32].T) + self.assertTrue(_same(out["features"], q8(np.clip(s, -6.28125, 6.28125)).astype(np.float32))) + + def test_table_edits(self): + """Provisional entries are table edits: other op orders, a pixel-level pre-network, plain q8 inputs.""" + rng = np.random.default_rng(7) + x = ME.random_inputs(rng, rw=24, rh=20) + base = T.TABLE["enc_ops"] + variants = { + "block_first": (base[0], base[1], base[2], + {**base[5], "C": 20, "heads": 1, "mlp": 80, "prefix": "blk."}, + base[3], base[4], base[6], base[7]), + "pixel_level": ( + base[0], + {**base[3], "src": "q"}, base[4], base[5], + {"op": "space_to_depth", "src": "x", "dst": "z"}, + {"op": "project", "src": "z", "w": "W", "b": "b", "groups": (32, 32, 32, 32), "n_out": 64}), + "plain_q8": ({"op": "quant", "mode": "q8", "src": "values", "dst": "q"},) + tuple(base[1:]), + } + feats = {} + for name, ops in variants.items(): + t = T.table(enc_ops=ops) + plan = T.encoder_plan(t) + p = L.random_params(np.random.default_rng(8), t)["enc0"] + self.assertEqual(set(p), set(T.encoder_shapes(t)), name) + out = L.encode(x, p, t=t) + self.assertTrue(is_fp8_exact(out["features"]), name) + feats[name] = out["features"] + self.assertEqual(plan[-1], ("project", "token", 64)) + self.assertNotIn("T", T.encoder_shapes(T.table(enc_ops=variants["plain_q8"]))) + self.assertEqual(T.encoder_plan(T.table(enc_ops=variants["pixel_level"]))[1], ("linear", "pixel", 32)) + + def test_plan_refuses_bad_lists(self): + base = T.TABLE["enc_ops"] + bad = [ + base[:-1], # no project at the end + (base[0], {**base[3], "src": "q", "n_in": 32}) + tuple(base[4:]), # 20 channels into a 32-input linear + (base[0], base[1], {**base[2], "hi": 99}) + tuple(base[3:]), # select outside the map + (base[0], {"op": "frobnicate", "src": "q"}, base[-1]), + ] + for ops in bad: + with self.assertRaises(ValueError): + T.encoder_plan(T.table(enc_ops=ops)) + + +class DecoderTest(unittest.TestCase): + def test_no_block_equals_model_m_decoder(self): + rng = np.random.default_rng(9) + t = T.table(dec_connect=(("pass", 0, 64),)) + for reset in (False, True): + x = MD.random_inputs(rng, rw=29, rh=19, reset=reset) + p = MD.random_params(rng) + want = MD.decode(x, p) + got = L.decode(x, p, 0, t) + for k in ("A", "B", "C", "latent", "codes_A", "codes_B", "codes_L", "h"): + self.assertTrue(_same(got[k], want[k]), (k, reset)) + + def test_default_decoder(self): + rng = np.random.default_rng(10) + x = MD.random_inputs(rng, rw=30, rh=18) + p = L.random_params(rng)["dec0"] + self.assertEqual(set(p), set(T.decoder_shapes())) + out = L.decode(x, p, 1) + front = out["front"] + self.assertTrue(_same(front[..., 32:], x["features"][..., 32:])) + bp = {k[4:]: v for k, v in p.items() if k.startswith("blk.")} + y = q8(L.swin_block(x["features"][..., :32], bp, T.block_offset(T.TABLE, "second", 1), 2, + T.M_BLOCK_CONSTS)).astype(np.float32) + self.assertTrue(_same(front[..., :32], y)) + xx = dict(x) + xx["features"] = front + want = MD.decode(xx, {k: p[k] for k in ("W", "b", "S", "T")}) + self.assertTrue(_same(out["A"], want["A"])) + self.assertTrue(_same(out["latent"], want["latent"])) + + def test_other_segments(self): + rng = np.random.default_rng(11) + x = MD.random_inputs(rng, rw=20, rh=12) + p = L.random_params(rng)["dec0"] + t = T.table(dec_connect=(("pass", 0, 32), ("block", 32, 64))) + out = L.decode(x, p, 0, t) + self.assertTrue(_same(out["front"][..., :32], x["features"][..., :32])) + with self.assertRaises(ValueError): + L.decode(x, p, 0, T.table(dec_connect=(("block", 0, 48), ("pass", 48, 64)))) + with self.assertRaises(NotImplementedError): + L.decode(x, p, 0, T.table(dec_block_level="pixel")) + with self.assertRaises(NotImplementedError): + L.reconstruct() + + +class NetworkTest(unittest.TestCase): + def _inputs(self, rng, rw, rh): + xe = ME.random_inputs(rng, rw=rw, rh=rh) + xd = MD.random_inputs(rng, rw=rw, rh=rh, fw=xe["feature_size"][0], fh=xe["feature_size"][1], + dw=xe["recon_size"][0], dh=xe["recon_size"][1]) + del xd["features"] + return xe, xd + + def test_reduces_to_model_m_chain(self): + """With M's encoder op list and no decoder block, the L network is Model M's oracle chain.""" + rng = np.random.default_rng(12) + xe, xd = self._inputs(rng, 40, 24) + t = T.table(enc_ops=M_ENC_OPS, dec_connect=(("pass", 0, 64),)) + p = L.random_params(rng, t) + p["enc0"] = ME.random_params(rng) + for parity in (0, 1): + got = L.network(xe, xd, p, parity, t) + x = ME.encode(xe, p["enc0"])["features"] + skips = {} + for s in T.SHARED_STAGES: + off = MS.window_offset(s, parity) + kind = MS.STAGES[s]["kind"] + if kind == "enc": + o = MS.run_stage(s, p[s], x, off) + skips[s], x = o["skip"], o["merged"] + elif kind == "dec": + o = MS.run_stage(s, p[s], x, off, skips["enc2" if s == "dec2" else "enc1"]) + x = o["out"] + else: + x = MS.run_stage(s, p[s], x, off)["out"] + key = "merged" if kind == "enc" else "out" + self.assertTrue(_same(got[s][key], x), (s, parity)) + xx = dict(xd) + xx["features"] = x + want = MD.decode(xx, p["dec0"]) + for k in ("A", "B", "C", "latent"): + self.assertTrue(_same(got["dec0"][k], want[k]), (k, parity)) + + def test_default_network_runs(self): + rng = np.random.default_rng(13) + xe, xd = self._inputs(rng, 36, 20) + p = L.random_params(rng) + out = L.network(xe, xd, p, 0) + fw, fh = xe["feature_size"] + self.assertEqual(out["dec1"]["out"].shape, (fh // 2, fw // 2, 64)) + for k in ("A", "B", "C", "latent"): + self.assertTrue(np.isfinite(out["dec0"][k]).all(), k) + + +class TableTest(unittest.TestCase): + def test_shared_stages_are_model_m(self): + self.assertEqual(tuple(T.SHARED_STAGES), tuple(k for k in MS.STAGES)) + + def test_macs(self): + self.assertEqual(T.block_macs_per_token(64, 2, 256), 47104) + self.assertEqual(T.block_macs_per_token(96, 4, 384), 114688) + self.assertEqual(T.block_macs_per_token(128, 4, 512), 184320) + self.assertEqual(T.block_macs_per_token(32, 2, 128), 16384) + r = T.frame_macs(T.TABLE, 1706, 960) + self.assertEqual(r["M"], 103089438720) + self.assertGreater(r["L"], r["M"]) + + def test_provisional_keys_cover_entries(self): + self.assertEqual(set(T.PROVISIONAL), {f"I{i}" for i in range(1, 11)}) + + def test_table_copy(self): + t = T.table(recon_ratio=(3, 1)) + self.assertEqual(t["recon_ratio"], (3, 1)) + self.assertEqual(T.TABLE["recon_ratio"], (3, 2)) + with self.assertRaises(KeyError): + T.table(nonsense=1) + + +if __name__ == "__main__": + unittest.main()