diff --git a/clean/l/README.md b/clean/l/README.md new file mode 100644 index 00000000..b40b284e --- /dev/null +++ b/clean/l/README.md @@ -0,0 +1,16 @@ +# Model L (issue #89) + +Work in progress for a new model choice, milestone 0.7.0. Nothing here is used by the runtime, the front end or the +launcher yet. + +| Path | Content | +|---|---| +| `reference/table.py` | Model L's structure table: the encoder's op list, the decoder's channel segments and block, window offsets, block constants, map names; parameter shapes from the table; matrix work per frame (computed, not measured). Entries in `PROVISIONAL` are pending confirmation and are changed here only | +| `reference/oracle.py` | NumPy oracle (float64, only the prescribed FP8 / FP16 roundings): `encode`, `decode`, `network`, synthetic `random_params`. The Swin stages it shares with Model M call Model M's oracle (`clean/reference/swin.py`); the output reconstruction is not written yet | +| `tests/test_l_reference.py` | self-checks on synthetic data; the main ones reduce Model L to Model M through the table and require Model M's oracle results byte for byte | + +``` +python -m unittest discover -s clean/l/tests -t . +``` + +Test data is synthetic. Model data never goes into this repository. diff --git a/clean/l/__init__.py b/clean/l/__init__.py new file mode 100644 index 00000000..548d2d44 --- /dev/null +++ b/clean/l/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: MIT diff --git a/clean/l/reference/__init__.py b/clean/l/reference/__init__.py new file mode 100644 index 00000000..548d2d44 --- /dev/null +++ b/clean/l/reference/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: MIT diff --git a/clean/l/reference/oracle.py b/clean/l/reference/oracle.py new file mode 100644 index 00000000..1d5fd933 --- /dev/null +++ b/clean/l/reference/oracle.py @@ -0,0 +1,315 @@ +# SPDX-License-Identifier: MIT +"""Model L NumPy oracle (issue #89): float64 arithmetic with only the prescribed roundings (q8, f16), as Model M's +oracle in clean/reference. Model L's structure comes from clean/l/reference/table.py; nothing here hard-codes it. + + enc = encode(x, params["enc0"], parity) # x: Model M's enc0 input dict (clean/reference/encoder.py) + out = network(x_enc, x_dec, params, parity) # encoder -> enc1 ... dec1 (Model M's stages) -> decoder + dec = decode(x, params["dec0"], parity) # x: Model M's dec0 input dict (features = dec1's output) + +The Swin stages shared with Model M call clean/reference/swin.py, so for them this oracle is Model M's oracle. +""" +import math + +import numpy as np + +from clean.reference import decoder as _dec +from clean.reference import encoder as _enc +from clean.reference import swin as _swin +from clean.reference.fp8 import f16, q8 + +from . import table as _table + +_erf = np.frompyfunc(math.erf, 1, 1) + + +def _round(x, mode): + if mode == "q8": + return q8(x) + if mode == "f16": + return f16(x) + if mode in (None, "none"): + return np.asarray(x, dtype=np.float64) + raise ValueError(f"unknown rounding {mode!r}") + + +# ---- Swin block with explicit constants ----------------------------------------------------------------------------- + +def gelu(x, kind): + if kind == "exact": + return 0.5 * x * (1.0 + _erf(x / math.sqrt(2.0)).astype(np.float64)) + return _swin.GELU_FUNCTIONS[kind](x) + + +def rmsnorm(v, gamma, C, eps_num): + eps = eps_num / C + return gamma * v / np.sqrt(np.mean(v * v, axis=-1, keepdims=True) + eps) + + +def soft_cap(S, cap): + Sc = np.clip(S, -cap, cap) + return Sc - Sc ** 3 / (3.0 * cap * cap) + + +def _block_tokens(xw, p, heads, k): + """One block on gathered windows xw [..., T, C] (T = window^2), constants k. Same step order as Model M's.""" + C = xw.shape[-1] + T = xw.shape[-2] + lead = xw.shape[:-2] + a = q8(rmsnorm(xw, p["gamma1"], C, k["eps_num"])) + u = q8(np.einsum("hdc,...tc->...htd", p["Wqk"], a)) + v = f16(np.einsum("hdc,...tc->...htd", p["Wv"], a)) + S = np.einsum("...htd,...hsd->...hts", u, u) + p["B"] + z = k["tau"] * soft_cap(S, k["cap"]) + z = z - z.max(axis=-1, keepdims=True) + A = np.exp(z) + A = f16(A / A.sum(axis=-1, keepdims=True)) + o = np.einsum("...hts,...hsd->...htd", A, v) + o = np.moveaxis(o, -3, -2).reshape(lead + (T, heads * k["head_dim"])) + x1 = xw + q8(o) @ p["Wp"].T + p["bp"] + m = q8(rmsnorm(x1, p["gamma2"], C, k["eps_num"])) + h = q8(gelu(m @ p["W1"].T + p["b1"], k["gelu"])) + return x1 + h @ p["W2"].T + p["b2"] + + +def window_grid(H, W, offset, win): + ox, oy = offset + nwx = (W + ox + win - 1) // win + nwy = (H + oy + win - 1) // win + cols = np.arange(nwx)[:, None] * win - ox + np.arange(win)[None, :] + rows = np.arange(nwy)[:, None] * win - oy + np.arange(win)[None, :] + return rows, cols + + +def swin_block(x, p, offset, heads, consts, max_tokens=1 << 16): + """One block on a whole map x [H, W, C] with mirrored borders (as clean/reference/swin.py swin_block); returns the + block output in float64, not yet rounded.""" + x = np.asarray(x, dtype=np.float64) + H, W, C = x.shape + win = int(consts["window"]) + p = {k: np.asarray(v, dtype=np.float64) for k, v in p.items()} + rows, cols = window_grid(H, W, offset, win) + out = np.full_like(x, np.nan) + rows_per_chunk = max(1, max_tokens // (win * win * cols.shape[0])) + for l0 in range(0, rows.shape[0], rows_per_chunk): + r = rows[l0:l0 + rows_per_chunk] + rr = _swin.mirror(r, H)[:, None, :, None] + cc = _swin.mirror(cols, W)[None, :, None, :] + xw = x[rr, cc] + nl, nk = xw.shape[:2] + x2 = _block_tokens(xw.reshape(nl, nk, win * win, C), p, heads, consts).reshape(nl, nk, win, win, C) + rv = (r >= 0) & (r < H) + cv = (cols >= 0) & (cols < W) + sel = rv[:, None, :, None] & cv[None, :, None, :] + R = np.broadcast_to(r[:, None, :, None], sel.shape) + K = np.broadcast_to(cols[None, :, None, :], sel.shape) + out[R[sel], K[sel]] = x2[sel] + if np.isnan(out).any(): + raise AssertionError("window grid does not cover the map") + return out + + +def _block_params(params, prefix): + n = len(prefix) + return {k[n:]: v for k, v in params.items() if k.startswith(prefix)} + + +# ---- encoder -------------------------------------------------------------------------------------------------------- + +def space_to_depth(v): + """[fh, fw, n] -> [fh/2, fw/2, 4n], sub-pixel p = dx + 2 dy, channel n * p + i.""" + fh, fw, n = v.shape + a = v.reshape(fh // 2, 2, fw // 2, 2, n) # [r, dy, c, dx, i] + return np.concatenate([a[:, p // 2, :, p % 2, :] for p in range(4)], axis=-1) + + +def project_groups(z, W, b, groups, clamp=_enc.FEATURE_CLAMP): + """Running sum s = b; for each channel group g: s = FP16(s + W[:, g] z[g]) (exact group sums); then clamp and + round to FP8. With groups (20, 20, 20, 20) this is clean/reference/encoder.py project.""" + W = np.asarray(W, dtype=np.float64) + z = np.asarray(z, dtype=np.float64) + s = np.broadcast_to(np.asarray(b, dtype=np.float64), z.shape[:-1] + (W.shape[0],)) + lo = 0 + for g in groups: + s = f16(s + z[..., lo:lo + g] @ W[:, lo:lo + g].T) + lo += g + return q8(np.clip(s, -clamp, clamp)) + + +def encode(x, params, parity=None, t=_table.TABLE, keep=False): + """Model L's input encoder. x: Model M's enc0 input dict; params: logical names of table.encoder_shapes(t). + Returns Model M's enc0 outputs ("features" [fh/2, fw/2, 64] FP8-exact, motion surfaces) and, with keep=True, every + named map ("maps").""" + _table.encoder_plan(t) + if parity is None: + parity = int(x["frame"]) & 1 + rw, rh, fw, fh, dw, dh = _enc.check_inputs(x) + pv_params = {"sigma1": params.get("sigma1", 0.0), "sigma2": params.get("sigma2", 0.0)} + maps = {"values": _enc.pixel_values(x, pv_params)} + feats = None + for op in t["enc_ops"]: + k = op["op"] + if k == "quant": + v = maps[op["src"]] + if op["mode"] == "dither": + d = _enc.dither_table(params["T"], x["frame"], fw, fh) + out = _enc.dither_fp8(v, d[..., None]) + elif op["mode"] == "q8": + out = q8(v) + else: + raise ValueError(f"unknown quant mode {op['mode']!r}") + elif k == "space_to_depth": + out = space_to_depth(maps[op["src"]]) + elif k == "select": + out = maps[op["src"]][..., op["lo"]:op["hi"]] + elif k == "linear": + W = np.asarray(params[op["w"]], dtype=np.float64) + b = np.asarray(params[op["b"]], dtype=np.float64) + out = _round(maps[op["src"]] @ W.T + b, op.get("out_round", "q8")) + elif k == "swin": + consts = _table.block_consts(t, op.get("name", "enc0." + op["prefix"])) + off = _table.block_offset(t, op["offsets"], parity) + y = swin_block(maps[op["src"]], _block_params(params, op["prefix"]), off, op["heads"], consts) + out = _round(y, op.get("out_round", "q8")) + elif k == "concat": + parts = [maps[s] for s in op["srcs"]] + c = sum(p.shape[-1] for p in parts) + pad = op.get("pad_to", c) + if pad > c: + parts.append(np.zeros(parts[0].shape[:-1] + (pad - c,))) + out = np.concatenate(parts, axis=-1) + elif k == "project": + out = project_groups(maps[op["src"]], params[op["w"]], params[op["b"]], op["groups"]) + feats = out + maps[op.get("dst", "features")] = out + m_near, m_far, m_center = _enc.motion_surfaces(x) + res = {"features": feats.astype(np.float32), + "m_near": f16(m_near).astype(np.float32), + "m_far": f16(m_far).astype(np.float32), + "m_center": f16(m_center).astype(np.float32)} + if keep: + res["maps"] = maps + return res + + +# ---- decoder -------------------------------------------------------------------------------------------------------- + +def decoder_front(features, params, parity, t=_table.TABLE): + """dec1's FP8 map [fh/2, fw/2, 64] -> the 64-channel map Model M's decoder projects (table entry dec_connect).""" + Z = np.asarray(features, dtype=np.float64) + segs = t["dec_connect"] + blk = [s for s in segs if s[0] == "block"] + if not blk: + return Z + if t["dec_block_level"] != "token": + raise NotImplementedError("decoder block level other than token: pending") + b = t["dec_block"] + lo = min(s[1] for s in blk) + hi = max(s[2] for s in blk) + if sum(s[2] - s[1] for s in blk) != hi - lo or hi - lo != b["C"]: + raise ValueError("block segments must be one contiguous range of the block's width") + consts = _table.block_consts(t, b.get("name", "dec0." + b["prefix"])) + off = _table.block_offset(t, b["offsets"], parity) + y = _round(swin_block(Z[..., lo:hi], _block_params(params, b["prefix"]), off, b["heads"], consts), + b.get("out_round", "q8")) + out = Z.copy() + for kind, s0, s1 in segs: + if kind == "block": + out[..., s0:s1] = y[..., s0 - lo:s1 - lo] + elif kind != "pass": + raise ValueError(f"unknown decoder segment {kind!r}") + return out + + +def decode(x, params, parity, t=_table.TABLE): + """Model L's coefficient decoder: decoder_front, then Model M's decoder (table entry dec_pixel = "m").""" + if t["dec_pixel"] != "m": + raise NotImplementedError("decoder per-pixel variant other than Model M's: pending") + xx = dict(x) + xx["features"] = decoder_front(x["features"], params, parity, t).astype(np.float32) + out = _dec.decode(xx, {k: params[k] for k in ("W", "b", "S", "T")}) + out["front"] = xx["features"] + return out + + +def reconstruct(*args, **kwargs): + raise NotImplementedError("Model L output reconstruction (table entry post_variant): pending") + + +# ---- whole network -------------------------------------------------------------------------------------------------- + +def network(x_enc, x_dec, params, parity, t=_table.TABLE, keep=False): + """Encoder -> the ten shared Swin stages (Model M's oracle, L's weights) -> decoder. + + x_enc: Model M's enc0 input dict. x_dec: Model M's dec0 input dict without "features" (motion, latent_prev, + jitter, jitter_prev, reset, feature_size, recon_size). params: {"enc0": ..., "enc1": ..., ..., "dec0": ...}. + Returns {"enc0": ..., stage outputs ..., "dec0": ...}.""" + res = {"enc0": encode(x_enc, params["enc0"], parity, t, keep)} + x = res["enc0"]["features"] + skips = {} + for s in _table.SHARED_STAGES: + cfg = _swin.STAGES[s] + off = _swin.window_offset(s, parity) + if cfg["kind"] == "enc": + o = _swin.run_stage(s, params[s], x, off) + skips[s] = o["skip"] + x = o["merged"] + elif cfg["kind"] == "dec": + skip = skips["enc2" if s == "dec2" else "enc1"] + o = _swin.run_stage(s, params[s], x, off, skip) + x = o["out"] + else: + o = _swin.run_stage(s, params[s], x, off) + x = o["out"] + res[s] = o + xd = dict(x_dec) + xd["features"] = x + res["dec0"] = decode(xd, params["dec0"], parity, t) + return res + + +# ---- synthetic parameters (tests only; never model data) ------------------------------------------------------------ + +def random_block_params(rng, C, heads, mlp, prefix="", weight_scale=1.0): + s = weight_scale + p = { + "gamma1": f16(rng.uniform(0.5, 1.5, C)), + "gamma2": f16(rng.uniform(0.5, 1.5, C)), + "Wqk": q8(rng.normal(0, s * 0.6 / math.sqrt(C), (heads, 32, C))), + "Wv": q8(rng.normal(0, s * 1.0 / math.sqrt(C), (heads, 32, C))), + "B": f16(rng.normal(0, 0.5, (heads, 16, 16))), + "Wp": q8(rng.normal(0, s * 0.5 / math.sqrt(32 * heads), (C, 32 * heads))), + "bp": f16(rng.normal(0, 0.1, C)), + "W1": q8(rng.normal(0, s * 1.0 / math.sqrt(C), (mlp, C))), + "b1": f16(rng.normal(0, 0.1, mlp)), + "W2": q8(rng.normal(0, s * 0.5 / math.sqrt(mlp), (C, mlp))), + "b2": f16(rng.normal(0, 0.1, C)), + } + return {prefix + k: v.astype(np.float32) for k, v in p.items()} + + +def random_params(rng, t=_table.TABLE): + """Synthetic parameters for every stage of Model L (shapes from the table and Model M's stages).""" + enc = {} + for name, shape in _table.encoder_shapes(t).items(): + if name == "T": + enc[name] = rng.integers(0, 128, size=shape).astype(np.float32) + elif name in ("sigma1", "sigma2"): + enc[name] = np.array(rng.uniform(0.5, 1.5), dtype=np.float32) + for op in t["enc_ops"]: + if op["op"] == "linear": + enc[op["w"]] = q8(rng.normal(0, 1.0 / math.sqrt(op["n_in"]), (op["n_out"], op["n_in"]))).astype(np.float32) + enc[op["b"]] = f16(rng.normal(0, 0.1, op["n_out"])).astype(np.float32) + elif op["op"] == "swin": + enc.update(random_block_params(rng, op["C"], op["heads"], op["mlp"], op["prefix"])) + elif op["op"] == "project": + n = sum(op["groups"]) + enc[op["w"]] = q8(rng.normal(0, 0.6 / math.sqrt(n), (op["n_out"], n))).astype(np.float32) + enc[op["b"]] = f16(rng.normal(0, 0.3, op["n_out"])).astype(np.float32) + dec = _dec.random_params(rng) + if any(s[0] == "block" for s in t["dec_connect"]): + b = t["dec_block"] + dec.update(random_block_params(rng, b["C"], b["heads"], b["mlp"], b["prefix"])) + p = {"enc0": enc, "dec0": dec} + for s in _table.SHARED_STAGES: + p[s] = {k: np.asarray(v, dtype=np.float32) for k, v in _swin.random_params(s, rng).items()} + return p diff --git a/clean/l/reference/table.py b/clean/l/reference/table.py new file mode 100644 index 00000000..2c3d5ce6 --- /dev/null +++ b/clean/l/reference/table.py @@ -0,0 +1,230 @@ +# SPDX-License-Identifier: MIT +"""Model L structure table (issue #89): the single source of Model L's shape for the oracle, the packers, the +runtime and the tests. + +Every consumer interprets this table instead of hard-coding L's structure, so a provisional entry can be changed +without touching code. Entries listed in PROVISIONAL are pending confirmation; each carries a short key (I1 ...). + +SHARED_STAGES run with Model M's code and stage table (clean/reference/swin.py STAGES) and L's own weights. +""" +import copy + +from clean.reference import swin as _swin + +# ---- stages shared with Model M ------------------------------------------------------------------------------------- + +SHARED_STAGES = ("enc1", "enc2", "b0", "b1", "b2", "b3", "b4", "b5", "dec2", "dec1") + +# ---- block constants ------------------------------------------------------------------------------------------------ + +# Model M's block constants (clean/reference/swin.py), the default for every L block. +M_BLOCK_CONSTS = {"tau": _swin.TAU, "cap": _swin.CAP, "eps_num": 2.0 ** -13, "gelu": "exact", + "window": _swin.WIN, "head_dim": _swin.HEAD_DIM} + +# ---- the table ------------------------------------------------------------------------------------------------------ + +C32_BLOCK = {"C": 32, "heads": 2, "mlp": 128} + +TABLE = { + # Input encoder: ops applied in order. Each op reads and writes named maps; "values" is the per-pixel input + # (clean/reference/encoder.py pixel_values: [fh, fw, 20] on the padded grid F). A map's level follows from its + # size: the pixel grid F, or the token grid F/2 after space_to_depth (2 x 2 pixels, sub-pixel p = dx + 2 dy, + # channel = n * p + i). The last op must be "project" (the 64 FP8 features enc1 reads). + "enc_ops": ( + {"op": "quant", "mode": "dither", "src": "values", "dst": "q"}, # I5 + {"op": "space_to_depth", "src": "q", "dst": "raw"}, # I2 + {"op": "select", "src": "raw", "dst": "x", "lo": 0, "hi": 20}, # I2 + {"op": "linear", "src": "x", "dst": "x", "w": "Wa", "b": "ba", "n_in": 20, "n_out": 32, + "out_round": "q8"}, # I1 + {"op": "linear", "src": "x", "dst": "x", "w": "Wb", "b": "bb", "n_in": 32, "n_out": 32, + "out_round": "q8"}, # I1 + {"op": "swin", "src": "x", "dst": "x", "prefix": "blk.", **C32_BLOCK, "offsets": "first", + "out_round": "q8"}, # I1, I6, I10 + {"op": "concat", "srcs": ("raw", "x"), "dst": "z", "pad_to": 128}, # I2 + {"op": "project", "src": "z", "w": "W", "b": "b", "groups": (32, 32, 32, 32), "n_out": 64}, # I2 + ), + # Coefficient decoder: dec1's 64 channels split into segments; "block" segments go through one block, "pass" + # segments are kept; the result (same channel order) feeds Model M's decoder (clean/reference/decoder.py). + "dec_connect": (("block", 0, 32), ("pass", 32, 64)), # I4 + "dec_block": {"prefix": "blk.", **C32_BLOCK, "offsets": "second", "out_round": "q8"}, # I6, I10 + "dec_block_level": "token", # I3 + "dec_pixel": "m", # I8 + "post_variant": "unfolded", # I8 + "recon_ratio": (3, 2), # I7 + # Window offsets (ox, oy) of an L block for frame parity 0 and 1. + "window_offsets": {"first": ((2, 2), (0, 2)), "second": ((0, 0), (2, 0))}, # I10 + # Per-block constants; a block's own entry wins over "default". + "consts": {"default": dict(M_BLOCK_CONSTS)}, # I6 + # Logical tensor name -> converter map name, per stage ("enc0", "dec0"); identity when absent. + "map_names": {}, # I9 +} + +PROVISIONAL = { + "I1": "enc_ops: order and kind of the ops", + "I2": "enc_ops: levels, select / concat segments, project groups", + "I3": "dec_block_level", + "I4": "dec_connect", + "I5": "enc_ops: quant mode", + "I6": "consts", + "I7": "recon_ratio", + "I8": "dec_pixel, post_variant", + "I9": "map_names", + "I10": "window_offsets", +} + + +def table(**overrides): + """A deep copy of TABLE with top-level entries replaced (tests and experiments).""" + t = copy.deepcopy(TABLE) + for k, v in overrides.items(): + if k not in t: + raise KeyError(f"unknown table entry {k!r}") + t[k] = v + return t + + +def block_consts(t, name): + c = dict(t["consts"]["default"]) + c.update(t["consts"].get(name, {})) + return c + + +def block_offset(t, kind, parity): + return tuple(t["window_offsets"][kind][int(parity) & 1]) + + +def map_name(t, stage, logical): + return t["map_names"].get(stage, {}).get(logical, logical) + + +# ---- parameter shapes ----------------------------------------------------------------------------------------------- + +def block_shapes(C, heads, mlp, head_dim=32, window=4): + t = window * window + return {"gamma1": (C,), "gamma2": (C,), "Wqk": (heads, head_dim, C), "Wv": (heads, head_dim, C), + "B": (heads, t, t), "Wp": (C, heads * head_dim), "bp": (C,), "W1": (mlp, C), "b1": (mlp,), + "W2": (C, mlp), "b2": (C,)} + + +def encoder_shapes(t=TABLE): + """Tensor shapes of the L encoder stage, logical names.""" + shapes = {} + for op in t["enc_ops"]: + if op["op"] == "quant": + shapes.update({"sigma1": (), "sigma2": ()}) + if op["mode"] == "dither": + shapes["T"] = (8, 16, 16) + elif op["op"] == "linear": + shapes.update({op["w"]: (op["n_out"], op["n_in"]), op["b"]: (op["n_out"],)}) + elif op["op"] == "swin": + for k, s in block_shapes(op["C"], op["heads"], op["mlp"]).items(): + shapes[op["prefix"] + k] = s + elif op["op"] == "project": + shapes.update({op["w"]: (op["n_out"], sum(op["groups"])), op["b"]: (op["n_out"],)}) + return shapes + + +def encoder_plan(t=TABLE): + """(level, channels) of every named map after each op, from the op list alone: [(op, level, channels), ...].""" + maps = {"values": ("pixel", 20)} + plan = [] + for op in t["enc_ops"]: + k = op["op"] + if k == "quant": + out = maps[op["src"]] + elif k == "space_to_depth": + lvl, c = maps[op["src"]] + if lvl != "pixel": + raise ValueError("space_to_depth needs a pixel-level map") + out = ("token", 4 * c) + elif k == "select": + lvl, c = maps[op["src"]] + if not 0 <= op["lo"] < op["hi"] <= c: + raise ValueError(f"select {op['lo']}..{op['hi']} outside {c} channels") + out = (lvl, op["hi"] - op["lo"]) + elif k == "linear": + lvl, c = maps[op["src"]] + if c != op["n_in"]: + raise ValueError(f"linear {op['w']}: input has {c} channels, op wants {op['n_in']}") + out = (lvl, op["n_out"]) + elif k == "swin": + lvl, c = maps[op["src"]] + if c != op["C"]: + raise ValueError(f"swin: input has {c} channels, block width {op['C']}") + out = (lvl, c) + elif k == "concat": + lv = {maps[s][0] for s in op["srcs"]} + if len(lv) != 1: + raise ValueError("concat of maps at different levels") + c = sum(maps[s][1] for s in op["srcs"]) + pad = op.get("pad_to", c) + if pad < c: + raise ValueError("concat: pad_to below the channel count") + out = (lv.pop(), pad) + elif k == "project": + lvl, c = maps[op["src"]] + if lvl != "token" or c != sum(op["groups"]): + raise ValueError(f"project: needs a token-level map of {sum(op['groups'])} channels, got {lvl} {c}") + out = ("token", op["n_out"]) + else: + raise ValueError(f"unknown encoder op {k!r}") + maps[op.get("dst", "features")] = out + plan.append((k, out[0], out[1])) + if not plan or plan[-1][0] != "project": + raise ValueError("the encoder's last op must be project") + return plan + + +def decoder_shapes(t=TABLE): + shapes = {"W": (68, 64), "b": (68,), "S": (), "T": ()} + if any(seg[0] == "block" for seg in t["dec_connect"]): + b = t["dec_block"] + for k, s in block_shapes(b["C"], b["heads"], b["mlp"]).items(): + shapes[b["prefix"] + k] = s + return shapes + + +# ---- work per frame (computed, not measured) ------------------------------------------------------------------------ + +def block_macs_per_token(C, heads, mlp, head_dim=32, window=4): + """Multiply-adds of the matrix parts of one block per token: QKV, scores + weighted sum, output projection, MLP.""" + t = window * window + return 2 * head_dim * heads * C + 2 * t * head_dim * heads + head_dim * heads * C + 2 * mlp * C + + +def frame_macs(t, render_w, render_h): + """Matrix multiply-adds of one frame for Model L and for Model M at the same render size (no window padding). + Returns {"M": ..., "L": ..., "parts": {...}}.""" + fw, fh = -(-render_w // 32) * 32, -(-render_h // 32) * 32 + n = {"pixel": fw * fh, "token": (fw // 2) * (fh // 2)} + n2, n3 = n["token"] // 4, n["token"] // 16 + S = _swin.STAGES + parts = {} + for s in SHARED_STAGES: + cfg = S[s] + tokens = {64: n["token"], 96: n2, 128: n3}[cfg["C"]] + m = block_macs_per_token(cfg["C"], cfg["heads"], cfg["mlp"]) * tokens + if cfg["kind"] == "enc": + m += 4 * cfg["C"] * cfg["C_out"] * (tokens // 4) + if cfg["kind"] == "dec": + m += cfg["C_low"] * 4 * cfg["C"] * (tokens // 4) + parts[s] = m + m_enc0 = 80 * 64 * n["token"] + m_dec0 = 64 * 68 * n["token"] + M = sum(parts.values()) + m_enc0 + m_dec0 + enc = 0 + for op, (_, lvl, _c) in zip(t["enc_ops"], encoder_plan(t)): + if op["op"] == "linear": + enc += op["n_in"] * op["n_out"] * n[lvl] + elif op["op"] == "swin": + enc += block_macs_per_token(op["C"], op["heads"], op["mlp"]) * n[lvl] + elif op["op"] == "project": + enc += sum(op["groups"]) * op["n_out"] * n["token"] + dec = m_dec0 + if any(seg[0] == "block" for seg in t["dec_connect"]): + b = t["dec_block"] + lvl = n["token"] if t["dec_block_level"] == "token" else render_w * render_h + dec += block_macs_per_token(b["C"], b["heads"], b["mlp"]) * lvl + L = sum(parts.values()) + enc + dec + parts.update({"enc0.M": m_enc0, "dec0.M": m_dec0, "enc0.L": enc, "dec0.L": dec}) + return {"M": M, "L": L, "parts": parts} diff --git a/clean/l/tests/__init__.py b/clean/l/tests/__init__.py new file mode 100644 index 00000000..548d2d44 --- /dev/null +++ b/clean/l/tests/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: MIT diff --git a/clean/l/tests/test_l_reference.py b/clean/l/tests/test_l_reference.py new file mode 100644 index 00000000..81103461 --- /dev/null +++ b/clean/l/tests/test_l_reference.py @@ -0,0 +1,316 @@ +# SPDX-License-Identifier: MIT +"""Model L oracle self-checks (no GPU, no compiler; synthetic data only). + +Run: python -m unittest discover -s clean/l/tests -t . + +The strongest checks reduce Model L to Model M through the table and require the L oracle to give Model M's oracle +results byte for byte: the block with M's constants, the encoder with M's op list, the decoder without a block, and +the whole network chain. +""" +import unittest + +import numpy as np + +from clean.l.reference import oracle as L +from clean.l.reference import table as T +from clean.reference import decoder as MD +from clean.reference import encoder as ME +from clean.reference import swin as MS +from clean.reference.fp8 import is_fp8_exact, q8 + +# Model M's input encoder written as an L op list. +M_ENC_OPS = ( + {"op": "quant", "mode": "dither", "src": "values", "dst": "q"}, + {"op": "space_to_depth", "src": "q", "dst": "z"}, + {"op": "project", "src": "z", "w": "W", "b": "b", "groups": (20, 20, 20, 20), "n_out": 64}, +) + + +def _same(a, b): + a = np.asarray(a) + b = np.asarray(b) + return a.shape == b.shape and a.dtype == b.dtype and a.tobytes() == b.tobytes() + + +class BlockTest(unittest.TestCase): + def test_block_equals_model_m_oracle(self): + """With Model M's constants the L block is clean/reference/swin.py swin_block, byte for byte.""" + rng = np.random.default_rng(1) + for C, heads, mlp in ((32, 2, 128), (64, 2, 256), (96, 4, 384)): + p = L.random_block_params(rng, C, heads, mlp) + for H, W in ((9, 13), (16, 8), (5, 7)): + x = q8(rng.normal(0, 1.5, (H, W, C))) + for off in ((0, 0), (2, 0), (2, 2), (0, 2), (1, 3)): + want = MS.swin_block(x, p, off, heads) + got = L.swin_block(x, p, off, heads, T.M_BLOCK_CONSTS) + self.assertTrue(_same(got, want), (C, H, W, off)) + + def test_constants_are_used(self): + rng = np.random.default_rng(2) + p = L.random_block_params(rng, 32, 2, 128) + x = q8(rng.normal(0, 1.5, (8, 8, 32))) + base = q8(L.swin_block(x, p, (2, 2), 2, T.M_BLOCK_CONSTS)) + for key, val in (("tau", 0.5), ("cap", 8.0), ("eps_num", 1e-2), ("gelu", "sigmoid")): + k = dict(T.M_BLOCK_CONSTS) + k[key] = val + self.assertFalse(_same(q8(L.swin_block(x, p, (2, 2), 2, k)), base), key) + + def test_block_slow_loop(self): + """Vectorised block vs a per-window, per-token loop (float64; outputs agree to rounding noise).""" + rng = np.random.default_rng(3) + C, heads = 32, 2 + p = {k: np.asarray(v, np.float64) for k, v in L.random_block_params(rng, C, heads, 128).items()} + H, W, off = 6, 7, (2, 2) + x = q8(rng.normal(0, 1.5, (H, W, C))) + got = L.swin_block(x, p, off, heads, T.M_BLOCK_CONSTS) + k = T.M_BLOCK_CONSTS + want = np.full_like(got, np.nan) + rows, cols = L.window_grid(H, W, off, 4) + for r in rows: + for c in cols: + toks = [(MS.mirror(rr, H), MS.mirror(cc, W)) for rr in r for cc in c] + xw = np.stack([x[a, b] for a, b in toks]) + a = q8(np.stack([L.rmsnorm(xw[i], p["gamma1"], C, k["eps_num"]) for i in range(16)])) + o = np.zeros((16, 32 * heads)) + for h in range(heads): + u = q8(np.array([[np.dot(p["Wqk"][h, d], a[i]) for d in range(32)] for i in range(16)])) + v = L.f16(np.array([[np.dot(p["Wv"][h, d], a[i]) for d in range(32)] for i in range(16)])) + for i in range(16): + S = np.array([np.dot(u[i], u[j]) for j in range(16)]) + p["B"][h, i] + z = k["tau"] * L.soft_cap(S, k["cap"]) + e = np.exp(z - z.max()) + A = L.f16(e / e.sum()) + o[i, 32 * h:32 * h + 32] = A @ v + x1 = xw + q8(o) @ p["Wp"].T + p["bp"] + m = q8(np.stack([L.rmsnorm(x1[i], p["gamma2"], C, k["eps_num"]) for i in range(16)])) + hdn = q8(L.gelu(m @ p["W1"].T + p["b1"], "exact")) + y = x1 + hdn @ p["W2"].T + p["b2"] + for i, (rr, cc) in enumerate((rr, cc) for rr in r for cc in c): + if 0 <= rr < H and 0 <= cc < W: + want[rr, cc] = y[i] + self.assertFalse(np.isnan(want).any()) + np.testing.assert_allclose(got, want, rtol=0, atol=1e-9) + + +class EncoderTest(unittest.TestCase): + def test_model_m_op_list_equals_model_m_encoder(self): + rng = np.random.default_rng(4) + t = T.table(enc_ops=M_ENC_OPS) + for reset in (False, True): + x = ME.random_inputs(rng, rw=37, rh=21, reset=reset) + p = ME.random_params(rng) + want = ME.encode(x, p) + got = L.encode(x, p, t=t) + for k in ("features", "m_near", "m_far", "m_center"): + self.assertTrue(_same(got[k], want[k]), (k, reset)) + + def test_default_encoder(self): + rng = np.random.default_rng(5) + x = ME.random_inputs(rng, rw=40, rh=24) + p = L.random_params(rng)["enc0"] + self.assertEqual(set(p), set(T.encoder_shapes())) + for name, shape in T.encoder_shapes().items(): + self.assertEqual(np.shape(p[name]), shape, name) + out = L.encode(x, p, keep=True) + fw, fh = x["feature_size"] + self.assertEqual(out["features"].shape, (fh // 2, fw // 2, 64)) + self.assertTrue(np.isfinite(out["features"]).all()) + self.assertTrue(is_fp8_exact(out["features"])) + maps = out["maps"] + self.assertEqual(maps["raw"].shape, (fh // 2, fw // 2, 80)) + self.assertEqual(maps["x"].shape, (fh // 2, fw // 2, 32)) + self.assertEqual(maps["z"].shape, (fh // 2, fw // 2, 128)) + self.assertTrue(np.all(maps["z"][..., 112:] == 0)) + again = L.encode(x, p) + self.assertTrue(_same(again["features"], out["features"])) + + def test_parts_by_hand(self): + """linear, select, concat and project of the default op list recomputed directly.""" + rng = np.random.default_rng(6) + x = ME.random_inputs(rng, rw=33, rh=17) + p = L.random_params(rng)["enc0"] + out = L.encode(x, p, parity=1, keep=True) + m = out["maps"] + fw, fh = x["feature_size"] + q = ME.dither_fp8(ME.pixel_values(x, p), ME.dither_table(p["T"], x["frame"], fw, fh)[..., None]) + self.assertTrue(_same(m["q"], q)) + raw = np.zeros((fh // 2, fw // 2, 80)) + for r in range(fh // 2): + for c in range(fw // 2): + for pp in range(4): + raw[r, c, 20 * pp:20 * pp + 20] = q[2 * r + pp // 2, 2 * c + pp % 2] + self.assertTrue(_same(m["raw"], raw)) + h = q8(raw[..., :20] @ np.asarray(p["Wa"], np.float64).T + p["ba"]) + h = q8(h @ np.asarray(p["Wb"], np.float64).T + p["bb"]) + bp = {k[4:]: v for k, v in p.items() if k.startswith("blk.")} + h = q8(L.swin_block(h, bp, T.block_offset(T.TABLE, "first", 1), 2, T.M_BLOCK_CONSTS)) + self.assertTrue(_same(m["x"], h)) + z = np.concatenate([raw, h, np.zeros(h.shape[:-1] + (16,))], axis=-1) + W = np.asarray(p["W"], np.float64) + s = np.broadcast_to(np.asarray(p["b"], np.float64), z.shape[:-1] + (64,)) + for g in range(4): + s = L.f16(s + z[..., 32 * g:32 * g + 32] @ W[:, 32 * g:32 * g + 32].T) + self.assertTrue(_same(out["features"], q8(np.clip(s, -6.28125, 6.28125)).astype(np.float32))) + + def test_table_edits(self): + """Provisional entries are table edits: other op orders, a pixel-level pre-network, plain q8 inputs.""" + rng = np.random.default_rng(7) + x = ME.random_inputs(rng, rw=24, rh=20) + base = T.TABLE["enc_ops"] + variants = { + "block_first": (base[0], base[1], base[2], + {**base[5], "C": 20, "heads": 1, "mlp": 80, "prefix": "blk."}, + base[3], base[4], base[6], base[7]), + "pixel_level": ( + base[0], + {**base[3], "src": "q"}, base[4], base[5], + {"op": "space_to_depth", "src": "x", "dst": "z"}, + {"op": "project", "src": "z", "w": "W", "b": "b", "groups": (32, 32, 32, 32), "n_out": 64}), + "plain_q8": ({"op": "quant", "mode": "q8", "src": "values", "dst": "q"},) + tuple(base[1:]), + } + feats = {} + for name, ops in variants.items(): + t = T.table(enc_ops=ops) + plan = T.encoder_plan(t) + p = L.random_params(np.random.default_rng(8), t)["enc0"] + self.assertEqual(set(p), set(T.encoder_shapes(t)), name) + out = L.encode(x, p, t=t) + self.assertTrue(is_fp8_exact(out["features"]), name) + feats[name] = out["features"] + self.assertEqual(plan[-1], ("project", "token", 64)) + self.assertNotIn("T", T.encoder_shapes(T.table(enc_ops=variants["plain_q8"]))) + self.assertEqual(T.encoder_plan(T.table(enc_ops=variants["pixel_level"]))[1], ("linear", "pixel", 32)) + + def test_plan_refuses_bad_lists(self): + base = T.TABLE["enc_ops"] + bad = [ + base[:-1], # no project at the end + (base[0], {**base[3], "src": "q", "n_in": 32}) + tuple(base[4:]), # 20 channels into a 32-input linear + (base[0], base[1], {**base[2], "hi": 99}) + tuple(base[3:]), # select outside the map + (base[0], {"op": "frobnicate", "src": "q"}, base[-1]), + ] + for ops in bad: + with self.assertRaises(ValueError): + T.encoder_plan(T.table(enc_ops=ops)) + + +class DecoderTest(unittest.TestCase): + def test_no_block_equals_model_m_decoder(self): + rng = np.random.default_rng(9) + t = T.table(dec_connect=(("pass", 0, 64),)) + for reset in (False, True): + x = MD.random_inputs(rng, rw=29, rh=19, reset=reset) + p = MD.random_params(rng) + want = MD.decode(x, p) + got = L.decode(x, p, 0, t) + for k in ("A", "B", "C", "latent", "codes_A", "codes_B", "codes_L", "h"): + self.assertTrue(_same(got[k], want[k]), (k, reset)) + + def test_default_decoder(self): + rng = np.random.default_rng(10) + x = MD.random_inputs(rng, rw=30, rh=18) + p = L.random_params(rng)["dec0"] + self.assertEqual(set(p), set(T.decoder_shapes())) + out = L.decode(x, p, 1) + front = out["front"] + self.assertTrue(_same(front[..., 32:], x["features"][..., 32:])) + bp = {k[4:]: v for k, v in p.items() if k.startswith("blk.")} + y = q8(L.swin_block(x["features"][..., :32], bp, T.block_offset(T.TABLE, "second", 1), 2, + T.M_BLOCK_CONSTS)).astype(np.float32) + self.assertTrue(_same(front[..., :32], y)) + xx = dict(x) + xx["features"] = front + want = MD.decode(xx, {k: p[k] for k in ("W", "b", "S", "T")}) + self.assertTrue(_same(out["A"], want["A"])) + self.assertTrue(_same(out["latent"], want["latent"])) + + def test_other_segments(self): + rng = np.random.default_rng(11) + x = MD.random_inputs(rng, rw=20, rh=12) + p = L.random_params(rng)["dec0"] + t = T.table(dec_connect=(("pass", 0, 32), ("block", 32, 64))) + out = L.decode(x, p, 0, t) + self.assertTrue(_same(out["front"][..., :32], x["features"][..., :32])) + with self.assertRaises(ValueError): + L.decode(x, p, 0, T.table(dec_connect=(("block", 0, 48), ("pass", 48, 64)))) + with self.assertRaises(NotImplementedError): + L.decode(x, p, 0, T.table(dec_block_level="pixel")) + with self.assertRaises(NotImplementedError): + L.reconstruct() + + +class NetworkTest(unittest.TestCase): + def _inputs(self, rng, rw, rh): + xe = ME.random_inputs(rng, rw=rw, rh=rh) + xd = MD.random_inputs(rng, rw=rw, rh=rh, fw=xe["feature_size"][0], fh=xe["feature_size"][1], + dw=xe["recon_size"][0], dh=xe["recon_size"][1]) + del xd["features"] + return xe, xd + + def test_reduces_to_model_m_chain(self): + """With M's encoder op list and no decoder block, the L network is Model M's oracle chain.""" + rng = np.random.default_rng(12) + xe, xd = self._inputs(rng, 40, 24) + t = T.table(enc_ops=M_ENC_OPS, dec_connect=(("pass", 0, 64),)) + p = L.random_params(rng, t) + p["enc0"] = ME.random_params(rng) + for parity in (0, 1): + got = L.network(xe, xd, p, parity, t) + x = ME.encode(xe, p["enc0"])["features"] + skips = {} + for s in T.SHARED_STAGES: + off = MS.window_offset(s, parity) + kind = MS.STAGES[s]["kind"] + if kind == "enc": + o = MS.run_stage(s, p[s], x, off) + skips[s], x = o["skip"], o["merged"] + elif kind == "dec": + o = MS.run_stage(s, p[s], x, off, skips["enc2" if s == "dec2" else "enc1"]) + x = o["out"] + else: + x = MS.run_stage(s, p[s], x, off)["out"] + key = "merged" if kind == "enc" else "out" + self.assertTrue(_same(got[s][key], x), (s, parity)) + xx = dict(xd) + xx["features"] = x + want = MD.decode(xx, p["dec0"]) + for k in ("A", "B", "C", "latent"): + self.assertTrue(_same(got["dec0"][k], want[k]), (k, parity)) + + def test_default_network_runs(self): + rng = np.random.default_rng(13) + xe, xd = self._inputs(rng, 36, 20) + p = L.random_params(rng) + out = L.network(xe, xd, p, 0) + fw, fh = xe["feature_size"] + self.assertEqual(out["dec1"]["out"].shape, (fh // 2, fw // 2, 64)) + for k in ("A", "B", "C", "latent"): + self.assertTrue(np.isfinite(out["dec0"][k]).all(), k) + + +class TableTest(unittest.TestCase): + def test_shared_stages_are_model_m(self): + self.assertEqual(tuple(T.SHARED_STAGES), tuple(k for k in MS.STAGES)) + + def test_macs(self): + self.assertEqual(T.block_macs_per_token(64, 2, 256), 47104) + self.assertEqual(T.block_macs_per_token(96, 4, 384), 114688) + self.assertEqual(T.block_macs_per_token(128, 4, 512), 184320) + self.assertEqual(T.block_macs_per_token(32, 2, 128), 16384) + r = T.frame_macs(T.TABLE, 1706, 960) + self.assertEqual(r["M"], 103089438720) + self.assertGreater(r["L"], r["M"]) + + def test_provisional_keys_cover_entries(self): + self.assertEqual(set(T.PROVISIONAL), {f"I{i}" for i in range(1, 11)}) + + def test_table_copy(self): + t = T.table(recon_ratio=(3, 1)) + self.assertEqual(t["recon_ratio"], (3, 1)) + self.assertEqual(T.TABLE["recon_ratio"], (3, 2)) + with self.assertRaises(KeyError): + T.table(nonsense=1) + + +if __name__ == "__main__": + unittest.main()