From f9ea20ce7396b788385931ebab70050b040bf066 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:56:06 -0500 Subject: [PATCH] Model L: structure table settled to the confirmed layout (encoder and decoder blocks per render pixel) The structure table, the oracle and the packers follow the confirmed layout of Model L's encoder and decoder: - encoder, per pixel of the padded grid: the quantized values, linear 20 -> 32 (FP16 sum, ReLU, FP8), linear 32 -> 32 (FP16 sum, FP8), a width-32 block, then the 2 x 2 gather of each token into 128 values and the projection to the 64 features (FP16 running sum per group of 32, clamp, FP8); - decoder: the projection of dec1's 64 channels to 4 x 32 per token (Model M's dec0 rounding over all rows), depth_to_space to a pixel map, a width-32 block, then the value positions of the coefficient functions, which run as Model M's. Parameters are grouped by the converter map's stages (enc0, enc0_block, dec0_block, dec0, post) with its tensor names. The remaining provisional entries (quantization mode, rounding of the linears and of the decoder projection, clamp, value positions, window offsets, block constants, the reconstruction) stay table entries. Packers: the encoder and decoder launch lists come from the table; new blobs dec0.project (expand section format at C = 32, groups of 32 input channels) and dec0.coef (Model M's dec0 blob without the projection). Tests (synthetic): the encoder with Model M's op list, the decoder with Model M's projection rows and no block, and the whole chain equal Model M's oracle byte for byte; the default layout recomputed step by step; the unpacked blobs reproduce the oracle's maps byte for byte. Matrix work per frame at 1440p Quality (computed, not measured): Model M 103.1 G, Model L 163.1 G. Refs #89 Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/l/README.md | 4 +- clean/l/reference/oracle.py | 231 ++++++++++++++++++----------- clean/l/reference/table.py | 235 +++++++++++++++++------------- clean/l/runner/l_pack.py | 166 +++++++++++++-------- clean/l/tests/test_l_pack.py | 104 ++++++++----- clean/l/tests/test_l_reference.py | 205 ++++++++++++++------------ 6 files changed, 559 insertions(+), 386 deletions(-) diff --git a/clean/l/README.md b/clean/l/README.md index 536e6858..b29177cc 100644 --- a/clean/l/README.md +++ b/clean/l/README.md @@ -5,8 +5,8 @@ launcher yet. | Path | Content | |---|---| -| `reference/table.py` | Model L's structure table: the encoder's op list, the decoder's channel segments and block, window offsets, block constants, map names; parameter shapes from the table; matrix work per frame (computed, not measured). Entries in `PROVISIONAL` are pending confirmation and are changed here only | -| `reference/oracle.py` | NumPy oracle (float64, only the prescribed FP8 / FP16 roundings): `encode`, `decode`, `network`, synthetic `random_params`. The Swin stages it shares with Model M call Model M's oracle (`clean/reference/swin.py`); the output reconstruction is not written yet | +| `reference/table.py` | Model L's structure table: the encoder's and the decoder's op lists (pixel-level linears and width-32 block, the 2 x 2 gather projection; the decoder projection, its width-32 block and the value positions), window offsets, block constants; parameter shapes per converter-map stage; matrix work per frame (computed, not measured). Entries in `PROVISIONAL` are pending confirmation and are changed here only | +| `reference/oracle.py` | NumPy oracle (float64, only the prescribed FP8 / FP16 roundings): `encode`, `decode`, `network`, synthetic `random_params`, parameters grouped by the converter map's stages; the decoder's coefficient functions are Model M's. The Swin stages it shares with Model M call Model M's oracle (`clean/reference/swin.py`); the output reconstruction is not written yet | | `runner/l_pack.py` | weight packing of Model L's own launches: the encoder's launch list derived from the table (`encoder_launches`) and one blob per launch (`pack_model_l`), each in a format Model M already uses (merge / expand section, Swin block section, enc0 and dec0 blobs); decoders for tests. The ten shared stages use `clean/runner/packing.py` unchanged | | `tests/test_l_pack.py` | packing round trips, the block packer against Model M's stage blobs, and the unpacked weights applied the kernels' way against the oracle's maps, byte for byte | | `tests/test_l_reference.py` | self-checks on synthetic data; the main ones reduce Model L to Model M through the table and require Model M's oracle results byte for byte | diff --git a/clean/l/reference/oracle.py b/clean/l/reference/oracle.py index 1d5fd933..21350286 100644 --- a/clean/l/reference/oracle.py +++ b/clean/l/reference/oracle.py @@ -2,9 +2,11 @@ """Model L NumPy oracle (issue #89): float64 arithmetic with only the prescribed roundings (q8, f16), as Model M's oracle in clean/reference. Model L's structure comes from clean/l/reference/table.py; nothing here hard-codes it. - enc = encode(x, params["enc0"], parity) # x: Model M's enc0 input dict (clean/reference/encoder.py) + enc = encode(x, params, parity) # x: Model M's enc0 input dict (clean/reference/encoder.py) out = network(x_enc, x_dec, params, parity) # encoder -> enc1 ... dec1 (Model M's stages) -> decoder - dec = decode(x, params["dec0"], parity) # x: Model M's dec0 input dict (features = dec1's output) + dec = decode(x, params, parity) # x: Model M's dec0 input dict (features = dec1's output) + +params: {stage: tensors} with the converter map's stages (enc0, enc0_block, enc1 ... dec1, dec0_block, dec0, post). The Swin stages shared with Model M call clean/reference/swin.py, so for them this oracle is Model M's oracle. """ @@ -108,11 +110,6 @@ def swin_block(x, p, offset, heads, consts, max_tokens=1 << 16): return out -def _block_params(params, prefix): - n = len(prefix) - return {k[n:]: v for k, v in params.items() if k.startswith(prefix)} - - # ---- encoder -------------------------------------------------------------------------------------------------------- def space_to_depth(v): @@ -122,9 +119,19 @@ def space_to_depth(v): return np.concatenate([a[:, p // 2, :, p % 2, :] for p in range(4)], axis=-1) +def depth_to_space(v): + """Inverse of space_to_depth: [h, w, 4n] -> [2h, 2w, n].""" + h, w, c = v.shape + n = c // 4 + out = np.empty((2 * h, 2 * w, n), dtype=np.asarray(v).dtype) + for p in range(4): + out[p // 2::2, p % 2::2, :] = v[..., n * p:n * (p + 1)] + return out + + def project_groups(z, W, b, groups, clamp=_enc.FEATURE_CLAMP): - """Running sum s = b; for each channel group g: s = FP16(s + W[:, g] z[g]) (exact group sums); then clamp and - round to FP8. With groups (20, 20, 20, 20) this is clean/reference/encoder.py project.""" + """Running sum s = b; for each channel group g: s = FP16(s + W[:, g] z[g]) (exact group sums); then clamp (None: + no clamp) and round to FP8. With groups (20, 20, 20, 20) this is clean/reference/encoder.py project.""" W = np.asarray(W, dtype=np.float64) z = np.asarray(z, dtype=np.float64) s = np.broadcast_to(np.asarray(b, dtype=np.float64), z.shape[:-1] + (W.shape[0],)) @@ -132,26 +139,38 @@ def project_groups(z, W, b, groups, clamp=_enc.FEATURE_CLAMP): for g in groups: s = f16(s + z[..., lo:lo + g] @ W[:, lo:lo + g].T) lo += g - return q8(np.clip(s, -clamp, clamp)) + if clamp is not None: + s = np.clip(s, -clamp, clamp) + return q8(s) -def encode(x, params, parity=None, t=_table.TABLE, keep=False): - """Model L's input encoder. x: Model M's enc0 input dict; params: logical names of table.encoder_shapes(t). - Returns Model M's enc0 outputs ("features" [fh/2, fw/2, 64] FP8-exact, motion surfaces) and, with keep=True, every - named map ("maps").""" - _table.encoder_plan(t) - if parity is None: - parity = int(x["frame"]) & 1 - rw, rh, fw, fh, dw, dh = _enc.check_inputs(x) - pv_params = {"sigma1": params.get("sigma1", 0.0), "sigma2": params.get("sigma2", 0.0)} - maps = {"values": _enc.pixel_values(x, pv_params)} - feats = None - for op in t["enc_ops"]: +def _out_round(y, mode): + if mode == "relu_q8": + return q8(np.maximum(y, 0.0)) + return _round(y, mode) + + +def linear(x, W, b, acc=None, out_round="q8"): + """out_round(acc(W x + b)) with exact sums; acc "f16" rounds the sum to FP16 first.""" + y = np.asarray(x, dtype=np.float64) @ np.asarray(W, dtype=np.float64).T + np.asarray(b, dtype=np.float64) + if acc == "f16": + y = f16(y) + elif acc not in (None, "none"): + raise ValueError(f"unknown accumulation {acc!r}") + return _out_round(y, out_round) + + +def _run_ops(ops, maps, params, parity, t, default_stage, frame=None): + """Interprets an op list over named maps; returns the last op's output.""" + last = None + for op in ops: k = op["op"] + st = params[op.get("params", default_stage)] if k == "quant": v = maps[op["src"]] + fw, fh = frame["feature_size"] if op["mode"] == "dither": - d = _enc.dither_table(params["T"], x["frame"], fw, fh) + d = _enc.dither_table(st["T"], frame["frame"], fw, fh) out = _enc.dither_fp8(v, d[..., None]) elif op["mode"] == "q8": out = q8(v) @@ -159,16 +178,16 @@ def encode(x, params, parity=None, t=_table.TABLE, keep=False): raise ValueError(f"unknown quant mode {op['mode']!r}") elif k == "space_to_depth": out = space_to_depth(maps[op["src"]]) + elif k == "depth_to_space": + out = depth_to_space(maps[op["src"]]) elif k == "select": out = maps[op["src"]][..., op["lo"]:op["hi"]] elif k == "linear": - W = np.asarray(params[op["w"]], dtype=np.float64) - b = np.asarray(params[op["b"]], dtype=np.float64) - out = _round(maps[op["src"]] @ W.T + b, op.get("out_round", "q8")) + out = linear(maps[op["src"]], st[op["w"]], st[op["b"]], op.get("acc"), op.get("out_round", "q8")) elif k == "swin": - consts = _table.block_consts(t, op.get("name", "enc0." + op["prefix"])) + consts = _table.block_consts(t, op["params"]) off = _table.block_offset(t, op["offsets"], parity) - y = swin_block(maps[op["src"]], _block_params(params, op["prefix"]), off, op["heads"], consts) + y = swin_block(maps[op["src"]], st, off, op["heads"], consts) out = _round(y, op.get("out_round", "q8")) elif k == "concat": parts = [maps[s] for s in op["srcs"]] @@ -178,9 +197,30 @@ def encode(x, params, parity=None, t=_table.TABLE, keep=False): parts.append(np.zeros(parts[0].shape[:-1] + (pad - c,))) out = np.concatenate(parts, axis=-1) elif k == "project": - out = project_groups(maps[op["src"]], params[op["w"]], params[op["b"]], op["groups"]) - feats = out - maps[op.get("dst", "features")] = out + out = project_groups(maps[op["src"]], st[op["w"]], st[op["b"]], op["groups"], op.get("clamp")) + elif k == "dec_project": + out = dec_project(maps[op["src"]], st[op["w"]], st[op["b"]], op["groups"], op.get("out_round", "q8")) + elif k == "pick": + out = np.asarray(maps[op["src"]])[..., list(op["values"])] + else: + raise ValueError(f"unknown op {k!r}") + maps[op.get("dst", "out")] = out + last = out + return last + + +def encode(x, params, parity=None, t=_table.TABLE, keep=False): + """Model L's input encoder. x: Model M's enc0 input dict; params: {stage: tensors} with the stages of + table.param_shapes(t) ("enc0", "enc0_block"). Returns Model M's enc0 outputs ("features" [fh/2, fw/2, 64] + FP8-exact, motion surfaces) and, with keep=True, every named map ("maps").""" + _table.encoder_plan(t) + if parity is None: + parity = int(x["frame"]) & 1 + _enc.check_inputs(x) + enc = params["enc0"] + pv_params = {"sigma1": enc.get("sigma1", 0.0), "sigma2": enc.get("sigma2", 0.0)} + maps = {"values": _enc.pixel_values(x, pv_params)} + feats = _run_ops(t["enc_ops"], maps, params, parity, t, "enc0", frame=x) m_near, m_far, m_center = _enc.motion_surfaces(x) res = {"features": feats.astype(np.float32), "m_near": f16(m_near).astype(np.float32), @@ -193,41 +233,49 @@ def encode(x, params, parity=None, t=_table.TABLE, keep=False): # ---- decoder -------------------------------------------------------------------------------------------------------- -def decoder_front(features, params, parity, t=_table.TABLE): - """dec1's FP8 map [fh/2, fw/2, 64] -> the 64-channel map Model M's decoder projects (table entry dec_connect).""" - Z = np.asarray(features, dtype=np.float64) - segs = t["dec_connect"] - blk = [s for s in segs if s[0] == "block"] - if not blk: - return Z - if t["dec_block_level"] != "token": - raise NotImplementedError("decoder block level other than token: pending") - b = t["dec_block"] - lo = min(s[1] for s in blk) - hi = max(s[2] for s in blk) - if sum(s[2] - s[1] for s in blk) != hi - lo or hi - lo != b["C"]: - raise ValueError("block segments must be one contiguous range of the block's width") - consts = _table.block_consts(t, b.get("name", "dec0." + b["prefix"])) - off = _table.block_offset(t, b["offsets"], parity) - y = _round(swin_block(Z[..., lo:hi], _block_params(params, b["prefix"]), off, b["heads"], consts), - b.get("out_round", "q8")) - out = Z.copy() - for kind, s0, s1 in segs: - if kind == "block": - out[..., s0:s1] = y[..., s0 - lo:s1 - lo] - elif kind != "pass": - raise ValueError(f"unknown decoder segment {kind!r}") - return out - - -def decode(x, params, parity, t=_table.TABLE): - """Model L's coefficient decoder: decoder_front, then Model M's decoder (table entry dec_pixel = "m").""" +def dec_project(Z, W, b, groups=(32, 32), out_round="q8"): + """Z [h, w, 64], W [4, 32, 64], b [4, 32] -> [h, w, 128] (channel 32 s + c): Model M's dec0 projection arithmetic + (clean/reference/decoder.py project) over all 4 x 32 rows.""" + Z = np.asarray(Z, dtype=np.float64) + W = np.asarray(W, dtype=np.float64).reshape(128, -1) + s = np.broadcast_to(np.asarray(b, dtype=np.float64).reshape(128), Z.shape[:-1] + (128,)) + lo = 0 + for g in groups: + s = f16(s + Z[..., lo:lo + g] @ W[:, lo:lo + g].T) + lo += g + return _round(s, out_round) + + +def coefficients(x, a, S, T): + """Model M's coefficient and latent functions (clean/reference/decoder.py decode after its projection) on the + per-pixel values a [rh, rw, 17].""" + rw, rh, fw, fh, dw, dh = _dec.check_inputs(x) + a = np.asarray(a, dtype=np.float32) + A = _dec.sigmoid16(a[..., 0:3]) + B = _dec.softmax3(a[..., 3:6]) + C = _dec.coefficient_c(a[..., 6], a[..., 7], a[..., 8], float(np.asarray(S)), float(np.asarray(T))) + p = _dec.previous_latent(x, rw, rh, dw, dh) + lat = _dec.latent_update(a, p) + cA, cB, cL = _dec.unorm8_code(A), _dec.unorm8_code(B), _dec.snorm8_code(lat) + return {"A": (cA / 255.0).astype(np.float32), "B": (cB / 255.0).astype(np.float32), + "C": C.astype(np.float32), "latent": (cL / 127.0).astype(np.float32), + "codes_A": cA, "codes_B": cB, "codes_L": cL, "p": p} + + +def decode(x, params, parity, t=_table.TABLE, keep=False): + """Model L's coefficient decoder. x: Model M's dec0 input dict (features = dec1's output [fh/2, fw/2, 64]); + params: {stage: tensors} ("dec0", "dec0_block").""" if t["dec_pixel"] != "m": raise NotImplementedError("decoder per-pixel variant other than Model M's: pending") - xx = dict(x) - xx["features"] = decoder_front(x["features"], params, parity, t).astype(np.float32) - out = _dec.decode(xx, {k: params[k] for k in ("W", "b", "S", "T")}) - out["front"] = xx["features"] + _table.decoder_plan(t) + rw, rh = _dec.check_inputs(x)[:2] + maps = {"features": np.asarray(x["features"], dtype=np.float64)} + a = _run_ops(t["dec_ops"], maps, params, parity, t, "dec0") + dec = params["dec0"] + out = coefficients(x, a[:rh, :rw], dec["S"], dec["T"]) + out["a"] = a[:rh, :rw].astype(np.float32) + if keep: + out["maps"] = maps return out @@ -241,9 +289,9 @@ def network(x_enc, x_dec, params, parity, t=_table.TABLE, keep=False): """Encoder -> the ten shared Swin stages (Model M's oracle, L's weights) -> decoder. x_enc: Model M's enc0 input dict. x_dec: Model M's dec0 input dict without "features" (motion, latent_prev, - jitter, jitter_prev, reset, feature_size, recon_size). params: {"enc0": ..., "enc1": ..., ..., "dec0": ...}. + jitter, jitter_prev, reset, feature_size, recon_size). params: {stage: tensors} for every stage. Returns {"enc0": ..., stage outputs ..., "dec0": ...}.""" - res = {"enc0": encode(x_enc, params["enc0"], parity, t, keep)} + res = {"enc0": encode(x_enc, params, parity, t, keep)} x = res["enc0"]["features"] skips = {} for s in _table.SHARED_STAGES: @@ -263,7 +311,7 @@ def network(x_enc, x_dec, params, parity, t=_table.TABLE, keep=False): res[s] = o xd = dict(x_dec) xd["features"] = x - res["dec0"] = decode(xd, params["dec0"], parity, t) + res["dec0"] = decode(xd, params, parity, t, keep) return res @@ -288,28 +336,33 @@ def random_block_params(rng, C, heads, mlp, prefix="", weight_scale=1.0): def random_params(rng, t=_table.TABLE): - """Synthetic parameters for every stage of Model L (shapes from the table and Model M's stages).""" - enc = {} - for name, shape in _table.encoder_shapes(t).items(): - if name == "T": - enc[name] = rng.integers(0, 128, size=shape).astype(np.float32) - elif name in ("sigma1", "sigma2"): - enc[name] = np.array(rng.uniform(0.5, 1.5), dtype=np.float32) - for op in t["enc_ops"]: - if op["op"] == "linear": - enc[op["w"]] = q8(rng.normal(0, 1.0 / math.sqrt(op["n_in"]), (op["n_out"], op["n_in"]))).astype(np.float32) - enc[op["b"]] = f16(rng.normal(0, 0.1, op["n_out"])).astype(np.float32) - elif op["op"] == "swin": - enc.update(random_block_params(rng, op["C"], op["heads"], op["mlp"], op["prefix"])) - elif op["op"] == "project": - n = sum(op["groups"]) - enc[op["w"]] = q8(rng.normal(0, 0.6 / math.sqrt(n), (op["n_out"], n))).astype(np.float32) - enc[op["b"]] = f16(rng.normal(0, 0.3, op["n_out"])).astype(np.float32) - dec = _dec.random_params(rng) - if any(s[0] == "block" for s in t["dec_connect"]): - b = t["dec_block"] - dec.update(random_block_params(rng, b["C"], b["heads"], b["mlp"], b["prefix"])) - p = {"enc0": enc, "dec0": dec} + """Synthetic parameters for every stage of Model L ({stage: tensors}; shapes from the table and Model M's + stages). Never model data.""" + p = {} + for stage, sh in _table.param_shapes(t).items(): + st = {} + if "Wqk" in sh: + st.update(random_block_params(rng, sh["gamma1"][0], sh["Wqk"][0], sh["W1"][0])) + for name, shape in sh.items(): + if name in st: + continue + if stage == "enc0" and name == "T": + st[name] = rng.integers(0, 128, size=shape).astype(np.float32) + elif name in ("sigma1", "sigma2"): + st[name] = np.array(rng.uniform(0.5, 1.5), dtype=np.float32) + elif stage == "dec0" and name == "S": + st[name] = np.array(rng.uniform(1.0, 4.0), dtype=np.float32) + elif stage == "dec0" and name == "T": + st[name] = np.array(rng.uniform(0.2, 1.5), dtype=np.float32) + elif name == "s": + st[name] = np.array(rng.uniform(0.5, 2.0), dtype=np.float32) + elif len(shape) >= 2: + fan_in = shape[-1] + scale = {"enc0": 0.6 if name == "W" else 1.0, "dec0": 0.25 * np.sqrt(fan_in)}.get(stage, 1.0) + st[name] = q8(rng.normal(0, scale / np.sqrt(fan_in), shape)).astype(np.float32) + else: + st[name] = f16(rng.normal(0, 0.3 if stage == "dec0" else 0.1, shape)).astype(np.float32) + p[stage] = st for s in _table.SHARED_STAGES: p[s] = {k: np.asarray(v, dtype=np.float32) for k, v in _swin.random_params(s, rng).items()} return p diff --git a/clean/l/reference/table.py b/clean/l/reference/table.py index 2c3d5ce6..18801492 100644 --- a/clean/l/reference/table.py +++ b/clean/l/reference/table.py @@ -3,9 +3,11 @@ runtime and the tests. Every consumer interprets this table instead of hard-coding L's structure, so a provisional entry can be changed -without touching code. Entries listed in PROVISIONAL are pending confirmation; each carries a short key (I1 ...). +without touching code. Entries listed in PROVISIONAL are pending confirmation; each carries a short key (I5 ...). -SHARED_STAGES run with Model M's code and stage table (clean/reference/swin.py STAGES) and L's own weights. +Parameters are grouped by the converter map's stages (data/preset-L.v1.map.json): "enc0", "enc0_block", the ten +shared stages, "dec0_block", "dec0", "post". SHARED_STAGES run with Model M's code and stage table +(clean/reference/swin.py STAGES) and L's own weights. """ import copy @@ -21,6 +23,10 @@ M_BLOCK_CONSTS = {"tau": _swin.TAU, "cap": _swin.CAP, "eps_num": 2.0 ** -13, "gelu": "exact", "window": _swin.WIN, "head_dim": _swin.HEAD_DIM} +# The value positions of the coefficient decoder (the map's perm "dec0_values"): value k of a render pixel is +# channel DEC0_VALUES[k] of its 32. +DEC0_VALUES = (0, 1, 10, 11, 12, 13, 14, 15, 16, 2, 3, 4, 5, 6, 7, 8, 9) + # ---- the table ------------------------------------------------------------------------------------------------------ C32_BLOCK = {"C": 32, "heads": 2, "mlp": 128} @@ -29,47 +35,54 @@ # Input encoder: ops applied in order. Each op reads and writes named maps; "values" is the per-pixel input # (clean/reference/encoder.py pixel_values: [fh, fw, 20] on the padded grid F). A map's level follows from its # size: the pixel grid F, or the token grid F/2 after space_to_depth (2 x 2 pixels, sub-pixel p = dx + 2 dy, - # channel = n * p + i). The last op must be "project" (the 64 FP8 features enc1 reads). + # channel = n p + i). Tensors are read from the stage named by "params" (default "enc0"). A linear's "acc" is the + # rounding of bias + products before "out_round" ("relu_q8" = q8(max(0, .))). The last op is "project" (the 64 FP8 + # features enc1 reads): running sum s = b, s = RN16(s + group) per channel group, clamp, FP8. "enc_ops": ( - {"op": "quant", "mode": "dither", "src": "values", "dst": "q"}, # I5 - {"op": "space_to_depth", "src": "q", "dst": "raw"}, # I2 - {"op": "select", "src": "raw", "dst": "x", "lo": 0, "hi": 20}, # I2 - {"op": "linear", "src": "x", "dst": "x", "w": "Wa", "b": "ba", "n_in": 20, "n_out": 32, - "out_round": "q8"}, # I1 - {"op": "linear", "src": "x", "dst": "x", "w": "Wb", "b": "bb", "n_in": 32, "n_out": 32, - "out_round": "q8"}, # I1 - {"op": "swin", "src": "x", "dst": "x", "prefix": "blk.", **C32_BLOCK, "offsets": "first", - "out_round": "q8"}, # I1, I6, I10 - {"op": "concat", "srcs": ("raw", "x"), "dst": "z", "pad_to": 128}, # I2 - {"op": "project", "src": "z", "w": "W", "b": "b", "groups": (32, 32, 32, 32), "n_out": 64}, # I2 + {"op": "quant", "mode": "dither", "src": "values", "dst": "q"}, # I5 + {"op": "linear", "src": "q", "dst": "x", "w": "Win1", "b": "bin1", "n_in": 20, "n_out": 32, + "acc": "f16", "out_round": "relu_q8"}, # I11 + {"op": "linear", "src": "x", "dst": "x", "w": "Win2", "b": "bin2", "n_in": 32, "n_out": 32, + "acc": "f16", "out_round": "q8"}, # I11 + {"op": "swin", "src": "x", "dst": "x", "params": "enc0_block", **C32_BLOCK, "offsets": "first", + "out_round": "q8"}, # I10, I6 + {"op": "space_to_depth", "src": "x", "dst": "z"}, + {"op": "project", "src": "z", "w": "W", "b": "b", "groups": (32, 32, 32, 32), "n_out": 64, + "clamp": 6.28125}, # I12 + ), + # Coefficient decoder: ops on dec1's map ("features", [fh/2, fw/2, 64]). "dec_project": rows W [4, 32, 64] with + # Model M's dec0 rounding (RN16 after each group of 32 input channels, then FP8), output [fh/2, fw/2, 4 x 32] + # (sub-pixel s = dx + 2 dy, channel 32 s + c); "depth_to_space" makes it a pixel map [fh, fw, 32]; "pick" takes + # the value positions of the coefficient functions, which then run as Model M's (clean/reference/decoder.py). + "dec_ops": ( + {"op": "dec_project", "src": "features", "dst": "y", "w": "W", "b": "b", "groups": (32, 32), + "out_round": "q8"}, # I13 + {"op": "depth_to_space", "src": "y", "dst": "p"}, + {"op": "swin", "src": "p", "dst": "p", "params": "dec0_block", **C32_BLOCK, "offsets": "second", + "out_round": "q8"}, # I10, I14 + {"op": "pick", "src": "p", "values": DEC0_VALUES}, # I14 ), - # Coefficient decoder: dec1's 64 channels split into segments; "block" segments go through one block, "pass" - # segments are kept; the result (same channel order) feeds Model M's decoder (clean/reference/decoder.py). - "dec_connect": (("block", 0, 32), ("pass", 32, 64)), # I4 - "dec_block": {"prefix": "blk.", **C32_BLOCK, "offsets": "second", "out_round": "q8"}, # I6, I10 - "dec_block_level": "token", # I3 - "dec_pixel": "m", # I8 - "post_variant": "unfolded", # I8 - "recon_ratio": (3, 2), # I7 - # Window offsets (ox, oy) of an L block for frame parity 0 and 1. - "window_offsets": {"first": ((2, 2), (0, 2)), "second": ((0, 0), (2, 0))}, # I10 - # Per-block constants; a block's own entry wins over "default". - "consts": {"default": dict(M_BLOCK_CONSTS)}, # I6 - # Logical tensor name -> converter map name, per stage ("enc0", "dec0"); identity when absent. - "map_names": {}, # I9 + "dec_pixel": "m", # I8 + "post_variant": "unfolded", # I8 + # Reconstruction variants and the rule that picks one per frame from the render and output sizes ("rule" names + # the comparison, "ratio" its factor). Only the 3/2 form (D = floor((3 O + 1) / 2)) exists in WrongGPU so far. + "recon": {"rule": "both_axes_at_least", "ratio": 3, "variants": ("3/2", "3/1")}, # I7 + # Window offsets (ox, oy) of an L block for frame parity 0 and 1; the blocks run on the padded grid F. + "window_offsets": {"first": ((2, 2), (0, 2)), "second": ((0, 0), (2, 0))}, # I10 + # Per-block constants; a block's own entry (key = its params stage) wins over "default". + "consts": {"default": dict(M_BLOCK_CONSTS)}, # I6 } PROVISIONAL = { - "I1": "enc_ops: order and kind of the ops", - "I2": "enc_ops: levels, select / concat segments, project groups", - "I3": "dec_block_level", - "I4": "dec_connect", - "I5": "enc_ops: quant mode", - "I6": "consts", - "I7": "recon_ratio", - "I8": "dec_pixel, post_variant", - "I9": "map_names", - "I10": "window_offsets", + "I5": "enc_ops quant mode (dither table or plain FP8 rounding)", + "I6": "consts of the width-32 blocks", + "I7": "recon: the 3/1 form and the exact rule", + "I8": "dec_pixel (coefficient functions as Model M's), post_variant", + "I10": "window_offsets of the width-32 blocks, and their grid (F)", + "I11": "acc / out_round of the input linears", + "I12": "project clamp and group structure", + "I13": "dec_project rounding", + "I14": "pick value positions and the rounding of the decoder block's output", } @@ -93,10 +106,6 @@ def block_offset(t, kind, parity): return tuple(t["window_offsets"][kind][int(parity) & 1]) -def map_name(t, stage, logical): - return t["map_names"].get(stage, {}).get(logical, logical) - - # ---- parameter shapes ----------------------------------------------------------------------------------------------- def block_shapes(C, heads, mlp, head_dim=32, window=4): @@ -106,84 +115,99 @@ def block_shapes(C, heads, mlp, head_dim=32, window=4): "W2": (C, mlp), "b2": (C,)} -def encoder_shapes(t=TABLE): - """Tensor shapes of the L encoder stage, logical names.""" - shapes = {} +def param_shapes(t=TABLE): + """{stage: {tensor: shape}} of Model L's own stages (enc0, its block, dec0, its block, post).""" + out = {"enc0": {}, "dec0": {"S": (), "T": ()}, "post": {"s": ()}} for op in t["enc_ops"]: + st = out.setdefault(op.get("params", "enc0"), {}) if op["op"] == "quant": - shapes.update({"sigma1": (), "sigma2": ()}) + st.update({"sigma1": (), "sigma2": ()}) if op["mode"] == "dither": - shapes["T"] = (8, 16, 16) + st["T"] = (8, 16, 16) elif op["op"] == "linear": - shapes.update({op["w"]: (op["n_out"], op["n_in"]), op["b"]: (op["n_out"],)}) + st.update({op["w"]: (op["n_out"], op["n_in"]), op["b"]: (op["n_out"],)}) elif op["op"] == "swin": - for k, s in block_shapes(op["C"], op["heads"], op["mlp"]).items(): - shapes[op["prefix"] + k] = s + st.update(block_shapes(op["C"], op["heads"], op["mlp"])) elif op["op"] == "project": - shapes.update({op["w"]: (op["n_out"], sum(op["groups"])), op["b"]: (op["n_out"],)}) - return shapes + st.update({op["w"]: (op["n_out"], sum(op["groups"])), op["b"]: (op["n_out"],)}) + for op in t["dec_ops"]: + st = out.setdefault(op.get("params", "dec0"), {}) + if op["op"] == "dec_project": + st.update({op["w"]: (4, 32, sum(op["groups"])), op["b"]: (4, 32)}) + elif op["op"] == "swin": + st.update(block_shapes(op["C"], op["heads"], op["mlp"])) + return out def encoder_plan(t=TABLE): - """(level, channels) of every named map after each op, from the op list alone: [(op, level, channels), ...].""" - maps = {"values": ("pixel", 20)} + """(op, level, channels) after each encoder op, from the op list alone.""" + return _plan(t["enc_ops"], {"values": ("pixel", 20)}, "encoder", last="project") + + +def decoder_plan(t=TABLE): + """(op, level, channels) after each decoder op, from the op list alone.""" + return _plan(t["dec_ops"], {"features": ("token", 64)}, "decoder", last="pick") + + +def _plan(ops, maps, what, last): plan = [] - for op in t["enc_ops"]: + for op in ops: k = op["op"] + src = maps.get(op.get("src")) + if k != "concat" and src is None: + raise ValueError(f"{what}: {k} reads unknown map {op.get('src')!r}") if k == "quant": - out = maps[op["src"]] + out = src elif k == "space_to_depth": - lvl, c = maps[op["src"]] - if lvl != "pixel": + if src[0] != "pixel": raise ValueError("space_to_depth needs a pixel-level map") - out = ("token", 4 * c) + out = ("token", 4 * src[1]) + elif k == "depth_to_space": + if src[0] != "token" or src[1] % 4: + raise ValueError("depth_to_space needs a token-level map of 4 n channels") + out = ("pixel", src[1] // 4) elif k == "select": - lvl, c = maps[op["src"]] - if not 0 <= op["lo"] < op["hi"] <= c: - raise ValueError(f"select {op['lo']}..{op['hi']} outside {c} channels") - out = (lvl, op["hi"] - op["lo"]) + if not 0 <= op["lo"] < op["hi"] <= src[1]: + raise ValueError(f"select {op['lo']}..{op['hi']} outside {src[1]} channels") + out = (src[0], op["hi"] - op["lo"]) elif k == "linear": - lvl, c = maps[op["src"]] - if c != op["n_in"]: - raise ValueError(f"linear {op['w']}: input has {c} channels, op wants {op['n_in']}") - out = (lvl, op["n_out"]) + if src[1] != op["n_in"]: + raise ValueError(f"linear {op['w']}: input has {src[1]} channels, op wants {op['n_in']}") + out = (src[0], op["n_out"]) elif k == "swin": - lvl, c = maps[op["src"]] - if c != op["C"]: - raise ValueError(f"swin: input has {c} channels, block width {op['C']}") - out = (lvl, c) + if src[1] != op["C"]: + raise ValueError(f"swin: input has {src[1]} channels, block width {op['C']}") + out = src elif k == "concat": - lv = {maps[s][0] for s in op["srcs"]} - if len(lv) != 1: + parts = [maps[s] for s in op["srcs"]] + if len({p[0] for p in parts}) != 1: raise ValueError("concat of maps at different levels") - c = sum(maps[s][1] for s in op["srcs"]) + c = sum(p[1] for p in parts) pad = op.get("pad_to", c) if pad < c: raise ValueError("concat: pad_to below the channel count") - out = (lv.pop(), pad) + out = (parts[0][0], pad) elif k == "project": - lvl, c = maps[op["src"]] - if lvl != "token" or c != sum(op["groups"]): - raise ValueError(f"project: needs a token-level map of {sum(op['groups'])} channels, got {lvl} {c}") + if src[0] != "token" or src[1] != sum(op["groups"]): + raise ValueError(f"project: needs a token-level map of {sum(op['groups'])} channels, got {src}") out = ("token", op["n_out"]) + elif k == "dec_project": + if src[0] != "token" or src[1] != sum(op["groups"]): + raise ValueError(f"dec_project: needs a token-level map of {sum(op['groups'])} channels, got {src}") + out = ("token", 128) + elif k == "pick": + if src[0] != "pixel" or max(op["values"]) >= src[1]: + raise ValueError("pick: needs a pixel-level map holding every value position") + out = ("pixel", len(op["values"])) else: - raise ValueError(f"unknown encoder op {k!r}") - maps[op.get("dst", "features")] = out + raise ValueError(f"unknown {what} op {k!r}") + maps[op.get("dst", "out")] = out plan.append((k, out[0], out[1])) - if not plan or plan[-1][0] != "project": - raise ValueError("the encoder's last op must be project") + if not plan or plan[-1][0] != last: + raise ValueError(f"the {what}'s last op must be {last}") return plan -def decoder_shapes(t=TABLE): - shapes = {"W": (68, 64), "b": (68,), "S": (), "T": ()} - if any(seg[0] == "block" for seg in t["dec_connect"]): - b = t["dec_block"] - for k, s in block_shapes(b["C"], b["heads"], b["mlp"]).items(): - shapes[b["prefix"] + k] = s - return shapes - - # ---- work per frame (computed, not measured) ------------------------------------------------------------------------ def block_macs_per_token(C, heads, mlp, head_dim=32, window=4): @@ -212,19 +236,22 @@ def frame_macs(t, render_w, render_h): m_enc0 = 80 * 64 * n["token"] m_dec0 = 64 * 68 * n["token"] M = sum(parts.values()) + m_enc0 + m_dec0 - enc = 0 - for op, (_, lvl, _c) in zip(t["enc_ops"], encoder_plan(t)): - if op["op"] == "linear": - enc += op["n_in"] * op["n_out"] * n[lvl] - elif op["op"] == "swin": - enc += block_macs_per_token(op["C"], op["heads"], op["mlp"]) * n[lvl] - elif op["op"] == "project": - enc += sum(op["groups"]) * op["n_out"] * n["token"] - dec = m_dec0 - if any(seg[0] == "block" for seg in t["dec_connect"]): - b = t["dec_block"] - lvl = n["token"] if t["dec_block_level"] == "token" else render_w * render_h - dec += block_macs_per_token(b["C"], b["heads"], b["mlp"]) * lvl + + def ops_macs(ops, plan): + total = 0 + for op, (_, lvl, _c) in zip(ops, plan): + if op["op"] == "linear": + total += op["n_in"] * op["n_out"] * n[lvl] + elif op["op"] == "swin": + total += block_macs_per_token(op["C"], op["heads"], op["mlp"]) * n[lvl] + elif op["op"] == "project": + total += sum(op["groups"]) * op["n_out"] * n["token"] + elif op["op"] == "dec_project": + total += sum(op["groups"]) * 128 * n["token"] + return total + + enc = ops_macs(t["enc_ops"], encoder_plan(t)) + dec = ops_macs(t["dec_ops"], decoder_plan(t)) L = sum(parts.values()) + enc + dec parts.update({"enc0.M": m_enc0, "dec0.M": m_dec0, "enc0.L": enc, "dec0.L": dec}) return {"M": M, "L": L, "parts": parts} diff --git a/clean/l/runner/l_pack.py b/clean/l/runner/l_pack.py index 5c578b8c..c237afab 100644 --- a/clean/l/runner/l_pack.py +++ b/clean/l/runner/l_pack.py @@ -1,18 +1,19 @@ # SPDX-License-Identifier: MIT """Model L weight packing (issue #89): the device blobs of Model L's own launches, from the structure table -(clean/l/reference/table.py) and the logical tensors of the oracle (clean/l/reference/oracle.py). +(clean/l/reference/table.py) and the parameters of the oracle ({stage: tensors}, clean/l/reference/oracle.py). -Model L's encoder and decoder are run as a chain of small FP8-map kernels, each with a blob in a format Model M -already uses (clean/FORMATS.md): +Model L's encoder and decoder run as a chain of small FP8-map kernels, each with a blob in a format Model M already +uses (clean/FORMATS.md). Maps are [tokens][C] FP8, channel fastest; pixel maps are on the padded grid F, token maps +on F/2. | Blob | Format | Kernel input -> output | |---|---|---| -| `enc0.values` | Model M's enc0 blob (clean/runner/enc0.py pack) with W = 0 and b = 0 | frame -> the quantised values as a token map [F/2][80] (space_to_depth order 20 p + i) | -| `enc0.` per linear | merge / expand section format: bias tile vector (rows R_L(C_out)), then fragments [mt < C_out/16][kc < C_in/16], rows R_L(C_out), K order K_L(C_in) | FP8 map [C_in] -> FP8 map [C_out]; a `select` feeding the linear is folded in as zero weights outside its channels | -| `enc0.` per block | Model M's Swin block section (packing.pack_block) at the block's width, padded to 256 | FP8 map -> FP8 map | -| `enc0.project` | bias tile vector (rows R_L(n_out)), then fragments [mt][kb], rows R_L(n_out), K order: groups of 32 channels, block 2i + s holds channel 32 i + 16 h + 8 s + j (dec0's order, clean/runner/dec0.py kord_decoder, over every group) | the concatenated maps -> the 64 FP8 features enc1 reads | -| `dec0.` | Model M's Swin block section at the decoder block's width | dec1's channels of the block segment -> same width | -| `dec0` | Model M's dec0 blob (clean/runner/dec0.py pack) | as Model M | +| `enc0.values` | Model M's enc0 blob (clean/runner/enc0.py pack) with W = 0 and b = 0 | frame -> the quantized values as a pixel map [F][32] (values 0..19, 12 zero bytes) | +| `enc0.` per linear | merge / expand section format: bias tile vector (rows R_L(C_out)), then fragments [mt < C_out/16][kc < C_in/16], rows R_L(C_out), K order K_L(C_in); inputs narrower than C_in (20 values) get zero weights | FP8 map [C_in] -> FP8 map [C_out] | +| `enc0_block`, `dec0_block` | Model M's Swin block section (packing.pack_block) at the block's width, padded to 256 | FP8 map -> FP8 map | +| `enc0.project` | bias tile vector (rows R_L(64)), then fragments [mt][kb], rows R_L(64), K order: block 2i + s of group i holds channel 32 i + 16 h + 8 s + j (dec0's order, clean/runner/dec0.py kord_decoder, over every group) | the 2 x 2 pixels of each token (channel 32 p + i, p = dx + 2 dy) -> the 64 FP8 features enc1 reads | +| `dec0.project` | expand section format: bias tile vector (rows R_E(32)), then fragments [mt < 8][kb < 4], rows R_E(32) (tile 2 s + g: sub-pixel s, channels 16 g ..), K order as `enc0.project` (two groups of 32) | dec1's map [F/2][64] -> pixel map [F][32] | +| `dec0.coef` | Model M's dec0 blob (clean/runner/dec0.py pack) with W = 0 and b = 0: the scalars S, T and the code tables | the decoder block's pixel map -> A, B, C, latent | The ten shared Swin stages use clean/runner/packing.py pack_stage unchanged. """ @@ -33,7 +34,7 @@ def kord_groups32(n): """K slot (h, j) of block kb holds channel 32 (kb // 2) + 16 h + 8 (kb % 2) + j: each group of 32 channels is two whole blocks, so a kernel can round its running sum after every second block.""" if n % 32: - raise ValueError(f"project input of {n} channels: groups of 32 only") + raise ValueError(f"input of {n} channels: groups of 32 only") K = np.empty(n, dtype=np.int64) for kb in range(n // 16): for h in range(2): @@ -67,10 +68,20 @@ def pack_project(W, b, groups, d_row="gfx12"): return P.pack_tile_vector(b, RL, drow) + P.pack_frags(W, RL, kord_groups32(n)) -def pack_block(tensors, prefix, C, heads, mlp, d_row="gfx12"): +def pack_dec_project(W, b, groups=(32, 32), d_row="gfx12"): + """W [4, 32, n], b [4, 32] -> bias (rows R_E(32)) + fragments (rows R_E(32), K order kord_groups32(n)).""" drow = P.resolve_d_row(d_row) - p = {k[len(prefix):]: v for k, v in tensors.items() if k.startswith(prefix)} - blob = P.pack_block(p, C, heads, mlp, drow) + W = np.asarray(W, dtype=np.float64).reshape(128, -1) + if any(g != 32 for g in groups) or sum(groups) != W.shape[1]: + raise ValueError(f"dec_project groups {groups}: groups of 32 covering {W.shape[1]} channels only") + RE = P.rows_expand(32, drow) + return P.pack_tile_vector(np.asarray(b, np.float32).reshape(128), RE, drow) + \ + P.pack_frags(W, RE, kord_groups32(W.shape[1])) + + +def pack_block(tensors, C, heads, mlp, d_row="gfx12"): + drow = P.resolve_d_row(d_row) + blob = P.pack_block(tensors, C, heads, mlp, drow) return blob + bytes((-len(blob)) % ALIGN) @@ -82,76 +93,98 @@ def pack_values(tensors, d_row="gfx12"): return _enc0.pack(t, d_row) -# ---- the encoder's launch list -------------------------------------------------------------------------------------- +def pack_coef(tensors, d_row="gfx12"): + """Model M's dec0 blob with zero projection: the scalars S, T and the code tables.""" + return _dec0.pack({"W": np.zeros((68, 64), np.float32), "b": np.zeros(68, np.float32), + "S": tensors["S"], "T": tensors["T"]}, d_row) + + +# ---- launch lists --------------------------------------------------------------------------------------------------- def encoder_launches(t=_table.TABLE): - """The kernels of the table's encoder, in order: [{"kind", "blob", "src", "dst", ...}]. kinds: "values" (quant - + space_to_depth into the raw map), "linear", "swin", "project". A select is folded into the linear it feeds.""" - ops = t["enc_ops"] - plan = _table.encoder_plan(t) - chans = {"values": 20} - launches = [] - selects = {} - i = 0 - while i < len(ops): - op = ops[i] + """The encoder's kernels in order: [{"kind", "blob", ...}]. kinds: "values" (quant into a pixel map of 32 + channels), "linear", "swin", "project" (with the space_to_depth gather folded in).""" + ops = list(t["enc_ops"]) + _table.encoder_plan(t) + width = {"values": 20} + out = [] + for i, op in enumerate(ops): k = op["op"] if k == "quant": - nxt = ops[i + 1] if i + 1 < len(ops) else None - if not nxt or nxt["op"] != "space_to_depth" or nxt["src"] != op["dst"]: - raise ValueError("the encoder's kernels need quant directly followed by its space_to_depth") - if op["mode"] != "dither": - raise ValueError("the encoder's value kernel quantises with the dither table") - launches.append({"kind": "values", "blob": "enc0.values", "dst": nxt["dst"]}) - chans[nxt["dst"]] = 80 - i += 2 - continue - if k == "select": - selects[op["dst"]] = (op["src"], op["lo"], op["hi"]) - chans[op["dst"]] = op["hi"] - op["lo"] + if op["mode"] != "dither" or i != 0: + raise ValueError("the encoder's value kernel quantizes the per-pixel values with the dither table") + out.append({"kind": "values", "blob": "enc0.values", "dst": op["dst"], "C": 32}) + width[op["dst"]] = 32 elif k == "linear": - src, lo = op["src"], 0 - if src in selects: - src, lo, _ = selects[src] - launches.append({"kind": "linear", "blob": "enc0." + op["w"], "src": src, "dst": op["dst"], - "C_in": chans[src], "C_out": op["n_out"], "lo": lo, "w": op["w"], "b": op["b"]}) - chans[op["dst"]] = op["n_out"] + c_in = width[op["src"]] + out.append({"kind": "linear", "blob": "enc0." + op["w"], "src": op["src"], "dst": op["dst"], + "C_in": c_in, "C_out": op["n_out"], "w": op["w"], "b": op["b"], "acc": op.get("acc"), + "out_round": op.get("out_round", "q8")}) + width[op["dst"]] = op["n_out"] elif k == "swin": - launches.append({"kind": "swin", "blob": "enc0." + op["prefix"].rstrip("."), "src": op["src"], - "dst": op["dst"], "prefix": op["prefix"], "C": op["C"], "heads": op["heads"], - "mlp": op["mlp"], "offsets": op["offsets"]}) - elif k == "concat": - chans[op["dst"]] = plan[i][2] - launches.append({"kind": "concat", "srcs": op["srcs"], "dst": op["dst"], "pad_to": plan[i][2]}) + out.append({"kind": "swin", "blob": op["params"], "src": op["src"], "dst": op["dst"], "C": op["C"], + "heads": op["heads"], "mlp": op["mlp"], "offsets": op["offsets"], "params": op["params"]}) + elif k == "space_to_depth": + nxt = ops[i + 1] if i + 1 < len(ops) else None + if not nxt or nxt["op"] != "project" or nxt["src"] != op["dst"]: + raise ValueError("space_to_depth must feed the project directly (one gather kernel)") + width[op["dst"]] = 4 * width[op["src"]] + out.append({"kind": "project", "blob": "enc0.project", "src": op["src"], "groups": nxt["groups"], + "w": nxt["w"], "b": nxt["b"], "clamp": nxt.get("clamp")}) elif k == "project": - launches.append({"kind": "project", "blob": "enc0.project", "src": op["src"], "groups": op["groups"]}) + continue # folded into the gather above else: raise ValueError(f"no kernel for encoder op {k!r}") - if k != "select": - selects.pop(op.get("dst"), None) # the name now holds this op's output - i += 1 - return launches + return out + + +def decoder_launches(t=_table.TABLE): + """The decoder's kernels in order: "dec_project" (with depth_to_space folded in), "swin", "coef" (pick + the + coefficient functions).""" + ops = list(t["dec_ops"]) + _table.decoder_plan(t) + out = [] + for i, op in enumerate(ops): + k = op["op"] + if k == "dec_project": + nxt = ops[i + 1] if i + 1 < len(ops) else None + if not nxt or nxt["op"] != "depth_to_space" or nxt["src"] != op["dst"]: + raise ValueError("dec_project must feed depth_to_space directly (one scatter kernel)") + out.append({"kind": "dec_project", "blob": "dec0.project", "w": op["w"], "b": op["b"], + "groups": op["groups"], "dst": nxt["dst"]}) + elif k == "depth_to_space": + continue + elif k == "swin": + out.append({"kind": "swin", "blob": op["params"], "src": op["src"], "dst": op["dst"], "C": op["C"], + "heads": op["heads"], "mlp": op["mlp"], "offsets": op["offsets"], "params": op["params"]}) + elif k == "pick": + out.append({"kind": "coef", "blob": "dec0.coef", "src": op["src"], "values": tuple(op["values"])}) + else: + raise ValueError(f"no kernel for decoder op {k!r}") + return out def pack_model_l(params, t=_table.TABLE, d_row="gfx12"): """{blob name: bytes} of Model L's own launches (the shared Swin stages: packing.pack_stage).""" - enc = params["enc0"] out = {} + enc = params["enc0"] for ln in encoder_launches(t): if ln["kind"] == "values": out[ln["blob"]] = pack_values(enc, d_row) elif ln["kind"] == "linear": - out[ln["blob"]] = pack_linear(enc[ln["w"]], enc[ln["b"]], ln["C_in"], ln["C_out"], ln["lo"], d_row) + out[ln["blob"]] = pack_linear(enc[ln["w"]], enc[ln["b"]], ln["C_in"], ln["C_out"], 0, d_row) elif ln["kind"] == "swin": - out[ln["blob"]] = pack_block(enc, ln["prefix"], ln["C"], ln["heads"], ln["mlp"], d_row) + out[ln["blob"]] = pack_block(params[ln["params"]], ln["C"], ln["heads"], ln["mlp"], d_row) elif ln["kind"] == "project": - op = t["enc_ops"][-1] - out[ln["blob"]] = pack_project(enc[op["w"]], enc[op["b"]], ln["groups"], d_row) + out[ln["blob"]] = pack_project(enc[ln["w"]], enc[ln["b"]], ln["groups"], d_row) dec = params["dec0"] - if any(s[0] == "block" for s in t["dec_connect"]): - b = t["dec_block"] - out["dec0." + b["prefix"].rstrip(".")] = pack_block(dec, b["prefix"], b["C"], b["heads"], b["mlp"], d_row) - out["dec0"] = _dec0.pack({k: dec[k] for k in ("W", "b", "S", "T")}, d_row) + for ln in decoder_launches(t): + if ln["kind"] == "dec_project": + out[ln["blob"]] = pack_dec_project(dec[ln["w"]], dec[ln["b"]], ln["groups"], d_row) + elif ln["kind"] == "swin": + out[ln["blob"]] = pack_block(params[ln["params"]], ln["C"], ln["heads"], ln["mlp"], d_row) + elif ln["kind"] == "coef": + out[ln["blob"]] = pack_coef(dec, d_row) return out @@ -190,3 +223,12 @@ def unpack_project(blob, n_in, n_out, d_row="gfx12"): RL = P.rows_layout_l(n_out, drow) b = unpack_tile_vector(blob[:4 * n_out], RL, drow) return unpack_frags(blob[4 * n_out:], RL, kord_groups32(n_in)), b + + +def unpack_dec_project(blob, n_in=64, d_row="gfx12"): + """(W [4, 32, n_in], b [4, 32]) from a dec0.project blob.""" + drow = P.resolve_d_row(d_row) + RE = P.rows_expand(32, drow) + b = unpack_tile_vector(blob[:4 * 128], RE, drow) + W = unpack_frags(blob[4 * 128:], RE, kord_groups32(n_in)) + return W.reshape(4, 32, n_in), b.reshape(4, 32) diff --git a/clean/l/tests/test_l_pack.py b/clean/l/tests/test_l_pack.py index 7a7917ad..7138aaaa 100644 --- a/clean/l/tests/test_l_pack.py +++ b/clean/l/tests/test_l_pack.py @@ -10,6 +10,7 @@ from clean.l.reference import oracle as L from clean.l.reference import table as T from clean.l.runner import l_pack as LP +from clean.reference import decoder as MD from clean.reference import encoder as ME from clean.reference import swin as MS from clean.reference.fp8 import q8 @@ -34,7 +35,7 @@ def test_groups32_blocks(self): class RoundTripTest(unittest.TestCase): def test_linear(self): rng = np.random.default_rng(1) - for C_in, C_out, n, lo in ((80, 32, 20, 0), (80, 32, 20, 60), (32, 32, 32, 0), (64, 16, 16, 16)): + for C_in, C_out, n, lo in ((32, 32, 20, 0), (80, 32, 20, 60), (32, 32, 32, 0), (64, 16, 16, 16)): W = q8(rng.normal(0, 0.5, (C_out, n))) b = rng.normal(0, 1, C_out).astype(np.float32) blob = LP.pack_linear(W, b, C_in, C_out, lo) @@ -48,7 +49,6 @@ def test_linear(self): LP.pack_linear(np.zeros((32, 20)), np.zeros(32), 80, 32, lo=70) def test_linear_is_merge_section_format(self): - """The linear blob is the patch-merge section format when the K order is the merge's.""" rng = np.random.default_rng(2) W = q8(rng.normal(0, 0.5, (96, 64))) b = rng.normal(0, 1, 96).astype(np.float32) @@ -69,71 +69,105 @@ def test_project(self): with self.assertRaises(ValueError): LP.pack_project(W, b, (20, 20, 20, 20, 48)) - def test_block_equals_model_m_packer(self): - """The L block packer is packing.pack_block: Model M's stage blobs come out byte for byte, and C = 32 has - the layout's size.""" + def test_dec_project(self): rng = np.random.default_rng(4) + W = q8(rng.normal(0, 0.3, (4, 32, 64))) + b = rng.normal(0, 1, (4, 32)).astype(np.float32) + blob = LP.pack_dec_project(W, b) + self.assertEqual(len(blob), 4 * 128 + 128 * 64) + Wu, bu = LP.unpack_dec_project(blob) + self.assertTrue(np.array_equal(Wu, W)) + self.assertTrue(np.array_equal(bu, b)) + + def test_dec_project_is_expand_section_format(self): + """Rows R_E(32): the expand section's row order at C = 32 (sub-pixel s, then layout L of its 32 channels).""" + rng = np.random.default_rng(5) + W = q8(rng.normal(0, 0.3, (4, 32, 64))) + b = rng.normal(0, 1, (4, 32)).astype(np.float32) + drow = P.resolve_d_row("gfx12") + RE = P.rows_expand(32, drow) + want = P.pack_tile_vector(b.reshape(128), RE, drow) + P.pack_frags(W.reshape(128, 64), RE, + LP.kord_groups32(64)) + self.assertEqual(LP.pack_dec_project(W, b), want) + + def test_block_equals_model_m_packer(self): + rng = np.random.default_rng(6) for stage in ("b0", "dec1"): cfg = MS.STAGES[stage] p = MS.random_params(stage, rng) want = P.pack_stage(stage, p)[:P.stage_sections(stage)[1]] - got = LP.pack_block({"x." + k: v for k, v in p.items()}, "x.", cfg["C"], cfg["heads"], cfg["mlp"]) + got = LP.pack_block({k: v for k, v in p.items() if k not in ("We", "be")}, cfg["C"], cfg["heads"], + cfg["mlp"]) self.assertEqual(got, want, stage) - p = L.random_block_params(rng, 32, 2, 128, "blk.") - blob = LP.pack_block(p, "blk.", 32, 2, 128) + p = L.random_block_params(rng, 32, 2, 128) + blob = LP.pack_block(p, 32, 2, 128) lay = P.block_layout(32, 2, 128) self.assertEqual(len(blob), lay["SIZE"] + (-lay["SIZE"]) % 256) self.assertEqual((lay["HPU"], lay["NMLP"], lay["MLP_BYTES"], lay["HEAD_BYTES"]), (8, 1, 8192, 3072)) - def test_values_blob(self): - rng = np.random.default_rng(5) - enc = L.random_params(rng)["enc0"] - blob = LP.pack_values(enc) + def test_values_and_coef_blobs(self): + rng = np.random.default_rng(7) + params = L.random_params(rng) + blob = LP.pack_values(params["enc0"]) self.assertEqual(len(blob), E0.BLOB_BYTES) self.assertEqual(blob[:256], bytes(256)) # zero bias - self.assertEqual(blob[E0.OFF_DITHER:E0.OFF_DITHER + 2048], np.asarray(enc["T"]).astype(np.uint8).tobytes()) + self.assertEqual(blob[E0.OFF_DITHER:E0.OFF_DITHER + 2048], + np.asarray(params["enc0"]["T"]).astype(np.uint8).tobytes()) + coef = LP.pack_coef(params["dec0"]) + self.assertEqual(len(coef), D0.BLOB_BYTES) + full = D0.pack({"W": np.zeros((68, 64), np.float32), "b": np.zeros(68, np.float32), + "S": params["dec0"]["S"], "T": params["dec0"]["T"]}) + self.assertEqual(coef, full) class ModelTest(unittest.TestCase): - def test_launch_list(self): + def test_launch_lists(self): ln = LP.encoder_launches() - self.assertEqual([x["kind"] for x in ln], ["values", "linear", "linear", "swin", "concat", "project"]) - self.assertEqual((ln[1]["src"], ln[1]["C_in"], ln[1]["lo"], ln[1]["C_out"]), ("raw", 80, 0, 32)) + self.assertEqual([x["kind"] for x in ln], ["values", "linear", "linear", "swin", "project"]) + self.assertEqual((ln[1]["src"], ln[1]["C_in"], ln[1]["C_out"], ln[1]["out_round"]), ("q", 32, 32, "relu_q8")) self.assertEqual((ln[2]["src"], ln[2]["C_in"]), ("x", 32)) + self.assertEqual(ln[3]["blob"], "enc0_block") + dn = LP.decoder_launches() + self.assertEqual([x["kind"] for x in dn], ["dec_project", "swin", "coef"]) + self.assertEqual(dn[2]["values"], T.DEC0_VALUES) def test_pack_model_l(self): - rng = np.random.default_rng(6) + rng = np.random.default_rng(8) params = L.random_params(rng) blobs = LP.pack_model_l(params) - self.assertEqual(sorted(blobs), sorted(["enc0.values", "enc0.Wa", "enc0.Wb", "enc0.blk", "enc0.project", - "dec0.blk", "dec0"])) - self.assertEqual(len(blobs["enc0.Wa"]), 4 * 32 + 32 * 80) - self.assertEqual(len(blobs["enc0.Wb"]), 4 * 32 + 32 * 32) - self.assertEqual(len(blobs["dec0"]), D0.BLOB_BYTES) - self.assertEqual(blobs["dec0"], D0.pack({k: params["dec0"][k] for k in ("W", "b", "S", "T")})) + self.assertEqual(sorted(blobs), sorted(["enc0.values", "enc0.Win1", "enc0.Win2", "enc0_block", "enc0.project", + "dec0.project", "dec0_block", "dec0.coef"])) + self.assertEqual(len(blobs["enc0.Win1"]), 4 * 32 + 32 * 32) + self.assertEqual(len(blobs["enc0.Win2"]), 4 * 32 + 32 * 32) + self.assertEqual(len(blobs["dec0.project"]), 4 * 128 + 128 * 64) self.assertEqual(blobs, LP.pack_model_l(params)) # deterministic def test_unpacked_weights_reproduce_the_oracle(self): - """The weights decoded from the blobs, applied to the oracle's maps the way the kernels apply them (whole - input maps, zero weights outside a select), give the oracle's maps byte for byte.""" - rng = np.random.default_rng(7) + """The weights decoded from the blobs, applied the way the kernels apply them (whole 32-channel input maps, + zero weights for the padding), give the oracle's maps byte for byte.""" + rng = np.random.default_rng(9) params = L.random_params(rng) x = ME.random_inputs(rng, rw=36, rh=20) - out = L.encode(x, params["enc0"], parity=0, keep=True) + out = L.encode(x, params, parity=0, keep=True) m = out["maps"] blobs = LP.pack_model_l(params) enc = params["enc0"] - Wa, ba = LP.unpack_linear(blobs["enc0.Wa"], 80, 32) - Wb, bb = LP.unpack_linear(blobs["enc0.Wb"], 32, 32) - h1 = q8(m["raw"] @ Wa.T + ba) - self.assertTrue(np.array_equal(h1, q8(m["raw"][..., :20] @ np.asarray(enc["Wa"], np.float64).T + enc["ba"]))) - h2 = q8(h1 @ Wb.T + bb) - bp = {k[4:]: v for k, v in enc.items() if k.startswith("blk.")} - h3 = q8(L.swin_block(h2, bp, T.block_offset(T.TABLE, "first", 0), 2, T.M_BLOCK_CONSTS)) + W1, b1 = LP.unpack_linear(blobs["enc0.Win1"], 32, 32) + W2, b2 = LP.unpack_linear(blobs["enc0.Win2"], 32, 32) + qpad = np.concatenate([m["q"], np.zeros(m["q"].shape[:-1] + (12,))], axis=-1) + h1 = L.linear(qpad, W1, b1, "f16", "relu_q8") + self.assertTrue(np.array_equal(h1, L.linear(m["q"], enc["Win1"], enc["bin1"], "f16", "relu_q8"))) + h2 = L.linear(h1, W2, b2, "f16", "q8") + h3 = q8(L.swin_block(h2, params["enc0_block"], T.block_offset(T.TABLE, "first", 0), 2, T.M_BLOCK_CONSTS)) self.assertTrue(np.array_equal(h3, m["x"])) Wp, bpj = LP.unpack_project(blobs["enc0.project"], 128, 64) - feats = L.project_groups(m["z"], Wp, bpj, (32, 32, 32, 32)) + feats = L.project_groups(L.space_to_depth(h3), Wp, bpj, (32, 32, 32, 32), 6.28125) self.assertEqual(feats.astype(np.float32).tobytes(), out["features"].tobytes()) + xd = MD.random_inputs(rng, rw=36, rh=20) + dout = L.decode(xd, params, 0, keep=True) + Wd, bd = LP.unpack_dec_project(blobs["dec0.project"]) + y = L.dec_project(xd["features"], Wd, bd) + self.assertTrue(np.array_equal(y, dout["maps"]["y"])) if __name__ == "__main__": diff --git a/clean/l/tests/test_l_reference.py b/clean/l/tests/test_l_reference.py index 81103461..c1cc9e5c 100644 --- a/clean/l/tests/test_l_reference.py +++ b/clean/l/tests/test_l_reference.py @@ -3,8 +3,8 @@ Run: python -m unittest discover -s clean/l/tests -t . -The strongest checks reduce Model L to Model M through the table and require the L oracle to give Model M's oracle -results byte for byte: the block with M's constants, the encoder with M's op list, the decoder without a block, and +The strongest checks reduce Model L to Model M through the table and require Model M's oracle results byte for byte: +the block with M's constants, the encoder with M's op list, the decoder with M's projection rows and no block, and the whole network chain. """ import unittest @@ -22,7 +22,13 @@ M_ENC_OPS = ( {"op": "quant", "mode": "dither", "src": "values", "dst": "q"}, {"op": "space_to_depth", "src": "q", "dst": "z"}, - {"op": "project", "src": "z", "w": "W", "b": "b", "groups": (20, 20, 20, 20), "n_out": 64}, + {"op": "project", "src": "z", "w": "W", "b": "b", "groups": (20, 20, 20, 20), "n_out": 64, "clamp": 6.28125}, +) +# Model M's decoder written as an L op list: the projection, then the value positions straight away (no block). +M_DEC_OPS = ( + {"op": "dec_project", "src": "features", "dst": "y", "w": "W", "b": "b", "groups": (32, 32)}, + {"op": "depth_to_space", "src": "y", "dst": "p"}, + {"op": "pick", "src": "p", "values": T.DEC0_VALUES}, ) @@ -32,6 +38,19 @@ def _same(a, b): return a.shape == b.shape and a.dtype == b.dtype and a.tobytes() == b.tobytes() +def m_dec_as_l(p): + """Model M's dec0 tensors (W [68, 64], b [68]) as L's [4, 32, 64] / [4, 32] with zeros outside the value rows.""" + W = np.zeros((4, 32, 64), np.float32) + b = np.zeros((4, 32), np.float32) + Wm = np.asarray(p["W"], np.float32).reshape(4, 17, 64) + bm = np.asarray(p["b"], np.float32).reshape(4, 17) + for s in range(4): + for k, c in enumerate(T.DEC0_VALUES): + W[s, c] = Wm[s, k] + b[s, c] = bm[s, k] + return {"W": W, "b": b, "S": p["S"], "T": p["T"]} + + class BlockTest(unittest.TestCase): def test_block_equals_model_m_oracle(self): """With Model M's constants the L block is clean/reference/swin.py swin_block, byte for byte.""" @@ -100,94 +119,84 @@ def test_model_m_op_list_equals_model_m_encoder(self): x = ME.random_inputs(rng, rw=37, rh=21, reset=reset) p = ME.random_params(rng) want = ME.encode(x, p) - got = L.encode(x, p, t=t) + got = L.encode(x, {"enc0": p}, t=t) for k in ("features", "m_near", "m_far", "m_center"): self.assertTrue(_same(got[k], want[k]), (k, reset)) def test_default_encoder(self): rng = np.random.default_rng(5) x = ME.random_inputs(rng, rw=40, rh=24) - p = L.random_params(rng)["enc0"] - self.assertEqual(set(p), set(T.encoder_shapes())) - for name, shape in T.encoder_shapes().items(): - self.assertEqual(np.shape(p[name]), shape, name) - out = L.encode(x, p, keep=True) + params = L.random_params(rng) + for stage, shapes in T.param_shapes().items(): + self.assertEqual(set(params[stage]), set(shapes), stage) + for name, shape in shapes.items(): + self.assertEqual(np.shape(params[stage][name]), shape, (stage, name)) + out = L.encode(x, params, keep=True) fw, fh = x["feature_size"] self.assertEqual(out["features"].shape, (fh // 2, fw // 2, 64)) self.assertTrue(np.isfinite(out["features"]).all()) self.assertTrue(is_fp8_exact(out["features"])) maps = out["maps"] - self.assertEqual(maps["raw"].shape, (fh // 2, fw // 2, 80)) - self.assertEqual(maps["x"].shape, (fh // 2, fw // 2, 32)) + self.assertEqual(maps["q"].shape, (fh, fw, 20)) + self.assertEqual(maps["x"].shape, (fh, fw, 32)) self.assertEqual(maps["z"].shape, (fh // 2, fw // 2, 128)) - self.assertTrue(np.all(maps["z"][..., 112:] == 0)) - again = L.encode(x, p) - self.assertTrue(_same(again["features"], out["features"])) + self.assertTrue(_same(L.encode(x, params)["features"], out["features"])) def test_parts_by_hand(self): - """linear, select, concat and project of the default op list recomputed directly.""" + """The default op list recomputed step by step.""" rng = np.random.default_rng(6) x = ME.random_inputs(rng, rw=33, rh=17) - p = L.random_params(rng)["enc0"] - out = L.encode(x, p, parity=1, keep=True) + params = L.random_params(rng) + enc = params["enc0"] + out = L.encode(x, params, parity=1, keep=True) m = out["maps"] fw, fh = x["feature_size"] - q = ME.dither_fp8(ME.pixel_values(x, p), ME.dither_table(p["T"], x["frame"], fw, fh)[..., None]) + q = ME.dither_fp8(ME.pixel_values(x, enc), ME.dither_table(enc["T"], x["frame"], fw, fh)[..., None]) self.assertTrue(_same(m["q"], q)) - raw = np.zeros((fh // 2, fw // 2, 80)) + h = q8(np.maximum(L.f16(q @ np.asarray(enc["Win1"], np.float64).T + enc["bin1"]), 0)) + h = q8(L.f16(h @ np.asarray(enc["Win2"], np.float64).T + enc["bin2"])) + h = q8(L.swin_block(h, params["enc0_block"], T.block_offset(T.TABLE, "first", 1), 2, T.M_BLOCK_CONSTS)) + self.assertTrue(_same(m["x"], h)) + z = np.zeros((fh // 2, fw // 2, 128)) for r in range(fh // 2): for c in range(fw // 2): - for pp in range(4): - raw[r, c, 20 * pp:20 * pp + 20] = q[2 * r + pp // 2, 2 * c + pp % 2] - self.assertTrue(_same(m["raw"], raw)) - h = q8(raw[..., :20] @ np.asarray(p["Wa"], np.float64).T + p["ba"]) - h = q8(h @ np.asarray(p["Wb"], np.float64).T + p["bb"]) - bp = {k[4:]: v for k, v in p.items() if k.startswith("blk.")} - h = q8(L.swin_block(h, bp, T.block_offset(T.TABLE, "first", 1), 2, T.M_BLOCK_CONSTS)) - self.assertTrue(_same(m["x"], h)) - z = np.concatenate([raw, h, np.zeros(h.shape[:-1] + (16,))], axis=-1) - W = np.asarray(p["W"], np.float64) - s = np.broadcast_to(np.asarray(p["b"], np.float64), z.shape[:-1] + (64,)) + for p in range(4): + z[r, c, 32 * p:32 * p + 32] = h[2 * r + p // 2, 2 * c + p % 2] + self.assertTrue(_same(m["z"], z)) + W = np.asarray(enc["W"], np.float64) + s = np.broadcast_to(np.asarray(enc["b"], np.float64), z.shape[:-1] + (64,)) for g in range(4): s = L.f16(s + z[..., 32 * g:32 * g + 32] @ W[:, 32 * g:32 * g + 32].T) self.assertTrue(_same(out["features"], q8(np.clip(s, -6.28125, 6.28125)).astype(np.float32))) def test_table_edits(self): - """Provisional entries are table edits: other op orders, a pixel-level pre-network, plain q8 inputs.""" + """Provisional entries are table edits: no ReLU, no FP16 accumulation, plain FP8 quantization, no clamp.""" rng = np.random.default_rng(7) x = ME.random_inputs(rng, rw=24, rh=20) base = T.TABLE["enc_ops"] variants = { - "block_first": (base[0], base[1], base[2], - {**base[5], "C": 20, "heads": 1, "mlp": 80, "prefix": "blk."}, - base[3], base[4], base[6], base[7]), - "pixel_level": ( - base[0], - {**base[3], "src": "q"}, base[4], base[5], - {"op": "space_to_depth", "src": "x", "dst": "z"}, - {"op": "project", "src": "z", "w": "W", "b": "b", "groups": (32, 32, 32, 32), "n_out": 64}), - "plain_q8": ({"op": "quant", "mode": "q8", "src": "values", "dst": "q"},) + tuple(base[1:]), + "no_relu": (base[0], dict(base[1], out_round="q8")) + tuple(base[2:]), + "no_f16": (base[0], dict(base[1], acc=None), dict(base[2], acc=None)) + tuple(base[3:]), + "plain_q8": (dict(base[0], mode="q8"),) + tuple(base[1:]), + "no_clamp": tuple(base[:-1]) + (dict(base[-1], clamp=None),), } - feats = {} + params = L.random_params(np.random.default_rng(8)) + ref = L.encode(x, params)["features"] for name, ops in variants.items(): t = T.table(enc_ops=ops) - plan = T.encoder_plan(t) - p = L.random_params(np.random.default_rng(8), t)["enc0"] - self.assertEqual(set(p), set(T.encoder_shapes(t)), name) - out = L.encode(x, p, t=t) + out = L.encode(x, params, t=t) self.assertTrue(is_fp8_exact(out["features"]), name) - feats[name] = out["features"] - self.assertEqual(plan[-1], ("project", "token", 64)) - self.assertNotIn("T", T.encoder_shapes(T.table(enc_ops=variants["plain_q8"]))) - self.assertEqual(T.encoder_plan(T.table(enc_ops=variants["pixel_level"]))[1], ("linear", "pixel", 32)) + if name != "no_clamp": + self.assertFalse(_same(out["features"], ref), name) + self.assertNotIn("T", T.param_shapes(T.table(enc_ops=variants["plain_q8"]))["enc0"]) def test_plan_refuses_bad_lists(self): base = T.TABLE["enc_ops"] bad = [ - base[:-1], # no project at the end - (base[0], {**base[3], "src": "q", "n_in": 32}) + tuple(base[4:]), # 20 channels into a 32-input linear - (base[0], base[1], {**base[2], "hi": 99}) + tuple(base[3:]), # select outside the map + base[:-1], # no project at the end + (base[0], dict(base[1], n_in=32)) + tuple(base[2:]), # 20 channels into a 32-input linear (base[0], {"op": "frobnicate", "src": "q"}, base[-1]), + tuple(base[:4]) + (base[5],), # project on a pixel-level map ] for ops in bad: with self.assertRaises(ValueError): @@ -195,48 +204,47 @@ def test_plan_refuses_bad_lists(self): class DecoderTest(unittest.TestCase): - def test_no_block_equals_model_m_decoder(self): + def test_model_m_op_list_equals_model_m_decoder(self): + """With Model M's projection rows and no block, the L decoder is Model M's decoder byte for byte.""" rng = np.random.default_rng(9) - t = T.table(dec_connect=(("pass", 0, 64),)) + t = T.table(dec_ops=M_DEC_OPS) for reset in (False, True): x = MD.random_inputs(rng, rw=29, rh=19, reset=reset) p = MD.random_params(rng) want = MD.decode(x, p) - got = L.decode(x, p, 0, t) - for k in ("A", "B", "C", "latent", "codes_A", "codes_B", "codes_L", "h"): + got = L.decode(x, {"dec0": m_dec_as_l(p)}, 0, t) + for k in ("A", "B", "C", "latent", "codes_A", "codes_B", "codes_L"): self.assertTrue(_same(got[k], want[k]), (k, reset)) def test_default_decoder(self): rng = np.random.default_rng(10) x = MD.random_inputs(rng, rw=30, rh=18) - p = L.random_params(rng)["dec0"] - self.assertEqual(set(p), set(T.decoder_shapes())) - out = L.decode(x, p, 1) - front = out["front"] - self.assertTrue(_same(front[..., 32:], x["features"][..., 32:])) - bp = {k[4:]: v for k, v in p.items() if k.startswith("blk.")} - y = q8(L.swin_block(x["features"][..., :32], bp, T.block_offset(T.TABLE, "second", 1), 2, - T.M_BLOCK_CONSTS)).astype(np.float32) - self.assertTrue(_same(front[..., :32], y)) - xx = dict(x) - xx["features"] = front - want = MD.decode(xx, {k: p[k] for k in ("W", "b", "S", "T")}) - self.assertTrue(_same(out["A"], want["A"])) - self.assertTrue(_same(out["latent"], want["latent"])) - - def test_other_segments(self): - rng = np.random.default_rng(11) - x = MD.random_inputs(rng, rw=20, rh=12) - p = L.random_params(rng)["dec0"] - t = T.table(dec_connect=(("pass", 0, 32), ("block", 32, 64))) - out = L.decode(x, p, 0, t) - self.assertTrue(_same(out["front"][..., :32], x["features"][..., :32])) - with self.assertRaises(ValueError): - L.decode(x, p, 0, T.table(dec_connect=(("block", 0, 48), ("pass", 48, 64)))) - with self.assertRaises(NotImplementedError): - L.decode(x, p, 0, T.table(dec_block_level="pixel")) + params = L.random_params(rng) + out = L.decode(x, params, 1, keep=True) + fw, fh = x["feature_size"] + m = out["maps"] + self.assertEqual(m["y"].shape, (fh // 2, fw // 2, 128)) + self.assertEqual(m["p"].shape, (fh, fw, 32)) + self.assertEqual(out["a"].shape, (18, 30, 17)) + y = L.dec_project(x["features"], params["dec0"]["W"], params["dec0"]["b"]) + p = q8(L.swin_block(L.depth_to_space(y), params["dec0_block"], T.block_offset(T.TABLE, "second", 1), 2, + T.M_BLOCK_CONSTS)) + self.assertTrue(_same(m["p"], p)) + self.assertTrue(_same(out["a"], p[:18, :30][..., list(T.DEC0_VALUES)].astype(np.float32))) + for k in ("A", "B", "C", "latent"): + self.assertTrue(np.isfinite(out[k]).all(), k) + + def test_depth_to_space_inverts_space_to_depth(self): + v = np.random.default_rng(11).normal(size=(6, 8, 5)) + self.assertTrue(_same(L.depth_to_space(L.space_to_depth(v)), v)) + + def test_pending(self): with self.assertRaises(NotImplementedError): L.reconstruct() + rng = np.random.default_rng(12) + x = MD.random_inputs(rng, rw=20, rh=12) + with self.assertRaises(NotImplementedError): + L.decode(x, L.random_params(rng), 0, T.table(dec_pixel="other")) class NetworkTest(unittest.TestCase): @@ -248,12 +256,14 @@ def _inputs(self, rng, rw, rh): return xe, xd def test_reduces_to_model_m_chain(self): - """With M's encoder op list and no decoder block, the L network is Model M's oracle chain.""" - rng = np.random.default_rng(12) + """With Model M's encoder and decoder op lists, the L network is Model M's oracle chain.""" + rng = np.random.default_rng(13) xe, xd = self._inputs(rng, 40, 24) - t = T.table(enc_ops=M_ENC_OPS, dec_connect=(("pass", 0, 64),)) + t = T.table(enc_ops=M_ENC_OPS, dec_ops=M_DEC_OPS) p = L.random_params(rng, t) p["enc0"] = ME.random_params(rng) + pm_dec = MD.random_params(rng) + p["dec0"] = m_dec_as_l(pm_dec) for parity in (0, 1): got = L.network(xe, xd, p, parity, t) x = ME.encode(xe, p["enc0"])["features"] @@ -273,12 +283,12 @@ def test_reduces_to_model_m_chain(self): self.assertTrue(_same(got[s][key], x), (s, parity)) xx = dict(xd) xx["features"] = x - want = MD.decode(xx, p["dec0"]) + want = MD.decode(xx, pm_dec) for k in ("A", "B", "C", "latent"): self.assertTrue(_same(got["dec0"][k], want[k]), (k, parity)) def test_default_network_runs(self): - rng = np.random.default_rng(13) + rng = np.random.default_rng(14) xe, xd = self._inputs(rng, 36, 20) p = L.random_params(rng) out = L.network(xe, xd, p, 0) @@ -292,6 +302,16 @@ class TableTest(unittest.TestCase): def test_shared_stages_are_model_m(self): self.assertEqual(tuple(T.SHARED_STAGES), tuple(k for k in MS.STAGES)) + def test_param_shapes_are_the_maps(self): + """Model L's own stages with the shapes of the preset-L converter map's records.""" + blk = {"gamma1": (32,), "gamma2": (32,), "Wqk": (2, 32, 32), "Wv": (2, 32, 32), "B": (2, 16, 16), + "Wp": (32, 64), "bp": (32,), "W1": (128, 32), "b1": (128,), "W2": (32, 128), "b2": (32,)} + want = {"enc0": {"W": (64, 128), "b": (64,), "T": (8, 16, 16), "sigma1": (), "sigma2": (), + "Win1": (32, 20), "bin1": (32,), "Win2": (32, 32), "bin2": (32,)}, + "enc0_block": blk, "dec0_block": blk, + "dec0": {"W": (4, 32, 64), "b": (4, 32), "S": (), "T": ()}, "post": {"s": ()}} + self.assertEqual(T.param_shapes(), want) + def test_macs(self): self.assertEqual(T.block_macs_per_token(64, 2, 256), 47104) self.assertEqual(T.block_macs_per_token(96, 4, 384), 114688) @@ -299,15 +319,12 @@ def test_macs(self): self.assertEqual(T.block_macs_per_token(32, 2, 128), 16384) r = T.frame_macs(T.TABLE, 1706, 960) self.assertEqual(r["M"], 103089438720) - self.assertGreater(r["L"], r["M"]) - - def test_provisional_keys_cover_entries(self): - self.assertEqual(set(T.PROVISIONAL), {f"I{i}" for i in range(1, 11)}) + self.assertEqual(round(r["L"] / 1e9), 163) def test_table_copy(self): - t = T.table(recon_ratio=(3, 1)) - self.assertEqual(t["recon_ratio"], (3, 1)) - self.assertEqual(T.TABLE["recon_ratio"], (3, 2)) + t = T.table(dec_pixel="x") + self.assertEqual(t["dec_pixel"], "x") + self.assertEqual(T.TABLE["dec_pixel"], "m") with self.assertRaises(KeyError): T.table(nonsense=1)