diff --git a/clean/l/README.md b/clean/l/README.md index 536e6858..b29177cc 100644 --- a/clean/l/README.md +++ b/clean/l/README.md @@ -5,8 +5,8 @@ launcher yet. | Path | Content | |---|---| -| `reference/table.py` | Model L's structure table: the encoder's op list, the decoder's channel segments and block, window offsets, block constants, map names; parameter shapes from the table; matrix work per frame (computed, not measured). Entries in `PROVISIONAL` are pending confirmation and are changed here only | -| `reference/oracle.py` | NumPy oracle (float64, only the prescribed FP8 / FP16 roundings): `encode`, `decode`, `network`, synthetic `random_params`. The Swin stages it shares with Model M call Model M's oracle (`clean/reference/swin.py`); the output reconstruction is not written yet | +| `reference/table.py` | Model L's structure table: the encoder's and the decoder's op lists (pixel-level linears and width-32 block, the 2 x 2 gather projection; the decoder projection, its width-32 block and the value positions), window offsets, block constants; parameter shapes per converter-map stage; matrix work per frame (computed, not measured). Entries in `PROVISIONAL` are pending confirmation and are changed here only | +| `reference/oracle.py` | NumPy oracle (float64, only the prescribed FP8 / FP16 roundings): `encode`, `decode`, `network`, synthetic `random_params`, parameters grouped by the converter map's stages; the decoder's coefficient functions are Model M's. The Swin stages it shares with Model M call Model M's oracle (`clean/reference/swin.py`); the output reconstruction is not written yet | | `runner/l_pack.py` | weight packing of Model L's own launches: the encoder's launch list derived from the table (`encoder_launches`) and one blob per launch (`pack_model_l`), each in a format Model M already uses (merge / expand section, Swin block section, enc0 and dec0 blobs); decoders for tests. The ten shared stages use `clean/runner/packing.py` unchanged | | `tests/test_l_pack.py` | packing round trips, the block packer against Model M's stage blobs, and the unpacked weights applied the kernels' way against the oracle's maps, byte for byte | | `tests/test_l_reference.py` | self-checks on synthetic data; the main ones reduce Model L to Model M through the table and require Model M's oracle results byte for byte | diff --git a/clean/l/reference/oracle.py b/clean/l/reference/oracle.py index 1d5fd933..21350286 100644 --- a/clean/l/reference/oracle.py +++ b/clean/l/reference/oracle.py @@ -2,9 +2,11 @@ """Model L NumPy oracle (issue #89): float64 arithmetic with only the prescribed roundings (q8, f16), as Model M's oracle in clean/reference. Model L's structure comes from clean/l/reference/table.py; nothing here hard-codes it. - enc = encode(x, params["enc0"], parity) # x: Model M's enc0 input dict (clean/reference/encoder.py) + enc = encode(x, params, parity) # x: Model M's enc0 input dict (clean/reference/encoder.py) out = network(x_enc, x_dec, params, parity) # encoder -> enc1 ... dec1 (Model M's stages) -> decoder - dec = decode(x, params["dec0"], parity) # x: Model M's dec0 input dict (features = dec1's output) + dec = decode(x, params, parity) # x: Model M's dec0 input dict (features = dec1's output) + +params: {stage: tensors} with the converter map's stages (enc0, enc0_block, enc1 ... dec1, dec0_block, dec0, post). The Swin stages shared with Model M call clean/reference/swin.py, so for them this oracle is Model M's oracle. """ @@ -108,11 +110,6 @@ def swin_block(x, p, offset, heads, consts, max_tokens=1 << 16): return out -def _block_params(params, prefix): - n = len(prefix) - return {k[n:]: v for k, v in params.items() if k.startswith(prefix)} - - # ---- encoder -------------------------------------------------------------------------------------------------------- def space_to_depth(v): @@ -122,9 +119,19 @@ def space_to_depth(v): return np.concatenate([a[:, p // 2, :, p % 2, :] for p in range(4)], axis=-1) +def depth_to_space(v): + """Inverse of space_to_depth: [h, w, 4n] -> [2h, 2w, n].""" + h, w, c = v.shape + n = c // 4 + out = np.empty((2 * h, 2 * w, n), dtype=np.asarray(v).dtype) + for p in range(4): + out[p // 2::2, p % 2::2, :] = v[..., n * p:n * (p + 1)] + return out + + def project_groups(z, W, b, groups, clamp=_enc.FEATURE_CLAMP): - """Running sum s = b; for each channel group g: s = FP16(s + W[:, g] z[g]) (exact group sums); then clamp and - round to FP8. With groups (20, 20, 20, 20) this is clean/reference/encoder.py project.""" + """Running sum s = b; for each channel group g: s = FP16(s + W[:, g] z[g]) (exact group sums); then clamp (None: + no clamp) and round to FP8. With groups (20, 20, 20, 20) this is clean/reference/encoder.py project.""" W = np.asarray(W, dtype=np.float64) z = np.asarray(z, dtype=np.float64) s = np.broadcast_to(np.asarray(b, dtype=np.float64), z.shape[:-1] + (W.shape[0],)) @@ -132,26 +139,38 @@ def project_groups(z, W, b, groups, clamp=_enc.FEATURE_CLAMP): for g in groups: s = f16(s + z[..., lo:lo + g] @ W[:, lo:lo + g].T) lo += g - return q8(np.clip(s, -clamp, clamp)) + if clamp is not None: + s = np.clip(s, -clamp, clamp) + return q8(s) -def encode(x, params, parity=None, t=_table.TABLE, keep=False): - """Model L's input encoder. x: Model M's enc0 input dict; params: logical names of table.encoder_shapes(t). - Returns Model M's enc0 outputs ("features" [fh/2, fw/2, 64] FP8-exact, motion surfaces) and, with keep=True, every - named map ("maps").""" - _table.encoder_plan(t) - if parity is None: - parity = int(x["frame"]) & 1 - rw, rh, fw, fh, dw, dh = _enc.check_inputs(x) - pv_params = {"sigma1": params.get("sigma1", 0.0), "sigma2": params.get("sigma2", 0.0)} - maps = {"values": _enc.pixel_values(x, pv_params)} - feats = None - for op in t["enc_ops"]: +def _out_round(y, mode): + if mode == "relu_q8": + return q8(np.maximum(y, 0.0)) + return _round(y, mode) + + +def linear(x, W, b, acc=None, out_round="q8"): + """out_round(acc(W x + b)) with exact sums; acc "f16" rounds the sum to FP16 first.""" + y = np.asarray(x, dtype=np.float64) @ np.asarray(W, dtype=np.float64).T + np.asarray(b, dtype=np.float64) + if acc == "f16": + y = f16(y) + elif acc not in (None, "none"): + raise ValueError(f"unknown accumulation {acc!r}") + return _out_round(y, out_round) + + +def _run_ops(ops, maps, params, parity, t, default_stage, frame=None): + """Interprets an op list over named maps; returns the last op's output.""" + last = None + for op in ops: k = op["op"] + st = params[op.get("params", default_stage)] if k == "quant": v = maps[op["src"]] + fw, fh = frame["feature_size"] if op["mode"] == "dither": - d = _enc.dither_table(params["T"], x["frame"], fw, fh) + d = _enc.dither_table(st["T"], frame["frame"], fw, fh) out = _enc.dither_fp8(v, d[..., None]) elif op["mode"] == "q8": out = q8(v) @@ -159,16 +178,16 @@ def encode(x, params, parity=None, t=_table.TABLE, keep=False): raise ValueError(f"unknown quant mode {op['mode']!r}") elif k == "space_to_depth": out = space_to_depth(maps[op["src"]]) + elif k == "depth_to_space": + out = depth_to_space(maps[op["src"]]) elif k == "select": out = maps[op["src"]][..., op["lo"]:op["hi"]] elif k == "linear": - W = np.asarray(params[op["w"]], dtype=np.float64) - b = np.asarray(params[op["b"]], dtype=np.float64) - out = _round(maps[op["src"]] @ W.T + b, op.get("out_round", "q8")) + out = linear(maps[op["src"]], st[op["w"]], st[op["b"]], op.get("acc"), op.get("out_round", "q8")) elif k == "swin": - consts = _table.block_consts(t, op.get("name", "enc0." + op["prefix"])) + consts = _table.block_consts(t, op["params"]) off = _table.block_offset(t, op["offsets"], parity) - y = swin_block(maps[op["src"]], _block_params(params, op["prefix"]), off, op["heads"], consts) + y = swin_block(maps[op["src"]], st, off, op["heads"], consts) out = _round(y, op.get("out_round", "q8")) elif k == "concat": parts = [maps[s] for s in op["srcs"]] @@ -178,9 +197,30 @@ def encode(x, params, parity=None, t=_table.TABLE, keep=False): parts.append(np.zeros(parts[0].shape[:-1] + (pad - c,))) out = np.concatenate(parts, axis=-1) elif k == "project": - out = project_groups(maps[op["src"]], params[op["w"]], params[op["b"]], op["groups"]) - feats = out - maps[op.get("dst", "features")] = out + out = project_groups(maps[op["src"]], st[op["w"]], st[op["b"]], op["groups"], op.get("clamp")) + elif k == "dec_project": + out = dec_project(maps[op["src"]], st[op["w"]], st[op["b"]], op["groups"], op.get("out_round", "q8")) + elif k == "pick": + out = np.asarray(maps[op["src"]])[..., list(op["values"])] + else: + raise ValueError(f"unknown op {k!r}") + maps[op.get("dst", "out")] = out + last = out + return last + + +def encode(x, params, parity=None, t=_table.TABLE, keep=False): + """Model L's input encoder. x: Model M's enc0 input dict; params: {stage: tensors} with the stages of + table.param_shapes(t) ("enc0", "enc0_block"). Returns Model M's enc0 outputs ("features" [fh/2, fw/2, 64] + FP8-exact, motion surfaces) and, with keep=True, every named map ("maps").""" + _table.encoder_plan(t) + if parity is None: + parity = int(x["frame"]) & 1 + _enc.check_inputs(x) + enc = params["enc0"] + pv_params = {"sigma1": enc.get("sigma1", 0.0), "sigma2": enc.get("sigma2", 0.0)} + maps = {"values": _enc.pixel_values(x, pv_params)} + feats = _run_ops(t["enc_ops"], maps, params, parity, t, "enc0", frame=x) m_near, m_far, m_center = _enc.motion_surfaces(x) res = {"features": feats.astype(np.float32), "m_near": f16(m_near).astype(np.float32), @@ -193,41 +233,49 @@ def encode(x, params, parity=None, t=_table.TABLE, keep=False): # ---- decoder -------------------------------------------------------------------------------------------------------- -def decoder_front(features, params, parity, t=_table.TABLE): - """dec1's FP8 map [fh/2, fw/2, 64] -> the 64-channel map Model M's decoder projects (table entry dec_connect).""" - Z = np.asarray(features, dtype=np.float64) - segs = t["dec_connect"] - blk = [s for s in segs if s[0] == "block"] - if not blk: - return Z - if t["dec_block_level"] != "token": - raise NotImplementedError("decoder block level other than token: pending") - b = t["dec_block"] - lo = min(s[1] for s in blk) - hi = max(s[2] for s in blk) - if sum(s[2] - s[1] for s in blk) != hi - lo or hi - lo != b["C"]: - raise ValueError("block segments must be one contiguous range of the block's width") - consts = _table.block_consts(t, b.get("name", "dec0." + b["prefix"])) - off = _table.block_offset(t, b["offsets"], parity) - y = _round(swin_block(Z[..., lo:hi], _block_params(params, b["prefix"]), off, b["heads"], consts), - b.get("out_round", "q8")) - out = Z.copy() - for kind, s0, s1 in segs: - if kind == "block": - out[..., s0:s1] = y[..., s0 - lo:s1 - lo] - elif kind != "pass": - raise ValueError(f"unknown decoder segment {kind!r}") - return out - - -def decode(x, params, parity, t=_table.TABLE): - """Model L's coefficient decoder: decoder_front, then Model M's decoder (table entry dec_pixel = "m").""" +def dec_project(Z, W, b, groups=(32, 32), out_round="q8"): + """Z [h, w, 64], W [4, 32, 64], b [4, 32] -> [h, w, 128] (channel 32 s + c): Model M's dec0 projection arithmetic + (clean/reference/decoder.py project) over all 4 x 32 rows.""" + Z = np.asarray(Z, dtype=np.float64) + W = np.asarray(W, dtype=np.float64).reshape(128, -1) + s = np.broadcast_to(np.asarray(b, dtype=np.float64).reshape(128), Z.shape[:-1] + (128,)) + lo = 0 + for g in groups: + s = f16(s + Z[..., lo:lo + g] @ W[:, lo:lo + g].T) + lo += g + return _round(s, out_round) + + +def coefficients(x, a, S, T): + """Model M's coefficient and latent functions (clean/reference/decoder.py decode after its projection) on the + per-pixel values a [rh, rw, 17].""" + rw, rh, fw, fh, dw, dh = _dec.check_inputs(x) + a = np.asarray(a, dtype=np.float32) + A = _dec.sigmoid16(a[..., 0:3]) + B = _dec.softmax3(a[..., 3:6]) + C = _dec.coefficient_c(a[..., 6], a[..., 7], a[..., 8], float(np.asarray(S)), float(np.asarray(T))) + p = _dec.previous_latent(x, rw, rh, dw, dh) + lat = _dec.latent_update(a, p) + cA, cB, cL = _dec.unorm8_code(A), _dec.unorm8_code(B), _dec.snorm8_code(lat) + return {"A": (cA / 255.0).astype(np.float32), "B": (cB / 255.0).astype(np.float32), + "C": C.astype(np.float32), "latent": (cL / 127.0).astype(np.float32), + "codes_A": cA, "codes_B": cB, "codes_L": cL, "p": p} + + +def decode(x, params, parity, t=_table.TABLE, keep=False): + """Model L's coefficient decoder. x: Model M's dec0 input dict (features = dec1's output [fh/2, fw/2, 64]); + params: {stage: tensors} ("dec0", "dec0_block").""" if t["dec_pixel"] != "m": raise NotImplementedError("decoder per-pixel variant other than Model M's: pending") - xx = dict(x) - xx["features"] = decoder_front(x["features"], params, parity, t).astype(np.float32) - out = _dec.decode(xx, {k: params[k] for k in ("W", "b", "S", "T")}) - out["front"] = xx["features"] + _table.decoder_plan(t) + rw, rh = _dec.check_inputs(x)[:2] + maps = {"features": np.asarray(x["features"], dtype=np.float64)} + a = _run_ops(t["dec_ops"], maps, params, parity, t, "dec0") + dec = params["dec0"] + out = coefficients(x, a[:rh, :rw], dec["S"], dec["T"]) + out["a"] = a[:rh, :rw].astype(np.float32) + if keep: + out["maps"] = maps return out @@ -241,9 +289,9 @@ def network(x_enc, x_dec, params, parity, t=_table.TABLE, keep=False): """Encoder -> the ten shared Swin stages (Model M's oracle, L's weights) -> decoder. x_enc: Model M's enc0 input dict. x_dec: Model M's dec0 input dict without "features" (motion, latent_prev, - jitter, jitter_prev, reset, feature_size, recon_size). params: {"enc0": ..., "enc1": ..., ..., "dec0": ...}. + jitter, jitter_prev, reset, feature_size, recon_size). params: {stage: tensors} for every stage. Returns {"enc0": ..., stage outputs ..., "dec0": ...}.""" - res = {"enc0": encode(x_enc, params["enc0"], parity, t, keep)} + res = {"enc0": encode(x_enc, params, parity, t, keep)} x = res["enc0"]["features"] skips = {} for s in _table.SHARED_STAGES: @@ -263,7 +311,7 @@ def network(x_enc, x_dec, params, parity, t=_table.TABLE, keep=False): res[s] = o xd = dict(x_dec) xd["features"] = x - res["dec0"] = decode(xd, params["dec0"], parity, t) + res["dec0"] = decode(xd, params, parity, t, keep) return res @@ -288,28 +336,33 @@ def random_block_params(rng, C, heads, mlp, prefix="", weight_scale=1.0): def random_params(rng, t=_table.TABLE): - """Synthetic parameters for every stage of Model L (shapes from the table and Model M's stages).""" - enc = {} - for name, shape in _table.encoder_shapes(t).items(): - if name == "T": - enc[name] = rng.integers(0, 128, size=shape).astype(np.float32) - elif name in ("sigma1", "sigma2"): - enc[name] = np.array(rng.uniform(0.5, 1.5), dtype=np.float32) - for op in t["enc_ops"]: - if op["op"] == "linear": - enc[op["w"]] = q8(rng.normal(0, 1.0 / math.sqrt(op["n_in"]), (op["n_out"], op["n_in"]))).astype(np.float32) - enc[op["b"]] = f16(rng.normal(0, 0.1, op["n_out"])).astype(np.float32) - elif op["op"] == "swin": - enc.update(random_block_params(rng, op["C"], op["heads"], op["mlp"], op["prefix"])) - elif op["op"] == "project": - n = sum(op["groups"]) - enc[op["w"]] = q8(rng.normal(0, 0.6 / math.sqrt(n), (op["n_out"], n))).astype(np.float32) - enc[op["b"]] = f16(rng.normal(0, 0.3, op["n_out"])).astype(np.float32) - dec = _dec.random_params(rng) - if any(s[0] == "block" for s in t["dec_connect"]): - b = t["dec_block"] - dec.update(random_block_params(rng, b["C"], b["heads"], b["mlp"], b["prefix"])) - p = {"enc0": enc, "dec0": dec} + """Synthetic parameters for every stage of Model L ({stage: tensors}; shapes from the table and Model M's + stages). Never model data.""" + p = {} + for stage, sh in _table.param_shapes(t).items(): + st = {} + if "Wqk" in sh: + st.update(random_block_params(rng, sh["gamma1"][0], sh["Wqk"][0], sh["W1"][0])) + for name, shape in sh.items(): + if name in st: + continue + if stage == "enc0" and name == "T": + st[name] = rng.integers(0, 128, size=shape).astype(np.float32) + elif name in ("sigma1", "sigma2"): + st[name] = np.array(rng.uniform(0.5, 1.5), dtype=np.float32) + elif stage == "dec0" and name == "S": + st[name] = np.array(rng.uniform(1.0, 4.0), dtype=np.float32) + elif stage == "dec0" and name == "T": + st[name] = np.array(rng.uniform(0.2, 1.5), dtype=np.float32) + elif name == "s": + st[name] = np.array(rng.uniform(0.5, 2.0), dtype=np.float32) + elif len(shape) >= 2: + fan_in = shape[-1] + scale = {"enc0": 0.6 if name == "W" else 1.0, "dec0": 0.25 * np.sqrt(fan_in)}.get(stage, 1.0) + st[name] = q8(rng.normal(0, scale / np.sqrt(fan_in), shape)).astype(np.float32) + else: + st[name] = f16(rng.normal(0, 0.3 if stage == "dec0" else 0.1, shape)).astype(np.float32) + p[stage] = st for s in _table.SHARED_STAGES: p[s] = {k: np.asarray(v, dtype=np.float32) for k, v in _swin.random_params(s, rng).items()} return p diff --git a/clean/l/reference/table.py b/clean/l/reference/table.py index 2c3d5ce6..18801492 100644 --- a/clean/l/reference/table.py +++ b/clean/l/reference/table.py @@ -3,9 +3,11 @@ runtime and the tests. Every consumer interprets this table instead of hard-coding L's structure, so a provisional entry can be changed -without touching code. Entries listed in PROVISIONAL are pending confirmation; each carries a short key (I1 ...). +without touching code. Entries listed in PROVISIONAL are pending confirmation; each carries a short key (I5 ...). -SHARED_STAGES run with Model M's code and stage table (clean/reference/swin.py STAGES) and L's own weights. +Parameters are grouped by the converter map's stages (data/preset-L.v1.map.json): "enc0", "enc0_block", the ten +shared stages, "dec0_block", "dec0", "post". SHARED_STAGES run with Model M's code and stage table +(clean/reference/swin.py STAGES) and L's own weights. """ import copy @@ -21,6 +23,10 @@ M_BLOCK_CONSTS = {"tau": _swin.TAU, "cap": _swin.CAP, "eps_num": 2.0 ** -13, "gelu": "exact", "window": _swin.WIN, "head_dim": _swin.HEAD_DIM} +# The value positions of the coefficient decoder (the map's perm "dec0_values"): value k of a render pixel is +# channel DEC0_VALUES[k] of its 32. +DEC0_VALUES = (0, 1, 10, 11, 12, 13, 14, 15, 16, 2, 3, 4, 5, 6, 7, 8, 9) + # ---- the table ------------------------------------------------------------------------------------------------------ C32_BLOCK = {"C": 32, "heads": 2, "mlp": 128} @@ -29,47 +35,54 @@ # Input encoder: ops applied in order. Each op reads and writes named maps; "values" is the per-pixel input # (clean/reference/encoder.py pixel_values: [fh, fw, 20] on the padded grid F). A map's level follows from its # size: the pixel grid F, or the token grid F/2 after space_to_depth (2 x 2 pixels, sub-pixel p = dx + 2 dy, - # channel = n * p + i). The last op must be "project" (the 64 FP8 features enc1 reads). + # channel = n p + i). Tensors are read from the stage named by "params" (default "enc0"). A linear's "acc" is the + # rounding of bias + products before "out_round" ("relu_q8" = q8(max(0, .))). The last op is "project" (the 64 FP8 + # features enc1 reads): running sum s = b, s = RN16(s + group) per channel group, clamp, FP8. "enc_ops": ( - {"op": "quant", "mode": "dither", "src": "values", "dst": "q"}, # I5 - {"op": "space_to_depth", "src": "q", "dst": "raw"}, # I2 - {"op": "select", "src": "raw", "dst": "x", "lo": 0, "hi": 20}, # I2 - {"op": "linear", "src": "x", "dst": "x", "w": "Wa", "b": "ba", "n_in": 20, "n_out": 32, - "out_round": "q8"}, # I1 - {"op": "linear", "src": "x", "dst": "x", "w": "Wb", "b": "bb", "n_in": 32, "n_out": 32, - "out_round": "q8"}, # I1 - {"op": "swin", "src": "x", "dst": "x", "prefix": "blk.", **C32_BLOCK, "offsets": "first", - "out_round": "q8"}, # I1, I6, I10 - {"op": "concat", "srcs": ("raw", "x"), "dst": "z", "pad_to": 128}, # I2 - {"op": "project", "src": "z", "w": "W", "b": "b", "groups": (32, 32, 32, 32), "n_out": 64}, # I2 + {"op": "quant", "mode": "dither", "src": "values", "dst": "q"}, # I5 + {"op": "linear", "src": "q", "dst": "x", "w": "Win1", "b": "bin1", "n_in": 20, "n_out": 32, + "acc": "f16", "out_round": "relu_q8"}, # I11 + {"op": "linear", "src": "x", "dst": "x", "w": "Win2", "b": "bin2", "n_in": 32, "n_out": 32, + "acc": "f16", "out_round": "q8"}, # I11 + {"op": "swin", "src": "x", "dst": "x", "params": "enc0_block", **C32_BLOCK, "offsets": "first", + "out_round": "q8"}, # I10, I6 + {"op": "space_to_depth", "src": "x", "dst": "z"}, + {"op": "project", "src": "z", "w": "W", "b": "b", "groups": (32, 32, 32, 32), "n_out": 64, + "clamp": 6.28125}, # I12 + ), + # Coefficient decoder: ops on dec1's map ("features", [fh/2, fw/2, 64]). "dec_project": rows W [4, 32, 64] with + # Model M's dec0 rounding (RN16 after each group of 32 input channels, then FP8), output [fh/2, fw/2, 4 x 32] + # (sub-pixel s = dx + 2 dy, channel 32 s + c); "depth_to_space" makes it a pixel map [fh, fw, 32]; "pick" takes + # the value positions of the coefficient functions, which then run as Model M's (clean/reference/decoder.py). + "dec_ops": ( + {"op": "dec_project", "src": "features", "dst": "y", "w": "W", "b": "b", "groups": (32, 32), + "out_round": "q8"}, # I13 + {"op": "depth_to_space", "src": "y", "dst": "p"}, + {"op": "swin", "src": "p", "dst": "p", "params": "dec0_block", **C32_BLOCK, "offsets": "second", + "out_round": "q8"}, # I10, I14 + {"op": "pick", "src": "p", "values": DEC0_VALUES}, # I14 ), - # Coefficient decoder: dec1's 64 channels split into segments; "block" segments go through one block, "pass" - # segments are kept; the result (same channel order) feeds Model M's decoder (clean/reference/decoder.py). - "dec_connect": (("block", 0, 32), ("pass", 32, 64)), # I4 - "dec_block": {"prefix": "blk.", **C32_BLOCK, "offsets": "second", "out_round": "q8"}, # I6, I10 - "dec_block_level": "token", # I3 - "dec_pixel": "m", # I8 - "post_variant": "unfolded", # I8 - "recon_ratio": (3, 2), # I7 - # Window offsets (ox, oy) of an L block for frame parity 0 and 1. - "window_offsets": {"first": ((2, 2), (0, 2)), "second": ((0, 0), (2, 0))}, # I10 - # Per-block constants; a block's own entry wins over "default". - "consts": {"default": dict(M_BLOCK_CONSTS)}, # I6 - # Logical tensor name -> converter map name, per stage ("enc0", "dec0"); identity when absent. - "map_names": {}, # I9 + "dec_pixel": "m", # I8 + "post_variant": "unfolded", # I8 + # Reconstruction variants and the rule that picks one per frame from the render and output sizes ("rule" names + # the comparison, "ratio" its factor). Only the 3/2 form (D = floor((3 O + 1) / 2)) exists in WrongGPU so far. + "recon": {"rule": "both_axes_at_least", "ratio": 3, "variants": ("3/2", "3/1")}, # I7 + # Window offsets (ox, oy) of an L block for frame parity 0 and 1; the blocks run on the padded grid F. + "window_offsets": {"first": ((2, 2), (0, 2)), "second": ((0, 0), (2, 0))}, # I10 + # Per-block constants; a block's own entry (key = its params stage) wins over "default". + "consts": {"default": dict(M_BLOCK_CONSTS)}, # I6 } PROVISIONAL = { - "I1": "enc_ops: order and kind of the ops", - "I2": "enc_ops: levels, select / concat segments, project groups", - "I3": "dec_block_level", - "I4": "dec_connect", - "I5": "enc_ops: quant mode", - "I6": "consts", - "I7": "recon_ratio", - "I8": "dec_pixel, post_variant", - "I9": "map_names", - "I10": "window_offsets", + "I5": "enc_ops quant mode (dither table or plain FP8 rounding)", + "I6": "consts of the width-32 blocks", + "I7": "recon: the 3/1 form and the exact rule", + "I8": "dec_pixel (coefficient functions as Model M's), post_variant", + "I10": "window_offsets of the width-32 blocks, and their grid (F)", + "I11": "acc / out_round of the input linears", + "I12": "project clamp and group structure", + "I13": "dec_project rounding", + "I14": "pick value positions and the rounding of the decoder block's output", } @@ -93,10 +106,6 @@ def block_offset(t, kind, parity): return tuple(t["window_offsets"][kind][int(parity) & 1]) -def map_name(t, stage, logical): - return t["map_names"].get(stage, {}).get(logical, logical) - - # ---- parameter shapes ----------------------------------------------------------------------------------------------- def block_shapes(C, heads, mlp, head_dim=32, window=4): @@ -106,84 +115,99 @@ def block_shapes(C, heads, mlp, head_dim=32, window=4): "W2": (C, mlp), "b2": (C,)} -def encoder_shapes(t=TABLE): - """Tensor shapes of the L encoder stage, logical names.""" - shapes = {} +def param_shapes(t=TABLE): + """{stage: {tensor: shape}} of Model L's own stages (enc0, its block, dec0, its block, post).""" + out = {"enc0": {}, "dec0": {"S": (), "T": ()}, "post": {"s": ()}} for op in t["enc_ops"]: + st = out.setdefault(op.get("params", "enc0"), {}) if op["op"] == "quant": - shapes.update({"sigma1": (), "sigma2": ()}) + st.update({"sigma1": (), "sigma2": ()}) if op["mode"] == "dither": - shapes["T"] = (8, 16, 16) + st["T"] = (8, 16, 16) elif op["op"] == "linear": - shapes.update({op["w"]: (op["n_out"], op["n_in"]), op["b"]: (op["n_out"],)}) + st.update({op["w"]: (op["n_out"], op["n_in"]), op["b"]: (op["n_out"],)}) elif op["op"] == "swin": - for k, s in block_shapes(op["C"], op["heads"], op["mlp"]).items(): - shapes[op["prefix"] + k] = s + st.update(block_shapes(op["C"], op["heads"], op["mlp"])) elif op["op"] == "project": - shapes.update({op["w"]: (op["n_out"], sum(op["groups"])), op["b"]: (op["n_out"],)}) - return shapes + st.update({op["w"]: (op["n_out"], sum(op["groups"])), op["b"]: (op["n_out"],)}) + for op in t["dec_ops"]: + st = out.setdefault(op.get("params", "dec0"), {}) + if op["op"] == "dec_project": + st.update({op["w"]: (4, 32, sum(op["groups"])), op["b"]: (4, 32)}) + elif op["op"] == "swin": + st.update(block_shapes(op["C"], op["heads"], op["mlp"])) + return out def encoder_plan(t=TABLE): - """(level, channels) of every named map after each op, from the op list alone: [(op, level, channels), ...].""" - maps = {"values": ("pixel", 20)} + """(op, level, channels) after each encoder op, from the op list alone.""" + return _plan(t["enc_ops"], {"values": ("pixel", 20)}, "encoder", last="project") + + +def decoder_plan(t=TABLE): + """(op, level, channels) after each decoder op, from the op list alone.""" + return _plan(t["dec_ops"], {"features": ("token", 64)}, "decoder", last="pick") + + +def _plan(ops, maps, what, last): plan = [] - for op in t["enc_ops"]: + for op in ops: k = op["op"] + src = maps.get(op.get("src")) + if k != "concat" and src is None: + raise ValueError(f"{what}: {k} reads unknown map {op.get('src')!r}") if k == "quant": - out = maps[op["src"]] + out = src elif k == "space_to_depth": - lvl, c = maps[op["src"]] - if lvl != "pixel": + if src[0] != "pixel": raise ValueError("space_to_depth needs a pixel-level map") - out = ("token", 4 * c) + out = ("token", 4 * src[1]) + elif k == "depth_to_space": + if src[0] != "token" or src[1] % 4: + raise ValueError("depth_to_space needs a token-level map of 4 n channels") + out = ("pixel", src[1] // 4) elif k == "select": - lvl, c = maps[op["src"]] - if not 0 <= op["lo"] < op["hi"] <= c: - raise ValueError(f"select {op['lo']}..{op['hi']} outside {c} channels") - out = (lvl, op["hi"] - op["lo"]) + if not 0 <= op["lo"] < op["hi"] <= src[1]: + raise ValueError(f"select {op['lo']}..{op['hi']} outside {src[1]} channels") + out = (src[0], op["hi"] - op["lo"]) elif k == "linear": - lvl, c = maps[op["src"]] - if c != op["n_in"]: - raise ValueError(f"linear {op['w']}: input has {c} channels, op wants {op['n_in']}") - out = (lvl, op["n_out"]) + if src[1] != op["n_in"]: + raise ValueError(f"linear {op['w']}: input has {src[1]} channels, op wants {op['n_in']}") + out = (src[0], op["n_out"]) elif k == "swin": - lvl, c = maps[op["src"]] - if c != op["C"]: - raise ValueError(f"swin: input has {c} channels, block width {op['C']}") - out = (lvl, c) + if src[1] != op["C"]: + raise ValueError(f"swin: input has {src[1]} channels, block width {op['C']}") + out = src elif k == "concat": - lv = {maps[s][0] for s in op["srcs"]} - if len(lv) != 1: + parts = [maps[s] for s in op["srcs"]] + if len({p[0] for p in parts}) != 1: raise ValueError("concat of maps at different levels") - c = sum(maps[s][1] for s in op["srcs"]) + c = sum(p[1] for p in parts) pad = op.get("pad_to", c) if pad < c: raise ValueError("concat: pad_to below the channel count") - out = (lv.pop(), pad) + out = (parts[0][0], pad) elif k == "project": - lvl, c = maps[op["src"]] - if lvl != "token" or c != sum(op["groups"]): - raise ValueError(f"project: needs a token-level map of {sum(op['groups'])} channels, got {lvl} {c}") + if src[0] != "token" or src[1] != sum(op["groups"]): + raise ValueError(f"project: needs a token-level map of {sum(op['groups'])} channels, got {src}") out = ("token", op["n_out"]) + elif k == "dec_project": + if src[0] != "token" or src[1] != sum(op["groups"]): + raise ValueError(f"dec_project: needs a token-level map of {sum(op['groups'])} channels, got {src}") + out = ("token", 128) + elif k == "pick": + if src[0] != "pixel" or max(op["values"]) >= src[1]: + raise ValueError("pick: needs a pixel-level map holding every value position") + out = ("pixel", len(op["values"])) else: - raise ValueError(f"unknown encoder op {k!r}") - maps[op.get("dst", "features")] = out + raise ValueError(f"unknown {what} op {k!r}") + maps[op.get("dst", "out")] = out plan.append((k, out[0], out[1])) - if not plan or plan[-1][0] != "project": - raise ValueError("the encoder's last op must be project") + if not plan or plan[-1][0] != last: + raise ValueError(f"the {what}'s last op must be {last}") return plan -def decoder_shapes(t=TABLE): - shapes = {"W": (68, 64), "b": (68,), "S": (), "T": ()} - if any(seg[0] == "block" for seg in t["dec_connect"]): - b = t["dec_block"] - for k, s in block_shapes(b["C"], b["heads"], b["mlp"]).items(): - shapes[b["prefix"] + k] = s - return shapes - - # ---- work per frame (computed, not measured) ------------------------------------------------------------------------ def block_macs_per_token(C, heads, mlp, head_dim=32, window=4): @@ -212,19 +236,22 @@ def frame_macs(t, render_w, render_h): m_enc0 = 80 * 64 * n["token"] m_dec0 = 64 * 68 * n["token"] M = sum(parts.values()) + m_enc0 + m_dec0 - enc = 0 - for op, (_, lvl, _c) in zip(t["enc_ops"], encoder_plan(t)): - if op["op"] == "linear": - enc += op["n_in"] * op["n_out"] * n[lvl] - elif op["op"] == "swin": - enc += block_macs_per_token(op["C"], op["heads"], op["mlp"]) * n[lvl] - elif op["op"] == "project": - enc += sum(op["groups"]) * op["n_out"] * n["token"] - dec = m_dec0 - if any(seg[0] == "block" for seg in t["dec_connect"]): - b = t["dec_block"] - lvl = n["token"] if t["dec_block_level"] == "token" else render_w * render_h - dec += block_macs_per_token(b["C"], b["heads"], b["mlp"]) * lvl + + def ops_macs(ops, plan): + total = 0 + for op, (_, lvl, _c) in zip(ops, plan): + if op["op"] == "linear": + total += op["n_in"] * op["n_out"] * n[lvl] + elif op["op"] == "swin": + total += block_macs_per_token(op["C"], op["heads"], op["mlp"]) * n[lvl] + elif op["op"] == "project": + total += sum(op["groups"]) * op["n_out"] * n["token"] + elif op["op"] == "dec_project": + total += sum(op["groups"]) * 128 * n["token"] + return total + + enc = ops_macs(t["enc_ops"], encoder_plan(t)) + dec = ops_macs(t["dec_ops"], decoder_plan(t)) L = sum(parts.values()) + enc + dec parts.update({"enc0.M": m_enc0, "dec0.M": m_dec0, "enc0.L": enc, "dec0.L": dec}) return {"M": M, "L": L, "parts": parts} diff --git a/clean/l/runner/l_pack.py b/clean/l/runner/l_pack.py index 5c578b8c..c237afab 100644 --- a/clean/l/runner/l_pack.py +++ b/clean/l/runner/l_pack.py @@ -1,18 +1,19 @@ # SPDX-License-Identifier: MIT """Model L weight packing (issue #89): the device blobs of Model L's own launches, from the structure table -(clean/l/reference/table.py) and the logical tensors of the oracle (clean/l/reference/oracle.py). +(clean/l/reference/table.py) and the parameters of the oracle ({stage: tensors}, clean/l/reference/oracle.py). -Model L's encoder and decoder are run as a chain of small FP8-map kernels, each with a blob in a format Model M -already uses (clean/FORMATS.md): +Model L's encoder and decoder run as a chain of small FP8-map kernels, each with a blob in a format Model M already +uses (clean/FORMATS.md). Maps are [tokens][C] FP8, channel fastest; pixel maps are on the padded grid F, token maps +on F/2. | Blob | Format | Kernel input -> output | |---|---|---| -| `enc0.values` | Model M's enc0 blob (clean/runner/enc0.py pack) with W = 0 and b = 0 | frame -> the quantised values as a token map [F/2][80] (space_to_depth order 20 p + i) | -| `enc0.` per linear | merge / expand section format: bias tile vector (rows R_L(C_out)), then fragments [mt < C_out/16][kc < C_in/16], rows R_L(C_out), K order K_L(C_in) | FP8 map [C_in] -> FP8 map [C_out]; a `select` feeding the linear is folded in as zero weights outside its channels | -| `enc0.` per block | Model M's Swin block section (packing.pack_block) at the block's width, padded to 256 | FP8 map -> FP8 map | -| `enc0.project` | bias tile vector (rows R_L(n_out)), then fragments [mt][kb], rows R_L(n_out), K order: groups of 32 channels, block 2i + s holds channel 32 i + 16 h + 8 s + j (dec0's order, clean/runner/dec0.py kord_decoder, over every group) | the concatenated maps -> the 64 FP8 features enc1 reads | -| `dec0.` | Model M's Swin block section at the decoder block's width | dec1's channels of the block segment -> same width | -| `dec0` | Model M's dec0 blob (clean/runner/dec0.py pack) | as Model M | +| `enc0.values` | Model M's enc0 blob (clean/runner/enc0.py pack) with W = 0 and b = 0 | frame -> the quantized values as a pixel map [F][32] (values 0..19, 12 zero bytes) | +| `enc0.` per linear | merge / expand section format: bias tile vector (rows R_L(C_out)), then fragments [mt < C_out/16][kc < C_in/16], rows R_L(C_out), K order K_L(C_in); inputs narrower than C_in (20 values) get zero weights | FP8 map [C_in] -> FP8 map [C_out] | +| `enc0_block`, `dec0_block` | Model M's Swin block section (packing.pack_block) at the block's width, padded to 256 | FP8 map -> FP8 map | +| `enc0.project` | bias tile vector (rows R_L(64)), then fragments [mt][kb], rows R_L(64), K order: block 2i + s of group i holds channel 32 i + 16 h + 8 s + j (dec0's order, clean/runner/dec0.py kord_decoder, over every group) | the 2 x 2 pixels of each token (channel 32 p + i, p = dx + 2 dy) -> the 64 FP8 features enc1 reads | +| `dec0.project` | expand section format: bias tile vector (rows R_E(32)), then fragments [mt < 8][kb < 4], rows R_E(32) (tile 2 s + g: sub-pixel s, channels 16 g ..), K order as `enc0.project` (two groups of 32) | dec1's map [F/2][64] -> pixel map [F][32] | +| `dec0.coef` | Model M's dec0 blob (clean/runner/dec0.py pack) with W = 0 and b = 0: the scalars S, T and the code tables | the decoder block's pixel map -> A, B, C, latent | The ten shared Swin stages use clean/runner/packing.py pack_stage unchanged. """ @@ -33,7 +34,7 @@ def kord_groups32(n): """K slot (h, j) of block kb holds channel 32 (kb // 2) + 16 h + 8 (kb % 2) + j: each group of 32 channels is two whole blocks, so a kernel can round its running sum after every second block.""" if n % 32: - raise ValueError(f"project input of {n} channels: groups of 32 only") + raise ValueError(f"input of {n} channels: groups of 32 only") K = np.empty(n, dtype=np.int64) for kb in range(n // 16): for h in range(2): @@ -67,10 +68,20 @@ def pack_project(W, b, groups, d_row="gfx12"): return P.pack_tile_vector(b, RL, drow) + P.pack_frags(W, RL, kord_groups32(n)) -def pack_block(tensors, prefix, C, heads, mlp, d_row="gfx12"): +def pack_dec_project(W, b, groups=(32, 32), d_row="gfx12"): + """W [4, 32, n], b [4, 32] -> bias (rows R_E(32)) + fragments (rows R_E(32), K order kord_groups32(n)).""" drow = P.resolve_d_row(d_row) - p = {k[len(prefix):]: v for k, v in tensors.items() if k.startswith(prefix)} - blob = P.pack_block(p, C, heads, mlp, drow) + W = np.asarray(W, dtype=np.float64).reshape(128, -1) + if any(g != 32 for g in groups) or sum(groups) != W.shape[1]: + raise ValueError(f"dec_project groups {groups}: groups of 32 covering {W.shape[1]} channels only") + RE = P.rows_expand(32, drow) + return P.pack_tile_vector(np.asarray(b, np.float32).reshape(128), RE, drow) + \ + P.pack_frags(W, RE, kord_groups32(W.shape[1])) + + +def pack_block(tensors, C, heads, mlp, d_row="gfx12"): + drow = P.resolve_d_row(d_row) + blob = P.pack_block(tensors, C, heads, mlp, drow) return blob + bytes((-len(blob)) % ALIGN) @@ -82,76 +93,98 @@ def pack_values(tensors, d_row="gfx12"): return _enc0.pack(t, d_row) -# ---- the encoder's launch list -------------------------------------------------------------------------------------- +def pack_coef(tensors, d_row="gfx12"): + """Model M's dec0 blob with zero projection: the scalars S, T and the code tables.""" + return _dec0.pack({"W": np.zeros((68, 64), np.float32), "b": np.zeros(68, np.float32), + "S": tensors["S"], "T": tensors["T"]}, d_row) + + +# ---- launch lists --------------------------------------------------------------------------------------------------- def encoder_launches(t=_table.TABLE): - """The kernels of the table's encoder, in order: [{"kind", "blob", "src", "dst", ...}]. kinds: "values" (quant - + space_to_depth into the raw map), "linear", "swin", "project". A select is folded into the linear it feeds.""" - ops = t["enc_ops"] - plan = _table.encoder_plan(t) - chans = {"values": 20} - launches = [] - selects = {} - i = 0 - while i < len(ops): - op = ops[i] + """The encoder's kernels in order: [{"kind", "blob", ...}]. kinds: "values" (quant into a pixel map of 32 + channels), "linear", "swin", "project" (with the space_to_depth gather folded in).""" + ops = list(t["enc_ops"]) + _table.encoder_plan(t) + width = {"values": 20} + out = [] + for i, op in enumerate(ops): k = op["op"] if k == "quant": - nxt = ops[i + 1] if i + 1 < len(ops) else None - if not nxt or nxt["op"] != "space_to_depth" or nxt["src"] != op["dst"]: - raise ValueError("the encoder's kernels need quant directly followed by its space_to_depth") - if op["mode"] != "dither": - raise ValueError("the encoder's value kernel quantises with the dither table") - launches.append({"kind": "values", "blob": "enc0.values", "dst": nxt["dst"]}) - chans[nxt["dst"]] = 80 - i += 2 - continue - if k == "select": - selects[op["dst"]] = (op["src"], op["lo"], op["hi"]) - chans[op["dst"]] = op["hi"] - op["lo"] + if op["mode"] != "dither" or i != 0: + raise ValueError("the encoder's value kernel quantizes the per-pixel values with the dither table") + out.append({"kind": "values", "blob": "enc0.values", "dst": op["dst"], "C": 32}) + width[op["dst"]] = 32 elif k == "linear": - src, lo = op["src"], 0 - if src in selects: - src, lo, _ = selects[src] - launches.append({"kind": "linear", "blob": "enc0." + op["w"], "src": src, "dst": op["dst"], - "C_in": chans[src], "C_out": op["n_out"], "lo": lo, "w": op["w"], "b": op["b"]}) - chans[op["dst"]] = op["n_out"] + c_in = width[op["src"]] + out.append({"kind": "linear", "blob": "enc0." + op["w"], "src": op["src"], "dst": op["dst"], + "C_in": c_in, "C_out": op["n_out"], "w": op["w"], "b": op["b"], "acc": op.get("acc"), + "out_round": op.get("out_round", "q8")}) + width[op["dst"]] = op["n_out"] elif k == "swin": - launches.append({"kind": "swin", "blob": "enc0." + op["prefix"].rstrip("."), "src": op["src"], - "dst": op["dst"], "prefix": op["prefix"], "C": op["C"], "heads": op["heads"], - "mlp": op["mlp"], "offsets": op["offsets"]}) - elif k == "concat": - chans[op["dst"]] = plan[i][2] - launches.append({"kind": "concat", "srcs": op["srcs"], "dst": op["dst"], "pad_to": plan[i][2]}) + out.append({"kind": "swin", "blob": op["params"], "src": op["src"], "dst": op["dst"], "C": op["C"], + "heads": op["heads"], "mlp": op["mlp"], "offsets": op["offsets"], "params": op["params"]}) + elif k == "space_to_depth": + nxt = ops[i + 1] if i + 1 < len(ops) else None + if not nxt or nxt["op"] != "project" or nxt["src"] != op["dst"]: + raise ValueError("space_to_depth must feed the project directly (one gather kernel)") + width[op["dst"]] = 4 * width[op["src"]] + out.append({"kind": "project", "blob": "enc0.project", "src": op["src"], "groups": nxt["groups"], + "w": nxt["w"], "b": nxt["b"], "clamp": nxt.get("clamp")}) elif k == "project": - launches.append({"kind": "project", "blob": "enc0.project", "src": op["src"], "groups": op["groups"]}) + continue # folded into the gather above else: raise ValueError(f"no kernel for encoder op {k!r}") - if k != "select": - selects.pop(op.get("dst"), None) # the name now holds this op's output - i += 1 - return launches + return out + + +def decoder_launches(t=_table.TABLE): + """The decoder's kernels in order: "dec_project" (with depth_to_space folded in), "swin", "coef" (pick + the + coefficient functions).""" + ops = list(t["dec_ops"]) + _table.decoder_plan(t) + out = [] + for i, op in enumerate(ops): + k = op["op"] + if k == "dec_project": + nxt = ops[i + 1] if i + 1 < len(ops) else None + if not nxt or nxt["op"] != "depth_to_space" or nxt["src"] != op["dst"]: + raise ValueError("dec_project must feed depth_to_space directly (one scatter kernel)") + out.append({"kind": "dec_project", "blob": "dec0.project", "w": op["w"], "b": op["b"], + "groups": op["groups"], "dst": nxt["dst"]}) + elif k == "depth_to_space": + continue + elif k == "swin": + out.append({"kind": "swin", "blob": op["params"], "src": op["src"], "dst": op["dst"], "C": op["C"], + "heads": op["heads"], "mlp": op["mlp"], "offsets": op["offsets"], "params": op["params"]}) + elif k == "pick": + out.append({"kind": "coef", "blob": "dec0.coef", "src": op["src"], "values": tuple(op["values"])}) + else: + raise ValueError(f"no kernel for decoder op {k!r}") + return out def pack_model_l(params, t=_table.TABLE, d_row="gfx12"): """{blob name: bytes} of Model L's own launches (the shared Swin stages: packing.pack_stage).""" - enc = params["enc0"] out = {} + enc = params["enc0"] for ln in encoder_launches(t): if ln["kind"] == "values": out[ln["blob"]] = pack_values(enc, d_row) elif ln["kind"] == "linear": - out[ln["blob"]] = pack_linear(enc[ln["w"]], enc[ln["b"]], ln["C_in"], ln["C_out"], ln["lo"], d_row) + out[ln["blob"]] = pack_linear(enc[ln["w"]], enc[ln["b"]], ln["C_in"], ln["C_out"], 0, d_row) elif ln["kind"] == "swin": - out[ln["blob"]] = pack_block(enc, ln["prefix"], ln["C"], ln["heads"], ln["mlp"], d_row) + out[ln["blob"]] = pack_block(params[ln["params"]], ln["C"], ln["heads"], ln["mlp"], d_row) elif ln["kind"] == "project": - op = t["enc_ops"][-1] - out[ln["blob"]] = pack_project(enc[op["w"]], enc[op["b"]], ln["groups"], d_row) + out[ln["blob"]] = pack_project(enc[ln["w"]], enc[ln["b"]], ln["groups"], d_row) dec = params["dec0"] - if any(s[0] == "block" for s in t["dec_connect"]): - b = t["dec_block"] - out["dec0." + b["prefix"].rstrip(".")] = pack_block(dec, b["prefix"], b["C"], b["heads"], b["mlp"], d_row) - out["dec0"] = _dec0.pack({k: dec[k] for k in ("W", "b", "S", "T")}, d_row) + for ln in decoder_launches(t): + if ln["kind"] == "dec_project": + out[ln["blob"]] = pack_dec_project(dec[ln["w"]], dec[ln["b"]], ln["groups"], d_row) + elif ln["kind"] == "swin": + out[ln["blob"]] = pack_block(params[ln["params"]], ln["C"], ln["heads"], ln["mlp"], d_row) + elif ln["kind"] == "coef": + out[ln["blob"]] = pack_coef(dec, d_row) return out @@ -190,3 +223,12 @@ def unpack_project(blob, n_in, n_out, d_row="gfx12"): RL = P.rows_layout_l(n_out, drow) b = unpack_tile_vector(blob[:4 * n_out], RL, drow) return unpack_frags(blob[4 * n_out:], RL, kord_groups32(n_in)), b + + +def unpack_dec_project(blob, n_in=64, d_row="gfx12"): + """(W [4, 32, n_in], b [4, 32]) from a dec0.project blob.""" + drow = P.resolve_d_row(d_row) + RE = P.rows_expand(32, drow) + b = unpack_tile_vector(blob[:4 * 128], RE, drow) + W = unpack_frags(blob[4 * 128:], RE, kord_groups32(n_in)) + return W.reshape(4, 32, n_in), b.reshape(4, 32) diff --git a/clean/l/tests/test_l_pack.py b/clean/l/tests/test_l_pack.py index 7a7917ad..7138aaaa 100644 --- a/clean/l/tests/test_l_pack.py +++ b/clean/l/tests/test_l_pack.py @@ -10,6 +10,7 @@ from clean.l.reference import oracle as L from clean.l.reference import table as T from clean.l.runner import l_pack as LP +from clean.reference import decoder as MD from clean.reference import encoder as ME from clean.reference import swin as MS from clean.reference.fp8 import q8 @@ -34,7 +35,7 @@ def test_groups32_blocks(self): class RoundTripTest(unittest.TestCase): def test_linear(self): rng = np.random.default_rng(1) - for C_in, C_out, n, lo in ((80, 32, 20, 0), (80, 32, 20, 60), (32, 32, 32, 0), (64, 16, 16, 16)): + for C_in, C_out, n, lo in ((32, 32, 20, 0), (80, 32, 20, 60), (32, 32, 32, 0), (64, 16, 16, 16)): W = q8(rng.normal(0, 0.5, (C_out, n))) b = rng.normal(0, 1, C_out).astype(np.float32) blob = LP.pack_linear(W, b, C_in, C_out, lo) @@ -48,7 +49,6 @@ def test_linear(self): LP.pack_linear(np.zeros((32, 20)), np.zeros(32), 80, 32, lo=70) def test_linear_is_merge_section_format(self): - """The linear blob is the patch-merge section format when the K order is the merge's.""" rng = np.random.default_rng(2) W = q8(rng.normal(0, 0.5, (96, 64))) b = rng.normal(0, 1, 96).astype(np.float32) @@ -69,71 +69,105 @@ def test_project(self): with self.assertRaises(ValueError): LP.pack_project(W, b, (20, 20, 20, 20, 48)) - def test_block_equals_model_m_packer(self): - """The L block packer is packing.pack_block: Model M's stage blobs come out byte for byte, and C = 32 has - the layout's size.""" + def test_dec_project(self): rng = np.random.default_rng(4) + W = q8(rng.normal(0, 0.3, (4, 32, 64))) + b = rng.normal(0, 1, (4, 32)).astype(np.float32) + blob = LP.pack_dec_project(W, b) + self.assertEqual(len(blob), 4 * 128 + 128 * 64) + Wu, bu = LP.unpack_dec_project(blob) + self.assertTrue(np.array_equal(Wu, W)) + self.assertTrue(np.array_equal(bu, b)) + + def test_dec_project_is_expand_section_format(self): + """Rows R_E(32): the expand section's row order at C = 32 (sub-pixel s, then layout L of its 32 channels).""" + rng = np.random.default_rng(5) + W = q8(rng.normal(0, 0.3, (4, 32, 64))) + b = rng.normal(0, 1, (4, 32)).astype(np.float32) + drow = P.resolve_d_row("gfx12") + RE = P.rows_expand(32, drow) + want = P.pack_tile_vector(b.reshape(128), RE, drow) + P.pack_frags(W.reshape(128, 64), RE, + LP.kord_groups32(64)) + self.assertEqual(LP.pack_dec_project(W, b), want) + + def test_block_equals_model_m_packer(self): + rng = np.random.default_rng(6) for stage in ("b0", "dec1"): cfg = MS.STAGES[stage] p = MS.random_params(stage, rng) want = P.pack_stage(stage, p)[:P.stage_sections(stage)[1]] - got = LP.pack_block({"x." + k: v for k, v in p.items()}, "x.", cfg["C"], cfg["heads"], cfg["mlp"]) + got = LP.pack_block({k: v for k, v in p.items() if k not in ("We", "be")}, cfg["C"], cfg["heads"], + cfg["mlp"]) self.assertEqual(got, want, stage) - p = L.random_block_params(rng, 32, 2, 128, "blk.") - blob = LP.pack_block(p, "blk.", 32, 2, 128) + p = L.random_block_params(rng, 32, 2, 128) + blob = LP.pack_block(p, 32, 2, 128) lay = P.block_layout(32, 2, 128) self.assertEqual(len(blob), lay["SIZE"] + (-lay["SIZE"]) % 256) self.assertEqual((lay["HPU"], lay["NMLP"], lay["MLP_BYTES"], lay["HEAD_BYTES"]), (8, 1, 8192, 3072)) - def test_values_blob(self): - rng = np.random.default_rng(5) - enc = L.random_params(rng)["enc0"] - blob = LP.pack_values(enc) + def test_values_and_coef_blobs(self): + rng = np.random.default_rng(7) + params = L.random_params(rng) + blob = LP.pack_values(params["enc0"]) self.assertEqual(len(blob), E0.BLOB_BYTES) self.assertEqual(blob[:256], bytes(256)) # zero bias - self.assertEqual(blob[E0.OFF_DITHER:E0.OFF_DITHER + 2048], np.asarray(enc["T"]).astype(np.uint8).tobytes()) + self.assertEqual(blob[E0.OFF_DITHER:E0.OFF_DITHER + 2048], + np.asarray(params["enc0"]["T"]).astype(np.uint8).tobytes()) + coef = LP.pack_coef(params["dec0"]) + self.assertEqual(len(coef), D0.BLOB_BYTES) + full = D0.pack({"W": np.zeros((68, 64), np.float32), "b": np.zeros(68, np.float32), + "S": params["dec0"]["S"], "T": params["dec0"]["T"]}) + self.assertEqual(coef, full) class ModelTest(unittest.TestCase): - def test_launch_list(self): + def test_launch_lists(self): ln = LP.encoder_launches() - self.assertEqual([x["kind"] for x in ln], ["values", "linear", "linear", "swin", "concat", "project"]) - self.assertEqual((ln[1]["src"], ln[1]["C_in"], ln[1]["lo"], ln[1]["C_out"]), ("raw", 80, 0, 32)) + self.assertEqual([x["kind"] for x in ln], ["values", "linear", "linear", "swin", "project"]) + self.assertEqual((ln[1]["src"], ln[1]["C_in"], ln[1]["C_out"], ln[1]["out_round"]), ("q", 32, 32, "relu_q8")) self.assertEqual((ln[2]["src"], ln[2]["C_in"]), ("x", 32)) + self.assertEqual(ln[3]["blob"], "enc0_block") + dn = LP.decoder_launches() + self.assertEqual([x["kind"] for x in dn], ["dec_project", "swin", "coef"]) + self.assertEqual(dn[2]["values"], T.DEC0_VALUES) def test_pack_model_l(self): - rng = np.random.default_rng(6) + rng = np.random.default_rng(8) params = L.random_params(rng) blobs = LP.pack_model_l(params) - self.assertEqual(sorted(blobs), sorted(["enc0.values", "enc0.Wa", "enc0.Wb", "enc0.blk", "enc0.project", - "dec0.blk", "dec0"])) - self.assertEqual(len(blobs["enc0.Wa"]), 4 * 32 + 32 * 80) - self.assertEqual(len(blobs["enc0.Wb"]), 4 * 32 + 32 * 32) - self.assertEqual(len(blobs["dec0"]), D0.BLOB_BYTES) - self.assertEqual(blobs["dec0"], D0.pack({k: params["dec0"][k] for k in ("W", "b", "S", "T")})) + self.assertEqual(sorted(blobs), sorted(["enc0.values", "enc0.Win1", "enc0.Win2", "enc0_block", "enc0.project", + "dec0.project", "dec0_block", "dec0.coef"])) + self.assertEqual(len(blobs["enc0.Win1"]), 4 * 32 + 32 * 32) + self.assertEqual(len(blobs["enc0.Win2"]), 4 * 32 + 32 * 32) + self.assertEqual(len(blobs["dec0.project"]), 4 * 128 + 128 * 64) self.assertEqual(blobs, LP.pack_model_l(params)) # deterministic def test_unpacked_weights_reproduce_the_oracle(self): - """The weights decoded from the blobs, applied to the oracle's maps the way the kernels apply them (whole - input maps, zero weights outside a select), give the oracle's maps byte for byte.""" - rng = np.random.default_rng(7) + """The weights decoded from the blobs, applied the way the kernels apply them (whole 32-channel input maps, + zero weights for the padding), give the oracle's maps byte for byte.""" + rng = np.random.default_rng(9) params = L.random_params(rng) x = ME.random_inputs(rng, rw=36, rh=20) - out = L.encode(x, params["enc0"], parity=0, keep=True) + out = L.encode(x, params, parity=0, keep=True) m = out["maps"] blobs = LP.pack_model_l(params) enc = params["enc0"] - Wa, ba = LP.unpack_linear(blobs["enc0.Wa"], 80, 32) - Wb, bb = LP.unpack_linear(blobs["enc0.Wb"], 32, 32) - h1 = q8(m["raw"] @ Wa.T + ba) - self.assertTrue(np.array_equal(h1, q8(m["raw"][..., :20] @ np.asarray(enc["Wa"], np.float64).T + enc["ba"]))) - h2 = q8(h1 @ Wb.T + bb) - bp = {k[4:]: v for k, v in enc.items() if k.startswith("blk.")} - h3 = q8(L.swin_block(h2, bp, T.block_offset(T.TABLE, "first", 0), 2, T.M_BLOCK_CONSTS)) + W1, b1 = LP.unpack_linear(blobs["enc0.Win1"], 32, 32) + W2, b2 = LP.unpack_linear(blobs["enc0.Win2"], 32, 32) + qpad = np.concatenate([m["q"], np.zeros(m["q"].shape[:-1] + (12,))], axis=-1) + h1 = L.linear(qpad, W1, b1, "f16", "relu_q8") + self.assertTrue(np.array_equal(h1, L.linear(m["q"], enc["Win1"], enc["bin1"], "f16", "relu_q8"))) + h2 = L.linear(h1, W2, b2, "f16", "q8") + h3 = q8(L.swin_block(h2, params["enc0_block"], T.block_offset(T.TABLE, "first", 0), 2, T.M_BLOCK_CONSTS)) self.assertTrue(np.array_equal(h3, m["x"])) Wp, bpj = LP.unpack_project(blobs["enc0.project"], 128, 64) - feats = L.project_groups(m["z"], Wp, bpj, (32, 32, 32, 32)) + feats = L.project_groups(L.space_to_depth(h3), Wp, bpj, (32, 32, 32, 32), 6.28125) self.assertEqual(feats.astype(np.float32).tobytes(), out["features"].tobytes()) + xd = MD.random_inputs(rng, rw=36, rh=20) + dout = L.decode(xd, params, 0, keep=True) + Wd, bd = LP.unpack_dec_project(blobs["dec0.project"]) + y = L.dec_project(xd["features"], Wd, bd) + self.assertTrue(np.array_equal(y, dout["maps"]["y"])) if __name__ == "__main__": diff --git a/clean/l/tests/test_l_reference.py b/clean/l/tests/test_l_reference.py index 81103461..c1cc9e5c 100644 --- a/clean/l/tests/test_l_reference.py +++ b/clean/l/tests/test_l_reference.py @@ -3,8 +3,8 @@ Run: python -m unittest discover -s clean/l/tests -t . -The strongest checks reduce Model L to Model M through the table and require the L oracle to give Model M's oracle -results byte for byte: the block with M's constants, the encoder with M's op list, the decoder without a block, and +The strongest checks reduce Model L to Model M through the table and require Model M's oracle results byte for byte: +the block with M's constants, the encoder with M's op list, the decoder with M's projection rows and no block, and the whole network chain. """ import unittest @@ -22,7 +22,13 @@ M_ENC_OPS = ( {"op": "quant", "mode": "dither", "src": "values", "dst": "q"}, {"op": "space_to_depth", "src": "q", "dst": "z"}, - {"op": "project", "src": "z", "w": "W", "b": "b", "groups": (20, 20, 20, 20), "n_out": 64}, + {"op": "project", "src": "z", "w": "W", "b": "b", "groups": (20, 20, 20, 20), "n_out": 64, "clamp": 6.28125}, +) +# Model M's decoder written as an L op list: the projection, then the value positions straight away (no block). +M_DEC_OPS = ( + {"op": "dec_project", "src": "features", "dst": "y", "w": "W", "b": "b", "groups": (32, 32)}, + {"op": "depth_to_space", "src": "y", "dst": "p"}, + {"op": "pick", "src": "p", "values": T.DEC0_VALUES}, ) @@ -32,6 +38,19 @@ def _same(a, b): return a.shape == b.shape and a.dtype == b.dtype and a.tobytes() == b.tobytes() +def m_dec_as_l(p): + """Model M's dec0 tensors (W [68, 64], b [68]) as L's [4, 32, 64] / [4, 32] with zeros outside the value rows.""" + W = np.zeros((4, 32, 64), np.float32) + b = np.zeros((4, 32), np.float32) + Wm = np.asarray(p["W"], np.float32).reshape(4, 17, 64) + bm = np.asarray(p["b"], np.float32).reshape(4, 17) + for s in range(4): + for k, c in enumerate(T.DEC0_VALUES): + W[s, c] = Wm[s, k] + b[s, c] = bm[s, k] + return {"W": W, "b": b, "S": p["S"], "T": p["T"]} + + class BlockTest(unittest.TestCase): def test_block_equals_model_m_oracle(self): """With Model M's constants the L block is clean/reference/swin.py swin_block, byte for byte.""" @@ -100,94 +119,84 @@ def test_model_m_op_list_equals_model_m_encoder(self): x = ME.random_inputs(rng, rw=37, rh=21, reset=reset) p = ME.random_params(rng) want = ME.encode(x, p) - got = L.encode(x, p, t=t) + got = L.encode(x, {"enc0": p}, t=t) for k in ("features", "m_near", "m_far", "m_center"): self.assertTrue(_same(got[k], want[k]), (k, reset)) def test_default_encoder(self): rng = np.random.default_rng(5) x = ME.random_inputs(rng, rw=40, rh=24) - p = L.random_params(rng)["enc0"] - self.assertEqual(set(p), set(T.encoder_shapes())) - for name, shape in T.encoder_shapes().items(): - self.assertEqual(np.shape(p[name]), shape, name) - out = L.encode(x, p, keep=True) + params = L.random_params(rng) + for stage, shapes in T.param_shapes().items(): + self.assertEqual(set(params[stage]), set(shapes), stage) + for name, shape in shapes.items(): + self.assertEqual(np.shape(params[stage][name]), shape, (stage, name)) + out = L.encode(x, params, keep=True) fw, fh = x["feature_size"] self.assertEqual(out["features"].shape, (fh // 2, fw // 2, 64)) self.assertTrue(np.isfinite(out["features"]).all()) self.assertTrue(is_fp8_exact(out["features"])) maps = out["maps"] - self.assertEqual(maps["raw"].shape, (fh // 2, fw // 2, 80)) - self.assertEqual(maps["x"].shape, (fh // 2, fw // 2, 32)) + self.assertEqual(maps["q"].shape, (fh, fw, 20)) + self.assertEqual(maps["x"].shape, (fh, fw, 32)) self.assertEqual(maps["z"].shape, (fh // 2, fw // 2, 128)) - self.assertTrue(np.all(maps["z"][..., 112:] == 0)) - again = L.encode(x, p) - self.assertTrue(_same(again["features"], out["features"])) + self.assertTrue(_same(L.encode(x, params)["features"], out["features"])) def test_parts_by_hand(self): - """linear, select, concat and project of the default op list recomputed directly.""" + """The default op list recomputed step by step.""" rng = np.random.default_rng(6) x = ME.random_inputs(rng, rw=33, rh=17) - p = L.random_params(rng)["enc0"] - out = L.encode(x, p, parity=1, keep=True) + params = L.random_params(rng) + enc = params["enc0"] + out = L.encode(x, params, parity=1, keep=True) m = out["maps"] fw, fh = x["feature_size"] - q = ME.dither_fp8(ME.pixel_values(x, p), ME.dither_table(p["T"], x["frame"], fw, fh)[..., None]) + q = ME.dither_fp8(ME.pixel_values(x, enc), ME.dither_table(enc["T"], x["frame"], fw, fh)[..., None]) self.assertTrue(_same(m["q"], q)) - raw = np.zeros((fh // 2, fw // 2, 80)) + h = q8(np.maximum(L.f16(q @ np.asarray(enc["Win1"], np.float64).T + enc["bin1"]), 0)) + h = q8(L.f16(h @ np.asarray(enc["Win2"], np.float64).T + enc["bin2"])) + h = q8(L.swin_block(h, params["enc0_block"], T.block_offset(T.TABLE, "first", 1), 2, T.M_BLOCK_CONSTS)) + self.assertTrue(_same(m["x"], h)) + z = np.zeros((fh // 2, fw // 2, 128)) for r in range(fh // 2): for c in range(fw // 2): - for pp in range(4): - raw[r, c, 20 * pp:20 * pp + 20] = q[2 * r + pp // 2, 2 * c + pp % 2] - self.assertTrue(_same(m["raw"], raw)) - h = q8(raw[..., :20] @ np.asarray(p["Wa"], np.float64).T + p["ba"]) - h = q8(h @ np.asarray(p["Wb"], np.float64).T + p["bb"]) - bp = {k[4:]: v for k, v in p.items() if k.startswith("blk.")} - h = q8(L.swin_block(h, bp, T.block_offset(T.TABLE, "first", 1), 2, T.M_BLOCK_CONSTS)) - self.assertTrue(_same(m["x"], h)) - z = np.concatenate([raw, h, np.zeros(h.shape[:-1] + (16,))], axis=-1) - W = np.asarray(p["W"], np.float64) - s = np.broadcast_to(np.asarray(p["b"], np.float64), z.shape[:-1] + (64,)) + for p in range(4): + z[r, c, 32 * p:32 * p + 32] = h[2 * r + p // 2, 2 * c + p % 2] + self.assertTrue(_same(m["z"], z)) + W = np.asarray(enc["W"], np.float64) + s = np.broadcast_to(np.asarray(enc["b"], np.float64), z.shape[:-1] + (64,)) for g in range(4): s = L.f16(s + z[..., 32 * g:32 * g + 32] @ W[:, 32 * g:32 * g + 32].T) self.assertTrue(_same(out["features"], q8(np.clip(s, -6.28125, 6.28125)).astype(np.float32))) def test_table_edits(self): - """Provisional entries are table edits: other op orders, a pixel-level pre-network, plain q8 inputs.""" + """Provisional entries are table edits: no ReLU, no FP16 accumulation, plain FP8 quantization, no clamp.""" rng = np.random.default_rng(7) x = ME.random_inputs(rng, rw=24, rh=20) base = T.TABLE["enc_ops"] variants = { - "block_first": (base[0], base[1], base[2], - {**base[5], "C": 20, "heads": 1, "mlp": 80, "prefix": "blk."}, - base[3], base[4], base[6], base[7]), - "pixel_level": ( - base[0], - {**base[3], "src": "q"}, base[4], base[5], - {"op": "space_to_depth", "src": "x", "dst": "z"}, - {"op": "project", "src": "z", "w": "W", "b": "b", "groups": (32, 32, 32, 32), "n_out": 64}), - "plain_q8": ({"op": "quant", "mode": "q8", "src": "values", "dst": "q"},) + tuple(base[1:]), + "no_relu": (base[0], dict(base[1], out_round="q8")) + tuple(base[2:]), + "no_f16": (base[0], dict(base[1], acc=None), dict(base[2], acc=None)) + tuple(base[3:]), + "plain_q8": (dict(base[0], mode="q8"),) + tuple(base[1:]), + "no_clamp": tuple(base[:-1]) + (dict(base[-1], clamp=None),), } - feats = {} + params = L.random_params(np.random.default_rng(8)) + ref = L.encode(x, params)["features"] for name, ops in variants.items(): t = T.table(enc_ops=ops) - plan = T.encoder_plan(t) - p = L.random_params(np.random.default_rng(8), t)["enc0"] - self.assertEqual(set(p), set(T.encoder_shapes(t)), name) - out = L.encode(x, p, t=t) + out = L.encode(x, params, t=t) self.assertTrue(is_fp8_exact(out["features"]), name) - feats[name] = out["features"] - self.assertEqual(plan[-1], ("project", "token", 64)) - self.assertNotIn("T", T.encoder_shapes(T.table(enc_ops=variants["plain_q8"]))) - self.assertEqual(T.encoder_plan(T.table(enc_ops=variants["pixel_level"]))[1], ("linear", "pixel", 32)) + if name != "no_clamp": + self.assertFalse(_same(out["features"], ref), name) + self.assertNotIn("T", T.param_shapes(T.table(enc_ops=variants["plain_q8"]))["enc0"]) def test_plan_refuses_bad_lists(self): base = T.TABLE["enc_ops"] bad = [ - base[:-1], # no project at the end - (base[0], {**base[3], "src": "q", "n_in": 32}) + tuple(base[4:]), # 20 channels into a 32-input linear - (base[0], base[1], {**base[2], "hi": 99}) + tuple(base[3:]), # select outside the map + base[:-1], # no project at the end + (base[0], dict(base[1], n_in=32)) + tuple(base[2:]), # 20 channels into a 32-input linear (base[0], {"op": "frobnicate", "src": "q"}, base[-1]), + tuple(base[:4]) + (base[5],), # project on a pixel-level map ] for ops in bad: with self.assertRaises(ValueError): @@ -195,48 +204,47 @@ def test_plan_refuses_bad_lists(self): class DecoderTest(unittest.TestCase): - def test_no_block_equals_model_m_decoder(self): + def test_model_m_op_list_equals_model_m_decoder(self): + """With Model M's projection rows and no block, the L decoder is Model M's decoder byte for byte.""" rng = np.random.default_rng(9) - t = T.table(dec_connect=(("pass", 0, 64),)) + t = T.table(dec_ops=M_DEC_OPS) for reset in (False, True): x = MD.random_inputs(rng, rw=29, rh=19, reset=reset) p = MD.random_params(rng) want = MD.decode(x, p) - got = L.decode(x, p, 0, t) - for k in ("A", "B", "C", "latent", "codes_A", "codes_B", "codes_L", "h"): + got = L.decode(x, {"dec0": m_dec_as_l(p)}, 0, t) + for k in ("A", "B", "C", "latent", "codes_A", "codes_B", "codes_L"): self.assertTrue(_same(got[k], want[k]), (k, reset)) def test_default_decoder(self): rng = np.random.default_rng(10) x = MD.random_inputs(rng, rw=30, rh=18) - p = L.random_params(rng)["dec0"] - self.assertEqual(set(p), set(T.decoder_shapes())) - out = L.decode(x, p, 1) - front = out["front"] - self.assertTrue(_same(front[..., 32:], x["features"][..., 32:])) - bp = {k[4:]: v for k, v in p.items() if k.startswith("blk.")} - y = q8(L.swin_block(x["features"][..., :32], bp, T.block_offset(T.TABLE, "second", 1), 2, - T.M_BLOCK_CONSTS)).astype(np.float32) - self.assertTrue(_same(front[..., :32], y)) - xx = dict(x) - xx["features"] = front - want = MD.decode(xx, {k: p[k] for k in ("W", "b", "S", "T")}) - self.assertTrue(_same(out["A"], want["A"])) - self.assertTrue(_same(out["latent"], want["latent"])) - - def test_other_segments(self): - rng = np.random.default_rng(11) - x = MD.random_inputs(rng, rw=20, rh=12) - p = L.random_params(rng)["dec0"] - t = T.table(dec_connect=(("pass", 0, 32), ("block", 32, 64))) - out = L.decode(x, p, 0, t) - self.assertTrue(_same(out["front"][..., :32], x["features"][..., :32])) - with self.assertRaises(ValueError): - L.decode(x, p, 0, T.table(dec_connect=(("block", 0, 48), ("pass", 48, 64)))) - with self.assertRaises(NotImplementedError): - L.decode(x, p, 0, T.table(dec_block_level="pixel")) + params = L.random_params(rng) + out = L.decode(x, params, 1, keep=True) + fw, fh = x["feature_size"] + m = out["maps"] + self.assertEqual(m["y"].shape, (fh // 2, fw // 2, 128)) + self.assertEqual(m["p"].shape, (fh, fw, 32)) + self.assertEqual(out["a"].shape, (18, 30, 17)) + y = L.dec_project(x["features"], params["dec0"]["W"], params["dec0"]["b"]) + p = q8(L.swin_block(L.depth_to_space(y), params["dec0_block"], T.block_offset(T.TABLE, "second", 1), 2, + T.M_BLOCK_CONSTS)) + self.assertTrue(_same(m["p"], p)) + self.assertTrue(_same(out["a"], p[:18, :30][..., list(T.DEC0_VALUES)].astype(np.float32))) + for k in ("A", "B", "C", "latent"): + self.assertTrue(np.isfinite(out[k]).all(), k) + + def test_depth_to_space_inverts_space_to_depth(self): + v = np.random.default_rng(11).normal(size=(6, 8, 5)) + self.assertTrue(_same(L.depth_to_space(L.space_to_depth(v)), v)) + + def test_pending(self): with self.assertRaises(NotImplementedError): L.reconstruct() + rng = np.random.default_rng(12) + x = MD.random_inputs(rng, rw=20, rh=12) + with self.assertRaises(NotImplementedError): + L.decode(x, L.random_params(rng), 0, T.table(dec_pixel="other")) class NetworkTest(unittest.TestCase): @@ -248,12 +256,14 @@ def _inputs(self, rng, rw, rh): return xe, xd def test_reduces_to_model_m_chain(self): - """With M's encoder op list and no decoder block, the L network is Model M's oracle chain.""" - rng = np.random.default_rng(12) + """With Model M's encoder and decoder op lists, the L network is Model M's oracle chain.""" + rng = np.random.default_rng(13) xe, xd = self._inputs(rng, 40, 24) - t = T.table(enc_ops=M_ENC_OPS, dec_connect=(("pass", 0, 64),)) + t = T.table(enc_ops=M_ENC_OPS, dec_ops=M_DEC_OPS) p = L.random_params(rng, t) p["enc0"] = ME.random_params(rng) + pm_dec = MD.random_params(rng) + p["dec0"] = m_dec_as_l(pm_dec) for parity in (0, 1): got = L.network(xe, xd, p, parity, t) x = ME.encode(xe, p["enc0"])["features"] @@ -273,12 +283,12 @@ def test_reduces_to_model_m_chain(self): self.assertTrue(_same(got[s][key], x), (s, parity)) xx = dict(xd) xx["features"] = x - want = MD.decode(xx, p["dec0"]) + want = MD.decode(xx, pm_dec) for k in ("A", "B", "C", "latent"): self.assertTrue(_same(got["dec0"][k], want[k]), (k, parity)) def test_default_network_runs(self): - rng = np.random.default_rng(13) + rng = np.random.default_rng(14) xe, xd = self._inputs(rng, 36, 20) p = L.random_params(rng) out = L.network(xe, xd, p, 0) @@ -292,6 +302,16 @@ class TableTest(unittest.TestCase): def test_shared_stages_are_model_m(self): self.assertEqual(tuple(T.SHARED_STAGES), tuple(k for k in MS.STAGES)) + def test_param_shapes_are_the_maps(self): + """Model L's own stages with the shapes of the preset-L converter map's records.""" + blk = {"gamma1": (32,), "gamma2": (32,), "Wqk": (2, 32, 32), "Wv": (2, 32, 32), "B": (2, 16, 16), + "Wp": (32, 64), "bp": (32,), "W1": (128, 32), "b1": (128,), "W2": (32, 128), "b2": (32,)} + want = {"enc0": {"W": (64, 128), "b": (64,), "T": (8, 16, 16), "sigma1": (), "sigma2": (), + "Win1": (32, 20), "bin1": (32,), "Win2": (32, 32), "bin2": (32,)}, + "enc0_block": blk, "dec0_block": blk, + "dec0": {"W": (4, 32, 64), "b": (4, 32), "S": (), "T": ()}, "post": {"s": ()}} + self.assertEqual(T.param_shapes(), want) + def test_macs(self): self.assertEqual(T.block_macs_per_token(64, 2, 256), 47104) self.assertEqual(T.block_macs_per_token(96, 4, 384), 114688) @@ -299,15 +319,12 @@ def test_macs(self): self.assertEqual(T.block_macs_per_token(32, 2, 128), 16384) r = T.frame_macs(T.TABLE, 1706, 960) self.assertEqual(r["M"], 103089438720) - self.assertGreater(r["L"], r["M"]) - - def test_provisional_keys_cover_entries(self): - self.assertEqual(set(T.PROVISIONAL), {f"I{i}" for i in range(1, 11)}) + self.assertEqual(round(r["L"] / 1e9), 163) def test_table_copy(self): - t = T.table(recon_ratio=(3, 1)) - self.assertEqual(t["recon_ratio"], (3, 1)) - self.assertEqual(T.TABLE["recon_ratio"], (3, 2)) + t = T.table(dec_pixel="x") + self.assertEqual(t["dec_pixel"], "x") + self.assertEqual(T.TABLE["dec_pixel"], "m") with self.assertRaises(KeyError): T.table(nonsense=1)