|
| 1 | +# SPDX-License-Identifier: MIT |
| 2 | +"""Model L weight packing (issue #89): the device blobs of Model L's own launches, from the structure table |
| 3 | +(clean/l/reference/table.py) and the logical tensors of the oracle (clean/l/reference/oracle.py). |
| 4 | +
|
| 5 | +Model L's encoder and decoder are run as a chain of small FP8-map kernels, each with a blob in a format Model M |
| 6 | +already uses (clean/FORMATS.md): |
| 7 | +
|
| 8 | +| Blob | Format | Kernel input -> output | |
| 9 | +|---|---|---| |
| 10 | +| `enc0.values` | Model M's enc0 blob (clean/runner/enc0.py pack) with W = 0 and b = 0 | frame -> the quantised values as a token map [F/2][80] (space_to_depth order 20 p + i) | |
| 11 | +| `enc0.<w>` per linear | merge / expand section format: bias tile vector (rows R_L(C_out)), then fragments [mt < C_out/16][kc < C_in/16], rows R_L(C_out), K order K_L(C_in) | FP8 map [C_in] -> FP8 map [C_out]; a `select` feeding the linear is folded in as zero weights outside its channels | |
| 12 | +| `enc0.<prefix>` per block | Model M's Swin block section (packing.pack_block) at the block's width, padded to 256 | FP8 map -> FP8 map | |
| 13 | +| `enc0.project` | bias tile vector (rows R_L(n_out)), then fragments [mt][kb], rows R_L(n_out), K order: groups of 32 channels, block 2i + s holds channel 32 i + 16 h + 8 s + j (dec0's order, clean/runner/dec0.py kord_decoder, over every group) | the concatenated maps -> the 64 FP8 features enc1 reads | |
| 14 | +| `dec0.<prefix>` | Model M's Swin block section at the decoder block's width | dec1's channels of the block segment -> same width | |
| 15 | +| `dec0` | Model M's dec0 blob (clean/runner/dec0.py pack) | as Model M | |
| 16 | +
|
| 17 | +The ten shared Swin stages use clean/runner/packing.py pack_stage unchanged. |
| 18 | +""" |
| 19 | +import numpy as np |
| 20 | + |
| 21 | +from clean.l.reference import table as _table |
| 22 | +from clean.reference.fp8 import fp8_decode |
| 23 | +from clean.runner import dec0 as _dec0 |
| 24 | +from clean.runner import enc0 as _enc0 |
| 25 | +from clean.runner import packing as P |
| 26 | + |
| 27 | +ALIGN = P.ALIGN |
| 28 | + |
| 29 | + |
| 30 | +# ---- orders --------------------------------------------------------------------------------------------------------- |
| 31 | + |
| 32 | +def kord_groups32(n): |
| 33 | + """K slot (h, j) of block kb holds channel 32 (kb // 2) + 16 h + 8 (kb % 2) + j: each group of 32 channels is two |
| 34 | + whole blocks, so a kernel can round its running sum after every second block.""" |
| 35 | + if n % 32: |
| 36 | + raise ValueError(f"project input of {n} channels: groups of 32 only") |
| 37 | + K = np.empty(n, dtype=np.int64) |
| 38 | + for kb in range(n // 16): |
| 39 | + for h in range(2): |
| 40 | + for j in range(8): |
| 41 | + K[16 * kb + 8 * h + j] = 32 * (kb // 2) + 16 * h + 8 * (kb % 2) + j |
| 42 | + return K |
| 43 | + |
| 44 | + |
| 45 | +# ---- blob packers --------------------------------------------------------------------------------------------------- |
| 46 | + |
| 47 | +def pack_linear(W, b, C_in, C_out, lo=0, d_row="gfx12"): |
| 48 | + """W [C_out, n] reading input channels lo .. lo + n of a C_in-wide map (zero weights elsewhere), bias [C_out].""" |
| 49 | + drow = P.resolve_d_row(d_row) |
| 50 | + W = np.asarray(W, dtype=np.float64) |
| 51 | + n = W.shape[1] |
| 52 | + if W.shape[0] != C_out or lo < 0 or lo + n > C_in or C_in % 16 or C_out % 16: |
| 53 | + raise ValueError(f"linear {W.shape} at {lo} of {C_in} -> {C_out}: bad shape (multiples of 16 needed)") |
| 54 | + Wx = np.zeros((C_out, C_in)) |
| 55 | + Wx[:, lo:lo + n] = W |
| 56 | + RL = P.rows_layout_l(C_out, drow) |
| 57 | + return P.pack_tile_vector(b, RL, drow) + P.pack_frags(Wx, RL, P.kord_layout_l(C_in)) |
| 58 | + |
| 59 | + |
| 60 | +def pack_project(W, b, groups, d_row="gfx12"): |
| 61 | + drow = P.resolve_d_row(d_row) |
| 62 | + W = np.asarray(W, dtype=np.float64) |
| 63 | + n_out, n = W.shape |
| 64 | + if any(g != 32 for g in groups) or sum(groups) != n: |
| 65 | + raise ValueError(f"project groups {groups}: groups of 32 covering {n} channels only") |
| 66 | + RL = P.rows_layout_l(n_out, drow) |
| 67 | + return P.pack_tile_vector(b, RL, drow) + P.pack_frags(W, RL, kord_groups32(n)) |
| 68 | + |
| 69 | + |
| 70 | +def pack_block(tensors, prefix, C, heads, mlp, d_row="gfx12"): |
| 71 | + drow = P.resolve_d_row(d_row) |
| 72 | + p = {k[len(prefix):]: v for k, v in tensors.items() if k.startswith(prefix)} |
| 73 | + blob = P.pack_block(p, C, heads, mlp, drow) |
| 74 | + return blob + bytes((-len(blob)) % ALIGN) |
| 75 | + |
| 76 | + |
| 77 | +def pack_values(tensors, d_row="gfx12"): |
| 78 | + """Model M's enc0 blob with zero projection: only the dither table and the two scalars are used.""" |
| 79 | + t = {"W": np.zeros((64, 80), np.float32), "b": np.zeros(64, np.float32), |
| 80 | + "T": tensors.get("T", np.zeros((8, 16, 16), np.float32)), |
| 81 | + "sigma1": tensors.get("sigma1", np.float32(0)), "sigma2": tensors.get("sigma2", np.float32(0))} |
| 82 | + return _enc0.pack(t, d_row) |
| 83 | + |
| 84 | + |
| 85 | +# ---- the encoder's launch list -------------------------------------------------------------------------------------- |
| 86 | + |
| 87 | +def encoder_launches(t=_table.TABLE): |
| 88 | + """The kernels of the table's encoder, in order: [{"kind", "blob", "src", "dst", ...}]. kinds: "values" (quant |
| 89 | + + space_to_depth into the raw map), "linear", "swin", "project". A select is folded into the linear it feeds.""" |
| 90 | + ops = t["enc_ops"] |
| 91 | + plan = _table.encoder_plan(t) |
| 92 | + chans = {"values": 20} |
| 93 | + launches = [] |
| 94 | + selects = {} |
| 95 | + i = 0 |
| 96 | + while i < len(ops): |
| 97 | + op = ops[i] |
| 98 | + k = op["op"] |
| 99 | + if k == "quant": |
| 100 | + nxt = ops[i + 1] if i + 1 < len(ops) else None |
| 101 | + if not nxt or nxt["op"] != "space_to_depth" or nxt["src"] != op["dst"]: |
| 102 | + raise ValueError("the encoder's kernels need quant directly followed by its space_to_depth") |
| 103 | + if op["mode"] != "dither": |
| 104 | + raise ValueError("the encoder's value kernel quantises with the dither table") |
| 105 | + launches.append({"kind": "values", "blob": "enc0.values", "dst": nxt["dst"]}) |
| 106 | + chans[nxt["dst"]] = 80 |
| 107 | + i += 2 |
| 108 | + continue |
| 109 | + if k == "select": |
| 110 | + selects[op["dst"]] = (op["src"], op["lo"], op["hi"]) |
| 111 | + chans[op["dst"]] = op["hi"] - op["lo"] |
| 112 | + elif k == "linear": |
| 113 | + src, lo = op["src"], 0 |
| 114 | + if src in selects: |
| 115 | + src, lo, _ = selects[src] |
| 116 | + launches.append({"kind": "linear", "blob": "enc0." + op["w"], "src": src, "dst": op["dst"], |
| 117 | + "C_in": chans[src], "C_out": op["n_out"], "lo": lo, "w": op["w"], "b": op["b"]}) |
| 118 | + chans[op["dst"]] = op["n_out"] |
| 119 | + elif k == "swin": |
| 120 | + launches.append({"kind": "swin", "blob": "enc0." + op["prefix"].rstrip("."), "src": op["src"], |
| 121 | + "dst": op["dst"], "prefix": op["prefix"], "C": op["C"], "heads": op["heads"], |
| 122 | + "mlp": op["mlp"], "offsets": op["offsets"]}) |
| 123 | + elif k == "concat": |
| 124 | + chans[op["dst"]] = plan[i][2] |
| 125 | + launches.append({"kind": "concat", "srcs": op["srcs"], "dst": op["dst"], "pad_to": plan[i][2]}) |
| 126 | + elif k == "project": |
| 127 | + launches.append({"kind": "project", "blob": "enc0.project", "src": op["src"], "groups": op["groups"]}) |
| 128 | + else: |
| 129 | + raise ValueError(f"no kernel for encoder op {k!r}") |
| 130 | + if k != "select": |
| 131 | + selects.pop(op.get("dst"), None) # the name now holds this op's output |
| 132 | + i += 1 |
| 133 | + return launches |
| 134 | + |
| 135 | + |
| 136 | +def pack_model_l(params, t=_table.TABLE, d_row="gfx12"): |
| 137 | + """{blob name: bytes} of Model L's own launches (the shared Swin stages: packing.pack_stage).""" |
| 138 | + enc = params["enc0"] |
| 139 | + out = {} |
| 140 | + for ln in encoder_launches(t): |
| 141 | + if ln["kind"] == "values": |
| 142 | + out[ln["blob"]] = pack_values(enc, d_row) |
| 143 | + elif ln["kind"] == "linear": |
| 144 | + out[ln["blob"]] = pack_linear(enc[ln["w"]], enc[ln["b"]], ln["C_in"], ln["C_out"], ln["lo"], d_row) |
| 145 | + elif ln["kind"] == "swin": |
| 146 | + out[ln["blob"]] = pack_block(enc, ln["prefix"], ln["C"], ln["heads"], ln["mlp"], d_row) |
| 147 | + elif ln["kind"] == "project": |
| 148 | + op = t["enc_ops"][-1] |
| 149 | + out[ln["blob"]] = pack_project(enc[op["w"]], enc[op["b"]], ln["groups"], d_row) |
| 150 | + dec = params["dec0"] |
| 151 | + if any(s[0] == "block" for s in t["dec_connect"]): |
| 152 | + b = t["dec_block"] |
| 153 | + out["dec0." + b["prefix"].rstrip(".")] = pack_block(dec, b["prefix"], b["C"], b["heads"], b["mlp"], d_row) |
| 154 | + out["dec0"] = _dec0.pack({k: dec[k] for k in ("W", "b", "S", "T")}, d_row) |
| 155 | + return out |
| 156 | + |
| 157 | + |
| 158 | +# ---- decoding (tests) ----------------------------------------------------------------------------------------------- |
| 159 | + |
| 160 | +def unpack_frags(raw, R, K): |
| 161 | + """Inverse of packing.frag_array: fragment bytes [MT][KC][256] -> the matrix M [len(R), len(K)] (FP8 values).""" |
| 162 | + MT, KC = len(R) // 16, len(K) // 16 |
| 163 | + a = np.frombuffer(bytes(raw), dtype=np.uint8).reshape(MT, KC, 2, 16, 8) # [mt, kc, h, r, j] |
| 164 | + P8 = a.transpose(0, 3, 1, 2, 4).reshape(16 * MT, 16 * KC) # [16 mt + r, 16 kc + 8 h + j] |
| 165 | + M = np.zeros((len(R), len(K)), dtype=np.float64) |
| 166 | + M[np.ix_(R, K)] = fp8_decode(P8) |
| 167 | + return M |
| 168 | + |
| 169 | + |
| 170 | +def unpack_tile_vector(raw, R, drow): |
| 171 | + v = np.frombuffer(bytes(raw), dtype=np.float32) |
| 172 | + out = np.empty(len(R), dtype=np.float32) |
| 173 | + for mt in range(len(R) // 16): |
| 174 | + for h in range(2): |
| 175 | + for k in range(8): |
| 176 | + out[R[16 * mt + drow[h][k]]] = v[16 * mt + 8 * h + k] |
| 177 | + return out |
| 178 | + |
| 179 | + |
| 180 | +def unpack_linear(blob, C_in, C_out, d_row="gfx12"): |
| 181 | + """(W [C_out, C_in], b [C_out]) from a linear blob.""" |
| 182 | + drow = P.resolve_d_row(d_row) |
| 183 | + RL = P.rows_layout_l(C_out, drow) |
| 184 | + b = unpack_tile_vector(blob[:4 * C_out], RL, drow) |
| 185 | + return unpack_frags(blob[4 * C_out:], RL, P.kord_layout_l(C_in)), b |
| 186 | + |
| 187 | + |
| 188 | +def unpack_project(blob, n_in, n_out, d_row="gfx12"): |
| 189 | + drow = P.resolve_d_row(d_row) |
| 190 | + RL = P.rows_layout_l(n_out, drow) |
| 191 | + b = unpack_tile_vector(blob[:4 * n_out], RL, drow) |
| 192 | + return unpack_frags(blob[4 * n_out:], RL, kord_groups32(n_in)), b |
0 commit comments