diff --git a/clean/l/FORMATS.md b/clean/l/FORMATS.md new file mode 100644 index 00000000..939c7039 --- /dev/null +++ b/clean/l/FORMATS.md @@ -0,0 +1,110 @@ +# Model L: kernels, maps, blobs and launch order (issue #89) + +This is the interface the Model L runtime frame builds against. +- A row marked **exists** has a merged HIP kernel. +- A row marked **planned** is the interface the kernel is being written to. Its names, arguments, grids and maps are + fixed here, and a change goes through this file first. +- Each D3D12 port declares the same code object, kernel name and arguments as its HIP kernel (`clean/dx12/kernels/l_*.hlsl`). + +The structure follows `clean/l/reference/table.py`. The launch list comes from `clean/l/runner/l_pack.py` +(`encoder_launches`, `decoder_launches`, with each map's `in_format` / `out_format`). The ten shared Swin stages +(enc1 ... dec1) use Model M's blobs, kernels and launch plan unchanged, with Model L's weights. + +## Maps + +Every map is `[rows][cols][C]`, channel fastest, as Model M's feature maps (`clean/FORMATS.md`, "Feature maps"). +- FP8 = E4M3, 1 byte per channel. FP16 = 2 bytes per channel, little-endian. +- F = fw x fh is the padded grid (multiples of 32). R = rw x rh is the render size. +- **Halo maps** have 2 extra pixels on each side: `[fh + 4][fw + 4][C]`. Map pixel (i, j) is grid position + (x, y) = (j - 2, i - 2). The halo pixels hold the per-pixel chain evaluated at those positions: the 20 values at + the mirrored render position, the dither of the unmirrored position. They are what the encoder block reads for + window members outside the grid. + +| Map | Shape | Element | Written by | Read by | +|---|---|---|---|---| +| `q` | `[fh + 4][fw + 4][32]` halo | FP8: values 0..19, bytes 20..31 zero | `lev/input_encoder` | `l_linear_32_32_relu` | +| `x1` | `[fh + 4][fw + 4][32]` halo | FP8 | `l_linear_32_32_relu` | `l_linear_32_32_f16` | +| `x2` | `[fh + 4][fw + 4][32]` halo | **FP16** (64 bytes per pixel) | `l_linear_32_32_f16` | `l_swin_block_c32_enc` | +| `x3` | `[fh][fw][32]` | FP8 | `l_swin_block_c32_enc` | `l_project_gather` | +| `enc0.features` | `[fh/2][fw/2][64]` | FP8 | `l_project_gather` | enc1 (Model M's stage) | +| m_near, m_far, m_center | as Model M | as Model M | `lev/input_encoder` | as Model M | +| `y` | `[fh][fw][32]` | FP8 | `l_dec_project` | `l_swin_block_c32_dec` | +| `y2` | `[fh][fw][32]` | **FP16** (64 bytes per pixel) | `l_swin_block_c32_dec` | `l_coef` | +| A, B, C, new latent | as Model M's `coef_decoder` outputs | as Model M | `l_coef` | as Model M | + +Extra bytes per frame: +- 32 (fh + 4)(fw + 4) for `q`; +- 32 (fh + 4)(fw + 4) for `x1`; +- 64 (fh + 4)(fw + 4) for `x2`; +- 32 fh fw for `x3`; +- 32 fh fw for `y`; +- 64 fh fw for `y2`. + +No kernel works in place. +- `x2` may reuse `q`'s memory (`q` is dead after `l_linear_32_32_relu`), if it is sized for `x2`. +- `y` and `y2` may reuse the encoder maps, which are dead after `l_project_gather`. + +## Blobs (`l_pack.pack_model_l`, names = the converter map's stage / tensor names) + +| Blob | Format | +|---|---| +| `enc0.values` | Model M's enc0 blob (`clean/runner/enc0.py` pack) with W = 0 and b = 0: dither table T, sigma1, sigma2 (the map's stored values) | +| `enc0.Win1`, `enc0.Win2` | merge / expand section format: bias tile vector (rows R_L(32)), then fragments [mt < 2][kc < 2], rows R_L(32), K order K_L(32). `Win1` has zero weights for inputs 20..31 | +| `enc0_block`, `dec0_block` | Model M's Swin block section (`packing.pack_block`) at C = 32, 2 heads, MLP 128, padded to 256 | +| `enc0.project` | bias tile vector (rows R_L(64)), then fragments [mt][kb], rows R_L(64). K order: block 2i + s of group i holds channel 32 i + 16 h + 8 s + j (dec0's order, `clean/runner/dec0.py` `kord_decoder`, over 4 groups) | +| `dec0.project` | bias tile vector (rows R_E(32)), then fragments [mt < 8][kb < 4], rows R_E(32) (tile 2 s + g: sub-pixel s, channels 16 g ...). K order as `enc0.project` (two groups of 32) | +| `dec0.coef` | Model M's dec0 blob (`clean/runner/dec0.py` pack) with W = 0 and b = 0: the scalars S and T | + +## Code objects + +| File | Source | Kernels | +|---|---|---| +| `l_enc_values.co` (loaded under the tag `lev`) | `clean/kernels/encoder.hip` with `-DWG_ENC_VALUES_OUT=1` | `input_encoder` | +| `l_ops.co` | `clean/l/kernels/l_ops.hip` | `l_linear_32_32_relu`, `l_linear_32_32_f16`, `l_project_gather`, `l_dec_project`, `l_coef` | +| `l_swin.co` | `clean/l/kernels/l_swin.hip` | `l_swin_block_c32[_nc]` (exists, #102: FP8 in / FP8 out, mirror border; block checks only), `l_swin_block_c32_enc[_nc]`, `l_swin_block_c32_dec[_nc]` | + +All are built with `clean/runner/build.py`'s `device_command`. The build's `-cuid` comes from the output file name, +so compare code objects only between outputs of the same name. `_nc` is used when bit 0 of the block blob's first +word is set, as for Model M's blocks. D3D12 ports exist for the `_nc` forms only. + +## Kernels + +Workgroups are 256 lanes (8 waves of 32) throughout. "Per-pixel" kernels handle 128 pixels (or tokens) per +workgroup, in raster order of their map. + +| Kernel | Status | Arguments | Grid | Does | +|---|---|---|---|---| +| `lev/input_encoder` | planned (rework of the old values-only variant) | as Model M's `input_encoder`, with `features` replaced by `q` | ceil(((fh + 4) / 2) ((fw + 4) / 2) / 128) (2 x 2 pixel groups of the halo map) | the 20 values and their dithered FP8 for every halo-map pixel, into `q`. The motion surfaces as Model M, only for pixels inside R. w = `enc0.values` | +| `l_linear_32_32_relu` | planned | `x, y, w, npix` | ceil(npix / 128), npix = (fh + 4)(fw + 4) | y = q8(max(0, RN16(b + W x))), FP8 -> FP8. w = `enc0.Win1` | +| `l_linear_32_32_f16` | planned | `x, y, w, npix` | as above | y = RN16(b + W x), FP8 -> **FP16**. w = `enc0.Win2` | +| `l_swin_block_c32_enc[_nc]` | planned | `x, xadd (null), out, w, H, W, ox, oy` | ceil(nwin / 8), nwin = ceil((W + ox) / 4) ceil((H + oy) / 4) | the encoder block. x = `x2` (FP16 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2)). out = `x3` (FP8, grid only). H = fh, W = fw. (ox, oy) = `window_offsets["first"]` for the frame parity. w = `enc0_block` | +| `l_project_gather` | planned | `x, features, w, tw, th` | ceil(tw th / 128), tw = fw / 2, th = fh / 2 | token (r, c) gathers pixels (2r + p / 2, 2c + p % 2), p = 0..3, as channels 32 p + i. Running FP16 sum per group of 32 from the bias, clamp +-6.28125, FP8 -> `enc0.features`. w = `enc0.project` | +| `l_dec_project` | planned | `z, y, w, tw, th` | ceil(tw th / 128) | per token of dec1's output z [th][tw][64]: 128 outputs (two RN16 groups of 32 inputs, then FP8). Output 32 s + k goes to pixel (2r + s / 2, 2c + s % 2), channel k, of `y`. w = `dec0.project` | +| `l_swin_block_c32_dec[_nc]` | planned | `x, xadd (null), out, w, H, W, ox, oy` | as the encoder block | the decoder block. x = `y` (FP8). A member outside the grid takes token-grid mirrored pixels with the sub-pixel kept. out = `y2` (**FP16**). (ox, oy) = `window_offsets["second"]`. w = `dec0_block` | +| `l_coef` | planned (new) | `y2, A, B, C, latent, w, params` | ceil((fh / 2)(fw / 2) / 128), lanes as Model M's `coef_decoder` | Model L's per-pixel functions on the 13 channels `table.L_DEC_VALUES` of `y2`: A, B, C in Model M's forms, latent = tanh(candidate). Writes the same buffers and formats as `coef_decoder`, pixels inside R only. **Reads no motion, previous latent, jitter or reset.** params = Model M's `WgDecParams` (only rw, rh, fw, fh are read). w = `dec0.coef` | + +## Launch order of one frame + +Model M's frame (`clean/FORMATS.md`, "Stage order per frame") with steps 2 and 4 replaced. Window offsets follow +the frame parity, as Model M's. + +2. Encoder: + 1. `lev/input_encoder` -> `q` (halo), m_near, m_far, m_center + 2. `l_linear_32_32_relu` q -> `x1` (halo) + 3. `l_linear_32_32_f16` x1 -> `x2` (halo, FP16) + 4. `l_swin_block_c32_enc[_nc]` x2 -> `x3` + 5. `l_project_gather` x3 -> `enc0.features` (what enc1 reads) +3. enc1 ... dec1: Model M's stages with Model L's weights, unchanged. +4. Decoder: + 1. `l_dec_project` dec1 output -> `y` + 2. `l_swin_block_c32_dec[_nc]` y -> `y2` (FP16) + 3. `l_coef` y2 -> A, B, C, new latent. Model L's latent recurrence runs only through the next frame's input + encoder, which samples this latent. + +The reconstruction (post) stage is still open. + +## Checks (GPU, synthetic data) + +- `clean/l/runner/l_c32_check.py`: the C = 32 block against the oracle. +- `clean/l/runner/l_d3d12_check.py` (with its D3D12 port): each D3D12 port against its HIP kernel, byte for byte. +- One check per new kernel follows with that kernel. diff --git a/clean/l/reference/oracle.py b/clean/l/reference/oracle.py index 21350286..5ae30fea 100644 --- a/clean/l/reference/oracle.py +++ b/clean/l/reference/oracle.py @@ -82,20 +82,42 @@ def window_grid(H, W, offset, win): return rows, cols -def swin_block(x, p, offset, heads, consts, max_tokens=1 << 16): - """One block on a whole map x [H, W, C] with mirrored borders (as clean/reference/swin.py swin_block); returns the - block output in float64, not yet rounded.""" +def border_index(border, n, halo=0): + """Grid position -> index into the block's input along one axis of n grid positions. + "mirror": about the edge pixel, edge not repeated (clean/reference/swin.py); "input": the input holds halo extra + positions on each side (position -halo is index 0); "coarse": the mirror of the 2 x 2 token grid (n / 2 tokens), + the sub-pixel kept.""" + if border == "mirror": + return lambda q: _swin.mirror(q, n) + if border == "input": + def f(q): + i = np.asarray(q) + halo + if (i < 0).any() or (i >= n + 2 * halo).any(): + raise AssertionError("window reaches beyond the input halo") + return i + return f + if border == "coarse": + return lambda q: 2 * _swin.mirror(np.asarray(q) // 2, n // 2) + np.asarray(q) % 2 + raise ValueError(f"unknown border {border!r}") + + +def swin_block(x, p, offset, heads, consts, border="mirror", halo=0, max_tokens=1 << 16): + """One block on a whole map x [H, W, C] (border "input": [H + 2 halo, W + 2 halo, C], the grid in the middle); + window members outside the grid per border_index. Returns the block output on the grid [H, W, C] in float64, not + yet rounded.""" x = np.asarray(x, dtype=np.float64) - H, W, C = x.shape + h = halo if border == "input" else 0 + H, W, C = x.shape[0] - 2 * h, x.shape[1] - 2 * h, x.shape[2] win = int(consts["window"]) p = {k: np.asarray(v, dtype=np.float64) for k, v in p.items()} rows, cols = window_grid(H, W, offset, win) - out = np.full_like(x, np.nan) + out = np.full((H, W, C), np.nan) + fr, fc = border_index(border, H, h), border_index(border, W, h) rows_per_chunk = max(1, max_tokens // (win * win * cols.shape[0])) for l0 in range(0, rows.shape[0], rows_per_chunk): r = rows[l0:l0 + rows_per_chunk] - rr = _swin.mirror(r, H)[:, None, :, None] - cc = _swin.mirror(cols, W)[None, :, None, :] + rr = fr(r)[:, None, :, None] + cc = fc(cols)[None, :, None, :] xw = x[rr, cc] nl, nk = xw.shape[:2] x2 = _block_tokens(xw.reshape(nl, nk, win * win, C), p, heads, consts).reshape(nl, nk, win, win, C) @@ -168,9 +190,9 @@ def _run_ops(ops, maps, params, parity, t, default_stage, frame=None): st = params[op.get("params", default_stage)] if k == "quant": v = maps[op["src"]] - fw, fh = frame["feature_size"] if op["mode"] == "dither": - d = _enc.dither_table(st["T"], frame["frame"], fw, fh) + h = maps.get("_halo", 0) + d = dither_table(st["T"], frame["frame"], v.shape[1], v.shape[0], h) out = _enc.dither_fp8(v, d[..., None]) elif op["mode"] == "q8": out = q8(v) @@ -187,7 +209,10 @@ def _run_ops(ops, maps, params, parity, t, default_stage, frame=None): elif k == "swin": consts = _table.block_consts(t, op["params"]) off = _table.block_offset(t, op["offsets"], parity) - y = swin_block(maps[op["src"]], st, off, op["heads"], consts) + border = op.get("border", "mirror") + y = swin_block(maps[op["src"]], st, off, op["heads"], consts, border, maps.get("_halo", 0)) + if border == "input": + maps["_halo"] = 0 out = _round(y, op.get("out_round", "q8")) elif k == "concat": parts = [maps[s] for s in op["srcs"]] @@ -209,6 +234,20 @@ def _run_ops(ops, maps, params, parity, t, default_stage, frame=None): return last +def dither_table(T, frame, w, h, halo=0): + """t = T[n][|y| mod 16][|x| mod 16] at grid positions x = -halo ... w - halo - 1 (y likewise), [h, w].""" + T = np.asarray(T) + Y, X = np.mgrid[0:h, 0:w] - halo + return T[int(frame)][np.abs(Y) % 16, np.abs(X) % 16] + + +def encoder_halo(t): + """Extra positions per side the encoder's per-pixel ops run on (a block with border "input"), else 0.""" + hs = [_table.block_halo(t, op["offsets"]) for op in t["enc_ops"] + if op["op"] == "swin" and op.get("border") == "input"] + return max(hs, default=0) + + def encode(x, params, parity=None, t=_table.TABLE, keep=False): """Model L's input encoder. x: Model M's enc0 input dict; params: {stage: tensors} with the stages of table.param_shapes(t) ("enc0", "enc0_block"). Returns Model M's enc0 outputs ("features" [fh/2, fw/2, 64] @@ -216,11 +255,24 @@ def encode(x, params, parity=None, t=_table.TABLE, keep=False): _table.encoder_plan(t) if parity is None: parity = int(x["frame"]) & 1 - _enc.check_inputs(x) + rw, rh = _enc.check_inputs(x)[:2] enc = params["enc0"] pv_params = {"sigma1": enc.get("sigma1", 0.0), "sigma2": enc.get("sigma2", 0.0)} - maps = {"values": _enc.pixel_values(x, pv_params)} + v = _enc.pixel_values(x, pv_params) + h = encoder_halo(t) + if h: + # The 20 values depend only on the mirrored render position (clean/reference/encoder.py reflect), which is + # a grid position, so the halo copies grid values; the dither is applied later at the unmirrored position. + # Only for render sizes near the padding (fw + h > 2 rw - 2) would the mirrored position be negative; reflect + # clamps it to 0 there (table.UNCONFIRMED "U1"). + fh, fw = v.shape[:2] + ry = _enc.reflect(np.arange(-h, fh + h), rh) + rx = _enc.reflect(np.arange(-h, fw + h), rw) + v = v[ry][:, rx] + maps = {"values": v, "_halo": h} feats = _run_ops(t["enc_ops"], maps, params, parity, t, "enc0", frame=x) + if maps.pop("_halo"): + raise ValueError("encoder: the per-pixel halo was not consumed by a block") m_near, m_far, m_center = _enc.motion_surfaces(x) res = {"features": feats.astype(np.float32), "m_near": f16(m_near).astype(np.float32), @@ -262,17 +314,47 @@ def coefficients(x, a, S, T): "codes_A": cA, "codes_B": cB, "codes_L": cL, "p": p} +def tanh32(v): + """tanh in float64, then one rounding to FP32.""" + return np.tanh(np.asarray(v, dtype=np.float64)).astype(np.float32) + + +def coefficients_l(a, S, T): + """Model L's per-pixel functions on a [rh, rw, 13] (table.L_DEC_VALUES order; FP16 block outputs held in FP32): + A, B and C with Model M's forms (C's gains from inputs clamped to [-20, 20]), latent = tanh(candidate). No gate, + no previous latent. Same outputs and codes as coefficients().""" + a = np.asarray(a, dtype=np.float32) + A = _dec.sigmoid16(a[..., 0:3]) + B = _dec.softmax3(a[..., 3:6]) + g = _dec.sigmoid16(np.clip(a[..., 6:8], np.float32(-20), np.float32(20))) + sx, sy = np.float32(S) * g[..., 0], np.float32(S) * g[..., 1] + rho = _dec.rho32(a[..., 8], float(np.asarray(T))) + C = np.stack([_dec.rn16(sx * sx), _dec.rn16(sy * sy), _dec.rn16((sx * sy) * rho)], axis=-1) + lat = tanh32(a[..., 9:13]) + cA, cB, cL = _dec.unorm8_code(A), _dec.unorm8_code(B), _dec.snorm8_code(lat) + return {"A": (cA / 255.0).astype(np.float32), "B": (cB / 255.0).astype(np.float32), + "C": C.astype(np.float32), "latent": (cL / 127.0).astype(np.float32), + "codes_A": cA, "codes_B": cB, "codes_L": cL} + + def decode(x, params, parity, t=_table.TABLE, keep=False): - """Model L's coefficient decoder. x: Model M's dec0 input dict (features = dec1's output [fh/2, fw/2, 64]); - params: {stage: tensors} ("dec0", "dec0_block").""" - if t["dec_pixel"] != "m": - raise NotImplementedError("decoder per-pixel variant other than Model M's: pending") + """Model L's coefficient decoder. x: Model M's dec0 input dict (features = dec1's output [fh/2, fw/2, 64]; with + dec_pixel "l" the motion / latent_prev / jitter / reset entries are not read); params: {stage: tensors} ("dec0", + "dec0_block").""" + kind = t["dec_pixel"] + if kind not in _table.DEC_PIXEL_VALUES: + raise NotImplementedError(f"decoder per-pixel variant {kind!r}: not known") _table.decoder_plan(t) + if len(t["dec_ops"][-1]["values"]) != _table.DEC_PIXEL_VALUES[kind]: + raise ValueError(f"dec_pixel {kind!r} needs {_table.DEC_PIXEL_VALUES[kind]} picked values") rw, rh = _dec.check_inputs(x)[:2] maps = {"features": np.asarray(x["features"], dtype=np.float64)} a = _run_ops(t["dec_ops"], maps, params, parity, t, "dec0") dec = params["dec0"] - out = coefficients(x, a[:rh, :rw], dec["S"], dec["T"]) + if kind == "l": + out = coefficients_l(a[:rh, :rw], dec["S"], dec["T"]) + else: + out = coefficients(x, a[:rh, :rw], dec["S"], dec["T"]) out["a"] = a[:rh, :rw].astype(np.float32) if keep: out["maps"] = maps diff --git a/clean/l/reference/table.py b/clean/l/reference/table.py index 18801492..db05219b 100644 --- a/clean/l/reference/table.py +++ b/clean/l/reference/table.py @@ -3,7 +3,8 @@ runtime and the tests. Every consumer interprets this table instead of hard-coding L's structure, so a provisional entry can be changed -without touching code. Entries listed in PROVISIONAL are pending confirmation; each carries a short key (I5 ...). +without touching code. Entries listed in PROVISIONAL are pending confirmation; each carries a short key (I7 ...). +UNCONFIRMED lists behaviour the reference had to choose without a confirmation (edge cases). Parameters are grouped by the converter map's stages (data/preset-L.v1.map.json): "enc0", "enc0_block", the ten shared stages, "dec0_block", "dec0", "post". SHARED_STAGES run with Model M's code and stage table @@ -23,9 +24,14 @@ M_BLOCK_CONSTS = {"tau": _swin.TAU, "cap": _swin.CAP, "eps_num": 2.0 ** -13, "gelu": "exact", "window": _swin.WIN, "head_dim": _swin.HEAD_DIM} -# The value positions of the coefficient decoder (the map's perm "dec0_values"): value k of a render pixel is -# channel DEC0_VALUES[k] of its 32. -DEC0_VALUES = (0, 1, 10, 11, 12, 13, 14, 15, 16, 2, 3, 4, 5, 6, 7, 8, 9) +# Model M's value positions (the map's perm "dec0_values", 17 values: A 0-2, B 3-5, C 6-8, latent gates 9-12, +# candidates 13-16 in clean/reference/decoder.py order): value k is channel M_DEC0_VALUES[k]. Model L does not use it. +M_DEC0_VALUES = (0, 1, 10, 11, 12, 13, 14, 15, 16, 2, 3, 4, 5, 6, 7, 8, 9) + +# Model L's value positions, 13 of the decoder block's 32 channels: value k is block channel L_DEC_VALUES[k], value +# order A0 A1 A2, B0 B1 B2, C x y rho, latent candidates 0-3 (no gates). The other 19 channels feed no output but +# stay in the block. The preset-L map from v1.1 on carries this list as perm "dec0_values" (v1 carried Model M's). +L_DEC_VALUES = (0, 1, 6, 7, 8, 9, 10, 11, 12, 2, 3, 4, 5) # ---- the table ------------------------------------------------------------------------------------------------------ @@ -37,52 +43,59 @@ # size: the pixel grid F, or the token grid F/2 after space_to_depth (2 x 2 pixels, sub-pixel p = dx + 2 dy, # channel = n p + i). Tensors are read from the stage named by "params" (default "enc0"). A linear's "acc" is the # rounding of bias + products before "out_round" ("relu_q8" = q8(max(0, .))). The last op is "project" (the 64 FP8 - # features enc1 reads): running sum s = b, s = RN16(s + group) per channel group, clamp, FP8. + # features enc1 reads): running sum s = b, s = RN16(s + group) per channel group, clamp, FP8. out_round "none" + # after acc "f16" keeps the FP16 value (the block then starts from FP16, not FP8). A swin op's "border" names how + # window members outside the grid get their values: "mirror" (clean/reference/swin.py, about the map size), + # "input" (the per-pixel ops before the block are evaluated at those positions: the 20 values at the mirrored + # render position, the dither at the unmirrored one) or "coarse" (the token-grid mirror, sub-pixel kept). "enc_ops": ( - {"op": "quant", "mode": "dither", "src": "values", "dst": "q"}, # I5 - {"op": "linear", "src": "q", "dst": "x", "w": "Win1", "b": "bin1", "n_in": 20, "n_out": 32, - "acc": "f16", "out_round": "relu_q8"}, # I11 - {"op": "linear", "src": "x", "dst": "x", "w": "Win2", "b": "bin2", "n_in": 32, "n_out": 32, - "acc": "f16", "out_round": "q8"}, # I11 - {"op": "swin", "src": "x", "dst": "x", "params": "enc0_block", **C32_BLOCK, "offsets": "first", - "out_round": "q8"}, # I10, I6 - {"op": "space_to_depth", "src": "x", "dst": "z"}, - {"op": "project", "src": "z", "w": "W", "b": "b", "groups": (32, 32, 32, 32), "n_out": 64, - "clamp": 6.28125}, # I12 + {"op": "quant", "mode": "dither", "src": "values", "dst": "q"}, + {"op": "linear", "src": "q", "dst": "x1", "w": "Win1", "b": "bin1", "n_in": 20, "n_out": 32, + "acc": "f16", "out_round": "relu_q8"}, + {"op": "linear", "src": "x1", "dst": "x2", "w": "Win2", "b": "bin2", "n_in": 32, "n_out": 32, + "acc": "f16", "out_round": "none"}, + {"op": "swin", "src": "x2", "dst": "x3", "params": "enc0_block", **C32_BLOCK, "offsets": "first", + "border": "input", "out_round": "q8"}, + {"op": "space_to_depth", "src": "x3", "dst": "z"}, + {"op": "project", "src": "z", "dst": "enc0.features", "w": "W", "b": "b", "groups": (32, 32, 32, 32), + "n_out": 64, "clamp": 6.28125}, ), # Coefficient decoder: ops on dec1's map ("features", [fh/2, fw/2, 64]). "dec_project": rows W [4, 32, 64] with # Model M's dec0 rounding (RN16 after each group of 32 input channels, then FP8), output [fh/2, fw/2, 4 x 32] - # (sub-pixel s = dx + 2 dy, channel 32 s + c); "depth_to_space" makes it a pixel map [fh, fw, 32]; "pick" takes - # the value positions of the coefficient functions, which then run as Model M's (clean/reference/decoder.py). + # (sub-pixel s = dx + 2 dy, channel 32 s + c); "depth_to_space" makes it a pixel map [fh, fw, 32]; the block's + # output stays FP16; "pick" takes the value positions of the per-pixel functions named by "dec_pixel": "l" + # (Model L's: A, B, C as Model M's, latent = tanh(candidate), no gate, no previous latent) or "m" (Model M's, + # clean/reference/decoder.py, 17 values). "dec_ops": ( - {"op": "dec_project", "src": "features", "dst": "y", "w": "W", "b": "b", "groups": (32, 32), - "out_round": "q8"}, # I13 - {"op": "depth_to_space", "src": "y", "dst": "p"}, - {"op": "swin", "src": "p", "dst": "p", "params": "dec0_block", **C32_BLOCK, "offsets": "second", - "out_round": "q8"}, # I10, I14 - {"op": "pick", "src": "p", "values": DEC0_VALUES}, # I14 + {"op": "dec_project", "src": "features", "dst": "yt", "w": "W", "b": "b", "groups": (32, 32), + "out_round": "q8"}, + {"op": "depth_to_space", "src": "yt", "dst": "y"}, + {"op": "swin", "src": "y", "dst": "y2", "params": "dec0_block", **C32_BLOCK, "offsets": "second", + "border": "coarse", "out_round": "f16"}, + {"op": "pick", "src": "y2", "values": L_DEC_VALUES}, ), - "dec_pixel": "m", # I8 + "dec_pixel": "l", "post_variant": "unfolded", # I8 # Reconstruction variants and the rule that picks one per frame from the render and output sizes ("rule" names # the comparison, "ratio" its factor). Only the 3/2 form (D = floor((3 O + 1) / 2)) exists in WrongGPU so far. "recon": {"rule": "both_axes_at_least", "ratio": 3, "variants": ("3/2", "3/1")}, # I7 - # Window offsets (ox, oy) of an L block for frame parity 0 and 1; the blocks run on the padded grid F. - "window_offsets": {"first": ((2, 2), (0, 2)), "second": ((0, 0), (2, 0))}, # I10 + # Window offsets (ox, oy) of an L block for frame parity 0 and 1, in pixels; the blocks run on the padded grid F + # and keep every grid position's output. + "window_offsets": {"first": ((2, 2), (0, 2)), "second": ((0, 0), (2, 0))}, # Per-block constants; a block's own entry (key = its params stage) wins over "default". - "consts": {"default": dict(M_BLOCK_CONSTS)}, # I6 + "consts": {"default": dict(M_BLOCK_CONSTS)}, } +DEC_PIXEL_VALUES = {"l": len(L_DEC_VALUES), "m": len(M_DEC0_VALUES)} + PROVISIONAL = { - "I5": "enc_ops quant mode (dither table or plain FP8 rounding)", - "I6": "consts of the width-32 blocks", "I7": "recon: the 3/1 form and the exact rule", - "I8": "dec_pixel (coefficient functions as Model M's), post_variant", - "I10": "window_offsets of the width-32 blocks, and their grid (F)", - "I11": "acc / out_round of the input linears", - "I12": "project clamp and group structure", - "I13": "dec_project rounding", - "I14": "pick value positions and the rounding of the decoder block's output", + "I8": "post_variant", +} + +UNCONFIRMED = { + "U1": "encoder halo: a mirrored render position below 0 (only when fw + halo > 2 rw - 2, i.e. render sizes near " + "the padding) is clamped to 0", } @@ -106,6 +119,14 @@ def block_offset(t, kind, parity): return tuple(t["window_offsets"][kind][int(parity) & 1]) +BORDERS = ("mirror", "input", "coarse") + + +def block_halo(t, kind): + """Pixels beyond the grid that a block's windows reach (grid sides multiples of the window): the largest offset.""" + return max(max(o) for o in t["window_offsets"][kind]) + + # ---- parameter shapes ----------------------------------------------------------------------------------------------- def block_shapes(C, heads, mlp, head_dim=32, window=4): @@ -177,6 +198,13 @@ def _plan(ops, maps, what, last): elif k == "swin": if src[1] != op["C"]: raise ValueError(f"swin: input has {src[1]} channels, block width {op['C']}") + border = op.get("border", "mirror") + if border not in BORDERS: + raise ValueError(f"swin: unknown border {border!r}") + if border != "mirror" and src[0] != "pixel": + raise ValueError(f"swin: border {border!r} needs a pixel-level map") + if border == "input" and any(p[0] not in ("quant", "linear", "select", "concat") for p in plan): + raise ValueError("swin: border 'input' needs only per-pixel ops before the block") out = src elif k == "concat": parts = [maps[s] for s in op["srcs"]] diff --git a/clean/l/runner/l_c32_check.py b/clean/l/runner/l_c32_check.py index 2084ade7..41af2eb6 100644 --- a/clean/l/runner/l_c32_check.py +++ b/clean/l/runner/l_c32_check.py @@ -104,19 +104,24 @@ def block(self, co, kernel, x, blob, offset, C): return fp8_decode(out) +CONSTS = dict(T.M_BLOCK_CONSTS, gelu="sigmoid") + + +def make_case(rng, H, W): + """Synthetic block parameters, their blob and an FP8 input map [H, W, 32] (CPU only).""" + p = L.random_block_params(rng, 32, 2, 128) + return p, LP.pack_block(p, 32, 2, 128), q8(rng.normal(0, 1.5, (H, W, 32))) + + def run(build, rocm, seeds=(89,)): g = Gpu(build, rocm) rngs = [np.random.default_rng(s) for s in seeds] - consts = dict(T.M_BLOCK_CONSTS, gelu="sigmoid") res = {"c32": [], "c32_nc_identical": None, "c64_template_identical": None} nc_same = [] for rng, (H, W, off) in ((r, c) for r in rngs for c in CASES): - p = L.random_block_params(rng, 32, 2, 128, "blk.") - blob = LP.pack_block(p, "blk.", 32, 2, 128) - x = q8(rng.normal(0, 1.5, (H, W, 32))) + p, blob, x = make_case(rng, H, W) got = g.block("l_swin.co", "l_swin_block_c32", x, blob, off, 32) - bp = {k[4:]: v for k, v in p.items()} - ref = q8(L.swin_block(x, bp, off, 2, consts)) + ref = q8(L.swin_block(x, p, off, 2, CONSTS)) m = metrics(got, ref) m.update({"case": [H, W, list(off)], "pass": passes(m)}) res["c32"].append(m) diff --git a/clean/l/runner/l_pack.py b/clean/l/runner/l_pack.py index c237afab..28de34f1 100644 --- a/clean/l/runner/l_pack.py +++ b/clean/l/runner/l_pack.py @@ -2,18 +2,18 @@ """Model L weight packing (issue #89): the device blobs of Model L's own launches, from the structure table (clean/l/reference/table.py) and the parameters of the oracle ({stage: tensors}, clean/l/reference/oracle.py). -Model L's encoder and decoder run as a chain of small FP8-map kernels, each with a blob in a format Model M already -uses (clean/FORMATS.md). Maps are [tokens][C] FP8, channel fastest; pixel maps are on the padded grid F, token maps -on F/2. +Model L's encoder and decoder run as a chain of small map kernels, each with a blob in a format Model M already +uses (clean/FORMATS.md). Maps are [tokens][C], channel fastest, FP8 or FP16 as each launch's in_format / +out_format says (map_format); pixel maps are on the padded grid F, token maps on F/2. | Blob | Format | Kernel input -> output | |---|---|---| | `enc0.values` | Model M's enc0 blob (clean/runner/enc0.py pack) with W = 0 and b = 0 | frame -> the quantized values as a pixel map [F][32] (values 0..19, 12 zero bytes) | -| `enc0.` per linear | merge / expand section format: bias tile vector (rows R_L(C_out)), then fragments [mt < C_out/16][kc < C_in/16], rows R_L(C_out), K order K_L(C_in); inputs narrower than C_in (20 values) get zero weights | FP8 map [C_in] -> FP8 map [C_out] | -| `enc0_block`, `dec0_block` | Model M's Swin block section (packing.pack_block) at the block's width, padded to 256 | FP8 map -> FP8 map | +| `enc0.` per linear | merge / expand section format: bias tile vector (rows R_L(C_out)), then fragments [mt < C_out/16][kc < C_in/16], rows R_L(C_out), K order K_L(C_in); inputs narrower than C_in (20 values) get zero weights | FP8 map [C_in] -> map [C_out]: FP8 (Win1, ReLU) or FP16 (Win2: the encoder block's input) | +| `enc0_block`, `dec0_block` | Model M's Swin block section (packing.pack_block) at the block's width, padded to 256 | enc0_block: FP16 map -> FP8 map, window members outside the grid from the per-pixel chain (border "input"); dec0_block: FP8 map -> FP16 map, token-grid mirror (border "coarse") | | `enc0.project` | bias tile vector (rows R_L(64)), then fragments [mt][kb], rows R_L(64), K order: block 2i + s of group i holds channel 32 i + 16 h + 8 s + j (dec0's order, clean/runner/dec0.py kord_decoder, over every group) | the 2 x 2 pixels of each token (channel 32 p + i, p = dx + 2 dy) -> the 64 FP8 features enc1 reads | | `dec0.project` | expand section format: bias tile vector (rows R_E(32)), then fragments [mt < 8][kb < 4], rows R_E(32) (tile 2 s + g: sub-pixel s, channels 16 g ..), K order as `enc0.project` (two groups of 32) | dec1's map [F/2][64] -> pixel map [F][32] | -| `dec0.coef` | Model M's dec0 blob (clean/runner/dec0.py pack) with W = 0 and b = 0: the scalars S, T and the code tables | the decoder block's pixel map -> A, B, C, latent | +| `dec0.coef` | Model M's dec0 blob (clean/runner/dec0.py pack) with W = 0 and b = 0: the scalars S, T and the code tables | the decoder block's FP16 pixel map -> A, B, C, latent (dec_pixel "l": 13 values, latent = tanh(candidate), no previous latent) | The ten shared Swin stages use clean/runner/packing.py pack_stage unchanged. """ @@ -101,36 +101,58 @@ def pack_coef(tensors, d_row="gfx12"): # ---- launch lists --------------------------------------------------------------------------------------------------- +def map_format(acc, out_round): + """Storage format of a kernel's output map: "fp8" (1 byte per channel) or "fp16" (2 bytes per channel).""" + if out_round in ("q8", "relu_q8"): + return "fp8" + if out_round == "f16" or (out_round in (None, "none") and acc == "f16"): + return "fp16" + raise ValueError(f"no map format for acc {acc!r} / out_round {out_round!r}") + + +def _swin_launch(op, in_format, out_format): + return {"kind": "swin", "blob": op["params"], "src": op["src"], "dst": op["dst"], "C": op["C"], + "heads": op["heads"], "mlp": op["mlp"], "offsets": op["offsets"], "params": op["params"], + "border": op.get("border", "mirror"), "in_format": in_format, "out_format": out_format} + + def encoder_launches(t=_table.TABLE): """The encoder's kernels in order: [{"kind", "blob", ...}]. kinds: "values" (quant into a pixel map of 32 channels), "linear", "swin", "project" (with the space_to_depth gather folded in).""" ops = list(t["enc_ops"]) _table.encoder_plan(t) width = {"values": 20} + fmt = {} out = [] for i, op in enumerate(ops): k = op["op"] if k == "quant": if op["mode"] != "dither" or i != 0: raise ValueError("the encoder's value kernel quantizes the per-pixel values with the dither table") - out.append({"kind": "values", "blob": "enc0.values", "dst": op["dst"], "C": 32}) + out.append({"kind": "values", "blob": "enc0.values", "dst": op["dst"], "C": 32, "out_format": "fp8"}) width[op["dst"]] = 32 + fmt[op["dst"]] = "fp8" elif k == "linear": c_in = width[op["src"]] + fo = map_format(op.get("acc"), op.get("out_round", "q8")) out.append({"kind": "linear", "blob": "enc0." + op["w"], "src": op["src"], "dst": op["dst"], "C_in": c_in, "C_out": op["n_out"], "w": op["w"], "b": op["b"], "acc": op.get("acc"), - "out_round": op.get("out_round", "q8")}) + "out_round": op.get("out_round", "q8"), "in_format": fmt[op["src"]], "out_format": fo}) width[op["dst"]] = op["n_out"] + fmt[op["dst"]] = fo elif k == "swin": - out.append({"kind": "swin", "blob": op["params"], "src": op["src"], "dst": op["dst"], "C": op["C"], - "heads": op["heads"], "mlp": op["mlp"], "offsets": op["offsets"], "params": op["params"]}) + fo = map_format("f16", op.get("out_round", "q8")) + out.append(_swin_launch(op, fmt[op["src"]], fo)) + fmt[op["dst"]] = fo + width[op["dst"]] = width[op["src"]] elif k == "space_to_depth": nxt = ops[i + 1] if i + 1 < len(ops) else None if not nxt or nxt["op"] != "project" or nxt["src"] != op["dst"]: raise ValueError("space_to_depth must feed the project directly (one gather kernel)") width[op["dst"]] = 4 * width[op["src"]] - out.append({"kind": "project", "blob": "enc0.project", "src": op["src"], "groups": nxt["groups"], - "w": nxt["w"], "b": nxt["b"], "clamp": nxt.get("clamp")}) + out.append({"kind": "project", "blob": "enc0.project", "src": op["src"], + "dst": nxt.get("dst", "enc0.features"), "groups": nxt["groups"], "w": nxt["w"], "b": nxt["b"], + "clamp": nxt.get("clamp"), "in_format": fmt[op["src"]], "out_format": "fp8"}) elif k == "project": continue # folded into the gather above else: @@ -143,6 +165,7 @@ def decoder_launches(t=_table.TABLE): coefficient functions).""" ops = list(t["dec_ops"]) _table.decoder_plan(t) + fmt = {} out = [] for i, op in enumerate(ops): k = op["op"] @@ -150,15 +173,19 @@ def decoder_launches(t=_table.TABLE): nxt = ops[i + 1] if i + 1 < len(ops) else None if not nxt or nxt["op"] != "depth_to_space" or nxt["src"] != op["dst"]: raise ValueError("dec_project must feed depth_to_space directly (one scatter kernel)") - out.append({"kind": "dec_project", "blob": "dec0.project", "w": op["w"], "b": op["b"], - "groups": op["groups"], "dst": nxt["dst"]}) + fmt[nxt["dst"]] = map_format("f16", op.get("out_round", "q8")) + out.append({"kind": "dec_project", "blob": "dec0.project", "src": op["src"], "dst": nxt["dst"], + "w": op["w"], "b": op["b"], "groups": op["groups"], "in_format": "fp8", + "out_format": fmt[nxt["dst"]]}) elif k == "depth_to_space": continue elif k == "swin": - out.append({"kind": "swin", "blob": op["params"], "src": op["src"], "dst": op["dst"], "C": op["C"], - "heads": op["heads"], "mlp": op["mlp"], "offsets": op["offsets"], "params": op["params"]}) + fo = map_format("f16", op.get("out_round", "q8")) + out.append(_swin_launch(op, fmt[op["src"]], fo)) + fmt[op["dst"]] = fo elif k == "pick": - out.append({"kind": "coef", "blob": "dec0.coef", "src": op["src"], "values": tuple(op["values"])}) + out.append({"kind": "coef", "blob": "dec0.coef", "src": op["src"], "values": tuple(op["values"]), + "dec_pixel": t["dec_pixel"], "in_format": fmt[op["src"]], "dst": "codes"}) else: raise ValueError(f"no kernel for decoder op {k!r}") return out diff --git a/clean/l/runtime/l_pack.cpp b/clean/l/runtime/l_pack.cpp index cb39b0a4..243692b7 100644 --- a/clean/l/runtime/l_pack.cpp +++ b/clean/l/runtime/l_pack.cpp @@ -36,10 +36,11 @@ int sum(const std::vector& v) { return n; } -Launch swin(const std::string& src, const std::string& offsets, const std::string& params) { +Launch swin(const std::string& src, const std::string& dst, const std::string& offsets, const std::string& params, + const std::string& border, const std::string& in_format, const std::string& out_format) { Launch l; - l.kind = "swin", l.blob = params, l.src = src, l.dst = src, l.C = 32, l.heads = 2, l.mlp = 128; - l.offsets = offsets, l.params = params; + l.kind = "swin", l.blob = params, l.src = src, l.dst = dst, l.C = 32, l.heads = 2, l.mlp = 128; + l.offsets = offsets, l.params = params, l.border = border, l.in_format = in_format, l.out_format = out_format; return l; } @@ -48,14 +49,17 @@ Launch swin(const std::string& src, const std::string& offsets, const std::strin const std::vector& encoder_launches() { static const std::vector v = [] { std::vector l(5); - l[0].kind = "values", l[0].blob = "enc0.values", l[0].dst = "q", l[0].C = 32; - l[1].kind = "linear", l[1].blob = "enc0.Win1", l[1].src = "q", l[1].dst = "x"; + l[0].kind = "values", l[0].blob = "enc0.values", l[0].dst = "q", l[0].C = 32, l[0].out_format = "fp8"; + l[1].kind = "linear", l[1].blob = "enc0.Win1", l[1].src = "q", l[1].dst = "x1"; l[1].C_in = 32, l[1].C_out = 32, l[1].w = "Win1", l[1].b = "bin1", l[1].acc = "f16", l[1].out_round = "relu_q8"; - l[2].kind = "linear", l[2].blob = "enc0.Win2", l[2].src = "x", l[2].dst = "x"; - l[2].C_in = 32, l[2].C_out = 32, l[2].w = "Win2", l[2].b = "bin2", l[2].acc = "f16", l[2].out_round = "q8"; - l[3] = swin("x", "first", "enc0_block"); - l[4].kind = "project", l[4].blob = "enc0.project", l[4].src = "x", l[4].groups = {32, 32, 32, 32}; - l[4].w = "W", l[4].b = "b", l[4].clamp = 6.28125f; + l[1].in_format = "fp8", l[1].out_format = "fp8"; + l[2].kind = "linear", l[2].blob = "enc0.Win2", l[2].src = "x1", l[2].dst = "x2"; + l[2].C_in = 32, l[2].C_out = 32, l[2].w = "Win2", l[2].b = "bin2", l[2].acc = "f16", l[2].out_round = "none"; + l[2].in_format = "fp8", l[2].out_format = "fp16"; + l[3] = swin("x2", "x3", "first", "enc0_block", "input", "fp16", "fp8"); + l[4].kind = "project", l[4].blob = "enc0.project", l[4].src = "x3", l[4].dst = "enc0.features"; + l[4].groups = {32, 32, 32, 32}, l[4].w = "W", l[4].b = "b", l[4].clamp = 6.28125f; + l[4].in_format = "fp8", l[4].out_format = "fp8"; return l; }(); return v; @@ -64,11 +68,12 @@ const std::vector& encoder_launches() { const std::vector& decoder_launches() { static const std::vector v = [] { std::vector l(3); - l[0].kind = "dec_project", l[0].blob = "dec0.project", l[0].dst = "p", l[0].w = "W", l[0].b = "b"; - l[0].groups = {32, 32}; - l[1] = swin("p", "second", "dec0_block"); - l[2].kind = "coef", l[2].blob = "dec0.coef", l[2].src = "p"; - l[2].values = {0, 1, 10, 11, 12, 13, 14, 15, 16, 2, 3, 4, 5, 6, 7, 8, 9}; // table.DEC0_VALUES + l[0].kind = "dec_project", l[0].blob = "dec0.project", l[0].src = "features", l[0].dst = "y"; + l[0].w = "W", l[0].b = "b", l[0].groups = {32, 32}, l[0].in_format = "fp8", l[0].out_format = "fp8"; + l[1] = swin("y", "y2", "second", "dec0_block", "coarse", "fp8", "fp16"); + l[2].kind = "coef", l[2].blob = "dec0.coef", l[2].src = "y2", l[2].dst = "codes"; + l[2].values = {0, 1, 6, 7, 8, 9, 10, 11, 12, 2, 3, 4, 5}; // table.L_DEC_VALUES (map perm "dec0_values" v1.1) + l[2].in_format = "fp16", l[2].dec_pixel = "l"; return l; }(); return v; diff --git a/clean/l/runtime/l_pack.h b/clean/l/runtime/l_pack.h index a74c7b5f..79497715 100644 --- a/clean/l/runtime/l_pack.h +++ b/clean/l/runtime/l_pack.h @@ -30,7 +30,11 @@ struct Launch { std::string offsets, params; // swin: window-offset kind ("first", "second"), the block's stage std::vector groups; // project, dec_project: input channel groups float clamp = 0.0f; // project: output clamp (0 = none) - std::vector values; // coef: the value positions (table.DEC0_VALUES) + std::vector values; // coef: the value positions (table.L_DEC_VALUES) + // Map formats ("fp8" / "fp16", l_pack.map_format; empty where a launch has no such map), the swin border + // ("mirror"; "input": src is a halo map; "coarse": token-grid mirror) and the coefficient variant ("dec_pixel"). + // src and dst are distinct map identities: no kernel works in place (clean/l/FORMATS.md). + std::string in_format, out_format, border, dec_pixel; }; // The default table's launches, in order (mirrors of l_pack.encoder_launches(TABLE) / decoder_launches(TABLE); the diff --git a/clean/l/runtime/tests/l_pack_dump.cpp b/clean/l/runtime/tests/l_pack_dump.cpp index 803d5f66..8f218faf 100644 --- a/clean/l/runtime/tests/l_pack_dump.cpp +++ b/clean/l/runtime/tests/l_pack_dump.cpp @@ -53,10 +53,11 @@ std::string join(const std::vector& v) { void print(const char* where, const wgrt::l::Launch& l) { char clamp[32] = ""; if (l.clamp != 0.0f) std::snprintf(clamp, sizeof clamp, "%.9g", l.clamp); - std::printf("%s|%s|%s|%s|%s|%d|%d|%s|%s|%s|%s|%d|%d|%d|%s|%s|%s|%s|%s\n", where, l.kind.c_str(), l.blob.c_str(), - l.src.c_str(), l.dst.c_str(), l.C_in, l.C_out, l.w.c_str(), l.b.c_str(), l.acc.c_str(), - l.out_round.c_str(), l.C, l.heads, l.mlp, l.offsets.c_str(), l.params.c_str(), join(l.groups).c_str(), - clamp, join(l.values).c_str()); + std::printf("%s|%s|%s|%s|%s|%d|%d|%s|%s|%s|%s|%d|%d|%d|%s|%s|%s|%s|%s|%s|%s|%s|%s\n", where, l.kind.c_str(), + l.blob.c_str(), l.src.c_str(), l.dst.c_str(), l.C_in, l.C_out, l.w.c_str(), l.b.c_str(), l.acc.c_str(), + l.out_round.c_str(), l.C, l.heads, l.mlp, l.offsets.c_str(), l.params.c_str(), l.in_format.c_str(), + l.out_format.c_str(), l.border.c_str(), l.dec_pixel.c_str(), join(l.groups).c_str(), clamp, + join(l.values).c_str()); } } // namespace diff --git a/clean/l/runtime/tests/test_l_runtime_pack.py b/clean/l/runtime/tests/test_l_runtime_pack.py index 4c7eaed8..46d62342 100644 --- a/clean/l/runtime/tests/test_l_runtime_pack.py +++ b/clean/l/runtime/tests/test_l_runtime_pack.py @@ -152,7 +152,7 @@ def without(stage, name=None, keep=None): class Launches(unittest.TestCase): FIELDS = (("kind", ""), ("blob", ""), ("src", ""), ("dst", ""), ("C_in", 0), ("C_out", 0), ("w", ""), ("b", ""), ("acc", ""), ("out_round", ""), ("C", 0), ("heads", 0), ("mlp", 0), ("offsets", ""), - ("params", "")) + ("params", ""), ("in_format", ""), ("out_format", ""), ("border", ""), ("dec_pixel", "")) def line(self, where, ln): f = [where] + [str(ln.get(k) if ln.get(k) is not None else d) for k, d in self.FIELDS] @@ -165,7 +165,26 @@ def test_default_table(self): want = [self.line("enc", ln) for ln in LP.encoder_launches()] want += [self.line("dec", ln) for ln in LP.decoder_launches()] self.assertEqual(dump("launches").splitlines(), want) - self.assertEqual(want[-1].split("|")[-1], ",".join(str(v) for v in T.DEC0_VALUES)) + self.assertEqual(want[-1].split("|")[-1], "0,1,6,7,8,9,10,11,12,2,3,4,5") # the exact 13-value sequence + + def test_map_identities(self): + """Every launch reads and writes distinct maps (no kernel works in place), each map is written once, and every + map read is written earlier (or is the stage input).""" + for launches, inputs in ((LP.encoder_launches(), {"values"}), (LP.decoder_launches(), {"features"})): + written = set(inputs) + for ln in launches: + src, dst = ln.get("src"), ln.get("dst") + self.assertTrue(dst and dst != src, ln) + self.assertNotIn(dst, written, ln) + if src: + self.assertIn(src, written, ln) + written.add(dst) + fmts = [(ln["kind"], ln.get("in_format", ""), ln.get("out_format", ""), ln.get("border", "")) + for ln in LP.encoder_launches() + LP.decoder_launches()] + self.assertEqual(fmts, [("values", "", "fp8", ""), ("linear", "fp8", "fp8", ""), ("linear", "fp8", "fp16", ""), + ("swin", "fp16", "fp8", "input"), ("project", "fp8", "fp8", ""), + ("dec_project", "fp8", "fp8", ""), ("swin", "fp8", "fp16", "coarse"), + ("coef", "fp16", "", "")]) if __name__ == "__main__": diff --git a/clean/l/tests/test_l_pack.py b/clean/l/tests/test_l_pack.py index 7138aaaa..2baf6b98 100644 --- a/clean/l/tests/test_l_pack.py +++ b/clean/l/tests/test_l_pack.py @@ -121,15 +121,31 @@ def test_values_and_coef_blobs(self): class ModelTest(unittest.TestCase): + def test_gpu_check_cases_pack(self): + """The C = 32 GPU check's case builder (clean/l/runner/l_c32_check.py) runs on the current packer API.""" + from clean.l.runner import l_c32_check as C32 + p, blob, x = C32.make_case(np.random.default_rng(3), 9, 13) + self.assertEqual(blob, LP.pack_block(p, 32, 2, 128)) + self.assertEqual(x.shape, (9, 13, 32)) + self.assertTrue(np.isfinite(L.swin_block(x, p, (2, 2), 2, C32.CONSTS)).all()) + def test_launch_lists(self): ln = LP.encoder_launches() self.assertEqual([x["kind"] for x in ln], ["values", "linear", "linear", "swin", "project"]) self.assertEqual((ln[1]["src"], ln[1]["C_in"], ln[1]["C_out"], ln[1]["out_round"]), ("q", 32, 32, "relu_q8")) - self.assertEqual((ln[2]["src"], ln[2]["C_in"]), ("x", 32)) + self.assertEqual((ln[2]["src"], ln[2]["C_in"]), ("x1", 32)) self.assertEqual(ln[3]["blob"], "enc0_block") + self.assertEqual([(x.get("in_format"), x.get("out_format")) for x in ln[1:4]], + [("fp8", "fp8"), ("fp8", "fp16"), ("fp16", "fp8")]) + self.assertEqual(ln[3]["border"], "input") dn = LP.decoder_launches() self.assertEqual([x["kind"] for x in dn], ["dec_project", "swin", "coef"]) - self.assertEqual(dn[2]["values"], T.DEC0_VALUES) + self.assertEqual((dn[0]["out_format"], dn[1]["in_format"], dn[1]["out_format"], dn[1]["border"]), + ("fp8", "fp8", "fp16", "coarse")) + self.assertEqual((dn[2]["values"], dn[2]["dec_pixel"], dn[2]["in_format"]), (T.L_DEC_VALUES, "l", "fp16")) + self.assertEqual(T.L_DEC_VALUES, (0, 1, 6, 7, 8, 9, 10, 11, 12, 2, 3, 4, 5)) # the exact sequence (map v1.1) + with self.assertRaises(ValueError): + LP.map_format(None, "none") def test_pack_model_l(self): rng = np.random.default_rng(8) @@ -157,9 +173,10 @@ def test_unpacked_weights_reproduce_the_oracle(self): qpad = np.concatenate([m["q"], np.zeros(m["q"].shape[:-1] + (12,))], axis=-1) h1 = L.linear(qpad, W1, b1, "f16", "relu_q8") self.assertTrue(np.array_equal(h1, L.linear(m["q"], enc["Win1"], enc["bin1"], "f16", "relu_q8"))) - h2 = L.linear(h1, W2, b2, "f16", "q8") - h3 = q8(L.swin_block(h2, params["enc0_block"], T.block_offset(T.TABLE, "first", 0), 2, T.M_BLOCK_CONSTS)) - self.assertTrue(np.array_equal(h3, m["x"])) + h2 = L.linear(h1, W2, b2, "f16", "none") + h3 = q8(L.swin_block(h2, params["enc0_block"], T.block_offset(T.TABLE, "first", 0), 2, T.M_BLOCK_CONSTS, + "input", T.block_halo(T.TABLE, "first"))) + self.assertTrue(np.array_equal(h3, m["x3"])) Wp, bpj = LP.unpack_project(blobs["enc0.project"], 128, 64) feats = L.project_groups(L.space_to_depth(h3), Wp, bpj, (32, 32, 32, 32), 6.28125) self.assertEqual(feats.astype(np.float32).tobytes(), out["features"].tobytes()) @@ -167,7 +184,7 @@ def test_unpacked_weights_reproduce_the_oracle(self): dout = L.decode(xd, params, 0, keep=True) Wd, bd = LP.unpack_dec_project(blobs["dec0.project"]) y = L.dec_project(xd["features"], Wd, bd) - self.assertTrue(np.array_equal(y, dout["maps"]["y"])) + self.assertTrue(np.array_equal(y, dout["maps"]["yt"])) if __name__ == "__main__": diff --git a/clean/l/tests/test_l_reference.py b/clean/l/tests/test_l_reference.py index c1cc9e5c..1c417055 100644 --- a/clean/l/tests/test_l_reference.py +++ b/clean/l/tests/test_l_reference.py @@ -28,7 +28,7 @@ M_DEC_OPS = ( {"op": "dec_project", "src": "features", "dst": "y", "w": "W", "b": "b", "groups": (32, 32)}, {"op": "depth_to_space", "src": "y", "dst": "p"}, - {"op": "pick", "src": "p", "values": T.DEC0_VALUES}, + {"op": "pick", "src": "p", "values": T.M_DEC0_VALUES}, ) @@ -38,6 +38,31 @@ def _same(a, b): return a.shape == b.shape and a.dtype == b.dtype and a.tobytes() == b.tobytes() +def _block_by_windows(value_at, H, W, off, p, heads=2): + """A block written window by window: value_at(y, x) gives any window member's input (inside or outside the + grid); outputs are kept on the H x W grid. Float64, not rounded.""" + p = {k: np.asarray(v, np.float64) for k, v in p.items()} + ox, oy = off + out = None + for y0 in range(-oy, H, 4): + for x0 in range(-ox, W, 4): + pos = [(y, x) for y in range(y0, y0 + 4) for x in range(x0, x0 + 4)] + y = L._block_tokens(np.stack([value_at(a, b) for a, b in pos]), p, heads, T.M_BLOCK_CONSTS) + if out is None: + out = np.full((H, W, y.shape[-1]), np.nan) + for i, (a, b) in enumerate(pos): + if 0 <= a < H and 0 <= b < W: + out[a, b] = y[i] + assert not np.isnan(out).any() + return out + + +def _render_mirror(q, n): + """The input encoder's position rule about the render size n (|q|, else 2 n - |q| - 2).""" + q = abs(q) + return q if q < n else 2 * n - q - 2 + + def m_dec_as_l(p): """Model M's dec0 tensors (W [68, 64], b [68]) as L's [4, 32, 64] / [4, 32] with zeros outside the value rows.""" W = np.zeros((4, 32, 64), np.float32) @@ -45,7 +70,7 @@ def m_dec_as_l(p): Wm = np.asarray(p["W"], np.float32).reshape(4, 17, 64) bm = np.asarray(p["b"], np.float32).reshape(4, 17) for s in range(4): - for k, c in enumerate(T.DEC0_VALUES): + for k, c in enumerate(T.M_DEC0_VALUES): W[s, c] = Wm[s, k] b[s, c] = bm[s, k] return {"W": W, "b": b, "S": p["S"], "T": p["T"]} @@ -137,26 +162,42 @@ def test_default_encoder(self): self.assertTrue(np.isfinite(out["features"]).all()) self.assertTrue(is_fp8_exact(out["features"])) maps = out["maps"] - self.assertEqual(maps["q"].shape, (fh, fw, 20)) - self.assertEqual(maps["x"].shape, (fh, fw, 32)) + h = T.block_halo(T.TABLE, "first") + self.assertEqual(h, 2) + self.assertEqual(maps["q"].shape, (fh + 2 * h, fw + 2 * h, 20)) # per-pixel ops run on the halo + self.assertEqual(maps["x3"].shape, (fh, fw, 32)) self.assertEqual(maps["z"].shape, (fh // 2, fw // 2, 128)) self.assertTrue(_same(L.encode(x, params)["features"], out["features"])) def test_parts_by_hand(self): - """The default op list recomputed step by step.""" + """The default op list recomputed step by step; the block's out-of-grid members evaluate the per-pixel chain + at the mirrored render position with the dither of the unmirrored position.""" rng = np.random.default_rng(6) - x = ME.random_inputs(rng, rw=33, rh=17) + rw, rh = 37, 19 # small sizes: positions fw + 1 and fh + 1 still mirror inside R + x = ME.random_inputs(rng, rw=rw, rh=rh) params = L.random_params(rng) enc = params["enc0"] - out = L.encode(x, params, parity=1, keep=True) - m = out["maps"] fw, fh = x["feature_size"] - q = ME.dither_fp8(ME.pixel_values(x, enc), ME.dither_table(enc["T"], x["frame"], fw, fh)[..., None]) - self.assertTrue(_same(m["q"], q)) - h = q8(np.maximum(L.f16(q @ np.asarray(enc["Win1"], np.float64).T + enc["bin1"]), 0)) - h = q8(L.f16(h @ np.asarray(enc["Win2"], np.float64).T + enc["bin2"])) - h = q8(L.swin_block(h, params["enc0_block"], T.block_offset(T.TABLE, "first", 1), 2, T.M_BLOCK_CONSTS)) - self.assertTrue(_same(m["x"], h)) + vals = ME.pixel_values(x, enc) + Tn = np.asarray(enc["T"])[int(x["frame"])] + Win1, Win2 = np.asarray(enc["Win1"], np.float64), np.asarray(enc["Win2"], np.float64) + + def chain(yy, xx): + v = vals[_render_mirror(yy, rh), _render_mirror(xx, rw)] + q = ME.dither_fp8(v, Tn[abs(yy) % 16, abs(xx) % 16]) + h1 = q8(np.maximum(L.f16(q @ Win1.T + enc["bin1"]), 0)) + return L.f16(h1 @ Win2.T + enc["bin2"]) # FP16, not FP8, into the block + + for parity in (0, 1): + out = L.encode(x, params, parity=parity, keep=True) + m = out["maps"] + q = np.stack([np.stack([ME.dither_fp8(vals[_render_mirror(a, rh), _render_mirror(b, rw)], + Tn[abs(a) % 16, abs(b) % 16]) + for b in range(-2, fw + 2)]) for a in range(-2, fh + 2)]) + self.assertTrue(_same(m["q"], q)) + h = q8(_block_by_windows(chain, fh, fw, T.block_offset(T.TABLE, "first", parity), params["enc0_block"])) + np.testing.assert_array_equal(m["x3"], h) + self.assertFalse(is_fp8_exact(chain(3, 4))) z = np.zeros((fh // 2, fw // 2, 128)) for r in range(fh // 2): for c in range(fw // 2): @@ -197,6 +238,9 @@ def test_plan_refuses_bad_lists(self): (base[0], dict(base[1], n_in=32)) + tuple(base[2:]), # 20 channels into a 32-input linear (base[0], {"op": "frobnicate", "src": "q"}, base[-1]), tuple(base[:4]) + (base[5],), # project on a pixel-level map + tuple(base[:3]) + (dict(base[3], border="wrap"),) + tuple(base[4:]), + tuple(base[:3]) + ({"op": "space_to_depth", "src": "x2", "dst": "z"}, + dict(base[3], src="z", C=128, border="input")) + tuple(base[4:]), ] for ops in bad: with self.assertRaises(ValueError): @@ -207,7 +251,7 @@ class DecoderTest(unittest.TestCase): def test_model_m_op_list_equals_model_m_decoder(self): """With Model M's projection rows and no block, the L decoder is Model M's decoder byte for byte.""" rng = np.random.default_rng(9) - t = T.table(dec_ops=M_DEC_OPS) + t = T.table(dec_ops=M_DEC_OPS, dec_pixel="m") for reset in (False, True): x = MD.random_inputs(rng, rw=29, rh=19, reset=reset) p = MD.random_params(rng) @@ -223,16 +267,54 @@ def test_default_decoder(self): out = L.decode(x, params, 1, keep=True) fw, fh = x["feature_size"] m = out["maps"] - self.assertEqual(m["y"].shape, (fh // 2, fw // 2, 128)) - self.assertEqual(m["p"].shape, (fh, fw, 32)) - self.assertEqual(out["a"].shape, (18, 30, 17)) + self.assertEqual(m["yt"].shape, (fh // 2, fw // 2, 128)) + self.assertEqual(m["y2"].shape, (fh, fw, 32)) + self.assertEqual(out["a"].shape, (18, 30, 13)) y = L.dec_project(x["features"], params["dec0"]["W"], params["dec0"]["b"]) - p = q8(L.swin_block(L.depth_to_space(y), params["dec0_block"], T.block_offset(T.TABLE, "second", 1), 2, - T.M_BLOCK_CONSTS)) - self.assertTrue(_same(m["p"], p)) - self.assertTrue(_same(out["a"], p[:18, :30][..., list(T.DEC0_VALUES)].astype(np.float32))) + hw = fw // 2 + + def member(yy, xx): # coarse mirror of the token, sub-pixel kept (rows stay inside) + q = xx // 2 + if not 0 <= q < hw: + q = min(abs(q), fw - abs(q) - 2) + return y[yy // 2, q, 32 * ((xx % 2) + 2 * (yy % 2)):][:32] + + self.assertTrue(_same(member(0, -1), member(0, 3)) and _same(member(1, fw), member(1, fw - 4))) + p = L.f16(_block_by_windows(member, fh, fw, T.block_offset(T.TABLE, "second", 1), params["dec0_block"])) + np.testing.assert_array_equal(m["y2"], p) # FP16 block output, no FP8 + self.assertFalse(is_fp8_exact(p)) + fine = L.f16(L.swin_block(L.depth_to_space(y), params["dec0_block"], T.block_offset(T.TABLE, "second", 1), + 2, T.M_BLOCK_CONSTS)) + self.assertFalse(_same(fine, p)) # not a fine-pixel mirror + self.assertTrue(_same(fine[:, 2:-2], p[:, 2:-2])) # ... only the edge windows differ + a = p[:18, :30][..., [0, 1, 6, 7, 8, 9, 10, 11, 12, 2, 3, 4, 5]].astype(np.float32) + self.assertTrue(_same(out["a"], a)) + # The per-pixel functions value by value. + S, Tc = np.float32(params["dec0"]["S"]), np.float32(params["dec0"]["T"]) + for (r, c) in ((0, 0), (5, 7), (17, 29)): + v = a[r, c] + sig = [np.float16(np.float32(1) / (np.float32(1) + np.exp(-v[i]))) for i in range(3)] + self.assertEqual(list(out["codes_A"][r, c]), [int(np.rint(np.float32(s) * np.float32(255))) for s in sig]) + e = np.exp(v[3:6] - v[3:6].max()) + self.assertTrue(np.allclose(out["B"][r, c], e / e.sum(), atol=0.5 / 255 + 1e-6)) + g = [np.float32(np.float16(np.float32(1) / (np.float32(1) + np.exp(-np.clip(v[i], -20, 20))))) + for i in (6, 7)] + sx, sy = S * g[0], S * g[1] + rho = np.float32(np.tanh(np.float64(Tc * v[8]))) + want_c = [np.float16(sx * sx), np.float16(sy * sy), np.float16((sx * sy) * rho)] + self.assertEqual([np.float16(q) for q in out["C"][r, c]], want_c) + want_l = [int(np.rint(np.clip(np.float32(np.tanh(np.float64(v[9 + i]))), -1, 1) * np.float32(127))) + for i in range(4)] + self.assertEqual(list(out["codes_L"][r, c]), want_l) + # No previous latent, motion or jitter is read. + x2 = dict(x, latent_prev=-np.asarray(x["latent_prev"]), motion=np.asarray(x["motion"]) + 5.0, + jitter=(0.3, -0.2), reset=not x["reset"]) + out2 = L.decode(x2, params, 1) for k in ("A", "B", "C", "latent"): self.assertTrue(np.isfinite(out[k]).all(), k) + self.assertTrue(_same(out[k], out2[k]), k) + with self.assertRaises(ValueError): # 13 values for "l", 17 for "m" + L.decode(x, params, 1, T.table(dec_pixel="m")) def test_depth_to_space_inverts_space_to_depth(self): v = np.random.default_rng(11).normal(size=(6, 8, 5)) @@ -259,7 +341,7 @@ def test_reduces_to_model_m_chain(self): """With Model M's encoder and decoder op lists, the L network is Model M's oracle chain.""" rng = np.random.default_rng(13) xe, xd = self._inputs(rng, 40, 24) - t = T.table(enc_ops=M_ENC_OPS, dec_ops=M_DEC_OPS) + t = T.table(enc_ops=M_ENC_OPS, dec_ops=M_DEC_OPS, dec_pixel="m") p = L.random_params(rng, t) p["enc0"] = ME.random_params(rng) pm_dec = MD.random_params(rng) @@ -324,7 +406,7 @@ def test_macs(self): def test_table_copy(self): t = T.table(dec_pixel="x") self.assertEqual(t["dec_pixel"], "x") - self.assertEqual(T.TABLE["dec_pixel"], "m") + self.assertEqual(T.TABLE["dec_pixel"], "l") with self.assertRaises(KeyError): T.table(nonsense=1)