diff --git a/clean/l/FORMATS.md b/clean/l/FORMATS.md index f1ef884d..7e80f6c7 100644 --- a/clean/l/FORMATS.md +++ b/clean/l/FORMATS.md @@ -60,7 +60,7 @@ No kernel works in place. | File | Source | Kernels | |---|---|---| | `l_enc_values.co` (loaded under the tag `lev`) | `clean/kernels/encoder.hip` with `-DWG_ENC_VALUES_OUT=1` | `input_encoder` | -| `l_ops.co` | `clean/l/kernels/l_ops.hip` | `l_linear_32_32_relu`, `l_linear_32_32_f16`, `l_project_gather`, `l_dec_project`, `l_coef` | +| `l_ops.co` | `clean/l/kernels/l_ops.hip` | `l_linear_32_32_relu`, `l_linear_32_32_f16`, `l_project_gather`, `l_dec_project`, `l_coef`, `l_coef_pack12` | | `l_post.co` | `clean/l/kernels/l_post.hip` (includes Model M's `clean/kernels/post.hip` for its functions; Model M's kernels come along unused) | `l_post_recon` (HDR) | | `l_swin.co` | `clean/l/kernels/l_swin.hip` | `l_swin_block_c32[_nc]` (exists, #102: FP8 in / FP8 out, mirror border; block checks only), `l_swin_block_c32_enc[_nc]`, `l_swin_block_c32_dec[_nc]` | @@ -83,6 +83,7 @@ workgroup, in raster order of their map. | `l_dec_project` | exists (GPU-checked) | `z, y, w, tw, th` | ceil(tw th / 128) | per token of dec1's output z [th][tw][64]: 128 outputs (two RN16 groups of 32 inputs, then FP8). Output 32 s + k goes to pixel (2r + s / 2, 2c + s % 2), channel k, of `y`. w = `dec0.project` | | `l_swin_block_c32_dec[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | as the encoder block | the decoder block. x = `y` (FP8). A member outside the grid takes token-grid mirrored pixels with the sub-pixel kept. out = `y2` (**FP16**). (ox, oy) = `window_offsets["second"]`. w = `dec0_block` | | `l_coef` | exists (GPU-checked) | `y2, A, B, C, latent, w, params` | ceil((fh / 2)(fw / 2) / 128), lanes as Model M's `coef_decoder` | Model L's per-pixel functions on the 13 channels `table.L_DEC_VALUES` of `y2`: A, B, C in Model M's forms, latent = tanh(candidate). Writes the same buffers and formats as `coef_decoder`, pixels inside R only. **Reads no motion, previous latent, jitter or reset.** params = Model M's `WgDecParams` (only rw, rh, fw, fh are read). w = `dec0.coef` | +| `l_coef_pack12` | exists (GPU-checked) | as `l_coef` (`B` unused, may be null) | as `l_coef` | `l_coef`'s values in Model M's dec0 "pack12" layout (`clean/kernels/decoder.hip` `WG_DEC_PACK12`, read by the output stage's `_p12` kernels): image A, 8 bytes per pixel = A0 A1 A2 B0 | B1 B2 C2 (FP16); image C, 4 bytes per pixel = C0 C1 (FP16); latent as `l_coef`. Only padding bytes are left out: unpacked, byte-identical to `l_coef` (`l_ops_check.py`) | ## Launch order of one frame diff --git a/clean/l/kernels/l_ops.hip b/clean/l/kernels/l_ops.hip index f5cee150..4a98378e 100644 --- a/clean/l/kernels/l_ops.hip +++ b/clean/l/kernels/l_ops.hip @@ -218,7 +218,13 @@ WG_KERNEL_N(l_dec_project, kThreads)(const u8* z, u8* y, const u8* w, int tw, in // Grid as coef_decoder: ceil((fh / 2)(fw / 2) / 128); lane (t, h) of wave i handles token t' = 16 i + t = (r, c), pixels // (2 c, 2 r + h) and (2 c + 1, 2 r + h). w = blob dec0.coef (Model M's dec0 blob: S, T at byte 320). No motion, // previous latent, jitter or reset is read. -WG_KERNEL_N(l_coef, kThreads)(const u8* y2, u32* A, u32* B, u32* C, u32* latent, const u8* w, LDecParams P) { +// PACK12 (kernel l_coef_pack12): A, B and C without padding bytes, in Model M's dec0 "pack12" layout +// (clean/kernels/decoder.hip WG_DEC_PACK12, read by the output stage's _p12 kernels): image A, 8 bytes per pixel = +// A0 A1 A2 B0 | B1 B2 C2 (FP16); image C, 4 bytes per pixel = C0 C1 (FP16); B unused. The latent as l_coef. Only +// padding bytes (zero in the base form) are left out: the values are the same. +namespace { +template +WG_FN void l_coef_body(const u8* y2, u32* A, u32* B, u32* C, u32* latent, const u8* w, const LDecParams& P) { const u32 tid = wg_lane_id(), hh = (tid >> 4) & 1u; const int tw = P.fw >> 1, ntok = (P.fh >> 1) * tw; const int tok = item_of(tid); @@ -235,10 +241,27 @@ WG_KERNEL_N(l_coef, kThreads)(const u8* y2, u32* A, u32* B, u32* C, u32* latent, l_values((const _Float16*)y2, P.fw, x, y, a); const LCodes o = l_pixel(a, S, T); const u32 i = (u32)(y * P.rw + x); - A[i] = o.A; - B[i] = o.B; - C[2 * i] = o.C0; - C[2 * i + 1] = o.C1; + if constexpr (PACK12) { + A[2 * i] = o.A | (o.B << 24); + A[2 * i + 1] = (o.B >> 8) | (o.C1 << 16); + C[i] = o.C0; + (void)B; + } else { + A[i] = o.A; + B[i] = o.B; + C[2 * i] = o.C0; + C[2 * i + 1] = o.C1; + } latent[i] = o.L; } } +} // namespace + +WG_KERNEL_N(l_coef, kThreads)(const u8* y2, u32* A, u32* B, u32* C, u32* latent, const u8* w, LDecParams P) { + l_coef_body(y2, A, B, C, latent, w, P); +} + +// Same arguments as l_coef; A is the 8-byte image A, C the 4-byte image C, B may be null. +WG_KERNEL_N(l_coef_pack12, kThreads)(const u8* y2, u32* A, u32* B, u32* C, u32* latent, const u8* w, LDecParams P) { + l_coef_body(y2, A, B, C, latent, w, P); +} diff --git a/clean/l/runner/l_ops_check.py b/clean/l/runner/l_ops_check.py index 7b0d6b27..2552d854 100644 --- a/clean/l/runner/l_ops_check.py +++ b/clean/l/runner/l_ops_check.py @@ -15,6 +15,8 @@ - l_linear_32_32_f16: RN16(b + Win2 x), FP16; - l_project_gather: the 2 x 2 gather and the 128 -> 64 projection (running FP16 sum per group of 32, clamp, FP8); - l_dec_project: the 64 -> 4 x 32 projection scattered to the pixel map; +- l_coef_pack12 (pack12 store form): its images, unpacked, byte-identical to l_coef's A, B, C and latent, with + sentinel-filled buffers (no byte past a buffer written, every pixel inside R written) and a negative control; - l_coef: Model L's per-pixel functions on an FP16 map against oracle.coefficients_l: stored codes of A, B and the latent within one step, C within 4 FP16 ulp (the oracle's exponential and tanh are exact; the kernel's use the hardware operations), with the counts that differ at all. @@ -162,6 +164,79 @@ def boundary_cases(g, rng, enc): return out +SENTINEL = 0xA5 +SLACK = 64 + + +def _launch_coef(g, kernel, yb, blob, sizes, P, grid, null_b=False): + """Run an l_coef form with output buffers of the given byte sizes (A, B, C, latent), each followed by SLACK bytes, + all filled with SENTINEL first. Returns the downloaded buffers including their slack.""" + h = g.hip + py, pw = h.malloc(len(yb)), h.malloc(len(blob)) + h.upload(py, yb) + h.upload(pw, blob) + outs = [] + for n in sizes: + if n is None: + outs.append(None) + continue + p = h.malloc(n + SLACK) + h.memset(p, SENTINEL, n + SLACK) + outs.append(p) + args = [ctypes.c_void_p(py)] + [ctypes.c_void_p(p or 0) for p in outs] + [ctypes.c_void_p(pw), P] + h.launch(h.get_function(g.mod, kernel), grid, 256, args) + h.sync() + res = [None if p is None else np.frombuffer(h.download(p, n + SLACK), np.uint8) for p, n in zip(outs, sizes)] + for p in [py, pw] + [p for p in outs if p]: + h.free(p) + return res + + +def unpack12(a8, c4): + """pack12 images (image A: 2 words per pixel, image C: 1 word) -> (A codes word, B codes word, C words [C0C1, + C2]) in the base form's layout (padding bytes 0).""" + w0, w1 = a8[0::2], a8[1::2] + A = w0 & 0xFFFFFF + B = (w0 >> 24) | ((w1 & 0xFFFF) << 8) + C = np.stack([c4, w1 >> 16], axis=-1).reshape(-1) + return A, B, C + + +def coef_pack12_case(g, rng, dec, fh, fw, rh, rw): + """l_coef and l_coef_pack12 on the same FP16 map: the pack12 images, unpacked, must equal the base form's A, B, + C and latent byte for byte; no byte past any buffer may be written; every pixel inside R must be written (the base + form's padding bytes are 0, not the sentinel); and a one-bit change in the pack12 image must be caught (negative + control).""" + from clean.runner.dec0 import DecParams + y2 = L.f16(rng.normal(0, 2.0, (fh, fw, 32))) + yb = y2.astype(np.float16).tobytes() + blob = LP.pack_coef(dec) + npx = rh * rw + P = DecParams(rw=rw, rh=rh, fw=fw, fh=fh) + grid = -(-((fh // 2) * (fw // 2)) // 128) + base = _launch_coef(g, "l_coef", yb, blob, (4 * npx, 4 * npx, 8 * npx, 4 * npx), P, grid) + p12 = _launch_coef(g, "l_coef_pack12", yb, blob, (8 * npx, None, 4 * npx, 4 * npx), P, grid) + m = {"case": [fh, fw, rh, rw], "pixels": npx} + m["slack_untouched"] = all(bool((b[-SLACK:] == SENTINEL).all()) for b in base + [p for p in p12 if p is not None]) + w = lambda b, n: b[:n].view(np.uint32) # noqa: E731 + bA, bB, bC, bL = w(base[0], 4 * npx), w(base[1], 4 * npx), w(base[2], 8 * npx), base[3][:4 * npx] + m["base_all_written"] = bool(((bA >> 24) == 0).all() and ((bB >> 24) == 0).all() and ((bC[1::2] >> 16) == 0).all()) + a8, c4, pL = w(p12[0], 8 * npx), w(p12[2], 4 * npx), p12[3][:4 * npx] + uA, uB, uC = unpack12(a8, c4) + m["A_identical"] = bool(np.array_equal(uA, bA)) + m["B_identical"] = bool(np.array_equal(uB, bB)) + m["C_identical"] = bool(np.array_equal(uC, bC)) + m["latent_identical"] = bool(np.array_equal(pL, bL)) + bad = a8.copy() + k = int(rng.integers(0, npx)) + bad[2 * k + 1] ^= np.uint32(1 << 20) # one bit of C2 of pixel k + nA, nB, nC = unpack12(bad, c4) + m["negative_control_caught"] = not np.array_equal(nC, bC) + m["pass"] = all(m[k_] for k_ in ("slack_untouched", "base_all_written", "A_identical", "B_identical", "C_identical", + "latent_identical", "negative_control_caught")) + return m + + def _fp8_metrics(got, ref, case): m = C32.metrics(got, ref) m.update({"case": case, "pass": C32.passes(m)}) @@ -212,6 +287,7 @@ def run(build, rocm, seeds=(89,)): res.setdefault("boundary", []).extend(boundary_cases(g, rng, enc)) for case in COEF_CASES: res.setdefault("coef", []).append(coef_case(g, rng, dec, *case)) + res.setdefault("coef_pack12", []).append(coef_pack12_case(g, rng, dec, *case)) res["coef_total"] = {k: {"values": sum(m[k]["values"] for m in res["coef"]), "differ": sum(m[k]["differ"] for m in res["coef"])} for k in ("codes_A", "codes_B", "C", "codes_L")} @@ -220,7 +296,11 @@ def run(build, rocm, seeds=(89,)): "differ": sum(m["differ"] for m in res[k]), "max_steps": max(m["max_steps"] for m in res[k])} res["f16_total"]["f16_differ"] = sum(m["f16_differ"] for m in res["f16"]) - res["pass"] = all(m["pass"] for k in ("relu", "f16", "project", "dec_project", "coef", "boundary") for m in res[k]) + kinds = ("relu", "f16", "project", "dec_project", "coef", "boundary", "coef_pack12") + p12 = res["coef_pack12"] + res["coef_pack12_total"] = {"cases": len(p12), "pixels": sum(m["pixels"] for m in p12), + "all_pass": all(m["pass"] for m in p12)} + res["pass"] = all(m["pass"] for k in kinds for m in res[k]) return res @@ -235,7 +315,7 @@ def main(argv=None): with open(a.json, "w") as f: json.dump(res, f, indent=1) print({k: v for k, v in res.items() if k.endswith("_total") or k == "pass"}) - for k in ("relu", "f16", "project", "dec_project", "coef", "boundary"): + for k in ("relu", "f16", "project", "dec_project", "coef", "boundary", "coef_pack12"): for m in res[k]: if not m["pass"]: print("FAIL", k, json.dumps(m))