diff --git a/clean/dx12/kernels/l_post.hlsl b/clean/dx12/kernels/l_post.hlsl index 1c987137..1bdc8227 100644 --- a/clean/dx12/kernels/l_post.hlsl +++ b/clean/dx12/kernels/l_post.hlsl @@ -19,9 +19,15 @@ // value is clamp(E(o), 0, 2.45) rounded by v_cvt_f16_f32 (wg_f2h); // - D(h) / e_prev is an IEEE division (wg_div). // Model M's DXILs are untouched: this file only adds the kernel (bound to l_post.co). +// l_post_recon_p12 (LP_P12 = 1): the same kernel on the pack12 coefficients of l_coef_pack12: A = image A8 (8 bytes per +// render pixel: codes A0 A1 A2 B0 | B1 B2, C2 FP16), C = image C4 (4 bytes per render pixel: C0, C1 FP16), B unused. // // @wg-kernel l_post_recon co=l_post.co block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p +// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1 #include "wg_platform.hlsli" +#ifndef LP_P12 +#define LP_P12 0 +#endif WG_ARGS_BEGIN uint color, A, B, C, hist_prev, m_near, m_center, m_far, w_p, hist_out, reduced, display; // pointers @@ -100,8 +106,15 @@ void load_texel(int idx, out uint t[kTexDw]) { const uint ui = (uint)idx; const float4 c = wg_ld4f(color, ui * 16u); +#if LP_P12 + const uint2 d = wg_ld64(A, ui * 8u); + const uint c01 = wg_ld32(C, ui * 4u); + const uint a = d.x & 0xffffffu, b = (d.x >> 24) | ((d.y & 0xffffu) << 8); + const uint2 k = uint2(c01, d.y >> 16); // C0 | C1 << 16, C2 (l_coef's C dword 0 and the low half of dword 1) +#else const uint b = wg_ld32(B, ui * 4u), a = wg_ld32(A, ui * 4u); const uint2 k = wg_ld64(C, ui * 8u); +#endif uint f[9]; f[0] = wg_f2h(enc_in(opq(g_e * c.x))); f[1] = wg_f2h(enc_in(opq(g_e * c.y))); @@ -297,7 +310,11 @@ void finish_pixel(LGather g, int x, int y, float px, float py, bool store, float float lpx(int xx) { return mad(opq((float)xx + 0.5f), P_rx, P_jx); } // fused(x + 1/2, rx, jx) float lpy(int yy) { return mad(P_ry, opq((float)yy + 0.5f), P_jy); } // fused(ry, y + 1/2, jy) +#if LP_P12 +WG_KERNEL(l_post_recon_p12) +#else WG_KERNEL(l_post_recon) +#endif { WG_SETUP(); g_z = wg_grid_ >> 31; diff --git a/clean/l/FORMATS.md b/clean/l/FORMATS.md index c711e444..d319d0c3 100644 --- a/clean/l/FORMATS.md +++ b/clean/l/FORMATS.md @@ -61,7 +61,7 @@ No kernel works in place. |---|---|---| | `l_enc_values.co` (loaded under the tag `lev`) | `clean/kernels/encoder.hip` with `-DWG_ENC_VALUES_OUT=1` | `input_encoder` | | `l_ops.co` | `clean/l/kernels/l_ops.hip` | `l_linear_32_32_relu`, `l_linear_32_32_f16`, `l_project_gather`, `l_dec_project`, `l_coef`, `l_coef_pack12` | -| `l_post.co` | `clean/l/kernels/l_post.hip` (includes Model M's `clean/kernels/post.hip` for its functions; Model M's kernels come along unused) | `l_post_recon` (HDR) | +| `l_post.co` | `clean/l/kernels/l_post.hip` (includes Model M's `clean/kernels/post.hip` for its functions; Model M's kernels come along unused) | `l_post_recon`, `l_post_recon_p12` (HDR) | | `l_swin.co` | `clean/l/kernels/l_swin.hip` | `l_swin_block_c32[_nc]` (exists, #102: FP8 in / FP8 out, mirror border; block checks only), `l_swin_block_c32_enc[_nc]`, `l_swin_block_c32_dec[_nc]` | All are built with `clean/runner/build.py`'s `device_command`. The build's `-cuid` comes from the output file name, @@ -141,6 +141,11 @@ the frame parity, as Model M's. - its blend is the fused form beta' = fused(alpha (1 - beta-bar) + beta-bar), H^ = fused((C - Hl) beta' + Hl); - the decode passes a NaN through. + `l_post_recon_p12` (`l_post.co`) is the same kernel on the pack12 coefficients of `l_coef_pack12`: `A` = image A8 + (8 bytes per render pixel), `C` = image C4 (4 bytes per render pixel), `B` unused (may be null); the other arguments, + the grid and the outputs as `l_post_recon`, byte for byte (`clean/l/runner/l_d3d12_ops_check.py` checks it on HIP). + The D3D12 ports of both are `clean/dx12/kernels/l_post.hlsl`. + The LDR kernel is planned, with the same arguments. Still open for the kernel-order model: the hardware exp2, reciprocal and divisions, and the Catmull-Rom weight forms, which the kernel takes from Model M's `post.hip` (part of the Model M fidelity decision). diff --git a/clean/l/kernels/l_post.hip b/clean/l/kernels/l_post.hip index a6d747f1..168cfe13 100644 --- a/clean/l/kernels/l_post.hip +++ b/clean/l/kernels/l_post.hip @@ -11,6 +11,8 @@ // 2^-126, FP16 weights into half accumulators, a float32 weight sum. The blend is item 3's fused form. The decode // passes a NaN through (a zero total gather weight gives C = D(NaN) = NaN; the NaN guard then substitutes it, as the // oracle). Arguments, layouts and grid are post_recon's; w[0] = Model L's post scale s. +// l_post_recon_p12: the same kernel on the pack12 coefficients of l_coef_pack12 (A8 / C4 images, B unused), which the +// D3D12 frame reads; the same texel values, so the same outputs byte for byte. #include "post.hip" namespace { @@ -48,6 +50,35 @@ WG_FN LTexel l_load_texel(const v4f* color, const u32* A, const u32* B, const v4 return t; } +// l_load_texel on the pack12 coefficients (l_coef_pack12, clean/l/FORMATS.md): image A8, 8 bytes per render pixel = +// codes A0 A1 A2 B0 | B1 B2, C2 (FP16); image C4, 4 bytes per render pixel = C0, C1 (FP16). The same values as +// l_load_texel on l_coef's images (only padding bytes are left out), so the same texel. +WG_FN LTexel l_load_texel12(const v4f* color, const u32* A8, const u32* C4, int idx, float e) { + LTexel t; + const v4f c = color[idx]; + t.f[0] = (h16)enc_in(e * c[0]); + t.f[1] = (h16)enc_in(e * c[1]); + t.f[2] = (h16)enc_in(e * c[2]); + const u32 d0 = A8[2 * idx], d1 = A8[2 * idx + 1], c01 = C4[idx]; + const u32 a = d0 & 0xffffffu, b = (d0 >> 24) | ((d1 & 0xffffu) << 8); +#pragma unroll + for (int i = 0; i < 3; ++i) { + t.f[3 + i] = (h16)(byte_f(b, i) / 255.0f); + t.f[6 + i] = (h16)(byte_f(a, i) / 255.0f); + } + t.kA = (h16)((float)__builtin_bit_cast(_Float16, (unsigned short)(c01 & 0xffffu)) * -0.721325f); + t.kB = (h16)((float)__builtin_bit_cast(_Float16, (unsigned short)(c01 >> 16)) * -0.721325f); + t.kC = (h16)((float)__builtin_bit_cast(_Float16, (unsigned short)(d1 >> 16)) * -1.44265f); + return t; +} + +// The texel of the kernel's coefficient form: l_coef's images (A, B, C) or pack12's (A8 in A, C4 in C, B unused). +template +WG_FN LTexel l_texel(const v4f* color, const u32* A, const u32* B, const void* C, int idx, float e) { + if (kP12) return l_load_texel12(color, A, (const u32*)C, idx, e); + return l_load_texel(color, A, B, (const v4h*)C, idx, e); +} + // Gathered sums of one D pixel: W = float32 sum of the unrounded weights; F = the half accumulators. struct LGather { float W; @@ -186,14 +217,11 @@ WG_FN void l_finish_pixel(const LGather& g, int x, int y, float px, float py, bo } } -} // namespace - -// Stage "post" for Model L, HDR. Arguments, layouts and grid as post_recon (clean/kernels/post.hip): grid -// ceil(dw / 32) * ceil(ceil(dh / 2) / 8) workgroups of 256 lanes; w = the post blob (w[0] = s). -WG_KERNEL_N(l_post_recon, kPostThreads)(const v4f* color, const u32* A, const u32* B, const v4h* C, - const v4h* hist_prev, const u32* m_near, const u32* m_center, - const u32* m_far, const float* w, v4h* hist_out, u32* reduced, - _Float16* display, WgPostParams p, const float* expo) { +// The kernel body; kP12 selects the coefficient form (l_texel). +template +WG_FN void l_post_recon_body(const v4f* color, const u32* A, const u32* B, const void* C, const v4h* hist_prev, + const u32* m_near, const u32* m_center, const u32* m_far, const float* w, v4h* hist_out, + u32* reduced, _Float16* display, WgPostParams p, const float* expo) { if (expo) { p.e = expo[0] == 0.0f ? 1.0f : expo[0]; p.e_prev = expo[1] == 0.0f ? 1.0f : expo[1]; @@ -224,7 +252,7 @@ WG_KERNEL_N(l_post_recon, kPostThreads)(const v4f* color, const u32* A, const u3 for (int i = lane; i < ncol * nrow; i += kPostThreads) { const int r = i / ncol, c = i - r * ncol; const int idx = clampi(r0 + r, 0, p.rh - 1) * p.rw + clampi(c0 + c, 0, p.rw - 1); - sh_tex[i] = l_load_texel(color, A, B, C, idx, p.e); + sh_tex[i] = l_texel(color, A, B, C, idx, p.e); } } wg_barrier(); @@ -259,7 +287,7 @@ WG_KERNEL_N(l_post_recon, kPostThreads)(const v4f* color, const u32* A, const u3 t = sh_tex[(b + j - r0) * ncol + (ax + i - c0)]; } else { const int idx = clampi(b + j, 0, p.rh - 1) * p.rw + clampi(ax + i, 0, p.rw - 1); - t = l_load_texel(color, A, B, C, idx, p.e); + t = l_texel(color, A, B, C, idx, p.e); } l_gather_add(g0, t, u, v0); l_gather_add(g1, t, u, v1); @@ -287,3 +315,24 @@ WG_KERNEL_N(l_post_recon, kPostThreads)(const v4f* color, const u32* A, const u3 reduced[(y0 / 2) * (p.dw / 2) + x / 2] = code; } } + +} // namespace + +// Stage "post" for Model L, HDR. Arguments, layouts and grid as post_recon (clean/kernels/post.hip): grid +// ceil(dw / 32) * ceil(ceil(dh / 2) / 8) workgroups of 256 lanes; w = the post blob (w[0] = s). +WG_KERNEL_N(l_post_recon, kPostThreads)(const v4f* color, const u32* A, const u32* B, const v4h* C, + const v4h* hist_prev, const u32* m_near, const u32* m_center, + const u32* m_far, const float* w, v4h* hist_out, u32* reduced, + _Float16* display, WgPostParams p, const float* expo) { + l_post_recon_body(color, A, B, C, hist_prev, m_near, m_center, m_far, w, hist_out, reduced, display, p, expo); +} + +// l_post_recon on the pack12 coefficients of l_coef_pack12: A = image A8 (8 bytes per render pixel), C = image C4 +// (4 bytes per render pixel), B unused (may be null). Everything else as l_post_recon; the same outputs. +WG_KERNEL_N(l_post_recon_p12, kPostThreads)(const v4f* color, const u32* A8, const u32* B, const u32* C4, + const v4h* hist_prev, const u32* m_near, const u32* m_center, + const u32* m_far, const float* w, v4h* hist_out, u32* reduced, + _Float16* display, WgPostParams p, const float* expo) { + l_post_recon_body(color, A8, B, C4, hist_prev, m_near, m_center, m_far, w, hist_out, reduced, display, p, + expo); +} diff --git a/clean/l/runner/l_d3d12_ops_check.py b/clean/l/runner/l_d3d12_ops_check.py index 55c7c543..1514d1a7 100644 --- a/clean/l/runner/l_d3d12_ops_check.py +++ b/clean/l/runner/l_d3d12_ops_check.py @@ -234,6 +234,16 @@ def _post_specials(rng, x): return x +def _pack12(data): + """l_coef's images A, B (4 bytes per pixel, codes in bytes 0 ... 2) and C (4 x FP16) in l_coef_pack12's layout: + A8 = A0 A1 A2 B0 | B1 B2, C2 (8 bytes per pixel); C4 = C0, C1 (4 bytes per pixel); B unused (passed as null).""" + a = np.frombuffer(data["A"], np.uint32) & 0xFFFFFF + b = np.frombuffer(data["B"], np.uint32) & 0xFFFFFF + c = np.frombuffer(data["C"], np.uint16).reshape(-1, 4).astype(np.uint32) + a8 = np.stack([a | ((b & 0xFF) << 24), (b >> 8) | (c[:, 2] << 16)], -1).astype(np.uint32) + return {"A": a8.tobytes(), "B": b"", "C": (c[:, 0] | (c[:, 1] << 16)).astype(np.uint32).tobytes()} + + def post_cases(kernel, rng): """l_post_recon (l_post.co): color, A, B, C, hist_prev, m_near, m_center, m_far, w (post blob, w[0] = s) -> full history [dh][dw] (RGBA16F), reduced codes [dh / 2][dw / 2], display (3 FP16 planes of pitch @@ -261,7 +271,10 @@ def post_cases(kernel, rng): defined = [np.concatenate([np.ones(n_hist, bool), np.zeros(POST_SLACK, bool)]), np.concatenate([red_def, np.zeros(POST_SLACK, bool)]), np.concatenate([disp_def.reshape(-1), np.zeros(POST_SLACK, bool)])] - args = [("buf", data[k]) for k in ("color", "A", "B", "C", "hist_prev", "m_near", "m_center", "m_far")] + if kernel == "l_post_recon_p12": + data = dict(data, **_pack12(data)) + args = [("null",) if data[k] == b"" else ("buf", data[k]) + for k in ("color", "A", "B", "C", "hist_prev", "m_near", "m_center", "m_far")] args += [("buf", RP.pack({"s": np.array(LPC.S)})), ("out", n_hist + POST_SLACK), ("out", n_red + POST_SLACK), ("out", n_disp + POST_SLACK), ("bytes", bytes(prm))] args += [("buf", ev.tobytes())] if expo else [("null",)] @@ -301,7 +314,8 @@ def enc_direct_cases(kernel, rng): KINDS = {"l_linear_32_32_relu": linear_cases, "l_linear_32_32_f16": linear_cases, "l_project_gather": project_cases, "l_dec_project": project_cases, "l_coef": coef_cases, "l_coef_pack12": coef_cases, "lev/input_encoder": enc_cases, "l_post_recon": post_cases, - "lev/input_encoder_direct": enc_direct_cases, "swin_block_c64_add_ex_nc": dec1_ex_cases} + "l_post_recon_p12": post_cases, "lev/input_encoder_direct": enc_direct_cases, + "swin_block_c64_add_ex_nc": dec1_ex_cases} CONTROL_KINDS = {"swin_block_c96_nc": control_cases} CO = "l_ops.co" @@ -451,6 +465,14 @@ def compare(got, want, dmask): for k in kernels} res["pass"] = r.returncode == 0 and caught is True and all(v["cases"] > 0 and v["identical"] == v["cases"] for v in res["per_kernel"].values()) + if "l_post_recon" in kernels and "l_post_recon_p12" in kernels: + # The same cases (same seeds) on both coefficient forms: on HIP, the pack12 kernel's raw output bytes must be + # l_post_recon's. + pairs = [(n, n.replace("l_post_recon_", "l_post_recon_p12_", 1)) for n in hip_out + if hip_out[n][0] == "l_post_recon"] + same = sum(hip_out[a][1] == hip_out[b][1] for a, b in pairs if b in hip_out) + res["p12_equals_base_on_hip"] = {"cases": len(pairs), "identical": same} + res["pass"] = res["pass"] and same == len(pairs) > 0 return res