From 2019c9c241c69d4da15eac2956e2932803e51056 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:52:37 -0500 Subject: [PATCH] Model L: D3D12 port of the coefficient step (l_coef), byte for byte against its HIP kernel Refs #89 Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/kernels/l_ops_coef.hlsl | 107 ++++++++++++++++++++++++++++ clean/l/runner/l_d3d12_ops_check.py | 37 +++++++++- 2 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 clean/dx12/kernels/l_ops_coef.hlsl diff --git a/clean/dx12/kernels/l_ops_coef.hlsl b/clean/dx12/kernels/l_ops_coef.hlsl new file mode 100644 index 00000000..c5effec6 --- /dev/null +++ b/clean/dx12/kernels/l_ops_coef.hlsl @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of Model L's coefficient step (issue #89): l_coef of l_ops.co (clean/l/kernels/l_ops.hip), byte for byte. +// y2 [fh][fw][32] FP16 (the decoder block's output) -> A, B (3 unorm8 codes each), C (two dwords: C[2 i] = FP16 sx^2 | +// sy^2 << 16, C[2 i + 1] = FP16 sxy rho in the low half) and the new latent (4 snorm8 codes), [rh][rw] each, pixels +// inside R only. Lane (t, h) of wave i handles token 16 i + t = +// (r, c) of the (fh / 2) x (fw / 2) grid, pixels (2 c, 2 r + h) and (2 c + 1, 2 r + h). Reads no motion, previous latent, +// jitter or reset. w = blob dec0.coef (S, T at byte 320). +// +// Every FP32 operation follows HIP's ISA of l_coef (llvm-objdump of l_ops.co), not its source, where they differ: +// - C0 = (f16)(sx sx) | (f16)(sy sy) << 16 and C1 = (f16)(sxy rho), each ONE rounding (v_fma_mixlo_f16 a, b, 0 = +// wg_mul_f16), not an FP32 product rounded again; +// - B: s = e2 + (e0 + e1) (commutes with the source's (e0 + e1) + e2), code = rint((e r) 255): opq() keeps the driver +// from regrouping the product with the constant 255; +// - tanh below 0.6: fma(p, t^2, 0) with an opaque zero (the driver turns mad(a, b, 0) into a b, which changes the sign +// of a zero), then fma(p2, t, t); +// - the clamp of the two spread inputs to +-20 keeps a NaN (compares, as HIP's v_cmp / v_cndmask on the FP16 value). +// The three 32-bit stores at the same per-lane offset into different buffers (A, B, latent) are the shape the driver was +// seen to merge (README "drop a store"): the pointers are read through wg_uniform(). +// +// @wg-kernel l_coef co=l_ops.co block=256 wave=32 args=p,p,p,p,p,p,b56 +#include "wg_platform.hlsli" + +WG_ARGS_BEGIN + uint y2, A, B, C, latent, w; // pointers + int P_rw, P_rh, P_fw, P_fh, P_reset, P_pad0; // LDecParams (Model M's WgDecParams; only rw, rh, fw, fh read) + float P_dw, P_dh, P_kx, P_ky, P_jx, P_jy, P_pjx, P_pjy; +WG_ARGS_END + +static uint g_z; // 0, opaque to the compiler (set from the grid size) +float opq(float x) { return asfloat(asuint(x) ^ g_z); } +float bits(uint b) { return asfloat(b); } + +// sigma(x) = 1 / (1 + 2^(x * -log2 e)): v_mul, v_exp, v_add 1.0, v_rcp +float l_sigma(float x) { return wg_rcp(1.0f + wg_exp2(x * bits(0xBFB8AA3Bu))); } + +// tanh(t): for |t| >= 0.6 (and not NaN), 1 - 2 / (2^(2 |t| log2 e) + 1), 1 from about 9.01 on, sign of t; below, the odd +// polynomial. Both sides are computed; the ISA's branch selects one, select() here (both are pure). +float l_tanh(float t) +{ + const float a = abs(t); + float big = mad(wg_rcp(wg_exp2(a * bits(0x4038AA3Bu)) + 1.0f), -2.0f, 1.0f); + big = select(bits(0x41102CB4u) > a || isnan(a), big, 1.0f); // v_cmp_nle 9.01, |t|; v_cndmask + big = asfloat((asuint(big) & 0x7fffffffu) | (asuint(t) & 0x80000000u)); // v_bfi copysign + const float t2 = t * t; + const float p = mad(mad(mad(bits(0x3C80F082u), t2, bits(0xBD563CAEu)), t2, bits(0x3E085941u)), t2, bits(0xBEAAA9EDu)); + const float p2 = mad(p, t2, asfloat(g_z)); // fma(p, t2, +0), the zero opaque + const float small = mad(p2, t, t); + return select(a >= bits(0x3F19999Au), big, small); // v_cmp_le 0.6, |t| (false for a NaN: the polynomial) +} + +uint unorm8_code(float v) { return wg_f2u(wg_rint(v * 255.0f)); } // v in [0, 1]: v_mul 255, v_rndne, v_cvt_u32 +uint snorm8_code(float v) { return (uint)wg_f2i(wg_rint(wg_med3f(v, -1.0f, 1.0f) * 127.0f)) & 0xffu; } + +// the clamp of a spread input (an FP16 value) to +-20, NaN kept: HIP's v >= 20 ? 20 : (v <= -20 ? -20 : v) +float clamp20(float v) { return select(v >= 20.0f, 20.0f, select(v <= -20.0f, -20.0f, v)); } + +WG_KERNEL(l_coef) +{ + WG_SETUP(); + g_z = wg_grid_ >> 31; + const uint tid = wg_lane_id(), hh = (tid >> 4) & 1u; + const int tw = P_fw >> 1, ntok = (P_fh >> 1) * tw; + const int tok = (int)(wg_block_id() * (WG_BLOCK / WG_WAVE) + (tid >> 5)) * 16 + (int)(tid & 15u); + if (tok >= ntok) return; + const int r = tok / tw, c = tok - r * tw; + const int y = 2 * r + (int)hh; + if (y >= P_rh) return; + const float S = wg_ldf(w, 320u), T = wg_ldf(w, 324u); + const uint pA = (uint)wg_uniform((int)A), pB = (uint)wg_uniform((int)B), pC = (uint)wg_uniform((int)C), + pL = (uint)wg_uniform((int)latent); + [unroll] for (int dx = 0; dx < 2; ++dx) { + const int x = 2 * c + dx; + if (x >= P_rw) continue; + // channels 0..15 of the pixel (8 dwords, two FP16 each); Model L's 13 values are channels + // a0..a12 = 0, 1, 6, 7, 8, 9, 10, 11, 12, 2, 3, 4, 5 (clean/l/reference/table.py L_DEC_VALUES) + const uint off = (uint)(y * P_fw + x) * 64u; + const uint4 lo = wg_ld128(y2, off), hi = wg_ld128(y2, off + 16u); + const float a0 = wg_h2f_lo(lo.x), a1 = wg_h2f_hi(lo.x), a2 = wg_h2f_lo(lo.w), a3 = wg_h2f_hi(lo.w); + const float a4 = wg_h2f_lo(hi.x), a5 = wg_h2f_hi(hi.x), a6 = wg_h2f_lo(hi.y), a7 = wg_h2f_hi(hi.y); + const float a8 = wg_h2f_lo(hi.z); + const float a9 = wg_h2f_lo(lo.y), a10 = wg_h2f_hi(lo.y), a11 = wg_h2f_lo(lo.z), a12 = wg_h2f_hi(lo.z); + // A: unorm codes of RN16(sigma(a0..a2)) + const uint cA = unorm8_code(wg_rn16(l_sigma(a0))) | (unorm8_code(wg_rn16(l_sigma(a1))) << 8) | + (unorm8_code(wg_rn16(l_sigma(a2))) << 16); + // B: softmax of a3..a5 + const float m = max(max(a3, a4), a5); + const float e0 = wg_exp2((a3 - m) * bits(0x3FB8AA3Bu)), e1 = wg_exp2((a4 - m) * bits(0x3FB8AA3Bu)), + e2 = wg_exp2((a5 - m) * bits(0x3FB8AA3Bu)); + const float rs = wg_rcp(e2 + (e0 + e1)); + const uint cB = unorm8_code(opq(e0 * rs)) | (unorm8_code(opq(e1 * rs)) << 8) | (unorm8_code(opq(e2 * rs)) << 16); + // C: spreads and correlation + const float g0 = wg_rn16(l_sigma(clamp20(a6))), g1 = wg_rn16(l_sigma(clamp20(a7))); + const float sx = S * g0, sy = S * g1; + const float sxy = sx * sy; + const float rho = l_tanh(a8 * T); + const uint c0 = wg_mul_f16(sx, sx) | (wg_mul_f16(sy, sy) << 16); + const uint c1 = wg_mul_f16(sxy, rho); + // latent: snorm codes of tanh(a9..a12) + const uint cL = snorm8_code(l_tanh(a9)) | (snorm8_code(l_tanh(a10)) << 8) | (snorm8_code(l_tanh(a11)) << 16) | + (snorm8_code(l_tanh(a12)) << 24); + const uint i = (uint)(y * P_rw + x); + wg_st32(pA, i * 4u, cA); + wg_st32(pB, i * 4u, cB); + wg_st64(pC, i * 8u, uint2(c0, c1)); + wg_st32(pL, i * 4u, cL); + } +} diff --git a/clean/l/runner/l_d3d12_ops_check.py b/clean/l/runner/l_d3d12_ops_check.py index e19470d7..57e9453b 100644 --- a/clean/l/runner/l_d3d12_ops_check.py +++ b/clean/l/runner/l_d3d12_ops_check.py @@ -101,8 +101,43 @@ def project_cases(kernel, rng): return cases +# (rw, rh, fw, fh) of the coefficient step: R inside the padded grid F (even). Odd rw / rh (a token's second column or +# row outside R), F wider than R by whole tokens, and 5 x 3 / 7 x 5 (partly filled waves). +COEF_FRAMES = [(24, 16, 24, 16), (67, 35, 68, 36), (7, 5, 8, 6), (100, 68, 100, 68), (5, 3, 6, 4), (13, 7, 20, 12)] + + +def _coef_map(rng, n): + """FP16 decoder map [n][32]: mostly N(0, 3), with values beyond the clamp (+-20 on the two spread inputs), beyond + tanh's saturation (9.01) and inside its polynomial range (|t| < 0.6), so every path of l_coef runs.""" + v = rng.normal(0, 3.0, (n, 32)) + pick = rng.random((n, 32)) + v = np.where(pick < 0.08, rng.uniform(20, 60, (n, 32)) * rng.choice([-1, 1], (n, 32)), v) + v = np.where((pick >= 0.08) & (pick < 0.2), rng.uniform(-0.6, 0.6, (n, 32)), v) + v = np.where((pick >= 0.2) & (pick < 0.25), rng.uniform(8, 10, (n, 32)) * rng.choice([-1, 1], (n, 32)), v) + return v.astype(np.float16).tobytes() + + +def coef_cases(kernel, rng): + """l_coef: y2 [fh][fw][32] FP16 -> A, B, latent [rh][rw] (4 bytes each), C [rh][rw] (8 bytes); blob dec0.coef + (Model M's dec0 blob with W = 0, b = 0: S, T at byte 320); LDecParams (56 bytes) by value.""" + from clean.runner import dec0 + cases = [] + for (rw, rh, fw, fh) in COEF_FRAMES: + npx = rw * rh + S, T = float(rng.uniform(0.5, 4.0)), float(rng.uniform(0.3, 3.0)) + blob = dec0.pack({"W": np.zeros((68, 64), np.float32), "b": np.zeros(68, np.float32), "S": S, "T": T}) + P = bytes(dec0.DecParams(rw=rw, rh=rh, fw=fw, fh=fh, reset=0, pad0=0, dw=2 * rw, dh=2 * rh, kx=0.5, ky=0.5, + jx=0.25, jy=-0.25, pjx=0.0, pjy=0.0)) + assert len(P) == 56 + ntok = (fh // 2) * (fw // 2) + cases.append({"kernel": kernel, "grid": -(-ntok // 128), "block": 256, + "args": [("buf", _coef_map(rng, fw * fh)), ("out", npx * 4), ("out", npx * 4), ("out", npx * 8), + ("out", npx * 4), ("buf", blob), ("bytes", P)]}) + return cases + + KINDS = {"l_linear_32_32_relu": linear_cases, "l_linear_32_32_f16": linear_cases, - "l_project_gather": project_cases, "l_dec_project": project_cases} + "l_project_gather": project_cases, "l_dec_project": project_cases, "l_coef": coef_cases} CONTROL_KINDS = {"swin_block_c96_nc": control_cases} CO = "l_ops.co"