From a0b5c4a8835082cded9b1f29e471f82105e358c8 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sat, 14 Nov 2026 05:56:02 -0500 Subject: [PATCH] Model L on D3D12: the folded decoder block's epilogue on the driver's conversions l_swin_block_c32_dec_coef12_nc's epilogue converted its 16 values per lane to FP16 and rounded five sigmoids and three products to FP16 with the software forms (wg_to_f16, wg_rn16, wg_mul_f16's wg_f2h). Now they are the driver's v_cvt_f16_f32 (the same bits; the products keep wg_mul_f16's round-to-odd FP32 value). The conversions need wave-uniform control flow, so every lane computes the per-pixel values and only the stores stay per lane. l_coef / l_coef_pack12 and the unfolded block keep their code and DXILs (byte-identical to main's), as does every other kernel built from these files. - clean/l/runner/l_fold_check.py: the cases cycle through large inputs (past the +-20 clamp and tanh's saturation), special FP8 codes (NaN) and extreme S, T (FP16 products from subnormal to infinite). - clean/l/runner/l_d3d12_profile.py: l_linear_32_32_fused in the encoder step. Refs #89 Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/hlsl/l_coef_px.hlsli | 67 ++++++++++++++++++++++++++++++ clean/dx12/kernels/fused_swin.hlsl | 14 +++++++ clean/dx12/kernels/l_swin.hlsl | 2 +- clean/dx12/kernels/swin_block.hlsl | 14 +++++++ clean/l/runner/l_d3d12_profile.py | 4 +- clean/l/runner/l_fold_check.py | 22 ++++++++-- 6 files changed, 117 insertions(+), 6 deletions(-) diff --git a/clean/dx12/hlsl/l_coef_px.hlsli b/clean/dx12/hlsl/l_coef_px.hlsli index a58877e0..42ec4eb3 100644 --- a/clean/dx12/hlsl/l_coef_px.hlsli +++ b/clean/dx12/hlsl/l_coef_px.hlsli @@ -73,3 +73,70 @@ void l_coef_px(uint4 lo, uint4 hi, float S, float T, uint pA, uint pB, uint pC, #endif wg_st32(pL, i * 4u, cL); } + +#if COEF_CVT_HW +// The folded decoder block's form (l_swin_block_c32_dec_coef12_nc; l_coef / l_coef_pack12 keep l_coef_px above): the +// same values, with the FP16 roundings by the driver's conversion (v_cvt_f16_f32; wg_platform.hlsli: the same bits as +// the software wg_f2h): the five RN16 of sigma in one conversion, the three FP16 products as their round-to-odd FP32 +// values (wg_mul_f16's) in another. The driver conversions need wave-uniform control flow: every lane computes +// l_coef_vals_hw, only the stores (l_coef_store12) are per lane. Needs COEF_PACK12. +struct LCoefVals { uint cA, cB, c0, c1, cL; }; + +// a * b as the round-to-odd FP32 value whose FP16 rounding is (f16)(a b) (wg_mul_f16 before its conversion) +float coef_mul_odd(float a, float b) +{ + const float p = a * b; + return asfloat(wg_round_odd(p, mad(a, b, -p))); +} +// wg_mul_f16's +0 for an exact zero product (a NaN product keeps its conversion) +bool coef_mul_zero(float a, float b) +{ + return ((asuint(a) & 0x7fffffffu) == 0u || (asuint(b) & 0x7fffffffu) == 0u) && !isnan(a * b); +} + +LCoefVals l_coef_vals_hw(uint4 lo, uint4 hi, float S, float T) +{ + const float a0 = wg_h2f_lo(lo.x), a1 = wg_h2f_hi(lo.x), a2 = wg_h2f_lo(lo.w), a3 = wg_h2f_hi(lo.w); + const float a4 = wg_h2f_lo(hi.x), a5 = wg_h2f_hi(hi.x), a6 = wg_h2f_lo(hi.y), a7 = wg_h2f_hi(hi.y); + const float a8 = wg_h2f_lo(hi.z); + const float a9 = wg_h2f_lo(lo.y), a10 = wg_h2f_hi(lo.y), a11 = wg_h2f_lo(lo.z), a12 = wg_h2f_hi(lo.z); + LCoefVals o; + // RN16 of sigma(a0), sigma(a1), sigma(a2), sigma(clamp20(a6)), sigma(clamp20(a7)): one conversion + v8f sg; + sg.v[0] = l_sigma(a0); sg.v[1] = l_sigma(a1); sg.v[2] = l_sigma(a2); + sg.v[3] = l_sigma(clamp20(a6)); sg.v[4] = l_sigma(clamp20(a7)); + sg.v[5] = 0.0f; sg.v[6] = 0.0f; sg.v[7] = 0.0f; + const uint4 hs = wg_to_f16_ags(sg); + // A: unorm codes of RN16(sigma(a0..a2)) + o.cA = unorm8_code(wg_h2f_lo(hs.x)) | (unorm8_code(wg_h2f_hi(hs.x)) << 8) | (unorm8_code(wg_h2f_lo(hs.y)) << 16); + // B: softmax of a3..a5 + const float m = max(max(a3, a4), a5); + const float e0 = wg_exp2((a3 - m) * bits(0x3FB8AA3Bu)), e1 = wg_exp2((a4 - m) * bits(0x3FB8AA3Bu)), + e2 = wg_exp2((a5 - m) * bits(0x3FB8AA3Bu)); + const float rs = wg_rcp(e2 + (e0 + e1)); + o.cB = unorm8_code(opq(e0 * rs)) | (unorm8_code(opq(e1 * rs)) << 8) | (unorm8_code(opq(e2 * rs)) << 16); + // C: spreads and correlation; the three FP16 products in one conversion + const float g0 = wg_h2f_hi(hs.y), g1 = wg_h2f_lo(hs.z); + const float sx = S * g0, sy = S * g1; + const float sxy = sx * sy; + const float rho = l_tanh(a8 * T); + v8f po; + po.v[0] = coef_mul_odd(sx, sx); po.v[1] = coef_mul_odd(sy, sy); po.v[2] = coef_mul_odd(sxy, rho); + po.v[3] = 0.0f; po.v[4] = 0.0f; po.v[5] = 0.0f; po.v[6] = 0.0f; po.v[7] = 0.0f; + const uint4 hp = wg_to_f16_ags(po); + const uint h0 = coef_mul_zero(sx, sx) ? 0u : (hp.x & 0xffffu), h1 = coef_mul_zero(sy, sy) ? 0u : (hp.x >> 16); + o.c0 = h0 | (h1 << 16); + o.c1 = coef_mul_zero(sxy, rho) ? 0u : (hp.y & 0xffffu); + // latent: snorm codes of tanh(a9..a12) + o.cL = snorm8_code(l_tanh(a9)) | (snorm8_code(l_tanh(a10)) << 8) | (snorm8_code(l_tanh(a11)) << 16) | + (snorm8_code(l_tanh(a12)) << 24); + return o; +} + +void l_coef_store12(LCoefVals o, uint pA, uint pC, uint pL, uint i) +{ + wg_st64(pA, i * 8u, uint2(o.cA | (o.cB << 24), (o.cB >> 8) | (o.c1 << 16))); + wg_st32(pC, i * 4u, o.c0); + wg_st32(pL, i * 4u, o.cL); +} +#endif diff --git a/clean/dx12/kernels/fused_swin.hlsl b/clean/dx12/kernels/fused_swin.hlsl index bff80ef6..8e34b400 100644 --- a/clean/dx12/kernels/fused_swin.hlsl +++ b/clean/dx12/kernels/fused_swin.hlsl @@ -1138,6 +1138,20 @@ WG_KERNEL(SW_ENTRY) #if SW_OUT_F16 // Output RN16(x2) as FP16, real window members only (HIP LStoreF16Epi). uint4 o16[KC]; +#if SW_OUT_COEF12 && COEF_CVT_HW + // the driver's conversions (the same bits as wg_to_f16 / the software forms): every lane computes, the hh = 0 lanes + // of real pixels inside the render rectangle store (the conversions need wave-uniform control flow) + [unroll] for (uint kh = 0; kh < KC; ++kh) o16[kh] = wg_to_f16_ags(acc[kh]); + { + g_z = wg_grid_ >> 31; + const float S = wg_ldf(coefW, 320u), T = wg_ldf(coefW, 324u); + const LCoefVals cv = l_coef_vals_hw(o16[0], o16[1], S, T); + if (dst >= 0 && hh == 0u && pr < coefRh && pc < coefRw) + l_coef_store12(cv, (uint)wg_uniform((int)coefA), (uint)wg_uniform((int)coefC), (uint)wg_uniform((int)coefL), + (uint)(pr * coefRw + pc)); + } + return; +#endif [unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16(acc[kf]); #if SW_OUT_COEF12 // l_coef_pack12 of the map: o16[0] / o16[1] of the hh = 0 lane are the pixel's channels 0..7 / 8..15 (the map's diff --git a/clean/dx12/kernels/l_swin.hlsl b/clean/dx12/kernels/l_swin.hlsl index cd51fff9..fbd7e187 100644 --- a/clean/dx12/kernels/l_swin.hlsl +++ b/clean/dx12/kernels/l_swin.hlsl @@ -16,5 +16,5 @@ // byte for byte the pair _dec_nc -> l_coef_pack12, which stays the fallback route); // the FP16 map is not written. Arguments: _dec_nc's (out unused), then A, C, latent, // dec0.coef, rw, rh -// @wg-kernel l_swin_block_c32_dec_coef12_nc co=- block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p,p,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=1,SW_SRC=2,SW_OUT_F16=1,SW_OUT_COEF12=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_coef12_nc +// @wg-kernel l_swin_block_c32_dec_coef12_nc co=- block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p,p,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=1,SW_SRC=2,SW_OUT_F16=1,SW_OUT_COEF12=1,COEF_CVT_HW=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_coef12_nc #include "swin_block.hlsl" diff --git a/clean/dx12/kernels/swin_block.hlsl b/clean/dx12/kernels/swin_block.hlsl index 7473e1e8..fda2ef1c 100644 --- a/clean/dx12/kernels/swin_block.hlsl +++ b/clean/dx12/kernels/swin_block.hlsl @@ -979,6 +979,20 @@ WG_KERNEL(SW_ENTRY) #if SW_OUT_F16 // Output RN16(x2) as FP16, real window members only (HIP LStoreF16Epi). uint4 o16[KC]; +#if SW_OUT_COEF12 && COEF_CVT_HW + // the driver's conversions (the same bits as wg_to_f16 / the software forms): every lane computes, the hh = 0 lanes + // of real pixels inside the render rectangle store (the conversions need wave-uniform control flow) + [unroll] for (uint kh = 0; kh < KC; ++kh) o16[kh] = wg_to_f16_ags(acc[kh]); + { + g_z = wg_grid_ >> 31; + const float S = wg_ldf(coefW, 320u), T = wg_ldf(coefW, 324u); + const LCoefVals cv = l_coef_vals_hw(o16[0], o16[1], S, T); + if (dst >= 0 && hh == 0u && pr < coefRh && pc < coefRw) + l_coef_store12(cv, (uint)wg_uniform((int)coefA), (uint)wg_uniform((int)coefC), (uint)wg_uniform((int)coefL), + (uint)(pr * coefRw + pc)); + } + return; +#endif [unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16(acc[kf]); #if SW_OUT_COEF12 // l_coef_pack12 of the map: o16[0] / o16[1] of the hh = 0 lane are the pixel's channels 0..7 / 8..15 (the map's diff --git a/clean/l/runner/l_d3d12_profile.py b/clean/l/runner/l_d3d12_profile.py index 597fbada..5393fc1f 100644 --- a/clean/l/runner/l_d3d12_profile.py +++ b/clean/l/runner/l_d3d12_profile.py @@ -28,8 +28,8 @@ sys.path.insert(0, ROOT) SHAPES = {"1080p-quality": ((1280, 720), (1920, 1080)), "1440p-quality": ((1706, 960), (2560, 1440))} -ENC = ("lev/input_encoder", "l_linear_32_32_relu", "l_linear_32_32_f16", "l_swin_block_c32_enc", "l_project_gather", - "input_encoder") +ENC = ("lev/input_encoder", "l_linear_32_32_relu", "l_linear_32_32_f16", "l_linear_32_32_fused", "l_swin_block_c32_enc", + "l_project_gather", "input_encoder") DEC = ("l_dec_project", "l_swin_block_c32_dec", "l_coef", "coef_decoder", "_add_dec0") POST = ("post_", "scale_rgb", "f16lut") SWIN = ("swin_", "patch_merge", "expand_", "bneck/") diff --git a/clean/l/runner/l_fold_check.py b/clean/l/runner/l_fold_check.py index 8199f835..fe66c509 100644 --- a/clean/l/runner/l_fold_check.py +++ b/clean/l/runner/l_fold_check.py @@ -54,10 +54,19 @@ def rects(H, W): return sorted({(W, H), (max(1, W - 3), max(1, H - 1)), (max(1, W // 2 + 1), max(1, H // 2 - 1))}) +# FP8 codes the network does not produce but the per-pixel functions must treat as the pair does: both NaN codes, +# +-448, +-0, subnormals +SPECIAL = np.array([0x7f, 0xff, 0x7e, 0xfe, 0x00, 0x80, 0x01, 0x81, 0x07, 0x87], np.uint8) + + def cases(rng): - """(H, W, offset, rw, rh, y bytes, block blob, coefficient blob, LDecParams bytes) of every case.""" + """(H, W, offset, rw, rh, y bytes, block blob, coefficient blob, LDecParams bytes) of every case. The cases cycle + through four kinds: the network's range; large inputs (outputs past the +-20 clamp and tanh's saturation); special + FP8 codes in a tenth of y (NaN outputs); extreme coefficient scalars S, T (FP16 products from subnormal to + infinite).""" from clean.runner import dec0 kind = "second" + n = 0 for (H, W) in GRIDS: for parity in (0, 1): off = T.block_offset(T.TABLE, kind, parity) @@ -65,9 +74,16 @@ def cases(rng): blob = LD._params_c32(rng) if not LD._nc(blob): continue # the folded kernel is the _nc form only - scale = float(rng.choice([1.5, 4.0])) - yb = fp8_encode(q8(rng.normal(0, scale, (H, W, 32)))).tobytes() + mode, n = n % 4, n + 1 + scale = 60.0 if mode == 1 else float(rng.choice([1.5, 4.0])) + y = fp8_encode(q8(rng.normal(0, scale, (H, W, 32)))).astype(np.uint8) + if mode == 2: + pick = rng.random(y.shape) < 0.1 + y[pick] = rng.choice(SPECIAL, int(pick.sum())) + yb = y.tobytes() S, T_ = float(rng.uniform(0.5, 4.0)), float(rng.uniform(0.3, 3.0)) + if mode == 3: + S, T_ = float(rng.choice([1e-3, 3e-3, 300.0, 1000.0])), float(rng.choice([1e-3, 50.0])) cw = dec0.pack({"W": np.zeros((68, 64), np.float32), "b": np.zeros(68, np.float32), "S": S, "T": T_}) P = bytes(dec0.DecParams(rw=rw, rh=rh, fw=W, fh=H, reset=0, pad0=0, dw=2 * rw, dh=2 * rh, kx=0.5, ky=0.5, jx=0.25, jy=-0.25, pjx=0.0, pjy=0.0))