diff --git a/clean/dx12/compare.py b/clean/dx12/compare.py index 2eb3a8b2..80b35af6 100644 --- a/clean/dx12/compare.py +++ b/clean/dx12/compare.py @@ -68,7 +68,7 @@ "expand_c96", "d1p12/swin_block_c64_add_dec0_nc", "w64/post_tf_f32_p12_diet_vsit"} # The native backend (--backend d3d12) runs the D3D12 fusions of WG_D3D12_FUSION (default: runtime.cpp's # kDefaultD3d12Fusion, mirrored here), which replace some launches (kernels/fused_swin.hlsl, fused_bneck.hlsl, fused_dec.hlsl). -DEFAULT_D3D12_FUSION = "pm,xe,mb,rv,xo,xd" +DEFAULT_D3D12_FUSION = "pm,xe,mb,rv,xo,xd,lc" # lc: Model L only def d3d12_fusion(): diff --git a/clean/dx12/hlsl/l_coef_px.hlsli b/clean/dx12/hlsl/l_coef_px.hlsli new file mode 100644 index 00000000..a58877e0 --- /dev/null +++ b/clean/dx12/hlsl/l_coef_px.hlsli @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// Model L's per-pixel coefficient functions (issue #89; HIP l_coef_body of clean/l/kernels/l_ops.hip), shared by +// kernels/l_ops_coef.hlsl (l_coef, l_coef_pack12) and the decoder block that ends in them (kernels/l_swin.hlsl, +// l_swin_block_c32_dec_coef12_nc): one pixel's channels 0..15 of the decoder block's FP16 output (y2) -> A, B, C and +// the latent, stored at pixel index i of the render rectangle. Every FP32 operation follows HIP's ISA of l_coef (see +// l_ops_coef.hlsl). Needs COEF_PACK12 (0: Model M's base layout, 1: pack12) and wg_platform.hlsli; uses g_z, an opaque +// zero the kernel sets from its grid size (g_z = wg_grid_ >> 31). + +static uint g_z; // 0, opaque to the compiler (set from the grid size) +float opq(float x) { return asfloat(asuint(x) ^ g_z); } +float bits(uint b) { return asfloat(b); } + +// sigma(x) = 1 / (1 + 2^(x * -log2 e)): v_mul, v_exp, v_add 1.0, v_rcp +float l_sigma(float x) { return wg_rcp(1.0f + wg_exp2(x * bits(0xBFB8AA3Bu))); } + +// tanh(t): for |t| >= 0.6 (and not NaN), 1 - 2 / (2^(2 |t| log2 e) + 1), 1 from about 9.01 on, sign of t; below, the odd +// polynomial. Both sides are computed; the ISA's branch selects one, select() here (both are pure). +float l_tanh(float t) +{ + const float a = abs(t); + float big = mad(wg_rcp(wg_exp2(a * bits(0x4038AA3Bu)) + 1.0f), -2.0f, 1.0f); + big = select(bits(0x41102CB4u) > a || isnan(a), big, 1.0f); // v_cmp_nle 9.01, |t|; v_cndmask + big = asfloat((asuint(big) & 0x7fffffffu) | (asuint(t) & 0x80000000u)); // v_bfi copysign + const float t2 = t * t; + const float p = mad(mad(mad(bits(0x3C80F082u), t2, bits(0xBD563CAEu)), t2, bits(0x3E085941u)), t2, bits(0xBEAAA9EDu)); + const float p2 = mad(p, t2, asfloat(g_z)); // fma(p, t2, +0), the zero opaque + const float small = mad(p2, t, t); + return select(a >= bits(0x3F19999Au), big, small); // v_cmp_le 0.6, |t| (false for a NaN: the polynomial) +} + +uint unorm8_code(float v) { return wg_f2u(wg_rint(v * 255.0f)); } // v in [0, 1]: v_mul 255, v_rndne, v_cvt_u32 +uint snorm8_code(float v) { return (uint)wg_f2i(wg_rint(wg_med3f(v, -1.0f, 1.0f) * 127.0f)) & 0xffu; } + +// the clamp of a spread input (an FP16 value) to +-20, NaN kept: HIP's v >= 20 ? 20 : (v <= -20 ? -20 : v) +float clamp20(float v) { return select(v >= 20.0f, 20.0f, select(v <= -20.0f, -20.0f, v)); } + +// One pixel: lo / hi = its channels 0..7 / 8..15 (four dwords each, two FP16 per dword, channel 2k in the low half); +// Model L's 13 values are channels a0..a12 = 0, 1, 6, 7, 8, 9, 10, 11, 12, 2, 3, 4, 5 (clean/l/reference/table.py +// L_DEC_VALUES). S, T: the coefficient blob's scalars (dec0.coef, bytes 320 and 324). pA, pB, pC, pL: the output +// buffers (pB unused with COEF_PACK12); i: the pixel's index in the render rectangle (y rw + x). +void l_coef_px(uint4 lo, uint4 hi, float S, float T, uint pA, uint pB, uint pC, uint pL, uint i) +{ + const float a0 = wg_h2f_lo(lo.x), a1 = wg_h2f_hi(lo.x), a2 = wg_h2f_lo(lo.w), a3 = wg_h2f_hi(lo.w); + const float a4 = wg_h2f_lo(hi.x), a5 = wg_h2f_hi(hi.x), a6 = wg_h2f_lo(hi.y), a7 = wg_h2f_hi(hi.y); + const float a8 = wg_h2f_lo(hi.z); + const float a9 = wg_h2f_lo(lo.y), a10 = wg_h2f_hi(lo.y), a11 = wg_h2f_lo(lo.z), a12 = wg_h2f_hi(lo.z); + // A: unorm codes of RN16(sigma(a0..a2)) + const uint cA = unorm8_code(wg_rn16(l_sigma(a0))) | (unorm8_code(wg_rn16(l_sigma(a1))) << 8) | + (unorm8_code(wg_rn16(l_sigma(a2))) << 16); + // B: softmax of a3..a5 + const float m = max(max(a3, a4), a5); + const float e0 = wg_exp2((a3 - m) * bits(0x3FB8AA3Bu)), e1 = wg_exp2((a4 - m) * bits(0x3FB8AA3Bu)), + e2 = wg_exp2((a5 - m) * bits(0x3FB8AA3Bu)); + const float rs = wg_rcp(e2 + (e0 + e1)); + const uint cB = unorm8_code(opq(e0 * rs)) | (unorm8_code(opq(e1 * rs)) << 8) | (unorm8_code(opq(e2 * rs)) << 16); + // C: spreads and correlation + const float g0 = wg_rn16(l_sigma(clamp20(a6))), g1 = wg_rn16(l_sigma(clamp20(a7))); + const float sx = S * g0, sy = S * g1; + const float sxy = sx * sy; + const float rho = l_tanh(a8 * T); + const uint c0 = wg_mul_f16(sx, sx) | (wg_mul_f16(sy, sy) << 16); + const uint c1 = wg_mul_f16(sxy, rho); + // latent: snorm codes of tanh(a9..a12) + const uint cL = snorm8_code(l_tanh(a9)) | (snorm8_code(l_tanh(a10)) << 8) | (snorm8_code(l_tanh(a11)) << 16) | + (snorm8_code(l_tanh(a12)) << 24); +#if COEF_PACK12 + wg_st64(pA, i * 8u, uint2(cA | (cB << 24), (cB >> 8) | (c1 << 16))); + wg_st32(pC, i * 4u, c0); +#else + wg_st32(pA, i * 4u, cA); + wg_st32(pB, i * 4u, cB); + wg_st64(pC, i * 8u, uint2(c0, c1)); +#endif + wg_st32(pL, i * 4u, cL); +} diff --git a/clean/dx12/kernels/fused_swin.hlsl b/clean/dx12/kernels/fused_swin.hlsl index 8527eba0..bff80ef6 100644 --- a/clean/dx12/kernels/fused_swin.hlsl +++ b/clean/dx12/kernels/fused_swin.hlsl @@ -450,6 +450,16 @@ void load_frags_g(uint base, uint off, out uint2 f[KC]) #if (SW_SRC != 0 || SW_OUT_F16) && (SW_ADD || SW_ECL > 0) #error "SW_SRC / SW_OUT_F16: plain blocks only" #endif +#ifndef SW_OUT_COEF12 +#define SW_OUT_COEF12 0 +#endif +#if SW_OUT_COEF12 +#if !SW_OUT_F16 || SW_C != 32 +#error "SW_OUT_COEF12: Model L's decoder block only (SW_OUT_F16, C = 32)" +#endif +#define COEF_PACK12 1 +#include "l_coef_px.hlsli" +#endif #if SW_SRC == 1 // LHaloF16Src: the lane's channels hh * C/2 + 8 k + j of the member at grid position (pr, pc) (inside the 2-pixel halo), // from the FP16 map [H + 4][W + 4][C] @@ -1129,6 +1139,18 @@ WG_KERNEL(SW_ENTRY) // Output RN16(x2) as FP16, real window members only (HIP LStoreF16Epi). uint4 o16[KC]; [unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16(acc[kf]); +#if SW_OUT_COEF12 + // l_coef_pack12 of the map: o16[0] / o16[1] of the hh = 0 lane are the pixel's channels 0..7 / 8..15 (the map's + // bytes dst * 64 + 0 .. 31, its lo / hi loads); Model L's 13 values are all among them + if (dst >= 0 && hh == 0u && pr < coefRh && pc < coefRw) { + g_z = wg_grid_ >> 31; + const float S = wg_ldf(coefW, 320u), T = wg_ldf(coefW, 324u); + const uint pA = (uint)wg_uniform((int)coefA), pC = (uint)wg_uniform((int)coefC), + pL = (uint)wg_uniform((int)coefL); + l_coef_px(o16[0], o16[1], S, T, pA, 0u, pC, pL, (uint)(pr * coefRw + pc)); + } + return; +#endif if (dst >= 0) { const uint ob16 = ((uint)dst * SW_C + hh * HALF) * 2u; [unroll] for (uint kg = 0; kg < KC; ++kg) wg_st128(out_p, ob16 + 16u * kg, o16[kg]); diff --git a/clean/dx12/kernels/fused_swin_gen.py b/clean/dx12/kernels/fused_swin_gen.py index 3dc860ce..c89eef05 100644 --- a/clean/dx12/kernels/fused_swin_gen.py +++ b/clean/dx12/kernels/fused_swin_gen.py @@ -66,6 +66,10 @@ #if SW_ECL > 0 uint aux; // unused by the expand kernels (null) #endif +#if SW_OUT_COEF12 + uint coefA, coefC, coefL, coefW; // pointers: l_coef_pack12's A, C, latent and coefficient blob (out_p unused) + int coefRw, coefRh; // the render rectangle +#endif WG_ARGS_END""" ARGS_NEW = """#ifndef SW_PM #define SW_PM 0 @@ -352,7 +356,7 @@ def main(): if "--check" in sys.argv: print("fused_swin.hlsl is STALE (run fused_swin_gen.py)") return 1 - with open(DST, "w", encoding="utf-8") as f: + with open(DST, "w", encoding="utf-8", newline="\n") as f: # LF on every OS (.gitattributes: no conversion) f.write(s) print("fused_swin.hlsl regenerated") return 0 diff --git a/clean/dx12/kernels/l_ops_coef.hlsl b/clean/dx12/kernels/l_ops_coef.hlsl index 318479ab..b9f28b7b 100644 --- a/clean/dx12/kernels/l_ops_coef.hlsl +++ b/clean/dx12/kernels/l_ops_coef.hlsl @@ -31,33 +31,9 @@ WG_ARGS_BEGIN float P_dw, P_dh, P_kx, P_ky, P_jx, P_jy, P_pjx, P_pjy; WG_ARGS_END -static uint g_z; // 0, opaque to the compiler (set from the grid size) -float opq(float x) { return asfloat(asuint(x) ^ g_z); } -float bits(uint b) { return asfloat(b); } - -// sigma(x) = 1 / (1 + 2^(x * -log2 e)): v_mul, v_exp, v_add 1.0, v_rcp -float l_sigma(float x) { return wg_rcp(1.0f + wg_exp2(x * bits(0xBFB8AA3Bu))); } - -// tanh(t): for |t| >= 0.6 (and not NaN), 1 - 2 / (2^(2 |t| log2 e) + 1), 1 from about 9.01 on, sign of t; below, the odd -// polynomial. Both sides are computed; the ISA's branch selects one, select() here (both are pure). -float l_tanh(float t) -{ - const float a = abs(t); - float big = mad(wg_rcp(wg_exp2(a * bits(0x4038AA3Bu)) + 1.0f), -2.0f, 1.0f); - big = select(bits(0x41102CB4u) > a || isnan(a), big, 1.0f); // v_cmp_nle 9.01, |t|; v_cndmask - big = asfloat((asuint(big) & 0x7fffffffu) | (asuint(t) & 0x80000000u)); // v_bfi copysign - const float t2 = t * t; - const float p = mad(mad(mad(bits(0x3C80F082u), t2, bits(0xBD563CAEu)), t2, bits(0x3E085941u)), t2, bits(0xBEAAA9EDu)); - const float p2 = mad(p, t2, asfloat(g_z)); // fma(p, t2, +0), the zero opaque - const float small = mad(p2, t, t); - return select(a >= bits(0x3F19999Au), big, small); // v_cmp_le 0.6, |t| (false for a NaN: the polynomial) -} - -uint unorm8_code(float v) { return wg_f2u(wg_rint(v * 255.0f)); } // v in [0, 1]: v_mul 255, v_rndne, v_cvt_u32 -uint snorm8_code(float v) { return (uint)wg_f2i(wg_rint(wg_med3f(v, -1.0f, 1.0f) * 127.0f)) & 0xffu; } - -// the clamp of a spread input (an FP16 value) to +-20, NaN kept: HIP's v >= 20 ? 20 : (v <= -20 ? -20 : v) -float clamp20(float v) { return select(v >= 20.0f, 20.0f, select(v <= -20.0f, -20.0f, v)); } +// the per-pixel functions (sigma, tanh, the codes, the clamp, the stores): hlsl/l_coef_px.hlsli, shared with the decoder +// block that ends in this step (l_swin.hlsl, l_swin_block_c32_dec_coef12_nc) +#include "l_coef_px.hlsli" #if COEF_PACK12 WG_KERNEL(l_coef_pack12) @@ -80,42 +56,9 @@ WG_KERNEL(l_coef) [unroll] for (int dx = 0; dx < 2; ++dx) { const int x = 2 * c + dx; if (x >= P_rw) continue; - // channels 0..15 of the pixel (8 dwords, two FP16 each); Model L's 13 values are channels - // a0..a12 = 0, 1, 6, 7, 8, 9, 10, 11, 12, 2, 3, 4, 5 (clean/l/reference/table.py L_DEC_VALUES) + // channels 0..15 of the pixel (8 dwords, two FP16 each) const uint off = (uint)(y * P_fw + x) * 64u; const uint4 lo = wg_ld128(y2, off), hi = wg_ld128(y2, off + 16u); - const float a0 = wg_h2f_lo(lo.x), a1 = wg_h2f_hi(lo.x), a2 = wg_h2f_lo(lo.w), a3 = wg_h2f_hi(lo.w); - const float a4 = wg_h2f_lo(hi.x), a5 = wg_h2f_hi(hi.x), a6 = wg_h2f_lo(hi.y), a7 = wg_h2f_hi(hi.y); - const float a8 = wg_h2f_lo(hi.z); - const float a9 = wg_h2f_lo(lo.y), a10 = wg_h2f_hi(lo.y), a11 = wg_h2f_lo(lo.z), a12 = wg_h2f_hi(lo.z); - // A: unorm codes of RN16(sigma(a0..a2)) - const uint cA = unorm8_code(wg_rn16(l_sigma(a0))) | (unorm8_code(wg_rn16(l_sigma(a1))) << 8) | - (unorm8_code(wg_rn16(l_sigma(a2))) << 16); - // B: softmax of a3..a5 - const float m = max(max(a3, a4), a5); - const float e0 = wg_exp2((a3 - m) * bits(0x3FB8AA3Bu)), e1 = wg_exp2((a4 - m) * bits(0x3FB8AA3Bu)), - e2 = wg_exp2((a5 - m) * bits(0x3FB8AA3Bu)); - const float rs = wg_rcp(e2 + (e0 + e1)); - const uint cB = unorm8_code(opq(e0 * rs)) | (unorm8_code(opq(e1 * rs)) << 8) | (unorm8_code(opq(e2 * rs)) << 16); - // C: spreads and correlation - const float g0 = wg_rn16(l_sigma(clamp20(a6))), g1 = wg_rn16(l_sigma(clamp20(a7))); - const float sx = S * g0, sy = S * g1; - const float sxy = sx * sy; - const float rho = l_tanh(a8 * T); - const uint c0 = wg_mul_f16(sx, sx) | (wg_mul_f16(sy, sy) << 16); - const uint c1 = wg_mul_f16(sxy, rho); - // latent: snorm codes of tanh(a9..a12) - const uint cL = snorm8_code(l_tanh(a9)) | (snorm8_code(l_tanh(a10)) << 8) | (snorm8_code(l_tanh(a11)) << 16) | - (snorm8_code(l_tanh(a12)) << 24); - const uint i = (uint)(y * P_rw + x); -#if COEF_PACK12 - wg_st64(pA, i * 8u, uint2(cA | (cB << 24), (cB >> 8) | (c1 << 16))); - wg_st32(pC, i * 4u, c0); -#else - wg_st32(pA, i * 4u, cA); - wg_st32(pB, i * 4u, cB); - wg_st64(pC, i * 8u, uint2(c0, c1)); -#endif - wg_st32(pL, i * 4u, cL); + l_coef_px(lo, hi, S, T, pA, pB, pC, pL, (uint)(y * P_rw + x)); } } diff --git a/clean/dx12/kernels/l_swin.hlsl b/clean/dx12/kernels/l_swin.hlsl index 6f6c9641..cd51fff9 100644 --- a/clean/dx12/kernels/l_swin.hlsl +++ b/clean/dx12/kernels/l_swin.hlsl @@ -12,4 +12,9 @@ // @wg-kernel l_swin_block_c32_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_ENTRY_OVERRIDE=l_swin_block_c32_nc // @wg-kernel l_swin_block_c32_enc_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=1,SW_SRC=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_enc_nc // @wg-kernel l_swin_block_c32_dec_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=1,SW_SRC=2,SW_OUT_F16=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_nc +// l_swin_block_c32_dec_coef12_nc D3D12 only: the decoder block with l_coef_pack12 as its epilogue (SW_OUT_COEF12; +// byte for byte the pair _dec_nc -> l_coef_pack12, which stays the fallback route); +// the FP16 map is not written. Arguments: _dec_nc's (out unused), then A, C, latent, +// dec0.coef, rw, rh +// @wg-kernel l_swin_block_c32_dec_coef12_nc co=- block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p,p,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=1,SW_SRC=2,SW_OUT_F16=1,SW_OUT_COEF12=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_coef12_nc #include "swin_block.hlsl" diff --git a/clean/dx12/kernels/swin_block.hlsl b/clean/dx12/kernels/swin_block.hlsl index cf79de3b..7473e1e8 100644 --- a/clean/dx12/kernels/swin_block.hlsl +++ b/clean/dx12/kernels/swin_block.hlsl @@ -12,6 +12,9 @@ // (pr + 2, pc + 2) (LHaloF16Src); 2 Model L's decoder block, the FP8 map with the token-grid // mirror, sub-pixel kept (LCoarseFp8Src) // SW_OUT_F16 1: the output is RN16(x2) as an FP16 map [H][W][C] (Model L's decoder block, LStoreF16Epi) +// SW_OUT_COEF12 1 (with SW_OUT_F16, C = 32): instead of the FP16 map, l_coef_pack12 of it as the epilogue: +// the lane that holds a pixel's channels 0..15 (hh = 0) runs hlsl/l_coef_px.hlsli on those +// FP16 values, the bytes the map would hold, for the pixels inside the render rectangle // HIP ISA: \dx12-m\isa\swin_block_c64_nc.s, swin_block_c96_nc.s, swin_block_c96_add_ex_nc.s. // Every FP32 operation follows that ISA: the RMSNorm sums of squares pair as fma(a_even, a_even, a_odd * a_odd); the // soft cap is fma(c^3, -1 / (3 cap^2), c), the max is over t * (tau log2 e) but the exponent is fma(t, tau log2 e, -max); @@ -30,6 +33,10 @@ WG_ARGS_BEGIN #if SW_ECL > 0 uint aux; // unused by the expand kernels (null) #endif +#if SW_OUT_COEF12 + uint coefA, coefC, coefL, coefW; // pointers: l_coef_pack12's A, C, latent and coefficient blob (out_p unused) + int coefRw, coefRh; // the render rectangle +#endif WG_ARGS_END // ---- BlockLayout (FORMATS.md, "Swin block section") --------------------------------------------------- @@ -403,6 +410,16 @@ void load_frags_g(uint base, uint off, out uint2 f[KC]) #if (SW_SRC != 0 || SW_OUT_F16) && (SW_ADD || SW_ECL > 0) #error "SW_SRC / SW_OUT_F16: plain blocks only" #endif +#ifndef SW_OUT_COEF12 +#define SW_OUT_COEF12 0 +#endif +#if SW_OUT_COEF12 +#if !SW_OUT_F16 || SW_C != 32 +#error "SW_OUT_COEF12: Model L's decoder block only (SW_OUT_F16, C = 32)" +#endif +#define COEF_PACK12 1 +#include "l_coef_px.hlsli" +#endif #if SW_SRC == 1 // LHaloF16Src: the lane's channels hh * C/2 + 8 k + j of the member at grid position (pr, pc) (inside the 2-pixel halo), // from the FP16 map [H + 4][W + 4][C] @@ -963,6 +980,18 @@ WG_KERNEL(SW_ENTRY) // Output RN16(x2) as FP16, real window members only (HIP LStoreF16Epi). uint4 o16[KC]; [unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16(acc[kf]); +#if SW_OUT_COEF12 + // l_coef_pack12 of the map: o16[0] / o16[1] of the hh = 0 lane are the pixel's channels 0..7 / 8..15 (the map's + // bytes dst * 64 + 0 .. 31, its lo / hi loads); Model L's 13 values are all among them + if (dst >= 0 && hh == 0u && pr < coefRh && pc < coefRw) { + g_z = wg_grid_ >> 31; + const float S = wg_ldf(coefW, 320u), T = wg_ldf(coefW, 324u); + const uint pA = (uint)wg_uniform((int)coefA), pC = (uint)wg_uniform((int)coefC), + pL = (uint)wg_uniform((int)coefL); + l_coef_px(o16[0], o16[1], S, T, pA, 0u, pC, pL, (uint)(pr * coefRw + pc)); + } + return; +#endif if (dst >= 0) { const uint ob16 = ((uint)dst * SW_C + hh * HALF) * 2u; [unroll] for (uint kg = 0; kg < KC; ++kg) wg_st128(out_p, ob16 + 16u * kg, o16[kg]); diff --git a/clean/dx12/tests/native_direct_check.py b/clean/dx12/tests/native_direct_check.py index 538bcf08..84fe0d88 100644 --- a/clean/dx12/tests/native_direct_check.py +++ b/clean/dx12/tests/native_direct_check.py @@ -30,7 +30,8 @@ POST='w64/post_tf_f32_p12_diet_vsit' -# The neural launches with the native D3D12 fusions of runtime.cpp kDefaultD3d12Fusion (pm,xe,mb,rv,xo,xd): enc2's block +# The neural launches with the native D3D12 fusions of runtime.cpp kDefaultD3d12Fusion (pm,xe,mb,rv,xo,xd; its lc is +# Model L's): enc2's block # with enc1's merge in reverse order (pm, rv), the bottleneck with enc2's merge and the overlap counters (mb, xo), dec2's # block with dec1's expand (xe) launched as xd on frames of at most kXdMaxPixels render pixels (WG_D3D12_XD_MAX_PIXELS) # else xo, and the fused dec1 + dec0 in its xd form on those frames. Set by main from --shape (COMMON above: the diff --git a/clean/dx12/vgpr_limits.json b/clean/dx12/vgpr_limits.json index 499d5f69..faf34036 100644 --- a/clean/dx12/vgpr_limits.json +++ b/clean/dx12/vgpr_limits.json @@ -3,6 +3,7 @@ "l_swin_block_c32_nc": 96, "l_swin_block_c32_enc_nc": 96, "l_swin_block_c32_dec_nc": 96, + "l_swin_block_c32_dec_coef12_nc": 96, "swin_block_c64_nc": 192, "swin_block_c64_add_ex_nc": 128, "swin_block_c96_nc": 192, diff --git a/clean/l/FORMATS.md b/clean/l/FORMATS.md index d319d0c3..fce40a7a 100644 --- a/clean/l/FORMATS.md +++ b/clean/l/FORMATS.md @@ -84,6 +84,7 @@ workgroup, in raster order of their map. | `l_swin_block_c32_dec[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | as the encoder block | the decoder block. x = `y` (FP8). A member outside the grid takes token-grid mirrored pixels with the sub-pixel kept. out = `y2` (**FP16**). (ox, oy) = `window_offsets["second"]`. w = `dec0_block` | | `l_coef` | exists (GPU-checked) | `y2, A, B, C, latent, w, params` | ceil((fh / 2)(fw / 2) / 128), lanes as Model M's `coef_decoder` | Model L's per-pixel functions on the 13 channels `table.L_DEC_VALUES` of `y2`: A, B, C in Model M's forms, latent = tanh(candidate). Writes the same buffers and formats as `coef_decoder`, pixels inside R only. **Reads no motion, previous latent, jitter or reset.** params = Model M's `WgDecParams` (only rw, rh, fw, fh are read). w = `dec0.coef` | | `l_coef_pack12` | exists (GPU-checked) | as `l_coef` (`B` unused, may be null) | as `l_coef` | `l_coef`'s values in Model M's dec0 "pack12" layout (`clean/kernels/decoder.hip` `WG_DEC_PACK12`, read by the output stage's `_p12` kernels): image A, 8 bytes per pixel = A0 A1 A2 B0 | B1 B2 C2 (FP16); image C, 4 bytes per pixel = C0 C1 (FP16); latent as `l_coef`. Only padding bytes are left out: unpacked, byte-identical to `l_coef` (`l_ops_check.py`) | +| `l_swin_block_c32_dec_coef12_nc` | exists (GPU-checked), D3D12 only | the decoder block's, `out` unused (null), then `A, C, latent, w_coef, rw, rh` | as the decoder block | the decoder block with `l_coef_pack12` as its epilogue: the same bytes in `A`, `C` and `latent`, `y2` not written. w = `dec0_block`, w_coef = `dec0.coef`. The native D3D12 frame runs it in place of the two (fusion `lc`, on by default); without `lc` or without the kernel in the build, the two run | ## Launch order of one frame @@ -156,4 +157,6 @@ the frame parity, as Model M's. - `clean/l/runner/l_ops_check.py`: the linears, both projections and `l_coef` against the oracle steps. - `clean/l/runner/l_enc_check.py`: `lev/input_encoder` (halo map and motion surfaces) against the oracle. - `clean/l/runner/l_d3d12_check.py` (with its D3D12 port): each D3D12 port against its HIP kernel, byte for byte. +- `clean/l/runner/l_fold_check.py`: `l_swin_block_c32_dec_coef12_nc` against `l_swin_block_c32_dec_nc` -> + `l_coef_pack12` on D3D12 and on HIP, byte for byte. - One check per new kernel follows with that kernel. diff --git a/clean/l/runner/l_fold_check.py b/clean/l/runner/l_fold_check.py new file mode 100644 index 00000000..8199f835 --- /dev/null +++ b/clean/l/runner/l_fold_check.py @@ -0,0 +1,269 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): the folded decoder block l_swin_block_c32_dec_coef12_nc (D3D12 only; clean/dx12/kernels/l_swin.hlsl, +swin_block.hlsl SW_OUT_COEF12) against the sequence it replaces, l_swin_block_c32_dec_nc -> l_coef_pack12, byte for byte. + + python clean/l/runner/l_fold_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --prepare + python clean/l/runner/l_fold_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --run + [--seeds N] --json out.json + +--prepare (CPU) builds into : l_swin.co, l_ops.co and swin.co (clean/l/runner/l_c32_check.py compile_kernels), the +D3D12 kernels (clean/dx12/build_dx12.py --only) and kernel_replay.exe. +--run (GPU, one GPU slot): every case is a decoder block grid H x W (both window offsets of the decoder's kind), a render +rectangle rw x rh inside it, a random block blob and coefficient scalars. On the same input bytes: +- the sequence on D3D12 (kernel_replay, two passes: the block writes y2, then l_coef_pack12 reads that file); +- the sequence on HIP (in this process); +- the folded kernel on D3D12. +Rules (as clean/l/runner/l_d3d12_check.py): +- the RAW downloaded bytes are compared (nothing is decoded and encoded again); +- every output buffer (A, C, latent) has 64 bytes of slack after the rectangle's pixels and starts filled with its run's + own sentinel (HIP 0xA5, the D3D12 sequence 0x5A, the folded kernel 0x3C): every rectangle byte must be identical + across the three, and every slack byte must have kept its own run's sentinel; +- a negative control (one case of the folded kernel again with half its workgroups) must be caught; +- Model M's swin_block_c96_nc (swin.co, its port already proven) runs through the same replay as a control. +Output: counts only, no timing. +""" +import argparse +import ctypes +import json +import os +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import table as T # noqa: E402 +from clean.l.runner import l_c32_check as C32 # noqa: E402 +from clean.l.runner import l_d3d12_check as LD # noqa: E402 +from clean.reference.fp8 import fp8_encode, q8 # noqa: E402 + +BLOCK, COEF, FOLD = "l_swin_block_c32_dec_nc", "l_coef_pack12", "l_swin_block_c32_dec_coef12_nc" +CONTROL = "swin_block_c96_nc" +HIP_SENTINEL, D3D12_SENTINEL, FOLD_SENTINEL = LD.HIP_SENTINEL, LD.D3D12_SENTINEL, 0x3C +SLACK = 64 +# Decoder block grids (H, W): sides multiples of 4 (the runtime's fh x fw), fewer windows than a workgroup's 8 in the +# last two. Each grid runs with the render rectangles below. +GRIDS = LD.L_BLOCK_GRIDS + + +def rects(H, W): + """Render rectangles (rw, rh) inside the grid: all of it, odd sides short of it, about half of it.""" + return sorted({(W, H), (max(1, W - 3), max(1, H - 1)), (max(1, W // 2 + 1), max(1, H // 2 - 1))}) + + +def cases(rng): + """(H, W, offset, rw, rh, y bytes, block blob, coefficient blob, LDecParams bytes) of every case.""" + from clean.runner import dec0 + kind = "second" + for (H, W) in GRIDS: + for parity in (0, 1): + off = T.block_offset(T.TABLE, kind, parity) + for (rw, rh) in rects(H, W): + blob = LD._params_c32(rng) + if not LD._nc(blob): + continue # the folded kernel is the _nc form only + scale = float(rng.choice([1.5, 4.0])) + yb = fp8_encode(q8(rng.normal(0, scale, (H, W, 32)))).tobytes() + S, T_ = float(rng.uniform(0.5, 4.0)), float(rng.uniform(0.3, 3.0)) + cw = dec0.pack({"W": np.zeros((68, 64), np.float32), "b": np.zeros(68, np.float32), "S": S, "T": T_}) + P = bytes(dec0.DecParams(rw=rw, rh=rh, fw=W, fh=H, reset=0, pad0=0, dw=2 * rw, dh=2 * rh, kx=0.5, + ky=0.5, jx=0.25, jy=-0.25, pjx=0.0, pjy=0.0)) + yield H, W, off, rw, rh, yb, blob, cw, P + + +def nwin_of(H, W, off): + return -(-(W + off[0]) // 4) * -(-(H + off[1]) // 4) + + +def out_sizes(rw, rh): + n = rw * rh + return [n * 8 + SLACK, n * 4 + SLACK, n * 4 + SLACK] # A (pack12 image A), C (image C), latent + + +def hip_sequence(g, ops, c): + """The block, then l_coef_pack12, on HIP: the three outputs' raw bytes (each buffer starts at HIP_SENTINEL).""" + H, W, off, rw, rh, yb, blob, cw, P = c + h = g.hip + owned = [] + + def buf(data): + p = h.malloc(max(len(data), 4)) + h.upload(p, data) + owned.append(p) + return p + + try: + px, pw, pcw = buf(yb), buf(blob), buf(cw) + py2 = h.malloc(H * W * 64) + owned.append(py2) + h.memset(py2, HIP_SENTINEL, H * W * 64) + outs = [] + for n in out_sizes(rw, rh): + p = h.malloc(n) + owned.append(p) + h.memset(p, HIP_SENTINEL, n) + outs.append((p, n)) + ox, oy = off + h.launch(h.get_function(g.mods["l_swin.co"], BLOCK), -(-nwin_of(H, W, off) // 8), 256, + [ctypes.c_void_p(px), ctypes.c_void_p(0), ctypes.c_void_p(py2), ctypes.c_void_p(pw), ctypes.c_int(H), + ctypes.c_int(W), ctypes.c_int(ox), ctypes.c_int(oy)]) + ntok = (H // 2) * (W // 2) + (pa, _), (pc, _), (pl, _) = outs + h.launch(h.get_function(ops, COEF), -(-ntok // 128), 256, + [ctypes.c_void_p(py2), ctypes.c_void_p(pa), ctypes.c_void_p(0), ctypes.c_void_p(pc), ctypes.c_void_p(pl), + ctypes.c_void_p(pcw), (ctypes.c_ubyte * len(P)).from_buffer_copy(P)]) + h.sync() + return [bytes(h.download(p, n)) for p, n in outs] + finally: + for p in owned: + h.free(p) + + +def write(path, data): + with open(path, "wb") as f: + f.write(data) + + +def replay(build, cdir, lines, tag): + path = os.path.join(cdir, f"cases_{tag}.txt") + with open(path, "w", newline="\n") as f: + f.write("\n".join(lines)) + env = {k: v for k, v in os.environ.items() + if not any(s in k.upper() for s in ("HIP", "LLVM", "ROCM", "HSA")) or k.upper() in ("HIP_PLATFORM",)} + env["WG_D3D12_ARENA_MB"] = "512" + r = subprocess.run([os.path.join(build, "kernel_replay.exe"), build, path], env=env, capture_output=True, text=True, + timeout=150, creationflags=0x08000000) + return r.returncode, r.stdout[-600:] + r.stderr[-300:] + + +def fold_lines(name, c, prefix="", grid=None): + H, W, off, rw, rh = c[:5] + nA, nC, nL = out_sizes(rw, rh) + return [f"case {prefix}{name}_fold", f"kernel {FOLD} {grid or -(-nwin_of(H, W, off) // 8)} 256", + f"arg buf {name}.y", "arg null", "arg null", f"arg buf {name}.w", f"arg u32 {H}", f"arg u32 {W}", + f"arg u32 {off[0]}", f"arg u32 {off[1]}", f"arg out {nA} {prefix}{name}.fA {FOLD_SENTINEL}", + f"arg out {nC} {prefix}{name}.fC {FOLD_SENTINEL}", f"arg out {nL} {prefix}{name}.fL {FOLD_SENTINEL}", + f"arg buf {name}.cw", f"arg u32 {rw}", f"arg u32 {rh}", ""] + + +def compare(got, want, defined, sent_got, sent_want): + """Raw bytes: identical over the first `defined` bytes; after them each buffer keeps its own sentinel.""" + a, b = np.frombuffer(got, np.uint8), np.frombuffer(want, np.uint8) + if a.size != b.size or a.size < defined: + return {"bytes": b.size, "identical": False, "differ": -1} + d = int(np.sum(a[:defined] != b[:defined])) + bad = int(np.sum(a[defined:] != sent_got) + np.sum(b[defined:] != sent_want)) + return {"bytes": int(b.size), "differ": d, "slack_bad": bad, "identical": d == 0 and bad == 0} + + +def run(build, rocm, seeds): + g = C32.Gpu(build, rocm) + with open(os.path.join(build, "l_ops.co"), "rb") as f: + ops = g.hip.load_module(f.read()) + cdir = os.path.join(build, "replay-fold") + os.makedirs(cdir, exist_ok=True) + pass1, pass2, all_cases, neg = [], [], {}, None + for si, seed in enumerate(seeds): + rng = np.random.default_rng(seed) + for ci, c in enumerate(cases(rng)): + name = f"s{si}_{ci}" + H, W, off, rw, rh, yb, blob, cw, P = c + all_cases[name] = (c, hip_sequence(g, ops, c)) + write(os.path.join(cdir, name + ".y"), yb) + write(os.path.join(cdir, name + ".w"), blob) + write(os.path.join(cdir, name + ".cw"), cw) + write(os.path.join(cdir, name + ".P"), P) + nA, nC, nL = out_sizes(rw, rh) + grid = -(-nwin_of(H, W, off) // 8) + # pass 1: the D3D12 block (y2) and the folded kernel + pass1 += [f"case {name}_block", "module l_swin.co", f"kernel {BLOCK} {grid} 256", f"arg buf {name}.y", + "arg null", f"arg out {H * W * 64} {name}.y2 {D3D12_SENTINEL}", f"arg buf {name}.w", + f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {off[0]}", f"arg u32 {off[1]}", ""] + pass1 += fold_lines(name, c) + if neg is None and grid >= 4 and (rw, rh) == (W, H): # the dropped windows hold rectangle pixels + neg = name + pass1 += fold_lines(name, c, prefix="negative_", grid=grid // 2) + # pass 2: l_coef_pack12 on the D3D12 block's y2 + pass2 += [f"case {name}_coef", "module l_ops.co", f"kernel {COEF} {-(-((H // 2) * (W // 2)) // 128)} 256", + f"arg buf {name}.y2", f"arg out {nA} {name}.sA {D3D12_SENTINEL}", "arg null", + f"arg out {nC} {name}.sC {D3D12_SENTINEL}", f"arg out {nL} {name}.sL {D3D12_SENTINEL}", + f"arg buf {name}.cw", f"arg bytes {name}.P", ""] + # control: Model M's swin_block_c96_nc through the same replay, against HIP + rng = np.random.default_rng(seeds[0]) + ctl = [] + for ci, (H, W, off, xb, nout, blob) in enumerate(LD.block_cases(CONTROL, rng)): + name = f"control_{ci}" + ctl.append((name, LD.hip_block_raw(g, "swin.co", CONTROL, xb, blob, H, W, off, nout))) + write(os.path.join(cdir, name + ".x"), xb) + write(os.path.join(cdir, name + ".w"), blob) + pass1 += [f"case {name}", "module swin.co", f"kernel {CONTROL} {-(-nwin_of(H, W, off) // 8)} 256", + f"arg buf {name}.x", "arg null", f"arg out {nout} {name}.out {D3D12_SENTINEL}", f"arg buf {name}.w", + f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {off[0]}", f"arg u32 {off[1]}", ""] + e1, t1 = replay(build, cdir, pass1, "pass1") + e2, t2 = replay(build, cdir, pass2, "pass2") if e1 == 0 else (-1, "pass 1 failed") + res = {"replay_exit": [e1, e2], "replay_tail": [t1, t2], "cases": []} + + def read(n): + p = os.path.join(cdir, n) + return open(p, "rb").read() if os.path.exists(p) else b"" + + for name, (c, hip) in all_cases.items(): + H, W, off, rw, rh = c[:5] + n = rw * rh + defined = [n * 8, n * 4, n * 4] + r = {"case": name, "grid": [H, W], "offset": list(off), "rect": [rw, rh], "fold_vs_d3d12_seq": [], + "fold_vs_hip_seq": [], "d3d12_seq_vs_hip_seq": []} + for k, s in enumerate("ACL"): + fold, seq = read(f"{name}.f{s}"), read(f"{name}.s{s}") + r["fold_vs_d3d12_seq"].append(compare(fold, seq, defined[k], FOLD_SENTINEL, D3D12_SENTINEL)) + r["fold_vs_hip_seq"].append(compare(fold, hip[k], defined[k], FOLD_SENTINEL, HIP_SENTINEL)) + r["d3d12_seq_vs_hip_seq"].append(compare(seq, hip[k], defined[k], D3D12_SENTINEL, HIP_SENTINEL)) + r["identical"] = all(o["identical"] for key in ("fold_vs_d3d12_seq", "fold_vs_hip_seq", "d3d12_seq_vs_hip_seq") + for o in r[key]) + res["cases"].append(r) + caught = None + if neg: + c = all_cases[neg][0] + n = c[3] * c[4] + outs = [compare(read(f"negative_{neg}.f{s}"), read(f"{neg}.s{s}"), d, FOLD_SENTINEL, D3D12_SENTINEL) + for s, d in zip("ACL", (n * 8, n * 4, n * 4))] + caught = all(o["differ"] >= 0 for o in outs) and any(not o["identical"] for o in outs) + res["negative_control"] = {"case": neg, "half_grid": True, "caught": caught, + "differ": sum(max(o["differ"], 0) for o in outs)} + ctl_ok = [read(f"{name}.out") == want for name, want in ctl] + res["control"] = {"kernel": CONTROL, "cases": len(ctl_ok), "identical": sum(ctl_ok)} + res["summary"] = {"cases": len(res["cases"]), "identical": sum(c["identical"] for c in res["cases"]), + "rect_bytes": sum(c["rect"][0] * c["rect"][1] * 16 for c in res["cases"])} + res["pass"] = (e1 == 0 and e2 == 0 and caught is True and res["summary"]["cases"] > 0 and + res["summary"]["identical"] == res["summary"]["cases"] and len(ctl_ok) > 0 and all(ctl_ok)) + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--prepare", action="store_true") + ap.add_argument("--run", action="store_true") + ap.add_argument("--seeds", type=int, default=2) + ap.add_argument("--json") + a = ap.parse_args(argv) + os.makedirs(a.build, exist_ok=True) + if a.prepare: + C32.compile_kernels(a.build, a.rocm, with_swin=True) + print("built", LD.prepare_replay(a.build, a.rocm, [BLOCK, COEF, FOLD, CONTROL])) + if a.run: + res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds))) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + print({k: v for k, v in res.items() if k not in ("cases", "replay_tail")}) + return 0 if res["pass"] else 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runtime/tests/test_l_runtime_frame.py b/clean/l/runtime/tests/test_l_runtime_frame.py index 0c2e1250..11213ca1 100644 --- a/clean/l/runtime/tests/test_l_runtime_frame.py +++ b/clean/l/runtime/tests/test_l_runtime_frame.py @@ -124,6 +124,16 @@ def blob_digests(): L_KERNELS = ("lev/input_encoder", "l_linear_32_32_relu", "l_linear_32_32_f16", "l_swin_block_c32_enc", "l_project_gather", "l_dec_project", "l_swin_block_c32_dec", "l_coef", "l_post_recon") +# native D3D12, fusion lc (the default): the decoder block with l_coef_pack12 as its epilogue, in place of the pair +FOLD = "l_swin_block_c32_dec_coef12_nc" +FOLD_BASE = "l_swin_block_c32_dec_coef12" +# the default fusions without lc: the pair (the fold's fallback route) +NO_FOLD = {"WG_D3D12_FUSION": "pm,xe,mb,rv,xo,xd"} + + +def latent_out(coef): + """The new latent a frame's coefficient store wrote (l_coef / l_coef_pack12, or the folded block).""" + return coef.args[10] if coef.kernel == FOLD else coef.args[4] def expected_args(t, f, digests): @@ -155,6 +165,9 @@ def expected_args(t, f, digests): "l_dec_project": [n["dec1.out"], n["l.narrow"], w["dec0.project"], tw, th], # dec1's output -> y "l_swin_block_c32_dec": [n["l.narrow"], "0", n["l.wide"], w["dec0_block"], str(fh), str(fw)] + do, # -> y2 "l_coef": [n["l.wide"], n["dec0.A"], n["dec0.B"], n["dec0.C"], n["dec0.latent"], w["dec0.coef"], None], + # the folded form: the block's arguments (no map out), then the pack12 store's images, latent, blob, rectangle + FOLD_BASE: [n["l.narrow"], "0", "0", w["dec0_block"], str(fh), str(fw)] + do + + [n["dec0.A"], n["dec0.C"], n["dec0.latent"], w["dec0.coef"], str(info["rw"]), str(info["rh"])], # the reconstruction (Model M's post_recon arguments): colour, A, B, C, the previous history, this frame's # motion surfaces, the post blob, the new history, the reduced history, the display image, P, exposure pair "l_post_recon": [n["input.color"], n["dec0.A"], n["dec0.B"], n["dec0.C"], n["state.history_prev"], @@ -185,7 +198,11 @@ def check_roles(t, f, digests, l_post=True): continue bad += ["%s argument %d: %s, expected %s" % (k, i, a, e) for i, (a, e) in enumerate(zip(ln.args, want)) if e is not None and a != e] - missing = [k for k in L_KERNELS if k not in seen and (l_post or k != "l_post_recon")] + fold = FOLD_BASE in seen + need = [k for k in L_KERNELS if not (fold and k in ("l_swin_block_c32_dec", "l_coef"))] + ([FOLD_BASE] if fold else []) + if fold and ({"l_swin_block_c32_dec", "l_coef"} & seen): + bad.append("the folded decoder block and the pair in one frame") + missing = [k for k in need if k not in seen and (l_post or k != "l_post_recon")] if bad or missing: raise AssertionError("; ".join(bad + ["not launched: " + k for k in missing])) @@ -318,8 +335,13 @@ def check_frame(self, t, f, roles=True, l_post=True): _, enc = k["l_swin_block_c32_enc"] gi, gather = k["l_project_gather"] di, dproj = k["l_dec_project"] - _, dec = k["l_swin_block_c32_dec"] - _, coef = k["l_coef"] + fold = FOLD_BASE in k + if fold: # one launch: the block with the coefficient store as its epilogue (no y2 map) + _, dec = k[FOLD_BASE] + coef = dec + else: + _, dec = k["l_swin_block_c32_dec"] + _, coef = k["l_coef"] q, x1, x2, x3, feat = lev.args[6], relu.args[1], f16.args[1], enc.args[2], gather.args[1] # the chain of maps (no launch reads what it writes) self.assertEqual(relu.args[0], q) @@ -334,14 +356,20 @@ def check_frame(self, t, f, roles=True, l_post=True): self.assertEqual(dproj.args[0], dec1.args[2]) # the decoder reads dec1's output self.assertEqual(dec.args[0], y) self.assertEqual(dec.args[1], "0") - self.assertEqual(coef.args[0], y2) + if fold: + self.assertEqual(y2, "0") # the map is not written + else: + self.assertEqual(coef.args[0], y2) for a, b in ((q, x1), (x1, x2), (x2, x3), (x3, feat), (dec1.args[2], y), (y, y2)): self.assertNotEqual(a, b) # every buffer holds its map (FORMATS.md "Maps") maps = ((q, 32 * npix), (x1, 32 * npix), (x2, 64 * npix), (x3, 32 * fh * fw), (feat, 64 * tw * th), - (y, 32 * fh * fw), (y2, 64 * fh * fw)) - coefs = ((coef.args[1], 4 * rw * rh), (coef.args[2], 4 * rw * rh), (coef.args[3], 8 * rw * rh), - (coef.args[4], 4 * rw * rh)) # A, B, C, the new latent: Model M's formats + (y, 32 * fh * fw)) + (() if fold else ((y2, 64 * fh * fw),)) + if fold: # pack12 image A (8 bytes a pixel), image C (4), the new latent + coefs = ((coef.args[8], 8 * rw * rh), (coef.args[9], 4 * rw * rh), (coef.args[10], 4 * rw * rh)) + else: + coefs = ((coef.args[1], 4 * rw * rh), (coef.args[2], 4 * rw * rh), (coef.args[3], 8 * rw * rh), + (coef.args[4], 4 * rw * rh)) # A, B, C, the new latent: Model M's formats for buf, need in maps + coefs: self.assertGreaterEqual(t.extent(buf), need, buf) # extents, offsets, parameters @@ -352,10 +380,13 @@ def check_frame(self, t, f, roles=True, l_post=True): for blk, kind in ((enc, "first"), (dec, "second")): ox, oy = T.block_offset(T.TABLE, kind, info["parity"]) self.assertEqual(blk.args[4:8], [str(fh), str(fw), str(ox), str(oy)]) - e, d = lev.params(12), coef.params(6) + e = lev.params(12) self.assertEqual(e[:4], [rw, rh, fw, fh]) self.assertEqual(e[5], info["reset"]) - self.assertEqual(d, [rw, rh, fw, fh, info["reset"]]) + if fold: + self.assertEqual(coef.args[12:14], [str(rw), str(rh)]) + else: + self.assertEqual(coef.params(6), [rw, rh, fw, fh, info["reset"]]) post = [ln for ln in ls if ln.kernel == "l_post_recon"] self.assertEqual(len(post), 1 if l_post else 0) # (native D3D12: Model M's fused pack12 post for now) if l_post: @@ -508,15 +539,22 @@ def check_d3d12_frame(self, t, f): info = f["info"] rw, rh = info["rw"], info["rh"] names = [ln.kernel for ln in f["launches"]] - self.assertIn("l_coef_pack12", names) + fold = FOLD in names + if fold: + self.assertNotIn("l_coef_pack12", names) + self.assertNotIn("l_swin_block_c32_dec_nc", names) + self.assertEqual(names[names.index(FOLD) - 1], "l_dec_project") + else: + self.assertIn("l_coef_pack12", names) self.assertNotIn("l_coef", names) self.assertTrue(all(not n.startswith(("d1p12/", "d0p12/")) and n != "input_encoder" for n in names), names) dec1 = [ln for ln in f["launches"] if ln.kernel.startswith("swin_block_c64_add")] self.assertEqual([ln.kernel for ln in dec1], ["swin_block_c64_add_ex_nc"]) # dec1's own block, not fused lev, coef = Arguments.check_frame(self, t, f, l_post=False) - self.assertGreaterEqual(t.extent(coef.args[1]), 8 * rw * rh) # pack12 image A: 8 bytes per pixel - self.assertGreaterEqual(t.extent(coef.args[3]), 4 * rw * rh) # pack12 image C: 4 bytes per pixel - i = names.index("l_coef_pack12") + a, c = (coef.args[8], coef.args[9]) if fold else (coef.args[1], coef.args[3]) + self.assertGreaterEqual(t.extent(a), 8 * rw * rh) # pack12 image A: 8 bytes per pixel + self.assertGreaterEqual(t.extent(c), 4 * rw * rh) # pack12 image C: 4 bytes per pixel + i = names.index(FOLD if fold else "l_coef_pack12") self.assertTrue(names[i + 1].startswith("w64/post_tf") and "_p12" in names[i + 1], names[i + 1:]) return lev, coef @@ -529,24 +567,36 @@ def test_frame(self): self.assertEqual(t.unknown(), []) self.assertEqual([f["info"]["reset"] for f in t.frames], [1, 0, 1, 0, 1, 1]) self.assertEqual(len({f["info"]["parity"] for f in t.frames}), 2) - prev = None - for f in t.frames: - self.assertIsNone(f["error"]) - lev, coef = self.check_d3d12_frame(t, f) - if prev: - self.assertEqual(lev.args[4], prev[1].args[4]) # the latent the last frame's store wrote - self.assertEqual(lev.args[5], prev[0].args[7]) # its near motion - prev = (lev, coef) + for env, ports, fold in ((None, full_ports(), True), (NO_FOLD, full_ports(), False), + (None, [kc for kc in full_ports() if kc[0] != FOLD], False)): + with self.subTest(fusion=env, fold_in_manifest=any(kc[0] == FOLD for kc in ports)): + # the fold (fusion lc, the default) and its fallback, the pair: without lc, or without the kernel + t = FT.Trace(self.l_run(ports, frames=6, rw=96, rh=64, flags=(sizes, "--reset=4"), env=env)) + self.assertEqual(t.create, "create ok model L") + self.assertEqual(t.unknown(), []) + self.assertEqual([f["info"]["reset"] for f in t.frames], [1, 0, 1, 0, 1, 1]) + self.assertEqual(len({f["info"]["parity"] for f in t.frames}), 2) + prev = None + for f in t.frames: + self.assertIsNone(f["error"]) + self.assertEqual(FOLD in [ln.kernel for ln in f["launches"]], fold) + lev, coef = self.check_d3d12_frame(t, f) + if prev: + self.assertEqual(lev.args[4], latent_out(prev[1])) # the latent the last frame's store wrote + self.assertEqual(lev.args[5], prev[0].args[7]) # its near motion + prev = (lev, coef) def test_failed_launch_stops_the_frame(self): - t = FT.Trace(self.l_run(full_ports(), frames=3, flags=("--fail=l_coef_pack12@1",))) - ok0, bad, ok2 = t.frames - self.assertEqual(bad["failed"], "l_coef_pack12") - names = [ln.kernel for ln in bad["launches"]] - self.assertEqual(names, [ln.kernel for ln in ok0["launches"]][:len(names)]) - self.assertTrue(names[-1].startswith("l_swin_block_c32_dec")) - self.assertIsNone(ok2["error"]) - self.check_d3d12_frame(t, ok2) + for env, kernel, before in ((None, FOLD, "l_dec_project"), (NO_FOLD, "l_coef_pack12", "l_swin_block_c32_dec")): + with self.subTest(kernel=kernel): + t = FT.Trace(self.l_run(full_ports(), frames=3, flags=("--fail=%s@1" % kernel,), env=env)) + ok0, bad, ok2 = t.frames + self.assertEqual(bad["failed"], kernel) + names = [ln.kernel for ln in bad["launches"]] + self.assertEqual(names, [ln.kernel for ln in ok0["launches"]][:len(names)]) + self.assertTrue(names[-1].startswith(before)) + self.assertIsNone(ok2["error"]) + self.check_d3d12_frame(t, ok2) def test_model_m_unchanged(self): """Model M on backend 3 traces the same in the build with Model L as in the build without it.""" diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index fe39bb6c..52e09ad3 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -125,7 +125,8 @@ std::string emu_lib_dir(const WgRtConfig& c) { // 10 faster; one 8-round run -11.5 us with every round faster). Re-measured in one process (runtime A/B, the same frames // alternating every 300 frames; inter-process noise was +-25 us): xd is +5.0 us at 1707x960 and +7.5 us at 1920x1080, // -2.25 us at 1280x720, so it now runs on frames of at most kXdMaxPixels render pixels only. -const char* const kDefaultD3d12Fusion = "pm,xe,mb,rv,xo,xd"; +// lc: Model L only (its decoder block with l_coef_pack12 as the epilogue; see fz_lc). +const char* const kDefaultD3d12Fusion = "pm,xe,mb,rv,xo,xd,lc"; // xd only up to this many render pixels per frame (see fz_xd): it is faster at 1280x720 and slower at 1707x960 and above constexpr long long kXdMaxPixels = 1200000; @@ -315,15 +316,21 @@ struct Runtime { // xd runs only on frames of at most xd_max_pixels render pixels (WG_D3D12_XD_MAX_PIXELS, default kXdMaxPixels; // 0 = any size): measured in one process against the same frames without it (runtime A/B, WG_AB_FUSION, // 3 x 4 windows of 300 frames each): 1280x720 -2.25 us, 1707x960 +5.0 us, 1920x1080 +7.5 us per frame + // lc Model L: its decoder block with l_coef_pack12 as the epilogue (l_swin_block_c32_dec_coef12_nc, l_swin.hlsl + // SW_OUT_COEF12; byte for byte the pair l_swin_block_c32_dec_nc -> l_coef_pack12): one launch less, and the + // block's FP16 map (y2, 64 bytes a pixel) is neither stored nor read back. Pack12 frames only; without lc, or + // with a manifest that lacks the kernel, the pair runs (the fallback route) + bool fz_lc = false, l_fold_built = false; long long xd_max_pixels = [] { const char* e = std::getenv("WG_D3D12_XD_MAX_PIXELS"); return e && *e ? std::atoll(e) : kXdMaxPixels; }(); // A/B diagnostics (WG_AB_*): the two fusion sets and the block length (0: off) struct FzSet { - bool pm = false, xe = false, mb = false, mb0 = false, rv = false, xo = false, xd = false; + bool pm = false, xe = false, mb = false, mb0 = false, rv = false, xo = false, xd = false, lc = false; template void apply(R* r) const { r->fz_pm = pm, r->fz_xe = xe, r->fz_mb = mb, r->fz_mb0 = mb0, r->fz_rv = rv, r->fz_xo = xo, r->fz_xd = xd; + r->fz_lc = lc; } }; FzSet fz_set[2]; @@ -352,7 +359,9 @@ struct Runtime { else if (f == "rv") z->rv = true; else if (f == "xo") z->xo = true; else if (f == "xd") z->xd = true; - else if (!f.empty()) return fail("WG_D3D12_FUSION: unknown fusion \"" + f + "\" (pm, xe, mb, rv, xo, xd or none)"); + else if (f == "lc") z->lc = true; + else if (!f.empty()) + return fail("WG_D3D12_FUSION: unknown fusion \"" + f + "\" (pm, xe, mb, rv, xo, xd, lc or none)"); } return true; } @@ -520,6 +529,8 @@ struct Runtime { if (model_l && c.backend == 3) { const std::string missing = l_native_missing(); if (!missing.empty()) return fail("Model L on native D3D12: not yet ported to D3D12: " + missing); + const std::vector have = manifest_kernels(); + l_fold_built = std::find(have.begin(), have.end(), "l_swin_block_c32_dec_coef12_nc") != have.end(); } #endif if (c.max_rw <= 0 || c.max_rh <= 0 || c.ow <= 0 || c.oh <= 0) return fail("invalid render or output size"); @@ -1362,7 +1373,7 @@ struct Runtime { kernels_text += std::string(" dec_rdiv=") + (dec_div.find('r') == std::string::npos ? "1" : "0"); kernels_text += std::string(" dec_bdiv=") + (dec_div.find('b') == std::string::npos ? "1" : "0"); kernels_text += std::string(" dec1_dec0=") + (dec1_dec0 ? "fused" : "separate"); - if (fz_pm || fz_xe || fz_mb || fz_rv || fz_xo || fz_xd) kernels_text += " d3d12_fusion=" + fz_text; + if (fz_pm || fz_xe || fz_mb || fz_rv || fz_xo || fz_xd || fz_lc) kernels_text += " d3d12_fusion=" + fz_text; kernels_text += " swin_mid=" + (mid_variant.empty() ? std::string("none") : mid_variant); kernels_text += std::string(" post_pack=") + (p12 ? "p12" : "none"); kernels_text += std::string(" pre_exposure=") + (pi_original ? "original" : "scale"); @@ -1375,6 +1386,7 @@ struct Runtime { if (native_direct_io) kernels_text += " native_io=direct_textures"; // Model L: its encoder and decoder (enc0= / dec0= above name Model M's, not loaded for Model L) if (model_l) kernels_text += " model=L l_kernels=l_enc_values.co,l_ops.co,l_swin.co,l_post.co l_post=l_post_recon"; + if (l_fold_built) kernels_text += std::string(" l_dec_fold=") + (fz_lc ? "on" : "off"); // (native D3D12 only) return kernels_text.c_str(); } @@ -1532,10 +1544,20 @@ struct Runtime { // Step 4 for Model L: the projection of dec1's output to 32 values per pixel (y), the decoder block (y2, FP16), and // Model L's per-pixel functions into A, B, C and the new latent (Model M's buffers and formats; no motion, previous - // latent, jitter or reset is read). + // latent, jitter or reset is read). Fusion lc (pack12 frames on native D3D12): the block with the per-pixel + // functions as its epilogue, y2 not stored. bool l_decoder(DecParams& p, int nxt, int fh, int fw, int f) { DevPtr wd = wdev["dec0.project"], wc = wdev["dec0.coef"]; int tw = fw / 2, th = fh / 2; + if (fz_lc && l_fold_built && p12_frame && l_nc[1]) { + DevPtr none = 0, wb = wdev["dec0_block"]; + int ox = 0, oy = 0, H = fh, W = fw, rw = p.rw, rh = p.rh; + l_offsets(true, f, &ox, &oy); + const int nwin = ((W + ox + 3) / 4) * ((H + oy + 3) / 4); + return launch("l_dec_project", cdiv(tw * th, 128), 256, {&out1, &l_narrow, &wd, &tw, &th}) && + launch("l_swin_block_c32_dec_coef12_nc", cdiv(nwin, 8), 256, + {&l_narrow, &none, &none, &wb, &H, &W, &ox, &oy, &A, &C, &latent[nxt], &wc, &rw, &rh}); + } return launch("l_dec_project", cdiv(tw * th, 128), 256, {&out1, &l_narrow, &wd, &tw, &th}) && l_block("dec", l_narrow, l_wide, wdev["dec0_block"], l_nc[1], fh, fw, true, f) && launch(p12_frame ? "l_coef_pack12" : "l_coef", cdiv(tw * th, 128), 256, @@ -1546,8 +1568,9 @@ struct Runtime { // its encoder (the direct form on the direct-resource route), linears, clamp-free blocks (the only D3D12 forms), // projections and pack12 coefficient store, and dec1's own block with its expand (Model M runs dec1 inside its fused // decoder there; Model L's decoder reads dec1's map). Model M's other kernels are its native frame's. - std::string l_native_missing() const { - std::string text, error, missing; + // The kernel names of /dx12/manifest.json (none if it is missing or unreadable). + std::vector manifest_kernels() const { + std::string text, error; wgc::Json manifest; std::vector have; if (read_text(build_dir + "/dx12/manifest.json", &text) && wgc::parse_json(text, manifest, error)) { @@ -1558,6 +1581,12 @@ struct Runtime { if (name && name->is_string()) have.push_back(name->text); } } + return have; + } + + std::string l_native_missing() const { + std::string missing; + const std::vector have = manifest_kernels(); const std::vector need = {native_direct_io ? "lev/input_encoder_direct" : "lev/input_encoder", "l_linear_32_32_relu", "l_linear_32_32_f16", "l_swin_block_c32_enc_nc", "l_project_gather", diff --git a/clean/tests/runtime_frame_trace.cpp b/clean/tests/runtime_frame_trace.cpp index 0c0de8c2..08c31159 100644 --- a/clean/tests/runtime_frame_trace.cpp +++ b/clean/tests/runtime_frame_trace.cpp @@ -73,6 +73,8 @@ const Signature kSignatures[] = { {"input_encoder", "ppppppppppppE"}, // Model M's encoder and Model L's values (q in place of the features) {"coef_decoder", "ppppppppD"}, // Model M's decoder (also its pack12 form) {"swin_block_*", "ppppiiii"}, // Model M's blocks: x, skip, out, w, H, W, ox, oy + {"l_swin_block_c32_dec_coef12_nc", "ppppiiiippppii"}, // the decoder block with l_coef_pack12 folded in: then + // A, C, latent, dec0.coef, rw, rh (native D3D12, fusion lc) {"l_swin_block_c32*", "ppppiiii"}, // Model L's blocks (clean/l/FORMATS.md) {"l_linear_32_32_relu", "pppi"}, {"l_linear_32_32_f16", "pppi"},