diff --git a/clean/dx12/compare.py b/clean/dx12/compare.py index c71184c9..5d2010a5 100644 --- a/clean/dx12/compare.py +++ b/clean/dx12/compare.py @@ -68,7 +68,7 @@ "expand_c96", "d1p12/swin_block_c64_add_dec0_nc", "w64/post_tf_f32_p12_diet_vsit"} # The native backend (--backend d3d12) runs the D3D12 fusions of WG_D3D12_FUSION (default: runtime.cpp's # kDefaultD3d12Fusion, mirrored here), which replace some launches (kernels/fused_swin.hlsl, fused_bneck.hlsl, fused_dec.hlsl). -DEFAULT_D3D12_FUSION = "pm,xe,mb,rv,xo,xd,lc,ll" # lc, ll: Model L only +DEFAULT_D3D12_FUSION = "pm,xe,mb,rv,xo,xd,lc,ll,le" # lc, ll, le: Model L only def d3d12_fusion(): diff --git a/clean/dx12/hlsl/l_lin_px.hlsli b/clean/dx12/hlsl/l_lin_px.hlsli new file mode 100644 index 00000000..78d41d9b --- /dev/null +++ b/clean/dx12/hlsl/l_lin_px.hlsli @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: MIT +// Model L's two input linears for one pixel per lane (issue #89; clean/l/kernels/l_ops.hip linear_32_32, +// then ), D3D12 only, shared by l_linear_32_32_fused (kernels/l_ops_linear.hlsl) and the encoder block +// with the linears as its prologue (kernels/swin_block.hlsl, SW_SRC 3). Lane (t, h) of a 16-pixel tile: its pixel's q +// fragments xf[kc] = q's bytes at pix * 32 + 16 h + 8 kc (FP8); output tile m = x2's channels 16 h + 8 m .. + 7. +// Blob (enc0.Win1 / enc0.Win2): bias tile vector (rows R_L(32)), then fragments [mt < 2][kc < 2], rows R_L(32), K order +// K_L(32) (clean/l/FORMATS.md). +#ifndef L_LIN_PX_HLSLI +#define L_LIN_PX_HLSLI + +// the blob's bias tile vectors and weight fragments of lane (t, h) +void lin_blob(uint wb, uint lane, uint hh, out v8f bias[2], out uint2 fr[2][2]) +{ + [unroll] for (uint mt = 0; mt < 2u; ++mt) { + bias[mt] = wg_load8f(wb, (hh * 8u + mt * 16u) * 4u); + [unroll] for (uint kc2 = 0; kc2 < 2u; ++kc2) fr[mt][kc2] = wg_ld64(wb, 128u + (mt * 2u + kc2) * 256u + lane * 8u); + } +} + +// output tile m before the rounding: b + W x +v8f lin_acc(v8f bias, uint2 fr[2], uint2 xf[2]) +{ + v8f acc = bias; + acc = wg_wmma_fp8(fr[0], xf[0], acc); + return wg_wmma_fp8(fr[1], xf[1], acc); +} + +// Both linears: x2's FP16 tiles y[m] (8 halves each) of the lane's pixel from its q fragments xf. x1 stays in +// registers: the relu linear's lane (t, h) holds x1's channels 16 h + 8 m .. + 7 as the two FP8 fragments that the f16 +// linear's lane (t, h) takes as its K blocks kc = m. The conversions by the driver's instructions (wg_platform.hlsli: +// the same bits as the software wg_rn16 / wg_pack_q8; wave-uniform control flow only): RN16 = v_cvt_f16_f32 then the +// exact widening, the pack v_cvt_pk_fp8_f32 after the clamp; FP16(RN16(acc)) = FP16(acc), one conversion (rounding to +// FP16 is idempotent, NaN payloads included). +void lin_pair(uint2 xf[2], v8f bias[2], uint2 fr[2][2], v8f bias2[2], uint2 fr2[2][2], out uint4 y[2]) +{ + uint2 x1[2]; + [unroll] for (uint m1 = 0; m1 < 2u; ++m1) { + const uint4 h = wg_to_f16_ags(lin_acc(bias[m1], fr[m1], xf)); + v8f r; + [unroll] for (uint j = 0; j < 4u; ++j) { + r.v[2 * j] = wg_fmax(wg_h2f_lo(h[j]), 0.0f); + r.v[2 * j + 1] = wg_fmax(wg_h2f_hi(h[j]), 0.0f); + } + x1[m1] = wg_pack_q8_ags(r); + } + [unroll] for (uint m = 0; m < 2u; ++m) y[m] = wg_to_f16_ags(lin_acc(bias2[m], fr2[m], x1)); +} + +#endif diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index 082c79b8..2a7febe3 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -254,6 +254,10 @@ "l_swin_block_c32_dec_nc", "l_swin.co" ], + [ + "l_swin_block_c32_enc_lin_nc", + "-" + ], [ "l_swin_block_c32_enc_nc", "l_swin.co" diff --git a/clean/dx12/kernels/fused_swin.hlsl b/clean/dx12/kernels/fused_swin.hlsl index 5b58734b..34bce87f 100644 --- a/clean/dx12/kernels/fused_swin.hlsl +++ b/clean/dx12/kernels/fused_swin.hlsl @@ -472,6 +472,30 @@ void sw_load_halo_f16(uint base, int pr, int pc, int W, uint hh, out v8f f[KC]) f[k].v[4] = wg_h2f_lo(q.z); f[k].v[5] = wg_h2f_hi(q.z); f[k].v[6] = wg_h2f_lo(q.w); f[k].v[7] = wg_h2f_hi(q.w); } } +#elif SW_SRC == 3 +#if SW_C != 32 +#error "SW_SRC 3: Model L's encoder block (C = 32)" +#endif +#include "l_lin_px.hlsli" +// LHaloLinSrc: the values of LHaloF16Src without the x2 map: the member at grid position (pr, pc) is pixel +// (pr + 2) (W + 4) + pc + 2 of the FP8 q halo map, through l_linear_32_32_fused's arithmetic (lane (t, h) = that +// kernel's lane of the pixel; every lane runs it, also for outside members: wave-uniform control flow) +void sw_load_halo_lin(uint base, int pr, int pc, int W, uint lane, uint hh, out v8f f[KC]) +{ + const uint pix = (uint)((pr + 2) * (W + 4) + pc + 2); + uint2 xq[2]; + [unroll] for (uint kc = 0; kc < 2u; ++kc) xq[kc] = wg_ld64(base, pix * 32u + hh * 16u + 8u * kc); + v8f b1[2], b2[2]; + uint2 f1[2][2], f2[2][2]; + lin_blob(linW1, lane, hh, b1, f1); + lin_blob(linW2, lane, hh, b2, f2); + uint4 y[2]; + lin_pair(xq, b1, f1, b2, f2, y); + [unroll] for (uint k = 0; k < KC; ++k) { + f[k].v[0] = wg_h2f_lo(y[k].x); f[k].v[1] = wg_h2f_hi(y[k].x); f[k].v[2] = wg_h2f_lo(y[k].y); f[k].v[3] = wg_h2f_hi(y[k].y); + f[k].v[4] = wg_h2f_lo(y[k].z); f[k].v[5] = wg_h2f_hi(y[k].z); f[k].v[6] = wg_h2f_lo(y[k].w); f[k].v[7] = wg_h2f_hi(y[k].w); + } +} #elif SW_SRC == 2 // LCoarseFp8Src: a member outside the grid takes the mirrored 2 x 2 group (mirror of the group index) and keeps its // sub-pixel; the byte offset of the lane's half of that pixel @@ -722,6 +746,9 @@ WG_KERNEL(SW_ENTRY) #elif SW_SRC == 1 v8f xf[KC]; sw_load_halo_f16(x, pr, pc, W, hh, xf); +#elif SW_SRC == 3 + v8f xf[KC]; + sw_load_halo_lin(x, pr, pc, W, lane, hh, xf); #elif SW_SRC == 2 load_frags_g(x, sw_coarse_src(pr, pc, H, W, hh), xa); #elif SW_PM @@ -786,7 +813,7 @@ WG_KERNEL(SW_ENTRY) v8f acc[KC]; { [unroll] for (uint k = 0; k < KC; ++k) { -#if SW_SRC == 1 +#if SW_SRC == 1 || SW_SRC == 3 acc[k] = xf[k]; #else acc[k] = sw_unpack_fp8(xa[k]); diff --git a/clean/dx12/kernels/fused_swin_gen.py b/clean/dx12/kernels/fused_swin_gen.py index c89eef05..528fbbd3 100644 --- a/clean/dx12/kernels/fused_swin_gen.py +++ b/clean/dx12/kernels/fused_swin_gen.py @@ -70,6 +70,9 @@ uint coefA, coefC, coefL, coefW; // pointers: l_coef_pack12's A, C, latent and coefficient blob (out_p unused) int coefRw, coefRh; // the render rectangle #endif +#if SW_SRC == 3 + uint linW1, linW2; // pointers: enc0.Win1, enc0.Win2 (x = the FP8 q halo map) +#endif WG_ARGS_END""" ARGS_NEW = """#ifndef SW_PM #define SW_PM 0 diff --git a/clean/dx12/kernels/l_ops_linear.hlsl b/clean/dx12/kernels/l_ops_linear.hlsl index fbbc71f5..e9ec0259 100644 --- a/clean/dx12/kernels/l_ops_linear.hlsl +++ b/clean/dx12/kernels/l_ops_linear.hlsl @@ -38,24 +38,9 @@ WG_ARGS_BEGIN int npix; WG_ARGS_END -#if LIN_FUSED // (helpers of the fused kernel only: the pair keeps its own code and DXIL) -// the blob's bias tile vectors and weight fragments of lane (t, h) -void lin_blob(uint wb, uint lane, uint hh, out v8f bias[2], out uint2 fr[2][2]) -{ - [unroll] for (uint mt = 0; mt < 2u; ++mt) { - bias[mt] = wg_load8f(wb, (hh * 8u + mt * 16u) * 4u); - [unroll] for (uint kc2 = 0; kc2 < 2u; ++kc2) fr[mt][kc2] = wg_ld64(wb, 128u + (mt * 2u + kc2) * 256u + lane * 8u); - } -} - -// output tile m before the rounding: b + W x -v8f lin_acc(v8f bias, uint2 fr[2], uint2 xf[2]) -{ - v8f acc = bias; - acc = wg_wmma_fp8(fr[0], xf[0], acc); - return wg_wmma_fp8(fr[1], xf[1], acc); -} - +#if LIN_FUSED // (helpers of the fused kernel only, shared with the encoder block's prologue: the pair keeps its own + // code and DXIL) +#include "l_lin_px.hlsli" #endif #if LIN_FUSED @@ -97,6 +82,7 @@ WG_KERNEL(l_linear_32_32_f16) // conversion (rounding to FP16 is idempotent, NaN payloads included) [unroll] for (uint m = 0; m < 2u; ++m) wg_st128(y, ((uint)pk * 32u + hh * 16u + 8u * m) * 2u, wg_to_f16_ags(lin_acc(bias2[m], fr2[m], x1))); + // (hlsl/l_lin_px.hlsli lin_pair is this code for the encoder block's prologue; kept inline here: the kernel's DXIL) #else // (the pair's own code, unchanged) [unroll] for (uint mt = 0; mt < 2u; ++mt) { diff --git a/clean/dx12/kernels/l_swin.hlsl b/clean/dx12/kernels/l_swin.hlsl index f160c527..2a86061e 100644 --- a/clean/dx12/kernels/l_swin.hlsl +++ b/clean/dx12/kernels/l_swin.hlsl @@ -13,9 +13,14 @@ // reads first the rows of x2 the linears wrote last, and its last-written rows of x3 are the // ones l_project_gather reads first (more cache hits at both hand-offs) // l_swin_block_c32_dec_nc the decoder block: FP8 map in with the token-grid mirror (SW_SRC 2), FP16 out (SW_OUT_F16) +// l_swin_block_c32_enc_lin_nc D3D12 only: the encoder block with the two input linears as its prologue (SW_SRC 3; +// byte for byte the pair l_linear_32_32_fused -> _enc_nc, which stays the fallback route): +// the FP8 q halo map in, x2 neither written nor read. Arguments: _enc_nc's (x = q), then +// enc0.Win1, enc0.Win2 // // @wg-kernel l_swin_block_c32_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_ENTRY_OVERRIDE=l_swin_block_c32_nc // @wg-kernel l_swin_block_c32_enc_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=3,SW_GELU_PAT=13333,SW_SRC=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_enc_nc,WG_REV_ORDER=1 +// @wg-kernel l_swin_block_c32_enc_lin_nc co=- block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=3,SW_GELU_PAT=13333,SW_SRC=3,SW_ENTRY_OVERRIDE=l_swin_block_c32_enc_lin_nc,WG_REV_ORDER=1 // @wg-kernel l_swin_block_c32_dec_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=1,SW_SRC=2,SW_OUT_F16=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_nc // l_swin_block_c32_dec_coef12_nc D3D12 only: the decoder block with l_coef_pack12 as its epilogue (SW_OUT_COEF12; // byte for byte the pair _dec_nc -> l_coef_pack12, which stays the fallback route); diff --git a/clean/dx12/kernels/swin_block.hlsl b/clean/dx12/kernels/swin_block.hlsl index e55a90a6..bced9859 100644 --- a/clean/dx12/kernels/swin_block.hlsl +++ b/clean/dx12/kernels/swin_block.hlsl @@ -10,7 +10,10 @@ // SW_SRC gather source (HIP's SRC hook; plain blocks only): 0 the FP8 map at the mirrored member // (default); 1 Model L's encoder block, an FP16 halo map [H + 4][W + 4][C] read directly at // (pr + 2, pc + 2) (LHaloF16Src); 2 Model L's decoder block, the FP8 map with the token-grid -// mirror, sub-pixel kept (LCoarseFp8Src) +// mirror, sub-pixel kept (LCoarseFp8Src); 3 (D3D12 only) Model L's encoder block with +// its two input linears as the prologue: the FP8 q halo map [H + 4][W + 4][32] at the same +// position through the linears (hlsl/l_lin_px.hlsli), the FP16 values mode 1 reads from x2, +// in registers (the x2 map is neither written nor read) // SW_OUT_F16 1: the output is RN16(x2) as an FP16 map [H][W][C] (Model L's decoder block, LStoreF16Epi) // SW_OUT_COEF12 1 (with SW_OUT_F16, C = 32): instead of the FP16 map, l_coef_pack12 of it as the epilogue: // the lane that holds a pixel's channels 0..15 (hh = 0) runs hlsl/l_coef_px.hlsli on those @@ -37,6 +40,9 @@ WG_ARGS_BEGIN uint coefA, coefC, coefL, coefW; // pointers: l_coef_pack12's A, C, latent and coefficient blob (out_p unused) int coefRw, coefRh; // the render rectangle #endif +#if SW_SRC == 3 + uint linW1, linW2; // pointers: enc0.Win1, enc0.Win2 (x = the FP8 q halo map) +#endif WG_ARGS_END // ---- BlockLayout (FORMATS.md, "Swin block section") --------------------------------------------------- @@ -432,6 +438,30 @@ void sw_load_halo_f16(uint base, int pr, int pc, int W, uint hh, out v8f f[KC]) f[k].v[4] = wg_h2f_lo(q.z); f[k].v[5] = wg_h2f_hi(q.z); f[k].v[6] = wg_h2f_lo(q.w); f[k].v[7] = wg_h2f_hi(q.w); } } +#elif SW_SRC == 3 +#if SW_C != 32 +#error "SW_SRC 3: Model L's encoder block (C = 32)" +#endif +#include "l_lin_px.hlsli" +// LHaloLinSrc: the values of LHaloF16Src without the x2 map: the member at grid position (pr, pc) is pixel +// (pr + 2) (W + 4) + pc + 2 of the FP8 q halo map, through l_linear_32_32_fused's arithmetic (lane (t, h) = that +// kernel's lane of the pixel; every lane runs it, also for outside members: wave-uniform control flow) +void sw_load_halo_lin(uint base, int pr, int pc, int W, uint lane, uint hh, out v8f f[KC]) +{ + const uint pix = (uint)((pr + 2) * (W + 4) + pc + 2); + uint2 xq[2]; + [unroll] for (uint kc = 0; kc < 2u; ++kc) xq[kc] = wg_ld64(base, pix * 32u + hh * 16u + 8u * kc); + v8f b1[2], b2[2]; + uint2 f1[2][2], f2[2][2]; + lin_blob(linW1, lane, hh, b1, f1); + lin_blob(linW2, lane, hh, b2, f2); + uint4 y[2]; + lin_pair(xq, b1, f1, b2, f2, y); + [unroll] for (uint k = 0; k < KC; ++k) { + f[k].v[0] = wg_h2f_lo(y[k].x); f[k].v[1] = wg_h2f_hi(y[k].x); f[k].v[2] = wg_h2f_lo(y[k].y); f[k].v[3] = wg_h2f_hi(y[k].y); + f[k].v[4] = wg_h2f_lo(y[k].z); f[k].v[5] = wg_h2f_hi(y[k].z); f[k].v[6] = wg_h2f_lo(y[k].w); f[k].v[7] = wg_h2f_hi(y[k].w); + } +} #elif SW_SRC == 2 // LCoarseFp8Src: a member outside the grid takes the mirrored 2 x 2 group (mirror of the group index) and keeps its // sub-pixel; the byte offset of the lane's half of that pixel @@ -596,6 +626,9 @@ WG_KERNEL(SW_ENTRY) #elif SW_SRC == 1 v8f xf[KC]; sw_load_halo_f16(x, pr, pc, W, hh, xf); +#elif SW_SRC == 3 + v8f xf[KC]; + sw_load_halo_lin(x, pr, pc, W, lane, hh, xf); #elif SW_SRC == 2 load_frags_g(x, sw_coarse_src(pr, pc, H, W, hh), xa); #else @@ -627,7 +660,7 @@ WG_KERNEL(SW_ENTRY) v8f acc[KC]; { [unroll] for (uint k = 0; k < KC; ++k) { -#if SW_SRC == 1 +#if SW_SRC == 1 || SW_SRC == 3 acc[k] = xf[k]; #else acc[k] = sw_unpack_fp8(xa[k]); diff --git a/clean/dx12/vgpr_limits.json b/clean/dx12/vgpr_limits.json index 4aea008a..6ce8f299 100644 --- a/clean/dx12/vgpr_limits.json +++ b/clean/dx12/vgpr_limits.json @@ -2,6 +2,7 @@ "_doc": "build_dx12.py VGPR check: the RGA (gfx1201) USED_VGPRs bound of every Swin / bottleneck kernel (a name with a swin_ part). Each bound is the largest count that keeps the kernel's present occupancy: VGPR file 1536 per SIMD lane, granule 8, a 256-lane wave32 workgroup = 4 waves on each of a CU's 2 SIMDs, so 2 workgroups per CU need <= 192, 3 need <= 128 (a 128-lane workgroup = 2 waves per SIMD: 4 per CU need <= 192). A kernel over its bound fails the build (a merge or an edit lost a workgroup per CU: swin_block_c96_add_xe_* at 215 VGPRs ran 1 per CU, +38 us per frame at 1440p Q). A new Swin / bottleneck kernel needs an entry. WG_DX12_VGPR_CHECK=0 skips the check.", "l_swin_block_c32_nc": 96, "l_swin_block_c32_enc_nc": 96, + "l_swin_block_c32_enc_lin_nc": 96, "l_swin_block_c32_dec_nc": 96, "l_swin_block_c32_dec_coef12_nc": 96, "swin_block_c64_nc": 192, diff --git a/clean/l/FORMATS.md b/clean/l/FORMATS.md index 2ed72cb1..74bff879 100644 --- a/clean/l/FORMATS.md +++ b/clean/l/FORMATS.md @@ -23,8 +23,8 @@ Every map is `[rows][cols][C]`, channel fastest, as Model M's feature maps (`cle | Map | Shape | Element | Written by | Read by | |---|---|---|---|---| | `q` | `[fh + 4][fw + 4][32]` halo | FP8: values 0..19, bytes 20..31 zero | `lev/input_encoder` | `l_linear_32_32_relu` | -| `x1` | `[fh + 4][fw + 4][32]` halo | FP8 (not stored by `l_linear_32_32_fused`) | `l_linear_32_32_relu` | `l_linear_32_32_f16` | -| `x2` | `[fh + 4][fw + 4][32]` halo | **FP16** (64 bytes per pixel) | `l_linear_32_32_f16` | `l_swin_block_c32_enc` | +| `x1` | `[fh + 4][fw + 4][32]` halo | FP8 (not stored by `l_linear_32_32_fused` or `l_swin_block_c32_enc_lin_nc`) | `l_linear_32_32_relu` | `l_linear_32_32_f16` | +| `x2` | `[fh + 4][fw + 4][32]` halo | **FP16** (64 bytes per pixel; not stored by `l_swin_block_c32_enc_lin_nc`) | `l_linear_32_32_f16` | `l_swin_block_c32_enc` | | `x3` | `[fh][fw][32]` | FP8 | `l_swin_block_c32_enc` | `l_project_gather` | | `enc0.features` | `[fh/2][fw/2][64]` | FP8 | `l_project_gather` | enc1 (Model M's stage) | | m_near, m_far, m_center | as Model M | as Model M | `lev/input_encoder` | as Model M | @@ -80,6 +80,7 @@ workgroup, in raster order of their map. | `l_linear_32_32_f16` | exists (GPU-checked) | `x, y, w, npix` | as above | y = RN16(b + W x), FP8 -> **FP16**. w = `enc0.Win2` | | `l_linear_32_32_fused` | exists (GPU-checked), D3D12 only | `x, y, w1, w2, npix` | as above | the two above in one kernel, `x1` kept in registers: the same bytes in `y` (`x2`), `x1` not stored. x = `q`, y = `x2` in two different buffers. w1 = `enc0.Win1`, w2 = `enc0.Win2`. The native D3D12 frame runs it in place of the two (fusion `ll`, on by default); without `ll` or without the kernel in the build, the two run | | `l_swin_block_c32_enc[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | ceil(nwin / 8), nwin = ceil((W + ox) / 4) ceil((H + oy) / 4) | the encoder block. x = `x2` (FP16 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2)). out = `x3` (FP8, grid only). H = fh, W = fw. (ox, oy) = `window_offsets["first"]` for the frame parity. w = `enc0_block` | +| `l_swin_block_c32_enc_lin_nc` | exists (GPU-checked), D3D12 only | the encoder block's, then `w1, w2` | as the encoder block | the encoder block with the two input linears as its prologue: the same bytes in `x3`, `x1` and `x2` not stored. x = `q` (FP8 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2), the linears run on it in registers). w1 = `enc0.Win1`, w2 = `enc0.Win2`. The native D3D12 frame runs it in place of the linears and the block (fusion `le`, on by default); without `le` or without the kernel in the build, the linears (fused with `ll`) and the block run | | `l_project_gather` | exists (GPU-checked) | `x, features, w, tw, th` | ceil(tw th / 128), tw = fw / 2, th = fh / 2 | token (r, c) gathers pixels (2r + p / 2, 2c + p % 2), p = 0..3, as channels 32 p + i. Running FP16 sum per group of 32 from the bias, clamp +-6.28125, FP8 -> `enc0.features`. w = `enc0.project` | | `l_dec_project` | exists (GPU-checked) | `z, y, w, tw, th` | ceil(tw th / 128) | per token of dec1's output z [th][tw][64]: 128 outputs (two RN16 groups of 32 inputs, then FP8). Output 32 s + k goes to pixel (2r + s / 2, 2c + s % 2), channel k, of `y`. w = `dec0.project` | | `l_swin_block_c32_dec[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | as the encoder block | the decoder block. x = `y` (FP8). A member outside the grid takes token-grid mirrored pixels with the sub-pixel kept. out = `y2` (**FP16**). (ox, oy) = `window_offsets["second"]`. w = `dec0_block` | diff --git a/clean/l/runner/l_enclin_check.py b/clean/l/runner/l_enclin_check.py new file mode 100644 index 00000000..5895c346 --- /dev/null +++ b/clean/l/runner/l_enclin_check.py @@ -0,0 +1,192 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): the encoder block with the two input linears as its prologue, l_swin_block_c32_enc_lin_nc +(D3D12 only; clean/dx12/kernels/swin_block.hlsl SW_SRC 3, declared in l_swin.hlsl), against the sequence it replaces, +the linears -> l_swin_block_c32_enc_nc, byte for byte. + + python clean/l/runner/l_enclin_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --prepare + python clean/l/runner/l_enclin_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --run + [--seeds N] --json out.json + +--prepare (CPU) builds into : l_ops.co, l_swin.co and swin.co (clean/l/runner/l_c32_check.py compile_kernels), the +D3D12 kernels (clean/dx12/build_dx12.py --only) and kernel_replay.exe. +--run (GPU, one GPU slot): every case is an encoder block case of clean/l/runner/l_d3d12_check.py (its grids, fewer +windows than a workgroup's 8 among them, both window offsets, the block blob) with an FP8 q halo map [H + 4][W + 4][32] +in place of x2 and the two linear blobs (enc0.Win1, enc0.Win2) at clean/l/runner/l_linfuse_check.py's weight scales, +special FP8 codes in q in every other case. On the same input bytes: +- the sequence on D3D12 (kernel_replay, two passes: l_linear_32_32_fused writes x2, then l_swin_block_c32_enc_nc reads + that file); +- the sequence on HIP (in this process: l_linear_32_32_relu, l_linear_32_32_f16, l_swin_block_c32_enc_nc); +- the fused kernel on D3D12. +The rules are clean/l/runner/l_fold_check.py's: raw bytes; the output (x3, 32 bytes a pixel) has 64 bytes of slack and +starts filled with its run's own sentinel (HIP 0xA5, the D3D12 sequence 0x5A, the fused kernel 0x3C): every pixel byte +identical across the three, every slack byte its own run's sentinel; a negative control (the fused kernel with half +its workgroups) must be caught; Model M's swin_block_c96_nc runs through the same replay as a control. +Output: counts only, no timing. +""" +import argparse +import ctypes +import json +import os +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.runner import l_c32_check as C32 # noqa: E402 +from clean.l.runner import l_d3d12_check as LD # noqa: E402 +from clean.l.runner import l_d3d12_ops_check as OPS # noqa: E402 +from clean.l.runner import l_linfuse_check as LF # noqa: E402 +from clean.l.runner.l_fold_check import (CONTROL, D3D12_SENTINEL, FOLD_SENTINEL, HIP_SENTINEL, SLACK, compare, # noqa: E402 + nwin_of, replay, write) + +RELU, F16, LIN = LF.RELU, LF.F16, LF.FUSED +ENC, ENCLIN = "l_swin_block_c32_enc_nc", "l_swin_block_c32_enc_lin_nc" + + +def cases(rng): + """(H, W, offset, q bytes, Win1 blob, Win2 blob, block blob, x3 bytes) of every case.""" + for ci, (H, W, off, _x2, nout, blob) in enumerate(LD.block_cases(ENC, rng)): + npix = (H + 4) * (W + 4) + s, b = LF.SCALES[ci % len(LF.SCALES)] + q = np.frombuffer(OPS._fp8_map(rng, npix * 32), np.uint8).copy() + if ci % 2: + pick = rng.random(q.size) < 0.1 + q[pick] = rng.choice(LF.SPECIAL, int(pick.sum())) + yield H, W, off, q.tobytes(), LF._blob(rng, s, b), LF._blob(rng, s, b), blob, nout + + +def hip_sequence(g, ops, c): + """relu, f16, then the encoder block, on HIP: x3's raw bytes (the output starts at HIP_SENTINEL).""" + H, W, off, q, w1, w2, blob, nout = c + npix = (H + 4) * (W + 4) + h = g.hip + owned = [] + + def buf(data): + p = h.malloc(max(len(data), 4)) + h.upload(p, data) + owned.append(p) + return p + + try: + pq, pw1, pw2 = buf(q), buf(w1), buf(w2) + px1, px2 = h.malloc(npix * 32), h.malloc(npix * 64) + owned += [px1, px2] + grid = -(-npix // 128) + for k, a, o, wb in ((RELU, pq, px1, pw1), (F16, px1, px2, pw2)): + h.launch(h.get_function(ops, k), grid, 256, + [ctypes.c_void_p(a), ctypes.c_void_p(o), ctypes.c_void_p(wb), ctypes.c_int(npix)]) + h.sync() + x2 = bytes(h.download(px2, npix * 64)) + finally: + for p in owned: + h.free(p) + return LD.hip_block_raw(g, "l_swin.co", ENC, x2, blob, H, W, off, nout + SLACK) + + +def enclin_lines(name, c, prefix="", grid=None): + H, W, off, _q, _w1, _w2, _blob, nout = c + return [f"case {prefix}{name}_enclin", f"kernel {ENCLIN} {grid or -(-nwin_of(H, W, off) // 8)} 256", + f"arg buf {name}.q", "arg null", f"arg out {nout + SLACK} {prefix}{name}.fx {FOLD_SENTINEL}", + f"arg buf {name}.w", f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {off[0]}", f"arg u32 {off[1]}", + f"arg buf {name}.w1", f"arg buf {name}.w2", ""] + + +def run(build, rocm, seeds): + g = C32.Gpu(build, rocm) + with open(os.path.join(build, "l_ops.co"), "rb") as f: + ops = g.hip.load_module(f.read()) + cdir = os.path.join(build, "replay-enclin") + os.makedirs(cdir, exist_ok=True) + pass1, pass2, all_cases, neg = [], [], {}, None + for si, seed in enumerate(seeds): + rng = np.random.default_rng(seed) + for ci, c in enumerate(cases(rng)): + name = f"s{si}_{ci}" + H, W, off, q, w1, w2, blob, nout = c + npix = (H + 4) * (W + 4) + all_cases[name] = (nout, hip_sequence(g, ops, c)) + for ext, data in ((".q", q), (".w1", w1), (".w2", w2), (".w", blob)): + write(os.path.join(cdir, name + ext), data) + grid = -(-nwin_of(H, W, off) // 8) + # pass 1: the D3D12 fused linears (x2) and the fused block + pass1 += [f"case {name}_lin", f"kernel {LIN} {-(-npix // 128)} 256", f"arg buf {name}.q", + f"arg out {npix * 64} {name}.x2 {D3D12_SENTINEL}", f"arg buf {name}.w1", f"arg buf {name}.w2", + f"arg u32 {npix}", ""] + pass1 += enclin_lines(name, c) + if neg is None and grid >= 4: + neg = name + pass1 += enclin_lines(name, c, prefix="negative_", grid=grid // 2) + # pass 2: the D3D12 encoder block on that x2 + pass2 += [f"case {name}_enc", "module l_swin.co", f"kernel {ENC} {grid} 256", f"arg buf {name}.x2", + "arg null", f"arg out {nout + SLACK} {name}.sx {D3D12_SENTINEL}", f"arg buf {name}.w", + f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {off[0]}", f"arg u32 {off[1]}", ""] + rng = np.random.default_rng(seeds[0]) + ctl = [] + for ci, (H, W, off, xb, nout, blob) in enumerate(LD.block_cases(CONTROL, rng)): + name = f"control_{ci}" + ctl.append((name, LD.hip_block_raw(g, "swin.co", CONTROL, xb, blob, H, W, off, nout))) + write(os.path.join(cdir, name + ".x"), xb) + write(os.path.join(cdir, name + ".w"), blob) + pass1 += [f"case {name}", "module swin.co", f"kernel {CONTROL} {-(-nwin_of(H, W, off) // 8)} 256", + f"arg buf {name}.x", "arg null", f"arg out {nout} {name}.out {D3D12_SENTINEL}", f"arg buf {name}.w", + f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {off[0]}", f"arg u32 {off[1]}", ""] + e1, t1 = replay(build, cdir, pass1, "pass1") + e2, t2 = replay(build, cdir, pass2, "pass2") if e1 == 0 else (-1, "pass 1 failed") + res = {"replay_exit": [e1, e2], "replay_tail": [t1, t2], "cases": []} + + def read(n): + p = os.path.join(cdir, n) + return open(p, "rb").read() if os.path.exists(p) else b"" + + for name, (nout, hip) in all_cases.items(): + fused, seq = read(f"{name}.fx"), read(f"{name}.sx") + r = {"case": name, "bytes": nout, + "fused_vs_d3d12_seq": compare(fused, seq, nout, FOLD_SENTINEL, D3D12_SENTINEL), + "fused_vs_hip_seq": compare(fused, hip, nout, FOLD_SENTINEL, HIP_SENTINEL), + "d3d12_seq_vs_hip_seq": compare(seq, hip, nout, D3D12_SENTINEL, HIP_SENTINEL)} + r["identical"] = all(r[k]["identical"] for k in ("fused_vs_d3d12_seq", "fused_vs_hip_seq", "d3d12_seq_vs_hip_seq")) + res["cases"].append(r) + caught = None + if neg: + nout = all_cases[neg][0] + o = compare(read(f"negative_{neg}.fx"), read(f"{neg}.sx"), nout, FOLD_SENTINEL, D3D12_SENTINEL) + caught = o["differ"] >= 0 and not o["identical"] + res["negative_control"] = {"case": neg, "half_grid": True, "caught": caught, "differ": max(o["differ"], 0)} + ctl_ok = [read(f"{name}.out") == want for name, want in ctl] + res["control"] = {"kernel": CONTROL, "cases": len(ctl_ok), "identical": sum(ctl_ok)} + res["summary"] = {"cases": len(res["cases"]), "identical": sum(c["identical"] for c in res["cases"]), + "output_bytes": sum(c["bytes"] for c in res["cases"])} + res["pass"] = (e1 == 0 and e2 == 0 and caught is True and res["summary"]["cases"] > 0 and + res["summary"]["identical"] == res["summary"]["cases"] and len(ctl_ok) > 0 and all(ctl_ok)) + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--prepare", action="store_true") + ap.add_argument("--run", action="store_true") + ap.add_argument("--seeds", type=int, default=2) + ap.add_argument("--json") + a = ap.parse_args(argv) + os.makedirs(a.build, exist_ok=True) + if a.prepare: + C32.compile_kernels(a.build, a.rocm, with_swin=True) + print("built", LD.prepare_replay(a.build, a.rocm, [LIN, ENC, ENCLIN, CONTROL])) + if a.run: + res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds))) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + print({k: v for k, v in res.items() if k not in ("cases", "replay_tail")}) + return 0 if res["pass"] else 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runtime/tests/test_l_runtime_frame.py b/clean/l/runtime/tests/test_l_runtime_frame.py index 8f4ef723..c702a087 100644 --- a/clean/l/runtime/tests/test_l_runtime_frame.py +++ b/clean/l/runtime/tests/test_l_runtime_frame.py @@ -129,9 +129,14 @@ def blob_digests(): FOLD_BASE = "l_swin_block_c32_dec_coef12" # native D3D12, fusion ll (the default): the two input linears in one kernel (x1 in registers; q in l.narrow) LIN = "l_linear_32_32_fused" -# the default fusions without lc and ll: the pairs (the fallback routes) +# native D3D12, fusion le (the default): the encoder block with the two input linears as its prologue (q in l.narrow, +# x1 and x2 in registers) +ENCLIN = "l_swin_block_c32_enc_lin_nc" +ENCLIN_BASE = "l_swin_block_c32_enc_lin" +# the default fusions without lc, ll and le: the pairs (the fallback routes); without le: the fused linears and the block NO_FOLD = {"WG_D3D12_FUSION": "pm,xe,mb,rv,xo,xd"} NO_LIN = {"WG_D3D12_FUSION": "pm,xe,mb,rv,xo,xd,lc"} +NO_ENCLIN = {"WG_D3D12_FUSION": "pm,xe,mb,rv,xo,xd,lc,ll"} def latent_out(coef): @@ -155,7 +160,8 @@ def expected_args(t, f, digests): npix, tw, th = str((fh + 4) * (fw + 4)), str(fw // 2), str(fh // 2) eo = [str(v) for v in T.block_offset(T.TABLE, "first", info["parity"])] do = [str(v) for v in T.block_offset(T.TABLE, "second", info["parity"])] - q = n["l.narrow"] if any(ln.kernel == LIN for ln in f["launches"]) else n["l.wide"] # fused linears: q not under x2 + # fused linears, or the linears in the block: q not under x2 + q = n["l.narrow"] if any(ln.kernel in (LIN, ENCLIN) for ln in f["launches"]) else n["l.wide"] return { # the input encoder's values: Model M's encoder inputs (the colour as the network reads it, here the game's, # pre-exposure 1), the state it reads, q into the wide map, the motion surfaces, the exposure pair @@ -166,6 +172,9 @@ def expected_args(t, f, digests): "l_linear_32_32_f16": [n["l.narrow"], n["l.wide"], w["enc0.Win2"], npix], # x1 -> x2 LIN: [n["l.narrow"], n["l.wide"], w["enc0.Win1"], w["enc0.Win2"], npix], # q -> x2 (x1 in registers) "l_swin_block_c32_enc": [n["l.wide"], "0", n["l.x3"], w["enc0_block"], str(fh), str(fw)] + eo, # x2 -> x3 + # the block with the linears as its prologue: q -> x3, then the two linear blobs + ENCLIN_BASE: [n["l.narrow"], "0", n["l.x3"], w["enc0_block"], str(fh), str(fw)] + eo + + [w["enc0.Win1"], w["enc0.Win2"]], "l_project_gather": [n["l.x3"], n["enc0.features"], w["enc0.project"], tw, th], # x3 -> enc1's input "l_dec_project": [n["dec1.out"], n["l.narrow"], w["dec0.project"], tw, th], # dec1's output -> y "l_swin_block_c32_dec": [n["l.narrow"], "0", n["l.wide"], w["dec0_block"], str(fh), str(fw)] + do, # -> y2 @@ -211,10 +220,14 @@ def check_roles(t, f, digests, l_post=True): continue bad += ["%s argument %d: %s, expected %s" % (k, i, a, e) for i, (a, e) in enumerate(zip(ln.args, want)) if e is not None and a != e] - fold, lin = FOLD_BASE in seen, LIN in seen + fold, lin, el = FOLD_BASE in seen, LIN in seen, ENCLIN_BASE in seen pair_d, pair_l = ("l_swin_block_c32_dec", "l_coef"), ("l_linear_32_32_relu", "l_linear_32_32_f16") - need = [k for k in L_KERNELS if not (fold and k in pair_d) and not (lin and k in pair_l)] + \ - ([FOLD_BASE] if fold else []) + ([LIN] if lin else []) + pair_e = pair_l + ("l_swin_block_c32_enc",) + need = [k for k in L_KERNELS if not (fold and k in pair_d) and not (lin and k in pair_l) and + not (el and k in pair_e)] + ([FOLD_BASE] if fold else []) + ([LIN] if lin else []) + \ + ([ENCLIN_BASE] if el else []) + if el and (lin or set(pair_e) & seen): + bad.append("the encoder block with the linears and the linears or the plain block in one frame") if fold and (set(pair_d) & seen): bad.append("the folded decoder block and the pair in one frame") if lin and (set(pair_l) & seen): @@ -347,11 +360,12 @@ def check_frame(self, t, f, roles=True, l_post=True): npix, tw, th = (fh + 4) * (fw + 4), fw // 2, fh // 2 k = {base(ln.kernel): (i, ln) for i, ln in enumerate(ls)} _, lev = k["lev/input_encoder"] + el = k[ENCLIN_BASE][1] if ENCLIN_BASE in k else None # the block with the linears (fusion le) lin = k[LIN][1] if LIN in k else None - if not lin: + if not lin and not el: _, relu = k["l_linear_32_32_relu"] _, f16 = k["l_linear_32_32_f16"] - _, enc = k["l_swin_block_c32_enc"] + enc = el if el else k["l_swin_block_c32_enc"][1] gi, gather = k["l_project_gather"] di, dproj = k["l_dec_project"] fold = FOLD_BASE in k @@ -363,7 +377,10 @@ def check_frame(self, t, f, roles=True, l_post=True): _, coef = k["l_coef"] q, x3, feat = lev.args[6], enc.args[2], gather.args[1] # the chain of maps (no launch reads what it writes) - if lin: # one launch, x1 in registers: no map + if el: # one launch for the linears and the block: no x1 or x2 map + x1 = x2 = None + self.assertEqual(el.args[0], q) + elif lin: # one launch, x1 in registers: no map x1, x2 = None, lin.args[1] self.assertEqual(lin.args[0], q) self.assertEqual(lin.args[4], str(npix)) @@ -373,7 +390,8 @@ def check_frame(self, t, f, roles=True, l_post=True): self.assertEqual(f16.args[0], x1) self.assertEqual(relu.args[3], str(npix)) self.assertEqual(f16.args[3], str(npix)) - self.assertEqual(enc.args[0], x2) + if not el: + self.assertEqual(enc.args[0], x2) self.assertEqual(enc.args[1], "0") # xadd: none self.assertEqual(gather.args[0], x3) self.assertTrue(ls[gi + 1].kernel.startswith("swin_block_c64")) @@ -387,14 +405,16 @@ def check_frame(self, t, f, roles=True, l_post=True): self.assertEqual(y2, "0") # the map is not written else: self.assertEqual(coef.args[0], y2) - for a, b in ((q, x1), (x1, x2), (q, x2 if lin else None), (x2, x3), (x3, feat), (dec1.args[2], y), (y, y2)): + for a, b in ((q, x1), (x1, x2), (q, x2 if lin else None), (x2, x3), (q, x3), (x3, feat), (dec1.args[2], y), + (y, y2)): if a is not None and b is not None: self.assertNotEqual(a, b) - if not lin: + if not lin and not el: self.assertEqual(q, x2) # the pair: x2 over q (q is dead once x1 is written) # every buffer holds its map (FORMATS.md "Maps") - maps = ((q, 32 * npix), (x2, 64 * npix), (x3, 32 * fh * fw), (feat, 64 * tw * th), - (y, 32 * fh * fw)) + (() if fold else ((y2, 64 * fh * fw),)) + (() if lin else ((x1, 32 * npix),)) + maps = ((q, 32 * npix), (x3, 32 * fh * fw), (feat, 64 * tw * th), (y, 32 * fh * fw)) + \ + (() if el else ((x2, 64 * npix),)) + (() if fold else ((y2, 64 * fh * fw),)) + \ + (() if lin or el else ((x1, 32 * npix),)) if fold: # pack12 image A (8 bytes a pixel), image C (4), the new latent coefs = ((coef.args[8], 8 * rw * rh), (coef.args[9], 4 * rw * rh), (coef.args[10], 4 * rw * rh)) else: @@ -577,7 +597,12 @@ def check_d3d12_frame(self, t, f): self.assertEqual(names[names.index(FOLD) - 1], "l_dec_project") else: self.assertIn("l_coef_pack12", names) - if LIN in names: + if ENCLIN in names: + self.assertNotIn(LIN, names) + self.assertNotIn("l_linear_32_32_relu", names) + self.assertNotIn("l_swin_block_c32_enc_nc", names) + self.assertTrue(names[names.index(ENCLIN) - 1].startswith("lev/input_encoder")) + elif LIN in names: self.assertNotIn("l_linear_32_32_relu", names) self.assertNotIn("l_linear_32_32_f16", names) self.assertTrue(names[names.index(LIN) - 1].startswith("lev/input_encoder")) @@ -605,14 +630,20 @@ def test_frame(self): self.assertEqual(t.unknown(), []) self.assertEqual([f["info"]["reset"] for f in t.frames], [1, 0, 1, 0, 1, 1]) self.assertEqual(len({f["info"]["parity"] for f in t.frames}), 2) - for env, ports, fold, lin in ((None, full_ports(), True, True), (NO_FOLD, full_ports(), False, False), - (None, [kc for kc in full_ports() if kc[0] != FOLD], False, True), - (NO_LIN, full_ports(), True, False), - (None, [kc for kc in full_ports() if kc[0] != LIN], True, False)): + without = lambda *ks: [kc for kc in full_ports() if kc[0] not in ks] # noqa: E731 + for env, ports, fold, lin, el in ((None, full_ports(), True, False, True), + (NO_FOLD, full_ports(), False, False, False), + (None, without(FOLD), False, False, True), + (NO_LIN, full_ports(), True, False, False), + (NO_ENCLIN, full_ports(), True, True, False), + (None, without(ENCLIN), True, True, False), + (None, without(ENCLIN, LIN), True, False, False), + (None, without(LIN), True, False, True)): with self.subTest(fusion=env, fold_in_manifest=any(kc[0] == FOLD for kc in ports), - lin_in_manifest=any(kc[0] == LIN for kc in ports)): - # the fusions (lc, ll: the default) and their fallbacks, the pairs: without the fusion, or without the - # kernel + lin_in_manifest=any(kc[0] == LIN for kc in ports), + enclin_in_manifest=any(kc[0] == ENCLIN for kc in ports)): + # the fusions (lc, ll, le: the default; le over ll) and their fallbacks: without the fusion, or without + # the kernel t = FT.Trace(self.l_run(ports, frames=6, rw=96, rh=64, flags=(sizes, "--reset=4"), env=env)) self.assertEqual(t.create, "create ok model L") self.assertEqual(t.unknown(), []) @@ -623,6 +654,7 @@ def test_frame(self): self.assertIsNone(f["error"]) self.assertEqual(FOLD in [ln.kernel for ln in f["launches"]], fold) self.assertEqual(LIN in [ln.kernel for ln in f["launches"]], lin) + self.assertEqual(ENCLIN in [ln.kernel for ln in f["launches"]], el) lev, coef = self.check_d3d12_frame(t, f) if prev: self.assertEqual(lev.args[4], latent_out(prev[1])) # the latent the last frame's store wrote diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index df14987c..ba3b6879 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -125,9 +125,9 @@ std::string emu_lib_dir(const WgRtConfig& c) { // 10 faster; one 8-round run -11.5 us with every round faster). Re-measured in one process (runtime A/B, the same frames // alternating every 300 frames; inter-process noise was +-25 us): xd is +5.0 us at 1707x960 and +7.5 us at 1920x1080, // -2.25 us at 1280x720, so it now runs on frames of at most kXdMaxPixels render pixels only. -// lc, ll: Model L only (its decoder block with l_coef_pack12 as the epilogue, its two input linears in one kernel; see -// fz_lc, fz_ll). -const char* const kDefaultD3d12Fusion = "pm,xe,mb,rv,xo,xd,lc,ll"; +// lc, ll, le: Model L only (its decoder block with l_coef_pack12 as the epilogue, its two input linears in one kernel, +// its encoder block with the two input linears as the prologue; see fz_lc, fz_ll, fz_le). +const char* const kDefaultD3d12Fusion = "pm,xe,mb,rv,xo,xd,lc,ll,le"; // xd only up to this many render pixels per frame (see fz_xd): it is faster at 1280x720 and slower at 1707x960 and above constexpr long long kXdMaxPixels = 1200000; @@ -339,16 +339,22 @@ struct Runtime { // in registers. The input encoder then writes q to l_narrow (x2 over q in l_wide would overwrite q that other // workgroups have not read yet). Without ll, or with a manifest that lacks the kernel, the pair runs bool fz_ll = false, l_lin_built = false; + // le Model L: its encoder block with the two input linears as the prologue (l_swin_block_c32_enc_lin_nc, + // l_swin.hlsl SW_SRC 3; byte for byte the linears -> l_swin_block_c32_enc_nc): one launch less, and x2 (64 bytes + // a pixel of the halo map) is neither stored nor read back: the block reads q (32 bytes) from l_narrow. Over ll + // when both are on. Without le, or with a manifest that lacks the kernel, the linears and the block run + bool fz_le = false, l_le_built = false; long long xd_max_pixels = [] { const char* e = std::getenv("WG_D3D12_XD_MAX_PIXELS"); return e && *e ? std::atoll(e) : kXdMaxPixels; }(); // A/B diagnostics (WG_AB_*): the two fusion sets and the block length (0: off) struct FzSet { - bool pm = false, xe = false, mb = false, mb0 = false, rv = false, xo = false, xd = false, lc = false, ll = false; + bool pm = false, xe = false, mb = false, mb0 = false, rv = false, xo = false, xd = false, lc = false, ll = false, + le = false; template void apply(R* r) const { r->fz_pm = pm, r->fz_xe = xe, r->fz_mb = mb, r->fz_mb0 = mb0, r->fz_rv = rv, r->fz_xo = xo, r->fz_xd = xd; - r->fz_lc = lc, r->fz_ll = ll; + r->fz_lc = lc, r->fz_ll = ll, r->fz_le = le; } }; FzSet fz_set[2]; @@ -379,8 +385,9 @@ struct Runtime { else if (f == "xd") z->xd = true; else if (f == "lc") z->lc = true; else if (f == "ll") z->ll = true; + else if (f == "le") z->le = true; else if (!f.empty()) - return fail("WG_D3D12_FUSION: unknown fusion \"" + f + "\" (pm, xe, mb, rv, xo, xd, lc, ll or none)"); + return fail("WG_D3D12_FUSION: unknown fusion \"" + f + "\" (pm, xe, mb, rv, xo, xd, lc, ll, le or none)"); } return true; } @@ -553,6 +560,7 @@ struct Runtime { const std::vector have = manifest_kernels(); l_fold_built = std::find(have.begin(), have.end(), "l_swin_block_c32_dec_coef12_nc") != have.end(); l_lin_built = std::find(have.begin(), have.end(), "l_linear_32_32_fused") != have.end(); + l_le_built = std::find(have.begin(), have.end(), "l_swin_block_c32_enc_lin_nc") != have.end(); } #endif if (c.max_rw <= 0 || c.max_rh <= 0 || c.ow <= 0 || c.oh <= 0) return fail("invalid render or output size"); @@ -1411,7 +1419,7 @@ struct Runtime { kernels_text += std::string(" dec_rdiv=") + (dec_div.find('r') == std::string::npos ? "1" : "0"); kernels_text += std::string(" dec_bdiv=") + (dec_div.find('b') == std::string::npos ? "1" : "0"); kernels_text += std::string(" dec1_dec0=") + (dec1_dec0 ? "fused" : "separate"); - if (fz_pm || fz_xe || fz_mb || fz_rv || fz_xo || fz_xd || fz_lc || fz_ll) kernels_text += " d3d12_fusion=" + fz_text; + if (fz_pm || fz_xe || fz_mb || fz_rv || fz_xo || fz_xd || fz_lc || fz_ll || fz_le) kernels_text += " d3d12_fusion=" + fz_text; kernels_text += " swin_mid=" + (mid_variant.empty() ? std::string("none") : mid_variant); kernels_text += std::string(" post_pack=") + (p12 ? "p12" : "none"); kernels_text += std::string(" pre_exposure=") + (pi_original ? "original" : "scale"); @@ -1429,6 +1437,7 @@ struct Runtime { (l_ldr ? "l_post_recon_ldr" : "l_post_recon"); if (l_fold_built) kernels_text += std::string(" l_dec_fold=") + (fz_lc ? "on" : "off"); // (native D3D12 only) if (l_lin_built) kernels_text += std::string(" l_lin_fused=") + (fz_ll ? "on" : "off"); + if (l_le_built) kernels_text += std::string(" l_enc_lin=") + (fz_le ? "on" : "off"); return kernels_text.c_str(); } @@ -1578,7 +1587,21 @@ struct Runtime { DevPtr wv = wdev["enc0.values"], w1 = wdev["enc0.Win1"], w2 = wdev["enc0.Win2"], wp = wdev["enc0.project"]; int npix = (fh + 4) * (fw + 4), tw = fw / 2, th = fh / 2; const bool fused = fz_ll && l_lin_built; // (native D3D12 only: l_lin_built) - DevPtr& q = fused ? l_narrow : l_wide; // q's map (fused: not under x2) + const bool in_block = fz_le && l_le_built && l_nc[0]; // fusion le: the linears in the block (its _nc form) + DevPtr& q = fused || in_block ? l_narrow : l_wide; // q's map (fused / in the block: not under x2) + if (in_block) { + DevPtr none = 0, wb = wdev["enc0_block"]; + int ox = 0, oy = 0; + l_offsets(false, f, &ox, &oy); + const int nwin = ((fw + ox + 3) / 4) * ((fh + oy + 3) / 4); + return launch(native_direct_io ? "lev/input_encoder_direct" : "lev/input_encoder", + cdiv(((fh + 4) / 2) * ((fw + 4) / 2), 128), 256, + {&cnet, &din, &min_, &reduced, &latent[cur], &mnear[cur], &q, &mnear[nxt], &mfar, &mcenter, + &expo, &wv, &p}) && + launch("l_swin_block_c32_enc_lin_nc", cdiv(nwin, 8), 256, + {&l_narrow, &none, &l_x3, &wb, &fh, &fw, &ox, &oy, &w1, &w2}) && + launch("l_project_gather", cdiv(tw * th, 128), 256, {&l_x3, &feat0, &wp, &tw, &th}); + } return launch(native_direct_io ? "lev/input_encoder_direct" : "lev/input_encoder", cdiv(((fh + 4) / 2) * ((fw + 4) / 2), 128), 256, {&cnet, &din, &min_, &reduced, &latent[cur], &mnear[cur], &q, &mnear[nxt], &mfar, &mcenter, diff --git a/clean/tests/runtime_frame_trace.cpp b/clean/tests/runtime_frame_trace.cpp index f3fefa39..165954b6 100644 --- a/clean/tests/runtime_frame_trace.cpp +++ b/clean/tests/runtime_frame_trace.cpp @@ -75,6 +75,8 @@ const Signature kSignatures[] = { {"swin_block_*", "ppppiiii"}, // Model M's blocks: x, skip, out, w, H, W, ox, oy {"l_swin_block_c32_dec_coef12_nc", "ppppiiiippppii"}, // the decoder block with l_coef_pack12 folded in: then // A, C, latent, dec0.coef, rw, rh (native D3D12, fusion lc) + {"l_swin_block_c32_enc_lin_nc", "ppppiiiipp"}, // the encoder block with the linears as its prologue (fusion + // le): x = q, then enc0.Win1, enc0.Win2 {"l_swin_block_c32*", "ppppiiii"}, // Model L's blocks (clean/l/FORMATS.md) {"l_linear_32_32_relu", "pppi"}, {"l_linear_32_32_f16", "pppi"},