From 2116f763b177cd6426723b4561de0a9e5795d337 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sat, 14 Nov 2026 09:56:03 -0500 Subject: [PATCH] Model L D3D12: dec1's block (swin_block_c64_add_ex_nc) with two windows per wave (the same bytes) swin_c64w2.hlsl (Model M's enc1 block in 128-lane workgroups, two windows per wave) takes the block input x + xadd (SW_ADD) and the fused expand (SW_ECL) under a kernel name switch (SW_W2_NAME); swin_block.hlsl's fused_expand runs with the workgroup's wave count (8, or 4 here). dec1's declaration (swin_c64ex.hlsl) moves to that form; its VGPR bound is 192 (4 workgroups of 4 waves per CU). Model M's DXILs are unchanged; the one-window line rebuilds main's dec1 byte for byte. Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/kernels/fused_swin.hlsl | 23 ++++++++-------- clean/dx12/kernels/swin_block.hlsl | 23 ++++++++-------- clean/dx12/kernels/swin_c64ex.hlsl | 15 +++++++--- clean/dx12/kernels/swin_c64w2.hlsl | 44 ++++++++++++++++++++++++++++-- clean/dx12/vgpr_limits.json | 2 +- 5 files changed, 77 insertions(+), 30 deletions(-) diff --git a/clean/dx12/kernels/fused_swin.hlsl b/clean/dx12/kernels/fused_swin.hlsl index 8e34b400..5b58734b 100644 --- a/clean/dx12/kernels/fused_swin.hlsl +++ b/clean/dx12/kernels/fused_swin.hlsl @@ -523,8 +523,9 @@ int wg_low0(int p0, int n) } #define EKC (SW_ECL / 16) -#if (4 * SW_C / 16) % 8 != 0 -#error "fused_expand: MT = 4 C / 16 must be a multiple of the 8 waves" +#define FE_NW (THREADS / 32) // waves per workgroup (8; 4 in swin_c64w2.hlsl's 128-lane form) +#if (4 * SW_C / 16) % FE_NW != 0 +#error "fused_expand: MT = 4 C / 16 must be a multiple of the waves" #endif // HIP fused_expand: the 8 windows' expanded block input e = q8(We y + be) into LDS records // [window][low token][sub-pixel][C] at e_lds. tail = expand weights (bias [4C] f32, then the fragments, in global memory). @@ -549,19 +550,19 @@ void fused_expand(uint tail, uint e_lds, int nwx, int nwin, uint tid, int blk, u } } uint be = tail + hh * 32u; - // MT = 24 = 3 x 8 waves: every wave runs exactly MT / 8 tiles, unrolled with all weight fragments and biases issued - // first (one exposed global latency instead of one per tile) + // MT = 24 = 3 x 8 waves: every wave runs exactly MT / FE_NW tiles, unrolled with all weight fragments and biases + // issued first (one exposed global latency instead of one per tile) const uint wv = (uint)wg_uniform((int)wave); - uint2 fa[MT / 8][EKC]; - v8f ba[MT / 8]; + uint2 fa[MT / FE_NW][EKC]; + v8f ba[MT / FE_NW]; const uint fwb = tail + WE + wv * (EKC * 256u) + (lane * 8u + sw_zero_bits()); - [unroll] for (uint it = 0; it < MT / 8; ++it) { - const uint mt = wv + 8u * it; - [unroll] for (uint kc = 0; kc < EKC; ++kc) fa[it][kc] = wg_ld64(fwb, (8u * it * EKC + kc) * 256u); + [unroll] for (uint it = 0; it < MT / FE_NW; ++it) { + const uint mt = wv + FE_NW * it; + [unroll] for (uint kc = 0; kc < EKC; ++kc) fa[it][kc] = wg_ld64(fwb, (FE_NW * it * EKC + kc) * 256u); ba[it] = wg_load8f(be, mt * 64u); } - [unroll] for (uint it2 = 0; it2 < MT / 8; ++it2) { - const uint mt = wv + 8u * it2; + [unroll] for (uint it2 = 0; it2 < MT / FE_NW; ++it2) { + const uint mt = wv + FE_NW * it2; uint k = mt / G, g = mt - k * G; v8f acc0 = ba[it2], acc1 = ba[it2]; [unroll] for (uint kc2 = 0; kc2 < EKC; ++kc2) { diff --git a/clean/dx12/kernels/swin_block.hlsl b/clean/dx12/kernels/swin_block.hlsl index fda2ef1c..e55a90a6 100644 --- a/clean/dx12/kernels/swin_block.hlsl +++ b/clean/dx12/kernels/swin_block.hlsl @@ -483,8 +483,9 @@ int wg_low0(int p0, int n) } #define EKC (SW_ECL / 16) -#if (4 * SW_C / 16) % 8 != 0 -#error "fused_expand: MT = 4 C / 16 must be a multiple of the 8 waves" +#define FE_NW (THREADS / 32) // waves per workgroup (8; 4 in swin_c64w2.hlsl's 128-lane form) +#if (4 * SW_C / 16) % FE_NW != 0 +#error "fused_expand: MT = 4 C / 16 must be a multiple of the waves" #endif // HIP fused_expand: the 8 windows' expanded block input e = q8(We y + be) into LDS records // [window][low token][sub-pixel][C] at e_lds. tail = expand weights (bias [4C] f32, then the fragments, in global memory). @@ -509,19 +510,19 @@ void fused_expand(uint tail, uint e_lds, int nwx, int nwin, uint tid, int blk, u } } uint be = tail + hh * 32u; - // MT = 24 = 3 x 8 waves: every wave runs exactly MT / 8 tiles, unrolled with all weight fragments and biases issued - // first (one exposed global latency instead of one per tile) + // MT = 24 = 3 x 8 waves: every wave runs exactly MT / FE_NW tiles, unrolled with all weight fragments and biases + // issued first (one exposed global latency instead of one per tile) const uint wv = (uint)wg_uniform((int)wave); - uint2 fa[MT / 8][EKC]; - v8f ba[MT / 8]; + uint2 fa[MT / FE_NW][EKC]; + v8f ba[MT / FE_NW]; const uint fwb = tail + WE + wv * (EKC * 256u) + (lane * 8u + sw_zero_bits()); - [unroll] for (uint it = 0; it < MT / 8; ++it) { - const uint mt = wv + 8u * it; - [unroll] for (uint kc = 0; kc < EKC; ++kc) fa[it][kc] = wg_ld64(fwb, (8u * it * EKC + kc) * 256u); + [unroll] for (uint it = 0; it < MT / FE_NW; ++it) { + const uint mt = wv + FE_NW * it; + [unroll] for (uint kc = 0; kc < EKC; ++kc) fa[it][kc] = wg_ld64(fwb, (FE_NW * it * EKC + kc) * 256u); ba[it] = wg_load8f(be, mt * 64u); } - [unroll] for (uint it2 = 0; it2 < MT / 8; ++it2) { - const uint mt = wv + 8u * it2; + [unroll] for (uint it2 = 0; it2 < MT / FE_NW; ++it2) { + const uint mt = wv + FE_NW * it2; uint k = mt / G, g = mt - k * G; v8f acc0 = ba[it2], acc1 = ba[it2]; [unroll] for (uint kc2 = 0; kc2 < EKC; ++kc2) { diff --git a/clean/dx12/kernels/swin_c64ex.hlsl b/clean/dx12/kernels/swin_c64ex.hlsl index 30fd1d84..46228f71 100644 --- a/clean/dx12/kernels/swin_c64ex.hlsl +++ b/clean/dx12/kernels/swin_c64ex.hlsl @@ -5,8 +5,15 @@ // Model L's frame (issue #89) launches it for dec1 (default swin_fuse_stages dec1,dec2: the L decoder reads dec1's map). // swin_block.hlsl instantiated at C = 64, 2 heads, MLP 256 (its C = 64 path, as swin_block_c64_nc) with the block input // x + xadd (SW_ADD) and the fused expand from the 96-channel low-resolution map (SW_ECL 96), as -// swin_block_c96_add_ex_nc is at C = 96. 256 lanes, one window per wave; x = dec2's output (96 channels at half size), -// xadd = the skip map, aux unused. +// swin_block_c96_add_ex_nc is at C = 96. x = dec2's output (96 channels at half size), xadd = the skip map, aux unused. // -// @wg-kernel swin_block_c64_add_ex_nc co=swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u,p defines=SW_C=64,SW_HEADS=2,SW_MLP=256,SW_ADD=1,SW_SAFE=1,SW_ECL=96,SW_ENTRY_OVERRIDE=swin_block_c64_add_ex_nc -#include "swin_block.hlsl" +// Two windows per wave in 128-lane workgroups (swin_c64w2.hlsl, the launch shape of Model M's enc1 block): the launch +// keeps HIP's geometry (8 windows per launched workgroup), each wave does two windows interleaved, every staged weight +// fragment read from LDS once for both, and a unit barrier waits for 4 waves instead of 8. Every window's arithmetic is +// that of swin_block.hlsl's C = 64 path (the same operations on the same operands in the same order). +// +// @wg-kernel swin_block_c64_add_ex_nc co=swin.co block=256 group=128 gscale=1 wave=32 args=p,p,p,p,u,u,u,u,p defines=SW_C=64,SW_HEADS=2,SW_MLP=256,SW_ADD=1,SW_SAFE=1,SW_ECL=96,SW_W2=1,SW_W2_NAME=swin_block_c64_add_ex_nc +// +// Launch-shape switch: the one-window-per-wave kernel of swin_block.hlsl (256-lane workgroups, HIP's shape) is the same +// line without "group=128 gscale=1", with SW_W2=0 and SW_ENTRY_OVERRIDE=swin_block_c64_add_ex_nc in place of SW_W2_NAME. +#include "swin_c64w2.hlsl" diff --git a/clean/dx12/kernels/swin_c64w2.hlsl b/clean/dx12/kernels/swin_c64w2.hlsl index 1de31997..4ba1a5ae 100644 --- a/clean/dx12/kernels/swin_c64w2.hlsl +++ b/clean/dx12/kernels/swin_c64w2.hlsl @@ -2,6 +2,9 @@ // swin_block_c64_nc (enc1's Swin block) with TWO windows per wave in 128-lane workgroups (tracks lev3-swin, r3-c64w2). // // @wg-kernel swin_block_c64_nc co=swin.co block=256 group=128 gscale=1 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=64,SW_HEADS=2,SW_MLP=256,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_W2=1 +// Model L's dec1 block swin_block_c64_add_ex_nc (swin_c64ex.hlsl) runs this kernel too (SW_W2_NAME), with the block +// input x + xadd (SW_ADD) and the fused expand (SW_ECL: swin_block.hlsl's fused_expand with the 4 waves; each wave then +// reads its two windows' records). // // Launch-shape switch: the line above runs this file's kernel (SW_W2 1: 128-lane workgroups, gscale 1). The one-window- // per-wave kernel of swin_block.hlsl (256-lane workgroups, HIP's shape) is the same line without "group=128 gscale=1" @@ -47,9 +50,12 @@ #include "swin_block.hlsl" #if SW_W2 -#if SW_C != 64 || THREADS != 128 || SW_ADD || SW_ECL > 0 || (GROUP != 2 && GROUP != 1) || SW_GELU != 3 || SW_NORM2_EARLY || SW_HEAD_PAIRS +#if SW_C != 64 || THREADS != 128 || (GROUP != 2 && GROUP != 1) || SW_GELU != 3 || SW_NORM2_EARLY || SW_HEAD_PAIRS || SW_SRC != 0 || SW_OUT_F16 #error "swin_c64w2.hlsl: the c64 block, 128 lanes" #endif +#ifndef SW_W2_NAME +#define SW_W2_NAME swin_block_c64_nc +#endif #define NWW 2 // windows per wave #ifndef SW_W2_MLP_STEP #define SW_W2_MLP_STEP 4 // MLP units per loop pass (NMLP = 4: the whole MLP unrolled) @@ -74,7 +80,7 @@ #define SW_W2_COMMIT SW_COMMIT_EARLY #endif -WG_KERNEL(swin_block_c64_nc) +WG_KERNEL(SW_W2_NAME) { WG_SETUP(); const uint tid = wg_lane_id(); @@ -91,6 +97,15 @@ WG_KERNEL(swin_block_c64_nc) // Gather: token t of each window, mirrored at the borders. int dst[NWW]; uint2 xa[NWW][KC]; +#if SW_ADD + uint2 xb[NWW][KC]; +#endif +#if SW_ECL > 0 + // the 8 windows' expanded input into the second staging buffer (swin_block.hlsl), read back after the barrier + const uint lowp = sw_win_low0(nwx, nwin, lane, (int)wg_block_id()); + fused_expand(w + L_TAIL, MAXU, nwx, nwin, tid, (int)wg_block_id(), lowp); + uint eslot[NWW]; +#endif [unroll] for (uint i = 0; i < NWW; ++i) { const int win0 = (int)(wg_block_id() * (THREADS / 32 * NWW) + wave_id * NWW + i); const int win = win0 < nwin ? win0 : nwin - 1; @@ -99,7 +114,17 @@ WG_KERNEL(swin_block_c64_nc) const bool real = win0 < nwin && pr >= 0 && pr < H && pc >= 0 && pc < W; dst[i] = real ? pr * W + pc : -1; const int mr = sw_mirror(pr, H), mc = sw_mirror(pc, W); +#if SW_ECL > 0 + const uint wb = wave_id * NWW + i; // the window in the workgroup (win: the same clamp) + const uint lpw = WaveReadLaneAt(lowp, wb); + const int low = ((mr >> 1) - (int)(lpw & 0xffffu)) * 2 + ((mc >> 1) - (int)(lpw >> 16)); + eslot[i] = wb * 16u + (uint)low * 4u + (uint)(((mr & 1) << 1) | (mc & 1)); +#else load_frags_g(x, (uint)(mr * W + mc) * SW_C + hh * HALF, xa[i]); +#endif +#if SW_ADD + load_frags_g(xadd, (uint)(mr * W + mc) * SW_C + hh * HALF, xb[i]); +#endif } v8f ng[KC], nb[KC]; { @@ -108,13 +133,26 @@ WG_KERNEL(swin_block_c64_nc) } st_commit(stage, 0u, HEAD_BYTES / 16, tid); wg_barrier_lds(); +#if SW_ECL > 0 + [unroll] for (uint ie = 0; ie < NWW; ++ie) + [unroll] for (uint je = 0; je < KC / 2; ++je) { + const uint4 v = sw_lds_ld128(MAXU + eslot[ie] * SW_C + hh * HALF + 16u * je); + xa[ie][2 * je] = v.xy; xa[ie][2 * je + 1] = v.zw; + } + wg_barrier_lds(); // read before the buffer they live in is refilled +#endif uint buf = 0u; // Steps 1-2: a = q8(RMSNorm(x; gamma1)); the residual accumulator starts as x + bp. uint2 af[NWW][KC]; v8f acc[NWW][KC]; [unroll] for (uint i1 = 0; i1 < NWW; ++i1) { - [unroll] for (uint k = 0; k < KC; ++k) acc[i1][k] = sw_unpack_fp8(xa[i1][k]); + [unroll] for (uint k = 0; k < KC; ++k) { + acc[i1][k] = sw_unpack_fp8(xa[i1][k]); +#if SW_ADD + acc[i1][k] = sw_add8(acc[i1][k], sw_unpack_fp8(xb[i1][k])); +#endif + } const float r = norm_r(acc[i1]); [unroll] for (uint k2 = 0; k2 < KC; ++k2) { v8f d; diff --git a/clean/dx12/vgpr_limits.json b/clean/dx12/vgpr_limits.json index faf34036..4aea008a 100644 --- a/clean/dx12/vgpr_limits.json +++ b/clean/dx12/vgpr_limits.json @@ -5,7 +5,7 @@ "l_swin_block_c32_dec_nc": 96, "l_swin_block_c32_dec_coef12_nc": 96, "swin_block_c64_nc": 192, - "swin_block_c64_add_ex_nc": 128, + "swin_block_c64_add_ex_nc": 192, "swin_block_c96_nc": 192, "swin_block_c96_add_ex_nc": 192, "swin_block_c96_pm_nc": 192,