From 3ce7024572a8ad404bac1a5e020418a611709891 Mon Sep 17 00:00:00 2001 From: Jonathan Shin <85975340+j0nsh1n@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:54:39 -0800 Subject: [PATCH 1/2] Bottleneck (D3D12): read the attention fragments in the K loop, not all 16 ahead (swin_bneck6_nc 256 -> 192 VGPRs on RADV) attention() in bneck.hlsl read all 16 fragments fq[8] / fv[8] from LDS (16 ds_read_b64, 32 registers) before the two WMMA chains. Under RADV/ACO (CU mode) that is the one spot, 193 registers (ACO_DEBUG=liveinfo), where the whole unfused body crosses the 192-VGPR line, so swin_bneck6_nc was allocated 256 VGPRs: 5 waves per SIMD and one workgroup per CU instead of 8 waves and two. clean/dx12/vgpr_limits.json bounds the bneck kernels at 192 for the Windows compiler (RGA), and bneck.hlsl's header says one workgroup per CU runs 25-27 % slower, so this is a cliff seen under ACO. Each K group's two fragments are now read right before their products. The aq and av chains keep their order (kc2 = 0 .. 7, alternating) and operands: no WMMA accumulation order or operand changes, only when the fragments are read. fused_bneck.hlsl is regenerated (the committed file equals the generator's output). On its own this helps the unfused kernel (WG_D3D12_FUSION without mb / xo); the fused kernels (mb, mbo: the default) stay at 256 VGPRs because of their merge prologue, which the next commit addresses. The measured result of both commits together on main is in the next commit. Checks (RX 9070 XT, gfx1201, Wine + vkd3d-proton + RADV, synthetic model, overclocked LACT profile; times are summed GPU dispatch times): - No-card gfx1201: swin_bneck6_nc 256 -> 192 VGPRs, 8 waves/SIMD, no spills (5748 -> 5877 instructions); mb / mbo unchanged at 256. - Byte identity on the run-stable dumps at 64x40 -> 96x60, 853x481 -> 1279x721 and 1706x960 -> 2560x1440, with and without the mb / xo fusions; a wrong fragment index (kc2 ^ 1) in attention() is caught. - Timing at 1706x960 -> 2560x1440 with both arms unfused (mb / xo off): the bottleneck 477.8 -> 377.4 us, the frame 2419.6 -> 2319.7 us. With the default fusion (mbo) 490.6 -> 490.3 us: no effect, no regression. Not tested: the Windows driver, RDNA 3 (the bneck DXIL does not compile on the no-card gfx1100 route here, dxil-spirv opcode 79). Signed-off-by: Jonathan Shin <85975340+j0nsh1n@users.noreply.github.com> --- clean/dx12/kernels/bneck.hlsl | 12 +++++------- clean/dx12/kernels/fused_bneck.hlsl | 12 +++++------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/clean/dx12/kernels/bneck.hlsl b/clean/dx12/kernels/bneck.hlsl index 81fd2555..90c820cc 100644 --- a/clean/dx12/kernels/bneck.hlsl +++ b/clean/dx12/kernels/bneck.hlsl @@ -211,15 +211,13 @@ void attention(uint U, uint sec, uint hd, uint2 af[KC], inout v8f acc[KC], uint uint4 vt[2]; [unroll] for (uint mt = 0; mt < 2; ++mt) { if (mt == 1u) stage_step_r(1u, 3u, nrows, ssrc, sdst, tid, sr); - uint2 fq[KC], fv[KC]; - [unroll] for (uint kc = 0; kc < KC; ++kc) { - fq[kc] = frag(U, (mt * KC + kc) * 2u, lb); - fv[kc] = frag(U, (mt * KC + kc) * 2u + 1u, lb); - } + // each K group's two fragments are read right before their products (the two chains keep their order and + // operands): all 16 read ahead were the kernel's one 193-register peak, which put the unfused body at 256 VGPRs v8f aq = wg_splat(0.0f), av = wg_splat(0.0f); [unroll] for (uint kc2 = 0; kc2 < KC; ++kc2) { - aq = wg_wmma_fp8(fq[kc2], af[kc2], aq); - av = wg_wmma_fp8(af[kc2], fv[kc2], av); + const uint2 fq = frag(U, (mt * KC + kc2) * 2u, lb), fv = frag(U, (mt * KC + kc2) * 2u + 1u, lb); + aq = wg_wmma_fp8(fq, af[kc2], aq); + av = wg_wmma_fp8(af[kc2], fv, av); } uf[mt] = wg_pack_q8_nc_ags(aq); vt[mt] = wg_to_f16_ags(av); diff --git a/clean/dx12/kernels/fused_bneck.hlsl b/clean/dx12/kernels/fused_bneck.hlsl index 730a88d2..f4fe6956 100644 --- a/clean/dx12/kernels/fused_bneck.hlsl +++ b/clean/dx12/kernels/fused_bneck.hlsl @@ -298,15 +298,13 @@ void attention(uint U, uint sec, uint hd, uint2 af[KC], inout v8f acc[KC], uint uint4 vt[2]; [unroll] for (uint mt = 0; mt < 2; ++mt) { if (mt == 1u) stage_step_r(1u, 3u, nrows, ssrc, sdst, tid, sr); - uint2 fq[KC], fv[KC]; - [unroll] for (uint kc = 0; kc < KC; ++kc) { - fq[kc] = frag(U, (mt * KC + kc) * 2u, lb); - fv[kc] = frag(U, (mt * KC + kc) * 2u + 1u, lb); - } + // each K group's two fragments are read right before their products (the two chains keep their order and + // operands): all 16 read ahead were the kernel's one 193-register peak, which put the unfused body at 256 VGPRs v8f aq = wg_splat(0.0f), av = wg_splat(0.0f); [unroll] for (uint kc2 = 0; kc2 < KC; ++kc2) { - aq = wg_wmma_fp8(fq[kc2], af[kc2], aq); - av = wg_wmma_fp8(af[kc2], fv[kc2], av); + const uint2 fq = frag(U, (mt * KC + kc2) * 2u, lb), fv = frag(U, (mt * KC + kc2) * 2u + 1u, lb); + aq = wg_wmma_fp8(fq, af[kc2], aq); + av = wg_wmma_fp8(af[kc2], fv, av); } uf[mt] = wg_pack_q8_nc_ags(aq); vt[mt] = wg_to_f16_ags(av); From b41650b2057be7d7f44d0f35ab8dd469bb671d04 Mon Sep 17 00:00:00 2001 From: Jonathan Shin <85975340+j0nsh1n@users.noreply.github.com> Date: Thu, 19 Nov 2026 13:54:39 -0800 Subject: [PATCH 2/2] Bottleneck (D3D12): fused patch-merge prologue at 192 VGPRs under ACO (mb / mbo 256 -> 192; -110 us GPU time per frame at 1440p on RADV) The persistent bottleneck kernels with enc2's patch merge fused in (bneck/swin_bneck6_mb_nc and _mbo_nc; mbo is the default) were allocated 256 VGPRs by RADV/ACO: 5 waves per SIMD and one workgroup per CU, instead of the 192 that clean/dx12/vgpr_limits.json bounds them at for the Windows compiler (RGA). The peak is not one spot: in the merge phase every fragment batch (MB_CH = 8 K groups x 2 tiles = 16 ds_read_b64, 32 registers) sits on a base level of 172-190 registers (zf[24] 48, the second weight stripe sv[6] 24 held since before the barrier, addresses), reaching 216-230 (ACO_DEBUG=liveinfo). Two changes to fused_bneck_gen.py, both only about when loads happen: - the second weight stripe's global loads take an address term that depends on merge pass 0's first packed result (tile 0) and is 0 at run time (res[0][0].x & (wg_grid_ >> 31), the opaque zero bneck.hlsl already uses as zb), so ACO cannot hoist them above that point; it interleaves them with the rest of pass 0, which keeps fewer values live; - MB_CH 8 -> 4 (fragment batches of 8 reads; 4 divides 24). The 24 K groups of every tile still accumulate in the same order from the bias, with the same operands: no WMMA accumulation order or operand changes. fused_bneck.hlsl is regenerated (the committed file equals the generator's output). With the previous commit (which removes the body's own 193-register spot) the kernels compile to 192 VGPRs / 8 waves per SIMD under ACO; this commit alone leaves mbo at 256, so both are needed. Checks on main 11d3bbc with these two commits (RX 9070 XT, gfx1201, Wine + vkd3d-proton + RADV, synthetic model, overclocked LACT profile; one runtime DLL built from main for both arms, only the three bottleneck DXIL files swapped): - No-card gfx1201: mb, mbo and nc 256 -> 192 VGPRs, 8 waves/SIMD, no spills (mbo 7278 -> 7495 instructions). - Byte identity at 64x40 -> 96x60, 853x481 -> 1279x721 and 1706x960 -> 2560x1440 on every dump a frame writes (63 per size; the unwritten fused-route buffers excluded, dec0.C compared on its active bytes), against two same-session base runs: equal. A wrong operand index in the merge's first accumulator is caught (29/63 equal). - Timing at 1706x960 -> 2560x1440, 5 interleaved rounds x 60 frames, shader clock median 3.05-3.09 GHz, no throttle. Summed GPU dispatch time per frame (frames kept when the unchanged input encoder is within 3 % of its run's minimum, 267 of 300 per arm): 2609.6 -> 2499.9 us (-109.7 us, -4.2 %; unfiltered -114.1 us); the bottleneck 487.2 -> 407.7 us, dec2's xo block 204.2 -> 187.4 us. Host frame time with the profiler on 2.700 -> 2.584 ms (-116 us, faster in 5 of 5 rounds); with it off -36 us (noisy, 4 of 5 rounds faster). Not tested: the Windows driver (where RGA already meets the 192 bound), and RDNA 3 (the bneck DXIL does not compile on the no-card gfx1100 route here). Signed-off-by: Jonathan Shin <85975340+j0nsh1n@users.noreply.github.com> --- clean/dx12/kernels/fused_bneck.hlsl | 7 +++++-- clean/dx12/kernels/fused_bneck_gen.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/clean/dx12/kernels/fused_bneck.hlsl b/clean/dx12/kernels/fused_bneck.hlsl index f4fe6956..5cfd0951 100644 --- a/clean/dx12/kernels/fused_bneck.hlsl +++ b/clean/dx12/kernels/fused_bneck.hlsl @@ -160,7 +160,7 @@ Grid grid_of(int H, int W, int ox, int oy) // BL | BR), 24 K groups, as patch_merge_c96. Output tile mt (16 channels) gives this lane's bytes hh 64 + 8 mt of the // merged token. #ifndef MB_CH -#define MB_CH 8u // K groups per fragment batch (a divisor of 24): 2 tiles x MB_CH fragments loaded before their multiplies +#define MB_CH 4u // K groups per fragment batch (a divisor of 24): 2 tiles x MB_CH fragments loaded before their multiplies #endif void merge_tiles(uint p, uint2 zf[24], inout uint2 res[8], uint bm, uint lb) { @@ -217,7 +217,6 @@ void merge_item(uint xo, int wf, int wl, Grid g, int H, int W, uint tid, uint wa merge_load(zo[0], zf); wg_barrier_lds(); [unroll] for (uint i1 = 0; i1 < 6; ++i1) st_lds4(tid * 2u + i1 * 512u, sv[i1]); - [unroll] for (uint i2 = 0; i2 < 6; ++i2) sv[i2] = wg_ld128(fw, 24576u + (tid + i2 * 256u) * 16u); wg_barrier_lds(); const uint bm = a_mw + hh * 32u; uint2 res[MB_WPW][8]; @@ -225,6 +224,10 @@ void merge_item(uint xo, int wf, int wl, Grid g, int H, int W, uint tid, uint wa if (j1 > 0u) merge_load(zo[j1], zf); merge_tiles(0u, zf, res[j1], bm, lb); } + // the second stripe's loads wait for pass 0's results through an address term that is 0 at run time: they are not + // hoisted over pass 0, where their 24 registers would be live under the fragment batches + const uint pin = res[0][0].x & (wg_grid_ >> 31); + [unroll] for (uint i2 = 0; i2 < 6; ++i2) sv[i2] = wg_ld128(fw, 24576u + pin + (tid + i2 * 256u) * 16u); wg_barrier_lds(); [unroll] for (uint i3 = 0; i3 < 6; ++i3) st_lds4(tid * 2u + i3 * 512u, sv[i3]); wg_barrier_lds(); diff --git a/clean/dx12/kernels/fused_bneck_gen.py b/clean/dx12/kernels/fused_bneck_gen.py index 874baf34..6ac8e261 100644 --- a/clean/dx12/kernels/fused_bneck_gen.py +++ b/clean/dx12/kernels/fused_bneck_gen.py @@ -84,7 +84,7 @@ def rep(old, new): // BL | BR), 24 K groups, as patch_merge_c96. Output tile mt (16 channels) gives this lane's bytes hh 64 + 8 mt of the // merged token. #ifndef MB_CH -#define MB_CH 8u // K groups per fragment batch (a divisor of 24): 2 tiles x MB_CH fragments loaded before their multiplies +#define MB_CH 4u // K groups per fragment batch (a divisor of 24): 2 tiles x MB_CH fragments loaded before their multiplies #endif void merge_tiles(uint p, uint2 zf[24], inout uint2 res[8], uint bm, uint lb) { @@ -141,7 +141,6 @@ def rep(old, new): merge_load(zo[0], zf); wg_barrier_lds(); [unroll] for (uint i1 = 0; i1 < 6; ++i1) st_lds4(tid * 2u + i1 * 512u, sv[i1]); - [unroll] for (uint i2 = 0; i2 < 6; ++i2) sv[i2] = wg_ld128(fw, 24576u + (tid + i2 * 256u) * 16u); wg_barrier_lds(); const uint bm = a_mw + hh * 32u; uint2 res[MB_WPW][8]; @@ -149,6 +148,10 @@ def rep(old, new): if (j1 > 0u) merge_load(zo[j1], zf); merge_tiles(0u, zf, res[j1], bm, lb); } + // the second stripe's loads wait for pass 0's results through an address term that is 0 at run time: they are not + // hoisted over pass 0, where their 24 registers would be live under the fragment batches + const uint pin = res[0][0].x & (wg_grid_ >> 31); + [unroll] for (uint i2 = 0; i2 < 6; ++i2) sv[i2] = wg_ld128(fw, 24576u + pin + (tid + i2 * 256u) * 16u); wg_barrier_lds(); [unroll] for (uint i3 = 0; i3 < 6; ++i3) st_lds4(tid * 2u + i3 * 512u, sv[i3]); wg_barrier_lds();