From a3769786cbac9ffaf8b4b12145610d58bce9b199 Mon Sep 17 00:00:00 2001 From: Jonathan Shin <85975340+j0nsh1n@users.noreply.github.com> Date: Thu, 19 Nov 2026 10:07:10 -0800 Subject: [PATCH] Model K (D3D12): 16-byte aligned loads in the narrow stage kernels (-24 us at 1440p on RADV, byte-identical) The narrow stage kernels of k_block.hlsl (k_stage_e2/e0/e1/d1/d2, k_stage2_e0/e1/d1/d0_hp/d0_hidden and the __rodd forms) load their weight, bias and activation fragments through RWByteAddressBuffer.Load4, which is known to be 4-byte aligned only. RADV/ACO therefore split every 16-byte load into four global_load_b32 with four address additions (k_stage2_d1: 576 dword loads, no wider one). Arena allocations and packed section starts are 256-byte aligned (the D3D12 arena, k_pack.cpp kAlign), and every derived base and byte offset these loads use is a multiple of 16 (k_common.hlsli already documents "16-byte aligned"), so clearing the low 4 address bits changes no address. LDS reads and all stores are not masked. KB_LD_A16 (default 0, set to 1 only in k_block.hlsl) selects the masked form in wgk_ld128_a16; wgk_ld8, kb_bias32 and kb_weight128 use it. k_block.hlsl is the only shader that includes k_common.hlsli or k_block.hlsli, so only its 13 DXIL files change in the 108-shader build with Model K; the other 95 are byte-identical. kSourceDigest in k_smx_selfcheck.h is recomputed with the repository's own smx_selfcheck_digest.digest(); the reference hashes in the header are HIP outputs and do not change (no arithmetic, summation order or WMMA accumulation order is touched). Checks (RX 9070 XT, gfx1201, Wine + vkd3d-proton + RADV, synthetic Model K): - Byte identity against the base build's output: all K buffers and the filter output, 85 / 51 / 51 files at 200x120, 1279x721 and 2560x1440, equal. - Create-time softmax self-check passes on the new kernels (smx=mix split=m, no fallback). A deliberately wrong mask (~31u) is refused at create. - No-card gfx1201: instructions -10.6 % (d1), -13.8 % (e1), -12 to -16 % (e2, d2, e0, d0_hp); k_stage2_d1's 576 global_load_b32 become 144 global_load_b128; VGPRs unchanged (240 / 192), no spills. - Timing (profiled GPU dispatch times), 4 interleaved rounds, all 240 timed frames per arm passed the 3 % input-duration filter; busy-sample shader clock median 2971-2974 MHz, no throttle; the card on an overclocked profile (core +150 MHz, -50 mV, 374 W cap): frame sum of kernels 1540.1 -> 1515.8 us (-24.3 us, -1.6 %); k_stage2_d1 -7.9, d0_hp -6.1, e2 -4.9, d2 -4.3, e0 -3.0 us, e1 +0.1 us (unchanged). - test_k_smx_selfcheck_digest, test_kernel_sets, test_k_wide_ng pass; line-endings check and git diff --check clean. Not tested: the native Windows AMD driver, and RDNA 3 (gfx1100) on the card; the no-card gfx1100 compile of these kernels fails here, so the effect there is unmeasured. Signed-off-by: Jonathan Shin <85975340+j0nsh1n@users.noreply.github.com> --- clean/k/dx12/hlsl/k_block.hlsli | 6 +++--- clean/k/dx12/hlsl/k_common.hlsli | 10 +++++++++- clean/k/dx12/kernels/k_block.hlsl | 1 + clean/k/runtime/k_smx_selfcheck.h | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/clean/k/dx12/hlsl/k_block.hlsli b/clean/k/dx12/hlsl/k_block.hlsli index e5c08c25..ecf3e5b4 100644 --- a/clean/k/dx12/hlsl/k_block.hlsli +++ b/clean/k/dx12/hlsl/k_block.hlsli @@ -165,8 +165,8 @@ static const int kb_head_row[40] = { // Rows 8*hh through 8*hh+7 of the FP32 bias tile. v8f kb_bias32(uint base, uint off_f) { - const uint4 lo = wg_ld128(base, off_f * 4u); - const uint4 hi = wg_ld128(base, off_f * 4u + 16u); + const uint4 lo = wgk_ld128_a16(base, off_f * 4u); + const uint4 hi = wgk_ld128_a16(base, off_f * 4u + 16u); v8f r; r.v[0] = asfloat(lo.x); r.v[1] = asfloat(lo.y); @@ -184,7 +184,7 @@ v8f kb_bias32(uint base, uint off_f) #if KB_BASE_FORM uint4 kb_weight128(uint base, uint off) { - return wg_ld128(base, off); + return wgk_ld128_a16(base, off); } #endif diff --git a/clean/k/dx12/hlsl/k_common.hlsli b/clean/k/dx12/hlsl/k_common.hlsli index eacbe45b..9cffe3b6 100644 --- a/clean/k/dx12/hlsl/k_common.hlsli +++ b/clean/k/dx12/hlsl/k_common.hlsli @@ -51,7 +51,15 @@ uint wgk_pin(uint v) { return WaveActiveBitOr(v & wgk_zero()); } // ---- 16-byte fragments ----------------------------------------------------------------------------------------------- // Global: base pointer (arena byte offset) + offset in halfs; 16-byte aligned. LDS: kb_lds (the kernel file declares // groupshared uint4 kb_lds[]) indexed in halfs (multiples of 8). -uint4 wgk_ld8(uint base, uint off_h) { return wg_ld128(base, off_h * 2u); } +// KB_LD_A16 (k_block.hlsl): the 16-byte loads of the narrow stage kernels with their alignment made visible to the driver +// (Load4 is known 4-byte aligned only and compiles to four dword loads). Every base is a 256-byte aligned runtime +// allocation (k_pack.cpp kAlign, the arena's own alignment) and every offset a multiple of 16 bytes, so clearing the +// address's low 4 bits changes no address. +#ifndef KB_LD_A16 +#define KB_LD_A16 0 +#endif +uint4 wgk_ld128_a16(uint base, uint off) { return wg_mem.Load4((base + off) & (KB_LD_A16 ? ~15u : ~0u)); } +uint4 wgk_ld8(uint base, uint off_h) { return wgk_ld128_a16(base, off_h * 2u); } void wgk_st8(uint base, uint off_h, uint4 v) { wg_st128(base, off_h * 2u, v); } uint4 wgk_zero8() { return uint4(0u, 0u, 0u, 0u); } v8f wgk_zero8f() { return wg_splat(0.0f); } diff --git a/clean/k/dx12/kernels/k_block.hlsl b/clean/k/dx12/kernels/k_block.hlsl index 33484bde..f70f0c5b 100644 --- a/clean/k/dx12/kernels/k_block.hlsl +++ b/clean/k/dx12/kernels/k_block.hlsl @@ -36,6 +36,7 @@ // @wg-kernel k_stage_e2__rodd co=k_block.co entry=k_stage_e2 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1 // @wg-kernel k_stage_d2__rodd co=k_block.co entry=k_stage_d2 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1 // @wg-kernel k_stage2_d1__rodd co=k_block.co entry=k_stage2_d1 block=128 wave=32 args=s184:p19.b32,p defines=K_SMX=1,KB_BIAS32=1,KB_BASE_FORM=1 +#define KB_LD_A16 1 // 16-byte aligned loads (hlsl/k_common.hlsli) #include "wg_platform.hlsli" #include "k_blkparams.hlsli" diff --git a/clean/k/runtime/k_smx_selfcheck.h b/clean/k/runtime/k_smx_selfcheck.h index 788f8302..3d67b91a 100644 --- a/clean/k/runtime/k_smx_selfcheck.h +++ b/clean/k/runtime/k_smx_selfcheck.h @@ -8,7 +8,7 @@ namespace k_smx_selfcheck { -constexpr char kSourceDigest[] = "a47546460b78b46b3ae0775661c6c4160ad20af09b6106c16322f8d164136264"; +constexpr char kSourceDigest[] = "08a22a023e75550a81652f66b3b12018a03b2e29b0ee6d5ac9d296a7b75f6ed3"; constexpr int kCount = 133; constexpr const char* kNames[kCount] = { "k_stage_e2@512x384/o4,4",