diff --git a/clean/k/dx12/hlsl/k_block.hlsli b/clean/k/dx12/hlsl/k_block.hlsli index e5c08c25..ecf3e5b4 100644 --- a/clean/k/dx12/hlsl/k_block.hlsli +++ b/clean/k/dx12/hlsl/k_block.hlsli @@ -165,8 +165,8 @@ static const int kb_head_row[40] = { // Rows 8*hh through 8*hh+7 of the FP32 bias tile. v8f kb_bias32(uint base, uint off_f) { - const uint4 lo = wg_ld128(base, off_f * 4u); - const uint4 hi = wg_ld128(base, off_f * 4u + 16u); + const uint4 lo = wgk_ld128_a16(base, off_f * 4u); + const uint4 hi = wgk_ld128_a16(base, off_f * 4u + 16u); v8f r; r.v[0] = asfloat(lo.x); r.v[1] = asfloat(lo.y); @@ -184,7 +184,7 @@ v8f kb_bias32(uint base, uint off_f) #if KB_BASE_FORM uint4 kb_weight128(uint base, uint off) { - return wg_ld128(base, off); + return wgk_ld128_a16(base, off); } #endif diff --git a/clean/k/dx12/hlsl/k_common.hlsli b/clean/k/dx12/hlsl/k_common.hlsli index eacbe45b..9cffe3b6 100644 --- a/clean/k/dx12/hlsl/k_common.hlsli +++ b/clean/k/dx12/hlsl/k_common.hlsli @@ -51,7 +51,15 @@ uint wgk_pin(uint v) { return WaveActiveBitOr(v & wgk_zero()); } // ---- 16-byte fragments ----------------------------------------------------------------------------------------------- // Global: base pointer (arena byte offset) + offset in halfs; 16-byte aligned. LDS: kb_lds (the kernel file declares // groupshared uint4 kb_lds[]) indexed in halfs (multiples of 8). -uint4 wgk_ld8(uint base, uint off_h) { return wg_ld128(base, off_h * 2u); } +// KB_LD_A16 (k_block.hlsl): the 16-byte loads of the narrow stage kernels with their alignment made visible to the driver +// (Load4 is known 4-byte aligned only and compiles to four dword loads). Every base is a 256-byte aligned runtime +// allocation (k_pack.cpp kAlign, the arena's own alignment) and every offset a multiple of 16 bytes, so clearing the +// address's low 4 bits changes no address. +#ifndef KB_LD_A16 +#define KB_LD_A16 0 +#endif +uint4 wgk_ld128_a16(uint base, uint off) { return wg_mem.Load4((base + off) & (KB_LD_A16 ? ~15u : ~0u)); } +uint4 wgk_ld8(uint base, uint off_h) { return wgk_ld128_a16(base, off_h * 2u); } void wgk_st8(uint base, uint off_h, uint4 v) { wg_st128(base, off_h * 2u, v); } uint4 wgk_zero8() { return uint4(0u, 0u, 0u, 0u); } v8f wgk_zero8f() { return wg_splat(0.0f); } diff --git a/clean/k/dx12/kernels/k_block.hlsl b/clean/k/dx12/kernels/k_block.hlsl index 33484bde..f70f0c5b 100644 --- a/clean/k/dx12/kernels/k_block.hlsl +++ b/clean/k/dx12/kernels/k_block.hlsl @@ -36,6 +36,7 @@ // @wg-kernel k_stage_e2__rodd co=k_block.co entry=k_stage_e2 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1 // @wg-kernel k_stage_d2__rodd co=k_block.co entry=k_stage_d2 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1 // @wg-kernel k_stage2_d1__rodd co=k_block.co entry=k_stage2_d1 block=128 wave=32 args=s184:p19.b32,p defines=K_SMX=1,KB_BIAS32=1,KB_BASE_FORM=1 +#define KB_LD_A16 1 // 16-byte aligned loads (hlsl/k_common.hlsli) #include "wg_platform.hlsli" #include "k_blkparams.hlsli" diff --git a/clean/k/runtime/k_smx_selfcheck.h b/clean/k/runtime/k_smx_selfcheck.h index 788f8302..3d67b91a 100644 --- a/clean/k/runtime/k_smx_selfcheck.h +++ b/clean/k/runtime/k_smx_selfcheck.h @@ -8,7 +8,7 @@ namespace k_smx_selfcheck { -constexpr char kSourceDigest[] = "a47546460b78b46b3ae0775661c6c4160ad20af09b6106c16322f8d164136264"; +constexpr char kSourceDigest[] = "08a22a023e75550a81652f66b3b12018a03b2e29b0ee6d5ac9d296a7b75f6ed3"; constexpr int kCount = 133; constexpr const char* kNames[kCount] = { "k_stage_e2@512x384/o4,4",