diff --git a/clean/dx12/hlsl/wg_platform.hlsli b/clean/dx12/hlsl/wg_platform.hlsli index 65b6eee5..4a6b1f36 100644 --- a/clean/dx12/hlsl/wg_platform.hlsli +++ b/clean/dx12/hlsl/wg_platform.hlsli @@ -68,10 +68,16 @@ uint wg_ld8(uint base, uint off) { uint a = base + off; return (wg_mem.Load(a & uint wg_ld16(uint base, uint off) { return (uint)wg_mem.Load(base + off); } uint wg_ld32(uint base, uint off) { return wg_mem.Load(base + off); } uint2 wg_ld64(uint base, uint off) { return wg_mem.Load2(base + off); } -uint4 wg_ld128(uint base, uint off) { return wg_mem.Load4(base + off); } +// WG_LD_A16 (kernel's defines=): 16-byte loads with the alignment made visible to the driver. Load4 is known 4-byte aligned +// only and compiles to four dword loads; a kernel that sets it guarantees that every base + offset of its wg_ld128 / +// wg_ld4f is a multiple of 16 bytes (arena allocations are 256-byte aligned, the packed sections are 16-byte multiples). +#ifndef WG_LD_A16 +#define WG_LD_A16 0 +#endif +uint4 wg_ld128(uint base, uint off) { return wg_mem.Load4((base + off) & (WG_LD_A16 ? ~15u : ~0u)); } uint4 wg_ld128_a16(uint base, uint off) { return wg_mem.Load4(base + off); } float wg_ldf(uint base, uint off) { return asfloat(wg_mem.Load(base + off)); } -float4 wg_ld4f(uint base, uint off) { return asfloat(wg_mem.Load4(base + off)); } +float4 wg_ld4f(uint base, uint off) { return asfloat(wg_mem.Load4((base + off) & (WG_LD_A16 ? ~15u : ~0u))); } void wg_st16(uint base, uint off, uint v) { wg_mem.Store(base + off, (uint16_t)v); } void wg_st32(uint base, uint off, uint v) { wg_mem.Store(base + off, v); } void wg_st64(uint base, uint off, uint2 v) { wg_mem.Store2(base + off, v); } diff --git a/clean/dx12/kernels/dec1_dec0.hlsl b/clean/dx12/kernels/dec1_dec0.hlsl index 909782b7..e08e1709 100644 --- a/clean/dx12/kernels/dec1_dec0.hlsl +++ b/clean/dx12/kernels/dec1_dec0.hlsl @@ -21,7 +21,7 @@ // both pixels of every lane and only the stores are conditional). dec0's FP16 latent update is done in FP32 with an FP16 // rounding after each step, not in half arithmetic (the driver contracts half mul + add into v_fma_mix_f32). // Bit-identical to the HIP kernel on the synthetic frames (clean/dx12/compare.py, both cases, every buffer). -// @wg-kernel d1p12/swin_block_c64_add_dec0_nc co=dec1_dec0__pack12.co block=256 wave=32 args=p,p,p,u,u,u,u,p,p,p,p,p,p,p,b56 defines=WG_GELU_T5=0 +// @wg-kernel d1p12/swin_block_c64_add_dec0_nc co=dec1_dec0__pack12.co block=256 wave=32 args=p,p,p,u,u,u,u,p,p,p,p,p,p,p,b56 defines=WG_GELU_T5=0,WG_LD_A16=1 // // DEC0_ONLY (port d1p12/dec0_q7; Model M on the FP16-form shared stages, WG_M_FORM=q7, where dec1 runs as the q7 block // l_q7_block_c64_dec and stores its map): the same kernel without the block. The epilogue's input o[0..3] (lane (t, hh): diff --git a/clean/dx12/kernels/swin_c64w2.hlsl b/clean/dx12/kernels/swin_c64w2.hlsl index 4ba1a5ae..1769335a 100644 --- a/clean/dx12/kernels/swin_c64w2.hlsl +++ b/clean/dx12/kernels/swin_c64w2.hlsl @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // swin_block_c64_nc (enc1's Swin block) with TWO windows per wave in 128-lane workgroups (tracks lev3-swin, r3-c64w2). // -// @wg-kernel swin_block_c64_nc co=swin.co block=256 group=128 gscale=1 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=64,SW_HEADS=2,SW_MLP=256,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_W2=1 +// @wg-kernel swin_block_c64_nc co=swin.co block=256 group=128 gscale=1 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=64,SW_HEADS=2,SW_MLP=256,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_W2=1,WG_LD_A16=1 // Model L's dec1 block swin_block_c64_add_ex_nc (swin_c64ex.hlsl) runs this kernel too (SW_W2_NAME), with the block // input x + xadd (SW_ADD) and the fused expand (SW_ECL: swin_block.hlsl's fused_expand with the 4 waves; each wave then // reads its two windows' records). diff --git a/clean/k/runtime/k_smx_selfcheck.h b/clean/k/runtime/k_smx_selfcheck.h index 3d67b91a..74309985 100644 --- a/clean/k/runtime/k_smx_selfcheck.h +++ b/clean/k/runtime/k_smx_selfcheck.h @@ -8,7 +8,7 @@ namespace k_smx_selfcheck { -constexpr char kSourceDigest[] = "08a22a023e75550a81652f66b3b12018a03b2e29b0ee6d5ac9d296a7b75f6ed3"; +constexpr char kSourceDigest[] = "f6ae0495ebdb3b688048890318dce0dd25231dca07406b8247532eab10e81fb1"; constexpr int kCount = 133; constexpr const char* kNames[kCount] = { "k_stage_e2@512x384/o4,4",