diff --git a/clean/k/dx12/hlsl/k_wide.hlsli b/clean/k/dx12/hlsl/k_wide.hlsli index 3b8b0b3a..aa6cac1d 100644 --- a/clean/k/dx12/hlsl/k_wide.hlsli +++ b/clean/k/dx12/hlsl/k_wide.hlsli @@ -154,7 +154,14 @@ uint4 kw_pin(uint4 v) } // global fragments: pointer (arena byte offset) + element (half) offset -uint4 kw_ldg(uint p, uint e) { return wg_ld128(p, e * 2u); } +// KW_LD_A16 (set by the wide-stage kernel files): 16-byte loads with the alignment made visible to the driver. Load4 is known +// 4-byte aligned only and compiles to four dword loads; every base is a 256-byte aligned runtime allocation (k_pack.cpp kAlign) and +// every offset a multiple of 16 bytes (HIP's own uint4 loads), so clearing the low 4 address bits changes no address. +#ifndef KW_LD_A16 +#define KW_LD_A16 0 +#endif +uint4 kw_ld128(uint base, uint off) { return wg_mem.Load4((base + off) & (KW_LD_A16 ? ~15u : ~0u)); } +uint4 kw_ldg(uint p, uint e) { return kw_ld128(p, e * 2u); } void kw_stg(uint p, uint e, uint4 v) { wg_st128(p, e * 2u, v); } // Mirror about the edge token, edge not repeated (spec 41; k_common.h wgk_mirror), branch free. diff --git a/clean/k/dx12/kernels/k_block_split.hlsl b/clean/k/dx12/kernels/k_block_split.hlsl index 2387060e..ccef0bd1 100644 --- a/clean/k/dx12/kernels/k_block_split.hlsl +++ b/clean/k/dx12/kernels/k_block_split.hlsl @@ -28,6 +28,7 @@ // @wg-kernel k_split_m_wo co=k_block_wide.co block=128 wave=32 args=s184:p19.b32,p,p,p,p entry=ks_wo defines=KS_PHASE=2,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4 // @wg-kernel k_split_m_w1 co=k_block_wide.co block=128 wave=32 args=s184:p19.b32,p,p,p,p entry=ks_w1 defines=KS_PHASE=3,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4 // @wg-kernel k_split_m_w2 co=k_block_wide.co block=128 wave=32 args=s184:p19.b32,p,p,p,p entry=ks_w2 defines=KS_PHASE=4,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4 +#define KW_LD_A16 1 // 16-byte aligned loads (hlsl/k_wide.hlsli) #include "wg_platform.hlsli" WG_ARGS_BEGIN @@ -111,7 +112,7 @@ uint ks_vidx(int row, int c) { return (uint)(256 + row * 8 + (c ^ (row & 7))) & static uint ksA_qkv; // product s = (ct KT + kt) 3 + {q, k, v} of the head's q / k / v chains (k_block_wide.hlsl's stream order of a round) -uint4 ks_wq(int s) { return wg_ld128(ksA_qkv, (uint)(((s % 3) * 2 * KT + s / 3) * 512)); } +uint4 ks_wq(int s) { return kw_ld128(ksA_qkv, (uint)(((s % 3) * 2 * KT + s / 3) * 512)); } #define KS_TAKE(s) rg[(s) % KS_RD] #define KS_NEXT(s) { if ((s) + KS_RD < SA) rg[(s) % KS_RD] = ks_wq((s) + KS_RD); } @@ -236,7 +237,7 @@ WG_KERNEL(ks_wo) const uint zr = ks_zrow(win, L.t, hh); uint4 wf[DT], zf[DT]; [unroll] for (int d0 = 0; d0 < DT; ++d0) { - wf[d0] = wg_ld128(wo, (uint)(d0 * 512)); + wf[d0] = kw_ld128(wo, (uint)(d0 * 512)); zf[d0] = kw_ldg(P_sz, zr + 16u * d0); } const uint4 xr = kw_ldg(P_x, L.mtok * KW_C + 8u * hh + 16u * ot); @@ -268,7 +269,7 @@ WG_KERNEL(ks_w1) uint4 a[KT]; uint4 wf[KS_HB][KT]; [unroll] for (int kt0 = 0; kt0 < KT; ++kt0) { - [unroll] for (int m0 = 0; m0 < KS_HB; ++m0) wf[m0][kt0] = wg_ld128(w1, (uint)((m0 * KT + kt0) * 512)); + [unroll] for (int m0 = 0; m0 < KS_HB; ++m0) wf[m0][kt0] = kw_ld128(w1, (uint)((m0 * KT + kt0) * 512)); a[kt0] = kw_ldg(P_sa, cr + 16u * kt0); } uint4 b1v[KS_HB]; @@ -312,8 +313,8 @@ WG_KERNEL(ks_w2) [unroll] for (int b = 0; b < NB2; ++b) { // the hidden tiles 0 .. HT - 1, in order uint4 rw[KS_B2], rh[KS_B2]; [unroll] for (int j0 = 0; j0 < KS_B2; ++j0) { - rw[j0] = wg_ld128(w2, (uint)((b * KS_B2 + j0) * KT * 512)); - rh[j0] = wg_ld128(hb, (uint)((b * KS_B2 + j0) * 32)); + rw[j0] = kw_ld128(w2, (uint)((b * KS_B2 + j0) * KT * 512)); + rh[j0] = kw_ld128(hb, (uint)((b * KS_B2 + j0) * 32)); } uint f = b == 0 ? b2v.x ^ x1.x : 0u; [unroll] for (int j1 = 0; j1 < KS_B2; ++j1) f ^= rw[j1].x ^ rh[j1].x; diff --git a/clean/k/dx12/kernels/k_block_wide.hlsl b/clean/k/dx12/kernels/k_block_wide.hlsl index 0303fd57..64793be0 100644 --- a/clean/k/dx12/kernels/k_block_wide.hlsl +++ b/clean/k/dx12/kernels/k_block_wide.hlsl @@ -20,6 +20,7 @@ // @wg-kernel k_stagew4_d4__rodd co=k_block_wide.co block=512 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=128,KW_HEADS=4,KW_IN=1,KW_CIN=160,KW_CNEXT=0,KW_NG=4,KW_RD=12 // @wg-kernel k_stagew2_d4__rodd co=k_block_wide.co block=256 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=128,KW_HEADS=4,KW_IN=1,KW_CIN=160,KW_CNEXT=0,KW_NG=2 // @wg-kernel k_stagew2_d3__rodd co=k_block_wide.co block=256 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=96,KW_HEADS=4,KW_IN=1,KW_CIN=128,KW_CNEXT=0,KW_NG=2,KW_RD=4 +#define KW_LD_A16 1 // 16-byte aligned loads (hlsl/k_wide.hlsli) #include "wg_platform.hlsli" WG_ARGS_BEGIN @@ -76,29 +77,29 @@ uint4 kw_wld(int s) { if (s < B_R) { // expand: product s = kt NK + k (tile wave + k NW) const int kt = s / max(NK, 1), k = s % max(NK, 1); - return wg_ld128(kwA_wa, (uint)((k * NW * KTI + kt) * 512)); + return kw_ld128(kwA_wa, (uint)((k * NW * KTI + kt) * 512)); } if (s < B_C) { const int r = (s - B_R) / RLEN, f = (s - B_R) % RLEN; if (f < SQ) // f = (ct KT + kt) 3 + {q, k, v}; head r NG + grp - return wg_ld128(kwA_qkv, (uint)((r * NG * 3 * 2 * KT + (f % 3) * 2 * KT + f / 3) * 512)); + return kw_ld128(kwA_qkv, (uint)((r * NG * 3 * 2 * KT + (f % 3) * 2 * KT + f / 3) * 512)); const int f2 = f - SQ; // f2 = j NOWN + i: z tile 2 r NG + j, owned tile i if ((KT % NG) == 0) // every owned tile present: grp + NG i, one address - return wg_ld128(kwA_wo[0], (uint)(((f2 % NOWN) * NG * DT + 2 * r * NG + f2 / NOWN) * 512)); - return wg_ld128(kwA_wo[f2 % NOWN], (uint)((2 * r * NG + f2 / NOWN) * 512)); + return kw_ld128(kwA_wo[0], (uint)(((f2 % NOWN) * NG * DT + 2 * r * NG + f2 / NOWN) * 512)); + return kw_ld128(kwA_wo[f2 % NOWN], (uint)((2 * r * NG + f2 / NOWN) * 512)); } if (s < B_M) { const int c = (s - B_C) / CLEN, f = (s - B_C) % CLEN; if (f < S1) { // hidden tile c CH + grp MM + m const int m = KW_ILV ? f % MM : f / KT, kt = KW_ILV ? f / MM : f % KT; - return wg_ld128(kwA_w1, (uint)(((c * CH + m) * KT + kt) * 512)); + return kw_ld128(kwA_w1, (uint)(((c * CH + m) * KT + kt) * 512)); } const int f2 = f - S1; // f2 = j NOWN + i: hidden tile c CH + j, owned tile i if ((KT % NG) == 0) - return wg_ld128(kwA_w2[0], (uint)(((c * CH + f2 / NOWN) * KT + (f2 % NOWN) * NG) * 512)); - return wg_ld128(kwA_w2[f2 % NOWN], (uint)((c * CH + f2 / NOWN) * KT * 512)); + return kw_ld128(kwA_w2[0], (uint)(((c * CH + f2 / NOWN) * KT + (f2 % NOWN) * NG) * 512)); + return kw_ld128(kwA_w2[f2 % NOWN], (uint)((c * CH + f2 / NOWN) * KT * 512)); } - return wg_ld128(kwA_wz, (uint)((s - B_M) * 512)); // merge: tile min(wave, NTM - 1), kt = s - B_M + return kw_ld128(kwA_wz, (uint)((s - B_M) * 512)); // merge: tile min(wave, NTM - 1), kt = s - B_M } // product s of the stream: its fragment, and the refill of its ring slot with product s + KW_RD diff --git a/clean/k/runtime/k_smx_selfcheck.h b/clean/k/runtime/k_smx_selfcheck.h index 74309985..c6085970 100644 --- a/clean/k/runtime/k_smx_selfcheck.h +++ b/clean/k/runtime/k_smx_selfcheck.h @@ -8,7 +8,7 @@ namespace k_smx_selfcheck { -constexpr char kSourceDigest[] = "f6ae0495ebdb3b688048890318dce0dd25231dca07406b8247532eab10e81fb1"; +constexpr char kSourceDigest[] = "c7a083f24edaa314370dd3d27ce234398ceae9148942e6ee9d3bb9506d72d474"; constexpr int kCount = 133; constexpr const char* kNames[kCount] = { "k_stage_e2@512x384/o4,4",