Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion clean/k/dx12/hlsl/k_wide.hlsli
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,14 @@ uint4 kw_pin(uint4 v)
}

// global fragments: pointer (arena byte offset) + element (half) offset
uint4 kw_ldg(uint p, uint e) { return wg_ld128(p, e * 2u); }
// KW_LD_A16 (set by the wide-stage kernel files): 16-byte loads with the alignment made visible to the driver. Load4 is known
// 4-byte aligned only and compiles to four dword loads; every base is a 256-byte aligned runtime allocation (k_pack.cpp kAlign) and
// every offset a multiple of 16 bytes (HIP's own uint4 loads), so clearing the low 4 address bits changes no address.
#ifndef KW_LD_A16
#define KW_LD_A16 0
#endif
uint4 kw_ld128(uint base, uint off) { return wg_mem.Load4((base + off) & (KW_LD_A16 ? ~15u : ~0u)); }
uint4 kw_ldg(uint p, uint e) { return kw_ld128(p, e * 2u); }
void kw_stg(uint p, uint e, uint4 v) { wg_st128(p, e * 2u, v); }

// Mirror about the edge token, edge not repeated (spec 41; k_common.h wgk_mirror), branch free.
Expand Down
11 changes: 6 additions & 5 deletions clean/k/dx12/kernels/k_block_split.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
// @wg-kernel k_split_m_wo co=k_block_wide.co block=128 wave=32 args=s184:p19.b32,p,p,p,p entry=ks_wo defines=KS_PHASE=2,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4
// @wg-kernel k_split_m_w1 co=k_block_wide.co block=128 wave=32 args=s184:p19.b32,p,p,p,p entry=ks_w1 defines=KS_PHASE=3,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4
// @wg-kernel k_split_m_w2 co=k_block_wide.co block=128 wave=32 args=s184:p19.b32,p,p,p,p entry=ks_w2 defines=KS_PHASE=4,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4
#define KW_LD_A16 1 // 16-byte aligned loads (hlsl/k_wide.hlsli)
#include "wg_platform.hlsli"

WG_ARGS_BEGIN
Expand Down Expand Up @@ -111,7 +112,7 @@ uint ks_vidx(int row, int c) { return (uint)(256 + row * 8 + (c ^ (row & 7))) &

static uint ksA_qkv;
// product s = (ct KT + kt) 3 + {q, k, v} of the head's q / k / v chains (k_block_wide.hlsl's stream order of a round)
uint4 ks_wq(int s) { return wg_ld128(ksA_qkv, (uint)(((s % 3) * 2 * KT + s / 3) * 512)); }
uint4 ks_wq(int s) { return kw_ld128(ksA_qkv, (uint)(((s % 3) * 2 * KT + s / 3) * 512)); }
#define KS_TAKE(s) rg[(s) % KS_RD]
#define KS_NEXT(s) { if ((s) + KS_RD < SA) rg[(s) % KS_RD] = ks_wq((s) + KS_RD); }

Expand Down Expand Up @@ -236,7 +237,7 @@ WG_KERNEL(ks_wo)
const uint zr = ks_zrow(win, L.t, hh);
uint4 wf[DT], zf[DT];
[unroll] for (int d0 = 0; d0 < DT; ++d0) {
wf[d0] = wg_ld128(wo, (uint)(d0 * 512));
wf[d0] = kw_ld128(wo, (uint)(d0 * 512));
zf[d0] = kw_ldg(P_sz, zr + 16u * d0);
}
const uint4 xr = kw_ldg(P_x, L.mtok * KW_C + 8u * hh + 16u * ot);
Expand Down Expand Up @@ -268,7 +269,7 @@ WG_KERNEL(ks_w1)
uint4 a[KT];
uint4 wf[KS_HB][KT];
[unroll] for (int kt0 = 0; kt0 < KT; ++kt0) {
[unroll] for (int m0 = 0; m0 < KS_HB; ++m0) wf[m0][kt0] = wg_ld128(w1, (uint)((m0 * KT + kt0) * 512));
[unroll] for (int m0 = 0; m0 < KS_HB; ++m0) wf[m0][kt0] = kw_ld128(w1, (uint)((m0 * KT + kt0) * 512));
a[kt0] = kw_ldg(P_sa, cr + 16u * kt0);
}
uint4 b1v[KS_HB];
Expand Down Expand Up @@ -312,8 +313,8 @@ WG_KERNEL(ks_w2)
[unroll] for (int b = 0; b < NB2; ++b) { // the hidden tiles 0 .. HT - 1, in order
uint4 rw[KS_B2], rh[KS_B2];
[unroll] for (int j0 = 0; j0 < KS_B2; ++j0) {
rw[j0] = wg_ld128(w2, (uint)((b * KS_B2 + j0) * KT * 512));
rh[j0] = wg_ld128(hb, (uint)((b * KS_B2 + j0) * 32));
rw[j0] = kw_ld128(w2, (uint)((b * KS_B2 + j0) * KT * 512));
rh[j0] = kw_ld128(hb, (uint)((b * KS_B2 + j0) * 32));
}
uint f = b == 0 ? b2v.x ^ x1.x : 0u;
[unroll] for (int j1 = 0; j1 < KS_B2; ++j1) f ^= rw[j1].x ^ rh[j1].x;
Expand Down
17 changes: 9 additions & 8 deletions clean/k/dx12/kernels/k_block_wide.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
// @wg-kernel k_stagew4_d4__rodd co=k_block_wide.co block=512 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=128,KW_HEADS=4,KW_IN=1,KW_CIN=160,KW_CNEXT=0,KW_NG=4,KW_RD=12
// @wg-kernel k_stagew2_d4__rodd co=k_block_wide.co block=256 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=128,KW_HEADS=4,KW_IN=1,KW_CIN=160,KW_CNEXT=0,KW_NG=2
// @wg-kernel k_stagew2_d3__rodd co=k_block_wide.co block=256 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=96,KW_HEADS=4,KW_IN=1,KW_CIN=128,KW_CNEXT=0,KW_NG=2,KW_RD=4
#define KW_LD_A16 1 // 16-byte aligned loads (hlsl/k_wide.hlsli)
#include "wg_platform.hlsli"

WG_ARGS_BEGIN
Expand Down Expand Up @@ -76,29 +77,29 @@ uint4 kw_wld(int s)
{
if (s < B_R) { // expand: product s = kt NK + k (tile wave + k NW)
const int kt = s / max(NK, 1), k = s % max(NK, 1);
return wg_ld128(kwA_wa, (uint)((k * NW * KTI + kt) * 512));
return kw_ld128(kwA_wa, (uint)((k * NW * KTI + kt) * 512));
}
if (s < B_C) {
const int r = (s - B_R) / RLEN, f = (s - B_R) % RLEN;
if (f < SQ) // f = (ct KT + kt) 3 + {q, k, v}; head r NG + grp
return wg_ld128(kwA_qkv, (uint)((r * NG * 3 * 2 * KT + (f % 3) * 2 * KT + f / 3) * 512));
return kw_ld128(kwA_qkv, (uint)((r * NG * 3 * 2 * KT + (f % 3) * 2 * KT + f / 3) * 512));
const int f2 = f - SQ; // f2 = j NOWN + i: z tile 2 r NG + j, owned tile i
if ((KT % NG) == 0) // every owned tile present: grp + NG i, one address
return wg_ld128(kwA_wo[0], (uint)(((f2 % NOWN) * NG * DT + 2 * r * NG + f2 / NOWN) * 512));
return wg_ld128(kwA_wo[f2 % NOWN], (uint)((2 * r * NG + f2 / NOWN) * 512));
return kw_ld128(kwA_wo[0], (uint)(((f2 % NOWN) * NG * DT + 2 * r * NG + f2 / NOWN) * 512));
return kw_ld128(kwA_wo[f2 % NOWN], (uint)((2 * r * NG + f2 / NOWN) * 512));
}
if (s < B_M) {
const int c = (s - B_C) / CLEN, f = (s - B_C) % CLEN;
if (f < S1) { // hidden tile c CH + grp MM + m
const int m = KW_ILV ? f % MM : f / KT, kt = KW_ILV ? f / MM : f % KT;
return wg_ld128(kwA_w1, (uint)(((c * CH + m) * KT + kt) * 512));
return kw_ld128(kwA_w1, (uint)(((c * CH + m) * KT + kt) * 512));
}
const int f2 = f - S1; // f2 = j NOWN + i: hidden tile c CH + j, owned tile i
if ((KT % NG) == 0)
return wg_ld128(kwA_w2[0], (uint)(((c * CH + f2 / NOWN) * KT + (f2 % NOWN) * NG) * 512));
return wg_ld128(kwA_w2[f2 % NOWN], (uint)((c * CH + f2 / NOWN) * KT * 512));
return kw_ld128(kwA_w2[0], (uint)(((c * CH + f2 / NOWN) * KT + (f2 % NOWN) * NG) * 512));
return kw_ld128(kwA_w2[f2 % NOWN], (uint)((c * CH + f2 / NOWN) * KT * 512));
}
return wg_ld128(kwA_wz, (uint)((s - B_M) * 512)); // merge: tile min(wave, NTM - 1), kt = s - B_M
return kw_ld128(kwA_wz, (uint)((s - B_M) * 512)); // merge: tile min(wave, NTM - 1), kt = s - B_M
}

// product s of the stream: its fragment, and the refill of its ring slot with product s + KW_RD
Expand Down
2 changes: 1 addition & 1 deletion clean/k/runtime/k_smx_selfcheck.h
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

namespace k_smx_selfcheck {

constexpr char kSourceDigest[] = "f6ae0495ebdb3b688048890318dce0dd25231dca07406b8247532eab10e81fb1";
constexpr char kSourceDigest[] = "c7a083f24edaa314370dd3d27ce234398ceae9148942e6ee9d3bb9506d72d474";
constexpr int kCount = 133;
constexpr const char* kNames[kCount] = {
"k_stage_e2@512x384/o4,4",
Expand Down
Loading