From c42f5459484cb819ad85ba5d3f13fc3c6171856f Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 19:21:27 -0500 Subject: [PATCH 1/2] Model K: streamline narrow stage operands and compact head Use prewidened FP32 bias planes and folded weight offsets in the two-window narrow kernels. Pack the 32 live compact-head rows into two WMMA tiles. Rebind the K softmax source digest after native D3D12 backend 3 k_smx_test=report reproduced all 133 stored hashes in each of the mix and round-to-odd forms with zero mismatches. Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/k/dx12/hlsl/k_block.hlsli | 228 ++++++++++++++++++++++++++++-- clean/k/dx12/hlsl/k_head_rows.inc | 6 + clean/k/dx12/kernels/k_block.hlsl | 29 +++- clean/k/runtime/k_pack.cpp | 22 +++ clean/k/runtime/k_pack.h | 4 + clean/k/runtime/k_smx_selfcheck.h | 2 +- clean/k/runtime/runtime_k.cpp | 85 ++++++++++- clean/k/runtime/runtime_k.h | 2 + 8 files changed, 353 insertions(+), 25 deletions(-) create mode 100644 clean/k/dx12/hlsl/k_head_rows.inc diff --git a/clean/k/dx12/hlsl/k_block.hlsli b/clean/k/dx12/hlsl/k_block.hlsli index 473877f2..0d7b4003 100644 --- a/clean/k/dx12/hlsl/k_block.hlsli +++ b/clean/k/dx12/hlsl/k_block.hlsli @@ -153,6 +153,50 @@ void wgk_store_hidden(uint base, uint off_h, float h[4]) wg_st64(base, off_h * 2u, uint2(wg_f2h(pr[0]) | (wg_f2h(pr[1]) << 16), wg_f2h(pr[2]) | (wg_f2h(pr[3]) << 16))); } +#ifndef KB_BIAS32 +#define KB_BIAS32 0 +#endif + +#if KB_BIAS32 +static const int kb_head_row[40] = { +#include "k_head_rows.inc" +}; + +// Rows 8*hh through 8*hh+7 of the FP32 bias tile. +v8f kb_bias32(uint base, uint off_f) +{ + const uint4 lo = wg_ld128(base, off_f * 4u); + const uint4 hi = wg_ld128(base, off_f * 4u + 16u); + v8f r; + r.v[0] = asfloat(lo.x); + r.v[1] = asfloat(lo.y); + r.v[2] = asfloat(lo.z); + r.v[3] = asfloat(lo.w); + r.v[4] = asfloat(hi.x); + r.v[5] = asfloat(hi.y); + r.v[6] = asfloat(hi.z); + r.v[7] = asfloat(hi.w); + return r; +} + +// Softmax starts with the bias row of this query, without the lane-half bias base. +v8f kb_smx32(uint base, uint off_f) +{ + return kb_bias32(base, off_f); +} +#endif + +#if KB_BASE_FORM +uint4 kb_weight128(uint base, uint off) +{ +#if KB_BASE_FORM == 2 + return wg_ld128(base + off, 0u); +#else + return wg_ld128(base, off); +#endif +} +#endif + // ---- the block ---------------------------------------------------------------------------------------------------- // Template parameters as k_block.hip (F16 = 0): C channels, HEADS, MIX (KMIX_*), IN (KIN_MAP / KIN_EMBED / KIN_EXPAND; // KIN_PADDED as well), CIN input channels of EMBED / EXPAND, CNEXT merge output channels (0: none), HEAD (32 -> 40 output @@ -160,7 +204,7 @@ void wgk_store_hidden(uint base, uint off_h, float h[4]) // pixel head), XG (decoder: x = f16(e + S) through P.y instead of LDS). blk: the workgroup's window group (windows // G blk .. G blk + G - 1): wg_block_id(). template -void k_block(uint hidden, uint blk) +void k_block(uint hidden, uint blk, uint bias32 = 0u) { const int KT = C / 16; const int DT = HEADS * 2; @@ -187,6 +231,22 @@ void k_block(uint hidden, uint blk) const int n16 = (int)(lane & 15u), hh = (int)(lane >> 4); const int nwin = P_nwx * P_nwy; const int t = 16 * (int)wave + n16; +#if KB_BIAS32 + const int OFF_BA = 0, OFF_BO = IN == KIN_EMBED ? C : (IN == KIN_EXPAND ? 4 * C : 0); + const int OFF_B1 = OFF_BO + C, OFF_B2 = OFF_B1 + 4 * C, OFF_BZ = OFF_B2 + C; + const int OFF_SMX = OFF_BZ + (HEAD ? 48 : CNEXT); + const uint kb_b32base = (bias32 + (uint)hh * 32u) ^ wgk_zero(); +#endif +#if KB_BASE_FORM + const uint weight_lane = lane * 16u; + const uint wb_wa = (P_wa + weight_lane) ^ wgk_zero(); + const uint wb_wqkv = (P_wqkv + weight_lane) ^ wgk_zero(); + const uint wb_wo = (P_wo + weight_lane) ^ wgk_zero(); + const uint wb_w1 = (P_w1 + weight_lane) ^ wgk_zero(); + const uint wb_w2 = (P_w2 + weight_lane) ^ wgk_zero(); + const uint wb_wz = (P_wz + weight_lane) ^ wgk_zero(); + const uint wb_mix = (P_mix + (uint)(t * 64 + 8 * hh) * 2u) ^ wgk_zero(); +#endif int wa[G], wb[G], gx[G], gy[G]; bool real[G]; uint mtok[G], xt[G], xl[G], ytok[G]; @@ -212,14 +272,31 @@ void k_block(uint hidden, uint blk) // 1. the block input uint4 n[G][C / 16]; if (IN == KIN_EMBED) { // e(t) = max(f16(We in(t) + be), 0) of the mirrored member - uint4 inp[G], ea[C / 16], eb[C / 16]; + uint4 inp[G], ea[C / 16]; +#if KB_BIAS32 + v8f eb[C / 16]; +#else + uint4 eb[C / 16]; +#endif [unroll] for (int g = 0; g < G; ++g) inp[g] = wgk_ld8(P_x, mtok[g] * 16u + 8u * (uint)hh); [unroll] for (int o0 = 0; o0 < KT; ++o0) { +#if KB_BASE_FORM + ea[o0] = kb_weight128(wb_wa, (uint)(o0 * 512)); +#else ea[o0] = wgk_ld8(P_wa, (uint)(o0 * 256) + lane * 8u); +#endif +#if KB_BIAS32 + eb[o0] = kb_bias32(kb_b32base, OFF_BA + 16 * o0); +#else eb[o0] = wgk_ld8(P_ba, (uint)(16 * o0 + 8 * hh)); +#endif } [unroll] for (int ot = 0; ot < KT; ++ot) { +#if KB_BIAS32 + const v8f b0 = eb[ot]; +#else const v8f b0 = wgk_widen8(eb[ot]); +#endif [unroll] for (int g = 0; g < G; ++g) { n[g][ot] = KB_MAXC8W(wgk_round8(wgk_mma(ea[ot], inp[g], b0)), 0u); kb_stl(xl[g] + (uint)(16 * ot), n[g][ot]); @@ -237,10 +314,20 @@ void k_block(uint hidden, uint blk) } [loop] for (int tt = (int)wave; tt < TOT; tt += 4) { // output tile tt of 4C: sub-position s, channels c0 .. uint4 wt[CIN / 16]; +#if KB_BIAS32 + const v8f b0 = kb_bias32(kb_b32base, OFF_BA + 16 * tt); +#else const uint4 bb = wgk_ld8(P_ba, (uint)(16 * tt + 8 * hh)); +#endif +#if KB_BASE_FORM + [unroll] for (int k1 = 0; k1 < KTI; ++k1) wt[k1] = kb_weight128(wb_wa, (uint)((tt * KTI + k1) * 512)); +#else [unroll] for (int k1 = 0; k1 < KTI; ++k1) wt[k1] = wgk_ld8(P_wa, (uint)((tt * KTI + k1) * 256) + lane * 8u); +#endif v8f e[G]; +#if !KB_BIAS32 const v8f b0 = wgk_widen8(bb); +#endif [unroll] for (int g = 0; g < G; ++g) e[g] = b0; [unroll] for (int kt = 0; kt < KTI; ++kt) [unroll] for (int g = 0; g < G; ++g) e[g] = wgk_mma(wt[kt], yb[g][kt], e[g]); @@ -293,31 +380,51 @@ void k_block(uint hidden, uint blk) // 2. token mixing, head by head; y accumulates the output projection v8f yacc[G][C / 16]; { +#if !KB_BIAS32 uint4 bo[C / 16]; [unroll] for (int o1 = 0; o1 < KT; ++o1) bo[o1] = wgk_ld8(P_bo, (uint)(16 * o1 + 8 * hh)); +#endif [unroll] for (int ot = 0; ot < KT; ++ot) { +#if KB_BIAS32 + const v8f b0 = kb_bias32(kb_b32base, OFF_BO + 16 * ot); +#else const v8f b0 = wgk_widen8(bo[ot]); +#endif [unroll] for (int g = 0; g < G; ++g) yacc[g][ot] = b0; } } +#if !KB_BASE_FORM const uint wl = lane * 8u; const uint pmix = KB_PREF ? P_mix + wgk_pin(pref) : P_mix; // KB_PREF: the touched words arrive before the heads +#endif #ifndef KB_GRPLATE #define KB_GRPLATE 0 #endif // (the head and MLP loops are left to the driver, which unrolls them: an opaque trip count that keeps them rolled // measured slower here, d1 +7 us / e1 +10 us with the MLP rolled one tile per iteration, no gain with two) [loop] for (int hd = 0; hd < HEADS; ++hd) { +#if KB_BASE_FORM + const uint wh = (uint)(hd * QKV * 2 * KT * 512); +#else const uint wh = wl + (uint)(hd * QKV * 2 * KT * 256); +#endif const uint kv = (uint)((NB == 2 ? (hd & 1) : 0) * (KH + VH)) + lane * 8u; // this lane's slot, window 0 uint4 q[G][2]; [unroll] for (int ct = 0; ct < 2; ++ct) { if (MIX == KMIX_SOFTMAX) { uint4 wq[C / 16], wk[C / 16]; [unroll] for (int k3 = 0; k3 < KT; ++k3) { +#if KB_BASE_FORM + wq[k3] = kb_weight128(wb_wqkv, wh + (uint)((0 * 2 * KT + ct * KT + k3) * 512)); +#else wq[k3] = wgk_ld8(P_wqkv, wh + (uint)((0 * 2 * KT + ct * KT + k3) * 256)); +#endif +#if KB_BASE_FORM + wk[k3] = kb_weight128(wb_wqkv, wh + (uint)((1 * 2 * KT + ct * KT + k3) * 512)); +#else wk[k3] = wgk_ld8(P_wqkv, wh + (uint)((1 * 2 * KT + ct * KT + k3) * 256)); +#endif } v8f aq[G], ak[G]; [unroll] for (int g = 0; g < G; ++g) { aq[g] = wgk_zero8f(); ak[g] = wgk_zero8f(); } @@ -335,7 +442,11 @@ void k_block(uint hidden, uint blk) // v^T: rows = channels 16 ct + lane % 16 (B columns), elements = tokens 16 wave + 8 hh + 0..7 uint4 wv[C / 16]; [unroll] for (int k4 = 0; k4 < KT; ++k4) +#if KB_BASE_FORM + wv[k4] = kb_weight128(wb_wqkv, wh + (uint)(((QKV - 1) * 2 * KT + ct * KT + k4) * 512)); +#else wv[k4] = wgk_ld8(P_wqkv, wh + (uint)(((QKV - 1) * 2 * KT + ct * KT + k4) * 256)); +#endif v8f av[G]; [unroll] for (int g = 0; g < G; ++g) av[g] = wgk_zero8f(); [unroll] for (int kt = 0; kt < KT; ++kt) @@ -346,9 +457,24 @@ void k_block(uint hidden, uint blk) } wg_barrier_lds(); +#if !KB_BASE_FORM const uint mrow = (uint)((hd * 64 + t) * 64 + 8 * hh); +#endif uint4 mx[4]; // A^T = M[h][t][s] (fixed) or the bias B[h][t][s] (softmax): the same for every window +#if KB_BIAS32 + if (MIX == KMIX_FIXED) { +#endif +#if KB_BASE_FORM + const uint mix_base = KB_PREF ? wb_mix + wgk_pin(pref) : wb_mix; + [unroll] for (int s0 = 0; s0 < 4; ++s0) + mx[s0] = kb_weight128(mix_base, (uint)((hd * 64 * 64 + 16 * s0) * 2)); +#else [unroll] for (int s0 = 0; s0 < 4; ++s0) mx[s0] = wgk_ld8(pmix, mrow + (uint)(16 * s0)); +#endif +#if KB_BIAS32 + } + const uint smx_row = OFF_SMX + (uint)((hd * 64 + t) * 64 + 8 * hh); +#endif uint4 z[G][2]; [unroll] for (int g = 0; g < G; ++g) { const uint lk = kv + (uint)g * KVW; @@ -361,7 +487,11 @@ void k_block(uint hidden, uint blk) [unroll] for (int f0 = 0; f0 < 8; ++f0) kf[f0] = kb_ldl(lk + (uint)(f0 * KB_FRAG)); uint4 gg[4]; [unroll] for (int st = 0; st < 4; ++st) { +#if KB_BIAS32 + v8f sc = kb_smx32(bias32, smx_row + (uint)(16 * st)); +#else v8f sc = wgk_widen8(mx[st]); // + B[h][t][s] +#endif [unroll] for (int ct = 0; ct < 2; ++ct) sc = wgk_mma(kf[2 * st + ct], q[g][ct], sc); const uint4 u = KB_CLAMP8W(wgk_round8(sc), WGK_SOFTMAX_C | 0x8000u, WGK_SOFTMAX_C); const uint4 u2 = wgk_mul8(u, u); @@ -388,11 +518,19 @@ void k_block(uint hidden, uint blk) if (!KB_GRPLATE) [unroll] for (int c1 = 0; c1 < 2; ++c1) [unroll] for (int o2 = 0; o2 < KT; ++o2) +#if KB_BASE_FORM + wo[c1][o2] = kb_weight128(wb_wo, (uint)((o2 * DT + 2 * hd + c1) * 512)); +#else wo[c1][o2] = wgk_ld8(P_wo, (uint)((o2 * DT + 2 * hd + c1) * 256) + lane * 8u); +#endif [unroll] for (int ct = 0; ct < 2; ++ct) { if (KB_GRPLATE) [unroll] for (int o2 = 0; o2 < KT; ++o2) +#if KB_BASE_FORM + wo[ct][o2] = kb_weight128(wb_wo, (uint)((o2 * DT + 2 * hd + ct) * 512)); +#else wo[ct][o2] = wgk_ld8(P_wo, (uint)((o2 * DT + 2 * hd + ct) * 256) + lane * 8u); +#endif [unroll] for (int ot = 0; ot < KT; ++ot) [unroll] for (int g = 0; g < G; ++g) yacc[g][ot] = wgk_mma(wo[ct][ot], z[g][ct], yacc[g][ot]); } @@ -435,31 +573,55 @@ void k_block(uint hidden, uint blk) // 4. MLP: per hidden tile p = f16(W1 a + b1), h = act(p), acc += W2 h (FP32, started from b2) v8f acc[G][C / 16]; { +#if !KB_BIAS32 uint4 b2v[C / 16]; [unroll] for (int o3 = 0; o3 < KT; ++o3) b2v[o3] = wgk_ld8(P_b2, (uint)(16 * o3 + 8 * hh)); +#endif [unroll] for (int ot = 0; ot < KT; ++ot) { +#if KB_BIAS32 + const v8f b0 = kb_bias32(kb_b32base, OFF_B2 + 16 * ot); +#else const v8f b0 = wgk_widen8(b2v[ot]); +#endif [unroll] for (int g = 0; g < G; ++g) acc[g][ot] = b0; } } [loop] for (int ht = 0; ht < HT; ++ht) { uint4 w1t[C / 16], w2t[C / 16]; +#if KB_BIAS32 + const v8f b0 = kb_bias32(kb_b32base, OFF_B1 + 16 * ht); +#else const uint4 b1v = wgk_ld8(P_b1, (uint)(16 * ht + 8 * hh)); +#endif +#if KB_BASE_FORM + [unroll] for (int k6 = 0; k6 < KT; ++k6) w1t[k6] = kb_weight128(wb_w1, (uint)((ht * KT + k6) * 512)); +#else [unroll] for (int k6 = 0; k6 < KT; ++k6) w1t[k6] = wgk_ld8(P_w1, (uint)((ht * KT + k6) * 256) + lane * 8u); +#endif #ifndef KB_W2LATE #define KB_W2LATE 0 #endif if (!KB_W2LATE) +#if KB_BASE_FORM + [unroll] for (int o4 = 0; o4 < KT; ++o4) w2t[o4] = kb_weight128(wb_w2, (uint)((ht * KT + o4) * 512)); +#else [unroll] for (int o4 = 0; o4 < KT; ++o4) w2t[o4] = wgk_ld8(P_w2, (uint)((ht * KT + o4) * 256) + lane * 8u); +#endif v8f p[G]; +#if !KB_BIAS32 const v8f b0 = wgk_widen8(b1v); +#endif [unroll] for (int g = 0; g < G; ++g) p[g] = b0; [unroll] for (int kt = 0; kt < KT; ++kt) [unroll] for (int g = 0; g < G; ++g) p[g] = wgk_mma(w1t[kt], a[g][kt], p[g]); uint4 hv[G]; [unroll] for (int g = 0; g < G; ++g) hv[g] = KB_ACT8W(wgk_round8(p[g])); if (KB_W2LATE) // W2's tiles just before their multiplies (fewer live registers across the first GEMM) +#if KB_BASE_FORM + [unroll] for (int o4 = 0; o4 < KT; ++o4) w2t[o4] = kb_weight128(wb_w2, (uint)((ht * KT + o4) * 512)); +#else [unroll] for (int o4 = 0; o4 < KT; ++o4) w2t[o4] = wgk_ld8(P_w2, (uint)((ht * KT + o4) * 256) + lane * 8u); +#endif [unroll] for (int ot = 0; ot < KT; ++ot) [unroll] for (int g = 0; g < G; ++g) acc[g][ot] = wgk_mma(w2t[ot], hv[g], acc[g][ot]); } @@ -492,7 +654,11 @@ void k_block(uint hidden, uint blk) const int mw = (P_w + 1) / 2, mh = (P_h + 1) / 2; [loop] for (int tt = (int)wave; tt < NTM; tt += 4) { v8f m[G]; +#if KB_BIAS32 + const v8f b0 = kb_bias32(kb_b32base, OFF_BZ + 16 * tt); +#else const v8f b0 = wgk_widen8(wgk_ld8(P_bz, (uint)(16 * tt + 8 * hh))); +#endif [unroll] for (int g = 0; g < G; ++g) m[g] = b0; [unroll] for (int k7 = 0; k7 < KTM; k7 += 4) { // in groups of 4 input tiles (HIP: unroll 4) uint4 wm[4], xm[G][4]; @@ -500,7 +666,11 @@ void k_block(uint hidden, uint blk) const int kt = k7 + i; const int s = (16 * kt) / C, c0 = 16 * kt - s * C; const int ft = (2 * gr + (s >> 1)) * 8 + 2 * gc + (s & 1); +#if KB_BASE_FORM + wm[i] = kb_weight128(wb_wz, (uint)((tt * KTM + kt) * 512)); +#else wm[i] = wgk_ld8(P_wz, (uint)((tt * KTM + kt) * 256) + lane * 8u); +#endif [unroll] for (int g = 0; g < G; ++g) xm[g][i] = kb_ldl(lds_x + (uint)(g * XH + ft * C + c0 + 8 * hh)); } [unroll] for (int i2 = 0; i2 < 4; ++i2) @@ -522,16 +692,35 @@ void k_block(uint hidden, uint blk) } } - if (HEAD) { // out = f16(Wh x2 + bh), 40 of 48 padded rows - uint4 heads[G][3]; - uint4 whd[3][C / 16], bhd[3]; - [unroll] for (int o6 = 0; o6 < 3; ++o6) { + if (HEAD) { // out = f16(Wh x2 + bh) + const int NT = KB_BIAS32 ? 2 : 3; + uint4 heads[G][NT], whd[NT][C / 16]; +#if KB_BIAS32 + v8f bhd[NT]; +#else + uint4 bhd[NT]; +#endif + [unroll] for (int o6 = 0; o6 < NT; ++o6) { +#if KB_BIAS32 + bhd[o6] = kb_bias32(kb_b32base, OFF_BZ + 16 * o6); +#else bhd[o6] = wgk_ld8(P_bz, (uint)(16 * o6 + 8 * hh)); - [unroll] for (int k8 = 0; k8 < KT; ++k8) whd[o6][k8] = wgk_ld8(P_wz, (uint)((o6 * KT + k8) * 256) + lane * 8u); +#endif +#if KB_BASE_FORM + [unroll] for (int k8 = 0; k8 < KT; ++k8) + whd[o6][k8] = kb_weight128(wb_wz, (uint)((o6 * KT + k8) * 512)); +#else + [unroll] for (int k8 = 0; k8 < KT; ++k8) + whd[o6][k8] = wgk_ld8(P_wz, (uint)((o6 * KT + k8) * 256) + lane * 8u); +#endif } - [unroll] for (int ot = 0; ot < 3; ++ot) { + [unroll] for (int ot = 0; ot < NT; ++ot) { v8f o[G]; +#if KB_BIAS32 + const v8f b0 = bhd[ot]; +#else const v8f b0 = wgk_widen8(bhd[ot]); +#endif [unroll] for (int g = 0; g < G; ++g) o[g] = b0; [unroll] for (int kt = 0; kt < KT; ++kt) [unroll] for (int g = 0; g < G; ++g) o[g] = wgk_mma(whd[ot][kt], x1[g][kt], o[g]); @@ -539,11 +728,11 @@ void k_block(uint hidden, uint blk) } uint pinh = 0u; [unroll] for (int g = 0; g < G; ++g) - [unroll] for (int o8 = 0; o8 < 3; ++o8) pinh ^= heads[g][o8].x; + [unroll] for (int o8 = 0; o8 < NT; ++o8) pinh ^= heads[g][o8].x; const uint phead = P_head + wgk_pin(pinh); // the head products before the predicated stores if (!PIX) { [unroll] for (int g = 0; g < G; ++g) - [unroll] for (int o7 = 0; o7 < 3; ++o7) + [unroll] for (int o7 = 0; o7 < NT; ++o7) if (real[g] && 16 * o7 + 8 * hh < 40) wgk_st8(phead, ytok[g] * 40u + (uint)(16 * o7 + 8 * hh), heads[g][o7]); } else { @@ -551,12 +740,20 @@ void k_block(uint hidden, uint blk) // 2 gy + (k >> 1), column 2 gx + (k & 1) holds (channel 10 k, 10 k + 1, 10 k + 9, 0). A lane of half hh has // channels 10 k, 10 k + 1 of slots k = hh and 2 + hh and gets the third value from its partner lane. [unroll] for (int g = 0; g < G; ++g) { +#if KB_BIAS32 + // Each half owns the three channels of both compact slots in the first packed tile. + const uint4 h0 = heads[g][0]; + const uint a0 = wgk_el(h0, kb_head_row[0] % 8), a1 = wgk_el(h0, kb_head_row[1] % 8); + const uint c0 = wgk_el(h0, kb_head_row[20] % 8), c1 = wgk_el(h0, kb_head_row[21] % 8); + const uint rcv = wgk_el(h0, kb_head_row[9] % 8) | (wgk_el(h0, kb_head_row[29] % 8) << 16); +#else const uint4 h0 = heads[g][0], h1 = heads[g][1], h2 = heads[g][2]; const uint a0 = hh ? wgk_el(h0, 2u) : wgk_el(h0, 0u), a1 = hh ? wgk_el(h0, 3u) : wgk_el(h0, 1u); const uint c0 = hh ? wgk_el(h1, 6u) : wgk_el(h1, 4u), c1 = hh ? wgk_el(h1, 7u) : wgk_el(h1, 5u); // to the partner: channel 9 + 10 k of its slots (hh = 0 holds 19, 39; hh = 1 holds 9, 29) const uint snd = (hh ? wgk_el(h0, 1u) : wgk_el(h1, 3u)) | ((hh ? wgk_el(h1, 5u) : wgk_el(h2, 7u)) << 16); const uint rcv = wgk_xor16_u(snd); +#endif if (real[g]) { const uint d = ((uint)(2 * gy[g]) * (uint)(2 * P_w) + (uint)(2 * gx[g] + hh)) * 4u; wg_st64(phead, d * 2u, uint2(a0 | (a1 << 16), rcv & 0xffffu)); @@ -572,10 +769,15 @@ void k_block(uint hidden, uint blk) float v[5]; [unroll] for (int j = 0; j < 5; ++j) { const int c = 10 * k + 3 + j; - const float own = hh == 0 ? wgk_elf(heads[0][c / 16], (uint)(c % 8)) : wgk_elf(heads[G - 1][c / 16], (uint)(c % 8)); - const float snd = hh == 0 ? wgk_elf(heads[G - 1][c / 16], (uint)(c % 8)) : wgk_elf(heads[0][c / 16], (uint)(c % 8)); +#if KB_BIAS32 + const int r = kb_head_row[c]; +#else + const int r = c; +#endif + const float own = hh == 0 ? wgk_elf(heads[0][r / 16], (uint)(r % 8)) : wgk_elf(heads[G - 1][r / 16], (uint)(r % 8)); + const float snd = hh == 0 ? wgk_elf(heads[G - 1][r / 16], (uint)(r % 8)) : wgk_elf(heads[0][r / 16], (uint)(r % 8)); const float other = wgk_xor16(snd); - v[j] = ((c % 16) / 8) == hh ? own : other; + v[j] = ((r % 16) / 8) == hh ? own : other; } // opaque: (0.2 s) est must stay two products (the driver regroups multiplication chains) hs[k] = asfloat(wgk_opq(asuint(0.2f * ((((v[0] + v[1]) + v[2]) + v[3]) + v[4])))); diff --git a/clean/k/dx12/hlsl/k_head_rows.inc b/clean/k/dx12/hlsl/k_head_rows.inc new file mode 100644 index 00000000..3923713a --- /dev/null +++ b/clean/k/dx12/hlsl/k_head_rows.inc @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: MIT +// Physical row for each logical d0 head channel. Unused rows are -1. +0, 1, -1, 6, 7, 16, 17, 18, -1, 2, +8, 9, -1, 19, 20, 21, 22, 23, -1, 10, +3, 4, -1, 14, 15, 24, 25, 26, -1, 5, +11, 12, -1, 27, 28, 29, 30, 31, -1, 13 diff --git a/clean/k/dx12/kernels/k_block.hlsl b/clean/k/dx12/kernels/k_block.hlsl index 1ce5ac9d..a5e8ae50 100644 --- a/clean/k/dx12/kernels/k_block.hlsl +++ b/clean/k/dx12/kernels/k_block.hlsl @@ -21,20 +21,20 @@ // Launch as HIP: 128 lanes (4 waves of 32) per workgroup, one workgroup per window (G = 1) or per two windows (G = 2). // // @wg-kernel k_stage_e2 co=k_block.co block=128 wave=32 args=s184:p19.b32 -// @wg-kernel k_stage2_e0 co=k_block.co block=128 wave=32 args=s184:p19.b32 -// @wg-kernel k_stage2_e1 co=k_block.co block=128 wave=32 args=s184:p19.b32 +// @wg-kernel k_stage2_e0 co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_BIAS32=1,KB_BASE_FORM=1 +// @wg-kernel k_stage2_e1 co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_BIAS32=1,KB_BASE_FORM=1 // @wg-kernel k_stage_d2 co=k_block.co block=128 wave=32 args=s184:p19.b32 -// @wg-kernel k_stage2_d1 co=k_block.co block=128 wave=32 args=s184:p19.b32 +// @wg-kernel k_stage2_d1 co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_BIAS32=1,KB_BASE_FORM=1 // @wg-kernel k_stage_e0 co=k_block.co block=128 wave=32 args=s184:p19.b32 defines=KB_LDS_HALFS=6144 // @wg-kernel k_stage_e1 co=k_block.co block=128 wave=32 args=s184:p19.b32 defines=KB_LDS_HALFS=4096 // @wg-kernel k_stage_d1 co=k_block.co block=128 wave=32 args=s184:p19.b32 -// @wg-kernel k_stage2_d0_hp co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_HIDDEN_ARG=1 +// @wg-kernel k_stage2_d0_hp co=k_block.co block=128 wave=32 args=s184:p19.b32,p,p defines=KB_HIDDEN_ARG=1,KB_BIAS32=1,KB_BASE_FORM=1 // @wg-kernel k_stage2_d0_hidden co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_HIDDEN_ARG=1 // The __rodd forms: the same stages with the round-to-odd softmax weights (hlsl/k_smx.hlsli K_SMX 1), launched by the // native D3D12 runtime when the mix-form probe (kernels/k_mix_probe.hlsl) fails on the adapter's driver. // @wg-kernel k_stage_e2__rodd co=k_block.co entry=k_stage_e2 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1 // @wg-kernel k_stage_d2__rodd co=k_block.co entry=k_stage_d2 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1 -// @wg-kernel k_stage2_d1__rodd co=k_block.co entry=k_stage2_d1 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1 +// @wg-kernel k_stage2_d1__rodd co=k_block.co entry=k_stage2_d1 block=128 wave=32 args=s184:p19.b32,p defines=K_SMX=1,KB_BIAS32=1,KB_BASE_FORM=1 #include "wg_platform.hlsli" #include "k_blkparams.hlsli" @@ -43,6 +43,9 @@ WG_ARGS_BEGIN #if KB_HIDDEN_ARG uint hidden_p; // the hidden history Hh written by the d0 epilogue #endif +#if KB_BIAS32 + uint P_b32; +#endif WG_ARGS_END #include "k_block.hlsli" @@ -50,7 +53,11 @@ WG_ARGS_END WG_KERNEL(k_stage2_d0_hp) { WG_SETUP(); +#if KB_BIAS32 + k_block<32, 2, KMIX_FIXED, KIN_EXPAND, 64, 0, true, 2, true, true, false>(hidden_p, wg_block_id(), P_b32); +#else k_block<32, 2, KMIX_FIXED, KIN_EXPAND, 64, 0, true, 2, true, true, false>(hidden_p, wg_block_id()); +#endif } WG_KERNEL(k_stage2_d0_hidden) { @@ -66,12 +73,20 @@ WG_KERNEL(k_stage_e2) WG_KERNEL(k_stage2_e0) { WG_SETUP(); +#if KB_BIAS32 + k_block<32, 2, KMIX_FIXED, KIN_EMBED, 16, 64, false, 2, false, false, false>(0u, wg_block_id(), P_b32); +#else k_block<32, 2, KMIX_FIXED, KIN_EMBED, 16, 64, false, 2, false, false, false>(0u, wg_block_id()); +#endif } WG_KERNEL(k_stage2_e1) { WG_SETUP(); +#if KB_BIAS32 + k_block<64, 2, KMIX_FIXED, KIN_MAP, 64, 64, false, 2, false, false, false>(0u, wg_block_id(), P_b32); +#else k_block<64, 2, KMIX_FIXED, KIN_MAP, 64, 64, false, 2, false, false, false>(0u, wg_block_id()); +#endif } WG_KERNEL(k_stage_d2) { @@ -81,7 +96,11 @@ WG_KERNEL(k_stage_d2) WG_KERNEL(k_stage2_d1) { WG_SETUP(); +#if KB_BIAS32 + k_block<64, 2, KMIX_SOFTMAX, KIN_EXPAND, 64, 0, false, 2, false, false, true>(0u, wg_block_id(), P_b32); +#else k_block<64, 2, KMIX_SOFTMAX, KIN_EXPAND, 64, 0, false, 2, false, false, true>(0u, wg_block_id()); +#endif } WG_KERNEL(k_stage_e0) { diff --git a/clean/k/runtime/k_pack.cpp b/clean/k/runtime/k_pack.cpp index 035d8676..b2cdfa8d 100644 --- a/clean/k/runtime/k_pack.cpp +++ b/clean/k/runtime/k_pack.cpp @@ -186,4 +186,26 @@ std::vector k_pack_stage(const std::string& stage, const StageTensors& return blob; } +std::vector k_pack_compact_head(const uint8_t* original, size_t channels) { + constexpr int row[40] = { +#include "../dx12/hlsl/k_head_rows.inc" + }; + const size_t kt_count = channels / 16; + const size_t old_bias = (48 * channels * 2 + kAlign - 1) / kAlign * kAlign; + const size_t new_bias = (32 * channels * 2 + kAlign - 1) / kAlign * kAlign; + std::vector out(new_bias + kAlign, 0); + for (size_t logical = 0; logical < 40; ++logical) { + if (row[logical] < 0) continue; + const size_t physical = (size_t)row[logical]; + for (size_t kt = 0; kt < kt_count; ++kt) + for (size_t half = 0; half < 2; ++half) { + const size_t from = (logical / 16 * kt_count + kt) * 512 + (16 * half + logical % 16) * 16; + const size_t to = (physical / 16 * kt_count + kt) * 512 + (16 * half + physical % 16) * 16; + std::memcpy(out.data() + to, original + from, 16); + } + std::memcpy(out.data() + new_bias + physical * 2, original + old_bias + logical * 2, 2); + } + return out; +} + } // namespace wgrt diff --git a/clean/k/runtime/k_pack.h b/clean/k/runtime/k_pack.h index e8822450..f81c7150 100644 --- a/clean/k/runtime/k_pack.h +++ b/clean/k/runtime/k_pack.h @@ -7,6 +7,7 @@ // clean/k/reference/params.py normalize; byte-identical blobs (clean/k/tests/test_k_runtime_pack.py). #pragma once +#include #include #include #include @@ -22,4 +23,7 @@ namespace wgrt { // k_stage_blob_bytes(*k_stage(stage)) bytes. On failure returns an empty vector with a message in *err. std::vector k_pack_stage(const std::string& stage, const StageTensors& t, std::string* err); +// Repack the 32 live rows of d0's head from the original gfx12 linear blob. +std::vector k_pack_compact_head(const uint8_t* original, size_t channels); + } // namespace wgrt diff --git a/clean/k/runtime/k_smx_selfcheck.h b/clean/k/runtime/k_smx_selfcheck.h index 01c8aaf0..1bcdb383 100644 --- a/clean/k/runtime/k_smx_selfcheck.h +++ b/clean/k/runtime/k_smx_selfcheck.h @@ -8,7 +8,7 @@ namespace k_smx_selfcheck { -constexpr char kSourceDigest[] = "ef7da006529622c2fa87d1d3371038cdf85467b1fb0010b361f334784b6fa92b"; +constexpr char kSourceDigest[] = "97e5b4d6fb6e9eab12891447f123cb92539000a4bb9bc05c0d688325afbfe59f"; constexpr int kCount = 133; constexpr const char* kNames[kCount] = { "k_stage_e2@512x384/o4,4", diff --git a/clean/k/runtime/runtime_k.cpp b/clean/k/runtime/runtime_k.cpp index 59c76a57..50366a25 100644 --- a/clean/k/runtime/runtime_k.cpp +++ b/clean/k/runtime/runtime_k.cpp @@ -138,6 +138,40 @@ StageSec stage_sections(const KStageInfo& s) { // order: embed, expand, block, return r; } +// Bias widening for the FP32 planes. Preserve the binary16 NaN payload and quiet signaling NaNs. +uint32_t k_bias32_bits(uint16_t h) { + const uint32_t payload = h & 0x3ffu; + if ((h & 0x7c00u) == 0x7c00u && payload) + return ((uint32_t)(h & 0x8000u) << 16) | 0x7fc00000u | (payload << 13); + return f32_bits(f16_to_f32(h)); +} + +std::vector k_narrow_bias32(const KStageInfo& s, const uint8_t* packed, + const uint8_t* compact_head = nullptr) { + const auto stage = stage_sections(s); + const auto block = block_sections(s); + std::vector plane; + const auto append = [&](const uint8_t* data, size_t count) { + for (size_t i = 0; i < count; ++i) { + uint16_t half; + std::memcpy(&half, data + i * sizeof(half), sizeof(half)); + plane.push_back(k_bias32_bits(half)); + } + }; + if (s.embed) append(packed + stage.embed + linear_bias(s.C, 16), s.C); + if (s.kind == 'd') append(packed + stage.expand + linear_bias(4 * s.C, s.C_prev), 4 * s.C); + append(packed + stage.block + block.bo, s.C); + append(packed + stage.block + block.b1, 4 * s.C); + append(packed + stage.block + block.b2, s.C); + if (s.kind == 'e') append(packed + stage.merge + linear_bias(s.C_next, 4 * s.C), s.C_next); + if (s.head) { + append(compact_head + linear_bias(32, s.C), 32); + plane.resize(plane.size() + 16, 0); + } + if (!s.fixed) append(packed + stage.block + block.mix, (size_t)s.heads * 64 * 64); + return plane; +} + // ---- network=fused8: FP8 forms of the level-0/1 stages (clean/k/kernels/k_block8.h) -------------------------------- static_assert(sizeof(KF8Params) == 104, "KF8Params of k_block8.h"); float blob_h(const std::vector& b, size_t off, size_t i) { @@ -662,6 +696,23 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e if (!alloc(&p, kv.second.size()) || !dev_->upload(p, kv.second.data(), kv.second.size())) return false; wdev_[kv.first] = p; } + if (c.backend == 2 || c.backend == 3) { + for (const char* name : {"e0", "e1", "d1", "d0"}) { + const KStageInfo& st = *k_stage(name); + const auto& weight = blobs_.at(name); + std::vector compact; + if (st.head) { + if (!compact_head_) continue; + compact = k_pack_compact_head(weight.data() + stage_sections(st).head, st.C); + if (!alloc(&d0_compact_head_, compact.size()) || + !dev_->upload(d0_compact_head_, compact.data(), compact.size())) return false; + } + const auto bias = k_narrow_bias32(st, weight.data(), compact.data()); + const size_t bytes = bias.size() * sizeof(bias[0]); + DevPtr& plane = narrow_bias_[name]; + if (!alloc(&plane, bytes) || !dev_->upload(plane, bias.data(), bytes)) return false; + } + } for (auto& kv : f8data_) { // network=fused8: the FP8 weights and FP32 biases; device pointers into KF8Params DevPtr p; if (!alloc(&p, kv.second.bytes.size()) || !dev_->upload(p, kv.second.bytes.data(), kv.second.bytes.size())) @@ -868,7 +919,7 @@ bool KRuntime::smx_check(bool rodd, bool all, std::vector in(in_bytes / 2), w(w_bytes / 2); - DevPtr din = 0, dw = 0, dy = 0, dm = 0; + DevPtr din = 0, dw = 0, dy = 0, dm = 0, db = 0; int loaded = -1; // the data set in din / dw auto load = [&](int set) { if (set == loaded) return true; @@ -881,6 +932,11 @@ bool KRuntime::smx_check(bool rodd, bool all, std::vectoralloc(&din, in_bytes) && dev_->alloc(&dw, w_bytes) && dev_->alloc(&dy, y_bytes) && dev_->alloc(&dm, std::max(m_bytes, 256)); + if (ok && narrow_bias_.count("d1")) { + const auto& d1 = *k_stage("d1"); + const size_t words = 10u * d1.C + (size_t)d1.heads * 64 * 64; + ok = dev_->alloc(&db, words * sizeof(uint32_t)); + } std::vector buf; for (int i = 0; ok && i < kSmxCount; ++i) { const SmxCase& c = kSmxCases[i]; @@ -896,11 +952,17 @@ bool KRuntime::smx_check(bool rodd, bool all, std::vectormemset(dy, 0, y_bytes) && dev_->memset(dm, 0, std::max(m_bytes, 256)) && + ok = load(set); + if (ok && use_bias) { + const auto bias = k_narrow_bias32(st, reinterpret_cast(w.data())); + ok = dev_->upload(db, bias.data(), bias.size() * sizeof(bias[0])); + } + ok = ok && dev_->memset(dy, 0, y_bytes) && dev_->memset(dm, 0, std::max(m_bytes, 256)) && launch(k.c_str(), (unsigned)(w2 ? cdiv(p.nwx * p.nwy, 2) : p.nwx * p.nwy), - (unsigned)(kThreads * (ng ? ng : 1)), &p) && + (unsigned)(kThreads * (ng ? ng : 1)), &p, use_bias ? &db : nullptr) && dev_->download(buf.data(), dy, yb) && (!mb || dev_->download(buf.data() + yb, dm, mb)); if (ok) sha.update(buf.data(), buf.size()); } @@ -911,7 +973,7 @@ bool KRuntime::smx_check(bool rodd, bool all, std::vectoremplace_back(smx_case_name(i, o.first, o.second, false), v); } } - for (DevPtr p : {din, dw, dy, dm}) + for (DevPtr p : {din, dw, dy, dm, db}) if (p) dev_->free(p); return ok || dfail(); } @@ -1141,8 +1203,19 @@ bool KRuntime::network(int f, const std::function& after) { // k_head=compact: p.head (the OUT buffer) receives the compact pixel head [2 Bh][2 Bw][4] instead of the head map const char* k0 = compact_head_ ? (f16_ ? "k_stage2h_d0_hp" : "k_stage2_d0_hp") : (f16_ ? "k_stage2h_d0_hidden" : "k_stage2_d0_hidden"); - if (!launch(k0, grid, kThreads, &p, &hidden)) return false; - } else if (!launch(stage_kernel(k, st.name).c_str(), grid, (unsigned)(kThreads * (ng ? ng : 1)), &p)) return false; + if (compact_head_ && !f16_ && d0_compact_head_) { + p.wz = d0_compact_head_; + if (!launch(k0, grid, kThreads, &p, &hidden, &narrow_bias_.at(st.name))) return false; + } else if (!launch(k0, grid, kThreads, &p, &hidden)) { + return false; + } + } else { + const std::string kernel = stage_kernel(k, st.name); + const auto bias_plane = narrow_bias_.find(st.name); + const bool use_bias = k == "k_stage2_e0" || k == "k_stage2_e1" || k == "k_stage2_d1"; + void* bias = use_bias && bias_plane != narrow_bias_.end() ? &bias_plane->second : nullptr; + if (!launch(kernel.c_str(), grid, (unsigned)(kThreads * (ng ? ng : 1)), &p, bias)) return false; + } if (after && !after(st.name)) return false; if (i < 10) { info_.offsets[i][0] = ox; // WgRtFrameInfo has ten rows: e0 ... d1 (d0's offset is (4, 4)) diff --git a/clean/k/runtime/runtime_k.h b/clean/k/runtime/runtime_k.h index 7fa1dff6..311ccd2a 100644 --- a/clean/k/runtime/runtime_k.h +++ b/clean/k/runtime/runtime_k.h @@ -164,6 +164,8 @@ class KRuntime { bool expo_v051_ = false; // runtime option k_expo_rule=v051 (A/B only): the 0.5.1 exposure rule int offsets_table_[8][2] = {}; std::map> blobs_; + std::map narrow_bias_; // FP32 bias planes for the D3D12 two-window narrow stages + DevPtr d0_compact_head_ = 0; // the compact head's two permuted tiles std::map wdev_, net_; // stage blobs; network maps (X0, S0..S4, A, B, OUT) std::map net_bytes_; DevPtr Hc_[2] = {0, 0}, L_[2] = {0, 0}, Hh_[2] = {0, 0}; From 731ba3689cd6642ce5d1e440a7a5dde570cde86f Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:03:24 -0500 Subject: [PATCH 2/2] Fix the Model K narrow D3D12 launch ABI Keep the original 48-row head in KBlkParams and pass the compact head and bias plane as separate D3D12 arguments. Require the narrow ports and their argument signatures for hybrid and native contexts. Refuse missing argument values before hybrid argument packing. Replay both d1 forms and the compact head against the original HIP arguments. Check the packed row permutation on the host and reject a compact-bias full-head shader at compile time. Rebind the softmax source digest to a47546460b78b46b3ae0775661c6c4160ad20af09b6106c16322f8d164136264. The native backend-3 k_smx_test=report reproduced 133/133 stored HIP hashes for each of mix and round-to-odd, with zero hash or probe mismatches. Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/k/dx12/hlsl/k_block.hlsli | 23 ++++----- clean/k/dx12/k_plan_bridge.py | 73 +++++++++++++++++++++++++++ clean/k/dx12/k_port_check.py | 16 ++++-- clean/k/dx12/kernels/k_block.hlsl | 15 ++++-- clean/k/runtime/k_smx_selfcheck.h | 2 +- clean/k/runtime/runtime_k.cpp | 18 ++++--- clean/k/tests/test_k_plan_bridge.py | 11 ++++ clean/k/tests/test_k_runtime_d3d12.py | 7 +++ clean/runtime/device.h | 6 +++ clean/runtime/device_d3d12.cpp | 9 +++- clean/runtime/device_dx12.cpp | 30 +++++++++-- 11 files changed, 174 insertions(+), 36 deletions(-) diff --git a/clean/k/dx12/hlsl/k_block.hlsli b/clean/k/dx12/hlsl/k_block.hlsli index 0d7b4003..e5c08c25 100644 --- a/clean/k/dx12/hlsl/k_block.hlsli +++ b/clean/k/dx12/hlsl/k_block.hlsli @@ -179,21 +179,12 @@ v8f kb_bias32(uint base, uint off_f) return r; } -// Softmax starts with the bias row of this query, without the lane-half bias base. -v8f kb_smx32(uint base, uint off_f) -{ - return kb_bias32(base, off_f); -} #endif #if KB_BASE_FORM uint4 kb_weight128(uint base, uint off) { -#if KB_BASE_FORM == 2 - return wg_ld128(base + off, 0u); -#else return wg_ld128(base, off); -#endif } #endif @@ -204,8 +195,12 @@ uint4 kb_weight128(uint base, uint off) // pixel head), XG (decoder: x = f16(e + S) through P.y instead of LDS). blk: the workgroup's window group (windows // G blk .. G blk + G - 1): wg_block_id(). template -void k_block(uint hidden, uint blk, uint bias32 = 0u) +void k_block(uint hidden, uint blk, uint bias32 = 0u, uint compact_head = 0u) { +#if KB_BIAS32 + // DXC lacks static_assert. A full head cannot use the compact row layout. + const int compact_bias_requires_pixel_head[!HEAD || PIX ? 1 : -1] = {0}; +#endif const int KT = C / 16; const int DT = HEADS * 2; const int HT = 4 * C / 16; @@ -234,7 +229,7 @@ void k_block(uint hidden, uint blk, uint bias32 = 0u) #if KB_BIAS32 const int OFF_BA = 0, OFF_BO = IN == KIN_EMBED ? C : (IN == KIN_EXPAND ? 4 * C : 0); const int OFF_B1 = OFF_BO + C, OFF_B2 = OFF_B1 + 4 * C, OFF_BZ = OFF_B2 + C; - const int OFF_SMX = OFF_BZ + (HEAD ? 48 : CNEXT); + const int OFF_SMX = OFF_BZ + CNEXT; const uint kb_b32base = (bias32 + (uint)hh * 32u) ^ wgk_zero(); #endif #if KB_BASE_FORM @@ -244,7 +239,7 @@ void k_block(uint hidden, uint blk, uint bias32 = 0u) const uint wb_wo = (P_wo + weight_lane) ^ wgk_zero(); const uint wb_w1 = (P_w1 + weight_lane) ^ wgk_zero(); const uint wb_w2 = (P_w2 + weight_lane) ^ wgk_zero(); - const uint wb_wz = (P_wz + weight_lane) ^ wgk_zero(); + const uint wb_wz = ((KB_BIAS32 && HEAD ? compact_head : P_wz) + weight_lane) ^ wgk_zero(); const uint wb_mix = (P_mix + (uint)(t * 64 + 8 * hh) * 2u) ^ wgk_zero(); #endif int wa[G], wb[G], gx[G], gy[G]; @@ -488,7 +483,7 @@ void k_block(uint hidden, uint blk, uint bias32 = 0u) uint4 gg[4]; [unroll] for (int st = 0; st < 4; ++st) { #if KB_BIAS32 - v8f sc = kb_smx32(bias32, smx_row + (uint)(16 * st)); + v8f sc = kb_bias32(bias32, smx_row + (uint)(16 * st)); #else v8f sc = wgk_widen8(mx[st]); // + B[h][t][s] #endif @@ -711,7 +706,7 @@ void k_block(uint hidden, uint blk, uint bias32 = 0u) whd[o6][k8] = kb_weight128(wb_wz, (uint)((o6 * KT + k8) * 512)); #else [unroll] for (int k8 = 0; k8 < KT; ++k8) - whd[o6][k8] = wgk_ld8(P_wz, (uint)((o6 * KT + k8) * 256) + lane * 8u); + whd[o6][k8] = wgk_ld8(KB_BIAS32 ? compact_head : P_wz, (uint)((o6 * KT + k8) * 256) + lane * 8u); #endif } [unroll] for (int ot = 0; ot < NT; ++ot) { diff --git a/clean/k/dx12/k_plan_bridge.py b/clean/k/dx12/k_plan_bridge.py index fe3d1f54..d1912719 100644 --- a/clean/k/dx12/k_plan_bridge.py +++ b/clean/k/dx12/k_plan_bridge.py @@ -13,8 +13,11 @@ import ctypes import json import os +from pathlib import Path import struct +from clean.k.runner.k_pack import block_sections, linear_sections, stage_sections + HIP_SENTINEL, D3D12_SENTINEL, SLACK = 0xA5, 0x5A, 64 @@ -43,6 +46,76 @@ def __getattr__(self, name): return getattr(self.hip, name) +def head_rows(): + """Physical rows for the 40 logical channels, shared with the C++ packer and shader.""" + path = Path(__file__).with_name("hlsl") / "k_head_rows.inc" + values = " ".join(line.split("//", 1)[0] for line in path.read_text(encoding="utf-8").splitlines()) + return tuple(int(value) for value in values.split(",")) + + +def _bias32(half): + """Widen FP16 bits as k_bias32_bits does, including NaN sign and payload.""" + payload = half & 0x3FF + if half & 0x7C00 == 0x7C00 and payload: + return (half & 0x8000) << 16 | 0x7FC00000 | payload << 13 + return struct.unpack(" its argument tokens, from /dx12/manifest.json.""" with open(os.path.join(build, "dx12", "manifest.json")) as f: diff --git a/clean/k/dx12/k_port_check.py b/clean/k/dx12/k_port_check.py index a18578a7..b1e086f4 100644 --- a/clean/k/dx12/k_port_check.py +++ b/clean/k/dx12/k_port_check.py @@ -53,7 +53,7 @@ CONTROL = "scale_rgb" # Model M, frame.co # family -> (HIP code objects, D3D12 kernels) NARROW_KERNELS = ["k_stage_e2", "k_stage_e0", "k_stage_e1", "k_stage_d2", "k_stage_d1", "k_stage2_e0", "k_stage2_e1", - "k_stage2_d1", "k_stage2_d0_hp", "k_stage2_d0_hidden"] + "k_stage2_d1", "k_stage2_d1__rodd", "k_stage2_d0_hp", "k_stage2_d0_hidden"] FAMILY_KERNELS = {"expo": (["k_io.co"], ["k_expo_scalars"]), "warm": ([], ["k_warm"]), "mixprobe": ([], ["k_mix_probe", "k_mix_probe__rodd"]), "narrow": (["k_block.co"], NARROW_KERNELS), @@ -359,7 +359,9 @@ def check_mixprobe(items, read): NARROW = {"k_stage_e2": ("e2", "", None), "k_stage_e0": ("e0", "", None), "k_stage_e1": ("e1", "", None), "k_stage_d2": ("d2", "", None), "k_stage_d1": ("d1", "", None), "k_stage2_e0": ("e0", "@w2", None), "k_stage2_e1": ("e1", "@w2", None), "k_stage2_d1": ("d1", "@w2", None), + "k_stage2_d1__rodd": ("d1", "@w2", None), "k_stage2_d0_hp": ("d0", "@w2", "hp"), "k_stage2_d0_hidden": ("d0", "@w2", "hidden")} +NARROW_BIAS32 = {"k_stage2_e0", "k_stage2_e1", "k_stage2_d1", "k_stage2_d1__rodd", "k_stage2_d0_hp"} # block-level sizes (W, H) and window offsets (spec 44: even): grids with fewer than a workgroup's windows among them NARROW_SIZES = [(8, 8), (24, 16), (17, 13), (40, 24), (9, 33), (12, 20)] NARROW_OFFSETS = [(0, 0), (2, 2), (4, 0), (0, 6), (6, 4)] @@ -433,6 +435,12 @@ def run_stages(table, co_name, tag, build, rocm, seeds, cdir, hip, grid_list=Non plan["launches"] = [(kernel, grid, block, args + [ctypes.c_void_p(hid)])] plan["outputs"] = [("hidden", (H, W, 4))] + ( [("head", (2 * H, 2 * W, 4))] if d0 == "hp" else [("head", (H, W, 40))]) + k, grid, block, hip_args = plan["launches"][0] + assert k == kernel.removesuffix("__rodd"), (k, kernel) + dx_args = hip_args + if kernel in NARROW_BIAS32: + dx_args = hip_args + PB.narrow_d3d12_args(plan, pdev, st, blob) + dx_plan = dict(plan, launches=[(kernel, grid, block, dx_args)]) outputs = [n for n, _ in plan["outputs"]] for n, p in plan["bufs"].items(): if p in pdev.inputs: @@ -444,11 +452,11 @@ def run_stages(table, co_name, tag, build, rocm, seeds, cdir, hip, grid_list=Non sizes = {n: pdev.sizes[plan["bufs"][n]] for n in outputs} got = {n: bytes(hip.h.download(plan["bufs"][n], sizes[n] + SLACK)) for n in outputs} defined = {n: min(int(np.prod(sh)) * itemsize(n), sizes[n]) for n, sh in plan["outputs"]} - lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, [co_name], + lines += PB.chain_lines(name, dx_plan, pdev, lambda k: tokens[k], cdir, [co_name], outputs) - if neg is None and plan["launches"][0][1] >= 4: # the dropped windows hold output pixels + if neg is None and grid >= 4: # the dropped windows hold output pixels neg = name - lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, + lines += PB.chain_lines(name, dx_plan, pdev, lambda k: tokens[k], cdir, [co_name], outputs, prefix="negative_", grid_div=2) items.append({"case": name, "kernel": kernel, "stage": stage, "size": list(size), "offset": list(offset), "specials": special, "outputs": outputs, "hip": got, diff --git a/clean/k/dx12/kernels/k_block.hlsl b/clean/k/dx12/kernels/k_block.hlsl index a5e8ae50..33484bde 100644 --- a/clean/k/dx12/kernels/k_block.hlsl +++ b/clean/k/dx12/kernels/k_block.hlsl @@ -17,7 +17,8 @@ // The FP16-accumulating forms (k_stageh_*) and the FP8 forms are not ported: the exact path only. // Basis: our HIP kernels (spec 41 block, spec 44 windows, spec 43 6.8 hidden history, spec 40 table); exactness and the // forms that differ (bit-identical) are listed in hlsl/k_block.hlsli and hlsl/k_common.hlsli. Arguments as HIP: -// KBlkParams by value (token s184:p19.b32, 27 constants), the d0 stages also the hidden history pointer (",p"). +// KBlkParams by value (token s184:p19.b32, 27 constants), the d0 stages also take the hidden history pointer. +// The narrow two-window ports take a FP32 bias plane, and d0_hp takes a separate compact head weight pointer. // Launch as HIP: 128 lanes (4 waves of 32) per workgroup, one workgroup per window (G = 1) or per two windows (G = 2). // // @wg-kernel k_stage_e2 co=k_block.co block=128 wave=32 args=s184:p19.b32 @@ -28,7 +29,7 @@ // @wg-kernel k_stage_e0 co=k_block.co block=128 wave=32 args=s184:p19.b32 defines=KB_LDS_HALFS=6144 // @wg-kernel k_stage_e1 co=k_block.co block=128 wave=32 args=s184:p19.b32 defines=KB_LDS_HALFS=4096 // @wg-kernel k_stage_d1 co=k_block.co block=128 wave=32 args=s184:p19.b32 -// @wg-kernel k_stage2_d0_hp co=k_block.co block=128 wave=32 args=s184:p19.b32,p,p defines=KB_HIDDEN_ARG=1,KB_BIAS32=1,KB_BASE_FORM=1 +// @wg-kernel k_stage2_d0_hp co=k_block.co block=128 wave=32 args=s184:p19.b32,p,p,p defines=KB_HIDDEN_ARG=1,KB_COMPACT_HEAD_ARG=1,KB_BIAS32=1,KB_BASE_FORM=1 // @wg-kernel k_stage2_d0_hidden co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_HIDDEN_ARG=1 // The __rodd forms: the same stages with the round-to-odd softmax weights (hlsl/k_smx.hlsli K_SMX 1), launched by the // native D3D12 runtime when the mix-form probe (kernels/k_mix_probe.hlsl) fails on the adapter's driver. @@ -46,6 +47,9 @@ WG_ARGS_BEGIN #if KB_BIAS32 uint P_b32; #endif +#if KB_COMPACT_HEAD_ARG + uint P_compact_head; +#endif WG_ARGS_END #include "k_block.hlsli" @@ -53,17 +57,20 @@ WG_ARGS_END WG_KERNEL(k_stage2_d0_hp) { WG_SETUP(); -#if KB_BIAS32 - k_block<32, 2, KMIX_FIXED, KIN_EXPAND, 64, 0, true, 2, true, true, false>(hidden_p, wg_block_id(), P_b32); +#if KB_COMPACT_HEAD_ARG + k_block<32, 2, KMIX_FIXED, KIN_EXPAND, 64, 0, true, 2, true, true, false>(hidden_p, wg_block_id(), P_b32, + P_compact_head); #else k_block<32, 2, KMIX_FIXED, KIN_EXPAND, 64, 0, true, 2, true, true, false>(hidden_p, wg_block_id()); #endif } +#if !KB_COMPACT_HEAD_ARG WG_KERNEL(k_stage2_d0_hidden) { WG_SETUP(); k_block<32, 2, KMIX_FIXED, KIN_EXPAND, 64, 0, true, 2, true, false, false>(hidden_p, wg_block_id()); } +#endif #else WG_KERNEL(k_stage_e2) { diff --git a/clean/k/runtime/k_smx_selfcheck.h b/clean/k/runtime/k_smx_selfcheck.h index 1bcdb383..788f8302 100644 --- a/clean/k/runtime/k_smx_selfcheck.h +++ b/clean/k/runtime/k_smx_selfcheck.h @@ -8,7 +8,7 @@ namespace k_smx_selfcheck { -constexpr char kSourceDigest[] = "97e5b4d6fb6e9eab12891447f123cb92539000a4bb9bc05c0d688325afbfe59f"; +constexpr char kSourceDigest[] = "a47546460b78b46b3ae0775661c6c4160ad20af09b6106c16322f8d164136264"; constexpr int kCount = 133; constexpr const char* kNames[kCount] = { "k_stage_e2@512x384/o4,4", diff --git a/clean/k/runtime/runtime_k.cpp b/clean/k/runtime/runtime_k.cpp index 50366a25..92e8d103 100644 --- a/clean/k/runtime/runtime_k.cpp +++ b/clean/k/runtime/runtime_k.cpp @@ -138,6 +138,7 @@ StageSec stage_sections(const KStageInfo& s) { // order: embed, expand, block, return r; } +// SMX-SELFCHECK-SOURCE BEGIN (the narrow d1 bias plane used by the self-check) // Bias widening for the FP32 planes. Preserve the binary16 NaN payload and quiet signaling NaNs. uint32_t k_bias32_bits(uint16_t h) { const uint32_t payload = h & 0x3ffu; @@ -164,13 +165,11 @@ std::vector k_narrow_bias32(const KStageInfo& s, const uint8_t* packed append(packed + stage.block + block.b1, 4 * s.C); append(packed + stage.block + block.b2, s.C); if (s.kind == 'e') append(packed + stage.merge + linear_bias(s.C_next, 4 * s.C), s.C_next); - if (s.head) { - append(compact_head + linear_bias(32, s.C), 32); - plane.resize(plane.size() + 16, 0); - } + if (s.head) append(compact_head + linear_bias(32, s.C), 32); if (!s.fixed) append(packed + stage.block + block.mix, (size_t)s.heads * 64 * 64); return plane; } +// SMX-SELFCHECK-SOURCE END // ---- network=fused8: FP8 forms of the level-0/1 stages (clean/k/kernels/k_block8.h) -------------------------------- static_assert(sizeof(KF8Params) == 104, "KF8Params of k_block8.h"); @@ -696,13 +695,17 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e if (!alloc(&p, kv.second.size()) || !dev_->upload(p, kv.second.data(), kv.second.size())) return false; wdev_[kv.first] = p; } - if (c.backend == 2 || c.backend == 3) { + if (!f16_ && (c.backend == 2 || c.backend == 3)) { for (const char* name : {"e0", "e1", "d1", "d0"}) { const KStageInfo& st = *k_stage(name); + if (f8data_.count(name) || (st.head && !compact_head_)) continue; + const std::string kernel = std::string("k_stage2_") + name + (st.head ? "_hp" : ""); + if (!dev_->require_d3d12_port(kernel, st.head ? "s184:p19.b32,p,p,p" : "s184:p19.b32,p")) return dfail(); + if (c.backend == 3 && std::strcmp(name, "d1") == 0 && + !dev_->require_d3d12_port(kernel + "__rodd", "s184:p19.b32,p")) return dfail(); const auto& weight = blobs_.at(name); std::vector compact; if (st.head) { - if (!compact_head_) continue; compact = k_pack_compact_head(weight.data() + stage_sections(st).head, st.C); if (!alloc(&d0_compact_head_, compact.size()) || !dev_->upload(d0_compact_head_, compact.data(), compact.size())) return false; @@ -1204,8 +1207,7 @@ bool KRuntime::network(int f, const std::function& after) { const char* k0 = compact_head_ ? (f16_ ? "k_stage2h_d0_hp" : "k_stage2_d0_hp") : (f16_ ? "k_stage2h_d0_hidden" : "k_stage2_d0_hidden"); if (compact_head_ && !f16_ && d0_compact_head_) { - p.wz = d0_compact_head_; - if (!launch(k0, grid, kThreads, &p, &hidden, &narrow_bias_.at(st.name))) return false; + if (!launch(k0, grid, kThreads, &p, &hidden, &narrow_bias_.at(st.name), &d0_compact_head_)) return false; } else if (!launch(k0, grid, kThreads, &p, &hidden)) { return false; } diff --git a/clean/k/tests/test_k_plan_bridge.py b/clean/k/tests/test_k_plan_bridge.py index 2873fbb0..ed8742db 100644 --- a/clean/k/tests/test_k_plan_bridge.py +++ b/clean/k/tests/test_k_plan_bridge.py @@ -49,6 +49,17 @@ def test_chain_step_of_a_block_launch(self): ["p", "p", "u", "u", "b4"], bufs, d, "case.l1") self.assertEqual(line, "step k 1 32 null @b_weights+8 4294967293 1065353216 B:case.l1.a4") + def test_compact_head_rows_cover_pixel_and_hidden_channels(self): + rows = PB.head_rows() + self.assertEqual(len(rows), 40) + used = {10 * slot + offset for slot in range(4) for offset in (0, 1, 3, 4, 5, 6, 7, 9)} + self.assertEqual(sorted(rows[ch] for ch in used), list(range(32))) + self.assertTrue(all(rows[ch] == -1 for ch in set(range(40)) - used)) + for half in range(2): + slot = 10 * half + channels = (slot, slot + 1, slot + 9, slot + 20, slot + 21, slot + 29) + self.assertEqual([rows[ch] for ch in channels], list(range(8 * half, 8 * half + 6))) + def test_argument_count_mismatch_is_refused(self): with tempfile.TemporaryDirectory() as d: with self.assertRaises(ValueError): diff --git a/clean/k/tests/test_k_runtime_d3d12.py b/clean/k/tests/test_k_runtime_d3d12.py index a0f060da..8bdc5e68 100644 --- a/clean/k/tests/test_k_runtime_d3d12.py +++ b/clean/k/tests/test_k_runtime_d3d12.py @@ -43,6 +43,7 @@ import subprocess import sys import unittest +from unittest.mock import patch import numpy as np @@ -207,6 +208,12 @@ def test_hybrid_hdr_auto_exposure(self): def test_hybrid_ldr_rows(self): self.check_hybrid(False, 0, True) + def test_hybrid_refuses_missing_compact_head_port(self): + selected = ",".join(stage for stage in STAGES if stage != "k_stage2_d0_hp") + with patch.dict(os.environ, {"WG_DX12_KERNELS": selected}): + with self.assertRaisesRegex(RuntimeError, "required port unavailable for k_stage2_d0_hp"): + self.make(2, True, False, "", OUT, MAX_RENDER) + def test_hybrid_head_full(self): self.check_hybrid(True, 0, True, "k_head=full") diff --git a/clean/runtime/device.h b/clean/runtime/device.h index 3cf5c021..0ed360f8 100644 --- a/clean/runtime/device.h +++ b/clean/runtime/device.h @@ -49,6 +49,12 @@ class Device { } // args: pointers to the argument values (hipModuleLaunchKernel convention). virtual bool launch(const std::string& kernel, unsigned grid, unsigned block, void** args) = 0; + // Require a D3D12 port with this argument ABI for the lifetime of the context. + virtual bool require_d3d12_port(const std::string& kernel, const char* signature) { + (void)signature; + err_ = "D3D12 port required for " + kernel; + return false; + } // Loads, verifies and compiles one kernel now (its first launch then compiles nothing); false with err() when its // shader is missing or corrupt. Devices that prepare nothing ahead: true. virtual bool prepare(const std::string& kernel) { (void)kernel; return true; } diff --git a/clean/runtime/device_d3d12.cpp b/clean/runtime/device_d3d12.cpp index 31bc79d2..5da26095 100644 --- a/clean/runtime/device_d3d12.cpp +++ b/clean/runtime/device_d3d12.cpp @@ -71,7 +71,7 @@ constexpr unsigned kProfileQueries = 4096; struct NativeArg { char kind; unsigned bytes; std::vector fields = {}; }; // 's': struct_args.h struct NativeKernel { - std::string name, file, co, rs = "buffer"; + std::string name, file, co, signature, rs = "buffer"; unsigned block = 0, wave = 0, dwords = 0; // group: lanes per D3D12 workgroup when the port splits each launched workgroup of `block` lanes into block / group // workgroups (manifest "group", build_dx12.py group=; 0 = block). The launch keeps the HIP geometry (grid x block). @@ -756,6 +756,12 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (variant_matches(i->second)) return &i->second; return range.first == range.second ? nullptr : &range.first->second; } + bool require_d3d12_port(const std::string& name, const char* signature) override { + const NativeKernel* port = find_port(name); + if (!port || !variant_matches(*port)) + return fail("native D3D12: required port unavailable for " + name); + return port->signature == signature || fail("native D3D12: incompatible argument signature for " + name); + } bool prepare(const std::string& name) override { NativeKernel* port = find_port(name); if (!port) return fail("native D3D12 unsupported kernel: " + name + "; no HIP fallback"); @@ -1450,6 +1456,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { !wgc::parse_digest_hex(hash, k.digest) || k.block == 0 || k.block > 1024 || (k.wave != 32 && k.wave != 64) || k.block % k.wave || k.dwords > kNativeArgs) return fail("native D3D12 manifest: invalid kernel metadata for " + k.name); + k.signature = arguments; unsigned words = 0; for (size_t start = 0; start < arguments.size();) { const size_t end = arguments.find(',', start); diff --git a/clean/runtime/device_dx12.cpp b/clean/runtime/device_dx12.cpp index d06e9c60..5ff8cd37 100644 --- a/clean/runtime/device_dx12.cpp +++ b/clean/runtime/device_dx12.cpp @@ -95,6 +95,7 @@ struct Arg { struct Kernel { std::string name, file, co; // co: file name of the HIP code object this port reproduces + std::string signature; unsigned block = 0, wave = 32, dwords = 0; unsigned group = 0; // lanes per D3D12 workgroup of a split port (manifest "group"; 0 = block) unsigned gscale = 1; // D3D12 workgroups per launched workgroup (manifest "gscale", default block / group) @@ -239,13 +240,27 @@ class Dx12Device : public Device { bool set_stream_priority(bool high) override { return fwd(hip_->set_stream_priority(high)); } void begin_teardown() override { hip_->begin_teardown(); } + bool require_d3d12_port(const std::string& name, const char* signature) override { + const Kernel* port = find_port(name); + if (!port || !arena_ || !variant_selected(*port)) + return fail("hybrid D3D12: required port unavailable for " + name); + if (port->signature != signature) + return fail("hybrid D3D12: incompatible argument signature for " + name); + required_ports_.insert(name); + return true; + } + // ---- launches ---------------------------------------------------------------------------------------------------- bool launch(const std::string& kernel, unsigned grid, unsigned block, void** args) override { Kernel* port = find_port(kernel); uint32_t c[kRuntimeConstants + kMaxArgDwords]; - if (!port || !arena_ || !variant_loaded(*port) || !pack_args(*port, args, c + kRuntimeConstants)) { - return to_hip() && fwd(hip_->launch(kernel, grid, block, args)); - } + if (!port || !arena_ || !variant_loaded(*port)) + return allow_hip_fallback(kernel) && to_hip() && fwd(hip_->launch(kernel, grid, block, args)); + if (!args && !port->args.empty()) return fail("hybrid D3D12 kernel " + kernel + ": null argument array"); + for (size_t i = 0; i < port->args.size(); ++i) + if (!args[i]) return fail("hybrid D3D12 kernel " + kernel + ": null argument value"); + if (!pack_args(*port, args, c + kRuntimeConstants)) + return allow_hip_fallback(kernel) && to_hip() && fwd(hip_->launch(kernel, grid, block, args)); Kernel& k = *port; if (block != k.block) return fail("D3D12 kernel " + kernel + " is built for " + std::to_string(k.block) + " lanes, launched with " + @@ -276,11 +291,16 @@ class Dx12Device : public Device { int stop_event) override { Kernel* port = find_port(kernel); if (!port || !arena_ || !variant_loaded(*port)) - return to_hip() && fwd(hip_->launch_timed(kernel, grid, block, args, start_event, stop_event)); + return allow_hip_fallback(kernel) && to_hip() && + fwd(hip_->launch_timed(kernel, grid, block, args, start_event, stop_event)); return Device::launch_timed(kernel, grid, block, args, start_event, stop_event); } private: + bool allow_hip_fallback(const std::string& kernel) { + return !required_ports_.count(kernel) || fail("hybrid D3D12: required port cannot run " + kernel); + } + bool fail(const std::string& m) { err_ = m; return false; @@ -532,6 +552,7 @@ class Dx12Device : public Device { return fail("D3D12 manifest " + path + ": incomplete kernel entry"); x.name = n->text; x.file = dir + fl->text; + x.signature = ar->text; if (co) x.co = co->text; // missing (an old manifest): the port is never used x.block = (unsigned)block; if (const wgc::Json* gp = k.find("group")) { @@ -784,6 +805,7 @@ class Dx12Device : public Device { uint64_t arena_bytes_ = 0, in_use_ = 0, fence_value_ = 0; std::map free_, used_; // offset -> bytes std::multimap kernels_; // by name; several ports of one name differ in "co" + std::set required_ports_; std::set untagged_cos_; // file names of the code objects loaded without a tag std::map tagged_cos_; // tag -> file name of the code object loaded under it unsigned module_gen_ = 0;