diff --git a/clean/k/dx12/hlsl/k_block.hlsli b/clean/k/dx12/hlsl/k_block.hlsli index 473877f2..e5c08c25 100644 --- a/clean/k/dx12/hlsl/k_block.hlsli +++ b/clean/k/dx12/hlsl/k_block.hlsli @@ -153,6 +153,41 @@ void wgk_store_hidden(uint base, uint off_h, float h[4]) wg_st64(base, off_h * 2u, uint2(wg_f2h(pr[0]) | (wg_f2h(pr[1]) << 16), wg_f2h(pr[2]) | (wg_f2h(pr[3]) << 16))); } +#ifndef KB_BIAS32 +#define KB_BIAS32 0 +#endif + +#if KB_BIAS32 +static const int kb_head_row[40] = { +#include "k_head_rows.inc" +}; + +// Rows 8*hh through 8*hh+7 of the FP32 bias tile. +v8f kb_bias32(uint base, uint off_f) +{ + const uint4 lo = wg_ld128(base, off_f * 4u); + const uint4 hi = wg_ld128(base, off_f * 4u + 16u); + v8f r; + r.v[0] = asfloat(lo.x); + r.v[1] = asfloat(lo.y); + r.v[2] = asfloat(lo.z); + r.v[3] = asfloat(lo.w); + r.v[4] = asfloat(hi.x); + r.v[5] = asfloat(hi.y); + r.v[6] = asfloat(hi.z); + r.v[7] = asfloat(hi.w); + return r; +} + +#endif + +#if KB_BASE_FORM +uint4 kb_weight128(uint base, uint off) +{ + return wg_ld128(base, off); +} +#endif + // ---- the block ---------------------------------------------------------------------------------------------------- // Template parameters as k_block.hip (F16 = 0): C channels, HEADS, MIX (KMIX_*), IN (KIN_MAP / KIN_EMBED / KIN_EXPAND; // KIN_PADDED as well), CIN input channels of EMBED / EXPAND, CNEXT merge output channels (0: none), HEAD (32 -> 40 output @@ -160,8 +195,12 @@ void wgk_store_hidden(uint base, uint off_h, float h[4]) // pixel head), XG (decoder: x = f16(e + S) through P.y instead of LDS). blk: the workgroup's window group (windows // G blk .. G blk + G - 1): wg_block_id(). template -void k_block(uint hidden, uint blk) +void k_block(uint hidden, uint blk, uint bias32 = 0u, uint compact_head = 0u) { +#if KB_BIAS32 + // DXC lacks static_assert. A full head cannot use the compact row layout. + const int compact_bias_requires_pixel_head[!HEAD || PIX ? 1 : -1] = {0}; +#endif const int KT = C / 16; const int DT = HEADS * 2; const int HT = 4 * C / 16; @@ -187,6 +226,22 @@ void k_block(uint hidden, uint blk) const int n16 = (int)(lane & 15u), hh = (int)(lane >> 4); const int nwin = P_nwx * P_nwy; const int t = 16 * (int)wave + n16; +#if KB_BIAS32 + const int OFF_BA = 0, OFF_BO = IN == KIN_EMBED ? C : (IN == KIN_EXPAND ? 4 * C : 0); + const int OFF_B1 = OFF_BO + C, OFF_B2 = OFF_B1 + 4 * C, OFF_BZ = OFF_B2 + C; + const int OFF_SMX = OFF_BZ + CNEXT; + const uint kb_b32base = (bias32 + (uint)hh * 32u) ^ wgk_zero(); +#endif +#if KB_BASE_FORM + const uint weight_lane = lane * 16u; + const uint wb_wa = (P_wa + weight_lane) ^ wgk_zero(); + const uint wb_wqkv = (P_wqkv + weight_lane) ^ wgk_zero(); + const uint wb_wo = (P_wo + weight_lane) ^ wgk_zero(); + const uint wb_w1 = (P_w1 + weight_lane) ^ wgk_zero(); + const uint wb_w2 = (P_w2 + weight_lane) ^ wgk_zero(); + const uint wb_wz = ((KB_BIAS32 && HEAD ? compact_head : P_wz) + weight_lane) ^ wgk_zero(); + const uint wb_mix = (P_mix + (uint)(t * 64 + 8 * hh) * 2u) ^ wgk_zero(); +#endif int wa[G], wb[G], gx[G], gy[G]; bool real[G]; uint mtok[G], xt[G], xl[G], ytok[G]; @@ -212,14 +267,31 @@ void k_block(uint hidden, uint blk) // 1. the block input uint4 n[G][C / 16]; if (IN == KIN_EMBED) { // e(t) = max(f16(We in(t) + be), 0) of the mirrored member - uint4 inp[G], ea[C / 16], eb[C / 16]; + uint4 inp[G], ea[C / 16]; +#if KB_BIAS32 + v8f eb[C / 16]; +#else + uint4 eb[C / 16]; +#endif [unroll] for (int g = 0; g < G; ++g) inp[g] = wgk_ld8(P_x, mtok[g] * 16u + 8u * (uint)hh); [unroll] for (int o0 = 0; o0 < KT; ++o0) { +#if KB_BASE_FORM + ea[o0] = kb_weight128(wb_wa, (uint)(o0 * 512)); +#else ea[o0] = wgk_ld8(P_wa, (uint)(o0 * 256) + lane * 8u); +#endif +#if KB_BIAS32 + eb[o0] = kb_bias32(kb_b32base, OFF_BA + 16 * o0); +#else eb[o0] = wgk_ld8(P_ba, (uint)(16 * o0 + 8 * hh)); +#endif } [unroll] for (int ot = 0; ot < KT; ++ot) { +#if KB_BIAS32 + const v8f b0 = eb[ot]; +#else const v8f b0 = wgk_widen8(eb[ot]); +#endif [unroll] for (int g = 0; g < G; ++g) { n[g][ot] = KB_MAXC8W(wgk_round8(wgk_mma(ea[ot], inp[g], b0)), 0u); kb_stl(xl[g] + (uint)(16 * ot), n[g][ot]); @@ -237,10 +309,20 @@ void k_block(uint hidden, uint blk) } [loop] for (int tt = (int)wave; tt < TOT; tt += 4) { // output tile tt of 4C: sub-position s, channels c0 .. uint4 wt[CIN / 16]; +#if KB_BIAS32 + const v8f b0 = kb_bias32(kb_b32base, OFF_BA + 16 * tt); +#else const uint4 bb = wgk_ld8(P_ba, (uint)(16 * tt + 8 * hh)); +#endif +#if KB_BASE_FORM + [unroll] for (int k1 = 0; k1 < KTI; ++k1) wt[k1] = kb_weight128(wb_wa, (uint)((tt * KTI + k1) * 512)); +#else [unroll] for (int k1 = 0; k1 < KTI; ++k1) wt[k1] = wgk_ld8(P_wa, (uint)((tt * KTI + k1) * 256) + lane * 8u); +#endif v8f e[G]; +#if !KB_BIAS32 const v8f b0 = wgk_widen8(bb); +#endif [unroll] for (int g = 0; g < G; ++g) e[g] = b0; [unroll] for (int kt = 0; kt < KTI; ++kt) [unroll] for (int g = 0; g < G; ++g) e[g] = wgk_mma(wt[kt], yb[g][kt], e[g]); @@ -293,31 +375,51 @@ void k_block(uint hidden, uint blk) // 2. token mixing, head by head; y accumulates the output projection v8f yacc[G][C / 16]; { +#if !KB_BIAS32 uint4 bo[C / 16]; [unroll] for (int o1 = 0; o1 < KT; ++o1) bo[o1] = wgk_ld8(P_bo, (uint)(16 * o1 + 8 * hh)); +#endif [unroll] for (int ot = 0; ot < KT; ++ot) { +#if KB_BIAS32 + const v8f b0 = kb_bias32(kb_b32base, OFF_BO + 16 * ot); +#else const v8f b0 = wgk_widen8(bo[ot]); +#endif [unroll] for (int g = 0; g < G; ++g) yacc[g][ot] = b0; } } +#if !KB_BASE_FORM const uint wl = lane * 8u; const uint pmix = KB_PREF ? P_mix + wgk_pin(pref) : P_mix; // KB_PREF: the touched words arrive before the heads +#endif #ifndef KB_GRPLATE #define KB_GRPLATE 0 #endif // (the head and MLP loops are left to the driver, which unrolls them: an opaque trip count that keeps them rolled // measured slower here, d1 +7 us / e1 +10 us with the MLP rolled one tile per iteration, no gain with two) [loop] for (int hd = 0; hd < HEADS; ++hd) { +#if KB_BASE_FORM + const uint wh = (uint)(hd * QKV * 2 * KT * 512); +#else const uint wh = wl + (uint)(hd * QKV * 2 * KT * 256); +#endif const uint kv = (uint)((NB == 2 ? (hd & 1) : 0) * (KH + VH)) + lane * 8u; // this lane's slot, window 0 uint4 q[G][2]; [unroll] for (int ct = 0; ct < 2; ++ct) { if (MIX == KMIX_SOFTMAX) { uint4 wq[C / 16], wk[C / 16]; [unroll] for (int k3 = 0; k3 < KT; ++k3) { +#if KB_BASE_FORM + wq[k3] = kb_weight128(wb_wqkv, wh + (uint)((0 * 2 * KT + ct * KT + k3) * 512)); +#else wq[k3] = wgk_ld8(P_wqkv, wh + (uint)((0 * 2 * KT + ct * KT + k3) * 256)); +#endif +#if KB_BASE_FORM + wk[k3] = kb_weight128(wb_wqkv, wh + (uint)((1 * 2 * KT + ct * KT + k3) * 512)); +#else wk[k3] = wgk_ld8(P_wqkv, wh + (uint)((1 * 2 * KT + ct * KT + k3) * 256)); +#endif } v8f aq[G], ak[G]; [unroll] for (int g = 0; g < G; ++g) { aq[g] = wgk_zero8f(); ak[g] = wgk_zero8f(); } @@ -335,7 +437,11 @@ void k_block(uint hidden, uint blk) // v^T: rows = channels 16 ct + lane % 16 (B columns), elements = tokens 16 wave + 8 hh + 0..7 uint4 wv[C / 16]; [unroll] for (int k4 = 0; k4 < KT; ++k4) +#if KB_BASE_FORM + wv[k4] = kb_weight128(wb_wqkv, wh + (uint)(((QKV - 1) * 2 * KT + ct * KT + k4) * 512)); +#else wv[k4] = wgk_ld8(P_wqkv, wh + (uint)(((QKV - 1) * 2 * KT + ct * KT + k4) * 256)); +#endif v8f av[G]; [unroll] for (int g = 0; g < G; ++g) av[g] = wgk_zero8f(); [unroll] for (int kt = 0; kt < KT; ++kt) @@ -346,9 +452,24 @@ void k_block(uint hidden, uint blk) } wg_barrier_lds(); +#if !KB_BASE_FORM const uint mrow = (uint)((hd * 64 + t) * 64 + 8 * hh); +#endif uint4 mx[4]; // A^T = M[h][t][s] (fixed) or the bias B[h][t][s] (softmax): the same for every window +#if KB_BIAS32 + if (MIX == KMIX_FIXED) { +#endif +#if KB_BASE_FORM + const uint mix_base = KB_PREF ? wb_mix + wgk_pin(pref) : wb_mix; + [unroll] for (int s0 = 0; s0 < 4; ++s0) + mx[s0] = kb_weight128(mix_base, (uint)((hd * 64 * 64 + 16 * s0) * 2)); +#else [unroll] for (int s0 = 0; s0 < 4; ++s0) mx[s0] = wgk_ld8(pmix, mrow + (uint)(16 * s0)); +#endif +#if KB_BIAS32 + } + const uint smx_row = OFF_SMX + (uint)((hd * 64 + t) * 64 + 8 * hh); +#endif uint4 z[G][2]; [unroll] for (int g = 0; g < G; ++g) { const uint lk = kv + (uint)g * KVW; @@ -361,7 +482,11 @@ void k_block(uint hidden, uint blk) [unroll] for (int f0 = 0; f0 < 8; ++f0) kf[f0] = kb_ldl(lk + (uint)(f0 * KB_FRAG)); uint4 gg[4]; [unroll] for (int st = 0; st < 4; ++st) { +#if KB_BIAS32 + v8f sc = kb_bias32(bias32, smx_row + (uint)(16 * st)); +#else v8f sc = wgk_widen8(mx[st]); // + B[h][t][s] +#endif [unroll] for (int ct = 0; ct < 2; ++ct) sc = wgk_mma(kf[2 * st + ct], q[g][ct], sc); const uint4 u = KB_CLAMP8W(wgk_round8(sc), WGK_SOFTMAX_C | 0x8000u, WGK_SOFTMAX_C); const uint4 u2 = wgk_mul8(u, u); @@ -388,11 +513,19 @@ void k_block(uint hidden, uint blk) if (!KB_GRPLATE) [unroll] for (int c1 = 0; c1 < 2; ++c1) [unroll] for (int o2 = 0; o2 < KT; ++o2) +#if KB_BASE_FORM + wo[c1][o2] = kb_weight128(wb_wo, (uint)((o2 * DT + 2 * hd + c1) * 512)); +#else wo[c1][o2] = wgk_ld8(P_wo, (uint)((o2 * DT + 2 * hd + c1) * 256) + lane * 8u); +#endif [unroll] for (int ct = 0; ct < 2; ++ct) { if (KB_GRPLATE) [unroll] for (int o2 = 0; o2 < KT; ++o2) +#if KB_BASE_FORM + wo[ct][o2] = kb_weight128(wb_wo, (uint)((o2 * DT + 2 * hd + ct) * 512)); +#else wo[ct][o2] = wgk_ld8(P_wo, (uint)((o2 * DT + 2 * hd + ct) * 256) + lane * 8u); +#endif [unroll] for (int ot = 0; ot < KT; ++ot) [unroll] for (int g = 0; g < G; ++g) yacc[g][ot] = wgk_mma(wo[ct][ot], z[g][ct], yacc[g][ot]); } @@ -435,31 +568,55 @@ void k_block(uint hidden, uint blk) // 4. MLP: per hidden tile p = f16(W1 a + b1), h = act(p), acc += W2 h (FP32, started from b2) v8f acc[G][C / 16]; { +#if !KB_BIAS32 uint4 b2v[C / 16]; [unroll] for (int o3 = 0; o3 < KT; ++o3) b2v[o3] = wgk_ld8(P_b2, (uint)(16 * o3 + 8 * hh)); +#endif [unroll] for (int ot = 0; ot < KT; ++ot) { +#if KB_BIAS32 + const v8f b0 = kb_bias32(kb_b32base, OFF_B2 + 16 * ot); +#else const v8f b0 = wgk_widen8(b2v[ot]); +#endif [unroll] for (int g = 0; g < G; ++g) acc[g][ot] = b0; } } [loop] for (int ht = 0; ht < HT; ++ht) { uint4 w1t[C / 16], w2t[C / 16]; +#if KB_BIAS32 + const v8f b0 = kb_bias32(kb_b32base, OFF_B1 + 16 * ht); +#else const uint4 b1v = wgk_ld8(P_b1, (uint)(16 * ht + 8 * hh)); +#endif +#if KB_BASE_FORM + [unroll] for (int k6 = 0; k6 < KT; ++k6) w1t[k6] = kb_weight128(wb_w1, (uint)((ht * KT + k6) * 512)); +#else [unroll] for (int k6 = 0; k6 < KT; ++k6) w1t[k6] = wgk_ld8(P_w1, (uint)((ht * KT + k6) * 256) + lane * 8u); +#endif #ifndef KB_W2LATE #define KB_W2LATE 0 #endif if (!KB_W2LATE) +#if KB_BASE_FORM + [unroll] for (int o4 = 0; o4 < KT; ++o4) w2t[o4] = kb_weight128(wb_w2, (uint)((ht * KT + o4) * 512)); +#else [unroll] for (int o4 = 0; o4 < KT; ++o4) w2t[o4] = wgk_ld8(P_w2, (uint)((ht * KT + o4) * 256) + lane * 8u); +#endif v8f p[G]; +#if !KB_BIAS32 const v8f b0 = wgk_widen8(b1v); +#endif [unroll] for (int g = 0; g < G; ++g) p[g] = b0; [unroll] for (int kt = 0; kt < KT; ++kt) [unroll] for (int g = 0; g < G; ++g) p[g] = wgk_mma(w1t[kt], a[g][kt], p[g]); uint4 hv[G]; [unroll] for (int g = 0; g < G; ++g) hv[g] = KB_ACT8W(wgk_round8(p[g])); if (KB_W2LATE) // W2's tiles just before their multiplies (fewer live registers across the first GEMM) +#if KB_BASE_FORM + [unroll] for (int o4 = 0; o4 < KT; ++o4) w2t[o4] = kb_weight128(wb_w2, (uint)((ht * KT + o4) * 512)); +#else [unroll] for (int o4 = 0; o4 < KT; ++o4) w2t[o4] = wgk_ld8(P_w2, (uint)((ht * KT + o4) * 256) + lane * 8u); +#endif [unroll] for (int ot = 0; ot < KT; ++ot) [unroll] for (int g = 0; g < G; ++g) acc[g][ot] = wgk_mma(w2t[ot], hv[g], acc[g][ot]); } @@ -492,7 +649,11 @@ void k_block(uint hidden, uint blk) const int mw = (P_w + 1) / 2, mh = (P_h + 1) / 2; [loop] for (int tt = (int)wave; tt < NTM; tt += 4) { v8f m[G]; +#if KB_BIAS32 + const v8f b0 = kb_bias32(kb_b32base, OFF_BZ + 16 * tt); +#else const v8f b0 = wgk_widen8(wgk_ld8(P_bz, (uint)(16 * tt + 8 * hh))); +#endif [unroll] for (int g = 0; g < G; ++g) m[g] = b0; [unroll] for (int k7 = 0; k7 < KTM; k7 += 4) { // in groups of 4 input tiles (HIP: unroll 4) uint4 wm[4], xm[G][4]; @@ -500,7 +661,11 @@ void k_block(uint hidden, uint blk) const int kt = k7 + i; const int s = (16 * kt) / C, c0 = 16 * kt - s * C; const int ft = (2 * gr + (s >> 1)) * 8 + 2 * gc + (s & 1); +#if KB_BASE_FORM + wm[i] = kb_weight128(wb_wz, (uint)((tt * KTM + kt) * 512)); +#else wm[i] = wgk_ld8(P_wz, (uint)((tt * KTM + kt) * 256) + lane * 8u); +#endif [unroll] for (int g = 0; g < G; ++g) xm[g][i] = kb_ldl(lds_x + (uint)(g * XH + ft * C + c0 + 8 * hh)); } [unroll] for (int i2 = 0; i2 < 4; ++i2) @@ -522,16 +687,35 @@ void k_block(uint hidden, uint blk) } } - if (HEAD) { // out = f16(Wh x2 + bh), 40 of 48 padded rows - uint4 heads[G][3]; - uint4 whd[3][C / 16], bhd[3]; - [unroll] for (int o6 = 0; o6 < 3; ++o6) { + if (HEAD) { // out = f16(Wh x2 + bh) + const int NT = KB_BIAS32 ? 2 : 3; + uint4 heads[G][NT], whd[NT][C / 16]; +#if KB_BIAS32 + v8f bhd[NT]; +#else + uint4 bhd[NT]; +#endif + [unroll] for (int o6 = 0; o6 < NT; ++o6) { +#if KB_BIAS32 + bhd[o6] = kb_bias32(kb_b32base, OFF_BZ + 16 * o6); +#else bhd[o6] = wgk_ld8(P_bz, (uint)(16 * o6 + 8 * hh)); - [unroll] for (int k8 = 0; k8 < KT; ++k8) whd[o6][k8] = wgk_ld8(P_wz, (uint)((o6 * KT + k8) * 256) + lane * 8u); +#endif +#if KB_BASE_FORM + [unroll] for (int k8 = 0; k8 < KT; ++k8) + whd[o6][k8] = kb_weight128(wb_wz, (uint)((o6 * KT + k8) * 512)); +#else + [unroll] for (int k8 = 0; k8 < KT; ++k8) + whd[o6][k8] = wgk_ld8(KB_BIAS32 ? compact_head : P_wz, (uint)((o6 * KT + k8) * 256) + lane * 8u); +#endif } - [unroll] for (int ot = 0; ot < 3; ++ot) { + [unroll] for (int ot = 0; ot < NT; ++ot) { v8f o[G]; +#if KB_BIAS32 + const v8f b0 = bhd[ot]; +#else const v8f b0 = wgk_widen8(bhd[ot]); +#endif [unroll] for (int g = 0; g < G; ++g) o[g] = b0; [unroll] for (int kt = 0; kt < KT; ++kt) [unroll] for (int g = 0; g < G; ++g) o[g] = wgk_mma(whd[ot][kt], x1[g][kt], o[g]); @@ -539,11 +723,11 @@ void k_block(uint hidden, uint blk) } uint pinh = 0u; [unroll] for (int g = 0; g < G; ++g) - [unroll] for (int o8 = 0; o8 < 3; ++o8) pinh ^= heads[g][o8].x; + [unroll] for (int o8 = 0; o8 < NT; ++o8) pinh ^= heads[g][o8].x; const uint phead = P_head + wgk_pin(pinh); // the head products before the predicated stores if (!PIX) { [unroll] for (int g = 0; g < G; ++g) - [unroll] for (int o7 = 0; o7 < 3; ++o7) + [unroll] for (int o7 = 0; o7 < NT; ++o7) if (real[g] && 16 * o7 + 8 * hh < 40) wgk_st8(phead, ytok[g] * 40u + (uint)(16 * o7 + 8 * hh), heads[g][o7]); } else { @@ -551,12 +735,20 @@ void k_block(uint hidden, uint blk) // 2 gy + (k >> 1), column 2 gx + (k & 1) holds (channel 10 k, 10 k + 1, 10 k + 9, 0). A lane of half hh has // channels 10 k, 10 k + 1 of slots k = hh and 2 + hh and gets the third value from its partner lane. [unroll] for (int g = 0; g < G; ++g) { +#if KB_BIAS32 + // Each half owns the three channels of both compact slots in the first packed tile. + const uint4 h0 = heads[g][0]; + const uint a0 = wgk_el(h0, kb_head_row[0] % 8), a1 = wgk_el(h0, kb_head_row[1] % 8); + const uint c0 = wgk_el(h0, kb_head_row[20] % 8), c1 = wgk_el(h0, kb_head_row[21] % 8); + const uint rcv = wgk_el(h0, kb_head_row[9] % 8) | (wgk_el(h0, kb_head_row[29] % 8) << 16); +#else const uint4 h0 = heads[g][0], h1 = heads[g][1], h2 = heads[g][2]; const uint a0 = hh ? wgk_el(h0, 2u) : wgk_el(h0, 0u), a1 = hh ? wgk_el(h0, 3u) : wgk_el(h0, 1u); const uint c0 = hh ? wgk_el(h1, 6u) : wgk_el(h1, 4u), c1 = hh ? wgk_el(h1, 7u) : wgk_el(h1, 5u); // to the partner: channel 9 + 10 k of its slots (hh = 0 holds 19, 39; hh = 1 holds 9, 29) const uint snd = (hh ? wgk_el(h0, 1u) : wgk_el(h1, 3u)) | ((hh ? wgk_el(h1, 5u) : wgk_el(h2, 7u)) << 16); const uint rcv = wgk_xor16_u(snd); +#endif if (real[g]) { const uint d = ((uint)(2 * gy[g]) * (uint)(2 * P_w) + (uint)(2 * gx[g] + hh)) * 4u; wg_st64(phead, d * 2u, uint2(a0 | (a1 << 16), rcv & 0xffffu)); @@ -572,10 +764,15 @@ void k_block(uint hidden, uint blk) float v[5]; [unroll] for (int j = 0; j < 5; ++j) { const int c = 10 * k + 3 + j; - const float own = hh == 0 ? wgk_elf(heads[0][c / 16], (uint)(c % 8)) : wgk_elf(heads[G - 1][c / 16], (uint)(c % 8)); - const float snd = hh == 0 ? wgk_elf(heads[G - 1][c / 16], (uint)(c % 8)) : wgk_elf(heads[0][c / 16], (uint)(c % 8)); +#if KB_BIAS32 + const int r = kb_head_row[c]; +#else + const int r = c; +#endif + const float own = hh == 0 ? wgk_elf(heads[0][r / 16], (uint)(r % 8)) : wgk_elf(heads[G - 1][r / 16], (uint)(r % 8)); + const float snd = hh == 0 ? wgk_elf(heads[G - 1][r / 16], (uint)(r % 8)) : wgk_elf(heads[0][r / 16], (uint)(r % 8)); const float other = wgk_xor16(snd); - v[j] = ((c % 16) / 8) == hh ? own : other; + v[j] = ((r % 16) / 8) == hh ? own : other; } // opaque: (0.2 s) est must stay two products (the driver regroups multiplication chains) hs[k] = asfloat(wgk_opq(asuint(0.2f * ((((v[0] + v[1]) + v[2]) + v[3]) + v[4])))); diff --git a/clean/k/dx12/hlsl/k_head_rows.inc b/clean/k/dx12/hlsl/k_head_rows.inc new file mode 100644 index 00000000..3923713a --- /dev/null +++ b/clean/k/dx12/hlsl/k_head_rows.inc @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: MIT +// Physical row for each logical d0 head channel. Unused rows are -1. +0, 1, -1, 6, 7, 16, 17, 18, -1, 2, +8, 9, -1, 19, 20, 21, 22, 23, -1, 10, +3, 4, -1, 14, 15, 24, 25, 26, -1, 5, +11, 12, -1, 27, 28, 29, 30, 31, -1, 13 diff --git a/clean/k/dx12/k_plan_bridge.py b/clean/k/dx12/k_plan_bridge.py index fe3d1f54..d1912719 100644 --- a/clean/k/dx12/k_plan_bridge.py +++ b/clean/k/dx12/k_plan_bridge.py @@ -13,8 +13,11 @@ import ctypes import json import os +from pathlib import Path import struct +from clean.k.runner.k_pack import block_sections, linear_sections, stage_sections + HIP_SENTINEL, D3D12_SENTINEL, SLACK = 0xA5, 0x5A, 64 @@ -43,6 +46,76 @@ def __getattr__(self, name): return getattr(self.hip, name) +def head_rows(): + """Physical rows for the 40 logical channels, shared with the C++ packer and shader.""" + path = Path(__file__).with_name("hlsl") / "k_head_rows.inc" + values = " ".join(line.split("//", 1)[0] for line in path.read_text(encoding="utf-8").splitlines()) + return tuple(int(value) for value in values.split(",")) + + +def _bias32(half): + """Widen FP16 bits as k_bias32_bits does, including NaN sign and payload.""" + payload = half & 0x3FF + if half & 0x7C00 == 0x7C00 and payload: + return (half & 0x8000) << 16 | 0x7FC00000 | payload << 13 + return struct.unpack(" its argument tokens, from /dx12/manifest.json.""" with open(os.path.join(build, "dx12", "manifest.json")) as f: diff --git a/clean/k/dx12/k_port_check.py b/clean/k/dx12/k_port_check.py index a18578a7..b1e086f4 100644 --- a/clean/k/dx12/k_port_check.py +++ b/clean/k/dx12/k_port_check.py @@ -53,7 +53,7 @@ CONTROL = "scale_rgb" # Model M, frame.co # family -> (HIP code objects, D3D12 kernels) NARROW_KERNELS = ["k_stage_e2", "k_stage_e0", "k_stage_e1", "k_stage_d2", "k_stage_d1", "k_stage2_e0", "k_stage2_e1", - "k_stage2_d1", "k_stage2_d0_hp", "k_stage2_d0_hidden"] + "k_stage2_d1", "k_stage2_d1__rodd", "k_stage2_d0_hp", "k_stage2_d0_hidden"] FAMILY_KERNELS = {"expo": (["k_io.co"], ["k_expo_scalars"]), "warm": ([], ["k_warm"]), "mixprobe": ([], ["k_mix_probe", "k_mix_probe__rodd"]), "narrow": (["k_block.co"], NARROW_KERNELS), @@ -359,7 +359,9 @@ def check_mixprobe(items, read): NARROW = {"k_stage_e2": ("e2", "", None), "k_stage_e0": ("e0", "", None), "k_stage_e1": ("e1", "", None), "k_stage_d2": ("d2", "", None), "k_stage_d1": ("d1", "", None), "k_stage2_e0": ("e0", "@w2", None), "k_stage2_e1": ("e1", "@w2", None), "k_stage2_d1": ("d1", "@w2", None), + "k_stage2_d1__rodd": ("d1", "@w2", None), "k_stage2_d0_hp": ("d0", "@w2", "hp"), "k_stage2_d0_hidden": ("d0", "@w2", "hidden")} +NARROW_BIAS32 = {"k_stage2_e0", "k_stage2_e1", "k_stage2_d1", "k_stage2_d1__rodd", "k_stage2_d0_hp"} # block-level sizes (W, H) and window offsets (spec 44: even): grids with fewer than a workgroup's windows among them NARROW_SIZES = [(8, 8), (24, 16), (17, 13), (40, 24), (9, 33), (12, 20)] NARROW_OFFSETS = [(0, 0), (2, 2), (4, 0), (0, 6), (6, 4)] @@ -433,6 +435,12 @@ def run_stages(table, co_name, tag, build, rocm, seeds, cdir, hip, grid_list=Non plan["launches"] = [(kernel, grid, block, args + [ctypes.c_void_p(hid)])] plan["outputs"] = [("hidden", (H, W, 4))] + ( [("head", (2 * H, 2 * W, 4))] if d0 == "hp" else [("head", (H, W, 40))]) + k, grid, block, hip_args = plan["launches"][0] + assert k == kernel.removesuffix("__rodd"), (k, kernel) + dx_args = hip_args + if kernel in NARROW_BIAS32: + dx_args = hip_args + PB.narrow_d3d12_args(plan, pdev, st, blob) + dx_plan = dict(plan, launches=[(kernel, grid, block, dx_args)]) outputs = [n for n, _ in plan["outputs"]] for n, p in plan["bufs"].items(): if p in pdev.inputs: @@ -444,11 +452,11 @@ def run_stages(table, co_name, tag, build, rocm, seeds, cdir, hip, grid_list=Non sizes = {n: pdev.sizes[plan["bufs"][n]] for n in outputs} got = {n: bytes(hip.h.download(plan["bufs"][n], sizes[n] + SLACK)) for n in outputs} defined = {n: min(int(np.prod(sh)) * itemsize(n), sizes[n]) for n, sh in plan["outputs"]} - lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, [co_name], + lines += PB.chain_lines(name, dx_plan, pdev, lambda k: tokens[k], cdir, [co_name], outputs) - if neg is None and plan["launches"][0][1] >= 4: # the dropped windows hold output pixels + if neg is None and grid >= 4: # the dropped windows hold output pixels neg = name - lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, + lines += PB.chain_lines(name, dx_plan, pdev, lambda k: tokens[k], cdir, [co_name], outputs, prefix="negative_", grid_div=2) items.append({"case": name, "kernel": kernel, "stage": stage, "size": list(size), "offset": list(offset), "specials": special, "outputs": outputs, "hip": got, diff --git a/clean/k/dx12/kernels/k_block.hlsl b/clean/k/dx12/kernels/k_block.hlsl index 1ce5ac9d..33484bde 100644 --- a/clean/k/dx12/kernels/k_block.hlsl +++ b/clean/k/dx12/kernels/k_block.hlsl @@ -17,24 +17,25 @@ // The FP16-accumulating forms (k_stageh_*) and the FP8 forms are not ported: the exact path only. // Basis: our HIP kernels (spec 41 block, spec 44 windows, spec 43 6.8 hidden history, spec 40 table); exactness and the // forms that differ (bit-identical) are listed in hlsl/k_block.hlsli and hlsl/k_common.hlsli. Arguments as HIP: -// KBlkParams by value (token s184:p19.b32, 27 constants), the d0 stages also the hidden history pointer (",p"). +// KBlkParams by value (token s184:p19.b32, 27 constants), the d0 stages also take the hidden history pointer. +// The narrow two-window ports take a FP32 bias plane, and d0_hp takes a separate compact head weight pointer. // Launch as HIP: 128 lanes (4 waves of 32) per workgroup, one workgroup per window (G = 1) or per two windows (G = 2). // // @wg-kernel k_stage_e2 co=k_block.co block=128 wave=32 args=s184:p19.b32 -// @wg-kernel k_stage2_e0 co=k_block.co block=128 wave=32 args=s184:p19.b32 -// @wg-kernel k_stage2_e1 co=k_block.co block=128 wave=32 args=s184:p19.b32 +// @wg-kernel k_stage2_e0 co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_BIAS32=1,KB_BASE_FORM=1 +// @wg-kernel k_stage2_e1 co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_BIAS32=1,KB_BASE_FORM=1 // @wg-kernel k_stage_d2 co=k_block.co block=128 wave=32 args=s184:p19.b32 -// @wg-kernel k_stage2_d1 co=k_block.co block=128 wave=32 args=s184:p19.b32 +// @wg-kernel k_stage2_d1 co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_BIAS32=1,KB_BASE_FORM=1 // @wg-kernel k_stage_e0 co=k_block.co block=128 wave=32 args=s184:p19.b32 defines=KB_LDS_HALFS=6144 // @wg-kernel k_stage_e1 co=k_block.co block=128 wave=32 args=s184:p19.b32 defines=KB_LDS_HALFS=4096 // @wg-kernel k_stage_d1 co=k_block.co block=128 wave=32 args=s184:p19.b32 -// @wg-kernel k_stage2_d0_hp co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_HIDDEN_ARG=1 +// @wg-kernel k_stage2_d0_hp co=k_block.co block=128 wave=32 args=s184:p19.b32,p,p,p defines=KB_HIDDEN_ARG=1,KB_COMPACT_HEAD_ARG=1,KB_BIAS32=1,KB_BASE_FORM=1 // @wg-kernel k_stage2_d0_hidden co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_HIDDEN_ARG=1 // The __rodd forms: the same stages with the round-to-odd softmax weights (hlsl/k_smx.hlsli K_SMX 1), launched by the // native D3D12 runtime when the mix-form probe (kernels/k_mix_probe.hlsl) fails on the adapter's driver. // @wg-kernel k_stage_e2__rodd co=k_block.co entry=k_stage_e2 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1 // @wg-kernel k_stage_d2__rodd co=k_block.co entry=k_stage_d2 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1 -// @wg-kernel k_stage2_d1__rodd co=k_block.co entry=k_stage2_d1 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1 +// @wg-kernel k_stage2_d1__rodd co=k_block.co entry=k_stage2_d1 block=128 wave=32 args=s184:p19.b32,p defines=K_SMX=1,KB_BIAS32=1,KB_BASE_FORM=1 #include "wg_platform.hlsli" #include "k_blkparams.hlsli" @@ -43,6 +44,12 @@ WG_ARGS_BEGIN #if KB_HIDDEN_ARG uint hidden_p; // the hidden history Hh written by the d0 epilogue #endif +#if KB_BIAS32 + uint P_b32; +#endif +#if KB_COMPACT_HEAD_ARG + uint P_compact_head; +#endif WG_ARGS_END #include "k_block.hlsli" @@ -50,13 +57,20 @@ WG_ARGS_END WG_KERNEL(k_stage2_d0_hp) { WG_SETUP(); +#if KB_COMPACT_HEAD_ARG + k_block<32, 2, KMIX_FIXED, KIN_EXPAND, 64, 0, true, 2, true, true, false>(hidden_p, wg_block_id(), P_b32, + P_compact_head); +#else k_block<32, 2, KMIX_FIXED, KIN_EXPAND, 64, 0, true, 2, true, true, false>(hidden_p, wg_block_id()); +#endif } +#if !KB_COMPACT_HEAD_ARG WG_KERNEL(k_stage2_d0_hidden) { WG_SETUP(); k_block<32, 2, KMIX_FIXED, KIN_EXPAND, 64, 0, true, 2, true, false, false>(hidden_p, wg_block_id()); } +#endif #else WG_KERNEL(k_stage_e2) { @@ -66,12 +80,20 @@ WG_KERNEL(k_stage_e2) WG_KERNEL(k_stage2_e0) { WG_SETUP(); +#if KB_BIAS32 + k_block<32, 2, KMIX_FIXED, KIN_EMBED, 16, 64, false, 2, false, false, false>(0u, wg_block_id(), P_b32); +#else k_block<32, 2, KMIX_FIXED, KIN_EMBED, 16, 64, false, 2, false, false, false>(0u, wg_block_id()); +#endif } WG_KERNEL(k_stage2_e1) { WG_SETUP(); +#if KB_BIAS32 + k_block<64, 2, KMIX_FIXED, KIN_MAP, 64, 64, false, 2, false, false, false>(0u, wg_block_id(), P_b32); +#else k_block<64, 2, KMIX_FIXED, KIN_MAP, 64, 64, false, 2, false, false, false>(0u, wg_block_id()); +#endif } WG_KERNEL(k_stage_d2) { @@ -81,7 +103,11 @@ WG_KERNEL(k_stage_d2) WG_KERNEL(k_stage2_d1) { WG_SETUP(); +#if KB_BIAS32 + k_block<64, 2, KMIX_SOFTMAX, KIN_EXPAND, 64, 0, false, 2, false, false, true>(0u, wg_block_id(), P_b32); +#else k_block<64, 2, KMIX_SOFTMAX, KIN_EXPAND, 64, 0, false, 2, false, false, true>(0u, wg_block_id()); +#endif } WG_KERNEL(k_stage_e0) { diff --git a/clean/k/runtime/k_pack.cpp b/clean/k/runtime/k_pack.cpp index 035d8676..b2cdfa8d 100644 --- a/clean/k/runtime/k_pack.cpp +++ b/clean/k/runtime/k_pack.cpp @@ -186,4 +186,26 @@ std::vector k_pack_stage(const std::string& stage, const StageTensors& return blob; } +std::vector k_pack_compact_head(const uint8_t* original, size_t channels) { + constexpr int row[40] = { +#include "../dx12/hlsl/k_head_rows.inc" + }; + const size_t kt_count = channels / 16; + const size_t old_bias = (48 * channels * 2 + kAlign - 1) / kAlign * kAlign; + const size_t new_bias = (32 * channels * 2 + kAlign - 1) / kAlign * kAlign; + std::vector out(new_bias + kAlign, 0); + for (size_t logical = 0; logical < 40; ++logical) { + if (row[logical] < 0) continue; + const size_t physical = (size_t)row[logical]; + for (size_t kt = 0; kt < kt_count; ++kt) + for (size_t half = 0; half < 2; ++half) { + const size_t from = (logical / 16 * kt_count + kt) * 512 + (16 * half + logical % 16) * 16; + const size_t to = (physical / 16 * kt_count + kt) * 512 + (16 * half + physical % 16) * 16; + std::memcpy(out.data() + to, original + from, 16); + } + std::memcpy(out.data() + new_bias + physical * 2, original + old_bias + logical * 2, 2); + } + return out; +} + } // namespace wgrt diff --git a/clean/k/runtime/k_pack.h b/clean/k/runtime/k_pack.h index e8822450..f81c7150 100644 --- a/clean/k/runtime/k_pack.h +++ b/clean/k/runtime/k_pack.h @@ -7,6 +7,7 @@ // clean/k/reference/params.py normalize; byte-identical blobs (clean/k/tests/test_k_runtime_pack.py). #pragma once +#include #include #include #include @@ -22,4 +23,7 @@ namespace wgrt { // k_stage_blob_bytes(*k_stage(stage)) bytes. On failure returns an empty vector with a message in *err. std::vector k_pack_stage(const std::string& stage, const StageTensors& t, std::string* err); +// Repack the 32 live rows of d0's head from the original gfx12 linear blob. +std::vector k_pack_compact_head(const uint8_t* original, size_t channels); + } // namespace wgrt diff --git a/clean/k/runtime/k_smx_selfcheck.h b/clean/k/runtime/k_smx_selfcheck.h index 01c8aaf0..788f8302 100644 --- a/clean/k/runtime/k_smx_selfcheck.h +++ b/clean/k/runtime/k_smx_selfcheck.h @@ -8,7 +8,7 @@ namespace k_smx_selfcheck { -constexpr char kSourceDigest[] = "ef7da006529622c2fa87d1d3371038cdf85467b1fb0010b361f334784b6fa92b"; +constexpr char kSourceDigest[] = "a47546460b78b46b3ae0775661c6c4160ad20af09b6106c16322f8d164136264"; constexpr int kCount = 133; constexpr const char* kNames[kCount] = { "k_stage_e2@512x384/o4,4", diff --git a/clean/k/runtime/runtime_k.cpp b/clean/k/runtime/runtime_k.cpp index 59c76a57..92e8d103 100644 --- a/clean/k/runtime/runtime_k.cpp +++ b/clean/k/runtime/runtime_k.cpp @@ -138,6 +138,39 @@ StageSec stage_sections(const KStageInfo& s) { // order: embed, expand, block, return r; } +// SMX-SELFCHECK-SOURCE BEGIN (the narrow d1 bias plane used by the self-check) +// Bias widening for the FP32 planes. Preserve the binary16 NaN payload and quiet signaling NaNs. +uint32_t k_bias32_bits(uint16_t h) { + const uint32_t payload = h & 0x3ffu; + if ((h & 0x7c00u) == 0x7c00u && payload) + return ((uint32_t)(h & 0x8000u) << 16) | 0x7fc00000u | (payload << 13); + return f32_bits(f16_to_f32(h)); +} + +std::vector k_narrow_bias32(const KStageInfo& s, const uint8_t* packed, + const uint8_t* compact_head = nullptr) { + const auto stage = stage_sections(s); + const auto block = block_sections(s); + std::vector plane; + const auto append = [&](const uint8_t* data, size_t count) { + for (size_t i = 0; i < count; ++i) { + uint16_t half; + std::memcpy(&half, data + i * sizeof(half), sizeof(half)); + plane.push_back(k_bias32_bits(half)); + } + }; + if (s.embed) append(packed + stage.embed + linear_bias(s.C, 16), s.C); + if (s.kind == 'd') append(packed + stage.expand + linear_bias(4 * s.C, s.C_prev), 4 * s.C); + append(packed + stage.block + block.bo, s.C); + append(packed + stage.block + block.b1, 4 * s.C); + append(packed + stage.block + block.b2, s.C); + if (s.kind == 'e') append(packed + stage.merge + linear_bias(s.C_next, 4 * s.C), s.C_next); + if (s.head) append(compact_head + linear_bias(32, s.C), 32); + if (!s.fixed) append(packed + stage.block + block.mix, (size_t)s.heads * 64 * 64); + return plane; +} +// SMX-SELFCHECK-SOURCE END + // ---- network=fused8: FP8 forms of the level-0/1 stages (clean/k/kernels/k_block8.h) -------------------------------- static_assert(sizeof(KF8Params) == 104, "KF8Params of k_block8.h"); float blob_h(const std::vector& b, size_t off, size_t i) { @@ -662,6 +695,27 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e if (!alloc(&p, kv.second.size()) || !dev_->upload(p, kv.second.data(), kv.second.size())) return false; wdev_[kv.first] = p; } + if (!f16_ && (c.backend == 2 || c.backend == 3)) { + for (const char* name : {"e0", "e1", "d1", "d0"}) { + const KStageInfo& st = *k_stage(name); + if (f8data_.count(name) || (st.head && !compact_head_)) continue; + const std::string kernel = std::string("k_stage2_") + name + (st.head ? "_hp" : ""); + if (!dev_->require_d3d12_port(kernel, st.head ? "s184:p19.b32,p,p,p" : "s184:p19.b32,p")) return dfail(); + if (c.backend == 3 && std::strcmp(name, "d1") == 0 && + !dev_->require_d3d12_port(kernel + "__rodd", "s184:p19.b32,p")) return dfail(); + const auto& weight = blobs_.at(name); + std::vector compact; + if (st.head) { + compact = k_pack_compact_head(weight.data() + stage_sections(st).head, st.C); + if (!alloc(&d0_compact_head_, compact.size()) || + !dev_->upload(d0_compact_head_, compact.data(), compact.size())) return false; + } + const auto bias = k_narrow_bias32(st, weight.data(), compact.data()); + const size_t bytes = bias.size() * sizeof(bias[0]); + DevPtr& plane = narrow_bias_[name]; + if (!alloc(&plane, bytes) || !dev_->upload(plane, bias.data(), bytes)) return false; + } + } for (auto& kv : f8data_) { // network=fused8: the FP8 weights and FP32 biases; device pointers into KF8Params DevPtr p; if (!alloc(&p, kv.second.bytes.size()) || !dev_->upload(p, kv.second.bytes.data(), kv.second.bytes.size())) @@ -868,7 +922,7 @@ bool KRuntime::smx_check(bool rodd, bool all, std::vector in(in_bytes / 2), w(w_bytes / 2); - DevPtr din = 0, dw = 0, dy = 0, dm = 0; + DevPtr din = 0, dw = 0, dy = 0, dm = 0, db = 0; int loaded = -1; // the data set in din / dw auto load = [&](int set) { if (set == loaded) return true; @@ -881,6 +935,11 @@ bool KRuntime::smx_check(bool rodd, bool all, std::vectoralloc(&din, in_bytes) && dev_->alloc(&dw, w_bytes) && dev_->alloc(&dy, y_bytes) && dev_->alloc(&dm, std::max(m_bytes, 256)); + if (ok && narrow_bias_.count("d1")) { + const auto& d1 = *k_stage("d1"); + const size_t words = 10u * d1.C + (size_t)d1.heads * 64 * 64; + ok = dev_->alloc(&db, words * sizeof(uint32_t)); + } std::vector buf; for (int i = 0; ok && i < kSmxCount; ++i) { const SmxCase& c = kSmxCases[i]; @@ -896,11 +955,17 @@ bool KRuntime::smx_check(bool rodd, bool all, std::vectormemset(dy, 0, y_bytes) && dev_->memset(dm, 0, std::max(m_bytes, 256)) && + ok = load(set); + if (ok && use_bias) { + const auto bias = k_narrow_bias32(st, reinterpret_cast(w.data())); + ok = dev_->upload(db, bias.data(), bias.size() * sizeof(bias[0])); + } + ok = ok && dev_->memset(dy, 0, y_bytes) && dev_->memset(dm, 0, std::max(m_bytes, 256)) && launch(k.c_str(), (unsigned)(w2 ? cdiv(p.nwx * p.nwy, 2) : p.nwx * p.nwy), - (unsigned)(kThreads * (ng ? ng : 1)), &p) && + (unsigned)(kThreads * (ng ? ng : 1)), &p, use_bias ? &db : nullptr) && dev_->download(buf.data(), dy, yb) && (!mb || dev_->download(buf.data() + yb, dm, mb)); if (ok) sha.update(buf.data(), buf.size()); } @@ -911,7 +976,7 @@ bool KRuntime::smx_check(bool rodd, bool all, std::vectoremplace_back(smx_case_name(i, o.first, o.second, false), v); } } - for (DevPtr p : {din, dw, dy, dm}) + for (DevPtr p : {din, dw, dy, dm, db}) if (p) dev_->free(p); return ok || dfail(); } @@ -1141,8 +1206,18 @@ bool KRuntime::network(int f, const std::function& after) { // k_head=compact: p.head (the OUT buffer) receives the compact pixel head [2 Bh][2 Bw][4] instead of the head map const char* k0 = compact_head_ ? (f16_ ? "k_stage2h_d0_hp" : "k_stage2_d0_hp") : (f16_ ? "k_stage2h_d0_hidden" : "k_stage2_d0_hidden"); - if (!launch(k0, grid, kThreads, &p, &hidden)) return false; - } else if (!launch(stage_kernel(k, st.name).c_str(), grid, (unsigned)(kThreads * (ng ? ng : 1)), &p)) return false; + if (compact_head_ && !f16_ && d0_compact_head_) { + if (!launch(k0, grid, kThreads, &p, &hidden, &narrow_bias_.at(st.name), &d0_compact_head_)) return false; + } else if (!launch(k0, grid, kThreads, &p, &hidden)) { + return false; + } + } else { + const std::string kernel = stage_kernel(k, st.name); + const auto bias_plane = narrow_bias_.find(st.name); + const bool use_bias = k == "k_stage2_e0" || k == "k_stage2_e1" || k == "k_stage2_d1"; + void* bias = use_bias && bias_plane != narrow_bias_.end() ? &bias_plane->second : nullptr; + if (!launch(kernel.c_str(), grid, (unsigned)(kThreads * (ng ? ng : 1)), &p, bias)) return false; + } if (after && !after(st.name)) return false; if (i < 10) { info_.offsets[i][0] = ox; // WgRtFrameInfo has ten rows: e0 ... d1 (d0's offset is (4, 4)) diff --git a/clean/k/runtime/runtime_k.h b/clean/k/runtime/runtime_k.h index 7fa1dff6..311ccd2a 100644 --- a/clean/k/runtime/runtime_k.h +++ b/clean/k/runtime/runtime_k.h @@ -164,6 +164,8 @@ class KRuntime { bool expo_v051_ = false; // runtime option k_expo_rule=v051 (A/B only): the 0.5.1 exposure rule int offsets_table_[8][2] = {}; std::map> blobs_; + std::map narrow_bias_; // FP32 bias planes for the D3D12 two-window narrow stages + DevPtr d0_compact_head_ = 0; // the compact head's two permuted tiles std::map wdev_, net_; // stage blobs; network maps (X0, S0..S4, A, B, OUT) std::map net_bytes_; DevPtr Hc_[2] = {0, 0}, L_[2] = {0, 0}, Hh_[2] = {0, 0}; diff --git a/clean/k/tests/test_k_plan_bridge.py b/clean/k/tests/test_k_plan_bridge.py index 2873fbb0..ed8742db 100644 --- a/clean/k/tests/test_k_plan_bridge.py +++ b/clean/k/tests/test_k_plan_bridge.py @@ -49,6 +49,17 @@ def test_chain_step_of_a_block_launch(self): ["p", "p", "u", "u", "b4"], bufs, d, "case.l1") self.assertEqual(line, "step k 1 32 null @b_weights+8 4294967293 1065353216 B:case.l1.a4") + def test_compact_head_rows_cover_pixel_and_hidden_channels(self): + rows = PB.head_rows() + self.assertEqual(len(rows), 40) + used = {10 * slot + offset for slot in range(4) for offset in (0, 1, 3, 4, 5, 6, 7, 9)} + self.assertEqual(sorted(rows[ch] for ch in used), list(range(32))) + self.assertTrue(all(rows[ch] == -1 for ch in set(range(40)) - used)) + for half in range(2): + slot = 10 * half + channels = (slot, slot + 1, slot + 9, slot + 20, slot + 21, slot + 29) + self.assertEqual([rows[ch] for ch in channels], list(range(8 * half, 8 * half + 6))) + def test_argument_count_mismatch_is_refused(self): with tempfile.TemporaryDirectory() as d: with self.assertRaises(ValueError): diff --git a/clean/k/tests/test_k_runtime_d3d12.py b/clean/k/tests/test_k_runtime_d3d12.py index a0f060da..8bdc5e68 100644 --- a/clean/k/tests/test_k_runtime_d3d12.py +++ b/clean/k/tests/test_k_runtime_d3d12.py @@ -43,6 +43,7 @@ import subprocess import sys import unittest +from unittest.mock import patch import numpy as np @@ -207,6 +208,12 @@ def test_hybrid_hdr_auto_exposure(self): def test_hybrid_ldr_rows(self): self.check_hybrid(False, 0, True) + def test_hybrid_refuses_missing_compact_head_port(self): + selected = ",".join(stage for stage in STAGES if stage != "k_stage2_d0_hp") + with patch.dict(os.environ, {"WG_DX12_KERNELS": selected}): + with self.assertRaisesRegex(RuntimeError, "required port unavailable for k_stage2_d0_hp"): + self.make(2, True, False, "", OUT, MAX_RENDER) + def test_hybrid_head_full(self): self.check_hybrid(True, 0, True, "k_head=full") diff --git a/clean/runtime/device.h b/clean/runtime/device.h index 3cf5c021..0ed360f8 100644 --- a/clean/runtime/device.h +++ b/clean/runtime/device.h @@ -49,6 +49,12 @@ class Device { } // args: pointers to the argument values (hipModuleLaunchKernel convention). virtual bool launch(const std::string& kernel, unsigned grid, unsigned block, void** args) = 0; + // Require a D3D12 port with this argument ABI for the lifetime of the context. + virtual bool require_d3d12_port(const std::string& kernel, const char* signature) { + (void)signature; + err_ = "D3D12 port required for " + kernel; + return false; + } // Loads, verifies and compiles one kernel now (its first launch then compiles nothing); false with err() when its // shader is missing or corrupt. Devices that prepare nothing ahead: true. virtual bool prepare(const std::string& kernel) { (void)kernel; return true; } diff --git a/clean/runtime/device_d3d12.cpp b/clean/runtime/device_d3d12.cpp index 31bc79d2..5da26095 100644 --- a/clean/runtime/device_d3d12.cpp +++ b/clean/runtime/device_d3d12.cpp @@ -71,7 +71,7 @@ constexpr unsigned kProfileQueries = 4096; struct NativeArg { char kind; unsigned bytes; std::vector fields = {}; }; // 's': struct_args.h struct NativeKernel { - std::string name, file, co, rs = "buffer"; + std::string name, file, co, signature, rs = "buffer"; unsigned block = 0, wave = 0, dwords = 0; // group: lanes per D3D12 workgroup when the port splits each launched workgroup of `block` lanes into block / group // workgroups (manifest "group", build_dx12.py group=; 0 = block). The launch keeps the HIP geometry (grid x block). @@ -756,6 +756,12 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (variant_matches(i->second)) return &i->second; return range.first == range.second ? nullptr : &range.first->second; } + bool require_d3d12_port(const std::string& name, const char* signature) override { + const NativeKernel* port = find_port(name); + if (!port || !variant_matches(*port)) + return fail("native D3D12: required port unavailable for " + name); + return port->signature == signature || fail("native D3D12: incompatible argument signature for " + name); + } bool prepare(const std::string& name) override { NativeKernel* port = find_port(name); if (!port) return fail("native D3D12 unsupported kernel: " + name + "; no HIP fallback"); @@ -1450,6 +1456,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { !wgc::parse_digest_hex(hash, k.digest) || k.block == 0 || k.block > 1024 || (k.wave != 32 && k.wave != 64) || k.block % k.wave || k.dwords > kNativeArgs) return fail("native D3D12 manifest: invalid kernel metadata for " + k.name); + k.signature = arguments; unsigned words = 0; for (size_t start = 0; start < arguments.size();) { const size_t end = arguments.find(',', start); diff --git a/clean/runtime/device_dx12.cpp b/clean/runtime/device_dx12.cpp index d06e9c60..5ff8cd37 100644 --- a/clean/runtime/device_dx12.cpp +++ b/clean/runtime/device_dx12.cpp @@ -95,6 +95,7 @@ struct Arg { struct Kernel { std::string name, file, co; // co: file name of the HIP code object this port reproduces + std::string signature; unsigned block = 0, wave = 32, dwords = 0; unsigned group = 0; // lanes per D3D12 workgroup of a split port (manifest "group"; 0 = block) unsigned gscale = 1; // D3D12 workgroups per launched workgroup (manifest "gscale", default block / group) @@ -239,13 +240,27 @@ class Dx12Device : public Device { bool set_stream_priority(bool high) override { return fwd(hip_->set_stream_priority(high)); } void begin_teardown() override { hip_->begin_teardown(); } + bool require_d3d12_port(const std::string& name, const char* signature) override { + const Kernel* port = find_port(name); + if (!port || !arena_ || !variant_selected(*port)) + return fail("hybrid D3D12: required port unavailable for " + name); + if (port->signature != signature) + return fail("hybrid D3D12: incompatible argument signature for " + name); + required_ports_.insert(name); + return true; + } + // ---- launches ---------------------------------------------------------------------------------------------------- bool launch(const std::string& kernel, unsigned grid, unsigned block, void** args) override { Kernel* port = find_port(kernel); uint32_t c[kRuntimeConstants + kMaxArgDwords]; - if (!port || !arena_ || !variant_loaded(*port) || !pack_args(*port, args, c + kRuntimeConstants)) { - return to_hip() && fwd(hip_->launch(kernel, grid, block, args)); - } + if (!port || !arena_ || !variant_loaded(*port)) + return allow_hip_fallback(kernel) && to_hip() && fwd(hip_->launch(kernel, grid, block, args)); + if (!args && !port->args.empty()) return fail("hybrid D3D12 kernel " + kernel + ": null argument array"); + for (size_t i = 0; i < port->args.size(); ++i) + if (!args[i]) return fail("hybrid D3D12 kernel " + kernel + ": null argument value"); + if (!pack_args(*port, args, c + kRuntimeConstants)) + return allow_hip_fallback(kernel) && to_hip() && fwd(hip_->launch(kernel, grid, block, args)); Kernel& k = *port; if (block != k.block) return fail("D3D12 kernel " + kernel + " is built for " + std::to_string(k.block) + " lanes, launched with " + @@ -276,11 +291,16 @@ class Dx12Device : public Device { int stop_event) override { Kernel* port = find_port(kernel); if (!port || !arena_ || !variant_loaded(*port)) - return to_hip() && fwd(hip_->launch_timed(kernel, grid, block, args, start_event, stop_event)); + return allow_hip_fallback(kernel) && to_hip() && + fwd(hip_->launch_timed(kernel, grid, block, args, start_event, stop_event)); return Device::launch_timed(kernel, grid, block, args, start_event, stop_event); } private: + bool allow_hip_fallback(const std::string& kernel) { + return !required_ports_.count(kernel) || fail("hybrid D3D12: required port cannot run " + kernel); + } + bool fail(const std::string& m) { err_ = m; return false; @@ -532,6 +552,7 @@ class Dx12Device : public Device { return fail("D3D12 manifest " + path + ": incomplete kernel entry"); x.name = n->text; x.file = dir + fl->text; + x.signature = ar->text; if (co) x.co = co->text; // missing (an old manifest): the port is never used x.block = (unsigned)block; if (const wgc::Json* gp = k.find("group")) { @@ -784,6 +805,7 @@ class Dx12Device : public Device { uint64_t arena_bytes_ = 0, in_use_ = 0, fence_value_ = 0; std::map free_, used_; // offset -> bytes std::multimap kernels_; // by name; several ports of one name differ in "co" + std::set required_ports_; std::set untagged_cos_; // file names of the code objects loaded without a tag std::map tagged_cos_; // tag -> file name of the code object loaded under it unsigned module_gen_ = 0;