diff --git a/clean/dx12/hlsl/l_coef_px.hlsli b/clean/dx12/hlsl/l_coef_px.hlsli index 12b4c1ac..4869b776 100644 --- a/clean/dx12/hlsl/l_coef_px.hlsli +++ b/clean/dx12/hlsl/l_coef_px.hlsli @@ -40,10 +40,11 @@ float clamp20(float v) { return select(v >= 20.0f, 20.0f, select(v <= -20.0f, -2 // buffers (pB unused with COEF_PACK12); i: the pixel's index in the render rectangle (y rw + x). void l_coef_px(uint4 lo, uint4 hi, float S, float T, uint pA, uint pB, uint pC, uint pL, uint i) { - const float a0 = wg_h2f_lo(lo.x), a1 = wg_h2f_hi(lo.x), a2 = wg_h2f_lo(lo.w), a3 = wg_h2f_hi(lo.w); - const float a4 = wg_h2f_lo(hi.x), a5 = wg_h2f_hi(hi.x), a6 = wg_h2f_lo(hi.y), a7 = wg_h2f_hi(hi.y); - const float a8 = wg_h2f_lo(hi.z); - const float a9 = wg_h2f_lo(lo.y), a10 = wg_h2f_hi(lo.y), a11 = wg_h2f_lo(lo.z), a12 = wg_h2f_hi(lo.z); + // f16tof32 directly, not wg_h2f_lo: its mask costs a v_mov_b16 per FP16 pair (see hyb_h2f_lo in wg_hyb.hlsli) + const float a0 = f16tof32(lo.x), a1 = wg_h2f_hi(lo.x), a2 = f16tof32(lo.w), a3 = wg_h2f_hi(lo.w); + const float a4 = f16tof32(hi.x), a5 = wg_h2f_hi(hi.x), a6 = f16tof32(hi.y), a7 = wg_h2f_hi(hi.y); + const float a8 = f16tof32(hi.z); + const float a9 = f16tof32(lo.y), a10 = wg_h2f_hi(lo.y), a11 = f16tof32(lo.z), a12 = wg_h2f_hi(lo.z); // A: unorm codes of RN16(sigma(a0..a2)) const uint cA = unorm8_code(wg_rn16(l_sigma(a0))) | (unorm8_code(wg_rn16(l_sigma(a1))) << 8) | (unorm8_code(wg_rn16(l_sigma(a2))) << 16); @@ -104,10 +105,11 @@ bool coef_mul_zero(float a, float b) LCoefVals l_coef_vals_hw(uint4 lo, uint4 hi, float S, float T) { - const float a0 = wg_h2f_lo(lo.x), a1 = wg_h2f_hi(lo.x), a2 = wg_h2f_lo(lo.w), a3 = wg_h2f_hi(lo.w); - const float a4 = wg_h2f_lo(hi.x), a5 = wg_h2f_hi(hi.x), a6 = wg_h2f_lo(hi.y), a7 = wg_h2f_hi(hi.y); - const float a8 = wg_h2f_lo(hi.z); - const float a9 = wg_h2f_lo(lo.y), a10 = wg_h2f_hi(lo.y), a11 = wg_h2f_lo(lo.z), a12 = wg_h2f_hi(lo.z); + // f16tof32 directly, not wg_h2f_lo: its mask costs a v_mov_b16 per FP16 pair (see hyb_h2f_lo in wg_hyb.hlsli) + const float a0 = f16tof32(lo.x), a1 = wg_h2f_hi(lo.x), a2 = f16tof32(lo.w), a3 = wg_h2f_hi(lo.w); + const float a4 = f16tof32(hi.x), a5 = wg_h2f_hi(hi.x), a6 = f16tof32(hi.y), a7 = wg_h2f_hi(hi.y); + const float a8 = f16tof32(hi.z); + const float a9 = f16tof32(lo.y), a10 = wg_h2f_hi(lo.y), a11 = f16tof32(lo.z), a12 = wg_h2f_hi(lo.z); LCoefVals o; // RN16 of sigma(a0), sigma(a1), sigma(a2), sigma(clamp20(a6)), sigma(clamp20(a7)): one conversion v8f sg; @@ -116,7 +118,8 @@ LCoefVals l_coef_vals_hw(uint4 lo, uint4 hi, float S, float T) sg.v[5] = 0.0f; sg.v[6] = 0.0f; sg.v[7] = 0.0f; const uint4 hs = wg_to_f16_ags(sg); // A: unorm codes of RN16(sigma(a0..a2)) - o.cA = unorm8_code(wg_h2f_lo(hs.x)) | (unorm8_code(wg_h2f_hi(hs.x)) << 8) | (unorm8_code(wg_h2f_lo(hs.y)) << 16); + // f16tof32 directly, not wg_h2f_lo: its mask costs a v_mov_b16 per FP16 pair (see hyb_h2f_lo in wg_hyb.hlsli) + o.cA = unorm8_code(f16tof32(hs.x)) | (unorm8_code(wg_h2f_hi(hs.x)) << 8) | (unorm8_code(f16tof32(hs.y)) << 16); // B: softmax of a3..a5 const float m = max(max(a3, a4), a5); const float e0 = wg_exp2((a3 - m) * bits(0x3FB8AA3Bu)), e1 = wg_exp2((a4 - m) * bits(0x3FB8AA3Bu)), @@ -124,7 +127,8 @@ LCoefVals l_coef_vals_hw(uint4 lo, uint4 hi, float S, float T) const float rs = wg_rcp(e2 + (e0 + e1)); o.cB = unorm8_code(opq(e0 * rs)) | (unorm8_code(opq(e1 * rs)) << 8) | (unorm8_code(opq(e2 * rs)) << 16); // C: spreads and correlation; the three FP16 products in one conversion - const float g0 = wg_h2f_hi(hs.y), g1 = wg_h2f_lo(hs.z); + // f16tof32 directly, not wg_h2f_lo: its mask costs a v_mov_b16 per FP16 pair (see hyb_h2f_lo in wg_hyb.hlsli) + const float g0 = wg_h2f_hi(hs.y), g1 = f16tof32(hs.z); const float sx = S * g0, sy = S * g1; const float sxy = sx * sy; const float rho = l_tanh(a8 * T); diff --git a/clean/dx12/hlsl/l_lin_px.hlsli b/clean/dx12/hlsl/l_lin_px.hlsli index 78d41d9b..3824fe15 100644 --- a/clean/dx12/hlsl/l_lin_px.hlsli +++ b/clean/dx12/hlsl/l_lin_px.hlsli @@ -38,7 +38,8 @@ void lin_pair(uint2 xf[2], v8f bias[2], uint2 fr[2][2], v8f bias2[2], uint2 fr2[ const uint4 h = wg_to_f16_ags(lin_acc(bias[m1], fr[m1], xf)); v8f r; [unroll] for (uint j = 0; j < 4u; ++j) { - r.v[2 * j] = wg_fmax(wg_h2f_lo(h[j]), 0.0f); + // f16tof32 directly, not wg_h2f_lo: its mask costs a v_mov_b16 per FP16 pair (see hyb_h2f_lo in wg_hyb.hlsli) + r.v[2 * j] = wg_fmax(f16tof32(h[j]), 0.0f); r.v[2 * j + 1] = wg_fmax(wg_h2f_hi(h[j]), 0.0f); } x1[m1] = wg_pack_q8_ags(r); diff --git a/clean/dx12/hlsl/wg_hyb.hlsli b/clean/dx12/hlsl/wg_hyb.hlsli index 99c86610..27de3e1e 100644 --- a/clean/dx12/hlsl/wg_hyb.hlsli +++ b/clean/dx12/hlsl/wg_hyb.hlsli @@ -35,12 +35,16 @@ // RN16 of 8 float32 values (the driver's round-to-nearest-even conversion), packed, and back to float32 (exact). uint4 hyb_rn16x8(v8f d) { return wg_to_f16_ags(d); } +// The FP16 in the low half of v as float32 (exact). f16tof32 reads only the low 16 bits, so this is wg_h2f_lo without +// its "& 0xffff": the same conversion (v_cvt_f32_f16 of the low half), but the mask made the driver copy the half +// into a register of its own first (one v_mov_b16 per FP16 pair at every hybrid chain end). +float hyb_h2f_lo(uint v) { return f16tof32(v); } v8f hyb_h2f8(uint4 h) { v8f r; const uint w[4] = { h.x, h.y, h.z, h.w }; [unroll] for (uint i = 0; i < 4; ++i) { - r.v[2 * i] = wg_h2f_lo(w[i]); + r.v[2 * i] = hyb_h2f_lo(w[i]); r.v[2 * i + 1] = wg_h2f_hi(w[i]); } return r; @@ -105,8 +109,8 @@ v8f hyb_softmax_e8(v8f s) const uint w[4] = { pb.x, pb.y, pb.z, pb.w }; v8f e; [unroll] for (uint i4 = 0; i4 < 4; ++i4) { - e.v[2 * i4] = wg_h2f_lo((w[i4] & 0x03ffu) << 5); - e.v[2 * i4 + 1] = wg_h2f_lo(((w[i4] >> 16) & 0x03ffu) << 5); + e.v[2 * i4] = hyb_h2f_lo((w[i4] & 0x03ffu) << 5); + e.v[2 * i4 + 1] = hyb_h2f_lo(((w[i4] >> 16) & 0x03ffu) << 5); } return e; } @@ -127,7 +131,7 @@ float hyb_rsq16_tab(uint sbits) const int k = (E - (int)p) >> 1; const uint off = WG_HYB_RSQ + 2u * (p * 1024u + (sbits & 1023u)); const uint rb = (wg_mem.Load(off & ~3u) >> (8u * (off & 2u))) & 0xffffu; - return wg_h2f_lo((uint)((int)rb - (k << 10))); + return hyb_h2f_lo((uint)((int)rb - (k << 10))); } // w = RN16(rcp32(sum)) for a positive normal FP16 row sum (bits) with a normal result (every softmax row), by table. @@ -136,7 +140,7 @@ float hyb_rcp16(uint sbits) const int E = (int)((sbits >> 10) & 31u) - 15; const uint off = WG_HYB_RCP + 2u * (sbits & 1023u); const uint rb = (wg_mem.Load(off & ~3u) >> (8u * (off & 2u))) & 0xffffu; - return wg_h2f_lo((uint)((int)rb - (E << 10))); + return hyb_h2f_lo((uint)((int)rb - (E << 10))); } // The merge clamp before the FP8 point: max first, then min, NaN kept (Q8 B4). diff --git a/clean/dx12/kernels/l_ops_linear.hlsl b/clean/dx12/kernels/l_ops_linear.hlsl index e9ec0259..87eb3671 100644 --- a/clean/dx12/kernels/l_ops_linear.hlsl +++ b/clean/dx12/kernels/l_ops_linear.hlsl @@ -73,7 +73,8 @@ WG_KERNEL(l_linear_32_32_f16) const uint4 h = wg_to_f16_ags(lin_acc(bias[m1], fr[m1], xf)); v8f r; [unroll] for (uint j = 0; j < 4u; ++j) { - r.v[2 * j] = wg_fmax(wg_h2f_lo(h[j]), 0.0f); + // f16tof32 directly, not wg_h2f_lo: its mask costs a v_mov_b16 per FP16 pair (see hyb_h2f_lo in wg_hyb.hlsli) + r.v[2 * j] = wg_fmax(f16tof32(h[j]), 0.0f); r.v[2 * j + 1] = wg_fmax(wg_h2f_hi(h[j]), 0.0f); } x1[m1] = wg_pack_q8_ags(r); diff --git a/clean/dx12/kernels/l_ops_project.hlsl b/clean/dx12/kernels/l_ops_project.hlsl index 270b02b0..723da953 100644 --- a/clean/dx12/kernels/l_ops_project.hlsl +++ b/clean/dx12/kernels/l_ops_project.hlsl @@ -37,7 +37,8 @@ v8f pj_rn16(v8f a) { const uint4 h = wg_to_f16_ags(a); v8f r; - [unroll] for (uint j = 0; j < 4u; ++j) { r.v[2 * j] = wg_h2f_lo(h[j]); r.v[2 * j + 1] = wg_h2f_hi(h[j]); } + // f16tof32 directly, not wg_h2f_lo: its mask costs a v_mov_b16 per FP16 pair (see hyb_h2f_lo in wg_hyb.hlsli) + [unroll] for (uint j = 0; j < 4u; ++j) { r.v[2 * j] = f16tof32(h[j]); r.v[2 * j + 1] = wg_h2f_hi(h[j]); } return r; }