Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 14 additions & 10 deletions clean/dx12/hlsl/l_coef_px.hlsli
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,11 @@ float clamp20(float v) { return select(v >= 20.0f, 20.0f, select(v <= -20.0f, -2
// buffers (pB unused with COEF_PACK12); i: the pixel's index in the render rectangle (y rw + x).
void l_coef_px(uint4 lo, uint4 hi, float S, float T, uint pA, uint pB, uint pC, uint pL, uint i)
{
const float a0 = wg_h2f_lo(lo.x), a1 = wg_h2f_hi(lo.x), a2 = wg_h2f_lo(lo.w), a3 = wg_h2f_hi(lo.w);
const float a4 = wg_h2f_lo(hi.x), a5 = wg_h2f_hi(hi.x), a6 = wg_h2f_lo(hi.y), a7 = wg_h2f_hi(hi.y);
const float a8 = wg_h2f_lo(hi.z);
const float a9 = wg_h2f_lo(lo.y), a10 = wg_h2f_hi(lo.y), a11 = wg_h2f_lo(lo.z), a12 = wg_h2f_hi(lo.z);
// f16tof32 directly, not wg_h2f_lo: its mask costs a v_mov_b16 per FP16 pair (see hyb_h2f_lo in wg_hyb.hlsli)
const float a0 = f16tof32(lo.x), a1 = wg_h2f_hi(lo.x), a2 = f16tof32(lo.w), a3 = wg_h2f_hi(lo.w);
const float a4 = f16tof32(hi.x), a5 = wg_h2f_hi(hi.x), a6 = f16tof32(hi.y), a7 = wg_h2f_hi(hi.y);
const float a8 = f16tof32(hi.z);
const float a9 = f16tof32(lo.y), a10 = wg_h2f_hi(lo.y), a11 = f16tof32(lo.z), a12 = wg_h2f_hi(lo.z);
// A: unorm codes of RN16(sigma(a0..a2))
const uint cA = unorm8_code(wg_rn16(l_sigma(a0))) | (unorm8_code(wg_rn16(l_sigma(a1))) << 8) |
(unorm8_code(wg_rn16(l_sigma(a2))) << 16);
Expand Down Expand Up @@ -104,10 +105,11 @@ bool coef_mul_zero(float a, float b)

LCoefVals l_coef_vals_hw(uint4 lo, uint4 hi, float S, float T)
{
const float a0 = wg_h2f_lo(lo.x), a1 = wg_h2f_hi(lo.x), a2 = wg_h2f_lo(lo.w), a3 = wg_h2f_hi(lo.w);
const float a4 = wg_h2f_lo(hi.x), a5 = wg_h2f_hi(hi.x), a6 = wg_h2f_lo(hi.y), a7 = wg_h2f_hi(hi.y);
const float a8 = wg_h2f_lo(hi.z);
const float a9 = wg_h2f_lo(lo.y), a10 = wg_h2f_hi(lo.y), a11 = wg_h2f_lo(lo.z), a12 = wg_h2f_hi(lo.z);
// f16tof32 directly, not wg_h2f_lo: its mask costs a v_mov_b16 per FP16 pair (see hyb_h2f_lo in wg_hyb.hlsli)
const float a0 = f16tof32(lo.x), a1 = wg_h2f_hi(lo.x), a2 = f16tof32(lo.w), a3 = wg_h2f_hi(lo.w);
const float a4 = f16tof32(hi.x), a5 = wg_h2f_hi(hi.x), a6 = f16tof32(hi.y), a7 = wg_h2f_hi(hi.y);
const float a8 = f16tof32(hi.z);
const float a9 = f16tof32(lo.y), a10 = wg_h2f_hi(lo.y), a11 = f16tof32(lo.z), a12 = wg_h2f_hi(lo.z);
LCoefVals o;
// RN16 of sigma(a0), sigma(a1), sigma(a2), sigma(clamp20(a6)), sigma(clamp20(a7)): one conversion
v8f sg;
Expand All @@ -116,15 +118,17 @@ LCoefVals l_coef_vals_hw(uint4 lo, uint4 hi, float S, float T)
sg.v[5] = 0.0f; sg.v[6] = 0.0f; sg.v[7] = 0.0f;
const uint4 hs = wg_to_f16_ags(sg);
// A: unorm codes of RN16(sigma(a0..a2))
o.cA = unorm8_code(wg_h2f_lo(hs.x)) | (unorm8_code(wg_h2f_hi(hs.x)) << 8) | (unorm8_code(wg_h2f_lo(hs.y)) << 16);
// f16tof32 directly, not wg_h2f_lo: its mask costs a v_mov_b16 per FP16 pair (see hyb_h2f_lo in wg_hyb.hlsli)
o.cA = unorm8_code(f16tof32(hs.x)) | (unorm8_code(wg_h2f_hi(hs.x)) << 8) | (unorm8_code(f16tof32(hs.y)) << 16);
// B: softmax of a3..a5
const float m = max(max(a3, a4), a5);
const float e0 = wg_exp2((a3 - m) * bits(0x3FB8AA3Bu)), e1 = wg_exp2((a4 - m) * bits(0x3FB8AA3Bu)),
e2 = wg_exp2((a5 - m) * bits(0x3FB8AA3Bu));
const float rs = wg_rcp(e2 + (e0 + e1));
o.cB = unorm8_code(opq(e0 * rs)) | (unorm8_code(opq(e1 * rs)) << 8) | (unorm8_code(opq(e2 * rs)) << 16);
// C: spreads and correlation; the three FP16 products in one conversion
const float g0 = wg_h2f_hi(hs.y), g1 = wg_h2f_lo(hs.z);
// f16tof32 directly, not wg_h2f_lo: its mask costs a v_mov_b16 per FP16 pair (see hyb_h2f_lo in wg_hyb.hlsli)
const float g0 = wg_h2f_hi(hs.y), g1 = f16tof32(hs.z);
const float sx = S * g0, sy = S * g1;
const float sxy = sx * sy;
const float rho = l_tanh(a8 * T);
Expand Down
3 changes: 2 additions & 1 deletion clean/dx12/hlsl/l_lin_px.hlsli
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ void lin_pair(uint2 xf[2], v8f bias[2], uint2 fr[2][2], v8f bias2[2], uint2 fr2[
const uint4 h = wg_to_f16_ags(lin_acc(bias[m1], fr[m1], xf));
v8f r;
[unroll] for (uint j = 0; j < 4u; ++j) {
r.v[2 * j] = wg_fmax(wg_h2f_lo(h[j]), 0.0f);
// f16tof32 directly, not wg_h2f_lo: its mask costs a v_mov_b16 per FP16 pair (see hyb_h2f_lo in wg_hyb.hlsli)
r.v[2 * j] = wg_fmax(f16tof32(h[j]), 0.0f);
r.v[2 * j + 1] = wg_fmax(wg_h2f_hi(h[j]), 0.0f);
}
x1[m1] = wg_pack_q8_ags(r);
Expand Down
14 changes: 9 additions & 5 deletions clean/dx12/hlsl/wg_hyb.hlsli
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,16 @@

// RN16 of 8 float32 values (the driver's round-to-nearest-even conversion), packed, and back to float32 (exact).
uint4 hyb_rn16x8(v8f d) { return wg_to_f16_ags(d); }
// The FP16 in the low half of v as float32 (exact). f16tof32 reads only the low 16 bits, so this is wg_h2f_lo without
// its "& 0xffff": the same conversion (v_cvt_f32_f16 of the low half), but the mask made the driver copy the half
// into a register of its own first (one v_mov_b16 per FP16 pair at every hybrid chain end).
float hyb_h2f_lo(uint v) { return f16tof32(v); }
v8f hyb_h2f8(uint4 h)
{
v8f r;
const uint w[4] = { h.x, h.y, h.z, h.w };
[unroll] for (uint i = 0; i < 4; ++i) {
r.v[2 * i] = wg_h2f_lo(w[i]);
r.v[2 * i] = hyb_h2f_lo(w[i]);
r.v[2 * i + 1] = wg_h2f_hi(w[i]);
}
return r;
Expand Down Expand Up @@ -105,8 +109,8 @@ v8f hyb_softmax_e8(v8f s)
const uint w[4] = { pb.x, pb.y, pb.z, pb.w };
v8f e;
[unroll] for (uint i4 = 0; i4 < 4; ++i4) {
e.v[2 * i4] = wg_h2f_lo((w[i4] & 0x03ffu) << 5);
e.v[2 * i4 + 1] = wg_h2f_lo(((w[i4] >> 16) & 0x03ffu) << 5);
e.v[2 * i4] = hyb_h2f_lo((w[i4] & 0x03ffu) << 5);
e.v[2 * i4 + 1] = hyb_h2f_lo(((w[i4] >> 16) & 0x03ffu) << 5);
}
return e;
}
Expand All @@ -127,7 +131,7 @@ float hyb_rsq16_tab(uint sbits)
const int k = (E - (int)p) >> 1;
const uint off = WG_HYB_RSQ + 2u * (p * 1024u + (sbits & 1023u));
const uint rb = (wg_mem.Load(off & ~3u) >> (8u * (off & 2u))) & 0xffffu;
return wg_h2f_lo((uint)((int)rb - (k << 10)));
return hyb_h2f_lo((uint)((int)rb - (k << 10)));
}

// w = RN16(rcp32(sum)) for a positive normal FP16 row sum (bits) with a normal result (every softmax row), by table.
Expand All @@ -136,7 +140,7 @@ float hyb_rcp16(uint sbits)
const int E = (int)((sbits >> 10) & 31u) - 15;
const uint off = WG_HYB_RCP + 2u * (sbits & 1023u);
const uint rb = (wg_mem.Load(off & ~3u) >> (8u * (off & 2u))) & 0xffffu;
return wg_h2f_lo((uint)((int)rb - (E << 10)));
return hyb_h2f_lo((uint)((int)rb - (E << 10)));
}

// The merge clamp before the FP8 point: max first, then min, NaN kept (Q8 B4).
Expand Down
3 changes: 2 additions & 1 deletion clean/dx12/kernels/l_ops_linear.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,8 @@ WG_KERNEL(l_linear_32_32_f16)
const uint4 h = wg_to_f16_ags(lin_acc(bias[m1], fr[m1], xf));
v8f r;
[unroll] for (uint j = 0; j < 4u; ++j) {
r.v[2 * j] = wg_fmax(wg_h2f_lo(h[j]), 0.0f);
// f16tof32 directly, not wg_h2f_lo: its mask costs a v_mov_b16 per FP16 pair (see hyb_h2f_lo in wg_hyb.hlsli)
r.v[2 * j] = wg_fmax(f16tof32(h[j]), 0.0f);
r.v[2 * j + 1] = wg_fmax(wg_h2f_hi(h[j]), 0.0f);
}
x1[m1] = wg_pack_q8_ags(r);
Expand Down
3 changes: 2 additions & 1 deletion clean/dx12/kernels/l_ops_project.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ v8f pj_rn16(v8f a)
{
const uint4 h = wg_to_f16_ags(a);
v8f r;
[unroll] for (uint j = 0; j < 4u; ++j) { r.v[2 * j] = wg_h2f_lo(h[j]); r.v[2 * j + 1] = wg_h2f_hi(h[j]); }
// f16tof32 directly, not wg_h2f_lo: its mask costs a v_mov_b16 per FP16 pair (see hyb_h2f_lo in wg_hyb.hlsli)
[unroll] for (uint j = 0; j < 4u; ++j) { r.v[2 * j] = f16tof32(h[j]); r.v[2 * j + 1] = wg_h2f_hi(h[j]); }
return r;
}

Expand Down
Loading