Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 67 additions & 0 deletions clean/dx12/hlsl/l_coef_px.hlsli
Original file line number Diff line number Diff line change
Expand Up @@ -73,3 +73,70 @@ void l_coef_px(uint4 lo, uint4 hi, float S, float T, uint pA, uint pB, uint pC,
#endif
wg_st32(pL, i * 4u, cL);
}

#if COEF_CVT_HW
// The folded decoder block's form (l_swin_block_c32_dec_coef12_nc; l_coef / l_coef_pack12 keep l_coef_px above): the
// same values, with the FP16 roundings by the driver's conversion (v_cvt_f16_f32; wg_platform.hlsli: the same bits as
// the software wg_f2h): the five RN16 of sigma in one conversion, the three FP16 products as their round-to-odd FP32
// values (wg_mul_f16's) in another. The driver conversions need wave-uniform control flow: every lane computes
// l_coef_vals_hw, only the stores (l_coef_store12) are per lane. Needs COEF_PACK12.
struct LCoefVals { uint cA, cB, c0, c1, cL; };

// a * b as the round-to-odd FP32 value whose FP16 rounding is (f16)(a b) (wg_mul_f16 before its conversion)
float coef_mul_odd(float a, float b)
{
const float p = a * b;
return asfloat(wg_round_odd(p, mad(a, b, -p)));
}
// wg_mul_f16's +0 for an exact zero product (a NaN product keeps its conversion)
bool coef_mul_zero(float a, float b)
{
return ((asuint(a) & 0x7fffffffu) == 0u || (asuint(b) & 0x7fffffffu) == 0u) && !isnan(a * b);
}

LCoefVals l_coef_vals_hw(uint4 lo, uint4 hi, float S, float T)
{
const float a0 = wg_h2f_lo(lo.x), a1 = wg_h2f_hi(lo.x), a2 = wg_h2f_lo(lo.w), a3 = wg_h2f_hi(lo.w);
const float a4 = wg_h2f_lo(hi.x), a5 = wg_h2f_hi(hi.x), a6 = wg_h2f_lo(hi.y), a7 = wg_h2f_hi(hi.y);
const float a8 = wg_h2f_lo(hi.z);
const float a9 = wg_h2f_lo(lo.y), a10 = wg_h2f_hi(lo.y), a11 = wg_h2f_lo(lo.z), a12 = wg_h2f_hi(lo.z);
LCoefVals o;
// RN16 of sigma(a0), sigma(a1), sigma(a2), sigma(clamp20(a6)), sigma(clamp20(a7)): one conversion
v8f sg;
sg.v[0] = l_sigma(a0); sg.v[1] = l_sigma(a1); sg.v[2] = l_sigma(a2);
sg.v[3] = l_sigma(clamp20(a6)); sg.v[4] = l_sigma(clamp20(a7));
sg.v[5] = 0.0f; sg.v[6] = 0.0f; sg.v[7] = 0.0f;
const uint4 hs = wg_to_f16_ags(sg);
// A: unorm codes of RN16(sigma(a0..a2))
o.cA = unorm8_code(wg_h2f_lo(hs.x)) | (unorm8_code(wg_h2f_hi(hs.x)) << 8) | (unorm8_code(wg_h2f_lo(hs.y)) << 16);
// B: softmax of a3..a5
const float m = max(max(a3, a4), a5);
const float e0 = wg_exp2((a3 - m) * bits(0x3FB8AA3Bu)), e1 = wg_exp2((a4 - m) * bits(0x3FB8AA3Bu)),
e2 = wg_exp2((a5 - m) * bits(0x3FB8AA3Bu));
const float rs = wg_rcp(e2 + (e0 + e1));
o.cB = unorm8_code(opq(e0 * rs)) | (unorm8_code(opq(e1 * rs)) << 8) | (unorm8_code(opq(e2 * rs)) << 16);
// C: spreads and correlation; the three FP16 products in one conversion
const float g0 = wg_h2f_hi(hs.y), g1 = wg_h2f_lo(hs.z);
const float sx = S * g0, sy = S * g1;
const float sxy = sx * sy;
const float rho = l_tanh(a8 * T);
v8f po;
po.v[0] = coef_mul_odd(sx, sx); po.v[1] = coef_mul_odd(sy, sy); po.v[2] = coef_mul_odd(sxy, rho);
po.v[3] = 0.0f; po.v[4] = 0.0f; po.v[5] = 0.0f; po.v[6] = 0.0f; po.v[7] = 0.0f;
const uint4 hp = wg_to_f16_ags(po);
const uint h0 = coef_mul_zero(sx, sx) ? 0u : (hp.x & 0xffffu), h1 = coef_mul_zero(sy, sy) ? 0u : (hp.x >> 16);
o.c0 = h0 | (h1 << 16);
o.c1 = coef_mul_zero(sxy, rho) ? 0u : (hp.y & 0xffffu);
// latent: snorm codes of tanh(a9..a12)
o.cL = snorm8_code(l_tanh(a9)) | (snorm8_code(l_tanh(a10)) << 8) | (snorm8_code(l_tanh(a11)) << 16) |
(snorm8_code(l_tanh(a12)) << 24);
return o;
}

void l_coef_store12(LCoefVals o, uint pA, uint pC, uint pL, uint i)
{
wg_st64(pA, i * 8u, uint2(o.cA | (o.cB << 24), (o.cB >> 8) | (o.c1 << 16)));
wg_st32(pC, i * 4u, o.c0);
wg_st32(pL, i * 4u, o.cL);
}
#endif
14 changes: 14 additions & 0 deletions clean/dx12/kernels/fused_swin.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -1138,6 +1138,20 @@ WG_KERNEL(SW_ENTRY)
#if SW_OUT_F16
// Output RN16(x2) as FP16, real window members only (HIP LStoreF16Epi).
uint4 o16[KC];
#if SW_OUT_COEF12 && COEF_CVT_HW
// the driver's conversions (the same bits as wg_to_f16 / the software forms): every lane computes, the hh = 0 lanes
// of real pixels inside the render rectangle store (the conversions need wave-uniform control flow)
[unroll] for (uint kh = 0; kh < KC; ++kh) o16[kh] = wg_to_f16_ags(acc[kh]);
{
g_z = wg_grid_ >> 31;
const float S = wg_ldf(coefW, 320u), T = wg_ldf(coefW, 324u);
const LCoefVals cv = l_coef_vals_hw(o16[0], o16[1], S, T);
if (dst >= 0 && hh == 0u && pr < coefRh && pc < coefRw)
l_coef_store12(cv, (uint)wg_uniform((int)coefA), (uint)wg_uniform((int)coefC), (uint)wg_uniform((int)coefL),
(uint)(pr * coefRw + pc));
}
return;
#endif
[unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16(acc[kf]);
#if SW_OUT_COEF12
// l_coef_pack12 of the map: o16[0] / o16[1] of the hh = 0 lane are the pixel's channels 0..7 / 8..15 (the map's
Expand Down
2 changes: 1 addition & 1 deletion clean/dx12/kernels/l_swin.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@
// byte for byte the pair _dec_nc -> l_coef_pack12, which stays the fallback route);
// the FP16 map is not written. Arguments: _dec_nc's (out unused), then A, C, latent,
// dec0.coef, rw, rh
// @wg-kernel l_swin_block_c32_dec_coef12_nc co=- block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p,p,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=1,SW_SRC=2,SW_OUT_F16=1,SW_OUT_COEF12=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_coef12_nc
// @wg-kernel l_swin_block_c32_dec_coef12_nc co=- block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p,p,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=1,SW_SRC=2,SW_OUT_F16=1,SW_OUT_COEF12=1,COEF_CVT_HW=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_coef12_nc
#include "swin_block.hlsl"
14 changes: 14 additions & 0 deletions clean/dx12/kernels/swin_block.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -979,6 +979,20 @@ WG_KERNEL(SW_ENTRY)
#if SW_OUT_F16
// Output RN16(x2) as FP16, real window members only (HIP LStoreF16Epi).
uint4 o16[KC];
#if SW_OUT_COEF12 && COEF_CVT_HW
// the driver's conversions (the same bits as wg_to_f16 / the software forms): every lane computes, the hh = 0 lanes
// of real pixels inside the render rectangle store (the conversions need wave-uniform control flow)
[unroll] for (uint kh = 0; kh < KC; ++kh) o16[kh] = wg_to_f16_ags(acc[kh]);
{
g_z = wg_grid_ >> 31;
const float S = wg_ldf(coefW, 320u), T = wg_ldf(coefW, 324u);
const LCoefVals cv = l_coef_vals_hw(o16[0], o16[1], S, T);
if (dst >= 0 && hh == 0u && pr < coefRh && pc < coefRw)
l_coef_store12(cv, (uint)wg_uniform((int)coefA), (uint)wg_uniform((int)coefC), (uint)wg_uniform((int)coefL),
(uint)(pr * coefRw + pc));
}
return;
#endif
[unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16(acc[kf]);
#if SW_OUT_COEF12
// l_coef_pack12 of the map: o16[0] / o16[1] of the hh = 0 lane are the pixel's channels 0..7 / 8..15 (the map's
Expand Down
4 changes: 2 additions & 2 deletions clean/l/runner/l_d3d12_profile.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@
sys.path.insert(0, ROOT)

SHAPES = {"1080p-quality": ((1280, 720), (1920, 1080)), "1440p-quality": ((1706, 960), (2560, 1440))}
ENC = ("lev/input_encoder", "l_linear_32_32_relu", "l_linear_32_32_f16", "l_swin_block_c32_enc", "l_project_gather",
"input_encoder")
ENC = ("lev/input_encoder", "l_linear_32_32_relu", "l_linear_32_32_f16", "l_linear_32_32_fused", "l_swin_block_c32_enc",
"l_project_gather", "input_encoder")
DEC = ("l_dec_project", "l_swin_block_c32_dec", "l_coef", "coef_decoder", "_add_dec0")
POST = ("post_", "scale_rgb", "f16lut")
SWIN = ("swin_", "patch_merge", "expand_", "bneck/")
Expand Down
22 changes: 19 additions & 3 deletions clean/l/runner/l_fold_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,20 +54,36 @@ def rects(H, W):
return sorted({(W, H), (max(1, W - 3), max(1, H - 1)), (max(1, W // 2 + 1), max(1, H // 2 - 1))})


# FP8 codes the network does not produce but the per-pixel functions must treat as the pair does: both NaN codes,
# +-448, +-0, subnormals
SPECIAL = np.array([0x7f, 0xff, 0x7e, 0xfe, 0x00, 0x80, 0x01, 0x81, 0x07, 0x87], np.uint8)


def cases(rng):
"""(H, W, offset, rw, rh, y bytes, block blob, coefficient blob, LDecParams bytes) of every case."""
"""(H, W, offset, rw, rh, y bytes, block blob, coefficient blob, LDecParams bytes) of every case. The cases cycle
through four kinds: the network's range; large inputs (outputs past the +-20 clamp and tanh's saturation); special
FP8 codes in a tenth of y (NaN outputs); extreme coefficient scalars S, T (FP16 products from subnormal to
infinite)."""
from clean.runner import dec0
kind = "second"
n = 0
for (H, W) in GRIDS:
for parity in (0, 1):
off = T.block_offset(T.TABLE, kind, parity)
for (rw, rh) in rects(H, W):
blob = LD._params_c32(rng)
if not LD._nc(blob):
continue # the folded kernel is the _nc form only
scale = float(rng.choice([1.5, 4.0]))
yb = fp8_encode(q8(rng.normal(0, scale, (H, W, 32)))).tobytes()
mode, n = n % 4, n + 1
scale = 60.0 if mode == 1 else float(rng.choice([1.5, 4.0]))
y = fp8_encode(q8(rng.normal(0, scale, (H, W, 32)))).astype(np.uint8)
if mode == 2:
pick = rng.random(y.shape) < 0.1
y[pick] = rng.choice(SPECIAL, int(pick.sum()))
yb = y.tobytes()
S, T_ = float(rng.uniform(0.5, 4.0)), float(rng.uniform(0.3, 3.0))
if mode == 3:
S, T_ = float(rng.choice([1e-3, 3e-3, 300.0, 1000.0])), float(rng.choice([1e-3, 50.0]))
cw = dec0.pack({"W": np.zeros((68, 64), np.float32), "b": np.zeros(68, np.float32), "S": S, "T": T_})
P = bytes(dec0.DecParams(rw=rw, rh=rh, fw=W, fh=H, reset=0, pad0=0, dw=2 * rw, dh=2 * rh, kx=0.5,
ky=0.5, jx=0.25, jy=-0.25, pjx=0.0, pjy=0.0))
Expand Down
Loading