Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 107 additions & 0 deletions clean/dx12/kernels/l_ops_coef.hlsl
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
// SPDX-License-Identifier: MIT
// D3D12 port of Model L's coefficient step (issue #89): l_coef of l_ops.co (clean/l/kernels/l_ops.hip), byte for byte.
// y2 [fh][fw][32] FP16 (the decoder block's output) -> A, B (3 unorm8 codes each), C (two dwords: C[2 i] = FP16 sx^2 |
// sy^2 << 16, C[2 i + 1] = FP16 sxy rho in the low half) and the new latent (4 snorm8 codes), [rh][rw] each, pixels
// inside R only. Lane (t, h) of wave i handles token 16 i + t =
// (r, c) of the (fh / 2) x (fw / 2) grid, pixels (2 c, 2 r + h) and (2 c + 1, 2 r + h). Reads no motion, previous latent,
// jitter or reset. w = blob dec0.coef (S, T at byte 320).
//
// Every FP32 operation follows HIP's ISA of l_coef (llvm-objdump of l_ops.co), not its source, where they differ:
// - C0 = (f16)(sx sx) | (f16)(sy sy) << 16 and C1 = (f16)(sxy rho), each ONE rounding (v_fma_mixlo_f16 a, b, 0 =
// wg_mul_f16), not an FP32 product rounded again;
// - B: s = e2 + (e0 + e1) (commutes with the source's (e0 + e1) + e2), code = rint((e r) 255): opq() keeps the driver
// from regrouping the product with the constant 255;
// - tanh below 0.6: fma(p, t^2, 0) with an opaque zero (the driver turns mad(a, b, 0) into a b, which changes the sign
// of a zero), then fma(p2, t, t);
// - the clamp of the two spread inputs to +-20 keeps a NaN (compares, as HIP's v_cmp / v_cndmask on the FP16 value).
// The three 32-bit stores at the same per-lane offset into different buffers (A, B, latent) are the shape the driver was
// seen to merge (README "drop a store"): the pointers are read through wg_uniform().
//
// @wg-kernel l_coef co=l_ops.co block=256 wave=32 args=p,p,p,p,p,p,b56
#include "wg_platform.hlsli"

WG_ARGS_BEGIN
uint y2, A, B, C, latent, w; // pointers
int P_rw, P_rh, P_fw, P_fh, P_reset, P_pad0; // LDecParams (Model M's WgDecParams; only rw, rh, fw, fh read)
float P_dw, P_dh, P_kx, P_ky, P_jx, P_jy, P_pjx, P_pjy;
WG_ARGS_END

static uint g_z; // 0, opaque to the compiler (set from the grid size)
float opq(float x) { return asfloat(asuint(x) ^ g_z); }
float bits(uint b) { return asfloat(b); }

// sigma(x) = 1 / (1 + 2^(x * -log2 e)): v_mul, v_exp, v_add 1.0, v_rcp
float l_sigma(float x) { return wg_rcp(1.0f + wg_exp2(x * bits(0xBFB8AA3Bu))); }

// tanh(t): for |t| >= 0.6 (and not NaN), 1 - 2 / (2^(2 |t| log2 e) + 1), 1 from about 9.01 on, sign of t; below, the odd
// polynomial. Both sides are computed; the ISA's branch selects one, select() here (both are pure).
float l_tanh(float t)
{
const float a = abs(t);
float big = mad(wg_rcp(wg_exp2(a * bits(0x4038AA3Bu)) + 1.0f), -2.0f, 1.0f);
big = select(bits(0x41102CB4u) > a || isnan(a), big, 1.0f); // v_cmp_nle 9.01, |t|; v_cndmask
big = asfloat((asuint(big) & 0x7fffffffu) | (asuint(t) & 0x80000000u)); // v_bfi copysign
const float t2 = t * t;
const float p = mad(mad(mad(bits(0x3C80F082u), t2, bits(0xBD563CAEu)), t2, bits(0x3E085941u)), t2, bits(0xBEAAA9EDu));
const float p2 = mad(p, t2, asfloat(g_z)); // fma(p, t2, +0), the zero opaque
const float small = mad(p2, t, t);
return select(a >= bits(0x3F19999Au), big, small); // v_cmp_le 0.6, |t| (false for a NaN: the polynomial)
}

uint unorm8_code(float v) { return wg_f2u(wg_rint(v * 255.0f)); } // v in [0, 1]: v_mul 255, v_rndne, v_cvt_u32
uint snorm8_code(float v) { return (uint)wg_f2i(wg_rint(wg_med3f(v, -1.0f, 1.0f) * 127.0f)) & 0xffu; }

// the clamp of a spread input (an FP16 value) to +-20, NaN kept: HIP's v >= 20 ? 20 : (v <= -20 ? -20 : v)
float clamp20(float v) { return select(v >= 20.0f, 20.0f, select(v <= -20.0f, -20.0f, v)); }

WG_KERNEL(l_coef)
{
WG_SETUP();
g_z = wg_grid_ >> 31;
const uint tid = wg_lane_id(), hh = (tid >> 4) & 1u;
const int tw = P_fw >> 1, ntok = (P_fh >> 1) * tw;
const int tok = (int)(wg_block_id() * (WG_BLOCK / WG_WAVE) + (tid >> 5)) * 16 + (int)(tid & 15u);
if (tok >= ntok) return;
const int r = tok / tw, c = tok - r * tw;
const int y = 2 * r + (int)hh;
if (y >= P_rh) return;
const float S = wg_ldf(w, 320u), T = wg_ldf(w, 324u);
const uint pA = (uint)wg_uniform((int)A), pB = (uint)wg_uniform((int)B), pC = (uint)wg_uniform((int)C),
pL = (uint)wg_uniform((int)latent);
[unroll] for (int dx = 0; dx < 2; ++dx) {
const int x = 2 * c + dx;
if (x >= P_rw) continue;
// channels 0..15 of the pixel (8 dwords, two FP16 each); Model L's 13 values are channels
// a0..a12 = 0, 1, 6, 7, 8, 9, 10, 11, 12, 2, 3, 4, 5 (clean/l/reference/table.py L_DEC_VALUES)
const uint off = (uint)(y * P_fw + x) * 64u;
const uint4 lo = wg_ld128(y2, off), hi = wg_ld128(y2, off + 16u);
const float a0 = wg_h2f_lo(lo.x), a1 = wg_h2f_hi(lo.x), a2 = wg_h2f_lo(lo.w), a3 = wg_h2f_hi(lo.w);
const float a4 = wg_h2f_lo(hi.x), a5 = wg_h2f_hi(hi.x), a6 = wg_h2f_lo(hi.y), a7 = wg_h2f_hi(hi.y);
const float a8 = wg_h2f_lo(hi.z);
const float a9 = wg_h2f_lo(lo.y), a10 = wg_h2f_hi(lo.y), a11 = wg_h2f_lo(lo.z), a12 = wg_h2f_hi(lo.z);
// A: unorm codes of RN16(sigma(a0..a2))
const uint cA = unorm8_code(wg_rn16(l_sigma(a0))) | (unorm8_code(wg_rn16(l_sigma(a1))) << 8) |
(unorm8_code(wg_rn16(l_sigma(a2))) << 16);
// B: softmax of a3..a5
const float m = max(max(a3, a4), a5);
const float e0 = wg_exp2((a3 - m) * bits(0x3FB8AA3Bu)), e1 = wg_exp2((a4 - m) * bits(0x3FB8AA3Bu)),
e2 = wg_exp2((a5 - m) * bits(0x3FB8AA3Bu));
const float rs = wg_rcp(e2 + (e0 + e1));
const uint cB = unorm8_code(opq(e0 * rs)) | (unorm8_code(opq(e1 * rs)) << 8) | (unorm8_code(opq(e2 * rs)) << 16);
// C: spreads and correlation
const float g0 = wg_rn16(l_sigma(clamp20(a6))), g1 = wg_rn16(l_sigma(clamp20(a7)));
const float sx = S * g0, sy = S * g1;
const float sxy = sx * sy;
const float rho = l_tanh(a8 * T);
const uint c0 = wg_mul_f16(sx, sx) | (wg_mul_f16(sy, sy) << 16);
const uint c1 = wg_mul_f16(sxy, rho);
// latent: snorm codes of tanh(a9..a12)
const uint cL = snorm8_code(l_tanh(a9)) | (snorm8_code(l_tanh(a10)) << 8) | (snorm8_code(l_tanh(a11)) << 16) |
(snorm8_code(l_tanh(a12)) << 24);
const uint i = (uint)(y * P_rw + x);
wg_st32(pA, i * 4u, cA);
wg_st32(pB, i * 4u, cB);
wg_st64(pC, i * 8u, uint2(c0, c1));
wg_st32(pL, i * 4u, cL);
}
}
37 changes: 36 additions & 1 deletion clean/l/runner/l_d3d12_ops_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,43 @@ def project_cases(kernel, rng):
return cases


# (rw, rh, fw, fh) of the coefficient step: R inside the padded grid F (even). Odd rw / rh (a token's second column or
# row outside R), F wider than R by whole tokens, and 5 x 3 / 7 x 5 (partly filled waves).
COEF_FRAMES = [(24, 16, 24, 16), (67, 35, 68, 36), (7, 5, 8, 6), (100, 68, 100, 68), (5, 3, 6, 4), (13, 7, 20, 12)]


def _coef_map(rng, n):
"""FP16 decoder map [n][32]: mostly N(0, 3), with values beyond the clamp (+-20 on the two spread inputs), beyond
tanh's saturation (9.01) and inside its polynomial range (|t| < 0.6), so every path of l_coef runs."""
v = rng.normal(0, 3.0, (n, 32))
pick = rng.random((n, 32))
v = np.where(pick < 0.08, rng.uniform(20, 60, (n, 32)) * rng.choice([-1, 1], (n, 32)), v)
v = np.where((pick >= 0.08) & (pick < 0.2), rng.uniform(-0.6, 0.6, (n, 32)), v)
v = np.where((pick >= 0.2) & (pick < 0.25), rng.uniform(8, 10, (n, 32)) * rng.choice([-1, 1], (n, 32)), v)
return v.astype(np.float16).tobytes()


def coef_cases(kernel, rng):
"""l_coef: y2 [fh][fw][32] FP16 -> A, B, latent [rh][rw] (4 bytes each), C [rh][rw] (8 bytes); blob dec0.coef
(Model M's dec0 blob with W = 0, b = 0: S, T at byte 320); LDecParams (56 bytes) by value."""
from clean.runner import dec0
cases = []
for (rw, rh, fw, fh) in COEF_FRAMES:
npx = rw * rh
S, T = float(rng.uniform(0.5, 4.0)), float(rng.uniform(0.3, 3.0))
blob = dec0.pack({"W": np.zeros((68, 64), np.float32), "b": np.zeros(68, np.float32), "S": S, "T": T})
P = bytes(dec0.DecParams(rw=rw, rh=rh, fw=fw, fh=fh, reset=0, pad0=0, dw=2 * rw, dh=2 * rh, kx=0.5, ky=0.5,
jx=0.25, jy=-0.25, pjx=0.0, pjy=0.0))
assert len(P) == 56
ntok = (fh // 2) * (fw // 2)
cases.append({"kernel": kernel, "grid": -(-ntok // 128), "block": 256,
"args": [("buf", _coef_map(rng, fw * fh)), ("out", npx * 4), ("out", npx * 4), ("out", npx * 8),
("out", npx * 4), ("buf", blob), ("bytes", P)]})
return cases


KINDS = {"l_linear_32_32_relu": linear_cases, "l_linear_32_32_f16": linear_cases,
"l_project_gather": project_cases, "l_dec_project": project_cases}
"l_project_gather": project_cases, "l_dec_project": project_cases, "l_coef": coef_cases}
CONTROL_KINDS = {"swin_block_c96_nc": control_cases}
CO = "l_ops.co"

Expand Down
Loading