Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions clean/dx12/kernels/l_ops_project.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
// No branch: a lane past the last token computes the last token (its inputs are clamped there, as HIP's) and stores
// that token's bytes to that token's place, the same bytes its own lane stores (a branch around the store let the
// compiler sink the matrix instructions into it, see l_ops_linear.hlsl).
// Conversions by the driver's instructions (wg_platform.hlsli; the same bits as the software wg_rn16 / wg_pack_q8, and
// this kernel has no branch): RN16 = v_cvt_f16_f32 then the exact widening, the FP8 pack v_cvt_pk_fp8_f32 after the
// clamp. The software forms were most of the kernels' instructions.
//
// @wg-kernel l_project_gather co=l_ops.co block=256 wave=32 args=p,p,p,u,u defines=PJ_DEC=0
// @wg-kernel l_dec_project co=l_ops.co block=256 wave=32 args=p,p,p,u,u defines=PJ_DEC=1
Expand All @@ -29,6 +32,15 @@ WG_ARGS_END
#define FRAG0 256u // 4 * 64: after the bias tile vector, rows R_L(64)
#endif

// RN16 of 8 values by the driver's conversion
v8f pj_rn16(v8f a)
{
const uint4 h = wg_to_f16_ags(a);
v8f r;
[unroll] for (uint j = 0; j < 4u; ++j) { r.v[2 * j] = wg_h2f_lo(h[j]); r.v[2 * j + 1] = wg_h2f_hi(h[j]); }
return r;
}

#if PJ_DEC
WG_KERNEL(l_dec_project)
#else
Expand Down Expand Up @@ -60,15 +72,15 @@ WG_KERNEL(l_project_gather)
[unroll] for (uint g = 0; g < KB / 2; ++g) {
acc = wg_wmma_fp8(fr[2u * g], zf[2u * g], acc);
acc = wg_wmma_fp8(fr[2u * g + 1u], zf[2u * g + 1u], acc);
[unroll] for (uint i = 0; i < 8u; ++i) acc.v[i] = wg_rn16(acc.v[i]);
acc = pj_rn16(acc);
}
#if PJ_DEC
const int s = (int)(mt >> 1), gg = (int)(mt & 1u);
const int pix = (2 * r + (s >> 1)) * (2 * tw) + 2 * c + (s & 1);
wg_st64(dst, (uint)pix * 32u + 16u * hh + 8u * (uint)gg, wg_pack_q8(acc));
wg_st64(dst, (uint)pix * 32u + 16u * hh + 8u * (uint)gg, wg_pack_q8_ags(acc));
#else
[unroll] for (uint j = 0; j < 8u; ++j) acc.v[j] = wg_fmin(wg_fmax(acc.v[j], -6.28125f), 6.28125f);
wg_st64(dst, (uint)tk * 64u + hh * 32u + 8u * mt, wg_pack_q8(acc));
wg_st64(dst, (uint)tk * 64u + hh * 32u + 8u * mt, wg_pack_q8_ags(acc));
#endif
}
}
42 changes: 33 additions & 9 deletions clean/l/runner/l_d3d12_ops_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,23 +84,47 @@ def control_cases(kernel, rng):
TOKENS = [(8, 12), (18, 34), (4, 4), (34, 50), (3, 5), (2, 9)]


def _project_blob(rng, rows, frags):
def _project_blob(rng, rows, frags, scale=0.2, bias_scale=0.5):
"""Projection blob: bias tile vector (`rows` floats), then `frags` fragments of 256 FP8 bytes."""
return rng.normal(0, 0.5, rows).astype(np.float32).tobytes() + fp8_encode(q8(rng.normal(0, 0.2, frags * 256))).tobytes()
return (rng.normal(0, bias_scale, rows).astype(np.float32).tobytes() +
fp8_encode(q8(rng.normal(0, scale, frags * 256))).tobytes())


# FP8 codes the network does not produce but the kernels' conversions must treat as HIP does: both NaN codes, +-448,
# +-0, subnormals
SPECIAL_FP8 = np.array([0x7f, 0xff, 0x7e, 0xfe, 0x00, 0x80, 0x01, 0x81, 0x07, 0x87], np.uint8)
# (weight, bias) scales of the projection cases: normal; large (sums past 448 and past FP16's 65504: the clamps,
# infinities); tiny (FP16 subnormal and zero results)
PROJECT_SCALES = [(0.2, 0.5), (24.0, 400.0), (2e-3, 1e-4)]


def _fp8_map_special(rng, n, special):
"""_fp8_map, with special FP8 codes in a tenth of the bytes when `special`."""
q = np.frombuffer(_fp8_map(rng, n), np.uint8).copy()
if special:
pick = rng.random(q.size) < 0.1
q[pick] = rng.choice(SPECIAL_FP8, int(pick.sum()))
return q.tobytes()


def project_cases(kernel, rng):
"""l_project_gather: x [2 th][2 tw][32] -> features [th][tw][64] (blob enc0.project: 64 bias rows, 4 x 8 fragments).
l_dec_project: z [th][tw][64] -> y [2 th][2 tw][32] (blob dec0.project: 128 bias rows, 8 x 4 fragments)."""
l_dec_project: z [th][tw][64] -> y [2 th][2 tw][32] (blob dec0.project: 128 bias rows, 8 x 4 fragments).
Every token grid at the three PROJECT_SCALES, every other case with special FP8 codes in the input."""
cases = []
for (th, tw) in TOKENS:
for ti, (th, tw) in enumerate(TOKENS):
ntok = th * tw
npx = 4 * ntok
if kernel == "l_project_gather":
args = [("buf", _fp8_map(rng, npx * 32)), ("out", ntok * 64), ("buf", _project_blob(rng, 64, 32))]
else:
args = [("buf", _fp8_map(rng, ntok * 64)), ("out", npx * 32), ("buf", _project_blob(rng, 128, 32))]
cases.append({"kernel": kernel, "grid": -(-ntok // 128), "block": 256, "args": args + [("u32", tw), ("u32", th)]})
for si, (s, b) in enumerate(PROJECT_SCALES):
special = (ti + si) % 2 == 1
if kernel == "l_project_gather":
args = [("buf", _fp8_map_special(rng, npx * 32, special)), ("out", ntok * 64),
("buf", _project_blob(rng, 64, 32, s, b))]
else:
args = [("buf", _fp8_map_special(rng, ntok * 64, special)), ("out", npx * 32),
("buf", _project_blob(rng, 128, 32, s, b))]
cases.append({"kernel": kernel, "grid": -(-ntok // 128), "block": 256,
"args": args + [("u32", tw), ("u32", th)]})
return cases


Expand Down
Loading