diff --git a/clean/dx12/kernels/l_ops_project.hlsl b/clean/dx12/kernels/l_ops_project.hlsl index 86a7c284..270b02b0 100644 --- a/clean/dx12/kernels/l_ops_project.hlsl +++ b/clean/dx12/kernels/l_ops_project.hlsl @@ -9,6 +9,9 @@ // No branch: a lane past the last token computes the last token (its inputs are clamped there, as HIP's) and stores // that token's bytes to that token's place, the same bytes its own lane stores (a branch around the store let the // compiler sink the matrix instructions into it, see l_ops_linear.hlsl). +// Conversions by the driver's instructions (wg_platform.hlsli; the same bits as the software wg_rn16 / wg_pack_q8, and +// this kernel has no branch): RN16 = v_cvt_f16_f32 then the exact widening, the FP8 pack v_cvt_pk_fp8_f32 after the +// clamp. The software forms were most of the kernels' instructions. // // @wg-kernel l_project_gather co=l_ops.co block=256 wave=32 args=p,p,p,u,u defines=PJ_DEC=0 // @wg-kernel l_dec_project co=l_ops.co block=256 wave=32 args=p,p,p,u,u defines=PJ_DEC=1 @@ -29,6 +32,15 @@ WG_ARGS_END #define FRAG0 256u // 4 * 64: after the bias tile vector, rows R_L(64) #endif +// RN16 of 8 values by the driver's conversion +v8f pj_rn16(v8f a) +{ + const uint4 h = wg_to_f16_ags(a); + v8f r; + [unroll] for (uint j = 0; j < 4u; ++j) { r.v[2 * j] = wg_h2f_lo(h[j]); r.v[2 * j + 1] = wg_h2f_hi(h[j]); } + return r; +} + #if PJ_DEC WG_KERNEL(l_dec_project) #else @@ -60,15 +72,15 @@ WG_KERNEL(l_project_gather) [unroll] for (uint g = 0; g < KB / 2; ++g) { acc = wg_wmma_fp8(fr[2u * g], zf[2u * g], acc); acc = wg_wmma_fp8(fr[2u * g + 1u], zf[2u * g + 1u], acc); - [unroll] for (uint i = 0; i < 8u; ++i) acc.v[i] = wg_rn16(acc.v[i]); + acc = pj_rn16(acc); } #if PJ_DEC const int s = (int)(mt >> 1), gg = (int)(mt & 1u); const int pix = (2 * r + (s >> 1)) * (2 * tw) + 2 * c + (s & 1); - wg_st64(dst, (uint)pix * 32u + 16u * hh + 8u * (uint)gg, wg_pack_q8(acc)); + wg_st64(dst, (uint)pix * 32u + 16u * hh + 8u * (uint)gg, wg_pack_q8_ags(acc)); #else [unroll] for (uint j = 0; j < 8u; ++j) acc.v[j] = wg_fmin(wg_fmax(acc.v[j], -6.28125f), 6.28125f); - wg_st64(dst, (uint)tk * 64u + hh * 32u + 8u * mt, wg_pack_q8(acc)); + wg_st64(dst, (uint)tk * 64u + hh * 32u + 8u * mt, wg_pack_q8_ags(acc)); #endif } } diff --git a/clean/l/runner/l_d3d12_ops_check.py b/clean/l/runner/l_d3d12_ops_check.py index 694261e8..6ce3c0c8 100644 --- a/clean/l/runner/l_d3d12_ops_check.py +++ b/clean/l/runner/l_d3d12_ops_check.py @@ -84,23 +84,47 @@ def control_cases(kernel, rng): TOKENS = [(8, 12), (18, 34), (4, 4), (34, 50), (3, 5), (2, 9)] -def _project_blob(rng, rows, frags): +def _project_blob(rng, rows, frags, scale=0.2, bias_scale=0.5): """Projection blob: bias tile vector (`rows` floats), then `frags` fragments of 256 FP8 bytes.""" - return rng.normal(0, 0.5, rows).astype(np.float32).tobytes() + fp8_encode(q8(rng.normal(0, 0.2, frags * 256))).tobytes() + return (rng.normal(0, bias_scale, rows).astype(np.float32).tobytes() + + fp8_encode(q8(rng.normal(0, scale, frags * 256))).tobytes()) + + +# FP8 codes the network does not produce but the kernels' conversions must treat as HIP does: both NaN codes, +-448, +# +-0, subnormals +SPECIAL_FP8 = np.array([0x7f, 0xff, 0x7e, 0xfe, 0x00, 0x80, 0x01, 0x81, 0x07, 0x87], np.uint8) +# (weight, bias) scales of the projection cases: normal; large (sums past 448 and past FP16's 65504: the clamps, +# infinities); tiny (FP16 subnormal and zero results) +PROJECT_SCALES = [(0.2, 0.5), (24.0, 400.0), (2e-3, 1e-4)] + + +def _fp8_map_special(rng, n, special): + """_fp8_map, with special FP8 codes in a tenth of the bytes when `special`.""" + q = np.frombuffer(_fp8_map(rng, n), np.uint8).copy() + if special: + pick = rng.random(q.size) < 0.1 + q[pick] = rng.choice(SPECIAL_FP8, int(pick.sum())) + return q.tobytes() def project_cases(kernel, rng): """l_project_gather: x [2 th][2 tw][32] -> features [th][tw][64] (blob enc0.project: 64 bias rows, 4 x 8 fragments). - l_dec_project: z [th][tw][64] -> y [2 th][2 tw][32] (blob dec0.project: 128 bias rows, 8 x 4 fragments).""" + l_dec_project: z [th][tw][64] -> y [2 th][2 tw][32] (blob dec0.project: 128 bias rows, 8 x 4 fragments). + Every token grid at the three PROJECT_SCALES, every other case with special FP8 codes in the input.""" cases = [] - for (th, tw) in TOKENS: + for ti, (th, tw) in enumerate(TOKENS): ntok = th * tw npx = 4 * ntok - if kernel == "l_project_gather": - args = [("buf", _fp8_map(rng, npx * 32)), ("out", ntok * 64), ("buf", _project_blob(rng, 64, 32))] - else: - args = [("buf", _fp8_map(rng, ntok * 64)), ("out", npx * 32), ("buf", _project_blob(rng, 128, 32))] - cases.append({"kernel": kernel, "grid": -(-ntok // 128), "block": 256, "args": args + [("u32", tw), ("u32", th)]}) + for si, (s, b) in enumerate(PROJECT_SCALES): + special = (ti + si) % 2 == 1 + if kernel == "l_project_gather": + args = [("buf", _fp8_map_special(rng, npx * 32, special)), ("out", ntok * 64), + ("buf", _project_blob(rng, 64, 32, s, b))] + else: + args = [("buf", _fp8_map_special(rng, ntok * 64, special)), ("out", npx * 32), + ("buf", _project_blob(rng, 128, 32, s, b))] + cases.append({"kernel": kernel, "grid": -(-ntok // 128), "block": 256, + "args": args + [("u32", tw), ("u32", th)]}) return cases