From 26f1ede61e094e3d658dd62652de4f52ee91aad0 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:19:33 -0500 Subject: [PATCH] Model L: D3D12 ports of the two projections, byte for byte against their HIP kernels l_ops_project.hlsl ports l_project_gather and l_dec_project of l_ops.co (clean/l/kernels/l_ops.hip, #115): two FP8 WMMAs then RN16 per group of 32 inputs, the gather's clamp, FP8 out; no branch (a lane past the last token stores its clamped token's bytes to that token's place). l_d3d12_ops_check.py gains their cases (6 token grids, two of them partly filled waves). Refs #89 Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/kernels/l_ops_project.hlsl | 74 +++++++++++++++++++++++++++ clean/l/runner/l_d3d12_ops_check.py | 27 +++++++++- 2 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 clean/dx12/kernels/l_ops_project.hlsl diff --git a/clean/dx12/kernels/l_ops_project.hlsl b/clean/dx12/kernels/l_ops_project.hlsl new file mode 100644 index 00000000..86a7c284 --- /dev/null +++ b/clean/dx12/kernels/l_ops_project.hlsl @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: MIT +// D3D12 ports of Model L's two projections (issue #89): l_project_gather and l_dec_project of l_ops.co +// (clean/l/kernels/l_ops.hip), byte for byte. Maps and blobs: clean/l/FORMATS.md. +// +// 256 lanes; wave i handles tokens 16 i .. 16 i + 15 of its workgroup's 128 (raster order), lane (t, h) token 16 i + t; +// accumulator element v of output tile mt is output channel 16 h + 8 mt' + v of the tile's layout (HIP: layout L / E). +// K blocks in the order of clean/l/runner/l_pack.py kord_groups32: block kb, lane half h = channel +// 32 (kb / 2) + 16 h + 8 (kb % 2) + j. Per group of 32 inputs: two FP8 WMMAs, then RN16 (running FP16 sum, bias first). +// No branch: a lane past the last token computes the last token (its inputs are clamped there, as HIP's) and stores +// that token's bytes to that token's place, the same bytes its own lane stores (a branch around the store let the +// compiler sink the matrix instructions into it, see l_ops_linear.hlsl). +// +// @wg-kernel l_project_gather co=l_ops.co block=256 wave=32 args=p,p,p,u,u defines=PJ_DEC=0 +// @wg-kernel l_dec_project co=l_ops.co block=256 wave=32 args=p,p,p,u,u defines=PJ_DEC=1 +#include "wg_platform.hlsli" + +WG_ARGS_BEGIN + uint src, dst, w; // gather: x (pixel map [2 th][2 tw][32] FP8), features [th][tw][64]; dec: z [th][tw][64], y pixel map + int tw, th; +WG_ARGS_END + +#if PJ_DEC +#define MT 8 +#define KB 4 +#define FRAG0 512u // 4 * 128: after the bias tile vector, rows R_E(32) x 4 sub-pixels +#else +#define MT 4 +#define KB 8 +#define FRAG0 256u // 4 * 64: after the bias tile vector, rows R_L(64) +#endif + +#if PJ_DEC +WG_KERNEL(l_dec_project) +#else +WG_KERNEL(l_project_gather) +#endif +{ + WG_SETUP(); + const uint tid = wg_lane_id(), lane = tid & 31u, hh = (tid >> 4) & 1u; + const int ntok = tw * th; + const int tok = (int)(wg_block_id() * (WG_BLOCK / WG_WAVE) + (tid >> 5)) * 16 + (int)(tid & 15u); + const int tk = select(tok < ntok, tok, ntok - 1); + const int r = tk / tw, c = tk - r * tw; + uint2 zf[KB]; +#if PJ_DEC + [unroll] for (uint kb = 0; kb < KB; ++kb) + zf[kb] = wg_ld64(src, (uint)tk * 64u + 32u * (kb >> 1) + 16u * hh + 8u * (kb & 1u)); +#else + [unroll] for (uint kb = 0; kb < KB; ++kb) { + const int p = (int)(kb >> 1); + const int pix = (2 * r + (p >> 1)) * (2 * tw) + 2 * c + (p & 1); + zf[kb] = wg_ld64(src, (uint)pix * 32u + 16u * hh + 8u * (kb & 1u)); + } +#endif + [unroll] for (uint mt = 0; mt < MT; ++mt) { + // the loads of this tile's multiplies before its first matrix instruction + v8f acc = wg_load8f(w, (hh * 8u + mt * 16u) * 4u); + uint2 fr[KB]; + [unroll] for (uint k = 0; k < KB; ++k) fr[k] = wg_ld64(w, FRAG0 + (mt * KB + k) * 256u + lane * 8u); + [unroll] for (uint g = 0; g < KB / 2; ++g) { + acc = wg_wmma_fp8(fr[2u * g], zf[2u * g], acc); + acc = wg_wmma_fp8(fr[2u * g + 1u], zf[2u * g + 1u], acc); + [unroll] for (uint i = 0; i < 8u; ++i) acc.v[i] = wg_rn16(acc.v[i]); + } +#if PJ_DEC + const int s = (int)(mt >> 1), gg = (int)(mt & 1u); + const int pix = (2 * r + (s >> 1)) * (2 * tw) + 2 * c + (s & 1); + wg_st64(dst, (uint)pix * 32u + 16u * hh + 8u * (uint)gg, wg_pack_q8(acc)); +#else + [unroll] for (uint j = 0; j < 8u; ++j) acc.v[j] = wg_fmin(wg_fmax(acc.v[j], -6.28125f), 6.28125f); + wg_st64(dst, (uint)tk * 64u + hh * 32u + 8u * mt, wg_pack_q8(acc)); +#endif + } +} diff --git a/clean/l/runner/l_d3d12_ops_check.py b/clean/l/runner/l_d3d12_ops_check.py index ef256f12..e19470d7 100644 --- a/clean/l/runner/l_d3d12_ops_check.py +++ b/clean/l/runner/l_d3d12_ops_check.py @@ -77,7 +77,32 @@ def control_cases(kernel, rng): return cases -KINDS = {"l_linear_32_32_relu": linear_cases, "l_linear_32_32_f16": linear_cases} +# (th, tw) token grids of the projections; the pixel maps are [2 th][2 tw]. 3 x 5 and 2 x 9: partly filled waves. +TOKENS = [(8, 12), (18, 34), (4, 4), (34, 50), (3, 5), (2, 9)] + + +def _project_blob(rng, rows, frags): + """Projection blob: bias tile vector (`rows` floats), then `frags` fragments of 256 FP8 bytes.""" + return rng.normal(0, 0.5, rows).astype(np.float32).tobytes() + fp8_encode(q8(rng.normal(0, 0.2, frags * 256))).tobytes() + + +def project_cases(kernel, rng): + """l_project_gather: x [2 th][2 tw][32] -> features [th][tw][64] (blob enc0.project: 64 bias rows, 4 x 8 fragments). + l_dec_project: z [th][tw][64] -> y [2 th][2 tw][32] (blob dec0.project: 128 bias rows, 8 x 4 fragments).""" + cases = [] + for (th, tw) in TOKENS: + ntok = th * tw + npx = 4 * ntok + if kernel == "l_project_gather": + args = [("buf", _fp8_map(rng, npx * 32)), ("out", ntok * 64), ("buf", _project_blob(rng, 64, 32))] + else: + args = [("buf", _fp8_map(rng, ntok * 64)), ("out", npx * 32), ("buf", _project_blob(rng, 128, 32))] + cases.append({"kernel": kernel, "grid": -(-ntok // 128), "block": 256, "args": args + [("u32", tw), ("u32", th)]}) + return cases + + +KINDS = {"l_linear_32_32_relu": linear_cases, "l_linear_32_32_f16": linear_cases, + "l_project_gather": project_cases, "l_dec_project": project_cases} CONTROL_KINDS = {"swin_block_c96_nc": control_cases} CO = "l_ops.co"