From 9458df934b47154e7ff617736624aba1a0f7a595 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sat, 14 Nov 2026 05:22:06 -0500 Subject: [PATCH] Model L on D3D12: the two input linears in one kernel l_linear_32_32_fused (D3D12 only): l_linear_32_32_relu and l_linear_32_32_f16 in one kernel, x1 kept in registers (the relu linear's lane stores exactly the two FP8 fragments the f16 linear's lane loads), byte for byte the pair. Its RN16, FP8 pack and FP16 conversions are the driver's instructions (the same bits as the software forms the pair keeps, which were most of the pair's instructions). - runtime: native D3D12 fusion "ll" (default on; Model L). The input encoder then writes q to l_narrow (x2 over q in l_wide would overwrite q that other workgroups have not read yet). Without ll, or without the kernel in the manifest, the pair runs. - clean/l/runner/l_linfuse_check.py: the fused kernel against the pair on D3D12 and on HIP, raw bytes, per-run sentinels in the slack, special FP8 codes and extreme weight scales, negative control, Model M control. - tests: the recorder's signature; the frame and role checks take either linear form. Refs #89 Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/compare.py | 2 +- clean/dx12/kernels/l_ops_linear.hlsl | 67 +++++- clean/dx12/tests/native_direct_check.py | 4 +- clean/l/FORMATS.md | 5 +- clean/l/runner/l_linfuse_check.py | 203 ++++++++++++++++++ clean/l/runtime/tests/test_l_runtime_frame.py | 71 ++++-- clean/runtime/runtime.cpp | 30 ++- clean/tests/runtime_frame_trace.cpp | 1 + 8 files changed, 349 insertions(+), 34 deletions(-) create mode 100644 clean/l/runner/l_linfuse_check.py diff --git a/clean/dx12/compare.py b/clean/dx12/compare.py index 80b35af6..c71184c9 100644 --- a/clean/dx12/compare.py +++ b/clean/dx12/compare.py @@ -68,7 +68,7 @@ "expand_c96", "d1p12/swin_block_c64_add_dec0_nc", "w64/post_tf_f32_p12_diet_vsit"} # The native backend (--backend d3d12) runs the D3D12 fusions of WG_D3D12_FUSION (default: runtime.cpp's # kDefaultD3d12Fusion, mirrored here), which replace some launches (kernels/fused_swin.hlsl, fused_bneck.hlsl, fused_dec.hlsl). -DEFAULT_D3D12_FUSION = "pm,xe,mb,rv,xo,xd,lc" # lc: Model L only +DEFAULT_D3D12_FUSION = "pm,xe,mb,rv,xo,xd,lc,ll" # lc, ll: Model L only def d3d12_fusion(): diff --git a/clean/dx12/kernels/l_ops_linear.hlsl b/clean/dx12/kernels/l_ops_linear.hlsl index 2b60c3f2..fbbc71f5 100644 --- a/clean/dx12/kernels/l_ops_linear.hlsl +++ b/clean/dx12/kernels/l_ops_linear.hlsl @@ -11,16 +11,56 @@ // the store let the D3D12 compiler sink the matrix instructions into it, where they ran with part of the wave masked // off: wrong values in a partly filled wave (a 35-pixel map, pixels 32..34; found by l_d3d12_ops_check.py). // +// l_linear_32_32_fused (D3D12 only): the two in one kernel, x1 kept in registers. The relu linear's lane (t, h) stores +// x1's channels 16 h + 8 m .. + 7 (m = 0, 1) as the two FP8 fragments that the f16 linear's lane (t, h) loads as its +// K blocks kc = m: the fragments pass in registers, the same bytes, and the x1 map is neither written nor read. x is +// the q map, y the x2 map: two different buffers (the pair runs q -> x1 -> x2 with x2 over q's buffer; here x2 would +// overwrite q that other workgroups have not read yet). Arguments: x, y, Win1, Win2, npix. Its conversions are the +// driver's instructions (RN16 and the final FP16 by v_cvt_f16_f32, the FP8 pack by v_cvt_pk_fp8_f32 after the clamp: +// wg_platform.hlsli, the same bits as the software forms the pair keeps); the software forms were most of the pair's +// instructions. Measured at 1080p Quality: 0.095 ms against the pair's 0.217 ms (the fused kernel with the software +// forms: 0.236 ms). +// // @wg-kernel l_linear_32_32_relu co=l_ops.co block=256 wave=32 args=p,p,p,u defines=LIN_RELU=1 // @wg-kernel l_linear_32_32_f16 co=l_ops.co block=256 wave=32 args=p,p,p,u defines=LIN_RELU=0 +// @wg-kernel l_linear_32_32_fused co=- block=256 wave=32 args=p,p,p,p,u defines=LIN_RELU=0,LIN_FUSED=1 #include "wg_platform.hlsli" +#ifndef LIN_FUSED +#define LIN_FUSED 0 +#endif + WG_ARGS_BEGIN uint x, y, w; // pointers: x [npix][32] FP8, y [npix][32] FP8 or FP16, w the blob (enc0.Win1 / enc0.Win2) +#if LIN_FUSED + uint w2; // the f16 linear's blob (enc0.Win2; w = enc0.Win1) +#endif int npix; WG_ARGS_END -#if LIN_RELU +#if LIN_FUSED // (helpers of the fused kernel only: the pair keeps its own code and DXIL) +// the blob's bias tile vectors and weight fragments of lane (t, h) +void lin_blob(uint wb, uint lane, uint hh, out v8f bias[2], out uint2 fr[2][2]) +{ + [unroll] for (uint mt = 0; mt < 2u; ++mt) { + bias[mt] = wg_load8f(wb, (hh * 8u + mt * 16u) * 4u); + [unroll] for (uint kc2 = 0; kc2 < 2u; ++kc2) fr[mt][kc2] = wg_ld64(wb, 128u + (mt * 2u + kc2) * 256u + lane * 8u); + } +} + +// output tile m before the rounding: b + W x +v8f lin_acc(v8f bias, uint2 fr[2], uint2 xf[2]) +{ + v8f acc = bias; + acc = wg_wmma_fp8(fr[0], xf[0], acc); + return wg_wmma_fp8(fr[1], xf[1], acc); +} + +#endif + +#if LIN_FUSED +WG_KERNEL(l_linear_32_32_fused) +#elif LIN_RELU WG_KERNEL(l_linear_32_32_relu) #else WG_KERNEL(l_linear_32_32_f16) @@ -35,6 +75,30 @@ WG_KERNEL(l_linear_32_32_f16) [unroll] for (uint kc = 0; kc < 2u; ++kc) xf[kc] = wg_ld64(x, (uint)pk * 32u + hh * 16u + 8u * kc); v8f bias[2]; uint2 fr[2][2]; +#if LIN_FUSED + lin_blob(w, lane, hh, bias, fr); + v8f bias2[2]; + uint2 fr2[2][2]; + lin_blob(w2, lane, hh, bias2, fr2); + // the relu linear: x1's fragments (the bytes its kernel stores at pk * 32 + hh * 16 + 8 m). The conversions by the + // driver's instructions (wg_platform.hlsli: the same bits as the pair's software wg_rn16 / wg_pack_q8; no branch in + // this kernel): RN16 = v_cvt_f16_f32 then the exact widening, the pack v_cvt_pk_fp8_f32 after the clamp. + uint2 x1[2]; + [unroll] for (uint m1 = 0; m1 < 2u; ++m1) { + const uint4 h = wg_to_f16_ags(lin_acc(bias[m1], fr[m1], xf)); + v8f r; + [unroll] for (uint j = 0; j < 4u; ++j) { + r.v[2 * j] = wg_fmax(wg_h2f_lo(h[j]), 0.0f); + r.v[2 * j + 1] = wg_fmax(wg_h2f_hi(h[j]), 0.0f); + } + x1[m1] = wg_pack_q8_ags(r); + } + // the f16 linear on them (its kernel's loads of x1 at pk * 32 + hh * 16 + 8 kc): FP16(RN16(acc)) = FP16(acc), one + // conversion (rounding to FP16 is idempotent, NaN payloads included) + [unroll] for (uint m = 0; m < 2u; ++m) + wg_st128(y, ((uint)pk * 32u + hh * 16u + 8u * m) * 2u, wg_to_f16_ags(lin_acc(bias2[m], fr2[m], x1))); +#else + // (the pair's own code, unchanged) [unroll] for (uint mt = 0; mt < 2u; ++mt) { bias[mt] = wg_load8f(w, (hh * 8u + mt * 16u) * 4u); [unroll] for (uint kc2 = 0; kc2 < 2u; ++kc2) fr[mt][kc2] = wg_ld64(w, 128u + (mt * 2u + kc2) * 256u + lane * 8u); @@ -51,4 +115,5 @@ WG_KERNEL(l_linear_32_32_f16) wg_st128(y, ((uint)pk * 32u + hh * 16u + 8u * m) * 2u, wg_to_f16(acc)); #endif } +#endif } diff --git a/clean/dx12/tests/native_direct_check.py b/clean/dx12/tests/native_direct_check.py index 84fe0d88..0bed5f9f 100644 --- a/clean/dx12/tests/native_direct_check.py +++ b/clean/dx12/tests/native_direct_check.py @@ -30,8 +30,8 @@ POST='w64/post_tf_f32_p12_diet_vsit' -# The neural launches with the native D3D12 fusions of runtime.cpp kDefaultD3d12Fusion (pm,xe,mb,rv,xo,xd; its lc is -# Model L's): enc2's block +# The neural launches with the native D3D12 fusions of runtime.cpp kDefaultD3d12Fusion (pm,xe,mb,rv,xo,xd; its lc and ll +# are Model L's): enc2's block # with enc1's merge in reverse order (pm, rv), the bottleneck with enc2's merge and the overlap counters (mb, xo), dec2's # block with dec1's expand (xe) launched as xd on frames of at most kXdMaxPixels render pixels (WG_D3D12_XD_MAX_PIXELS) # else xo, and the fused dec1 + dec0 in its xd form on those frames. Set by main from --shape (COMMON above: the diff --git a/clean/l/FORMATS.md b/clean/l/FORMATS.md index fce40a7a..2ed72cb1 100644 --- a/clean/l/FORMATS.md +++ b/clean/l/FORMATS.md @@ -23,7 +23,7 @@ Every map is `[rows][cols][C]`, channel fastest, as Model M's feature maps (`cle | Map | Shape | Element | Written by | Read by | |---|---|---|---|---| | `q` | `[fh + 4][fw + 4][32]` halo | FP8: values 0..19, bytes 20..31 zero | `lev/input_encoder` | `l_linear_32_32_relu` | -| `x1` | `[fh + 4][fw + 4][32]` halo | FP8 | `l_linear_32_32_relu` | `l_linear_32_32_f16` | +| `x1` | `[fh + 4][fw + 4][32]` halo | FP8 (not stored by `l_linear_32_32_fused`) | `l_linear_32_32_relu` | `l_linear_32_32_f16` | | `x2` | `[fh + 4][fw + 4][32]` halo | **FP16** (64 bytes per pixel) | `l_linear_32_32_f16` | `l_swin_block_c32_enc` | | `x3` | `[fh][fw][32]` | FP8 | `l_swin_block_c32_enc` | `l_project_gather` | | `enc0.features` | `[fh/2][fw/2][64]` | FP8 | `l_project_gather` | enc1 (Model M's stage) | @@ -78,6 +78,7 @@ workgroup, in raster order of their map. | `lev/input_encoder` | exists (GPU-checked) | as Model M's `input_encoder`, with `features` replaced by `q` | ceil(((fh + 4) / 2) ((fw + 4) / 2) / 128) (2 x 2 pixel groups of the halo map) | the 20 values and their dithered FP8 for every halo-map pixel, into `q`. The motion surfaces as Model M, only for pixels inside R. w = `enc0.values` | | `l_linear_32_32_relu` | exists (GPU-checked) | `x, y, w, npix` | ceil(npix / 128), npix = (fh + 4)(fw + 4) | y = q8(max(0, RN16(b + W x))), FP8 -> FP8. w = `enc0.Win1` | | `l_linear_32_32_f16` | exists (GPU-checked) | `x, y, w, npix` | as above | y = RN16(b + W x), FP8 -> **FP16**. w = `enc0.Win2` | +| `l_linear_32_32_fused` | exists (GPU-checked), D3D12 only | `x, y, w1, w2, npix` | as above | the two above in one kernel, `x1` kept in registers: the same bytes in `y` (`x2`), `x1` not stored. x = `q`, y = `x2` in two different buffers. w1 = `enc0.Win1`, w2 = `enc0.Win2`. The native D3D12 frame runs it in place of the two (fusion `ll`, on by default); without `ll` or without the kernel in the build, the two run | | `l_swin_block_c32_enc[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | ceil(nwin / 8), nwin = ceil((W + ox) / 4) ceil((H + oy) / 4) | the encoder block. x = `x2` (FP16 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2)). out = `x3` (FP8, grid only). H = fh, W = fw. (ox, oy) = `window_offsets["first"]` for the frame parity. w = `enc0_block` | | `l_project_gather` | exists (GPU-checked) | `x, features, w, tw, th` | ceil(tw th / 128), tw = fw / 2, th = fh / 2 | token (r, c) gathers pixels (2r + p / 2, 2c + p % 2), p = 0..3, as channels 32 p + i. Running FP16 sum per group of 32 from the bias, clamp +-6.28125, FP8 -> `enc0.features`. w = `enc0.project` | | `l_dec_project` | exists (GPU-checked) | `z, y, w, tw, th` | ceil(tw th / 128) | per token of dec1's output z [th][tw][64]: 128 outputs (two RN16 groups of 32 inputs, then FP8). Output 32 s + k goes to pixel (2r + s / 2, 2c + s % 2), channel k, of `y`. w = `dec0.project` | @@ -159,4 +160,6 @@ the frame parity, as Model M's. - `clean/l/runner/l_d3d12_check.py` (with its D3D12 port): each D3D12 port against its HIP kernel, byte for byte. - `clean/l/runner/l_fold_check.py`: `l_swin_block_c32_dec_coef12_nc` against `l_swin_block_c32_dec_nc` -> `l_coef_pack12` on D3D12 and on HIP, byte for byte. +- `clean/l/runner/l_linfuse_check.py`: `l_linear_32_32_fused` against `l_linear_32_32_relu` -> `l_linear_32_32_f16` + on D3D12 and on HIP, byte for byte. - One check per new kernel follows with that kernel. diff --git a/clean/l/runner/l_linfuse_check.py b/clean/l/runner/l_linfuse_check.py new file mode 100644 index 00000000..97bc2132 --- /dev/null +++ b/clean/l/runner/l_linfuse_check.py @@ -0,0 +1,203 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): the fused input linears l_linear_32_32_fused (D3D12 only; clean/dx12/kernels/l_ops_linear.hlsl) +against the sequence it replaces, l_linear_32_32_relu -> l_linear_32_32_f16, byte for byte. + + python clean/l/runner/l_linfuse_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --prepare + python clean/l/runner/l_linfuse_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --run + [--seeds N] --json out.json + +--prepare (CPU) builds into : l_ops.co and swin.co (clean/l/runner/l_c32_check.py compile_kernels), the D3D12 +kernels (clean/dx12/build_dx12.py --only) and kernel_replay.exe. +--run (GPU, one GPU slot): every case is a pixel map (npix pixels, partly filled waves and workgroups among them), an +FP8 input map q and the two blobs (enc0.Win1, enc0.Win2), at three weight scales (results past 448 and past FP16's +range, results in FP16's subnormal range) and with special FP8 codes in q (NaN, +-448, +-0, subnormals): the fused +kernel's conversions are the driver's, the pair's software. On the same input bytes: +- the sequence on D3D12 (kernel_replay, two passes: the relu linear writes x1, then the f16 linear reads that file); +- the sequence on HIP (in this process); +- the fused kernel on D3D12. +The rules are clean/l/runner/l_fold_check.py's: raw bytes; the output (x2, 64 bytes a pixel) has 64 bytes of slack and +starts filled with its run's own sentinel (HIP 0xA5, the D3D12 sequence 0x5A, the fused kernel 0x3C): every pixel byte +identical across the three, every slack byte its own run's sentinel; a negative control (the fused kernel with half its +workgroups) must be caught; Model M's swin_block_c96_nc runs through the same replay as a control. +Output: counts only, no timing. +""" +import argparse +import ctypes +import json +import os +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.runner import l_c32_check as C32 # noqa: E402 +from clean.l.runner import l_d3d12_check as LD # noqa: E402 +from clean.l.runner import l_d3d12_ops_check as OPS # noqa: E402 +from clean.reference.fp8 import fp8_encode, q8 # noqa: E402 +from clean.l.runner.l_fold_check import (CONTROL, D3D12_SENTINEL, FOLD_SENTINEL, HIP_SENTINEL, SLACK, compare, # noqa: E402 + nwin_of, replay, write) + +RELU, F16, FUSED = "l_linear_32_32_relu", "l_linear_32_32_f16", "l_linear_32_32_fused" +# (rows, cols) of the maps: the ops check's (a 35-pixel map among them: a partly filled wave), the halo maps of the +# runtime's smallest frames, 1 x 1, and one more workgroup than a whole number (129 pixels) +MAPS = OPS.MAPS + [(36, 36), (1, 1), (3, 43), (68, 132)] + + +def _blob(rng, scale, bias_scale): + """enc0.Win1 / Win2 layout (bias tile vector of 32 floats, 4 fragments of 256 FP8 bytes) at a weight scale.""" + bias = rng.normal(0, bias_scale, 32).astype(np.float32).tobytes() + return bias + fp8_encode(q8(rng.normal(0, scale, 4 * 256))).tobytes() + + +# FP8 codes the network does not produce but the conversions must treat as the pair does: both NaN codes, +-448, +-0, +# subnormals +SPECIAL = np.array([0x7f, 0xff, 0x7e, 0xfe, 0x00, 0x80, 0x01, 0x81, 0x07, 0x87], np.uint8) +# weight / bias scales: the ops check's; large (sums past 448 and past FP16's 65504: infinities); tiny (FP16 subnormal +# and zero results) +SCALES = [(0.25, 0.5), (24.0, 400.0), (2e-3, 1e-4)] + + +def cases(rng): + """(npix, q bytes, Win1 blob, Win2 blob) of every case: each map at each weight scale, every other case with + special FP8 codes in a tenth of q's bytes.""" + for ci, (rows, cols) in enumerate(MAPS): + n = rows * cols + for si, (s, b) in enumerate(SCALES): + q = np.frombuffer(OPS._fp8_map(rng, n * 32), np.uint8).copy() + if (ci + si) % 2: + pick = rng.random(q.size) < 0.1 + q[pick] = rng.choice(SPECIAL, int(pick.sum())) + yield n, q.tobytes(), _blob(rng, s, b), _blob(rng, s, b) + + +def hip_sequence(g, ops, c): + """relu, then f16, on HIP: x2's raw bytes (the output starts at HIP_SENTINEL).""" + n, q, w1, w2 = c + h = g.hip + owned = [] + + def buf(data): + p = h.malloc(max(len(data), 4)) + h.upload(p, data) + owned.append(p) + return p + + try: + pq, pw1, pw2 = buf(q), buf(w1), buf(w2) + px1, py = h.malloc(n * 32), h.malloc(n * 64 + SLACK) + owned += [px1, py] + h.memset(px1, HIP_SENTINEL, n * 32) + h.memset(py, HIP_SENTINEL, n * 64 + SLACK) + grid = -(-n // 128) + for k, a, b, wb in ((RELU, pq, px1, pw1), (F16, px1, py, pw2)): + h.launch(h.get_function(ops, k), grid, 256, + [ctypes.c_void_p(a), ctypes.c_void_p(b), ctypes.c_void_p(wb), ctypes.c_int(n)]) + h.sync() + return bytes(h.download(py, n * 64 + SLACK)) + finally: + for p in owned: + h.free(p) + + +def fused_lines(name, n, prefix="", grid=None): + return [f"case {prefix}{name}_fused", f"kernel {FUSED} {grid or -(-n // 128)} 256", f"arg buf {name}.q", + f"arg out {n * 64 + SLACK} {prefix}{name}.fy {FOLD_SENTINEL}", f"arg buf {name}.w1", f"arg buf {name}.w2", + f"arg u32 {n}", ""] + + +def run(build, rocm, seeds): + g = C32.Gpu(build, rocm) + with open(os.path.join(build, "l_ops.co"), "rb") as f: + ops = g.hip.load_module(f.read()) + cdir = os.path.join(build, "replay-linfuse") + os.makedirs(cdir, exist_ok=True) + pass1, pass2, all_cases, neg = [], [], {}, None + for si, seed in enumerate(seeds): + rng = np.random.default_rng(seed) + for ci, c in enumerate(cases(rng)): + name = f"s{si}_{ci}" + n, q, w1, w2 = c + all_cases[name] = (n, hip_sequence(g, ops, c)) + write(os.path.join(cdir, name + ".q"), q) + write(os.path.join(cdir, name + ".w1"), w1) + write(os.path.join(cdir, name + ".w2"), w2) + grid = -(-n // 128) + # pass 1: the D3D12 relu linear (x1) and the fused kernel + pass1 += [f"case {name}_relu", "module l_ops.co", f"kernel {RELU} {grid} 256", f"arg buf {name}.q", + f"arg out {n * 32} {name}.x1 {D3D12_SENTINEL}", f"arg buf {name}.w1", f"arg u32 {n}", ""] + pass1 += fused_lines(name, n) + if neg is None and grid >= 4: + neg = name + pass1 += fused_lines(name, n, prefix="negative_", grid=grid // 2) + # pass 2: the D3D12 f16 linear on that x1 + pass2 += [f"case {name}_f16", "module l_ops.co", f"kernel {F16} {grid} 256", f"arg buf {name}.x1", + f"arg out {n * 64 + SLACK} {name}.sy {D3D12_SENTINEL}", f"arg buf {name}.w2", f"arg u32 {n}", ""] + rng = np.random.default_rng(seeds[0]) + ctl = [] + for ci, (H, W, off, xb, nout, blob) in enumerate(LD.block_cases(CONTROL, rng)): + name = f"control_{ci}" + ctl.append((name, LD.hip_block_raw(g, "swin.co", CONTROL, xb, blob, H, W, off, nout))) + write(os.path.join(cdir, name + ".x"), xb) + write(os.path.join(cdir, name + ".w"), blob) + pass1 += [f"case {name}", "module swin.co", f"kernel {CONTROL} {-(-nwin_of(H, W, off) // 8)} 256", + f"arg buf {name}.x", "arg null", f"arg out {nout} {name}.out {D3D12_SENTINEL}", f"arg buf {name}.w", + f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {off[0]}", f"arg u32 {off[1]}", ""] + e1, t1 = replay(build, cdir, pass1, "pass1") + e2, t2 = replay(build, cdir, pass2, "pass2") if e1 == 0 else (-1, "pass 1 failed") + res = {"replay_exit": [e1, e2], "replay_tail": [t1, t2], "cases": []} + + def read(n): + p = os.path.join(cdir, n) + return open(p, "rb").read() if os.path.exists(p) else b"" + + for name, (n, hip) in all_cases.items(): + fused, seq = read(f"{name}.fy"), read(f"{name}.sy") + r = {"case": name, "npix": n, + "fused_vs_d3d12_seq": compare(fused, seq, n * 64, FOLD_SENTINEL, D3D12_SENTINEL), + "fused_vs_hip_seq": compare(fused, hip, n * 64, FOLD_SENTINEL, HIP_SENTINEL), + "d3d12_seq_vs_hip_seq": compare(seq, hip, n * 64, D3D12_SENTINEL, HIP_SENTINEL)} + r["identical"] = all(r[k]["identical"] for k in ("fused_vs_d3d12_seq", "fused_vs_hip_seq", "d3d12_seq_vs_hip_seq")) + res["cases"].append(r) + caught = None + if neg: + n = all_cases[neg][0] + o = compare(read(f"negative_{neg}.fy"), read(f"{neg}.sy"), n * 64, FOLD_SENTINEL, D3D12_SENTINEL) + caught = o["differ"] >= 0 and not o["identical"] + res["negative_control"] = {"case": neg, "half_grid": True, "caught": caught, "differ": max(o["differ"], 0)} + ctl_ok = [read(f"{name}.out") == want for name, want in ctl] + res["control"] = {"kernel": CONTROL, "cases": len(ctl_ok), "identical": sum(ctl_ok)} + res["summary"] = {"cases": len(res["cases"]), "identical": sum(c["identical"] for c in res["cases"]), + "pixel_bytes": sum(c["npix"] * 64 for c in res["cases"])} + res["pass"] = (e1 == 0 and e2 == 0 and caught is True and res["summary"]["cases"] > 0 and + res["summary"]["identical"] == res["summary"]["cases"] and len(ctl_ok) > 0 and all(ctl_ok)) + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--prepare", action="store_true") + ap.add_argument("--run", action="store_true") + ap.add_argument("--seeds", type=int, default=2) + ap.add_argument("--json") + a = ap.parse_args(argv) + os.makedirs(a.build, exist_ok=True) + if a.prepare: + C32.compile_kernels(a.build, a.rocm, with_swin=True) + print("built", LD.prepare_replay(a.build, a.rocm, [RELU, F16, FUSED, CONTROL])) + if a.run: + res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds))) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + print({k: v for k, v in res.items() if k not in ("cases", "replay_tail")}) + return 0 if res["pass"] else 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runtime/tests/test_l_runtime_frame.py b/clean/l/runtime/tests/test_l_runtime_frame.py index 11213ca1..616c77c8 100644 --- a/clean/l/runtime/tests/test_l_runtime_frame.py +++ b/clean/l/runtime/tests/test_l_runtime_frame.py @@ -127,8 +127,11 @@ def blob_digests(): # native D3D12, fusion lc (the default): the decoder block with l_coef_pack12 as its epilogue, in place of the pair FOLD = "l_swin_block_c32_dec_coef12_nc" FOLD_BASE = "l_swin_block_c32_dec_coef12" -# the default fusions without lc: the pair (the fold's fallback route) +# native D3D12, fusion ll (the default): the two input linears in one kernel (x1 in registers; q in l.narrow) +LIN = "l_linear_32_32_fused" +# the default fusions without lc and ll: the pairs (the fallback routes) NO_FOLD = {"WG_D3D12_FUSION": "pm,xe,mb,rv,xo,xd"} +NO_LIN = {"WG_D3D12_FUSION": "pm,xe,mb,rv,xo,xd,lc"} def latent_out(coef): @@ -152,14 +155,16 @@ def expected_args(t, f, digests): npix, tw, th = str((fh + 4) * (fw + 4)), str(fw // 2), str(fh // 2) eo = [str(v) for v in T.block_offset(T.TABLE, "first", info["parity"])] do = [str(v) for v in T.block_offset(T.TABLE, "second", info["parity"])] + q = n["l.narrow"] if any(ln.kernel == LIN for ln in f["launches"]) else n["l.wide"] # fused linears: q not under x2 return { # the input encoder's values: Model M's encoder inputs (the colour as the network reads it, here the game's, # pre-exposure 1), the state it reads, q into the wide map, the motion surfaces, the exposure pair "lev/input_encoder": [n["input.color"], n["input.depth"], n["input.motion"], n["post.reduced"], - n["state.latent_prev"], n["state.motion_history"], n["l.wide"], n["enc0.m_near"], + n["state.latent_prev"], n["state.motion_history"], q, n["enc0.m_near"], n["enc0.m_far"], n["enc0.m_center"], n["exposure.pair"], w["enc0.values"], None], "l_linear_32_32_relu": [n["l.wide"], n["l.narrow"], w["enc0.Win1"], npix], # q -> x1 "l_linear_32_32_f16": [n["l.narrow"], n["l.wide"], w["enc0.Win2"], npix], # x1 -> x2 + LIN: [n["l.narrow"], n["l.wide"], w["enc0.Win1"], w["enc0.Win2"], npix], # q -> x2 (x1 in registers) "l_swin_block_c32_enc": [n["l.wide"], "0", n["l.x3"], w["enc0_block"], str(fh), str(fw)] + eo, # x2 -> x3 "l_project_gather": [n["l.x3"], n["enc0.features"], w["enc0.project"], tw, th], # x3 -> enc1's input "l_dec_project": [n["dec1.out"], n["l.narrow"], w["dec0.project"], tw, th], # dec1's output -> y @@ -198,10 +203,14 @@ def check_roles(t, f, digests, l_post=True): continue bad += ["%s argument %d: %s, expected %s" % (k, i, a, e) for i, (a, e) in enumerate(zip(ln.args, want)) if e is not None and a != e] - fold = FOLD_BASE in seen - need = [k for k in L_KERNELS if not (fold and k in ("l_swin_block_c32_dec", "l_coef"))] + ([FOLD_BASE] if fold else []) - if fold and ({"l_swin_block_c32_dec", "l_coef"} & seen): + fold, lin = FOLD_BASE in seen, LIN in seen + pair_d, pair_l = ("l_swin_block_c32_dec", "l_coef"), ("l_linear_32_32_relu", "l_linear_32_32_f16") + need = [k for k in L_KERNELS if not (fold and k in pair_d) and not (lin and k in pair_l)] + \ + ([FOLD_BASE] if fold else []) + ([LIN] if lin else []) + if fold and (set(pair_d) & seen): bad.append("the folded decoder block and the pair in one frame") + if lin and (set(pair_l) & seen): + bad.append("the fused linears and the pair in one frame") missing = [k for k in need if k not in seen and (l_post or k != "l_post_recon")] if bad or missing: raise AssertionError("; ".join(bad + ["not launched: " + k for k in missing])) @@ -330,8 +339,10 @@ def check_frame(self, t, f, roles=True, l_post=True): npix, tw, th = (fh + 4) * (fw + 4), fw // 2, fh // 2 k = {base(ln.kernel): (i, ln) for i, ln in enumerate(ls)} _, lev = k["lev/input_encoder"] - _, relu = k["l_linear_32_32_relu"] - _, f16 = k["l_linear_32_32_f16"] + lin = k[LIN][1] if LIN in k else None + if not lin: + _, relu = k["l_linear_32_32_relu"] + _, f16 = k["l_linear_32_32_f16"] _, enc = k["l_swin_block_c32_enc"] gi, gather = k["l_project_gather"] di, dproj = k["l_dec_project"] @@ -342,10 +353,18 @@ def check_frame(self, t, f, roles=True, l_post=True): else: _, dec = k["l_swin_block_c32_dec"] _, coef = k["l_coef"] - q, x1, x2, x3, feat = lev.args[6], relu.args[1], f16.args[1], enc.args[2], gather.args[1] + q, x3, feat = lev.args[6], enc.args[2], gather.args[1] # the chain of maps (no launch reads what it writes) - self.assertEqual(relu.args[0], q) - self.assertEqual(f16.args[0], x1) + if lin: # one launch, x1 in registers: no map + x1, x2 = None, lin.args[1] + self.assertEqual(lin.args[0], q) + self.assertEqual(lin.args[4], str(npix)) + else: + x1, x2 = relu.args[1], f16.args[1] + self.assertEqual(relu.args[0], q) + self.assertEqual(f16.args[0], x1) + self.assertEqual(relu.args[3], str(npix)) + self.assertEqual(f16.args[3], str(npix)) self.assertEqual(enc.args[0], x2) self.assertEqual(enc.args[1], "0") # xadd: none self.assertEqual(gather.args[0], x3) @@ -360,11 +379,14 @@ def check_frame(self, t, f, roles=True, l_post=True): self.assertEqual(y2, "0") # the map is not written else: self.assertEqual(coef.args[0], y2) - for a, b in ((q, x1), (x1, x2), (x2, x3), (x3, feat), (dec1.args[2], y), (y, y2)): - self.assertNotEqual(a, b) + for a, b in ((q, x1), (x1, x2), (q, x2 if lin else None), (x2, x3), (x3, feat), (dec1.args[2], y), (y, y2)): + if a is not None and b is not None: + self.assertNotEqual(a, b) + if not lin: + self.assertEqual(q, x2) # the pair: x2 over q (q is dead once x1 is written) # every buffer holds its map (FORMATS.md "Maps") - maps = ((q, 32 * npix), (x1, 32 * npix), (x2, 64 * npix), (x3, 32 * fh * fw), (feat, 64 * tw * th), - (y, 32 * fh * fw)) + (() if fold else ((y2, 64 * fh * fw),)) + maps = ((q, 32 * npix), (x2, 64 * npix), (x3, 32 * fh * fw), (feat, 64 * tw * th), + (y, 32 * fh * fw)) + (() if fold else ((y2, 64 * fh * fw),)) + (() if lin else ((x1, 32 * npix),)) if fold: # pack12 image A (8 bytes a pixel), image C (4), the new latent coefs = ((coef.args[8], 8 * rw * rh), (coef.args[9], 4 * rw * rh), (coef.args[10], 4 * rw * rh)) else: @@ -373,8 +395,6 @@ def check_frame(self, t, f, roles=True, l_post=True): for buf, need in maps + coefs: self.assertGreaterEqual(t.extent(buf), need, buf) # extents, offsets, parameters - self.assertEqual(relu.args[3], str(npix)) - self.assertEqual(f16.args[3], str(npix)) self.assertEqual(gather.args[3:5], [str(tw), str(th)]) self.assertEqual(dproj.args[3:5], [str(tw), str(th)]) for blk, kind in ((enc, "first"), (dec, "second")): @@ -546,6 +566,12 @@ def check_d3d12_frame(self, t, f): self.assertEqual(names[names.index(FOLD) - 1], "l_dec_project") else: self.assertIn("l_coef_pack12", names) + if LIN in names: + self.assertNotIn("l_linear_32_32_relu", names) + self.assertNotIn("l_linear_32_32_f16", names) + self.assertTrue(names[names.index(LIN) - 1].startswith("lev/input_encoder")) + else: + self.assertIn("l_linear_32_32_relu", names) self.assertNotIn("l_coef", names) self.assertTrue(all(not n.startswith(("d1p12/", "d0p12/")) and n != "input_encoder" for n in names), names) dec1 = [ln for ln in f["launches"] if ln.kernel.startswith("swin_block_c64_add")] @@ -567,10 +593,14 @@ def test_frame(self): self.assertEqual(t.unknown(), []) self.assertEqual([f["info"]["reset"] for f in t.frames], [1, 0, 1, 0, 1, 1]) self.assertEqual(len({f["info"]["parity"] for f in t.frames}), 2) - for env, ports, fold in ((None, full_ports(), True), (NO_FOLD, full_ports(), False), - (None, [kc for kc in full_ports() if kc[0] != FOLD], False)): - with self.subTest(fusion=env, fold_in_manifest=any(kc[0] == FOLD for kc in ports)): - # the fold (fusion lc, the default) and its fallback, the pair: without lc, or without the kernel + for env, ports, fold, lin in ((None, full_ports(), True, True), (NO_FOLD, full_ports(), False, False), + (None, [kc for kc in full_ports() if kc[0] != FOLD], False, True), + (NO_LIN, full_ports(), True, False), + (None, [kc for kc in full_ports() if kc[0] != LIN], True, False)): + with self.subTest(fusion=env, fold_in_manifest=any(kc[0] == FOLD for kc in ports), + lin_in_manifest=any(kc[0] == LIN for kc in ports)): + # the fusions (lc, ll: the default) and their fallbacks, the pairs: without the fusion, or without the + # kernel t = FT.Trace(self.l_run(ports, frames=6, rw=96, rh=64, flags=(sizes, "--reset=4"), env=env)) self.assertEqual(t.create, "create ok model L") self.assertEqual(t.unknown(), []) @@ -580,6 +610,7 @@ def test_frame(self): for f in t.frames: self.assertIsNone(f["error"]) self.assertEqual(FOLD in [ln.kernel for ln in f["launches"]], fold) + self.assertEqual(LIN in [ln.kernel for ln in f["launches"]], lin) lev, coef = self.check_d3d12_frame(t, f) if prev: self.assertEqual(lev.args[4], latent_out(prev[1])) # the latent the last frame's store wrote diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index 52e09ad3..9499f5a2 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -125,8 +125,9 @@ std::string emu_lib_dir(const WgRtConfig& c) { // 10 faster; one 8-round run -11.5 us with every round faster). Re-measured in one process (runtime A/B, the same frames // alternating every 300 frames; inter-process noise was +-25 us): xd is +5.0 us at 1707x960 and +7.5 us at 1920x1080, // -2.25 us at 1280x720, so it now runs on frames of at most kXdMaxPixels render pixels only. -// lc: Model L only (its decoder block with l_coef_pack12 as the epilogue; see fz_lc). -const char* const kDefaultD3d12Fusion = "pm,xe,mb,rv,xo,xd,lc"; +// lc, ll: Model L only (its decoder block with l_coef_pack12 as the epilogue, its two input linears in one kernel; see +// fz_lc, fz_ll). +const char* const kDefaultD3d12Fusion = "pm,xe,mb,rv,xo,xd,lc,ll"; // xd only up to this many render pixels per frame (see fz_xd): it is faster at 1280x720 and slower at 1707x960 and above constexpr long long kXdMaxPixels = 1200000; @@ -321,16 +322,21 @@ struct Runtime { // block's FP16 map (y2, 64 bytes a pixel) is neither stored nor read back. Pack12 frames only; without lc, or // with a manifest that lacks the kernel, the pair runs (the fallback route) bool fz_lc = false, l_fold_built = false; + // ll Model L: its two input linears in one kernel (l_linear_32_32_fused, l_ops_linear.hlsl; byte for byte the pair + // l_linear_32_32_relu -> l_linear_32_32_f16): one launch less, and x1 (32 bytes a pixel of the halo map) stays + // in registers. The input encoder then writes q to l_narrow (x2 over q in l_wide would overwrite q that other + // workgroups have not read yet). Without ll, or with a manifest that lacks the kernel, the pair runs + bool fz_ll = false, l_lin_built = false; long long xd_max_pixels = [] { const char* e = std::getenv("WG_D3D12_XD_MAX_PIXELS"); return e && *e ? std::atoll(e) : kXdMaxPixels; }(); // A/B diagnostics (WG_AB_*): the two fusion sets and the block length (0: off) struct FzSet { - bool pm = false, xe = false, mb = false, mb0 = false, rv = false, xo = false, xd = false, lc = false; + bool pm = false, xe = false, mb = false, mb0 = false, rv = false, xo = false, xd = false, lc = false, ll = false; template void apply(R* r) const { r->fz_pm = pm, r->fz_xe = xe, r->fz_mb = mb, r->fz_mb0 = mb0, r->fz_rv = rv, r->fz_xo = xo, r->fz_xd = xd; - r->fz_lc = lc; + r->fz_lc = lc, r->fz_ll = ll; } }; FzSet fz_set[2]; @@ -360,8 +366,9 @@ struct Runtime { else if (f == "xo") z->xo = true; else if (f == "xd") z->xd = true; else if (f == "lc") z->lc = true; + else if (f == "ll") z->ll = true; else if (!f.empty()) - return fail("WG_D3D12_FUSION: unknown fusion \"" + f + "\" (pm, xe, mb, rv, xo, xd, lc or none)"); + return fail("WG_D3D12_FUSION: unknown fusion \"" + f + "\" (pm, xe, mb, rv, xo, xd, lc, ll or none)"); } return true; } @@ -531,6 +538,7 @@ struct Runtime { if (!missing.empty()) return fail("Model L on native D3D12: not yet ported to D3D12: " + missing); const std::vector have = manifest_kernels(); l_fold_built = std::find(have.begin(), have.end(), "l_swin_block_c32_dec_coef12_nc") != have.end(); + l_lin_built = std::find(have.begin(), have.end(), "l_linear_32_32_fused") != have.end(); } #endif if (c.max_rw <= 0 || c.max_rh <= 0 || c.ow <= 0 || c.oh <= 0) return fail("invalid render or output size"); @@ -1373,7 +1381,7 @@ struct Runtime { kernels_text += std::string(" dec_rdiv=") + (dec_div.find('r') == std::string::npos ? "1" : "0"); kernels_text += std::string(" dec_bdiv=") + (dec_div.find('b') == std::string::npos ? "1" : "0"); kernels_text += std::string(" dec1_dec0=") + (dec1_dec0 ? "fused" : "separate"); - if (fz_pm || fz_xe || fz_mb || fz_rv || fz_xo || fz_xd || fz_lc) kernels_text += " d3d12_fusion=" + fz_text; + if (fz_pm || fz_xe || fz_mb || fz_rv || fz_xo || fz_xd || fz_lc || fz_ll) kernels_text += " d3d12_fusion=" + fz_text; kernels_text += " swin_mid=" + (mid_variant.empty() ? std::string("none") : mid_variant); kernels_text += std::string(" post_pack=") + (p12 ? "p12" : "none"); kernels_text += std::string(" pre_exposure=") + (pi_original ? "original" : "scale"); @@ -1387,6 +1395,7 @@ struct Runtime { // Model L: its encoder and decoder (enc0= / dec0= above name Model M's, not loaded for Model L) if (model_l) kernels_text += " model=L l_kernels=l_enc_values.co,l_ops.co,l_swin.co,l_post.co l_post=l_post_recon"; if (l_fold_built) kernels_text += std::string(" l_dec_fold=") + (fz_lc ? "on" : "off"); // (native D3D12 only) + if (l_lin_built) kernels_text += std::string(" l_lin_fused=") + (fz_ll ? "on" : "off"); return kernels_text.c_str(); } @@ -1532,12 +1541,15 @@ struct Runtime { bool l_encoder(EncParams& p, DevPtr cnet, DevPtr din, DevPtr min_, int cur, int nxt, int fh, int fw, int f) { DevPtr wv = wdev["enc0.values"], w1 = wdev["enc0.Win1"], w2 = wdev["enc0.Win2"], wp = wdev["enc0.project"]; int npix = (fh + 4) * (fw + 4), tw = fw / 2, th = fh / 2; + const bool fused = fz_ll && l_lin_built; // (native D3D12 only: l_lin_built) + DevPtr& q = fused ? l_narrow : l_wide; // q's map (fused: not under x2) return launch(native_direct_io ? "lev/input_encoder_direct" : "lev/input_encoder", cdiv(((fh + 4) / 2) * ((fw + 4) / 2), 128), 256, - {&cnet, &din, &min_, &reduced, &latent[cur], &mnear[cur], &l_wide, &mnear[nxt], &mfar, &mcenter, + {&cnet, &din, &min_, &reduced, &latent[cur], &mnear[cur], &q, &mnear[nxt], &mfar, &mcenter, &expo, &wv, &p}) && - launch("l_linear_32_32_relu", cdiv(npix, 128), 256, {&l_wide, &l_narrow, &w1, &npix}) && - launch("l_linear_32_32_f16", cdiv(npix, 128), 256, {&l_narrow, &l_wide, &w2, &npix}) && + (fused ? launch("l_linear_32_32_fused", cdiv(npix, 128), 256, {&l_narrow, &l_wide, &w1, &w2, &npix}) + : launch("l_linear_32_32_relu", cdiv(npix, 128), 256, {&l_wide, &l_narrow, &w1, &npix}) && + launch("l_linear_32_32_f16", cdiv(npix, 128), 256, {&l_narrow, &l_wide, &w2, &npix})) && l_block("enc", l_wide, l_x3, wdev["enc0_block"], l_nc[0], fh, fw, false, f) && launch("l_project_gather", cdiv(tw * th, 128), 256, {&l_x3, &feat0, &wp, &tw, &th}); } diff --git a/clean/tests/runtime_frame_trace.cpp b/clean/tests/runtime_frame_trace.cpp index 08c31159..350abc86 100644 --- a/clean/tests/runtime_frame_trace.cpp +++ b/clean/tests/runtime_frame_trace.cpp @@ -78,6 +78,7 @@ const Signature kSignatures[] = { {"l_swin_block_c32*", "ppppiiii"}, // Model L's blocks (clean/l/FORMATS.md) {"l_linear_32_32_relu", "pppi"}, {"l_linear_32_32_f16", "pppi"}, + {"l_linear_32_32_fused", "ppppi"}, // the two in one: q, x2, Win1, Win2, npix (native D3D12, fusion ll) {"l_project_gather", "pppii"}, {"l_dec_project", "pppii"}, {"l_coef", "ppppppD"},