diff --git a/clean/dx12/compare.py b/clean/dx12/compare.py index 80b35af6..c71184c9 100644 --- a/clean/dx12/compare.py +++ b/clean/dx12/compare.py @@ -68,7 +68,7 @@ "expand_c96", "d1p12/swin_block_c64_add_dec0_nc", "w64/post_tf_f32_p12_diet_vsit"} # The native backend (--backend d3d12) runs the D3D12 fusions of WG_D3D12_FUSION (default: runtime.cpp's # kDefaultD3d12Fusion, mirrored here), which replace some launches (kernels/fused_swin.hlsl, fused_bneck.hlsl, fused_dec.hlsl). -DEFAULT_D3D12_FUSION = "pm,xe,mb,rv,xo,xd,lc" # lc: Model L only +DEFAULT_D3D12_FUSION = "pm,xe,mb,rv,xo,xd,lc,ll" # lc, ll: Model L only def d3d12_fusion(): diff --git a/clean/dx12/kernels/l_ops_linear.hlsl b/clean/dx12/kernels/l_ops_linear.hlsl index 2b60c3f2..fbbc71f5 100644 --- a/clean/dx12/kernels/l_ops_linear.hlsl +++ b/clean/dx12/kernels/l_ops_linear.hlsl @@ -11,16 +11,56 @@ // the store let the D3D12 compiler sink the matrix instructions into it, where they ran with part of the wave masked // off: wrong values in a partly filled wave (a 35-pixel map, pixels 32..34; found by l_d3d12_ops_check.py). // +// l_linear_32_32_fused (D3D12 only): the two in one kernel, x1 kept in registers. The relu linear's lane (t, h) stores +// x1's channels 16 h + 8 m .. + 7 (m = 0, 1) as the two FP8 fragments that the f16 linear's lane (t, h) loads as its +// K blocks kc = m: the fragments pass in registers, the same bytes, and the x1 map is neither written nor read. x is +// the q map, y the x2 map: two different buffers (the pair runs q -> x1 -> x2 with x2 over q's buffer; here x2 would +// overwrite q that other workgroups have not read yet). Arguments: x, y, Win1, Win2, npix. Its conversions are the +// driver's instructions (RN16 and the final FP16 by v_cvt_f16_f32, the FP8 pack by v_cvt_pk_fp8_f32 after the clamp: +// wg_platform.hlsli, the same bits as the software forms the pair keeps); the software forms were most of the pair's +// instructions. Measured at 1080p Quality: 0.095 ms against the pair's 0.217 ms (the fused kernel with the software +// forms: 0.236 ms). +// // @wg-kernel l_linear_32_32_relu co=l_ops.co block=256 wave=32 args=p,p,p,u defines=LIN_RELU=1 // @wg-kernel l_linear_32_32_f16 co=l_ops.co block=256 wave=32 args=p,p,p,u defines=LIN_RELU=0 +// @wg-kernel l_linear_32_32_fused co=- block=256 wave=32 args=p,p,p,p,u defines=LIN_RELU=0,LIN_FUSED=1 #include "wg_platform.hlsli" +#ifndef LIN_FUSED +#define LIN_FUSED 0 +#endif + WG_ARGS_BEGIN uint x, y, w; // pointers: x [npix][32] FP8, y [npix][32] FP8 or FP16, w the blob (enc0.Win1 / enc0.Win2) +#if LIN_FUSED + uint w2; // the f16 linear's blob (enc0.Win2; w = enc0.Win1) +#endif int npix; WG_ARGS_END -#if LIN_RELU +#if LIN_FUSED // (helpers of the fused kernel only: the pair keeps its own code and DXIL) +// the blob's bias tile vectors and weight fragments of lane (t, h) +void lin_blob(uint wb, uint lane, uint hh, out v8f bias[2], out uint2 fr[2][2]) +{ + [unroll] for (uint mt = 0; mt < 2u; ++mt) { + bias[mt] = wg_load8f(wb, (hh * 8u + mt * 16u) * 4u); + [unroll] for (uint kc2 = 0; kc2 < 2u; ++kc2) fr[mt][kc2] = wg_ld64(wb, 128u + (mt * 2u + kc2) * 256u + lane * 8u); + } +} + +// output tile m before the rounding: b + W x +v8f lin_acc(v8f bias, uint2 fr[2], uint2 xf[2]) +{ + v8f acc = bias; + acc = wg_wmma_fp8(fr[0], xf[0], acc); + return wg_wmma_fp8(fr[1], xf[1], acc); +} + +#endif + +#if LIN_FUSED +WG_KERNEL(l_linear_32_32_fused) +#elif LIN_RELU WG_KERNEL(l_linear_32_32_relu) #else WG_KERNEL(l_linear_32_32_f16) @@ -35,6 +75,30 @@ WG_KERNEL(l_linear_32_32_f16) [unroll] for (uint kc = 0; kc < 2u; ++kc) xf[kc] = wg_ld64(x, (uint)pk * 32u + hh * 16u + 8u * kc); v8f bias[2]; uint2 fr[2][2]; +#if LIN_FUSED + lin_blob(w, lane, hh, bias, fr); + v8f bias2[2]; + uint2 fr2[2][2]; + lin_blob(w2, lane, hh, bias2, fr2); + // the relu linear: x1's fragments (the bytes its kernel stores at pk * 32 + hh * 16 + 8 m). The conversions by the + // driver's instructions (wg_platform.hlsli: the same bits as the pair's software wg_rn16 / wg_pack_q8; no branch in + // this kernel): RN16 = v_cvt_f16_f32 then the exact widening, the pack v_cvt_pk_fp8_f32 after the clamp. + uint2 x1[2]; + [unroll] for (uint m1 = 0; m1 < 2u; ++m1) { + const uint4 h = wg_to_f16_ags(lin_acc(bias[m1], fr[m1], xf)); + v8f r; + [unroll] for (uint j = 0; j < 4u; ++j) { + r.v[2 * j] = wg_fmax(wg_h2f_lo(h[j]), 0.0f); + r.v[2 * j + 1] = wg_fmax(wg_h2f_hi(h[j]), 0.0f); + } + x1[m1] = wg_pack_q8_ags(r); + } + // the f16 linear on them (its kernel's loads of x1 at pk * 32 + hh * 16 + 8 kc): FP16(RN16(acc)) = FP16(acc), one + // conversion (rounding to FP16 is idempotent, NaN payloads included) + [unroll] for (uint m = 0; m < 2u; ++m) + wg_st128(y, ((uint)pk * 32u + hh * 16u + 8u * m) * 2u, wg_to_f16_ags(lin_acc(bias2[m], fr2[m], x1))); +#else + // (the pair's own code, unchanged) [unroll] for (uint mt = 0; mt < 2u; ++mt) { bias[mt] = wg_load8f(w, (hh * 8u + mt * 16u) * 4u); [unroll] for (uint kc2 = 0; kc2 < 2u; ++kc2) fr[mt][kc2] = wg_ld64(w, 128u + (mt * 2u + kc2) * 256u + lane * 8u); @@ -51,4 +115,5 @@ WG_KERNEL(l_linear_32_32_f16) wg_st128(y, ((uint)pk * 32u + hh * 16u + 8u * m) * 2u, wg_to_f16(acc)); #endif } +#endif } diff --git a/clean/dx12/tests/native_direct_check.py b/clean/dx12/tests/native_direct_check.py index 84fe0d88..0bed5f9f 100644 --- a/clean/dx12/tests/native_direct_check.py +++ b/clean/dx12/tests/native_direct_check.py @@ -30,8 +30,8 @@ POST='w64/post_tf_f32_p12_diet_vsit' -# The neural launches with the native D3D12 fusions of runtime.cpp kDefaultD3d12Fusion (pm,xe,mb,rv,xo,xd; its lc is -# Model L's): enc2's block +# The neural launches with the native D3D12 fusions of runtime.cpp kDefaultD3d12Fusion (pm,xe,mb,rv,xo,xd; its lc and ll +# are Model L's): enc2's block # with enc1's merge in reverse order (pm, rv), the bottleneck with enc2's merge and the overlap counters (mb, xo), dec2's # block with dec1's expand (xe) launched as xd on frames of at most kXdMaxPixels render pixels (WG_D3D12_XD_MAX_PIXELS) # else xo, and the fused dec1 + dec0 in its xd form on those frames. Set by main from --shape (COMMON above: the diff --git a/clean/l/FORMATS.md b/clean/l/FORMATS.md index fce40a7a..2ed72cb1 100644 --- a/clean/l/FORMATS.md +++ b/clean/l/FORMATS.md @@ -23,7 +23,7 @@ Every map is `[rows][cols][C]`, channel fastest, as Model M's feature maps (`cle | Map | Shape | Element | Written by | Read by | |---|---|---|---|---| | `q` | `[fh + 4][fw + 4][32]` halo | FP8: values 0..19, bytes 20..31 zero | `lev/input_encoder` | `l_linear_32_32_relu` | -| `x1` | `[fh + 4][fw + 4][32]` halo | FP8 | `l_linear_32_32_relu` | `l_linear_32_32_f16` | +| `x1` | `[fh + 4][fw + 4][32]` halo | FP8 (not stored by `l_linear_32_32_fused`) | `l_linear_32_32_relu` | `l_linear_32_32_f16` | | `x2` | `[fh + 4][fw + 4][32]` halo | **FP16** (64 bytes per pixel) | `l_linear_32_32_f16` | `l_swin_block_c32_enc` | | `x3` | `[fh][fw][32]` | FP8 | `l_swin_block_c32_enc` | `l_project_gather` | | `enc0.features` | `[fh/2][fw/2][64]` | FP8 | `l_project_gather` | enc1 (Model M's stage) | @@ -78,6 +78,7 @@ workgroup, in raster order of their map. | `lev/input_encoder` | exists (GPU-checked) | as Model M's `input_encoder`, with `features` replaced by `q` | ceil(((fh + 4) / 2) ((fw + 4) / 2) / 128) (2 x 2 pixel groups of the halo map) | the 20 values and their dithered FP8 for every halo-map pixel, into `q`. The motion surfaces as Model M, only for pixels inside R. w = `enc0.values` | | `l_linear_32_32_relu` | exists (GPU-checked) | `x, y, w, npix` | ceil(npix / 128), npix = (fh + 4)(fw + 4) | y = q8(max(0, RN16(b + W x))), FP8 -> FP8. w = `enc0.Win1` | | `l_linear_32_32_f16` | exists (GPU-checked) | `x, y, w, npix` | as above | y = RN16(b + W x), FP8 -> **FP16**. w = `enc0.Win2` | +| `l_linear_32_32_fused` | exists (GPU-checked), D3D12 only | `x, y, w1, w2, npix` | as above | the two above in one kernel, `x1` kept in registers: the same bytes in `y` (`x2`), `x1` not stored. x = `q`, y = `x2` in two different buffers. w1 = `enc0.Win1`, w2 = `enc0.Win2`. The native D3D12 frame runs it in place of the two (fusion `ll`, on by default); without `ll` or without the kernel in the build, the two run | | `l_swin_block_c32_enc[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | ceil(nwin / 8), nwin = ceil((W + ox) / 4) ceil((H + oy) / 4) | the encoder block. x = `x2` (FP16 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2)). out = `x3` (FP8, grid only). H = fh, W = fw. (ox, oy) = `window_offsets["first"]` for the frame parity. w = `enc0_block` | | `l_project_gather` | exists (GPU-checked) | `x, features, w, tw, th` | ceil(tw th / 128), tw = fw / 2, th = fh / 2 | token (r, c) gathers pixels (2r + p / 2, 2c + p % 2), p = 0..3, as channels 32 p + i. Running FP16 sum per group of 32 from the bias, clamp +-6.28125, FP8 -> `enc0.features`. w = `enc0.project` | | `l_dec_project` | exists (GPU-checked) | `z, y, w, tw, th` | ceil(tw th / 128) | per token of dec1's output z [th][tw][64]: 128 outputs (two RN16 groups of 32 inputs, then FP8). Output 32 s + k goes to pixel (2r + s / 2, 2c + s % 2), channel k, of `y`. w = `dec0.project` | @@ -159,4 +160,6 @@ the frame parity, as Model M's. - `clean/l/runner/l_d3d12_check.py` (with its D3D12 port): each D3D12 port against its HIP kernel, byte for byte. - `clean/l/runner/l_fold_check.py`: `l_swin_block_c32_dec_coef12_nc` against `l_swin_block_c32_dec_nc` -> `l_coef_pack12` on D3D12 and on HIP, byte for byte. +- `clean/l/runner/l_linfuse_check.py`: `l_linear_32_32_fused` against `l_linear_32_32_relu` -> `l_linear_32_32_f16` + on D3D12 and on HIP, byte for byte. - One check per new kernel follows with that kernel. diff --git a/clean/l/runner/l_linfuse_check.py b/clean/l/runner/l_linfuse_check.py new file mode 100644 index 00000000..97bc2132 --- /dev/null +++ b/clean/l/runner/l_linfuse_check.py @@ -0,0 +1,203 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): the fused input linears l_linear_32_32_fused (D3D12 only; clean/dx12/kernels/l_ops_linear.hlsl) +against the sequence it replaces, l_linear_32_32_relu -> l_linear_32_32_f16, byte for byte. + + python clean/l/runner/l_linfuse_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --prepare + python clean/l/runner/l_linfuse_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --run + [--seeds N] --json out.json + +--prepare (CPU) builds into : l_ops.co and swin.co (clean/l/runner/l_c32_check.py compile_kernels), the D3D12 +kernels (clean/dx12/build_dx12.py --only) and kernel_replay.exe. +--run (GPU, one GPU slot): every case is a pixel map (npix pixels, partly filled waves and workgroups among them), an +FP8 input map q and the two blobs (enc0.Win1, enc0.Win2), at three weight scales (results past 448 and past FP16's +range, results in FP16's subnormal range) and with special FP8 codes in q (NaN, +-448, +-0, subnormals): the fused +kernel's conversions are the driver's, the pair's software. On the same input bytes: +- the sequence on D3D12 (kernel_replay, two passes: the relu linear writes x1, then the f16 linear reads that file); +- the sequence on HIP (in this process); +- the fused kernel on D3D12. +The rules are clean/l/runner/l_fold_check.py's: raw bytes; the output (x2, 64 bytes a pixel) has 64 bytes of slack and +starts filled with its run's own sentinel (HIP 0xA5, the D3D12 sequence 0x5A, the fused kernel 0x3C): every pixel byte +identical across the three, every slack byte its own run's sentinel; a negative control (the fused kernel with half its +workgroups) must be caught; Model M's swin_block_c96_nc runs through the same replay as a control. +Output: counts only, no timing. +""" +import argparse +import ctypes +import json +import os +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.runner import l_c32_check as C32 # noqa: E402 +from clean.l.runner import l_d3d12_check as LD # noqa: E402 +from clean.l.runner import l_d3d12_ops_check as OPS # noqa: E402 +from clean.reference.fp8 import fp8_encode, q8 # noqa: E402 +from clean.l.runner.l_fold_check import (CONTROL, D3D12_SENTINEL, FOLD_SENTINEL, HIP_SENTINEL, SLACK, compare, # noqa: E402 + nwin_of, replay, write) + +RELU, F16, FUSED = "l_linear_32_32_relu", "l_linear_32_32_f16", "l_linear_32_32_fused" +# (rows, cols) of the maps: the ops check's (a 35-pixel map among them: a partly filled wave), the halo maps of the +# runtime's smallest frames, 1 x 1, and one more workgroup than a whole number (129 pixels) +MAPS = OPS.MAPS + [(36, 36), (1, 1), (3, 43), (68, 132)] + + +def _blob(rng, scale, bias_scale): + """enc0.Win1 / Win2 layout (bias tile vector of 32 floats, 4 fragments of 256 FP8 bytes) at a weight scale.""" + bias = rng.normal(0, bias_scale, 32).astype(np.float32).tobytes() + return bias + fp8_encode(q8(rng.normal(0, scale, 4 * 256))).tobytes() + + +# FP8 codes the network does not produce but the conversions must treat as the pair does: both NaN codes, +-448, +-0, +# subnormals +SPECIAL = np.array([0x7f, 0xff, 0x7e, 0xfe, 0x00, 0x80, 0x01, 0x81, 0x07, 0x87], np.uint8) +# weight / bias scales: the ops check's; large (sums past 448 and past FP16's 65504: infinities); tiny (FP16 subnormal +# and zero results) +SCALES = [(0.25, 0.5), (24.0, 400.0), (2e-3, 1e-4)] + + +def cases(rng): + """(npix, q bytes, Win1 blob, Win2 blob) of every case: each map at each weight scale, every other case with + special FP8 codes in a tenth of q's bytes.""" + for ci, (rows, cols) in enumerate(MAPS): + n = rows * cols + for si, (s, b) in enumerate(SCALES): + q = np.frombuffer(OPS._fp8_map(rng, n * 32), np.uint8).copy() + if (ci + si) % 2: + pick = rng.random(q.size) < 0.1 + q[pick] = rng.choice(SPECIAL, int(pick.sum())) + yield n, q.tobytes(), _blob(rng, s, b), _blob(rng, s, b) + + +def hip_sequence(g, ops, c): + """relu, then f16, on HIP: x2's raw bytes (the output starts at HIP_SENTINEL).""" + n, q, w1, w2 = c + h = g.hip + owned = [] + + def buf(data): + p = h.malloc(max(len(data), 4)) + h.upload(p, data) + owned.append(p) + return p + + try: + pq, pw1, pw2 = buf(q), buf(w1), buf(w2) + px1, py = h.malloc(n * 32), h.malloc(n * 64 + SLACK) + owned += [px1, py] + h.memset(px1, HIP_SENTINEL, n * 32) + h.memset(py, HIP_SENTINEL, n * 64 + SLACK) + grid = -(-n // 128) + for k, a, b, wb in ((RELU, pq, px1, pw1), (F16, px1, py, pw2)): + h.launch(h.get_function(ops, k), grid, 256, + [ctypes.c_void_p(a), ctypes.c_void_p(b), ctypes.c_void_p(wb), ctypes.c_int(n)]) + h.sync() + return bytes(h.download(py, n * 64 + SLACK)) + finally: + for p in owned: + h.free(p) + + +def fused_lines(name, n, prefix="", grid=None): + return [f"case {prefix}{name}_fused", f"kernel {FUSED} {grid or -(-n // 128)} 256", f"arg buf {name}.q", + f"arg out {n * 64 + SLACK} {prefix}{name}.fy {FOLD_SENTINEL}", f"arg buf {name}.w1", f"arg buf {name}.w2", + f"arg u32 {n}", ""] + + +def run(build, rocm, seeds): + g = C32.Gpu(build, rocm) + with open(os.path.join(build, "l_ops.co"), "rb") as f: + ops = g.hip.load_module(f.read()) + cdir = os.path.join(build, "replay-linfuse") + os.makedirs(cdir, exist_ok=True) + pass1, pass2, all_cases, neg = [], [], {}, None + for si, seed in enumerate(seeds): + rng = np.random.default_rng(seed) + for ci, c in enumerate(cases(rng)): + name = f"s{si}_{ci}" + n, q, w1, w2 = c + all_cases[name] = (n, hip_sequence(g, ops, c)) + write(os.path.join(cdir, name + ".q"), q) + write(os.path.join(cdir, name + ".w1"), w1) + write(os.path.join(cdir, name + ".w2"), w2) + grid = -(-n // 128) + # pass 1: the D3D12 relu linear (x1) and the fused kernel + pass1 += [f"case {name}_relu", "module l_ops.co", f"kernel {RELU} {grid} 256", f"arg buf {name}.q", + f"arg out {n * 32} {name}.x1 {D3D12_SENTINEL}", f"arg buf {name}.w1", f"arg u32 {n}", ""] + pass1 += fused_lines(name, n) + if neg is None and grid >= 4: + neg = name + pass1 += fused_lines(name, n, prefix="negative_", grid=grid // 2) + # pass 2: the D3D12 f16 linear on that x1 + pass2 += [f"case {name}_f16", "module l_ops.co", f"kernel {F16} {grid} 256", f"arg buf {name}.x1", + f"arg out {n * 64 + SLACK} {name}.sy {D3D12_SENTINEL}", f"arg buf {name}.w2", f"arg u32 {n}", ""] + rng = np.random.default_rng(seeds[0]) + ctl = [] + for ci, (H, W, off, xb, nout, blob) in enumerate(LD.block_cases(CONTROL, rng)): + name = f"control_{ci}" + ctl.append((name, LD.hip_block_raw(g, "swin.co", CONTROL, xb, blob, H, W, off, nout))) + write(os.path.join(cdir, name + ".x"), xb) + write(os.path.join(cdir, name + ".w"), blob) + pass1 += [f"case {name}", "module swin.co", f"kernel {CONTROL} {-(-nwin_of(H, W, off) // 8)} 256", + f"arg buf {name}.x", "arg null", f"arg out {nout} {name}.out {D3D12_SENTINEL}", f"arg buf {name}.w", + f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {off[0]}", f"arg u32 {off[1]}", ""] + e1, t1 = replay(build, cdir, pass1, "pass1") + e2, t2 = replay(build, cdir, pass2, "pass2") if e1 == 0 else (-1, "pass 1 failed") + res = {"replay_exit": [e1, e2], "replay_tail": [t1, t2], "cases": []} + + def read(n): + p = os.path.join(cdir, n) + return open(p, "rb").read() if os.path.exists(p) else b"" + + for name, (n, hip) in all_cases.items(): + fused, seq = read(f"{name}.fy"), read(f"{name}.sy") + r = {"case": name, "npix": n, + "fused_vs_d3d12_seq": compare(fused, seq, n * 64, FOLD_SENTINEL, D3D12_SENTINEL), + "fused_vs_hip_seq": compare(fused, hip, n * 64, FOLD_SENTINEL, HIP_SENTINEL), + "d3d12_seq_vs_hip_seq": compare(seq, hip, n * 64, D3D12_SENTINEL, HIP_SENTINEL)} + r["identical"] = all(r[k]["identical"] for k in ("fused_vs_d3d12_seq", "fused_vs_hip_seq", "d3d12_seq_vs_hip_seq")) + res["cases"].append(r) + caught = None + if neg: + n = all_cases[neg][0] + o = compare(read(f"negative_{neg}.fy"), read(f"{neg}.sy"), n * 64, FOLD_SENTINEL, D3D12_SENTINEL) + caught = o["differ"] >= 0 and not o["identical"] + res["negative_control"] = {"case": neg, "half_grid": True, "caught": caught, "differ": max(o["differ"], 0)} + ctl_ok = [read(f"{name}.out") == want for name, want in ctl] + res["control"] = {"kernel": CONTROL, "cases": len(ctl_ok), "identical": sum(ctl_ok)} + res["summary"] = {"cases": len(res["cases"]), "identical": sum(c["identical"] for c in res["cases"]), + "pixel_bytes": sum(c["npix"] * 64 for c in res["cases"])} + res["pass"] = (e1 == 0 and e2 == 0 and caught is True and res["summary"]["cases"] > 0 and + res["summary"]["identical"] == res["summary"]["cases"] and len(ctl_ok) > 0 and all(ctl_ok)) + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--prepare", action="store_true") + ap.add_argument("--run", action="store_true") + ap.add_argument("--seeds", type=int, default=2) + ap.add_argument("--json") + a = ap.parse_args(argv) + os.makedirs(a.build, exist_ok=True) + if a.prepare: + C32.compile_kernels(a.build, a.rocm, with_swin=True) + print("built", LD.prepare_replay(a.build, a.rocm, [RELU, F16, FUSED, CONTROL])) + if a.run: + res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds))) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + print({k: v for k, v in res.items() if k not in ("cases", "replay_tail")}) + return 0 if res["pass"] else 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runtime/tests/test_l_runtime_frame.py b/clean/l/runtime/tests/test_l_runtime_frame.py index 11213ca1..616c77c8 100644 --- a/clean/l/runtime/tests/test_l_runtime_frame.py +++ b/clean/l/runtime/tests/test_l_runtime_frame.py @@ -127,8 +127,11 @@ def blob_digests(): # native D3D12, fusion lc (the default): the decoder block with l_coef_pack12 as its epilogue, in place of the pair FOLD = "l_swin_block_c32_dec_coef12_nc" FOLD_BASE = "l_swin_block_c32_dec_coef12" -# the default fusions without lc: the pair (the fold's fallback route) +# native D3D12, fusion ll (the default): the two input linears in one kernel (x1 in registers; q in l.narrow) +LIN = "l_linear_32_32_fused" +# the default fusions without lc and ll: the pairs (the fallback routes) NO_FOLD = {"WG_D3D12_FUSION": "pm,xe,mb,rv,xo,xd"} +NO_LIN = {"WG_D3D12_FUSION": "pm,xe,mb,rv,xo,xd,lc"} def latent_out(coef): @@ -152,14 +155,16 @@ def expected_args(t, f, digests): npix, tw, th = str((fh + 4) * (fw + 4)), str(fw // 2), str(fh // 2) eo = [str(v) for v in T.block_offset(T.TABLE, "first", info["parity"])] do = [str(v) for v in T.block_offset(T.TABLE, "second", info["parity"])] + q = n["l.narrow"] if any(ln.kernel == LIN for ln in f["launches"]) else n["l.wide"] # fused linears: q not under x2 return { # the input encoder's values: Model M's encoder inputs (the colour as the network reads it, here the game's, # pre-exposure 1), the state it reads, q into the wide map, the motion surfaces, the exposure pair "lev/input_encoder": [n["input.color"], n["input.depth"], n["input.motion"], n["post.reduced"], - n["state.latent_prev"], n["state.motion_history"], n["l.wide"], n["enc0.m_near"], + n["state.latent_prev"], n["state.motion_history"], q, n["enc0.m_near"], n["enc0.m_far"], n["enc0.m_center"], n["exposure.pair"], w["enc0.values"], None], "l_linear_32_32_relu": [n["l.wide"], n["l.narrow"], w["enc0.Win1"], npix], # q -> x1 "l_linear_32_32_f16": [n["l.narrow"], n["l.wide"], w["enc0.Win2"], npix], # x1 -> x2 + LIN: [n["l.narrow"], n["l.wide"], w["enc0.Win1"], w["enc0.Win2"], npix], # q -> x2 (x1 in registers) "l_swin_block_c32_enc": [n["l.wide"], "0", n["l.x3"], w["enc0_block"], str(fh), str(fw)] + eo, # x2 -> x3 "l_project_gather": [n["l.x3"], n["enc0.features"], w["enc0.project"], tw, th], # x3 -> enc1's input "l_dec_project": [n["dec1.out"], n["l.narrow"], w["dec0.project"], tw, th], # dec1's output -> y @@ -198,10 +203,14 @@ def check_roles(t, f, digests, l_post=True): continue bad += ["%s argument %d: %s, expected %s" % (k, i, a, e) for i, (a, e) in enumerate(zip(ln.args, want)) if e is not None and a != e] - fold = FOLD_BASE in seen - need = [k for k in L_KERNELS if not (fold and k in ("l_swin_block_c32_dec", "l_coef"))] + ([FOLD_BASE] if fold else []) - if fold and ({"l_swin_block_c32_dec", "l_coef"} & seen): + fold, lin = FOLD_BASE in seen, LIN in seen + pair_d, pair_l = ("l_swin_block_c32_dec", "l_coef"), ("l_linear_32_32_relu", "l_linear_32_32_f16") + need = [k for k in L_KERNELS if not (fold and k in pair_d) and not (lin and k in pair_l)] + \ + ([FOLD_BASE] if fold else []) + ([LIN] if lin else []) + if fold and (set(pair_d) & seen): bad.append("the folded decoder block and the pair in one frame") + if lin and (set(pair_l) & seen): + bad.append("the fused linears and the pair in one frame") missing = [k for k in need if k not in seen and (l_post or k != "l_post_recon")] if bad or missing: raise AssertionError("; ".join(bad + ["not launched: " + k for k in missing])) @@ -330,8 +339,10 @@ def check_frame(self, t, f, roles=True, l_post=True): npix, tw, th = (fh + 4) * (fw + 4), fw // 2, fh // 2 k = {base(ln.kernel): (i, ln) for i, ln in enumerate(ls)} _, lev = k["lev/input_encoder"] - _, relu = k["l_linear_32_32_relu"] - _, f16 = k["l_linear_32_32_f16"] + lin = k[LIN][1] if LIN in k else None + if not lin: + _, relu = k["l_linear_32_32_relu"] + _, f16 = k["l_linear_32_32_f16"] _, enc = k["l_swin_block_c32_enc"] gi, gather = k["l_project_gather"] di, dproj = k["l_dec_project"] @@ -342,10 +353,18 @@ def check_frame(self, t, f, roles=True, l_post=True): else: _, dec = k["l_swin_block_c32_dec"] _, coef = k["l_coef"] - q, x1, x2, x3, feat = lev.args[6], relu.args[1], f16.args[1], enc.args[2], gather.args[1] + q, x3, feat = lev.args[6], enc.args[2], gather.args[1] # the chain of maps (no launch reads what it writes) - self.assertEqual(relu.args[0], q) - self.assertEqual(f16.args[0], x1) + if lin: # one launch, x1 in registers: no map + x1, x2 = None, lin.args[1] + self.assertEqual(lin.args[0], q) + self.assertEqual(lin.args[4], str(npix)) + else: + x1, x2 = relu.args[1], f16.args[1] + self.assertEqual(relu.args[0], q) + self.assertEqual(f16.args[0], x1) + self.assertEqual(relu.args[3], str(npix)) + self.assertEqual(f16.args[3], str(npix)) self.assertEqual(enc.args[0], x2) self.assertEqual(enc.args[1], "0") # xadd: none self.assertEqual(gather.args[0], x3) @@ -360,11 +379,14 @@ def check_frame(self, t, f, roles=True, l_post=True): self.assertEqual(y2, "0") # the map is not written else: self.assertEqual(coef.args[0], y2) - for a, b in ((q, x1), (x1, x2), (x2, x3), (x3, feat), (dec1.args[2], y), (y, y2)): - self.assertNotEqual(a, b) + for a, b in ((q, x1), (x1, x2), (q, x2 if lin else None), (x2, x3), (x3, feat), (dec1.args[2], y), (y, y2)): + if a is not None and b is not None: + self.assertNotEqual(a, b) + if not lin: + self.assertEqual(q, x2) # the pair: x2 over q (q is dead once x1 is written) # every buffer holds its map (FORMATS.md "Maps") - maps = ((q, 32 * npix), (x1, 32 * npix), (x2, 64 * npix), (x3, 32 * fh * fw), (feat, 64 * tw * th), - (y, 32 * fh * fw)) + (() if fold else ((y2, 64 * fh * fw),)) + maps = ((q, 32 * npix), (x2, 64 * npix), (x3, 32 * fh * fw), (feat, 64 * tw * th), + (y, 32 * fh * fw)) + (() if fold else ((y2, 64 * fh * fw),)) + (() if lin else ((x1, 32 * npix),)) if fold: # pack12 image A (8 bytes a pixel), image C (4), the new latent coefs = ((coef.args[8], 8 * rw * rh), (coef.args[9], 4 * rw * rh), (coef.args[10], 4 * rw * rh)) else: @@ -373,8 +395,6 @@ def check_frame(self, t, f, roles=True, l_post=True): for buf, need in maps + coefs: self.assertGreaterEqual(t.extent(buf), need, buf) # extents, offsets, parameters - self.assertEqual(relu.args[3], str(npix)) - self.assertEqual(f16.args[3], str(npix)) self.assertEqual(gather.args[3:5], [str(tw), str(th)]) self.assertEqual(dproj.args[3:5], [str(tw), str(th)]) for blk, kind in ((enc, "first"), (dec, "second")): @@ -546,6 +566,12 @@ def check_d3d12_frame(self, t, f): self.assertEqual(names[names.index(FOLD) - 1], "l_dec_project") else: self.assertIn("l_coef_pack12", names) + if LIN in names: + self.assertNotIn("l_linear_32_32_relu", names) + self.assertNotIn("l_linear_32_32_f16", names) + self.assertTrue(names[names.index(LIN) - 1].startswith("lev/input_encoder")) + else: + self.assertIn("l_linear_32_32_relu", names) self.assertNotIn("l_coef", names) self.assertTrue(all(not n.startswith(("d1p12/", "d0p12/")) and n != "input_encoder" for n in names), names) dec1 = [ln for ln in f["launches"] if ln.kernel.startswith("swin_block_c64_add")] @@ -567,10 +593,14 @@ def test_frame(self): self.assertEqual(t.unknown(), []) self.assertEqual([f["info"]["reset"] for f in t.frames], [1, 0, 1, 0, 1, 1]) self.assertEqual(len({f["info"]["parity"] for f in t.frames}), 2) - for env, ports, fold in ((None, full_ports(), True), (NO_FOLD, full_ports(), False), - (None, [kc for kc in full_ports() if kc[0] != FOLD], False)): - with self.subTest(fusion=env, fold_in_manifest=any(kc[0] == FOLD for kc in ports)): - # the fold (fusion lc, the default) and its fallback, the pair: without lc, or without the kernel + for env, ports, fold, lin in ((None, full_ports(), True, True), (NO_FOLD, full_ports(), False, False), + (None, [kc for kc in full_ports() if kc[0] != FOLD], False, True), + (NO_LIN, full_ports(), True, False), + (None, [kc for kc in full_ports() if kc[0] != LIN], True, False)): + with self.subTest(fusion=env, fold_in_manifest=any(kc[0] == FOLD for kc in ports), + lin_in_manifest=any(kc[0] == LIN for kc in ports)): + # the fusions (lc, ll: the default) and their fallbacks, the pairs: without the fusion, or without the + # kernel t = FT.Trace(self.l_run(ports, frames=6, rw=96, rh=64, flags=(sizes, "--reset=4"), env=env)) self.assertEqual(t.create, "create ok model L") self.assertEqual(t.unknown(), []) @@ -580,6 +610,7 @@ def test_frame(self): for f in t.frames: self.assertIsNone(f["error"]) self.assertEqual(FOLD in [ln.kernel for ln in f["launches"]], fold) + self.assertEqual(LIN in [ln.kernel for ln in f["launches"]], lin) lev, coef = self.check_d3d12_frame(t, f) if prev: self.assertEqual(lev.args[4], latent_out(prev[1])) # the latent the last frame's store wrote diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index 52e09ad3..9499f5a2 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -125,8 +125,9 @@ std::string emu_lib_dir(const WgRtConfig& c) { // 10 faster; one 8-round run -11.5 us with every round faster). Re-measured in one process (runtime A/B, the same frames // alternating every 300 frames; inter-process noise was +-25 us): xd is +5.0 us at 1707x960 and +7.5 us at 1920x1080, // -2.25 us at 1280x720, so it now runs on frames of at most kXdMaxPixels render pixels only. -// lc: Model L only (its decoder block with l_coef_pack12 as the epilogue; see fz_lc). -const char* const kDefaultD3d12Fusion = "pm,xe,mb,rv,xo,xd,lc"; +// lc, ll: Model L only (its decoder block with l_coef_pack12 as the epilogue, its two input linears in one kernel; see +// fz_lc, fz_ll). +const char* const kDefaultD3d12Fusion = "pm,xe,mb,rv,xo,xd,lc,ll"; // xd only up to this many render pixels per frame (see fz_xd): it is faster at 1280x720 and slower at 1707x960 and above constexpr long long kXdMaxPixels = 1200000; @@ -321,16 +322,21 @@ struct Runtime { // block's FP16 map (y2, 64 bytes a pixel) is neither stored nor read back. Pack12 frames only; without lc, or // with a manifest that lacks the kernel, the pair runs (the fallback route) bool fz_lc = false, l_fold_built = false; + // ll Model L: its two input linears in one kernel (l_linear_32_32_fused, l_ops_linear.hlsl; byte for byte the pair + // l_linear_32_32_relu -> l_linear_32_32_f16): one launch less, and x1 (32 bytes a pixel of the halo map) stays + // in registers. The input encoder then writes q to l_narrow (x2 over q in l_wide would overwrite q that other + // workgroups have not read yet). Without ll, or with a manifest that lacks the kernel, the pair runs + bool fz_ll = false, l_lin_built = false; long long xd_max_pixels = [] { const char* e = std::getenv("WG_D3D12_XD_MAX_PIXELS"); return e && *e ? std::atoll(e) : kXdMaxPixels; }(); // A/B diagnostics (WG_AB_*): the two fusion sets and the block length (0: off) struct FzSet { - bool pm = false, xe = false, mb = false, mb0 = false, rv = false, xo = false, xd = false, lc = false; + bool pm = false, xe = false, mb = false, mb0 = false, rv = false, xo = false, xd = false, lc = false, ll = false; template void apply(R* r) const { r->fz_pm = pm, r->fz_xe = xe, r->fz_mb = mb, r->fz_mb0 = mb0, r->fz_rv = rv, r->fz_xo = xo, r->fz_xd = xd; - r->fz_lc = lc; + r->fz_lc = lc, r->fz_ll = ll; } }; FzSet fz_set[2]; @@ -360,8 +366,9 @@ struct Runtime { else if (f == "xo") z->xo = true; else if (f == "xd") z->xd = true; else if (f == "lc") z->lc = true; + else if (f == "ll") z->ll = true; else if (!f.empty()) - return fail("WG_D3D12_FUSION: unknown fusion \"" + f + "\" (pm, xe, mb, rv, xo, xd, lc or none)"); + return fail("WG_D3D12_FUSION: unknown fusion \"" + f + "\" (pm, xe, mb, rv, xo, xd, lc, ll or none)"); } return true; } @@ -531,6 +538,7 @@ struct Runtime { if (!missing.empty()) return fail("Model L on native D3D12: not yet ported to D3D12: " + missing); const std::vector have = manifest_kernels(); l_fold_built = std::find(have.begin(), have.end(), "l_swin_block_c32_dec_coef12_nc") != have.end(); + l_lin_built = std::find(have.begin(), have.end(), "l_linear_32_32_fused") != have.end(); } #endif if (c.max_rw <= 0 || c.max_rh <= 0 || c.ow <= 0 || c.oh <= 0) return fail("invalid render or output size"); @@ -1373,7 +1381,7 @@ struct Runtime { kernels_text += std::string(" dec_rdiv=") + (dec_div.find('r') == std::string::npos ? "1" : "0"); kernels_text += std::string(" dec_bdiv=") + (dec_div.find('b') == std::string::npos ? "1" : "0"); kernels_text += std::string(" dec1_dec0=") + (dec1_dec0 ? "fused" : "separate"); - if (fz_pm || fz_xe || fz_mb || fz_rv || fz_xo || fz_xd || fz_lc) kernels_text += " d3d12_fusion=" + fz_text; + if (fz_pm || fz_xe || fz_mb || fz_rv || fz_xo || fz_xd || fz_lc || fz_ll) kernels_text += " d3d12_fusion=" + fz_text; kernels_text += " swin_mid=" + (mid_variant.empty() ? std::string("none") : mid_variant); kernels_text += std::string(" post_pack=") + (p12 ? "p12" : "none"); kernels_text += std::string(" pre_exposure=") + (pi_original ? "original" : "scale"); @@ -1387,6 +1395,7 @@ struct Runtime { // Model L: its encoder and decoder (enc0= / dec0= above name Model M's, not loaded for Model L) if (model_l) kernels_text += " model=L l_kernels=l_enc_values.co,l_ops.co,l_swin.co,l_post.co l_post=l_post_recon"; if (l_fold_built) kernels_text += std::string(" l_dec_fold=") + (fz_lc ? "on" : "off"); // (native D3D12 only) + if (l_lin_built) kernels_text += std::string(" l_lin_fused=") + (fz_ll ? "on" : "off"); return kernels_text.c_str(); } @@ -1532,12 +1541,15 @@ struct Runtime { bool l_encoder(EncParams& p, DevPtr cnet, DevPtr din, DevPtr min_, int cur, int nxt, int fh, int fw, int f) { DevPtr wv = wdev["enc0.values"], w1 = wdev["enc0.Win1"], w2 = wdev["enc0.Win2"], wp = wdev["enc0.project"]; int npix = (fh + 4) * (fw + 4), tw = fw / 2, th = fh / 2; + const bool fused = fz_ll && l_lin_built; // (native D3D12 only: l_lin_built) + DevPtr& q = fused ? l_narrow : l_wide; // q's map (fused: not under x2) return launch(native_direct_io ? "lev/input_encoder_direct" : "lev/input_encoder", cdiv(((fh + 4) / 2) * ((fw + 4) / 2), 128), 256, - {&cnet, &din, &min_, &reduced, &latent[cur], &mnear[cur], &l_wide, &mnear[nxt], &mfar, &mcenter, + {&cnet, &din, &min_, &reduced, &latent[cur], &mnear[cur], &q, &mnear[nxt], &mfar, &mcenter, &expo, &wv, &p}) && - launch("l_linear_32_32_relu", cdiv(npix, 128), 256, {&l_wide, &l_narrow, &w1, &npix}) && - launch("l_linear_32_32_f16", cdiv(npix, 128), 256, {&l_narrow, &l_wide, &w2, &npix}) && + (fused ? launch("l_linear_32_32_fused", cdiv(npix, 128), 256, {&l_narrow, &l_wide, &w1, &w2, &npix}) + : launch("l_linear_32_32_relu", cdiv(npix, 128), 256, {&l_wide, &l_narrow, &w1, &npix}) && + launch("l_linear_32_32_f16", cdiv(npix, 128), 256, {&l_narrow, &l_wide, &w2, &npix})) && l_block("enc", l_wide, l_x3, wdev["enc0_block"], l_nc[0], fh, fw, false, f) && launch("l_project_gather", cdiv(tw * th, 128), 256, {&l_x3, &feat0, &wp, &tw, &th}); } diff --git a/clean/tests/runtime_frame_trace.cpp b/clean/tests/runtime_frame_trace.cpp index 08c31159..350abc86 100644 --- a/clean/tests/runtime_frame_trace.cpp +++ b/clean/tests/runtime_frame_trace.cpp @@ -78,6 +78,7 @@ const Signature kSignatures[] = { {"l_swin_block_c32*", "ppppiiii"}, // Model L's blocks (clean/l/FORMATS.md) {"l_linear_32_32_relu", "pppi"}, {"l_linear_32_32_f16", "pppi"}, + {"l_linear_32_32_fused", "ppppi"}, // the two in one: q, x2, Win1, Win2, npix (native D3D12, fusion ll) {"l_project_gather", "pppii"}, {"l_dec_project", "pppii"}, {"l_coef", "ppppppD"},