diff --git a/clean/dx12/compare.py b/clean/dx12/compare.py
index 80b35af6..c71184c9 100644
--- a/clean/dx12/compare.py
+++ b/clean/dx12/compare.py
@@ -68,7 +68,7 @@
"expand_c96", "d1p12/swin_block_c64_add_dec0_nc", "w64/post_tf_f32_p12_diet_vsit"}
# The native backend (--backend d3d12) runs the D3D12 fusions of WG_D3D12_FUSION (default: runtime.cpp's
# kDefaultD3d12Fusion, mirrored here), which replace some launches (kernels/fused_swin.hlsl, fused_bneck.hlsl, fused_dec.hlsl).
-DEFAULT_D3D12_FUSION = "pm,xe,mb,rv,xo,xd,lc" # lc: Model L only
+DEFAULT_D3D12_FUSION = "pm,xe,mb,rv,xo,xd,lc,ll" # lc, ll: Model L only
def d3d12_fusion():
diff --git a/clean/dx12/kernels/l_ops_linear.hlsl b/clean/dx12/kernels/l_ops_linear.hlsl
index 2b60c3f2..fbbc71f5 100644
--- a/clean/dx12/kernels/l_ops_linear.hlsl
+++ b/clean/dx12/kernels/l_ops_linear.hlsl
@@ -11,16 +11,56 @@
// the store let the D3D12 compiler sink the matrix instructions into it, where they ran with part of the wave masked
// off: wrong values in a partly filled wave (a 35-pixel map, pixels 32..34; found by l_d3d12_ops_check.py).
//
+// l_linear_32_32_fused (D3D12 only): the two in one kernel, x1 kept in registers. The relu linear's lane (t, h) stores
+// x1's channels 16 h + 8 m .. + 7 (m = 0, 1) as the two FP8 fragments that the f16 linear's lane (t, h) loads as its
+// K blocks kc = m: the fragments pass in registers, the same bytes, and the x1 map is neither written nor read. x is
+// the q map, y the x2 map: two different buffers (the pair runs q -> x1 -> x2 with x2 over q's buffer; here x2 would
+// overwrite q that other workgroups have not read yet). Arguments: x, y, Win1, Win2, npix. Its conversions are the
+// driver's instructions (RN16 and the final FP16 by v_cvt_f16_f32, the FP8 pack by v_cvt_pk_fp8_f32 after the clamp:
+// wg_platform.hlsli, the same bits as the software forms the pair keeps); the software forms were most of the pair's
+// instructions. Measured at 1080p Quality: 0.095 ms against the pair's 0.217 ms (the fused kernel with the software
+// forms: 0.236 ms).
+//
// @wg-kernel l_linear_32_32_relu co=l_ops.co block=256 wave=32 args=p,p,p,u defines=LIN_RELU=1
// @wg-kernel l_linear_32_32_f16 co=l_ops.co block=256 wave=32 args=p,p,p,u defines=LIN_RELU=0
+// @wg-kernel l_linear_32_32_fused co=- block=256 wave=32 args=p,p,p,p,u defines=LIN_RELU=0,LIN_FUSED=1
#include "wg_platform.hlsli"
+#ifndef LIN_FUSED
+#define LIN_FUSED 0
+#endif
+
WG_ARGS_BEGIN
uint x, y, w; // pointers: x [npix][32] FP8, y [npix][32] FP8 or FP16, w the blob (enc0.Win1 / enc0.Win2)
+#if LIN_FUSED
+ uint w2; // the f16 linear's blob (enc0.Win2; w = enc0.Win1)
+#endif
int npix;
WG_ARGS_END
-#if LIN_RELU
+#if LIN_FUSED // (helpers of the fused kernel only: the pair keeps its own code and DXIL)
+// the blob's bias tile vectors and weight fragments of lane (t, h)
+void lin_blob(uint wb, uint lane, uint hh, out v8f bias[2], out uint2 fr[2][2])
+{
+ [unroll] for (uint mt = 0; mt < 2u; ++mt) {
+ bias[mt] = wg_load8f(wb, (hh * 8u + mt * 16u) * 4u);
+ [unroll] for (uint kc2 = 0; kc2 < 2u; ++kc2) fr[mt][kc2] = wg_ld64(wb, 128u + (mt * 2u + kc2) * 256u + lane * 8u);
+ }
+}
+
+// output tile m before the rounding: b + W x
+v8f lin_acc(v8f bias, uint2 fr[2], uint2 xf[2])
+{
+ v8f acc = bias;
+ acc = wg_wmma_fp8(fr[0], xf[0], acc);
+ return wg_wmma_fp8(fr[1], xf[1], acc);
+}
+
+#endif
+
+#if LIN_FUSED
+WG_KERNEL(l_linear_32_32_fused)
+#elif LIN_RELU
WG_KERNEL(l_linear_32_32_relu)
#else
WG_KERNEL(l_linear_32_32_f16)
@@ -35,6 +75,30 @@ WG_KERNEL(l_linear_32_32_f16)
[unroll] for (uint kc = 0; kc < 2u; ++kc) xf[kc] = wg_ld64(x, (uint)pk * 32u + hh * 16u + 8u * kc);
v8f bias[2];
uint2 fr[2][2];
+#if LIN_FUSED
+ lin_blob(w, lane, hh, bias, fr);
+ v8f bias2[2];
+ uint2 fr2[2][2];
+ lin_blob(w2, lane, hh, bias2, fr2);
+ // the relu linear: x1's fragments (the bytes its kernel stores at pk * 32 + hh * 16 + 8 m). The conversions by the
+ // driver's instructions (wg_platform.hlsli: the same bits as the pair's software wg_rn16 / wg_pack_q8; no branch in
+ // this kernel): RN16 = v_cvt_f16_f32 then the exact widening, the pack v_cvt_pk_fp8_f32 after the clamp.
+ uint2 x1[2];
+ [unroll] for (uint m1 = 0; m1 < 2u; ++m1) {
+ const uint4 h = wg_to_f16_ags(lin_acc(bias[m1], fr[m1], xf));
+ v8f r;
+ [unroll] for (uint j = 0; j < 4u; ++j) {
+ r.v[2 * j] = wg_fmax(wg_h2f_lo(h[j]), 0.0f);
+ r.v[2 * j + 1] = wg_fmax(wg_h2f_hi(h[j]), 0.0f);
+ }
+ x1[m1] = wg_pack_q8_ags(r);
+ }
+ // the f16 linear on them (its kernel's loads of x1 at pk * 32 + hh * 16 + 8 kc): FP16(RN16(acc)) = FP16(acc), one
+ // conversion (rounding to FP16 is idempotent, NaN payloads included)
+ [unroll] for (uint m = 0; m < 2u; ++m)
+ wg_st128(y, ((uint)pk * 32u + hh * 16u + 8u * m) * 2u, wg_to_f16_ags(lin_acc(bias2[m], fr2[m], x1)));
+#else
+ // (the pair's own code, unchanged)
[unroll] for (uint mt = 0; mt < 2u; ++mt) {
bias[mt] = wg_load8f(w, (hh * 8u + mt * 16u) * 4u);
[unroll] for (uint kc2 = 0; kc2 < 2u; ++kc2) fr[mt][kc2] = wg_ld64(w, 128u + (mt * 2u + kc2) * 256u + lane * 8u);
@@ -51,4 +115,5 @@ WG_KERNEL(l_linear_32_32_f16)
wg_st128(y, ((uint)pk * 32u + hh * 16u + 8u * m) * 2u, wg_to_f16(acc));
#endif
}
+#endif
}
diff --git a/clean/dx12/tests/native_direct_check.py b/clean/dx12/tests/native_direct_check.py
index 84fe0d88..0bed5f9f 100644
--- a/clean/dx12/tests/native_direct_check.py
+++ b/clean/dx12/tests/native_direct_check.py
@@ -30,8 +30,8 @@
POST='w64/post_tf_f32_p12_diet_vsit'
-# The neural launches with the native D3D12 fusions of runtime.cpp kDefaultD3d12Fusion (pm,xe,mb,rv,xo,xd; its lc is
-# Model L's): enc2's block
+# The neural launches with the native D3D12 fusions of runtime.cpp kDefaultD3d12Fusion (pm,xe,mb,rv,xo,xd; its lc and ll
+# are Model L's): enc2's block
# with enc1's merge in reverse order (pm, rv), the bottleneck with enc2's merge and the overlap counters (mb, xo), dec2's
# block with dec1's expand (xe) launched as xd on frames of at most kXdMaxPixels render pixels (WG_D3D12_XD_MAX_PIXELS)
# else xo, and the fused dec1 + dec0 in its xd form on those frames. Set by main from --shape (COMMON above: the
diff --git a/clean/l/FORMATS.md b/clean/l/FORMATS.md
index fce40a7a..2ed72cb1 100644
--- a/clean/l/FORMATS.md
+++ b/clean/l/FORMATS.md
@@ -23,7 +23,7 @@ Every map is `[rows][cols][C]`, channel fastest, as Model M's feature maps (`cle
| Map | Shape | Element | Written by | Read by |
|---|---|---|---|---|
| `q` | `[fh + 4][fw + 4][32]` halo | FP8: values 0..19, bytes 20..31 zero | `lev/input_encoder` | `l_linear_32_32_relu` |
-| `x1` | `[fh + 4][fw + 4][32]` halo | FP8 | `l_linear_32_32_relu` | `l_linear_32_32_f16` |
+| `x1` | `[fh + 4][fw + 4][32]` halo | FP8 (not stored by `l_linear_32_32_fused`) | `l_linear_32_32_relu` | `l_linear_32_32_f16` |
| `x2` | `[fh + 4][fw + 4][32]` halo | **FP16** (64 bytes per pixel) | `l_linear_32_32_f16` | `l_swin_block_c32_enc` |
| `x3` | `[fh][fw][32]` | FP8 | `l_swin_block_c32_enc` | `l_project_gather` |
| `enc0.features` | `[fh/2][fw/2][64]` | FP8 | `l_project_gather` | enc1 (Model M's stage) |
@@ -78,6 +78,7 @@ workgroup, in raster order of their map.
| `lev/input_encoder` | exists (GPU-checked) | as Model M's `input_encoder`, with `features` replaced by `q` | ceil(((fh + 4) / 2) ((fw + 4) / 2) / 128) (2 x 2 pixel groups of the halo map) | the 20 values and their dithered FP8 for every halo-map pixel, into `q`. The motion surfaces as Model M, only for pixels inside R. w = `enc0.values` |
| `l_linear_32_32_relu` | exists (GPU-checked) | `x, y, w, npix` | ceil(npix / 128), npix = (fh + 4)(fw + 4) | y = q8(max(0, RN16(b + W x))), FP8 -> FP8. w = `enc0.Win1` |
| `l_linear_32_32_f16` | exists (GPU-checked) | `x, y, w, npix` | as above | y = RN16(b + W x), FP8 -> **FP16**. w = `enc0.Win2` |
+| `l_linear_32_32_fused` | exists (GPU-checked), D3D12 only | `x, y, w1, w2, npix` | as above | the two above in one kernel, `x1` kept in registers: the same bytes in `y` (`x2`), `x1` not stored. x = `q`, y = `x2` in two different buffers. w1 = `enc0.Win1`, w2 = `enc0.Win2`. The native D3D12 frame runs it in place of the two (fusion `ll`, on by default); without `ll` or without the kernel in the build, the two run |
| `l_swin_block_c32_enc[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | ceil(nwin / 8), nwin = ceil((W + ox) / 4) ceil((H + oy) / 4) | the encoder block. x = `x2` (FP16 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2)). out = `x3` (FP8, grid only). H = fh, W = fw. (ox, oy) = `window_offsets["first"]` for the frame parity. w = `enc0_block` |
| `l_project_gather` | exists (GPU-checked) | `x, features, w, tw, th` | ceil(tw th / 128), tw = fw / 2, th = fh / 2 | token (r, c) gathers pixels (2r + p / 2, 2c + p % 2), p = 0..3, as channels 32 p + i. Running FP16 sum per group of 32 from the bias, clamp +-6.28125, FP8 -> `enc0.features`. w = `enc0.project` |
| `l_dec_project` | exists (GPU-checked) | `z, y, w, tw, th` | ceil(tw th / 128) | per token of dec1's output z [th][tw][64]: 128 outputs (two RN16 groups of 32 inputs, then FP8). Output 32 s + k goes to pixel (2r + s / 2, 2c + s % 2), channel k, of `y`. w = `dec0.project` |
@@ -159,4 +160,6 @@ the frame parity, as Model M's.
- `clean/l/runner/l_d3d12_check.py` (with its D3D12 port): each D3D12 port against its HIP kernel, byte for byte.
- `clean/l/runner/l_fold_check.py`: `l_swin_block_c32_dec_coef12_nc` against `l_swin_block_c32_dec_nc` ->
`l_coef_pack12` on D3D12 and on HIP, byte for byte.
+- `clean/l/runner/l_linfuse_check.py`: `l_linear_32_32_fused` against `l_linear_32_32_relu` -> `l_linear_32_32_f16`
+ on D3D12 and on HIP, byte for byte.
- One check per new kernel follows with that kernel.
diff --git a/clean/l/runner/l_linfuse_check.py b/clean/l/runner/l_linfuse_check.py
new file mode 100644
index 00000000..97bc2132
--- /dev/null
+++ b/clean/l/runner/l_linfuse_check.py
@@ -0,0 +1,203 @@
+# SPDX-License-Identifier: MIT
+"""Model L (issue #89): the fused input linears l_linear_32_32_fused (D3D12 only; clean/dx12/kernels/l_ops_linear.hlsl)
+against the sequence it replaces, l_linear_32_32_relu -> l_linear_32_32_f16, byte for byte.
+
+ python clean/l/runner/l_linfuse_check.py --build
--rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --prepare
+ python clean/l/runner/l_linfuse_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --run
+ [--seeds N] --json out.json
+
+--prepare (CPU) builds into : l_ops.co and swin.co (clean/l/runner/l_c32_check.py compile_kernels), the D3D12
+kernels (clean/dx12/build_dx12.py --only) and kernel_replay.exe.
+--run (GPU, one GPU slot): every case is a pixel map (npix pixels, partly filled waves and workgroups among them), an
+FP8 input map q and the two blobs (enc0.Win1, enc0.Win2), at three weight scales (results past 448 and past FP16's
+range, results in FP16's subnormal range) and with special FP8 codes in q (NaN, +-448, +-0, subnormals): the fused
+kernel's conversions are the driver's, the pair's software. On the same input bytes:
+- the sequence on D3D12 (kernel_replay, two passes: the relu linear writes x1, then the f16 linear reads that file);
+- the sequence on HIP (in this process);
+- the fused kernel on D3D12.
+The rules are clean/l/runner/l_fold_check.py's: raw bytes; the output (x2, 64 bytes a pixel) has 64 bytes of slack and
+starts filled with its run's own sentinel (HIP 0xA5, the D3D12 sequence 0x5A, the fused kernel 0x3C): every pixel byte
+identical across the three, every slack byte its own run's sentinel; a negative control (the fused kernel with half its
+workgroups) must be caught; Model M's swin_block_c96_nc runs through the same replay as a control.
+Output: counts only, no timing.
+"""
+import argparse
+import ctypes
+import json
+import os
+import sys
+
+import numpy as np
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE)))
+sys.path.insert(0, ROOT)
+
+from clean.l.runner import l_c32_check as C32 # noqa: E402
+from clean.l.runner import l_d3d12_check as LD # noqa: E402
+from clean.l.runner import l_d3d12_ops_check as OPS # noqa: E402
+from clean.reference.fp8 import fp8_encode, q8 # noqa: E402
+from clean.l.runner.l_fold_check import (CONTROL, D3D12_SENTINEL, FOLD_SENTINEL, HIP_SENTINEL, SLACK, compare, # noqa: E402
+ nwin_of, replay, write)
+
+RELU, F16, FUSED = "l_linear_32_32_relu", "l_linear_32_32_f16", "l_linear_32_32_fused"
+# (rows, cols) of the maps: the ops check's (a 35-pixel map among them: a partly filled wave), the halo maps of the
+# runtime's smallest frames, 1 x 1, and one more workgroup than a whole number (129 pixels)
+MAPS = OPS.MAPS + [(36, 36), (1, 1), (3, 43), (68, 132)]
+
+
+def _blob(rng, scale, bias_scale):
+ """enc0.Win1 / Win2 layout (bias tile vector of 32 floats, 4 fragments of 256 FP8 bytes) at a weight scale."""
+ bias = rng.normal(0, bias_scale, 32).astype(np.float32).tobytes()
+ return bias + fp8_encode(q8(rng.normal(0, scale, 4 * 256))).tobytes()
+
+
+# FP8 codes the network does not produce but the conversions must treat as the pair does: both NaN codes, +-448, +-0,
+# subnormals
+SPECIAL = np.array([0x7f, 0xff, 0x7e, 0xfe, 0x00, 0x80, 0x01, 0x81, 0x07, 0x87], np.uint8)
+# weight / bias scales: the ops check's; large (sums past 448 and past FP16's 65504: infinities); tiny (FP16 subnormal
+# and zero results)
+SCALES = [(0.25, 0.5), (24.0, 400.0), (2e-3, 1e-4)]
+
+
+def cases(rng):
+ """(npix, q bytes, Win1 blob, Win2 blob) of every case: each map at each weight scale, every other case with
+ special FP8 codes in a tenth of q's bytes."""
+ for ci, (rows, cols) in enumerate(MAPS):
+ n = rows * cols
+ for si, (s, b) in enumerate(SCALES):
+ q = np.frombuffer(OPS._fp8_map(rng, n * 32), np.uint8).copy()
+ if (ci + si) % 2:
+ pick = rng.random(q.size) < 0.1
+ q[pick] = rng.choice(SPECIAL, int(pick.sum()))
+ yield n, q.tobytes(), _blob(rng, s, b), _blob(rng, s, b)
+
+
+def hip_sequence(g, ops, c):
+ """relu, then f16, on HIP: x2's raw bytes (the output starts at HIP_SENTINEL)."""
+ n, q, w1, w2 = c
+ h = g.hip
+ owned = []
+
+ def buf(data):
+ p = h.malloc(max(len(data), 4))
+ h.upload(p, data)
+ owned.append(p)
+ return p
+
+ try:
+ pq, pw1, pw2 = buf(q), buf(w1), buf(w2)
+ px1, py = h.malloc(n * 32), h.malloc(n * 64 + SLACK)
+ owned += [px1, py]
+ h.memset(px1, HIP_SENTINEL, n * 32)
+ h.memset(py, HIP_SENTINEL, n * 64 + SLACK)
+ grid = -(-n // 128)
+ for k, a, b, wb in ((RELU, pq, px1, pw1), (F16, px1, py, pw2)):
+ h.launch(h.get_function(ops, k), grid, 256,
+ [ctypes.c_void_p(a), ctypes.c_void_p(b), ctypes.c_void_p(wb), ctypes.c_int(n)])
+ h.sync()
+ return bytes(h.download(py, n * 64 + SLACK))
+ finally:
+ for p in owned:
+ h.free(p)
+
+
+def fused_lines(name, n, prefix="", grid=None):
+ return [f"case {prefix}{name}_fused", f"kernel {FUSED} {grid or -(-n // 128)} 256", f"arg buf {name}.q",
+ f"arg out {n * 64 + SLACK} {prefix}{name}.fy {FOLD_SENTINEL}", f"arg buf {name}.w1", f"arg buf {name}.w2",
+ f"arg u32 {n}", ""]
+
+
+def run(build, rocm, seeds):
+ g = C32.Gpu(build, rocm)
+ with open(os.path.join(build, "l_ops.co"), "rb") as f:
+ ops = g.hip.load_module(f.read())
+ cdir = os.path.join(build, "replay-linfuse")
+ os.makedirs(cdir, exist_ok=True)
+ pass1, pass2, all_cases, neg = [], [], {}, None
+ for si, seed in enumerate(seeds):
+ rng = np.random.default_rng(seed)
+ for ci, c in enumerate(cases(rng)):
+ name = f"s{si}_{ci}"
+ n, q, w1, w2 = c
+ all_cases[name] = (n, hip_sequence(g, ops, c))
+ write(os.path.join(cdir, name + ".q"), q)
+ write(os.path.join(cdir, name + ".w1"), w1)
+ write(os.path.join(cdir, name + ".w2"), w2)
+ grid = -(-n // 128)
+ # pass 1: the D3D12 relu linear (x1) and the fused kernel
+ pass1 += [f"case {name}_relu", "module l_ops.co", f"kernel {RELU} {grid} 256", f"arg buf {name}.q",
+ f"arg out {n * 32} {name}.x1 {D3D12_SENTINEL}", f"arg buf {name}.w1", f"arg u32 {n}", ""]
+ pass1 += fused_lines(name, n)
+ if neg is None and grid >= 4:
+ neg = name
+ pass1 += fused_lines(name, n, prefix="negative_", grid=grid // 2)
+ # pass 2: the D3D12 f16 linear on that x1
+ pass2 += [f"case {name}_f16", "module l_ops.co", f"kernel {F16} {grid} 256", f"arg buf {name}.x1",
+ f"arg out {n * 64 + SLACK} {name}.sy {D3D12_SENTINEL}", f"arg buf {name}.w2", f"arg u32 {n}", ""]
+ rng = np.random.default_rng(seeds[0])
+ ctl = []
+ for ci, (H, W, off, xb, nout, blob) in enumerate(LD.block_cases(CONTROL, rng)):
+ name = f"control_{ci}"
+ ctl.append((name, LD.hip_block_raw(g, "swin.co", CONTROL, xb, blob, H, W, off, nout)))
+ write(os.path.join(cdir, name + ".x"), xb)
+ write(os.path.join(cdir, name + ".w"), blob)
+ pass1 += [f"case {name}", "module swin.co", f"kernel {CONTROL} {-(-nwin_of(H, W, off) // 8)} 256",
+ f"arg buf {name}.x", "arg null", f"arg out {nout} {name}.out {D3D12_SENTINEL}", f"arg buf {name}.w",
+ f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {off[0]}", f"arg u32 {off[1]}", ""]
+ e1, t1 = replay(build, cdir, pass1, "pass1")
+ e2, t2 = replay(build, cdir, pass2, "pass2") if e1 == 0 else (-1, "pass 1 failed")
+ res = {"replay_exit": [e1, e2], "replay_tail": [t1, t2], "cases": []}
+
+ def read(n):
+ p = os.path.join(cdir, n)
+ return open(p, "rb").read() if os.path.exists(p) else b""
+
+ for name, (n, hip) in all_cases.items():
+ fused, seq = read(f"{name}.fy"), read(f"{name}.sy")
+ r = {"case": name, "npix": n,
+ "fused_vs_d3d12_seq": compare(fused, seq, n * 64, FOLD_SENTINEL, D3D12_SENTINEL),
+ "fused_vs_hip_seq": compare(fused, hip, n * 64, FOLD_SENTINEL, HIP_SENTINEL),
+ "d3d12_seq_vs_hip_seq": compare(seq, hip, n * 64, D3D12_SENTINEL, HIP_SENTINEL)}
+ r["identical"] = all(r[k]["identical"] for k in ("fused_vs_d3d12_seq", "fused_vs_hip_seq", "d3d12_seq_vs_hip_seq"))
+ res["cases"].append(r)
+ caught = None
+ if neg:
+ n = all_cases[neg][0]
+ o = compare(read(f"negative_{neg}.fy"), read(f"{neg}.sy"), n * 64, FOLD_SENTINEL, D3D12_SENTINEL)
+ caught = o["differ"] >= 0 and not o["identical"]
+ res["negative_control"] = {"case": neg, "half_grid": True, "caught": caught, "differ": max(o["differ"], 0)}
+ ctl_ok = [read(f"{name}.out") == want for name, want in ctl]
+ res["control"] = {"kernel": CONTROL, "cases": len(ctl_ok), "identical": sum(ctl_ok)}
+ res["summary"] = {"cases": len(res["cases"]), "identical": sum(c["identical"] for c in res["cases"]),
+ "pixel_bytes": sum(c["npix"] * 64 for c in res["cases"])}
+ res["pass"] = (e1 == 0 and e2 == 0 and caught is True and res["summary"]["cases"] > 0 and
+ res["summary"]["identical"] == res["summary"]["cases"] and len(ctl_ok) > 0 and all(ctl_ok))
+ return res
+
+
+def main(argv=None):
+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+ ap.add_argument("--build", required=True)
+ ap.add_argument("--rocm", required=True)
+ ap.add_argument("--prepare", action="store_true")
+ ap.add_argument("--run", action="store_true")
+ ap.add_argument("--seeds", type=int, default=2)
+ ap.add_argument("--json")
+ a = ap.parse_args(argv)
+ os.makedirs(a.build, exist_ok=True)
+ if a.prepare:
+ C32.compile_kernels(a.build, a.rocm, with_swin=True)
+ print("built", LD.prepare_replay(a.build, a.rocm, [RELU, F16, FUSED, CONTROL]))
+ if a.run:
+ res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds)))
+ if a.json:
+ with open(a.json, "w") as f:
+ json.dump(res, f, indent=1)
+ print({k: v for k, v in res.items() if k not in ("cases", "replay_tail")})
+ return 0 if res["pass"] else 1
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/clean/l/runtime/tests/test_l_runtime_frame.py b/clean/l/runtime/tests/test_l_runtime_frame.py
index 11213ca1..616c77c8 100644
--- a/clean/l/runtime/tests/test_l_runtime_frame.py
+++ b/clean/l/runtime/tests/test_l_runtime_frame.py
@@ -127,8 +127,11 @@ def blob_digests():
# native D3D12, fusion lc (the default): the decoder block with l_coef_pack12 as its epilogue, in place of the pair
FOLD = "l_swin_block_c32_dec_coef12_nc"
FOLD_BASE = "l_swin_block_c32_dec_coef12"
-# the default fusions without lc: the pair (the fold's fallback route)
+# native D3D12, fusion ll (the default): the two input linears in one kernel (x1 in registers; q in l.narrow)
+LIN = "l_linear_32_32_fused"
+# the default fusions without lc and ll: the pairs (the fallback routes)
NO_FOLD = {"WG_D3D12_FUSION": "pm,xe,mb,rv,xo,xd"}
+NO_LIN = {"WG_D3D12_FUSION": "pm,xe,mb,rv,xo,xd,lc"}
def latent_out(coef):
@@ -152,14 +155,16 @@ def expected_args(t, f, digests):
npix, tw, th = str((fh + 4) * (fw + 4)), str(fw // 2), str(fh // 2)
eo = [str(v) for v in T.block_offset(T.TABLE, "first", info["parity"])]
do = [str(v) for v in T.block_offset(T.TABLE, "second", info["parity"])]
+ q = n["l.narrow"] if any(ln.kernel == LIN for ln in f["launches"]) else n["l.wide"] # fused linears: q not under x2
return {
# the input encoder's values: Model M's encoder inputs (the colour as the network reads it, here the game's,
# pre-exposure 1), the state it reads, q into the wide map, the motion surfaces, the exposure pair
"lev/input_encoder": [n["input.color"], n["input.depth"], n["input.motion"], n["post.reduced"],
- n["state.latent_prev"], n["state.motion_history"], n["l.wide"], n["enc0.m_near"],
+ n["state.latent_prev"], n["state.motion_history"], q, n["enc0.m_near"],
n["enc0.m_far"], n["enc0.m_center"], n["exposure.pair"], w["enc0.values"], None],
"l_linear_32_32_relu": [n["l.wide"], n["l.narrow"], w["enc0.Win1"], npix], # q -> x1
"l_linear_32_32_f16": [n["l.narrow"], n["l.wide"], w["enc0.Win2"], npix], # x1 -> x2
+ LIN: [n["l.narrow"], n["l.wide"], w["enc0.Win1"], w["enc0.Win2"], npix], # q -> x2 (x1 in registers)
"l_swin_block_c32_enc": [n["l.wide"], "0", n["l.x3"], w["enc0_block"], str(fh), str(fw)] + eo, # x2 -> x3
"l_project_gather": [n["l.x3"], n["enc0.features"], w["enc0.project"], tw, th], # x3 -> enc1's input
"l_dec_project": [n["dec1.out"], n["l.narrow"], w["dec0.project"], tw, th], # dec1's output -> y
@@ -198,10 +203,14 @@ def check_roles(t, f, digests, l_post=True):
continue
bad += ["%s argument %d: %s, expected %s" % (k, i, a, e) for i, (a, e) in enumerate(zip(ln.args, want))
if e is not None and a != e]
- fold = FOLD_BASE in seen
- need = [k for k in L_KERNELS if not (fold and k in ("l_swin_block_c32_dec", "l_coef"))] + ([FOLD_BASE] if fold else [])
- if fold and ({"l_swin_block_c32_dec", "l_coef"} & seen):
+ fold, lin = FOLD_BASE in seen, LIN in seen
+ pair_d, pair_l = ("l_swin_block_c32_dec", "l_coef"), ("l_linear_32_32_relu", "l_linear_32_32_f16")
+ need = [k for k in L_KERNELS if not (fold and k in pair_d) and not (lin and k in pair_l)] + \
+ ([FOLD_BASE] if fold else []) + ([LIN] if lin else [])
+ if fold and (set(pair_d) & seen):
bad.append("the folded decoder block and the pair in one frame")
+ if lin and (set(pair_l) & seen):
+ bad.append("the fused linears and the pair in one frame")
missing = [k for k in need if k not in seen and (l_post or k != "l_post_recon")]
if bad or missing:
raise AssertionError("; ".join(bad + ["not launched: " + k for k in missing]))
@@ -330,8 +339,10 @@ def check_frame(self, t, f, roles=True, l_post=True):
npix, tw, th = (fh + 4) * (fw + 4), fw // 2, fh // 2
k = {base(ln.kernel): (i, ln) for i, ln in enumerate(ls)}
_, lev = k["lev/input_encoder"]
- _, relu = k["l_linear_32_32_relu"]
- _, f16 = k["l_linear_32_32_f16"]
+ lin = k[LIN][1] if LIN in k else None
+ if not lin:
+ _, relu = k["l_linear_32_32_relu"]
+ _, f16 = k["l_linear_32_32_f16"]
_, enc = k["l_swin_block_c32_enc"]
gi, gather = k["l_project_gather"]
di, dproj = k["l_dec_project"]
@@ -342,10 +353,18 @@ def check_frame(self, t, f, roles=True, l_post=True):
else:
_, dec = k["l_swin_block_c32_dec"]
_, coef = k["l_coef"]
- q, x1, x2, x3, feat = lev.args[6], relu.args[1], f16.args[1], enc.args[2], gather.args[1]
+ q, x3, feat = lev.args[6], enc.args[2], gather.args[1]
# the chain of maps (no launch reads what it writes)
- self.assertEqual(relu.args[0], q)
- self.assertEqual(f16.args[0], x1)
+ if lin: # one launch, x1 in registers: no map
+ x1, x2 = None, lin.args[1]
+ self.assertEqual(lin.args[0], q)
+ self.assertEqual(lin.args[4], str(npix))
+ else:
+ x1, x2 = relu.args[1], f16.args[1]
+ self.assertEqual(relu.args[0], q)
+ self.assertEqual(f16.args[0], x1)
+ self.assertEqual(relu.args[3], str(npix))
+ self.assertEqual(f16.args[3], str(npix))
self.assertEqual(enc.args[0], x2)
self.assertEqual(enc.args[1], "0") # xadd: none
self.assertEqual(gather.args[0], x3)
@@ -360,11 +379,14 @@ def check_frame(self, t, f, roles=True, l_post=True):
self.assertEqual(y2, "0") # the map is not written
else:
self.assertEqual(coef.args[0], y2)
- for a, b in ((q, x1), (x1, x2), (x2, x3), (x3, feat), (dec1.args[2], y), (y, y2)):
- self.assertNotEqual(a, b)
+ for a, b in ((q, x1), (x1, x2), (q, x2 if lin else None), (x2, x3), (x3, feat), (dec1.args[2], y), (y, y2)):
+ if a is not None and b is not None:
+ self.assertNotEqual(a, b)
+ if not lin:
+ self.assertEqual(q, x2) # the pair: x2 over q (q is dead once x1 is written)
# every buffer holds its map (FORMATS.md "Maps")
- maps = ((q, 32 * npix), (x1, 32 * npix), (x2, 64 * npix), (x3, 32 * fh * fw), (feat, 64 * tw * th),
- (y, 32 * fh * fw)) + (() if fold else ((y2, 64 * fh * fw),))
+ maps = ((q, 32 * npix), (x2, 64 * npix), (x3, 32 * fh * fw), (feat, 64 * tw * th),
+ (y, 32 * fh * fw)) + (() if fold else ((y2, 64 * fh * fw),)) + (() if lin else ((x1, 32 * npix),))
if fold: # pack12 image A (8 bytes a pixel), image C (4), the new latent
coefs = ((coef.args[8], 8 * rw * rh), (coef.args[9], 4 * rw * rh), (coef.args[10], 4 * rw * rh))
else:
@@ -373,8 +395,6 @@ def check_frame(self, t, f, roles=True, l_post=True):
for buf, need in maps + coefs:
self.assertGreaterEqual(t.extent(buf), need, buf)
# extents, offsets, parameters
- self.assertEqual(relu.args[3], str(npix))
- self.assertEqual(f16.args[3], str(npix))
self.assertEqual(gather.args[3:5], [str(tw), str(th)])
self.assertEqual(dproj.args[3:5], [str(tw), str(th)])
for blk, kind in ((enc, "first"), (dec, "second")):
@@ -546,6 +566,12 @@ def check_d3d12_frame(self, t, f):
self.assertEqual(names[names.index(FOLD) - 1], "l_dec_project")
else:
self.assertIn("l_coef_pack12", names)
+ if LIN in names:
+ self.assertNotIn("l_linear_32_32_relu", names)
+ self.assertNotIn("l_linear_32_32_f16", names)
+ self.assertTrue(names[names.index(LIN) - 1].startswith("lev/input_encoder"))
+ else:
+ self.assertIn("l_linear_32_32_relu", names)
self.assertNotIn("l_coef", names)
self.assertTrue(all(not n.startswith(("d1p12/", "d0p12/")) and n != "input_encoder" for n in names), names)
dec1 = [ln for ln in f["launches"] if ln.kernel.startswith("swin_block_c64_add")]
@@ -567,10 +593,14 @@ def test_frame(self):
self.assertEqual(t.unknown(), [])
self.assertEqual([f["info"]["reset"] for f in t.frames], [1, 0, 1, 0, 1, 1])
self.assertEqual(len({f["info"]["parity"] for f in t.frames}), 2)
- for env, ports, fold in ((None, full_ports(), True), (NO_FOLD, full_ports(), False),
- (None, [kc for kc in full_ports() if kc[0] != FOLD], False)):
- with self.subTest(fusion=env, fold_in_manifest=any(kc[0] == FOLD for kc in ports)):
- # the fold (fusion lc, the default) and its fallback, the pair: without lc, or without the kernel
+ for env, ports, fold, lin in ((None, full_ports(), True, True), (NO_FOLD, full_ports(), False, False),
+ (None, [kc for kc in full_ports() if kc[0] != FOLD], False, True),
+ (NO_LIN, full_ports(), True, False),
+ (None, [kc for kc in full_ports() if kc[0] != LIN], True, False)):
+ with self.subTest(fusion=env, fold_in_manifest=any(kc[0] == FOLD for kc in ports),
+ lin_in_manifest=any(kc[0] == LIN for kc in ports)):
+ # the fusions (lc, ll: the default) and their fallbacks, the pairs: without the fusion, or without the
+ # kernel
t = FT.Trace(self.l_run(ports, frames=6, rw=96, rh=64, flags=(sizes, "--reset=4"), env=env))
self.assertEqual(t.create, "create ok model L")
self.assertEqual(t.unknown(), [])
@@ -580,6 +610,7 @@ def test_frame(self):
for f in t.frames:
self.assertIsNone(f["error"])
self.assertEqual(FOLD in [ln.kernel for ln in f["launches"]], fold)
+ self.assertEqual(LIN in [ln.kernel for ln in f["launches"]], lin)
lev, coef = self.check_d3d12_frame(t, f)
if prev:
self.assertEqual(lev.args[4], latent_out(prev[1])) # the latent the last frame's store wrote
diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp
index 52e09ad3..9499f5a2 100644
--- a/clean/runtime/runtime.cpp
+++ b/clean/runtime/runtime.cpp
@@ -125,8 +125,9 @@ std::string emu_lib_dir(const WgRtConfig& c) {
// 10 faster; one 8-round run -11.5 us with every round faster). Re-measured in one process (runtime A/B, the same frames
// alternating every 300 frames; inter-process noise was +-25 us): xd is +5.0 us at 1707x960 and +7.5 us at 1920x1080,
// -2.25 us at 1280x720, so it now runs on frames of at most kXdMaxPixels render pixels only.
-// lc: Model L only (its decoder block with l_coef_pack12 as the epilogue; see fz_lc).
-const char* const kDefaultD3d12Fusion = "pm,xe,mb,rv,xo,xd,lc";
+// lc, ll: Model L only (its decoder block with l_coef_pack12 as the epilogue, its two input linears in one kernel; see
+// fz_lc, fz_ll).
+const char* const kDefaultD3d12Fusion = "pm,xe,mb,rv,xo,xd,lc,ll";
// xd only up to this many render pixels per frame (see fz_xd): it is faster at 1280x720 and slower at 1707x960 and above
constexpr long long kXdMaxPixels = 1200000;
@@ -321,16 +322,21 @@ struct Runtime {
// block's FP16 map (y2, 64 bytes a pixel) is neither stored nor read back. Pack12 frames only; without lc, or
// with a manifest that lacks the kernel, the pair runs (the fallback route)
bool fz_lc = false, l_fold_built = false;
+ // ll Model L: its two input linears in one kernel (l_linear_32_32_fused, l_ops_linear.hlsl; byte for byte the pair
+ // l_linear_32_32_relu -> l_linear_32_32_f16): one launch less, and x1 (32 bytes a pixel of the halo map) stays
+ // in registers. The input encoder then writes q to l_narrow (x2 over q in l_wide would overwrite q that other
+ // workgroups have not read yet). Without ll, or with a manifest that lacks the kernel, the pair runs
+ bool fz_ll = false, l_lin_built = false;
long long xd_max_pixels = [] {
const char* e = std::getenv("WG_D3D12_XD_MAX_PIXELS");
return e && *e ? std::atoll(e) : kXdMaxPixels;
}();
// A/B diagnostics (WG_AB_*): the two fusion sets and the block length (0: off)
struct FzSet {
- bool pm = false, xe = false, mb = false, mb0 = false, rv = false, xo = false, xd = false, lc = false;
+ bool pm = false, xe = false, mb = false, mb0 = false, rv = false, xo = false, xd = false, lc = false, ll = false;
template void apply(R* r) const {
r->fz_pm = pm, r->fz_xe = xe, r->fz_mb = mb, r->fz_mb0 = mb0, r->fz_rv = rv, r->fz_xo = xo, r->fz_xd = xd;
- r->fz_lc = lc;
+ r->fz_lc = lc, r->fz_ll = ll;
}
};
FzSet fz_set[2];
@@ -360,8 +366,9 @@ struct Runtime {
else if (f == "xo") z->xo = true;
else if (f == "xd") z->xd = true;
else if (f == "lc") z->lc = true;
+ else if (f == "ll") z->ll = true;
else if (!f.empty())
- return fail("WG_D3D12_FUSION: unknown fusion \"" + f + "\" (pm, xe, mb, rv, xo, xd, lc or none)");
+ return fail("WG_D3D12_FUSION: unknown fusion \"" + f + "\" (pm, xe, mb, rv, xo, xd, lc, ll or none)");
}
return true;
}
@@ -531,6 +538,7 @@ struct Runtime {
if (!missing.empty()) return fail("Model L on native D3D12: not yet ported to D3D12: " + missing);
const std::vector have = manifest_kernels();
l_fold_built = std::find(have.begin(), have.end(), "l_swin_block_c32_dec_coef12_nc") != have.end();
+ l_lin_built = std::find(have.begin(), have.end(), "l_linear_32_32_fused") != have.end();
}
#endif
if (c.max_rw <= 0 || c.max_rh <= 0 || c.ow <= 0 || c.oh <= 0) return fail("invalid render or output size");
@@ -1373,7 +1381,7 @@ struct Runtime {
kernels_text += std::string(" dec_rdiv=") + (dec_div.find('r') == std::string::npos ? "1" : "0");
kernels_text += std::string(" dec_bdiv=") + (dec_div.find('b') == std::string::npos ? "1" : "0");
kernels_text += std::string(" dec1_dec0=") + (dec1_dec0 ? "fused" : "separate");
- if (fz_pm || fz_xe || fz_mb || fz_rv || fz_xo || fz_xd || fz_lc) kernels_text += " d3d12_fusion=" + fz_text;
+ if (fz_pm || fz_xe || fz_mb || fz_rv || fz_xo || fz_xd || fz_lc || fz_ll) kernels_text += " d3d12_fusion=" + fz_text;
kernels_text += " swin_mid=" + (mid_variant.empty() ? std::string("none") : mid_variant);
kernels_text += std::string(" post_pack=") + (p12 ? "p12" : "none");
kernels_text += std::string(" pre_exposure=") + (pi_original ? "original" : "scale");
@@ -1387,6 +1395,7 @@ struct Runtime {
// Model L: its encoder and decoder (enc0= / dec0= above name Model M's, not loaded for Model L)
if (model_l) kernels_text += " model=L l_kernels=l_enc_values.co,l_ops.co,l_swin.co,l_post.co l_post=l_post_recon";
if (l_fold_built) kernels_text += std::string(" l_dec_fold=") + (fz_lc ? "on" : "off"); // (native D3D12 only)
+ if (l_lin_built) kernels_text += std::string(" l_lin_fused=") + (fz_ll ? "on" : "off");
return kernels_text.c_str();
}
@@ -1532,12 +1541,15 @@ struct Runtime {
bool l_encoder(EncParams& p, DevPtr cnet, DevPtr din, DevPtr min_, int cur, int nxt, int fh, int fw, int f) {
DevPtr wv = wdev["enc0.values"], w1 = wdev["enc0.Win1"], w2 = wdev["enc0.Win2"], wp = wdev["enc0.project"];
int npix = (fh + 4) * (fw + 4), tw = fw / 2, th = fh / 2;
+ const bool fused = fz_ll && l_lin_built; // (native D3D12 only: l_lin_built)
+ DevPtr& q = fused ? l_narrow : l_wide; // q's map (fused: not under x2)
return launch(native_direct_io ? "lev/input_encoder_direct" : "lev/input_encoder",
cdiv(((fh + 4) / 2) * ((fw + 4) / 2), 128), 256,
- {&cnet, &din, &min_, &reduced, &latent[cur], &mnear[cur], &l_wide, &mnear[nxt], &mfar, &mcenter,
+ {&cnet, &din, &min_, &reduced, &latent[cur], &mnear[cur], &q, &mnear[nxt], &mfar, &mcenter,
&expo, &wv, &p}) &&
- launch("l_linear_32_32_relu", cdiv(npix, 128), 256, {&l_wide, &l_narrow, &w1, &npix}) &&
- launch("l_linear_32_32_f16", cdiv(npix, 128), 256, {&l_narrow, &l_wide, &w2, &npix}) &&
+ (fused ? launch("l_linear_32_32_fused", cdiv(npix, 128), 256, {&l_narrow, &l_wide, &w1, &w2, &npix})
+ : launch("l_linear_32_32_relu", cdiv(npix, 128), 256, {&l_wide, &l_narrow, &w1, &npix}) &&
+ launch("l_linear_32_32_f16", cdiv(npix, 128), 256, {&l_narrow, &l_wide, &w2, &npix})) &&
l_block("enc", l_wide, l_x3, wdev["enc0_block"], l_nc[0], fh, fw, false, f) &&
launch("l_project_gather", cdiv(tw * th, 128), 256, {&l_x3, &feat0, &wp, &tw, &th});
}
diff --git a/clean/tests/runtime_frame_trace.cpp b/clean/tests/runtime_frame_trace.cpp
index 08c31159..350abc86 100644
--- a/clean/tests/runtime_frame_trace.cpp
+++ b/clean/tests/runtime_frame_trace.cpp
@@ -78,6 +78,7 @@ const Signature kSignatures[] = {
{"l_swin_block_c32*", "ppppiiii"}, // Model L's blocks (clean/l/FORMATS.md)
{"l_linear_32_32_relu", "pppi"},
{"l_linear_32_32_f16", "pppi"},
+ {"l_linear_32_32_fused", "ppppi"}, // the two in one: q, x2, Win1, Win2, npix (native D3D12, fusion ll)
{"l_project_gather", "pppii"},
{"l_dec_project", "pppii"},
{"l_coef", "ppppppD"},