Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion clean/dx12/compare.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@
"expand_c96", "d1p12/swin_block_c64_add_dec0_nc", "w64/post_tf_f32_p12_diet_vsit"}
# The native backend (--backend d3d12) runs the D3D12 fusions of WG_D3D12_FUSION (default: runtime.cpp's
# kDefaultD3d12Fusion, mirrored here), which replace some launches (kernels/fused_swin.hlsl, fused_bneck.hlsl, fused_dec.hlsl).
DEFAULT_D3D12_FUSION = "pm,xe,mb,rv,xo,xd,lc" # lc: Model L only
DEFAULT_D3D12_FUSION = "pm,xe,mb,rv,xo,xd,lc,ll" # lc, ll: Model L only


def d3d12_fusion():
Expand Down
67 changes: 66 additions & 1 deletion clean/dx12/kernels/l_ops_linear.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,56 @@
// the store let the D3D12 compiler sink the matrix instructions into it, where they ran with part of the wave masked
// off: wrong values in a partly filled wave (a 35-pixel map, pixels 32..34; found by l_d3d12_ops_check.py).
//
// l_linear_32_32_fused (D3D12 only): the two in one kernel, x1 kept in registers. The relu linear's lane (t, h) stores
// x1's channels 16 h + 8 m .. + 7 (m = 0, 1) as the two FP8 fragments that the f16 linear's lane (t, h) loads as its
// K blocks kc = m: the fragments pass in registers, the same bytes, and the x1 map is neither written nor read. x is
// the q map, y the x2 map: two different buffers (the pair runs q -> x1 -> x2 with x2 over q's buffer; here x2 would
// overwrite q that other workgroups have not read yet). Arguments: x, y, Win1, Win2, npix. Its conversions are the
// driver's instructions (RN16 and the final FP16 by v_cvt_f16_f32, the FP8 pack by v_cvt_pk_fp8_f32 after the clamp:
// wg_platform.hlsli, the same bits as the software forms the pair keeps); the software forms were most of the pair's
// instructions. Measured at 1080p Quality: 0.095 ms against the pair's 0.217 ms (the fused kernel with the software
// forms: 0.236 ms).
//
// @wg-kernel l_linear_32_32_relu co=l_ops.co block=256 wave=32 args=p,p,p,u defines=LIN_RELU=1
// @wg-kernel l_linear_32_32_f16 co=l_ops.co block=256 wave=32 args=p,p,p,u defines=LIN_RELU=0
// @wg-kernel l_linear_32_32_fused co=- block=256 wave=32 args=p,p,p,p,u defines=LIN_RELU=0,LIN_FUSED=1
#include "wg_platform.hlsli"

#ifndef LIN_FUSED
#define LIN_FUSED 0
#endif

WG_ARGS_BEGIN
uint x, y, w; // pointers: x [npix][32] FP8, y [npix][32] FP8 or FP16, w the blob (enc0.Win1 / enc0.Win2)
#if LIN_FUSED
uint w2; // the f16 linear's blob (enc0.Win2; w = enc0.Win1)
#endif
int npix;
WG_ARGS_END

#if LIN_RELU
#if LIN_FUSED // (helpers of the fused kernel only: the pair keeps its own code and DXIL)
// the blob's bias tile vectors and weight fragments of lane (t, h)
void lin_blob(uint wb, uint lane, uint hh, out v8f bias[2], out uint2 fr[2][2])
{
[unroll] for (uint mt = 0; mt < 2u; ++mt) {
bias[mt] = wg_load8f(wb, (hh * 8u + mt * 16u) * 4u);
[unroll] for (uint kc2 = 0; kc2 < 2u; ++kc2) fr[mt][kc2] = wg_ld64(wb, 128u + (mt * 2u + kc2) * 256u + lane * 8u);
}
}

// output tile m before the rounding: b + W x
v8f lin_acc(v8f bias, uint2 fr[2], uint2 xf[2])
{
v8f acc = bias;
acc = wg_wmma_fp8(fr[0], xf[0], acc);
return wg_wmma_fp8(fr[1], xf[1], acc);
}

#endif

#if LIN_FUSED
WG_KERNEL(l_linear_32_32_fused)
#elif LIN_RELU
WG_KERNEL(l_linear_32_32_relu)
#else
WG_KERNEL(l_linear_32_32_f16)
Expand All @@ -35,6 +75,30 @@ WG_KERNEL(l_linear_32_32_f16)
[unroll] for (uint kc = 0; kc < 2u; ++kc) xf[kc] = wg_ld64(x, (uint)pk * 32u + hh * 16u + 8u * kc);
v8f bias[2];
uint2 fr[2][2];
#if LIN_FUSED
lin_blob(w, lane, hh, bias, fr);
v8f bias2[2];
uint2 fr2[2][2];
lin_blob(w2, lane, hh, bias2, fr2);
// the relu linear: x1's fragments (the bytes its kernel stores at pk * 32 + hh * 16 + 8 m). The conversions by the
// driver's instructions (wg_platform.hlsli: the same bits as the pair's software wg_rn16 / wg_pack_q8; no branch in
// this kernel): RN16 = v_cvt_f16_f32 then the exact widening, the pack v_cvt_pk_fp8_f32 after the clamp.
uint2 x1[2];
[unroll] for (uint m1 = 0; m1 < 2u; ++m1) {
const uint4 h = wg_to_f16_ags(lin_acc(bias[m1], fr[m1], xf));
v8f r;
[unroll] for (uint j = 0; j < 4u; ++j) {
r.v[2 * j] = wg_fmax(wg_h2f_lo(h[j]), 0.0f);
r.v[2 * j + 1] = wg_fmax(wg_h2f_hi(h[j]), 0.0f);
}
x1[m1] = wg_pack_q8_ags(r);
}
// the f16 linear on them (its kernel's loads of x1 at pk * 32 + hh * 16 + 8 kc): FP16(RN16(acc)) = FP16(acc), one
// conversion (rounding to FP16 is idempotent, NaN payloads included)
[unroll] for (uint m = 0; m < 2u; ++m)
wg_st128(y, ((uint)pk * 32u + hh * 16u + 8u * m) * 2u, wg_to_f16_ags(lin_acc(bias2[m], fr2[m], x1)));
#else
// (the pair's own code, unchanged)
[unroll] for (uint mt = 0; mt < 2u; ++mt) {
bias[mt] = wg_load8f(w, (hh * 8u + mt * 16u) * 4u);
[unroll] for (uint kc2 = 0; kc2 < 2u; ++kc2) fr[mt][kc2] = wg_ld64(w, 128u + (mt * 2u + kc2) * 256u + lane * 8u);
Expand All @@ -51,4 +115,5 @@ WG_KERNEL(l_linear_32_32_f16)
wg_st128(y, ((uint)pk * 32u + hh * 16u + 8u * m) * 2u, wg_to_f16(acc));
#endif
}
#endif
}
4 changes: 2 additions & 2 deletions clean/dx12/tests/native_direct_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,8 @@
POST='w64/post_tf_f32_p12_diet_vsit'


# The neural launches with the native D3D12 fusions of runtime.cpp kDefaultD3d12Fusion (pm,xe,mb,rv,xo,xd; its lc is
# Model L's): enc2's block
# The neural launches with the native D3D12 fusions of runtime.cpp kDefaultD3d12Fusion (pm,xe,mb,rv,xo,xd; its lc and ll
# are Model L's): enc2's block
# with enc1's merge in reverse order (pm, rv), the bottleneck with enc2's merge and the overlap counters (mb, xo), dec2's
# block with dec1's expand (xe) launched as xd on frames of at most kXdMaxPixels render pixels (WG_D3D12_XD_MAX_PIXELS)
# else xo, and the fused dec1 + dec0 in its xd form on those frames. Set by main from --shape (COMMON above: the
Expand Down
5 changes: 4 additions & 1 deletion clean/l/FORMATS.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ Every map is `[rows][cols][C]`, channel fastest, as Model M's feature maps (`cle
| Map | Shape | Element | Written by | Read by |
|---|---|---|---|---|
| `q` | `[fh + 4][fw + 4][32]` halo | FP8: values 0..19, bytes 20..31 zero | `lev/input_encoder` | `l_linear_32_32_relu` |
| `x1` | `[fh + 4][fw + 4][32]` halo | FP8 | `l_linear_32_32_relu` | `l_linear_32_32_f16` |
| `x1` | `[fh + 4][fw + 4][32]` halo | FP8 (not stored by `l_linear_32_32_fused`) | `l_linear_32_32_relu` | `l_linear_32_32_f16` |
| `x2` | `[fh + 4][fw + 4][32]` halo | **FP16** (64 bytes per pixel) | `l_linear_32_32_f16` | `l_swin_block_c32_enc` |
| `x3` | `[fh][fw][32]` | FP8 | `l_swin_block_c32_enc` | `l_project_gather` |
| `enc0.features` | `[fh/2][fw/2][64]` | FP8 | `l_project_gather` | enc1 (Model M's stage) |
Expand Down Expand Up @@ -78,6 +78,7 @@ workgroup, in raster order of their map.
| `lev/input_encoder` | exists (GPU-checked) | as Model M's `input_encoder`, with `features` replaced by `q` | ceil(((fh + 4) / 2) ((fw + 4) / 2) / 128) (2 x 2 pixel groups of the halo map) | the 20 values and their dithered FP8 for every halo-map pixel, into `q`. The motion surfaces as Model M, only for pixels inside R. w = `enc0.values` |
| `l_linear_32_32_relu` | exists (GPU-checked) | `x, y, w, npix` | ceil(npix / 128), npix = (fh + 4)(fw + 4) | y = q8(max(0, RN16(b + W x))), FP8 -> FP8. w = `enc0.Win1` |
| `l_linear_32_32_f16` | exists (GPU-checked) | `x, y, w, npix` | as above | y = RN16(b + W x), FP8 -> **FP16**. w = `enc0.Win2` |
| `l_linear_32_32_fused` | exists (GPU-checked), D3D12 only | `x, y, w1, w2, npix` | as above | the two above in one kernel, `x1` kept in registers: the same bytes in `y` (`x2`), `x1` not stored. x = `q`, y = `x2` in two different buffers. w1 = `enc0.Win1`, w2 = `enc0.Win2`. The native D3D12 frame runs it in place of the two (fusion `ll`, on by default); without `ll` or without the kernel in the build, the two run |
| `l_swin_block_c32_enc[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | ceil(nwin / 8), nwin = ceil((W + ox) / 4) ceil((H + oy) / 4) | the encoder block. x = `x2` (FP16 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2)). out = `x3` (FP8, grid only). H = fh, W = fw. (ox, oy) = `window_offsets["first"]` for the frame parity. w = `enc0_block` |
| `l_project_gather` | exists (GPU-checked) | `x, features, w, tw, th` | ceil(tw th / 128), tw = fw / 2, th = fh / 2 | token (r, c) gathers pixels (2r + p / 2, 2c + p % 2), p = 0..3, as channels 32 p + i. Running FP16 sum per group of 32 from the bias, clamp +-6.28125, FP8 -> `enc0.features`. w = `enc0.project` |
| `l_dec_project` | exists (GPU-checked) | `z, y, w, tw, th` | ceil(tw th / 128) | per token of dec1's output z [th][tw][64]: 128 outputs (two RN16 groups of 32 inputs, then FP8). Output 32 s + k goes to pixel (2r + s / 2, 2c + s % 2), channel k, of `y`. w = `dec0.project` |
Expand Down Expand Up @@ -159,4 +160,6 @@ the frame parity, as Model M's.
- `clean/l/runner/l_d3d12_check.py` (with its D3D12 port): each D3D12 port against its HIP kernel, byte for byte.
- `clean/l/runner/l_fold_check.py`: `l_swin_block_c32_dec_coef12_nc` against `l_swin_block_c32_dec_nc` ->
`l_coef_pack12` on D3D12 and on HIP, byte for byte.
- `clean/l/runner/l_linfuse_check.py`: `l_linear_32_32_fused` against `l_linear_32_32_relu` -> `l_linear_32_32_f16`
on D3D12 and on HIP, byte for byte.
- One check per new kernel follows with that kernel.
203 changes: 203 additions & 0 deletions clean/l/runner/l_linfuse_check.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,203 @@
# SPDX-License-Identifier: MIT
"""Model L (issue #89): the fused input linears l_linear_32_32_fused (D3D12 only; clean/dx12/kernels/l_ops_linear.hlsl)
against the sequence it replaces, l_linear_32_32_relu -> l_linear_32_32_f16, byte for byte.

python clean/l/runner/l_linfuse_check.py --build <dir> --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --prepare
python clean/l/runner/l_linfuse_check.py --build <dir> --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --run
[--seeds N] --json out.json

--prepare (CPU) builds into <dir>: l_ops.co and swin.co (clean/l/runner/l_c32_check.py compile_kernels), the D3D12
kernels (clean/dx12/build_dx12.py --only) and kernel_replay.exe.
--run (GPU, one GPU slot): every case is a pixel map (npix pixels, partly filled waves and workgroups among them), an
FP8 input map q and the two blobs (enc0.Win1, enc0.Win2), at three weight scales (results past 448 and past FP16's
range, results in FP16's subnormal range) and with special FP8 codes in q (NaN, +-448, +-0, subnormals): the fused
kernel's conversions are the driver's, the pair's software. On the same input bytes:
- the sequence on D3D12 (kernel_replay, two passes: the relu linear writes x1, then the f16 linear reads that file);
- the sequence on HIP (in this process);
- the fused kernel on D3D12.
The rules are clean/l/runner/l_fold_check.py's: raw bytes; the output (x2, 64 bytes a pixel) has 64 bytes of slack and
starts filled with its run's own sentinel (HIP 0xA5, the D3D12 sequence 0x5A, the fused kernel 0x3C): every pixel byte
identical across the three, every slack byte its own run's sentinel; a negative control (the fused kernel with half its
workgroups) must be caught; Model M's swin_block_c96_nc runs through the same replay as a control.
Output: counts only, no timing.
"""
import argparse
import ctypes
import json
import os
import sys

import numpy as np

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE)))
sys.path.insert(0, ROOT)

from clean.l.runner import l_c32_check as C32 # noqa: E402
from clean.l.runner import l_d3d12_check as LD # noqa: E402
from clean.l.runner import l_d3d12_ops_check as OPS # noqa: E402
from clean.reference.fp8 import fp8_encode, q8 # noqa: E402
from clean.l.runner.l_fold_check import (CONTROL, D3D12_SENTINEL, FOLD_SENTINEL, HIP_SENTINEL, SLACK, compare, # noqa: E402
nwin_of, replay, write)

RELU, F16, FUSED = "l_linear_32_32_relu", "l_linear_32_32_f16", "l_linear_32_32_fused"
# (rows, cols) of the maps: the ops check's (a 35-pixel map among them: a partly filled wave), the halo maps of the
# runtime's smallest frames, 1 x 1, and one more workgroup than a whole number (129 pixels)
MAPS = OPS.MAPS + [(36, 36), (1, 1), (3, 43), (68, 132)]


def _blob(rng, scale, bias_scale):
"""enc0.Win1 / Win2 layout (bias tile vector of 32 floats, 4 fragments of 256 FP8 bytes) at a weight scale."""
bias = rng.normal(0, bias_scale, 32).astype(np.float32).tobytes()
return bias + fp8_encode(q8(rng.normal(0, scale, 4 * 256))).tobytes()


# FP8 codes the network does not produce but the conversions must treat as the pair does: both NaN codes, +-448, +-0,
# subnormals
SPECIAL = np.array([0x7f, 0xff, 0x7e, 0xfe, 0x00, 0x80, 0x01, 0x81, 0x07, 0x87], np.uint8)
# weight / bias scales: the ops check's; large (sums past 448 and past FP16's 65504: infinities); tiny (FP16 subnormal
# and zero results)
SCALES = [(0.25, 0.5), (24.0, 400.0), (2e-3, 1e-4)]


def cases(rng):
"""(npix, q bytes, Win1 blob, Win2 blob) of every case: each map at each weight scale, every other case with
special FP8 codes in a tenth of q's bytes."""
for ci, (rows, cols) in enumerate(MAPS):
n = rows * cols
for si, (s, b) in enumerate(SCALES):
q = np.frombuffer(OPS._fp8_map(rng, n * 32), np.uint8).copy()
if (ci + si) % 2:
pick = rng.random(q.size) < 0.1
q[pick] = rng.choice(SPECIAL, int(pick.sum()))
yield n, q.tobytes(), _blob(rng, s, b), _blob(rng, s, b)


def hip_sequence(g, ops, c):
"""relu, then f16, on HIP: x2's raw bytes (the output starts at HIP_SENTINEL)."""
n, q, w1, w2 = c
h = g.hip
owned = []

def buf(data):
p = h.malloc(max(len(data), 4))
h.upload(p, data)
owned.append(p)
return p

try:
pq, pw1, pw2 = buf(q), buf(w1), buf(w2)
px1, py = h.malloc(n * 32), h.malloc(n * 64 + SLACK)
owned += [px1, py]
h.memset(px1, HIP_SENTINEL, n * 32)
h.memset(py, HIP_SENTINEL, n * 64 + SLACK)
grid = -(-n // 128)
for k, a, b, wb in ((RELU, pq, px1, pw1), (F16, px1, py, pw2)):
h.launch(h.get_function(ops, k), grid, 256,
[ctypes.c_void_p(a), ctypes.c_void_p(b), ctypes.c_void_p(wb), ctypes.c_int(n)])
h.sync()
return bytes(h.download(py, n * 64 + SLACK))
finally:
for p in owned:
h.free(p)


def fused_lines(name, n, prefix="", grid=None):
return [f"case {prefix}{name}_fused", f"kernel {FUSED} {grid or -(-n // 128)} 256", f"arg buf {name}.q",
f"arg out {n * 64 + SLACK} {prefix}{name}.fy {FOLD_SENTINEL}", f"arg buf {name}.w1", f"arg buf {name}.w2",
f"arg u32 {n}", ""]


def run(build, rocm, seeds):
g = C32.Gpu(build, rocm)
with open(os.path.join(build, "l_ops.co"), "rb") as f:
ops = g.hip.load_module(f.read())
cdir = os.path.join(build, "replay-linfuse")
os.makedirs(cdir, exist_ok=True)
pass1, pass2, all_cases, neg = [], [], {}, None
for si, seed in enumerate(seeds):
rng = np.random.default_rng(seed)
for ci, c in enumerate(cases(rng)):
name = f"s{si}_{ci}"
n, q, w1, w2 = c
all_cases[name] = (n, hip_sequence(g, ops, c))
write(os.path.join(cdir, name + ".q"), q)
write(os.path.join(cdir, name + ".w1"), w1)
write(os.path.join(cdir, name + ".w2"), w2)
grid = -(-n // 128)
# pass 1: the D3D12 relu linear (x1) and the fused kernel
pass1 += [f"case {name}_relu", "module l_ops.co", f"kernel {RELU} {grid} 256", f"arg buf {name}.q",
f"arg out {n * 32} {name}.x1 {D3D12_SENTINEL}", f"arg buf {name}.w1", f"arg u32 {n}", ""]
pass1 += fused_lines(name, n)
if neg is None and grid >= 4:
neg = name
pass1 += fused_lines(name, n, prefix="negative_", grid=grid // 2)
# pass 2: the D3D12 f16 linear on that x1
pass2 += [f"case {name}_f16", "module l_ops.co", f"kernel {F16} {grid} 256", f"arg buf {name}.x1",
f"arg out {n * 64 + SLACK} {name}.sy {D3D12_SENTINEL}", f"arg buf {name}.w2", f"arg u32 {n}", ""]
rng = np.random.default_rng(seeds[0])
ctl = []
for ci, (H, W, off, xb, nout, blob) in enumerate(LD.block_cases(CONTROL, rng)):
name = f"control_{ci}"
ctl.append((name, LD.hip_block_raw(g, "swin.co", CONTROL, xb, blob, H, W, off, nout)))
write(os.path.join(cdir, name + ".x"), xb)
write(os.path.join(cdir, name + ".w"), blob)
pass1 += [f"case {name}", "module swin.co", f"kernel {CONTROL} {-(-nwin_of(H, W, off) // 8)} 256",
f"arg buf {name}.x", "arg null", f"arg out {nout} {name}.out {D3D12_SENTINEL}", f"arg buf {name}.w",
f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {off[0]}", f"arg u32 {off[1]}", ""]
e1, t1 = replay(build, cdir, pass1, "pass1")
e2, t2 = replay(build, cdir, pass2, "pass2") if e1 == 0 else (-1, "pass 1 failed")
res = {"replay_exit": [e1, e2], "replay_tail": [t1, t2], "cases": []}

def read(n):
p = os.path.join(cdir, n)
return open(p, "rb").read() if os.path.exists(p) else b""

for name, (n, hip) in all_cases.items():
fused, seq = read(f"{name}.fy"), read(f"{name}.sy")
r = {"case": name, "npix": n,
"fused_vs_d3d12_seq": compare(fused, seq, n * 64, FOLD_SENTINEL, D3D12_SENTINEL),
"fused_vs_hip_seq": compare(fused, hip, n * 64, FOLD_SENTINEL, HIP_SENTINEL),
"d3d12_seq_vs_hip_seq": compare(seq, hip, n * 64, D3D12_SENTINEL, HIP_SENTINEL)}
r["identical"] = all(r[k]["identical"] for k in ("fused_vs_d3d12_seq", "fused_vs_hip_seq", "d3d12_seq_vs_hip_seq"))
res["cases"].append(r)
caught = None
if neg:
n = all_cases[neg][0]
o = compare(read(f"negative_{neg}.fy"), read(f"{neg}.sy"), n * 64, FOLD_SENTINEL, D3D12_SENTINEL)
caught = o["differ"] >= 0 and not o["identical"]
res["negative_control"] = {"case": neg, "half_grid": True, "caught": caught, "differ": max(o["differ"], 0)}
ctl_ok = [read(f"{name}.out") == want for name, want in ctl]
res["control"] = {"kernel": CONTROL, "cases": len(ctl_ok), "identical": sum(ctl_ok)}
res["summary"] = {"cases": len(res["cases"]), "identical": sum(c["identical"] for c in res["cases"]),
"pixel_bytes": sum(c["npix"] * 64 for c in res["cases"])}
res["pass"] = (e1 == 0 and e2 == 0 and caught is True and res["summary"]["cases"] > 0 and
res["summary"]["identical"] == res["summary"]["cases"] and len(ctl_ok) > 0 and all(ctl_ok))
return res


def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--build", required=True)
ap.add_argument("--rocm", required=True)
ap.add_argument("--prepare", action="store_true")
ap.add_argument("--run", action="store_true")
ap.add_argument("--seeds", type=int, default=2)
ap.add_argument("--json")
a = ap.parse_args(argv)
os.makedirs(a.build, exist_ok=True)
if a.prepare:
C32.compile_kernels(a.build, a.rocm, with_swin=True)
print("built", LD.prepare_replay(a.build, a.rocm, [RELU, F16, FUSED, CONTROL]))
if a.run:
res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds)))
if a.json:
with open(a.json, "w") as f:
json.dump(res, f, indent=1)
print({k: v for k, v in res.items() if k not in ("cases", "replay_tail")})
return 0 if res["pass"] else 1
return 0


if __name__ == "__main__":
sys.exit(main())
Loading
Loading