Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 54 additions & 0 deletions clean/dx12/kernels/l_ops_linear.hlsl
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
// SPDX-License-Identifier: MIT
// D3D12 ports of Model L's input linears (issue #89): l_linear_32_32_relu and l_linear_32_32_f16 of l_ops.co
// (clean/l/kernels/l_ops.hip, linear_32_32<RELU>), byte for byte. Maps and blobs: clean/l/FORMATS.md.
//
// One 32 -> 32 linear on a pixel map: acc = b + W x (the bias tile vector as the accumulator start, one WMMA per K block
// of 16 inputs, two K blocks), then RN16. relu: y = q8(max(0, acc)), FP8 out; f16: y = acc, FP16 out. 256 lanes; wave i
// handles pixels 16 i .. 16 i + 15 of its workgroup's 128, lane (t, h) pixel 16 i + t, output channels 16 h + 8 mt + v.
// Blob: bias tile vector (rows R_L(32)), then fragments [mt < 2][kc < 2], rows R_L(32), K order K_L(32).
// No branch at all: a lane past the end of the map (HIP: `continue`, no store) computes pixel npix - 1 (its input is
// clamped there, as HIP's) and stores it to pixel npix - 1, the same bytes the lane of that pixel stores. A branch around
// the store let the D3D12 compiler sink the matrix instructions into it, where they ran with part of the wave masked
// off: wrong values in a partly filled wave (a 35-pixel map, pixels 32..34; found by l_d3d12_ops_check.py).
//
// @wg-kernel l_linear_32_32_relu co=l_ops.co block=256 wave=32 args=p,p,p,u defines=LIN_RELU=1
// @wg-kernel l_linear_32_32_f16 co=l_ops.co block=256 wave=32 args=p,p,p,u defines=LIN_RELU=0
#include "wg_platform.hlsli"

WG_ARGS_BEGIN
uint x, y, w; // pointers: x [npix][32] FP8, y [npix][32] FP8 or FP16, w the blob (enc0.Win1 / enc0.Win2)
int npix;
WG_ARGS_END

#if LIN_RELU
WG_KERNEL(l_linear_32_32_relu)
#else
WG_KERNEL(l_linear_32_32_f16)
#endif
{
WG_SETUP();
const uint tid = wg_lane_id(), lane = tid & 31u, hh = (tid >> 4) & 1u;
const int pix = (int)(wg_block_id() * (WG_BLOCK / WG_WAVE) + (tid >> 5)) * 16 + (int)(tid & 15u);
const int pk = select(pix < npix, pix, npix - 1);
// every load the matrix instructions need, before the first of them
uint2 xf[2];
[unroll] for (uint kc = 0; kc < 2u; ++kc) xf[kc] = wg_ld64(x, (uint)pk * 32u + hh * 16u + 8u * kc);
v8f bias[2];
uint2 fr[2][2];
[unroll] for (uint mt = 0; mt < 2u; ++mt) {
bias[mt] = wg_load8f(w, (hh * 8u + mt * 16u) * 4u);
[unroll] for (uint kc2 = 0; kc2 < 2u; ++kc2) fr[mt][kc2] = wg_ld64(w, 128u + (mt * 2u + kc2) * 256u + lane * 8u);
}
[unroll] for (uint m = 0; m < 2u; ++m) {
v8f acc = bias[m];
acc = wg_wmma_fp8(fr[m][0], xf[0], acc);
acc = wg_wmma_fp8(fr[m][1], xf[1], acc);
[unroll] for (uint i = 0; i < 8u; ++i) acc.v[i] = wg_rn16(acc.v[i]);
#if LIN_RELU
[unroll] for (uint j = 0; j < 8u; ++j) acc.v[j] = wg_fmax(acc.v[j], 0.0f);
wg_st64(y, (uint)pk * 32u + hh * 16u + 8u * m, wg_pack_q8(acc));
#else
wg_st128(y, ((uint)pk * 32u + hh * 16u + 8u * m) * 2u, wg_to_f16(acc));
#endif
}
}
10 changes: 8 additions & 2 deletions clean/l/runner/l_d3d12_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,17 @@


def prepare(build, rocm):
from clean.runner.hip import hip_child_environment
C32.compile_kernels(build, rocm, with_swin=True)
return prepare_replay(build, rocm, D3D12_KERNELS + CONTROL_KERNELS)


def prepare_replay(build, rocm, kernels):
"""The D3D12 ports `kernels` (and native_fill, the device's memset) into <build>/dx12, and kernel_replay.exe."""
from clean.runner.hip import hip_child_environment
env, _ = hip_child_environment(rocm_dir=rocm)
names = list(kernels) + ([] if "native_fill" in kernels else ["native_fill"])
subprocess.check_call([sys.executable, os.path.join(ROOT, "clean", "dx12", "build_dx12.py"), "--out", build,
"--only", ",".join(D3D12_KERNELS + CONTROL_KERNELS), "-j", "1"], env=env, cwd=ROOT)
"--only", ",".join(names), "-j", "1"], env=env, cwd=ROOT)
rt = os.path.join(ROOT, "clean", "runtime")
conv = os.path.join(ROOT, "clean", "converter", "src")
exe = os.path.join(build, "kernel_replay.exe")
Expand Down
235 changes: 235 additions & 0 deletions clean/l/runner/l_d3d12_ops_check.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,235 @@
# SPDX-License-Identifier: MIT
"""Model L (issue #89): the D3D12 ports of the l_ops.co kernels against their HIP kernels on the same synthetic inputs,
byte for byte (clean/l/kernels/l_ops.hip; ports clean/dx12/kernels/l_ops_*.hlsl).

python clean/l/runner/l_d3d12_ops_check.py --build <dir> --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --prepare
python clean/l/runner/l_d3d12_ops_check.py --build <dir> --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --run
[--seeds N] [--kernels a,b] --json out.json

--prepare (CPU) builds into <dir>: l_ops.co (clean/l/runner/l_c32_check.py compile_kernels), the D3D12 ports
(clean/dx12/build_dx12.py --only, with WG_TOOLCHAINS / WG_DXC_DIR as BUILDING.md describes) and kernel_replay.exe.
--run (GPU, one GPU slot) runs every case on HIP in this process and the same inputs through kernel_replay on the native
D3D12 device in a child process, then compares every output buffer. The rules (as clean/l/runner/l_d3d12_check.py):
- the RAW downloaded bytes of both back ends are compared (nothing is decoded and encoded again);
- each output buffer starts filled with its back end's own non-zero sentinel (HIP 0xA5, D3D12 0x5A), so a byte that
either path, or both, leaves unwritten differs;
- a negative control (one case again on D3D12 with half its workgroups) must be caught;
- Model M's swin_block_c96_nc (swin.co, its port already proven) runs through the same engine as a control
(--no-control skips it).
Output: counts only, no timing.
"""
import argparse
import ctypes
import json
import os
import subprocess
import sys

import numpy as np

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE)))
sys.path.insert(0, ROOT)

from clean.l.runner import l_c32_check as C32 # noqa: E402
from clean.l.runner.l_d3d12_check import D3D12_SENTINEL, HIP_SENTINEL, prepare_replay # noqa: E402
from clean.reference.fp8 import fp8_encode, q8 # noqa: E402

# (rows, cols) of the pixel maps; for the halo maps of the encoder these are (fh + 4, fw + 4) sizes
MAPS = [(16, 24), (36, 68), (8, 8), (68, 100), (5, 7)]


def _fp8_map(rng, n, scale=1.5):
return fp8_encode(q8(rng.normal(0, scale, n))).tobytes()


def _linear_blob(rng):
"""enc0.Win1 / enc0.Win2 layout: bias tile vector (32 floats), then 4 fragments of 256 FP8 bytes."""
bias = (rng.normal(0, 0.5, 32)).astype(np.float32).tobytes()
frags = fp8_encode(q8(rng.normal(0, 0.25, 4 * 256))).tobytes()
return bias + frags


def linear_cases(kernel, rng):
out_bytes_per = 32 if kernel == "l_linear_32_32_relu" else 64
cases = []
for (rows, cols) in MAPS:
n = rows * cols
cases.append({"kernel": kernel, "grid": -(-n // 128), "block": 256,
"args": [("buf", _fp8_map(rng, n * 32)), ("out", n * out_bytes_per), ("buf", _linear_blob(rng)),
("u32", n)]})
return cases


def control_cases(kernel, rng):
"""Control: Model M's swin_block_c96_nc (swin.co), whose D3D12 port is already proven byte for byte (whole frames,
clean/dx12/compare.py), through this harness's engine: if it is identical here too, the harness is sound."""
from clean.l.runner.l_d3d12_check import _params_c96
cases = []
for (H, W, (ox, oy)) in C32.CASES:
blob = _params_c96(rng)
if not np.frombuffer(blob[:4], np.uint32)[0] & 1:
continue # only the _nc form is ported
nwin = -(-(W + ox) // 4) * -(-(H + oy) // 4)
cases.append({"kernel": kernel, "co": "swin.co", "grid": -(-nwin // 8), "block": 256,
"args": [("buf", _fp8_map(rng, H * W * 96)), ("null",), ("out", H * W * 96), ("buf", blob),
("u32", H), ("u32", W), ("u32", ox), ("u32", oy)]})
return cases


KINDS = {"l_linear_32_32_relu": linear_cases, "l_linear_32_32_f16": linear_cases}
CONTROL_KINDS = {"swin_block_c96_nc": control_cases}
CO = "l_ops.co"


def prepare(build, rocm, kernels):
C32.compile_kernels(build, rocm, with_swin=True) # swin.co (the control); l_ops.co: with #115's compile_kernels
return prepare_replay(build, rocm, kernels)


def hip_run(g, mod, case):
"""One case on HIP: the outputs' downloaded bytes, as they are. Outputs start filled with HIP_SENTINEL."""
h = g.hip
args, owned, outs = [], [], []
for a in case["args"]:
if a[0] == "buf":
p = h.malloc(max(len(a[1]), 4))
h.upload(p, a[1])
owned.append(p)
args.append(ctypes.c_void_p(p))
elif a[0] == "out":
p = h.malloc(a[1])
h.memset(p, HIP_SENTINEL, a[1])
owned.append(p)
outs.append((p, a[1]))
args.append(ctypes.c_void_p(p))
elif a[0] == "null":
args.append(ctypes.c_void_p(0))
elif a[0] == "u32":
args.append(ctypes.c_int(a[1]))
elif a[0] == "bytes":
args.append((ctypes.c_ubyte * len(a[1])).from_buffer_copy(a[1]))
try:
h.launch(h.get_function(mod, case["kernel"]), case["grid"], case["block"], args)
h.sync()
return [bytes(h.download(p, n)) for p, n in outs]
finally:
for p in owned:
h.free(p)


def replay_lines(name, case, cdir, grid=None, prefix=""):
lines = [f"case {prefix}{name}", f"module {case.get('co', CO)}",
f"kernel {case['kernel']} {grid or case['grid']} {case['block']}"]
oi = 0
for i, a in enumerate(case["args"]):
if a[0] in ("buf", "bytes"):
f = f"{name}.a{i}"
if not prefix:
with open(os.path.join(cdir, f), "wb") as fh:
fh.write(a[1])
lines.append(f"arg {a[0]} {f}")
elif a[0] == "out":
lines.append(f"arg out {a[1]} {prefix}{name}.o{oi} {D3D12_SENTINEL}")
oi += 1
elif a[0] == "null":
lines.append("arg null")
elif a[0] == "u32":
lines.append(f"arg u32 {a[1]}")
return lines + [""]


def run(build, rocm, seeds, kernels):
g = C32.Gpu(build, rocm)
mods = {}

def module(co):
if co not in mods:
with open(os.path.join(build, co), "rb") as f:
mods[co] = g.hip.load_module(f.read())
return mods[co]

kinds = dict(KINDS, **CONTROL_KINDS)
cdir = os.path.join(build, "replay-ops")
os.makedirs(cdir, exist_ok=True)
lines, hip_out, neg = [], {}, None
for kernel in kernels:
for si, seed in enumerate(seeds):
rng = np.random.default_rng(seed)
for ci, case in enumerate(kinds[kernel](kernel, rng)):
name = f"{kernel}_s{si}_{ci}"
hip_out[name] = (kernel, hip_run(g, module(case.get("co", CO)), case))
for oi, data in enumerate(hip_out[name][1]): # kept next to the D3D12 outputs, for a look at a difference
with open(os.path.join(cdir, f"{name}.hip.o{oi}"), "wb") as fh:
fh.write(data)
lines += replay_lines(name, case, cdir)
if neg is None and case["grid"] >= 4:
neg = name
lines += replay_lines(name, case, cdir, grid=case["grid"] // 2, prefix="negative_")
with open(os.path.join(cdir, "cases.txt"), "w", newline="\n") as f:
f.write("\n".join(lines))
env = {k: v for k, v in os.environ.items()
if not any(s in k.upper() for s in ("HIP", "LLVM", "ROCM", "HSA")) or k.upper() in ("HIP_PLATFORM",)}
env["WG_D3D12_ARENA_MB"] = "512"
r = subprocess.run([os.path.join(build, "kernel_replay.exe"), build, os.path.join(cdir, "cases.txt")], env=env,
capture_output=True, text=True, timeout=150, creationflags=0x08000000)
res = {"replay_exit": r.returncode, "replay_tail": r.stdout[-600:] + r.stderr[-300:], "cases": []}

def read(path):
return open(path, "rb").read() if os.path.exists(path) else b""

for name, (kernel, wants) in hip_out.items():
c = {"case": name, "kernel": kernel, "outputs": []}
for oi, want in enumerate(wants):
got = read(os.path.join(cdir, f"{name}.o{oi}"))
a, b = np.frombuffer(got, np.uint8), np.frombuffer(want, np.uint8)
c["outputs"].append({"bytes": len(want), "identical": got == want,
"differ": int(np.sum(a != b)) if a.size == b.size else -1,
"hip_sentinel_bytes": int(np.sum(b == HIP_SENTINEL)),
"d3d12_sentinel_bytes": int(np.sum(a == D3D12_SENTINEL)) if a.size else -1})
c["identical"] = all(o["identical"] for o in c["outputs"])
res["cases"].append(c)
caught = None
if neg:
wants = hip_out[neg][1]
gots = [read(os.path.join(cdir, f"negative_{neg}.o{oi}")) for oi in range(len(wants))]
caught = all(len(gt) == len(w) for gt, w in zip(gots, wants)) and any(gt != w for gt, w in zip(gots, wants))
res["negative_control"] = {"case": neg, "half_grid": True, "caught": caught,
"differ": sum(int(np.sum(np.frombuffer(gt, np.uint8) != np.frombuffer(w, np.uint8)))
for gt, w in zip(gots, wants) if len(gt) == len(w))}
res["per_kernel"] = {k: {"cases": sum(c["kernel"] == k for c in res["cases"]),
"identical": sum(c["identical"] for c in res["cases"] if c["kernel"] == k),
"bytes": sum(o["bytes"] for c in res["cases"] if c["kernel"] == k for o in c["outputs"])}
for k in kernels}
res["pass"] = r.returncode == 0 and caught is True and all(v["cases"] > 0 and v["identical"] == v["cases"]
for v in res["per_kernel"].values())
return res


def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--build", required=True)
ap.add_argument("--rocm", required=True)
ap.add_argument("--prepare", action="store_true")
ap.add_argument("--run", action="store_true")
ap.add_argument("--seeds", type=int, default=2)
ap.add_argument("--kernels", default=",".join(KINDS))
ap.add_argument("--no-control", action="store_true", help="skip Model M's swin_block_c96_nc control")
ap.add_argument("--json")
a = ap.parse_args(argv)
kernels = [k for k in a.kernels.split(",") if k] + ([] if a.no_control else list(CONTROL_KINDS))
os.makedirs(a.build, exist_ok=True)
if a.prepare:
print("built", prepare(a.build, a.rocm, kernels))
if a.run:
res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds)), kernels)
if a.json:
with open(a.json, "w") as f:
json.dump(res, f, indent=1)
print({k: v for k, v in res.items() if k not in ("cases", "replay_tail")})
return 0 if res["pass"] else 1
return 0


if __name__ == "__main__":
sys.exit(main())
Loading