diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index c44c4a4b..82a0a2c5 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -22,7 +22,7 @@ and rcp() (fdiv 1.0, x) is the hardware v_rcp_f32; it is reassembled (dxil_asm.cpp, DXC's IDxcAssembler) and signed by dxv. Any other float division ('/', which would be an IEEE division) or float remainder is refused: use wg_div. -VGPR check: every Swin / bottleneck kernel built (a name with a swin_ part) is run through RGA (gfx1201) and its +VGPR check: every Swin / bottleneck kernel built (a name with a swin_ or l_swin_ part) is run through RGA (gfx1201) and its USED_VGPRs compared with its bound in clean/dx12/vgpr_limits.json (the count that keeps its present workgroups per CU); over the bound, or no bound listed, fails the build. WG_DX12_VGPR_CHECK=0 skips it. """ @@ -201,7 +201,7 @@ def rga_isa(dxil, isa_dir, stem): def vgpr_checked(name): - return re.search(r"(^|/)swin_", name) is not None + return re.search(r"(^|/|l_)swin_", name) is not None # l_swin_*: Model L's blocks (#89) def rga_vgprs(dxil): diff --git a/clean/dx12/kernels/fused_swin.hlsl b/clean/dx12/kernels/fused_swin.hlsl index 752ac702..0576790d 100644 --- a/clean/dx12/kernels/fused_swin.hlsl +++ b/clean/dx12/kernels/fused_swin.hlsl @@ -449,6 +449,8 @@ float norm_r(v8f a[KC]) ss = ss + wg_xor16(ss); #if SW_C == 64 return wg_rsq((ss + asfloat(0x39000000u)) * asfloat(0x3c800000u)); // (ss + 2^-13) * (1 / 64) +#elif SW_C == 32 + return wg_rsq((ss + asfloat(0x39000000u)) * asfloat(0x3d000000u)); // (ss + 2^-13) * (1 / 32) (Model L, #89) #else return wg_rsq((ss + asfloat(0x39000000u)) * asfloat(0x3c2aaaabu)); // (ss + 2^-13) * (1 / 96) #endif diff --git a/clean/dx12/kernels/l_swin.hlsl b/clean/dx12/kernels/l_swin.hlsl new file mode 100644 index 00000000..d4e0f8e2 --- /dev/null +++ b/clean/dx12/kernels/l_swin.hlsl @@ -0,0 +1,7 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of Model L's C = 32 Swin block (clean/l/kernels/l_swin.hip, l_swin_block_c32_nc of l_swin.co; issue #89): +// swin_block.hlsl instantiated at C = 32, 2 heads, MLP 128 (the same template as the HIP kernel's), with HIP's +// transcendental GELU chain (SW_GELU 0, no table) and HIP's MLP unit cut (HPU 8 = all 8 hidden tiles in one unit). +// +// @wg-kernel l_swin_block_c32_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_ENTRY_OVERRIDE=l_swin_block_c32_nc +#include "swin_block.hlsl" diff --git a/clean/dx12/kernels/swin_block.hlsl b/clean/dx12/kernels/swin_block.hlsl index 6ec87fde..5f8e6a01 100644 --- a/clean/dx12/kernels/swin_block.hlsl +++ b/clean/dx12/kernels/swin_block.hlsl @@ -397,6 +397,8 @@ float norm_r(v8f a[KC]) ss = ss + wg_xor16(ss); #if SW_C == 64 return wg_rsq((ss + asfloat(0x39000000u)) * asfloat(0x3c800000u)); // (ss + 2^-13) * (1 / 64) +#elif SW_C == 32 + return wg_rsq((ss + asfloat(0x39000000u)) * asfloat(0x3d000000u)); // (ss + 2^-13) * (1 / 32) (Model L, #89) #else return wg_rsq((ss + asfloat(0x39000000u)) * asfloat(0x3c2aaaabu)); // (ss + 2^-13) * (1 / 96) #endif diff --git a/clean/dx12/tests/kernel_replay.cpp b/clean/dx12/tests/kernel_replay.cpp new file mode 100644 index 00000000..c66816a0 --- /dev/null +++ b/clean/dx12/tests/kernel_replay.cpp @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: MIT +// Kernel replay on the native D3D12 Device (issue #89): runs single kernels by their launch names on synthetic inputs +// given in files, so that a D3D12 port can be compared byte for byte with its HIP kernel on the same inputs (driver: +// clean/l/runner/l_d3d12_check.py). Uses only the project's Device API (clean/runtime/device.h). +// +// kernel_replay +// +// cases.txt, one case per block of lines (blank line between cases; paths relative to the cases file): +// case +// module [] (zero or more) +// kernel +// arg buf device buffer with the file's bytes +// arg out [fill] device buffer of that size, every byte set to fill (0..255, default 0) +// before the launch, written to afterwards +// arg null null pointer +// arg u32 32-bit scalar (decimal, may be negative) +// arg bytes a struct passed by value (the file's bytes) +#include "../../runtime/device.h" + +#include +#include +#include +#include +#include +#include +#include + +using namespace wgrt; + +namespace { + +bool read_file(const std::string& path, std::vector* out) { + std::ifstream f(path, std::ios::binary); + if (!f) return false; + out->assign(std::istreambuf_iterator(f), std::istreambuf_iterator()); + return true; +} + +bool write_file(const std::string& path, const std::vector& data) { + std::ofstream f(path, std::ios::binary); + if (!f) return false; + f.write(reinterpret_cast(data.data()), (std::streamsize)data.size()); + return (bool)f; +} + +struct Arg { + std::string kind, file; + size_t bytes = 0; + int64_t value = 0; + DevPtr ptr = 0; + uint32_t u = 0; + std::vector blob; +}; + +struct Case { + std::string name, kernel; + unsigned grid = 0, block = 0; + std::vector> modules; + std::vector args; +}; + +std::string dir_of(const std::string& path) { + const size_t s = path.find_last_of("/\\"); + return s == std::string::npos ? std::string(".") : path.substr(0, s); +} + +int run_case(Device* d, Case& c, const std::string& base) { + for (auto& m : c.modules) { + const bool ok = m.second.empty() ? d->load_module(m.first) : d->load_module_tagged(m.first, m.second); + if (!ok) { std::printf("%s: module %s: %s\n", c.name.c_str(), m.first.c_str(), d->err().c_str()); return 1; } + } + std::vector argv; + std::vector owned; + for (auto& a : c.args) { + if (a.kind == "buf") { + std::vector data; + if (!read_file(base + "/" + a.file, &data)) { std::printf("%s: cannot read %s\n", c.name.c_str(), a.file.c_str()); return 1; } + if (!d->alloc(&a.ptr, data.size() ? data.size() : 4) || !d->upload(a.ptr, data.data(), data.size())) { + std::printf("%s: buffer %s: %s\n", c.name.c_str(), a.file.c_str(), d->err().c_str()); + return 1; + } + owned.push_back(a.ptr); + argv.push_back(&a.ptr); + } else if (a.kind == "out") { + if (!d->alloc(&a.ptr, a.bytes) || !d->memset(a.ptr, (int)(a.value & 0xff), a.bytes)) { + std::printf("%s: output: %s\n", c.name.c_str(), d->err().c_str()); + return 1; + } + owned.push_back(a.ptr); + argv.push_back(&a.ptr); + } else if (a.kind == "null") { + a.ptr = 0; + argv.push_back(&a.ptr); + } else if (a.kind == "u32") { + a.u = (uint32_t)a.value; + argv.push_back(&a.u); + } else if (a.kind == "bytes") { + if (!read_file(base + "/" + a.file, &a.blob)) { std::printf("%s: cannot read %s\n", c.name.c_str(), a.file.c_str()); return 1; } + argv.push_back(a.blob.data()); + } + } + if (!d->launch(c.kernel, c.grid, c.block, argv.data())) { + std::printf("%s: launch %s: %s\n", c.name.c_str(), c.kernel.c_str(), d->err().c_str()); + return 1; + } + if (!d->sync()) { std::printf("%s: sync: %s\n", c.name.c_str(), d->err().c_str()); return 1; } + for (auto& a : c.args) { + if (a.kind != "out") continue; + std::vector data(a.bytes); + if (!d->download(data.data(), a.ptr, a.bytes) || !write_file(base + "/" + a.file, data)) { + std::printf("%s: download %s: %s\n", c.name.c_str(), a.file.c_str(), d->err().c_str()); + return 1; + } + } + for (DevPtr p : owned) d->free(p); + std::printf("%s: ok (%s)\n", c.name.c_str(), c.kernel.c_str()); + return 0; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 3) { std::printf("usage: kernel_replay \n"); return 2; } + std::string err; + auto d = make_d3d12_device(argv[1], &err); + if (!d) { std::printf("device: %s\n", err.c_str()); return 1; } + std::printf("device: %s\n", d->name().c_str()); + std::ifstream f(argv[2]); + if (!f) { std::printf("cannot read %s\n", argv[2]); return 2; } + const std::string base = dir_of(argv[2]); + std::vector cases; + std::string line; + while (std::getline(f, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + std::istringstream s(line); + std::string w; + if (!(s >> w)) continue; + if (w == "case") { cases.emplace_back(); s >> cases.back().name; continue; } + if (cases.empty()) { std::printf("line before the first case: %s\n", line.c_str()); return 2; } + Case& c = cases.back(); + if (w == "module") { + std::string m, tag; + s >> m >> tag; + c.modules.emplace_back(m, tag); + } else if (w == "kernel") { + s >> c.kernel >> c.grid >> c.block; + } else if (w == "arg") { + Arg a; + s >> a.kind; + if (a.kind == "buf" || a.kind == "bytes") s >> a.file; + else if (a.kind == "out") { + s >> a.bytes >> a.file; + if (!(s >> a.value)) a.value = 0; // the fill byte (a sentinel), default 0 + } + else if (a.kind == "u32") s >> a.value; + else if (a.kind != "null") { std::printf("unknown arg kind %s\n", a.kind.c_str()); return 2; } + c.args.push_back(a); + } else { + std::printf("unknown line: %s\n", line.c_str()); + return 2; + } + } + int fails = 0; + for (auto& c : cases) fails += run_case(d.get(), c, base); + if (!d->sync()) { std::printf("final sync: %s\n", d->err().c_str()); return 1; } + std::printf("kernel_replay: %d of %zu cases failed\n", fails, cases.size()); + return fails ? 1 : 0; +} diff --git a/clean/dx12/vgpr_limits.json b/clean/dx12/vgpr_limits.json index 8db13de3..d4251e04 100644 --- a/clean/dx12/vgpr_limits.json +++ b/clean/dx12/vgpr_limits.json @@ -1,5 +1,6 @@ { "_doc": "build_dx12.py VGPR check: the RGA (gfx1201) USED_VGPRs bound of every Swin / bottleneck kernel (a name with a swin_ part). Each bound is the largest count that keeps the kernel's present occupancy: VGPR file 1536 per SIMD lane, granule 8, a 256-lane wave32 workgroup = 4 waves on each of a CU's 2 SIMDs, so 2 workgroups per CU need <= 192, 3 need <= 128 (a 128-lane workgroup = 2 waves per SIMD: 4 per CU need <= 192). A kernel over its bound fails the build (a merge or an edit lost a workgroup per CU: swin_block_c96_add_xe_* at 215 VGPRs ran 1 per CU, +38 us per frame at 1440p Q). A new Swin / bottleneck kernel needs an entry. WG_DX12_VGPR_CHECK=0 skips the check.", + "l_swin_block_c32_nc": 96, "swin_block_c64_nc": 192, "swin_block_c96_nc": 192, "swin_block_c96_add_ex_nc": 192, diff --git a/clean/l/runner/l_d3d12_check.py b/clean/l/runner/l_d3d12_check.py new file mode 100644 index 00000000..83f0ec15 --- /dev/null +++ b/clean/l/runner/l_d3d12_check.py @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): each D3D12 port against its HIP kernel on the same synthetic inputs, byte for byte. + + python clean/l/runner/l_d3d12_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --prepare + python clean/l/runner/l_d3d12_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --run + [--seeds N] --json out.json + +--prepare (CPU) builds into : the HIP code objects (l_swin.co), the D3D12 ports (clean/dx12/build_dx12.py +--only , needs WG_TOOLCHAINS / WG_DXC_DIR as BUILDING.md describes) and +clean/dx12/tests/kernel_replay.exe (host clang++ of the HIP SDK). +--run (GPU, one GPU slot) runs every case on HIP in this process, writes the same inputs for kernel_replay, runs it on +the native D3D12 device in a child process, and compares the outputs: the RAW downloaded bytes of both (nothing is +decoded and encoded again), each output buffer filled beforehand with its own non-zero sentinel (HIP 0xA5, D3D12 +0x5A), so a byte either path leaves unwritten shows up as a difference. Output: counts only, no timing. + +Kernels: l_swin_block_c32_nc (the D3D12 port has only the `_nc` form, as Model M's ports). +""" +import argparse +import ctypes +import json +import os +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import oracle as L # noqa: E402 +from clean.l.runner import l_c32_check as C32 # noqa: E402 +from clean.reference.fp8 import fp8_encode, q8 # noqa: E402 +from clean.runner import packing as P # noqa: E402 + +D3D12_KERNELS = ["l_swin_block_c32_nc", "native_fill"] # native_fill: the device's memset +# Control: Model M's swin_block_c96_nc, whose port is already proven byte for byte (clean/dx12/compare.py on whole +# frames): if it matches here too, the harness is sound. +CONTROL_KERNELS = ["swin_block_c96_nc"] +CASES = C32.CASES + [(96, 160, (2, 2)), (240, 432, (0, 2))] + + +def prepare(build, rocm): + from clean.runner.hip import hip_child_environment + C32.compile_kernels(build, rocm, with_swin=True) + env, _ = hip_child_environment(rocm_dir=rocm) + subprocess.check_call([sys.executable, os.path.join(ROOT, "clean", "dx12", "build_dx12.py"), "--out", build, + "--only", ",".join(D3D12_KERNELS + CONTROL_KERNELS), "-j", "1"], env=env, cwd=ROOT) + rt = os.path.join(ROOT, "clean", "runtime") + conv = os.path.join(ROOT, "clean", "converter", "src") + exe = os.path.join(build, "kernel_replay.exe") + cmd = [os.path.join(rocm, "bin", "clang++.exe"), "-std=c++17", "-O2", "-D_CRT_SECURE_NO_WARNINGS", + "-fms-runtime-lib=static", os.path.join(ROOT, "clean", "dx12", "tests", "kernel_replay.cpp"), + os.path.join(rt, "device.cpp"), os.path.join(rt, "device_d3d12.cpp"), os.path.join(rt, "embedded.cpp"), + os.path.join(conv, "json.cpp"), os.path.join(conv, "sha256.cpp"), "-o", exe] + subprocess.check_call(cmd, env=env, cwd=build) + return exe + + +def _write(path, data): + with open(path, "wb") as f: + f.write(data) + + +def _params_c32(rng): + p = L.random_block_params(rng, 32, 2, 128) + blob = P.pack_block(p, 32, 2, 128, P.resolve_d_row("gfx12")) + return blob + bytes((-len(blob)) % P.ALIGN) + + +def _params_c96(rng): + from clean.reference import swin as MS + return P.pack_stage("enc2", MS.random_params("enc2", rng))[:P.stage_sections("enc2")[1]] + + +KINDS = {"l_swin_block_c32_nc": ("l_swin.co", 32, _params_c32), "swin_block_c96_nc": ("swin.co", 96, _params_c96)} + +# Each back end's output buffer starts filled with its own non-zero sentinel byte, different from the other's: a byte +# that one path (or both) never writes then differs between the two outputs instead of comparing equal as zeros. +HIP_SENTINEL, D3D12_SENTINEL = 0xA5, 0x5A + + +def hip_block_raw(g, co, kernel, xb, blob, H, W, off, C): + """The block on HIP; returns the output buffer's downloaded bytes as they are (never decoded and encoded again, so + for example the two FP8 NaN codes stay distinct). The output starts filled with HIP_SENTINEL.""" + h = g.hip + n = H * W * C + px, pw, po = h.malloc(len(xb)), h.malloc(len(blob)), h.malloc(n) + try: + h.upload(px, xb) + h.upload(pw, blob) + h.memset(po, HIP_SENTINEL, n) + ox, oy = off + nwin = -(-(W + ox) // 4) * -(-(H + oy) // 4) + args = [ctypes.c_void_p(px), ctypes.c_void_p(0), ctypes.c_void_p(po), ctypes.c_void_p(pw), + ctypes.c_int(H), ctypes.c_int(W), ctypes.c_int(ox), ctypes.c_int(oy)] + h.launch(h.get_function(g.mods[co], kernel), -(-nwin // 8), 256, args) + h.sync() + return bytes(h.download(po, n)) + finally: + for p in (px, pw, po): + h.free(p) + + +def run(build, rocm, seeds, kernels): + g = C32.Gpu(build, rocm) + cdir = os.path.join(build, "replay") + os.makedirs(cdir, exist_ok=True) + lines, hip_out = [], {} + neg = None # (case name, kernel) of the negative control + for kernel in kernels: + co, C, gen = KINDS[kernel] + for si, seed in enumerate(seeds): + rng = np.random.default_rng(seed) + for ci, (H, W, off) in enumerate(CASES): + name = f"{kernel}_s{si}_{ci}" + blob = gen(rng) + if not np.frombuffer(blob[:4], np.uint32)[0] & 1: + continue # only the _nc forms are ported + x = q8(rng.normal(0, 1.5, (H, W, C))) + xb = fp8_encode(x).tobytes() + hip_out[name] = (kernel, hip_block_raw(g, co, kernel, xb, blob, H, W, off, C)) + _write(os.path.join(cdir, name + ".x"), xb) + _write(os.path.join(cdir, name + ".w"), blob) + ox, oy = off + nwin = -(-(W + ox) // 4) * -(-(H + oy) // 4) + lines += [f"case {name}", f"module {co}", f"kernel {kernel} {-(-nwin // 8)} 256", + f"arg buf {name}.x", "arg null", f"arg out {H * W * C} {name}.out {D3D12_SENTINEL}", f"arg buf {name}.w", + f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {ox}", f"arg u32 {oy}", ""] + if neg is None and nwin > 8: + # Negative control: this case again on D3D12 with half its workgroups. The windows never processed + # keep the D3D12 sentinel, so the comparison must find them: the check can fail. + neg = (name, kernel) + lines += [f"case negative_{name}", f"module {co}", f"kernel {kernel} {max(1, -(-nwin // 8) // 2)} 256", + f"arg buf {name}.x", "arg null", f"arg out {H * W * C} negative_{name}.out {D3D12_SENTINEL}", + f"arg buf {name}.w", f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {ox}", f"arg u32 {oy}", ""] + with open(os.path.join(cdir, "cases.txt"), "w", newline="\n") as f: + f.write("\n".join(lines)) + env = {k: v for k, v in os.environ.items() + if not any(s in k.upper() for s in ("HIP", "LLVM", "ROCM", "HSA")) or k.upper() in ("HIP_PLATFORM",)} + env["WG_D3D12_ARENA_MB"] = "512" + r = subprocess.run([os.path.join(build, "kernel_replay.exe"), build, os.path.join(cdir, "cases.txt")], env=env, + capture_output=True, text=True, timeout=150, creationflags=0x08000000) + res = {"replay_exit": r.returncode, "replay_tail": r.stdout[-600:] + r.stderr[-300:], "cases": []} + for name, (kernel, want) in hip_out.items(): + path = os.path.join(cdir, name + ".out") + got = open(path, "rb").read() if os.path.exists(path) else b"" + a = np.frombuffer(got, np.uint8) if got else np.zeros(0, np.uint8) + b = np.frombuffer(want, np.uint8) + res["cases"].append({"case": name, "kernel": kernel, "bytes": len(want), + "identical": got == want, + "differ": int(np.sum(a != b)) if a.size == b.size else -1, + # bytes still equal to their buffer's sentinel (a value the kernel may also write) + "hip_sentinel_bytes": int(np.sum(b == HIP_SENTINEL)), + "d3d12_sentinel_bytes": int(np.sum(a == D3D12_SENTINEL)) if a.size else -1}) + res["per_kernel"] = {k: {"cases": sum(c["kernel"] == k for c in res["cases"]), + "identical": sum(c["identical"] for c in res["cases"] if c["kernel"] == k)} + for k in kernels} + caught = None + if neg: + path = os.path.join(cdir, "negative_" + neg[0] + ".out") + got = open(path, "rb").read() if os.path.exists(path) else b"" + want = hip_out[neg[0]][1] + a, b = np.frombuffer(got, np.uint8), np.frombuffer(want, np.uint8) + caught = len(got) == len(want) and got != want + res["negative_control"] = {"case": neg[0], "kernel": neg[1], "half_grid": True, "caught": caught, + "differ": int(np.sum(a != b)) if a.size == b.size else -1, + "d3d12_sentinel_bytes": int(np.sum(a == D3D12_SENTINEL)) if a.size else -1} + res["pass"] = r.returncode == 0 and caught is not False and all(v["cases"] > 0 and v["identical"] == v["cases"] + for v in res["per_kernel"].values()) + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--prepare", action="store_true") + ap.add_argument("--run", action="store_true") + ap.add_argument("--seeds", type=int, default=2) + ap.add_argument("--no-control", action="store_true", help="skip Model M's swin_block_c96_nc control") + ap.add_argument("--json") + a = ap.parse_args(argv) + os.makedirs(a.build, exist_ok=True) + if a.prepare: + print("built", prepare(a.build, a.rocm)) + if a.run: + kernels = ["l_swin_block_c32_nc"] + ([] if a.no_control else CONTROL_KERNELS) + res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds)), kernels) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + print({k: v for k, v in res.items() if k not in ("cases", "replay_tail")}) + return 0 if res["pass"] else 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main())