diff --git a/clean/README.md b/clean/README.md index e322ce6a..a329289d 100644 --- a/clean/README.md +++ b/clean/README.md @@ -133,6 +133,34 @@ objects and the converter map into `build`; the FidelityFX front end loads it (c other than `gfx12`, set `WG_D_ROW` to the reported table (for example `set WG_D_ROW=[[0,2,4,...],[1,3,...]]`) before running. +## Linux (gfx1100 and gfx1201) + +`--rocm` names an SDK in the layout of AMD's installer: `bin/clang`, the device library in `amdgcn/bitcode` and +`libamdhip64.so` in `lib`. A distribution's ROCm that keeps these elsewhere is used through a folder of links in that +layout. + +``` +python clean/runner/build.py --rocm --out build --no-runtime --no-model-k # gfx1201, all code objects +python clean/runner/build.py --arch gfx1100 --rocm --out build-gfx1100 # the gfx11 default sources +python clean/runner/build.py --arch gfx1100 --rocm --out build-gfx1100 --sources post.hip frame.hip +``` + +Targets are matched by exact name (`runner/family.py`): gfx1200 and gfx1201 are gfx12, gfx1100 and gfx1101 are gfx11, +anything else is refused. A gfx11 build compiles the Model M kernels of `FAMILY_SOURCES` in `build.py` (with `--variants`, the +swin variants of the family) and no Model K or L kernels, post_w64 or motion variants or resource gate; `--sources` builds a +chosen list for either family. Both write `target.json` next to the code objects, and `Runner` takes its default accumulator +row table from it (`WG_D_ROW` and the `d_row` argument still win). Unless `--no-runtime` or `--sources` is given, a gfx11 build +also makes `libwg_runtime.so` (nothing built in: it reads its code objects from the folder as `.`, copies of the +files that the build writes next to them, and the runtime takes the row table of its device's target; `--runtime-only` does +this step alone). The runtime's HIP gate takes the same four targets (`runtime/target_family.h`). + +On a gfx11 card (headless, one GPU job at a time): `python clean/runner/selftest.py --build --rocm ` (the +half-wave exchange and the software FP8 conversions; the matrix and hardware-conversion probes are gfx12 only), +`WG_GPU_TEST=1 WG_BUILD= WG_ROCM= python -m unittest clean.tests.test_gpu_stages clean.tests.test_gpu_q8` (the +stages without matrix instructions, and the conversions on a set of input chunks) and +`python clean/runner/q8_exhaustive.py --build --rocm ` (the conversions over all 2^32 FP32 patterns, about two +minutes). + ## Spec team: stages post and filter (spec 06) ``` diff --git a/clean/dx12/README.md b/clean/dx12/README.md index 35330d61..10a31542 100644 --- a/clean/dx12/README.md +++ b/clean/dx12/README.md @@ -125,6 +125,12 @@ WG_KERNEL(exposure_auto_diet) { WG_SETUP(); ... } // entry name = HIP kernel n builds (one manifest entry per name and `co`, for example `w64/post_tf_f32_p12_diet_vsit` of `post_w64__orig.co` and of `post_w64.co`): the device runs the port whose code object is loaded. Every port of a name but the first sets `file=`, its DXIL file name (`.dxil`). +* `gfx=<11|12|11,12>` (optional, default 12) = the GPU families the declaration is built for (`build_dx12.py --gfx`, + default 12): `--gfx 11` (RDNA 3, `WG_GFX=11` in every kernel) builds only the declarations that list 11 into + `/dx12-gfx11/` with a `"gfx": 11` key in its manifest (a gfx12 manifest has no `gfx` key), without RGA. The + gfx11 helpers (`wg_wmma_f16_k8` / `_k16`, `wg_d_row`, no FP8 matrix instruction) are in `wg_gfx11.hlsli`, which a gfx11 kernel includes + instead of `wg_platform.hlsli` (it includes that itself); the platform layer and `wg_ags.hlsli` hold no gfx11 code, because + model K's softmax self-check digest covers them. * `args` = the HIP kernel's arguments in order: `p` pointer (DWORD: arena byte offset), `u` any 32-bit scalar, `b` struct by value (N bytes = N/4 DWORDs, copied verbatim). At most 57 DWORDs. The `WG_ARGS_BEGIN` block must declare exactly these DWORDs in order, **32-bit scalars only** (no vectors / arrays / structs: cbuffer packing). diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index 3781dcd4..a483b438 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -32,6 +32,11 @@ and rcp() (fdiv 1.0, x) is the hardware v_rcp_f32; it is reassembled (dxil_asm.cpp, DXC's IDxcAssembler) and signed by dxv. Any other float division ('/', which would be an IEEE division) or float remainder is refused: use wg_div. +GPU family (--gfx 12, the default, or 11 for RDNA 3): a gfx11 build defines WG_GFX=11 for every kernel, writes /dx12-gfx11/ +and a "gfx": 11 key in its manifest, and builds only the kernels whose declaration says gfx=11 (a comma list, default +gfx=12: every existing declaration is a gfx12 kernel, so a build without --gfx is the same set as before). RGA is not used +for gfx11 (no VGPR check, no --isa). + VGPR check: every Swin / bottleneck kernel built (a name with a swin_ or l_swin_ part) is run through RGA (gfx1201) and its USED_VGPRs compared with its bound in clean/dx12/vgpr_limits.json (the count that keeps its present workgroups per CU); over the bound, or no bound listed, fails the build. WG_DX12_VGPR_CHECK=0 skips it. @@ -210,6 +215,9 @@ def kernel_decls(path): raise SystemExit(f"{path}: {name}: gscale {d['gscale']}") # probe=1: a measurement probe, built only when --only names it (never part of a model's kernel set) d["probe"] = kv.get("probe", "0") == "1" + d["gfx"] = {int(x) for x in kv.get("gfx", "12").split(",")} + if not d["gfx"] <= {11, 12}: + raise SystemExit(f"{path}: {name}: gfx={kv.get('gfx')} (11, 12 or 11,12)") d["file"] = kv.get("file", "") # DXIL file stem of a second port of the same name (module doc) if d["file"] and not re.fullmatch(r"[A-Za-z0-9_]+", d["file"]): raise SystemExit(f"{path}: {name}: file={d['file']} (letters, digits and _ only)") @@ -258,14 +266,15 @@ def source_files(model_k=False, only_k=False): return out -def kernel_set(model_l=False, model_k=False): +def kernel_set(model_l=False, model_k=False, gfx=12): """(name, code object) of every kernel a full build compiles (no --only), sorted: Model M's set, plus Model L's with model_l, plus Model K's with model_k. From the sources' declarations alone (nothing is compiled). Frame - generation's kernels and measurement probes (probe=1) are never in it (is_frame_gen).""" + generation's kernels and measurement probes (probe=1) are never in it (is_frame_gen). gfx: the GPU family whose + kernels are listed.""" keys = set() for path, k_source in source_files(model_k): for d in kernel_decls(path): - if is_frame_gen(d) or d["probe"]: + if is_frame_gen(d) or d["probe"] or gfx not in d["gfx"]: continue if k_source or is_model_k(d): if model_k: @@ -431,7 +440,10 @@ def text(path): def label(path): if gen_dir and os.path.normcase(os.path.abspath(path)).startswith(os.path.normcase(os.path.abspath(gen_dir))): return "gen/" + os.path.relpath(path, gen_dir).replace(os.sep, "/") - return os.path.relpath(path, root).replace(os.sep, "/") + try: + return os.path.relpath(path, root).replace(os.sep, "/") + except ValueError: # another drive (Windows): no relative path exists; a source outside the repo is named by its file + return "ext/" + os.path.basename(path) seen, order, missing = set(), [], set() @@ -452,7 +464,8 @@ def walk(path): walk(src) h = hashlib.sha256() - h.update(b"decl\n" + json.dumps(decl, sort_keys=True).encode() + b"\n") + # the gfx set (a set, not JSON) is left out: a build's family is in its defines, and a gfx12 digest stays what it was + h.update(b"decl\n" + json.dumps({k: v for k, v in decl.items() if k != "gfx"}, sort_keys=True).encode() + b"\n") for path in sorted(order, key=label): h.update(b"file " + label(path).encode() + b"\n" + text(path) + b"\n") h.update(b"missing " + ",".join(sorted(missing)).encode() + b"\n") @@ -575,6 +588,9 @@ def main(argv=None): ap.add_argument("--model-k", action="store_true", help="also build Model K's kernels (clean/k/dx12/kernels; off by default: Model M's and Model L's " "kernel sets unchanged)") + ap.add_argument("--gfx", type=int, choices=(11, 12), default=12, + help="GPU family of the build: 12 = RDNA 4 (default, /dx12), 11 = RDNA 3 (WG_GFX=11, " + "/dx12-gfx11, only kernels declared gfx=11, no RGA)") ap.add_argument("--define", action="append", default=[], metavar="NAME=VALUE", help="an extra preprocessor define for every kernel built (test variants: negative-control mutants, " "test counters; never for a shipped build)") @@ -586,7 +602,7 @@ def main(argv=None): "the cheap check before a build (kernel_sets.json, tests/test_kernel_sets.py)") a = ap.parse_args(argv) if a.list: - print(json.dumps([list(k) for k in kernel_set(a.model_l, a.model_k)], indent=1)) + print(json.dumps([list(k) for k in kernel_set(a.model_l, a.model_k, a.gfx)], indent=1)) return 0 if not a.out: ap.error("--out is required") @@ -595,7 +611,9 @@ def main(argv=None): if not re.fullmatch(r"[A-Z][A-Z0-9_]*=-?[0-9A-Za-z_]+", kv): ap.error(f"--define {kv!r} (NAME=VALUE)") extra_defs[kv.split("=", 1)[0]] = kv.split("=", 1)[1] - dst = os.path.join(a.out, "dx12") + if a.gfx == 11 and a.isa: + ap.error("--isa needs RGA, which a gfx11 build does not use") + dst = os.path.join(a.out, "dx12" if a.gfx == 12 else "dx12-gfx11") os.makedirs(dst, exist_ok=True) asm = build_asm_tool(a.out) only = set(a.only.split(",")) if a.only else None @@ -623,6 +641,8 @@ def main(argv=None): jobs = [] # (source, declaration, output path) in declaration order (sorted file name, then line) for src, k_source in source_files(a.model_k or bool(only)): for d in kernel_decls(src): + if a.gfx not in d["gfx"]: + continue if only and d["name"] not in only: continue if not only and (k_source or is_model_k(d)) and not a.model_k: @@ -656,7 +676,8 @@ def main(argv=None): def build_one(job): src, d, out, k_source = job - defs = {"WG_BLOCK": d["group"], "WG_WAVE": d["wave"], **d["defines"], **extra_defs} + defs = {"WG_BLOCK": d["group"], "WG_WAVE": d["wave"], **({"WG_GFX": a.gfx} if a.gfx != 12 else {}), + **d["defines"], **extra_defs} if d["group"] != d["block"]: defs["WG_LAUNCH_BLOCK"] = d["block"] info = {} @@ -707,14 +728,19 @@ def build_one(job): if only and not (only <= names): raise SystemExit(f"not declared in clean/dx12/kernels or clean/k/dx12/kernels: {sorted(only - names)}") try: - vgpr_check(built, {d["name"] for _, d, _, _ in jobs if hyb_scanned(d["name"]) and not d["probe"]}) - isa_check(built) + if a.gfx == 12: + vgpr_check(built, {d["name"] for _, d, _, _ in jobs if hyb_scanned(d["name"]) and not d["probe"]}) + isa_check(built) finally: if scan_dir: shutil.rmtree(scan_dir, ignore_errors=True) + manifest = {"format": FORMAT, "dxc": os.path.basename(os.path.dirname(os.path.dirname(DXC_DIR)))} + if a.gfx != 12: + manifest["gfx"] = a.gfx # absent = 12 + manifest["test_only"] = test_only + manifest["kernels"] = [kernels[k] for k in sorted(kernels)] with open(man_path, "w") as f: - json.dump({"format": FORMAT, "dxc": os.path.basename(os.path.dirname(os.path.dirname(DXC_DIR))), - "test_only": test_only, "kernels": [kernels[k] for k in sorted(kernels)]}, f, indent=1) + json.dump(manifest, f, indent=1) print(f"{len(kernels)} kernel(s) in {man_path}" + (" (TEST-ONLY: never embedded or packaged)" if test_only else "")) return 0 diff --git a/clean/dx12/gelu_lut/mk_headers.py b/clean/dx12/gelu_lut/mk_headers.py index 14738a42..fcb146d9 100644 --- a/clean/dx12/gelu_lut/mk_headers.py +++ b/clean/dx12/gelu_lut/mk_headers.py @@ -53,7 +53,11 @@ // codes clamped to the slots that matter: positive w to WG_GELU_LUT_HP, negative w to -WG_GELU_LUT_HN (RTZ // conversion first, then the clamp in FP16: the same codes as clamping the 16-bit codes; inf / NaN land on the // clamp slots) +#if WG_GFX == 11 // RADV rounds a (half) cast to nearest even and f32tof16 toward zero + half2 c = half2(asfloat16((uint16_t)f32tof16(w0)), asfloat16((uint16_t)f32tof16(w1))); +#else half2 c = half2((half)w0, (half)w1); +#endif c = max(min(c, half2(WG_GELU_LUT_HP, WG_GELU_LUT_HP)), half2(-WG_GELU_LUT_HN, -WG_GELU_LUT_HN)); const uint d = ((uint)asuint16(c.x) | ((uint)asuint16(c.y) << 16)) & (0xffe0ffe0u | ((wg_bid_ >> 31) << 16)); #else diff --git a/clean/dx12/hlsl/wg_gelu_lut.hlsli b/clean/dx12/hlsl/wg_gelu_lut.hlsli index 7c7c84f8..5f3dce37 100644 --- a/clean/dx12/hlsl/wg_gelu_lut.hlsli +++ b/clean/dx12/hlsl/wg_gelu_lut.hlsli @@ -41,7 +41,11 @@ uint2 wg_gelu_lut_off2(float h0, float h1) // codes clamped to the slots that matter: positive w to WG_GELU_LUT_HP, negative w to -WG_GELU_LUT_HN (RTZ // conversion first, then the clamp in FP16: the same codes as clamping the 16-bit codes; inf / NaN land on the // clamp slots) +#if WG_GFX == 11 // RADV rounds a (half) cast to nearest even and f32tof16 toward zero + half2 c = half2(asfloat16((uint16_t)f32tof16(w0)), asfloat16((uint16_t)f32tof16(w1))); +#else half2 c = half2((half)w0, (half)w1); +#endif c = max(min(c, half2(WG_GELU_LUT_HP, WG_GELU_LUT_HP)), half2(-WG_GELU_LUT_HN, -WG_GELU_LUT_HN)); const uint d = ((uint)asuint16(c.x) | ((uint)asuint16(c.y) << 16)) & (0xffe0ffe0u | ((wg_bid_ >> 31) << 16)); #else diff --git a/clean/dx12/hlsl/wg_gfx11.hlsli b/clean/dx12/hlsl/wg_gfx11.hlsli new file mode 100644 index 00000000..8d772fed --- /dev/null +++ b/clean/dx12/hlsl/wg_gfx11.hlsli @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: MIT +// RDNA 3 (gfx11) layer over the platform layer of the D3D12 ports (WrongGPU's own code). A kernel that is built for gfx11 +// (build_dx12.py --gfx 11, declaration gfx=11) includes THIS header first instead of "wg_platform.hlsli"; it includes that +// one itself. wg_platform.hlsli and wg_ags.hlsli stay as they are: they are part of model K's softmax self-check digest +// (clean/k/dx12/smx_selfcheck_digest.py), so every gfx11 definition lives here and in the gfx11 headers of the kernels. +// +// WG_GFX is 12 (RDNA 4, the default: this header then only includes the platform layer) or 11 (RDNA 3, defined by +// build_dx12.py --gfx 11). Under 11: +// * the matrix instruction takes F16 only, with its own fragment layout (below); the FP8 matrix instruction, the FP8 +// conversion and the _ags packs of the platform layer do not exist on RDNA 3 and are hidden again (a shader that uses +// one fails to compile, not to translate); +// * the gfx12-shaped wg_wmma_f16 of the platform layer is renamed away while it is included and replaced by HIP's gfx11 +// form (operands in the gfx12 shape, one half-wave exchange per dword, then the K = 16 chain). +#pragma once + +#ifndef WG_WAVE +#define WG_WAVE 32 +#endif +#ifndef WG_GFX +#define WG_GFX 12 +#endif +#if WG_GFX != 11 && WG_GFX != 12 +#error "WG_GFX is 11 (RDNA 3) or 12 (RDNA 4)" +#endif +#if WG_GFX == 11 && WG_WAVE != 32 +#error "WG_GFX 11 kernels run in wave32" +#endif + +#if WG_GFX == 11 +#define wg_wmma_f16 wg_wmma_f16_gfx12_shape +#endif +#include "wg_platform.hlsli" +#if WG_GFX == 11 +#undef wg_wmma_f16 + +// Hidden on RDNA 3 (the identifiers are undeclared, so a use is a compile error). +#undef WG_AGS_FP8_F32 +#undef WG_AGS_T_FP8_B +#define wg_wmma_fp8 wg_no_fp8_matrix_instruction_on_gfx11 +#define wg_pack_q8_nc_ags wg_no_fp8_convert_on_gfx11 +#define wg_pack_q8_ags wg_no_fp8_convert_on_gfx11 +#define wg_pack_q8_inner_ags wg_no_fp8_convert_on_gfx11 + +// RDNA 3 (measured on gfx1100 through vkd3d-proton's raw fragments): the F16 instruction (format 0x5) takes 8 K values +// per call from the channel's fragment (K 0 .. 7; the K 8 .. 15 slots are zero) and, per lane l (half h = l / 16, +// c = l % 16): +// A: accumulator row r reads its A values from lane 16 (r & 1) + r -> hold row r's values in lanes r and r + 16; +// B: column c reads lane l itself -> lanes c and c + 16 hold column c; +// C / D: element i of lane l is row 2 i + h, column c (wg_d_row). +// So an operand is replicated across the two halves of the wave, where gfx12 splits K between them. A K = 16 step is two +// chained instructions (the second takes K 8 .. 15 and the first's result as its C): the result equals HIP's one- +// instruction v_wmma_f32_16x16x16_f16 bit for bit. +bool wg_in_upper_half() { return (WaveGetLaneIndex() & 16u) != 0u; } +uint4 wg_xor16_4(uint4 v) { return uint4(wg_xor16_u(v.x), wg_xor16_u(v.y), wg_xor16_u(v.z), wg_xor16_u(v.w)); } +// One instruction: D = A x B + C over K 0 .. 7. a / b: the 8 halves of this lane's row / column, replicated across halves. +v8f wg_wmma_f16_k8(uint4 a, uint4 b, v8f c) { return wg_v8f_frag(wg_ags_mma(WG_AGS_F16_F32, wg_frag4(a), wg_frag4(b), wg_frag_v8f(c))); } +// K = 16: a0 / b0 hold K 0 .. 7, a1 / b1 K 8 .. 15 (replicated as above); two chained instructions. +v8f wg_wmma_f16_k16(uint4 a0, uint4 b0, uint4 a1, uint4 b1, v8f c) +{ + return wg_wmma_f16_k8(a1, b1, wg_wmma_f16_k8(a0, b0, c)); +} +// Row of D held by accumulator element i of this lane (the gfx11 table: 2 i + h; packing.py D_ROW_LAYOUTS["gfx11"]). +uint wg_d_row(uint i) { return 2u * i + (WaveGetLaneIndex() >> 4); } +// HIP's wg_wmma_f16 on gfx11: operands in the gfx12 shape (lane l: row / column l % 16, half h holds K 8 h .. 8 h + 7); +// one half-wave exchange per dword, then the K = 16 chain. +v8f wg_wmma_f16(uint4 a, uint4 b, v8f c) +{ + const uint4 ya = wg_xor16_4(a), yb = wg_xor16_4(b); + const bool hi = wg_in_upper_half(); + return wg_wmma_f16_k16(hi ? ya : a, hi ? yb : b, hi ? a : ya, hi ? b : yb, c); +} +#else +// Row of D held by accumulator element i of this lane (the gfx12 table: v + 8 h, h = lane / 16). +uint wg_d_row(uint i) { return i + 8u * (WaveGetLaneIndex() >> 4); } +#endif diff --git a/clean/dx12/kernels/native_game_helpers.hlsl b/clean/dx12/kernels/native_game_helpers.hlsl index 134ad3a1..f2641c15 100644 --- a/clean/dx12/kernels/native_game_helpers.hlsl +++ b/clean/dx12/kernels/native_game_helpers.hlsl @@ -3,9 +3,11 @@ // Original WrongGPU code; basis: own io_probe, exposure.hip exposure_pair_diet, // spec 07/10 and the existing strict build arithmetic contract. No alternate I/O path. // @wg-kernel native_driver_probe co=- block=32 wave=32 args=p defines=NATIVE_DRIVER_PROBE=1 +// @wg-kernel native_driver_probe_f16 co=- block=32 wave=32 args=p defines=NATIVE_DRIVER_PROBE_F16=1 gfx=11 +// @wg-kernel native_round_probe co=- block=32 wave=32 args=p defines=NATIVE_ROUND_PROBE=1 gfx=11 // @wg-kernel exposure_pair_diet co=exposure.co block=32 wave=32 args=p,p,u,u,u,u defines=NATIVE_EXPOSURE_PAIR=1 // @wg-kernel exposure_pair_diet_direct co=exposure.co entry=exposure_pair_diet block=32 wave=32 args=p,p,u,u,u,u rs=direct defines=NATIVE_EXPOSURE_PAIR=1,WG_DIRECT_TEXTURES=1 -#include "wg_platform.hlsli" +#include "wg_gfx11.hlsli" #if defined(WG_DIRECT_TEXTURES) #include "native_direct.hlsli" #endif @@ -32,6 +34,41 @@ WG_KERNEL(native_driver_probe) { } } #endif +#if defined(NATIVE_DRIVER_PROBE_F16) +WG_ARGS_BEGIN + uint dst; +WG_ARGS_END +// RDNA 3 form of the probe (no FP8 matrix instruction): D = A x B + C with FP16 operands as a K = 16 step of two chained +// instructions, A and B in the replicated fragment layout of wmma_probe.h (namespace gfx11). Offsets: kOutF16 0, kInC 2048, +// kInAF16 3072, kInBF16 3584, kInAF16K2 4096, kInBF16K2 4608. All loads come before the first matrix call. +WG_KERNEL(native_driver_probe_f16) { + WG_SETUP(); + uint lane=wg_lane_id(); + uint4 a0=wg_ld128(dst,3072u+lane*16u),b0=wg_ld128(dst,3584u+lane*16u); + uint4 a1=wg_ld128(dst,4096u+lane*16u),b1=wg_ld128(dst,4608u+lane*16u); + v8f c; + [unroll] for(uint i=0;i<8u;++i) c.v[i]=wg_ldf(dst,2048u+(i*32u+lane)*4u); + v8f h=wg_wmma_f16_k16(a0,b0,a1,b1,c); + [unroll] for(uint k=0;k<8u;++k) wg_stf(dst,(k*32u+lane)*4u,h.v[k]); +} +#endif +#if defined(NATIVE_ROUND_PROBE) +WG_ARGS_BEGIN + uint dst; +WG_ARGS_END +// Developer-only shortcut check (WG_DEV_GFX11, runtime/wmma_probe.h namespace round16): the F32 -> F16 -> F32 pair of the +// driver's matrix conversion, exactly as the encoder's round_f16 writes it. A compiler that folds the pair away returns +// the input unchanged. Float 32 i + lane of the buffer at byte 1024 is element i of lane `lane`; the results go to byte 0. +WG_KERNEL(native_round_probe) { + WG_SETUP(); + uint lane=wg_lane_id(); + v8f a; + [unroll] for(uint i=0;i<8u;++i) a.v[i]=wg_ldf(dst,1024u+(i*32u+lane)*4u); + wg_frag hf=wg_ags_convert(wg_frag_v8f(a),WG_AGS_T_F32_ACC,WG_AGS_T_F16_ACC); + wg_frag bf=wg_ags_convert(hf,WG_AGS_T_F16_ACC,WG_AGS_T_F32_ACC); + [unroll] for(uint k=0;k<8u;++k) wg_stf(dst,(k*32u+lane)*4u,asfloat(bf.v[k])); +} +#endif #if defined(NATIVE_EXPOSURE_PAIR) WG_ARGS_BEGIN uint ex,computed; diff --git a/clean/dx12/kernels/swin_common.hlsli b/clean/dx12/kernels/swin_common.hlsli index 93a65346..877f4c57 100644 --- a/clean/dx12/kernels/swin_common.hlsli +++ b/clean/dx12/kernels/swin_common.hlsli @@ -76,8 +76,12 @@ float sw_hmax8(v8f a) // v_cvt_pk_f32_fp8); 0: software (about 8 instructions per value). Same bits as HIP, NaN codes included (checked by the // whole-kernel differential test with random and full-range FP8 inputs; this direction is not in probe/arith3). #ifndef SW_UNPACK_AGS +#if WG_GFX == 11 // the FP8 B conversion of the channel does not exist on RDNA 3 +#define SW_UNPACK_AGS 0 +#else #define SW_UNPACK_AGS 1 #endif +#endif v8f sw_unpack_fp8(uint2 q) { #if SW_UNPACK_AGS diff --git a/clean/dx12/tests/build_kernel_replay.py b/clean/dx12/tests/build_kernel_replay.py new file mode 100644 index 00000000..9dde652c --- /dev/null +++ b/clean/dx12/tests/build_kernel_replay.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: MIT +"""Cross-builds the headless test hosts on Linux with llvm-mingw (static, runs under Wine): + kernel_replay.exe clean/dx12/tests/kernel_replay.cpp + the runtime's device sources (runs launches on the native + D3D12 device; the device refuses a card that is not RDNA 4 until the family gate is opened) + native_probe_test.exe the borrowed-device path with its driver-channel probe (a device made here, handed to the runtime); + -DWG_DEV_GFX11 in --cxx makes it (and kernel_replay) accept an RX 7900 series board (runtime/rdna4.h) + pso_sweep.exe pso_sweep.cpp (PSO creation per kernel with the runtime's root signatures; no runtime code) + gfx11_probe_host.exe the host of test_gfx11_helpers.py + frame_host.exe a whole frame on the runtime library's native D3D12 executor (the host of frame_parity.py; loads + wronggpu_runtime.dll, itself a Windows build of clean/runtime) + python3 build_kernel_replay.py --cxx --out [--only pso_sweep,...] +""" +import argparse +import os +import shlex +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +RT = os.path.join(os.path.dirname(os.path.dirname(HERE)), "runtime") +CONV = os.path.join(os.path.dirname(os.path.dirname(HERE)), "converter", "src") +TARGETS = { + "kernel_replay": [os.path.join(HERE, "kernel_replay.cpp"), *(os.path.join(RT, f) for f in + ("device.cpp", "device_d3d12.cpp", "embedded.cpp")), os.path.join(CONV, "json.cpp"), + os.path.join(CONV, "sha256.cpp")], + "native_probe_test": [os.path.join(HERE, "native_probe_test.cpp"), *(os.path.join(RT, f) for f in + ("device.cpp", "device_d3d12.cpp", "embedded.cpp")), os.path.join(CONV, "json.cpp"), + os.path.join(CONV, "sha256.cpp")], + "pso_sweep": [os.path.join(HERE, "pso_sweep.cpp")], + "gfx11_probe_host": [os.path.join(HERE, "gfx11_probe_host.cpp")], + "frame_host": [os.path.join(HERE, "frame_host.cpp")], +} + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--cxx", required=True, help="llvm-mingw g++ command (flags allowed)") + ap.add_argument("--out", required=True) + ap.add_argument("--only", help="comma list of: " + ", ".join(TARGETS)) + a = ap.parse_args() + os.makedirs(a.out, exist_ok=True) + for name in (a.only.split(",") if a.only else TARGETS): + cmd = [*shlex.split(a.cxx), "-std=c++17", "-O2", "-fms-extensions", "-static", "-static-libgcc", "-static-libstdc++", + "-D_CRT_SECURE_NO_WARNINGS", *TARGETS[name], "-o", os.path.join(a.out, name + ".exe")] + print(" ".join(shlex.quote(c) for c in cmd[:1]), "...", name, flush=True) + subprocess.check_call(cmd) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/dx12/tests/frame_host.cpp b/clean/dx12/tests/frame_host.cpp new file mode 100644 index 00000000..bba209bb --- /dev/null +++ b/clean/dx12/tests/frame_host.cpp @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: MIT +// A whole Model M frame on the runtime library's native D3D12 executor (backend 3), driven from a plain C++ host so that it runs +// headless under Wine on Linux (no Python on the Windows side). WrongGPU's own code. Synthetic model and inputs only: the model +// folder (raw tensors) and the input files are written by frame_parity.py; nothing here reads a model.wgcm. +// +// frame_host.exe +// +// holds wronggpu_runtime.dll (clean/runner/build.py, a Windows build) and dx12\manifest.json (the D3D12 kernels). +// , one item per line (paths relative to the spec file; '#' starts a comment): +// model the folder with the raw tensors (model_format 1) +// config a WgRtConfig field by name (see wg_runtime.h); block_threads / block_wpg / d_row as +// " " +// frame +// float32 arrays [rh][rw][3], [rh][rw], [rh][rw][2] +// dump after each frame write the named buffer to /f_.bin +// time after the last frame, timed frames of the last frame's inputs (no reset): +// the GPU times in ms go to /times.txt, one per line +// The runtime library is loaded at run time, as a game does; `kernels` and `info` of the last frame go to /report.txt. +#define WIN32_LEAN_AND_MEAN +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../runtime/wg_runtime.h" + +namespace { + +struct Frame { + int rw = 0, rh = 0, reset = 0; + float scale[2] = {}, bias[2] = {}, jitter[2] = {}, exposure = 1, pre_exposure = 1; + std::string color, depth, motion; +}; + +std::vector read_file(const std::string& path) { + std::ifstream f(path, std::ios::binary); + if (!f) { std::fprintf(stderr, "cannot read %s\n", path.c_str()); std::exit(2); } + return std::vector((std::istreambuf_iterator(f)), std::istreambuf_iterator()); +} +void write_file(const std::string& path, const void* data, size_t bytes) { + std::ofstream f(path, std::ios::binary); + f.write(static_cast(data), static_cast(bytes)); +} +std::string folder_of(const std::string& path) { + const size_t at = path.find_last_of("/\\"); + return at == std::string::npos ? std::string() : path.substr(0, at + 1); +} + +template T load(HMODULE lib, const char* name) { + T f = reinterpret_cast(GetProcAddress(lib, name)); + if (!f) { std::fprintf(stderr, "runtime library has no %s\n", name); std::exit(2); } + return f; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 4) { std::fprintf(stderr, "usage: frame_host.exe \n"); return 2; } + const std::string build = argv[1], spec = argv[2], out = std::string(argv[3]) + "\\", base = folder_of(spec); + HMODULE lib = LoadLibraryA((build + "\\wronggpu_runtime.dll").c_str()); + if (!lib) { std::fprintf(stderr, "cannot load %s\\wronggpu_runtime.dll (error %lu)\n", build.c_str(), GetLastError()); return 2; } + auto create = load(lib, "wg_rt_create"); + auto destroy = load(lib, "wg_rt_destroy"); + auto rt_error = load(lib, "wg_rt_error"); + auto upload = load(lib, "wg_rt_upload_inputs"); + auto frame_fn = load(lib, "wg_rt_frame"); + auto timed = load(lib, "wg_rt_frame_timed"); + auto sync = load(lib, "wg_rt_sync"); + auto info = load(lib, "wg_rt_info"); + auto buffer = load(lib, "wg_rt_buffer"); + auto download = load(lib, "wg_rt_download"); + auto kernels = load(lib, "wg_rt_kernels"); + + WgRtConfig cfg; + std::memset(&cfg, 0, sizeof cfg); + cfg.backend = 3; + cfg.model_format = 1; + std::vector strings; // owns every string the config points to + strings.reserve(64); + auto keep = [&](const std::string& s) { strings.push_back(s); return strings.back().c_str(); }; + cfg.build_dir = keep(build); + std::vector frames; + std::vector dumps; + unsigned repeats = 0; + std::ifstream in(spec); + if (!in) { std::fprintf(stderr, "cannot read %s\n", spec.c_str()); return 2; } + for (std::string line; std::getline(in, line);) { + if (const size_t hash = line.find('#'); hash != std::string::npos) line.resize(hash); + std::istringstream s(line); + std::string key; + if (!(s >> key)) continue; + if (key == "model") { std::string d; s >> d; cfg.model_dir = keep(d.size() > 1 && d[1] == ':' ? d : base + d); } + else if (key == "dump") { std::string n; s >> n; dumps.push_back(n); } + else if (key == "time") s >> repeats; + else if (key == "frame") { + Frame f; + s >> f.rw >> f.rh >> f.reset >> f.scale[0] >> f.scale[1] >> f.bias[0] >> f.bias[1] >> f.jitter[0] >> f.jitter[1] >> f.exposure >> + f.pre_exposure >> f.color >> f.depth >> f.motion; + if (!s) { std::fprintf(stderr, "bad frame line: %s\n", line.c_str()); return 2; } + frames.push_back(f); + } else if (key == "config") { + std::string field; + s >> field; + int i = 0; + std::string v; + auto indexed = [&](int* array, int n) { s >> i >> v; if (i < 0 || i >= n) { std::fprintf(stderr, "bad index\n"); std::exit(2); } array[i] = std::atoi(v.c_str()); }; + if (field == "block_threads") indexed(cfg.block_threads, 5); + else if (field == "block_wpg") indexed(cfg.block_wpg, 5); + else if (field == "d_row") indexed(cfg.d_row, 16); + else { + s >> v; +#define INT(name) if (field == #name) cfg.name = std::atoi(v.c_str()); else +#define STR(name) if (field == #name) cfg.name = keep(v); else + INT(max_rw) INT(max_rh) INT(ow) INT(oh) INT(depth_inverted) INT(auto_exposure) INT(swin_threads) INT(gemm_threads) + INT(nt) INT(wpw64) INT(wpw96) INT(wpw128) INT(swin_force_clamp) INT(swin_fuse) INT(post_wave) INT(recon_output) INT(hdr) + STR(swin_code_object) STR(map_path) STR(enc0_variant) STR(dec0_variant) STR(exposure_variant) STR(post_kernel) + STR(filter_kernel) STR(swin_stage_variants) STR(swin_fuse_stages) STR(runtime_options) STR(swin_bneck) STR(dec1_dec0) + STR(swin_mid) STR(post_pack) + if (field == "alpha") cfg.alpha = static_cast(std::atof(v.c_str())); + else { std::fprintf(stderr, "unknown config field %s\n", field.c_str()); return 2; } + } + } else { std::fprintf(stderr, "unknown spec line: %s\n", line.c_str()); return 2; } + } + if (frames.empty() || !cfg.model_dir) { std::fprintf(stderr, "spec needs a model and a frame\n"); return 2; } + + void* rt = nullptr; + char err[2048] = {}; + if (create(&cfg, &rt, err, sizeof err) != 0) { std::printf("create FAILED: %s\n", err); return 1; } + std::printf("created\n"); + auto fail = [&](const char* what) { std::printf("%s FAILED: %s\n", what, rt_error(rt)); destroy(rt); return 1; }; + WgRtFrame fr; + for (size_t n = 0; n < frames.size(); ++n) { + const Frame& f = frames[n]; + const std::vector color = read_file(base + f.color), depth = read_file(base + f.depth), motion = read_file(base + f.motion); + const size_t px = static_cast(f.rw) * f.rh; + if (color.size() != px * 12 || depth.size() != px * 4 || motion.size() != px * 8) { std::printf("input size mismatch\n"); return 2; } + if (upload(rt, f.rw, f.rh, reinterpret_cast(color.data()), reinterpret_cast(depth.data()), + reinterpret_cast(motion.data())) != 0) return fail("upload"); + std::memset(&fr, 0, sizeof fr); + fr.rw = f.rw; fr.rh = f.rh; fr.reset = f.reset; fr.exposure = f.exposure; fr.pre_exposure = f.pre_exposure; + for (int i = 0; i < 2; ++i) { fr.mv_scale[i] = f.scale[i]; fr.mv_bias[i] = f.bias[i]; fr.jitter[i] = f.jitter[i]; } + if (frame_fn(rt, &fr) != 0 || sync(rt) != 0) return fail("frame"); + for (const std::string& name : dumps) { + uintptr_t ptr = 0; + size_t bytes = 0; + if (buffer(rt, name.c_str(), &ptr, &bytes) != 0) continue; // a buffer this route does not have + std::vector data(bytes); + if (download(rt, ptr, data.data(), bytes) != 0) return fail(("download " + name).c_str()); + write_file(out + "f" + std::to_string(n) + "_" + name + ".bin", data.data(), bytes); + } + std::printf("frame %zu done\n", n); + std::fflush(stdout); + } + if (repeats) { + fr.reset = 0; + std::string times; + for (unsigned i = 0; i < repeats; ++i) { + float ms = 0; + if (timed(rt, &fr, &ms) != 0) return fail("timed frame"); + times += std::to_string(ms) + "\n"; + } + write_file(out + "times.txt", times.data(), times.size()); + } + WgRtFrameInfo fi; + std::string report = std::string("kernels ") + kernels(rt) + "\n"; + if (info(rt, &fi) == 0) report += "frame_index " + std::to_string(fi.index) + " render " + std::to_string(fi.rw) + "x" + std::to_string(fi.rh) + "\n"; + write_file(out + "report.txt", report.data(), report.size()); + destroy(rt); + std::printf("DONE\n"); + return 0; +} diff --git a/clean/dx12/tests/frame_parity.py b/clean/dx12/tests/frame_parity.py new file mode 100644 index 00000000..8ec0a0ff --- /dev/null +++ b/clean/dx12/tests/frame_parity.py @@ -0,0 +1,209 @@ +# SPDX-License-Identifier: MIT +"""Whole-frame parity of the native D3D12 executor with the HIP whole frame on the same synthetic inputs, for a headless run +on Linux: the D3D12 side is clean/dx12/tests/frame_host.exe under Wine (a Windows build of the runtime library), the HIP side +and the comparison run natively in Python (clean/runner/pipeline.py, as compare.py does for a Windows machine). Synthetic +model and frames only (clean/runner/synthetic_model.py, clean/runner/output_filter_original_check.py frame_input). + + python frame_parity.py prepare --out --render 1706x960 --output 2560x1440 [--frames 3] [--seed 1] + /model (raw tensors), /in (frame inputs), /spec.txt (the host's spec, see frame_host.cpp), /case.json + python frame_parity.py ref --out --name hip|emu --build [--rocm ] [--time N] + the HIP (or lane-emulator) whole frame: //f_.bin, times.txt, report.txt + (run frame_host.exe on /spec.txt with --out /d3d12 under Wine, see wine_headless_run.sh) + python frame_parity.py compare --out --a hip --b d3d12 [--json report.json] + per named buffer: bytes that differ; for the output image: the differences in values; the GPU times of both + +Not a gate: the output standard of an RDNA 3 route is the maintainer's open decision, this only measures. +""" +import argparse +import json +import os +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +DX12 = os.path.dirname(HERE) +ROOT = os.path.dirname(os.path.dirname(DX12)) +for p in (ROOT, DX12): + if p not in sys.path: + sys.path.insert(0, p) + +import compare as C # noqa: E402 (clean/dx12/compare.py: the buffer list and the fusion rule) +from clean.runner import output_filter_original_check as OC # noqa: E402 +from clean.runner import synthetic_model as SM # noqa: E402 +from clean.runner.pipeline import Pipeline # noqa: E402 + +FLAGS = {"depth_inverted": True, "auto_exposure": True} + + +def size(text): + w, h = text.lower().split("x") + return int(w), int(h) + + +def prepare(a): + render, output = size(a.render), size(a.output) + os.makedirs(os.path.join(a.out, "in"), exist_ok=True) + model = os.path.join(a.out, "model") + os.makedirs(model, exist_ok=True) + SM.write_raw(model, SM.synthetic_tensors(a.seed)) + lines = ["model model", f"config max_rw {render[0]}", f"config max_rh {render[1]}", f"config ow {output[0]}", + f"config oh {output[1]}", f"config depth_inverted {int(FLAGS['depth_inverted'])}", + f"config auto_exposure {int(FLAGS['auto_exposure'])}", "config alpha 0.18"] + from clean.runner import variants as V + geo = V.DEFAULT_GEOMETRY # the values Pipeline passes (the native executor ignores the HIP launch geometry) + lines += [f"config gemm_threads {geo['gemm_threads']}", f"config nt {geo['nt']}", + f"config wpw64 {geo['wpw']['swin_block_c64']}", f"config wpw96 {geo['wpw']['swin_block_c96']}", + f"config wpw128 {geo['wpw']['swin_block_c128']}"] + for k, kernel in enumerate(("swin_block_c64", "swin_block_c64_add", "swin_block_c96", "swin_block_c96_add", "swin_block_c128")): + lines += [f"config block_threads {k} {geo['threads'][kernel]}", f"config block_wpg {k} {geo.get('wpg', {}).get(kernel, 0)}"] + for i in range(a.frames): + x = OC.frame_input(i, render, reset=i == 0) + for name, dtype in (("color", np.float32), ("depth", np.float32), ("motion", np.float32)): + np.ascontiguousarray(x[name], dtype=dtype).tofile(os.path.join(a.out, "in", f"f{i}_{name}.bin")) + v = lambda key, n: " ".join(repr(float(t)) for t in np.asarray(x[key], np.float64).reshape(-1)[:n]) + lines.append(f"frame {render[0]} {render[1]} {int(x['reset'])} {v('mv_scale', 2)} 0.0 0.0 {v('jitter', 2)} " + f"{float(x['exposure'])!r} {float(x['pre_exposure'])!r} in/f{i}_color.bin in/f{i}_depth.bin in/f{i}_motion.bin") + lines += [f"dump {n}" for n in C.BUFFERS] + ["dump filter.out"] + with open(os.path.join(a.out, "spec.txt"), "w") as f: + f.write("\n".join(lines) + "\n") + with open(os.path.join(a.out, "case.json"), "w") as f: + json.dump({"render": render, "output": output, "frames": a.frames, "seed": a.seed}, f) + print(f"prepared {a.frames} frames {render} -> {output} in {a.out}") + + +def case(out): + with open(os.path.join(out, "case.json")) as f: + return json.load(f) + + +def reference(a): + c = case(a.out) + render, output = tuple(c["render"]), tuple(c["output"]) + dest = os.path.join(a.out, a.name) + os.makedirs(dest, exist_ok=True) + p = Pipeline(a.build, a.rocm or "", os.path.join(a.out, "model"), render, output, FLAGS, + backend="emu" if a.name == "emu" else "hip", model_format="raw", check_packing=False) + try: + for i in range(c["frames"]): + x = OC.frame_input(i, render, reset=i == 0) + p.frame(x) + for name, raw in C.raw_buffers(p).items(): + raw.tofile(os.path.join(dest, f"f{i}_{name}.bin")) + p.raw("filter.out").tofile(os.path.join(dest, f"f{i}_filter.out.bin")) + with open(os.path.join(dest, "report.txt"), "w") as f: + f.write("kernels " + " ".join(f"{k}={v}" for k, v in p.kernels().items()) + "\n") + if a.time: + ms = [p.time_frame(OC.frame_input(c["frames"] - 1, render, reset=False), a.time) for _ in range(3)] + with open(os.path.join(dest, "times.txt"), "w") as f: + f.write("\n".join(repr(m) for m in ms) + "\n") + print("whole frame median ms (3 runs):", ms) + finally: + OC.close(p) + + +def kernels_of(path): + with open(os.path.join(path, "report.txt")) as f: + line = next(l for l in f if l.startswith("kernels ")) + return dict(kv.split("=", 1) for kv in line[len("kernels "):].split()) + + +def load(path, i, name): + f = os.path.join(path, f"f{i}_{name}.bin") + return np.fromfile(f, np.uint8) if os.path.exists(f) else None + + +def output_image(path, i, kernels, output): + raw = load(path, i, "filter.out") + if kernels.get("out") == "rgba16f": + return raw.view(np.float16).reshape(output[1], output[0], 4)[..., :3].astype(np.float32) + return raw.view(np.float32).reshape(output[1], output[0], 3).copy() + + +def f16_steps(x, y): + """The distance of two float32 images in FP16 steps (as clean/tests/test_post_emu.f16_steps).""" + def key(v): + b = v.astype(np.float16).view(np.uint16).astype(np.int32) + return np.where(b & 0x8000, 0x8000 - (b & 0x7fff), 0x8000 + b) + return np.abs(key(x) - key(y)) + + +def times(path): + f = os.path.join(path, "times.txt") + if not os.path.exists(f): + return None + v = [float(t) for t in open(f).read().split()] + return {"median_ms": float(np.median(v)), "min_ms": float(np.min(v)), "n": len(v)} + + +def compare(a): + c = case(a.out) + output = tuple(c["output"]) + pa, pb = os.path.join(a.out, a.a), os.path.join(a.out, a.b) + ka, kb = kernels_of(pa), kernels_of(pb) + unwritten = C.fusion_unwritten(kb) | C.fusion_unwritten(ka) + stale = {f"{s}.out" for s in OC.STALE} + report = {"a": a.a, "b": a.b, "kernels_a": ka, "kernels_b": kb, "frames": []} + print(f"{a.a} kernels: {ka}\n{a.b} kernels: {kb}") + for i in range(c["frames"]): + reset = i == 0 + rows = {} + for name in C.BUFFERS: + x, y = load(pa, i, name), load(pb, i, name) + if x is None or y is None: + continue + if name == "exposure.pair": + x, y = x[:C.EXPOSURE_BYTES], y[:C.EXPOSURE_BYTES] + n = min(x.size, y.size) + skipped = name in stale or name in unwritten or (reset and name in ("state.latent_prev", "state.motion_history", "state.history_prev")) \ + or (name == "post.display" and "fused" in (ka.get("filter_last"), kb.get("filter_last"))) \ + or (name in ("dec0.A", "dec0.B", "dec0.C") and ka.get("dec0") != kb.get("dec0")) # another decoder format + d = int(np.count_nonzero(x[:n] != y[:n])) + rows[name] = {"bytes": int(n), "differ": d, "skipped": bool(skipped)} + ia, ib = output_image(pa, i, ka, output), output_image(pb, i, kb, output) + steps = f16_steps(ia, ib) + px = np.any(steps > 0, axis=-1) + scale = float(np.sqrt(np.mean(ia.astype(np.float64) ** 2))) or 1.0 + out = {"nonfinite_a": int((~np.isfinite(ia)).sum()), "nonfinite_b": int((~np.isfinite(ib)).sum()), + "pixels_differ_pct": float(px.mean() * 100), "max_f16_steps": int(steps.max()), + "median_f16_steps_where_differ": float(np.median(steps[steps > 0])) if (steps > 0).any() else 0.0, + "max_abs": float(np.max(np.abs(ia - ib))), "rms_diff_over_rms": float(np.sqrt(np.mean((ia.astype(np.float64) - ib) ** 2)) / scale), + "bytes_equal": bool(np.array_equal(load(pa, i, "filter.out"), load(pb, i, "filter.out")))} + report["frames"].append({"frame": i, "reset": reset, "buffers": rows, "output": out}) + print(f"frame {i}{' (reset)' if reset else ''}: output pixels differing {out['pixels_differ_pct']:.3f} %, max {out['max_f16_steps']} FP16 steps, " + f"RMS diff / RMS {out['rms_diff_over_rms']:.2e}, byte-equal {out['bytes_equal']}") + for name, r in rows.items(): + print(f" {name:22s} {r['bytes']:>10d} B differ {r['differ']:>9d} ({100.0 * r['differ'] / max(r['bytes'], 1):6.3f} %){' (not counted: unwritten, stale or another format)' if r['skipped'] else ''}") + report["times"] = {a.a: times(pa), a.b: times(pb)} + print("GPU times:", report["times"]) + if a.json: + with open(a.json, "w") as f: + json.dump(report, f, indent=1) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + p = sub.add_parser("prepare") + p.add_argument("--out", required=True) + p.add_argument("--render", required=True) + p.add_argument("--output", required=True) + p.add_argument("--frames", type=int, default=3) + p.add_argument("--seed", type=int, default=1) + p = sub.add_parser("ref") + p.add_argument("--out", required=True) + p.add_argument("--name", choices=("hip", "emu"), required=True) + p.add_argument("--build", required=True) + p.add_argument("--rocm", default="") + p.add_argument("--time", type=int, default=0) + p = sub.add_parser("compare") + p.add_argument("--out", required=True) + p.add_argument("--a", default="hip") + p.add_argument("--b", default="d3d12") + p.add_argument("--json") + a = ap.parse_args() + {"prepare": prepare, "ref": reference, "compare": compare}[a.cmd](a) + + +if __name__ == "__main__": + main() diff --git a/clean/dx12/tests/gfx11_helpers.hlsl b/clean/dx12/tests/gfx11_helpers.hlsl new file mode 100644 index 00000000..2a2f1ee8 --- /dev/null +++ b/clean/dx12/tests/gfx11_helpers.hlsl @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: MIT +// Probe kernels of the gfx11 (RDNA 3) platform helpers (wg_platform.hlsli with WG_GFX 11), one entry point each, run by +// test_gfx11_helpers.py through gfx11_probe_host.cpp. Lane l of a 32-lane workgroup; every array is in the arena at the +// byte offset of its argument. Fragments are 8 halves per lane (uint4), accumulators 8 floats per lane stored element-major: +// element i of lane l at (i * 32 + l) * 4, matrix e at + e * 1024. +#include "wg_gfx11.hlsli" +WG_ARGS_BEGIN + uint pa, pb, pc, po, n, pa2, pb2; +WG_ARGS_END + +uint4 load_frag(uint base, uint e, uint l) { return wg_ld128(base, (e * 32u + l) * 16u); } +v8f load_acc(uint base, uint e, uint l) +{ + v8f c; + [unroll] for (uint i = 0; i < 8u; ++i) c.v[i] = wg_ldf(base, ((e * 8u + i) * 32u + l) * 4u); + return c; +} +void store_acc(uint base, uint e, uint l, v8f d) +{ + [unroll] for (uint i = 0; i < 8u; ++i) wg_stf(base, ((e * 8u + i) * 32u + l) * 4u, d.v[i]); +} + +// One raw instruction (K 0 .. 7), operands already replicated across the halves. +WG_KERNEL(k8) +{ + WG_SETUP(); + const uint l = wg_lane_id(); + [loop] for (uint e = 0; e < n; ++e) + store_acc(po, e, l, wg_wmma_f16_k8(load_frag(pa, e, l), load_frag(pb, e, l), load_acc(pc, e, l))); +} + +// K = 16: two chained instructions from replicated fragments (K 0 .. 7 at pa / pb, K 8 .. 15 at pa2 / pb2). +WG_KERNEL(k16) +{ + WG_SETUP(); + const uint l = wg_lane_id(); + [loop] for (uint e = 0; e < n; ++e) + store_acc(po, e, l, wg_wmma_f16_k16(load_frag(pa, e, l), load_frag(pb, e, l), load_frag(pa2, e, l), + load_frag(pb2, e, l), load_acc(pc, e, l))); +} + +// HIP's wg_wmma_f16 shape: lane l row / column l % 16, half h holds K 8 h .. 8 h + 7. +WG_KERNEL(f16_gfx12_shape) +{ + WG_SETUP(); + const uint l = wg_lane_id(); + [loop] for (uint e = 0; e < n; ++e) + store_acc(po, e, l, wg_wmma_f16(load_frag(pa, e, l), load_frag(pb, e, l), load_acc(pc, e, l))); +} + +// The driver's F32 -> F16 conversion (0x32) of 8 floats per lane: out = 4 dwords per lane. +WG_KERNEL(convert) +{ + WG_SETUP(); + const uint l = wg_lane_id(); + wg_st128(po, l * 16u, wg_to_f16_ags(load_acc(pa, 0u, l))); +} + +// wg_xor16 of one float per lane and of a uint4 (wg_xor16_4); the accumulator row table of the lane. +WG_KERNEL(xor16) +{ + WG_SETUP(); + const uint l = wg_lane_id(); + wg_stf(po, l * 4u, wg_xor16(wg_ldf(pa, l * 4u))); + wg_st128(po, 128u + l * 16u, wg_xor16_4(wg_ld128(pb, l * 16u))); +} +WG_KERNEL(d_row) +{ + WG_SETUP(); + const uint l = wg_lane_id(); + [unroll] for (uint i = 0; i < 8u; ++i) wg_st32(po, (i * 32u + l) * 4u, wg_d_row(i)); +} + +// wg_xor16 after a matrix instruction (the exchange of the accumulator rows 2 i and 2 i + 1). +WG_KERNEL(xor16_after_mma) +{ + WG_SETUP(); + const uint l = wg_lane_id(); + v8f d = wg_wmma_f16_k8(load_frag(pa, 0u, l), load_frag(pb, 0u, l), load_acc(pc, 0u, l)); + [unroll] for (uint i = 0; i < 8u; ++i) d.v[i] += wg_xor16(d.v[i]); + store_acc(po, 0u, l, d); +} + +// Rounding of FP32 -> FP16 in a shader that contains a matrix instruction (RADV): f32tof16, a (half) cast, the form the +// gfx11 branch of wg_gelu_lut_off2 uses, and the driver's conversion. 8 floats per lane at pa; out: three regions of +// 8 dwords per lane at po (f32tof16, cast, asfloat16(f32tof16)), then the convert (4 dwords per lane) at +3072 and the +// matrix result at +4096. +WG_KERNEL(round_modes) +{ + WG_SETUP(); + const uint l = wg_lane_id(); + const v8f c = load_acc(pa, 0u, l); + const v8f z = wg_wmma_f16_k8(uint4(0, 0, 0, 0), uint4(0, 0, 0, 0), c); // keeps the shader a matrix shader + const uint4 r = wg_to_f16_ags(c); + wg_st128(po, 3u * 1024u + l * 16u, r); + [unroll] for (uint k = 0; k < 8u; ++k) { + wg_st32(po, (k * 32u + l) * 4u, f32tof16(c.v[k])); + const half h = (half)c.v[k]; + wg_st32(po, 1024u + (k * 32u + l) * 4u, (uint)asuint16(h)); + const half g = asfloat16((uint16_t)f32tof16(c.v[k])); + wg_st32(po, 2048u + (k * 32u + l) * 4u, (uint)asuint16(g)); + wg_stf(po, 4096u + (k * 32u + l) * 4u, z.v[k]); + } +} diff --git a/clean/dx12/tests/gfx11_hip_ref.hip b/clean/dx12/tests/gfx11_hip_ref.hip new file mode 100644 index 00000000..2f100df7 --- /dev/null +++ b/clean/dx12/tests/gfx11_hip_ref.hip @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: MIT +// HIP reference of the gfx11 helper tests (test_gfx11_helpers.py): D = A x B + C by the native v_wmma_f32_16x16x16_f16 +// of gfx1100 (the HIP code's wg_wmma16_f16, clean/kernels/wg_platform.h), K = 16, one 16x16 matrix per workgroup. +// gfx11_hip_ref in: int32 N, A[N][16][16] f16, B[N][16][16] f16, C[N][16][16] f32 (row-major) +// out: D[N][16][16] f32 (row-major) +// A lane holds row (A) / column (B) l % 16 with K 0 .. 15 in order; element i of lane l of D is row 2 i + l / 16. +#include +#include +#include +typedef _Float16 v16h __attribute__((ext_vector_type(16))); +typedef float v8f __attribute__((ext_vector_type(8))); +__global__ void k(const _Float16* A, const _Float16* B, const float* C, float* D) { + const int n = blockIdx.x, l = threadIdx.x; + const _Float16* a = A + n * 256; const _Float16* b = B + n * 256; const float* c = C + n * 256; float* d = D + n * 256; + v16h fa, fb; for (int i = 0; i < 16; ++i) { fa[i] = a[(l % 16) * 16 + i]; fb[i] = b[i * 16 + (l % 16)]; } + v8f fc; for (int i = 0; i < 8; ++i) fc[i] = c[(2 * i + l / 16) * 16 + l % 16]; + const v8f r = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(fa, fb, fc); + for (int i = 0; i < 8; ++i) d[(2 * i + l / 16) * 16 + l % 16] = r[i]; +} +int main(int argc, char** argv) { + if (argc < 3) return 1; + FILE* f = fopen(argv[1], "rb"); if (!f) return 1; + int N = 0; if (fread(&N, 4, 1, f) != 1) return 1; + std::vector<_Float16> A(N * 256), B(N * 256); std::vector C(N * 256), D(N * 256); + if (fread(A.data(), 2, A.size(), f) != A.size() || fread(B.data(), 2, B.size(), f) != B.size() || + fread(C.data(), 4, C.size(), f) != C.size()) return 1; + fclose(f); + _Float16 *dA, *dB; float *dC, *dD; + hipMalloc(&dA, N * 512); hipMalloc(&dB, N * 512); hipMalloc(&dC, N * 1024); hipMalloc(&dD, N * 1024); + hipMemcpy(dA, A.data(), N * 512, hipMemcpyHostToDevice); hipMemcpy(dB, B.data(), N * 512, hipMemcpyHostToDevice); + hipMemcpy(dC, C.data(), N * 1024, hipMemcpyHostToDevice); + hipDeviceProp_t p; hipGetDeviceProperties(&p, 0); fprintf(stderr, "device %s %s\n", p.name, p.gcnArchName); + k<<>>(dA, dB, dC, dD); hipDeviceSynchronize(); + hipMemcpy(D.data(), dD, N * 1024, hipMemcpyDeviceToHost); + f = fopen(argv[2], "wb"); fwrite(D.data(), 4, D.size(), f); fclose(f); + return 0; +} diff --git a/clean/dx12/tests/gfx11_probe_host.cpp b/clean/dx12/tests/gfx11_probe_host.cpp new file mode 100644 index 00000000..17ecf929 --- /dev/null +++ b/clean/dx12/tests/gfx11_probe_host.cpp @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: MIT +// Probe host of the gfx11 helper tests (test_gfx11_helpers.py): runs ONE compute kernel (a DXIL file built by +// clean/dx12/build_dx12.py's compile_hlsl) on the native D3D12 device with one fixed root signature of the buffer kind (two root +// UAVs over one arena, root constants = group base, grid, then the kernel's arguments, one raw UAV table at space +// 0x7FFF0ADE for the driver's matrix channel) and writes the arena back. Windows build (llvm-mingw), run under Wine: +// gfx11_probe_host.exe [ ...] +// Prints "PSO hr=... OK|FAILED", then "DONE" after a clean run. It does not build each kernel's own root signature from the manifest (kernel_replay.cpp does). +#include +#include +#include +#include +#include +#include +#include +#include +#define CK(x) do { HRESULT h_ = (x); if (FAILED(h_)) { printf("FAIL %s hr=0x%08lx\n", #x, (unsigned long)h_); return 2; } } while (0) +static std::vector rd(const char* p) { std::ifstream f(p, std::ios::binary); return std::vector((std::istreambuf_iterator(f)), {}); } +int main(int argc, char** argv) { + if (argc < 6) return 1; + auto dxil = rd(argv[1]); auto in = rd(argv[2]); + UINT64 bytes = strtoull(argv[4], 0, 0); UINT groups = atoi(argv[5]); + std::vector rc = {0, groups}; + for (int i = 6; i < argc; ++i) rc.push_back((UINT)strtoul(argv[i], 0, 0)); + HMODULE m = LoadLibraryA("d3d12.dll"); + auto create = (PFN_D3D12_CREATE_DEVICE)GetProcAddress(m, "D3D12CreateDevice"); + auto ser = (PFN_D3D12_SERIALIZE_ROOT_SIGNATURE)GetProcAddress(m, "D3D12SerializeRootSignature"); + ID3D12Device* dev; CK(create(nullptr, D3D_FEATURE_LEVEL_12_0, __uuidof(ID3D12Device), (void**)&dev)); + auto buf = [&](D3D12_HEAP_TYPE t, UINT64 sz, D3D12_RESOURCE_FLAGS fl, D3D12_RESOURCE_STATES st) { + D3D12_HEAP_PROPERTIES hp{}; hp.Type = t; + D3D12_RESOURCE_DESC d{}; d.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; d.Width = sz; d.Height = d.DepthOrArraySize = d.MipLevels = 1; + d.SampleDesc.Count = 1; d.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; d.Flags = fl; + ID3D12Resource* r = nullptr; HRESULT h = dev->CreateCommittedResource(&hp, D3D12_HEAP_FLAG_NONE, &d, st, nullptr, __uuidof(ID3D12Resource), (void**)&r); + if (FAILED(h)) { printf("FAIL buffer 0x%lx\n", (unsigned long)h); exit(2); } return r; }; + ID3D12Resource* arena = buf(D3D12_HEAP_TYPE_DEFAULT, bytes, D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, D3D12_RESOURCE_STATE_COMMON); + ID3D12Resource* chan = buf(D3D12_HEAP_TYPE_DEFAULT, 256, D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, D3D12_RESOURCE_STATE_COMMON); + ID3D12Resource* up = buf(D3D12_HEAP_TYPE_UPLOAD, bytes, D3D12_RESOURCE_FLAG_NONE, D3D12_RESOURCE_STATE_GENERIC_READ); + ID3D12Resource* rb = buf(D3D12_HEAP_TYPE_READBACK, bytes, D3D12_RESOURCE_FLAG_NONE, D3D12_RESOURCE_STATE_COPY_DEST); + { void* p; D3D12_RANGE r{0,0}; up->Map(0, &r, &p); memset(p, 0, bytes); memcpy(p, in.data(), in.size() < bytes ? in.size() : bytes); up->Unmap(0, nullptr); } + D3D12_DESCRIPTOR_RANGE rg{}; rg.RangeType = D3D12_DESCRIPTOR_RANGE_TYPE_UAV; rg.NumDescriptors = 1; rg.RegisterSpace = 0x7FFF0ADE; + rg.OffsetInDescriptorsFromTableStart = 0; + D3D12_ROOT_PARAMETER pr[4] = {}; + pr[0].ParameterType = pr[1].ParameterType = D3D12_ROOT_PARAMETER_TYPE_UAV; pr[1].Descriptor.ShaderRegister = 1; + pr[2].ParameterType = D3D12_ROOT_PARAMETER_TYPE_32BIT_CONSTANTS; pr[2].Constants.Num32BitValues = (UINT)rc.size(); + pr[3].ParameterType = D3D12_ROOT_PARAMETER_TYPE_DESCRIPTOR_TABLE; pr[3].DescriptorTable.NumDescriptorRanges = 1; pr[3].DescriptorTable.pDescriptorRanges = &rg; + D3D12_ROOT_SIGNATURE_DESC sd{}; sd.NumParameters = 4; sd.pParameters = pr; + ID3DBlob *blob = nullptr, *err = nullptr; CK(ser(&sd, D3D_ROOT_SIGNATURE_VERSION_1, &blob, &err)); + ID3D12RootSignature* rs; CK(dev->CreateRootSignature(0, blob->GetBufferPointer(), blob->GetBufferSize(), __uuidof(ID3D12RootSignature), (void**)&rs)); + D3D12_COMPUTE_PIPELINE_STATE_DESC pd{}; pd.pRootSignature = rs; pd.CS = {dxil.data(), dxil.size()}; + ID3D12PipelineState* pso = nullptr; HRESULT h = dev->CreateComputePipelineState(&pd, __uuidof(ID3D12PipelineState), (void**)&pso); + printf("PSO hr=0x%08lx %s\n", (unsigned long)h, SUCCEEDED(h) ? "OK" : "FAILED"); fflush(stdout); + if (FAILED(h)) return 3; + D3D12_DESCRIPTOR_HEAP_DESC hd{}; hd.Type = D3D12_DESCRIPTOR_HEAP_TYPE_CBV_SRV_UAV; hd.NumDescriptors = 1; hd.Flags = D3D12_DESCRIPTOR_HEAP_FLAG_SHADER_VISIBLE; + ID3D12DescriptorHeap* dh; CK(dev->CreateDescriptorHeap(&hd, __uuidof(ID3D12DescriptorHeap), (void**)&dh)); + D3D12_UNORDERED_ACCESS_VIEW_DESC uv{}; uv.Format = DXGI_FORMAT_R32_TYPELESS; uv.ViewDimension = D3D12_UAV_DIMENSION_BUFFER; uv.Buffer.NumElements = 64; uv.Buffer.Flags = D3D12_BUFFER_UAV_FLAG_RAW; + dev->CreateUnorderedAccessView(chan, nullptr, &uv, dh->GetCPUDescriptorHandleForHeapStart()); + D3D12_COMMAND_QUEUE_DESC qd{}; qd.Type = D3D12_COMMAND_LIST_TYPE_COMPUTE; + ID3D12CommandQueue* q; CK(dev->CreateCommandQueue(&qd, __uuidof(ID3D12CommandQueue), (void**)&q)); + ID3D12CommandAllocator* al; CK(dev->CreateCommandAllocator(D3D12_COMMAND_LIST_TYPE_COMPUTE, __uuidof(ID3D12CommandAllocator), (void**)&al)); + ID3D12GraphicsCommandList* cl; CK(dev->CreateCommandList(0, D3D12_COMMAND_LIST_TYPE_COMPUTE, al, nullptr, __uuidof(ID3D12GraphicsCommandList), (void**)&cl)); + auto bar = [&](ID3D12Resource* r, D3D12_RESOURCE_STATES a, D3D12_RESOURCE_STATES b) { + D3D12_RESOURCE_BARRIER x{}; x.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; x.Transition.pResource = r; x.Transition.StateBefore = a; x.Transition.StateAfter = b; + x.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES; cl->ResourceBarrier(1, &x); }; + bar(arena, D3D12_RESOURCE_STATE_COMMON, D3D12_RESOURCE_STATE_COPY_DEST); + cl->CopyBufferRegion(arena, 0, up, 0, bytes); + bar(arena, D3D12_RESOURCE_STATE_COPY_DEST, D3D12_RESOURCE_STATE_UNORDERED_ACCESS); + bar(chan, D3D12_RESOURCE_STATE_COMMON, D3D12_RESOURCE_STATE_UNORDERED_ACCESS); + ID3D12DescriptorHeap* hs[] = {dh}; cl->SetDescriptorHeaps(1, hs); + cl->SetComputeRootSignature(rs); cl->SetPipelineState(pso); + cl->SetComputeRootUnorderedAccessView(0, arena->GetGPUVirtualAddress()); + cl->SetComputeRootUnorderedAccessView(1, arena->GetGPUVirtualAddress()); + cl->SetComputeRoot32BitConstants(2, (UINT)rc.size(), rc.data(), 0); + cl->SetComputeRootDescriptorTable(3, dh->GetGPUDescriptorHandleForHeapStart()); + LARGE_INTEGER t0,t1,fq; QueryPerformanceFrequency(&fq); + cl->Dispatch(groups, 1, 1); + bar(arena, D3D12_RESOURCE_STATE_UNORDERED_ACCESS, D3D12_RESOURCE_STATE_COPY_SOURCE); + cl->CopyBufferRegion(rb, 0, arena, 0, bytes); + CK(cl->Close()); + ID3D12CommandList* ls[] = {cl}; QueryPerformanceCounter(&t0); q->ExecuteCommandLists(1, ls); + ID3D12Fence* f; CK(dev->CreateFence(0, D3D12_FENCE_FLAG_NONE, __uuidof(ID3D12Fence), (void**)&f)); + HANDLE ev = CreateEventA(nullptr, FALSE, FALSE, nullptr); q->Signal(f, 1); f->SetEventOnCompletion(1, ev); WaitForSingleObject(ev, 20000); QueryPerformanceCounter(&t1); printf("wall_ms %.3f\n", 1000.0*(t1.QuadPart-t0.QuadPart)/fq.QuadPart); + printf("fence %llu removed=0x%lx\n", (unsigned long long)f->GetCompletedValue(), (unsigned long)dev->GetDeviceRemovedReason()); + void* p; D3D12_RANGE r{0, (SIZE_T)bytes}; CK(rb->Map(0, &r, &p)); + std::ofstream o(argv[3], std::ios::binary); o.write((char*)p, bytes); o.close(); + printf("DONE\n"); + return 0; +} diff --git a/clean/dx12/tests/native_probe_test.cpp b/clean/dx12/tests/native_probe_test.cpp index 4998782b..2f2be62c 100644 --- a/clean/dx12/tests/native_probe_test.cpp +++ b/clean/dx12/tests/native_probe_test.cpp @@ -8,6 +8,8 @@ // ../../runtime/{device,device_dx12,device_d3d12,embedded,model,no_model_k,runtime}.cpp // ../../runtime/interop/{convert,native_candidate}.cpp ../../converter/src/{json,sha256}.cpp -o native_probe_test.exe // With WG_TEST_DRIVER_PROBE_FAIL=1 the create must fail with "probe mismatch at float 7". Exit status 0 on PASS. +// Built with -DWG_DEV_GFX11 it takes an RX 7900 series board too (runtime/rdna4.h) and the build dir then holds +// dx12-gfx11\manifest.json with native_driver_probe_f16 (build_dx12.py --gfx 11). #define WIN32_LEAN_AND_MEAN #define NOMINMAX #include @@ -46,7 +48,8 @@ int main(int argc, char** argv) { DXGI_ADAPTER_DESC1 desc{}; char n[256] = {}; if (SUCCEEDED(a->GetDesc1(&desc))) WideCharToMultiByte(CP_UTF8, 0, desc.Description, -1, n, sizeof n - 1, nullptr, nullptr); - if (wgrt::rdna4_identity(wgrt::read_adapter_identity(desc.AdapterLuid))) { // the driver's ids (rdna4_win.h) + const wgrt::AdapterIdentity identity = wgrt::read_adapter_identity(desc.AdapterLuid); // the driver's ids (rdna4_win.h) + if (wgrt::rdna4_identity(identity) || wgrt::dev_gfx11_identity(identity)) { // (the second: WG_DEV_GFX11 builds only) adapter = a; name = n; break; diff --git a/clean/dx12/tests/pso_sweep.cpp b/clean/dx12/tests/pso_sweep.cpp new file mode 100644 index 00000000..bc7f686d --- /dev/null +++ b/clean/dx12/tests/pso_sweep.cpp @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: MIT +// PSO-creation sweep: does the driver stack create a compute pipeline for each DXIL file, with the root signature the +// runtime gives that kernel? Windows build (llvm-mingw, build_kernel_replay.py), run under Wine headless. +// pso_sweep.exe list: one kernel per line " " +// Prints "PSO hr=0x........ OK|FAILED" per line, then "SWEEP DONE". The two root signatures are those of +// clean/runtime/device_d3d12.cpp (create_root_signature: buffer; create_direct_objects: direct), written out here so that +// the sweep needs neither the runtime nor its gates (the runtime refuses a card that is not RDNA 4). +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const UINT kArgs = 57, kConstants = 2, kGeluT5Slot = 7; + +int main(int argc, char** argv) { + if (argc != 2) { std::printf("usage: pso_sweep \n"); return 2; } + HMODULE m = LoadLibraryA("d3d12.dll"); + if (!m) { std::printf("no d3d12.dll\n"); return 2; } + auto create = (PFN_D3D12_CREATE_DEVICE)GetProcAddress(m, "D3D12CreateDevice"); + auto ser = (PFN_D3D12_SERIALIZE_ROOT_SIGNATURE)GetProcAddress(m, "D3D12SerializeRootSignature"); + ID3D12Device* dev = nullptr; + HRESULT h = create(nullptr, D3D_FEATURE_LEVEL_12_0, __uuidof(ID3D12Device), (void**)&dev); + if (FAILED(h)) { std::printf("D3D12CreateDevice hr=0x%08lx\n", (unsigned long)h); return 2; } + + D3D12_DESCRIPTOR_RANGE ranges[4] = {}; + D3D12_ROOT_PARAMETER pr[4] = {}; + pr[0].ParameterType = pr[1].ParameterType = D3D12_ROOT_PARAMETER_TYPE_UAV; + pr[1].Descriptor.ShaderRegister = 1; + pr[2].ParameterType = D3D12_ROOT_PARAMETER_TYPE_32BIT_CONSTANTS; + pr[2].Constants.Num32BitValues = kConstants + kArgs; + pr[3].ParameterType = D3D12_ROOT_PARAMETER_TYPE_DESCRIPTOR_TABLE; + D3D12_ROOT_SIGNATURE_DESC sd{}; + sd.NumParameters = 4; sd.pParameters = pr; + auto make_rs = [&](ID3D12RootSignature** out) { + ID3DBlob *blob = nullptr, *err = nullptr; + HRESULT r = ser(&sd, D3D_ROOT_SIGNATURE_VERSION_1, &blob, &err); + if (SUCCEEDED(r)) r = dev->CreateRootSignature(0, blob->GetBufferPointer(), blob->GetBufferSize(), __uuidof(ID3D12RootSignature), (void**)out); + return r; + }; + // buffer: channel UAV table (u0, space 0x7FFF0ADE) + the GELU T5 SRV (t0, space 1) at slot 7 + ranges[0].RangeType = D3D12_DESCRIPTOR_RANGE_TYPE_UAV; ranges[0].NumDescriptors = 1; ranges[0].RegisterSpace = 0x7FFF0ADE; + ranges[1].RangeType = D3D12_DESCRIPTOR_RANGE_TYPE_SRV; ranges[1].NumDescriptors = 1; ranges[1].RegisterSpace = 1; + ranges[1].OffsetInDescriptorsFromTableStart = kGeluT5Slot; + pr[3].DescriptorTable.NumDescriptorRanges = 2; pr[3].DescriptorTable.pDescriptorRanges = ranges; + ID3D12RootSignature *rs_buffer = nullptr, *rs_direct = nullptr; + if (FAILED(h = make_rs(&rs_buffer))) { std::printf("buffer root signature hr=0x%08lx\n", (unsigned long)h); return 2; } + // direct: channel UAV, four SRVs (t0..t3), output UAV (u3), CBV (b1) + D3D12_DESCRIPTOR_RANGE d[4] = {}; + d[0] = ranges[0]; d[0].OffsetInDescriptorsFromTableStart = 0; + d[1].RangeType = D3D12_DESCRIPTOR_RANGE_TYPE_SRV; d[1].NumDescriptors = 4; d[1].OffsetInDescriptorsFromTableStart = 1; + d[2].RangeType = D3D12_DESCRIPTOR_RANGE_TYPE_UAV; d[2].NumDescriptors = 1; d[2].BaseShaderRegister = 3; d[2].OffsetInDescriptorsFromTableStart = 5; + d[3].RangeType = D3D12_DESCRIPTOR_RANGE_TYPE_CBV; d[3].NumDescriptors = 1; d[3].BaseShaderRegister = 1; d[3].OffsetInDescriptorsFromTableStart = 6; + pr[3].DescriptorTable.NumDescriptorRanges = 4; pr[3].DescriptorTable.pDescriptorRanges = d; + if (FAILED(h = make_rs(&rs_direct))) { std::printf("direct root signature hr=0x%08lx\n", (unsigned long)h); return 2; } + + std::ifstream list(argv[1]); + std::string line; + while (std::getline(list, line)) { + if (!line.empty() && line.back() == '\r') line.pop_back(); + std::istringstream s(line); + std::string name, rs, file; + if (!(s >> name >> rs >> file)) continue; + std::ifstream f(file, std::ios::binary); + std::vector code((std::istreambuf_iterator(f)), {}); + D3D12_COMPUTE_PIPELINE_STATE_DESC pd{}; + pd.pRootSignature = rs == "direct" ? rs_direct : rs_buffer; + pd.CS = {code.data(), code.size()}; + ID3D12PipelineState* pso = nullptr; + HRESULT r = code.empty() ? E_FAIL : dev->CreateComputePipelineState(&pd, __uuidof(ID3D12PipelineState), (void**)&pso); + std::printf("PSO %s %s hr=0x%08lx %s\n", name.c_str(), rs.c_str(), (unsigned long)r, SUCCEEDED(r) ? "OK" : "FAILED"); + std::fflush(stdout); + if (pso) pso->Release(); + } + std::printf("SWEEP DONE\n"); + return 0; +} diff --git a/clean/dx12/tests/pso_sweep.py b/clean/dx12/tests/pso_sweep.py new file mode 100644 index 00000000..ebfc165f --- /dev/null +++ b/clean/dx12/tests/pso_sweep.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: MIT +"""PSO-creation sweep over a folder of built D3D12 kernels (/manifest.json + the DXIL files): creates each kernel's +pipeline with the runtime's root signature on the card (pso_sweep.exe under Wine, wine_headless_run.sh) and prints a table. +Each kernel is also classified from its DXIL disassembly (dxc -dumpbin): does it contain an FP8 matrix op or an FP8 +conversion of the driver channel (code words of wg_ags.hlsli)? On RDNA 3 a PSO with one is expected to fail, any other to +succeed; a surprise returns 1. + python3 pso_sweep.py --dxil-dir /dx12 --exe pso_sweep.exe --run wine_headless_run.sh [] +Environment: WG_DXC_DIR (dxc.exe, as build_dx12.py), plus whatever --run needs.""" +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.dirname(HERE)) +import build_dx12 as B # noqa: E402 + +# phase 1 code words 0x50000000 | 1 << 24 | imm << 8 | op: FP8 matrix op (op 0x28, format 0x7), FP8 -> F32 convert (op 0x32, +# source type 0x1a), F32 -> FP8 convert (op 0x32, source type 0x26, the destination type 0x1a is the data value) +FP8_MMA = 0x50000000 | 1 << 24 | 0x7 << 8 | 0x28 +FP8_UNPACK = 0x50000000 | 1 << 24 | 0x1A << 8 | 0x32 +CONVERT_F32 = 0x50000000 | 1 << 24 | 0x26 << 8 | 0x32 + + +def has_fp8(dxil): + r = subprocess.run([os.path.join(B.DXC_DIR, "dxc.exe"), "-dumpbin", dxil], capture_output=True, text=True) + text = r.stdout + consts = lambda v: re.search(r"(? F16 converts use the same code word with data 0x24; an FP8 target has the data value 26 next to it + return consts(CONVERT_F32) and re.search(r"i32 %d, i32 26," % CONVERT_F32, text) is not None + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--dxil-dir", required=True) + ap.add_argument("--exe", required=True) + ap.add_argument("--run", required=True, help="the headless Wine runner script") + ap.add_argument("--work", help="folder for the list file (default: next to the DXIL folder)") + ap.add_argument("--timeout", type=int, default=1800) + a = ap.parse_args() + with open(os.path.join(a.dxil_dir, "manifest.json")) as f: + kernels = json.load(f)["kernels"] + work = a.work or os.path.dirname(os.path.abspath(a.dxil_dir)) + lst = os.path.join(work, "pso_sweep.list") + got = {} + todo = [k for k in kernels if k.get("rs", "buffer") != "tex"] # the runtime has no root signature of the tex kind + log = "" + while todo: # a driver crash ends the process: the kernel being created is recorded as CRASH, the rest run again + with open(lst, "w") as f: + for k in todo: + f.write(f"{k['file']} {k.get('rs', 'buffer')} {os.path.abspath(os.path.join(a.dxil_dir, k['file']))}\n") + r = subprocess.run([a.run, "--", os.path.abspath(a.exe), lst], capture_output=True, text=True, timeout=a.timeout) + log = r.stdout + r.stderr + for line in r.stdout.splitlines(): + m = re.match(r"PSO (\S+) (\S+) hr=(0x[0-9a-f]+) (OK|FAILED)", line) + if m: + got[m.group(1)] = (m.group(3), m.group(4) == "OK") + todo = [k for k in todo if k["file"] not in got] + if "SWEEP DONE" in r.stdout: + break + if not todo or "PSO " not in r.stdout and "D3D12CreateDevice" in log: + print(log[-600:]) + return 2 + got[todo[0]["file"]] = ("CRASH", False) # the first kernel without a line + todo = todo[1:] + bad = 0 + print(f"{'kernel':44s} {'rs':7s} {'fp8':4s} {'hr':12s} result") + for k in kernels: + fp8 = has_fp8(os.path.join(a.dxil_dir, k["file"])) + if k.get("rs") == "tex": + print(f"{k['name']:44s} tex {'yes' if fp8 else 'no':4s} {'-':12s} not tried (no root signature of this kind in the runtime)") + continue + hr, ok = got.get(k["file"], ("-", False)) + surprise = ok == fp8 and hr != "CRASH" + bad += surprise + print(f"{k['name']:44s} {k.get('rs', 'buffer'):7s} {'yes' if fp8 else 'no':4s} {hr:12s} {'OK' if ok else ('CRASH' if hr == 'CRASH' else 'FAILED')}" + f"{' <-- unexpected' if surprise else ''}") + n_ok = sum(1 for v in got.values() if v[1]) + n_crash = sum(1 for v in got.values() if v[0] == "CRASH") + print(f"{len(kernels)} kernels: {n_ok} created, {len(got) - n_ok - n_crash} failed, {n_crash} crashed the driver, " + f"{len(kernels) - len(got)} not tried, {bad} unexpected") + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/dx12/tests/test_build_gfx.py b/clean/dx12/tests/test_build_gfx.py new file mode 100644 index 00000000..3775a3e2 --- /dev/null +++ b/clean/dx12/tests/test_build_gfx.py @@ -0,0 +1,122 @@ +# SPDX-License-Identifier: MIT +"""build_dx12.py --gfx: the GPU family of a build (declarations, output folder, manifest key, defines). Nothing is +compiled: compile_hlsl and the assembler tool are replaced, so the test runs everywhere in a second.""" +import json +import os +import sys +import tempfile +import unittest +from unittest import mock + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.dirname(HERE)) +import build_dx12 as B # noqa: E402 + +SRC = """// @wg-kernel old_kernel co=a.co block=32 wave=32 args=p +// @wg-kernel both_kernel co=b.co block=32 wave=32 args=p gfx=11,12 +// @wg-kernel rdna3_kernel co=- block=32 wave=32 args=p,u gfx=11 entry=k11 +""" + + +class TestBuildGfx(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.kdir = os.path.join(self.tmp.name, "kernels") + os.makedirs(self.kdir) + with open(os.path.join(self.kdir, "t.hlsl"), "w") as f: + f.write(SRC) + self.out = os.path.join(self.tmp.name, "out") + self.calls = [] + + def fake_compile(src, entry, out_dxil, defines=None, **kw): + self.calls.append((entry, dict(defines or {}))) + with open(out_dxil, "wb") as f: + f.write(b"DXIL" + entry.encode()) + return out_dxil + + for p in (mock.patch.object(B, "KERNEL_DIR", self.kdir), mock.patch.object(B, "K_KERNEL_DIR", "/nonexistent"), + mock.patch.object(B, "build_asm_tool", lambda out: "asm"), + mock.patch.object(B, "compile_hlsl", fake_compile), + mock.patch.object(B, "vgpr_check", lambda built, required=(): self.calls.append(("vgpr_check", len(built))))): + p.start() + self.addCleanup(p.stop) + + def tearDown(self): + self.tmp.cleanup() + + def manifest(self, folder): + with open(os.path.join(self.out, folder, "manifest.json")) as f: + return json.load(f) + + def build(self, *args): + with mock.patch("sys.stdout"): + self.assertEqual(B.main(["--out", self.out, *args]), 0) + + def test_declarations_default_to_gfx12(self): + decls = {d["name"]: d["gfx"] for d in B.kernel_decls(os.path.join(self.kdir, "t.hlsl"))} + self.assertEqual(decls, {"old_kernel": {12}, "both_kernel": {11, 12}, "rdna3_kernel": {11}}) + + def test_bad_family_in_a_declaration_is_refused(self): + path = os.path.join(self.kdir, "bad.hlsl") + with open(path, "w") as f: + f.write("// @wg-kernel x co=a.co block=32 wave=32 args=p gfx=13\n") + with self.assertRaises(SystemExit): + B.kernel_decls(path) + + def test_kernel_sets_by_family(self): + self.assertEqual(B.kernel_set(), [("both_kernel", "b.co"), ("old_kernel", "a.co")]) + self.assertEqual(B.kernel_set(gfx=12), B.kernel_set()) + self.assertEqual(B.kernel_set(gfx=11), [("both_kernel", "b.co"), ("rdna3_kernel", "-")]) + + def test_gfx12_build_is_as_before(self): + self.build() + man = self.manifest("dx12") + self.assertEqual(list(man), ["format", "dxc", "test_only", "kernels"]) # no gfx key: a gfx12 manifest keeps its bytes + self.assertEqual([k["name"] for k in man["kernels"]], ["both_kernel", "old_kernel"]) + for _, defs in [c for c in self.calls if c[0] != "vgpr_check"]: + self.assertNotIn("WG_GFX", defs) + self.assertIn(("vgpr_check", 2), self.calls) + self.assertFalse(os.path.exists(os.path.join(self.out, "dx12-gfx11"))) + + def test_a_source_on_another_drive_still_builds(self): + real = os.path.relpath + tmp = self.tmp.name + + def cross_drive(path, start=os.curdir): # Windows: relpath raises across drives + if path.startswith(tmp) != str(start).startswith(tmp): + raise ValueError(f"path is on mount 'C:', start on mount 'D:'") + return real(path, start) + + with mock.patch.object(B.os.path, "relpath", cross_drive): + self.build() + self.assertEqual([k["name"] for k in self.manifest("dx12")["kernels"]], ["both_kernel", "old_kernel"]) + + def test_gfx11_build(self): + self.build("--gfx", "11") + man = self.manifest("dx12-gfx11") + self.assertEqual(man["gfx"], 11) + self.assertEqual([k["name"] for k in man["kernels"]], ["both_kernel", "rdna3_kernel"]) + self.assertEqual(sorted(os.listdir(os.path.join(self.out, "dx12-gfx11"))), + ["both_kernel.dxil", "manifest.json", "rdna3_kernel.dxil"]) + self.assertEqual(sorted(e for e, _ in self.calls), ["both_kernel", "k11"]) + for _, defs in self.calls: + self.assertEqual(defs["WG_GFX"], 11) + self.assertNotIn(("vgpr_check", 2), self.calls) # RGA is not used for gfx11 + self.assertFalse(os.path.exists(os.path.join(self.out, "dx12", "manifest.json"))) + + def test_gfx11_only_picks_gfx11_kernels(self): + with self.assertRaises(SystemExit): + self.build("--gfx", "11", "--only", "old_kernel") # a gfx12-only kernel is not declared for gfx11 + + def test_isa_is_refused_for_gfx11(self): + with self.assertRaises(SystemExit): + self.build("--gfx", "11", "--isa") + + def test_list_follows_the_family(self): + with mock.patch("builtins.print") as pr: + B.main(["--list", "--gfx", "11"]) + self.assertEqual(json.loads(pr.call_args[0][0]), [["both_kernel", "b.co"], ["rdna3_kernel", "-"]]) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/dx12/tests/test_gfx11_helpers.py b/clean/dx12/tests/test_gfx11_helpers.py new file mode 100644 index 00000000..51203d43 --- /dev/null +++ b/clean/dx12/tests/test_gfx11_helpers.py @@ -0,0 +1,386 @@ +# SPDX-License-Identifier: MIT +"""The gfx11 (RDNA 3) platform helpers of hlsl/wg_gfx11.hlsli (WG_GFX 11), run headless on the card through +vkd3d-proton under Wine and compared with numpy and with a small HIP kernel on the same card (the native +v_wmma_f32_16x16x16_f16 of gfx1100): fragment layout of the raw F16 matrix instruction, the K = 16 step as two chained +instructions, the accumulator row table, the F32 -> F16 conversion and the half-wave exchange. The layout and the +numbers are those measured on the card (RX 7900 XT, vkd3d-proton with extended emulation, RADV). + +The card tests need a prefix and tools; each test is skipped without them (the compile checks need only DXC): + WG_GFX11_WORK scratch folder (never a memory-backed one) for the DXIL, the host and the data + WG_GFX11_WRUN runner script: `