diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index e1e9ac90..05dd92fa 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -7,6 +7,9 @@ Model L's kernels (#89: those that reproduce one of Model L's code objects, co=l_*.co, direct-route kernels named l_*, and MODEL_L_ONLY) are built only with --model-l or when --only names them, so that a build without the option is Model M's kernel set unchanged. Frame generation's kernels (#96: co=fg_*.co or named fg_*) are built only when --only names them, never by a full build. +Model K's kernels (clean/k/dx12/kernels, co=k_*.co: #15) are built only with --model-k or when --only names them, so +that a build without the option is the same kernel set as before. A kernel that takes a struct with pointers in it by +value (a HIP kernel's parameter block) declares the argument as s: (clean/runtime/struct_args.h). Every kernel source declares its kernels in header lines (clean/dx12/README.md): @@ -48,6 +51,11 @@ HERE = os.path.dirname(os.path.abspath(__file__)) KERNEL_DIR = os.path.join(HERE, "kernels") HLSL_DIR = os.path.join(HERE, "hlsl") +# Model K's ports (clean/k/dx12): their kernels are built only with --model-k (or when --only names them); their +# sources may include both hlsl folders (clean/k/dx12/hlsl first). +K_DX12_DIR = os.path.join(os.path.dirname(HERE), "k", "dx12") +K_KERNEL_DIR = os.path.join(K_DX12_DIR, "kernels") +K_HLSL_DIR = os.path.join(K_DX12_DIR, "hlsl") # Pinned tools (BUILDING.md): WG_TOOLCHAINS = a folder holding dxc-v1.10.2605.24\ and rga-2.15\ (default \toolchains); # WG_DXC_DIR / WG_RGA point at the tools directly. TOOLCHAINS = os.environ.get("WG_TOOLCHAINS", os.path.join(os.path.dirname(os.path.dirname(HERE)), "toolchains")) @@ -108,8 +116,17 @@ def fdiv(m): return text -def compile_hlsl(src, entry, out_dxil, defines=None, includes=(), asm_tool=None, keep_listing=None, allow_fdiv=False): - """One entry point of an HLSL file in the exact arithmetic mode -> signed DXIL container at out_dxil.""" +# Binary64 operations of the D3D11.1 extended doubles (double fma = DXIL opcode 47, double division), which D3D12 reports +# in D3D12_FEATURE_DATA_D3D12_OPTIONS.DoublePrecisionFloatShaderOps (no separate field): such a port is marked +# "fp64": "ext" in the manifest, and the devices drop it when the adapter lacks the feature (the kernel then has no +# port: HIP in the hybrid device, a refused context on the native one). +_EXT_DOUBLES = re.compile(r"@dx\.op\.tertiary\.f64\(i32 47,|\bfdiv (fast )?(<\d+ x )?double\b") + + +def compile_hlsl(src, entry, out_dxil, defines=None, includes=(), asm_tool=None, keep_listing=None, allow_fdiv=False, + info=None): + """One entry point of an HLSL file in the exact arithmetic mode -> signed DXIL container at out_dxil. info (a dict), + if given, receives "ext_doubles": whether the kernel uses extended binary64 instructions.""" dxc, dxv = os.path.join(DXC_DIR, "dxc.exe"), os.path.join(DXC_DIR, "dxv.exe") with tempfile.TemporaryDirectory(prefix="wgdx12_") as tmp: ll = os.path.join(tmp, "a.ll") @@ -120,7 +137,9 @@ def compile_hlsl(src, entry, out_dxil, defines=None, includes=(), asm_tool=None, raise SystemExit(f"dxc failed for {src} ({entry}):\n{r.stdout}{r.stderr}") with open(ll, encoding="utf-8", errors="replace") as f: text = exact_listing(f.read(), f"{os.path.basename(src)}:{entry}", allow_fdiv) - ll2 = os.path.join(tmp, "b.ll") + if info is not None: + info["ext_doubles"] = bool(_EXT_DOUBLES.search(text)) + ll2 =os.path.join(tmp, "b.ll") with open(ll2, "w", encoding="utf-8") as f: f.write(text) if keep_listing: @@ -194,21 +213,58 @@ def is_frame_gen(decl): return decl["co"].startswith("fg_") or decl["name"].startswith("fg_") -def kernel_set(model_l=False): +def is_model_k(decl): + """A port of one of Model K's code objects (clean/k/kernels: k_block.co, k_block_wide.co, k_io.co, k_io_exact.co, + k_act.co ...) or a Model K kernel with no code object (co "-" or a helper, named k_*). All of them live in + clean/k/dx12/kernels: Model M's and Model L's sets never contain one.""" + return decl["co"].startswith("k_") or decl["name"].startswith("k_") or decl["name"].startswith("kx/") + + +def source_files(model_k=False, only_k=False): + """The .hlsl sources of a build, (path, is_k_source): clean/dx12/kernels, plus clean/k/dx12/kernels with model_k.""" + out = [] + if not only_k and os.path.isdir(KERNEL_DIR): + out += [(os.path.join(KERNEL_DIR, fn), False) for fn in sorted(os.listdir(KERNEL_DIR)) if fn.endswith(".hlsl")] + if (model_k or only_k) and os.path.isdir(K_KERNEL_DIR): + out += [(os.path.join(K_KERNEL_DIR, fn), True) for fn in sorted(os.listdir(K_KERNEL_DIR)) if fn.endswith(".hlsl")] + return out + + +def kernel_set(model_l=False, model_k=False): """(name, code object) of every kernel a full build compiles (no --only), sorted: Model M's set, plus Model L's - with model_l. From the sources' declarations alone (nothing is compiled). Frame generation's kernels are never in - it (is_frame_gen).""" + with model_l, plus Model K's with model_k. From the sources' declarations alone (nothing is compiled). Frame + generation's kernels are never in it (is_frame_gen).""" keys = set() - for fn in sorted(os.listdir(KERNEL_DIR)) if os.path.isdir(KERNEL_DIR) else []: - if fn.endswith(".hlsl"): - for d in kernel_decls(os.path.join(KERNEL_DIR, fn)): - if is_frame_gen(d): - continue - if model_l or not is_model_l(d): + for path, k_source in source_files(model_k): + for d in kernel_decls(path): + if is_frame_gen(d): + continue + if k_source or is_model_k(d): + if model_k: keys.add((d["name"], d["co"])) + elif model_l or not is_model_l(d): + keys.add((d["name"], d["co"])) return sorted(keys) +def struct_dwords(a, where): + """Root constants of a struct argument s: (clean/runtime/struct_args.h): p n pointers of 8 bytes, one + constant each (the arena offset); b n bytes copied; x n bytes skipped. The fields add up to N bytes.""" + m = re.fullmatch(r"s(\d+):((?:[pbx]\d+)(?:\.[pbx]\d+)*)", a) + if not m: + raise SystemExit(f"{where}: struct argument {a!r} (s:.<...>)") + total, seen, n = int(m.group(1)), 0, 0 + for f in m.group(2).split("."): + k, c = f[0], int(f[1:]) + if c <= 0 or (k != "p" and c % 4): + raise SystemExit(f"{where}: struct field {f!r} in {a!r}") + seen += 8 * c if k == "p" else c + n += c if k == "p" else (c // 4 if k == "b" else 0) + if seen != total: + raise SystemExit(f"{where}: struct fields of {a!r} give {seen} bytes, not {total}") + return n + + def arg_dwords(args, where): n = 0 for a in args.split(","): @@ -216,8 +272,10 @@ def arg_dwords(args, where): n += 1 elif re.fullmatch(r"b\d+", a) and int(a[1:]) % 4 == 0 and int(a[1:]) > 0: n += int(a[1:]) // 4 + elif a.startswith("s"): + n += struct_dwords(a, where) else: - raise SystemExit(f"{where}: argument kind {a!r} (p, u, b with N a multiple of 4)") + raise SystemExit(f"{where}: argument kind {a!r} (p, u, b with N a multiple of 4, s:)") if n > MAX_ARG_DWORDS: raise SystemExit(f"{where}: {n} argument DWORDs, at most {MAX_ARG_DWORDS} fit the root signature") return n @@ -297,12 +355,15 @@ def main(argv=None): help="kernels compiled in parallel (default min(12, CPUs), env WG_DX12_JOBS; 1 = one at a time)") ap.add_argument("--model-l", action="store_true", help="also build Model L's kernels (co=l_*.co; off by default: Model M's kernel set unchanged)") + ap.add_argument("--model-k", action="store_true", + help="also build Model K's kernels (clean/k/dx12/kernels; off by default: Model M's and Model L's " + "kernel sets unchanged)") ap.add_argument("--list", action="store_true", help="print the kernels a build with these options compiles (name and code object, JSON) and exit: " "the cheap check before a build (kernel_sets.json, tests/test_kernel_sets.py)") a = ap.parse_args(argv) if a.list: - print(json.dumps([list(k) for k in kernel_set(a.model_l)], indent=1)) + print(json.dumps([list(k) for k in kernel_set(a.model_l, a.model_k)], indent=1)) return 0 if not a.out: ap.error("--out is required") @@ -323,14 +384,13 @@ def main(argv=None): if fn.startswith("fused_") and fn.endswith("_gen.py"): subprocess.run([sys.executable, os.path.join(KERNEL_DIR, fn)], check=True) jobs = [] # (source, declaration, output path) in declaration order (sorted file name, then line) - for fn in sorted(os.listdir(KERNEL_DIR)) if os.path.isdir(KERNEL_DIR) else []: - if not fn.endswith(".hlsl"): - continue - src = os.path.join(KERNEL_DIR, fn) + for src, k_source in source_files(a.model_k or bool(only)): for d in kernel_decls(src): if only and d["name"] not in only: continue - if not only and not a.model_l and is_model_l(d): + if not only and (k_source or is_model_k(d)) and not a.model_k: + continue + if not only and not k_source and not is_model_k(d) and not a.model_l and is_model_l(d): continue if not only and is_frame_gen(d): continue @@ -342,14 +402,16 @@ def main(argv=None): if any(j[2] == out for j in jobs): raise SystemExit(f"kernel {d['name']} ({d['co']}): DXIL file {os.path.basename(out)} already used " "(a second port of a name needs file=)") - jobs.append((src, d, out)) + jobs.append((src, d, out, k_source)) def build_one(job): - src, d, out = job + src, d, out, k_source = job defs = {"WG_BLOCK": d["group"], "WG_WAVE": d["wave"], **d["defines"]} if d["group"] != d["block"]: defs["WG_LAUNCH_BLOCK"] = d["block"] - compile_hlsl(src, d["entry"], out, defs, asm_tool=asm, keep_listing=out[:-5] + ".ll" if a.listing else None) + info = {} + compile_hlsl(src, d["entry"], out, defs, includes=(K_HLSL_DIR,) if k_source and os.path.isdir(K_HLSL_DIR) else (), asm_tool=asm, + keep_listing=out[:-5] + ".ll" if a.listing else None, info=info) with open(out, "rb") as f: sha = hashlib.sha256(f.read()).hexdigest() if a.isa: @@ -360,6 +422,8 @@ def build_one(job): if d["group"] != d["block"]: r["group"] = d["group"] r["gscale"] = d["gscale"] + if info.get("ext_doubles"): + r["fp64"] = "ext" return r # Every kernel is an independent chain of child processes (dxc -> dxil_asm -> dxv [-> rga]) with its own temporary @@ -369,7 +433,7 @@ def build_one(job): with concurrent.futures.ThreadPoolExecutor(max_workers=n_jobs) as pool: futures = [pool.submit(build_one, j) for j in jobs] try: - for (src, d, out), fut in zip(jobs, futures): + for (src, d, out, _k), fut in zip(jobs, futures): kernels[(d["name"], d["co"])] = fut.result() built.append((d["name"], out)) print(f"{d['name']:40s} -> {os.path.basename(out)} block {d['block']} wave {d['wave']} " @@ -380,7 +444,7 @@ def build_one(job): raise names = {n for n, _ in kernels} if only and not (only <= names): - raise SystemExit(f"not declared in clean/dx12/kernels: {sorted(only - names)}") + raise SystemExit(f"not declared in clean/dx12/kernels or clean/k/dx12/kernels: {sorted(only - names)}") vgpr_check(built) with open(man_path, "w") as f: json.dump({"format": FORMAT, "dxc": os.path.basename(os.path.dirname(os.path.dirname(DXC_DIR))), diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index 2b651688..b2b49543 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -286,5 +286,19 @@ "swin_block_c64_add_ex_nc", "swin.co" ] + ], + "model_k": [ + [ + "k_expo_scalars", + "k_io.co" + ], + [ + "k_expo_scalars_direct", + "k_io.co" + ], + [ + "k_warm", + "k_block_wide.co" + ] ] } diff --git a/clean/dx12/tests/kernel_replay.cpp b/clean/dx12/tests/kernel_replay.cpp index cf989632..427cd926 100644 --- a/clean/dx12/tests/kernel_replay.cpp +++ b/clean/dx12/tests/kernel_replay.cpp @@ -32,7 +32,11 @@ // module [] (zero or more) // buf [fill] device buffer: the file's bytes (padded with fill when the file is // shorter), or every byte set to fill (0..255, default 0) with "-" -// step ... a launch; each arg is @ (a buffer's pointer) or a 32-bit decimal +// step ... a launch; each arg is @ (a buffer's pointer), a 32-bit decimal +// value, B: (a struct passed by value: the file's bytes) or +// S:[:=@[,=@...]] (a struct with pointers +// inside, manifest argument kind s:...: the file's bytes with the 8 +// bytes at each byte offset replaced by that buffer's device pointer) // save after the run, the buffer's bytes are written to // time after the run (and the saves), the steps run more times; the mean // wall time per run (submission to completion, synchronised after each @@ -404,13 +408,43 @@ int run_chain(Device* d, Chain& c, const std::string& base) { // the argument values of every step, resolved once std::vector> ptrs(c.steps.size()); std::vector> vals(c.steps.size()); + std::vector>> blobs(c.steps.size()); std::vector> argv(c.steps.size()); for (size_t i = 0; rc == 0 && i < c.steps.size(); ++i) { auto& s = c.steps[i]; ptrs[i].reserve(s.args.size()); vals[i].reserve(s.args.size()); + blobs[i].reserve(s.args.size()); for (auto& a : s.args) { - if (!a.empty() && a[0] == '@') { + if (a.size() > 2 && (a[0] == 'B' || a[0] == 'S') && a[1] == ':') { + // B: / S:[:=@,...] + const size_t colon = a[0] == 'S' ? a.find(':', 2) : std::string::npos; + const std::string file = a.substr(2, colon == std::string::npos ? std::string::npos : colon - 2); + std::vector blob; + if (!read_file(base + "/" + file, &blob)) { + std::printf("%s: step %zu: cannot read %s\n", c.name.c_str(), i, file.c_str()); + rc = 1; + break; + } + bool ok = true; + for (size_t at = colon == std::string::npos ? std::string::npos : colon + 1; ok && at != std::string::npos;) { + const size_t comma = a.find(',', at); + const std::string item = a.substr(at, comma == std::string::npos ? std::string::npos : comma - at); + const size_t eq = item.find("=@"); + ChainBuf* b = eq == std::string::npos ? nullptr : find(item.substr(eq + 2)); + const size_t off = eq == std::string::npos ? 0 : (size_t)std::stoull(item.substr(0, eq)); + if (!b || off + 8 > blob.size()) { ok = false; break; } + for (int k = 0; k < 8; ++k) blob[off + k] = (uint8_t)((uint64_t)b->ptr >> (8 * k)); + at = comma == std::string::npos ? std::string::npos : comma + 1; + } + if (!ok) { + std::printf("%s: step %zu: bad struct argument %s\n", c.name.c_str(), i, a.c_str()); + rc = 1; + break; + } + blobs[i].push_back(std::move(blob)); + argv[i].push_back(blobs[i].back().data()); + } else if (!a.empty() && a[0] == '@') { ChainBuf* b = find(a.substr(1)); if (!b) { std::printf("%s: step %zu: unknown buffer %s\n", c.name.c_str(), i, a.c_str()); rc = 1; break; } ptrs[i].push_back(b->ptr); diff --git a/clean/dx12/tests/test_kernel_sets.py b/clean/dx12/tests/test_kernel_sets.py index a9925bf1..7cc88493 100644 --- a/clean/dx12/tests/test_kernel_sets.py +++ b/clean/dx12/tests/test_kernel_sets.py @@ -1,7 +1,8 @@ # SPDX-License-Identifier: MIT """The D3D12 kernel sets, from the sources' declarations (clean/dx12/build_dx12.py kernel_set; nothing is compiled): the cheap check before a build, in seconds. kernel_sets.json lists them: "model_m" = what a build without --model-l -compiles (Model M's kernel set; Model L's kernels never enter it), "model_l" = what --model-l adds (#89). +compiles (Model M's kernel set; Model L's and Model K's kernels never enter it), "model_l" = what --model-l adds +(#89), "model_k" = what --model-k adds (clean/k/dx12/kernels, #15). A kernel added, removed or moved between the sets changes kernel_sets.json in the same change; regenerate it with python clean/dx12/tests/test_kernel_sets.py --write @@ -22,7 +23,9 @@ def current(): m = B.kernel_set(False) full = B.kernel_set(True) - return {"model_m": [list(k) for k in m], "model_l": [list(k) for k in full if k not in set(m)]} + k = B.kernel_set(False, True) + return {"model_m": [list(x) for x in m], "model_l": [list(x) for x in full if x not in set(m)], + "model_k": [list(x) for x in k if x not in set(m)]} class TestKernelSets(unittest.TestCase): @@ -30,7 +33,7 @@ def test_sets_match_the_list(self): with open(SETS) as f: want = json.load(f) got = current() - for name in ("model_m", "model_l"): + for name in ("model_m", "model_l", "model_k"): w, g = set(map(tuple, want[name])), set(map(tuple, got[name])) self.assertEqual(g, w, "%s: added %s, removed %s (update kernel_sets.json: python " "clean/dx12/tests/test_kernel_sets.py --write)" % (name, sorted(g - w), sorted(w - g))) @@ -48,6 +51,30 @@ def test_model_l_adds_only_model_l_kernels(self): self.assertFalse([k for k in extra if not (k[1].startswith("l_") or k[0].startswith("l_") or k[0] in B.MODEL_L_ONLY)]) + def test_model_k_adds_only_model_k_kernels(self): + """--model-k adds Model K's kernels to Model M's set and nothing else; with --model-l it is all three sets.""" + m, k = set(B.kernel_set(False)), set(B.kernel_set(False, True)) + self.assertTrue(m <= k) + extra = k - m + self.assertTrue(extra) + self.assertFalse([x for x in extra if not (x[1].startswith("k_") or x[0].startswith("k_") or x[0].startswith("kx/"))]) + self.assertEqual(set(B.kernel_set(True, True)), set(B.kernel_set(True)) | extra) + + def test_no_k_kernel_in_the_other_sets(self): + """Neither Model M's nor Model L's set holds a Model K kernel (a K port is declared in clean/k/dx12/kernels).""" + for full in (B.kernel_set(False), B.kernel_set(True)): + self.assertFalse([x for x in full if x[1].startswith("k_") or x[0].startswith("k_") or x[0].startswith("kx/")]) + + def test_struct_argument_tokens(self): + """s:: pointers are one root constant each, copied bytes one per four, skipped bytes none.""" + self.assertEqual(B.struct_dwords("s184:p19.b32", "t"), 27) + self.assertEqual(B.struct_dwords("s160:p13.b56", "t"), 27) + self.assertEqual(B.struct_dwords("s32:p1.x8.b16", "t"), 5) + self.assertEqual(B.arg_dwords("p,u,s184:p19.b32,b36", "t"), 1 + 1 + 27 + 9) + for bad in ("s16:p1", "s16:p1.b4", "s12:p1.b3", "s8:q1", "s0:p0", "s8", "s8:p1.x4", "s12:p1.b4.x"): + with self.assertRaises(SystemExit, msg=bad): + B.struct_dwords(bad, "t") + def test_frame_gen_kernels_are_in_no_set(self): """Frame generation's ports (#96, co=fg_*.co or named fg_*) are declared but enter neither set: only --only builds them, so the Model M and Model L builds stay as they are.""" diff --git a/clean/k/dx12/README.md b/clean/k/dx12/README.md new file mode 100644 index 00000000..20f33cba --- /dev/null +++ b/clean/k/dx12/README.md @@ -0,0 +1,37 @@ + +# Model K on Direct3D 12 + +The D3D12 ports of Model K's HIP kernels (issue #15). Each port reproduces one HIP kernel of `clean/k/kernels` bit for +bit; the HIP kernel stays the reference. The ports are built only with `--model-k` +(`python clean/dx12/build_dx12.py --out --model-k`), so a build without it is Model M's kernel set, and +`--model-l` Model L's, unchanged (`clean/dx12/kernel_sets.json`, `clean/dx12/tests/test_kernel_sets.py`). + +| Path | What | +|---|---| +| `kernels/*.hlsl` | the ports, declared with `// @wg-kernel` header lines like the other D3D12 kernels (`co=` names the HIP code object whose kernel the port reproduces) | +| `hlsl/*.hlsli` | headers shared by the K ports (included with `clean/dx12/hlsl`) | +| `k_port_check.py` | the byte-for-byte check of a family of ports against HIP, below | + +A HIP kernel that takes a struct with pointers inside by value declares that argument as `s:` +(`clean/runtime/struct_args.h`): the hybrid and the native device turn each pointer into an arena offset and pass the +other bytes as they are. + +## The port check + +```bat +python clean\k\dx12\k_port_check.py --build --rocm "%ROCM_PATH%" --prepare --family expo,warm +python clean\k\dx12\k_port_check.py --build --rocm "%ROCM_PATH%" --run --family expo,warm --json out.json +``` + +`--prepare` (CPU) builds the HIP code objects the families need, the D3D12 ports and `kernel_replay.exe`; `--run` (GPU, +under the machine's GPU lock) runs every case on HIP in the process and on D3D12 through `kernel_replay` on the same +input bytes, and compares the raw bytes. Rules, as for the Model L checks: every output buffer has 64 bytes of slack +and starts filled with its run's own sentinel (HIP `0xA5`, D3D12 `0x5A`); the defined bytes must be identical and the +slack must keep its sentinel; special values are among the inputs; a negative control (the port given another input +than HIP) must be caught; Model M's `scale_rgb` runs through the same replay as a control. Output: counts and verdicts. + +Families: `expo` (`k_expo_scalars`) against `k_io.co`; `warm` (`k_warm`, a cache warm-up with no HIP kernel: checked +against its definition, through the struct argument path). + +Known limit, shared with Model M's ports: D3D12 flushes FP32 denormals (inputs and results) to zero and HIP keeps them. +`k_expo_scalars` only meets one with a pre-exposure above about 8.5e37, so the check's inputs keep clear of them. diff --git a/clean/k/dx12/__init__.py b/clean/k/dx12/__init__.py new file mode 100644 index 00000000..548d2d44 --- /dev/null +++ b/clean/k/dx12/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: MIT diff --git a/clean/k/dx12/k_port_check.py b/clean/k/dx12/k_port_check.py new file mode 100644 index 00000000..c7e2b0a9 --- /dev/null +++ b/clean/k/dx12/k_port_check.py @@ -0,0 +1,387 @@ +# SPDX-License-Identifier: MIT +"""Model K (issue #15): byte-for-byte check of a K kernel's D3D12 port against its HIP kernel, one family at a time. + + python clean/k/dx12/k_port_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --prepare [--family expo] + python clean/k/dx12/k_port_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --run + [--family expo,warm] [--seeds N] --json out.json + +--prepare (CPU) builds into : the HIP code objects the families need (clean/k/runner/k_build.py's build list, and +Model M's frame.co for the control), the D3D12 ports (clean/dx12/build_dx12.py --only), and kernel_replay.exe. +--run (GPU, one GPU slot) runs every case on HIP in this process and on D3D12 through kernel_replay on the same input +bytes. The rules are the Model L checks' (clean/l/runner/l_fold_check.py): +- the RAW downloaded bytes are compared, nothing is decoded and encoded again; +- every output buffer has 64 bytes of slack after the bytes the kernel defines and starts filled with its run's own + sentinel (HIP 0xA5, D3D12 0x5A): every defined byte identical, every slack byte its own run's sentinel; +- special values (zero, negative zero, subnormal, the largest and smallest FP16, beyond the FP16 range, infinity, NaN) + are among the inputs of every family that reads values; +- a negative control (the port run on an input that differs from the one HIP ran: the wrong result) must be caught; + the half-grid control (the port again with half its workgroups) is used by the families whose grid has more than one + workgroup; +- Model M's scale_rgb (frame.co, its port already proven) runs through the same replay as a control. +The families: expo = k_expo_scalars against k_io.co (HIP); warm = k_warm (D3D12 only, no HIP kernel: it is checked +against its own definition, the dummy word is written exactly when the key equals a lane's loaded value). +One known limit, shared with Model M's ports: D3D12 flushes FP32 denormals (inputs and results) to zero, HIP keeps them. +k_expo_scalars only meets one with a pre-exposure above about 8.5e37 (kappa = RN(1 / pi) * E), so the inputs here keep +every FP32 input and result out of the denormal range (the FP16 specials are in: they are FP32-normal or zero). +Output: counts and verdicts only, no timing. +""" +import argparse +import ctypes +import json +import os +import struct +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +HIP_SENTINEL, D3D12_SENTINEL = 0xA5, 0x5A +SLACK = 64 +CONTROL = "scale_rgb" # Model M, frame.co +# family -> (HIP code objects, D3D12 kernels) +FAMILY_KERNELS = {"expo": (["k_io.co"], ["k_expo_scalars"]), "warm": ([], ["k_warm"])} +CONTROL_CO = "frame.co" + + +def write(path, data): + with open(path, "wb") as f: + f.write(data) + + +def f32_bits(x): + return struct.pack("= 4 else None + slack_ok = got[4:] == bytes([D3D12_SENTINEL]) * (len(got) - 4) + want_hit = it["hit"] and it["key"] in set(it["acc"]) + if want_hit: + right = word == it["key"] and slack_ok + else: + right = got[:4] == bytes([D3D12_SENTINEL]) * 4 and slack_ok + good += bool(right) + if not right: + fails.append(it["case"]) + res["results"]["warm"] = {"cases": len(state["warm"][0]), "as_defined": good, "failures": fails[:5]} + ok = ok and good == len(state["warm"][0]) and good > 0 + ctl_got = read("ctl.out") + ctl = compare(ctl_got, h[0], 1000 * 4, D3D12_SENTINEL, HIP_SENTINEL) + res["control"] = {"kernel": CONTROL, "identical": bool(ctl["identical"]), "differ": ctl["differ"], + "first_word": ctl.get("first_word"), "slack_bad": ctl["slack_bad"]} + ok = ok and ctl["identical"] + res["pass"] = bool(ok) + if json_out: + with open(json_out, "w") as f: + json.dump(res, f, indent=1) + print(json.dumps({k: v for k, v in res.items() if k != "replay_tail"}, indent=1)) + return 0 if ok else 1 + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--family", default="expo,warm") + ap.add_argument("--prepare", action="store_true") + ap.add_argument("--run", action="store_true") + ap.add_argument("--seeds", type=int, default=2) + ap.add_argument("--json") + a = ap.parse_args(argv) + families = a.family.split(",") + bad = [f for f in families if f not in FAMILY_KERNELS] + if bad: + ap.error(f"unknown family {bad} (choose from {sorted(FAMILY_KERNELS)})") + if a.prepare: + prepare(a.build, a.rocm, families) + if a.run: + return run(a.build, a.rocm, families, list(range(a.seeds)), a.json) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/k/dx12/kernels/k_expo.hlsl b/clean/k/dx12/kernels/k_expo.hlsl new file mode 100644 index 00000000..4ed6106e --- /dev/null +++ b/clean/k/dx12/kernels/k_expo.hlsl @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of model K's k_expo_scalars (clean/k/kernels/k_io.hip; HIP build: k_io.co). WrongGPU's own code. +// Spec 42 draft 3 section 4 / spec 43 section 3.7, exactly the HIP kernel's operations: the state st = [E_cur, E_prev, +// E_aux, E_cur kept for the next frame] (raw values), each read as FP16 (nearest even, wg_f2h; the plain f32tof16 +// rounds toward zero on this driver) with 0 -> 1, then kappa = inv_pi e1 E_cur (1 when 0), rho = e1 E_cur / (e1 E_prev), +// lambda = e2 E_aux / (e1 E_cur), kinv = RN(1 / kappa); the divisions correctly rounded as HIP's (wg_div; it flushes a +// denormal quotient to zero, which these E values, FP16 numbers >= 2^-24 times the factors, do not produce in practice: +// the port check clean/k/dx12/k_port_check.py covers the runtime's cases). One lane writes xs = (kappa, rho, lambda, +// kinv), which the input / output stage kernels read through KFrameIn.xs. +// Args as the HIP kernel: st, src, measured, xs (pointers), KExpoArgs (36 bytes, runtime_k.cpp). +// k_expo_scalars_direct (rs=direct, the native executor's direct I/O; launched once runtime_k.cpp names it): E_cur of +// src_mode 1 is the game's exposure texel t3 (native_direct_exposure, the value the HIP executor copies into src[0]); +// the src pointer is unused, everything else identical. +// @wg-kernel k_expo_scalars co=k_io.co block=32 wave=32 args=p,p,p,p,b36 +// @wg-kernel k_expo_scalars_direct co=k_io.co entry=k_expo_scalars block=32 wave=32 args=p,p,p,p,b36 rs=direct defines=WG_DIRECT_TEXTURES=1 +#include "wg_platform.hlsli" +#ifndef WG_DIRECT_TEXTURES +#define WG_DIRECT_TEXTURES 0 +#endif +#if WG_DIRECT_TEXTURES +#include "native_direct.hlsli" +#endif + +WG_ARGS_BEGIN + uint st, src, measured, xs; // pointers + float A_value; // KExpoArgs (36 bytes) + int A_src_mode, A_aux, A_first, A_hdr, A_reset; + float A_e1, A_e2, A_inv_pi; +WG_ARGS_END + +float kx_ev(float e) +{ + const float h = wg_h2f(wg_f2h(e)); + return h == 0.0f ? 1.0f : h; +} + +WG_KERNEL(k_expo_scalars) +{ + WG_SETUP(); + if (wg_lane_id() != 0u) + return; + float Ecr = 1.0f, Epr = 1.0f, Ear = 1.0f; + if (A_hdr != 0) { +#if WG_DIRECT_TEXTURES + Ecr = A_src_mode == 1 ? native_direct_exposure() : (A_src_mode == 0 ? A_value : 1.0f); +#else + Ecr = A_src_mode == 1 ? wg_ldf(src, 0u) : (A_src_mode == 0 ? A_value : 1.0f); +#endif + Ear = A_aux != 0 ? wg_ldf(measured, 0u) : Ecr; + Epr = (A_first != 0 || A_reset != 0) ? Ecr : wg_ldf(st, 12u); + } + wg_stf(st, 0u, Ecr); + wg_stf(st, 4u, Epr); + wg_stf(st, 8u, Ear); + wg_stf(st, 12u, Ecr); + const float e1 = A_hdr != 0 ? A_e1 : 1.0f, e2 = A_hdr != 0 ? A_e2 : 1.0f; + const float Ec = kx_ev(Ecr), Ep = kx_ev(Epr), Ea = kx_ev(Ear); + const float ce = e1 * Ec; + float kappa = 1.0f, rho = 1.0f; + if (A_hdr != 0) { + kappa = A_inv_pi * ce; + if (kappa == 0.0f) + kappa = 1.0f; + rho = wg_div(ce, e1 * Ep); + } + const float lam = wg_div(e2 * Ea, ce); + wg_stf(xs, 0u, kappa); + wg_stf(xs, 4u, rho); + wg_stf(xs, 8u, lam); + wg_stf(xs, 12u, wg_div(1.0f, kappa)); +} diff --git a/clean/k/dx12/kernels/k_warm.hlsl b/clean/k/dx12/kernels/k_warm.hlsl new file mode 100644 index 00000000..98d4ad42 --- /dev/null +++ b/clean/k/dx12/kernels/k_warm.hlsl @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: MIT +// k_warm: cache warm-up of model K's weights on the D3D12 executors (no HIP counterpart; runtime_k.cpp KRuntime::warm, +// runtime option k_warm). WrongGPU's own code. Launched unordered beside a stage whose grid leaves most of the GPU idle +// (dev_->unordered_next()), it reads one 4-byte word per `line` bytes of up to twelve byte ranges (the weight blobs of +// the latency-bound stages that follow), so their first weight loads hit the caches instead of memory. +// It changes nothing anyone reads: the XOR of the loaded words goes to the dummy word only when it equals key (the +// store keeps the loads alive; the dummy is a buffer of its own that no kernel reads). +// Args: KWarmParams by value (runtime_k.cpp): r[12], dummy (pointers), n[12] (bytes), line, key. +// @wg-kernel k_warm co=k_block_wide.co block=256 wave=32 args=s160:p13.b56 +#include "wg_platform.hlsli" + +WG_ARGS_BEGIN + uint W_r0, W_r1, W_r2, W_r3, W_r4, W_r5, W_r6, W_r7, W_r8, W_r9, W_r10, W_r11, W_dummy; // pointers (0 = null) + uint W_n0, W_n1, W_n2, W_n3, W_n4, W_n5, W_n6, W_n7, W_n8, W_n9, W_n10, W_n11, W_line, W_key; +WG_ARGS_END + +uint kwarm_range(uint r, uint n, uint first, uint stride) +{ + uint acc = 0u; + [loop] for (uint o = first; o < n; o += stride) + acc ^= wg_ld32(r, o); + return acc; +} + +WG_KERNEL(k_warm) +{ + WG_SETUP(); + const uint first = (wg_block_id() * WG_BLOCK + wg_lane_id()) * W_line; + const uint stride = wg_grid_size() * WG_BLOCK * W_line; + uint acc = kwarm_range(W_r0, W_n0, first, stride); + acc ^= kwarm_range(W_r1, W_n1, first, stride); + acc ^= kwarm_range(W_r2, W_n2, first, stride); + acc ^= kwarm_range(W_r3, W_n3, first, stride); + acc ^= kwarm_range(W_r4, W_n4, first, stride); + acc ^= kwarm_range(W_r5, W_n5, first, stride); + acc ^= kwarm_range(W_r6, W_n6, first, stride); + acc ^= kwarm_range(W_r7, W_n7, first, stride); + acc ^= kwarm_range(W_r8, W_n8, first, stride); + acc ^= kwarm_range(W_r9, W_n9, first, stride); + acc ^= kwarm_range(W_r10, W_n10, first, stride); + acc ^= kwarm_range(W_r11, W_n11, first, stride); + if (acc == W_key) + wg_st32(W_dummy, 0u, acc); +} diff --git a/clean/runtime/device_d3d12.cpp b/clean/runtime/device_d3d12.cpp index e8ad66d9..9d738819 100644 --- a/clean/runtime/device_d3d12.cpp +++ b/clean/runtime/device_d3d12.cpp @@ -4,6 +4,7 @@ // No HIP object, imported memory, external semaphore or order_mark is used. Module paths select arithmetic variants // by identity only: the native device neither reads nor loads their HIP code objects. #include "device_d3d12.h" +#include "struct_args.h" #if defined(_WIN32) #include "interop/d3d12_include.h" @@ -66,13 +67,14 @@ constexpr unsigned kNativeSlots = 8, kMaxEvents = 65536; constexpr uint64_t kTransientSubmitBytes = 64ull << 20; constexpr unsigned kProfileQueries = 4096; -struct NativeArg { char kind; unsigned bytes; }; +struct NativeArg { char kind; unsigned bytes; std::vector fields = {}; }; // 's': struct_args.h struct NativeKernel { std::string name, file, co, rs = "buffer"; unsigned block = 0, wave = 0, dwords = 0; // group: lanes per D3D12 workgroup when the port splits each launched workgroup of `block` lanes into block / group // workgroups (manifest "group", build_dx12.py group=; 0 = block). The launch keeps the HIP geometry (grid x block). unsigned group = 0, gscale = 1; // gscale: D3D12 workgroups per launched workgroup (manifest "gscale", default block / group) + bool ext_doubles = false; // manifest "fp64": "ext": binary64 fma / division (DoublePrecisionFloatShaderOps) std::vector args; wgc::Digest digest{}; NativeCom pso; @@ -199,6 +201,18 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (borrowed_device || borrowed_queue) { if (!borrow_device(borrowed_device, borrowed_queue)) return false; } else if (!create_device()) return false; + // ports needing binary64 fma / division (model K's exact input stage) are dropped on an adapter without them: the + // kernel then has no port and a context that needs it is refused (no silent fallback). D3D12 folds the D3D11.1 + // extended doubles (DFMA / DDIV / DRCP) into DoublePrecisionFloatShaderOps (d3d12.h has no separate field) + D3D12_FEATURE_DATA_D3D12_OPTIONS options{}; + const bool ext_doubles = SUCCEEDED(device_->CheckFeatureSupport(D3D12_FEATURE_D3D12_OPTIONS, &options, sizeof options)) && + options.DoublePrecisionFloatShaderOps; + for (auto i = kernels_.begin(); i != kernels_.end();) { + if (i->second.ext_doubles && !ext_doubles) { + log(true, "native D3D12: " + i->first + " needs binary64 fma (DoublePrecisionFloatShaderOps), which this adapter lacks: no port"); + i = kernels_.erase(i); + } else ++i; + } if (!create_objects()) return false; if (!profile_init()) return false; return reserve_events(64) && (!borrowed_ || driver_probe()) && gelu_lut_upload() && gelu_t5_upload(); @@ -1034,6 +1048,14 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { DevPtr pointer = 0; std::memcpy(&pointer, args[n], sizeof pointer); if (pointer && !range(pointer, 1, "kernel pointer")) return false; constants[d++] = static_cast(pointer); + } else if (a.kind == 's') { + const int w = pack_struct_arg(a.fields, args[n], constants + d, [&](uint64_t p, uint32_t* off) { + if (!range(p, 1, "kernel struct pointer")) return false; + *off = static_cast(p); + return true; + }); + if (w < 0) return false; + d += static_cast(w); } else { std::memcpy(constants + d, args[n], a.bytes); d += a.bytes / 4; } @@ -1252,6 +1274,10 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { k.gscale = (unsigned)sn; } } + if (const auto* fp = entry.find("fp64")) { + if (!fp->is_string() || fp->text != "ext") return fail("native D3D12 manifest: invalid fp64 for " + k.name); + k.ext_doubles = true; + } const auto* signature = entry.find("rs"); if (signature) { if (!signature->is_string() || (signature->text != "buffer" && signature->text != "tex" && signature->text != "direct")) @@ -1268,6 +1294,18 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { const size_t end = arguments.find(',', start); const std::string token = arguments.substr(start, end == std::string::npos ? end : end - start); unsigned bytes = 4; char kind = token.empty() ? 0 : token[0]; + if (kind == 's') { // struct with pointers inside (struct_args.h) + NativeArg sa{'s', 0}; + unsigned sw = 0; + if (!parse_struct_arg(token, &sa.bytes, &sw, &sa.fields)) + return fail("native D3D12 manifest: bad struct argument for " + k.name); + words += sw; k.args.push_back(std::move(sa)); + if (words > kNativeArgs) return fail("native D3D12 manifest: too many argument DWORDs"); + if (end == std::string::npos) break; + start = end + 1; + if (start == arguments.size()) return fail("native D3D12 manifest: trailing argument comma"); + continue; + } if (token != "p" && token != "u") { if (kind != 'b' || token.size() < 2) return fail("native D3D12 manifest: bad argument for " + k.name); uint64_t parsed = 0; diff --git a/clean/runtime/device_dx12.cpp b/clean/runtime/device_dx12.cpp index 021da362..d06e9c60 100644 --- a/clean/runtime/device_dx12.cpp +++ b/clean/runtime/device_dx12.cpp @@ -21,6 +21,7 @@ #include "device.h" #include "gelu_lut_table.h" +#include "struct_args.h" #include "gelu_t5_table.h" #include "interop/d3d12_include.h" @@ -87,8 +88,9 @@ constexpr uint64_t kOrderMark = 128; // arena byte offset of the HIP ordering m constexpr int kAllocators = 8; struct Arg { - char kind; // 'p' pointer, 'u' 32-bit scalar, 'b' bytes - unsigned size; // bytes for 'b' + char kind; // 'p' pointer, 'u' 32-bit scalar, 'b' bytes, 's' struct with pointers (struct_args.h) + unsigned size; // bytes for 'b' and 's' + std::vector fields = {}; // 's' }; struct Kernel { @@ -357,6 +359,20 @@ class Dx12Device : public Device { out[d++] = p ? (uint32_t)(p - arena_hip_) : 0u; } else if (a.kind == 'u') { std::memcpy(&out[d++], args[i], 4); + } else if (a.kind == 's') { + const uint64_t lo = arena_hip_ + kAlign, hi = arena_hip_ + arena_bytes_; + const int n = pack_struct_arg(a.fields, args[i], out + d, [&](uint64_t p, uint32_t* off) { + if (p < lo || p >= hi) return false; + *off = (uint32_t)(p - arena_hip_); + return true; + }); + if (n < 0) { + if (!k.fallback_noted) + log(false, "D3D12: " + k.name + " has a struct pointer outside the arena; it runs on HIP"), + k.fallback_noted = true; + return false; + } + d += (unsigned)n; } else { std::memcpy(&out[d], args[i], a.size); d += a.size / 4; @@ -471,6 +487,11 @@ class Dx12Device : public Device { if (FAILED(device_->CheckFeatureSupport(D3D12_FEATURE_SHADER_MODEL, &sm, sizeof sm)) || sm.HighestShaderModel < D3D_SHADER_MODEL_6_6) return fail("D3D12 backend: the driver does not support shader model 6.6"); + // binary64 fma / division (manifest "fp64": "ext", e.g. model K's exact input stage): ports that need them are + // dropped by read_manifest when the adapter lacks them. D3D12 has no separate cap bit for the D3D11.1 extended + // doubles (DFMA / DDIV / DRCP): DoublePrecisionFloatShaderOps includes them (d3d12.h has no other field) + D3D12_FEATURE_DATA_D3D12_OPTIONS o = {}; + ext_doubles_ = SUCCEEDED(device_->CheckFeatureSupport(D3D12_FEATURE_D3D12_OPTIONS, &o, sizeof o)) && o.DoublePrecisionFloatShaderOps; return true; } @@ -540,6 +561,13 @@ class Dx12Device : public Device { } else if (s.size() > 1 && s[0] == 'b' && std::atoi(s.c_str() + 1) > 0 && std::atoi(s.c_str() + 1) % 4 == 0) { x.args.push_back({'b', (unsigned)std::atoi(s.c_str() + 1)}); count += x.args.back().size / 4; + } else if (!s.empty() && s[0] == 's') { + Arg sa{'s', 0}; + unsigned words = 0; + if (!parse_struct_arg(s, &sa.size, &words, &sa.fields)) + return fail("D3D12 manifest: kernel " + x.name + ": struct argument '" + s + "'"); + x.args.push_back(std::move(sa)); + count += words; } else { return fail("D3D12 manifest: kernel " + x.name + ": argument kind '" + s + "'"); } @@ -547,6 +575,11 @@ class Dx12Device : public Device { if (count != x.dwords || count > kMaxArgDwords) return fail("D3D12 manifest: kernel " + x.name + ": argument DWORDs do not add up"); seen.insert(x.name); + if (const wgc::Json* fp = k.find("fp64"); fp && fp->is_string() && fp->text == "ext" && !ext_doubles_) { + log(true, "D3D12: " + x.name + " needs binary64 fma / division (DoublePrecisionFloatShaderOps), which this " + "adapter lacks: no port, it runs on HIP"); + continue; + } if (all || want.count(x.name)) { std::string nm = x.name; kernels_.emplace(nm, std::move(x)); @@ -755,6 +788,7 @@ class Dx12Device : public Device { std::map tagged_cos_; // tag -> file name of the code object loaded under it unsigned module_gen_ = 0; std::string adapter_name_; + bool ext_doubles_ = false; // binary64 incl. fma / division: DoublePrecisionFloatShaderOps (create_device) bool on_dx_ = false; // the D3D12 queue owns the work since the last switch from HIP (a list is open) bool need_barrier_ = false; // a D3D12 dispatch since that switch: the next one needs a UAV barrier first unsigned dispatches_ = 0; // dispatches in the open list diff --git a/clean/runtime/struct_args.h b/clean/runtime/struct_args.h new file mode 100644 index 00000000..b4429bcb --- /dev/null +++ b/clean/runtime/struct_args.h @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: MIT +// Struct arguments of the D3D12 kernel ports with pointers inside (manifest argument kind "s", clean/dx12/build_dx12.py). +// WrongGPU's own code. +// +// A HIP kernel that takes a struct by value with 64-bit device pointers in it (model K's KBlkParams, KInParams, +// KOutParams) cannot be passed to its D3D12 port as plain bytes ("b"): a D3D12 port addresses the arena by 32-bit +// byte offsets, and the hybrid device's pointers are HIP addresses. The argument token +// +// s:.... e.g. s184:p19.b32 (KBlkParams: 27 constants), +// s312:p4.b12.x24.b52.x12.b24.x4.p4.x8.p4.b8.p2.b56 (KOutParams without cbx..zby, +// kappa / rho / lam, pad_ and exp2tab: 52 constants instead of 78) +// +// describes the N bytes of the struct as consecutive fields: p = n pointers (8 bytes each; each becomes ONE root +// constant, the arena byte offset, 0 = null), b = n bytes copied as they are (n a multiple of 4), x = n bytes the +// port does not read (skipped: no root constant). The fields must add up to exactly N bytes. The port's HLSL struct +// is the compacted layout: one uint per pointer, the copied bytes, nothing for the skipped ones. +#pragma once + +#include +#include +#include + +namespace wgrt { + +struct StructField { + char kind; // 'p', 'b' or 'x' + unsigned count; // pointers ('p') or bytes ('b', 'x') +}; + +// Parses an "s:" token. *bytes = N (the HIP struct's size), *dwords = the root constants it takes. +inline bool parse_struct_arg(const std::string& token, unsigned* bytes, unsigned* dwords, + std::vector* fields) { + fields->clear(); + const size_t colon = token.find(':'); + if (token.size() < 4 || token[0] != 's' || colon == std::string::npos || colon < 2) return false; + auto number = [](const std::string& s, size_t a, size_t b, unsigned* v) { + if (a >= b || b - a > 6) return false; + unsigned n = 0; + for (size_t i = a; i < b; ++i) { + if (s[i] < '0' || s[i] > '9') return false; + n = n * 10 + (unsigned)(s[i] - '0'); + } + *v = n; + return n > 0; + }; + unsigned total = 0; + if (!number(token, 1, colon, &total)) return false; + unsigned seen = 0, words = 0; + for (size_t a = colon + 1; a <= token.size();) { + size_t b = token.find('.', a); + if (b == std::string::npos) b = token.size(); + if (b <= a + 1) return false; + const char kind = token[a]; + unsigned n = 0; + if ((kind != 'p' && kind != 'b' && kind != 'x') || !number(token, a + 1, b, &n)) return false; + if (kind == 'p') { + seen += 8 * n; + words += n; + } else { + if (n % 4) return false; + seen += n; + if (kind == 'b') words += n / 4; + } + fields->push_back({kind, n}); + a = b + 1; + } + if (seen != total || fields->empty()) return false; + *bytes = total; + *dwords = words; + return true; +} + +// Packs one struct argument (src: the HIP struct's bytes) into root constants. translate(pointer, &offset) turns a +// non-null device pointer into its arena byte offset (false: not an arena pointer). Returns the constants written, or +// -1 when a pointer could not be translated. +template +int pack_struct_arg(const std::vector& fields, const void* src, uint32_t* out, Translate translate) { + const unsigned char* s = static_cast(src); + int d = 0; + for (const StructField& f : fields) { + if (f.kind == 'p') { + for (unsigned i = 0; i < f.count; ++i, s += 8) { + uint64_t p = 0; + for (int k = 0; k < 8; ++k) p |= (uint64_t)s[k] << (8 * k); + uint32_t off = 0; + if (p && !translate(p, &off)) return -1; + out[d++] = p ? off : 0u; + } + } else if (f.kind == 'b') { + for (unsigned i = 0; i < f.count / 4; ++i, s += 4) + out[d++] = (uint32_t)s[0] | (uint32_t)s[1] << 8 | (uint32_t)s[2] << 16 | (uint32_t)s[3] << 24; + } else { + s += f.count; + } + } + return d; +} + +} // namespace wgrt diff --git a/clean/tests/test_struct_args.cpp b/clean/tests/test_struct_args.cpp new file mode 100644 index 00000000..92342280 --- /dev/null +++ b/clean/tests/test_struct_args.cpp @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: MIT +// CPU test of clean/runtime/struct_args.h: the parser of the manifest's struct argument token s: and the packer +// that turns a HIP struct (64-bit device pointers inside) into root constants. Prints "ok" and exits 0, or the first +// failure and exits 1. +#include "../runtime/struct_args.h" + +#include +#include +#include +#include + +using namespace wgrt; + +static int failures = 0; +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::printf("FAIL line %d: %s\n", __LINE__, #cond); \ + ++failures; \ + } \ + } while (0) + +static bool parse(const char* t, unsigned* bytes, unsigned* dwords, std::vector* f) { + return parse_struct_arg(t, bytes, dwords, f); +} + +int main() { + unsigned bytes = 0, dwords = 0; + std::vector f; + CHECK(parse("s184:p19.b32", &bytes, &dwords, &f) && bytes == 184 && dwords == 27 && f.size() == 2); + CHECK(parse("s32:p1.x8.b16", &bytes, &dwords, &f) && bytes == 32 && dwords == 5 && f.size() == 3); + CHECK(parse("s16:p2", &bytes, &dwords, &f) && dwords == 2); + const char* bad[] = {"", "s", "s8", "s8:", "s8:p", "s8:q1", "s16:p1", "s16:p1.b4", "s12:p1.b3", "s12:p1.b4.x", "s0:p0", + "x8:p1", "s8:p1.", "s8:.p1", "s8p1", "s1234567:p1", "s12:p1.x3.b1", "s8:p1.p1"}; + for (const char* b : bad) CHECK(!parse(b, &bytes, &dwords, &f)); + + // pack: p2.b8.x8.p1 = 16 + 8 + 8 + 8 = 40 bytes -> 2 + 2 + 0 + 1 root constants + CHECK(parse("s40:p2.b8.x8.p1", &bytes, &dwords, &f) && dwords == 5); + unsigned char src[40]; + for (int i = 0; i < 40; ++i) src[i] = (unsigned char)(0xA0 + i); + auto put64 = [&](int at, uint64_t v) { for (int k = 0; k < 8; ++k) src[at + k] = (unsigned char)(v >> (8 * k)); }; + put64(0, 0x1000); // translated by the callback + put64(8, 0); // null stays 0 and is not translated + put64(32, 0x2468); + uint32_t out[8] = {}; + int calls = 0; + int n = pack_struct_arg(f, src, out, [&](uint64_t p, uint32_t* off) { + ++calls; + *off = (uint32_t)(p - 0x1000) + 64; + return true; + }); + CHECK(n == 5); + CHECK(calls == 2); + CHECK(out[0] == 64 && out[1] == 0); + CHECK(out[2] == ((uint32_t)src[16] | (uint32_t)src[17] << 8 | (uint32_t)src[18] << 16 | (uint32_t)src[19] << 24)); + CHECK(out[3] == ((uint32_t)src[20] | (uint32_t)src[21] << 8 | (uint32_t)src[22] << 16 | (uint32_t)src[23] << 24)); + CHECK(out[4] == (0x2468 - 0x1000) + 64); // the skipped bytes 24..31 produce nothing + // a pointer that cannot be translated fails the whole argument + CHECK(pack_struct_arg(f, src, out, [](uint64_t, uint32_t*) { return false; }) == -1); + if (failures == 0) std::printf("ok\n"); + return failures ? 1 : 0; +} diff --git a/clean/tests/test_struct_args.py b/clean/tests/test_struct_args.py new file mode 100644 index 00000000..fea406db --- /dev/null +++ b/clean/tests/test_struct_args.py @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: MIT +"""The D3D12 kernel ports' struct arguments (clean/runtime/struct_args.h): the token s: and the packing of a +HIP struct with pointers into root constants. No GPU: clean/tests/test_struct_args.cpp is built with the host C++ +compiler (CXX, clang++ or g++; on Windows MSVC cl after them; skipped without one) and run. + +Run: python -m unittest clean.tests.test_struct_args +""" +import subprocess +import unittest + +from clean.tests import host_cxx + + +class StructArgs(unittest.TestCase): + def test_parse_and_pack(self): + exe = host_cxx.build("test_struct_args", ["clean/tests/test_struct_args.cpp"]) + r = subprocess.run([exe], capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertIn("ok", r.stdout) + + +if __name__ == "__main__": + unittest.main()