diff --git a/clean/dx12/README.md b/clean/dx12/README.md index a46b265e..4a70db2b 100644 --- a/clean/dx12/README.md +++ b/clean/dx12/README.md @@ -109,7 +109,10 @@ WG_KERNEL(exposure_auto_diet) { WG_SETUP(); ... } // entry name = HIP kernel n * `co` (required) = file name of the HIP code object whose build of the kernel the port reproduces. The device uses the port only while the runtime has exactly that code object loaded (untagged, or under the name's tag); any other build variant of the same kernel name (`post_w64.co` / `post_w64__q5.co` under `w64`, a `swin.co` variant, - `dec1_dec0__pack12__div*.co` under `d1p12`, ...) runs on HIP, logged once. + `dec1_dec0__pack12__div*.co` under `d1p12`, ...) runs on HIP, logged once. One name may have ports of several + builds (one manifest entry per name and `co`, for example `w64/post_tf_f32_p12_diet_vsit` of `post_w64__orig.co` + and of `post_w64.co`): the device runs the port whose code object is loaded. Every port of a name but the first + sets `file=`, its DXIL file name (`.dxil`). * `args` = the HIP kernel's arguments in order: `p` pointer (DWORD: arena byte offset), `u` any 32-bit scalar, `b` struct by value (N bytes = N/4 DWORDs, copied verbatim). At most 57 DWORDs. The `WG_ARGS_BEGIN` block must declare exactly these DWORDs in order, **32-bit scalars only** (no vectors / arrays / structs: cbuffer packing). diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index 82a0a2c5..03617313 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -13,7 +13,10 @@ the tag. co is the file name of the HIP code object whose build of the kernel this port reproduces ("swin.co", "post_w64__orig.co"): the device runs the port only while the runtime has exactly that code object loaded (untagged, or under the name's tag), so a different build variant of the same kernel name keeps running on HIP. co=- marks a -D3D12-only kernel with no HIP counterpart (the device self-test's kernels): it always runs on D3D12. args lists the HIP kernel's arguments in order: p = pointer (a uint byte offset into the arena; 0 = null), +D3D12-only kernel with no HIP counterpart (the device self-test's kernels): it always runs on D3D12. One name may have +ports of several builds of its HIP kernel (for example w64/post_tf_f32_p12_diet_vsit of post_w64__orig.co and of +post_w64.co): one manifest entry per (name, co), and the device runs the one whose code object is loaded. Every port but +the first of a name sets file= (its DXIL is .dxil; default __). args lists the HIP kernel's arguments in order: p = pointer (a uint byte offset into the arena; 0 = null), u = any 32-bit scalar, b = a struct of N bytes passed by value (N / 4 uint constants). Arithmetic mode ("exact", README.md): DXC compiles with IEEE strictness (-Gis: no fast-math flags, so no contraction, no @@ -158,6 +161,9 @@ def kernel_decls(path): d["gscale"] = int(kv.get("gscale", d["block"] // d["group"])) if d["gscale"] < 1 or d["gscale"] > d["block"] // d["group"]: raise SystemExit(f"{path}: {name}: gscale {d['gscale']}") + d["file"] = kv.get("file", "") # DXIL file stem of a second port of the same name (module doc) + if d["file"] and not re.fullmatch(r"[A-Za-z0-9_]+", d["file"]): + raise SystemExit(f"{path}: {name}: file={d['file']} (letters, digits and _ only)") d["rs"] = kv.get("rs", "buffer") if d["rs"] not in ("buffer", "tex", "direct"): raise SystemExit(f"{path}: {name}: unsupported root-signature kind {d['rs']}") @@ -261,8 +267,8 @@ def main(argv=None): old = {} if only and os.path.exists(man_path): # partial build: keep the other kernels' entries with open(man_path) as f: - old = {k["name"]: k for k in json.load(f)["kernels"]} - kernels = dict(old) + old = {(k["name"], k.get("co", "")): k for k in json.load(f)["kernels"]} + kernels = dict(old) # (name, co) -> manifest entry built = [] # generated kernels (kernels/fused_*_gen.py: the fused forms of swin_block.hlsl / bneck.hlsl) are regenerated from # the live sources first, so that no build compiles a stale copy @@ -277,10 +283,15 @@ def main(argv=None): for d in kernel_decls(src): if only and d["name"] not in only: continue - if d["name"] in kernels and d["name"] not in old: - raise SystemExit(f"kernel {d['name']} declared twice") - kernels[d["name"]] = None # placeholder: the declared-twice check sees it before any compile starts - jobs.append((src, d, os.path.join(dst, output_name(d["name"])))) + key = (d["name"], d["co"]) + if key in kernels and key not in old: + raise SystemExit(f"kernel {d['name']} ({d['co']}) declared twice") + kernels[key] = None # placeholder: the declared-twice check sees it before any compile starts + out = os.path.join(dst, d["file"] + ".dxil" if d["file"] else output_name(d["name"])) + if any(j[2] == out for j in jobs): + raise SystemExit(f"kernel {d['name']} ({d['co']}): DXIL file {os.path.basename(out)} already used " + "(a second port of a name needs file=)") + jobs.append((src, d, out)) def build_one(job): src, d, out = job @@ -308,7 +319,7 @@ def build_one(job): futures = [pool.submit(build_one, j) for j in jobs] try: for (src, d, out), fut in zip(jobs, futures): - kernels[d["name"]] = fut.result() + kernels[(d["name"], d["co"])] = fut.result() built.append((d["name"], out)) print(f"{d['name']:40s} -> {os.path.basename(out)} block {d['block']} wave {d['wave']} " f"args {d['args']}", flush=True) @@ -316,8 +327,9 @@ def build_one(job): for fut in futures: fut.cancel() # kernels not started yet; the running ones finish their current child process raise - if only and not (only <= set(kernels)): - raise SystemExit(f"not declared in clean/dx12/kernels: {sorted(only - set(kernels))}") + names = {n for n, _ in kernels} + if only and not (only <= names): + raise SystemExit(f"not declared in clean/dx12/kernels: {sorted(only - names)}") vgpr_check(built) with open(man_path, "w") as f: json.dump({"format": FORMAT, "dxc": os.path.basename(os.path.dirname(os.path.dirname(DXC_DIR))), diff --git a/clean/runtime/device_d3d12.cpp b/clean/runtime/device_d3d12.cpp index 0a74cccc..e8ad66d9 100644 --- a/clean/runtime/device_d3d12.cpp +++ b/clean/runtime/device_d3d12.cpp @@ -604,10 +604,19 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { tagged_cos_[tag] = co; return true; } void unordered_next() override { unordered_next_ = true; } + // The port of `name` that the loaded code objects select. One name can have ports of several builds of its HIP kernel + // (manifest entries with the same name and different "co", for example post_w64.co and post_w64__orig.co); without + // a selected one, the first port (variant_loaded then names what it requires). + NativeKernel* find_port(const std::string& name) { + auto range = kernels_.equal_range(name); + for (auto i = range.first; i != range.second; ++i) + if (variant_matches(i->second)) return &i->second; + return range.first == range.second ? nullptr : &range.first->second; + } bool launch(const std::string& name, unsigned grid, unsigned block, void** args) override { - auto i = kernels_.find(name); - if (i == kernels_.end()) return fail("native D3D12 unsupported kernel: " + name + "; no HIP fallback"); - NativeKernel& k = i->second; + NativeKernel* port = find_port(name); + if (!port) return fail("native D3D12 unsupported kernel: " + name + "; no HIP fallback"); + NativeKernel& k = *port; // Legacy tex ports have a different ABI and remain unsupported. A direct port needs this frame's typed // descriptors before variant/argument inspection, including for a caller with no argument array. const bool direct = k.rs == "direct"; @@ -1278,7 +1287,10 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { } if (words != k.dwords) return fail("native D3D12 manifest: argument DWORD count mismatch for " + k.name); k.file = directory + filename; const std::string key = k.name; - if (!kernels_.emplace(key, std::move(k)).second) return fail("native D3D12 manifest: duplicate kernel " + key); + auto same = kernels_.equal_range(key); // one port per (name, code object) + for (auto i = same.first; i != same.second; ++i) + if (i->second.co == k.co) return fail("native D3D12 manifest: duplicate kernel " + key + " (" + k.co + ")"); + kernels_.emplace(key, std::move(k)); } return true; } @@ -1659,7 +1671,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { std::map free_; std::map used_; std::map hosts_; - std::map kernels_; + std::multimap kernels_; // by name; several ports of one name differ in "co" std::set untagged_cos_; std::map tagged_cos_; std::vector events_; diff --git a/clean/runtime/device_dx12.cpp b/clean/runtime/device_dx12.cpp index e4752e35..021da362 100644 --- a/clean/runtime/device_dx12.cpp +++ b/clean/runtime/device_dx12.cpp @@ -239,12 +239,12 @@ class Dx12Device : public Device { // ---- launches ---------------------------------------------------------------------------------------------------- bool launch(const std::string& kernel, unsigned grid, unsigned block, void** args) override { - auto it = kernels_.find(kernel); + Kernel* port = find_port(kernel); uint32_t c[kRuntimeConstants + kMaxArgDwords]; - if (it == kernels_.end() || !arena_ || !variant_loaded(it->second) || !pack_args(it->second, args, c + kRuntimeConstants)) { + if (!port || !arena_ || !variant_loaded(*port) || !pack_args(*port, args, c + kRuntimeConstants)) { return to_hip() && fwd(hip_->launch(kernel, grid, block, args)); } - Kernel& k = it->second; + Kernel& k = *port; if (block != k.block) return fail("D3D12 kernel " + kernel + " is built for " + std::to_string(k.block) + " lanes, launched with " + std::to_string(block)); @@ -272,8 +272,8 @@ class Dx12Device : public Device { } bool launch_timed(const std::string& kernel, unsigned grid, unsigned block, void** args, int start_event, int stop_event) override { - auto it = kernels_.find(kernel); - if (it == kernels_.end() || !arena_ || !variant_loaded(it->second)) + Kernel* port = find_port(kernel); + if (!port || !arena_ || !variant_loaded(*port)) return to_hip() && fwd(hip_->launch_timed(kernel, grid, block, args, start_event, stop_event)); return Device::launch_timed(kernel, grid, block, args, start_event, stop_event); } @@ -297,6 +297,24 @@ class Dx12Device : public Device { static std::string base_name(const std::string& path) { return path.substr(path.find_last_of("/\\") + 1); } // Is the HIP code object this port reproduces the one the runtime loaded for the kernel's name? + // Whether the runtime's loaded code objects select port k (variant_loaded without its cache and note). + bool variant_selected(const Kernel& k) const { + if (k.co == "-") return true; + const size_t slash = k.name.find('/'); + if (slash == std::string::npos) return !k.co.empty() && untagged_cos_.count(k.co); + auto t = tagged_cos_.find(k.name.substr(0, slash)); + return !k.co.empty() && t != tagged_cos_.end() && t->second == k.co; + } + // The port of `kernel` that the loaded code objects select. One name can have ports of several builds of its HIP + // kernel (manifest entries with the same name and different "co"); without a selected one, the first port + // (variant_loaded then notes once that the kernel runs on HIP). + Kernel* find_port(const std::string& kernel) { + auto range = kernels_.equal_range(kernel); + for (auto i = range.first; i != range.second; ++i) + if (variant_selected(i->second)) return &i->second; + return range.first == range.second ? nullptr : &range.first->second; + } + bool variant_loaded(Kernel& k) { if (k.co_gen != module_gen_) { k.co_gen = module_gen_; @@ -732,7 +750,7 @@ class Dx12Device : public Device { DevPtr arena_hip_ = 0; uint64_t arena_bytes_ = 0, in_use_ = 0, fence_value_ = 0; std::map free_, used_; // offset -> bytes - std::map kernels_; + std::multimap kernels_; // by name; several ports of one name differ in "co" std::set untagged_cos_; // file names of the code objects loaded without a tag std::map tagged_cos_; // tag -> file name of the code object loaded under it unsigned module_gen_ = 0;