Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion clean/dx12/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,10 @@ WG_KERNEL(exposure_auto_diet) { WG_SETUP(); ... } // entry name = HIP kernel n
* `co` (required) = file name of the HIP code object whose build of the kernel the port reproduces. The device uses
the port only while the runtime has exactly that code object loaded (untagged, or under the name's tag); any other
build variant of the same kernel name (`post_w64.co` / `post_w64__q5.co` under `w64`, a `swin.co` variant,
`dec1_dec0__pack12__div*.co` under `d1p12`, ...) runs on HIP, logged once.
`dec1_dec0__pack12__div*.co` under `d1p12`, ...) runs on HIP, logged once. One name may have ports of several
builds (one manifest entry per name and `co`, for example `w64/post_tf_f32_p12_diet_vsit` of `post_w64__orig.co`
and of `post_w64.co`): the device runs the port whose code object is loaded. Every port of a name but the first
sets `file=<stem>`, its DXIL file name (`<stem>.dxil`).
* `args` = the HIP kernel's arguments in order: `p` pointer (DWORD: arena byte offset), `u` any 32-bit scalar,
`b<N>` struct by value (N bytes = N/4 DWORDs, copied verbatim). At most 57 DWORDs. The `WG_ARGS_BEGIN` block must
declare exactly these DWORDs in order, **32-bit scalars only** (no vectors / arrays / structs: cbuffer packing).
Expand Down
32 changes: 22 additions & 10 deletions clean/dx12/build_dx12.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,10 @@
the tag. co is the file name of the HIP code object whose build of the kernel this port reproduces ("swin.co",
"post_w64__orig.co"): the device runs the port only while the runtime has exactly that code object loaded (untagged, or
under the name's tag), so a different build variant of the same kernel name keeps running on HIP. co=- marks a
D3D12-only kernel with no HIP counterpart (the device self-test's kernels): it always runs on D3D12. args lists the HIP kernel's arguments in order: p = pointer (a uint byte offset into the arena; 0 = null),
D3D12-only kernel with no HIP counterpart (the device self-test's kernels): it always runs on D3D12. One name may have
ports of several builds of its HIP kernel (for example w64/post_tf_f32_p12_diet_vsit of post_w64__orig.co and of
post_w64.co): one manifest entry per (name, co), and the device runs the one whose code object is loaded. Every port but
the first of a name sets file=<stem> (its DXIL is <stem>.dxil; default <tag>__<kernel>). args lists the HIP kernel's arguments in order: p = pointer (a uint byte offset into the arena; 0 = null),
u = any 32-bit scalar, b<N> = a struct of N bytes passed by value (N / 4 uint constants).

Arithmetic mode ("exact", README.md): DXC compiles with IEEE strictness (-Gis: no fast-math flags, so no contraction, no
Expand Down Expand Up @@ -158,6 +161,9 @@ def kernel_decls(path):
d["gscale"] = int(kv.get("gscale", d["block"] // d["group"]))
if d["gscale"] < 1 or d["gscale"] > d["block"] // d["group"]:
raise SystemExit(f"{path}: {name}: gscale {d['gscale']}")
d["file"] = kv.get("file", "") # DXIL file stem of a second port of the same name (module doc)
if d["file"] and not re.fullmatch(r"[A-Za-z0-9_]+", d["file"]):
raise SystemExit(f"{path}: {name}: file={d['file']} (letters, digits and _ only)")
d["rs"] = kv.get("rs", "buffer")
if d["rs"] not in ("buffer", "tex", "direct"):
raise SystemExit(f"{path}: {name}: unsupported root-signature kind {d['rs']}")
Expand Down Expand Up @@ -261,8 +267,8 @@ def main(argv=None):
old = {}
if only and os.path.exists(man_path): # partial build: keep the other kernels' entries
with open(man_path) as f:
old = {k["name"]: k for k in json.load(f)["kernels"]}
kernels = dict(old)
old = {(k["name"], k.get("co", "")): k for k in json.load(f)["kernels"]}
kernels = dict(old) # (name, co) -> manifest entry
built = []
# generated kernels (kernels/fused_*_gen.py: the fused forms of swin_block.hlsl / bneck.hlsl) are regenerated from
# the live sources first, so that no build compiles a stale copy
Expand All @@ -277,10 +283,15 @@ def main(argv=None):
for d in kernel_decls(src):
if only and d["name"] not in only:
continue
if d["name"] in kernels and d["name"] not in old:
raise SystemExit(f"kernel {d['name']} declared twice")
kernels[d["name"]] = None # placeholder: the declared-twice check sees it before any compile starts
jobs.append((src, d, os.path.join(dst, output_name(d["name"]))))
key = (d["name"], d["co"])
if key in kernels and key not in old:
raise SystemExit(f"kernel {d['name']} ({d['co']}) declared twice")
kernels[key] = None # placeholder: the declared-twice check sees it before any compile starts
out = os.path.join(dst, d["file"] + ".dxil" if d["file"] else output_name(d["name"]))
if any(j[2] == out for j in jobs):
raise SystemExit(f"kernel {d['name']} ({d['co']}): DXIL file {os.path.basename(out)} already used "
"(a second port of a name needs file=<stem>)")
jobs.append((src, d, out))

def build_one(job):
src, d, out = job
Expand Down Expand Up @@ -308,16 +319,17 @@ def build_one(job):
futures = [pool.submit(build_one, j) for j in jobs]
try:
for (src, d, out), fut in zip(jobs, futures):
kernels[d["name"]] = fut.result()
kernels[(d["name"], d["co"])] = fut.result()
built.append((d["name"], out))
print(f"{d['name']:40s} -> {os.path.basename(out)} block {d['block']} wave {d['wave']} "
f"args {d['args']}", flush=True)
except BaseException:
for fut in futures:
fut.cancel() # kernels not started yet; the running ones finish their current child process
raise
if only and not (only <= set(kernels)):
raise SystemExit(f"not declared in clean/dx12/kernels: {sorted(only - set(kernels))}")
names = {n for n, _ in kernels}
if only and not (only <= names):
raise SystemExit(f"not declared in clean/dx12/kernels: {sorted(only - names)}")
vgpr_check(built)
with open(man_path, "w") as f:
json.dump({"format": FORMAT, "dxc": os.path.basename(os.path.dirname(os.path.dirname(DXC_DIR))),
Expand Down
22 changes: 17 additions & 5 deletions clean/runtime/device_d3d12.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -604,10 +604,19 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext {
tagged_cos_[tag] = co; return true;
}
void unordered_next() override { unordered_next_ = true; }
// The port of `name` that the loaded code objects select. One name can have ports of several builds of its HIP kernel
// (manifest entries with the same name and different "co", for example post_w64.co and post_w64__orig.co); without
// a selected one, the first port (variant_loaded then names what it requires).
NativeKernel* find_port(const std::string& name) {
auto range = kernels_.equal_range(name);
for (auto i = range.first; i != range.second; ++i)
if (variant_matches(i->second)) return &i->second;
return range.first == range.second ? nullptr : &range.first->second;
}
bool launch(const std::string& name, unsigned grid, unsigned block, void** args) override {
auto i = kernels_.find(name);
if (i == kernels_.end()) return fail("native D3D12 unsupported kernel: " + name + "; no HIP fallback");
NativeKernel& k = i->second;
NativeKernel* port = find_port(name);
if (!port) return fail("native D3D12 unsupported kernel: " + name + "; no HIP fallback");
NativeKernel& k = *port;
// Legacy tex ports have a different ABI and remain unsupported. A direct port needs this frame's typed
// descriptors before variant/argument inspection, including for a caller with no argument array.
const bool direct = k.rs == "direct";
Expand Down Expand Up @@ -1278,7 +1287,10 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext {
}
if (words != k.dwords) return fail("native D3D12 manifest: argument DWORD count mismatch for " + k.name);
k.file = directory + filename; const std::string key = k.name;
if (!kernels_.emplace(key, std::move(k)).second) return fail("native D3D12 manifest: duplicate kernel " + key);
auto same = kernels_.equal_range(key); // one port per (name, code object)
for (auto i = same.first; i != same.second; ++i)
if (i->second.co == k.co) return fail("native D3D12 manifest: duplicate kernel " + key + " (" + k.co + ")");
kernels_.emplace(key, std::move(k));
}
return true;
}
Expand Down Expand Up @@ -1659,7 +1671,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext {
std::map<uint64_t, uint64_t> free_;
std::map<uint64_t, NativeAllocation> used_;
std::map<uintptr_t, Host> hosts_;
std::map<std::string, NativeKernel> kernels_;
std::multimap<std::string, NativeKernel> kernels_; // by name; several ports of one name differ in "co"
std::set<std::string> untagged_cos_;
std::map<std::string, std::string> tagged_cos_;
std::vector<NativeEvent> events_;
Expand Down
30 changes: 24 additions & 6 deletions clean/runtime/device_dx12.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -239,12 +239,12 @@ class Dx12Device : public Device {

// ---- launches ----------------------------------------------------------------------------------------------------
bool launch(const std::string& kernel, unsigned grid, unsigned block, void** args) override {
auto it = kernels_.find(kernel);
Kernel* port = find_port(kernel);
uint32_t c[kRuntimeConstants + kMaxArgDwords];
if (it == kernels_.end() || !arena_ || !variant_loaded(it->second) || !pack_args(it->second, args, c + kRuntimeConstants)) {
if (!port || !arena_ || !variant_loaded(*port) || !pack_args(*port, args, c + kRuntimeConstants)) {
return to_hip() && fwd(hip_->launch(kernel, grid, block, args));
}
Kernel& k = it->second;
Kernel& k = *port;
if (block != k.block)
return fail("D3D12 kernel " + kernel + " is built for " + std::to_string(k.block) + " lanes, launched with " +
std::to_string(block));
Expand Down Expand Up @@ -272,8 +272,8 @@ class Dx12Device : public Device {
}
bool launch_timed(const std::string& kernel, unsigned grid, unsigned block, void** args, int start_event,
int stop_event) override {
auto it = kernels_.find(kernel);
if (it == kernels_.end() || !arena_ || !variant_loaded(it->second))
Kernel* port = find_port(kernel);
if (!port || !arena_ || !variant_loaded(*port))
return to_hip() && fwd(hip_->launch_timed(kernel, grid, block, args, start_event, stop_event));
return Device::launch_timed(kernel, grid, block, args, start_event, stop_event);
}
Expand All @@ -297,6 +297,24 @@ class Dx12Device : public Device {

static std::string base_name(const std::string& path) { return path.substr(path.find_last_of("/\\") + 1); }
// Is the HIP code object this port reproduces the one the runtime loaded for the kernel's name?
// Whether the runtime's loaded code objects select port k (variant_loaded without its cache and note).
bool variant_selected(const Kernel& k) const {
if (k.co == "-") return true;
const size_t slash = k.name.find('/');
if (slash == std::string::npos) return !k.co.empty() && untagged_cos_.count(k.co);
auto t = tagged_cos_.find(k.name.substr(0, slash));
return !k.co.empty() && t != tagged_cos_.end() && t->second == k.co;
}
// The port of `kernel` that the loaded code objects select. One name can have ports of several builds of its HIP
// kernel (manifest entries with the same name and different "co"); without a selected one, the first port
// (variant_loaded then notes once that the kernel runs on HIP).
Kernel* find_port(const std::string& kernel) {
auto range = kernels_.equal_range(kernel);
for (auto i = range.first; i != range.second; ++i)
if (variant_selected(i->second)) return &i->second;
return range.first == range.second ? nullptr : &range.first->second;
}

bool variant_loaded(Kernel& k) {
if (k.co_gen != module_gen_) {
k.co_gen = module_gen_;
Expand Down Expand Up @@ -732,7 +750,7 @@ class Dx12Device : public Device {
DevPtr arena_hip_ = 0;
uint64_t arena_bytes_ = 0, in_use_ = 0, fence_value_ = 0;
std::map<uint64_t, uint64_t> free_, used_; // offset -> bytes
std::map<std::string, Kernel> kernels_;
std::multimap<std::string, Kernel> kernels_; // by name; several ports of one name differ in "co"
std::set<std::string> untagged_cos_; // file names of the code objects loaded without a tag
std::map<std::string, std::string> tagged_cos_; // tag -> file name of the code object loaded under it
unsigned module_gen_ = 0;
Expand Down
Loading