Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 88 additions & 24 deletions clean/dx12/build_dx12.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@
Model L's kernels (#89: those that reproduce one of Model L's code objects, co=l_*.co, direct-route kernels named l_*, and MODEL_L_ONLY) are built only with --model-l
or when --only names them, so that a build without the option is Model M's kernel set unchanged. Frame generation's
kernels (#96: co=fg_*.co or named fg_*) are built only when --only names them, never by a full build.
Model K's kernels (clean/k/dx12/kernels, co=k_*.co: #15) are built only with --model-k or when --only names them, so
that a build without the option is the same kernel set as before. A kernel that takes a struct with pointers in it by
value (a HIP kernel's parameter block) declares the argument as s<N>:<fields> (clean/runtime/struct_args.h).

Every kernel source declares its kernels in header lines (clean/dx12/README.md):

Expand Down Expand Up @@ -48,6 +51,11 @@
HERE = os.path.dirname(os.path.abspath(__file__))
KERNEL_DIR = os.path.join(HERE, "kernels")
HLSL_DIR = os.path.join(HERE, "hlsl")
# Model K's ports (clean/k/dx12): their kernels are built only with --model-k (or when --only names them); their
# sources may include both hlsl folders (clean/k/dx12/hlsl first).
K_DX12_DIR = os.path.join(os.path.dirname(HERE), "k", "dx12")
K_KERNEL_DIR = os.path.join(K_DX12_DIR, "kernels")
K_HLSL_DIR = os.path.join(K_DX12_DIR, "hlsl")
# Pinned tools (BUILDING.md): WG_TOOLCHAINS = a folder holding dxc-v1.10.2605.24\ and rga-2.15\ (default <repo>\toolchains);
# WG_DXC_DIR / WG_RGA point at the tools directly.
TOOLCHAINS = os.environ.get("WG_TOOLCHAINS", os.path.join(os.path.dirname(os.path.dirname(HERE)), "toolchains"))
Expand Down Expand Up @@ -108,8 +116,17 @@ def fdiv(m):
return text


def compile_hlsl(src, entry, out_dxil, defines=None, includes=(), asm_tool=None, keep_listing=None, allow_fdiv=False):
"""One entry point of an HLSL file in the exact arithmetic mode -> signed DXIL container at out_dxil."""
# Binary64 operations of the D3D11.1 extended doubles (double fma = DXIL opcode 47, double division), which D3D12 reports
# in D3D12_FEATURE_DATA_D3D12_OPTIONS.DoublePrecisionFloatShaderOps (no separate field): such a port is marked
# "fp64": "ext" in the manifest, and the devices drop it when the adapter lacks the feature (the kernel then has no
# port: HIP in the hybrid device, a refused context on the native one).
_EXT_DOUBLES = re.compile(r"@dx\.op\.tertiary\.f64\(i32 47,|\bfdiv (fast )?(<\d+ x )?double\b")


def compile_hlsl(src, entry, out_dxil, defines=None, includes=(), asm_tool=None, keep_listing=None, allow_fdiv=False,
info=None):
"""One entry point of an HLSL file in the exact arithmetic mode -> signed DXIL container at out_dxil. info (a dict),
if given, receives "ext_doubles": whether the kernel uses extended binary64 instructions."""
dxc, dxv = os.path.join(DXC_DIR, "dxc.exe"), os.path.join(DXC_DIR, "dxv.exe")
with tempfile.TemporaryDirectory(prefix="wgdx12_") as tmp:
ll = os.path.join(tmp, "a.ll")
Expand All @@ -120,7 +137,9 @@ def compile_hlsl(src, entry, out_dxil, defines=None, includes=(), asm_tool=None,
raise SystemExit(f"dxc failed for {src} ({entry}):\n{r.stdout}{r.stderr}")
with open(ll, encoding="utf-8", errors="replace") as f:
text = exact_listing(f.read(), f"{os.path.basename(src)}:{entry}", allow_fdiv)
ll2 = os.path.join(tmp, "b.ll")
if info is not None:
info["ext_doubles"] = bool(_EXT_DOUBLES.search(text))
ll2 =os.path.join(tmp, "b.ll")
with open(ll2, "w", encoding="utf-8") as f:
f.write(text)
if keep_listing:
Expand Down Expand Up @@ -194,30 +213,69 @@ def is_frame_gen(decl):
return decl["co"].startswith("fg_") or decl["name"].startswith("fg_")


def kernel_set(model_l=False):
def is_model_k(decl):
"""A port of one of Model K's code objects (clean/k/kernels: k_block.co, k_block_wide.co, k_io.co, k_io_exact.co,
k_act.co ...) or a Model K kernel with no code object (co "-" or a helper, named k_*). All of them live in
clean/k/dx12/kernels: Model M's and Model L's sets never contain one."""
return decl["co"].startswith("k_") or decl["name"].startswith("k_") or decl["name"].startswith("kx/")


def source_files(model_k=False, only_k=False):
"""The .hlsl sources of a build, (path, is_k_source): clean/dx12/kernels, plus clean/k/dx12/kernels with model_k."""
out = []
if not only_k and os.path.isdir(KERNEL_DIR):
out += [(os.path.join(KERNEL_DIR, fn), False) for fn in sorted(os.listdir(KERNEL_DIR)) if fn.endswith(".hlsl")]
if (model_k or only_k) and os.path.isdir(K_KERNEL_DIR):
out += [(os.path.join(K_KERNEL_DIR, fn), True) for fn in sorted(os.listdir(K_KERNEL_DIR)) if fn.endswith(".hlsl")]
return out


def kernel_set(model_l=False, model_k=False):
"""(name, code object) of every kernel a full build compiles (no --only), sorted: Model M's set, plus Model L's
with model_l. From the sources' declarations alone (nothing is compiled). Frame generation's kernels are never in
it (is_frame_gen)."""
with model_l, plus Model K's with model_k. From the sources' declarations alone (nothing is compiled). Frame
generation's kernels are never in it (is_frame_gen)."""
keys = set()
for fn in sorted(os.listdir(KERNEL_DIR)) if os.path.isdir(KERNEL_DIR) else []:
if fn.endswith(".hlsl"):
for d in kernel_decls(os.path.join(KERNEL_DIR, fn)):
if is_frame_gen(d):
continue
if model_l or not is_model_l(d):
for path, k_source in source_files(model_k):
for d in kernel_decls(path):
if is_frame_gen(d):
continue
if k_source or is_model_k(d):
if model_k:
keys.add((d["name"], d["co"]))
elif model_l or not is_model_l(d):
keys.add((d["name"], d["co"]))
return sorted(keys)


def struct_dwords(a, where):
"""Root constants of a struct argument s<N>:<fields> (clean/runtime/struct_args.h): p<n> n pointers of 8 bytes, one
constant each (the arena offset); b<n> n bytes copied; x<n> n bytes skipped. The fields add up to N bytes."""
m = re.fullmatch(r"s(\d+):((?:[pbx]\d+)(?:\.[pbx]\d+)*)", a)
if not m:
raise SystemExit(f"{where}: struct argument {a!r} (s<N>:<p|b|x><n>.<...>)")
total, seen, n = int(m.group(1)), 0, 0
for f in m.group(2).split("."):
k, c = f[0], int(f[1:])
if c <= 0 or (k != "p" and c % 4):
raise SystemExit(f"{where}: struct field {f!r} in {a!r}")
seen += 8 * c if k == "p" else c
n += c if k == "p" else (c // 4 if k == "b" else 0)
if seen != total:
raise SystemExit(f"{where}: struct fields of {a!r} give {seen} bytes, not {total}")
return n


def arg_dwords(args, where):
n = 0
for a in args.split(","):
if a in ("p", "u"):
n += 1
elif re.fullmatch(r"b\d+", a) and int(a[1:]) % 4 == 0 and int(a[1:]) > 0:
n += int(a[1:]) // 4
elif a.startswith("s"):
n += struct_dwords(a, where)
else:
raise SystemExit(f"{where}: argument kind {a!r} (p, u, b<N> with N a multiple of 4)")
raise SystemExit(f"{where}: argument kind {a!r} (p, u, b<N> with N a multiple of 4, s<N>:<fields>)")
if n > MAX_ARG_DWORDS:
raise SystemExit(f"{where}: {n} argument DWORDs, at most {MAX_ARG_DWORDS} fit the root signature")
return n
Expand Down Expand Up @@ -297,12 +355,15 @@ def main(argv=None):
help="kernels compiled in parallel (default min(12, CPUs), env WG_DX12_JOBS; 1 = one at a time)")
ap.add_argument("--model-l", action="store_true",
help="also build Model L's kernels (co=l_*.co; off by default: Model M's kernel set unchanged)")
ap.add_argument("--model-k", action="store_true",
help="also build Model K's kernels (clean/k/dx12/kernels; off by default: Model M's and Model L's "
"kernel sets unchanged)")
ap.add_argument("--list", action="store_true",
help="print the kernels a build with these options compiles (name and code object, JSON) and exit: "
"the cheap check before a build (kernel_sets.json, tests/test_kernel_sets.py)")
a = ap.parse_args(argv)
if a.list:
print(json.dumps([list(k) for k in kernel_set(a.model_l)], indent=1))
print(json.dumps([list(k) for k in kernel_set(a.model_l, a.model_k)], indent=1))
return 0
if not a.out:
ap.error("--out is required")
Expand All @@ -323,14 +384,13 @@ def main(argv=None):
if fn.startswith("fused_") and fn.endswith("_gen.py"):
subprocess.run([sys.executable, os.path.join(KERNEL_DIR, fn)], check=True)
jobs = [] # (source, declaration, output path) in declaration order (sorted file name, then line)
for fn in sorted(os.listdir(KERNEL_DIR)) if os.path.isdir(KERNEL_DIR) else []:
if not fn.endswith(".hlsl"):
continue
src = os.path.join(KERNEL_DIR, fn)
for src, k_source in source_files(a.model_k or bool(only)):
for d in kernel_decls(src):
if only and d["name"] not in only:
continue
if not only and not a.model_l and is_model_l(d):
if not only and (k_source or is_model_k(d)) and not a.model_k:
continue
if not only and not k_source and not is_model_k(d) and not a.model_l and is_model_l(d):
continue
if not only and is_frame_gen(d):
continue
Expand All @@ -342,14 +402,16 @@ def main(argv=None):
if any(j[2] == out for j in jobs):
raise SystemExit(f"kernel {d['name']} ({d['co']}): DXIL file {os.path.basename(out)} already used "
"(a second port of a name needs file=<stem>)")
jobs.append((src, d, out))
jobs.append((src, d, out, k_source))

def build_one(job):
src, d, out = job
src, d, out, k_source = job
defs = {"WG_BLOCK": d["group"], "WG_WAVE": d["wave"], **d["defines"]}
if d["group"] != d["block"]:
defs["WG_LAUNCH_BLOCK"] = d["block"]
compile_hlsl(src, d["entry"], out, defs, asm_tool=asm, keep_listing=out[:-5] + ".ll" if a.listing else None)
info = {}
compile_hlsl(src, d["entry"], out, defs, includes=(K_HLSL_DIR,) if k_source and os.path.isdir(K_HLSL_DIR) else (), asm_tool=asm,
keep_listing=out[:-5] + ".ll" if a.listing else None, info=info)
with open(out, "rb") as f:
sha = hashlib.sha256(f.read()).hexdigest()
if a.isa:
Expand All @@ -360,6 +422,8 @@ def build_one(job):
if d["group"] != d["block"]:
r["group"] = d["group"]
r["gscale"] = d["gscale"]
if info.get("ext_doubles"):
r["fp64"] = "ext"
return r

# Every kernel is an independent chain of child processes (dxc -> dxil_asm -> dxv [-> rga]) with its own temporary
Expand All @@ -369,7 +433,7 @@ def build_one(job):
with concurrent.futures.ThreadPoolExecutor(max_workers=n_jobs) as pool:
futures = [pool.submit(build_one, j) for j in jobs]
try:
for (src, d, out), fut in zip(jobs, futures):
for (src, d, out, _k), fut in zip(jobs, futures):
kernels[(d["name"], d["co"])] = fut.result()
built.append((d["name"], out))
print(f"{d['name']:40s} -> {os.path.basename(out)} block {d['block']} wave {d['wave']} "
Expand All @@ -380,7 +444,7 @@ def build_one(job):
raise
names = {n for n, _ in kernels}
if only and not (only <= names):
raise SystemExit(f"not declared in clean/dx12/kernels: {sorted(only - names)}")
raise SystemExit(f"not declared in clean/dx12/kernels or clean/k/dx12/kernels: {sorted(only - names)}")
vgpr_check(built)
with open(man_path, "w") as f:
json.dump({"format": FORMAT, "dxc": os.path.basename(os.path.dirname(os.path.dirname(DXC_DIR))),
Expand Down
14 changes: 14 additions & 0 deletions clean/dx12/kernel_sets.json
Original file line number Diff line number Diff line change
Expand Up @@ -286,5 +286,19 @@
"swin_block_c64_add_ex_nc",
"swin.co"
]
],
"model_k": [
[
"k_expo_scalars",
"k_io.co"
],
[
"k_expo_scalars_direct",
"k_io.co"
],
[
"k_warm",
"k_block_wide.co"
]
]
}
38 changes: 36 additions & 2 deletions clean/dx12/tests/kernel_replay.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,11 @@
// module <code object name> [<tag>] (zero or more)
// buf <id> <bytes> <file | -> [fill] device buffer: the file's bytes (padded with fill when the file is
// shorter), or every byte set to fill (0..255, default 0) with "-"
// step <launch name> <grid> <block> <arg>... a launch; each arg is @<id> (a buffer's pointer) or a 32-bit decimal
// step <launch name> <grid> <block> <arg>... a launch; each arg is @<id> (a buffer's pointer), a 32-bit decimal
// value, B:<file> (a struct passed by value: the file's bytes) or
// S:<file>[:<offset>=@<id>[,<offset>=@<id>...]] (a struct with pointers
// inside, manifest argument kind s<N>:...: the file's bytes with the 8
// bytes at each byte offset replaced by that buffer's device pointer)
// save <id> <file> after the run, the buffer's bytes are written to <file>
// time <runs> after the run (and the saves), the steps run <runs> more times; the mean
// wall time per run (submission to completion, synchronised after each
Expand Down Expand Up @@ -404,13 +408,43 @@ int run_chain(Device* d, Chain& c, const std::string& base) {
// the argument values of every step, resolved once
std::vector<std::vector<DevPtr>> ptrs(c.steps.size());
std::vector<std::vector<uint32_t>> vals(c.steps.size());
std::vector<std::vector<std::vector<uint8_t>>> blobs(c.steps.size());
std::vector<std::vector<void*>> argv(c.steps.size());
for (size_t i = 0; rc == 0 && i < c.steps.size(); ++i) {
auto& s = c.steps[i];
ptrs[i].reserve(s.args.size());
vals[i].reserve(s.args.size());
blobs[i].reserve(s.args.size());
for (auto& a : s.args) {
if (!a.empty() && a[0] == '@') {
if (a.size() > 2 && (a[0] == 'B' || a[0] == 'S') && a[1] == ':') {
// B:<file> / S:<file>[:<offset>=@<id>,...]
const size_t colon = a[0] == 'S' ? a.find(':', 2) : std::string::npos;
const std::string file = a.substr(2, colon == std::string::npos ? std::string::npos : colon - 2);
std::vector<uint8_t> blob;
if (!read_file(base + "/" + file, &blob)) {
std::printf("%s: step %zu: cannot read %s\n", c.name.c_str(), i, file.c_str());
rc = 1;
break;
}
bool ok = true;
for (size_t at = colon == std::string::npos ? std::string::npos : colon + 1; ok && at != std::string::npos;) {
const size_t comma = a.find(',', at);
const std::string item = a.substr(at, comma == std::string::npos ? std::string::npos : comma - at);
const size_t eq = item.find("=@");
ChainBuf* b = eq == std::string::npos ? nullptr : find(item.substr(eq + 2));
const size_t off = eq == std::string::npos ? 0 : (size_t)std::stoull(item.substr(0, eq));
if (!b || off + 8 > blob.size()) { ok = false; break; }
for (int k = 0; k < 8; ++k) blob[off + k] = (uint8_t)((uint64_t)b->ptr >> (8 * k));
at = comma == std::string::npos ? std::string::npos : comma + 1;
}
if (!ok) {
std::printf("%s: step %zu: bad struct argument %s\n", c.name.c_str(), i, a.c_str());
rc = 1;
break;
}
blobs[i].push_back(std::move(blob));
argv[i].push_back(blobs[i].back().data());
} else if (!a.empty() && a[0] == '@') {
ChainBuf* b = find(a.substr(1));
if (!b) { std::printf("%s: step %zu: unknown buffer %s\n", c.name.c_str(), i, a.c_str()); rc = 1; break; }
ptrs[i].push_back(b->ptr);
Expand Down
33 changes: 30 additions & 3 deletions clean/dx12/tests/test_kernel_sets.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
# SPDX-License-Identifier: MIT
"""The D3D12 kernel sets, from the sources' declarations (clean/dx12/build_dx12.py kernel_set; nothing is compiled):
the cheap check before a build, in seconds. kernel_sets.json lists them: "model_m" = what a build without --model-l
compiles (Model M's kernel set; Model L's kernels never enter it), "model_l" = what --model-l adds (#89).
compiles (Model M's kernel set; Model L's and Model K's kernels never enter it), "model_l" = what --model-l adds
(#89), "model_k" = what --model-k adds (clean/k/dx12/kernels, #15).

A kernel added, removed or moved between the sets changes kernel_sets.json in the same change; regenerate it with
python clean/dx12/tests/test_kernel_sets.py --write
Expand All @@ -22,15 +23,17 @@
def current():
m = B.kernel_set(False)
full = B.kernel_set(True)
return {"model_m": [list(k) for k in m], "model_l": [list(k) for k in full if k not in set(m)]}
k = B.kernel_set(False, True)
return {"model_m": [list(x) for x in m], "model_l": [list(x) for x in full if x not in set(m)],
"model_k": [list(x) for x in k if x not in set(m)]}


class TestKernelSets(unittest.TestCase):
def test_sets_match_the_list(self):
with open(SETS) as f:
want = json.load(f)
got = current()
for name in ("model_m", "model_l"):
for name in ("model_m", "model_l", "model_k"):
w, g = set(map(tuple, want[name])), set(map(tuple, got[name]))
self.assertEqual(g, w, "%s: added %s, removed %s (update kernel_sets.json: python "
"clean/dx12/tests/test_kernel_sets.py --write)" % (name, sorted(g - w), sorted(w - g)))
Expand All @@ -48,6 +51,30 @@ def test_model_l_adds_only_model_l_kernels(self):
self.assertFalse([k for k in extra if not (k[1].startswith("l_") or k[0].startswith("l_") or k[0] in B.MODEL_L_ONLY)])


def test_model_k_adds_only_model_k_kernels(self):
"""--model-k adds Model K's kernels to Model M's set and nothing else; with --model-l it is all three sets."""
m, k = set(B.kernel_set(False)), set(B.kernel_set(False, True))
self.assertTrue(m <= k)
extra = k - m
self.assertTrue(extra)
self.assertFalse([x for x in extra if not (x[1].startswith("k_") or x[0].startswith("k_") or x[0].startswith("kx/"))])
self.assertEqual(set(B.kernel_set(True, True)), set(B.kernel_set(True)) | extra)

def test_no_k_kernel_in_the_other_sets(self):
"""Neither Model M's nor Model L's set holds a Model K kernel (a K port is declared in clean/k/dx12/kernels)."""
for full in (B.kernel_set(False), B.kernel_set(True)):
self.assertFalse([x for x in full if x[1].startswith("k_") or x[0].startswith("k_") or x[0].startswith("kx/")])

def test_struct_argument_tokens(self):
"""s<N>:<fields>: pointers are one root constant each, copied bytes one per four, skipped bytes none."""
self.assertEqual(B.struct_dwords("s184:p19.b32", "t"), 27)
self.assertEqual(B.struct_dwords("s160:p13.b56", "t"), 27)
self.assertEqual(B.struct_dwords("s32:p1.x8.b16", "t"), 5)
self.assertEqual(B.arg_dwords("p,u,s184:p19.b32,b36", "t"), 1 + 1 + 27 + 9)
for bad in ("s16:p1", "s16:p1.b4", "s12:p1.b3", "s8:q1", "s0:p0", "s8", "s8:p1.x4", "s12:p1.b4.x"):
with self.assertRaises(SystemExit, msg=bad):
B.struct_dwords(bad, "t")

def test_frame_gen_kernels_are_in_no_set(self):
"""Frame generation's ports (#96, co=fg_*.co or named fg_*) are declared but enter neither set: only --only
builds them, so the Model M and Model L builds stay as they are."""
Expand Down
Loading
Loading