diff --git a/clean/l/runner/l_frame_check.py b/clean/l/runner/l_frame_check.py index 34aa605e..0a9cce95 100644 --- a/clean/l/runner/l_frame_check.py +++ b/clean/l/runner/l_frame_check.py @@ -41,6 +41,21 @@ Output: counts only, no timing. The summary gives per stage the values differing, the largest difference (FP8 steps, FP16 ulps or codes, as the stage's metric has it), the non-finite values on each side and the largest finite magnitude on each side (saturation shows as one side stopping where the other goes on). + +Native D3D12 (runtime backend 3), after --run: + + python clean/l/runner/l_frame_check.py --build --run --backend d3d12 # GPU, inside a GPU slot + python clean/l/runner/l_frame_check.py --build --compare-d3d12 --json out.json + +/dx12 holds the D3D12 ports (python clean/dx12/build_dx12.py --out ). The same frames run on D3D12 with the +runtime's default kernels there (Model L: its encoder and decoder ports, the pack12 store l_coef_pack12, Model M's +pack12 post path; Model M: its native frame), and every stage output both backends write is compared with the HIP +run's ("default"), byte for byte: the D3D12 ports reproduce their HIP kernels byte for byte, and Model M's pack12 post +path the banded one bit for bit. The pack12 images (A: A0 A1 A2 B0 | B1 B2 C2, C: C0 C1; clean/FORMATS.md) are +unpacked and compared with the HIP run's A, B and C. Not compared: b0 ... b4 (the bottleneck megakernels write only +b5), the display image (the fused pack12 post writes none), and on the first frame the previous frame's state. An +output that is all zero on D3D12 and not on HIP is reported as not written there (the D3D12 fusions keep enc1's merged +map, and Model M's dec2 / dec1 outputs, on chip), not as differing. """ import argparse import copy @@ -159,6 +174,207 @@ def frame_inputs(rng, rw, rh, reset, special=True): "jitter": x["jitter"], "exposure": 1.0, "pre_exposure": PRE_EXPOSURE, "reset": reset} +def run_d3d12(build, special=True, stages=False): + """The frame sequence on native D3D12 (backend 3), also the raw pack12 images: "d3d12" with the default kernels, + "d3d12_stages" with the D3D12 fusions off (WG_D3D12_FUSION=none), so that every stage output is written except b0 ... + b4 (the bottleneck runs as its megakernel there: the plain C = 128 block has no D3D12 port). One configuration + per process: the runtime library reads WG_D3D12_FUSION from the environment when it is loaded.""" + if stages: + os.environ["WG_D3D12_FUSION"] = "none" # before the runtime library is loaded + from clean.runner.pipeline import Pipeline + d = model_dir(build) + setups = {"L": dict(map_path=l_map(build), check_packing=False, kernels={"options": "model=L"}), + "M": dict(check_packing=True, kernels={})} + flags = {"L": {"depth_inverted": False, "auto_exposure": True, "hdr": True}, # as the HIP run's + "M": {"depth_inverted": False, "auto_exposure": True}} + for model, kw in setups.items(): + for config in ("d3d12_stages",) if stages else ("d3d12",): + p = Pipeline(build, "", d, MAX_RENDER, OUTPUT, flags[model], backend="d3d12", **kw) + rng = np.random.default_rng(SEED) + frames = [] + for (rw, rh), reset in SEQUENCE: + fr = frame_inputs(rng, rw, rh, reset, special) + out = p.frame(fr, debug=True) + raw = {n: p._get(n, np.uint8).copy() for n in ("dec0.A", "dec0.C")} + frames.append({"inputs": fr, "out": out, "raw": raw}) + rec = {"model": model, "config": config, "kernels": p.kernels(), "notes": p.notes(), + "model_source": p.model_source, "dw": p.dw, "dh": p.dh, "ow": p.ow, "oh": p.oh, + "taps": p.output_taps(), "frames": frames} + with open(os.path.join(build, f"lframe_{model}_{config}.pkl"), "wb") as f: + pickle.dump(rec, f) + print(model, config, "frames", len(frames), "post", rec["kernels"].get("post"), "pack", + rec["kernels"].get("post_pack"), "bneck", rec["kernels"].get("swin_bneck")) + del p + + +def unpack12(raw_a, raw_c, rw, rh): + """The pack12 images -> A, B codes [rh, rw, 3] (uint8) and C [rh, rw, 3] (FP16 as float32).""" + a = np.frombuffer(raw_a.tobytes()[:rw * rh * 8], np.uint8).reshape(rh, rw, 8) + c = np.frombuffer(raw_c.tobytes()[:rw * rh * 4], np.uint8).reshape(rh, rw, 4) + A = a[..., 0:3] + B = np.stack([a[..., 3], a[..., 4], a[..., 5]], -1) + C = np.stack([c[..., 0:2].copy().view(np.float16)[..., 0], c[..., 2:4].copy().view(np.float16)[..., 0], + a[..., 6:8].copy().view(np.float16)[..., 0]], -1).astype(np.float32) + return A, B, C + + +POST_OUTPUTS = ("post.history", "post.reduced", "state.history_prev", "out") + + +def compare_backends(hip, dx, same_post=True): + """Per frame: {stage output: values differing (0 = byte-identical)} between the HIP and the D3D12 run. same_post + False (the backends run different reconstruction kernels): the post-dependent outputs are left out.""" + res = [] + for k, (fh_, fd_) in enumerate(zip(hip["frames"], dx["frames"])): + a, b = fh_["out"], fd_["out"] + rh, rw = fh_["inputs"]["color"].shape[:2] + m, compared, unwritten = {}, [], [] + + def walk(x, y, prefix=""): + for key in x: + name = prefix + key + if name in ("b0", "b1", "b2", "b3", "b4", "dec0.A", "dec0.B", "dec0.C", "post.display", "filter", + "state.info") or (k == 0 and name in UNWRITTEN_ON_FIRST) or ( + not same_post and name in POST_OUTPUTS): + continue + u, v = x[key], y.get(key) + if isinstance(u, dict): + walk(u, v, name + ".") + elif isinstance(u, np.ndarray) and isinstance(v, np.ndarray): + if u.shape != v.shape or u.dtype != v.dtype: + m[name] = -1 # not comparable: counts as differing + else: # values whose bit patterns differ + iv = np.dtype("u%d" % u.dtype.itemsize) + n = int(np.sum(np.ascontiguousarray(u).view(iv) != np.ascontiguousarray(v).view(iv))) + if n and not np.any(v) and np.any(u): # all zero on D3D12 only: not written there + unwritten.append(name) + else: + m[name] = n + compared.append(name) + elif isinstance(u, (int, float)) and isinstance(v, (int, float)): + m[name] = int(u != v) + walk(a, b) + A, B, C = unpack12(fd_["raw"]["dec0.A"], fd_["raw"]["dec0.C"], rw, rh) + dec = a["dec0"] + m["dec0.A"] = int(np.sum(A != np.rint(dec["A"] * 255.0).astype(np.uint8))) + m["dec0.B"] = int(np.sum(B != np.rint(dec["B"] * 255.0).astype(np.uint8))) + m["dec0.C"] = int(np.sum(~((C == dec["C"]) | (np.isnan(C) & np.isnan(dec["C"]))))) + if same_post: + m["out"] = int(np.sum(a["out"].view(np.uint32) != b["out"].view(np.uint32))) + res.append({"frame": k, "render": [rw, rh], "differ": m, "unwritten_d3d12": unwritten, + "identical": all(v == 0 for v in m.values())}) + return res + + +def fill_pack12(rec): + """A copy of a D3D12 record with dec0's A, B, C taken from the unpacked pack12 images (the debug view has none); + with the true-fusion post (one launch, reconstruction and filter) no display image, which that kernel does not + write.""" + out = copy.deepcopy(rec) + tf = "post_tf" in str(rec["kernels"].get("post_last", "")) + for fd in out["frames"]: + if tf: + fd["out"]["post"]["display"] = None + rh, rw = fd["inputs"]["color"].shape[:2] + A, B, C = unpack12(fd["raw"]["dec0.A"], fd["raw"]["dec0.C"], rw, rh) + fd["out"]["dec0"].update(A=(A / 255.0).astype(np.float32), B=(B / 255.0).astype(np.float32), C=C) + return out + + +def check_m_post(dx, params): + """Model L's D3D12 frame runs Model M's fused pack12 post: its history and reduced codes against Model M's post + reference (clean/reference/post.py) on the frame's own inputs (the unpacked pack12 coefficients), under + test_post_emu.py's acceptance as in --compare.""" + from clean.reference import post as Po + s = float(np.asarray(params["post"]["s"]).reshape(-1)[0]) + D = (dx["dw"], dx["dh"]) + res, prev, prev_size = [], None, None + for k, fd in enumerate(dx["frames"]): + fr, out = fd["inputs"], fd["out"] + rh, rw = fr["color"].shape[:2] + reset = prev is None or bool(fr["reset"]) or prev_size != (rw, rh) + A, B, C = unpack12(fd["raw"]["dec0.A"], fd["raw"]["dec0.C"], rw, rh) + xp = {"color": (fr["color"] / np.float32(fr["pre_exposure"])).astype(np.float32), + "exposure": out["exposure"]["e"], "exposure_prev": out["exposure"]["e_prev"], + "A": (A / 255.0).astype(np.float32), "B": (B / 255.0).astype(np.float32), "C": C, + "history": out["state"]["history_prev"] if not reset else np.zeros((D[1], D[0], 3), np.float32), + "m_near": out["enc0"]["m_near"], "m_center": out["enc0"]["m_center"], "m_far": out["enc0"]["m_far"], + "jitter": fr["jitter"], "reset": reset, "recon_size": D} + ref = Po.reconstruct(xp, s) + h = post_fp16(out["post"]["history"], ref["history"]) + dc = codes(out["post"]["reduced"], ref["reduced"], 1023.0) + dc["pass"] = dc["max_code_diff"] <= 1 and dc["differ"] <= 0.01 * dc["values"] + res.append({"frame": k, "history": h, "reduced": dc, "pass": h["pass"] and dc["pass"]}) + prev, prev_size = out, (rw, rh) + return res + + +def compare_d3d12(build, json_path): + with open(os.path.join(model_dir(build), "params.pkl"), "rb") as f: + params = pickle.load(f) + res = {} + for model in ("L", "M"): + recs = {} + for config in ("default", "d3d12"): + with open(os.path.join(build, f"lframe_{model}_{config}.pkl"), "rb") as f: + recs[config] = pickle.load(f) + # the HIP run's reconstruction: Model M's (bit-identical to the D3D12 fused pack12 post) or Model L's own + hip_post = recs["default"]["kernels"].get("l_post") or recs["default"]["kernels"].get("post_last") + same_post = hip_post != "l_post_recon" + frames = compare_backends(recs["default"], recs["d3d12"], same_post) + res[model] = {"kernels_d3d12": recs["d3d12"]["kernels"], "frames": frames, + "unwritten_d3d12": sorted({n for f in frames for n in f["unwritten_d3d12"]}), + "outputs_compared": sorted({n for f in frames for n in f["differ"]}), + "identical_frames": sum(f["identical"] for f in frames), + "differing": sorted({k for f in frames for k, v in f["differ"].items() if v}), + "post_compared_with_hip": same_post} + if not same_post: + # Model L: HIP runs l_post_recon, D3D12 Model M's fused pack12 post, and each frame's encoder reads the + # previous frame's reconstruction, so the two runs part after the first frame. The D3D12 frame is checked + # on its own: every stage of the run with all outputs written against the reference on that run's inputs + # (as --compare), and the default kernels against that run byte for byte. + with open(os.path.join(build, f"lframe_{model}_d3d12_stages.pkl"), "rb") as f: + st = fill_pack12(pickle.load(f)) + per_stage = compare_model(st, params[model], post_ref="M", bneck_unit=True) + fused = compare_backends(st, recs["d3d12"]) + res[model]["d3d12_per_stage"] = {"frames": per_stage, "summary": summary(per_stage), + "passed": sum(m["pass"] for m in per_stage)} + res[model]["d3d12_default_vs_stages"] = { + "frames": fused, "identical_frames": sum(f["identical"] for f in fused), + "differing": sorted({k for f in fused for k, v in f["differ"].items() if v}), + "unwritten_default": sorted({n for f in fused for n in f["unwritten_d3d12"]})} + mp = check_m_post(recs["d3d12"], params[model]) + res[model]["m_post_reference"] = {"frames": mp, "passed": sum(f["pass"] for f in mp)} + n_frames = {m: len(res[m]["frames"]) for m in ("L", "M")} + + def model_pass(m): + r = res[m] + if r["post_compared_with_hip"]: + return r["identical_frames"] == n_frames[m] + return (r["d3d12_per_stage"]["passed"] == n_frames[m] and r["m_post_reference"]["passed"] == n_frames[m] and + r["d3d12_default_vs_stages"]["identical_frames"] == n_frames[m]) + res["pass"] = all(model_pass(m) for m in ("L", "M")) + with open(json_path, "w") as f: + json.dump(res, f, indent=1, default=lambda o: o.item() if hasattr(o, "item") else str(o)) + for model in ("L", "M"): + print(model, "D3D12 against HIP: frames byte-identical", res[model]["identical_frames"], "of", + len(res[model]["frames"]), "outputs compared", len(res[model]["outputs_compared"]), "differing", + res[model]["differing"], "not written on D3D12 (all zero; fused kernels)", res[model]["unwritten_d3d12"]) + if "d3d12_per_stage" in res[model]: + r = res[model] + print(" (HIP runs l_post_recon, D3D12 Model M's fused pack12 post: the runs part after frame 0)") + print(" D3D12 per stage against the reference (fusions off; b0-b5 as one unit):", + r["d3d12_per_stage"]["passed"], + "of", len(r["frames"]), "frames pass;", "failing:", + sorted({s for f in r["d3d12_per_stage"]["frames"] for s, ok in f["stage_pass"].items() if not ok})) + print(" D3D12 default kernels against that run: byte-identical", r["d3d12_default_vs_stages"][ + "identical_frames"], "of", len(r["frames"]), "differing", r["d3d12_default_vs_stages"]["differing"], + "not written by the defaults", r["d3d12_default_vs_stages"]["unwritten_default"]) + print(" its post against Model M's post reference (default kernels):", r["m_post_reference"]["passed"], + "of", len(r["frames"]), "frames pass") + return 0 if res["pass"] else 1 + + def run(build, rocm, special=True): from clean.runner.hip import clean_process_environment clean_process_environment(rocm) @@ -257,7 +473,10 @@ def post_fp16(got, ref, model="M"): return m -def compare_model(rec, params): +def compare_model(rec, params, post_ref=None, bneck_unit=False): + """post_ref "M": the post against Model M's reference whatever the model (Model L's native D3D12 frame runs Model + M's fused pack12 post for now). bneck_unit: b0 ... b5 as one unit (a bottleneck megakernel writes only b5): the + reference chained from the runtime's enc2.merged, b5 under the per-stage rule.""" from clean.l.reference import oracle as L from clean.l.reference import table as T from clean.reference import decoder as MD @@ -323,6 +542,9 @@ def compare_model(rec, params): kind = S.STAGES[s]["kind"] skip = skips.get("enc2" if s == "dec2" else "enc1") if kind == "dec" else None ref = S.run_stage(s, params[s], x, off, skip, gelu="sigmoid") + if bneck_unit and s in ("b0", "b1", "b2", "b3", "b4"): + x = ref["out"] # the reference's own chain (the runtime does not write b0 ... b4) + continue m[s] = {key: fp8_stage(out[s][key], ref[key]) for key in out[s]} if kind == "enc": skips[s] = out[s]["skip"] @@ -348,7 +570,8 @@ def compare_model(rec, params): "m_near": out["enc0"]["m_near"], "m_center": out["enc0"]["m_center"], "m_far": out["enc0"]["m_far"], "jitter": fr["jitter"], "reset": eff_reset, "recon_size": D} s_post = float(np.asarray(params["post"]["s"]).reshape(-1)[0]) - ref = R.reconstruct(xp, s_post) if model == "L" else Po.reconstruct(xp, s_post) + l_recon = model == "L" and post_ref != "M" + ref = R.reconstruct(xp, s_post) if l_recon else Po.reconstruct(xp, s_post) m["post"] = {"history": post_fp16(out["post"]["history"], ref["history"], model)} if out["post"]["display"] is not None: m["post"]["display"] = post_fp16(out["post"]["display"], ref["display"], model) @@ -359,7 +582,7 @@ def compare_model(rec, params): disp, e = out["post"]["display"], out["exposure"]["e"] if rec["taps"] is not None: fo = Po.output_filter_original(disp, e, (rec["ow"], rec["oh"]), rec["taps"]) - elif model == "L": + elif l_recon: fo = R.output_filter(disp, e, (rec["ow"], rec["oh"])) else: fo = Po.output_filter(disp, e, (rec["ow"], rec["oh"])) @@ -369,7 +592,7 @@ def compare_model(rec, params): stage_pass = {s: (all(v["pass"] for v in m[s].values() if isinstance(v, dict) and "pass" in v) if isinstance(m[s], dict) else True) - for s in ["enc0"] + list(T.SHARED_STAGES) + ["post"]} + for s in ["enc0"] + list(T.SHARED_STAGES) + ["post"] if s in m} stage_pass["dec0"] = m["dec0"]["pass"] stage_pass["state"] = m["state"]["pass"] if "filter" in m: @@ -481,6 +704,10 @@ def main(argv=None): g.add_argument("--prepare", action="store_true") g.add_argument("--run", action="store_true") g.add_argument("--compare", action="store_true") + g.add_argument("--compare-d3d12", action="store_true") + ap.add_argument("--backend", choices=("hip", "d3d12", "d3d12-stages"), default="hip", + help="--run: HIP (both bottleneck forms), native D3D12 with its default kernels, or native D3D12 " + "with its fusions off (d3d12-stages; run both D3D12 forms before --compare-d3d12)") ap.add_argument("--json") ap.add_argument("--plain", action="store_true", help="--run: frames without random_inputs' special values (NaN / huge / negative colours, NaN " @@ -497,8 +724,13 @@ def main(argv=None): prepare(a.build, a.rocm, a.model, a.map) return 0 if a.run: - run(a.build, a.rocm, special=not a.plain) + if a.backend.startswith("d3d12"): + run_d3d12(a.build, special=not a.plain, stages=a.backend == "d3d12-stages") + else: + run(a.build, a.rocm, special=not a.plain) return 0 + if a.compare_d3d12: + return compare_d3d12(a.build, a.json or os.path.join(a.build, "lframe_d3d12.json")) return compare(a.build, a.json or os.path.join(a.build, "lframe_check.json")) diff --git a/clean/l/runtime/tests/test_l_runtime_frame.py b/clean/l/runtime/tests/test_l_runtime_frame.py index 33def5aa..0c2e1250 100644 --- a/clean/l/runtime/tests/test_l_runtime_frame.py +++ b/clean/l/runtime/tests/test_l_runtime_frame.py @@ -27,8 +27,11 @@ python -m unittest discover -s clean/l/runtime/tests -t clean/l/runtime/tests -p test_l_runtime_frame.py """ import copy +import glob import hashlib +import json import os +import re import shutil import struct import subprocess @@ -84,12 +87,18 @@ def map_path(): return os.path.join(model_dir(), "maps", "preset-L.v1.map.json") -def run(options="", frames=4, build_files=(), env=None, model=None, exe=None, flags=(), rw=RW, rh=RH, hdr=1): +def run(options="", frames=4, build_files=(), env=None, model=None, exe=None, flags=(), rw=RW, rh=RH, hdr=1, + manifest=None): """The trace program's output lines (WG_* removed from the environment, then env applied). hdr: WgRtConfig.hdr - (Model L's reconstruction kernel is HDR; Model M does not read the flag).""" + (Model L's reconstruction kernel is HDR; Model M does not read the flag). manifest: [(kernel, code object)] for + /dx12/manifest.json (the D3D12 ports a build has; backend 3).""" build = tempfile.mkdtemp(prefix="wg_lfb_") for n in build_files: open(os.path.join(build, n), "wb").close() + if manifest is not None: + os.makedirs(os.path.join(build, "dx12")) + with open(os.path.join(build, "dx12", "manifest.json"), "w") as fh: + json.dump({"format": 1, "kernels": [{"name": k, "co": co} for k, co in manifest]}, fh) e = {k: v for k, v in os.environ.items() if not k.upper().startswith("WG_")} e.update(env or {}) try: @@ -154,12 +163,19 @@ def expected_args(t, f, digests): } -def check_roles(t, f, digests): +def base(kernel): + """A Model L launch's kernel without its clamp-free suffix; the pack12 coefficient store (native D3D12) as l_coef, + whose arguments it shares.""" + k = kernel.replace("_nc", "") + return "l_coef" if k == "l_coef_pack12" else k + + +def check_roles(t, f, digests, l_post=True): """Raises AssertionError naming every Model L launch argument that is not the expected buffer or blob.""" exp = expected_args(t, f, digests) seen, bad = set(), [] for ln in f["launches"]: - k = ln.kernel.replace("_nc", "") + k = base(ln.kernel) if k not in exp: continue seen.add(k) @@ -169,11 +185,47 @@ def check_roles(t, f, digests): continue bad += ["%s argument %d: %s, expected %s" % (k, i, a, e) for i, (a, e) in enumerate(zip(ln.args, want)) if e is not None and a != e] - missing = [k for k in L_KERNELS if k not in seen] + missing = [k for k in L_KERNELS if k not in seen and (l_post or k != "l_post_recon")] if bad or missing: raise AssertionError("; ".join(bad + ["not launched: " + k for k in missing])) +def declared_ports(): + """[(kernel, code object)] of every D3D12 port declared in clean/dx12/kernels (the "@wg-kernel" lines that + clean/dx12/build_dx12.py builds into the manifest): what a full build's manifest names.""" + out = [] + for path in sorted(glob.glob(os.path.join(host_cxx.ROOT, "clean", "dx12", "kernels", "*.hlsl"))): + with open(path, encoding="utf-8") as fh: + for m in re.finditer(r"@wg-kernel\s+(\S+)\s+co=(\S+)", fh.read()): + if m.groups() not in out: + out.append(m.groups()) + return out + + +# Model L's D3D12 kernels the runtime checks at create (runtime.cpp l_native_missing), with their code objects +L_D3D12 = [("lev/input_encoder", "l_enc_values.co"), ("l_linear_32_32_relu", "l_ops.co"), + ("l_linear_32_32_f16", "l_ops.co"), ("l_swin_block_c32_enc_nc", "l_swin.co"), ("l_project_gather", "l_ops.co"), + ("swin_block_c64_add_ex_nc", "swin.co"), ("l_dec_project", "l_ops.co"), ("l_swin_block_c32_dec_nc", "l_swin.co"), + ("l_coef_pack12", "l_ops.co")] + + +def full_ports(): + """The declared ports plus any of Model L's not declared yet: a build where every port Model L needs has landed. + (Model M's wave64 post ports come in both filter builds, keyed by name and code object: Model L's file has no + output-filter weights, so it takes the analytic one, post_w64.co.)""" + have = declared_ports() + return have + [kc for kc in L_D3D12 if kc[0] not in [k for k, _ in have]] + + +class Ports(unittest.TestCase): + def test_analytic_post_declared(self): + """The analytic-filter build of Model M's fused pack12 post (the post Model L's native frame runs) is among the + declared ports.""" + have = declared_ports() + for k in ("w64/post_tf_f32_p12_diet_vsit", "w64/post_tf4_dl_f32_p12_diet_vsit"): + self.assertIn((k, "post_w64.co"), have) + + def cdiv(a, b): return -(-a // b) @@ -254,12 +306,12 @@ def test_model_m_unchanged(self): class Arguments(unittest.TestCase): """The recorder's view of every Model L launch (clean/l/FORMATS.md, "Maps" and "Kernels").""" - def check_frame(self, t, f, roles=True): + def check_frame(self, t, f, roles=True, l_post=True): info, ls = f["info"], f["launches"] rw, rh = info["rw"], info["rh"] fw, fh = cdiv(rw, 32) * 32, cdiv(rh, 32) * 32 npix, tw, th = (fh + 4) * (fw + 4), fw // 2, fh // 2 - k = {ln.kernel.replace("_nc", ""): (i, ln) for i, ln in enumerate(ls)} + k = {base(ln.kernel): (i, ln) for i, ln in enumerate(ls)} _, lev = k["lev/input_encoder"] _, relu = k["l_linear_32_32_relu"] _, f16 = k["l_linear_32_32_f16"] @@ -305,13 +357,14 @@ def check_frame(self, t, f, roles=True): self.assertEqual(e[5], info["reset"]) self.assertEqual(d, [rw, rh, fw, fh, info["reset"]]) post = [ln for ln in ls if ln.kernel == "l_post_recon"] - self.assertEqual(len(post), 1) - self.assertEqual(post[0].params(12), [rw, rh, (3 * 120 + 1) // 2, (3 * 72 + 1) // 2, info["reset"]]) - self.assertEqual(post[0].grid, cdiv((3 * 120 + 1) // 2, 32) * cdiv(cdiv((3 * 72 + 1) // 2, 2), 8)) - self.assertTrue(ls[ls.index(post[0]) + 1].kernel.endswith("post_filter_c1_s4")) # Model M's filter next + self.assertEqual(len(post), 1 if l_post else 0) # (native D3D12: Model M's fused pack12 post for now) + if l_post: + self.assertEqual(post[0].params(12), [rw, rh, (3 * 120 + 1) // 2, (3 * 72 + 1) // 2, info["reset"]]) + self.assertEqual(post[0].grid, cdiv((3 * 120 + 1) // 2, 32) * cdiv(cdiv((3 * 72 + 1) // 2, 2), 8)) + self.assertTrue(ls[ls.index(post[0]) + 1].kernel.endswith("post_filter_c1_s4")) # Model M's filter next self.assertTrue(lev.args[11].startswith("b")) # the values blob: an allocation if roles: # every argument against the named buffers and blobs - check_roles(t, f, blob_digests()) + check_roles(t, f, blob_digests(), l_post) return lev, coef def test_sizes_reset_history(self): @@ -426,6 +479,89 @@ def test_swapped_weights_in_the_runtime(self): self.assertEqual(msg.count("argument"), 2, msg) +class D3D12(unittest.TestCase): + """Model L on native D3D12 (backend 3): Model M's native frame with Model L's encoder and decoder, the pack12 + coefficient store for Model M's pack12 post path; each missing D3D12 port refused by name, never replaced.""" + + def l_run(self, ports, frames=3, flags=(), **kw): + return run("model=L", frames=frames, build_files=FT.SHIPPING, flags=("--backend=3",) + tuple(flags), + manifest=ports, **kw) + + def test_missing_ports_named(self): + have = [k for k, _ in declared_ports()] + missing = [k for k, _ in L_D3D12 if k not in have] + got = self.l_run(declared_ports(), frames=1) + if not missing: + self.assertEqual(got[0], "create ok model L") + return + self.assertEqual(got, ["ERR create: Model L on native D3D12: not yet ported to D3D12: " + ", ".join(missing)]) + + def test_each_port_named(self): + for k, _ in L_D3D12: + with self.subTest(port=k): + got = self.l_run([kc for kc in full_ports() if kc[0] != k], frames=1) + self.assertEqual(got, ["ERR create: Model L on native D3D12: not yet ported to D3D12: " + k]) + + def check_d3d12_frame(self, t, f): + """The #111 checks (map chain, extents, offsets, parameters, every argument's buffer or blob), then what the + native frame adds: the pack12 store's images, dec1's own block, Model M's pack12 post path after it.""" + info = f["info"] + rw, rh = info["rw"], info["rh"] + names = [ln.kernel for ln in f["launches"]] + self.assertIn("l_coef_pack12", names) + self.assertNotIn("l_coef", names) + self.assertTrue(all(not n.startswith(("d1p12/", "d0p12/")) and n != "input_encoder" for n in names), names) + dec1 = [ln for ln in f["launches"] if ln.kernel.startswith("swin_block_c64_add")] + self.assertEqual([ln.kernel for ln in dec1], ["swin_block_c64_add_ex_nc"]) # dec1's own block, not fused + lev, coef = Arguments.check_frame(self, t, f, l_post=False) + self.assertGreaterEqual(t.extent(coef.args[1]), 8 * rw * rh) # pack12 image A: 8 bytes per pixel + self.assertGreaterEqual(t.extent(coef.args[3]), 4 * rw * rh) # pack12 image C: 4 bytes per pixel + i = names.index("l_coef_pack12") + self.assertTrue(names[i + 1].startswith("w64/post_tf") and "_p12" in names[i + 1], names[i + 1:]) + return lev, coef + + def test_frame(self): + """Sizes aligned, non-aligned and changing, a game reset; history handed from frame to frame.""" + # upscaling sizes the pack12 post path takes (D = 180 x 108): aligned, non-aligned, a change of size + sizes = "--sizes=96x64,96x64,80x48,80x48,80x48,77x45" + t = FT.Trace(self.l_run(full_ports(), frames=6, rw=96, rh=64, flags=(sizes, "--reset=4"))) + self.assertEqual(t.create, "create ok model L") + self.assertEqual(t.unknown(), []) + self.assertEqual([f["info"]["reset"] for f in t.frames], [1, 0, 1, 0, 1, 1]) + self.assertEqual(len({f["info"]["parity"] for f in t.frames}), 2) + prev = None + for f in t.frames: + self.assertIsNone(f["error"]) + lev, coef = self.check_d3d12_frame(t, f) + if prev: + self.assertEqual(lev.args[4], prev[1].args[4]) # the latent the last frame's store wrote + self.assertEqual(lev.args[5], prev[0].args[7]) # its near motion + prev = (lev, coef) + + def test_failed_launch_stops_the_frame(self): + t = FT.Trace(self.l_run(full_ports(), frames=3, flags=("--fail=l_coef_pack12@1",))) + ok0, bad, ok2 = t.frames + self.assertEqual(bad["failed"], "l_coef_pack12") + names = [ln.kernel for ln in bad["launches"]] + self.assertEqual(names, [ln.kernel for ln in ok0["launches"]][:len(names)]) + self.assertTrue(names[-1].startswith("l_swin_block_c32_dec")) + self.assertIsNone(ok2["error"]) + self.check_d3d12_frame(t, ok2) + + def test_model_m_unchanged(self): + """Model M on backend 3 traces the same in the build with Model L as in the build without it.""" + a = run(frames=2, build_files=FT.SHIPPING, flags=("--backend=3",), manifest=full_ports()) + b = run(frames=2, build_files=FT.SHIPPING, flags=("--backend=3",), manifest=full_ports(), + exe=FT.trace_exe()) + self.assertEqual([re.sub(r"wg_lfb_\w+", "B", x) for x in a], [re.sub(r"wg_lfb_\w+", "B", x) for x in b]) + self.assertTrue(a[0].startswith("create ok model M"), a[0]) + + def test_hip_unchanged(self): + """On HIP Model L keeps the base coefficient format (l_coef) and refuses pack12.""" + self.assertTrue(run("model=L", build_files=FT.SHIPPING, env={"WG_POST_PACK": "p12"})[0].startswith( + "ERR create: Model L: post_pack=p12")) + + class Refusals(unittest.TestCase): def test_paths(self): self.assertTrue(run("model=L", env={"WG_DEC1_DEC0": "fused"})[0].startswith( diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index a70080e0..fe39bb6c 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -500,7 +500,8 @@ struct Runtime { ModelId model_id = ModelId::M; // runtime option model=K (clean/k/runtime/model_k.h); default model M if (std::string merr; !model_from_options(c, &model_id, &merr)) return fail(merr); if (c.backend < 0 || c.backend > 3) return fail("unknown runtime backend"); - const bool native_ok = model_info(model_id).native_d3d12; // model_select.h (Model M only today) + // model_select.h (Model M); Model L on native D3D12 in builds with it (its missing D3D12 ports are refused below) + const bool native_ok = model_info(model_id).native_d3d12 || (model_id == ModelId::L && model_l_built()); if (native_direct_io && (!external_io || c.backend != 3 || !native_ok)) return fail("native direct resources require external_io, backend 3 and " + native_d3d12_models()); if (c.backend == 3 && !native_ok) return fail("native D3D12 supports " + native_d3d12_models() + " only"); @@ -514,7 +515,13 @@ struct Runtime { #endif model_l = model_id == ModelId::L; // (model_from_options accepts it only in builds with WG_MODEL_L) if (model_l && (c.backend == 1 || c.backend == 2)) - return fail("Model L runs on the HIP backend (backend 0) only for now"); + return fail("Model L runs on the HIP backend (backend 0) and native D3D12 (backend 3) only"); +#ifdef WG_MODEL_L + if (model_l && c.backend == 3) { + const std::string missing = l_native_missing(); + if (!missing.empty()) return fail("Model L on native D3D12: not yet ported to D3D12: " + missing); + } +#endif if (c.max_rw <= 0 || c.max_rh <= 0 || c.ow <= 0 || c.oh <= 0) return fail("invalid render or output size"); // post_core's fallback colour read uses a 32-bit byte offset (mul24, kernels/post.hip): the render colour image // must stay within 4 GB (16 rw rh <= 2^32, 16 rw < 2^24; any D3D12 texture, at most 16384 x 16384, does) @@ -541,16 +548,23 @@ struct Runtime { if (c.wpw128) wpw[2] = c.wpw128; if (!choose_kernels(c)) return false; if (model_l && recon_output) return fail("Model L: recon=output (D = O) is Model M's experimental mode"); - // Model L's reconstruction kernel l_post_recon is HDR; its LDR form (11/11/10-bit display codes) is not written yet - if (model_l && !c.hdr) + // Model L's reconstruction kernel l_post_recon is HDR; its LDR form (11/11/10-bit display codes) is not written yet. + // (Native D3D12 runs Model M's pack12 post for Model L, which takes both.) + if (model_l && !c.hdr && c.backend != 3) return fail("Model L: LDR colour (WgRtConfig.hdr = 0) needs Model L's LDR reconstruction kernel, which is not " "written yet (l_post_recon is HDR)"); if (model_l && enc0_co != "encoder.co") return fail("Model L: enc0 variants are Model M's encoder; Model L's runs from l_enc_values.co"); // Model L's coefficient kernel (l_coef) writes Model M's coef_decoder formats, not pack12: the _p12 post kernels // are not for it (choose_kernels does not choose pack12 by default for Model L; asked for, it is refused here) - if (model_l && p12) return fail("Model L: post_pack=p12 (the pack12 decoder) is Model M's; Model L's l_coef " - "writes the base coefficient format"); + // On native D3D12 Model L stores its coefficients in pack12 (l_coef_pack12, lossless) for Model M's pack12 post + // path, the only post path ported there; on HIP it writes the base format (l_coef) for the post_recon path. + if (model_l && p12 && c.backend != 3) + return fail("Model L: post_pack=p12 (the pack12 decoder) is Model M's; Model L's l_coef writes the base " + "coefficient format (pack12 is Model L's native D3D12 form only)"); + if (model_l && !p12 && c.backend == 3) + return fail("Model L on native D3D12 needs the pack12 post path (post_pack=p12; Model M's decoder__pack12.co " + "selects it)"); bool any = false; for (int i = 0; i < 16; ++i) any |= c.d_row[i] != 0; drow = drow_gfx12(); @@ -603,6 +617,9 @@ struct Runtime { uint32_t flags = 0; std::memcpy(&flags, blobs[i ? "dec0_block" : "enc0_block"].data(), 4); l_nc[i] = (flags & 1u) != 0; + if (!l_nc[i] && c.backend == 3) + return fail(std::string("Model L on native D3D12: ") + (i ? "dec0_block" : "enc0_block") + + "'s weights need the clamped block, which is not ported to D3D12 (only the _nc forms are)"); } } #endif @@ -872,8 +889,10 @@ struct Runtime { #ifdef WG_MODEL_L // Model L reconstructs with its own kernel, l_post_recon (l_post.co; clean/l/FORMATS.md step 5), which takes Model // M's post_recon arguments and buffers: the unbanded post_recon route (8-byte history, base coefficient format, - // no pack12), then Model M's output filter (unbanded). Model M's post kernels are not for Model L. - if (model_l) { + // no pack12), then Model M's output filter (unbanded). Model M's post kernels are not for Model L. On native D3D12 + // Model L's frames run Model M's pack12 post path for now (l_coef_pack12 feeds it); frames that path does not take + // run l_post_recon there too (its D3D12 port, then the analytic filter). + if (model_l && c.backend != 3) { if ((c.post_kernel && *c.post_kernel) || choose(nullptr, "WG_POST_KERNEL", "") != "") return fail("Model L: its reconstruction is l_post_recon; a post kernel (post_kernel / WG_POST_KERNEL) " "chooses among Model M's"); @@ -891,7 +910,8 @@ struct Runtime { { const bool post_given = (c.post_kernel && *c.post_kernel) || choose(nullptr, "WG_POST_KERNEL", "") != ""; const bool dec0_given = (c.dec0_variant && *c.dec0_variant) || choose(nullptr, "WG_DEC0_VARIANT", "") != ""; - p12_default = !model_l && module_available(p12_path) && !post_given && !dec0_given && !recon_output && + p12_default = (!model_l || c.backend == 3) && module_available(p12_path) && !post_given && !dec0_given && + !recon_output && choose(c.dec1_dec0, "WG_DEC1_DEC0", "separate") != "fused"; } const std::string pack = choose(c.post_pack, "WG_POST_PACK", p12_default ? "p12" : "none"); @@ -1118,13 +1138,16 @@ struct Runtime { if (enc0_co != "encoder.co" || exposure_co != "exposure.co" || dec0_co != "decoder.co" || swin_co != "swin.co" || !stage_geo.empty()) return fail("native direct resources require the original encoder, exposure, decoder and Swin code-object variants"); - if (!output_original || output_quality5 || output_quality9 || !has_taps || post_pre != "w64/") - return fail("native direct resources require post_w64__orig.co with the original 42-DWORD output-filter arguments"); + // the wave64 fused post: the "original" filter build with the model file's weights (post_w64__orig.co, 42 DWORDs), + // or the analytic filter build (post_w64.co, 22 DWORDs: no weights; a model file without them, as Model L's) + if (output_quality5 || output_quality9 || post_pre != "w64/" || (output_original && !has_taps)) + return fail("native direct resources require the wave64 fused post: post_w64__orig.co with the model's " + "output-filter weights, or the analytic post_w64.co"); if (recon_output || !p12 || !post_band || post_fused || out_f16 || post_wave != 64 || post_kernel != "post_recon2_r_band_p12" || filter_kernel != "post_filter_c1_s4_band" || !post_tf || !post_diet || !tf_vdiet || !post_sel || !post_idx || !post_trim) return fail("native direct resources support only post_tf_f32_p12_diet_vsit; alternate post/TF routes are unsupported"); - if (!d1d0_p12 || !dec_div.empty() || dec1_dec0 || bneck_variant != "bmegahg" || + if ((!d1d0_p12 && !model_l) || !dec_div.empty() || dec1_dec0 || bneck_variant != "bmegahg" || bneck_threads != 256 || bneck_groups != 128 || !mid_variant.empty() || swin_resident || force_clamp || fuse) return fail("native direct resources require default pack12 decoder, bmegahg bottleneck and nonresident unclamped Swin routes"); if (gemm_threads != 256 || nt != 2 || wpw[0] != 1 || wpw[1] != 1 || wpw[2] != 1) @@ -1497,7 +1520,8 @@ struct Runtime { bool l_encoder(EncParams& p, DevPtr cnet, DevPtr din, DevPtr min_, int cur, int nxt, int fh, int fw, int f) { DevPtr wv = wdev["enc0.values"], w1 = wdev["enc0.Win1"], w2 = wdev["enc0.Win2"], wp = wdev["enc0.project"]; int npix = (fh + 4) * (fw + 4), tw = fw / 2, th = fh / 2; - return launch("lev/input_encoder", cdiv(((fh + 4) / 2) * ((fw + 4) / 2), 128), 256, + return launch(native_direct_io ? "lev/input_encoder_direct" : "lev/input_encoder", + cdiv(((fh + 4) / 2) * ((fw + 4) / 2), 128), 256, {&cnet, &din, &min_, &reduced, &latent[cur], &mnear[cur], &l_wide, &mnear[nxt], &mfar, &mcenter, &expo, &wv, &p}) && launch("l_linear_32_32_relu", cdiv(npix, 128), 256, {&l_wide, &l_narrow, &w1, &npix}) && @@ -1514,7 +1538,34 @@ struct Runtime { int tw = fw / 2, th = fh / 2; return launch("l_dec_project", cdiv(tw * th, 128), 256, {&out1, &l_narrow, &wd, &tw, &th}) && l_block("dec", l_narrow, l_wide, wdev["dec0_block"], l_nc[1], fh, fw, true, f) && - launch("l_coef", cdiv(tw * th, 128), 256, {&l_wide, &A, &B, &C, &latent[nxt], &wc, &p}); + launch(p12_frame ? "l_coef_pack12" : "l_coef", cdiv(tw * th, 128), 256, + {&l_wide, &A, &B, &C, &latent[nxt], &wc, &p}); + } + + // Model L's kernels on native D3D12 that /dx12/manifest.json lacks, comma-separated ("" = all there): + // its encoder (the direct form on the direct-resource route), linears, clamp-free blocks (the only D3D12 forms), + // projections and pack12 coefficient store, and dec1's own block with its expand (Model M runs dec1 inside its fused + // decoder there; Model L's decoder reads dec1's map). Model M's other kernels are its native frame's. + std::string l_native_missing() const { + std::string text, error, missing; + wgc::Json manifest; + std::vector have; + if (read_text(build_dir + "/dx12/manifest.json", &text) && wgc::parse_json(text, manifest, error)) { + const auto* kernels = manifest.find("kernels"); + if (kernels && kernels->is_array()) + for (const auto& k : kernels->items) { + const auto* name = k.find("name"); + if (name && name->is_string()) have.push_back(name->text); + } + } + const std::vector need = {native_direct_io ? "lev/input_encoder_direct" : "lev/input_encoder", + "l_linear_32_32_relu", "l_linear_32_32_f16", + "l_swin_block_c32_enc_nc", "l_project_gather", + "swin_block_c64_add_ex_nc", "l_dec_project", + "l_swin_block_c32_dec_nc", "l_coef_pack12"}; + for (const std::string& k : need) + if (std::find(have.begin(), have.end(), k) == have.end()) missing += (missing.empty() ? "" : ", ") + k; + return missing; } #endif diff --git a/clean/tests/runtime_frame_trace.cpp b/clean/tests/runtime_frame_trace.cpp index 071cf224..0c0de8c2 100644 --- a/clean/tests/runtime_frame_trace.cpp +++ b/clean/tests/runtime_frame_trace.cpp @@ -7,6 +7,7 @@ // // rw x rh is the maximum render size. Flags: // --format= model_format (default 1: the converter's raw layout; 2: model.wgcm, with --map) +// --backend= WgRtConfig.backend (default 0; the recording device stands in for every backend) // --map= the converter map (map_path) // --sizes=WxH,... the render size of frame 0, 1, ... (repeated; default: the maximum every frame) // --reset=f,... frames the game asks for a reset (WgRtFrame.reset = 1) @@ -79,6 +80,7 @@ const Signature kSignatures[] = { {"l_dec_project", "pppii"}, {"l_coef", "ppppppD"}, {"l_post_recon", "ppppppppppppPp"}, // Model M's post_recon arguments (clean/l/FORMATS.md step 5) + {"l_coef_pack12", "ppppppD"}, // the same arguments (B unused) }; const char* kinds_of(const std::string& kernel) { @@ -266,6 +268,8 @@ int main(int argc, char** argv) { const std::string s = argv[a]; if (s.compare(0, 9, "--format=") == 0) { cfg.model_format = std::atoi(s.c_str() + 9); + } else if (s.compare(0, 10, "--backend=") == 0) { + cfg.backend = std::atoi(s.c_str() + 10); } else if (s.compare(0, 6, "--map=") == 0) { map_path = s.substr(6); } else if (s.compare(0, 8, "--sizes=") == 0) {