From e74a610305afff4bfe027f7468ed1eca4613fcfb Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:19:48 -0500 Subject: [PATCH 1/5] Model L: the hybrid form as the native D3D12 default (hybrid_full), with its checks and speed levers Model L runs the hybrid block form by default on native D3D12 (no form variable needed; WG_L_FORM=hybrid gives the same bytes): FP8 matrix chains accumulated in float32 with one FP16 rounding at each chain end, the original's FP16 norm, softmax and GELU steps by exact tables and FP16 forms. The HIP twins (hybrid_twin) are the byte reference. - Kernels: the hybrid C32 blocks with the input linears as their prologue and the pack12 store as their epilogue; the shared-stage hybrid blocks, merges and expands; the bottleneck b0 ... b5 in one persistent-queue launch (swin_bneck6_hyb, bneck.hlsl's schedule around swin_block_c128_hyb's body). - Tables: GELU16 with 16-bit entries, rsqrt and rcp, reserved in the arena for Model L hybrid contexts only (Model M and K keep their arena layout); a missing table range refuses create in one line. - No silent fallback: every kernel the route launches is required by name at create; a refused route is never replaced by another one. - Driver contraction barrier: the norm's squares pass an OR with a zero the compiler cannot prove; build_dx12.py's fused-square scan fails a build where a hybrid kernel's ISA has a fused square, and each hybrid kernel's scan mutant (no barrier; for the C96 blocks a substitute mutant) must fail it. - Checks: per-stage oracle comparison with dump kernels (random weights with finite codes plus one NaN-input leg reported separately, and real weights), D3D12 == HIP bytes for every block, merge and expand, the network byte leg (HIP twins vs D3D12, both legs), default leg (each environment case in its own process), CPU recorder frame tests. Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/build_dx12.py | 133 +- clean/dx12/hlsl/hyb_block.hlsli | 168 +++ clean/dx12/hlsl/l_coef_px.hlsli | 8 + clean/dx12/hlsl/l_q7_common.hlsli | 62 + clean/dx12/hlsl/l_q7_op8.hlsli | 96 ++ clean/dx12/hlsl/wg_hyb.hlsli | 158 +++ clean/dx12/hyb_isa_scan.py | 362 ++++++ clean/dx12/kernel_sets.json | 120 ++ clean/dx12/kernels/dec1_dec0.hlsl | 31 + clean/dx12/kernels/fused_dec.hlsl | 23 + clean/dx12/kernels/fused_swin.hlsl | 288 ++++- clean/dx12/kernels/fused_swin_gen.py | 3 + clean/dx12/kernels/hyb_bneck.hlsl | 162 +++ clean/dx12/kernels/hyb_gemm.hlsl | 186 +++ clean/dx12/kernels/hyb_swin.hlsl | 31 + clean/dx12/kernels/l_hyb_probe.hlsl | 60 + clean/dx12/kernels/l_post.hlsl | 27 + clean/dx12/kernels/l_q7.hlsl | 474 +++++++ clean/dx12/kernels/l_q7_s2a_probe.hlsl | 74 ++ clean/dx12/kernels/l_q7w.hlsl | 343 ++++++ clean/dx12/kernels/l_swin.hlsl | 36 + clean/dx12/kernels/swin_block.hlsl | 291 ++++- clean/dx12/tests/kernel_replay.cpp | 7 + clean/dx12/tests/test_hyb_isa_scan.py | 101 ++ clean/dx12/tests/test_kernel_sets.py | 5 +- clean/dx12/vgpr_limits.json | 21 + clean/kernels/post.hip | 19 + clean/kernels/swin.hip | 70 +- clean/kernels/swin_hyb.hip | 37 + clean/kernels/wg_hyb.h | 154 +++ clean/l/kernels/l_hyb_probe.hip | 31 + clean/l/kernels/l_post.hip | 44 +- clean/l/kernels/l_post_c12_probe.hip | 25 + clean/l/kernels/l_q7.hip | 401 ++++++ clean/l/kernels/l_q7_common.h | 69 ++ clean/l/kernels/l_q7_op8.h | 106 ++ clean/l/kernels/l_q7_s2a_probe.hip | 61 + clean/l/kernels/l_q7w.hip | 345 ++++++ clean/l/kernels/l_swin.hip | 4 + clean/l/kernels/l_swin_hyb.hip | 22 + clean/l/kernels/l_wmma_probe.hip | 32 + .../l/records/l_time_1440q_hybrid_d3d12.json | 293 +++++ clean/l/reference/hyb_tables.py | 129 ++ clean/l/reference/oracle.py | 97 +- clean/l/reference/recon.py | 16 +- clean/l/reference/table.py | 25 +- clean/l/runner/l_d3d12_check.py | 45 + clean/l/runner/l_d3d12_ops_check.py | 25 +- clean/l/runner/l_d3d12_profile.py | 6 +- clean/l/runner/l_frame_check.py | 57 +- clean/l/runner/l_hyb_block_check.py | 207 ++++ clean/l/runner/l_hyb_bneck_check.py | 178 +++ clean/l/runner/l_hyb_bneck_guard_check.py | 169 +++ clean/l/runner/l_hyb_default_check.py | 129 ++ clean/l/runner/l_hyb_pack.py | 43 + clean/l/runner/l_hyb_probe_check.py | 178 +++ clean/l/runner/l_hyb_refuse_check.py | 131 ++ clean/l/runner/l_hyb_shared_check.py | 223 ++++ clean/l/runner/l_hyb_stage_check.py | 559 +++++++++ clean/l/runner/l_hyb_stage_pack.py | 39 + clean/l/runner/l_mm8_f32_check.py | 430 +++++++ clean/l/runner/l_pack.py | 96 ++ clean/l/runner/l_post_c12_check.py | 194 +++ clean/l/runner/l_post_check.py | 62 +- clean/l/runner/l_q7_check.py | 454 +++++++ clean/l/runner/l_q7_negative_control.py | 1 + clean/l/runner/l_q7_s2a_bounds.py | 225 ++++ clean/l/runner/l_q7_s2a_check.py | 423 +++++++ clean/l/runner/l_q7_s2a_fallbacks.py | 141 +++ clean/l/runner/l_q7w_check.py | 348 ++++++ clean/l/runtime/l_hyb_pack.h | 56 + clean/l/runtime/l_pack.cpp | 52 + clean/l/runtime/l_pack.h | 6 + clean/l/runtime/l_q7_pack.h | 122 ++ clean/l/runtime/tests/l_hyb_pack_dump.cpp | 125 ++ clean/l/runtime/tests/test_l_runtime_frame.py | 1084 ++++++++++++----- .../runtime/tests/test_l_runtime_hyb_pack.py | 183 +++ clean/l/tests/test_hyb_tables.py | 51 + clean/l/tests/test_hyb_tables_cpp.py | 58 + clean/l/tests/test_l_mm8_kernel.py | 91 ++ clean/l/tests/test_l_q7_known.py | 7 +- clean/l/tests/test_l_q7_mask.py | 43 + clean/l/tests/test_l_q7_pack.py | 97 ++ clean/l/tests/test_l_recon.py | 2 +- clean/l/tests/test_l_reference.py | 14 +- clean/l/tests/test_l_shared_q7.py | 105 ++ clean/runner/build.py | 41 +- clean/runner/m_q7_check.py | 279 +++++ clean/runtime/device.h | 23 + clean/runtime/device_d3d12.cpp | 138 ++- clean/runtime/hyb_tables.h | 117 ++ clean/runtime/m_q7_pack.h | 79 ++ clean/runtime/runtime.cpp | 518 +++++++- clean/tests/check_no_dev_diagnostics.py | 24 + clean/tests/hyb_dump.cpp | 28 + clean/tests/runtime_frame_trace.cpp | 102 ++ clean/tests/test_dev_diagnostics_off.py | 25 + clean/tests/test_runtime_frame_trace.py | 9 +- 98 files changed, 12670 insertions(+), 422 deletions(-) create mode 100644 clean/dx12/hlsl/hyb_block.hlsli create mode 100644 clean/dx12/hlsl/l_q7_common.hlsli create mode 100644 clean/dx12/hlsl/l_q7_op8.hlsli create mode 100644 clean/dx12/hlsl/wg_hyb.hlsli create mode 100644 clean/dx12/hyb_isa_scan.py create mode 100644 clean/dx12/kernels/hyb_bneck.hlsl create mode 100644 clean/dx12/kernels/hyb_gemm.hlsl create mode 100644 clean/dx12/kernels/hyb_swin.hlsl create mode 100644 clean/dx12/kernels/l_hyb_probe.hlsl create mode 100644 clean/dx12/kernels/l_q7.hlsl create mode 100644 clean/dx12/kernels/l_q7_s2a_probe.hlsl create mode 100644 clean/dx12/kernels/l_q7w.hlsl create mode 100644 clean/dx12/tests/test_hyb_isa_scan.py create mode 100644 clean/kernels/swin_hyb.hip create mode 100644 clean/kernels/wg_hyb.h create mode 100644 clean/l/kernels/l_hyb_probe.hip create mode 100644 clean/l/kernels/l_post_c12_probe.hip create mode 100644 clean/l/kernels/l_q7.hip create mode 100644 clean/l/kernels/l_q7_common.h create mode 100644 clean/l/kernels/l_q7_op8.h create mode 100644 clean/l/kernels/l_q7_s2a_probe.hip create mode 100644 clean/l/kernels/l_q7w.hip create mode 100644 clean/l/kernels/l_swin_hyb.hip create mode 100644 clean/l/kernels/l_wmma_probe.hip create mode 100644 clean/l/records/l_time_1440q_hybrid_d3d12.json create mode 100644 clean/l/reference/hyb_tables.py create mode 100644 clean/l/runner/l_hyb_block_check.py create mode 100644 clean/l/runner/l_hyb_bneck_check.py create mode 100644 clean/l/runner/l_hyb_bneck_guard_check.py create mode 100644 clean/l/runner/l_hyb_default_check.py create mode 100644 clean/l/runner/l_hyb_pack.py create mode 100644 clean/l/runner/l_hyb_probe_check.py create mode 100644 clean/l/runner/l_hyb_refuse_check.py create mode 100644 clean/l/runner/l_hyb_shared_check.py create mode 100644 clean/l/runner/l_hyb_stage_check.py create mode 100644 clean/l/runner/l_hyb_stage_pack.py create mode 100644 clean/l/runner/l_mm8_f32_check.py create mode 100644 clean/l/runner/l_post_c12_check.py create mode 100644 clean/l/runner/l_q7_check.py create mode 100644 clean/l/runner/l_q7_s2a_bounds.py create mode 100644 clean/l/runner/l_q7_s2a_check.py create mode 100644 clean/l/runner/l_q7_s2a_fallbacks.py create mode 100644 clean/l/runner/l_q7w_check.py create mode 100644 clean/l/runtime/l_hyb_pack.h create mode 100644 clean/l/runtime/l_q7_pack.h create mode 100644 clean/l/runtime/tests/l_hyb_pack_dump.cpp create mode 100644 clean/l/runtime/tests/test_l_runtime_hyb_pack.py create mode 100644 clean/l/tests/test_hyb_tables.py create mode 100644 clean/l/tests/test_hyb_tables_cpp.py create mode 100644 clean/l/tests/test_l_mm8_kernel.py create mode 100644 clean/l/tests/test_l_q7_mask.py create mode 100644 clean/l/tests/test_l_q7_pack.py create mode 100644 clean/l/tests/test_l_shared_q7.py create mode 100644 clean/runner/m_q7_check.py create mode 100644 clean/runtime/hyb_tables.h create mode 100644 clean/runtime/m_q7_pack.h create mode 100644 clean/tests/hyb_dump.cpp diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index 0c5bbccb..7777762d 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -35,6 +35,15 @@ VGPR check: every Swin / bottleneck kernel built (a name with a swin_ or l_swin_ part) is run through RGA (gfx1201) and its USED_VGPRs compared with its bound in clean/dx12/vgpr_limits.json (the count that keeps its present workgroups per CU); over the bound, or no bound listed, fails the build. WG_DX12_VGPR_CHECK=0 skips it. +ISA rules of the hybrid kernels: every hybrid kernel built (a name with a _hyb part) is checked in the same RGA run by +hyb_isa_scan.py: rule 1, no fused multiply-add of a square (a driver contracting the norm's RN16(x x) into its sum, +copies of a register included); rule 2, both sources of every FP8 conversion are FP16 values (no RN16 dropped or folded +into the FP8 pack). A finding fails the build. No skip switch (WG_DX12_VGPR_CHECK=0 does not skip them; RGA is required +when hybrid kernels are built). Negative controls, fail-closed: every model hybrid kernel K built (probe=0) needs its +mutant K_scanmut (the same kernel without the contraction barrier and without the RN16 before the FP8 pack, +HYB_SCAN_MUTANT=1, probe=1), built beside every build of hybrid kernels into /dx12/scanmut (never in the manifest); +the build fails when it is not declared, gives no ISA, or PASSES rule 2, or rule 1 (unless K has no norm tree: +HYB_NO_NORM). """ import argparse import concurrent.futures @@ -184,6 +193,8 @@ def kernel_decls(path): d["gscale"] = int(kv.get("gscale", d["block"] // d["group"])) if d["gscale"] < 1 or d["gscale"] > d["block"] // d["group"]: raise SystemExit(f"{path}: {name}: gscale {d['gscale']}") + # probe=1: a measurement probe, built only when --only names it (never part of a model's kernel set) + d["probe"] = kv.get("probe", "0") == "1" d["file"] = kv.get("file", "") # DXIL file stem of a second port of the same name (module doc) if d["file"] and not re.fullmatch(r"[A-Za-z0-9_]+", d["file"]): raise SystemExit(f"{path}: {name}: file={d['file']} (letters, digits and _ only)") @@ -202,8 +213,10 @@ def kernel_decls(path): def is_model_l(decl): """A port of one of Model L's code objects (#89: l_swin.co, l_ops.co, l_post.co, l_enc_values.co), a Model L kernel - of the direct route with no code object (co "-", named l_*), or a kernel only Model L launches (MODEL_L_ONLY).""" - return decl["co"].startswith("l_") or decl["name"].startswith("l_") or decl["name"] in MODEL_L_ONLY + of the direct route with no code object (co "-", named l_*), a kernel only Model L launches (MODEL_L_ONLY), or a + kernel of the hybrid form (named *_hyb: WG_L_FORM = hybrid, Model L's route only so far).""" + return (decl["co"].startswith("l_") or decl["name"].startswith("l_") or decl["name"] in MODEL_L_ONLY + or decl["name"].endswith("_hyb")) def is_frame_gen(decl): @@ -233,11 +246,11 @@ def source_files(model_k=False, only_k=False): def kernel_set(model_l=False, model_k=False): """(name, code object) of every kernel a full build compiles (no --only), sorted: Model M's set, plus Model L's with model_l, plus Model K's with model_k. From the sources' declarations alone (nothing is compiled). Frame - generation's kernels are never in it (is_frame_gen).""" + generation's kernels and measurement probes (probe=1) are never in it (is_frame_gen).""" keys = set() for path, k_source in source_files(model_k): for d in kernel_decls(path): - if is_frame_gen(d): + if is_frame_gen(d) or d["probe"]: continue if k_source or is_model_k(d): if model_k: @@ -305,32 +318,98 @@ def vgpr_checked(name): return re.search(r"(^|/|l_)swin_", name) is not None # l_swin_*: Model L's blocks (#89) -def rga_vgprs(dxil): - """RGA's USED_VGPRs of one kernel (gfx1201), None when RGA gives no statistics.""" +def rga_vgprs(dxil, isa=False): + """RGA's USED_VGPRs of one kernel (gfx1201), None when RGA gives no statistics; with isa: (VGPRs, ISA text or + None).""" with tempfile.TemporaryDirectory(prefix="wgrga_") as tmp: - subprocess.run([RGA, "-s", "dx12", "-c", "gfx1201", "--cs-blob", dxil, "-a", os.path.join(tmp, "s.csv")], - capture_output=True, text=True) + cmd = [RGA, "-s", "dx12", "-c", "gfx1201", "--cs-blob", dxil, "-a", os.path.join(tmp, "s.csv")] + if isa: + cmd += ["--isa", os.path.join(tmp, "x.txt")] + subprocess.run(cmd, capture_output=True, text=True) + v, text = None, None for f in os.listdir(tmp): if f.endswith("_s_comp.csv"): with open(os.path.join(tmp, f), newline="") as fh: rows = list(csv.DictReader(fh)) - return int(rows[0]["USED_VGPRs"]) if rows else None - return None + v = int(rows[0]["USED_VGPRs"]) if rows else None + elif f.endswith("_x_comp.txt"): + with open(os.path.join(tmp, f), encoding="utf-8", errors="replace") as fh: + text = fh.read() + return (v, text) if isa else v -def vgpr_check(built): - """built: [(kernel name, dxil path)] of this build. Fails when a Swin / bottleneck kernel exceeds its bound.""" - todo = [(n, p) for n, p in built if vgpr_checked(n)] - if not todo or os.environ.get("WG_DX12_VGPR_CHECK", "1") == "0": +def hyb_scanned(name): + return "_hyb" in name and not name.endswith("_scanmut") + + +# Model hybrid kernels with no norm tree (no square to fuse): their mutants must fail rule 2 only. +HYB_NO_NORM = {"patch_merge_c64_hyb", "patch_merge_c96_hyb", "expand_c96_hyb", "expand_c128_hyb"} + + +def scan_mutant(name): + return name.endswith("_scanmut") + + +def vgpr_check(built, required=()): + """built: [(kernel name, dxil path)] of this build. Fails when a Swin / bottleneck kernel exceeds its bound, when a + hybrid kernel breaks an ISA rule, and when a kernel of `required` (the model hybrid kernels built) has no built + mutant or its mutant passes the rules it must fail.""" + vg_on = os.environ.get("WG_DX12_VGPR_CHECK", "1") != "0" + todo = [(n, p) for n, p in built if (vg_on and vgpr_checked(n)) or hyb_scanned(n) or scan_mutant(n)] + if not todo and not required: return if not os.path.exists(RGA): - raise SystemExit(f"VGPR check: RGA not found at {RGA} (WG_RGA, or WG_DX12_VGPR_CHECK=0 to skip)") + raise SystemExit(f"VGPR check / fused-square scan: RGA not found at {RGA} (WG_RGA; WG_DX12_VGPR_CHECK=0 skips " + "the VGPR check only, never the hybrid kernels' scan)") with open(VGPR_LIMITS, encoding="utf-8") as f: limits = {k: v for k, v in json.load(f).items() if not k.startswith("_")} # RGA runs are compiles too: at most 6 at a time, fewer when WG_DX12_JOBS asks for fewer with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, min(6, int(os.environ.get("WG_DX12_JOBS", 6))))) as ex: - counts = dict(zip([n for n, _ in todo], ex.map(rga_vgprs, [p for _, p in todo]))) + res = dict(zip([n for n, _ in todo], ex.map(lambda np_: rga_vgprs(np_[1], isa=hyb_scanned(np_[0]) or + scan_mutant(np_[0])), todo))) bad = [] + if HERE not in sys.path: # python -I keeps the script's folder off sys.path + sys.path.insert(0, HERE) + from hyb_isa_scan import fused_squares, fp8_sources_not_fp16 # clean/dx12 (this folder) + for n in sorted(n for n in res if hyb_scanned(n)): + text = res[n][1] + hits = fused_squares(text) if text is not None else None + conv = fp8_sources_not_fp16(text) if text is not None else None + print(f"ISA-RULES {n:40s} fused squares {len(hits) if hits is not None else '?'}, FP8 sources not FP16 " + f"{len(conv) if conv is not None else '?'}", flush=True) + if hits is None: + bad.append(f"{n}: RGA gave no ISA (hybrid ISA rules)") + continue + if hits: + bad.append(f"{n}: {len(hits)} fused square(s), e.g. {hits[0][1]} (the driver contracts RN16(x x) into a " + "sum: hyb_isa_scan.py rule 1, hlsl/hyb_block.hlsli HYB_OPAQUE0)") + if conv: + bad.append(f"{n}: {len(conv)} FP8 conversion(s) with a source that is not an FP16 value, e.g. {conv[0][1]} " + "(an RN16 dropped or folded into the FP8 pack: hyb_isa_scan.py rule 2)") + for n in sorted(required): # fail-closed: every model hybrid kernel has its built mutant + if n + "_scanmut" not in res: + bad.append(f"{n}: its ISA rules' mutant {n}_scanmut was not built (declare it beside the kernel: every " + "model hybrid kernel has one)") + for n in sorted(n for n in res if scan_mutant(n)): # the mutants: must FAIL rule 2, and rule 1 with a norm tree + text = res[n][1] + base = n[:-len("_scanmut")] + hits = fused_squares(text) if text is not None else None + conv = fp8_sources_not_fp16(text) if text is not None else None + need1 = base not in HYB_NO_NORM + print(f"ISA-RULES {n:40s} fused squares {len(hits) if hits is not None else '?'}" + f"{' (must be > 0)' if need1 else ''}, FP8 sources not FP16 {len(conv) if conv is not None else '?'} " + "(must be > 0)", flush=True) + if text is None: + bad.append(f"{n}: RGA gave no ISA for the mutant: the ISA rules prove nothing for {base}") + continue + if need1 and not hits: + bad.append(f"{n}: the mutant (no contraction barrier) PASSES rule 1 (0 fused squares): the rule no longer " + f"proves anything for {base}") + if not conv: + bad.append(f"{n}: the mutant (no RN16 before the FP8 pack) PASSES rule 2: the rule no longer proves " + f"anything for {base}") + counts = {n: (r[0] if (hyb_scanned(n) or scan_mutant(n)) else r) for n, r in res.items() + if vg_on and vgpr_checked(n) and not scan_mutant(n)} for n in sorted(counts): v, lim = counts[n], limits.get(n) print(f"VGPR {n:40s} {v if v is not None else '?':>4} / {lim if lim is not None else '-'}", flush=True) @@ -342,7 +421,7 @@ def vgpr_check(built): bad.append(f"{n}: {v} VGPRs > {lim} (one workgroup per CU less: restore the bound or re-measure and " "change it in vgpr_limits.json)") if bad: - raise SystemExit("VGPR check failed:\n " + "\n ".join(bad)) + raise SystemExit("VGPR check / fused-square scan failed:\n " + "\n ".join(bad)) def main(argv=None): @@ -406,7 +485,7 @@ def main(argv=None): continue if not only and not k_source and not is_model_k(d) and not a.model_l and is_model_l(d): continue - if not only and is_frame_gen(d): + if not only and (is_frame_gen(d) or d["probe"]): continue key = (d["name"], d["co"]) if key in kernels and key not in old: @@ -417,6 +496,14 @@ def main(argv=None): raise SystemExit(f"kernel {d['name']} ({d['co']}): DXIL file {os.path.basename(out)} already used " "(a second port of a name needs file=)") jobs.append((src, d, out, k_source)) + # the fused-square scan's mutants beside every build of hybrid kernels (outside the manifest, /dx12/scanmut) + if any(hyb_scanned(j[1]["name"]) for j in jobs): + have = {j[1]["name"] for j in jobs} + for src, k_source in source_files(False): + for d in kernel_decls(src): + if scan_mutant(d["name"]) and d["name"] not in have: + os.makedirs(os.path.join(dst, "scanmut"), exist_ok=True) + jobs.append((src, d, os.path.join(dst, "scanmut", output_name(d["name"])), k_source)) def build_one(job): src, d, out, k_source = job @@ -448,7 +535,11 @@ def build_one(job): futures = [pool.submit(build_one, j) for j in jobs] try: for (src, d, out, _k), fut in zip(jobs, futures): - kernels[(d["name"], d["co"])] = fut.result() + r_ = fut.result() + if not scan_mutant(d["name"]): + kernels[(d["name"], d["co"])] = r_ + else: + kernels.pop((d["name"], d["co"]), None) built.append((d["name"], out)) print(f"{d['name']:40s} -> {os.path.basename(out)} block {d['block']} wave {d['wave']} " f"args {d['args']}", flush=True) @@ -456,10 +547,10 @@ def build_one(job): for fut in futures: fut.cancel() # kernels not started yet; the running ones finish their current child process raise - names = {n for n, _ in kernels} + names = {n for n, _ in kernels} | {n for n, _ in built} if only and not (only <= names): raise SystemExit(f"not declared in clean/dx12/kernels or clean/k/dx12/kernels: {sorted(only - names)}") - vgpr_check(built) + vgpr_check(built, {d["name"] for _, d, _, _ in jobs if hyb_scanned(d["name"]) and not d["probe"]}) with open(man_path, "w") as f: json.dump({"format": FORMAT, "dxc": os.path.basename(os.path.dirname(os.path.dirname(DXC_DIR))), "kernels": [kernels[k] for k in sorted(kernels)]}, f, indent=1) diff --git a/clean/dx12/hlsl/hyb_block.hlsli b/clean/dx12/hlsl/hyb_block.hlsli new file mode 100644 index 00000000..1d8c99e5 --- /dev/null +++ b/clean/dx12/hlsl/hyb_block.hlsli @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: MIT +// The hybrid block form's norm and softmax row in the Swin block templates' register layout (SW_HYB=1; included by +// kernels/swin_block.hlsl after its layout macros): lane (t, hh) of a wave holds token t's channels hh C/2 + 8 k + v in +// v8f tiles k = 0 .. KC - 1, and the scores S(t, s) of keys s = 8 hh + v (gfx12 D rows, packing.resolve_d_row). +// The sums follow the original's fixed FP16 orders (clean/l/reference/oracle.py norm_q7, softmax_q7) exactly; the +// partner half (hh ^ 1) arrives by lane exchange, and both lanes of a token compute the same sums. +// Arithmetic in native FP16 (half2, packed): v_pk_add_f16 / v_pk_mul_f16 are correctly rounded single operations (the +// original's FP16 steps). Contraction: the only multiply feeding an add is the norm's square; `precise` does NOT stop +// the driver from fusing it (gfx1201), so the square passes an OR with a zero it cannot prove (HYB_OPAQUE0), and +// build_dx12.py's fused-square scan (hyb_isa_scan.py, with its *_scanmut mutants) fails a build where it fuses anyway. +// The softmax's two fused steps are float32 (hlsl/wg_hyb.hlsli hyb_softmax_e8: exact under any lowering). +#pragma once + +typedef float16_t2 h2; +struct hv8 { h2 p[4]; }; // 8 FP16 values, element 2 i + j in p[i][j] (the order of a v8f) + +hv8 hyb_to_h(v8f a) // exact for FP16 values; otherwise RN16 (the driver's conversion, checked by the probe) +{ + hv8 r; + [unroll] for (uint i = 0; i < 4; ++i) r.p[i] = h2((float16_t)a.v[2 * i], (float16_t)a.v[2 * i + 1]); + return r; +} +v8f hyb_to_f(hv8 a) +{ + v8f r; + [unroll] for (uint i = 0; i < 4; ++i) { r.v[2 * i] = (float)a.p[i].x; r.v[2 * i + 1] = (float)a.p[i].y; } + return r; +} +// RN16(a + b) of 8 FP16 values held as float32 (exact: one float rounding first, 24 >= 2 x 11 + 2) +v8f hyb_add16(v8f a, v8f b) +{ + v8f s; + [unroll] for (uint i = 0; i < 8; ++i) s.v[i] = a.v[i] + b.v[i]; + return hyb_round16(s); +} +hv8 hyb_hadd(hv8 a, hv8 b) { hv8 r; [unroll] for (uint i = 0; i < 4; ++i) r.p[i] = a.p[i] + b.p[i]; return r; } +#ifndef HYB_OPAQUE0 +#error "hyb_block.hlsli: define HYB_OPAQUE0 (a uint zero the compiler cannot prove, e.g. from a kernel argument)" +#endif +hv8 hyb_hmul(hv8 a, hv8 b) { hv8 r; [unroll] for (uint i = 0; i < 4; ++i) r.p[i] = a.p[i] * b.p[i]; return r; } +uint hyb_h2u(h2 v) { return (uint)asuint16(v.x) | ((uint)asuint16(v.y) << 16); } +h2 hyb_u2h(uint u) { return h2(asfloat16((uint16_t)(u & 0xffffu)), asfloat16((uint16_t)(u >> 16))); } +hv8 hyb_hxor16(hv8 a) +{ + hv8 r; + [unroll] for (uint i = 0; i < 4; ++i) r.p[i] = hyb_u2h(wg_xor16_u(hyb_h2u(a.p[i]))); + return r; +} +hv8 hyb_hsel(bool c, hv8 a, hv8 b) { hv8 r; [unroll] for (uint i = 0; i < 4; ++i) r.p[i] = select(c, a.p[i], b.p[i]); return r; } +uint4 hyb_hpack(hv8 a) { return uint4(hyb_h2u(a.p[0]), hyb_h2u(a.p[1]), hyb_h2u(a.p[2]), hyb_h2u(a.p[3])); } + +// _group of oracle.norm_q7 over 1 .. 4 partial tiles: p0; p0 + p1; (p0 + p1) + p2; (p0 + p1) + (p2 + p3) +hv8 hyb_group(hv8 p[4], uint n) +{ + if (n == 1u) return p[0]; + if (n == 2u) return hyb_hadd(p[0], p[1]); + if (n == 3u) return hyb_hadd(hyb_hadd(p[0], p[1]), p[2]); + return hyb_hadd(hyb_hadd(p[0], p[1]), hyb_hadd(p[2], p[3])); +} + +// The norm of the lane's token: x = this lane's KC tiles (FP16 values), gp = g' of the same channels (FP16 values). +// y = RN16(x RN16(r g')) with r = RN16(rsqrt32(S')) and S' = RN16(S + 2^-13), S = the FP16 tree of the squares. +void hyb_norm(v8f x[KC], v8f gp[KC], uint hh, out v8f y[KC]) +{ + hv8 xh[KC], q0h[KC], qph[KC]; // this lane's squares, the partner lane's (its half of the token's channels) +#if !(KC == 2 || KC == 4 || KC == 8) + hv8 T[2 * KC]; + const bool h0 = hh == 0u; +#endif + [unroll] for (uint k = 0; k < KC; ++k) { + xh[k] = hyb_to_h(x[k]); + [unroll] for (uint i = 0; i < 4; ++i) { + precise h2 sq = xh[k].p[i] * xh[k].p[i]; // q = RN16(x x): never fused into the tree's adds + // contraction barrier: the driver fuses a square feeding an add into v_pk_fma_f16 despite precise (seen in + // the ISA of the select-free tree: x x + q' on one lane of a token only, RN16(x x) skipped); an OR with + // a zero it cannot prove leaves no multiply-add pattern (one v_or_b32 per register) + q0h[k].p[i] = hyb_u2h(hyb_h2u(sq) | HYB_OPAQUE0); + } + qph[k] = hyb_hxor16(q0h[k]); +#if !(KC == 2 || KC == 4 || KC == 8) +#if defined(HYB_SCAN_MUTANT) && HYB_SCAN_MUTANT + // the fused-square scan's SUBSTITUTE mutant for KC 6 (never in a model build; wrong values by design): with + // the selects, every square reaches the tree only through a v_cndmask or the lane exchange (RGA: no square + // feeds an add, so even without the barrier the driver forms no multiply-add and the plain mutant shows 0). + // Here the tree takes the lane's own squares directly, a square feeds an add with no barrier, and the scan + // must find the fused square in this kernel's ISA. + T[k] = q0h[k]; + T[KC + k] = q0h[k]; +#else + T[k] = hyb_hsel(h0, q0h[k], qph[k]); // tile j = channels 8 j .. 8 j + 7: low half j < KC + T[KC + k] = hyb_hsel(h0, qph[k], q0h[k]); // high half +#endif +#endif + } + // element e = 2 t + odd of tile j is the original's Q[j] of lane t, half odd: pair sums (j, j + 2), groups of the + // even tiles j = 0, 4, 8, 12 (left) and the odd ones j = 1, 5, 9, 13 (right) +#if KC == 2 || KC == 4 || KC == 8 + // Select-free form (lever: the same FP16 sums, no per-lane tile selects). FP16 addition is commutative (a + b == b + a + // bit for bit), and for KC = 2, 4, 8 every add of the tree whose operands the selects would swap between the two + // lanes of a token adds the same two values in either order: + // KC 2: pairs (T0, T2) = {q0, q0'} and (T1, T3) = {q1, q1'}; + // KC 4, 8: the left / right groups are (the group of this lane's own pairs) + (the group of the partner's pairs), and + // the partner lane computes the same two groups and adds them in the other order. + // (KC 6: the three-pair group mixes own and partner tiles asymmetrically; the selects stay.) + hv8 L; +#if KC == 2 + L = hyb_hadd(hyb_hadd(q0h[0], qph[0]), hyb_hadd(q0h[1], qph[1])); +#else + { + hv8 ol[KC / 4], orr[KC / 4], pl2[KC / 4], pr2[KC / 4]; + [unroll] for (uint j2 = 0; j2 < KC; j2 += 4u) { + ol[j2 / 4u] = hyb_hadd(q0h[j2], q0h[j2 + 2u]); + orr[j2 / 4u] = hyb_hadd(q0h[j2 + 1u], q0h[j2 + 3u]); + pl2[j2 / 4u] = hyb_hadd(qph[j2], qph[j2 + 2u]); + pr2[j2 / 4u] = hyb_hadd(qph[j2 + 1u], qph[j2 + 3u]); + } +#if KC == 4 + L = hyb_hadd(hyb_hadd(ol[0], pl2[0]), hyb_hadd(orr[0], pr2[0])); +#else + L = hyb_hadd(hyb_hadd(hyb_hadd(ol[0], ol[1]), hyb_hadd(pl2[0], pl2[1])), + hyb_hadd(hyb_hadd(orr[0], orr[1]), hyb_hadd(pr2[0], pr2[1]))); +#endif + } +#endif +#else + const uint tiles = 2u * KC; + hv8 pl[4], pr[4]; + uint nl = 0u, nr = 0u; + [unroll] for (uint j = 0; j < tiles; j += 4u) { + pl[nl++] = hyb_hadd(T[j], T[j + 2u]); + pr[nr++] = hyb_hadd(T[j + 1u], T[j + 3u]); + } + const hv8 L = hyb_hadd(hyb_group(pl, nl), hyb_group(pr, nr)); // L[2 t + odd] +#endif + // across the original's lanes t, per half: ((t0 + t1) + (t2 + t3)); then even + odd, + 2^-13 (packed pairs: even + // in .x, odd in .y) + const h2 u01 = L.p[0] + L.p[1], u23 = L.p[2] + L.p[3]; // (t0 + t1), (t2 + t3) for both halves + const h2 hv = u01 + u23; // half(0), half(1) + const float16_t S = hv.x + hv.y; + const float16_t S2 = S + (float16_t)asfloat16((uint16_t)0x0800u); // + 2^-13 + const float16_t r = (float16_t)hyb_rsq16((uint)asuint16(S2)); + const h2 r2 = h2(r, r); + [unroll] for (uint k2 = 0; k2 < KC; ++k2) { + const hv8 g = hyb_to_h(gp[k2]); + hv8 yh; + [unroll] for (uint i2 = 0; i2 < 4; ++i2) yh.p[i2] = xh[k2].p[i2] * (r2 * g.p[i2]); + y[k2] = hyb_to_f(yh); + } +} + +// The attention weights of the lane's 8 keys from its 8 scores s (float32 chain values started at B): +// RN16(s), e = hyb_softmax_e8, the row sum in the original's order (a_t = e_2t + e_8+2t, b_t = e_2t+1 + e_9+2t; +// a = (a0 + a1) + (a2 + a3), b likewise; sum = a + b), w = RN16(rcp32(sum)), P = RN16(e w) as packed FP16. +uint4 hyb_softmax_p(v8f s, uint hh) +{ + const hv8 e = hyb_to_h(hyb_softmax_e8(hyb_round16(s))); // exact (e is FP16) + const hv8 ep = hyb_hxor16(e); + // p[t] = (a_t, b_t) = e (keys 2t, 2t+1 of this half) + the partner half's: FP16 addition is commutative, so + // e + ep on both lanes is the original's RN16(e_{2t} + e_{8+2t}) (no per-lane select) + const hv8 A = hyb_hadd(e, ep); + const h2 ab = (A.p[0] + A.p[1]) + (A.p[2] + A.p[3]); // (a, b) + const float16_t sum = ab.x + ab.y; + const float16_t w = (float16_t)hyb_rcp16((uint)asuint16(sum)); + const h2 w2 = h2(w, w); + hv8 P; + [unroll] for (uint i = 0; i < 4; ++i) P.p[i] = e.p[i] * w2; + return hyb_hpack(P); +} diff --git a/clean/dx12/hlsl/l_coef_px.hlsli b/clean/dx12/hlsl/l_coef_px.hlsli index 42ec4eb3..12b4c1ac 100644 --- a/clean/dx12/hlsl/l_coef_px.hlsli +++ b/clean/dx12/hlsl/l_coef_px.hlsli @@ -86,7 +86,15 @@ struct LCoefVals { uint cA, cB, c0, c1, cL; }; float coef_mul_odd(float a, float b) { const float p = a * b; +#if defined(SW_HYB) && SW_HYB + // the hybrid twins (l_swin_block_c32_dec_coef12_hyb): the product's exact error term is a deliberate fused + // multiply-add (TwoProduct), here of squares (C0 = sx sx, C1 = sy sy); its second operand passes the opaque zero + // (opq(b) == b bit for bit) so that build_dx12.py's fused-square scan, which fails on any fused square in a hybrid + // kernel, keeps its rule without an exemption. The same bits as below (the M-form kernels keep that form unchanged). + return asfloat(wg_round_odd(p, mad(a, opq(b), -p))); +#else return asfloat(wg_round_odd(p, mad(a, b, -p))); +#endif } // wg_mul_f16's +0 for an exact zero product (a NaN product keeps its conversion) bool coef_mul_zero(float a, float b) diff --git a/clean/dx12/hlsl/l_q7_common.hlsli b/clean/dx12/hlsl/l_q7_common.hlsli new file mode 100644 index 00000000..6e14999b --- /dev/null +++ b/clean/dx12/hlsl/l_q7_common.hlsli @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: MIT +// The FP16 form's exact arithmetic for the D3D12 ports of Model L's q7 kernels (clean/l/kernels/l_q7.hip, l_q7w.hip): +// one rounding to FP16 (software, nearest even), exact FP16 additions (error-free sums, the tail picking the side at +// an FP16 midpoint), and one FP8 op over 32 terms computed exactly. Porting notes: clean/dx12/kernels/l_q7.hlsl. +#pragma once + +float h16(float v) { return wg_rn16(v); } +uint bits16(float v) { return wg_f2h(v); } +float from16(uint b) { return f16tof32(b & 0xffffu); } +float fp8v(uint code) { return wg_e4m3_to_f32(code & 0xffu); } +bool fin(float v) { return (asuint(v) & 0x7f800000u) != 0x7f800000u; } +// Output stores: the plain codes. This driver generates NaNs with the sign set where HIP does not; the checks compare +// with only a NaN's sign bit masked (the accepted port exception), since a canonical NaN would cost about 3 VALU per +// stored dword (RGA ISA, measured 2026-11-15) and real data has no NaN. +uint out8(float v) { return wg_f32_to_e4m3(v); } +uint out16(float v) { return wg_f2h(v); } +int q7_mirror(int p, int n) +{ + p = p < 0 ? -p : p; + return p >= n ? 2 * (n - 1) - p : p; +} + +void two_sum(float a, float b, out float s, out float e) +{ + s = a + b; + float bb = s - a; + e = (a - (s - bb)) + (b - bb); +} + +float rn16_pair(float z, float t) +{ + if (!fin(z) || t == 0.0f) return h16(z); + uint b = asuint(z); + bool mid; + if (abs(z) >= asfloat(0x38800000u)) { + mid = (b & 0x1fffu) == 0x1000u; + } else { + float q = z * 33554432.0f; + int qi = (int)q; + mid = (float)qi == q && (qi & 1) != 0; + } + if (!mid) return h16(z); + bool up = t > 0.0f; + uint nb = (up == (z > 0.0f)) ? b + 1u : b - 1u; + return h16(asfloat(nb)); +} + +float add16(float a, float b) +{ + float s, e; + two_sum(a, b, s, e); + return rn16_pair(s, e); +} +float mul16(float a, float b) { return h16(a * b); } +float fma16(float a, float b, float c) { return add16(a * b, c); } + +float gelu16(float h) +{ + float y = min(max(h, -2.0f), 2.0f); + const float A = 0.412109375f, B = 0.0810546875f; + return mul16(h, add16(0.5f, mul16(y, add16(A, -mul16(B, abs(y)))))); +} diff --git a/clean/dx12/hlsl/l_q7_op8.hlsli b/clean/dx12/hlsl/l_q7_op8.hlsli new file mode 100644 index 00000000..caa10d18 --- /dev/null +++ b/clean/dx12/hlsl/l_q7_op8.hlsli @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: MIT +// One FP8 matrix op of the q7 D3D12 ports, exact (l_q7_op8.h), and the 32-value FP8 row loader it is fed with: +// apart from the FP16 helpers (l_q7_common.hlsli) so that the FP8 op (S2a) and the FP16 steps (S2b) +// change in separate files. Include after l_q7_common.hlsli. +#pragma once + +float op8(float start, float a[32], float wv[32]) +{ + float acc = 0.0f, aacc = 0.0f; + [unroll] for (uint k = 0; k < 32; ++k) { + float p = a[k] * wv[k]; + acc = acc + p; + aacc = aacc + abs(p); + } + if (!fin(acc) || !fin(start)) return h16(acc + start); + float s = acc, c = 0.0f; + if (!(aacc < 63.0f)) { + s = 0.0f; + [unroll] for (uint k2 = 0; k2 < 32; ++k2) { + float e; + two_sum(s, a[k2] * wv[k2], s, e); + c = c + e; + } + } + float u, e1, v, e2, z, e3, t; + two_sum(s, start, u, e1); + two_sum(e1, c, v, e2); + two_sum(u, v, z, e3); + t = e3 + e2; + float z2, t2; + two_sum(z, t, z2, t2); + return rn16_pair(z2, t2); +} + +// ---- S2a: the FP8 matrix instruction under a precomputed bound (l_q7_op8.h, same names) ------ +// Wave32 only (wg_wmma_fp8 of wg_platform.hlsli; no wave64 matrix code until an on-device layout probe passes for it). +// The calls must stay in wave-uniform control flow: q7_tile_sum is called unconditionally, the per-lane fallback +// branch comes after the sums. +#ifndef Q7_S2A_MUTANT +#define Q7_S2A_MUTANT 0 +#endif +#if Q7_S2A_MUTANT == 1 +#define Q7_S2A_LIMIT 128.0f +#else +#define Q7_S2A_LIMIT 63.0f +#endif + +v8f q7_tile_sum(uint2 a0, uint2 b0, uint2 a1, uint2 b1) +{ + v8f zero; + [unroll] for (uint i = 0; i < 8; ++i) zero.v[i] = 0.0f; + v8f lo = wg_wmma_fp8(a0, b0, zero); +#if Q7_S2A_MUTANT == 2 + return wg_wmma_fp8(a1, b1, lo); +#else + v8f hi = wg_wmma_fp8(a1, b1, zero); + v8f s; + [unroll] for (uint j = 0; j < 8; ++j) s.v[j] = lo.v[j] + hi.v[j]; + return s; +#endif +} +uint q7_umax(uint a, uint b) { return a > b ? a : b; } +uint q7_maxcode(uint m, uint code) { return q7_umax(m, code & 0x7fu); } +uint q7_maxcode4(uint m, uint word) +{ + m = q7_maxcode(m, word); + m = q7_maxcode(m, word >> 8); + m = q7_maxcode(m, word >> 16); + return q7_maxcode(m, word >> 24); +} +float q7_amax(uint m0, uint m1) { return fp8v(q7_umax(m0, m1)); } +bool q7_bounded(float wsum, float amax) { return wsum * amax < Q7_S2A_LIMIT; } +float q7_join(float S, float start) { return add16(S == 0.0f ? 0.0f : S, start); } +// (q7_combine: HLSL has no lambdas; the ports write y = q7_bounded(..) ? q7_join(S, start) : inline) + +// 32 FP8 values from global memory (byte offset a multiple of 4) +void q7_load8(uint base, uint off, out float v[32]) +{ + [unroll] for (uint i = 0; i < 8; ++i) { + uint u = wg_ld32(base, off + 4u * i); + [unroll] for (uint j = 0; j < 4; ++j) v[4u * i + j] = fp8v(u >> (8u * j)); + } +} + +// 32 FP16 weights from global memory (byte offset a multiple of 16): exact values, one native conversion each (Q7F-1) +void q7_load16(uint base, uint off, out float v[32]) +{ + [unroll] for (uint i = 0; i < 4; ++i) { + uint4 u = wg_ld128(base, off + 16u * i); + uint q[4] = { u.x, u.y, u.z, u.w }; + [unroll] for (uint j = 0; j < 4; ++j) { + v[8u * i + 2u * j] = f16tof32(q[j] & 0xffffu); + v[8u * i + 2u * j + 1u] = f16tof32(q[j] >> 16); + } + } +} diff --git a/clean/dx12/hlsl/wg_hyb.hlsli b/clean/dx12/hlsl/wg_hyb.hlsli new file mode 100644 index 00000000..99c86610 --- /dev/null +++ b/clean/dx12/hlsl/wg_hyb.hlsli @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: MIT +// Element-wise steps of Model L's / Model M's HYBRID block form (WG_L_FORM=hybrid; the block templates with SW_HYB=1): +// the original's FP16 forms (PRESET-L-P3-Q7, clean/l/reference/oracle.py norm_q7, softmax_q7, gelu_q7) around FP8 +// matrix chains that accumulate in float32 over the whole chain with ONE FP16 rounding at its end. +// +// Exactness (clean/l/reference/hyb_tables.py, every fact over all possible FP16 inputs): +// - FP16 x FP16 and FP16 + FP16 done in float32 and rounded once to FP16 (round to nearest even) are the correctly +// rounded FP16 results (24 >= 2 x 11 + 2: double rounding is innocuous), so a driver fusing the float32 operation +// into its conversion gives the same bits; +// - the softmax weight e(u'): float32 fused multiply-adds with exact products, one FP16 rounding per step, equal the +// FMA16 form for every clamped u' (softmax_e_f32_equals_fma16); +// - GELU16 -> FP8, the norm's rsqrt and the softmax normaliser are tables generated from the oracle (exact by +// construction), read from the arena at WG_HYB_BASE (the runtime fills them, runtime/hyb_tables.h); +// - the chain-end double rounding float32 -> FP16 -> FP8 is two conversions (RN16, then the saturating FP8 pack of +// the FP16 value): the build's ISA rule 2 (clean/dx12/hyb_isa_scan.py) requires both sources of every FP8 +// conversion to be FP16 values on every path, and each kernel's mutant (HYB_SCAN_MUTANT: no RN16 here) must fail it. +// Round to nearest even FP16: the native float16_t conversion (hyb_round16) or the driver's packed conversion +// (wg_to_f16_ags); f32tof16 rounds toward zero on this driver and is not used. +#pragma once + +#ifndef WG_HYB_BASE +#define WG_HYB_BASE (WG_GELU_LUT_BASE + 16384u) // after the 16 KB GELU table (runtime/hyb_tables.h kHybBase) +#endif +#define WG_HYB_GELU (WG_HYB_BASE) // 65,536 bytes: q8(GELU16(h)) by the FP16 bits of h +#define WG_HYB_RSQ (WG_HYB_BASE + 65536u) // 2 x 1024 FP16 (u16): r by exponent parity and mantissa of S' +#define WG_HYB_RCP (WG_HYB_RSQ + 4096u) // 1024 FP16 (u16): w by the mantissa of the row sum +#define WG_HYB_GELU16 (WG_HYB_RCP + 2048u) // 65,536 u16: the GELU table again, entry = the FP8 byte (high byte 0) +#define WG_HYB_BYTES (65536u + 4096u + 2048u + 131072u) + +#define WG_HYB_SCORE_C asfloat(0x3c8cc000u) // FP16 0x2466 as float32 (0.0972232 / sqrt(32) rounded) +#define WG_HYB_SCORE_CLAMP asfloat(0x3f0e6000u) // FP16 0x3873 = 0.55615234375 +#define WG_HYB_POLY_T asfloat(0x3f6d6000u) // FP16 0x3B6B = 0.92724609375 +#define WG_HYB_POLY_P asfloat(0x3fb00000u) // FP16 0x3D80 = 1.375 +#define WG_HYB_MERGE_CLAMP 6.28125f + +// RN16 of 8 float32 values (the driver's round-to-nearest-even conversion), packed, and back to float32 (exact). +uint4 hyb_rn16x8(v8f d) { return wg_to_f16_ags(d); } +v8f hyb_h2f8(uint4 h) +{ + v8f r; + const uint w[4] = { h.x, h.y, h.z, h.w }; + [unroll] for (uint i = 0; i < 4; ++i) { + r.v[2 * i] = wg_h2f_lo(w[i]); + r.v[2 * i + 1] = wg_h2f_hi(w[i]); + } + return r; +} +// RN16 per value by the driver's packed conversion (wg_to_f16_ags: v_cvt_f16_f32 in the default round-to-nearest-even +// mode), then back to float32 (exact). NOT (float)(float16_t)x: the driver lowers that cast to v_cvt_pk_rtz_f16_f32 under +// s_round_mode 0xc, i.e. round toward zero (RGA, v2 build). The probe's adversarial set checks every FP16 / FP8 midpoint. +#if defined(HYB_SCAN_MUTANT) && HYB_SCAN_MUTANT +// the ISA rules' mutant (*_scanmut, never in a model build; wrong values by design): the RN16 removed, so the FP8 pack +// takes the float32 value (hyb_isa_scan.py rule 2 must fail) +v8f hyb_round16(v8f d) { return d; } +#else +v8f hyb_round16(v8f d) { return hyb_h2f8(wg_to_f16_ags(d)); } +#endif + +// An FP8 point of FP16 values r (lever, exact): when no value of the wave exceeds 448 in magnitude, the saturating +// pack's clamp and NaN keep change nothing, so the driver's plain conversion gives the same bytes (every NaN -> 0xff on +// both paths; the driver's max drops NaN, a wave of NaNs only takes the clamped path). Wave-uniform branch. +bool hyb_wave_in448(v8f r) +{ + float m = 0.0f; + [unroll] for (uint i = 0; i < 8; ++i) m = max(m, abs(r.v[i])); + return WaveActiveAllTrue(m <= 448.0f); +} +uint2 hyb_q8(v8f r) +{ + if (hyb_wave_in448(r)) return wg_pack_q8_nc_ags(r); + return wg_pack_q8_ags(r); +} +// The chain end before an FP8 point: RN16, then the saturating FP8 pack of the FP16 value (two roundings, by design). +uint2 hyb_q8_of_chain(v8f acc) { return hyb_q8(hyb_round16(acc)); } + +// GELU16 then the FP8 point, by table: h = the FP16 bits of 8 RN16'd W1 chain ends (hyb_rn16x8). Speed lever 2: the +// 16-bit-entry copy of the table, one u16 load per value at h << 1 (the byte table needed a dword address, a shift +// by the byte position and a mask per value); the same bytes (runtime/hyb_tables.h, checked by test_hyb_tables_cpp). +uint2 hyb_gelu_q8(uint4 h) +{ + const uint w[4] = { h.x, h.y, h.z, h.w }; + uint b[8]; + [unroll] for (uint i = 0; i < 4; ++i) { + b[2 * i] = (uint)wg_mem.Load(WG_HYB_GELU16 + ((w[i] & 0xffffu) << 1)); + b[2 * i + 1] = (uint)wg_mem.Load(WG_HYB_GELU16 + ((w[i] >> 16) << 1)); + } + return uint2(b[0] | (b[1] << 8) | (b[2] << 16) | (b[3] << 24), b[4] | (b[5] << 8) | (b[6] << 16) | (b[7] << 24)); +} + +// The softmax weight e of 8 FP16 scores s (float32 values of FP16 numbers): u = RN16(s c), clamp to +-c' (a NaN takes +// the lower clamp, as min / max return the number), t = RN16(fma(u, -u, t0)), p = RN16(fma(u, t, p0)), +// bits(e) = (bits(p) & 0x3FF) << 5. Returns e as float32 values (exact). +v8f hyb_softmax_e8(v8f s) +{ + v8f u; + [unroll] for (uint i = 0; i < 8; ++i) u.v[i] = s.v[i] * WG_HYB_SCORE_C; + u = hyb_round16(u); + [unroll] for (uint i1 = 0; i1 < 8; ++i1) u.v[i1] = min(max(u.v[i1], -WG_HYB_SCORE_CLAMP), WG_HYB_SCORE_CLAMP); + v8f t; + [unroll] for (uint i2 = 0; i2 < 8; ++i2) t.v[i2] = mad(u.v[i2], -u.v[i2], WG_HYB_POLY_T); + t = hyb_round16(t); + v8f p; + [unroll] for (uint i3 = 0; i3 < 8; ++i3) p.v[i3] = mad(u.v[i3], t.v[i3], WG_HYB_POLY_P); + const uint4 pb = hyb_rn16x8(p); + const uint w[4] = { pb.x, pb.y, pb.z, pb.w }; + v8f e; + [unroll] for (uint i4 = 0; i4 < 4; ++i4) { + e.v[2 * i4] = wg_h2f_lo((w[i4] & 0x03ffu) << 5); + e.v[2 * i4 + 1] = wg_h2f_lo(((w[i4] >> 16) & 0x03ffu) << 5); + } + return e; +} + +// r = RN16(rsqrt32(S')) for a positive normal FP16 S' (bits) in the norm's range (S' >= 2^-13), by table; S' = inf +// (the FP16 sum of squares overflowed) gives 0 and a NaN S' gives NaN, as rsqrt32 (oracle _rsqrt32, HIP 1 / sqrt): +// the table alone would read inf as 2^16 and a NaN as a finite number. +float hyb_rsq16_tab(uint sbits); +float hyb_rsq16(uint sbits) +{ + const float r = hyb_rsq16_tab(sbits); + return (sbits & 0x7c00u) == 0x7c00u ? ((sbits & 0x03ffu) != 0u ? asfloat(0x7fc00000u) : 0.0f) : r; +} +float hyb_rsq16_tab(uint sbits) +{ + const int E = (int)((sbits >> 10) & 31u) - 15; + const uint p = (uint)E & 1u; + const int k = (E - (int)p) >> 1; + const uint off = WG_HYB_RSQ + 2u * (p * 1024u + (sbits & 1023u)); + const uint rb = (wg_mem.Load(off & ~3u) >> (8u * (off & 2u))) & 0xffffu; + return wg_h2f_lo((uint)((int)rb - (k << 10))); +} + +// w = RN16(rcp32(sum)) for a positive normal FP16 row sum (bits) with a normal result (every softmax row), by table. +float hyb_rcp16(uint sbits) +{ + const int E = (int)((sbits >> 10) & 31u) - 15; + const uint off = WG_HYB_RCP + 2u * (sbits & 1023u); + const uint rb = (wg_mem.Load(off & ~3u) >> (8u * (off & 2u))) & 0xffffu; + return wg_h2f_lo((uint)((int)rb - (E << 10))); +} + +// The merge clamp before the FP8 point: max first, then min, NaN kept (Q8 B4). +float hyb_merge_clamp(float x) { return isnan(x) ? x : min(max(x, -WG_HYB_MERGE_CLAMP), WG_HYB_MERGE_CLAMP); } +// The merges' FP8 point: FP8(clamp(RN16(chain), +-6.28125)), NaN kept (oracle run_stage_q7: q8(clamp(mm8(...)))). +uint2 hyb_merge_q8(v8f acc) +{ + v8f r = hyb_round16(acc); + [unroll] for (uint i = 0; i < 8; ++i) r.v[i] = hyb_merge_clamp(r.v[i]); + return wg_pack_q8_ags(r); +} + +// The decoder expansion's two mirrors (Q10b): the fine one for S, the coarse one for E (c = floor(p / 2)). +int hyb_mirror_fine(int p, int n) { return min(n - 1, min(abs(p), 2 * n - 2 - abs(p))); } +int hyb_mirror_coarse(int p, int n, int nc) +{ + const int c = p >> 1; // floor(p / 2) for negative p too (arithmetic shift) + return min(nc - 1, min(abs(c), 2 * (n >> 1) - 2 - abs(c))); +} diff --git a/clean/dx12/hyb_isa_scan.py b/clean/dx12/hyb_isa_scan.py new file mode 100644 index 00000000..4772b168 --- /dev/null +++ b/clean/dx12/hyb_isa_scan.py @@ -0,0 +1,362 @@ +# SPDX-License-Identifier: MIT +"""ISA rules of the hybrid kernels' D3D12 code (the hybrid check design: the byte legs and these rules catch a driver that +contracts an FP16 product into an add, or folds the FP16 rounding before an FP8 point into the FP8 conversion). + +1. Fused squares. The hybrid form's FP16 sums follow the original's order with every square rounded first + (q = RN16(x x)); the driver fuses a square feeding an add into one instruction when it can see the pattern, even under + `precise` (seen on gfx1201: v_pk_fma_f16 v11, v17, v17, v11 in the select-free norm tree). The kernels hide the + pattern with an OR against a zero the compiler cannot prove (hlsl/hyb_block.hlsli HYB_OPAQUE0); a future driver that + sees through it would bring the fusion back. The rule fails on any fused multiply-add whose two multiplicands hold + the same value half (a square): + v_pk_fma_f16, v_fma_f16, v_fmac_f16, v_pk_fmac_f16, v_fmaak_f16, v_fma_mix{lo,hi}_f16, v_fma_mix_f32, v_fma_f32, + v_fmac_f32, v_fmaak_f32, v_fma_dx9_zero_f32, v_fmac_dx9_zero_f32, and the dual-issue (VOPD) halves v_dual_fmac_f32, + v_dual_fmaak_f32 + "The same value": the same register, or registers that a copy (v_mov_b32 / v_mov_b16 / v_dual_mov_b32 of a VGPR) + made equal on every path to the instruction (must-copies, through the basic blocks: a compiler that copies x to a + second register and fuses x * copy is caught); the same op_sel / op_sel_hi bits for both sources; neg / abs + modifiers ignored (-x x is a square too). FP16 forms fail with any addend. The float32 / mixed forms fail when the + addend is a VGPR (a running sum, as in the norm tree; v_fmac always adds its destination): the softmax's float32 + steps e(u') compute -u u + constant with a literal / SGPR addend, exact by design (hlsl/wg_hyb.hlsli + hyb_softmax_e8), and are not flagged. +2. Conversions before FP8 points. Every FP8 point of the hybrid form is FP8(RN16(float32 value)): two roundings by + design (wg_hyb.hlsli hyb_q8_of_chain). In the ISA, both sources of every v_cvt_pk_fp8_f32 must be FP16 values: written + by v_cvt_f32_f16 (the exact widening after the RN16), or by the saturating pack's steps on such values only + (v_mov_b32, v_cndmask_b32, v_max / v_min / v_max3 / v_min3 / v_maxmin / v_minmax / v_med3 (_num)_f32 with constants: + the ReLU and the clamps; the NaN quieting v_or_b32 with 0x400000), on every path to the conversion (a dataflow over + the basic blocks). Anything else + (a WMMA result, an FMA, a load, an instruction the rule does not know) fails: a driver that drops the RN16 or folds + it into the FP8 conversion leaves the float32 value as the source. + +Each rule has its negative control: every model hybrid kernel's mutant (*_scanmut: no contraction barrier, no RN16 +before the FP8 pack, HYB_SCAN_MUTANT=1; build_dx12.py) must FAIL rule 2, and rule 1 too unless the kernel has no norm +tree; tests/test_hyb_isa_scan.py checks both rules on constructed ISA (a copied-register square, a removed widening). + + python clean/dx12/hyb_isa_scan.py ... exit 1 when a rule fails +build_dx12.py runs both rules on every *_hyb* kernel it builds (no skip switch). +""" +import os +import re +import sys + +FMA_OPS = {"v_pk_fma_f16", "v_fma_f16", "v_fmac_f16", "v_pk_fmac_f16", "v_fmaak_f16", "v_fma_mixlo_f16", + "v_fma_mixhi_f16", "v_fma_mix_f32", "v_fma_f32", "v_fmac_f32", "v_fmaak_f32", "v_fma_dx9_zero_f32", + "v_fmac_dx9_zero_f32"} +F16_FMA = {"v_pk_fma_f16", "v_fma_f16", "v_fmac_f16", "v_pk_fmac_f16", "v_fmaak_f16"} +FP8_PACK = {"v_cvt_pk_fp8_f32", "v_cvt_sr_fp8_f32"} +WIDEN = {"v_cvt_f32_f16"} +# ops that keep a value an FP16 value when every VGPR source is one (constants: the pack's bounds) +KEEP = {"v_mov_b32", "v_cndmask_b32", "v_max_num_f32", "v_min_num_f32", "v_maxmin_num_f32", "v_minmax_num_f32", + "v_med3_num_f32", "v_max3_num_f32", "v_min3_num_f32", "v_max_f32", "v_min_f32", "v_maxmin_f32", + "v_minmax_f32", "v_med3_f32", "v_max3_f32", "v_min3_f32"} +QUIET_NAN = "0x400000" +NO_DEST = re.compile(r"^(s_|v_cmp_|v_cmpx_|v_readlane|v_readfirstlane|.*_store|ds_write|exp\b|v_nop)") +VREG = re.compile(r"^v(\d+)(?:\.(l|h))?$") +VRANGE = re.compile(r"^v\[(\d+):(\d+)\]$") +MOD = re.compile(r"\b[a-z_]+:(?:\[[^\]]*\]|[^\s,]+)") +LABEL = re.compile(r"^\s*([A-Za-z_][\w.$]*):\s*(?://.*)?$") + + +class Ins: + __slots__ = ("line", "op", "ops", "mods", "text") + + def __init__(self, line, op, ops, mods, text): + self.line, self.op, self.ops, self.mods, self.text = line, op, ops, mods, text + + +def _norm_op(op): + op = re.sub(r"_e(32|64|64_dpp|32_dpp)$", "", op) + return op.replace("v_dual_", "v_", 1) if op.startswith("v_dual_") else op + + +def _split_one(line, text): + text = text.strip() + if not text: + return None + head, _, rest = text.partition(" ") + # key:value modifiers (op_sel:[0,1,0], quad_perm:[...], offset:16; their brackets hold commas) first + mods = [m.group(0) for m in MOD.finditer(rest)] + rest = MOD.sub(" ", rest) + parts = [p.strip() for p in rest.split(",")] if rest.strip() else [] + ops = [] + for p in parts: + toks = p.split() + if not toks: + continue + ops.append(toks[0]) + mods += toks[1:] # bare modifiers (clamp, ...) + return Ins(line, _norm_op(head), ops, " ".join(mods), text) + + +def parse(text): + """[(label or None, Ins or None)] in text order (labels as their own entries); VOPD lines give two Ins.""" + out = [] + for n, raw in enumerate(text.splitlines(), 1): + m = LABEL.match(raw) + if m: + out.append((m.group(1), None)) + continue + code = raw.split("//", 1)[0].strip() + if not code or code.startswith((";", ".")): + continue + for half in code.split("::"): + ins = _split_one(n, half) + if ins: + out.append((None, ins)) + return out + + +def _reg(op): + return op.strip().lstrip("-").strip("|").strip() + + +def _vregs(tok): + """32-bit VGPR numbers a token names (a register, a half, a range); [] for anything else.""" + t = _reg(tok) + m = VREG.match(t) + if m: + return [int(m.group(1))] + m = VRANGE.match(t) + if m: + return list(range(int(m.group(1)), int(m.group(2)) + 1)) + return [] + + +def _dest(ins): + if NO_DEST.match(ins.op) or not ins.ops: + return [] + return _vregs(ins.ops[0]) + + +def blocks(items): + """Basic blocks [(label set, [Ins], successor labels, falls through)] of a parsed program.""" + bl, cur, labels = [], [], [] + + def close(succ, fall): + bl.append((labels[:], cur[:], succ, fall)) + + for lab, ins in items: + if lab is not None: + if cur: + close([], True) + cur.clear() + labels.clear() + labels.append(lab) + continue + cur.append(ins) + if ins.op == "s_branch" and ins.ops: + close([ins.ops[0]], False) + elif ins.op.startswith("s_cbranch") and ins.ops: + close([ins.ops[0]], True) + elif ins.op in ("s_endpgm", "s_setpc_b64", "s_trap"): + close([], False) + else: + continue + cur.clear() + labels.clear() + if cur or labels: + close([], False) + return bl + + +def _solve(bl, entry, transfer, meet): + """Forward must-analysis over the blocks: in-state of each block (None = not reached yet).""" + idx = {} + for i, (labs, _, _, _) in enumerate(bl): + for lab in labs: + idx[lab] = i + preds = [[] for _ in bl] + for i, (_, _, succ, fall) in enumerate(bl): + for s in succ: + if s in idx: + preds[idx[s]].append(i) + if fall and i + 1 < len(bl): + preds[i + 1].append(i) + ins_state = [None] * len(bl) + out_state = [None] * len(bl) + changed = True + while changed: + changed = False + for i, (_, body, _, _) in enumerate(bl): + ps = [out_state[p] for p in preds[i] if out_state[p] is not None] + st = entry() if i == 0 else (meet(ps) if ps else None) + if st is None: + continue + if ins_state[i] != st or out_state[i] is None: + ins_state[i] = st + o = st + for ins in body: + o = transfer(o, ins, None) + if o != out_state[i]: + out_state[i] = o + changed = True + return ins_state + + +# ---- rule 1: fused squares (with must-copies) ---- + +def _alias_kill(al, d): + al = dict(al) + for k in [k for k in al if k == d or k.startswith(d + ".")]: + del al[k] + members = sorted(k for k, r in al.items() if r == d or r.startswith(d + ".")) + if members: + for k in members: + r = al[k] + al[k] = members[0] + r[len(d):] if r != d else members[0] + return al + + +def _root(al, tok): + t = _reg(tok) + if t in al: + return al[t] + base, dot, half = t.partition(".") + if dot and base in al: + return al[base] + "." + half + return t + + +def _alias_transfer(al, ins, hits): + if hits is not None and ins.op in FMA_OPS and len(ins.ops) >= 3: + _check_fma(al, ins, hits) + dregs = _dest(ins) + if not dregs: + return al + tok = _reg(ins.ops[0]) + src = _reg(ins.ops[1]) if len(ins.ops) > 1 else "" + al = dict(al) + for r in dregs: + al = _alias_kill(al, "v%d" % r) + if ins.op in ("v_mov_b32", "v_mov_b16") and len(ins.ops) == 2 and (VREG.match(src)) and not ins.mods: + if src.split(".")[0] != tok.split(".")[0]: + al[tok] = _root(al, src) + return al + + +def _alias_meet(states): + first = states[0] + return {k: v for k, v in first.items() if all(s.get(k) == v for s in states[1:])} + + +def _bits(mods, key): + m = re.search(key + r":\[([01,\s]+)\]", mods) + return [int(b) for b in m.group(1).replace(" ", "").split(",")] if m else None + + +def _check_fma(al, ins, hits): + ops = ins.ops + a, b = _reg(ops[1]), _reg(ops[2]) + if not (a.startswith("v") and b.startswith("v")): + return + if a != b and _root(al, a) != _root(al, b): + return + sel, hi = _bits(ins.mods, "op_sel"), _bits(ins.mods, "op_sel_hi") + if sel and len(sel) > 1 and sel[0] != sel[1]: + return + if hi and len(hi) > 1 and hi[0] != hi[1]: + return + if ins.op not in F16_FMA: + if "fmaak" in ins.op: + return # float32: a literal addend + addend = _reg(ops[0]) if "fmac" in ins.op else (_reg(ops[3]) if len(ops) > 3 else "") + if not re.match(r"v[0-9]|v\[", addend): + return + hits.append((ins.line, ins.text)) + + +def fused_squares(text): + """[(line number, instruction text)] of fused multiply-adds whose two multiplicands are one value half.""" + bl = blocks(parse(text)) + if not bl: + return [] + states = _solve(bl, dict, _alias_transfer, _alias_meet) + hits = [] + for st, (_, body, _, _) in zip(states, bl): + if st is None: + st = {} + for ins in body: + st = _alias_transfer(st, ins, hits) + return sorted(set(hits)) + + +# ---- rule 2: FP16 values into the FP8 conversions ---- + +ALL = None # the meet's top: every register an FP16 value (blocks not reached yet) + + +def _exact_transfer(ex, ins, bad): + srcs = ins.ops[1:] + if bad is not None and ins.op in FP8_PACK: + for s in srcs[:2]: + rs = _vregs(s) + if rs and not all(r in ex for r in rs): + bad.append((ins.line, ins.text)) + break + dregs = _dest(ins) + if not dregs: + return ex + vsrc = [r for s in srcs for r in _vregs(s)] + if ins.op in WIDEN: + keep = True + elif ins.op in KEEP: + keep = all(r in ex for r in vsrc) and not ("v_cndmask" in ins.op and ins.mods and "neg" in ins.mods) + elif ins.op == "v_or_b32": + lits = [_reg(s) for s in srcs if not _vregs(s)] + keep = len(vsrc) == 1 and lits == [QUIET_NAN] and vsrc[0] in ex + else: + keep = False + if any(t in ins.mods for t in ("clamp", "mul:", "div:")): + keep = False + ex = set(ex) + full = VREG.match(_reg(ins.ops[0])) + if keep and len(dregs) == 1 and full and not full.group(2): # a whole 32-bit register (a half write: not) + ex.add(dregs[0]) + else: + ex.difference_update(dregs) + return frozenset(ex) + + +def _exact_meet(states): + out = states[0] + for s in states[1:]: + out = out & s + return out + + +def fp8_sources_not_fp16(text): + """[(line number, instruction text)] of FP8 conversions with a source that is not an FP16 value on every path.""" + bl = blocks(parse(text)) + if not bl: + return [] + states = _solve(bl, frozenset, _exact_transfer, _exact_meet) + bad = [] + for st, (_, body, _, _) in zip(states, bl): + if st is None: + continue # unreachable + for ins in body: + st = _exact_transfer(st, ins, bad) + return sorted(set(bad)) + + +def fp8_conversions(text): + return sum(1 for _, ins in parse(text) if ins is not None and ins.op in FP8_PACK) + + +def main(argv): + files = [] + for p in argv: + files += [os.path.join(p, f) for f in sorted(os.listdir(p)) if f.endswith(".txt")] if os.path.isdir(p) else [p] + bad = 0 + for f in files: + text = open(f, encoding="utf-8", errors="replace").read() + hits, conv = fused_squares(text), fp8_sources_not_fp16(text) + print(f"{os.path.basename(f)}: {len(hits)} fused square(s), {len(conv)} of {fp8_conversions(text)} FP8 " + "conversion(s) with a source that is not an FP16 value") + for ln, ins in (hits[:4] + conv[:4]): + print(f" line {ln}: {ins}") + bad += len(hits) + len(conv) + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index 492c509f..881354c2 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -16,6 +16,10 @@ "color_divide", "frame.co" ], + [ + "d1p12/dec0_q7", + "dec1_dec0__pack12.co" + ], [ "d1p12/swin_block_c64_add_dec0_nc", "dec1_dec0__pack12.co" @@ -186,6 +190,14 @@ ] ], "model_l": [ + [ + "expand_c128_hyb", + "-" + ], + [ + "expand_c96_hyb", + "-" + ], [ "l_coef", "l_ops.co" @@ -246,14 +258,74 @@ "l_project_gather", "l_ops.co" ], + [ + "l_q7_block_c128", + "l_q7w.co" + ], + [ + "l_q7_block_c32_dec", + "l_q7.co" + ], + [ + "l_q7_block_c32_enc", + "l_q7.co" + ], + [ + "l_q7_block_c64", + "l_q7w.co" + ], + [ + "l_q7_block_c64_dec", + "l_q7w.co" + ], + [ + "l_q7_block_c96", + "l_q7w.co" + ], + [ + "l_q7_block_c96_dec", + "l_q7w.co" + ], + [ + "l_q7_expand_c128", + "l_q7w.co" + ], + [ + "l_q7_expand_c96", + "l_q7w.co" + ], + [ + "l_q7_merge_c64", + "l_q7w.co" + ], + [ + "l_q7_merge_c96", + "l_q7w.co" + ], + [ + "l_swin_block_c32_dec_coef12_hyb", + "-" + ], [ "l_swin_block_c32_dec_coef12_nc", "-" ], + [ + "l_swin_block_c32_dec_hyb", + "-" + ], [ "l_swin_block_c32_dec_nc", "l_swin.co" ], + [ + "l_swin_block_c32_enc_hyb", + "-" + ], + [ + "l_swin_block_c32_enc_lin_hyb", + "-" + ], [ "l_swin_block_c32_enc_lin_nc", "-" @@ -266,6 +338,22 @@ "l_swin_block_c32_nc", "l_swin.co" ], + [ + "lc12/l_post_recon", + "l_post__lc12.co" + ], + [ + "lc12/l_post_recon_ldr", + "l_post__lc12.co" + ], + [ + "lc12/l_post_recon_p12", + "l_post__lc12.co" + ], + [ + "lc12/l_post_recon_p12_ldr", + "l_post__lc12.co" + ], [ "lev/input_encoder", "l_enc_values.co" @@ -282,9 +370,41 @@ "lev/input_encoder_q20", "-" ], + [ + "patch_merge_c64_hyb", + "-" + ], + [ + "patch_merge_c96_hyb", + "-" + ], + [ + "swin_block_c128_hyb", + "-" + ], [ "swin_block_c64_add_ex_nc", "swin.co" + ], + [ + "swin_block_c64_add_hyb", + "-" + ], + [ + "swin_block_c64_hyb", + "-" + ], + [ + "swin_block_c96_add_hyb", + "-" + ], + [ + "swin_block_c96_hyb", + "-" + ], + [ + "swin_bneck6_hyb", + "-" ] ], "model_k": [ diff --git a/clean/dx12/kernels/dec1_dec0.hlsl b/clean/dx12/kernels/dec1_dec0.hlsl index 8ea9aebb..909782b7 100644 --- a/clean/dx12/kernels/dec1_dec0.hlsl +++ b/clean/dx12/kernels/dec1_dec0.hlsl @@ -22,6 +22,14 @@ // rounding after each step, not in half arithmetic (the driver contracts half mul + add into v_fma_mix_f32). // Bit-identical to the HIP kernel on the synthetic frames (clean/dx12/compare.py, both cases, every buffer). // @wg-kernel d1p12/swin_block_c64_add_dec0_nc co=dec1_dec0__pack12.co block=256 wave=32 args=p,p,p,u,u,u,u,p,p,p,p,p,p,p,b56 defines=WG_GELU_T5=0 +// +// DEC0_ONLY (port d1p12/dec0_q7; Model M on the FP16-form shared stages, WG_M_FORM=q7, where dec1 runs as the q7 block +// l_q7_block_c64_dec and stores its map): the same kernel without the block. The epilogue's input o[0..3] (lane (t, hh): +// channels 32 hh .. 32 hh + 31 of token t, the bytes dec1 would store at token * 64 + 32 hh) is read from that stored +// map (a_x = dec1's output, FP8 [H][W][64]) instead of being packed from the block's accumulators; windows, prc and the +// whole epilogue are this kernel's own, so the result is d0p12/coef_decoder's on HIP (the fused kernel's contract). +// a_xadd and a_w are not read. Launched with the fused kernel's arguments and grid. +// @wg-kernel d1p12/dec0_q7 co=dec1_dec0__pack12.co entry=swin_block_c64_add_dec0_nc block=256 wave=32 args=p,p,p,u,u,u,u,p,p,p,p,p,p,p,b56 defines=WG_GELU_T5=0,DEC0_ONLY=1 #include "wg_platform.hlsli" #include "wg_gelu_lut.hlsli" #include "dec1_dec0_diet.hlsli" @@ -456,6 +464,24 @@ WG_KERNEL(swin_block_c64_add_dec0_nc) const uint oz0 = 0u - (wg_grid_ >> 31); const uint vw32 = a_w + hh * 32u + oz0, vw128 = a_w + hh * 128u + oz0, vwb = a_w + (t * 2u + hh) * 32u + oz0; +#ifndef DEC0_ONLY +#define DEC0_ONLY 0 +#endif +#if DEC0_ONLY + // dec0 only: the window's tokens as below (same windows, mirror and prc), their stored FP8 codes as o[0..3] + uint prc; + uint2 xo[KC]; + { + const int win = win0 < nwin ? win0 : nwin - 1; + const int wy = (int)((uint)win / (uint)nwx), wx = win - wy * nwx; + const int pr = 4 * wy - a_oy + (int)(t >> 2), pc = 4 * wx - a_ox + (int)(t & 3u); + const bool real = (win0 < nwin) & ((uint)pr < (uint)a_H) & ((uint)pc < (uint)a_W); + prc = real ? ((uint)pr << 16) | (uint)pc : 0xffffffffu; + const uint at = (uint)(wg_mirror(pr, a_H) * a_W + wg_mirror(pc, a_W)) * (uint)C_ + hh * (uint)HALF; + const uint4 d0 = wg_ld128(a_x, at), d1 = wg_ld128(a_x, at + 16u); + xo[0] = d0.xy; xo[1] = d0.zw; xo[2] = d1.xy; xo[3] = d1.zw; + } +#else fetch_head(L_HEAD(0), tid); // unit 0 #if DEC_GL_USED { @@ -716,6 +742,7 @@ WG_KERNEL(swin_block_c64_add_dec0_nc) } } +#endif // !DEC0_ONLY // ---- dec0 as the epilogue (dec1_dec0.hip Dec0Epi::run) ---- const bool real = prc != 0xffffffffu; const int rr = real ? (int)(prc >> 16) : 0, cc = real ? (int)(prc & 0xffffu) : 0; @@ -732,6 +759,9 @@ WG_KERNEL(swin_block_c64_add_dec0_nc) } // dec0's B fragments from the block output, exactly the bytes dec1 would store; lane halves swap two 8-byte groups. uint2 o[KC]; +#if DEC0_ONLY + [unroll] for (uint k = 0; k < KC; ++k) o[k] = xo[k]; +#else // wave-uniform skip of the clamp: when every value of the wave is NaN or |x| <= 448 (the driver's max drops NaN), // the clamp is the identity and the plain conversion gives the same bytes (NaN -> 0xff on both sides) { @@ -745,6 +775,7 @@ WG_KERNEL(swin_block_c64_add_dec0_nc) [unroll] for (uint k = 0; k < KC; ++k) o[k] = dec_pack_q8_ags(acc[k]); } } +#endif const bool h1 = hh != 0u; uint snd0 = h1 ? o[0].x : o[2].x, snd1 = h1 ? o[0].y : o[2].y, snd2 = h1 ? o[1].x : o[3].x, snd3 = h1 ? o[1].y : o[3].y; const uint2 r01 = uint2(wg_xor16_u(snd0), wg_xor16_u(snd1)), r23 = uint2(wg_xor16_u(snd2), wg_xor16_u(snd3)); diff --git a/clean/dx12/kernels/fused_dec.hlsl b/clean/dx12/kernels/fused_dec.hlsl index 55488495..31324ce5 100644 --- a/clean/dx12/kernels/fused_dec.hlsl +++ b/clean/dx12/kernels/fused_dec.hlsl @@ -477,6 +477,24 @@ WG_KERNEL(swin_block_c64_add_dec0_nc) const uint oz0 = 0u - (wg_grid_ >> 31); const uint vw32 = a_w + hh * 32u + oz0, vw128 = a_w + hh * 128u + oz0, vwb = a_w + (t * 2u + hh) * 32u + oz0; +#ifndef DEC0_ONLY +#define DEC0_ONLY 0 +#endif +#if DEC0_ONLY + // dec0 only: the window's tokens as below (same windows, mirror and prc), their stored FP8 codes as o[0..3] + uint prc; + uint2 xo[KC]; + { + const int win = win0 < nwin ? win0 : nwin - 1; + const int wy = (int)((uint)win / (uint)nwx), wx = win - wy * nwx; + const int pr = 4 * wy - a_oy + (int)(t >> 2), pc = 4 * wx - a_ox + (int)(t & 3u); + const bool real = (win0 < nwin) & ((uint)pr < (uint)a_H) & ((uint)pc < (uint)a_W); + prc = real ? ((uint)pr << 16) | (uint)pc : 0xffffffffu; + const uint at = (uint)(wg_mirror(pr, a_H) * a_W + wg_mirror(pc, a_W)) * (uint)C_ + hh * (uint)HALF; + const uint4 d0 = wg_ld128(a_x, at), d1 = wg_ld128(a_x, at + 16u); + xo[0] = d0.xy; xo[1] = d0.zw; xo[2] = d1.xy; xo[3] = d1.zw; + } +#else fetch_head(L_HEAD(0), tid); // unit 0 #if DEC_GL_USED { @@ -738,6 +756,7 @@ WG_KERNEL(swin_block_c64_add_dec0_nc) } } +#endif // !DEC0_ONLY // ---- dec0 as the epilogue (dec1_dec0.hip Dec0Epi::run) ---- const bool real = prc != 0xffffffffu; const int rr = real ? (int)(prc >> 16) : 0, cc = real ? (int)(prc & 0xffffu) : 0; @@ -754,6 +773,9 @@ WG_KERNEL(swin_block_c64_add_dec0_nc) } // dec0's B fragments from the block output, exactly the bytes dec1 would store; lane halves swap two 8-byte groups. uint2 o[KC]; +#if DEC0_ONLY + [unroll] for (uint k = 0; k < KC; ++k) o[k] = xo[k]; +#else // wave-uniform skip of the clamp: when every value of the wave is NaN or |x| <= 448 (the driver's max drops NaN), // the clamp is the identity and the plain conversion gives the same bytes (NaN -> 0xff on both sides) { @@ -767,6 +789,7 @@ WG_KERNEL(swin_block_c64_add_dec0_nc) [unroll] for (uint k = 0; k < KC; ++k) o[k] = dec_pack_q8_ags(acc[k]); } } +#endif const bool h1 = hh != 0u; uint snd0 = h1 ? o[0].x : o[2].x, snd1 = h1 ? o[0].y : o[2].y, snd2 = h1 ? o[1].x : o[3].x, snd3 = h1 ? o[1].y : o[3].y; const uint2 r01 = uint2(wg_xor16_u(snd0), wg_xor16_u(snd1)), r23 = uint2(wg_xor16_u(snd2), wg_xor16_u(snd3)); diff --git a/clean/dx12/kernels/fused_swin.hlsl b/clean/dx12/kernels/fused_swin.hlsl index d3157eea..0f24598b 100644 --- a/clean/dx12/kernels/fused_swin.hlsl +++ b/clean/dx12/kernels/fused_swin.hlsl @@ -35,7 +35,107 @@ // @wg-kernel swin_block_c96_add_xe_xd_nc co=swin.co entry=swin_block_c96_add_xe_nc block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p,p,p,u,u,u,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=1,SW_SAFE=1,SW_ECL=128,SW_XE=1,FZ_XO=1,FZ_XD=1,SW_HEAD_PAIRS=0 #include "wg_platform.hlsli" #include "wg_gelu_lut.hlsli" +// SW_HYB 1: the HYBRID block form (WG_L_FORM / WG_M_FORM = hybrid; hlsl/wg_hyb.hlsli, hlsl/hyb_block.hlsli): float32 FP8 +// chains with one FP16 rounding at each chain's end, the original's FP16 norm / softmax / GELU and its start values; +// the weights in this file's layout with the original's values (FP16 biases, g', B; plain FP8 matrices). Default 0. +#ifndef SW_HYB +#define SW_HYB 0 +#endif +#ifndef SW_HYB_MUT_ORDER +#define SW_HYB_MUT_ORDER 0 // check mutants only: 1 one W2 chunk pair reversed, 2 the pair of every W2 group, + // 3 the odd W2 groups in a second accumulator added at the end (one extra f32 rounding), + // 4 the W2 chain end moved by one float32 ulp before its RN16 +#endif +#ifndef SW_HYB_DUMP +#define SW_HYB_DUMP 0 // check variant only (probe kernels, l_hyb_stage_check.py): every stage's values to `dumpp` +#endif +#if SW_HYB_DUMP && !SW_HYB +#error "SW_HYB_DUMP: hybrid kernels only" +#endif +#if SW_HYB +// The hybrid kernels at every width (C = 32, 64, 96, 128) run the plain schedule (the C = 32 knob set): no scheduling +// knob of the M-form kernels, no GELU table of theirs (hyb_gelu_q8 has its own), and every output pack through +// hyb_q8_of_chain (no clamp-skip shortcut: FP8(x2) without the RN16 is not the hybrid output). Set here, before the +// defaults below; setting one of them on a kernel line stops the build (the check after this block). +#ifndef SW_GELU +#define SW_GELU 0 +#endif +#ifndef SW_CLAMP_SKIP +#define SW_CLAMP_SKIP 0 +#endif +#ifndef SW_NORM1_EARLY +#define SW_NORM1_EARLY 0 +#endif +#ifndef SW_NORM2_EARLY +#define SW_NORM2_EARLY 0 +#endif +#ifndef SW_NORM_R2 +#define SW_NORM_R2 0 +#endif +#ifndef SW_SMAX_ADDST +#define SW_SMAX_ADDST 0 +#endif +#ifndef SW_SMAX_STAGED +#define SW_SMAX_STAGED 0 +#endif +#ifndef SW_HEAD_UNROLL +#define SW_HEAD_UNROLL 0 +#endif +#ifndef SW_HEAD_PAIRS +#define SW_HEAD_PAIRS 0 +#endif +#ifndef SW_MLP_PAIRS +#define SW_MLP_PAIRS 0 +#endif +#ifndef SW_MLP_STEP +#define SW_MLP_STEP 1 +#endif +#ifndef SW_REV_PHI +#define SW_REV_PHI 0 +#endif +#ifndef SW_COMMIT_EARLY +#define SW_COMMIT_EARLY 0 +#endif +#ifndef SW_N2_BATCH +#define SW_N2_BATCH 0 +#endif +#ifndef SW_GELU_LATE0 +#define SW_GELU_LATE0 0 +#endif +#endif +#if SW_HYB && (SW_GELU != 0 || SW_HEAD_UNROLL || SW_HEAD_PAIRS || SW_MLP_PAIRS || SW_MLP_STEP != 1 || SW_COMMIT_EARLY || \ + SW_N2_BATCH) +#error "SW_HYB: the hybrid kernels run the plain schedule (no M-form scheduling knobs)" +#endif +// (SW_SRC 3 and SW_OUT_COEF12 are allowed: the linears' prologue gives the block the same FP16 values as the x2 map, +// and the pack12 epilogue reads the same FP16 output, converted by the driver's RNE as the hybrid output point does) +#if SW_HYB &&(SW_ECL > 0 || SW_NORM1_EARLY || SW_NORM2_EARLY || SW_NORM_R2 || SW_SMAX_STAGED || SW_SMAX_ADDST || \ + SW_REV_PHI || SW_CLAMP_SKIP) +#error "SW_HYB: only the plain block path (no fused expand, no scheduling knobs) has the hybrid form so far" +#endif +#if SW_HYB && SW_OUT_COEF12 && !COEF_CVT_HW +#error "SW_HYB with SW_OUT_COEF12: COEF_CVT_HW 1 (the driver's RNE conversion of the hybrid output point)" +#endif +// SW_MEGA 1 (kernels/hyb_bneck.hlsl only): the body below becomes the function sw_item(), one item (8 windows) of the +// bottleneck's persistent queue; its arguments are the queue kernel's (bneck.hlsl's), and the block's x, out_p, w, ox, +// oy are statics the queue sets per item (wg_bid_ = the item's index in its block). The input gather and the output +// store use the coherent view (the maps are written by other workgroups of the same launch). The same instructions on +// the same values otherwise: b5's bytes equal six swin_block_c128_hyb launches. +#ifndef SW_MEGA +#define SW_MEGA 0 +#endif +#if SW_MEGA && (!SW_HYB || SW_C != 128 || SW_ADD || SW_ECL > 0 || SW_SRC || SW_OUT_F16 || SW_OUT_COEF12 || SW_HYB_DUMP) +#error "SW_MEGA: the hybrid C128 bottleneck block only (plain FP8 map in and out, no dump)" +#endif +#if SW_MEGA +WG_ARGS_BEGIN + uint a_x, a_out, a_tmp, a_sync, a_w0, a_w1, a_w2, a_w3, a_w4, a_w5; // pointers (bneck.hlsl's arguments) + int H, W, a_oxe, a_oye, a_oxo, a_oyo, a_groups; +WG_ARGS_END +static uint x, out_p, w; // the current item's block: input map, output map, weight section +static int ox, oy; // its window offsets +#else #ifndef SW_PM #define SW_PM 0 #endif @@ -78,6 +178,7 @@ WG_ARGS_END #else #define FZ_EST(b, o, v) wg_st128(b, o, v) #endif +#endif // ---- BlockLayout (FORMATS.md, "Swin block section") --------------------------------------------------- #define KC (SW_C / 16) @@ -120,6 +221,8 @@ WG_ARGS_END #define HPU 8 // MTH 16, limit 8 #elif SW_C == 96 #define HPU 4 // MTH 24, limit 5 +#elif SW_C == 128 && SW_HYB +#define HPU 4 // MTH 32, limit 4 (the hybrid bottleneck block; the M-form C = 128 blocks are bneck.hlsl) #else #error "SW_C: 64 or 96" #endif @@ -427,13 +530,57 @@ uint L_mlp(uint u) { return L_STREAM + SW_HEADS * HEAD_BYTES + u * MLP_BYTES; } // Inner FP8 pack (HIP pack_in) with the driver conversion (wave-uniform control flow); the output pack is sw_pack_q8. uint2 pack_in(v8f d) { -#if SW_SAFE +#if SW_HYB + return wg_pack_q8_ags(d); +#elif SW_SAFE return wg_pack_q8_nc_ags(d); #else return wg_pack_q8_inner_ags(d); #endif } +#if SW_HYB +#include "wg_hyb.hlsli" +// a zero the compiler cannot prove (the grid height is never negative): hyb_block.hlsli's contraction barrier. +// HYB_SCAN_MUTANT=1 (the fused-square scan's mutant, *_scanmut, never in a model build): a literal zero, i.e. the +// barrier removed; build_dx12.py requires the scan to FAIL on it +#if HYB_SCAN_MUTANT +#define HYB_OPAQUE0 0u +#else +// the unknown bit in BOTH FP16 halves (bits 0 and 16): with bit 0 only, the compiler proved the high half untouched +// and fused it (v_and_or_b32 on the low half, v_fma_f16 on .h) +#define HYB_OPAQUE0 (((uint)H >> 31u) * 0x10001u) +#endif +#include "hyb_block.hlsli" +#endif +#if SW_HYB_DUMP +// Stage dump (l_hyb_stage_check.py): per real window and lane, HD_WORDS 32-bit words, every value in the lane's own +// register layout (float32 bits; P as its packed FP16 words). Fields: X the block input (after the skip add), Y1 norm1's +// output; per head (60 words): UQ RN16 of the q/k chain (2 tiles), V RN16 of the v chain (2 tiles), S RN16 of the score +// chain, P (4 words), O RN16 of the P V chain (2 tiles); Z RN16 of the Wp chain, Y2 norm2's output, H RN16 of the W1 +// chains (MTH tiles), OUT RN16 of the W2 chain (the block output before its FP8 / FP16 point). Only values the kernel +// computes anyway; the stores are its only addition. +#define HD_HEAD (16u * KC) +#define HD_Z (HD_HEAD + 60u * SW_HEADS) +#define HD_Y2 (HD_Z + 8u * KC) +#define HD_H (HD_Y2 + 8u * KC) +#define HD_OUT (HD_H + 8u * MTH) +#define HD_WORDS (HD_OUT + 8u * KC) +static uint hd_base; +static bool hd_on; +void hd_st4(uint word, uint4 a) { if (hd_on) wg_st128(dumpp, (hd_base + word) * 4u, a); } +void hd_st8(uint word, v8f a) +{ + hd_st4(word, uint4(asuint(a.v[0]), asuint(a.v[1]), asuint(a.v[2]), asuint(a.v[3]))); + hd_st4(word + 4u, uint4(asuint(a.v[4]), asuint(a.v[5]), asuint(a.v[6]), asuint(a.v[7]))); +} +#define HD_ST8(word, a) hd_st8(word, a) +#define HD_ST4(word, a) hd_st4(word, a) +#else +#define HD_ST8(word, a) +#define HD_ST4(word, a) +#endif + // ---- Stager: global -> registers (fetch) -> LDS (commit), `chunks` 16-byte chunks (a compile-time // constant at every call, so the row tests fold): lane tid moves chunks tid, tid + THREADS, ...; in a partial last row the // lanes past the end re-load / re-store the unit's last chunk (same value: no per-lane branch). @@ -464,7 +611,11 @@ void st_commit(Stager s, uint dst, uint chunks, uint tid) void load_frags_g(uint base, uint off, out uint2 f[KC]) { [unroll] for (uint i = 0; i < KC / 2; ++i) { +#if SW_MEGA + uint4 v = wg_mem_gc.Load4(base + off + 16u * i); // written by other workgroups of this launch +#else uint4 v = wg_ld128(base, off + 16u * i); +#endif f[2 * i] = v.xy; f[2 * i + 1] = v.zw; } } @@ -475,8 +626,8 @@ void load_frags_g(uint base, uint off, out uint2 f[KC]) #ifndef SW_OUT_F16 #define SW_OUT_F16 0 #endif -#if (SW_SRC != 0 || SW_OUT_F16) && (SW_ADD || SW_ECL > 0) -#error "SW_SRC / SW_OUT_F16: plain blocks only" +#if (SW_SRC != 0 || (SW_OUT_F16 && !SW_HYB)) && (SW_ADD || SW_ECL > 0) +#error "SW_SRC / SW_OUT_F16: plain blocks only (the hybrid C32 decoder: SW_OUT_F16 with SW_ADD, the skip)" #endif #ifndef SW_OUT_COEF12 #define SW_OUT_COEF12 0 @@ -541,9 +692,10 @@ void sw_load_halo_lin(uint base, int pr, int pc, int W, uint lane, uint hh, out f[k].v[4] = wg_h2f_lo(y[k].z); f[k].v[5] = wg_h2f_hi(y[k].z); f[k].v[6] = wg_h2f_lo(y[k].w); f[k].v[7] = wg_h2f_hi(y[k].w); } } -#elif SW_SRC == 2 +#elif SW_SRC == 2 || (SW_HYB && SW_ADD) // LCoarseFp8Src: a member outside the grid takes the mirrored 2 x 2 group (mirror of the group index) and keeps its -// sub-pixel; the byte offset of the lane's half of that pixel +// sub-pixel; the byte offset of the lane's half of that pixel (also the hybrid decoder blocks' expanded map E: the +// table's shared_dec_border "coarse"; their skip map S keeps the fine mirror) uint sw_coarse_src(int pr, int pc, int H, int W, uint hh) { const int r = 2 * sw_mirror(pr >> 1, H >> 1) + (pr & 1), c = 2 * sw_mirror(pc >> 1, W >> 1) + (pc & 1); @@ -749,14 +901,26 @@ void fz_wait_b5(int nwx, int nwin, uint tid) #define SW_ENTRY SW_ENTRY_OVERRIDE #endif +#if SW_MEGA +// the body's global stores (its output map, read by other workgroups of this launch) through the coherent view +#define wg_st128(b, o, v) wg_mem_gc.Store4((b) + (o), (v)) +void sw_item() // one queue item (hyb_bneck.hlsl): the block on windows 8 wg_bid_ .. + 7 of the current block +#else WG_KERNEL(SW_ENTRY) +#endif { +#if !SW_MEGA WG_SETUP(); +#endif const uint tid = wg_lane_id(); const uint lane = tid & 31u, t = lane & 15u, hh = lane >> 4; const int nwx = (W + ox + 3) >> 2, nwy = (H + oy + 3) >> 2, nwin = nwx * nwy; const uint wave_id = (uint)wg_uniform((int)(tid >> 5)); const int win0 = (int)(wg_block_id() * (THREADS / 32) + wave_id); +#if SW_HYB_DUMP + hd_on = win0 < nwin; + hd_base = ((uint)win0 * 32u + lane) * HD_WORDS; +#endif Stager stage; st_fetch(stage, w + L_head(0), HEAD_BYTES / 16, tid); // unit 0 @@ -794,7 +958,7 @@ WG_KERNEL(SW_ENTRY) #elif SW_SRC == 3 v8f xf[KC]; sw_load_halo_lin(x, pr, pc, W, lane, hh, xf); -#elif SW_SRC == 2 +#elif SW_SRC == 2 || (SW_HYB && SW_ADD) load_frags_g(x, sw_coarse_src(pr, pc, H, W, hh), xa); #elif SW_PM // enc1's patch merge of this lane's (mirrored) token: lane half hh holds the 2 x 64 bytes of input row 2 mr + hh @@ -865,8 +1029,25 @@ WG_KERNEL(SW_ENTRY) #endif #if SW_ADD acc[k] = sw_add8(acc[k], sw_unpack_fp8(xb[k])); +#if SW_HYB + acc[k] = hyb_round16(acc[k]); // x = RN16(dec(P) + dec(S)) #endif +#endif + } +#if SW_HYB + // y = the original's norm (FP16 tree); a = FP8(y); the Wp chain starts at RN16(bp + x) + { + const uint g1 = w + L_G1 + hh * (HALF * 4), bp = w + L_BP + hh * 32u; + v8f gp[KC], y[KC]; + [unroll] for (uint k1 = 0; k1 < KC; ++k1) gp[k1] = wg_load8f(g1, 32u * k1); + hyb_norm(acc, gp, hh, y); + [unroll] for (uint kd = 0; kd < KC; ++kd) { HD_ST8(8u * kd, acc[kd]); HD_ST8(8u * (KC + kd), y[kd]); } + [unroll] for (uint k3 = 0; k3 < KC; ++k3) { + af[k3] = hyb_q8(y[k3]); + acc[k3] = hyb_add16(acc[k3], wg_load8f(bp, 64u * k3)); + } } +#else const float r = norm_r(acc); #if SW_NORM_R2 & 1 const float r2 = sw_opaque_copy(r); @@ -889,6 +1070,7 @@ WG_KERNEL(SW_ENTRY) af[k2] = pack_in(d); acc[k2] = sw_add8(acc[k2], b); } +#endif } // Steps 3-7, one head at a time; one staged unit per head (QKV fragments, then the head's slice of Wp). @@ -929,13 +1111,29 @@ WG_KERNEL(SW_ENTRY) accq = wg_wmma_fp8(fq[kc2], af[kc2], accq); accv = wg_wmma_fp8(af[kc2], fv[kc2], accv); } +#if SW_HYB + uf[mt] = hyb_q8_of_chain(accq); + HD_ST8(HD_HEAD + 60u * hd + 8u * mt, hyb_round16(accq)); +#else uf[mt] = pack_in(accq); +#endif vt[mt] = wg_to_f16_ags(accv); +#if SW_HYB_DUMP + HD_ST8(HD_HEAD + 60u * hd + 16u + 8u * mt, hyb_h2f8(vt[mt])); +#endif } // Lane (t, h) holds the logits S(t, s) of 8 keys; the position bias is packed in the same order. #if SW_C != 64 const v8f bias = wg_load8f(biasl, hd * 1024u); #endif +#if SW_HYB + // s = one 32-term FP8 op started at B (the position bias, FP16 values); the original's weights P + v8f s = wg_wmma_fp8(uf[0], uf[0], bias); + s = wg_wmma_fp8(uf[1], uf[1], s); + const uint4 p = hyb_softmax_p(s, hh); + HD_ST8(HD_HEAD + 60u * hd + 32u, hyb_round16(s)); + HD_ST4(HD_HEAD + 60u * hd + 40u, p); +#else v8f s = wg_wmma_fp8(uf[0], uf[0], sw_zero8()); // S = U^T U s = wg_wmma_fp8(uf[1], uf[1], s); // soft cap: c = clamp(S + B, -cap, cap); t = c - c^3 / (3 cap^2) (fused); z = t tau log2(e). Written stage by stage @@ -993,10 +1191,19 @@ WG_KERNEL(SW_ENTRY) [unroll] for (uint va = 0; va < 8; ++va) nz[va] = min(nz[va], 1u); [unroll] for (uint vb = 0; vb < 8; ++vb) po.v[vb] = asfloat(qs[vb] | nz[vb]); const uint4 p = wg_to_f16_ags(po); +#endif #endif // o = A V (rows value channels, columns tokens) uint2 of[2]; +#if SW_HYB + [unroll] for (uint dt = 0; dt < 2; ++dt) { + const v8f pv = wg_wmma_f16(vt[dt], p, sw_zero8()); + of[dt] = hyb_q8_of_chain(pv); + HD_ST8(HD_HEAD + 60u * hd + 44u + 8u * dt, hyb_round16(pv)); + } +#else [unroll] for (uint dt = 0; dt < 2; ++dt) of[dt] = pack_in(wg_wmma_f16(vt[dt], p, sw_zero8())); +#endif // x1 += Wp[:, head hd] q8(o_hd) uint2 fp[KC][2]; [unroll] for (uint m = 0; m < KC; ++m) { @@ -1034,6 +1241,21 @@ WG_KERNEL(SW_ENTRY) // Step 8: m = q8(RMSNorm(x1; gamma2)), then b2 into the residual accumulator. uint2 mf[KC]; +#if SW_HYB + { + // z = RN16(the Wp chain); m = FP8(norm(z; g2')); the W2 chain starts at RN16(b2 + z) + [unroll] for (uint kz = 0; kz < KC; ++kz) acc[kz] = hyb_round16(acc[kz]); + const uint g2 = w + L_G2 + hh * (HALF * 4), b2 = w + L_B2 + hh * 32u; + v8f gp[KC], y[KC]; + [unroll] for (uint k1 = 0; k1 < KC; ++k1) gp[k1] = wg_load8f(g2, 32u * k1); + hyb_norm(acc, gp, hh, y); + [unroll] for (uint kd = 0; kd < KC; ++kd) { HD_ST8(HD_Z + 8u * kd, acc[kd]); HD_ST8(HD_Y2 + 8u * kd, y[kd]); } + [unroll] for (uint k3 = 0; k3 < KC; ++k3) { + mf[k3] = hyb_q8(y[k3]); + acc[k3] = hyb_add16(acc[k3], wg_load8f(b2, 64u * k3)); + } + } +#else { const float r = norm_r(acc); #if SW_NORM_R2 & 2 @@ -1064,9 +1286,14 @@ WG_KERNEL(SW_ENTRY) acc[k] = sw_add8(acc[k], b); } } +#endif // Steps 9-10: x2 = x1 + W2 q8(GELU(W1 m + b1)) + b2, GROUP hidden tiles of 16 at a time. const uint b1 = w + L_B1 + hh * 32u; +#if SW_HYB && SW_HYB_MUT_ORDER == 3 + v8f acc2[KC]; // check mutant 3: the odd tile groups' W2 chunks accumulate here (from 0), added at the end + [unroll] for (uint ma = 0; ma < KC; ++ma) acc2[ma] = sw_zero8(); +#endif #if SW_REV_PHI v8f racc[KC]; [unroll] for (uint rk = 0; rk < KC; ++rk) racc[rk] = sw_perm8(acc[rk], 7u, 7u); @@ -1133,7 +1360,7 @@ WG_KERNEL(SW_ENTRY) #endif [unroll] for (uint g4 = 0; g4 < GROUP; ++g4) [unroll] for (uint m = 0; m < KC; ++m) f2[g4][m] = sw_lfrag_b(fb, g4 * 2u * KC + KC + m); -#if SW_GELU != 0 +#if SW_GELU != 0 && !SW_HYB #if !SW_SAFE #error "the GELU tables give HIP's pack_in bytes (no clamp)" #endif @@ -1156,7 +1383,12 @@ WG_KERNEL(SW_ENTRY) [unroll] for (uint g7 = 0; g7 < GROUP; ++g7) hnext[g7] = hcur[g7]; } uint2 hf[GROUP]; -#if SW_GELU != 0 +#if SW_HYB + [unroll] for (uint gh = 0; gh < GROUP; ++gh) { + hf[gh] = hyb_gelu_q8(hyb_rn16x8(hcur[gh])); + HD_ST8(HD_H + 8u * (u * HPU + j0 + gh), hyb_round16(hcur[gh])); + } +#elif SW_GELU != 0 #if SW_GELU_EARLY if ((SW_GELU_LATE0 >> ((j0 / GROUP) % 2u)) & 1u) gp = sw_gelu_pre(hcur); sw_gelu_fin(hcur, gp, hf); @@ -1186,8 +1418,27 @@ WG_KERNEL(SW_ENTRY) if (j0 == GROUP * (SW_STAGE_LATE - 1) && (SW_MLP_STEP != NMLP || more)) st_fetch(stage, wd + (hf[0].x & sw_zero_bits()) + L_mlp(more ? u + 1u : u), MLP_BYTES / 16, tid); #endif +#if SW_HYB && SW_HYB_MUT_ORDER + // check mutant (never shipped): the first unit's first tile group accumulates its W2 chunks in reverse order + // (same products, same start): inside the f32chain tolerance, but not HIP's bytes (l_hyb_stage_check.py) +#if SW_HYB_MUT_ORDER == 3 + if ((j0 / GROUP) & 1u) { // 3: odd groups into the second accumulator + [unroll] for (uint m5 = 0; m5 < KC; ++m5) + [unroll] for (uint g11 = 0; g11 < GROUP; ++g11) acc2[m5] = wg_wmma_fp8(f2[g11][m5], hf[g11], acc2[m5]); + } else +#endif + if (SW_HYB_MUT_ORDER == 2 || (SW_HYB_MUT_ORDER == 1 && u == 0u && j0 == 0u)) { // 1: first group; 2: every group + [unroll] for (uint m3 = 0; m3 < KC; ++m3) + [unroll] for (uint g9 = 0; g9 < GROUP; ++g9) + acc[m3] = wg_wmma_fp8(f2[GROUP - 1u - g9][m3], hf[GROUP - 1u - g9], acc[m3]); + } else { + [unroll] for (uint m4 = 0; m4 < KC; ++m4) + [unroll] for (uint g10 = 0; g10 < GROUP; ++g10) acc[m4] = wg_wmma_fp8(f2[g10][m4], hf[g10], acc[m4]); + } +#else [unroll] for (uint m2 = 0; m2 < KC; ++m2) [unroll] for (uint g6 = 0; g6 < GROUP; ++g6) acc[m2] = wg_wmma_fp8(f2[g6][m2], hf[g6], acc[m2]); +#endif [unroll] for (uint g8 = 0; g8 < GROUP; ++g8) hcur[g8] = hnext[g8]; #if SW_COMMIT_EARLY if (more && j0 == GROUP * (SW_COMMIT_EARLY - 1)) st_commit(stage, (buf ^ 1u) * MAXU + 8u * SW_MDZ, MLP_BYTES / 16, tid); @@ -1207,6 +1458,18 @@ WG_KERNEL(SW_ENTRY) #if SW_REV_PHI [unroll] for (uint rk2 = 0; rk2 < KC; ++rk2) acc[rk2] = sw_unperm8(racc[rk2], 7u, 7u); #endif +#if SW_HYB && SW_HYB_MUT_ORDER == 3 + [unroll] for (uint mb = 0; mb < KC; ++mb) [unroll] for (uint vb = 0; vb < 8; ++vb) acc[mb].v[vb] = acc[mb].v[vb] + acc2[mb].v[vb]; +#endif +#if SW_HYB && SW_HYB_MUT_ORDER == 4 + // check mutant 4: the W2 chain end moved by one float32 ulp (last mantissa bit flipped) before its RN16: far inside + // the f32chain tolerance, visible in the bytes wherever the value sits on an FP16 rounding tie + [unroll] for (uint mc = 0; mc < KC; ++mc) + [unroll] for (uint vc = 0; vc < 8; ++vc) acc[mc].v[vc] = asfloat(asuint(acc[mc].v[vc]) ^ 1u); +#endif +#if SW_HYB_DUMP + [unroll] for (uint kdo = 0; kdo < KC; ++kdo) HD_ST8(HD_OUT + 8u * kdo, hyb_round16(acc[kdo])); +#endif #if SW_OUT_F16 // Output RN16(x2) as FP16, real window members only (HIP LStoreF16Epi). @@ -1225,7 +1488,12 @@ WG_KERNEL(SW_ENTRY) } return; #endif +#if SW_HYB + // the driver's RNE conversion (the chain-end RN16 of the hybrid helpers; HIP's LStoreF16Epi v_cvt_f16_f32) + [unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16_ags(acc[kf]); +#else [unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16(acc[kf]); +#endif #if SW_OUT_COEF12 // l_coef_pack12 of the map: o16[0] / o16[1] of the hh = 0 lane are the pixel's channels 0..7 / 8..15 (the map's // bytes dst * 64 + 0 .. 31, its lo / hi loads); Model L's 13 values are all among them @@ -1255,7 +1523,11 @@ WG_KERNEL(SW_ENTRY) } else #endif { +#if SW_HYB + [unroll] for (uint k = 0; k < KC; ++k) o[k] = hyb_q8_of_chain(acc[k]); +#else [unroll] for (uint k = 0; k < KC; ++k) o[k] = sw_pack_q8_v2(acc[k]); +#endif } if (dst >= 0 && (SW_XE == 0 || FZ_KEEP_MAPS)) { // xe: dec2.out only read by the expand const uint ob = (uint)dst * SW_C + hh * HALF; diff --git a/clean/dx12/kernels/fused_swin_gen.py b/clean/dx12/kernels/fused_swin_gen.py index 528fbbd3..202bde27 100644 --- a/clean/dx12/kernels/fused_swin_gen.py +++ b/clean/dx12/kernels/fused_swin_gen.py @@ -73,6 +73,9 @@ #if SW_SRC == 3 uint linW1, linW2; // pointers: enc0.Win1, enc0.Win2 (x = the FP8 q halo map) #endif +#if SW_HYB_DUMP + uint dumpp; // pointer: the stage dump (HD_WORDS words per lane of every real window) +#endif WG_ARGS_END""" ARGS_NEW = """#ifndef SW_PM #define SW_PM 0 diff --git a/clean/dx12/kernels/hyb_bneck.hlsl b/clean/dx12/kernels/hyb_bneck.hlsl new file mode 100644 index 00000000..08bf6675 --- /dev/null +++ b/clean/dx12/kernels/hyb_bneck.hlsl @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: MIT +// The HYBRID form's bottleneck in one launch (WG_L_FORM = hybrid, native D3D12): b0 .. b5 as a persistent queue of +// items, bneck.hlsl's schedule (items of 8 windows in block order; one claimed ahead; per-row counters of every block; +// an item waits for the rows of the previous block it reads and overwrites; the bounded wait; the last workgroup resets +// the sync words), with swin_block.hlsl's hybrid C128 block body as the item (SW_MEGA: sw_item(), the same instructions +// as swin_block_c128_hyb on the same values). b5's output equals six swin_block_c128_hyb launches byte for byte; b0 .. +// b4's outputs are not kept (ping-pong in out / tmp, as swin_bneck6_nc). +// Arguments (bneck.hlsl's): x (b0's input, enc2's merged map), out (b5's output), tmp (scratch map), sync (the zeroed +// sync words: [0] queue, [1] finished workgroups, [12] the frame's abort word, [13] zero (the 8-byte frame guard is +// [12 .. 13]), [15] wait timeouts (never reset), [16 + k NR + r] row counters), the six blocks' hybrid weight sections, +// H, W, the even and odd blocks' window offsets, the workgroup count. +// A wait that times out (BNECK_SPIN_LIMIT polls on one row) REFUSES the frame: it adds 1 to [12] and [15], and every +// wait of the launch then stops at once (no item waits on rows that may never come); the launch completes, its maps +// are not valid. The runtime reads [12 .. 13] right after this launch into the frame guard (Device::guard_begin): +// every later launch and copy of the frame, and of every frame after it until the runtime has seen the count [15] +// move, is skipped on the GPU (no output, no history); the runtime then refuses one frame, starts the next one over +// (reset) and clears [12]. The kernel never clears [12]. +// swin_bneck6_hyb the twin (Model L hybrid route) +// swin_bneck6_hyb_scanmut the fused-square scan's mutant (probe=1, outside the manifest: the scan must FAIL on it) +// swin_bneck6_hyb_mut check mutant (probe=1): order mutant 4 (W2 chain end one float32 ulp off); the byte check +// against the six launches must FAIL on it +// swin_bneck6_hyb_tmo timeout control (probe=1): BNECK_SPIN_LIMIT 1 (a wait gives up at its first poll that +// finds a producer not done): the check requires [12] and [15] nonzero after it (and zero +// after the twin) +// @wg-kernel swin_bneck6_hyb co=- block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,u,u,u,u,u,u,u defines=SW_C=128,SW_HEADS=4,SW_MLP=512,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HYB=1,SW_MEGA=1,SW_ENTRY_OVERRIDE=swin_bneck6_hyb +// @wg-kernel swin_bneck6_hyb_scanmut co=- probe=1 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,u,u,u,u,u,u,u defines=SW_C=128,SW_HEADS=4,SW_MLP=512,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HYB=1,SW_MEGA=1,HYB_SCAN_MUTANT=1,SW_ENTRY_OVERRIDE=swin_bneck6_hyb_scanmut +// @wg-kernel swin_bneck6_hyb_mut co=- probe=1 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,u,u,u,u,u,u,u defines=SW_C=128,SW_HEADS=4,SW_MLP=512,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HYB=1,SW_MEGA=1,SW_HYB_MUT_ORDER=4,SW_ENTRY_OVERRIDE=swin_bneck6_hyb_mut +// @wg-kernel swin_bneck6_hyb_tmo co=- probe=1 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,u,u,u,u,u,u,u defines=SW_C=128,SW_HEADS=4,SW_MLP=512,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HYB=1,SW_MEGA=1,BNECK_SPIN_LIMIT=1u,SW_ENTRY_OVERRIDE=swin_bneck6_hyb_tmo +#include "swin_block.hlsl" + +#define BQ_SYNC_HEAD 16 +#define BQ_ABORT 48u // byte offset of sync word 12: the frame's abort word +#define BQ_TIMEOUTS 60u // byte offset of sync word 15: wait timeouts (never reset) +#ifndef BNECK_SPIN_LIMIT +#define BNECK_SPIN_LIMIT (1u << 18) // polls per row, roughly 0.1 s (bneck.hlsl's bounded wait) +#endif + +// The claim slot: LDS element 0 of the block body's staging buffers (sw_lds_q, every byte in use by the C128 body), +// written and read only while the body is not running (between barriers), and read before the next item's body starts +// (the barrier ahead of sw_item). + +struct BqGrid { int oy, ox, nwx, nwy, nwin, items; }; +BqGrid bq_grid(int h, int wd, int gx, int gy) +{ + BqGrid g; + g.ox = gx; + g.oy = gy; + g.nwx = (wd + gx + 3) >> 2; + g.nwy = (h + gy + 3) >> 2; + g.nwin = g.nwx * g.nwy; + g.items = (g.nwin + 7) / 8; + return g; +} + +uint bq_weights(int k) +{ + return k == 0 ? a_w0 : k == 1 ? a_w1 : k == 2 ? a_w2 : k == 3 ? a_w3 : k == 4 ? a_w4 : a_w5; +} + +// Current value of a sync word (an atomic, so that the spin loop cannot be hoisted). +uint bq_sync_read(uint off) +{ + uint v; + wg_mem_gc.InterlockedOr(off, 0u, v); + return v; +} + +WG_KERNEL(SW_ENTRY) +{ + WG_SETUP(); + const uint tid = wg_lane_id(); + const BqGrid ge = bq_grid(H, W, a_oxe, a_oye), go = bq_grid(H, W, a_oxo, a_oyo); + const int total = 3 * (ge.items + go.items); + const int NR = ge.nwy > go.nwy ? ge.nwy : go.nwy; + const uint rows = a_sync + BQ_SYNC_HEAD * 4u; + + if (tid == 0u) sw_lds_q[0] = (uint64_t)wg_atomic_add(a_sync, 0u, 1u); + wg_barrier_lds(); + int cur = WaveReadLaneFirst((int)(uint)sw_lds_q[0]); + [loop] while (cur < total) { + int k = 0, local = cur; + [loop] while (k < 5 && local >= ((k & 1) ? go.items : ge.items)) { + local -= (k & 1) ? go.items : ge.items; + ++k; + } + BqGrid g = ge; + if (k & 1) g = go; + // the item's block (sw_item's statics): even blocks write tmp, odd blocks out (b5 -> out) + // every per-item value through WaveReadLaneFirst: the driver formed the selects (v_cndmask) and the row + // division in VGPRs and kept them across the body (200 VGPRs, against the block kernel's 192) + x = WaveReadLaneFirst(k == 0 ? a_x : ((k & 1) ? a_tmp : a_out)); + out_p = WaveReadLaneFirst((k & 1) ? a_out : a_tmp); + w = WaveReadLaneFirst(bq_weights(k)); + ox = WaveReadLaneFirst(g.ox); + oy = WaveReadLaneFirst(g.oy); + wg_bid_ = WaveReadLaneFirst((uint)local); + const int wf = local * 8, wl = WaveReadLaneFirst((wf + 8 < g.nwin ? wf + 8 : g.nwin) - 1); + const int r0 = WaveReadLaneFirst((int)((uint)wf / (uint)g.nwx)), + r1 = WaveReadLaneFirst((int)((uint)wl / (uint)g.nwx)); // wf, wl >= 0, nwx > 0 + // the next item, claimed now (one ahead, as bneck.hlsl) + uint after = (uint)total; + if (tid == 0u) after = wg_atomic_add(a_sync, 0u, 1u); + after = WaveReadLaneFirst(after); // wave 0's lane 0 (only it stores it): a scalar across the body, not a VGPR + if (k > 0 && tid == 0u) { + // rows of block k - 1 this item depends on (writers of its input rows, readers of the rows it overwrites): + // bneck.hlsl's range, mirror included + BqGrid p = go; + if (k & 1) p = ge; + const int a = 4 * r0 - g.oy, b = 4 * r1 - g.oy + 3; + const int lo = a < 0 ? 0 : (b >= H && 2 * (H - 1) - b < a ? 2 * (H - 1) - b : a); + const int mm = b > -a ? b : -a; + const int hi = mm < H - 1 ? mm : H - 1; + int q0 = (lo + p.oy) >> 2, q1 = (hi + p.oy) >> 2; + if (lo <= 3) q0 = 0; + if (hi >= H - 4 || q1 > p.nwy - 1) q1 = p.nwy - 1; + const uint rc = rows + (uint)((k - 1) * NR) * 4u; + // bounded wait: a launch whose workgroups stop being scheduled still completes. A wait of + // BNECK_SPIN_LIMIT polls on one row refuses the frame (the abort word; the frame guard skips the rest of + // it); once the abort word is set (by this or another workgroup), no item waits any more (checked every + // 64th poll of a wait): the remaining items run on what is there, into maps nothing reads. + bool stop = false; + [loop] for (int q = q0; q <= q1 && !stop; ++q) { + uint lim = 0; + [loop] while (bq_sync_read(rc + (uint)q * 4u) < (uint)p.nwx) { + ++lim; + if (lim == BNECK_SPIN_LIMIT) { + wg_atomic_add(a_sync, BQ_ABORT, 1u); + wg_atomic_add(a_sync, BQ_TIMEOUTS, 1u); + stop = true; + break; + } + if ((lim & 63u) == 0u && bq_sync_read(a_sync + BQ_ABORT) != 0u) { + stop = true; + break; + } + } + } + } + wg_barrier(); + sw_item(); + wg_barrier(); // every wave's stores are complete (device scope) before the counters move; the LDS is free + if (tid == 0u) { + sw_lds_q[0] = (uint64_t)after; + const uint rc = rows + (uint)(k * NR) * 4u; + [loop] for (int rr = r0; rr <= r1; ++rr) { + const int s0 = rr * g.nwx > wf ? rr * g.nwx : wf; + const int s1 = (rr + 1) * g.nwx - 1 < wl ? (rr + 1) * g.nwx - 1 : wl; + wg_atomic_add(rc, (uint)rr * 4u, (uint)(s1 - s0 + 1)); + } + } + wg_barrier_lds(); + cur = WaveReadLaneFirst((int)(uint)sw_lds_q[0]); + } + if (tid == 0u) { + wg_fence_device(); + if (wg_atomic_add(a_sync, 4u, 1u) == (uint)a_groups - 1u) { // last workgroup: every other one has finished + wg_st32_gc(a_sync, 0u, 0u); + for (int j = 0; j < 6 * NR; ++j) wg_st32_gc(rows, (uint)j * 4u, 0u); + wg_st32_gc(a_sync, 4u, 0u); + } + } +} diff --git a/clean/dx12/kernels/hyb_gemm.hlsl b/clean/dx12/kernels/hyb_gemm.hlsl new file mode 100644 index 00000000..ad980dea --- /dev/null +++ b/clean/dx12/kernels/hyb_gemm.hlsl @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: MIT +// The HYBRID form's patch merges and expands (WG_L_FORM = hybrid; the unfused shared stages, hyb_swin.hlsl's blocks): +// the M-form kernels' WMMA chains (swin_gemm.hlsl: the bias as the first accumulator, the K groups in order, the same +// fragments of the same packed section) with the hybrid FP8 points (oracle run_stage_q7): +// patch_merge_c64_hyb / _c96_hyb merged = FP8(clamp(RN16(bm + Wm z), +-6.28125)) (hyb_merge_q8, NaN kept) +// expand_c96_hyb / _c128_hyb E = FP8(RN16(be + We y)) (hyb_q8_of_chain) +// The weights are the stage's merge / expand section of the hybrid blob (packing.pack_merge / pack_expand layout with +// the original's values: FP16 biases stored as float32, plain FP8 matrices; nothing folded). Arguments and grids as the +// M-form kernels: in, out, w (the section), H, W (merge: the input map's size; expand: the low-resolution size), grid +// ceil(tokens / (8 waves x NT x 16)) of 256 lanes (tokens: merged H/2 x W/2, expand Hl x Wl). +// Staging (performance only, the bytes do not depend on it): the merge stages the fragments of SW_PM_SPLIT output-tile +// groups one group at a time, the expand those of one sub-pixel k at a time when SW_EX_SPLIT (expand_c128: the four +// sub-pixels' 48 KB do not fit D3D12's 32 KB). +// HIP twins: the same names in swin_hyb.co (clean/kernels/swin_hyb.hip: swin.hip's patch_merge / expand with HYB). +// +// @wg-kernel patch_merge_c64_hyb co=- block=256 wave=32 args=p,p,p,u,u defines=SW_PM_C=64,SW_PM_CO=96,SW_PM_SPLIT=1 +// @wg-kernel patch_merge_c96_hyb co=- block=256 wave=32 args=p,p,p,u,u defines=SW_PM_C=96,SW_PM_CO=128,SW_PM_SPLIT=4 +// @wg-kernel expand_c96_hyb co=- block=256 wave=32 args=p,p,p,u,u defines=SW_EX_CL=96,SW_EX_C=64,SW_EX_SPLIT=0 +// @wg-kernel expand_c128_hyb co=- block=256 wave=32 args=p,p,p,u,u defines=SW_EX_CL=128,SW_EX_C=96,SW_EX_SPLIT=1 +// *_scanmut the ISA rules' mutants (probe=1, outside the manifest; build_dx12.py requires one per model hybrid +// kernel): no RN16 before the FP8 pack (HYB_SCAN_MUTANT=1), rule 2 must fail (no norm tree here: rule 1 +// has nothing to find) +// @wg-kernel patch_merge_c64_hyb_scanmut co=- probe=1 block=256 wave=32 args=p,p,p,u,u defines=SW_PM_C=64,SW_PM_CO=96,SW_PM_SPLIT=1,HYB_SCAN_MUTANT=1,SW_ENTRY_OVERRIDE=patch_merge_c64_hyb_scanmut +// @wg-kernel patch_merge_c96_hyb_scanmut co=- probe=1 block=256 wave=32 args=p,p,p,u,u defines=SW_PM_C=96,SW_PM_CO=128,SW_PM_SPLIT=4,HYB_SCAN_MUTANT=1,SW_ENTRY_OVERRIDE=patch_merge_c96_hyb_scanmut +// @wg-kernel expand_c96_hyb_scanmut co=- probe=1 block=256 wave=32 args=p,p,p,u,u defines=SW_EX_CL=96,SW_EX_C=64,SW_EX_SPLIT=0,HYB_SCAN_MUTANT=1,SW_ENTRY_OVERRIDE=expand_c96_hyb_scanmut +// @wg-kernel expand_c128_hyb_scanmut co=- probe=1 block=256 wave=32 args=p,p,p,u,u defines=SW_EX_CL=128,SW_EX_C=96,SW_EX_SPLIT=1,HYB_SCAN_MUTANT=1,SW_ENTRY_OVERRIDE=expand_c128_hyb_scanmut +#include "wg_platform.hlsli" + +WG_ARGS_BEGIN + uint in_p, out_p, w; // pointers: input map, output map, weights (bias first, then the fragments) + int H, W; // patch merge: the input map's size; expand: the low-resolution size (Hl, Wl) +WG_ARGS_END + +#define NT 2 +#define THREADS WG_BLOCK +#define SW_CLAMP_SKIP 0 // every FP8 point through the hybrid packs (no clamp-skip shortcut) + +#if defined(SW_PM_C) +#define KZ (SW_PM_C / 4) // 4C / 16 input groups +#define MO (SW_PM_CO / 16) // output tiles +#define MOP (MO / SW_PM_SPLIT) // output tiles per pass +#define WM (4 * SW_PM_CO) // byte offset of the fragments (after the bias) +#define SW_LDS_BYTES (MOP * KZ * 256) +#else +#define KC (SW_EX_CL / 16) +#define G (SW_EX_C / 16) +#define WE (16 * SW_EX_C) +#if SW_EX_SPLIT +#define SW_LDS_BYTES (G * KC * 256) // one sub-pixel's fragments +#else +#define SW_LDS_BYTES (4 * G * KC * 256) +#endif +#endif +#include "swin_common.hlsli" +#include "speed_swin.hlsli" +#include "wg_gelu_lut.hlsli" +#include "wg_hyb.hlsli" + +// `bytes` (a multiple of THREADS x 16) from global memory to LDS byte 0, then a barrier. +void copy_to_lds(uint src, uint bytes, uint tid) +{ + [loop] for (uint i = tid; i < bytes / 16u; i += THREADS) sw_lds_st128(i * 16u, wg_ld128(src, i * 16u)); + wg_barrier_lds(); +} + +#if defined(SW_PM_C) +#if SW_PM_C == 64 +#define SW_ENTRY patch_merge_c64_hyb +#else +#define SW_ENTRY patch_merge_c96_hyb +#endif +#ifdef SW_ENTRY_OVERRIDE +#undef SW_ENTRY +#define SW_ENTRY SW_ENTRY_OVERRIDE +#endif +// merged(i, j) = FP8(clamp(RN16(bm + Wm concat(TL, TR, BL, BR)))); lane half h holds the 2C bytes of input row 2i + h. +WG_KERNEL(SW_ENTRY) +{ + WG_SETUP(); + const uint tid = wg_lane_id(); + const uint lane = tid & 31u, t = lane & 15u, hh = lane >> 4; + const int Wm = W >> 1, ntok = (H >> 1) * Wm; + const int tok0 = (int)(wg_block_id() * (THREADS / 32) + (tid >> 5)) * NT * 16 + (int)t; + uint2 zf[NT][KZ]; + int tok[NT]; + [unroll] for (int i = 0; i < NT; ++i) { + tok[i] = tok0 + 16 * i; + const int tk = tok[i] < ntok ? tok[i] : ntok - 1; + const int r = tk / Wm, c = tk - r * Wm; + const uint off = (uint)((2 * r + (int)hh) * W + 2 * c) * SW_PM_C; + [unroll] for (uint q = 0; q < KZ / 2; ++q) { + uint4 v = wg_ld128(in_p, off + 16u * q); + zf[i][2 * q] = v.xy; zf[i][2 * q + 1] = v.zw; + } + } + const uint bm = w + hh * 32u; + [unroll] for (uint pass = 0; pass < SW_PM_SPLIT; ++pass) { + if (pass > 0u) wg_barrier_lds(); // every wave is done with the previous pass's fragments + copy_to_lds(w + WM + pass * SW_LDS_BYTES, SW_LDS_BYTES, tid); + const uint mop = MOP + sw_zero(); + [loop] for (uint mp = 0; mp < mop; ++mp) { + const uint mt = pass * MOP + mp; + uint2 f[KZ]; + [unroll] for (uint g = 0; g < KZ; ++g) f[g] = sw_lfrag_b(sw_lane_base_m(0u, lane), mp * KZ + g); + const v8f b = wg_load8f(bm, mt * 64u); + v8f acc[NT]; + [unroll] for (int i2 = 0; i2 < NT; ++i2) acc[i2] = b; + [unroll] for (uint g2 = 0; g2 < KZ; ++g2) + [unroll] for (int i3 = 0; i3 < NT; ++i3) acc[i3] = wg_wmma_fp8(f[g2], zf[i3][g2], acc[i3]); + uint2 res[NT]; + [unroll] for (int i4 = 0; i4 < NT; ++i4) res[i4] = hyb_merge_q8(acc[i4]); + [unroll] for (int i5 = 0; i5 < NT; ++i5) + if (tok[i5] < ntok) wg_st64(out_p, (uint)tok[i5] * SW_PM_CO + hh * (SW_PM_CO / 2) + 8u * mt, res[i5]); + } + } +} +#else +#if SW_EX_CL == 96 +#define SW_ENTRY expand_c96_hyb +#else +#define SW_ENTRY expand_c128_hyb +#endif +#ifdef SW_ENTRY_OVERRIDE +#undef SW_ENTRY +#define SW_ENTRY SW_ENTRY_OVERRIDE +#endif +// E = FP8(RN16(be + We y)) per low-resolution token, as an FP8 map E [2 Hl][2 Wl][C] (sub-pixel k at +// (2r + k / 2, 2c + k % 2)). +WG_KERNEL(SW_ENTRY) +{ + WG_SETUP(); + const uint tid = wg_lane_id(); + const uint lane = tid & 31u, t = lane & 15u, hh = lane >> 4; + const int Hl = H, Wl = W, ntok = Hl * Wl; + const int tok0 = (int)(wg_block_id() * (THREADS / 32) + (tid >> 5)) * NT * 16 + (int)t; + uint2 yf[NT][KC]; + int tok[NT]; + [unroll] for (int i = 0; i < NT; ++i) { + tok[i] = tok0 + 16 * i; + const int tk = tok[i] < ntok ? tok[i] : ntok - 1; + const uint off = (uint)tk * SW_EX_CL + hh * (SW_EX_CL / 2); + [unroll] for (uint q = 0; q < KC / 2; ++q) { + uint4 v = wg_ld128(in_p, off + 16u * q); + yf[i][2 * q] = v.xy; yf[i][2 * q + 1] = v.zw; + } + } +#if !SW_EX_SPLIT + copy_to_lds(w + WE, SW_LDS_BYTES, tid); +#endif + uint ob[NT]; + [unroll] for (int i6 = 0; i6 < NT; ++i6) { + const int r = tok[i6] / Wl, c = tok[i6] - r * Wl; + ob[i6] = (uint)((2 * r) * (2 * Wl) + 2 * c) * SW_EX_C + hh * (SW_EX_C / 2); + } + const uint be = w + hh * 32u; + [loop] for (uint k = 0; k < 4; ++k) { +#if SW_EX_SPLIT + // sub-pixel k's fragments (tiles k G .. k G + G - 1, contiguous): wait until every wave is done with k - 1's + if (k > 0u) wg_barrier_lds(); + copy_to_lds(w + WE + k * SW_LDS_BYTES, SW_LDS_BYTES, tid); + const uint fk = 0u; // fragment index base inside the staged part +#else + const uint fk = k * G * KC; +#endif + uint2 res[NT][G]; + [unroll] for (uint g = 0; g < G; ++g) { + const uint mt = k * G + g; + uint2 f[KC]; + [unroll] for (uint kc = 0; kc < KC; ++kc) f[kc] = sw_lfrag_b(sw_lane_base_m(0u, lane), fk + g * KC + kc); + const v8f b = wg_load8f(be, mt * 64u); + v8f acc[NT]; + [unroll] for (int i2 = 0; i2 < NT; ++i2) acc[i2] = b; + [unroll] for (uint kc2 = 0; kc2 < KC; ++kc2) + [unroll] for (int i3 = 0; i3 < NT; ++i3) acc[i3] = wg_wmma_fp8(f[kc2], yf[i3][kc2], acc[i3]); + [unroll] for (int i4 = 0; i4 < NT; ++i4) res[i4][g] = hyb_q8_of_chain(acc[i4]); + } + [unroll] for (int i5 = 0; i5 < NT; ++i5) { + if (tok[i5] < ntok) { + const uint dst = ob[i5] + (uint)((int)(k >> 1) * (2 * Wl) + (int)(k & 1u)) * SW_EX_C; + [unroll] for (uint q2 = 0; q2 < G / 2; ++q2) + wg_st128(out_p, dst + 16u * q2, uint4(res[i5][2 * q2], res[i5][2 * q2 + 1])); + } + } + } +} +#endif diff --git a/clean/dx12/kernels/hyb_swin.hlsl b/clean/dx12/kernels/hyb_swin.hlsl new file mode 100644 index 00000000..ef4df9bc --- /dev/null +++ b/clean/dx12/kernels/hyb_swin.hlsl @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: MIT +// The HYBRID form's Swin blocks of the shared stages (WG_L_FORM = hybrid; SW_HYB in swin_block.hlsl): swin_block.hlsl +// instantiated with SW_HYB=1 at Model M's stage shapes, one window per wave in 256-lane workgroups (HIP's shape; grid = +// ceil(windows / 8)), the plain schedule (swin_block.hlsl sets the hybrid knob set), unfused: the merges and expands are +// their own kernels (hyb_gemm.hlsl). +// swin_block_c64_hyb enc1 swin_block_c64_add_hyb dec1 (x = E, the expanded map; xadd = enc1's output) +// swin_block_c96_hyb enc2 swin_block_c96_add_hyb dec2 (x = E; xadd = enc2's output) +// swin_block_c128_hyb b0 .. b5 (one launch per block) +// Arguments as the M-form blocks: x, xadd (null for the encoder / bottleneck blocks), out, w (the stage's hybrid blob, +// clean/l/runner/l_hyb_pack.py and its C++ packer), H, W, ox, oy. The decoder blocks read E with the coarse mirror +// (the table's shared_dec_border) and the skip map with the fine mirror; the member input is RN16(E + S). +// HIP twins: the same names in swin_hyb.co (clean/kernels/swin_hyb.hip). +// +// @wg-kernel swin_block_c64_hyb co=- block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=64,SW_HEADS=2,SW_MLP=256,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HYB=1,SW_ENTRY_OVERRIDE=swin_block_c64_hyb +// @wg-kernel swin_block_c64_add_hyb co=- block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=64,SW_HEADS=2,SW_MLP=256,SW_ADD=1,SW_SAFE=1,SW_ECL=0,SW_HYB=1,SW_ENTRY_OVERRIDE=swin_block_c64_add_hyb +// @wg-kernel swin_block_c96_hyb co=- block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HYB=1,SW_ENTRY_OVERRIDE=swin_block_c96_hyb +// @wg-kernel swin_block_c96_add_hyb co=- block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=1,SW_SAFE=1,SW_ECL=0,SW_HYB=1,SW_ENTRY_OVERRIDE=swin_block_c96_add_hyb +// @wg-kernel swin_block_c128_hyb co=- block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=128,SW_HEADS=4,SW_MLP=512,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HYB=1,SW_ENTRY_OVERRIDE=swin_block_c128_hyb +// swin_block_c64_hyb_scanmut / _c128_hyb_scanmut the fused-square scan's mutants (probe=1, outside the manifest; see +// l_swin.hlsl l_swin_block_c32_enc_hyb_scanmut): the select-free norm trees KC 4 and 8 +// @wg-kernel swin_block_c64_hyb_scanmut co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=64,SW_HEADS=2,SW_MLP=256,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HYB=1,HYB_SCAN_MUTANT=1,SW_ENTRY_OVERRIDE=swin_block_c64_hyb_scanmut +// @wg-kernel swin_block_c64_add_hyb_scanmut co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=64,SW_HEADS=2,SW_MLP=256,SW_ADD=1,SW_SAFE=1,SW_ECL=0,SW_HYB=1,HYB_SCAN_MUTANT=1,SW_ENTRY_OVERRIDE=swin_block_c64_add_hyb_scanmut +// swin_block_c96_hyb_scanmut / _c96_add_hyb_scanmut the same for KC 6, as SUBSTITUTE mutants: the KC 6 tree keeps its +// per-lane selects, so no square ever feeds an add (RGA: every square's first consumer is a +// v_cndmask_b16 or v_permlanex16_b32) and the plain barrier-free build forms no multiply-add; +// with HYB_SCAN_MUTANT, hyb_block.hlsli also bypasses the selects so that a square does feed +// an add, and the scan must find it (wrong values by design; never in a model build) +// @wg-kernel swin_block_c96_hyb_scanmut co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HYB=1,HYB_SCAN_MUTANT=1,SW_ENTRY_OVERRIDE=swin_block_c96_hyb_scanmut +// @wg-kernel swin_block_c96_add_hyb_scanmut co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=1,SW_SAFE=1,SW_ECL=0,SW_HYB=1,HYB_SCAN_MUTANT=1,SW_ENTRY_OVERRIDE=swin_block_c96_add_hyb_scanmut +// @wg-kernel swin_block_c128_hyb_scanmut co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=128,SW_HEADS=4,SW_MLP=512,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HYB=1,HYB_SCAN_MUTANT=1,SW_ENTRY_OVERRIDE=swin_block_c128_hyb_scanmut +#include "swin_block.hlsl" diff --git a/clean/dx12/kernels/l_hyb_probe.hlsl b/clean/dx12/kernels/l_hyb_probe.hlsl new file mode 100644 index 00000000..231e7ea5 --- /dev/null +++ b/clean/dx12/kernels/l_hyb_probe.hlsl @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: MIT +// Exhaustive probe of the hybrid block form's element-wise helpers (hlsl/wg_hyb.hlsli), not used by the runtime: +// every thread takes 8 consecutive FP16 bit patterns i = 8 t .. 8 t + 7 (t = global thread, 8192 threads = all 65,536) +// and writes, at byte offsets of `dst` (the expected bytes come from clean/l/reference/hyb_tables.py and oracle.py; +// driver: clean/l/runner/l_hyb_probe_check.py): +// [0, 65536) hyb_gelu_q8 (h = i) FP8 bytes +// [65536, 196608) hyb_softmax_e8 (s = i) e as FP16 bits (u16) +// [196608, 327680) hyb_rsq16 (S' bits = i) FP16 bits (u16; compared where S' >= 2^-13) +// [327680, 458752) hyb_rcp16 (sum bits = i) FP16 bits (u16; compared where both are normal) +// [458752, 720896) hyb_merge_clamp (x = i as FP16 value) float32 bits +// and, for the chain-end double rounding, nf float32 values at `src` (nf a multiple of 8): hyb_q8_of_chain bytes at +// [720896, 720896 + nf). +// @wg-kernel l_hyb_probe co=- probe=1 block=256 wave=32 args=p,p,u +#include "wg_platform.hlsli" +WG_ARGS_BEGIN + uint src, dst; + uint nf; +WG_ARGS_END +#include "wg_gelu_lut.hlsli" +#include "wg_hyb.hlsli" + +WG_KERNEL(l_hyb_probe) +{ + WG_SETUP(); + const uint t = wg_block_id() * 256u + wg_lane_id(); + const uint i0 = 8u * t; + uint4 h; + h.x = i0 | ((i0 + 1u) << 16); + h.y = (i0 + 2u) | ((i0 + 3u) << 16); + h.z = (i0 + 4u) | ((i0 + 5u) << 16); + h.w = (i0 + 6u) | ((i0 + 7u) << 16); + // GELU16 -> FP8 + const uint2 g = hyb_gelu_q8(h); + wg_st32(dst, i0, g.x); + wg_st32(dst, i0 + 4u, g.y); + // softmax weight of the 8 scores + const v8f s = hyb_h2f8(h); + const uint4 e = hyb_rn16x8(hyb_softmax_e8(s)); + wg_st32(dst, 65536u + 2u * i0, e.x); + wg_st32(dst, 65536u + 2u * i0 + 4u, e.y); + wg_st32(dst, 65536u + 2u * i0 + 8u, e.z); + wg_st32(dst, 65536u + 2u * i0 + 12u, e.w); + // rsqrt / rcp tables (the 16-bit results packed in pairs) and the merge clamp + [unroll] for (uint j = 0; j < 8; j += 2) { + const uint a = i0 + j, b = i0 + j + 1u; + wg_st32(dst, 196608u + 2u * a, wg_f2h_bits(hyb_rsq16(a)) | (wg_f2h_bits(hyb_rsq16(b)) << 16)); + wg_st32(dst, 327680u + 2u * a, wg_f2h_bits(hyb_rcp16(a)) | (wg_f2h_bits(hyb_rcp16(b)) << 16)); + } + [unroll] for (uint k = 0; k < 8; ++k) wg_st32(dst, 458752u + 4u * (i0 + k), asuint(hyb_merge_clamp(s.v[k]))); + // chain-end double rounding on the given float32 values (8 per thread; the conversion runs in wave-uniform control + // flow on a clamped index, only the stores depend on the range) + const uint ic = min(i0, nf - 8u); + v8f x; + [unroll] for (uint m = 0; m < 8; ++m) x.v[m] = wg_ldf(src, 4u * (ic + m)); + const uint2 q = hyb_q8_of_chain(x); + if (i0 < nf) { + wg_st32(dst, 720896u + i0, q.x); + wg_st32(dst, 720896u + i0 + 4u, q.y); + } +} diff --git a/clean/dx12/kernels/l_post.hlsl b/clean/dx12/kernels/l_post.hlsl index d214e775..993e4da1 100644 --- a/clean/dx12/kernels/l_post.hlsl +++ b/clean/dx12/kernels/l_post.hlsl @@ -34,10 +34,23 @@ // @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1 // @wg-kernel l_post_recon_ldr co=l_post.co entry=l_post_recon_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_LDR=1 // @wg-kernel l_post_recon_p12_ldr co=l_post.co entry=l_post_recon_p12_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_LDR=1 +// +// LP_C12 = 1 (Q11 part C / 0.6.5 hybrid_full: runtime WG_L_POST=c12, and WG_L_FORM=hybrid on native D3D12; port of +// l_post__lc12.co = l_post.hip with WG_LP_C1 = 1, WG_CR_C2 = 1, tag "lc12"): the gather exponent in the machine-code form +// eps = fma(h(u v), C, fma(h(v v), B, h(h(u u) A))) (C1) and the Catmull-Rom weights' fused forms (C2: f2 = f f, +// f3 = f f2, w0 = fused(-(f + f3), 1/2, f2), w1 = fused(f3, 1.5, -RN(2.5 f2)) + 1, w3 = RN(RN(f3 - f2) 1/2), +// w2 = ((1 - w0) - w1) - w3). The default entries above are unchanged (LP_C12 = 0). +// @wg-kernel lc12/l_post_recon co=l_post__lc12.co entry=l_post_recon block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_C12=1 +// @wg-kernel lc12/l_post_recon_p12 co=l_post__lc12.co entry=l_post_recon_p12 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_C12=1 +// @wg-kernel lc12/l_post_recon_ldr co=l_post__lc12.co entry=l_post_recon_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_LDR=1,LP_C12=1 +// @wg-kernel lc12/l_post_recon_p12_ldr co=l_post__lc12.co entry=l_post_recon_p12_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_LDR=1,LP_C12=1 #include "wg_platform.hlsli" #ifndef LP_P12 #define LP_P12 0 #endif +#ifndef LP_C12 +#define LP_C12 0 +#endif #ifndef LP_LDR #define LP_LDR 0 #endif @@ -251,6 +264,11 @@ LGather gather_zero() float tap_weight(uint t[kTexDw], half uu, half vv, half uv) { const half kA = h_bits(t[4] >> 16), kB = h_bits(t[5]), kC = h_bits(t[5] >> 16); +#if LP_C12 + const half ta = mulh(uu, kA); // (#231's product: opqh(uu kA), or mad(uu, kA, -0) with LP_FMANZ) + const half s = mad(vv, kB, ta); // C1: the row term fused into the sum + const half eps = mad(uv, kC, s); +#else #if LP_MUT == 1 const half ta = mulh(uu, kB), tb = mulh(vv, kB); // mutant 1: wrong factor #else @@ -258,6 +276,7 @@ float tap_weight(uint t[kTexDw], half uu, half vv, half uv) #endif const half s = tb + ta; const half eps = mad(uv, kC, s); +#endif const float w = exp2((float)opqh(eps)); return w >= kMinNormal ? w : 0.0f; } @@ -275,6 +294,14 @@ void gather_acc(inout LGather g, uint t[kTexDw], float w, uint hwb) float4 cr_weights(float f) { float4 w; +#if LP_C12 + const float f2 = opq(f * f), f3 = opq(f * f2); + w.x = mad(-(f + f3), 0.5f, f2); + w.y = mad(f3, 1.5f, -opq(2.5f * f2)) + 1.0f; + w.w = opq(opq(f3 - f2) * 0.5f); + w.z = ((1.0f - w.x) - w.y) - w.w; + return w; +#endif w.x = f * (-0.5f + f * (1.0f - 0.5f * f)); w.y = 1.0f + f * f * (-2.5f + 1.5f * f); w.z = f * (0.5f + f * (2.0f - 1.5f * f)); diff --git a/clean/dx12/kernels/l_q7.hlsl b/clean/dx12/kernels/l_q7.hlsl new file mode 100644 index 00000000..32cfb5d0 --- /dev/null +++ b/clean/dx12/kernels/l_q7.hlsl @@ -0,0 +1,474 @@ +// SPDX-License-Identifier: MIT +// D3D12 ports of Model L's FP16-form width-32 blocks (clean/l/kernels/l_q7.hip, l_q7.co; issue #89): the same +// arithmetic step for step, so the same bytes (clean/l/runner/l_q7_check.py compares both back ends and the oracle). +// +// Porting notes (clean/dx12/README.md "Arithmetic mode", "Driver exceptions"): +// - every FP16 rounding is wg_rn16 / wg_f2h (software, nearest even: HIP's (_Float16) x); no half arithmetic; +// - every product that is rounded or summed is exact in float32 (FP8 x FP8, FP16 x FP16), so a fusion of a product +// into the following addition by the driver cannot change a result; +// - no FP32 denormal occurs: the smallest nonzero values are 2^-18 (FP8 products), 2^-48 (FP16 products) and the +// errors of sums of multiples of those; +// - the window's rows live in groupshared memory and are read into local arrays with constant indices; the outer +// loops' trip counts carry an opaque zero (wg_grid_ >> 31) so that the driver keeps them as loops; +// - no byte stores: the FP8 output is packed four to a dword and written as two 8-byte stores per lane; +// - Q7F-2 (S2a): the FP8 ops on the matrix instruction (l_q7_op8.hlsli, wave32), the bounded joins select-stored, a +// failed output marked in a bitmask and recomputed by op8 in a loop after the phase's matrix calls. +// +// @wg-kernel l_q7_block_c32_enc co=l_q7.co block=32 wave=32 args=p,p,p,p,p,u,u,u,u +// @wg-kernel l_q7_block_c32_dec co=l_q7.co block=32 wave=32 args=p,p,p,p,p,u,u,u,u +#include "wg_platform.hlsli" + +WG_ARGS_BEGIN + uint x, skip, out_, w, tab; + int H, W, ox, oy; +WG_ARGS_END + +#define Q7_C 32 +#define Q7_T 16 +// q7 blob offsets (clean/l/runner/l_pack.py pack_q7_block for C 32, 2 heads, MLP 128), bytes +#define OFF_G1 0u +#define OFF_G2 64u +#define OFF_BP 128u +#define OFF_B1 192u +#define OFF_B2 448u +#define OFF_B 512u +#define OFF_WQK 1536u +#define OFF_WV 3584u +#define OFF_WP 5632u +#define OFF_W1 7680u +#define OFF_W2 11776u +// the same matrices as FP16 (Q7F-1: exact copies, read without an FP8 decode) +#define OFF_WQK16 15872u +#define OFF_WV16 19968u +#define OFF_WP16 24064u +#define OFF_W1_16 28160u +#define OFF_W2_16 36352u + +// FP8 activations (a8, u, o8, m8, g8) are packed codes, 4 per dword (byte i = element i), token-major with the +// channels contiguous: 8 consecutive codes of a token (a uint2) are one operand fragment of the FP8 matrix instruction +// (wg_wmma_fp8), as the blob's row-major FP8 rows are for the weights (S2a: the hardware sums drop in +// without a format change). +groupshared float g_x[Q7_T][Q7_C]; +groupshared uint g_a8[Q7_T][Q7_C / 4]; +groupshared uint g_u[2][Q7_T][8]; +groupshared float g_v[2][Q7_T][32]; +groupshared float g_pw[2][Q7_T][Q7_T]; +groupshared uint g_o8[Q7_T][16]; +groupshared float g_z[Q7_T][Q7_C]; // the Wp chain, after norm 2 the MLP accumulators (each lane its own 16) +groupshared uint g_m8[Q7_T][Q7_C / 4]; +groupshared uint g_g8[Q7_T][8]; +groupshared float g_r[Q7_T]; +// S2a (Q7F-2): max code of the values each lane wrote, [token][writer half] (a chunk's amax = the max of its writers'), +// and sum |u| per head / token / writer half (the scores' wsum) +groupshared uint g_ma[Q7_T][2]; +groupshared uint g_mu[2][Q7_T][2]; +groupshared uint g_mo[Q7_T][2]; +groupshared uint g_mm[Q7_T][2]; +groupshared uint g_mg[Q7_T][2]; +groupshared float g_su[2][Q7_T][2]; +// sum |w| tables (chunk-major [K / 32][rows] float32) +#define WS_WQK 44544u +#define WS_WV 44800u +#define WS_WP 45056u +#define WS_W1 45312u +#define WS_W2 45824u + +uint q7_zero() { return wg_grid_ >> 31; } // 0, not provable by the compiler + +#include "l_q7_common.hlsli" +#include "l_q7_op8.hlsli" +float cd8(uint w4, uint j) { return fp8v(w4 >> (8u * j)); } // element j of a packed dword +float ldh(uint off) { return from16(wg_ld16(w, off)); } // an FP16 value of the blob +// 32 FP8 weights of a blob row (byte offset a multiple of 4) +void load_row8(uint off, out float wv[32]) +{ + [unroll] for (uint i = 0; i < 8; ++i) { + uint u = wg_ld32(w, off + 4u * i); + [unroll] for (uint j = 0; j < 4; ++j) wv[4u * i + j] = fp8v(u >> (8u * j)); + } +} + +// r = RN16(1 / sqrt(S')) of one token by the table: the even / odd lane tree (l_q7.hip norm_r). +float norm_r(float xv[32]) +{ + float q[32]; + [unroll] for (uint c = 0; c < 32; ++c) q[c] = mul16(xv[c], xv[c]); + float hv[2]; + [unroll] for (uint odd = 0; odd < 2; ++odd) { + float e[4]; + [unroll] for (uint t = 0; t < 4; ++t) { + uint c0 = 2u * t + odd; + e[t] = add16(add16(q[c0], q[c0 + 16u]), add16(q[c0 + 8u], q[c0 + 24u])); + } + hv[odd] = add16(add16(e[0], e[1]), add16(e[2], e[3])); + } + float s = add16(add16(hv[0], hv[1]), asfloat(0x39000000u)); // + 2^-13 + return from16(wg_ld16(tab, 2u * bits16(s))); +} + +// ---- S2a (Q7F-2) helpers: the matrix-instruction sums are joined branch-free (select stores); an output whose bound +// fails is marked in a bitmask and recomputed exactly by op8 in a separate loop after the phase's matrix calls (the +// driver wants the calls in wave-uniform, branch-free surroundings, wg_ags.hlsli) +float score_e(float s) // the scores' exp polynomial (l_q7.hip) +{ + const float cs = 0.017181396484375f, lim = 0.55615234375f; // 0x2466, 0x3873 + float uu = min(max(mul16(s, cs), -lim), lim); + float tt = fma16(uu, -uu, 0.92724609375f); // 0x3B6B + float p = fma16(uu, tt, 1.375f); // 0x3D80 + return from16((bits16(p) & 0x3ffu) << 5); +} +void ws8(uint off, out float ws[8]) // 8 consecutive float32 of the blob (32-byte aligned) +{ + uint4 a = wg_ld128(w, off), b = wg_ld128(w, off + 16u); + ws[0] = asfloat(a.x); ws[1] = asfloat(a.y); ws[2] = asfloat(a.z); ws[3] = asfloat(a.w); + ws[4] = asfloat(b.x); ws[5] = asfloat(b.y); ws[6] = asfloat(b.z); ws[7] = asfloat(b.w); +} +uint pack4(uint c0, uint c1, uint c2, uint c3) { return c0 | (c1 << 8) | (c2 << 16) | (c3 << 24); } +uint setb(uint word, uint i, uint code) { return (word & ~(0xffu << (8u * i))) | ((code & 0xffu) << (8u * i)); } +uint max16(uint4 q) { return q7_maxcode4(q7_maxcode4(q7_maxcode4(q7_maxcode4(0u, q.x), q.y), q.z), q.w); } +float sum16(uint4 q) // sum |code values| of 16 codes (exact) +{ + uint a[4] = { q.x, q.y, q.z, q.w }; + float s = 0.0f; + [unroll] for (uint i = 0; i < 16; ++i) s = s + abs(cd8(a[i >> 2], i & 3u)); + return s; +} +// the lane's 16 outputs of a 32-row pair of tiles: output i is row 16 (i >> 3) + 8 hf + (i & 7) +uint orow(uint i, uint hf) { return 16u * (i >> 3) + 8u * hf + (i & 7u); } + +// The block on one window (g_x already holds the 16 members). Out: the FP16 block output of member t, channel +// orow(i, hf) in g_z[t][orow(i, hf)]. +void block(uint t, uint hf) +{ + const uint Z = q7_zero(); + float xa[32], wv[32]; + // norm 1 -> FP8 a8 + if (hf == 0u) { + [unroll] for (uint c = 0; c < 32; ++c) xa[c] = g_x[t][c]; + g_r[t] = norm_r(xa); + } + wg_barrier_lds(); + { + uint m = 0u; + [unroll] for (uint i = 0; i < 4; ++i) { + uint pk = 0u; + [unroll] for (uint jj = 0; jj < 4; ++jj) { + uint c = 16u * hf + 4u * i + jj; + pk |= wg_f32_to_e4m3(mul16(g_x[t][c], mul16(g_r[t], ldh(OFF_G1 + 2u * c)))) << (8u * jj); + } + g_a8[t][4u * hf + i] = pk; + m = q7_maxcode4(m, pk); + } + g_ma[t][hf] = m; + } + wg_barrier_lds(); + // u = FP8(Wqk a8), v = Wv a8: 64 rows each = 4 tiles (tiles 2h, 2h + 1 = head h), one chunk + { + float amx = q7_amax(g_ma[t][0], g_ma[t][1]); + uint2 b0 = uint2(g_a8[t][2u * hf], g_a8[t][2u * hf + 1u]), b1 = uint2(g_a8[t][4u + 2u * hf], g_a8[t][5u + 2u * hf]); + uint fq = 0u, fv = 0u; // fallback bits 8 tile + j + [unroll] for (uint tile = 0; tile < 4; ++tile) { + uint r0 = 16u * tile, ra = OFF_WQK + (r0 + t) * 32u + 8u * hf; + v8f S = q7_tile_sum(wg_ld64(w, ra), b0, wg_ld64(w, ra + 16u), b1); + float ws[8]; + ws8(WS_WQK + 4u * (r0 + 8u * hf), ws); + uint cd[8]; + [unroll] for (uint j = 0; j < 8; ++j) { + cd[j] = wg_f32_to_e4m3(q7_join(S.v[j], 0.0f)); + fq |= q7_bounded(ws[j], amx) ? 0u : (1u << (8u * tile + j)); + } + uint dw = ((r0 & 31u) + 8u * hf) >> 2; + g_u[tile >> 1][t][dw] = pack4(cd[0], cd[1], cd[2], cd[3]); + g_u[tile >> 1][t][dw + 1u] = pack4(cd[4], cd[5], cd[6], cd[7]); + } + [unroll] for (uint tile2 = 0; tile2 < 4; ++tile2) { + uint r0 = 16u * tile2, ra = OFF_WV + (r0 + t) * 32u + 8u * hf; + v8f S = q7_tile_sum(wg_ld64(w, ra), b0, wg_ld64(w, ra + 16u), b1); + float ws[8]; + ws8(WS_WV + 4u * (r0 + 8u * hf), ws); + [unroll] for (uint j = 0; j < 8; ++j) { + uint r = r0 + 8u * hf + j; + g_v[r >> 5][t][r & 31u] = q7_join(S.v[j], 0.0f); + fv |= q7_bounded(ws[j], amx) ? 0u : (1u << (8u * tile2 + j)); + } + } + if ((fq | fv) != 0u) { + [unroll] for (uint c1 = 0; c1 < 32; ++c1) xa[c1] = cd8(g_a8[t][c1 >> 2], c1 & 3u); + while (fq != 0u) { + uint b = firstbitlow(fq); + fq &= fq - 1u; + uint r = 16u * (b >> 3) + 8u * hf + (b & 7u); + q7_load16(w, OFF_WQK16 + 2u * r * 32u, wv); + uint code = wg_f32_to_e4m3(op8(0.0f, xa, wv)); + g_u[r >> 5][t][(r & 31u) >> 2] = setb(g_u[r >> 5][t][(r & 31u) >> 2], r & 3u, code); + } + while (fv != 0u) { + uint b = firstbitlow(fv); + fv &= fv - 1u; + uint r = 16u * (b >> 3) + 8u * hf + (b & 7u); + q7_load16(w, OFF_WV16 + 2u * r * 32u, wv); + g_v[r >> 5][t][r & 31u] = op8(0.0f, xa, wv); + } + } + [unroll] for (uint h = 0; h < 2; ++h) { // this lane's 16 final codes of head h + uint4 q = uint4(g_u[h][t][2u * hf], g_u[h][t][2u * hf + 1u], g_u[h][t][4u + 2u * hf], g_u[h][t][5u + 2u * hf]); + g_mu[h][t][hf] = max16(q); + g_su[h][t][hf] = sum16(q); + } + } + wg_barrier_lds(); + // scores s[h][t][j] = RN16(B + u_t . u_j): per head one tile, A = keys (row j), B = queries (token t), the same + // fragments; the lane's outputs: query t, keys 8 hf + jj; wsum = sum |u_j|, amax = max |u_t|; e into g_pw + { + uint fs = 0u; + [unroll] for (uint h = 0; h < 2; ++h) { + uint2 u0 = uint2(g_u[h][t][2u * hf], g_u[h][t][2u * hf + 1u]); + uint2 u1 = uint2(g_u[h][t][4u + 2u * hf], g_u[h][t][5u + 2u * hf]); + v8f S = q7_tile_sum(u0, u0, u1, u1); + float amx = q7_amax(g_mu[h][t][0], g_mu[h][t][1]); + [unroll] for (uint jj = 0; jj < 8; ++jj) { + uint j = 8u * hf + jj; + float bij = ldh(OFF_B + 2u * ((h * 16u + t) * 16u + j)); + g_pw[h][t][j] = score_e(q7_join(S.v[jj], bij)); + fs |= q7_bounded(g_su[h][j][0] + g_su[h][j][1], amx) ? 0u : (1u << (8u * h + jj)); + } + } + while (fs != 0u) { + uint b = firstbitlow(fs); + fs &= fs - 1u; + uint h = b >> 3, j = 8u * hf + (b & 7u); + [unroll] for (uint k = 0; k < 32; ++k) { + xa[k] = cd8(g_u[h][t][k >> 2], k & 3u); + wv[k] = cd8(g_u[h][j][k >> 2], k & 3u); + } + g_pw[h][t][j] = score_e(op8(ldh(OFF_B + 2u * ((h * 16u + t) * 16u + j)), xa, wv)); + } + } + wg_barrier_lds(); + // the attention weights of head hf, query t (e in g_pw's own row) + { + float e[16]; + [unroll] for (uint j2 = 0; j2 < 16; ++j2) e[j2] = g_pw[hf][t][j2]; + float a[2]; + [unroll] for (uint lo = 0; lo < 2; ++lo) { + float pr[4]; + [unroll] for (uint k3 = 0; k3 < 4; ++k3) pr[k3] = add16(e[lo + 2u * k3], e[lo + 8u + 2u * k3]); + a[lo] = add16(add16(pr[0], pr[1]), add16(pr[2], pr[3])); + } + float wsum = from16(wg_ld16(tab, 2u * (65536u + bits16(add16(a[0], a[1]))))); + [unroll] for (uint j3 = 0; j3 < 16; ++j3) g_pw[hf][t][j3] = mul16(e[j3], wsum); + } + wg_barrier_lds(); + // o = RN16(P v) of head hf (16 FP16 x FP16 products, summed exactly), FP8 + float pw[16]; + [unroll] for (uint j4 = 0; j4 < 16; ++j4) pw[j4] = g_pw[hf][t][j4]; + uint mo = 0u; + [loop] for (uint q4 = 0; q4 < 8u + Z; ++q4) { + uint pk = 0u; + [unroll] for (uint jj = 0; jj < 4; ++jj) { + uint d2 = 4u * q4 + jj; + float s = 0.0f, c = 0.0f; + [unroll] for (uint j5 = 0; j5 < 16; ++j5) { + float e; + two_sum(s, pw[j5] * g_v[hf][j5][d2], s, e); + c = c + e; + } + float z, tl; + two_sum(s, c, z, tl); + pk |= wg_f32_to_e4m3(rn16_pair(z, tl)) << (8u * jj); + } + g_o8[t][8u * hf + q4] = pk; + mo = q7_maxcode4(mo, pk); + } + g_mo[t][hf] = mo; // chunk hf of o8 is this lane's alone + wg_barrier_lds(); + // z = Wp chain (head 0, then head 1) from RN16(bp + x): 32 rows = 2 tiles, K 64 = 2 chunks; acc in g_z (the + // bounded joins are select-stored, a failed output keeps its start for the fallback) + [unroll] for (uint i2 = 0; i2 < 16; ++i2) { + uint c = orow(i2, hf); + g_z[t][c] = add16(ldh(OFF_BP + 2u * c), g_x[t][c]); + } + [unroll] for (uint hd = 0; hd < 2; ++hd) { + float amx = q7_amax(g_mo[t][hd], 0u); + uint2 o0 = uint2(g_o8[t][8u * hd + 2u * hf], g_o8[t][8u * hd + 2u * hf + 1u]); + uint2 o1 = uint2(g_o8[t][8u * hd + 4u + 2u * hf], g_o8[t][8u * hd + 5u + 2u * hf]); + uint fp = 0u; + [unroll] for (uint tile = 0; tile < 2; ++tile) { + uint r0 = 16u * tile, ra = OFF_WP + (r0 + t) * 64u + 32u * hd + 8u * hf; + v8f S = q7_tile_sum(wg_ld64(w, ra), o0, wg_ld64(w, ra + 16u), o1); + float ws[8]; + ws8(WS_WP + 4u * (32u * hd + r0 + 8u * hf), ws); + [unroll] for (uint j = 0; j < 8; ++j) { + uint c = r0 + 8u * hf + j; + float a0 = g_z[t][c]; + bool bd = q7_bounded(ws[j], amx); + g_z[t][c] = bd ? q7_join(S.v[j], a0) : a0; + fp |= bd ? 0u : (1u << (8u * tile + j)); + } + } + if (fp != 0u) { + [unroll] for (uint k4 = 0; k4 < 32; ++k4) xa[k4] = cd8(g_o8[t][8u * hd + (k4 >> 2)], k4 & 3u); + while (fp != 0u) { + uint b = firstbitlow(fp); + fp &= fp - 1u; + uint c = 16u * (b >> 3) + 8u * hf + (b & 7u); + q7_load16(w, OFF_WP16 + 2u * (c * 64u + 32u * hd), wv); + g_z[t][c] = op8(g_z[t][c], xa, wv); + } + } + } + wg_barrier_lds(); + // norm 2 -> FP8 m8; then the MLP accumulators RN16(b2 + z) in place of z (each lane its own channels) + if (hf == 0u) { + [unroll] for (uint c3 = 0; c3 < 32; ++c3) xa[c3] = g_z[t][c3]; + g_r[t] = norm_r(xa); + } + wg_barrier_lds(); + { + uint m = 0u; + [unroll] for (uint i3 = 0; i3 < 4; ++i3) { + uint pk = 0u; + [unroll] for (uint jj = 0; jj < 4; ++jj) { + uint c = 16u * hf + 4u * i3 + jj; + pk |= wg_f32_to_e4m3(mul16(g_z[t][c], mul16(g_r[t], ldh(OFF_G2 + 2u * c)))) << (8u * jj); + } + g_m8[t][4u * hf + i3] = pk; + m = q7_maxcode4(m, pk); + } + g_mm[t][hf] = m; + } + wg_barrier_lds(); // every lane has read z for its m8 before acc overwrites it + [unroll] for (uint i6 = 0; i6 < 16; ++i6) { + uint c = orow(i6, hf); + g_z[t][c] = add16(ldh(OFF_B2 + 2u * c), g_z[t][c]); + } + // MLP: 4 hidden slices of 32, ascending: W1 slice (2 tiles, K 32) -> GELU -> FP8 g8, then W2's chunk (2 tiles) + float amx_m = q7_amax(g_mm[t][0], g_mm[t][1]); + uint2 m0 = uint2(g_m8[t][2u * hf], g_m8[t][2u * hf + 1u]), m1 = uint2(g_m8[t][4u + 2u * hf], g_m8[t][5u + 2u * hf]); + [loop] for (uint sl = 0; sl < 4u + Z; ++sl) { + uint f1 = 0u; + [unroll] for (uint tile = 0; tile < 2; ++tile) { + uint r0 = 32u * sl + 16u * tile, ra = OFF_W1 + (r0 + t) * 32u + 8u * hf; + v8f S = q7_tile_sum(wg_ld64(w, ra), m0, wg_ld64(w, ra + 16u), m1); + float ws[8]; + ws8(WS_W1 + 4u * (r0 + 8u * hf), ws); + uint cd[8]; + [unroll] for (uint j = 0; j < 8; ++j) { + uint jr = r0 + 8u * hf + j; + cd[j] = wg_f32_to_e4m3(gelu16(q7_join(S.v[j], ldh(OFF_B1 + 2u * jr)))); + f1 |= q7_bounded(ws[j], amx_m) ? 0u : (1u << (8u * tile + j)); + } + uint dw = (16u * tile + 8u * hf) >> 2; + g_g8[t][dw] = pack4(cd[0], cd[1], cd[2], cd[3]); + g_g8[t][dw + 1u] = pack4(cd[4], cd[5], cd[6], cd[7]); + } + if (f1 != 0u) { + [unroll] for (uint c4 = 0; c4 < 32; ++c4) xa[c4] = cd8(g_m8[t][c4 >> 2], c4 & 3u); + while (f1 != 0u) { + uint b = firstbitlow(f1); + f1 &= f1 - 1u; + uint jl = 16u * (b >> 3) + 8u * hf + (b & 7u), jr = 32u * sl + jl; + q7_load16(w, OFF_W1_16 + 2u * jr * 32u, wv); + uint code = wg_f32_to_e4m3(gelu16(op8(ldh(OFF_B1 + 2u * jr), xa, wv))); + g_g8[t][jl >> 2] = setb(g_g8[t][jl >> 2], jl & 3u, code); + } + } + g_mg[t][hf] = max16(uint4(g_g8[t][2u * hf], g_g8[t][2u * hf + 1u], g_g8[t][4u + 2u * hf], g_g8[t][5u + 2u * hf])); + wg_barrier_lds(); + float amx_g = q7_amax(g_mg[t][0], g_mg[t][1]); + uint2 q0 = uint2(g_g8[t][2u * hf], g_g8[t][2u * hf + 1u]), q1 = uint2(g_g8[t][4u + 2u * hf], g_g8[t][5u + 2u * hf]); + uint f2 = 0u; + [unroll] for (uint tile2 = 0; tile2 < 2; ++tile2) { + uint r0 = 16u * tile2, ra = OFF_W2 + (r0 + t) * 128u + 32u * sl + 8u * hf; + v8f S = q7_tile_sum(wg_ld64(w, ra), q0, wg_ld64(w, ra + 16u), q1); + float ws[8]; + ws8(WS_W2 + 4u * (32u * sl + r0 + 8u * hf), ws); + [unroll] for (uint j = 0; j < 8; ++j) { + uint c = r0 + 8u * hf + j; + float a0 = g_z[t][c]; + bool bd = q7_bounded(ws[j], amx_g); + g_z[t][c] = bd ? q7_join(S.v[j], a0) : a0; + f2 |= bd ? 0u : (1u << (8u * tile2 + j)); + } + } + if (f2 != 0u) { + float gv[32]; + [unroll] for (uint k5 = 0; k5 < 32; ++k5) gv[k5] = cd8(g_g8[t][k5 >> 2], k5 & 3u); + while (f2 != 0u) { + uint b = firstbitlow(f2); + f2 &= f2 - 1u; + uint c = 16u * (b >> 3) + 8u * hf + (b & 7u); + q7_load16(w, OFF_W2_16 + 2u * (c * 128u + 32u * sl), wv); + g_z[t][c] = op8(g_z[t][c], gv, wv); + } + } + wg_barrier_lds(); + } +} + +void window_member(uint t, out int pr, out int pc) +{ + int nwx = (W + ox + 3) / 4; + int win = (int)wg_block_id(); + int wy = win / nwx, wx = win % nwx; + pr = 4 * wy - oy + (int)(t >> 2); + pc = 4 * wx - ox + (int)(t & 3u); +} + +// Encoder block: x = the FP16 halo map [H + 4][W + 4][32], out = the FP8 map [H][W][32] (l_q7.hip). +WG_KERNEL(l_q7_block_c32_enc) +{ + WG_SETUP(); + uint lane = wg_lane_id(), t = lane & 15u, hf = lane >> 4; + int pr, pc; + window_member(t, pr, pc); + uint src = ((uint)(pr + 2) * (uint)(W + 4) + (uint)(pc + 2)) * 64u + 32u * hf; + [unroll] for (uint i = 0; i < 8; ++i) { + uint v = wg_ld32(x, src + 4u * i); + g_x[t][16u * hf + 2u * i] = wg_h2f_lo(v); + g_x[t][16u * hf + 2u * i + 1u] = wg_h2f_hi(v); + } + wg_barrier_lds(); + block(t, hf); + if (pr >= 0 && pr < H && pc >= 0 && pc < W) { + uint ob[4]; // channels 8 hf ... + 7, then 16 + 8 hf ... + 7 (orow) + [unroll] for (uint q = 0; q < 4; ++q) { + uint c = 16u * (q >> 1) + 8u * hf + 4u * (q & 1u); + ob[q] = out8(g_z[t][c]) | (out8(g_z[t][c + 1u]) << 8) | (out8(g_z[t][c + 2u]) << 16) | + (out8(g_z[t][c + 3u]) << 24); + } + uint ob0 = ((uint)pr * (uint)W + (uint)pc) * 32u + 8u * hf; + wg_st64(out_, ob0, uint2(ob[0], ob[1])); + wg_st64(out_, ob0 + 16u, uint2(ob[2], ob[3])); + } +} + +// Decoder block: x = the FP8 projection map [H][W][32] (coarse mirror, sub-pixel kept), skip = the encoder block's FP8 +// map [H][W][32] (fine mirror), the member's input RN16(P + S); out = the FP16 map [H][W][32] (l_q7.hip). +WG_KERNEL(l_q7_block_c32_dec) +{ + WG_SETUP(); + uint lane = wg_lane_id(), t = lane & 15u, hf = lane >> 4; + int pr, pc; + window_member(t, pr, pc); + int r = 2 * q7_mirror(pr >> 1, H >> 1) + (pr & 1), c = 2 * q7_mirror(pc >> 1, W >> 1) + (pc & 1); + int sr = q7_mirror(pr, H), sc = q7_mirror(pc, W); + uint4 pp = wg_ld128(x, ((uint)r * (uint)W + (uint)c) * 32u + 16u * hf); + uint4 sp = wg_ld128(skip, ((uint)sr * (uint)W + (uint)sc) * 32u + 16u * hf); + uint pa[4] = { pp.x, pp.y, pp.z, pp.w }; + uint sa[4] = { sp.x, sp.y, sp.z, sp.w }; + [unroll] for (uint i = 0; i < 16; ++i) + g_x[t][16u * hf + i] = h16(fp8v(pa[i >> 2] >> (8u * (i & 3u))) + fp8v(sa[i >> 2] >> (8u * (i & 3u)))); + wg_barrier_lds(); + block(t, hf); + if (pr >= 0 && pr < H && pc >= 0 && pc < W) { + uint ob[8]; // channels 8 hf ... + 7, then 16 + 8 hf ... + 7 (orow) + [unroll] for (uint q = 0; q < 8; ++q) { + uint c = 16u * (q >> 2) + 8u * hf + 2u * (q & 3u); + ob[q] = out16(g_z[t][c]) | (out16(g_z[t][c + 1u]) << 16); + } + uint o = (((uint)pr * (uint)W + (uint)pc) * 32u + 8u * hf) * 2u; + wg_st128(out_, o, uint4(ob[0], ob[1], ob[2], ob[3])); + wg_st128(out_, o + 32u, uint4(ob[4], ob[5], ob[6], ob[7])); + } +} diff --git a/clean/dx12/kernels/l_q7_s2a_probe.hlsl b/clean/dx12/kernels/l_q7_s2a_probe.hlsl new file mode 100644 index 00000000..fe7fc61c --- /dev/null +++ b/clean/dx12/kernels/l_q7_s2a_probe.hlsl @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of the S2a primitive's measurement probe (clean/l/kernels/l_q7_s2a_probe.hip; the primitive in +// clean/dx12/hlsl/l_q7_op8.hlsli), not used by the runtime: built only when named (probe=1), compared with the HIP +// probe and the exact model by clean/l/runner/l_q7_s2a_check.py. Mutant ports: _m1 ... _m4 (l_q7_s2a_probe.hip). +// +// @wg-kernel l_q7_s2a_probe co=l_q7_s2a_probe.co block=32 wave=32 args=p,p,p,p,p,p,p probe=1 +// @wg-kernel l_q7_s2a_probe_m1 co=l_q7_s2a_probe.co entry=l_q7_s2a_probe block=32 wave=32 args=p,p,p,p,p,p,p probe=1 defines=Q7_S2A_MUTANT=1 +// @wg-kernel l_q7_s2a_probe_m2 co=l_q7_s2a_probe.co entry=l_q7_s2a_probe block=32 wave=32 args=p,p,p,p,p,p,p probe=1 defines=Q7_S2A_MUTANT=2 +// @wg-kernel l_q7_s2a_probe_m3 co=l_q7_s2a_probe.co entry=l_q7_s2a_probe block=32 wave=32 args=p,p,p,p,p,p,p probe=1 defines=Q7_S2A_MUTANT=3 +// @wg-kernel l_q7_s2a_probe_m4 co=l_q7_s2a_probe.co entry=l_q7_s2a_probe block=32 wave=32 args=p,p,p,p,p,p,p probe=1 defines=Q7_S2A_MUTANT=4 +#include "wg_platform.hlsli" + +WG_ARGS_BEGIN + uint a, w, w16, st, wsum, out_, path; +WG_ARGS_END + +groupshared uint g_m[32]; + +#include "l_q7_common.hlsli" +#include "l_q7_op8.hlsli" + +WG_KERNEL(l_q7_s2a_probe) +{ + WG_SETUP(); + uint lane = wg_lane_id(), t = lane & 15u, hf = lane >> 4; + uint base = 16u * wg_block_id(); + uint i = base + t; + uint2 a0 = wg_ld64(w, i * 32u + 8u * hf), a1 = wg_ld64(w, i * 32u + 16u + 8u * hf); + uint2 b0 = wg_ld64(a, i * 32u + 8u * hf), b1 = wg_ld64(a, i * 32u + 16u + 8u * hf); + uint m = q7_maxcode4(0u, b0.x); + m = q7_maxcode4(m, b0.y); + m = q7_maxcode4(m, b1.x); + m = q7_maxcode4(m, b1.y); + g_m[lane] = m; + wg_barrier_lds(); +#if Q7_S2A_MUTANT == 3 + float amax = q7_amax(g_m[lane], g_m[lane]); +#else + float amax = q7_amax(g_m[t], g_m[t + 16u]); +#endif + bool diag = (t >> 3) == hf; + float sti = wg_ldf(st, 4u * i); +#if Q7_S2A_MUTANT == 4 + v8f c, zero; + [unroll] for (uint j = 0; j < 8; ++j) { + c.v[j] = (8u * hf + j == t) ? sti : 0.0f; + zero.v[j] = 0.0f; + } + v8f lo = wg_wmma_fp8(a0, b0, c); + v8f hi = wg_wmma_fp8(a1, b1, zero); + v8f S; + [unroll] for (uint j2 = 0; j2 < 8; ++j2) S.v[j2] = lo.v[j2] + hi.v[j2]; + float start = 0.0f; +#else + v8f S = q7_tile_sum(a0, b0, a1, b1); + float start = sti; +#endif + float s = S.v[0]; + [unroll] for (uint j3 = 1; j3 < 8; ++j3) s = (t & 7u) == j3 ? S.v[j3] : s; + if (diag) { + float ws = wg_ldf(wsum, 4u * i); + float y; + if (q7_bounded(ws, amax)) { + y = q7_join(s, start); + } else { + float av[32], wv[32]; + q7_load8(a, i * 32u, av); + q7_load16(w16, i * 64u, wv); + y = op8(sti, av, wv); + } + wg_stf(out_, 4u * i, y); + wg_st32(path, 4u * i, q7_bounded(ws, amax) ? 1u : 0u); + } +} diff --git a/clean/dx12/kernels/l_q7w.hlsl b/clean/dx12/kernels/l_q7w.hlsl new file mode 100644 index 00000000..ba26bd83 --- /dev/null +++ b/clean/dx12/kernels/l_q7w.hlsl @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: MIT +// D3D12 ports of Model L's shared-stage kernels in the FP16 form (clean/l/kernels/l_q7w.hip, l_q7w.co; issue #89): +// the same arithmetic step for step (l_q7_common.hlsli; porting notes in l_q7.hlsl), so the same bytes +// (clean/l/runner/l_q7w_check.py --d3d12). One source, one build per kernel: Q7_KIND 0 = block (FP8 map in, mirror +// border), 1 = decoder block (E coarse mirror + S fine mirror), 2 = merge, 3 = expansion. Groupshared FP16 values are +// 16-bit words and FP8 values packed codes (4 per dword), so that the width-128 block fits in 32 KB. +// +// @wg-kernel l_q7_block_c64 co=l_q7w.co block=32 wave=32 args=p,p,p,p,p,u,u,u,u defines=Q7_KIND=0,Q7_C=64,Q7_HEADS=2,Q7_ENTRY=l_q7_block_c64 +// @wg-kernel l_q7_block_c96 co=l_q7w.co block=32 wave=32 args=p,p,p,p,p,u,u,u,u defines=Q7_KIND=0,Q7_C=96,Q7_HEADS=4,Q7_ENTRY=l_q7_block_c96 +// @wg-kernel l_q7_block_c128 co=l_q7w.co block=32 wave=32 args=p,p,p,p,p,u,u,u,u defines=Q7_KIND=0,Q7_C=128,Q7_HEADS=4,Q7_ENTRY=l_q7_block_c128 +// @wg-kernel l_q7_block_c64_dec co=l_q7w.co block=32 wave=32 args=p,p,p,p,p,u,u,u,u defines=Q7_KIND=1,Q7_C=64,Q7_HEADS=2,Q7_ENTRY=l_q7_block_c64_dec +// @wg-kernel l_q7_block_c96_dec co=l_q7w.co block=32 wave=32 args=p,p,p,p,p,u,u,u,u defines=Q7_KIND=1,Q7_C=96,Q7_HEADS=4,Q7_ENTRY=l_q7_block_c96_dec +// @wg-kernel l_q7_merge_c64 co=l_q7w.co block=32 wave=32 args=p,p,p,u,u defines=Q7_KIND=2,Q7_C=64,Q7_COUT=96,Q7_ENTRY=l_q7_merge_c64 +// @wg-kernel l_q7_merge_c96 co=l_q7w.co block=32 wave=32 args=p,p,p,u,u defines=Q7_KIND=2,Q7_C=96,Q7_COUT=128,Q7_ENTRY=l_q7_merge_c96 +// @wg-kernel l_q7_expand_c128 co=l_q7w.co block=32 wave=32 args=p,p,p,u,u defines=Q7_KIND=3,Q7_CL=128,Q7_C=96,Q7_ENTRY=l_q7_expand_c128 +// @wg-kernel l_q7_expand_c96 co=l_q7w.co block=32 wave=32 args=p,p,p,u,u defines=Q7_KIND=3,Q7_CL=96,Q7_C=64,Q7_ENTRY=l_q7_expand_c96 +#include "wg_platform.hlsli" + +#if Q7_KIND <= 1 +WG_ARGS_BEGIN + uint x, skip, out_, w, tab; + int H, W, ox, oy; +WG_ARGS_END +#else +WG_ARGS_BEGIN + uint in_, out_, w; + int H, W; +WG_ARGS_END +#endif + +#include "l_q7_common.hlsli" +#include "l_q7_op8.hlsli" + +uint q7_zero() { return wg_grid_ >> 31; } +float ldh(uint off) { return from16(wg_ld16(w, off)); } + +#if Q7_KIND <= 1 +// ---- block -------------------------------------------------------------------------------------------------------- +#define C Q7_C +#define HEADS Q7_HEADS +#define MLP (4 * C) +#define CH (C / 2) +#define HPL (HEADS / 2) +#define NC (C / 32) +// blob offsets (l_pack.py q7_block_offsets) +#define O_G1 0u +#define O_G2 (2u * C) +#define O_BP (4u * C) +#define O_B1 (6u * C) +#define O_B2 (6u * C + 2u * MLP) +#define O_B (8u * C + 2u * MLP) +#define O_WQK (O_B + 2u * HEADS * 256u) +#define O_WV (O_WQK + HEADS * 32u * C) +#define O_WP (O_WV + HEADS * 32u * C) +#define O_W1 (O_WP + C * 32u * HEADS) +#define O_W2 (O_W1 + MLP * C) +// the same matrices as FP16 (Q7F-1) +#define O_END8 (O_W2 + C * MLP) +#define O_WQK16 O_END8 +#define O_WV16 (O_WQK16 + 2u * HEADS * 32u * C) +#define O_WP16 (O_WV16 + 2u * HEADS * 32u * C) +#define O_W1_16 (O_WP16 + 2u * C * 32u * HEADS) +#define O_W2_16 (O_W1_16 + 2u * MLP * C) + +groupshared uint16_t g_x[16][C]; +groupshared uint16_t g_z[16][C]; +groupshared uint16_t g_v[HEADS][16][32]; +groupshared uint16_t g_pw[HEADS][16][16]; +// FP8 activations: packed codes, 4 per dword (byte i = element i), token-major with the channels contiguous, so 8 +// consecutive codes of a token are one FP8 matrix-instruction operand fragment (S2a; l_q7.hlsl) +groupshared uint g_a8[16][C / 4]; +groupshared uint g_u[HEADS][16][8]; +groupshared uint g_o8[16][8 * HEADS]; +groupshared uint g_g8[16][8]; +groupshared float g_r[16]; + +float cd8(uint w4, uint j) { return fp8v(w4 >> (8u * j)); } +float gx(uint t, uint c) { return from16(g_x[t][c]); } +float gz(uint t, uint c) { return from16(g_z[t][c]); } + +// the Q8 B3 tree, one half (odd) of lane tt's tile channels 8 j + 2 tt + odd +float lane_half(float q[C], uint tt, uint odd) +{ + float g[2]; + [unroll] for (uint grp = 0; grp < 2; ++grp) { + float pr[4]; + [unroll] for (uint i = 0; i < C / 32; ++i) + pr[i] = add16(q[8u * (4u * i + grp) + 2u * tt + odd], q[8u * (4u * i + 2u + grp) + 2u * tt + odd]); +#if C == 32 + g[grp] = pr[0]; +#elif C == 64 + g[grp] = add16(pr[0], pr[1]); +#elif C == 96 + g[grp] = add16(add16(pr[0], pr[1]), pr[2]); +#else + g[grp] = add16(add16(pr[0], pr[1]), add16(pr[2], pr[3])); +#endif + } + return add16(g[0], g[1]); +} + +float norm_r(float xv[C]) +{ + float q[C]; + [unroll] for (uint c = 0; c < C; ++c) q[c] = mul16(xv[c], xv[c]); + float hv[2]; + [unroll] for (uint odd = 0; odd < 2; ++odd) { + float e[4]; + [unroll] for (uint tt = 0; tt < 4; ++tt) e[tt] = lane_half(q, tt, odd); + hv[odd] = add16(add16(e[0], e[1]), add16(e[2], e[3])); + } + float s = add16(add16(hv[0], hv[1]), asfloat(0x39000000u)); + return from16(wg_ld16(tab, 2u * bits16(s))); +} + +// chain over n chunks of 32: a = FP8 codes in a groupshared row (passed as values), row = FP8 codes in the blob +float chain_a8(float start, uint t, uint row, uint n) +{ + float a[32], wv[32]; + [loop] for (uint ch = 0; ch < n + q7_zero(); ++ch) { + [unroll] for (uint k = 0; k < 32; ++k) a[k] = cd8(g_a8[t][8u * ch + (k >> 2)], k & 3u); + q7_load16(w, row + 64u * ch, wv); + start = op8(start, a, wv); + } + return start; +} + +void block(uint t, uint hf) +{ + const uint Z = q7_zero(); + float wv[32], a[32]; + if (hf == 0u) { + float xv[C]; + [unroll] for (uint c = 0; c < C; ++c) xv[c] = gx(t, c); + g_r[t] = norm_r(xv); + } + wg_barrier_lds(); + [loop] for (uint i = 0; i < CH / 4u + Z; ++i) { + uint pk = 0u; + [unroll] for (uint jj = 0; jj < 4; ++jj) { + uint c = CH * hf + 4u * i + jj; + pk |= wg_f32_to_e4m3(mul16(gx(t, c), mul16(g_r[t], ldh(O_G1 + 2u * c)))) << (8u * jj); + } + g_a8[t][CH / 4u * hf + i] = pk; + } + wg_barrier_lds(); + [loop] for (uint hh = 0; hh < HPL + Z; ++hh) { + uint hd = HPL * hf + hh; + [loop] for (uint d4 = 0; d4 < 8u + Z; ++d4) { + uint pk = 0u; + [unroll] for (uint jj = 0; jj < 4; ++jj) { + uint d = 4u * d4 + jj; + pk |= wg_f32_to_e4m3(chain_a8(0.0f, t, O_WQK16 + 2u * (hd * 32u + d) * C, NC)) << (8u * jj); + g_v[hd][t][d] = (uint16_t)bits16(chain_a8(0.0f, t, O_WV16 + 2u * (hd * 32u + d) * C, NC)); + } + g_u[hd][t][d4] = pk; + } + } + wg_barrier_lds(); + [loop] for (uint hh2 = 0; hh2 < HPL + Z; ++hh2) { + uint hd = HPL * hf + hh2; + [unroll] for (uint k = 0; k < 32; ++k) a[k] = cd8(g_u[hd][t][k >> 2], k & 3u); + [loop] for (uint j = 0; j < 16u + Z; ++j) { + [unroll] for (uint k2 = 0; k2 < 32; ++k2) wv[k2] = cd8(g_u[hd][j][k2 >> 2], k2 & 3u); + float s = op8(ldh(O_B + 2u * ((hd * 16u + t) * 16u + j)), a, wv); + const float cs = 0.017181396484375f, lim = 0.55615234375f; + float uu = min(max(mul16(s, cs), -lim), lim); + float tt = fma16(uu, -uu, 0.92724609375f); + float p = fma16(uu, tt, 1.375f); + g_pw[hd][t][j] = (uint16_t)((bits16(p) & 0x3ffu) << 5); + } + float e[16]; + [unroll] for (uint j2 = 0; j2 < 16; ++j2) e[j2] = from16(g_pw[hd][t][j2]); + float sm[2]; + [unroll] for (uint lo = 0; lo < 2; ++lo) { + float pr[4]; + [unroll] for (uint k3 = 0; k3 < 4; ++k3) pr[k3] = add16(e[lo + 2u * k3], e[lo + 8u + 2u * k3]); + sm[lo] = add16(add16(pr[0], pr[1]), add16(pr[2], pr[3])); + } + float wsum = from16(wg_ld16(tab, 2u * (65536u + bits16(add16(sm[0], sm[1]))))); + [unroll] for (uint j3 = 0; j3 < 16; ++j3) g_pw[hd][t][j3] = (uint16_t)bits16(mul16(e[j3], wsum)); + } + wg_barrier_lds(); + [loop] for (uint hh3 = 0; hh3 < HPL + Z; ++hh3) { + uint hd = HPL * hf + hh3; + float pw[16]; + [unroll] for (uint j4 = 0; j4 < 16; ++j4) pw[j4] = from16(g_pw[hd][t][j4]); + [loop] for (uint q4 = 0; q4 < 8u + Z; ++q4) { + uint pk = 0u; + [unroll] for (uint jj = 0; jj < 4; ++jj) { + uint d2 = 4u * q4 + jj; + float s = 0.0f, c = 0.0f; + [unroll] for (uint j5 = 0; j5 < 16; ++j5) { + float e; + two_sum(s, pw[j5] * from16(g_v[hd][j5][d2]), s, e); + c = c + e; + } + float z, tl; + two_sum(s, c, z, tl); + pk |= wg_f32_to_e4m3(rn16_pair(z, tl)) << (8u * jj); + } + g_o8[t][8u * hd + q4] = pk; + } + } + wg_barrier_lds(); + [loop] for (uint i2 = 0; i2 < CH + Z; ++i2) { + uint c = CH * hf + i2; + float acc = add16(ldh(O_BP + 2u * c), gx(t, c)); + [loop] for (uint hd2 = 0; hd2 < HEADS + Z; ++hd2) { + [unroll] for (uint k4 = 0; k4 < 32; ++k4) a[k4] = cd8(g_o8[t][8u * hd2 + (k4 >> 2)], k4 & 3u); + q7_load16(w, O_WP16 + 2u * (c * 32u * HEADS + 32u * hd2), wv); + acc = op8(acc, a, wv); + } + g_z[t][c] = (uint16_t)bits16(acc); + } + wg_barrier_lds(); + if (hf == 0u) { + float zv[C]; + [unroll] for (uint c2 = 0; c2 < C; ++c2) zv[c2] = gz(t, c2); + g_r[t] = norm_r(zv); + } + wg_barrier_lds(); + [loop] for (uint i3 = 0; i3 < CH / 4u + Z; ++i3) { + uint pk = 0u; + [unroll] for (uint jj = 0; jj < 4; ++jj) { + uint c = CH * hf + 4u * i3 + jj; + float zc = gz(t, c); + pk |= wg_f32_to_e4m3(mul16(zc, mul16(g_r[t], ldh(O_G2 + 2u * c)))) << (8u * jj); + g_z[t][c] = (uint16_t)bits16(add16(ldh(O_B2 + 2u * c), zc)); + } + g_a8[t][CH / 4u * hf + i3] = pk; + } + wg_barrier_lds(); + [loop] for (uint sl = 0; sl < MLP / 32u + Z; ++sl) { + [loop] for (uint i4 = 0; i4 < 4u + Z; ++i4) { + uint pk = 0u; + [unroll] for (uint jj = 0; jj < 4; ++jj) { + uint j6 = 32u * sl + 16u * hf + 4u * i4 + jj; + pk |= wg_f32_to_e4m3(gelu16(chain_a8(ldh(O_B1 + 2u * j6), t, O_W1_16 + 2u * j6 * C, NC))) << (8u * jj); + } + g_g8[t][4u * hf + i4] = pk; + } + wg_barrier_lds(); + [unroll] for (uint k5 = 0; k5 < 32; ++k5) a[k5] = cd8(g_g8[t][k5 >> 2], k5 & 3u); + [loop] for (uint i5 = 0; i5 < CH + Z; ++i5) { + uint c = CH * hf + i5; + q7_load16(w, O_W2_16 + 2u * (c * MLP + 32u * sl), wv); + g_z[t][c] = (uint16_t)bits16(op8(gz(t, c), a, wv)); + } + wg_barrier_lds(); + } +} + +WG_KERNEL(Q7_ENTRY) +{ + WG_SETUP(); + uint lane = wg_lane_id(), t = lane & 15u, hf = lane >> 4; + int nwx = (W + ox + 3) / 4; + int win = (int)wg_block_id(); + int pr = 4 * (win / nwx) - oy + (int)(t >> 2), pc = 4 * (win % nwx) - ox + (int)(t & 3u); +#if Q7_KIND == 1 + int r = 2 * q7_mirror(pr >> 1, H >> 1) + (pr & 1), c = 2 * q7_mirror(pc >> 1, W >> 1) + (pc & 1); + int sr = q7_mirror(pr, H), sc = q7_mirror(pc, W); + uint eo = ((uint)r * (uint)W + (uint)c) * C + CH * hf, so = ((uint)sr * (uint)W + (uint)sc) * C + CH * hf; + [loop] for (uint i = 0; i < CH / 4u + q7_zero(); ++i) { + uint ev = wg_ld32(x, eo + 4u * i), sv = wg_ld32(skip, so + 4u * i); + [unroll] for (uint j = 0; j < 4; ++j) + g_x[t][CH * hf + 4u * i + j] = (uint16_t)bits16(fp8v(ev >> (8u * j)) + fp8v(sv >> (8u * j))); + } +#else + uint xo = ((uint)q7_mirror(pr, H) * (uint)W + (uint)q7_mirror(pc, W)) * C + CH * hf; + [loop] for (uint i = 0; i < CH / 4u + q7_zero(); ++i) { + uint xv = wg_ld32(x, xo + 4u * i); + [unroll] for (uint j = 0; j < 4; ++j) g_x[t][CH * hf + 4u * i + j] = (uint16_t)bits16(fp8v(xv >> (8u * j))); + } +#endif + wg_barrier_lds(); + block(t, hf); + if (pr >= 0 && pr < H && pc >= 0 && pc < W) { + uint oo = ((uint)pr * (uint)W + (uint)pc) * C + CH * hf; + [loop] for (uint q = 0; q < CH / 4u + q7_zero(); ++q) { + uint cb = CH * hf + 4u * q; + wg_st32(out_, oo + 4u * q, out8(gz(t, cb)) | (out8(gz(t, cb + 1u)) << 8) | (out8(gz(t, cb + 2u)) << 16) | + (out8(gz(t, cb + 3u)) << 24)); + } + } +} + +#elif Q7_KIND == 2 +// ---- merge: in [H][W][C] -> out [H/2][W/2][COUT] --------------------------------------------------------------------- +WG_KERNEL(Q7_ENTRY) +{ + WG_SETUP(); + const uint C = Q7_C, COUT = Q7_COUT; + uint lane = wg_lane_id(); + uint w2 = (uint)W / 2u, px = wg_block_id(), r = px / w2, c = px % w2; + if (4u * lane < COUT) { + uint rows = (2u * COUT + 15u) & ~15u; + float a[32], wv[32]; + uint ob = 0u; + [loop] for (uint j = 0; j < 4u + q7_zero(); ++j) { + uint n = 4u * lane + j; + float acc = ldh(2u * n); + [loop] for (uint ch = 0; ch < 4u * C / 32u + q7_zero(); ++ch) { + uint s = (32u * ch) / C, i0 = (32u * ch) % C; // a chunk never crosses a sub-pixel (C % 32 == 0) + q7_load8(in_, ((2u * r + (s >> 1)) * (uint)W + (2u * c + (s & 1u))) * C + i0, a); + q7_load16(w, rows + COUT * 4u * C + 2u * (n * 4u * C + 32u * ch), wv); + acc = op8(acc, a, wv); + } + if (!isnan(acc)) acc = min(max(acc, -6.28125f), 6.28125f); + ob |= out8(acc) << (8u * j); + } + wg_st32(out_, (r * w2 + c) * COUT + 4u * lane, ob); + } +} + +#else +// ---- expansion: in [h][w][CL] -> out [2h][2w][C] ---------------------------------------------------------------------- +WG_KERNEL(Q7_ENTRY) +{ + WG_SETUP(); + const uint C = Q7_C, CL = Q7_CL; + uint lane = wg_lane_id(); + uint wd = (uint)W, px = wg_block_id(), r = px / wd, c = px % wd; + uint rows = (2u * 4u * C + 15u) & ~15u; + float a[32], wv[32]; + [loop] for (uint g = 0; g < C / 32u + q7_zero(); ++g) { + uint n0 = 128u * g + 4u * lane, s = n0 / C, chn = n0 % C; + uint ob = 0u; + [loop] for (uint j = 0; j < 4u + q7_zero(); ++j) { + uint n = n0 + j; + float acc = ldh(2u * n); + [loop] for (uint ch = 0; ch < CL / 32u + q7_zero(); ++ch) { + q7_load8(in_, (r * wd + c) * CL + 32u * ch, a); + q7_load16(w, rows + 4u * C * CL + 2u * (n * CL + 32u * ch), wv); + acc = op8(acc, a, wv); + } + ob |= out8(acc) << (8u * j); + } + wg_st32(out_, ((2u * r + (s >> 1)) * (2u * wd) + (2u * c + (s & 1u))) * C + chn, ob); + } +} +#endif diff --git a/clean/dx12/kernels/l_swin.hlsl b/clean/dx12/kernels/l_swin.hlsl index 3f14f168..9dacb5b0 100644 --- a/clean/dx12/kernels/l_swin.hlsl +++ b/clean/dx12/kernels/l_swin.hlsl @@ -31,4 +31,40 @@ // the FP16 map is not written. Arguments: _dec_nc's (out unused), then A, C, latent, // dec0.coef, rw, rh // @wg-kernel l_swin_block_c32_dec_coef12_nc co=- block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p,p,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=3,SW_GELU_PAT=13333,SW_SRC=2,SW_OUT_F16=1,SW_OUT_COEF12=1,COEF_CVT_HW=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_coef12_nc,SW_SMAX_ADDST=1,SW_SMAX_STAGED=1 +// l_swin_block_c32_enc_hyb / _dec_hyb the HYBRID block form (SW_HYB, WG_L_FORM=hybrid): _enc_nc's and _dec_nc's shapes +// (the decoder WITH the skip: xadd = the encoder block's FP8 map x3 by the fine mirror, x by +// the coarse one, member input RN16(E + S), as l_q7_block_c32_dec and the oracle) +// (the linears and l_coef_pack12 as their own launches) with the original's FP16 norm, softmax, +// GELU and start values around float32 FP8 chains, one FP16 rounding at each chain's end +// @wg-kernel l_swin_block_c32_enc_hyb co=- block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_SRC=1,SW_HYB=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_enc_hyb,WG_REV_ORDER=1 +// @wg-kernel l_swin_block_c32_dec_hyb co=- block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=1,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_OUT_F16=1,SW_HYB=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_hyb +// l_swin_block_c32_enc_lin_hyb / _dec_coef12_hyb the hybrid twins of the fusions le / lc (D3D12 only, speed lever 3): +// _enc_hyb with the two input linears as its prologue (SW_SRC 3: the q map in its 20-byte +// form, enc0.Win1, enc0.Win2 as the last arguments) and _dec_hyb with l_coef_pack12 as its +// epilogue (SW_OUT_COEF12, COEF_CVT_HW); the same bytes as the launches they replace +// @wg-kernel l_swin_block_c32_enc_lin_hyb co=- block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_SRC=3,SW_HYB=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_enc_lin_hyb,WG_REV_ORDER=1 +// @wg-kernel l_swin_block_c32_dec_coef12_hyb co=- block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p,p,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=1,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_OUT_F16=1,SW_OUT_COEF12=1,COEF_CVT_HW=1,SW_HYB=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_coef12_hyb +// ..._enc_lin_hyb_dump / _dec_coef12_hyb_dump (+ _mutdump: SW_HYB_MUT_ORDER=4) their stage dumps (probe=1, check only: +// l_hyb_stage_check.py --set fused, the oracle per stage); _scanmut: the fused-square scan's +// mutants (no contraction barrier, built beside every hybrid build, outside the manifest) +// @wg-kernel l_swin_block_c32_enc_lin_hyb_dump co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_SRC=3,SW_HYB=1,SW_HYB_DUMP=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_enc_lin_hyb_dump,WG_REV_ORDER=1 +// @wg-kernel l_swin_block_c32_enc_lin_hyb_mutdump co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_SRC=3,SW_HYB=1,SW_HYB_DUMP=1,SW_HYB_MUT_ORDER=4,SW_ENTRY_OVERRIDE=l_swin_block_c32_enc_lin_hyb_mutdump,WG_REV_ORDER=1 +// @wg-kernel l_swin_block_c32_enc_lin_hyb_scanmut co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_SRC=3,SW_HYB=1,HYB_SCAN_MUTANT=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_enc_lin_hyb_scanmut,WG_REV_ORDER=1 +// @wg-kernel l_swin_block_c32_dec_coef12_hyb_dump co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p,p,u,u,p defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=1,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_OUT_F16=1,SW_OUT_COEF12=1,COEF_CVT_HW=1,SW_HYB=1,SW_HYB_DUMP=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_coef12_hyb_dump +// @wg-kernel l_swin_block_c32_dec_coef12_hyb_mutdump co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p,p,u,u,p defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=1,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_OUT_F16=1,SW_OUT_COEF12=1,COEF_CVT_HW=1,SW_HYB=1,SW_HYB_DUMP=1,SW_HYB_MUT_ORDER=4,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_coef12_hyb_mutdump +// @wg-kernel l_swin_block_c32_dec_coef12_hyb_scanmut co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p,p,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=1,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_OUT_F16=1,SW_OUT_COEF12=1,COEF_CVT_HW=1,SW_HYB=1,HYB_SCAN_MUTANT=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_coef12_hyb_scanmut +// l_swin_block_c32_enc_hyb_dump / _dec_hyb_dump check variants (probe=1, never in a model build): the same kernels plus +// the stage dump (SW_HYB_DUMP, the extra pointer argument; l_hyb_stage_check.py) +// @wg-kernel l_swin_block_c32_enc_hyb_dump co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u,p defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_SRC=1,SW_HYB=1,SW_HYB_DUMP=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_enc_hyb_dump,WG_REV_ORDER=1 +// @wg-kernel l_swin_block_c32_dec_hyb_dump co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u,p defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=1,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_OUT_F16=1,SW_HYB=1,SW_HYB_DUMP=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_hyb_dump +// l_swin_block_c32_enc_hyb_mut / _dec_hyb_mut the byte-identity mutant (probe=1, check only): SW_HYB_MUT_ORDER=4, the odd W2 +// groups in a second accumulator added at the end (l_hyb_block_check.py --mutant must FAIL HIP == D3D12) +// @wg-kernel l_swin_block_c32_enc_hyb_mut co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_SRC=1,SW_HYB=1,SW_HYB_MUT_ORDER=4,SW_ENTRY_OVERRIDE=l_swin_block_c32_enc_hyb_mut,WG_REV_ORDER=1 +// @wg-kernel l_swin_block_c32_dec_hyb_mut co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=1,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_OUT_F16=1,SW_HYB=1,SW_HYB_MUT_ORDER=4,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_hyb_mut +// @wg-kernel l_swin_block_c32_enc_hyb_mutdump co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u,p defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_SRC=1,SW_HYB=1,SW_HYB_MUT_ORDER=4,SW_HYB_DUMP=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_enc_hyb_mutdump,WG_REV_ORDER=1 +// l_swin_block_c32_enc_hyb_scanmut the fused-square scan's mutant (probe=1; build_dx12.py builds it beside every build +// of hybrid kernels, outside the manifest, and requires hyb_isa_scan.py to FAIL on it): +// _enc_hyb without the contraction barrier (HYB_SCAN_MUTANT=1) +// @wg-kernel l_swin_block_c32_enc_hyb_scanmut co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_SRC=1,SW_HYB=1,HYB_SCAN_MUTANT=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_enc_hyb_scanmut,WG_REV_ORDER=1 +// @wg-kernel l_swin_block_c32_dec_hyb_scanmut co=- probe=1 block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=1,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_OUT_F16=1,SW_HYB=1,HYB_SCAN_MUTANT=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_hyb_scanmut #include "swin_block.hlsl" diff --git a/clean/dx12/kernels/swin_block.hlsl b/clean/dx12/kernels/swin_block.hlsl index d119627e..755ae841 100644 --- a/clean/dx12/kernels/swin_block.hlsl +++ b/clean/dx12/kernels/swin_block.hlsl @@ -31,7 +31,107 @@ // @wg-kernel swin_block_c96_add_ex_nc co=swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=1,SW_SAFE=1,SW_ECL=128 #include "wg_platform.hlsli" #include "wg_gelu_lut.hlsli" +// SW_HYB 1: the HYBRID block form (WG_L_FORM / WG_M_FORM = hybrid; hlsl/wg_hyb.hlsli, hlsl/hyb_block.hlsli): float32 FP8 +// chains with one FP16 rounding at each chain's end, the original's FP16 norm / softmax / GELU and its start values; +// the weights in this file's layout with the original's values (FP16 biases, g', B; plain FP8 matrices). Default 0. +#ifndef SW_HYB +#define SW_HYB 0 +#endif +#ifndef SW_HYB_MUT_ORDER +#define SW_HYB_MUT_ORDER 0 // check mutants only: 1 one W2 chunk pair reversed, 2 the pair of every W2 group, + // 3 the odd W2 groups in a second accumulator added at the end (one extra f32 rounding), + // 4 the W2 chain end moved by one float32 ulp before its RN16 +#endif +#ifndef SW_HYB_DUMP +#define SW_HYB_DUMP 0 // check variant only (probe kernels, l_hyb_stage_check.py): every stage's values to `dumpp` +#endif +#if SW_HYB_DUMP && !SW_HYB +#error "SW_HYB_DUMP: hybrid kernels only" +#endif +#if SW_HYB +// The hybrid kernels at every width (C = 32, 64, 96, 128) run the plain schedule (the C = 32 knob set): no scheduling +// knob of the M-form kernels, no GELU table of theirs (hyb_gelu_q8 has its own), and every output pack through +// hyb_q8_of_chain (no clamp-skip shortcut: FP8(x2) without the RN16 is not the hybrid output). Set here, before the +// defaults below; setting one of them on a kernel line stops the build (the check after this block). +#ifndef SW_GELU +#define SW_GELU 0 +#endif +#ifndef SW_CLAMP_SKIP +#define SW_CLAMP_SKIP 0 +#endif +#ifndef SW_NORM1_EARLY +#define SW_NORM1_EARLY 0 +#endif +#ifndef SW_NORM2_EARLY +#define SW_NORM2_EARLY 0 +#endif +#ifndef SW_NORM_R2 +#define SW_NORM_R2 0 +#endif +#ifndef SW_SMAX_ADDST +#define SW_SMAX_ADDST 0 +#endif +#ifndef SW_SMAX_STAGED +#define SW_SMAX_STAGED 0 +#endif +#ifndef SW_HEAD_UNROLL +#define SW_HEAD_UNROLL 0 +#endif +#ifndef SW_HEAD_PAIRS +#define SW_HEAD_PAIRS 0 +#endif +#ifndef SW_MLP_PAIRS +#define SW_MLP_PAIRS 0 +#endif +#ifndef SW_MLP_STEP +#define SW_MLP_STEP 1 +#endif +#ifndef SW_REV_PHI +#define SW_REV_PHI 0 +#endif +#ifndef SW_COMMIT_EARLY +#define SW_COMMIT_EARLY 0 +#endif +#ifndef SW_N2_BATCH +#define SW_N2_BATCH 0 +#endif +#ifndef SW_GELU_LATE0 +#define SW_GELU_LATE0 0 +#endif +#endif +#if SW_HYB && (SW_GELU != 0 || SW_HEAD_UNROLL || SW_HEAD_PAIRS || SW_MLP_PAIRS || SW_MLP_STEP != 1 || SW_COMMIT_EARLY || \ + SW_N2_BATCH) +#error "SW_HYB: the hybrid kernels run the plain schedule (no M-form scheduling knobs)" +#endif +// (SW_SRC 3 and SW_OUT_COEF12 are allowed: the linears' prologue gives the block the same FP16 values as the x2 map, +// and the pack12 epilogue reads the same FP16 output, converted by the driver's RNE as the hybrid output point does) +#if SW_HYB &&(SW_ECL > 0 || SW_NORM1_EARLY || SW_NORM2_EARLY || SW_NORM_R2 || SW_SMAX_STAGED || SW_SMAX_ADDST || \ + SW_REV_PHI || SW_CLAMP_SKIP) +#error "SW_HYB: only the plain block path (no fused expand, no scheduling knobs) has the hybrid form so far" +#endif +#if SW_HYB && SW_OUT_COEF12 && !COEF_CVT_HW +#error "SW_HYB with SW_OUT_COEF12: COEF_CVT_HW 1 (the driver's RNE conversion of the hybrid output point)" +#endif +// SW_MEGA 1 (kernels/hyb_bneck.hlsl only): the body below becomes the function sw_item(), one item (8 windows) of the +// bottleneck's persistent queue; its arguments are the queue kernel's (bneck.hlsl's), and the block's x, out_p, w, ox, +// oy are statics the queue sets per item (wg_bid_ = the item's index in its block). The input gather and the output +// store use the coherent view (the maps are written by other workgroups of the same launch). The same instructions on +// the same values otherwise: b5's bytes equal six swin_block_c128_hyb launches. +#ifndef SW_MEGA +#define SW_MEGA 0 +#endif +#if SW_MEGA && (!SW_HYB || SW_C != 128 || SW_ADD || SW_ECL > 0 || SW_SRC || SW_OUT_F16 || SW_OUT_COEF12 || SW_HYB_DUMP) +#error "SW_MEGA: the hybrid C128 bottleneck block only (plain FP8 map in and out, no dump)" +#endif +#if SW_MEGA +WG_ARGS_BEGIN + uint a_x, a_out, a_tmp, a_sync, a_w0, a_w1, a_w2, a_w3, a_w4, a_w5; // pointers (bneck.hlsl's arguments) + int H, W, a_oxe, a_oye, a_oxo, a_oyo, a_groups; +WG_ARGS_END +static uint x, out_p, w; // the current item's block: input map, output map, weight section +static int ox, oy; // its window offsets +#else WG_ARGS_BEGIN uint x, xadd, out_p, w; // pointers int H, W, ox, oy; @@ -45,7 +145,11 @@ WG_ARGS_BEGIN #if SW_SRC == 3 uint linW1, linW2; // pointers: enc0.Win1, enc0.Win2 (x = the FP8 q halo map) #endif +#if SW_HYB_DUMP + uint dumpp; // pointer: the stage dump (HD_WORDS words per lane of every real window) +#endif WG_ARGS_END +#endif // ---- BlockLayout (FORMATS.md, "Swin block section") --------------------------------------------------- #define KC (SW_C / 16) @@ -88,6 +192,8 @@ WG_ARGS_END #define HPU 8 // MTH 16, limit 8 #elif SW_C == 96 #define HPU 4 // MTH 24, limit 5 +#elif SW_C == 128 && SW_HYB +#define HPU 4 // MTH 32, limit 4 (the hybrid bottleneck block; the M-form C = 128 blocks are bneck.hlsl) #else #error "SW_C: 64 or 96" #endif @@ -395,13 +501,57 @@ uint L_mlp(uint u) { return L_STREAM + SW_HEADS * HEAD_BYTES + u * MLP_BYTES; } // Inner FP8 pack (HIP pack_in) with the driver conversion (wave-uniform control flow); the output pack is sw_pack_q8. uint2 pack_in(v8f d) { -#if SW_SAFE +#if SW_HYB + return wg_pack_q8_ags(d); +#elif SW_SAFE return wg_pack_q8_nc_ags(d); #else return wg_pack_q8_inner_ags(d); #endif } +#if SW_HYB +#include "wg_hyb.hlsli" +// a zero the compiler cannot prove (the grid height is never negative): hyb_block.hlsli's contraction barrier. +// HYB_SCAN_MUTANT=1 (the fused-square scan's mutant, *_scanmut, never in a model build): a literal zero, i.e. the +// barrier removed; build_dx12.py requires the scan to FAIL on it +#if HYB_SCAN_MUTANT +#define HYB_OPAQUE0 0u +#else +// the unknown bit in BOTH FP16 halves (bits 0 and 16): with bit 0 only, the compiler proved the high half untouched +// and fused it (v_and_or_b32 on the low half, v_fma_f16 on .h) +#define HYB_OPAQUE0 (((uint)H >> 31u) * 0x10001u) +#endif +#include "hyb_block.hlsli" +#endif +#if SW_HYB_DUMP +// Stage dump (l_hyb_stage_check.py): per real window and lane, HD_WORDS 32-bit words, every value in the lane's own +// register layout (float32 bits; P as its packed FP16 words). Fields: X the block input (after the skip add), Y1 norm1's +// output; per head (60 words): UQ RN16 of the q/k chain (2 tiles), V RN16 of the v chain (2 tiles), S RN16 of the score +// chain, P (4 words), O RN16 of the P V chain (2 tiles); Z RN16 of the Wp chain, Y2 norm2's output, H RN16 of the W1 +// chains (MTH tiles), OUT RN16 of the W2 chain (the block output before its FP8 / FP16 point). Only values the kernel +// computes anyway; the stores are its only addition. +#define HD_HEAD (16u * KC) +#define HD_Z (HD_HEAD + 60u * SW_HEADS) +#define HD_Y2 (HD_Z + 8u * KC) +#define HD_H (HD_Y2 + 8u * KC) +#define HD_OUT (HD_H + 8u * MTH) +#define HD_WORDS (HD_OUT + 8u * KC) +static uint hd_base; +static bool hd_on; +void hd_st4(uint word, uint4 a) { if (hd_on) wg_st128(dumpp, (hd_base + word) * 4u, a); } +void hd_st8(uint word, v8f a) +{ + hd_st4(word, uint4(asuint(a.v[0]), asuint(a.v[1]), asuint(a.v[2]), asuint(a.v[3]))); + hd_st4(word + 4u, uint4(asuint(a.v[4]), asuint(a.v[5]), asuint(a.v[6]), asuint(a.v[7]))); +} +#define HD_ST8(word, a) hd_st8(word, a) +#define HD_ST4(word, a) hd_st4(word, a) +#else +#define HD_ST8(word, a) +#define HD_ST4(word, a) +#endif + // ---- Stager: global -> registers (fetch) -> LDS (commit), `chunks` 16-byte chunks (a compile-time // constant at every call, so the row tests fold): lane tid moves chunks tid, tid + THREADS, ...; in a partial last row the // lanes past the end re-load / re-store the unit's last chunk (same value: no per-lane branch). @@ -432,7 +582,11 @@ void st_commit(Stager s, uint dst, uint chunks, uint tid) void load_frags_g(uint base, uint off, out uint2 f[KC]) { [unroll] for (uint i = 0; i < KC / 2; ++i) { +#if SW_MEGA + uint4 v = wg_mem_gc.Load4(base + off + 16u * i); // written by other workgroups of this launch +#else uint4 v = wg_ld128(base, off + 16u * i); +#endif f[2 * i] = v.xy; f[2 * i + 1] = v.zw; } } @@ -443,8 +597,8 @@ void load_frags_g(uint base, uint off, out uint2 f[KC]) #ifndef SW_OUT_F16 #define SW_OUT_F16 0 #endif -#if (SW_SRC != 0 || SW_OUT_F16) && (SW_ADD || SW_ECL > 0) -#error "SW_SRC / SW_OUT_F16: plain blocks only" +#if (SW_SRC != 0 || (SW_OUT_F16 && !SW_HYB)) && (SW_ADD || SW_ECL > 0) +#error "SW_SRC / SW_OUT_F16: plain blocks only (the hybrid C32 decoder: SW_OUT_F16 with SW_ADD, the skip)" #endif #ifndef SW_OUT_COEF12 #define SW_OUT_COEF12 0 @@ -509,9 +663,10 @@ void sw_load_halo_lin(uint base, int pr, int pc, int W, uint lane, uint hh, out f[k].v[4] = wg_h2f_lo(y[k].z); f[k].v[5] = wg_h2f_hi(y[k].z); f[k].v[6] = wg_h2f_lo(y[k].w); f[k].v[7] = wg_h2f_hi(y[k].w); } } -#elif SW_SRC == 2 +#elif SW_SRC == 2 || (SW_HYB && SW_ADD) // LCoarseFp8Src: a member outside the grid takes the mirrored 2 x 2 group (mirror of the group index) and keeps its -// sub-pixel; the byte offset of the lane's half of that pixel +// sub-pixel; the byte offset of the lane's half of that pixel (also the hybrid decoder blocks' expanded map E: the +// table's shared_dec_border "coarse"; their skip map S keeps the fine mirror) uint sw_coarse_src(int pr, int pc, int H, int W, uint hh) { const int r = 2 * sw_mirror(pr >> 1, H >> 1) + (pr & 1), c = 2 * sw_mirror(pc >> 1, W >> 1) + (pc & 1); @@ -634,14 +789,26 @@ void fused_expand(uint tail, uint e_lds, int nwx, int nwin, uint tid, int blk, u #define SW_ENTRY SW_ENTRY_OVERRIDE #endif +#if SW_MEGA +// the body's global stores (its output map, read by other workgroups of this launch) through the coherent view +#define wg_st128(b, o, v) wg_mem_gc.Store4((b) + (o), (v)) +void sw_item() // one queue item (hyb_bneck.hlsl): the block on windows 8 wg_bid_ .. + 7 of the current block +#else WG_KERNEL(SW_ENTRY) +#endif { +#if !SW_MEGA WG_SETUP(); +#endif const uint tid = wg_lane_id(); const uint lane = tid & 31u, t = lane & 15u, hh = lane >> 4; const int nwx = (W + ox + 3) >> 2, nwy = (H + oy + 3) >> 2, nwin = nwx * nwy; const uint wave_id = (uint)wg_uniform((int)(tid >> 5)); const int win0 = (int)(wg_block_id() * (THREADS / 32) + wave_id); +#if SW_HYB_DUMP + hd_on = win0 < nwin; + hd_base = ((uint)win0 * 32u + lane) * HD_WORDS; +#endif Stager stage; st_fetch(stage, w + L_head(0), HEAD_BYTES / 16, tid); // unit 0 @@ -676,7 +843,7 @@ WG_KERNEL(SW_ENTRY) #elif SW_SRC == 3 v8f xf[KC]; sw_load_halo_lin(x, pr, pc, W, lane, hh, xf); -#elif SW_SRC == 2 +#elif SW_SRC == 2 || (SW_HYB && SW_ADD) load_frags_g(x, sw_coarse_src(pr, pc, H, W, hh), xa); #else load_frags_g(x, src, xa); @@ -714,8 +881,25 @@ WG_KERNEL(SW_ENTRY) #endif #if SW_ADD acc[k] = sw_add8(acc[k], sw_unpack_fp8(xb[k])); +#if SW_HYB + acc[k] = hyb_round16(acc[k]); // x = RN16(dec(P) + dec(S)) +#endif #endif } +#if SW_HYB + // y = the original's norm (FP16 tree); a = FP8(y); the Wp chain starts at RN16(bp + x) + { + const uint g1 = w + L_G1 + hh * (HALF * 4), bp = w + L_BP + hh * 32u; + v8f gp[KC], y[KC]; + [unroll] for (uint k1 = 0; k1 < KC; ++k1) gp[k1] = wg_load8f(g1, 32u * k1); + hyb_norm(acc, gp, hh, y); + [unroll] for (uint kd = 0; kd < KC; ++kd) { HD_ST8(8u * kd, acc[kd]); HD_ST8(8u * (KC + kd), y[kd]); } + [unroll] for (uint k3 = 0; k3 < KC; ++k3) { + af[k3] = hyb_q8(y[k3]); + acc[k3] = hyb_add16(acc[k3], wg_load8f(bp, 64u * k3)); + } + } +#else const float r = norm_r(acc); #if SW_NORM_R2 & 1 const float r2 = sw_opaque_copy(r); @@ -738,6 +922,7 @@ WG_KERNEL(SW_ENTRY) af[k2] = pack_in(d); acc[k2] = sw_add8(acc[k2], b); } +#endif } // Steps 3-7, one head at a time; one staged unit per head (QKV fragments, then the head's slice of Wp). @@ -778,13 +963,29 @@ WG_KERNEL(SW_ENTRY) accq = wg_wmma_fp8(fq[kc2], af[kc2], accq); accv = wg_wmma_fp8(af[kc2], fv[kc2], accv); } +#if SW_HYB + uf[mt] = hyb_q8_of_chain(accq); + HD_ST8(HD_HEAD + 60u * hd + 8u * mt, hyb_round16(accq)); +#else uf[mt] = pack_in(accq); +#endif vt[mt] = wg_to_f16_ags(accv); +#if SW_HYB_DUMP + HD_ST8(HD_HEAD + 60u * hd + 16u + 8u * mt, hyb_h2f8(vt[mt])); +#endif } // Lane (t, h) holds the logits S(t, s) of 8 keys; the position bias is packed in the same order. #if SW_C != 64 const v8f bias = wg_load8f(biasl, hd * 1024u); #endif +#if SW_HYB + // s = one 32-term FP8 op started at B (the position bias, FP16 values); the original's weights P + v8f s = wg_wmma_fp8(uf[0], uf[0], bias); + s = wg_wmma_fp8(uf[1], uf[1], s); + const uint4 p = hyb_softmax_p(s, hh); + HD_ST8(HD_HEAD + 60u * hd + 32u, hyb_round16(s)); + HD_ST4(HD_HEAD + 60u * hd + 40u, p); +#else v8f s = wg_wmma_fp8(uf[0], uf[0], sw_zero8()); // S = U^T U s = wg_wmma_fp8(uf[1], uf[1], s); // soft cap: c = clamp(S + B, -cap, cap); t = c - c^3 / (3 cap^2) (fused); z = t tau log2(e). Written stage by stage @@ -842,10 +1043,19 @@ WG_KERNEL(SW_ENTRY) [unroll] for (uint va = 0; va < 8; ++va) nz[va] = min(nz[va], 1u); [unroll] for (uint vb = 0; vb < 8; ++vb) po.v[vb] = asfloat(qs[vb] | nz[vb]); const uint4 p = wg_to_f16_ags(po); +#endif #endif // o = A V (rows value channels, columns tokens) uint2 of[2]; +#if SW_HYB + [unroll] for (uint dt = 0; dt < 2; ++dt) { + const v8f pv = wg_wmma_f16(vt[dt], p, sw_zero8()); + of[dt] = hyb_q8_of_chain(pv); + HD_ST8(HD_HEAD + 60u * hd + 44u + 8u * dt, hyb_round16(pv)); + } +#else [unroll] for (uint dt = 0; dt < 2; ++dt) of[dt] = pack_in(wg_wmma_f16(vt[dt], p, sw_zero8())); +#endif // x1 += Wp[:, head hd] q8(o_hd) uint2 fp[KC][2]; [unroll] for (uint m = 0; m < KC; ++m) { @@ -883,6 +1093,21 @@ WG_KERNEL(SW_ENTRY) // Step 8: m = q8(RMSNorm(x1; gamma2)), then b2 into the residual accumulator. uint2 mf[KC]; +#if SW_HYB + { + // z = RN16(the Wp chain); m = FP8(norm(z; g2')); the W2 chain starts at RN16(b2 + z) + [unroll] for (uint kz = 0; kz < KC; ++kz) acc[kz] = hyb_round16(acc[kz]); + const uint g2 = w + L_G2 + hh * (HALF * 4), b2 = w + L_B2 + hh * 32u; + v8f gp[KC], y[KC]; + [unroll] for (uint k1 = 0; k1 < KC; ++k1) gp[k1] = wg_load8f(g2, 32u * k1); + hyb_norm(acc, gp, hh, y); + [unroll] for (uint kd = 0; kd < KC; ++kd) { HD_ST8(HD_Z + 8u * kd, acc[kd]); HD_ST8(HD_Y2 + 8u * kd, y[kd]); } + [unroll] for (uint k3 = 0; k3 < KC; ++k3) { + mf[k3] = hyb_q8(y[k3]); + acc[k3] = hyb_add16(acc[k3], wg_load8f(b2, 64u * k3)); + } + } +#else { const float r = norm_r(acc); #if SW_NORM_R2 & 2 @@ -913,9 +1138,14 @@ WG_KERNEL(SW_ENTRY) acc[k] = sw_add8(acc[k], b); } } +#endif // Steps 9-10: x2 = x1 + W2 q8(GELU(W1 m + b1)) + b2, GROUP hidden tiles of 16 at a time. const uint b1 = w + L_B1 + hh * 32u; +#if SW_HYB && SW_HYB_MUT_ORDER == 3 + v8f acc2[KC]; // check mutant 3: the odd tile groups' W2 chunks accumulate here (from 0), added at the end + [unroll] for (uint ma = 0; ma < KC; ++ma) acc2[ma] = sw_zero8(); +#endif #if SW_REV_PHI v8f racc[KC]; [unroll] for (uint rk = 0; rk < KC; ++rk) racc[rk] = sw_perm8(acc[rk], 7u, 7u); @@ -982,7 +1212,7 @@ WG_KERNEL(SW_ENTRY) #endif [unroll] for (uint g4 = 0; g4 < GROUP; ++g4) [unroll] for (uint m = 0; m < KC; ++m) f2[g4][m] = sw_lfrag_b(fb, g4 * 2u * KC + KC + m); -#if SW_GELU != 0 +#if SW_GELU != 0 && !SW_HYB #if !SW_SAFE #error "the GELU tables give HIP's pack_in bytes (no clamp)" #endif @@ -1005,7 +1235,12 @@ WG_KERNEL(SW_ENTRY) [unroll] for (uint g7 = 0; g7 < GROUP; ++g7) hnext[g7] = hcur[g7]; } uint2 hf[GROUP]; -#if SW_GELU != 0 +#if SW_HYB + [unroll] for (uint gh = 0; gh < GROUP; ++gh) { + hf[gh] = hyb_gelu_q8(hyb_rn16x8(hcur[gh])); + HD_ST8(HD_H + 8u * (u * HPU + j0 + gh), hyb_round16(hcur[gh])); + } +#elif SW_GELU != 0 #if SW_GELU_EARLY if ((SW_GELU_LATE0 >> ((j0 / GROUP) % 2u)) & 1u) gp = sw_gelu_pre(hcur); sw_gelu_fin(hcur, gp, hf); @@ -1035,8 +1270,27 @@ WG_KERNEL(SW_ENTRY) if (j0 == GROUP * (SW_STAGE_LATE - 1) && (SW_MLP_STEP != NMLP || more)) st_fetch(stage, wd + (hf[0].x & sw_zero_bits()) + L_mlp(more ? u + 1u : u), MLP_BYTES / 16, tid); #endif +#if SW_HYB && SW_HYB_MUT_ORDER + // check mutant (never shipped): the first unit's first tile group accumulates its W2 chunks in reverse order + // (same products, same start): inside the f32chain tolerance, but not HIP's bytes (l_hyb_stage_check.py) +#if SW_HYB_MUT_ORDER == 3 + if ((j0 / GROUP) & 1u) { // 3: odd groups into the second accumulator + [unroll] for (uint m5 = 0; m5 < KC; ++m5) + [unroll] for (uint g11 = 0; g11 < GROUP; ++g11) acc2[m5] = wg_wmma_fp8(f2[g11][m5], hf[g11], acc2[m5]); + } else +#endif + if (SW_HYB_MUT_ORDER == 2 || (SW_HYB_MUT_ORDER == 1 && u == 0u && j0 == 0u)) { // 1: first group; 2: every group + [unroll] for (uint m3 = 0; m3 < KC; ++m3) + [unroll] for (uint g9 = 0; g9 < GROUP; ++g9) + acc[m3] = wg_wmma_fp8(f2[GROUP - 1u - g9][m3], hf[GROUP - 1u - g9], acc[m3]); + } else { + [unroll] for (uint m4 = 0; m4 < KC; ++m4) + [unroll] for (uint g10 = 0; g10 < GROUP; ++g10) acc[m4] = wg_wmma_fp8(f2[g10][m4], hf[g10], acc[m4]); + } +#else [unroll] for (uint m2 = 0; m2 < KC; ++m2) [unroll] for (uint g6 = 0; g6 < GROUP; ++g6) acc[m2] = wg_wmma_fp8(f2[g6][m2], hf[g6], acc[m2]); +#endif [unroll] for (uint g8 = 0; g8 < GROUP; ++g8) hcur[g8] = hnext[g8]; #if SW_COMMIT_EARLY if (more && j0 == GROUP * (SW_COMMIT_EARLY - 1)) st_commit(stage, (buf ^ 1u) * MAXU + 8u * SW_MDZ, MLP_BYTES / 16, tid); @@ -1056,6 +1310,18 @@ WG_KERNEL(SW_ENTRY) #if SW_REV_PHI [unroll] for (uint rk2 = 0; rk2 < KC; ++rk2) acc[rk2] = sw_unperm8(racc[rk2], 7u, 7u); #endif +#if SW_HYB && SW_HYB_MUT_ORDER == 3 + [unroll] for (uint mb = 0; mb < KC; ++mb) [unroll] for (uint vb = 0; vb < 8; ++vb) acc[mb].v[vb] = acc[mb].v[vb] + acc2[mb].v[vb]; +#endif +#if SW_HYB && SW_HYB_MUT_ORDER == 4 + // check mutant 4: the W2 chain end moved by one float32 ulp (last mantissa bit flipped) before its RN16: far inside + // the f32chain tolerance, visible in the bytes wherever the value sits on an FP16 rounding tie + [unroll] for (uint mc = 0; mc < KC; ++mc) + [unroll] for (uint vc = 0; vc < 8; ++vc) acc[mc].v[vc] = asfloat(asuint(acc[mc].v[vc]) ^ 1u); +#endif +#if SW_HYB_DUMP + [unroll] for (uint kdo = 0; kdo < KC; ++kdo) HD_ST8(HD_OUT + 8u * kdo, hyb_round16(acc[kdo])); +#endif #if SW_OUT_F16 // Output RN16(x2) as FP16, real window members only (HIP LStoreF16Epi). @@ -1074,7 +1340,12 @@ WG_KERNEL(SW_ENTRY) } return; #endif +#if SW_HYB + // the driver's RNE conversion (the chain-end RN16 of the hybrid helpers; HIP's LStoreF16Epi v_cvt_f16_f32) + [unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16_ags(acc[kf]); +#else [unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16(acc[kf]); +#endif #if SW_OUT_COEF12 // l_coef_pack12 of the map: o16[0] / o16[1] of the hh = 0 lane are the pixel's channels 0..7 / 8..15 (the map's // bytes dst * 64 + 0 .. 31, its lo / hi loads); Model L's 13 values are all among them @@ -1104,7 +1375,11 @@ WG_KERNEL(SW_ENTRY) } else #endif { +#if SW_HYB + [unroll] for (uint k = 0; k < KC; ++k) o[k] = hyb_q8_of_chain(acc[k]); +#else [unroll] for (uint k = 0; k < KC; ++k) o[k] = sw_pack_q8_v2(acc[k]); +#endif } if (dst >= 0) { const uint ob = (uint)dst * SW_C + hh * HALF; diff --git a/clean/dx12/tests/kernel_replay.cpp b/clean/dx12/tests/kernel_replay.cpp index 96d123f2..45869470 100644 --- a/clean/dx12/tests/kernel_replay.cpp +++ b/clean/dx12/tests/kernel_replay.cpp @@ -666,6 +666,13 @@ int main(int argc, char** argv) { return 2; } } + // The hybrid kernels (a name with a _hyb part) read the hybrid tables: reserve them as the runtime does for a Model L + // hybrid context (Device::hyb_reserve, before any allocation), by kernel name, never by an environment variable + bool hyb = false; + for (auto& c : cases) hyb = hyb || c.kernel.find("_hyb") != std::string::npos; + for (auto& c : chains) + for (auto& st : c.steps) hyb = hyb || st.kernel.find("_hyb") != std::string::npos; + if (hyb && !d->hyb_reserve()) { std::printf("hybrid tables: %s\n", d->err().c_str()); return 1; } int fails = 0; for (auto& c : cases) fails += run_case(d.get(), c, base); for (auto& c : chains) fails += run_chain(d.get(), c, base); diff --git a/clean/dx12/tests/test_hyb_isa_scan.py b/clean/dx12/tests/test_hyb_isa_scan.py new file mode 100644 index 00000000..b0599249 --- /dev/null +++ b/clean/dx12/tests/test_hyb_isa_scan.py @@ -0,0 +1,101 @@ +# SPDX-License-Identifier: MIT +"""The hybrid kernels' ISA rules (clean/dx12/hyb_isa_scan.py) on constructed gfx12 ISA: each rule finds what it must +(the negative controls: a square fused through a copied register, an FP8 conversion of a float32 value whose RN16 was +removed) and passes what it must not flag. No GPU, no RGA. +Run: python clean/dx12/tests/test_hyb_isa_scan.py +""" +import os +import sys +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.dirname(HERE)) +import hyb_isa_scan as S # noqa: E402 + + +def isa(*lines): + return "\n".join("\t" + x if not x.endswith(":") else x for x in lines) + "\n\ts_endpgm\n" + + +class FusedSquares(unittest.TestCase): + def found(self, *lines): + return len(S.fused_squares(isa(*lines))) + + def test_square(self): + self.assertEqual(self.found("v_pk_fma_f16 v1, v4, v4, v1"), 1) + self.assertEqual(self.found("v_fmac_f16_e32 v1, -v4, v4"), 1) + self.assertEqual(self.found("v_fma_f32 v1, v4, v4, v2"), 1) + self.assertEqual(self.found("v_pk_fma_f16 v1, v4, v6, v1"), 0) + + def test_copied_register(self): + """The negative control the review asked for: x copied to a second register, then x * copy fused.""" + self.assertEqual(self.found("v_mov_b32_e32 v5, v4", "v_pk_fma_f16 v1, v4, v5, v1"), 1) + self.assertEqual(self.found("v_mov_b32_e32 v5, v4", "v_mov_b32_e32 v6, v5", "v_pk_fma_f16 v1, v6, v4, v1"), 1) + self.assertEqual(self.found("v_mov_b16_e32 v5.l, v4.h", "v_fma_f16 v1.l, v4.h, v5.l, v1.l"), 1) + # the copy overwritten, or its source overwritten: not one value any more + self.assertEqual(self.found("v_mov_b32_e32 v5, v4", "v_add_f32_e32 v5, v6, v7", "v_pk_fma_f16 v1, v4, v5, v1"), 0) + self.assertEqual(self.found("v_mov_b32_e32 v5, v4", "v_add_f32_e32 v4, v6, v7", "v_pk_fma_f16 v1, v4, v5, v1"), 0) + # a lane move (DPP) is not a copy of the lane's value + self.assertEqual(self.found("v_mov_b32_dpp v5, v4 quad_perm:[1,0,3,2] row_mask:0xf bank_mask:0xf", + "v_pk_fma_f16 v1, v4, v5, v1"), 0) + + def test_copies_through_blocks(self): + both = isa("s_cbranch_scc0 L1", "v_mov_b32_e32 v5, v4", "s_branch L2", "L1:", "v_mov_b32_e32 v5, v4", "L2:", + "v_pk_fma_f16 v1, v4, v5, v1") + one = isa("s_cbranch_scc0 L1", "v_mov_b32_e32 v5, v4", "L1:", "v_pk_fma_f16 v1, v4, v5, v1") + self.assertEqual(len(S.fused_squares(both)), 1) + self.assertEqual(len(S.fused_squares(one)), 0) + + def test_dual_issue(self): + self.assertEqual(self.found("v_dual_mov_b32 v5, v4 :: v_dual_fmac_f32 v1, v6, v6"), 1) + self.assertEqual(self.found("v_mov_b32_e32 v5, v4", "v_dual_add_f32 v8, v2, v3 :: v_dual_fmac_f32 v1, v4, v5"), 1) + + def test_not_flagged(self): + self.assertEqual(self.found("v_pk_fma_f16 v1, v4, v4, v1 op_sel:[0,1,0] op_sel_hi:[1,1,1]"), 0) # two halves + self.assertEqual(self.found("v_fma_f32 v1, -v4, v4, 0x3f6d6000"), 0) # float32, literal addend (softmax) + self.assertEqual(self.found("v_fma_f32 v1, -v4, v4, s3"), 0) + self.assertEqual(self.found("v_fmaak_f32 v1, v4, v4, 0x3f800000"), 0) + + +class Fp8Sources(unittest.TestCase): + def bad(self, *lines): + return len(S.fp8_sources_not_fp16(isa(*lines))) + + WIDEN = ("v_cvt_f16_f32_e32 v2.l, v1", "v_cvt_f16_f32_e32 v2.h, v9", "v_cvt_f32_f16_e32 v3, v2.l", + "v_cvt_f32_f16_e32 v4, v2.h") + + def test_fp16_sources(self): + self.assertEqual(self.bad(*self.WIDEN, "v_cvt_pk_fp8_f32 v5.l, v3, v4"), 0) + # the saturating pack's steps on FP16 values: clamp, NaN quieting, select + self.assertEqual(self.bad(*self.WIDEN, "v_maxmin_num_f32 v6, s6, v3, 0x43e00000", "v_or_b32_e32 v7, 0x400000, v3", + "v_cndmask_b32_e64 v8, v6, v7, s4", "v_max3_num_f32 v9, 0, v4, 0xc3e00000", + "v_cvt_pk_fp8_f32 v5.h, v8, v9 op_sel:[0,0,1]"), 0) + self.assertEqual(self.bad("v_mov_b32_e32 v3, 0", "v_cvt_pk_fp8_f32 v5.l, v3, 0"), 0) # constants + + def test_removed_widening(self): + """The negative control the review asked for: the RN16 removed, the FP8 pack takes the float32 value.""" + self.assertEqual(self.bad("v_fma_f32 v1, v2, v3, v4", "v_cvt_pk_fp8_f32 v5.l, v1, v1"), 1) + self.assertEqual(self.bad("v_fma_f32 v1, v2, v3, v4", "v_maxmin_num_f32 v6, s6, v1, 0x43e00000", + "v_cvt_pk_fp8_f32 v5.l, v6, v6"), 1) + self.assertEqual(self.bad(*self.WIDEN, "v_or_b32_e32 v7, 0x1, v3", "v_cvt_pk_fp8_f32 v5.l, v7, v4"), 1) + # one source FP16, the other not + self.assertEqual(self.bad(*self.WIDEN, "v_add_f32_e32 v6, v3, v4", "v_cvt_pk_fp8_f32 v5.l, v3, v6"), 1) + # a register range written over the widened value (a WMMA result), an op the rule does not know + self.assertEqual(self.bad(*self.WIDEN, "v_wmma_f32_16x16x16_fp8_fp8 v[0:7], v[8:9], v[10:11], v[0:7]", + "v_cvt_pk_fp8_f32 v5.l, v3, v4"), 1) + self.assertEqual(self.bad(*self.WIDEN, "v_permlanex16_b32 v6, v3, s0, s1", "v_cvt_pk_fp8_f32 v5.l, v6, v4"), 1) + # loads are not FP16 values + self.assertEqual(self.bad("global_load_b32 v3, v1, s[0:1]", "v_cvt_pk_fp8_f32 v5.l, v3, v3"), 1) + + def test_paths(self): + both = isa("s_cbranch_scc0 L1", *self.WIDEN, "s_branch L2", "L1:", *self.WIDEN, "L2:", + "v_cvt_pk_fp8_f32 v5.l, v3, v4") + one = isa("s_cbranch_scc0 L1", *self.WIDEN, "L1:", "v_cvt_pk_fp8_f32 v5.l, v3, v4") + loop = isa(*self.WIDEN, "L0:", "v_cvt_pk_fp8_f32 v5.l, v3, v4", "v_add_f32_e32 v3, v3, v3", "s_cbranch_scc0 L0") + self.assertEqual(len(S.fp8_sources_not_fp16(both)), 0) + self.assertEqual(len(S.fp8_sources_not_fp16(one)), 1) + self.assertEqual(len(S.fp8_sources_not_fp16(loop)), 1) # the back edge brings a float32 v3 + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/dx12/tests/test_kernel_sets.py b/clean/dx12/tests/test_kernel_sets.py index 7cc88493..52617fde 100644 --- a/clean/dx12/tests/test_kernel_sets.py +++ b/clean/dx12/tests/test_kernel_sets.py @@ -48,7 +48,10 @@ def test_model_l_adds_only_model_l_kernels(self): self.assertTrue(m <= full) extra = full - m self.assertTrue(extra) - self.assertFalse([k for k in extra if not (k[1].startswith("l_") or k[0].startswith("l_") or k[0] in B.MODEL_L_ONLY)]) + # Model L's kernels: its code objects, the l_* direct-route kernels, MODEL_L_ONLY, and the hybrid twins (*_hyb: + # Model L's route only; build_dx12.py is_model_l) + self.assertFalse([k for k in extra if not (k[1].startswith("l_") or k[0].startswith("l_") or k[0] in B.MODEL_L_ONLY + or k[0].endswith("_hyb"))]) def test_model_k_adds_only_model_k_kernels(self): diff --git a/clean/dx12/vgpr_limits.json b/clean/dx12/vgpr_limits.json index 6ce8f299..492da741 100644 --- a/clean/dx12/vgpr_limits.json +++ b/clean/dx12/vgpr_limits.json @@ -4,6 +4,27 @@ "l_swin_block_c32_enc_nc": 96, "l_swin_block_c32_enc_lin_nc": 96, "l_swin_block_c32_dec_nc": 96, + "l_swin_block_c32_enc_hyb": 96, + "l_swin_block_c32_dec_hyb": 96, + "l_swin_block_c32_enc_lin_hyb": 96, + "l_swin_block_c32_dec_coef12_hyb": 96, + "l_swin_block_c32_enc_lin_hyb_dump": 256, + "l_swin_block_c32_enc_lin_hyb_mutdump": 256, + "l_swin_block_c32_dec_coef12_hyb_dump": 256, + "l_swin_block_c32_dec_coef12_hyb_mutdump": 256, + "l_swin_block_c32_enc_hyb_dump": 256, + "l_swin_block_c32_dec_hyb_dump": 256, + "l_swin_block_c32_enc_hyb_mut": 128, + "l_swin_block_c32_dec_hyb_mut": 128, + "l_swin_block_c32_enc_hyb_mutdump": 256, + "swin_block_c64_hyb": 192, + "swin_block_c64_add_hyb": 192, + "swin_block_c96_hyb": 256, + "swin_block_c96_add_hyb": 192, + "swin_block_c128_hyb": 256, + "swin_bneck6_hyb": 256, + "swin_bneck6_hyb_mut": 256, + "swin_bneck6_hyb_tmo": 256, "l_swin_block_c32_dec_coef12_nc": 96, "swin_block_c64_nc": 192, "swin_block_c64_add_ex_nc": 192, diff --git a/clean/kernels/post.hip b/clean/kernels/post.hip index ca6896a0..ce3d956a 100644 --- a/clean/kernels/post.hip +++ b/clean/kernels/post.hip @@ -262,7 +262,26 @@ WG_FN PostTexel load_texel12(const v4f* color, const u32* A8, const u32* C4, int } // Catmull-Rom (cubic convolution, a = -1/2) weights for taps -1, 0, 1, 2 at fraction f. +// WG_CR_C2 = 1 (Q11 part C evaluation builds only, l_post__morig.co): the original's form, correction C2 +// (handoff PRESET-M-C1C2-machine-code.md, L part 3 item 3), float32 one rounding per operation (contraction is off): +// f2 = f f, f3 = f f2, w0 = fused(-(f + f3), 1/2, f2), w1 = fused(f3, 1.5, -RN(2.5 f2)) + 1, w3 = RN(RN(f3 - f2) 1/2), +// w2 = ((1 - w0) - w1) - w3. WG_CR_C2 = 2: the check's mutant (w1 with the product 2.5 f2 fused instead). +#ifndef WG_CR_C2 +#define WG_CR_C2 0 +#endif WG_FN void cr_weights(float f, float w[4]) { +#if WG_CR_C2 + const float f2 = f * f, f3 = f * f2; + w[0] = __builtin_fmaf(-(f + f3), 0.5f, f2); +#if WG_CR_C2 == 2 + w[1] = __builtin_fmaf(f3, 1.5f, __builtin_fmaf(-2.5f, f2, 1.0f)); +#else + w[1] = __builtin_fmaf(f3, 1.5f, -(2.5f * f2)) + 1.0f; +#endif + w[3] = (f3 - f2) * 0.5f; + w[2] = ((1.0f - w[0]) - w[1]) - w[3]; + return; +#endif w[0] = f * (-0.5f + f * (1.0f - 0.5f * f)); w[1] = 1.0f + f * f * (-2.5f + 1.5f * f); w[2] = f * (0.5f + f * (2.0f - 1.5f * f)); diff --git a/clean/kernels/swin.hip b/clean/kernels/swin.hip index 9cbde7a7..7bf8c77a 100644 --- a/clean/kernels/swin.hip +++ b/clean/kernels/swin.hip @@ -17,6 +17,7 @@ // from there; the Swin block streams them in double-buffered units of at most 8 KB, so every weight byte is read // from global memory once per 8 or 16 windows instead of once per window. #include "wg_platform.h" +#include "wg_hyb.h" // the HYBRID block form (swin_block<..., HYB = true>) // Build-time knobs (build.py variants; defaults are the shipped configuration): // WG_SWIN_THREADS default lanes per Swin block workgroup (128, 256 or 512); WG_THREADS_ overrides one @@ -461,12 +462,16 @@ struct SwinFp8Src { }; template + class EPI = SwinStoreEpi, bool RESIDENT = false, class SRC = SwinFp8Src, bool HYB = false> WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8* __restrict__ out, const u8* __restrict__ w, int H, int W, int ox, int oy, u8* __restrict__ aux WG_PROF_ARG_DECL, const void* epi = nullptr, int nwg = 0) { typedef BlockLayout L; constexpr int KC = L::KC, HALF = C / 2, MAXU = L::MAXU; + // HYB: the hybrid block form (clean/kernels/wg_hyb.h; the twin of the D3D12 SW_HYB kernels, same bytes): float32 FP8 + // chains with one FP16 rounding at each chain's end, the original's norm / softmax / GELU and start values. Only the + // plain block path has it. + static_assert(!HYB || (MCO == 0 && ECL == 0 && !RESIDENT && !WG_AV8 && !WG_SMAX16), "HYB: plain block path only"); // RESIDENT (kernels *_r): the block's whole weight stream (every head and MLP unit) stays in LDS for the whole // launch; each wave runs windows wave_id, wave_id + total waves, ... with no staging and no barrier after the first // copy. Same arithmetic per window as the staged kernel (bit-identical). @@ -531,7 +536,12 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8* if constexpr (SRC::kCustom) { SRC::template load(x, pr, pc, H, W, hh, xf[i]); } else { - if constexpr (ECL == 0) load_frags(x + src, xa[i]); + if constexpr (HYB && ADD) { // hybrid decoder: E by the coarse mirror (shared_dec_border), S by the fine one + const int cr = 2 * wg_mirror(pr >> 1, H >> 1) + (pr & 1), cc = 2 * wg_mirror(pc >> 1, W >> 1) + (pc & 1); + load_frags(x + ((size_t)(cr * W + cc) * C) + hh * HALF, xa[i]); + } else if constexpr (ECL == 0) { + load_frags(x + src, xa[i]); + } if (ADD) load_frags(xadd + src, xb[i]); } #endif @@ -576,7 +586,21 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8* if constexpr (SRC::kCustom) acc[i][k] = xf[i][k]; else acc[i][k] = wg_unpack_fp8(xa[i][k]); if (ADD) acc[i][k] += wg_unpack_fp8(xb[i][k]); - if (!(WG_NORM_DOT4 && !ADD && !SRC::kCustom)) ss += hsum8(acc[i][k] * acc[i][k]); + if constexpr (HYB) { + if (ADD) acc[i][k] = hyb_round16(acc[i][k]); // x = RN16(dec(P) + dec(S)) + } else if (!(WG_NORM_DOT4 && !ADD && !SRC::kCustom)) { + ss += hsum8(acc[i][k] * acc[i][k]); + } + } + if constexpr (HYB) { // y = the original's norm; a = FP8(y); the Wp chain starts at RN16(bp + x) + v8f y[KC]; + hyb_norm(acc[i], g1, hh, y); +#pragma unroll + for (int k = 0; k < KC; ++k) { + af[i][k] = wg_pack_q8(y[k]); + acc[i][k] = hyb_round16(acc[i][k] + WG_BIAS8(bp + k * 16)); + } + continue; } if (WG_NORM_DOT4 && !ADD && !SRC::kCustom) { #pragma unroll @@ -640,7 +664,8 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8* } #pragma unroll for (int i = 0; i < WPW; ++i) { - uf[i][mt] = pack_in(accq[i]); + if constexpr (HYB) uf[i][mt] = hyb_q8_of_chain(accq[i]); + else uf[i][mt] = pack_in(accq[i]); #if WG_AV8 vt[i][mt] = pack_in(accv[i]); #else @@ -660,6 +685,14 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8* v2i of[WPW][2]; #pragma unroll for (int i = 0; i < WPW; ++i) { + if constexpr (HYB) { // s = one 32-term FP8 op started at B; the original's weights; o = FP8(RN16(P V)) + v8f sh = wg_wmma_fp8(uf[i][0], uf[i][0], bias); + sh = wg_wmma_fp8(uf[i][1], uf[i][1], sh); + const v8h ph = hyb_softmax_p(sh, hh); +#pragma unroll + for (int dt = 0; dt < 2; ++dt) of[i][dt] = hyb_q8_of_chain(wg_wmma_f16(vt[i][dt], ph, wg_splat(0.0f))); + continue; + } v8f s = wg_wmma_fp8(uf[i][0], uf[i][0], wg_splat(0.0f)); // S = U^T U is symmetric s = wg_wmma_fp8(uf[i][1], uf[i][1], s); #if WG_SMAX16 && !WG_AV8 && !(defined(WG_ABLATE_CAP) && WG_ABLATE_CAP) && !(defined(WG_ABLATE_SMAX) && WG_ABLATE_SMAX) @@ -768,6 +801,18 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8* const float* b2 = (const float*)(w + L::B2) + hh * 8; #pragma unroll for (int i = 0; i < WPW; ++i) { + if constexpr (HYB) { // z = RN16(the Wp chain); m = FP8(norm(z)); the W2 chain starts at RN16(b2 + z) + v8f y[KC]; +#pragma unroll + for (int k = 0; k < KC; ++k) acc[i][k] = hyb_round16(acc[i][k]); + hyb_norm(acc[i], g2, hh, y); +#pragma unroll + for (int k = 0; k < KC; ++k) { + mf[i][k] = wg_pack_q8(y[k]); + acc[i][k] = hyb_round16(acc[i][k] + WG_BIAS8(b2 + k * 16)); + } + continue; + } float ss = 0.0f; #pragma unroll for (int k = 0; k < KC; ++k) ss += hsum8(acc[i][k] * acc[i][k]); @@ -854,6 +899,10 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8* for (int g = 0; g < GROUP; ++g) #pragma unroll for (int i = 0; i < WPW; ++i) { + if constexpr (HYB) { // GELU16 of RN16(h), the W1 chain started at b1 + hf[g][i] = hyb_gelu_q8(hyb_round16(hacc[g][i])); + continue; + } #if (WG_GELU_MODE == 3 || WG_GELU_MODE == 4) && !(defined(WG_ABLATE_PACK) && WG_ABLATE_PACK) && !(defined(WG_ABLATE_GELU) && WG_ABLATE_GELU) hf[g][i] = wg_hidden_mix(hacc[g][i]); continue; @@ -922,7 +971,7 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8* if (dst[i] < 0) continue; v2i o[KC]; #pragma unroll - for (int k = 0; k < KC; ++k) o[k] = wg_pack_q8(acc[i][k]); + for (int k = 0; k < KC; ++k) o[k] = HYB ? hyb_q8_of_chain(acc[i][k]) : wg_pack_q8(acc[i][k]); #if defined(WG_ABLATE_STORE) && WG_ABLATE_STORE int chk = 0; // timing-only: store only if an impossible pattern shows up, so the packed values stay alive #pragma unroll @@ -963,7 +1012,8 @@ constexpr int kGemmThreads = WG_GEMM_THREADS; // patch merge / expand workgroup // Patch merge (spec 02): merged(i, j) = q8(Wm concat(TL, TR, BL, BR) + bm) from the q8 block output s [H][W][C]. // Lane half h loads the 2C contiguous bytes of row 2i + h (left and right token), i.e. z[h*2C, (h+1)*2C). // NT groups of 16 merged tokens per wave; Wm is staged in LDS once per workgroup. -template +// HYB (the hybrid form, kernels patch_merge_c*_hyb in swin_hyb.hip): FP8(clamp(RN16(chain), +-6.28125)) (hyb_merge_q8). +template WG_FN void patch_merge(const u8* __restrict__ s, u8* __restrict__ out, const u8* __restrict__ w, int H, int W) { constexpr int KZ = C / 4, MO = CO / 16; // 4C / 16 input groups, CO / 16 output tiles constexpr int BM = 0, WM = 4 * CO, WBYTES = MO * KZ * 256; @@ -998,13 +1048,15 @@ WG_FN void patch_merge(const u8* __restrict__ s, u8* __restrict__ out, const u8* } #pragma unroll for (int i = 0; i < NT; ++i) - if (tok[i] < ntok) *(v2i*)(out + (size_t)tok[i] * CO + hh * (CO / 2) + 8 * mt) = wg_pack_q8(acc[i]); + if (tok[i] < ntok) + *(v2i*)(out + (size_t)tok[i] * CO + hh * (CO / 2) + 8 * mt) = HYB ? hyb_merge_q8(acc[i]) : wg_pack_q8(acc[i]); } } // Expand (spec 02): e = q8(We y + be) for each low-resolution token, written as an FP8 map e [2Hl][2Wl][C] // (sub-pixel k goes to (2r + k/2, 2c + k%2)). The decoder block then adds the skip map (ADD = true). -template +// HYB (the hybrid form, kernels expand_c*_hyb in swin_hyb.hip): FP8(RN16(chain)) (hyb_q8_of_chain). +template WG_FN void expand(const u8* __restrict__ y, u8* __restrict__ e, const u8* __restrict__ w, int Hl, int Wl) { constexpr int KC = CL / 16, G = C / 16; constexpr int BE = 0, WE = 16 * C, WBYTES = 4 * G * KC * 256; @@ -1040,7 +1092,7 @@ WG_FN void expand(const u8* __restrict__ y, u8* __restrict__ e, const u8* __rest for (int i = 0; i < NT; ++i) acc[i] = wg_wmma_fp8(f, yf[i][kc], acc[i]); } #pragma unroll - for (int i = 0; i < NT; ++i) res[i][g] = wg_pack_q8(acc[i]); + for (int i = 0; i < NT; ++i) res[i][g] = HYB ? hyb_q8_of_chain(acc[i]) : wg_pack_q8(acc[i]); } #pragma unroll for (int i = 0; i < NT; ++i) { diff --git a/clean/kernels/swin_hyb.hip b/clean/kernels/swin_hyb.hip new file mode 100644 index 00000000..e342f48a --- /dev/null +++ b/clean/kernels/swin_hyb.hip @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: MIT +// The HYBRID form's shared-stage kernels (WG_L_FORM = hybrid): the HIP twins of the D3D12 ports in +// clean/dx12/kernels/hyb_swin.hlsl (blocks, SW_HYB=1) and hyb_gemm.hlsl (merges, expands), the same bytes. swin.hip's +// templates with HYB = true (swin_block: one window per wave, 256 lanes; patch_merge / expand: WG_GEMM_THREADS lanes, +// WG_NT token groups). Its own code object (swin_hyb.co), so swin.co stays as it is. +// swin_block_c64_hyb, swin_block_c64_add_hyb, swin_block_c96_hyb, swin_block_c96_add_hyb, swin_block_c128_hyb +// (x, xadd | null, out, w, H, W, ox, oy); grid ceil(windows / 8) x 256 +// patch_merge_c64_hyb, patch_merge_c96_hyb (s, out, w, H, W); expand_c96_hyb, expand_c128_hyb (y, e, w, Hl, Wl); +// grid ceil(tokens / (8 x WG_NT x 16)) x 256 +#define WG_SWIN_NO_EXPORTS 1 +#include "swin.hip" + +static_assert(WG_GEMM_THREADS == 256 && WG_NT == 2, "swin_hyb: the D3D12 twins' merge / expand geometry"); + +#define WG_HYB_BLOCK(name, C, HEADS, MLP, ADD, MINW) \ + WG_KERNEL_NW(name, 256, MINW)(const u8* x, const u8* xadd, u8* out, const u8* w, int H, int W, int ox, int oy) { \ + swin_block( \ + x, xadd, out, w, H, W, ox, oy, nullptr); \ + } +WG_HYB_BLOCK(swin_block_c64_hyb, 64, 2, 256, false, WG_MINW_C64) +WG_HYB_BLOCK(swin_block_c64_add_hyb, 64, 2, 256, true, WG_MINW_C64) +WG_HYB_BLOCK(swin_block_c96_hyb, 96, 4, 384, false, WG_MINW_C96) +WG_HYB_BLOCK(swin_block_c96_add_hyb, 96, 4, 384, true, WG_MINW_C96) +WG_HYB_BLOCK(swin_block_c128_hyb, 128, 4, 512, false, WG_MINW_C128) + +WG_KERNEL_N(patch_merge_c64_hyb, WG_GEMM_THREADS)(const u8* s, u8* out, const u8* w, int H, int W) { + patch_merge<64, 96, WG_NT, true>(s, out, w, H, W); +} +WG_KERNEL_N(patch_merge_c96_hyb, WG_GEMM_THREADS)(const u8* s, u8* out, const u8* w, int H, int W) { + patch_merge<96, 128, WG_NT, true>(s, out, w, H, W); +} +WG_KERNEL_N(expand_c96_hyb, WG_GEMM_THREADS)(const u8* y, u8* e, const u8* w, int Hl, int Wl) { + expand<96, 64, WG_NT, true>(y, e, w, Hl, Wl); +} +WG_KERNEL_N(expand_c128_hyb, WG_GEMM_THREADS)(const u8* y, u8* e, const u8* w, int Hl, int Wl) { + expand<128, 96, WG_NT, true>(y, e, w, Hl, Wl); +} diff --git a/clean/kernels/wg_hyb.h b/clean/kernels/wg_hyb.h new file mode 100644 index 00000000..ac8c08bb --- /dev/null +++ b/clean/kernels/wg_hyb.h @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: MIT +// HIP twin of clean/dx12/hlsl/wg_hyb.hlsli: the element-wise steps of the HYBRID block form (SW_HYB / WG_L_FORM=hybrid), +// the same values computed directly instead of from the D3D12 arena tables. +// - RN16: the (_Float16) conversion (v_cvt_f16_f32, round to nearest even). +// - FP16 x FP16 and FP16 +- FP16 in float32 rounded once to FP16 are exact (24 >= 2 x 11 + 2), and every product here +// is of two FP16 values (exact in float32), so clang's default contraction into fma gives the same bits. +// - rsqrt / rcp: 1 / sqrtf(x) and 1 / x with HIP's default correctly rounded division and square root, as the oracle's +// float32 (clean/l/reference/oracle.py _rsqrt32, _rcp32). +// - the softmax weight's two fma steps: the exhaustive fact softmax_e_f32_equals_fma16 (clean/l/reference/hyb_tables.py). +// Every helper is checked for all FP16 inputs against the oracle by clean/l/runner/l_hyb_probe_check.py --hip. +#pragma once +#include "wg_platform.h" + +#define WG_HYB_SCORE_C 0.0171813964843750f // FP16 0x2466 +#define WG_HYB_SCORE_CLAMP 0.55615234375f // FP16 0x3873 +#define WG_HYB_POLY_T 0.92724609375f // FP16 0x3B6B +#define WG_HYB_POLY_P 1.375f // FP16 0x3D80 +#define WG_HYB_GELU_A 0.412109375f // FP16 0x3698 +#define WG_HYB_GELU_B 0.0810546875f // FP16 0x2D30 +#define WG_HYB_MERGE_CLAMP 6.28125f + +WG_FN float hyb_rn16(float x) { return (float)(_Float16)x; } +WG_FN v8f hyb_round16(v8f d) { + v8f r; + for (int i = 0; i < 8; ++i) r[i] = hyb_rn16(d[i]); + return r; +} +WG_FN unsigned hyb_bits16(float x) { return (unsigned)__builtin_bit_cast(unsigned short, (_Float16)x); } +WG_FN float hyb_from16(unsigned b) { return (float)__builtin_bit_cast(_Float16, (unsigned short)b); } + +// the chain end before an FP8 point: RN16, then the saturating FP8 pack (two roundings, by design) +WG_FN v2i hyb_q8_of_chain(v8f acc) { return wg_pack_q8(hyb_round16(acc)); } + +// GELU16 (oracle gelu_q7): every step its own FP16 rounding; NaN takes y = -2 (fmax returns the number) +WG_FN float hyb_gelu16(float h) { + const float y = __builtin_fminf(__builtin_fmaxf(h, -2.0f), 2.0f); + const float inner = hyb_rn16(WG_HYB_GELU_A - hyb_rn16(WG_HYB_GELU_B * __builtin_fabsf(y))); + return hyb_rn16(h * hyb_rn16(0.5f + hyb_rn16(y * inner))); +} +WG_FN v2i hyb_gelu_q8(v8f h) { + v8f g; + for (int i = 0; i < 8; ++i) g[i] = hyb_gelu16(h[i]); + return wg_pack_q8(g); +} + +// the softmax weight e of 8 FP16 scores (float values of FP16 numbers) +WG_FN v8f hyb_softmax_e8(v8f s) { + v8f e; + for (int i = 0; i < 8; ++i) { + float u = hyb_rn16(s[i] * WG_HYB_SCORE_C); + u = __builtin_fminf(__builtin_fmaxf(u, -WG_HYB_SCORE_CLAMP), WG_HYB_SCORE_CLAMP); + const float t = hyb_rn16(__builtin_fmaf(u, -u, WG_HYB_POLY_T)); + const float p = hyb_rn16(__builtin_fmaf(u, t, WG_HYB_POLY_P)); + e[i] = hyb_from16((hyb_bits16(p) & 0x03ffu) << 5); + } + return e; +} + +WG_FN float hyb_rsq16(float s) { return hyb_rn16(1.0f / __builtin_sqrtf(s)); } // S' >= 2^-13 +WG_FN float hyb_rcp16(float x) { return hyb_rn16(1.0f / x); } // normal sum, normal result + +WG_FN float hyb_merge_clamp(float x) { + return x != x ? x : __builtin_fminf(__builtin_fmaxf(x, -WG_HYB_MERGE_CLAMP), WG_HYB_MERGE_CLAMP); +} + +// The merges' FP8 point: FP8(clamp(RN16(chain), +-6.28125)), NaN kept (oracle run_stage_q7: q8(clamp(mm8(...)))). +WG_FN v2i hyb_merge_q8(v8f acc) { + v8f r = hyb_round16(acc); +#pragma unroll + for (int i = 0; i < 8; ++i) r[i] = hyb_merge_clamp(r[i]); + return wg_pack_q8(r); +} +WG_FN int hyb_mirror_fine(int p, int n) { + const int a = p < 0 ? -p : p; + const int m = 2 * n - 2 - a; + return (n - 1) < (a < m ? a : m) ? (n - 1) : (a < m ? a : m); +} +WG_FN int hyb_mirror_coarse(int p, int n, int nc) { + const int c = p >> 1; + const int a = c < 0 ? -c : c; + const int m = 2 * (n >> 1) - 2 - a; + const int v = a < m ? a : m; + return (nc - 1) < v ? (nc - 1) : v; +} + +// ---- the block's norm and softmax row in swin.hip's register layout (swin_block<..., HYB = true>) ------------------- +// Lane (t, hh) holds token t's channels hh C/2 + 8 k + v in tiles k, and the scores of keys 8 hh + v (gfx12 D rows). +// The sums follow the original's fixed FP16 orders (oracle norm_q7, softmax_q7), the partner half by lane exchange; both +// lanes of a token compute the same sums. Same arithmetic as clean/dx12/hlsl/hyb_block.hlsli (FP16 steps exact). +WG_FN unsigned hyb_xor16_u(unsigned v) { return __builtin_bit_cast(unsigned, wg_xor16(__builtin_bit_cast(float, v))); } +WG_FN float hyb_add16(float a, float b) { return hyb_rn16(a + b); } // exact: FP16 operands, one float rounding first + +template +WG_FN void hyb_norm(const v8f* x, const float* gp, unsigned hh, v8f* y) { +#pragma clang fp contract(off) + v8f T[2 * KC]; +#pragma unroll + for (int k = 0; k < KC; ++k) { + v8f q, qp; +#pragma unroll + for (int v = 0; v < 8; ++v) q[v] = hyb_rn16(x[k][v] * x[k][v]); // q = RN16(x x), its own rounding +#pragma unroll + for (int v = 0; v < 8; ++v) qp[v] = wg_xor16(q[v]); + T[k] = hh == 0 ? q : qp; + T[KC + k] = hh == 0 ? qp : q; + } + constexpr int tiles = 2 * KC; + v8f pl[4], pr[4]; + int n = 0; +#pragma unroll + for (int j = 0; j < tiles; j += 4, ++n) +#pragma unroll + for (int v = 0; v < 8; ++v) { + pl[n][v] = hyb_add16(T[j][v], T[j + 2][v]); + pr[n][v] = hyb_add16(T[j + 1][v], T[j + 3][v]); + } + auto group = [&](const v8f* p, int m, int v) { + if (m == 1) return p[0][v]; + if (m == 2) return hyb_add16(p[0][v], p[1][v]); + if (m == 3) return hyb_add16(hyb_add16(p[0][v], p[1][v]), p[2][v]); + return hyb_add16(hyb_add16(p[0][v], p[1][v]), hyb_add16(p[2][v], p[3][v])); + }; + float L[8]; +#pragma unroll + for (int v = 0; v < 8; ++v) L[v] = hyb_add16(group(pl, n, v), group(pr, n, v)); + const float h0 = hyb_add16(hyb_add16(L[0], L[2]), hyb_add16(L[4], L[6])); + const float h1 = hyb_add16(hyb_add16(L[1], L[3]), hyb_add16(L[5], L[7])); + const float S2 = hyb_add16(hyb_add16(h0, h1), 0x1p-13f); + const float r = hyb_rsq16(S2); +#pragma unroll + for (int k = 0; k < KC; ++k) +#pragma unroll + for (int v = 0; v < 8; ++v) y[k][v] = hyb_rn16(x[k][v] * hyb_rn16(r * gp[8 * k + v])); +} + +// the attention weights P of the lane's 8 keys from its 8 score chain values (started at B), as FP16 +WG_FN v8h hyb_softmax_p(v8f s, unsigned hh) { +#pragma clang fp contract(off) + const v8f e = hyb_softmax_e8(hyb_round16(s)); + v8f ep; +#pragma unroll + for (int v = 0; v < 8; ++v) ep[v] = wg_xor16(e[v]); + const v8f lo = hh == 0 ? e : ep, hi = hh == 0 ? ep : e; + float A[8]; +#pragma unroll + for (int v = 0; v < 8; ++v) A[v] = hyb_add16(lo[v], hi[v]); // v = 2 t: a_t, v = 2 t + 1: b_t + const float a = hyb_add16(hyb_add16(A[0], A[2]), hyb_add16(A[4], A[6])); + const float b = hyb_add16(hyb_add16(A[1], A[3]), hyb_add16(A[5], A[7])); + const float w = hyb_rcp16(hyb_add16(a, b)); + v8h p; +#pragma unroll + for (int v = 0; v < 8; ++v) p[v] = (_Float16)(e[v] * w); + return p; +} diff --git a/clean/l/kernels/l_hyb_probe.hip b/clean/l/kernels/l_hyb_probe.hip new file mode 100644 index 00000000..c4dea18b --- /dev/null +++ b/clean/l/kernels/l_hyb_probe.hip @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: MIT +// HIP twin of clean/dx12/kernels/l_hyb_probe.hlsl (same buffer layout): the hybrid helpers of clean/kernels/wg_hyb.h +// for every FP16 input, and the chain-end double rounding of nf float32 values (clean/l/runner/l_hyb_probe_check.py +// --hip). Not used by the runtime. +#include "wg_platform.h" +#include "wg_hyb.h" + +WG_KERNEL_NW(l_hyb_probe, 256, 8)(const float* src, unsigned char* dst, unsigned nf) { + const unsigned t = wg_block_id() * 256u + wg_lane_id(); + const unsigned i0 = 8u * t; + v8f s; + for (int k = 0; k < 8; ++k) s[k] = hyb_from16(i0 + k); + const v2i g = hyb_gelu_q8(s); + *(v2i*)(dst + i0) = g; + const v8f e = hyb_softmax_e8(s); + unsigned short* e16 = (unsigned short*)(dst + 65536); + unsigned short* r16 = (unsigned short*)(dst + 196608); + unsigned short* c16 = (unsigned short*)(dst + 327680); + float* cl = (float*)(dst + 458752); + for (int k = 0; k < 8; ++k) { + e16[i0 + k] = (unsigned short)hyb_bits16(e[k]); + r16[i0 + k] = (unsigned short)hyb_bits16(hyb_rsq16(s[k])); + c16[i0 + k] = (unsigned short)hyb_bits16(hyb_rcp16(s[k])); + cl[i0 + k] = hyb_merge_clamp(s[k]); + } + const unsigned ic = i0 < nf - 8u ? i0 : nf - 8u; + v8f x; + for (int m = 0; m < 8; ++m) x[m] = src[ic + m]; + const v2i q = hyb_q8_of_chain(x); + if (i0 < nf) *(v2i*)(dst + 720896u + i0) = q; +} diff --git a/clean/l/kernels/l_post.hip b/clean/l/kernels/l_post.hip index 9b198442..d352772b 100644 --- a/clean/l/kernels/l_post.hip +++ b/clean/l/kernels/l_post.hip @@ -22,11 +22,29 @@ // l_post_filter_ldr / l_post_filter_ldr_h: the output stage for that display image (Q2): Model M's 5-tap filter // (post.hip filter_row_c1, lz5) with no clamp to [1e-4, 3], no decode and no exposure; 3 x f32, or RGBA16F rows with pi // folded in (framebuffer_half(v pi), alpha 1). +// +// Q11 part C evaluation switches (all 0 by default: Model L's kernel as above, unchanged), built together as +// l_post__morig.co = Model M's ORIGINAL reconstruction (handoff preset-M/from-spec/Q11C-M-POST-ITEMS.md), which the +// runtime runs for Model M with WG_M_POST=orig (HIP only), with M's scale s and M's coefficients: +// WG_LP_M = 1: M's three steps (HDR; the LDR forms are the same for both presets): D1 a 3 x 3 gather (columns +// ax - 1 ... ax + 1, rows b - 1 ... b + 1); D2 the two candidates combined in the ENCODED domain, +// H = fused(w1 H1 + (w2 H2)) (w1 H1 when w2 = 0), then one decode H_lin = (D(H) / e_prev) e; D3 an invalid or reset +// candidate is E(e_prev colour) (no 32752 cap). +// WG_LP_C1 = 1: the gather exponent in the machine-code form (PRESET-M-C1C2-machine-code.md, for M and L): +// eps = fused_half(h(u v), C, fused_half(h(v v), B, h(h(u u) A))) (0: item 3's form, Model L's kernel; one +// rounding more, the probe's near-miss mutant). +// WG_CR_C2 (post.hip cr_weights): the Catmull-Rom weights' C2 form. +#ifndef WG_LP_M +#define WG_LP_M 0 +#endif +#ifndef WG_LP_C1 +#define WG_LP_C1 0 +#endif #include "post.hip" namespace { -constexpr int kLWin = 4; // taps -1 ... 2 +constexpr int kLWin = WG_LP_M ? 3 : 4; // taps -1 ... 2 (Model L), -1 ... 1 (WG_LP_M) typedef _Float16 h16; // D(.) with a NaN passed through. @@ -111,11 +129,22 @@ WG_FN void l_gather_zero(LGather& g) { // One tap: eps = fused_half(h(u v), C, h(h(h(u u) A) + h(h(v v) B))), w = 2^eps (float32, +0 below 2^-126), // W += w, F_k = fused_half(h(w), f_k, F_k). One FP16 operation per statement (contraction is off in post.hip). -WG_FN void l_gather_add(LGather& g, const LTexel& t, h16 u, h16 v) { +// The tap's exponent (WG_LP_C1: the machine-code form; also called by the C1/C2 probe, l_post_c12_probe.hip). +WG_FN h16 l_eps(h16 u, h16 v, h16 kA, h16 kB, h16 kC) { const h16 uu = u * u, vv = v * v, uv = u * v; - const h16 ta = uu * t.kA, tb = vv * t.kB; +#if WG_LP_C1 == 1 + const h16 ta = uu * kA; + const h16 s = __builtin_fmaf16(vv, kB, ta); + return __builtin_fmaf16(uv, kC, s); +#else + const h16 ta = uu * kA, tb = vv * kB; const h16 s = ta + tb; - const h16 eps = __builtin_fmaf16(uv, t.kC, s); + return __builtin_fmaf16(uv, kC, s); +#endif +} + +WG_FN void l_gather_add(LGather& g, const LTexel& t, h16 u, h16 v) { + const h16 eps = l_eps(u, v, t.kA, t.kB, t.kC); float w = wg_exp2((float)eps); if (!(w >= 0x1p-126f)) w = 0.0f; // flush to zero (also a NaN exponent gives 0, as 2^NaN never occurs here) g.W += w; @@ -170,6 +199,7 @@ WG_FN void l_post_core(const LGather& g, int x, int y, int fx, int fy, const flo const float X = (float)x + 0.5f, Y = (float)y + 0.5f; const int ncand = sl.w2 != 0.0f ? 2 : 1; const float rp = kLdr ? 1.0f : 1.0f / p.e_prev; // IEEE division + constexpr bool kEnc = WG_LP_M && !kLdr; // D2 / D3: candidates in the encoded domain float L[2][3]; for (int n = 0; n < ncand; ++n) { const int k = n == 0 ? sl.first : sl.second; @@ -179,11 +209,11 @@ WG_FN void l_post_core(const LGather& g, int x, int y, int fx, int fy, const flo float h[3]; history_cr(hist_prev, p.dw, p.dh, qx, qy, h); #pragma unroll - for (int i = 0; i < 3; ++i) L[n][i] = kLdr ? h[i] : l_dec(h[i]) * rp; + for (int i = 0; i < 3; ++i) L[n][i] = (kLdr || kEnc) ? h[i] : l_dec(h[i]) * rp; } else { const v4f c = color[fy * p.rw + fx]; #pragma unroll - for (int i = 0; i < 3; ++i) L[n][i] = c[i]; + for (int i = 0; i < 3; ++i) L[n][i] = kEnc ? enc_e(p.e_prev * c[i]) : c[i]; } } float Hl[3], Cl[3]; @@ -191,7 +221,7 @@ WG_FN void l_post_core(const LGather& g, int x, int y, int fx, int fy, const flo #pragma unroll for (int i = 0; i < 3; ++i) { const float comb = ncand == 2 ? __builtin_fmaf(sl.w1, L[0][i], sl.w2 * L[1][i]) : sl.w1 * L[0][i]; - Hl[i] = kLdr ? comb : comb * p.e; + Hl[i] = kLdr ? comb : (kEnc ? (l_dec(comb) * rp) * p.e : comb * p.e); hl_inf |= __builtin_fabsf(Hl[i]) == __builtin_inff(); Cl[i] = kLdr ? sl.fbar[i] : l_dec(sl.fbar[i]); } diff --git a/clean/l/kernels/l_post_c12_probe.hip b/clean/l/kernels/l_post_c12_probe.hip new file mode 100644 index 00000000..18f39d40 --- /dev/null +++ b/clean/l/kernels/l_post_c12_probe.hip @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: MIT +// Q11 part C: probe of the reconstruction's two fused-rounding forms, bit for bit (clean/l/runner/l_post_c12_check.py). +// Built as l_post_c12_probe.co (WG_LP_C1 = 1, WG_CR_C2 = 1: the forms of l_post__morig.co) and +// l_post_c12_probe__mut.co (WG_LP_C1 = 0: item 3's exponent, Model L's kernel; WG_CR_C2 = 2: w1 with 2.5 f2 fused): +// the check must find the mutant's values different. Evaluation only; not in any release build. +#include "l_post.hip" + +// eps[i] = l_eps(u[i], v[i], kA[i], kB[i], kC[i]) (FP16 bits in, FP16 bits out) +WG_KERNEL_N(l_probe_eps, 256)(const unsigned short* in, unsigned short* out, int n) { + const int i = (int)(wg_block_id() * 256u + wg_lane_id()); + if (i >= n) return; + const unsigned short* q = in + 5 * i; + const h16 e = l_eps(__builtin_bit_cast(h16, q[0]), __builtin_bit_cast(h16, q[1]), __builtin_bit_cast(h16, q[2]), + __builtin_bit_cast(h16, q[3]), __builtin_bit_cast(h16, q[4])); + out[i] = __builtin_bit_cast(unsigned short, e); +} + +// w[4 i .. 4 i + 3] = cr_weights(f[i]) (float32) +WG_KERNEL_N(l_probe_cr, 256)(const float* f, float* w, int n) { + const int i = (int)(wg_block_id() * 256u + wg_lane_id()); + if (i >= n) return; + float r[4]; + cr_weights(f[i], r); + for (int k = 0; k < 4; ++k) w[4 * i + k] = r[k]; +} diff --git a/clean/l/kernels/l_q7.hip b/clean/l/kernels/l_q7.hip new file mode 100644 index 00000000..e616095f --- /dev/null +++ b/clean/l/kernels/l_q7.hip @@ -0,0 +1,401 @@ +// SPDX-License-Identifier: MIT +// Model L (issue #89): the width-32 Swin block in its FP16 form (clean/l/reference/oracle.py _block_tokens_q7, +// table consts form "q7", FP8 op model "exact"), for Model L's encoder block and decoder block. +// +// Arithmetic (equal to the oracle bit for bit): +// - one FP8 matrix op (up to 32 FP8 x FP8 products and an FP16 start, one FP16 result) is computed exactly: the +// products (exact in float32) are summed in float32 in ascending k with the float32 sum of |products| alongside; +// while that stays below 63 the sum is exact, otherwise a TwoSum chain gives it exactly as a head and an error sum; +// the start is added with error-free sums and the head rounded to FP16, the tail's sign picking the side where the +// head is exactly an FP16 midpoint (clean/l/runner/l_mm8_f32_check.py op_kernel, tests test_l_mm8_kernel); +// - Q7F-2 (S2a): the FP8 ops run on the FP8 matrix instruction, 16 rows x the window's 16 tokens per 32-wide chunk +// (l_q7_op8.h q7_tile_sum), while the precomputed bound sum |w| x max |a| < 63 holds; an output whose bound fails +// takes the exact scalar op8 above (rare; l_q7_s2a_fallbacks.py counts them on a recorded run); +// - every FP16 addition is exact the same way (an FP16 sum is not always exact in float32); every FP16 product is +// exact in float32 and rounded once; the FMA16 steps round the exact sum once; +// - the norm's r = RN16(1 / sqrt(S')) and the attention weights' w = RN16(1 / sum) are table lookups by the FP16 bit +// pattern (clean/l/runner/l_pack.py q7_tables, computed as the oracle computes them); +// - FP8 rounding: nearest even, saturating at +-448, NaN kept (wg_f32_to_e4m3). +// Weights: the q7 blob (clean/l/runner/l_pack.py pack_q7_block, natural order): FP8 rows (matrix instruction), FP16 +// rows (fallback), sum |w| tables (bound). +// +// Geometry: one 4 x 4 window per workgroup of 32 lanes (one wave); lane = token t (0 ... 15, raster 4 row + col) + +// 16 x half. Window (wy, wx) covers grid rows 4 wy - oy ... and columns 4 wx - ox ...; grid = nwy x nwx workgroups, +// nwx = ceil((W + ox) / 4), nwy = ceil((H + oy) / 4). Only grid positions inside H x W are written. +// +// Build: clang -x hip ... -I clean/kernels clean/l/kernels/l_q7.hip -o l_q7.co +#pragma clang fp contract(off) +#include "wg_platform.h" +#include "l_q7_op8.h" + +namespace q7 { + +using namespace q7c; + +constexpr int C = 32, HEADS = 2, MLP = 128, T = 16; +// q7 blob offsets (pack_q7_block for C 32, 2 heads, MLP 128) +constexpr int OFF_G1 = 0, OFF_G2 = 64, OFF_BP = 128, OFF_B1 = 192, OFF_B2 = 448, OFF_B = 512; +constexpr int OFF_WQK = 1536, OFF_WV = 3584, OFF_WP = 5632, OFF_W1 = 7680, OFF_W2 = 11776; +// the same matrices as FP16 (Q7F-1: exact copies, read without an FP8 decode) +constexpr int OFF_WQK16 = 15872, OFF_WV16 = 19968, OFF_WP16 = 24064, OFF_W1_16 = 28160, OFF_W2_16 = 36352; + +// RMS norm of one token (FP16 values x[32]) -> r = RN16(1 / sqrt(S')) by the table: the even / odd lane tree. +WG_FN float norm_r(const float* x, const u16* rsqrt_tab) { + float q[C]; + for (int c = 0; c < C; ++c) q[c] = mul16(x[c], x[c]); + float hv[2]; + for (int odd = 0; odd < 2; ++odd) { + float e[4]; + for (int t = 0; t < 4; ++t) { + const int c0 = 2 * t + odd; // tiles j = 0 .. 3: channels c0, c0 + 8, c0 + 16, c0 + 24 + e[t] = add16(add16(q[c0], q[c0 + 16]), add16(q[c0 + 8], q[c0 + 24])); + } + hv[odd] = add16(add16(e[0], e[1]), add16(e[2], e[3])); + } + const float s = add16(add16(hv[0], hv[1]), 0x1p-13f); + return from16(rsqrt_tab[bits16(s)]); +} + +// sum |w| tables of the FP8 matrices (Q7F-2, chunk-major [K / 32][rows] float32) +constexpr int WS_WQK = 44544, WS_WV = 44800, WS_WP = 45056, WS_W1 = 45312, WS_W2 = 45824; + +// FP8 activations are kept as packed codes, token-major with the channels contiguous (a8, u, o8, m8, g8): 8 +// consecutive codes of one token are one operand fragment of the FP8 matrix instruction (S2a), as the blob's +// row-major FP8 rows are for the weights. Beside each, the max code of the values each lane wrote (ma, mu, mo, mm, mg: +// [token][writer half]), so a token chunk's amax is the max of its writers' (the same 32 values the sums use); su: +// sum |u| of each head's query / key per writer half (exact: at most 32 FP8 magnitudes), the scores' wsum. +struct Lds { + float x[T][C], v[HEADS][T][32], pw[HEADS][T][T], z[T][C], r[T]; + u8 a8[T][C], u[HEADS][T][32], o8[T][32 * HEADS], m8[T][C], g8[T][32]; + u32 ma[T][2], mu[HEADS][T][2], mo[T][2], mm[T][2], mg[T][2]; + float su[HEADS][T][2]; +}; + +WG_FN void dec8(const u8* p, float* a) { // 32 FP8 codes -> values + for (int k = 0; k < 32; ++k) a[k] = fp8v(p[k]); +} +// The exact fallback of one output (rare: the bound failed): op8 on 32 FP8 activation codes and an FP16 weight row, +// or on two code rows (the scores). +WG_FN_NOINLINE float fb_w(float start, const u8* act, const _Float16* row) { + float av[32]; + dec8(act, av); + return op8(start, av, [&](int k) { return (float)row[k]; }, 32); +} +WG_FN_NOINLINE float fb_uu(float start, const u8* q, const u8* key) { + float av[32], kv[32]; + dec8(q, av); + dec8(key, kv); + return op8(start, av, [&](int k) { return kv[k]; }, 32); +} +// Q7_HYB (Q11 part C evaluation, code object l_q7__hyb.co, runtime WG_L_FORM=hybrid; NOT the original): every FP8 chain +// of the C32 block accumulates in float32 (the products, exact in float32, added one at a time from the start, k and +// chunks ascending) and is rounded to FP16 once at its end: the single-chunk chains, the scores (32 terms from B), Wp +// across both heads and W2 across the four MLP slices. Everything else as the FP16 form. Oracle: +// clean/l/reference/oracle.py FP8 op model "f32chain" (as l_q7w.hip Q7_HYB). Q7_HYB_MUTANT (check only, +// l_q7__hybmut.co): W2 rounded to FP16 after every slice (the per-chunk rounding) - the check must catch it. +#ifndef Q7_HYB +#define Q7_HYB 0 +#endif +#ifndef Q7_HYB_MUTANT +#define Q7_HYB_MUTANT 0 +#endif +#if Q7_HYB +WG_FN float hyb_w(float acc, const u8* act, const _Float16* row) { // float32, not rounded + for (int k = 0; k < 32; ++k) acc = acc + fp8v(act[k]) * (float)row[k]; + return acc; +} +WG_FN float hyb_uu(float acc, const u8* q, const u8* key) { + for (int k = 0; k < 32; ++k) acc = acc + fp8v(q[k]) * fp8v(key[k]); + return acc; +} +#endif +// The lane's 16 outputs of a 32-row pair of tiles: output i is row 16 (i >> 3) + 8 hf + (i & 7). +WG_FN int orow(int i, u32 hf) { return 16 * (i >> 3) + 8 * (int)hf + (i & 7); } + +// The block on one window. In: x (FP16 values) for the 16 members already in L.x. Out: the FP16 block output of +// member t, channel orow(i, hf) in out16[i]. +WG_FN void block(Lds& L, const u8* w, const u16* tab, u32 t, u32 hf, float* out16) { + const _Float16* wh = (const _Float16*)w; + const u8* wb = w; + // norm 1 -> FP8 a8 + if (hf == 0) L.r[t] = norm_r(L.x[t], tab); + wg_barrier_lds(); + { + u32 m = 0; + for (int i = 0; i < 16; ++i) { + const int c = 16 * hf + i; + const float g = (float)wh[OFF_G1 / 2 + c]; + const u8 code = e8(mul16(L.x[t][c], mul16(L.r[t], g))); + L.a8[t][c] = code; + m = q7_maxcode(m, code); + } + L.ma[t][hf] = m; + } + wg_barrier_lds(); + // u = FP8(Wqk a8), v = Wv a8: 64 rows each = 4 tiles of 16 rows (tiles 2h, 2h + 1 = head h), one chunk + { + const float amx = q7_amax(L.ma[t][0], L.ma[t][1]); + const u8* bt = &L.a8[t][8 * hf]; + u32 mq[HEADS] = {0, 0}; + float sq[HEADS] = {0.0f, 0.0f}; +#pragma unroll + for (int tile = 0; tile < 4; ++tile) { + const int r0 = 16 * tile; + const v8f S = q7_tile_sum(wb + OFF_WQK + (r0 + (int)t) * C + 8 * hf, bt); + const float* ws = (const float*)(wb + WS_WQK) + r0 + 8 * hf; +#pragma unroll + for (int j = 0; j < 8; ++j) { + const int r = r0 + 8 * (int)hf + j; +#if Q7_HYB + const float y = h16(hyb_w(0.0f, L.a8[t], (const _Float16*)(wb + OFF_WQK16) + r * C)); + (void)S; (void)ws; (void)amx; +#else + const float y = q7_combine(S[j], 0.0f, ws[j], amx, [&](float s0) { + return fb_w(s0, L.a8[t], (const _Float16*)(wb + OFF_WQK16) + r * C); + }); +#endif + const u8 code = e8(y); + L.u[r >> 5][t][r & 31] = code; + mq[tile >> 1] = q7_maxcode(mq[tile >> 1], code); + sq[tile >> 1] = sq[tile >> 1] + __builtin_fabsf(fp8v(code)); + } + } +#pragma unroll + for (int tile = 0; tile < 4; ++tile) { + const int r0 = 16 * tile; + const v8f S = q7_tile_sum(wb + OFF_WV + (r0 + (int)t) * C + 8 * hf, bt); + const float* ws = (const float*)(wb + WS_WV) + r0 + 8 * hf; +#pragma unroll + for (int j = 0; j < 8; ++j) { + const int r = r0 + 8 * (int)hf + j; +#if Q7_HYB + L.v[r >> 5][t][r & 31] = h16(hyb_w(0.0f, L.a8[t], (const _Float16*)(wb + OFF_WV16) + r * C)); +#else + L.v[r >> 5][t][r & 31] = q7_combine(S[j], 0.0f, ws[j], amx, [&](float s0) { + return fb_w(s0, L.a8[t], (const _Float16*)(wb + OFF_WV16) + r * C); + }); +#endif + } + } + for (int h = 0; h < HEADS; ++h) { + L.mu[h][t][hf] = mq[h]; + L.su[h][t][hf] = sq[h]; + } + } + wg_barrier_lds(); + // scores s[h][t][j] = RN16(B + u_t . u_j): per head one tile, A = keys (row j), B = queries (token t), the same + // fragments; the lane's outputs are query t, keys 8 hf + jj; wsum = sum |u_j|, amax = max |u_t| +#pragma unroll + for (int h = 0; h < HEADS; ++h) { + const u8* up = &L.u[h][t][8 * hf]; + const v8f S = q7_tile_sum(up, up); + const float amx = q7_amax(L.mu[h][t][0], L.mu[h][t][1]); +#pragma unroll + for (int jj = 0; jj < 8; ++jj) { + const int j = 8 * (int)hf + jj; + const float bij = (float)wh[OFF_B / 2 + (h * T + (int)t) * T + j]; +#if Q7_HYB + const float s = h16(hyb_uu(bij, L.u[h][t], L.u[h][j])); +#else + const float s = q7_combine(S[jj], bij, L.su[h][j][0] + L.su[h][j][1], amx, [&](float s0) { + return fb_uu(s0, L.u[h][t], L.u[h][j]); + }); +#endif + const float cs = 0.017181396484375f, lim = 0.55615234375f; // 0x2466, 0x3873 + const float uu = __builtin_fminf(__builtin_fmaxf(mul16(s, cs), -lim), lim); + const float tt = fma16(uu, -uu, 0.92724609375f); // 0x3B6B + const float p = fma16(uu, tt, 1.375f); // 0x3D80 + L.pw[h][t][j] = from16((bits16(p) & 0x3ffu) << 5); + } + } + wg_barrier_lds(); + // the attention weights of head hf, query t (e in pw's row) + { + float e[T]; + for (int j = 0; j < T; ++j) e[j] = L.pw[hf][t][j]; + float a[2]; + for (int lo = 0; lo < 2; ++lo) { + float pr[4]; + for (int k = 0; k < 4; ++k) pr[k] = add16(e[lo + 2 * k], e[lo + 8 + 2 * k]); + a[lo] = add16(add16(pr[0], pr[1]), add16(pr[2], pr[3])); + } + const float wsum = from16(tab[65536 + bits16(add16(a[0], a[1]))]); + for (int j = 0; j < T; ++j) L.pw[hf][t][j] = mul16(e[j], wsum); + } + wg_barrier_lds(); + // o = RN16(P v) of head hf (16 FP16 x FP16 products, summed exactly), FP8 + { + u32 m = 0; + for (int d = 0; d < 32; ++d) { + float s = 0.0f, c = 0.0f; + for (int j = 0; j < T; ++j) { + float e; + two_sum(s, L.pw[hf][t][j] * L.v[hf][j][d], s, e); + c = c + e; + } + float z, tl; + two_sum(s, c, z, tl); + const u8 code = e8(rn16_pair(z, tl)); + L.o8[t][32 * hf + d] = code; + m = q7_maxcode(m, code); + } + L.mo[t][hf] = m; // chunk hf of o8 is this lane's alone + } + wg_barrier_lds(); + // z = Wp chain (head 0, then head 1) from RN16(bp + x): 32 rows = 2 tiles, K 64 = 2 chunks + float acc[16]; + for (int i = 0; i < 16; ++i) { + const int c = orow(i, hf); + acc[i] = add16((float)wh[OFF_BP / 2 + c], L.x[t][c]); + } +#pragma unroll + for (int hd = 0; hd < HEADS; ++hd) { + const float amx = q7_amax(L.mo[t][hd], 0u); +#pragma unroll + for (int tile = 0; tile < 2; ++tile) { + const int r0 = 16 * tile; + const v8f S = q7_tile_sum(wb + OFF_WP + (r0 + (int)t) * 32 * HEADS + 32 * hd + 8 * hf, &L.o8[t][32 * hd + 8 * hf]); + const float* ws = (const float*)(wb + WS_WP) + 32 * hd + r0 + 8 * hf; +#pragma unroll + for (int j = 0; j < 8; ++j) { + const int i = 8 * tile + j, c = r0 + 8 * (int)hf + j; +#if Q7_HYB + acc[i] = hyb_w(acc[i], L.o8[t] + 32 * hd, (const _Float16*)(wb + OFF_WP16) + c * 32 * HEADS + 32 * hd); +#else + acc[i] = q7_combine(S[j], acc[i], ws[j], amx, [&](float s0) { + return fb_w(s0, L.o8[t] + 32 * hd, (const _Float16*)(wb + OFF_WP16) + c * 32 * HEADS + 32 * hd); + }); +#endif + } + } + } +#if Q7_HYB + for (int i = 0; i < 16; ++i) acc[i] = h16(acc[i]); // Wp's chain over both heads: one FP16 rounding +#endif + for (int i = 0; i < 16; ++i) L.z[t][orow(i, hf)] = acc[i]; + wg_barrier_lds(); + // norm 2 -> FP8 m8 + if (hf == 0) L.r[t] = norm_r(L.z[t], tab); + wg_barrier_lds(); + { + u32 m = 0; + for (int i = 0; i < 16; ++i) { + const int c = 16 * hf + i; + const u8 code = e8(mul16(L.z[t][c], mul16(L.r[t], (float)wh[OFF_G2 / 2 + c]))); + L.m8[t][c] = code; + m = q7_maxcode(m, code); + } + L.mm[t][hf] = m; + } + for (int i = 0; i < 16; ++i) { + const int c = orow(i, hf); + acc[i] = add16((float)wh[OFF_B2 / 2 + c], L.z[t][c]); + } + wg_barrier_lds(); + // MLP: 4 hidden slices of 32, ascending: W1 slice (2 tiles, K 32) -> GELU -> FP8 g8, then W2's chunk (2 tiles) + const float amx_m = q7_amax(L.mm[t][0], L.mm[t][1]); + for (int sl = 0; sl < MLP / 32; ++sl) { + u32 m = 0; +#pragma unroll + for (int tile = 0; tile < 2; ++tile) { + const int r0 = 32 * sl + 16 * tile; + const v8f S = q7_tile_sum(wb + OFF_W1 + (r0 + (int)t) * C + 8 * hf, &L.m8[t][8 * hf]); + const float* ws = (const float*)(wb + WS_W1) + r0 + 8 * hf; +#pragma unroll + for (int j = 0; j < 8; ++j) { + const int jr = r0 + 8 * (int)hf + j; +#if Q7_HYB + const float hval = h16(hyb_w((float)wh[OFF_B1 / 2 + jr], L.m8[t], (const _Float16*)(wb + OFF_W1_16) + jr * C)); +#else + const float hval = q7_combine(S[j], (float)wh[OFF_B1 / 2 + jr], ws[j], amx_m, [&](float s0) { + return fb_w(s0, L.m8[t], (const _Float16*)(wb + OFF_W1_16) + jr * C); + }); +#endif + const u8 code = e8(gelu16(hval)); + L.g8[t][jr - 32 * sl] = code; + m = q7_maxcode(m, code); + } + } + L.mg[t][hf] = m; + wg_barrier_lds(); + const float amx_g = q7_amax(L.mg[t][0], L.mg[t][1]); +#pragma unroll + for (int tile = 0; tile < 2; ++tile) { + const int r0 = 16 * tile; + const v8f S = q7_tile_sum(wb + OFF_W2 + (r0 + (int)t) * MLP + 32 * sl + 8 * hf, &L.g8[t][8 * hf]); + const float* ws = (const float*)(wb + WS_W2) + 32 * sl + r0 + 8 * hf; +#pragma unroll + for (int j = 0; j < 8; ++j) { + const int i = 8 * tile + j, c = r0 + 8 * (int)hf + j; +#if Q7_HYB + acc[i] = hyb_w(acc[i], L.g8[t], (const _Float16*)(wb + OFF_W2_16) + c * MLP + 32 * sl); +#if Q7_HYB_MUTANT + acc[i] = h16(acc[i]); +#endif +#else + acc[i] = q7_combine(S[j], acc[i], ws[j], amx_g, [&](float s0) { + return fb_w(s0, L.g8[t], (const _Float16*)(wb + OFF_W2_16) + c * MLP + 32 * sl); + }); +#endif + } + } + wg_barrier_lds(); + } +#if Q7_HYB + for (int i = 0; i < 16; ++i) acc[i] = h16(acc[i]); // W2's chain over the four slices: one FP16 rounding +#endif + for (int i = 0; i < 16; ++i) out16[i] = acc[i]; +} + +WG_FN void window_member(int H, int W, int ox, int oy, u32 t, int& pr, int& pc) { + const int nwx = (W + ox + 3) / 4; + const int win = (int)wg_block_id(); + const int wy = win / nwx, wx = win % nwx; + pr = 4 * wy - oy + (int)(t >> 2); + pc = 4 * wx - ox + (int)(t & 3u); +} + +} // namespace q7 + +// Encoder block: x = the FP16 halo map [H + 4][W + 4][32] (grid position (pr, pc) at map pixel (pr + 2, pc + 2)), +// skip unused (may be null), out = the FP8 map [H][W][32] (also the decoder block's skip). w = q7 blob, tab = tables. +WG_KERNEL_NW(l_q7_block_c32_enc, 32, 1)(const u8* x, const u8* skip, u8* out, const u8* w, const u16* tab, int H, + int W, int ox, int oy) { + (void)skip; + WG_LDS q7::Lds L; + const u32 lane = wg_lane_id(), t = lane & 15u, hf = lane >> 4; + int pr, pc; + q7::window_member(H, W, ox, oy, t, pr, pc); + const _Float16* src = (const _Float16*)x + ((size_t)(pr + 2) * (W + 4) + (pc + 2)) * q7::C; + for (int i = 0; i < 16; ++i) L.x[t][16 * hf + i] = (float)src[16 * hf + i]; + wg_barrier_lds(); + float o[16]; + q7::block(L, w, tab, t, hf, o); + if (pr >= 0 && pr < H && pc >= 0 && pc < W) { + u8* dst = out + ((size_t)pr * W + pc) * q7::C; + for (int i = 0; i < 16; ++i) dst[q7::orow(i, hf)] = q7::out8(o[i]); + } +} + +// Decoder block: x = the FP8 projection map [H][W][32] (a member outside the grid takes the coarse mirror of its 2 x 2 +// group, sub-pixel kept), skip = the encoder block's FP8 map [H][W][32] (a member outside the grid takes the mirror +// of its own position), the member's input RN16(P + S); out = the FP16 map [H][W][32]. +WG_KERNEL_NW(l_q7_block_c32_dec, 32, 1)(const u8* x, const u8* skip, u8* out, const u8* w, const u16* tab, int H, + int W, int ox, int oy) { + WG_LDS q7::Lds L; + const u32 lane = wg_lane_id(), t = lane & 15u, hf = lane >> 4; + int pr, pc; + q7::window_member(H, W, ox, oy, t, pr, pc); + const int r = 2 * q7::mirror(pr >> 1, H >> 1) + (pr & 1), c = 2 * q7::mirror(pc >> 1, W >> 1) + (pc & 1); + const int sr = q7::mirror(pr, H), sc = q7::mirror(pc, W); + const u8* pp = x + ((size_t)r * W + c) * q7::C + 16 * hf; + const u8* sp = skip + ((size_t)sr * W + sc) * q7::C + 16 * hf; + for (int i = 0; i < 16; ++i) L.x[t][16 * hf + i] = q7::h16(q7::fp8v(pp[i]) + q7::fp8v(sp[i])); // exact sum, one rounding + wg_barrier_lds(); + float o[16]; + q7::block(L, w, tab, t, hf, o); + if (pr >= 0 && pr < H && pc >= 0 && pc < W) { + u16* dst = (u16*)out + ((size_t)pr * W + pc) * q7::C; + for (int i = 0; i < 16; ++i) dst[q7::orow(i, hf)] = q7::out16(o[i]); + } +} diff --git a/clean/l/kernels/l_q7_common.h b/clean/l/kernels/l_q7_common.h new file mode 100644 index 00000000..a1901d5b --- /dev/null +++ b/clean/l/kernels/l_q7_common.h @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: MIT +// Model L (issue #89): the FP16 form's arithmetic shared by the q7 kernels (l_q7.hip, l_q7w.hip), in ONE place so +// that a native packed-FP16 form (S2b) can replace the helpers here without touching the kernels. The +// D3D12 counterpart is clean/dx12/hlsl/l_q7_common.hlsli (same functions, same bits). +// - h16: one rounding to FP16 (nearest even); add16 / mul16 / fma16: the FP16 steps, each the exact result rounded +// once (an FP16 sum is not always exact in float32: error-free sums, the tail picking the side at a midpoint); +// - the FP8 matrix op is in l_q7_op8.h (S2a), apart from these FP16 steps (S2b); +// - out8 / out16: output stores (plain codes; a NaN's sign is masked by the checks). +// Include after wg_platform.h, with "#pragma clang fp contract(off)" in the including file. +#pragma once + +typedef unsigned short u16; + +namespace q7c { + +WG_FN float h16(float x) { return (float)(_Float16)x; } +WG_FN u32 bits16(float x) { return (u32)__builtin_bit_cast(u16, (_Float16)x); } +WG_FN float from16(u32 b) { return (float)__builtin_bit_cast(_Float16, (u16)b); } +WG_FN float fp8v(u32 code) { return wg_e4m3_to_f32(code); } +WG_FN u8 e8(float x) { return (u8)wg_f32_to_e4m3(x); } +WG_FN bool finite(float x) { return __builtin_isfinite(x); } +WG_FN int mirror(int p, int n) { // about the edge, edge not repeated + p = p < 0 ? -p : p; + return p >= n ? 2 * (n - 1) - p : p; +} +// Output stores: the plain codes (a NaN's sign is back-end dependent; the checks mask it, l_q7_common.hlsli) +WG_FN u8 out8(float x) { return (u8)wg_f32_to_e4m3(x); } +WG_FN u16 out16(float x) { return (u16)bits16(x); } + +WG_FN void two_sum(float a, float b, float& s, float& e) { + s = a + b; + const float bb = s - a; + e = (a - (s - bb)) + (b - bb); +} + +// RN16 of the exact value z + t (z = fl(z + t), |t| at most half an ulp of z): where z is exactly an FP16 midpoint +// the tail's sign picks the side (no tail: ties to even). +WG_FN float rn16_pair(float z, float t) { + if (!finite(z) || t == 0.0f) return h16(z); + const u32 b = __builtin_bit_cast(u32, z); + bool mid; + if (__builtin_fabsf(z) >= 0x1p-14f) { + mid = (b & 0x1fffu) == 0x1000u; + } else { + const float q = z * 0x1p25f; // exact + const int qi = (int)q; + mid = (float)qi == q && (qi & 1); + } + if (!mid) return h16(z); + const bool up = t > 0.0f; // towards +inf + const u32 nb = (up == (z > 0.0f)) ? b + 1u : b - 1u; // next float32 on the tail's side + return h16(__builtin_bit_cast(float, nb)); +} + +WG_FN float add16(float a, float b) { // RN16(a + b), exact + float s, e; + two_sum(a, b, s, e); + return rn16_pair(s, e); +} +WG_FN float mul16(float a, float b) { return h16(a * b); } // FP16 x FP16: exact in float32 +WG_FN float fma16(float a, float b, float c) { return add16(a * b, c); } // a * b exact: RN16 of the exact sum + +WG_FN float gelu16(float h) { + const float y = __builtin_fminf(__builtin_fmaxf(h, -2.0f), 2.0f); + const float A = 0.412109375f, B = 0.0810546875f; // 0x3698, 0x2D30 + return mul16(h, add16(0.5f, mul16(y, add16(A, -mul16(B, __builtin_fabsf(y)))))); +} + +} // namespace q7c diff --git a/clean/l/kernels/l_q7_op8.h b/clean/l/kernels/l_q7_op8.h new file mode 100644 index 00000000..5cbe49ac --- /dev/null +++ b/clean/l/kernels/l_q7_op8.h @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: MIT +// Model L (issue #89): one FP8 matrix op of the q7 kernels (up to 32 FP8 x FP8 products and an FP16 start, one FP16 +// result), computed exactly (clean/l/runner/l_mm8_f32_check.py op_kernel). Kept apart from the FP16 helpers +// (l_q7_common.h) so that the FP8 op (S2a: the matrix instruction with a precomputed bound) and the FP16 +// steps (S2b: native packed FP16) change in separate files. D3D12 counterpart: clean/dx12/hlsl/l_q7_op8.hlsli. +#pragma once + +#include "l_q7_common.h" + +namespace q7c { + +// RN16(start + sum_k a[k] w(k)), exact. a: FP8 values (float), w(k): FP8 value of the weight. The products (exact in +// float32) are summed in float32 in ascending k with the float32 sum of |products| alongside; while that stays below +// 63 the sum is exact, otherwise a TwoSum chain gives it exactly as a head and an error sum; the start is added with +// error-free sums and the head rounded to FP16, the tail's sign picking the side at an FP16 midpoint. +template +WG_FN float op8(float start, const float* a, WF w, int K) { + float acc = 0.0f, aacc = 0.0f; + for (int k = 0; k < K; ++k) { + const float p = a[k] * w(k); + acc = acc + p; + aacc = aacc + __builtin_fabsf(p); + } + if (!finite(acc) || !finite(start)) return h16(acc + start); + float s = acc, c = 0.0f; + if (!(aacc < 63.0f)) { + s = 0.0f; + for (int k = 0; k < K; ++k) { + float e; + two_sum(s, a[k] * w(k), s, e); + c = c + e; + } + } + float u, e1, v, e2, z, e3, t; + two_sum(s, start, u, e1); + two_sum(e1, c, v, e2); + two_sum(u, v, z, e3); + t = e3 + e2; + two_sum(z, t, z, t); + return rn16_pair(z, t); +} +WG_FN float op8(float start, const float* a, const float* w) { // 32 terms, both as values + return op8(start, a, [&](int k) { return w[k]; }, 32); +} + +// ---- S2a: the FP8 matrix instruction under a precomputed bound -------------------------------- +// +// One 32-wide chunk of 16 rows (A) x 16 tokens (B) = two v_wmma_f32_16x16x16_fp8_fp8 (wave32 layout: A lane l row +// l & 15, B lane l token l & 15, both K values k0 + 8 (l >> 4) ... + 7 and k0 + 16 + 8 (l >> 4) ... + 7; D lane l +// token l & 15, rows 8 (l >> 4) + j), each started from 0, their results added in float32. Exactness: every product +// of two FP8 values is a multiple of 2^-18; while sum |a w| < 64 every partial sum, in any order, is a multiple of +// 2^-18 below 2^6 (24 bits): exact in float32 (measured: 0 inexact with a zero accumulator, by the WMMA FP8 probe). +// The kernels bound sum |a w| <= wsum x amax with wsum = sum |w| of the row's chunk (blob table, l_pack.wsum_table) +// and amax = max |a| over the SAME 32 values of the token's chunk (from the codes: max (code & 0x7f), so a NaN code +// 0x7f wins and gives NaN). bounded: wsum x amax < 63 in float32 (a NaN fails it); otherwise the exact scalar op8. +// The start stays out of the accumulator (measured inexact there) and is added afterwards with one exact FP16 +// addition, equal to op8's tail for an exact sum (op8's float32 sum of an all-zero chunk is +0, so a -0 sum is +0). +// Mutants for the probe only (l_q7_s2a_check.py; never set in a kernel build): 1 the bound limit 128 instead of 63, +// 2 the first instruction's result fed as the second's accumulator. +#ifndef Q7_S2A_MUTANT +#define Q7_S2A_MUTANT 0 +#endif +#if Q7_S2A_MUTANT == 1 +constexpr float Q7_S2A_LIMIT = 128.0f; +#else +constexpr float Q7_S2A_LIMIT = 63.0f; +#endif + +// The chunk's exact product sums (when bounded) of the lane's 8 outputs; a0 / b0 the lane's 8 FP8 codes of the +// first instruction, a1 / b1 of the second (+16 K values). +WG_FN v8f q7_tile_sum(v2i a0, v2i b0, v2i a1, v2i b1) { + const v8f zero = {}; + const v8f lo = wg_wmma_fp8(a0, b0, zero); +#if Q7_S2A_MUTANT == 2 + return wg_wmma_fp8(a1, b1, lo); +#else + const v8f hi = wg_wmma_fp8(a1, b1, zero); + return lo + hi; +#endif +} +// The same from the lane's operand addresses (8-byte aligned; the second instruction's codes at +16 bytes). +WG_FN v8f q7_tile_sum(const u8* a_lane, const u8* b_lane) { + return q7_tile_sum(*(const v2i*)a_lane, *(const v2i*)b_lane, *(const v2i*)(a_lane + 16), + *(const v2i*)(b_lane + 16)); +} +// amax from codes: fold each FP8 code into m (start 0); q7_amax of the two halves' m (or one m covering all 32). +WG_FN u32 q7_umax(u32 a, u32 b) { return a > b ? a : b; } +WG_FN u32 q7_maxcode(u32 m, u32 code) { return q7_umax(m, code & 0x7fu); } +WG_FN u32 q7_maxcode4(u32 m, u32 word) { // four packed codes + m = q7_maxcode(m, word); + m = q7_maxcode(m, word >> 8); + m = q7_maxcode(m, word >> 16); + return q7_maxcode(m, word >> 24); +} +WG_FN float q7_amax(u32 m0, u32 m1) { return fp8v(q7_umax(m0, m1)); } +WG_FN bool q7_bounded(float wsum, float amax) { return wsum * amax < Q7_S2A_LIMIT; } +// RN16(start + S) for an exact chunk sum S (op8's result for the same chunk) +WG_FN float q7_join(float S, float start) { return add16(S == 0.0f ? 0.0f : S, start); } +// One output: the bounded join, else fallback(start) (the exact scalar op8 on the FP16 weights and decoded codes). +template +WG_FN float q7_combine(float S, float start, float wsum, float amax, FB fallback) { + if (q7_bounded(wsum, amax)) return q7_join(S, start); + return fallback(start); +} + +} // namespace q7c diff --git a/clean/l/kernels/l_q7_s2a_probe.hip b/clean/l/kernels/l_q7_s2a_probe.hip new file mode 100644 index 00000000..ac0f8d83 --- /dev/null +++ b/clean/l/kernels/l_q7_s2a_probe.hip @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: MIT +// Model L (issue #89): measurement probe of the S2a primitive (clean/l/kernels/l_q7_op8.h: q7_tile_sum, q7_maxcode4, +// q7_amax, q7_combine with the exact op8 as the fallback), not used by the runtime (clean/l/runner/l_q7_s2a_check.py). +// 16 single-chunk ops per workgroup (one wave of 32 lanes) on the tile's diagonal: A row m = op base + m's weight row, +// B token m = op base + m's activations; op i's result is D[row i][token i]. amax: each lane folds the 16 codes of its +// own B operands (both instructions), the token's two lanes l, l ^ 16 are combined (the same 32 values the sums use). +// a, w: [n][32] FP8 codes; w16: [n][32] FP16 bits (the weight row's exact copy, the fallback's operand); st, wsum: [n] +// float32; out: [n] float32; path: [n] (1 bounded = the instruction's sum, 0 = the exact scalar fallback). n % 16 == 0. +// Mutants (-DQ7_S2A_MUTANT=N, probe builds only): 1, 2 in l_q7_op8.h; 3 amax from the lane's own 16 codes only; +// 4 the start fed into the first instruction's accumulator (joined with 0 afterwards). +#pragma clang fp contract(off) +#include "wg_platform.h" +#include "l_q7_op8.h" + +using namespace q7c; + +WG_KERNEL_NW(l_q7_s2a_probe, 32, 1)(const u8* a, const u8* w, const u16* w16, const float* st, const float* wsum, + float* out, u32* path) { + WG_LDS u32 lm[32]; + const int base = 16 * (int)wg_block_id(); + const u32 lane = wg_lane_id(), t = lane & 15u, hf = lane >> 4; + const u8* pw = w + (size_t)(base + (int)t) * 32 + 8 * hf; + const u8* pa = a + (size_t)(base + (int)t) * 32 + 8 * hf; + const v2i a0 = *(const v2i*)pw, a1 = *(const v2i*)(pw + 16); + const v2i b0 = *(const v2i*)pa, b1 = *(const v2i*)(pa + 16); + u32 m = q7_maxcode4(0u, (u32)b0[0]); + m = q7_maxcode4(m, (u32)b0[1]); + m = q7_maxcode4(m, (u32)b1[0]); + m = q7_maxcode4(m, (u32)b1[1]); + lm[lane] = m; + wg_barrier_lds(); +#if Q7_S2A_MUTANT == 3 + const float amax = q7_amax(lm[lane], lm[lane]); +#else + const float amax = q7_amax(lm[t], lm[t + 16u]); +#endif + const bool diag = (t >> 3) == hf; + const int i = base + (int)t; +#if Q7_S2A_MUTANT == 4 + v8f c = {}; + for (int j = 0; j < 8; ++j) c[j] = (8 * (int)hf + j == (int)t) ? st[i] : 0.0f; + const v8f zero = {}; + const v8f S = wg_wmma_fp8(a0, b0, c) + wg_wmma_fp8(a1, b1, zero); + const float start = 0.0f; +#else + const v8f S = q7_tile_sum(a0, b0, a1, b1); + const float start = diag ? st[i] : 0.0f; +#endif + if (diag) { + const float s = S[t & 7u]; + const float ws = wsum[i]; + const u8* act = a + (size_t)i * 32; + const u16* row = w16 + (size_t)i * 32; + out[i] = q7_combine(s, start, ws, amax, [&](float) { // (mutant 4: the fallback keeps the real start) + float av[32]; + for (int k = 0; k < 32; ++k) av[k] = fp8v(act[k]); + return op8(st[i], av, [&](int k) { return from16(row[k]); }, 32); + }); + path[i] = q7_bounded(ws, amax) ? 1u : 0u; + } +} diff --git a/clean/l/kernels/l_q7w.hip b/clean/l/kernels/l_q7w.hip new file mode 100644 index 00000000..c35926b8 --- /dev/null +++ b/clean/l/kernels/l_q7w.hip @@ -0,0 +1,345 @@ +// SPDX-License-Identifier: MIT +// Model L (issue #89): the shared stages (enc1, enc2, b0-b5, dec2, dec1) in their FP16 form, as Model L's own kernel +// variants (Model M's kernels are not changed): clean/l/reference/oracle.py run_stage_q7 (table shared_form "q7"). +// The arithmetic of every step is l_q7.hip's (exact FP8 ops, exact FP16 additions, table rsqrt / rcp), here for the +// widths C = 64, 96, 128 (heads C / 32 rounded to 2 or 4 as Model M's stages: 2, 4, 4) and MLP 4 C. +// +// Kernels (one 4 x 4 window, or one pixel, per workgroup of 32 lanes): +// l_q7_block_c x = FP8 map [H][W][C] (the block's input is its FP8 values), border mirror; out = FP8 map +// l_q7_block_c_dec x = the expansion's FP8 fine map E [H][W][C] (outside the grid: the coarse mirror, sub-pixel +// kept; table shared_dec_border "coarse"), skip = the encoder stage's FP8 map S [H][W][C] (the +// fine mirror); the member's input RN16(E + S); out = FP8 map +// l_q7_merge_c in = FP8 map [H][W][C] (H, W even), out = FP8 [H/2][W/2][C_out]: the chain over the 2 x 2 +// patch (sub-pixel s = dx + 2 dy, input channel C s + i) from the bias, clamped to +-6.28125 +// (a NaN kept), FP8. Grid = (H/2) (W/2) workgroups. +// l_q7_expand_c in = FP8 [h][w][Cl], out = the FP8 fine map [2h][2w][C] (row C s + c of the expansion goes to +// sub-pixel s, channel c): the chain over Cl from the bias, FP8. Grid = h w workgroups. +// Block weights: the q7 blob (l_pack.py pack_q7_block); merge / expansion weights: the q7 linear blob (l_pack.py +// pack_q7_linear: FP16 bias [n], then the FP8 rows [n][K] at q7_linear_offsets). +// +// Build: clang -x hip ... -I clean/kernels clean/l/kernels/l_q7w.hip -o l_q7w.co +#pragma clang fp contract(off) +#include "wg_platform.h" +#include "l_q7_op8.h" + +namespace q7w { + +using namespace q7c; + +constexpr int T = 16; + +WG_FN void row16(const u8* p, float* w) { // 32 FP16 weights -> values (Q7F-1: no FP8 decode) + const _Float16* h = (const _Float16*)p; + for (int k = 0; k < 32; ++k) w[k] = (float)h[k]; +} + +WG_FN void row8(const u8* p, float* w) { // 32 FP8 codes -> values + for (int k = 0; k < 32; ++k) w[k] = fp8v(p[k]); +} + +// Q7_HYB (Model M's HYBRID evaluation form, Q11 part C, runtime WG_M_FORM=hybrid; code object l_q7w__hyb.co; NOT +// the original): every FP8 chain accumulates in float32 over all its chunks (the products, exact in float32, added one +// at a time, chunks and k ascending, from the start widened exactly), ONE FP16 rounding at the end of the chain; the +// scores likewise (32 terms from B); W2's chain runs over all MLP slices in float32 (Lds::zf). Everything else as the +// FP16 form. Oracle: clean/l/reference/oracle.py FP8 op model "f32chain". Q7_HYB_MUTANT (check only, l_q7w__hybmut.co): +// the Wqk chains keep an FP16 rounding after every chunk (the original's per-chunk rounding) - the check must catch it. +#ifndef Q7_HYB +#define Q7_HYB 0 +#endif +#ifndef Q7_HYB_MUTANT +#define Q7_HYB_MUTANT 0 +#endif + +// float32 chain over K = 32 n terms, not rounded (Q7_HYB); per_chunk_rn16: the mutant's FP16 rounding per chunk +WG_FN float chainf(float acc, const u8* a8, const u8* row, int n, bool per_chunk_rn16 = false) { + float a[32], w[32]; + for (int ch = 0; ch < n; ++ch) { + row8(a8 + 32 * ch, a); + row16(row + 64 * ch, w); + for (int k = 0; k < 32; ++k) acc = acc + a[k] * w[k]; // contraction off: the product (exact) then one rounding + if (per_chunk_rn16) acc = h16(acc); + } + return acc; +} + +// The chain over K = 32 n terms (ascending chunks): a8 = FP8 codes, row = FP8 codes, both contiguous +WG_FN float chain8(float start, const u8* a8, const u8* row, int n) { +#if Q7_HYB + return h16(chainf(start, a8, row, n)); +#endif + float a[32], w[32]; + for (int ch = 0; ch < n; ++ch) { + row8(a8 + 32 * ch, a); + row16(row + 64 * ch, w); + start = op8(start, a, w); + } + return start; +} + +// Blob offsets (l_pack.py q7_block_offsets) +template +struct Off { + static constexpr int MLP = 4 * C; + static constexpr int G1 = 0, G2 = 2 * C, BP = 4 * C, B1 = 6 * C, B2 = 6 * C + 2 * MLP, B = 8 * C + 2 * MLP; + static constexpr int WQK = B + 2 * HEADS * 256, WV = WQK + HEADS * 32 * C, WP = WV + HEADS * 32 * C, + W1 = WP + C * 32 * HEADS, W2 = W1 + MLP * C; + static constexpr int END8 = W2 + C * MLP; // FP16 copies (Q7F-1), the same order + static constexpr int WQK16 = END8, WV16 = WQK16 + 2 * HEADS * 32 * C, WP16 = WV16 + 2 * HEADS * 32 * C, + W1_16 = WP16 + 2 * C * 32 * HEADS, W2_16 = W1_16 + 2 * MLP * C; +}; + +// norm tree of Q8 B3 for width C: one half (odd 0 / 1) of lane t's tile channels 8 j + 2 t + odd +template +WG_FN float lane_half(const float* q, int t, int odd) { + constexpr int J = C / 8, NP = J / 4; // pairs per group: 1 (C 32), 2, 3, 4 (C 128) + float g[2]; + for (int grp = 0; grp < 2; ++grp) { // grp 0: even tiles, grp 1: odd tiles + float pr[4]; + for (int i = 0; i < NP; ++i) { + const int j0 = 4 * i + grp, j1 = 4 * i + 2 + grp; + pr[i] = add16(q[8 * j0 + 2 * t + odd], q[8 * j1 + 2 * t + odd]); + } + float s; + if (NP == 1) s = pr[0]; + else if (NP == 2) s = add16(pr[0], pr[1]); + else if (NP == 3) s = add16(add16(pr[0], pr[1]), pr[2]); + else s = add16(add16(pr[0], pr[1]), add16(pr[2], pr[3])); + g[grp] = s; + } + return add16(g[0], g[1]); +} + +template +WG_FN float norm_r(const _Float16* x, const u16* tab) { + float q[C]; + for (int c = 0; c < C; ++c) q[c] = mul16((float)x[c], (float)x[c]); + float hv[2]; + for (int odd = 0; odd < 2; ++odd) { + float e[4]; + for (int t = 0; t < 4; ++t) e[t] = lane_half(q, t, odd); + hv[odd] = add16(add16(e[0], e[1]), add16(e[2], e[3])); + } + const float s = add16(add16(hv[0], hv[1]), 0x1p-13f); + return from16(tab[bits16(s)]); +} + +template +struct Lds { + _Float16 x[T][C], z[T][C], v[HEADS][T][32], pw[HEADS][T][T]; + u8 a8[T][C], u[HEADS][T][32], o8[T][32 * HEADS], g8[T][32]; +#if Q7_HYB + float zf[T][C]; // W2's float32 chain across the MLP slices +#endif + float r[T]; +}; + +// The block on one window (L.x holds the 16 members' FP16 inputs). Out: lane (t, hf)'s channels C/2 hf ... of member +// t, FP16, in L.z[t][...]. +template +WG_FN void block(Lds& L, const u8* w, const u16* tab, int t, int hf) { + using O = Off; + constexpr int CH = C / 2, HPL = HEADS / 2, NC = C / 32; + const _Float16* wh = (const _Float16*)w; + if (hf == 0) L.r[t] = norm_r(L.x[t], tab); + wg_barrier_lds(); + for (int i = 0; i < CH; ++i) { + const int c = CH * hf + i; + L.a8[t][c] = (u8)e8(mul16((float)L.x[t][c], mul16(L.r[t], (float)wh[O::G1 / 2 + c]))); + } + wg_barrier_lds(); + for (int hh = 0; hh < HPL; ++hh) { + const int hd = HPL * hf + hh; + for (int d = 0; d < 32; ++d) { +#if Q7_HYB && Q7_HYB_MUTANT + L.u[hd][t][d] = (u8)e8(h16(chainf(0.0f, L.a8[t], w + O::WQK16 + 2 * (hd * 32 + d) * C, NC, true))); +#else + L.u[hd][t][d] = (u8)e8(chain8(0.0f, L.a8[t], w + O::WQK16 + 2 * (hd * 32 + d) * C, NC)); +#endif + L.v[hd][t][d] = (_Float16)chain8(0.0f, L.a8[t], w + O::WV16 + 2 * (hd * 32 + d) * C, NC); + } + } + wg_barrier_lds(); + for (int hh = 0; hh < HPL; ++hh) { + const int hd = HPL * hf + hh; + float e[T], a[32], b[32]; + row8(L.u[hd][t], a); + for (int j = 0; j < T; ++j) { + row8(L.u[hd][j], b); +#if Q7_HYB + float s = (float)wh[O::B / 2 + (hd * T + t) * T + j]; + for (int k = 0; k < 32; ++k) s = s + a[k] * b[k]; + s = h16(s); +#else + const float s = op8((float)wh[O::B / 2 + (hd * T + t) * T + j], a, b); +#endif + const float cs = 0.017181396484375f, lim = 0.55615234375f; + const float uu = __builtin_fminf(__builtin_fmaxf(mul16(s, cs), -lim), lim); + const float tt = fma16(uu, -uu, 0.92724609375f); + const float p = fma16(uu, tt, 1.375f); + e[j] = from16((bits16(p) & 0x3ffu) << 5); + } + float sm[2]; + for (int lo = 0; lo < 2; ++lo) { + float pr[4]; + for (int k = 0; k < 4; ++k) pr[k] = add16(e[lo + 2 * k], e[lo + 8 + 2 * k]); + sm[lo] = add16(add16(pr[0], pr[1]), add16(pr[2], pr[3])); + } + const float wsum = from16(tab[65536 + bits16(add16(sm[0], sm[1]))]); + for (int j = 0; j < T; ++j) L.pw[hd][t][j] = (_Float16)mul16(e[j], wsum); + } + wg_barrier_lds(); + for (int hh = 0; hh < HPL; ++hh) { + const int hd = HPL * hf + hh; + for (int d = 0; d < 32; ++d) { + float s = 0.0f, c = 0.0f; + for (int j = 0; j < T; ++j) { + float e; + two_sum(s, (float)L.pw[hd][t][j] * (float)L.v[hd][j][d], s, e); + c = c + e; + } + float z, tl; + two_sum(s, c, z, tl); + L.o8[t][32 * hd + d] = (u8)e8(rn16_pair(z, tl)); + } + } + wg_barrier_lds(); + for (int i = 0; i < CH; ++i) { + const int c = CH * hf + i; + float acc = add16((float)wh[O::BP / 2 + c], (float)L.x[t][c]); + acc = chain8(acc, L.o8[t], w + O::WP16 + 2 * c * 32 * HEADS, HEADS); + L.z[t][c] = (_Float16)acc; + } + wg_barrier_lds(); + if (hf == 0) L.r[t] = norm_r(L.z[t], tab); + wg_barrier_lds(); + for (int i = 0; i < CH; ++i) { // m8 in a8's place; the MLP accumulators in z's + const int c = CH * hf + i; + const float zc = (float)L.z[t][c]; + L.a8[t][c] = (u8)e8(mul16(zc, mul16(L.r[t], (float)wh[O::G2 / 2 + c]))); + L.z[t][c] = (_Float16)add16((float)wh[O::B2 / 2 + c], zc); +#if Q7_HYB + L.zf[t][c] = (float)L.z[t][c]; +#endif + } + wg_barrier_lds(); + for (int sl = 0; sl < O::MLP / 32; ++sl) { + for (int i = 0; i < 16; ++i) { + const int j = 32 * sl + 16 * hf + i; + const float hv = chain8((float)wh[O::B1 / 2 + j], L.a8[t], w + O::W1_16 + 2 * j * C, NC); + L.g8[t][16 * hf + i] = (u8)e8(gelu16(hv)); + } + wg_barrier_lds(); + for (int i = 0; i < CH; ++i) { + const int c = CH * hf + i; +#if Q7_HYB + L.zf[t][c] = chainf(L.zf[t][c], L.g8[t], w + O::W2_16 + 2 * (c * O::MLP + 32 * sl), 1); + if (sl == O::MLP / 32 - 1) L.z[t][c] = (_Float16)L.zf[t][c]; // the chain's one FP16 rounding +#else + L.z[t][c] = (_Float16)chain8((float)L.z[t][c], L.g8[t], w + O::W2_16 + 2 * (c * O::MLP + 32 * sl), 1); +#endif + } + wg_barrier_lds(); + } +} + +WG_FN void window_member(int W, int ox, int oy, int t, int& pr, int& pc) { + const int nwx = (W + ox + 3) / 4; + const int win = (int)wg_block_id(); + const int wy = win / nwx, wx = win % nwx; + pr = 4 * wy - oy + (t >> 2); + pc = 4 * wx - ox + (t & 3); +} + +template +WG_FN void block_kernel(const u8* x, const u8* skip, u8* out, const u8* w, const u16* tab, int H, int W, int ox, + int oy) { + WG_LDS Lds L; + constexpr int CH = C / 2; + const int lane = (int)wg_lane_id(), t = lane & 15, hf = lane >> 4; + int pr, pc; + window_member(W, ox, oy, t, pr, pc); + if (DEC) { + const int r = 2 * mirror(pr >> 1, H >> 1) + (pr & 1), c = 2 * mirror(pc >> 1, W >> 1) + (pc & 1); + const int sr = mirror(pr, H), sc = mirror(pc, W); + const u8* ep = x + ((size_t)r * W + c) * C + CH * hf; + const u8* sp = skip + ((size_t)sr * W + sc) * C + CH * hf; + for (int i = 0; i < CH; ++i) L.x[t][CH * hf + i] = (_Float16)h16(fp8v(ep[i]) + fp8v(sp[i])); + } else { + const u8* xp = x + ((size_t)mirror(pr, H) * W + mirror(pc, W)) * C + CH * hf; + for (int i = 0; i < CH; ++i) L.x[t][CH * hf + i] = (_Float16)fp8v(xp[i]); + } + wg_barrier_lds(); + block(L, w, tab, t, hf); + if (pr >= 0 && pr < H && pc >= 0 && pc < W) { + u8* dst = out + ((size_t)pr * W + pc) * C + CH * hf; + for (int i = 0; i < CH; ++i) dst[i] = out8((float)L.z[t][CH * hf + i]); + } +} + +// Linear blob offsets (l_pack.py q7_linear_offsets): bias FP16 [n] at 0, rows FP8 [n][K] at align16(2 n) +WG_FN int lin_rows(int n) { return (2 * n + 15) & ~15; } + +template +WG_FN void merge_kernel(const u8* in, u8* out, const u8* w, int H, int W) { + const int lane = (int)wg_lane_id(); + const int px = (int)wg_block_id(), w2 = W / 2, r = px / w2, c = px % w2; + if (4 * lane >= COUT) return; + const _Float16* bias = (const _Float16*)w; + const u8* rows16 = w + lin_rows(COUT) + (size_t)COUT * 4 * C; // the FP16 rows (Q7F-1) + u8 z[4 * C]; + for (int s = 0; s < 4; ++s) { + const u8* src = in + ((size_t)(2 * r + (s >> 1)) * W + (2 * c + (s & 1))) * C; + for (int i = 0; i < C; ++i) z[C * s + i] = src[i]; + } + u8 o[4]; + for (int j = 0; j < 4; ++j) { + const int n = 4 * lane + j; + float acc = chain8((float)bias[n], z, rows16 + (size_t)2 * n * 4 * C, 4 * C / 32); + if (!__builtin_isnan(acc)) acc = __builtin_fminf(__builtin_fmaxf(acc, -6.28125f), 6.28125f); + o[j] = out8(acc); + } + u8* dst = out + ((size_t)r * w2 + c) * COUT + 4 * lane; + for (int j = 0; j < 4; ++j) dst[j] = o[j]; +} + +template +WG_FN void expand_kernel(const u8* in, u8* out, const u8* w, int h, int wd) { + const int lane = (int)wg_lane_id(); + const int px = (int)wg_block_id(), r = px / wd, c = px % wd; + const _Float16* bias = (const _Float16*)w; + const u8* rows16 = w + lin_rows(4 * C) + (size_t)4 * C * CL; // the FP16 rows (Q7F-1) + const u8* src = in + ((size_t)r * wd + c) * CL; + for (int g = 0; g < C / 32; ++g) { // 4 C outputs in groups of 128: lane's 4 consecutive rows + const int n0 = 128 * g + 4 * lane, s = n0 / C, ch = n0 % C; + u8 o[4]; + for (int j = 0; j < 4; ++j) o[j] = out8(chain8((float)bias[n0 + j], src, rows16 + (size_t)2 * (n0 + j) * CL, CL / 32)); + u8* dst = out + ((size_t)(2 * r + (s >> 1)) * (2 * wd) + (2 * c + (s & 1))) * C + ch; + for (int j = 0; j < 4; ++j) dst[j] = o[j]; + } +} + +} // namespace q7w + +#define Q7W_BLOCK(name, C, HEADS, DEC) \ + WG_KERNEL_NW(name, 32, 1)(const u8* x, const u8* skip, u8* out, const u8* w, const u16* tab, int H, int W, int ox, \ + int oy) { \ + q7w::block_kernel(x, skip, out, w, tab, H, W, ox, oy); \ + } +Q7W_BLOCK(l_q7_block_c64, 64, 2, false) +Q7W_BLOCK(l_q7_block_c96, 96, 4, false) +Q7W_BLOCK(l_q7_block_c128, 128, 4, false) +Q7W_BLOCK(l_q7_block_c64_dec, 64, 2, true) +Q7W_BLOCK(l_q7_block_c96_dec, 96, 4, true) + +WG_KERNEL_NW(l_q7_merge_c64, 32, 1)(const u8* in, u8* out, const u8* w, int H, int W) { + q7w::merge_kernel<64, 96>(in, out, w, H, W); +} +WG_KERNEL_NW(l_q7_merge_c96, 32, 1)(const u8* in, u8* out, const u8* w, int H, int W) { + q7w::merge_kernel<96, 128>(in, out, w, H, W); +} +WG_KERNEL_NW(l_q7_expand_c128, 32, 1)(const u8* in, u8* out, const u8* w, int h, int wd) { + q7w::expand_kernel<128, 96>(in, out, w, h, wd); +} +WG_KERNEL_NW(l_q7_expand_c96, 32, 1)(const u8* in, u8* out, const u8* w, int h, int wd) { + q7w::expand_kernel<96, 64>(in, out, w, h, wd); +} diff --git a/clean/l/kernels/l_swin.hip b/clean/l/kernels/l_swin.hip index baf2c2c2..03a55b8a 100644 --- a/clean/l/kernels/l_swin.hip +++ b/clean/l/kernels/l_swin.hip @@ -14,6 +14,7 @@ // x, xadd (unused, may be null), out, w, H, W, ox, oy: as swin_block_c64. `_nc` when bit 0 of the blob's flags word is // set (the weights prove every inner FP8 pack in range), with identical results. +#ifndef L_SWIN_HELPERS_ONLY // l_swin_hyb.hip includes this file for its source / epilogue types only WG_KERNEL_NW(l_swin_block_c32, 256, WG_MINW_C32)(const u8* x, const u8* xadd, u8* out, const u8* w, int H, int W, int ox, int oy) { swin_block<32, 2, 128, 1, false, WG_MLP_GROUP, false, 256>(x, xadd, out, w, H, W, ox, oy, nullptr); @@ -22,6 +23,7 @@ WG_KERNEL_NW(l_swin_block_c32_nc, 256, WG_MINW_C32)(const u8* x, const u8* xadd, int ox, int oy) { swin_block<32, 2, 128, 1, false, WG_MLP_GROUP, true, 256>(x, xadd, out, w, H, W, ox, oy, nullptr); } +#endif // ---- Model L's two blocks (clean/l/FORMATS.md) --------------------------------------------------------------------- @@ -85,6 +87,7 @@ struct LStoreF16Epi { swin_block<32, 2, 128, 1, false, WG_MLP_GROUP, NC, 256, 0, 0, EPI, false, SRC>(x, xadd, OUTARG, w, H, W, ox, oy, \ nullptr, out); \ } +#ifndef L_SWIN_HELPERS_ONLY L_BLOCK_KERNEL(l_swin_block_c32_enc, false, SwinStoreEpi, LHaloF16Src, out) L_BLOCK_KERNEL(l_swin_block_c32_enc_nc, true, SwinStoreEpi, LHaloF16Src, out) L_BLOCK_KERNEL(l_swin_block_c32_dec, false, LStoreF16Epi, LCoarseFp8Src, nullptr) @@ -96,3 +99,4 @@ WG_KERNEL_NW(l_swin_block_c64, 256, WG_MINW_C64)(const u8* x, const u8* xadd, u8 int ox, int oy) { swin_block<64, 2, 256, 1, false, WG_MLP_GROUP, false, 256>(x, xadd, out, w, H, W, ox, oy, nullptr); } +#endif // L_SWIN_HELPERS_ONLY diff --git a/clean/l/kernels/l_swin_hyb.hip b/clean/l/kernels/l_swin_hyb.hip new file mode 100644 index 00000000..c69f1ddc --- /dev/null +++ b/clean/l/kernels/l_swin_hyb.hip @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: MIT +// Model L's C = 32 blocks in the HYBRID form (WG_L_FORM=hybrid; swin_block<..., HYB = true>, clean/kernels/wg_hyb.h): +// the HIP twins of the D3D12 ports l_swin_block_c32_enc_hyb / _dec_hyb (clean/dx12/kernels/l_swin.hlsl, SW_HYB=1), the +// same bytes. Shapes of l_swin_block_c32_enc_nc / _dec_nc (clean/l/kernels/l_swin.hip). Its own code object +// (l_swin_hyb.co), so the default objects stay as they are. +#define L_SWIN_HELPERS_ONLY 1 +#include "l_swin.hip" + +#define L_HYB_KERNEL(name, EPI, SRC, OUTARG) \ + WG_KERNEL_NW(name, 256, WG_MINW_C32)(const u8* x, const u8* xadd, u8* out, const u8* w, int H, int W, int ox, \ + int oy) { \ + swin_block<32, 2, 128, 1, false, WG_MLP_GROUP, true, 256, 0, 0, EPI, false, SRC, true>(x, xadd, OUTARG, w, H, W, \ + ox, oy, nullptr, out); \ + } +L_HYB_KERNEL(l_swin_block_c32_enc_hyb, SwinStoreEpi, LHaloF16Src, out) +// the decoder: ADD (xadd = the skip map S, fine mirror; x = E by the coarse mirror in swin_block's HYB && ADD gather), +// member input RN16(E + S) (oracle swin_block skip, l_q7_block_c32_dec) +WG_KERNEL_NW(l_swin_block_c32_dec_hyb, 256, WG_MINW_C32)(const u8* x, const u8* xadd, u8* out, const u8* w, int H, int W, + int ox, int oy) { + swin_block<32, 2, 128, 1, true, WG_MLP_GROUP, true, 256, 0, 0, LStoreF16Epi, false, SwinFp8Src, true>( + x, xadd, nullptr, w, H, W, ox, oy, nullptr, out); +} diff --git a/clean/l/kernels/l_wmma_probe.hip b/clean/l/kernels/l_wmma_probe.hip new file mode 100644 index 00000000..23b02b7b --- /dev/null +++ b/clean/l/kernels/l_wmma_probe.hip @@ -0,0 +1,32 @@ +// SPDX-License-Identifier: MIT +// Model L (issue #89): measurement probe, not used by the runtime. One 32-term FP8 op per output computed with RDNA 4's +// FP8 matrix instruction (two 16 x 16 x 16 steps, float32 accumulator started at the op's start value), to record +// whether the instruction gives the exact sum for these shapes (clean/l/runner/l_mm8_f32_check.py --wmma). +// +// a, b: [n][32] FP8 codes (op i = sum_k a[i][k] b[i][k]); st: [n] float32 start values; out: [n] float32 results. +// Workgroup = one wave of 32 lanes computes the 16 x 16 tile of ops base ... base + 15 against each other and keeps +// the diagonal (op i's own row and column). Operand layout (gfx12, wave32): A lane l holds row l & 15, K values +// 8 (l >> 4) ... + 7; B lane l holds column l & 15, the same K values; D lane l holds column l & 15, rows +// 8 (l >> 4) ... + 7. +#include "wg_platform.h" + +WG_KERNEL_NW(l_wmma_probe, 32, 1)(const u8* a, const u8* b, const float* st, float* out, int n) { + const int base = 16 * (int)wg_block_id(); + const u32 lane = wg_lane_id(); + const int r = (int)(lane & 15u), kh = (int)(lane >> 4); + const int op = base + r < n ? base + r : n - 1; + v8f c; + for (int j = 0; j < 8; ++j) { + const int m = 8 * kh + j; + c[j] = (m == r && base + m < n) ? st[base + m] : 0.0f; + } + for (int half = 0; half < 2; ++half) { + const v2i qa = *(const v2i*)(a + (size_t)op * 32 + 16 * half + 8 * kh); + const v2i qb = *(const v2i*)(b + (size_t)op * 32 + 16 * half + 8 * kh); + c = wg_wmma_fp8(qa, qb, c); + } + for (int j = 0; j < 8; ++j) { + const int m = 8 * kh + j; + if (m == r && base + m < n) out[base + m] = c[j]; + } +} diff --git a/clean/l/records/l_time_1440q_hybrid_d3d12.json b/clean/l/records/l_time_1440q_hybrid_d3d12.json new file mode 100644 index 00000000..02c77478 --- /dev/null +++ b/clean/l/records/l_time_1440q_hybrid_d3d12.json @@ -0,0 +1,293 @@ +{ + "what": "Model L, 1440p Quality, harness, D3D12: frame GPU time per configuration, 5 interleaved rounds (median of each round's frames), the direct-route benchmark (clean/runner/direct_bench.py) on the D3D12 interop test harness, executor dx12", + "label": "Model L, 1440p Quality, harness, D3D12", + "shape": "1440p Quality", + "render": [ + 1707, + 960 + ], + "output": [ + 2560, + 1440 + ], + "frames_per_round": 1500, + "configs": { + "main": { + "build": "main at bf0654e9 (Model L's previous native route, the older per-chunk form)", + "env": {}, + "executor": [ + "dx12", + "dx12", + "dx12", + "dx12", + "dx12" + ], + "total_ms": [ + 3.822, + 3.88, + 3.8825, + 3.886, + 3.891 + ], + "network_ms": [ + 3.8155, + 3.874, + 3.876, + 3.8795, + 3.8845 + ], + "parts_ms": [ + { + "enc0": 0.815, + "Swin": 1.228, + "dec0": 0.6475, + "post": 1.124 + }, + { + "enc0": 0.8255, + "Swin": 1.249, + "dec0": 0.66, + "post": 1.1395 + }, + { + "enc0": 0.826, + "Swin": 1.251, + "dec0": 0.6605, + "post": 1.139 + }, + { + "enc0": 0.8265, + "Swin": 1.2515, + "dec0": 0.661, + "post": 1.1405 + }, + { + "enc0": 0.827, + "Swin": 1.252, + "dec0": 0.662, + "post": 1.143 + } + ], + "total_median_ms": 3.882, + "parts_median_ms": { + "enc0": 0.826, + "Swin": 1.251, + "dec0": 0.661, + "post": 1.139 + } + }, + "hyb": { + "build": "this branch, WG_L_FORM=hybrid", + "env": { + "WG_L_FORM": "hybrid" + }, + "executor": [ + "dx12", + "dx12", + "dx12", + "dx12", + "dx12" + ], + "total_ms": [ + 4.8085, + 4.843, + 4.856, + 4.854, + 4.8505 + ], + "network_ms": [ + 4.7985, + 4.8325, + 4.846, + 4.844, + 4.8405 + ], + "parts_ms": [ + { + "enc0": 1.0235, + "Swin": 1.8065, + "dec0": 0.894, + "post": 1.075 + }, + { + "enc0": 1.032, + "Swin": 1.8195, + "dec0": 0.9, + "post": 1.081 + }, + { + "enc0": 1.032, + "Swin": 1.827, + "dec0": 0.9025, + "post": 1.084 + }, + { + "enc0": 1.032, + "Swin": 1.826, + "dec0": 0.902, + "post": 1.0835 + }, + { + "enc0": 1.0325, + "Swin": 1.8245, + "dec0": 0.901, + "post": 1.0825 + } + ], + "total_median_ms": 4.851, + "parts_median_ms": { + "enc0": 1.032, + "Swin": 1.825, + "dec0": 0.901, + "post": 1.083 + } + }, + "hybdef": { + "build": "this branch, no form variable (the release default)", + "env": {}, + "executor": [ + "dx12", + "dx12", + "dx12", + "dx12", + "dx12" + ], + "total_ms": [ + 4.82, + 4.837, + 4.857, + 4.855, + 4.8595 + ], + "network_ms": [ + 4.81, + 4.827, + 4.847, + 4.845, + 4.8495 + ], + "parts_ms": [ + { + "enc0": 1.025, + "Swin": 1.812, + "dec0": 0.8965, + "post": 1.076 + }, + { + "enc0": 1.0305, + "Swin": 1.818, + "dec0": 0.8995, + "post": 1.08 + }, + { + "enc0": 1.0325, + "Swin": 1.827, + "dec0": 0.9035, + "post": 1.084 + }, + { + "enc0": 1.033, + "Swin": 1.826, + "dec0": 0.903, + "post": 1.0835 + }, + { + "enc0": 1.0325, + "Swin": 1.8285, + "dec0": 0.903, + "post": 1.0845 + } + ], + "total_median_ms": 4.855, + "parts_median_ms": { + "enc0": 1.032, + "Swin": 1.826, + "dec0": 0.903, + "post": 1.083 + } + }, + "ng": { + "build": "this branch with the bottleneck's frame guard compiled out (timing comparison only, never shipped: the runtime's guard calls removed, swin_bneck6_hyb without the abort word)", + "env": {}, + "executor": [ + "dx12", + "dx12", + "dx12", + "dx12", + "dx12" + ], + "total_ms": [ + 4.817, + 4.8235, + 4.857, + 4.85, + 4.8625 + ], + "network_ms": [ + 4.8105, + 4.817, + 4.8505, + 4.8435, + 4.8565 + ], + "parts_ms": [ + { + "enc0": 1.0265, + "Swin": 1.8085, + "dec0": 0.8975, + "post": 1.078 + }, + { + "enc0": 1.028, + "Swin": 1.811, + "dec0": 0.8985, + "post": 1.0795 + }, + { + "enc0": 1.034, + "Swin": 1.8265, + "dec0": 0.905, + "post": 1.0855 + }, + { + "enc0": 1.033, + "Swin": 1.823, + "dec0": 0.9035, + "post": 1.084 + }, + { + "enc0": 1.034, + "Swin": 1.8285, + "dec0": 0.906, + "post": 1.0865 + } + ], + "total_median_ms": 4.85, + "parts_median_ms": { + "enc0": 1.033, + "Swin": 1.823, + "dec0": 0.903, + "post": 1.084 + } + } + }, + "frame_guard_cost": { + "what": "the bottleneck wait timeout's frame guard (D3D12 predication, read at frame start and after the bottleneck, lifted at the end, and the kernel's abort-word check): the release default minus the same build with the guard compiled out, per interleaved round", + "total_ms_per_round": [ + 0.003, + 0.0135, + 0.0, + 0.005, + -0.003 + ], + "total_ms_median": 0.003, + "network_ms_per_round": [ + -0.0005, + 0.01, + -0.0035, + 0.0015, + -0.007 + ], + "network_ms_median": -0.0005 + } +} diff --git a/clean/l/reference/hyb_tables.py b/clean/l/reference/hyb_tables.py new file mode 100644 index 00000000..d70ffd03 --- /dev/null +++ b/clean/l/reference/hyb_tables.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: MIT +"""Tables and exhaustive facts behind the fast hybrid kernels' element-wise steps (WG_L_FORM=hybrid, SW_HYB): every +step whose input is one FP16 value is checked here for ALL 65,536 inputs against the oracle forms of +clean/l/reference/oracle.py (norm_q7, softmax_q7, gelu_q7). + +- gelu16_q8: 65,536 bytes, index = the FP16 bits of h: q8(gelu_q7(h)) (the MLP's GELU16 then the FP8 point); a NaN + result is written as 0xFF, the hardware FP8 conversion's NaN on HIP and D3D12 (it does not keep a NaN's sign), so + the D3D12 table and the HIP computation give the same bytes. +- rsq16: the norm's r = RN16(rsqrt32(S')) for every positive finite FP16 S', as 2 x 1024 entries (exponent parity x + mantissa) plus an exponent shift: exact for the norm's range S' >= 2^-13 (checked for every FP16 value there). +- rcp16: the softmax normaliser w = RN16(rcp32(sum)) for every positive normal FP16 sum, as 1024 entries (mantissa) plus + an exponent shift; exact wherever the result is normal (checked). +- softmax_e: e(u') of softmax_q7 for every clamped u' = min(max(RN16(s c), -c'), c'): computed in float32 with one FP16 + rounding per step (t = RN16(fma32(u', -u', t0)), p = RN16(fma32(u', t, p0))) equals the FMA16 form for every u' + (softmax_e_f32_equals_fma16), so no table is needed; the products u' u' and u' t are exact in float32, so every + lowering (fused or not) gives the same float32 value. +""" +import numpy as np + +from clean.l.reference import oracle as O +from clean.reference.fp8 import fp8_encode, q8 + +ALL16 = np.arange(65536, dtype=np.uint32).astype(np.uint16) +H = ALL16.view(np.float16).astype(np.float64) + + +def f16(x): + return np.asarray(x, np.float64).astype(np.float16).astype(np.float64) + + +def gelu16_q8(): + """q8(gelu_q7(h)) for every FP16 bit pattern h (NaN in, NaN code out).""" + with np.errstate(invalid="ignore", over="ignore"): + g = O.gelu_q7(H) + out = np.asarray(fp8_encode(q8(g)), np.uint8) + out[np.isnan(g)] = 0xFF + return out + + +def _rsq_ref(s): + return f16(O._rsqrt32(s)) + + +def rsq16_table(): + """[2][1024] FP16 bits: r for S' = 1.m x 2^(e) with e even (row 0: e = 0) and odd (row 1: e = 1).""" + m = np.arange(1024) + base = 1.0 + m / 1024.0 + t0 = _rsq_ref(base).astype(np.float16).view(np.uint16) + t1 = _rsq_ref(base * 2.0).astype(np.float16).view(np.uint16) + return np.stack([t0, t1]) + + +def rsq16_lookup(sbits, tab): + """r bits for positive normal FP16 S' bits via the table: S' = 1.m 2^E, E = 2k + p -> r = T[p][m] x 2^-k.""" + sbits = np.asarray(sbits, np.int64) + E = ((sbits >> 10) & 31) - 15 + m = sbits & 1023 + p = E & 1 + k = (E - p) // 2 + rb = tab[p, m].astype(np.int64) + return (rb - (k << 10)).astype(np.uint16) # r normal in the norm's range: exponent field minus k + + +def rcp16_table(): + m = np.arange(1024) + return f16(O._rcp32(1.0 + m / 1024.0)).astype(np.float16).view(np.uint16) + + +def rcp16_lookup(sbits, tab): + sbits = np.asarray(sbits, np.int64) + E = ((sbits >> 10) & 31) - 15 + m = sbits & 1023 + rb = tab[m].astype(np.int64) + return (rb - (E << 10)).astype(np.uint16) + + +def softmax_e_ref(u): + """e of softmax_q7 for clamped u' (float64 values), by the oracle's own steps.""" + t = f16(u * -u + O.Q7_POLY_T) + p = f16(u * t + O.Q7_POLY_P) + pb = p.astype(np.float16).view(np.uint16) + return ((pb & 0x03FF) << 5).astype(np.uint16) + + +def softmax_e_f32(u): + """the kernels' form: float32 fused multiply-adds (exact products), one FP16 rounding per step.""" + u32 = u.astype(np.float32) + t32 = (u.astype(np.float64) * -u.astype(np.float64) + O.Q7_POLY_T).astype(np.float32) # fma32: exact, then RN32 + t = f16(t32) + p32 = (u.astype(np.float64) * t + O.Q7_POLY_P).astype(np.float32) + p = f16(p32) + pb = p.astype(np.float16).view(np.uint16) + del u32 + return ((pb & 0x03FF) << 5).astype(np.uint16) + + +def clamped_u(): + """every FP16 value u' can take: |u'| <= c' (FP16 0x3873).""" + c = O.Q7_SCORE_CLAMP + return H[np.isfinite(H) & (np.abs(H) <= c)] + + +def facts(): + """The exhaustive facts the kernels rely on (all must be True).""" + out = {} + with np.errstate(invalid="ignore", over="ignore", divide="ignore"): + # rsqrt: every FP16 S' >= 2^-13 (the norm adds 2^-13 to a non-negative sum) + sb = ALL16[(H >= 2.0 ** -13) & np.isfinite(H)] + ref = _rsq_ref(sb.view(np.float16).astype(np.float64)).astype(np.float16).view(np.uint16) + out["rsq16_table_exact"] = bool(np.array_equal(rsq16_lookup(sb, rsq16_table()), ref)) + # rcp: every positive normal FP16 sum whose reciprocal is normal + sb = ALL16[(H >= 2.0 ** -14) & np.isfinite(H)] + ref = f16(O._rcp32(sb.view(np.float16).astype(np.float64))) + ok = ref >= 2.0 ** -14 + out["rcp16_table_exact"] = bool(np.array_equal(rcp16_lookup(sb[ok], rcp16_table()), + ref[ok].astype(np.float16).view(np.uint16))) + u = clamped_u() + out["softmax_e_f32_equals_fma16"] = bool(np.array_equal(softmax_e_f32(u), softmax_e_ref(u))) + # the softmax weight range: every e from a clamped u' is a positive normal FP16 (never 0, never Inf / NaN) + e = softmax_e_ref(u).view(np.float16).astype(np.float64) + out["softmax_e_positive_normal"] = bool(np.all((e >= 2.0 ** -14) & np.isfinite(e))) + g = gelu16_q8() + out["gelu16_q8_nan_maps_to_nan"] = bool(np.all((g[np.isnan(H)] & 0x7F) == 0x7F)) + return out + + +if __name__ == "__main__": + for k, v in facts().items(): + print(k, v) diff --git a/clean/l/reference/oracle.py b/clean/l/reference/oracle.py index f0d609f4..8b31c167 100644 --- a/clean/l/reference/oracle.py +++ b/clean/l/reference/oracle.py @@ -89,8 +89,25 @@ def _block_tokens(xw, p, heads, k): # FP16; block 1's result is block 2's start value. # The attention product P V (FP16 operands, 16 terms) is summed in float64 and rounded once (its accumulation precision # does not move an FP16 result). +# - "f32chain" (NOT the original: Model M's HYBRID evaluation form, Q11 part C): the whole chain of a linear (all its +# 32-wide chunks) accumulates in float32, starting from the FP16 start value widened exactly; the products (exact +# in float32: FP8 x FP8) are added one at a time, chunks ascending and k ascending inside a chunk, each addition +# rounded to float32 (RNE); ONE FP16 rounding at the end of the chain. A chain split over several calls (W2 over +# the MLP slices) keeps its float32 accumulator between them (mm8 final=False). -MM8_MODELS = ("exact", "trunc13") +MM8_MODELS = ("exact", "trunc13", "f32chain") + + +def _f32chain(start, a8, W8): + """The float32 sequential chain: start [..., n] (float32-exact values), a8 [..., K], W8 [n, K] -> float32 [..., n] + (not rounded to FP16).""" + a = np.asarray(a8, np.float32) + w = np.asarray(W8, np.float32) + acc = np.array(np.broadcast_to(np.asarray(start, np.float32), a.shape[:-1] + (w.shape[0],)), np.float32) + with np.errstate(over="ignore", invalid="ignore"): + for k in range(a.shape[-1]): + acc = (acc + a[..., k:k + 1] * w[:, k]).astype(np.float32) + return acc def _trunc13_block(start, prods): @@ -130,9 +147,13 @@ def _op8(start, a8, W8, model): return out -def mm8(start, a8, W8, model="exact"): +def mm8(start, a8, W8, model="exact", final=True): """The chain of FP8 matrix ops over the 32-wide chunks of K, ascending: a8 [..., K], W8 [n, K], start [..., n] or - [n] (FP16) -> FP16 [..., n].""" + [n] (FP16) -> FP16 [..., n]. Model "f32chain": float32 over the whole chain, one FP16 rounding at the end; with + final=False the float32 accumulator is returned unrounded (the chain continues in the next call).""" + if model == "f32chain": + acc = _f32chain(start, a8, W8) + return f16(acc.astype(np.float64)) if final else acc a8 = np.asarray(a8, np.float64) W8 = np.asarray(W8, np.float64) acc = start @@ -257,6 +278,13 @@ def _score_op(B, u, model): with np.errstate(over="ignore", invalid="ignore"): if model == "exact": return f16(B + np.einsum("...htd,...hsd->...hts", u, u)) + if model == "f32chain": # 32 terms from B, float32 sequential, one rounding + acc = np.array(np.broadcast_to(np.asarray(B, np.float32), u.shape[:-3] + (u.shape[-3], u.shape[-2], + u.shape[-2])), np.float32) + uf = np.asarray(u, np.float32) + for d in range(uf.shape[-1]): + acc = (acc + uf[..., :, None, d] * uf[..., None, :, d]).astype(np.float32) + return f16(acc.astype(np.float64)) if model != "trunc13": raise ValueError(f"unknown FP8 op model {model!r}") prods = u[..., :, None, :] * u[..., None, :, :] # [..., h, T, T, 32] @@ -282,20 +310,23 @@ def _block_tokens_q7(xw, p, heads, k): lead = xw.shape[:-2] x = np.asarray(xw, np.float64) W = {n: q8(p[n]) for n in ("Wqk", "Wv", "Wp", "W1", "W2")} + V = {n: f16(p[n]) for n in ("bp", "b1", "b2", "B")} # FP16 vectors as the device holds them (no change for FP16) with np.errstate(over="ignore", invalid="ignore"): a8 = q8(norm_q7(x, gamma_q7(p["gamma1"]))) u = np.stack([q8(mm8(0.0, a8, W["Wqk"][h], model)) for h in range(heads)], axis=-3) # [..., h, T, 32] v = np.stack([mm8(0.0, a8, W["Wv"][h], model) for h in range(heads)], axis=-3) - s = _score_op(p["B"], u, model) + s = _score_op(V["B"], u, model) o = f16(np.einsum("...hts,...hsd->...htd", softmax_q7(s), v)) o8 = q8(np.moveaxis(o, -3, -2).reshape(lead + (T, heads * 32))) - z = mm8(f16(p["bp"] + x), o8, W["Wp"], model) + z = mm8(f16(V["bp"] + x), o8, W["Wp"], model) m8 = q8(norm_q7(z, gamma_q7(p["gamma2"]))) - acc = f16(p["b2"] + z) - for sl in range(W["W1"].shape[0] // 32): + acc = f16(V["b2"] + z) + nsl = W["W1"].shape[0] // 32 + for sl in range(nsl): r = slice(32 * sl, 32 * sl + 32) - hs = mm8(p["b1"][r], m8, W["W1"][r], model) - acc = mm8(acc, q8(gelu_q7(hs)), W["W2"][:, r], model) + hs = mm8(V["b1"][r], m8, W["W1"][r], model) + # f32chain: W2's chain runs over all slices in float32, rounded to FP16 once after the last + acc = mm8(acc, q8(gelu_q7(hs)), W["W2"][:, r], model, final=(model != "f32chain" or sl == nsl - 1)) return acc @@ -340,6 +371,7 @@ def swin_block(x, p, offset, heads, consts, border="mirror", halo=0, max_tokens= p = {k: np.asarray(v, dtype=np.float64) for k, v in p.items()} rows, cols = window_grid(H, W, offset, win) out = np.full((H, W, C), np.nan) + covered = np.zeros((H, W), bool) # not the NaN fill: a NaN input member legitimately gives NaN outputs fr, fc = border_index(border, H, h), border_index(border, W, h) if consts.get("mm8") == "trunc13": max_tokens = min(max_tokens, 4096) @@ -359,7 +391,8 @@ def swin_block(x, p, offset, heads, consts, border="mirror", halo=0, max_tokens= R = np.broadcast_to(r[:, None, :, None], sel.shape) K = np.broadcast_to(cols[None, :, None, :], sel.shape) out[R[sel], K[sel]] = x2[sel] - if np.isnan(out).any(): + covered[R[sel], K[sel]] = True + if not covered.all(): raise AssertionError("window grid does not cover the map") return out @@ -619,6 +652,44 @@ def reconstruct(x, params, t=_table.TABLE, ldr=False): return _recon.reconstruct(x, params["post"]["s"], t, ldr) +# ---- the shared stages in their FP16 form --------------------------------------------------------------------------- + +def run_stage_q7(stage, p, x, offset, skip=None, t=_table.TABLE): + """One shared stage (enc1, enc2, b0-b5, dec2, dec1; Model M's shapes, L's weights) in the FP16 form (table + shared_form "q7", Q8 Part B, Q10). x: the stage's FP8 input map (enc / b: [H, W, C]; dec: [H/2, W/2, C_low]); + skip (dec): the matching encoder stage's FP8 block output [H, W, C]. Encoder: block (border "mirror") -> FP8 output + = the skip; merged = FP8(clamp(the Wm chain over the 2 x 2 patch of the FP8 output (space_to_depth order), started + from bm), +-MERGE_CLAMP)). Bottleneck: FP8(block). Decoder: E = FP8(the We chain started from be) on the coarse + grid, depth_to_space to [H, W, C]; the block's member input RN16(E + S) (swin_block skip: S by the fine mirror, E by + the table's shared_dec_border), output FP8. Returns Model M's run_stage keys (float32 FP8 values).""" + cfg = _swin.STAGES[stage] + k = _table.block_consts(t, stage) + k["mm8"] = t.get("mm8", "exact") + model = k["mm8"] + if k.get("form") != "q7": + raise ValueError("run_stage_q7: the stage's consts must have form 'q7'") + x = np.asarray(x, np.float64) + if cfg["kind"] == "dec": + e = q8(mm8(f16(p["be"]), x, q8(p["We"]), model)) # [H/2, W/2, 4C] + y = swin_block(depth_to_space(e), p, offset, cfg["heads"], k, t["shared_dec_border"], + skip=np.asarray(skip, np.float64)) + else: + y = swin_block(x, p, offset, cfg["heads"], k, "mirror") + out = q8(y) + if cfg["kind"] == "enc": + m = mm8(f16(p["bm"]), space_to_depth(out), q8(p["Wm"]), model) + merged = q8(np.minimum(np.maximum(m, -_table.MERGE_CLAMP), _table.MERGE_CLAMP)) + return {"skip": out.astype(np.float32), "merged": merged.astype(np.float32)} + return {"out": out.astype(np.float32)} + + +def run_shared_stage(stage, p, x, offset, skip=None, t=_table.TABLE): + """A shared stage by the table's shared_form: "m" Model M's oracle, "q7" run_stage_q7.""" + if t.get("shared_form", "m") == "q7": + return run_stage_q7(stage, p, x, offset, skip, t) + return _swin.run_stage(stage, p, x, offset, skip) + + # ---- whole network -------------------------------------------------------------------------------------------------- def network(x_enc, x_dec, params, parity, t=_table.TABLE, keep=False): @@ -634,15 +705,15 @@ def network(x_enc, x_dec, params, parity, t=_table.TABLE, keep=False): cfg = _swin.STAGES[s] off = _swin.window_offset(s, parity) if cfg["kind"] == "enc": - o = _swin.run_stage(s, params[s], x, off) + o = run_shared_stage(s, params[s], x, off, t=t) skips[s] = o["skip"] x = o["merged"] elif cfg["kind"] == "dec": skip = skips["enc2" if s == "dec2" else "enc1"] - o = _swin.run_stage(s, params[s], x, off, skip) + o = run_shared_stage(s, params[s], x, off, skip, t=t) x = o["out"] else: - o = _swin.run_stage(s, params[s], x, off) + o = run_shared_stage(s, params[s], x, off, t=t) x = o["out"] res[s] = o xd = dict(x_dec) diff --git a/clean/l/reference/recon.py b/clean/l/reference/recon.py index 7ae5980a..de08cbdf 100644 --- a/clean/l/reference/recon.py +++ b/clean/l/reference/recon.py @@ -133,7 +133,7 @@ def target_positions_f32(rw, rh, dw, dh, jx, jy): return px, py -def gather_half(feat, C, px, py, rw, rh, window): +def gather_half(feat, C, px, py, rw, rh, window, c1="item3"): """Step 2 in the model's precision (spec side, confirmed for Model L and Model M): - per render pixel: A = h(f32(a -0.721325)), B = h(f32(b -0.721325)), C = h(f32(c -1.44265)); the nine features in FP16 (feat must already be FP16 values); @@ -143,7 +143,9 @@ def gather_half(feat, C, px, py, rw, rh, window): - w = 2^eps in float32, flushed to +0 below 2^-126; W = float32 sum of the unrounded w in tap order; - F_k = fused_half(h(w), f_k, F_k), from +0; r = 1 / W, f-bar_k = float32(r F_k). Tap order: i (columns) outer, j (rows) inner. Fused half multiply-adds are evaluated in float64 and rounded once - to FP16 (exact unless the addends' exponents differ by more than about 30, far outside these values).""" + to FP16 (exact unless the addends' exponents differ by more than about 30, far outside these values). + c1="machine": the exponent in the machine-code form (PRESET-M-C1C2-machine-code.md): the row term fused too, + eps = fused_half(h(u v), C, fused_half(h(v v), B, h(h(u u) A))).""" ax, b = _post.kernel_window(px, py) Cc = np.asarray(C, np.float64) kA = _h(_f32(Cc[..., 0] * _f32(-0.721325))) @@ -165,7 +167,10 @@ def gather_half(feat, C, px, py, rw, rh, window): sy = np.clip(b + j, 0, rh - 1) v = _h(dyv + j) a_, b_, c_ = kA[sy][:, sx], kB[sy][:, sx], kC[sy][:, sx] - s_ = _h(_h(_h(u * u) * a_) + _h(_h(v * v) * b_)) + if c1 == "machine": + s_ = _h(_h(v * v) * b_ + _h(_h(u * u) * a_)) + else: + s_ = _h(_h(_h(u * u) * a_) + _h(_h(v * v) * b_)) eps = _h(_h(u * v) * c_ + s_) w = _f32(np.exp2(eps)) w = np.where(w < 2.0 ** -126, 0.0, w) @@ -206,7 +211,7 @@ def reconstruct(x, s, t=_table.TABLE, ldr=False): feat = np.concatenate([ec, np.asarray(x["B"], np.float64), np.asarray(x["A"], np.float64)], axis=2) if rc.get("gather", "float64") == "half": px, py = target_positions_f32(rw, rh, dw, dh, jx, jy) - W, fbar = gather_half(_h(feat), x["C"], px, py, rw, rh, window) + W, fbar = gather_half(_h(feat), x["C"], px, py, rw, rh, window, rc.get("c1", "item3")) else: px, py = _post.target_positions(rw, rh, dw, dh, jx, jy) W, fbar = gather(feat, x["C"], px, py, rw, rh, window) @@ -282,7 +287,8 @@ def along(qx, qy): # alpha = sat(f32(f32(s W) g-bar)), beta' = fused(alpha f32(1 - beta-bar) + beta-bar), # H^ = fused(f32(C - Hl) beta' + Hl), O^ likewise. C_lin = r32(dec(cbar)) - H_lin = r32(H * e) + # encoded combination (Model M's original): one decode of the combined H, H_lin = (D(H) / e_prev) e + H_lin = r32(H * e) if linear else r32(r32(r32(dec(H)) / e_prev) * e) alpha = _post.clamp(r32(r32(np.float32(s) * W) * gbar), 0.0, 1.0) bh2 = fma(alpha, r32(1.0 - bh), bh)[..., None] bo2 = fma(alpha, r32(1.0 - bo), bo)[..., None] diff --git a/clean/l/reference/table.py b/clean/l/reference/table.py index abbfc7b4..b3a236c4 100644 --- a/clean/l/reference/table.py +++ b/clean/l/reference/table.py @@ -108,18 +108,32 @@ # "skip": each window member's input is RN16(src member + skip member); the skip member is read with the mirror # about the grid size (fine positions), whatever the op's border (the decoder: "coarse" for the projection). "skip_map": "x3", + # The ten shared stages (enc1, enc2, b0-b5, dec2, dec1, L's own weights): "m" = Model M's oracle (clean/reference/ + # swin.py run_stage); "q7" = their FP16 form (oracle.run_stage_q7: the blocks as L's own, the merges' FP8 op chains + # from the bias with the clamp to +-6.28125 then FP8, the expansions' chains from the bias then FP8, the decoder + # blocks' input RN16(E + S) with S, the encoder block's FP8 output, read with the fine mirror). + "shared_form": "q7", + # The window border of a shared decoder stage's expansion E for members outside the grid (form "q7"): "coarse" = + # as dec0's projection, the coarse mirror (spec Q10b: FACT for dec2 and dec1; S takes the fine mirror), the + # member's own sub-pixel kept (Q10b: INFERENCE, UNCONFIRMED U2); "mirror" = the fine mirror (Model M's oracle). + "shared_dec_border": "coarse", } +MERGE_CLAMP = 6.28125 # RN16(2 pi): the merges' clamp (Q8 B4) + DEC_PIXEL_VALUES = {"l": len(L_DEC_VALUES), "m": len(M_DEC0_VALUES)} -PROVISIONAL = { - "S8": "the ten shared stages run Model M's oracle (clean/reference/swin.py, Model M's block arithmetic); their FP16 " - "form is pending a decision", -} +PROVISIONAL = {} # (S8 ended: the shared stages' FP16 form is the default since Model L's q7 kernels run them) # Model M's reconstruction in the same terms (clean/reference/post.py reconstruct): for reductions and tests. M_RECON = {"rule": "d_over_render_at_least", "ratio": 3.0, "variants": ("3/2", "3/1"), "window": 3, "history_combine": "encoded", "fallback": "encoded_colour", "gather": "float64", "blend": "float64"} +# Model M's ORIGINAL reconstruction as the spec side states it (handoff preset-M/from-spec/Q11C-M-POST-ITEMS.md; Q11 +# part C evaluation, kernel l_post__morig.co): Model L's form (half-precision gather, float32 blend) with M's three +# steps (window 3, encoded-domain combination, encoded fallback) and the gather exponent in the machine-code form +# ("c1": "machine", PRESET-M-C1C2-machine-code.md). C2 (the Catmull-Rom weights' fused forms) is a float32 rounding +# matter below this oracle's float64 history sample; the C1/C2 probe checks it bit for bit. +M_RECON_ORIG = dict(M_RECON, gather="half", blend="float32", c1="machine") def recon_size(output_w, output_h): @@ -142,6 +156,9 @@ def recon_variant(t, render_w, render_h, output_w, output_h): UNCONFIRMED = { "U1": "encoder halo: a mirrored render position below 0 (only when fw + halo > 2 rw - 2, i.e. render sizes near " "the padding) is clamped to 0", + "U2": "decoder stages (dec0's projection, dec2, dec1): an out-of-grid member takes the expansion output of its own " + "sub-pixel (the parity of its unmirrored fine position) from the coarse-mirrored token (spec Q10b: the coarse " + "mirror for E and the fine mirror for S are FACT, the kept sub-pixel is INFERENCE)", } diff --git a/clean/l/runner/l_d3d12_check.py b/clean/l/runner/l_d3d12_check.py index 03d18083..d78e33c1 100644 --- a/clean/l/runner/l_d3d12_check.py +++ b/clean/l/runner/l_d3d12_check.py @@ -55,6 +55,51 @@ def prepare(build, rocm): return prepare_replay(build, rocm, D3D12_KERNELS + CONTROL_KERNELS) +# ---- the default leg (standing rule, oversight 2026-11-15) ----------------------------------------------------------- +# Every release-path check also runs once with NO model / form environment variable set (the shipped default); its +# output must equal the explicit setting's byte for byte. FORM_VARS: the variables that choose a model's form or route. +FORM_VARS = ("WG_L_FORM", "WG_M_FORM", "WG_L_POST", "WG_M_POST", "WG_HYB_TABLES", "WG_SWIN_FUSE_STAGES") + + +def default_env(base=None): + """base (default os.environ) without any FORM_VARS variable.""" + return {k: v for k, v in (base if base is not None else os.environ).items() if k.upper() not in FORM_VARS} + + +def replay_outputs(cases_path): + """The output file names of a kernel_replay cases file (arg out ... ...; a chain's save ).""" + names = [] + for ln in open(cases_path, encoding="utf-8"): + p = ln.split() + if len(p) >= 4 and p[0] == "arg" and p[1] == "out": + names.append(p[3]) + elif len(p) == 3 and p[0] == "save": + names.append(p[2]) + return names + + +def replay_default_leg(build, cases_path, env, timeout=150): + """Re-run kernel_replay on cases_path with env minus FORM_VARS and compare every output file with the explicit + run's (already on disk); the explicit outputs are restored afterwards. Returns the record's default-leg entry.""" + cdir = os.path.dirname(cases_path) + names = replay_outputs(cases_path) + before = {n: open(os.path.join(cdir, n), "rb").read() for n in names if os.path.exists(os.path.join(cdir, n))} + r = subprocess.run([os.path.join(build, "kernel_replay.exe"), build, cases_path], env=default_env(env), + capture_output=True, text=True, timeout=timeout, creationflags=0x08000000) + differ = [] + for n in names: + p = os.path.join(cdir, n) + got = open(p, "rb").read() if os.path.exists(p) else None + if got != before.get(n): + differ.append(n) + if n in before: + with open(p, "wb") as f: + f.write(before[n]) + return {"env_removed": [k for k in FORM_VARS if k in env], "replay_exit": r.returncode, + "outputs": len(names), "identical": len(names) - len(differ), "differ": differ[:8], + "pass": r.returncode == 0 and not differ, "tail": "" if r.returncode == 0 else (r.stdout[-400:] + r.stderr[-200:])} + + def prepare_replay(build, rocm, kernels): """The D3D12 ports `kernels` (and native_fill, the device's memset) into /dx12, and kernel_replay.exe.""" from clean.runner.hip import hip_child_environment diff --git a/clean/l/runner/l_d3d12_ops_check.py b/clean/l/runner/l_d3d12_ops_check.py index 0d897cfa..027270b0 100644 --- a/clean/l/runner/l_d3d12_ops_check.py +++ b/clean/l/runner/l_d3d12_ops_check.py @@ -369,15 +369,26 @@ def enc_direct_cases(kernel, rng): return cases +def lc12_post_cases(kernel, rng): + """lc12/l_post_recon[_p12][_ldr]: the C1/C2 reconstruction (D3D12 port LP_C12 = 1 of l_post__lc12.co, tag lc12) + against l_post__lc12.co's kernel on HIP, on post_cases' inputs.""" + base = kernel.split("/", 1)[1] + for c in post_cases(base, rng): + c = dict(c) + c.update({"co": "l_post__lc12.co", "tag": "lc12", "hip_kernel": base, "kernel": kernel}) + yield c + + +LC12_KERNELS = ("lc12/l_post_recon", "lc12/l_post_recon_p12", "lc12/l_post_recon_ldr", "lc12/l_post_recon_p12_ldr") KINDS = {"l_linear_32_32_relu": linear_cases, "l_linear_32_32_f16": linear_cases, "l_project_gather": project_cases, "l_dec_project": project_cases, "l_coef": coef_cases, "l_coef_pack12": coef_cases, "lev/input_encoder": enc_cases, "l_post_recon": post_cases, "l_post_recon_p12": post_cases, "l_post_recon_ldr": post_cases, "l_post_recon_p12_ldr": post_cases, "l_post_filter_ldr": filter_ldr_cases, "lev/input_encoder_direct": enc_direct_cases, - "swin_block_c64_add_ex_nc": dec1_ex_cases} + "swin_block_c64_add_ex_nc": dec1_ex_cases, **{k: lc12_post_cases for k in LC12_KERNELS}} CONTROL_KINDS = {"swin_block_c96_nc": control_cases} # kernels that get a negative control of their own (besides the run's first case) -NEG_KERNELS = ("l_post_recon", "l_post_recon_p12", "l_post_recon_ldr", "l_post_recon_p12_ldr", "l_post_filter_ldr") +NEG_KERNELS = ("l_post_recon", "l_post_recon_p12", "l_post_recon_ldr", "l_post_recon_p12_ldr", "l_post_filter_ldr") + LC12_KERNELS CO = "l_ops.co" @@ -536,6 +547,16 @@ def negative(nm): for k in kernels} res["pass"] = r.returncode == 0 and caught is True and all(v["cases"] > 0 and v["identical"] == v["cases"] for v in res["per_kernel"].values()) + # The C1/C2 forms against the default reconstruction on HIP, the same cases: they must differ somewhere (the lc12 + # build is not l_post.co); the default kernels' cases come from the same seeds (both in --kernels). + for k in LC12_KERNELS: + b = k.split("/", 1)[1] + if k in kernels and b in kernels: + kn, bn = k.replace("/", "_"), b + pairs = [(n, bn + n[len(kn):]) for n in hip_out if hip_out[n][0] == k] + differ = sum(hip_out[a][1] != hip_out[c][1] for a, c in pairs if c in hip_out) + res.setdefault("lc12_differs_from_default_on_hip", {})[k] = {"cases": len(pairs), "differing": differ} + res["pass"] = res["pass"] and differ > 0 for sfx in ("", "_ldr"): if "l_post_recon" + sfx in kernels and "l_post_recon_p12" + sfx in kernels: # The same cases (same seeds) on both coefficient forms: on HIP, the pack12 kernel's raw output bytes must diff --git a/clean/l/runner/l_d3d12_profile.py b/clean/l/runner/l_d3d12_profile.py index f2f7fcdb..453fb06b 100644 --- a/clean/l/runner/l_d3d12_profile.py +++ b/clean/l/runner/l_d3d12_profile.py @@ -29,10 +29,10 @@ SHAPES = {"1080p-quality": ((1280, 720), (1920, 1080)), "1440p-quality": ((1706, 960), (2560, 1440))} ENC = ("lev/input_encoder", "l_linear_32_32_relu", "l_linear_32_32_f16", "l_linear_32_32_fused", "l_swin_block_c32_enc", - "l_project_gather", "input_encoder") -DEC = ("l_dec_project", "l_swin_block_c32_dec", "l_coef", "coef_decoder", "_add_dec0") + "l_q7_block_c32_enc", "l_project_gather", "input_encoder") +DEC = ("l_dec_project", "l_swin_block_c32_dec", "l_q7_block_c32_dec", "l_coef", "coef_decoder", "_add_dec0") POST = ("post_", "scale_rgb", "f16lut") -SWIN = ("swin_", "patch_merge", "expand_", "bneck/") +SWIN = ("swin_", "patch_merge", "expand_", "bneck/", "l_q7_block_c", "l_q7_merge", "l_q7_expand") # l_q7_*: FP16 form def step(name): diff --git a/clean/l/runner/l_frame_check.py b/clean/l/runner/l_frame_check.py index 5febcf53..fd85dd6b 100644 --- a/clean/l/runner/l_frame_check.py +++ b/clean/l/runner/l_frame_check.py @@ -220,6 +220,12 @@ def frame_inputs(rng, rw, rh, reset, special=True, index=None): BNECK_UNWRITTEN = ("b0.out", "b1.out", "b2.out", "b3.out", "b4.out") # the megakernel writes only b5's output +def q7_route(rec): + """A recording of Model L's FP16-form route (kernels l_form=q7): every stage runs on its own, nothing stays on + chip (no bottleneck megakernel, no fused merge), so b0 ... b4 and enc1's merged map are written.""" + return (rec.get("kernels") or {}).get("l_form") == "q7" + + def fill_sentinels(p, value, first): """Every output this frame writes set to the run's sentinel (before the frame).""" for n in RAW_OUTPUTS: @@ -228,6 +234,17 @@ def fill_sentinels(p, value, first): p.fill(n, value) +def with_l_x3(p, raw): + """Model L: also the encoder block's map l.x3 (the decoder block's skip; a scratch map, not a byte-compared output), + for --compare to run the decoder block on the runtime's own inputs.""" + if (p.kernels() or {}).get("model") == "L": + try: + raw["l.x3"] = p.raw("l.x3") + except Exception: # a runtime without the map: --compare falls back to the oracle's skip + pass + return raw + + def raw_outputs(p): """{output: its defined bytes} after a frame. Pack12 frames (Model L on D3D12, Model M's native frame): image A 8 bytes and image C 4 bytes per render pixel, B unused (wg_rt_buffer reports the base layout's 8 bytes of C).""" @@ -360,7 +377,7 @@ def run_d3d12(build, special=True, stages=False): fr = frame_inputs(rng, rw, rh, reset, special, index=i) fill_sentinels(p, SENTINELS[config], i == 0) out = p.frame(fr, debug=True) - frames.append({"inputs": fr, "out": out, "raw": raw_outputs(p)}) + frames.append({"inputs": fr, "out": out, "raw": with_l_x3(p, raw_outputs(p))}) rec = {"model": model, "config": config, "sentinel": SENTINELS[config], "kernels": p.kernels(), "ldr": LDR and model == "L", "notes": p.notes(), @@ -394,11 +411,12 @@ def compare_backends(hip, dx, same_post=True): keep on chip (enc1's merged map; Model M's dec2 / dec1 outputs, which its fused decoder reads on chip) and, with the fused pack12 post, the display image (that kernel writes none; Model L's own post writes it), on the first frame the previous frame's state on both.""" - fused_onchip = ("enc1.merged",) + (("dec2.out", "dec1.out") if hip["model"] == "M" else ()) + q7 = q7_route(hip) and q7_route(dx) + fused_onchip = (() if q7 else ("enc1.merged",)) + (("dec2.out", "dec1.out") if hip["model"] == "M" else ()) no_display = "post_tf" in str(dx["kernels"].get("post_last", "")) def unwritten(k, n): - if n in BNECK_UNWRITTEN or (k == 0 and n in UNWRITTEN_ON_FIRST): + if (n in BNECK_UNWRITTEN and not q7) or (k == 0 and n in UNWRITTEN_ON_FIRST): return "both" if n in fused_onchip or (n == "post.display" and no_display): return "b" @@ -411,10 +429,12 @@ def compare_d3d12_configs(st, df): """D3D12 default kernels (df) against the run with the fusions off (st), raw defined bytes. Not written by design: b0 ... b4 on both (the megakernel runs in both), the display image on both (the fused pack12 post), enc1's merged map with the defaults (fusion pm keeps it on chip), on the first frame the previous frame's state on both.""" + q7 = q7_route(st) and q7_route(df) + def unwritten(k, n): - if n in BNECK_UNWRITTEN or n == "post.display" or (k == 0 and n in UNWRITTEN_ON_FIRST): + if (n in BNECK_UNWRITTEN and not q7) or n == "post.display" or (k == 0 and n in UNWRITTEN_ON_FIRST): return "both" - if n == "enc1.merged": + if n == "enc1.merged" and not q7: return "b" return None return raw_compare(st, df, st["sentinel"], df["sentinel"], unwritten), unwritten @@ -490,11 +510,12 @@ def compare_d3d12(build, json_path): "post_compared_with_hip": same_post} if same_post: # the record-level negative controls on this comparison dflt = recs["default"] + q7 = q7_route(dflt) and q7_route(recs["d3d12"]) + onchip = ("dec2.out", "dec1.out") if q7 else ("enc1.merged", "dec2.out", "dec1.out") res[model]["negative_controls"] = negative_controls( dflt, recs["d3d12"], dflt["sentinel"], recs["d3d12"]["sentinel"], - lambda k, n: ("both" if n in BNECK_UNWRITTEN or (k == 0 and n in UNWRITTEN_ON_FIRST) else - "b" if n in ("enc1.merged", "dec2.out", "dec1.out") or (n == "post.display" and - no_display) else None), + lambda k, n: ("both" if (n in BNECK_UNWRITTEN and not q7) or (k == 0 and n in UNWRITTEN_ON_FIRST) else + "b" if n in onchip or (n == "post.display" and no_display) else None), convert_b=True) if not same_post: # Model L: HIP runs l_post_recon, D3D12 Model M's fused pack12 post, and each frame's encoder reads the @@ -565,7 +586,7 @@ def run(build, rocm, special=True): fr = frame_inputs(rng, rw, rh, reset, special, index=i) fill_sentinels(p, SENTINELS[config], i == 0) out = p.frame(fr, debug=True) - frames.append({"inputs": fr, "out": out, "raw": raw_outputs(p)}) + frames.append({"inputs": fr, "out": out, "raw": with_l_x3(p, raw_outputs(p))}) rec = {"model": model, "config": config, "sentinel": SENTINELS[config], "kernels": p.kernels(), "ldr": LDR and model == "L", "notes": p.notes(), @@ -777,7 +798,14 @@ def compare_model(rec, params, post_ref=None, bneck_unit=False, table=None): x0["reduced_history"] = np.zeros_like(reduced) ref = L.encode(x0, params, f, t=table) if model == "L" else ME.encode(x0, params["enc0"]) enc_skip = ref.get("skip") if model == "L" else None # the encoder block output the decoder block reads + x3 = fd.get("raw", {}).get("l.x3") + if model == "L" and x3 is not None and x3.size == fh * fw * 32: + # the runtime's own encoder block map (recorded l.x3): the decoder block on the runtime's own inputs + from clean.reference.fp8 import fp8_decode + enc_skip = fp8_decode(np.asarray(x3, np.uint8)).reshape(fh, fw, 32) m["enc0"] = {"features": fp8_stage(out["enc0"]["features"], ref["features"])} + if model == "L" and enc_skip is not None and enc_skip is not ref.get("skip"): + m["enc0"]["x3"] = fp8_stage(enc_skip, ref["skip"]) # the encoder block map against the oracle's for s in ("m_near", "m_far", "m_center"): m["enc0"][s] = motion_surface(out["enc0"][s], ref[s]) @@ -787,7 +815,10 @@ def compare_model(rec, params, post_ref=None, bneck_unit=False, table=None): off = S.window_offset(s, f) kind = S.STAGES[s]["kind"] skip = skips.get("enc2" if s == "dec2" else "enc1") if kind == "dec" else None - ref = S.run_stage(s, params[s], x, off, skip, gelu="sigmoid") + if model == "L" and (table or T.TABLE).get("shared_form", "m") == "q7": + ref = L.run_stage_q7(s, params[s], x, off, skip, table or T.TABLE) # the FP16 form (L's q7 kernels) + else: + ref = S.run_stage(s, params[s], x, off, skip, gelu="sigmoid") if bneck_unit and s in ("b0", "b1", "b2", "b3", "b4"): x = ref["out"] # the reference's own chain (the runtime does not write b0 ... b4) continue @@ -900,8 +931,12 @@ def compare(build, json_path): # the default kernels against the six-launch bottleneck, raw defined bytes: b0 ... b4 are not written by the # default megakernel; on the first frame the previous frame's state is written by neither (a reset frame does # not read it) + # Model L's FP16-form route (kernels "l_form=q7") runs every stage on its own in both configurations: no + # megakernel, so b0 ... b4 are written by both and compared + on_q7 = q7_route(recs["default"]) + def unwritten(k, n): - if n in BNECK_UNWRITTEN: + if n in BNECK_UNWRITTEN and not on_q7: return "b" return "both" if k == 0 and n in UNWRITTEN_ON_FIRST else None st_, df_ = recs["stages"], recs["default"] diff --git a/clean/l/runner/l_hyb_block_check.py b/clean/l/runner/l_hyb_block_check.py new file mode 100644 index 00000000..6172be5c --- /dev/null +++ b/clean/l/runner/l_hyb_block_check.py @@ -0,0 +1,207 @@ +# SPDX-License-Identifier: MIT +"""Block-level check of the HYBRID C32 kernels (the first leg of the hybrid check design; the per-stage check with the +SW_HYB_DUMP variant follows): random parameters in the hybrid values (l_hyb_pack.py), the encoder (FP16 halo map in) +and decoder (FP8 map in, FP16 out) blocks on the L block grids, with special values in every other case. + + 1. HIP (l_swin_hyb.co) == D3D12 (l_swin_block_c32_*_hyb) raw bytes; each output starts with its own sentinel; a + half-grid D3D12 run is the negative control (must differ). + 2. Encoder only, informational: the block output against the oracle's FP16 form with f32chain chains (border + "input", halo 2): counts of identical / differing FP8 codes (the per-stage check gives the pass / fail). + + python clean/l/runner/l_hyb_block_check.py --build [--prepare] [--run] [--seeds N] [--json out.json] + [--mutant] also the order mutants l_swin_block_c32_*_hyb_mut (SW_HYB_MUT_ORDER=1, same build): each mutant + kernel must differ from HIP in at least one case, or the command fails (exit 1) + [--mutant-unchanged] self-check of that leg: the mutant leg run on the unmodified kernels; the command must + then FAIL (it does unless the exit condition is broken) +""" +import argparse +import ctypes +import json +import os +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import oracle as O # noqa: E402 +from clean.l.reference import table as T # noqa: E402 +from clean.l.runner import l_d3d12_check as LD # noqa: E402 +from clean.l.runner.l_hyb_pack import pack_block_hyb # noqa: E402 +from clean.reference.fp8 import fp8_decode, fp8_encode, q8 # noqa: E402 + +KERNELS = {"l_swin_block_c32_enc_hyb": "enc", "l_swin_block_c32_dec_hyb": "dec"} + + +def cases(rng, seeds): + for seed in range(seeds): + for kernel, kind in KERNELS.items(): + tkind = "first" if kind == "enc" else "second" + h = T.block_halo(T.TABLE, tkind) + for gi, (H, W) in enumerate(LD.L_BLOCK_GRIDS): + for parity in (0, 1): + p = O.random_block_params(rng, 32, 2, 128) + blob = pack_block_hyb(p, 32, 2, 128) + off = T.block_offset(T.TABLE, tkind, parity) + special = (gi + parity + seed) % 2 == 1 + scale = 60.0 if special else 1.5 + if kind == "enc": + x = rng.normal(0, scale, (H + 2 * h, W + 2 * h, 32)).astype(np.float16) + if special: + pick = rng.random(x.shape) < 0.05 + x[pick] = rng.choice(LD.SPECIAL_F16, int(pick.sum())) + yield kernel, H, W, off, x.tobytes(), H * W * 32, blob, p, None + else: + y = fp8_encode(q8(rng.normal(0, scale, (H, W, 32)))).astype(np.uint8) + if special: + pick = rng.random(y.shape) < 0.05 + y[pick] = rng.choice(LD.SPECIAL_FP8, int(pick.sum())) + s = fp8_encode(q8(rng.normal(0, scale, (H, W, 32)))).astype(np.uint8) # the skip S + if special: + pick = rng.random(s.shape) < 0.05 + s[pick] = rng.choice(LD.SPECIAL_FP8, int(pick.sum())) + yield kernel, H, W, off, y.tobytes(), H * W * 64, blob, p, s.tobytes() + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", default=os.path.join(r"C:\Program Files\AMD\ROCm", "7.2")) + ap.add_argument("--prepare", action="store_true") + ap.add_argument("--run", action="store_true") + ap.add_argument("--seeds", type=int, default=1) + ap.add_argument("--mutant", action="store_true", + help="also the order mutants: each mutant kernel must differ from HIP in at least one case (exit 1 " + "otherwise)") + ap.add_argument("--mutant-unchanged", action="store_true", + help="self-check of the mutant leg: run it on the UNMODIFIED kernels; the command must then fail") + ap.add_argument("--json") + a = ap.parse_args(argv) + if a.prepare: + from clean.runner import build as B + LD.prepare_replay(a.build, a.rocm, list(KERNELS)) + B.run(B.device_command(B.find_clang(a.rocm), a.rocm, os.path.join(ROOT, "clean", "l", "kernels", "l_swin_hyb.hip"), + os.path.join(a.build, "l_swin_hyb.co"), + extra=["-I", os.path.join(ROOT, "clean", "kernels")])) + if not a.run: + return 0 + from clean.runner.hip import Hip + h = Hip(a.rocm) + mod = h.load_module(open(os.path.join(a.build, "l_swin_hyb.co"), "rb").read()) + rng = np.random.default_rng(7) + cdir = os.path.join(a.build, "hyb_block") + os.makedirs(cdir, exist_ok=True) + lines, hip_out, meta = [], {}, [] + for n, (kernel, H, W, off, xb, nbytes, blob, p, sb) in enumerate(cases(rng, a.seeds)): + # HIP + px, pw, po = h.malloc(len(xb)), h.malloc(len(blob)), h.malloc(nbytes) + ps = h.malloc(len(sb)) if sb else 0 + if sb: + h.upload(ps, sb) + h.upload(px, xb) + h.upload(pw, blob) + h.memset(po, LD.HIP_SENTINEL, nbytes) + ox, oy = off + nwin = -(-(W + ox) // 4) * -(-(H + oy) // 4) + h.launch(h.get_function(mod, kernel), -(-nwin // 8), 256, + [ctypes.c_void_p(px), ctypes.c_void_p(ps), ctypes.c_void_p(po), ctypes.c_void_p(pw), ctypes.c_int(H), + ctypes.c_int(W), ctypes.c_int(ox), ctypes.c_int(oy)]) + h.sync() + hip_out[n] = bytes(h.download(po, nbytes)) + with open(os.path.join(cdir, f"h{n}.bin"), "wb") as f: # kept for diagnosis + f.write(hip_out[n]) + for q in (px, pw, po) + ((ps,) if sb else ()): + h.free(q) + # D3D12 case (same inputs; the decoder's FP16 output goes through the out pointer as the HIP epilogue does) + for nm, data in ((f"x{n}.bin", xb), (f"w{n}.bin", blob)) + (((f"s{n}.bin", sb),) if sb else ()): + with open(os.path.join(cdir, nm), "wb") as f: + f.write(data) + grid = -(-nwin // 8) + sarg = f"arg buf s{n}.bin" if sb else "arg null" + lines += [f"case c{n}", f"kernel {kernel} {grid} 256", f"arg buf x{n}.bin", sarg, + f"arg out {nbytes} o{n}.bin {LD.D3D12_SENTINEL}", f"arg buf w{n}.bin", f"arg u32 {H}", f"arg u32 {W}", + f"arg u32 {ox}", f"arg u32 {oy}", ""] + if n == 0: # negative control: half the workgroups + lines += [f"case neg{n}", f"kernel {kernel} {max(1, grid // 2)} 256", f"arg buf x{n}.bin", sarg, + f"arg out {nbytes} neg{n}.bin {LD.D3D12_SENTINEL}", f"arg buf w{n}.bin", f"arg u32 {H}", + f"arg u32 {W}", f"arg u32 {ox}", f"arg u32 {oy}", ""] + meta.append((n, kernel, H, W, off, p, xb, sb)) + with open(os.path.join(cdir, "cases.txt"), "w") as f: + f.write("\n".join(lines)) + a.mutant = a.mutant or a.mutant_unchanged + if a.mutant: # the same cases on the mutant kernels, outputs m.bin (the negative-control cases dropped) + import re + out, keep = [], True + for ln in lines: + if ln.startswith("case "): + keep = not ln.startswith("case neg") + if keep: + if not a.mutant_unchanged: + ln = re.sub(r"^kernel (\S+_hyb) ", r"kernel \1_mut ", ln) + ln = re.sub(r" o(\d+)\.bin ", r" m\1.bin ", ln) + out.append(ln) + with open(os.path.join(cdir, "cases_mut.txt"), "w") as f: + f.write("\n".join(out)) + env = dict(os.environ, WG_L_FORM="hybrid") + res = {"cases": []} + caught_by_kernel = {} + for label, cfile, pre in (("d3d12", "cases.txt", "o"),) + ((("mutant", "cases_mut.txt", "m"),) if a.mutant else ()): + r = subprocess.run([os.path.join(a.build, "kernel_replay.exe"), a.build, os.path.join(cdir, cfile)], env=env, + capture_output=True, text=True, timeout=150, creationflags=0x08000000) + if r.returncode: + raise SystemExit(f"kernel_replay ({label}) failed: {r.stdout[-800:]}{r.stderr[-400:]}") + same = 0 + for n, kernel, H, W, off, p, xb, sb in meta: + d = open(os.path.join(cdir, f"{pre}{n}.bin"), "rb").read() + eq = d == hip_out[n] + same += eq + if label == "mutant" and not eq: + caught_by_kernel[kernel] = caught_by_kernel.get(kernel, 0) + 1 + if label == "d3d12": + c = {"case": n, "kernel": kernel, "H": H, "W": W, "identical": eq, + "differing_bytes": int(np.count_nonzero(np.frombuffer(d, np.uint8) != np.frombuffer(hip_out[n], np.uint8)))} + if kernel.endswith("enc_hyb"): # informational oracle comparison (FP8 codes) + x = np.frombuffer(xb, np.float16).reshape(H + 4, W + 4, 32).astype(np.float64) + k = dict(T.consts(T.TABLE)) if hasattr(T, "consts") else {"window": 4, "head_dim": 32} + k.update({"form": "q7", "mm8": "f32chain", "head_dim": 32, "window": 4}) + want = fp8_encode(q8(O.swin_block(x, p, off, 2, k, border="input", halo=2))).astype(np.uint8) + got = np.frombuffer(d, np.uint8).reshape(H, W, 32) + bad = (got != want) & ~(((got & 0x7F) == 0x7F) & ((want & 0x7F) == 0x7F)) + c["oracle_fp8_differ"] = int(bad.sum()) + c["oracle_fp8_total"] = int(want.size) + elif kernel.endswith("dec_hyb"): # informational: the FP16 output against the oracle WITH the skip + k = {"form": "q7", "mm8": "f32chain", "head_dim": 32, "window": 4} + xd = fp8_decode(np.frombuffer(xb, np.uint8).reshape(H, W, 32)).astype(np.float64) + sk = fp8_decode(np.frombuffer(sb, np.uint8).reshape(H, W, 32)).astype(np.float64) + want = O.f16(O.swin_block(xd, p, off, 2, k, border="coarse", skip=sk)) + got = np.frombuffer(d, np.float16).reshape(H, W, 32).astype(np.float64) + bad = ~((got == want) | (np.isnan(got) & np.isnan(want))) + c["oracle_f16_differ"] = int(bad.sum()) + c["oracle_f16_total"] = int(want.size) + res["cases"].append(c) + res[f"{label}_identical"] = f"{same}/{len(meta)}" + if label == "d3d12": # the default leg: no form variable set (the shipped default) == WG_L_FORM=hybrid + res["default_leg"] = LD.replay_default_leg(a.build, os.path.join(cdir, cfile), env) + neg = open(os.path.join(cdir, "neg0.bin"), "rb").read() + res["negative_control_caught"] = neg != hip_out[0] + res["pass_bytes"] = (res["d3d12_identical"] == f"{len(meta)}/{len(meta)}" and res["negative_control_caught"] + and res["default_leg"]["pass"]) + if a.mutant: + # fail closed: EVERY mutant kernel (one per kernel checked) must differ from HIP in at least one case + kernels = sorted({kernel for _, kernel, *_ in meta}) + res["mutant_caught_by_kernel"] = {k: caught_by_kernel.get(k, 0) for k in kernels} + res["mutant_caught_by_bytes"] = all(caught_by_kernel.get(k, 0) > 0 for k in kernels) + res["mutant_unchanged_self_check"] = a.mutant_unchanged + res["pass_bytes"] = res["pass_bytes"] and res["mutant_caught_by_bytes"] + print(json.dumps({k: v for k, v in res.items() if k != "cases"}, indent=1)) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + return 0 if res["pass_bytes"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_hyb_bneck_check.py b/clean/l/runner/l_hyb_bneck_check.py new file mode 100644 index 00000000..e8db110c --- /dev/null +++ b/clean/l/runner/l_hyb_bneck_check.py @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: MIT +"""Byte check of the HYBRID bottleneck in one launch (speed lever 4): swin_bneck6_hyb (clean/dx12/kernels/hyb_bneck.hlsl, +bneck.hlsl's persistent queue around swin_block_c128_hyb's body) against six swin_block_c128_hyb launches on the same +input map and the same six random hybrid blobs, on native D3D12 (kernel_replay chains). + + 1. b5's output of the queue kernel == the six launches' b5 output, raw bytes, for every case and every workgroup + count (1: one workgroup runs every item in queue order; 7, 128: the runtime's count; 300: more than fit at once), + every output buffer starting from a sentinel fill. + 2. The sync words are all zero after every launch (the last workgroup resets the queue, the finish counter and every + row counter; word 15 counts bounded-wait timeouts and is never reset: it must stay 0). + 3. Mutant: swin_bneck6_hyb_mut (order mutant 4: the W2 chain end one float32 ulp off before its RN16) must differ. + 4. The default leg (standing rule): kernel_replay with no form variable == WG_L_FORM=hybrid, every saved byte. + 5. Timeout control: swin_bneck6_hyb_tmo (BNECK_SPIN_LIMIT 1: a wait gives up at its first poll of a producer that + is not done) on the two largest grids, 128 workgroups: the launch completes, the abort word (sync word 12) and the + timeout count (word 15) are equal and nonzero (a timed-out wait refuses the frame: the runtime's frame guard reads + word 12 right after the launch), and every other sync word is reset. The twin's launches leave both at 0 (2.). + The block arithmetic itself is swin_block_c128_hyb's (checked against the oracle by l_hyb_stage_check.py / the S3 + checks); the queue adds no arithmetic, so the six launches are the reference. Inputs: random FP8 maps, special codes + (NaN included: both sides are ours) in every other case; grids with partial windows, both parities' offsets. + + python clean/l/runner/l_hyb_bneck_check.py --build [--json out.json] [--time N] + (: a build with swin_block_c128_hyb, swin_bneck6_hyb, swin_bneck6_hyb_mut, swin_bneck6_hyb_tmo and + kernel_replay.exe; GPU) +""" +import argparse +import json +import os +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import oracle as O # noqa: E402 +from clean.l.runner import l_d3d12_check as LD # noqa: E402 +from clean.l.runner.l_hyb_pack import pack_block_hyb # noqa: E402 +from clean.reference import swin as SW # noqa: E402 +from clean.reference.fp8 import fp8_encode, q8 # noqa: E402 + +C, HEADS, MLP = 128, 4, 512 +GRIDS = [(8, 8), (12, 20), (18, 10), (30, 54), (120, 216)] # (H8, W8); 120 x 216: the 1440p Quality bottleneck +GROUPS = (1, 7, 128, 300) +SENT = 0xAB + + +def nwin_of(H, W, ox, oy): + return ((W + ox + 3) // 4) * ((H + oy + 3) // 4) + + +def chain_lines(cdir, n, H, W, offs, special, rng, timed, tmo=False): + """The chains of one case: six launches (ref), the queue kernel at each workgroup count, its mutant; and (tmo) the + timeout control's chain, returned apart (its bytes depend on the timing: not in the default leg's comparison).""" + blobs = [pack_block_hyb(O.random_block_params(rng, C, HEADS, MLP), C, HEADS, MLP) for _ in range(6)] + x = fp8_encode(q8(rng.normal(0, 40.0 if special else 1.5, (H, W, C)))).astype(np.uint8) + if special: + pick = rng.random(x.shape) < 0.05 + x[pick] = rng.choice(LD.SPECIAL_FP8, int(pick.sum())) + for i, b in enumerate(blobs): + with open(os.path.join(cdir, f"w{n}_{i}.bin"), "wb") as f: + f.write(b) + with open(os.path.join(cdir, f"x{n}.bin"), "wb") as f: + f.write(x.tobytes()) + nb = H * W * C + NR = max((H + oy + 3) // 4 for _, oy in offs) + nsync = 4 * (16 + 7 * NR + 16) + wl = [f"buf w{i} {len(blobs[i])} w{n}_{i}.bin" for i in range(6)] + ln = [f"chain r{n}", f"buf x {nb} x{n}.bin"] + wl + [f"buf o{i} {nb} - {SENT}" for i in range(6)] + src = "@x" + for i in range(6): + ox, oy = offs[i & 1] + ln.append(f"step swin_block_c128_hyb {-(-nwin_of(H, W, ox, oy) // 8)} 256 {src} null @o{i} @w{i} " + f"{H} {W} {ox} {oy}") + src = f"@o{i}" + ln += [f"save o5 r{n}.bin"] + ([f"time {timed}"] if timed else []) + [""] + (oxe, oye), (oxo, oyo) = offs + legs = [("swin_bneck6_hyb", "m", g) for g in GROUPS] + [("swin_bneck6_hyb_mut", "u", 128)] + if tmo: + legs.append(("swin_bneck6_hyb_tmo", "t", 128)) + tl = [] + for kern, tag, groups in legs: + nm = f"{tag}{n}_{groups}" + if tag == "t": + ln, tl = tl, ln + ln += [f"chain {nm}", f"buf x {nb} x{n}.bin"] + wl + \ + [f"buf out {nb} - {SENT}", f"buf tmp {nb} - {SENT}", f"buf sync {nsync} - 0", + f"step {kern} {groups} 256 @x @out @tmp @sync @w0 @w1 @w2 @w3 @w4 @w5 {H} {W} {oxe} {oye} {oxo} {oyo} " + f"{groups}", f"save out {nm}.bin", f"save sync {nm}_sync.bin"] + \ + ([f"time {timed}"] if timed and tag == "m" and groups == 128 else []) + [""] + if tag == "t": + ln, tl = tl, ln + return ln, tl + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--json") + ap.add_argument("--seeds", type=int, default=1) + ap.add_argument("--time", type=int, default=0, help="also time the 1440p case: N runs of each chain") + a = ap.parse_args(argv) + rng = np.random.default_rng(29) + cdir = os.path.join(a.build, "hyb_bneck") + os.makedirs(cdir, exist_ok=True) + lines, tlines, meta, n = [], [], [], 0 + for seed in range(a.seeds): + for gi, (H, W) in enumerate(GRIDS): + for par in (0, 1): + offs = (SW.window_offset("b0", par), SW.window_offset("b1", par)) + special = (gi + par + seed) % 2 == 1 + timed = a.time if (H, W) == GRIDS[-1] and par == 0 and seed == 0 else 0 + tmo = gi >= len(GRIDS) - 2 and seed == 0 + ln, tl = chain_lines(cdir, n, H, W, offs, special, rng, timed, tmo) + lines += ln + tlines += tl + meta.append({"case": n, "H": H, "W": W, "offsets": [list(o) for o in offs], "special": special, + "tmo": tmo}) + n += 1 + cases = os.path.join(cdir, "cases.txt") + with open(cases, "w") as f: + f.write("\n".join(lines)) + env = dict(os.environ, WG_L_FORM="hybrid") + r = subprocess.run([os.path.join(a.build, "kernel_replay.exe"), a.build, cases], env=env, capture_output=True, + text=True, timeout=160, creationflags=0x08000000) + if r.returncode: + raise SystemExit(f"kernel_replay failed: {r.stdout[-1200:]}{r.stderr[-400:]}") + tcases = os.path.join(cdir, "cases_tmo.txt") # the timeout control: its own run (bounded: every wait gives up) + with open(tcases, "w") as f: + f.write("\n".join(tlines)) + rt = subprocess.run([os.path.join(a.build, "kernel_replay.exe"), a.build, tcases], env=env, capture_output=True, + text=True, timeout=160, creationflags=0x08000000) + if rt.returncode: + raise SystemExit(f"kernel_replay (timeout control) failed: {rt.stdout[-1200:]}{rt.stderr[-400:]}") + times = [ln for ln in r.stdout.splitlines() if ": time " in ln] + rd = lambda nm: open(os.path.join(cdir, nm), "rb").read() # noqa: E731 + res = {"cases": [], "groups": list(GROUPS), "times": times} + for m in meta: + k = m["case"] + ref = rd(f"r{k}.bin") + e = dict(m, ref_sentinel_left=int(np.sum(np.frombuffer(ref, np.uint8) == SENT))) + e["identical"] = {g: rd(f"m{k}_{g}.bin") == ref for g in GROUPS} + e["sync_zero"] = {g: not np.any(np.frombuffer(rd(f"m{k}_{g}_sync.bin"), np.uint32)) for g in GROUPS} + sync15 = [int(np.frombuffer(rd(f"m{k}_{g}_sync.bin"), np.uint32)[15]) for g in GROUPS] + e["timeouts"] = sum(sync15) + mu = np.frombuffer(rd(f"u{k}_128.bin"), np.uint8) + e["mutant_differ_bytes"] = int(np.sum(mu != np.frombuffer(ref, np.uint8))) + if m["tmo"]: + s = np.frombuffer(rd(f"t{k}_128_sync.bin"), np.uint32).copy() + ab, cnt = int(s[12]), int(s[15]) + s[12] = s[15] = 0 + e["timeout_control"] = {"abort": ab, "count": cnt, "others_zero": not np.any(s), + "pass": ab > 0 and ab == cnt and not np.any(s)} + res["cases"].append(e) + dl = LD.replay_default_leg(a.build, cases, env, timeout=160) + res["default_leg"] = dl + cs = res["cases"] + res["queue_identical"] = sum(all(c["identical"].values()) for c in cs) + res["sync_all_zero"] = all(all(c["sync_zero"].values()) for c in cs) + res["mutant_caught"] = sum(c["mutant_differ_bytes"] > 0 for c in cs) + tc = [c["timeout_control"] for c in cs if "timeout_control" in c] + res["timeout_control"] = {"cases": len(tc), "pass": sum(t["pass"] for t in tc), + "abort": [t["abort"] for t in tc], "count": [t["count"] for t in tc]} + res["pass"] = (res["queue_identical"] == len(cs) and res["sync_all_zero"] and res["mutant_caught"] > 0 + and dl["pass"] and len(tc) > 0 and res["timeout_control"]["pass"] == len(tc)) + print(json.dumps({k: v for k, v in res.items() if k != "cases"}, indent=1)) + for c in cs: + print(c["case"], c["H"], c["W"], c["offsets"], "special" if c["special"] else "", c["identical"], + "sync0" if all(c["sync_zero"].values()) else "SYNC!", "mutant", c["mutant_differ_bytes"]) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1, default=str) + return 0 if res["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_hyb_bneck_guard_check.py b/clean/l/runner/l_hyb_bneck_guard_check.py new file mode 100644 index 00000000..023eae0f --- /dev/null +++ b/clean/l/runner/l_hyb_bneck_guard_check.py @@ -0,0 +1,169 @@ +# SPDX-License-Identifier: MIT +"""The hybrid bottleneck's wait timeout REFUSES the frame and writes no history (review P1), on native D3D12 with a +developer runtime (build.py --dev-diagnostics: the hook WG_DEV_BNECK_ABORT= writes, right after that frame's +swin_bneck6_hyb launch, the words a timed-out wait writes: the abort word and the timeout count; hyb_bneck.hlsl). + +Per leg (a Model L context, frames 0 .. 4 at 128 x 72 -> 192 x 108, random inputs): + hook: WG_DEV_BNECK_ABORT=2. Five properties (oversight 2026-11-16 04:50Z): + 1. no output write: before frame 2 the display and the output are filled with a sentinel byte; after frame 2 + (synchronised) every byte is still the sentinel; + 2. no history write: the same for both history buffers and both latent buffers; + 3. the guard lifted before the recording ends: after frame 2, with the abort word still set, a fill of a scratch + buffer (a launch of the runtime's own list) takes effect, byte for byte; + 4. frame 3 refused with exactly one log line (the runtime's message: one line, naming the timeout); + 5. frame 4 a clean reset: info reset 1, and it writes the output, its history and its latent state again. + control: the same frames without the hook: frame 2 writes them (the check sees writes when there are any), no frame + is refused, frame 4 is no reset. + release (--release-build): a release runtime with WG_DEV_BNECK_ABORT=2 set behaves as the control (the hook is not + in a release build; check_no_dev_diagnostics.py refuses a library that carries its name). +The hook and control legs run with WG_L_FORM=hybrid and with no form variable set (the shipped default), each context +in its own process (the library's getenv reads the environment copied at load: see l_hyb_refuse_check.py). +The kernel side (a real timed-out wait sets those words) is l_hyb_bneck_check.py's swin_bneck6_hyb_tmo control. + + python clean/l/runner/l_hyb_bneck_guard_check.py --build --models + [--release-build ] [--json out.json] + (GPU: inside a GPU slot) +""" +import argparse +import json +import os +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.runner.l_d3d12_check import FORM_VARS # noqa: E402 + +RW, RH, OW, OH = 128, 72, 192, 108 +ABORT = 2 +WATCH = ("post.history", "state.history_prev", "dec0.latent", "state.latent_prev", "post.display", "filter.out") +SENT = 0x5A +SCRATCH, SCRATCH_SENT = "post.reduced", 0x33 + + +def child(build, models): + from clean.runner.pipeline import Pipeline + p = Pipeline(build, None, models, (RW, RH), (OW, OH), {"hdr": True}, backend="d3d12", + map_path=os.path.join(ROOT, "data", "preset-L.v1.map.json"), check_packing=False, + kernels={"options": "model=L"}) + kern = p.kernels() or {} + rng = np.random.default_rng(5) + out = {"kernels": {k: kern.get(k) for k in ("l_form", "l_hyb_d3d12", "swin_bneck")}, "frames": []} + for f in range(5): + x = {"color": rng.uniform(0, 1, (RH, RW, 3)).astype(np.float32), + "depth": rng.uniform(0, 1, (RH, RW)).astype(np.float32), + "motion": rng.uniform(-1, 1, (RH, RW, 2)).astype(np.float32), "reset": f == 0, + "mv_scale": (1.0, 1.0), "jitter": (0.0, 0.0)} + fills = f in (ABORT, 4) + if fills: + for nm in WATCH: + p.fill(nm, SENT) + e = {"frame": f} + try: + p.frame(x) + e["ok"] = True + except RuntimeError as err: + e["ok"], e["error"] = False, str(err) + e["reset"] = int(p.info().get("reset", -1)) + if fills: + e["untouched"] = {nm: bool(np.all(p.raw(nm) == SENT)) for nm in WATCH} + if f == ABORT: # the abort word is still set here (cleared before frame 4): work after the frame must run + p.fill(SCRATCH, SCRATCH_SENT) + e["after_frame_runs"] = bool(np.all(p.raw(SCRATCH) == SCRATCH_SENT)) + out["frames"].append(e) + return out + + +def run_leg(build, models, form, hook): + cenv = {k: v for k, v in os.environ.items() if k.upper() not in FORM_VARS and k.upper() != "WG_DEV_BNECK_ABORT"} + if form is not None: + cenv["WG_L_FORM"] = form + if hook: + cenv["WG_DEV_BNECK_ABORT"] = str(ABORT) + cmd = [sys.executable, "-I", os.path.abspath(__file__), "--child", "--build", build, "--models", models] + cp = subprocess.run(cmd, env=cenv, capture_output=True, text=True, timeout=150, creationflags=0x08000000) + try: + return json.loads(cp.stdout.strip().splitlines()[-1]) + except (ValueError, IndexError): + return {"child_failed": cp.stdout[-400:] + cp.stderr[-800:]} + + +def written(u): + """A frame wrote its history, its latent state and its output: one of each double-buffered pair (the other is the + previous frame's, read only) and the output.""" + return (not (u["post.history"] and u["state.history_prev"]) and not (u["dec0.latent"] and u["state.latent_prev"]) + and not u["filter.out"]) + + +def judge(r, hook): + if "child_failed" in r: + return False, "child failed" + fr = r["frames"] + if r["kernels"].get("l_hyb_d3d12") != "on" or r["kernels"].get("swin_bneck") != "hyb6": + return False, "not the hybrid route with swin_bneck6_hyb" + if hook: + a, refused, after = fr[ABORT], fr[ABORT + 1], fr[ABORT + 2] + if not all(e["ok"] for e in fr[:ABORT + 1]): + return False, "a frame before the refusal failed" + if not all(a["untouched"].values()): + return False, "the timed-out frame wrote " + ",".join(k for k, v in a["untouched"].items() if not v) + if not a.get("after_frame_runs"): + return False, "work after the timed-out frame was skipped (the guard was not lifted)" + if refused["ok"] or "timed out" not in refused.get("error", "") or "refused" not in refused["error"]: + return False, "the next frame was not refused" + if len([ln for ln in refused["error"].splitlines() if ln.strip()]) != 1: + return False, "the refusal is not exactly one log line" + if not (after["ok"] and after["reset"] == 1 and written(after["untouched"])): + return False, "the frame after the refusal is not a reset that writes" + return True, "" + if not all(e["ok"] for e in fr): + return False, "a frame failed without the hook" + if not fr[ABORT].get("after_frame_runs"): + return False, "the scratch fill after frame 2 did not take effect" + if not written(fr[ABORT]["untouched"]) or fr[4]["reset"] != 0: + return False, "the control frame wrote nothing (or a reset without the hook)" + return True, "" + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--models", required=True) + ap.add_argument("--release-build", help="also: a release runtime with the hook variable set (must be the control)") + ap.add_argument("--json") + ap.add_argument("--child", action="store_true", help=argparse.SUPPRESS) + a = ap.parse_args(argv) + if a.child: + print(json.dumps(child(a.build, a.models))) + return 0 + res = {"legs": []} + for form in ("hybrid", None): + for hook in (True, False): + r = run_leg(a.build, a.models, form, hook) + good, why = judge(r, hook) + res["legs"].append({"form": form or "(default)", "hook": hook, "pass": good, "why": why, "result": r}) + if a.release_build: # the hook variable set, judged as the control + r = run_leg(a.release_build, a.models, None, True) + good, why = judge(r, False) + res["legs"].append({"form": "(default)", "hook": True, "release": True, "pass": good, "why": why, "result": r}) + res["pass"] = all(leg["pass"] for leg in res["legs"]) + for leg in res["legs"]: + r = leg["result"] + print(leg["form"], "release+hook" if leg.get("release") else "hook" if leg["hook"] else "control", + "PASS" if leg["pass"] else "FAIL", leg["why"]) + for e in r.get("frames", []): + print(" ", e["frame"], "ok" if e["ok"] else "REFUSED", "reset", e["reset"], + ("untouched " + ",".join(k for k, v in e["untouched"].items() if v)) if "untouched" in e else "", + (e.get("error") or "")[:160]) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + return 0 if res["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_hyb_default_check.py b/clean/l/runner/l_hyb_default_check.py new file mode 100644 index 00000000..51d90547 --- /dev/null +++ b/clean/l/runner/l_hyb_default_check.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: MIT +"""Model L's shipped default on native D3D12 (0.6.5: WG_L_FORM unset = hybrid_full) is the hybrid route WITH its arena +tables, byte for byte the explicit WG_L_FORM=hybrid (oversight 2026-11-15 standing rule: every release-path check also +runs once with no model / form variable set). Found by code reading: the D3D12 device used to reserve and fill the +hybrid tables (GELU16 -> FP8, rsqrt, rcp; runtime/hyb_tables.h) only for an explicit WG_L_FORM=hybrid, so the default +route would have read whatever buffer the arena put there. + +Cases (each its own Model L context, the same frames): + 1. default: no form / post variable set: create must give l_form=hybrid, l_hyb_d3d12=on, hyb_tables=on, l_post_c12=on; + 2. explicit: WG_L_FORM=hybrid: the same kernels text; every frame's output equal to case 1's, byte for byte; + 3. WG_HYB_TABLES=off with no form variable. A developer build (WG_DEV_DIAGNOSTICS, library marker + "wg-build dev-diagnostics=1") honours the diagnostic: create must REFUSE with the runtime's one line (no other route + runs a frame). A release build ignores it (review finding: a stray variable never refuses Model L for a player): + create must succeed with hyb_tables=on and every frame equal to case 1's. (The refusal itself is also tested on + the CPU recorder, clean/l/runtime/tests/test_l_runtime_frame.py test_hybrid_tables_refused_never_falls_back.) + + python clean/l/runner/l_hyb_default_check.py --build --models [--frames N] [--json out.json] + (GPU: inside a GPU slot) + +Every case runs in its OWN process with its environment set before the runtime library loads: the library links the +C runtime statically, and its getenv reads the environment copied at load, so a variable changed later in the same +process is invisible to it (an in-process second leg would silently repeat the first leg's environment). +""" +import argparse +import hashlib +import json +import os +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.runner.l_d3d12_check import FORM_VARS # noqa: E402 +from clean.runner.pipeline import Pipeline # noqa: E402 + +RW, RH, OW, OH = 256, 144, 384, 216 + + +def frames(n): + rng = np.random.default_rng(7) + out = [] + for k in range(n): + out.append({"color": rng.uniform(0, 4, (RH, RW, 3)).astype(np.float32), + "depth": rng.uniform(0, 1, (RH, RW)).astype(np.float32), + "motion": rng.uniform(-2, 2, (RH, RW, 2)).astype(np.float32), "reset": k == 0, + "mv_scale": (1.0, 1.0), "jitter": (float(rng.uniform(-0.5, 0.5)), float(rng.uniform(-0.5, 0.5)))}) + return out + + +def run(build, models, env, xs): + """(created, error, kernels, [sha256 per frame]) of one Model L context on native D3D12 under env (FORM_VARS set + exactly as given, every other one removed), in a child process (see the module text).""" + cenv = {k: v for k, v in os.environ.items() if k.upper() not in FORM_VARS} + cenv.update(env) + cp = subprocess.run([sys.executable, "-I", os.path.abspath(__file__), "--child", "--build", build, "--models", models, + "--frames", str(len(xs))], env=cenv, capture_output=True, text=True, timeout=150) + try: + r = json.loads(cp.stdout.strip().splitlines()[-1]) + except (ValueError, IndexError): + return False, "child failed: " + cp.stdout[-300:] + cp.stderr[-600:], {}, [] + return r["created"], r["error"], r["kernels"], r["shas"] + + +def child(build, models, xs): + """One context in this process, under the environment it was started with; one JSON line on stdout.""" + try: + p = Pipeline(build, None, models, (RW, RH), (OW, OH), {"hdr": True}, backend="d3d12", + map_path=os.path.join(ROOT, "data", "preset-L.v1.map.json"), check_packing=False, + kernels={"options": "model=L"}) + except RuntimeError as e: + print(json.dumps({"created": False, "error": str(e), "kernels": {}, "shas": []})) + return 0 + kern = p.kernels() or {} + shas = [hashlib.sha256(np.ascontiguousarray(p.frame(dict(x))["out"]).tobytes()).hexdigest() for x in xs] + print(json.dumps({"created": True, "error": "", "kernels": kern, "shas": shas})) + return 0 + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--models", required=True) + ap.add_argument("--frames", type=int, default=4) + ap.add_argument("--json") + ap.add_argument("--child", action="store_true", help=argparse.SUPPRESS) + a = ap.parse_args(argv) + xs = frames(a.frames) + if a.child: + return child(a.build, a.models, xs) + keys = ("l_form", "l_hyb_d3d12", "hyb_tables", "l_post_c12", "hyb_twins") + res = {"cases": []} + ok_d, err_d, kd, sd = run(a.build, a.models, {}, xs) + want = {"l_form": "hybrid", "l_hyb_d3d12": "on", "hyb_tables": "on", "l_post_c12": "on"} + res["cases"].append({"case": "default (no form variable)", "created": ok_d, "error": err_d[-300:], + "kernels": {k: kd.get(k) for k in keys}, "frames": len(sd), + "pass": ok_d and all(kd.get(k) == v for k, v in want.items())}) + ok_e, err_e, ke, se = run(a.build, a.models, {"WG_L_FORM": "hybrid"}, xs) + res["cases"].append({"case": "explicit WG_L_FORM=hybrid", "created": ok_e, "error": err_e[-300:], + "kernels": {k: ke.get(k) for k in keys}, "frames": len(se), + "frames_identical_to_default": sum(x == y for x, y in zip(sd, se)), + "pass": ok_e and all(ke.get(k) == v for k, v in want.items()) and len(se) == len(xs) + and se == sd}) + ok_o, err_o, ko, so = run(a.build, a.models, {"WG_HYB_TABLES": "off"}, xs) + lib = open(os.path.join(a.build, "wronggpu_runtime.dll"), "rb").read() + dev = b"wg-build dev-diagnostics=1" in lib + if dev: + passed = (not ok_o) and "no hybrid tables" in err_o and "Model L hybrid (default)" in err_o and not so + else: + passed = ok_o and ko.get("hyb_tables") == "on" and so == sd and len(so) == len(xs) + res["cases"].append({"case": "WG_HYB_TABLES=off, no form variable (%s build)" % ("developer" if dev else "release"), + "created": ok_o, "error": err_o[-400:], + "error_lines": len([ln for ln in err_o.splitlines() if ln.strip()]), + "hyb_tables": ko.get("hyb_tables"), "frames_identical_to_default": sum(x == y for x, y in zip(so, sd)), + "expect": "refused" if dev else "ignored", "pass": passed}) + res["frame_sha256_default"] = [s[:16] for s in sd] + res["pass"] = all(c["pass"] for c in res["cases"]) + print(json.dumps(res, indent=1)) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + return 0 if res["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_hyb_pack.py b/clean/l/runner/l_hyb_pack.py new file mode 100644 index 00000000..75987e97 --- /dev/null +++ b/clean/l/runner/l_hyb_pack.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: MIT +"""Weights of the HYBRID block kernels (SW_HYB / HYB = true): the Swin block section of the M-form kernels +(clean/runner/packing.py pack_block, the layout their fragments load) filled with the original's values, as the oracle +uses them (clean/l/reference/oracle.py _block_tokens_q7): + gamma1, gamma2 -> g' = RN16(float32(gamma) sqrtf(C)) (gamma_q7; the kernels multiply r g' themselves) + bp, b1, b2, B -> FP16 values (stored as float32 in the header: exact) + Wqk, Wv, Wp, W1, W2 -> FP8 values (q8 of the natural value)""" +import numpy as np + +from clean.l.reference import oracle as O +from clean.reference.fp8 import q8 +from clean.runner import packing as P + + +def hyb_block_values(p): + """The block's parameters in the values the hybrid kernels read (same keys as the M-form packer).""" + out = dict(p) + out["gamma1"] = O.gamma_q7(p["gamma1"]).astype(np.float32) + out["gamma2"] = O.gamma_q7(p["gamma2"]).astype(np.float32) + for n in ("bp", "b1", "b2", "B"): + out[n] = O.f16(p[n]).astype(np.float32) + for n in ("Wqk", "Wv", "Wp", "W1", "W2"): + out[n] = q8(np.asarray(p[n], np.float64)).astype(np.float32) + return out + + +def pack_block_hyb(p, C, heads, mlp, drow="gfx12"): + """The packed block section (bytes) for the hybrid kernels.""" + return P.pack_block(hyb_block_values(p), C, heads, mlp, P.resolve_d_row(drow)) + + +def pack_merge_hyb(Wm, bm, C, C_out, drow="gfx12"): + """The merge section (bytes) for patch_merge_c*_hyb: packing.pack_merge's layout, bm as FP16 values (float32), Wm + plain FP8 values.""" + return P.pack_merge(q8(np.asarray(Wm, np.float64)).astype(np.float32), O.f16(bm).astype(np.float32), C, C_out, + P.resolve_d_row(drow)) + + +def pack_expand_hyb(We, be, C, C_low, drow="gfx12"): + """The expand section (bytes) for expand_c*_hyb: packing.pack_expand's layout, be as FP16 values (float32), We plain + FP8 values.""" + return P.pack_expand(q8(np.asarray(We, np.float64)).astype(np.float32), O.f16(be).astype(np.float32), C, C_low, + P.resolve_d_row(drow)) diff --git a/clean/l/runner/l_hyb_probe_check.py b/clean/l/runner/l_hyb_probe_check.py new file mode 100644 index 00000000..5ecc0e17 --- /dev/null +++ b/clean/l/runner/l_hyb_probe_check.py @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: MIT +"""GPU check of the hybrid block form's element-wise helpers on native D3D12 (kernels/l_hyb_probe.hlsl, +hlsl/wg_hyb.hlsli) against the oracle, for every FP16 input of each single-input helper and an adversarial float32 set +for the chain-end double rounding (float32 -> FP16 -> FP8). Runs with WG_L_FORM=hybrid so the device fills the tables. + + python clean/l/runner/l_hyb_probe_check.py --build [--rocm ] [--prepare] [--run] [--json out.json] + [--mutant gelu|rsq|chainfold] (a deliberately wrong expectation: the check must FAIL; chainfold = one + rounding float32 -> FP8 at the chain end, as a folded conversion would give) + +--prepare builds the probe's DXIL and kernel_replay.exe (host compile); --run runs the probe (one GPU slot).""" +import argparse +import json +import os +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import hyb_tables as T # noqa: E402 +from clean.l.reference import oracle as O # noqa: E402 +from clean.reference.fp8 import fp8_encode, q8 # noqa: E402 + +OFF_GELU, OFF_E, OFF_RSQ, OFF_RCP, OFF_CLAMP, OFF_CHAIN = 0, 65536, 196608, 327680, 458752, 720896 + + +def chain_inputs(seed=1): + """float32 values at and around FP8 midpoints (and the FP16 values next to them), the saturation edge, specials.""" + rng = np.random.default_rng(seed) + codes = np.arange(0, 0x7E, dtype=np.uint8) # positive finite e4m3, 0 .. 448 - 1 step + from clean.reference.fp8 import fp8_decode + v = fp8_decode(codes).astype(np.float64) + mids = (v[:-1] + v[1:]) / 2 # every midpoint between neighbours + out = [] + for m in mids: + m32 = np.float32(m) + out += [m32, np.nextafter(m32, np.float32(0)), np.nextafter(m32, np.float32(np.inf))] + h = np.float16(m) # FP16 values next to the midpoint + for d in (-2, -1, 1, 2): + hb = (int(np.array([h], np.float16).view(np.uint16)[0]) + d) & 0xFFFF + hv = np.array([hb], np.uint16).view(np.float16)[0].astype(np.float32) + out += [hv, np.nextafter(hv, np.float32(0)), np.nextafter(hv, np.float32(np.inf))] + # the float32 midpoint between two FP16 values next to the FP8 midpoint + hb2 = (hb + 1) & 0xFFFF + hv2 = np.array([hb2], np.uint16).view(np.float16)[0].astype(np.float32) + mid16 = np.float32((np.float64(hv) + np.float64(hv2)) / 2) + out += [mid16, np.nextafter(mid16, np.float32(0)), np.nextafter(mid16, np.float32(np.inf))] + out += [np.float32(x) for x in (448.0, 464.0, 480.0, 1e6, 65504.0, 65520.0, 0.0, -0.0, 1e-30, 2.0 ** -10, 2.0 ** -9, + 3 * 2.0 ** -11, np.inf, -np.inf, np.nan)] + out += list(rng.standard_normal(20000).astype(np.float32) * np.float32(4.0) ** rng.integers(-8, 5, 20000)) + x = np.array(out, np.float32) + x = np.concatenate([x, -x]) + x = x[: len(x) - len(x) % 8] + return x + + +def expected(chain): + H = T.H + want = {} + want["gelu"] = T.gelu16_q8() + with np.errstate(invalid="ignore", over="ignore"): + u = np.fmin(np.fmax(T.f16(H * O.Q7_SCORE_SCALE), -O.Q7_SCORE_CLAMP), O.Q7_SCORE_CLAMP) + want["e"] = T.softmax_e_ref(u) + valid_rsq = (H >= 2.0 ** -13) & np.isfinite(H) + with np.errstate(divide="ignore", invalid="ignore"): + want["rsq"] = T._rsq_ref(H).astype(np.float16).view(np.uint16) + rc = T.f16(O._rcp32(H)) + valid_rcp = (H >= 2.0 ** -14) & np.isfinite(H) & (rc >= 2.0 ** -14) + want["rcp"] = rc.astype(np.float16).view(np.uint16) + with np.errstate(invalid="ignore"): + cl = np.where(np.isnan(H), H, np.fmin(np.fmax(H, -6.28125), 6.28125)).astype(np.float32) + want["clamp"] = cl.view(np.uint32) + with np.errstate(invalid="ignore", over="ignore"): + want["chain"] = np.asarray(fp8_encode(q8(chain.astype(np.float16).astype(np.float64))), np.uint8) + return want, valid_rsq, valid_rcp + + +def compare(raw, chain, mutant=None): + want, vr, vc = expected(chain) + if mutant == "gelu": + want["gelu"] = want["gelu"].copy() + want["gelu"][0x3C00] ^= 1 + if mutant == "chainfold": + with np.errstate(invalid="ignore", over="ignore"): + want["chain"] = np.asarray(fp8_encode(q8(chain.astype(np.float64))), np.uint8) + if mutant == "rsq": + want["rsq"] = want["rsq"].copy() + want["rsq"][0x3C00] ^= 1 + got = { + "gelu": np.frombuffer(raw, np.uint8, 65536, OFF_GELU), + "e": np.frombuffer(raw, np.uint16, 65536, OFF_E), + "rsq": np.frombuffer(raw, np.uint16, 65536, OFF_RSQ), + "rcp": np.frombuffer(raw, np.uint16, 65536, OFF_RCP), + "clamp": np.frombuffer(raw, np.uint32, 65536, OFF_CLAMP), + "chain": np.frombuffer(raw, np.uint8, len(chain), OFF_CHAIN), + } + res = {} + for k in got: + g, w = got[k], want[k] + mask = vr if k == "rsq" else (vc if k == "rcp" else np.ones(len(w), bool)) + if k == "clamp": # NaN: any NaN + gn = (g & 0x7F800000) == 0x7F800000 + same = (g == w) | (gn & ((g & 0x7FFFFF) != 0) & ((w & 0x7F800000) == 0x7F800000) & ((w & 0x7FFFFF) != 0)) + elif k in ("chain", "gelu"): # FP8 NaN: either code (the hardware conversion writes 0xFF for every NaN) + same = (g == w) | (((g & 0x7F) == 0x7F) & ((w & 0x7F) == 0x7F)) + else: + same = g == w + bad = np.nonzero(mask & ~same)[0] + res[k] = {"compared": int(mask.sum()), "differ": int(bad.size), "first": [int(i) for i in bad[:8]]} + res["pass"] = all(v["differ"] == 0 for v in res.values() if isinstance(v, dict)) + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", default=os.path.join(r"C:\Program Files\AMD\ROCm", "7.2"), help="the official HIP SDK") + ap.add_argument("--prepare", action="store_true") + ap.add_argument("--run", action="store_true") + ap.add_argument("--mutant", choices=("gelu", "rsq", "chainfold")) + ap.add_argument("--hip", action="store_true", help="the HIP twin (clean/l/kernels/l_hyb_probe.hip) instead") + ap.add_argument("--json") + a = ap.parse_args(argv) + os.makedirs(a.build, exist_ok=True) + if a.prepare and a.hip: + from clean.runner import build as B + B.run(B.device_command(B.find_clang(a.rocm), a.rocm, os.path.join(ROOT, "clean", "l", "kernels", "l_hyb_probe.hip"), + os.path.join(a.build, "l_hyb_probe.co"), + extra=["-I", os.path.join(ROOT, "clean", "kernels")])) + elif a.prepare: + from clean.l.runner.l_d3d12_check import prepare_replay + prepare_replay(a.build, a.rocm, ["l_hyb_probe"]) + chain = chain_inputs() + cdir = os.path.join(a.build, "hyb_probe_hip" if a.hip else "hyb_probe") + os.makedirs(cdir, exist_ok=True) + src = os.path.join(cdir, "chain.bin") + out = os.path.join(cdir, "out.bin") + if a.run and a.hip: + import ctypes + from clean.runner.hip import Hip + h = Hip(a.rocm) + fn = h.get_function(h.load_module(open(os.path.join(a.build, "l_hyb_probe.co"), "rb").read()), "l_hyb_probe") + nbytes = OFF_CHAIN + len(chain) + ps, pd = h.malloc(chain.nbytes), h.malloc(nbytes) + h.upload(ps, chain.tobytes()) + h.memset(pd, 165, nbytes) + h.launch(fn, 32, 256, [ctypes.c_void_p(ps), ctypes.c_void_p(pd), ctypes.c_uint(len(chain))]) + h.sync() + with open(out, "wb") as f: + f.write(h.download(pd, nbytes)) + elif a.run: + chain.tofile(src) + nbytes = OFF_CHAIN + len(chain) + with open(os.path.join(cdir, "cases.txt"), "w") as f: + f.write("case hyb_probe\nkernel l_hyb_probe 32 256\n" + f"arg buf chain.bin\narg out {nbytes} out.bin 165\narg u32 {len(chain)}\n") + env = dict(os.environ) + env["WG_L_FORM"] = "hybrid" + r = subprocess.run([os.path.join(a.build, "kernel_replay.exe"), a.build, os.path.join(cdir, "cases.txt")], + env=env, capture_output=True, text=True, timeout=150, creationflags=0x08000000) + if r.returncode: + raise SystemExit(f"kernel_replay failed ({r.returncode}):\n{r.stdout[-1500:]}{r.stderr[-800:]}") + raw = open(out, "rb").read() + res = compare(raw, chain, a.mutant) + res["chain_inputs"] = int(len(chain)) + res["mutant"] = a.mutant + print(json.dumps(res, indent=1)) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + return 0 if (res["pass"] != bool(a.mutant)) else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_hyb_refuse_check.py b/clean/l/runner/l_hyb_refuse_check.py new file mode 100644 index 00000000..b2ae78a7 --- /dev/null +++ b/clean/l/runner/l_hyb_refuse_check.py @@ -0,0 +1,131 @@ +# SPDX-License-Identifier: MIT +"""The hybrid route never falls back (oversight 2026-11-15 18:36Z, hard rule; executor level: #225): when native D3D12 +cannot take WG_L_FORM=hybrid, the runtime REFUSES at create with one line naming what is missing, and no other route +(the q7 / scalar reference, the M-form, the HIP executor) runs a frame in its place. + +Cases, each on a copy of a build folder whose D3D12 manifest lacks something the hybrid route needs: + 1. one hybrid twin (swin_block_c128_hyb) removed: create must fail with the route's manifest message; + 2. the C32 decoder twin removed: create must fail likewise; + 3. control: the unmodified build creates on the hybrid route (kernels text l_form=hybrid, + l_hyb_d3d12=on), so the refusals above are the missing kernels and nothing else. +Every case runs twice (standing rule, oversight 2026-11-15): with WG_L_FORM=hybrid and with NO form variable set (the +shipped default); both legs must give the same outcome. + +Every case runs in its OWN process with its environment set before the runtime library loads: the library links the +C runtime statically, and its getenv reads the environment copied at load, so a variable changed later in the same +process is invisible to it (an in-process second leg would silently repeat the first leg's environment). + + python clean/l/runner/l_hyb_refuse_check.py --build --models --work + [--json out.json] (GPU: inside a GPU slot) +""" +import argparse +import json +import os +import shutil +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.runner.l_d3d12_check import FORM_VARS # noqa: E402 +from clean.runner.pipeline import Pipeline # noqa: E402 + +RW, RH, OW, OH = 128, 72, 192, 108 + + +def strip_manifest(src, dst, drop): + """A copy of build folder src at dst (files linked or copied) whose dx12/manifest.json lacks the kernels `drop`.""" + if os.path.exists(dst): # no recursive delete here: the caller clears the work folder (guarded_delete) + raise SystemExit(f"work folder {dst} exists: remove it first") + shutil.copytree(src, dst, ignore=shutil.ignore_patterns("lframe_model", "hyb_*", "*-chk")) + mp = os.path.join(dst, "dx12", "manifest.json") + m = json.load(open(mp)) + ks = m["kernels"] if isinstance(m, dict) and "kernels" in m else m + keep = [k for k in ks if k.get("name") not in drop] + assert len(keep) == len(ks) - len(drop), "kernels to drop not found in the manifest" + if isinstance(m, dict) and "kernels" in m: + m["kernels"] = keep + else: + m = keep + json.dump(m, open(mp, "w"), indent=1) + return dst + + +def try_create(build, models, frame=False, form="hybrid"): + """(created, error text, kernels dict) of a Model L context on native D3D12 with WG_L_FORM=form (None: no form + variable set, the shipped default), in a child process (see the module text).""" + cenv = {k: v for k, v in os.environ.items() if k.upper() not in FORM_VARS} + if form is not None: + cenv["WG_L_FORM"] = form + cmd = [sys.executable, "-I", os.path.abspath(__file__), "--child", "--build", build, "--models", models, "--work", "-"] + cp = subprocess.run(cmd + (["--frame"] if frame else []), env=cenv, capture_output=True, text=True, timeout=150) + try: + r = json.loads(cp.stdout.strip().splitlines()[-1]) + except (ValueError, IndexError): + return False, "child failed: " + cp.stdout[-300:] + cp.stderr[-600:], {} + return r["created"], r["error"], r["kernels"] + + +def child_create(build, models, frame=False): + """One context in this process, under its start environment: (created, error text, kernels dict).""" + try: + p = Pipeline(build, None, models, (RW, RH), (OW, OH), {"hdr": True}, backend="d3d12", + map_path=os.path.join(ROOT, "data", "preset-L.v1.map.json"), check_packing=False, + kernels={"options": "model=L"}) + except RuntimeError as e: + return False, str(e), {} + kern = p.kernels() or {} + if frame: + rng = np.random.default_rng(1) + x = {"color": rng.uniform(0, 1, (RH, RW, 3)).astype(np.float32), + "depth": rng.uniform(0, 1, (RH, RW)).astype(np.float32), + "motion": np.zeros((RH, RW, 2), np.float32), "reset": True} + p.frame(x) + return True, "", kern + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--models", required=True) + ap.add_argument("--work", required=True) + ap.add_argument("--json") + ap.add_argument("--child", action="store_true", help=argparse.SUPPRESS) + ap.add_argument("--frame", action="store_true", help=argparse.SUPPRESS) + a = ap.parse_args(argv) + if a.child: + ok, err, kern = child_create(a.build, a.models, a.frame) + print(json.dumps({"created": ok, "error": err, "kernels": kern})) + return 0 + res = {"cases": []} + for leg, form in (("explicit", "hybrid"), ("default", None)): + for name, drop in (("no_c128_twin", ["swin_block_c128_hyb"]), ("no_c32_dec_twin", ["l_swin_block_c32_dec_hyb"])): + b = strip_manifest(a.build, os.path.join(a.work, leg, name), drop) # one folder per leg: the first + # leg's runtime DLL stays loaded in this process + ok, err, _ = try_create(b, a.models, form=form) + lines = [ln for ln in err.splitlines() if ln.strip()] + # the refusal names the form as resolved (nothing set: the release default) + label = "WG_L_FORM=hybrid" if form else "Model L hybrid (default)" + refused = (not ok) and (label + " on native D3D12") in err and any(d in err for d in drop) + res["cases"].append({"case": name, "leg": leg, "created": ok, "error": err[-400:], "error_lines": len(lines), + "pass": refused}) + ok, err, kern = try_create(a.build, a.models, form=form) + res["cases"].append({"case": "control", "leg": leg, "created": ok, "error": err[-400:], + "kernels": {k: kern.get(k) for k in ("l_form", "l_hyb_d3d12", "hyb_twins", "l_dec_skip", + "l_post_c12", "hyb_tables")}, + "pass": ok and kern.get("l_form") == "hybrid" and kern.get("l_hyb_d3d12") == "on" + and kern.get("hyb_tables") == "on"}) + res["pass"] = all(c["pass"] for c in res["cases"]) + print(json.dumps(res, indent=1)) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + return 0 if res["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_hyb_shared_check.py b/clean/l/runner/l_hyb_shared_check.py new file mode 100644 index 00000000..3d577f06 --- /dev/null +++ b/clean/l/runner/l_hyb_shared_check.py @@ -0,0 +1,223 @@ +# SPDX-License-Identifier: MIT +"""Kernel-level check of the HYBRID form's shared-stage kernels (S3: clean/dx12/kernels/hyb_swin.hlsl, hyb_gemm.hlsl; +HIP twins clean/kernels/swin_hyb.hip -> swin_hyb.co): random parameters in the hybrid values (l_hyb_pack.py), FP8 +input maps with special codes in every other case, both window offsets of each table kind. + + 1. HIP (swin_hyb.co) == D3D12 raw bytes for every case; each output starts with its own sentinel; a half-grid D3D12 + run of the first case of every kernel is the negative control (must differ). + 2. Informational: the output's FP8 codes against the oracle's hybrid form (form q7, mm8 f32chain): blocks by + oracle.swin_block (encoder / bottleneck border "mirror"; decoder E by "coarse", the skip map by the fine mirror), + merges FP8(clamp(mm8)), expands FP8(mm8): counts of differing codes (both NaN = same). The per-stage check + (SW_HYB_DUMP, the written tolerance) gives the pass / fail; a gross count here means a wrong index or mirror. + + python clean/l/runner/l_hyb_shared_check.py --build [--prepare] [--run] [--seeds N] [--kernels a,b] + [--json out.json] [--no-oracle] +""" +import argparse +import ctypes +import json +import os +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import oracle as O # noqa: E402 +from clean.l.reference import table as T # noqa: E402 +from clean.l.runner import l_d3d12_check as LD # noqa: E402 +from clean.l.runner.l_hyb_pack import pack_block_hyb, pack_expand_hyb, pack_merge_hyb # noqa: E402 +from clean.reference.fp8 import fp8_decode, fp8_encode, q8 # noqa: E402 + +# name: (kind, C, heads, mlp, extra) - kind block / add / merge / expand; extra = C_out (merge) or C_low (expand) +KERNELS = { + "swin_block_c64_hyb": ("block", 64, 2, 256, None), + "swin_block_c64_add_hyb": ("add", 64, 2, 256, None), + "swin_block_c96_hyb": ("block", 96, 4, 384, None), + "swin_block_c96_add_hyb": ("add", 96, 4, 384, None), + "swin_block_c128_hyb": ("block", 128, 4, 512, None), + "patch_merge_c64_hyb": ("merge", 64, 0, 0, 96), + "patch_merge_c96_hyb": ("merge", 96, 0, 0, 128), + "expand_c96_hyb": ("expand", 64, 0, 0, 96), + "expand_c128_hyb": ("expand", 96, 0, 0, 128), +} +GRIDS = [(48, 40), (18, 10), (8, 8), (4, 6)] # even sides (merges, decoder blocks); partial windows; 48 x 40: 2+ groups + + +def _fp8_map(rng, shape, special, scale): + y = fp8_encode(q8(rng.normal(0, scale, shape))).astype(np.uint8) + if special: + pick = rng.random(y.shape) < 0.05 + y[pick] = rng.choice(LD.SPECIAL_FP8, int(pick.sum())) + return y + + +def cases(rng, seeds, names): + """(name, launch args (list of ("buf", bytes) / ("null",) / ("out", n) / ("u32", v)), grid, oracle inputs)""" + for seed in range(seeds): + for name in names: + kind, C, heads, mlp, extra = KERNELS[name] + for gi, (H, W) in enumerate(GRIDS): + for parity in (0, 1): + special = (gi + parity + seed) % 2 == 1 + scale = 40.0 if special else 1.5 + if kind in ("block", "add"): + tkind = "second" if C == 128 else "first" + off = T.block_offset(T.TABLE, tkind, parity) + p = O.random_block_params(rng, C, heads, mlp) + blob = pack_block_hyb(p, C, heads, mlp) + x = _fp8_map(rng, (H, W, C), special, scale) + s = _fp8_map(rng, (H, W, C), special, scale) if kind == "add" else None + ox, oy = off + nwin = -(-(W + ox) // 4) * -(-(H + oy) // 4) + args = [("buf", x.tobytes()), ("buf", s.tobytes()) if s is not None else ("null",), + ("out", H * W * C), ("buf", blob), ("u32", H), ("u32", W), ("u32", ox), ("u32", oy)] + yield name, args, -(-nwin // 8), {"kind": kind, "p": p, "x": x, "s": s, "off": off, "C": C, + "heads": heads, "H": H, "W": W} + elif kind == "merge": + Co = extra + Wm = rng.normal(0, 1.0 / np.sqrt(4 * C), (Co, 4 * C)) + bm = rng.normal(0, 0.1, Co) + x = _fp8_map(rng, (H, W, C), special, scale) + args = [("buf", x.tobytes()), ("out", (H // 2) * (W // 2) * Co), + ("buf", pack_merge_hyb(Wm, bm, C, Co)), ("u32", H), ("u32", W)] + yield name, args, -(-((H // 2) * (W // 2)) // 256), {"kind": kind, "x": x, "Wm": Wm, "bm": bm} + else: + Cl = extra + Hl, Wl = H // 2, W // 2 + We = rng.normal(0, 1.0 / np.sqrt(Cl), (4 * C, Cl)) + be = rng.normal(0, 0.1, 4 * C) + y = _fp8_map(rng, (Hl, Wl, Cl), special, scale) + args = [("buf", y.tobytes()), ("out", H * W * C), ("buf", pack_expand_hyb(We, be, C, Cl)), + ("u32", Hl), ("u32", Wl)] + yield name, args, -(-(Hl * Wl) // 256), {"kind": kind, "x": y, "We": We, "be": be} + + +def oracle_codes(o): + """The oracle's FP8 codes of a case (the hybrid form: q7, f32chain).""" + k = dict(T.block_consts(T.TABLE, "enc1")) + k["mm8"] = "f32chain" + if o["kind"] in ("block", "add"): + x = fp8_decode(o["x"]).astype(np.float64) + if o["kind"] == "add": + y = O.swin_block(x, o["p"], o["off"], o["heads"], k, T.TABLE["shared_dec_border"], + skip=fp8_decode(o["s"]).astype(np.float64)) + else: + y = O.swin_block(x, o["p"], o["off"], o["heads"], k, "mirror") + return fp8_encode(q8(y)).astype(np.uint8) + x = fp8_decode(o["x"]).astype(np.float64) + if o["kind"] == "merge": + m = O.mm8(O.f16(o["bm"]), O.space_to_depth(x), q8(o["Wm"]), "f32chain") + return fp8_encode(q8(np.minimum(np.maximum(m, -T.MERGE_CLAMP), T.MERGE_CLAMP))).astype(np.uint8) + e = q8(O.mm8(O.f16(o["be"]), x, q8(o["We"]), "f32chain")) + return fp8_encode(O.depth_to_space(e)).astype(np.uint8) + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", default=os.path.join(r"C:\Program Files\AMD\ROCm", "7.2")) + ap.add_argument("--prepare", action="store_true") + ap.add_argument("--run", action="store_true") + ap.add_argument("--seeds", type=int, default=1) + ap.add_argument("--kernels", default=",".join(KERNELS)) + ap.add_argument("--no-oracle", action="store_true") + ap.add_argument("--json") + a = ap.parse_args(argv) + names = [n for n in a.kernels.split(",") if n] + for n in names: + if n not in KERNELS: + raise SystemExit(f"unknown kernel {n}") + if a.prepare: + from clean.runner import build as B + LD.prepare_replay(a.build, a.rocm, list(KERNELS)) + B.run(B.device_command(B.find_clang(a.rocm), a.rocm, os.path.join(ROOT, "clean", "kernels", "swin_hyb.hip"), + os.path.join(a.build, "swin_hyb.co"), extra=["-I", os.path.join(ROOT, "clean", "kernels")])) + if not a.run: + return 0 + from clean.runner.hip import Hip + h = Hip(a.rocm) + mod = h.load_module(open(os.path.join(a.build, "swin_hyb.co"), "rb").read()) + rng = np.random.default_rng(11) + cdir = os.path.join(a.build, "hyb_shared") + os.makedirs(cdir, exist_ok=True) + lines, hip_out, meta, negs = [], {}, [], {} + for n, (name, args, grid, o) in enumerate(cases(rng, a.seeds, names)): + ptrs, cargs, out_ptr, nout, rl = [], [], None, 0, [f"case c{n}", f"kernel {name} {grid} 256"] + for j, arg in enumerate(args): + if arg[0] == "buf": + p = h.malloc(len(arg[1])) + h.upload(p, arg[1]) + ptrs.append(p) + cargs.append(ctypes.c_void_p(p)) + fn = f"a{n}_{j}.bin" + with open(os.path.join(cdir, fn), "wb") as f: + f.write(arg[1]) + rl.append(f"arg buf {fn}") + elif arg[0] == "null": + cargs.append(ctypes.c_void_p(0)) + rl.append("arg null") + elif arg[0] == "out": + nout = arg[1] + out_ptr = h.malloc(nout) + h.memset(out_ptr, LD.HIP_SENTINEL, nout) + ptrs.append(out_ptr) + cargs.append(ctypes.c_void_p(out_ptr)) + rl.append(f"arg out {nout} o{n}.bin {LD.D3D12_SENTINEL}") + else: + cargs.append(ctypes.c_int(arg[1])) + rl.append(f"arg u32 {arg[1]}") + h.launch(h.get_function(mod, name), grid, 256, cargs) + h.sync() + hip_out[n] = bytes(h.download(out_ptr, nout)) + for p in ptrs: + h.free(p) + lines += rl + [""] + if name not in negs and grid > 1: # negative control: half the workgroups + negs[name] = n + lines += [f"case neg{n}", f"kernel {name} {grid // 2} 256"] + \ + [x.replace(f"o{n}.bin", f"neg{n}.bin") for x in rl[2:]] + [""] + meta.append((n, name, o)) + with open(os.path.join(cdir, "cases.txt"), "w") as f: + f.write("\n".join(lines)) + r = subprocess.run([os.path.join(a.build, "kernel_replay.exe"), a.build, os.path.join(cdir, "cases.txt")], + env=dict(os.environ, WG_L_FORM="hybrid"), capture_output=True, text=True, timeout=150, + creationflags=0x08000000) + if r.returncode: + raise SystemExit(f"kernel_replay failed: {r.stdout[-800:]}{r.stderr[-400:]}") + res = {"cases": [], "per_kernel": {}} + # the default leg (standing rule): no form variable set == WG_L_FORM=hybrid, every output byte + res["default_leg"] = LD.replay_default_leg(a.build, os.path.join(cdir, "cases.txt"), dict(os.environ, WG_L_FORM="hybrid")) + for n, name, o in meta: + d = open(os.path.join(cdir, f"o{n}.bin"), "rb").read() + eq = d == hip_out[n] + c = {"case": n, "kernel": name, "identical": eq, + "differing_bytes": int(np.count_nonzero(np.frombuffer(d, np.uint8) != np.frombuffer(hip_out[n], np.uint8)))} + if not a.no_oracle: + want = oracle_codes(o).ravel() + got = np.frombuffer(d, np.uint8) + bad = (got != want) & ~(((got & 0x7F) == 0x7F) & ((want & 0x7F) == 0x7F)) + c["oracle_fp8_differ"], c["oracle_fp8_total"] = int(bad.sum()), int(want.size) + res["cases"].append(c) + k = res["per_kernel"].setdefault(name, {"identical": 0, "cases": 0, "oracle_differ": 0, "oracle_total": 0}) + k["identical"] += eq + k["cases"] += 1 + k["oracle_differ"] += c.get("oracle_fp8_differ", 0) + k["oracle_total"] += c.get("oracle_fp8_total", 0) + for name, n in negs.items(): + res["per_kernel"][name]["negative_control_caught"] = open(os.path.join(cdir, f"neg{n}.bin"), "rb").read() != hip_out[n] + res["pass_bytes"] = all(k["identical"] == k["cases"] and k.get("negative_control_caught", False) + for k in res["per_kernel"].values()) and res["default_leg"]["pass"] + print(json.dumps(res["per_kernel"], indent=1)) + print("pass_bytes", res["pass_bytes"]) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + return 0 if res["pass_bytes"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_hyb_stage_check.py b/clean/l/runner/l_hyb_stage_check.py new file mode 100644 index 00000000..8a00ca70 --- /dev/null +++ b/clean/l/runner/l_hyb_stage_check.py @@ -0,0 +1,559 @@ +# SPDX-License-Identifier: MIT +"""Per-stage check of the HYBRID C32 kernels (the hybrid check design's per-stage leg): the dump variants +l_swin_block_c32_enc_hyb_dump / _dec_hyb_dump (SW_HYB_DUMP) write every stage's values; each stage is recomputed by +the oracle's FP16 form from the kernel's OWN inputs to that stage and compared: + + element-wise (norm_q7, softmax_q7, the gather and its mirrors, the output point): identical bits (NaN = NaN); + chains (q/k, v, scores, P V, Wp, W1, W2): identical to RN16 of the f32chain oracle, except where the exact value E + lies within delta = bound(hardware) + bound(oracle) of an FP16 rounding boundary; the kernel's value must always lie + in [RN16(E - bound_hw), RN16(E + bound_hw)] (the measured probe-1 rule, never widened): per 8-product step of the + kernel's K order, 9 g + ulp32(step result) / 2 with g = 2^(e - 24), e the exponent of the step's largest term (the + incoming exact partial included); the oracle's bound is ulp32 / 2 of each of its sequential partials. + Pass (oversight 2026-11-15 16:00Z): no bound exceedance, every difference inside the allowance, actual difference + rate <= 1e-4 per stage; the allowance share (<= 1e-3) counts only on the real-weights run (--real). + NaN (oversight decision on the NaN finding, permanent until the 0.6.6 fix): the random legs draw finite codes only + (FINITE_FP8 / FINITE_F16: no FP8 NaN code, no FP16 NaN); ONE dedicated NaN-input leg (one extra seed, every case with + the full special sets, NaN codes included) is reported separately in the record's "nan_input": listed, neither pass + nor fail (the original keeps a NaN through enc0's ReLU + FP8 step where ours gives 0; spec 03 maps NaN inputs to 0). + +Layouts (gfx12, packing.py with D rows v + 8 h): lane (t, hh), element v: + X, Y1, Z, Y2, OUT: token t, channel hh C/2 + 8 k + v (tile k) UQ: token t, dim 16 mt + 8 hh + v + V: value dim 16 mt + t, token 8 hh + v S, P: query t, key 8 hh + v + O: token t, dim 16 dt + 8 hh + v H: token t, hidden 16 j + 8 hh + v + + python clean/l/runner/l_hyb_stage_check.py --build [--prepare] [--run] [--seeds N] [--json out.json] +""" +import argparse +import json +import os +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import oracle as O # noqa: E402 +from clean.l.reference import table as T # noqa: E402 +from clean.l.runner import l_d3d12_check as LD # noqa: E402 +from clean.l.runner.l_hyb_pack import pack_block_hyb # noqa: E402 +from clean.l.runner import l_pack as LP # noqa: E402 +from clean.l.runner.l_enclin_check import q20 # noqa: E402 +from clean.l.runner.l_ops_check import unpack12 # noqa: E402 +from clean.l.runner.l_c32_check import f16_steps # noqa: E402 +from clean.reference.fp8 import fp8_decode, fp8_encode, q8 # noqa: E402 + + +# the random legs' special codes: finite only (the NaN codes 0x7f / 0xff and the FP16 NaN go to the NaN leg alone) +FINITE_FP8 = np.array([c for c in LD.SPECIAL_FP8 if c & 0x7F != 0x7F], np.uint8) +FINITE_F16 = LD.SPECIAL_F16[~np.isnan(LD.SPECIAL_F16)] + + +def configure(c, heads, mlp): + """The block shape and the dump layout (swin_block.hlsl HD_*) of the kernel being checked.""" + global C, HEADS, MLP, KC, MTH, HD_HEAD, HD_Z, HD_Y2, HD_H, HD_OUT, HD_WORDS + C, HEADS, MLP = c, heads, mlp + KC, MTH = C // 16, MLP // 16 + HD_HEAD = 16 * KC + HD_Z = HD_HEAD + 60 * HEADS + HD_Y2 = HD_Z + 8 * KC + HD_H = HD_Y2 + 8 * KC + HD_OUT = HD_H + 8 * MTH + HD_WORDS = HD_OUT + 8 * KC + + +configure(32, 2, 128) +# name: (kind, C, heads, mlp). Kinds: enc = FP16 halo map in, FP8 out (Model L's enc0); dec = RN16(E + S) in (E coarse +# mirror, S fine), FP16 out (dec0); blk = FP8 map in (fine mirror), FP8 out (enc1, enc2, b0..b5); add = RN16(E + S) in, +# FP8 out (dec2, dec1). +KERNELS = {"l_swin_block_c32_enc_hyb_dump": ("enc", 32, 2, 128), "l_swin_block_c32_dec_hyb_dump": ("dec", 32, 2, 128)} +S3_KERNELS = {"swin_block_c64_hyb_dump": ("blk", 64, 2, 256), "swin_block_c64_add_hyb_dump": ("add", 64, 2, 256), + "swin_block_c96_hyb_dump": ("blk", 96, 4, 384), "swin_block_c96_add_hyb_dump": ("add", 96, 4, 384), + "swin_block_c128_hyb_dump": ("blk", 128, 4, 512)} +# --set fused (speed lever 3): the hybrid twins of Model L's fusions. enclin = the encoder block with the two input +# linears as its prologue (q map in its 20-byte form + enc0.Win1 / Win2 in; its X stage = x2 = the linears' output, +# checked as the chain lin2 from the oracle's x1); deccoef = the decoder block with l_coef_pack12 as its epilogue (no +# FP16 map out: the pack12 images A / C / latent, checked against oracle.coefficients_l of the block's own output +# stage, l_ops_check's rule: codes within 1, C within 4 FP16 steps) +FUSED_KERNELS = {"l_swin_block_c32_enc_lin_hyb_dump": ("enclin", 32, 2, 128), + "l_swin_block_c32_dec_coef12_hyb_dump": ("deccoef", 32, 2, 128)} +FUSED_MUT = {"l_swin_block_c32_enc_lin_hyb_mutdump": ("enclin", 32, 2, 128), + "l_swin_block_c32_dec_coef12_hyb_mutdump": ("deccoef", 32, 2, 128)} +SHARE_MAX = 1e-3 # the allowance share: judged on the real-weights recorded-frame run only (--real) +DIFF_MAX = 1e-4 # actual one-step differences per stage (oversight 2026-11-15 16:00Z: hard rule) + + +def f16(x): + return O.f16(np.asarray(x, np.float64)) + + +def same(a, b): + """Elementwise identical FP16 values (NaN equals NaN; +0 and -0 differ).""" + a, b = np.asarray(a, np.float64), np.asarray(b, np.float64) + na, nb = np.isnan(a), np.isnan(b) + eq = (a == b) & (np.signbit(a) == np.signbit(b)) + return np.where(na | nb, na & nb, eq) + + +# ---- the dump in token / channel order --------------------------------------------------------------------------- + +def lane_words(dump, win): + """[32 lanes, HD_WORDS] float32 values of one window.""" + w = np.frombuffer(dump, np.uint32).reshape(-1, 32, HD_WORDS)[win] + return w.view(np.float32).astype(np.float64), w + + +def layout_l(f, base): + """[16 tokens, C] from a layout-L field at word `base` (KC tiles of 8).""" + out = np.empty((16, C)) + for hh in range(2): + for t in range(16): + for k in range(KC): + out[t, hh * (C // 2) + 8 * k:hh * (C // 2) + 8 * k + 8] = f[16 * hh + t, base + 8 * k:base + 8 * k + 8] + return out + + +def stages(dump, win): + f, u = lane_words(dump, win) + d = {"X": layout_l(f, 0), "Y1": layout_l(f, 8 * KC), "Z": layout_l(f, HD_Z), "Y2": layout_l(f, HD_Y2), + "OUT": layout_l(f, HD_OUT), "UQ": [], "V": [], "S": [], "P": [], "O": []} + for hd in range(HEADS): + b = HD_HEAD + 60 * hd + uq, v, s, p, o = np.empty((16, 32)), np.empty((16, 32)), np.empty((16, 16)), np.empty((16, 16)), np.empty((16, 32)) + for hh in range(2): + for t in range(16): + ln = 16 * hh + t + for mt in range(2): + uq[t, 16 * mt + 8 * hh:16 * mt + 8 * hh + 8] = f[ln, b + 8 * mt:b + 8 * mt + 8] + v[8 * hh:8 * hh + 8, 16 * mt + t] = f[ln, b + 16 + 8 * mt:b + 24 + 8 * mt] # [token, dim] + o[t, 16 * mt + 8 * hh:16 * mt + 8 * hh + 8] = f[ln, b + 44 + 8 * mt:b + 52 + 8 * mt] + s[t, 8 * hh:8 * hh + 8] = f[ln, b + 32:b + 40] + pw = u[ln, b + 40:b + 44] + h16 = np.stack([pw & 0xFFFF, pw >> 16], axis=-1).reshape(8).astype(np.uint16) + p[t, 8 * hh:8 * hh + 8] = h16.view(np.float16).astype(np.float64) + for n, a in (("UQ", uq), ("V", v), ("S", s), ("P", p), ("O", o)): + d[n].append(a) + hmap = np.empty((16, MLP)) + for hh in range(2): + for t in range(16): + for j in range(MTH): + hmap[t, 16 * j + 8 * hh:16 * j + 8 * hh + 8] = f[16 * hh + t, HD_H + 8 * j:HD_H + 8 * j + 8] + d["H"] = hmap + return d + + +# ---- chain comparison -------------------------------------------------------------------------------------------- + +class Tally: + def __init__(self): + self.s = {} + + def add(self, stage, **kw): + t = self.s.setdefault(stage, {"outputs": 0, "near": 0, "differ": 0, "differ_outside": 0, "violations": 0, + "max_steps": 0}) + for k, v in kw.items(): + t[k] = max(t[k], v) if k == "max_steps" else t[k] + v + + +def order_l(K): + """The kernel's K order of a layout-L operand (q/k, v, W1 chains): instruction kc, step 1 = channels 8 kc + j, step 2 + = channels K/2 + 8 kc + j (K slot (h, j) of group kc holds channel h K/2 + 8 kc + j).""" + return np.array([h * (K // 2) + 8 * kc + j for kc in range(K // 16) for h in range(2) for j in range(8)]) + + +def _half_ulp32(x): + """ulp32(|x|) / 2 for float64 values (0 for 0).""" + m, e = np.frexp(np.abs(x)) + return np.where(m == 0, 0.0, np.ldexp(1.0, e - 25)) + + +def _hw_bound(s0, pk): + """The probe-1 bound of one chain: start s0 [...], products pk [..., K] in the kernel's order.""" + K = pk.shape[-1] + steps = pk.reshape(pk.shape[:-1] + (K // 8, 8)) + acc_in = s0[..., None] + np.concatenate([np.zeros(steps.shape[:-2] + (1,)), + np.cumsum(steps.sum(axis=-1), axis=-1)[..., :-1]], axis=-1) + big = np.maximum(np.abs(acc_in), np.abs(steps).max(axis=-1)) + return (9.0 * _half_ulp32(big) + _half_ulp32(acc_in + steps.sum(axis=-1))).sum(axis=-1) + + +def chain(tally, stage, start, a, Wm, got, order=None, oracle_exact=False, split=None): + """start [..., n] (or [n]), a [..., K], Wm [n, K] (values; products exact in float64), got [..., n] (the kernel's + RN16 chain end); order: the kernel's K order (default ascending). Records the outcome under `stage`. + Hardware bound (probe 1): per 8-product step, 9 g (g = half ulp32 of the step's largest term, the incoming exact + partial included) + half ulp32 of the step result. Oracle bound: half ulp32 of every sequential partial (K order).""" + a = np.asarray(a, np.float64) + Wm = np.asarray(Wm, np.float64) + K = a.shape[-1] + st = np.broadcast_to(np.asarray(start, np.float64), a.shape[:-1] + (Wm.shape[0],)) + with np.errstate(over="ignore", invalid="ignore"): + prods = a[..., None, :] * Wm # [..., n, K] + E = st + prods.sum(axis=-1) + fin = np.isfinite(E) & np.all(np.isfinite(prods), axis=-1) & np.isfinite(st) + pk = np.where(fin[..., None], prods, 0.0) + s0 = np.where(fin, st, 0.0) + if split is None: + b_hw = _hw_bound(s0, pk if order is None else pk[..., order]) + else: # the order mutant 3: split[0] from the start, split[1] from 0, one float32 add at the end + b_hw = (_hw_bound(s0, pk[..., split[0]]) + _hw_bound(np.zeros_like(s0), pk[..., split[1]]) + + _half_ulp32(E)) + b_or = 0.0 if oracle_exact else _half_ulp32(s0[..., None] + np.cumsum(pk, axis=-1)).sum(axis=-1) + oracle = f16(E) if oracle_exact else O.mm8(st.astype(np.float32), a, Wm, "f32chain") + lo, hi = f16(E - b_hw), f16(E + b_hw) + near = fin & ~same(f16(E - b_hw - b_or), f16(E + b_hw + b_or)) + got = np.asarray(got, np.float64) + differ = ~same(got, oracle) + inside = (got >= lo) & (got <= hi) + viol = (fin & ~inside) | (~fin & differ) + steps = np.where(fin & differ & np.isfinite(got) & np.isfinite(oracle), + np.abs(got.astype(np.float16).view(np.int16).astype(np.int64) + - np.asarray(oracle, np.float64).astype(np.float16).view(np.int16).astype(np.int64)), 0) + tally.add(stage, outputs=int(got.size), near=int(near.sum()), differ=int(differ.sum()), + differ_outside=int((differ & ~near).sum()), violations=int(viol.sum()), max_steps=int(steps.max(initial=0))) + + +def elementwise(tally, stage, want, got): + ok = same(want, got) + tally.add(stage, outputs=int(np.size(got)), differ=int((~ok).sum()), differ_outside=int((~ok).sum()), + violations=int((~ok).sum())) + + +# ---- one window -------------------------------------------------------------------------------------------------- + +def check_window(tally, d, p, x_tokens, out_tokens, kind, mutant=False, lin=None): + """d: stages(); p: the block's random parameters (natural values); x_tokens [16, C]: the oracle's gather of the + block input (FP16 values); out_tokens: the kernel's output for the window's real tokens (dict token -> values). + lin (kind enclin): (x1 tokens [16, 32], Win2 [32, 32], bin2): the X stage is the second linear's chain.""" + W = {n: q8(np.asarray(p[n], np.float64)) for n in ("Wqk", "Wv", "Wp", "W1", "W2")} + V = {n: f16(p[n]) for n in ("bp", "b1", "b2", "B")} + if lin is not None: + x1t, w2, b2 = lin + chain(tally, "lin2", f16(np.asarray(b2, np.float64)), x1t, q8(np.asarray(w2, np.float64)), d["X"], + order=order_l(32)) + else: + elementwise(tally, "gather", x_tokens, d["X"]) + elementwise(tally, "norm1", O.norm_q7(d["X"], O.gamma_q7(p["gamma1"])), d["Y1"]) + a8 = q8(d["Y1"]) + for hd in range(HEADS): + chain(tally, "qk", 0.0, a8, W["Wqk"][hd], d["UQ"][hd], order=order_l(C)) + chain(tally, "v", 0.0, a8, W["Wv"][hd], d["V"][hd], order=order_l(C)) + u8 = q8(d["UQ"][hd]) + chain(tally, "scores", V["B"][hd], u8, u8, d["S"][hd]) # S[t, s] = B[t, s] + u_t . u_s + elementwise(tally, "softmax", O.softmax_q7(d["S"][hd]), d["P"][hd]) + chain(tally, "pv", 0.0, d["P"][hd], d["V"][hd].T, d["O"][hd], oracle_exact=True) + o8 = q8(np.concatenate([d["O"][hd] for hd in range(HEADS)], axis=-1)) + chain(tally, "wp", f16(V["bp"] + d["X"]), o8, W["Wp"], d["Z"]) + elementwise(tally, "norm2", O.norm_q7(d["Z"], O.gamma_q7(p["gamma2"])), d["Y2"]) + chain(tally, "w1", V["b1"], q8(d["Y2"]), W["W1"], d["H"], order=order_l(C)) + split = None + if mutant == 3: # SW_HYB_MUT_ORDER 3 (kept for the record; the probe kernels now build mutant 4): hidden groups of 32 (GROUP 2 tiles), odd groups in the second accumulator + idx = np.arange(MLP) + split = (idx[(idx // 32) % 2 == 0], idx[(idx // 32) % 2 == 1]) + chain(tally, "w2", f16(V["b2"] + d["Z"]), q8(O.gelu_q7(d["H"])), W["W2"], d["OUT"], split=split) + if out_tokens: + idx = sorted(out_tokens) + got = np.stack([out_tokens[t] for t in idx]) + want = d["OUT"][idx] + if kind not in ("dec", "deccoef"): + elementwise(tally, "output_fp8", fp8_decode(fp8_encode(q8(want))), fp8_decode(got.astype(np.uint8))) + else: + elementwise(tally, "output_f16", want, got) + + +def gather(kind, xin, H, W, off, win, skip=None): + ox, oy = off + nwx = (W + ox + 3) // 4 + wy, wx = divmod(win, nwx) + xt, toks = np.empty((16, C)), {} + for t in range(16): + pr, pc = 4 * wy - oy + t // 4, 4 * wx - ox + t % 4 + if kind == "enc": # FP16 halo map [H + 4][W + 4][C] at (pr + 2, pc + 2) + xt[t] = xin[pr + 2, pc + 2].astype(np.float64) + elif kind == "enclin": # xin = the oracle's x1 halo map [H + 4][W + 4][32] (lin2's input) + xt[t] = xin[pr + 2, pc + 2] + elif kind == "blk": # FP8 map, the fine mirror + xt[t] = fp8_decode(xin[int(O._swin.mirror(pr, H)), int(O._swin.mirror(pc, W))]).astype(np.float64) + else: # RN16(E + S): E by the coarse mirror (sub-pixel kept), S by the fine one + m = lambda q, n: 2 * int(O._swin.mirror(q >> 1, n >> 1)) + (q & 1) # noqa: E731 + fm = lambda q, n: int(O._swin.mirror(q, n)) # noqa: E731 + xt[t] = f16(fp8_decode(xin[m(pr, H), m(pc, W)]).astype(np.float64) + + fp8_decode(skip[fm(pr, H), fm(pc, W)]).astype(np.float64)) + if 0 <= pr < H and 0 <= pc < W: + toks[t] = (pr, pc) + return xt, toks + + +# ---- the fused twins' inputs and outputs (--set fused) ----------------------------------------------------------- + +def lin_extra(qb, lp): + """kind enclin: the oracle's x1 halo map (hybrid form: f32 chain, RN16 at its end, ReLU, FP8) from the q codes + qb [H + 4][W + 4][32] (bytes 20..31 zero) and the linears lp (Win1 [32, 20], bin1, Win2, bin2); the 20-byte q form + and the two blobs for the kernel.""" + qv = fp8_decode(qb).astype(np.float64) + x1 = O.linear(qv[..., :20], lp["Win1"], lp["bin1"], "f16", "relu_q8", "f32chain") + return {"x1": np.asarray(x1, np.float64), "q20": q20(qb), "w2": lp["Win2"], "b2": lp["bin2"], + "blob1": LP.pack_linear(lp["Win1"], lp["bin1"], 32, 32), "blob2": LP.pack_linear(lp["Win2"], lp["bin2"], 32, 32)} + + +def case_lines(cdir, n, kernel, kind, H, W, ox, oy, nwin, xin, skip, nbytes, p, ex): + """The kernel_replay lines of one case (and its input files): the block kinds as before; enclin with the q20 map + and the two linear blobs; deccoef with the pack12 images (A: 8, C: 4, latent: 4 bytes a pixel of the rw x rh + rectangle) in place of the FP16 map and its coefficient blob.""" + files = [(f"w{n}.bin", pack_block_hyb(p, C, HEADS, MLP))] + files.append((f"x{n}.bin", ex["q20"] if kind == "enclin" else xin.tobytes())) + if skip is not None: + files.append((f"s{n}.bin", skip.tobytes())) + if kind == "enclin": + files += [(f"l1{n}.bin", ex["blob1"]), (f"l2{n}.bin", ex["blob2"])] + if kind == "deccoef": + files.append((f"cw{n}.bin", LP.pack_coef(ex["coef"]))) + for nm, data in files: + with open(os.path.join(cdir, nm), "wb") as f: + f.write(data) + out = (["arg null"] if kind == "deccoef" else [f"arg out {nbytes} o{n}.bin {LD.D3D12_SENTINEL}"]) + ln = [f"case c{n}", f"kernel {kernel} {-(-nwin // 8)} 256", f"arg buf x{n}.bin", + f"arg buf s{n}.bin" if skip is not None else "arg null"] + out + \ + [f"arg buf w{n}.bin", f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {ox}", f"arg u32 {oy}"] + if kind == "deccoef": + npx = ex["rw"] * ex["rh"] + ln += [f"arg out {8 * npx} cA{n}.bin 0", f"arg out {4 * npx} cC{n}.bin 0", f"arg out {4 * npx} cL{n}.bin 0", + f"arg buf cw{n}.bin", f"arg u32 {ex['rw']}", f"arg u32 {ex['rh']}"] + if kind == "enclin": + ln += [f"arg buf l1{n}.bin", f"arg buf l2{n}.bin"] + return ln + [f"arg out {nwin * 32 * HD_WORDS * 4} d{n}.bin 0", ""] + + +def coef_check(tally, cdir, n, y2, ex): + """kind deccoef: the pack12 images against oracle.coefficients_l of the block's own output stage y2 (the dumped + RN16 outputs, every real pixel; checked against the oracle per stage), l_ops_check coef_case's rule.""" + rw, rh = ex["rw"], ex["rh"] + npx = rw * rh + a8 = np.fromfile(os.path.join(cdir, f"cA{n}.bin"), np.uint32)[:2 * npx] + c4 = np.fromfile(os.path.join(cdir, f"cC{n}.bin"), np.uint32)[:npx] + lt = np.fromfile(os.path.join(cdir, f"cL{n}.bin"), np.int8)[:4 * npx].reshape(rh, rw, 4) + uA, uB, uC = unpack12(a8, c4) + ga = np.stack([(uA >> (8 * i)) & 0xFF for i in range(3)], -1).reshape(rh, rw, 3) + gb = np.stack([(uB >> (8 * i)) & 0xFF for i in range(3)], -1).reshape(rh, rw, 3) + cw = uC.reshape(-1, 2) + gc = np.stack([cw[:, 0] & 0xFFFF, cw[:, 0] >> 16, cw[:, 1] & 0xFFFF], -1).astype(np.uint16).view(np.float16) + gc = gc.reshape(rh, rw, 3) + a = y2[:rh, :rw][..., list(T.L_DEC_VALUES)].astype(np.float32) + ref = O.coefficients_l(a, ex["coef"]["S"], ex["coef"]["T"]) + for got, k in ((ga, "codes_A"), (gb, "codes_B"), (lt, "codes_L")): + dd = np.abs(got.astype(int) - np.asarray(ref[k]).astype(int)) + tally.add("coef12_" + k, outputs=int(dd.size), differ=int(np.sum(dd > 0)), differ_outside=int(np.sum(dd > 1)), + violations=int(np.sum(dd > 1)), max_steps=int(dd.max(initial=0))) + st = f16_steps(gc, np.asarray(ref["C"]).astype(np.float16)) + tally.add("coef12_C", outputs=int(st.size), differ=int(np.sum(st > 0)), differ_outside=int(np.sum(st > 4)), + violations=int(np.sum(st > 4)), max_steps=int(st.max(initial=0))) + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", default=os.path.join(r"C:\Program Files\AMD\ROCm", "7.2")) + ap.add_argument("--prepare", action="store_true") + ap.add_argument("--run", action="store_true") + ap.add_argument("--seeds", type=int, default=1) + ap.add_argument("--json") + ap.add_argument("--replay-only", action="store_true", help="the GPU replay only (run it inside the GPU slot)") + ap.add_argument("--no-replay", action="store_true", help="analyse the outputs of an earlier --replay-only (CPU)") + ap.add_argument("--real", action="store_true", help="real L weights / recorded frames: the allowance share counts") + ap.add_argument("--real-inputs", help="a directory of -.npz (real L weights and recorded-frame block " + "inputs: x2, x3, y, parity, enc0_block, dec0_block; private, never in the repository); implies --real") + ap.add_argument("--set", choices=("c32", "s3", "fused"), default="c32", help="c32: Model L's two blocks; s3: the " + "shared stages' hybrid blocks (C64, C96 with and without the skip, C128); fused: the twins of the " + "fusions le / lc (encoder with the linears, decoder with the pack12 epilogue)") + ap.add_argument("--mutant", action="store_true", + help="the order mutant's dump kernel (SW_HYB_MUT_ORDER=2, encoder): the tolerance leg must PASS it") + a = ap.parse_args(argv) + if a.set == "fused": + kernels = FUSED_MUT if a.mutant else FUSED_KERNELS + else: + kernels = ({"l_swin_block_c32_enc_hyb_mutdump": ("enc", 32, 2, 128)} if a.mutant + else S3_KERNELS if a.set == "s3" else KERNELS) + if a.prepare: + LD.prepare_replay(a.build, a.rocm, list(kernels)) + if not a.run: + return 0 + rng = np.random.default_rng(13) + cdir = os.path.join(a.build, "hyb_stage") + os.makedirs(cdir, exist_ok=True) + lines, meta, nan_leg = [], [], set() + n = 0 + # the random legs (finite codes), then the one NaN-input leg (seed a.seeds, every case special, NaN codes included) + for seed in range(a.seeds + 1): + is_nan = seed == a.seeds + sp8 = LD.SPECIAL_FP8 if is_nan else FINITE_FP8 + sp16 = LD.SPECIAL_F16 if is_nan else FINITE_F16 + for kernel, spec in kernels.items(): + kind = spec[0] + configure(*spec[1:]) + tkind = "first" if kind in ("enc", "blk", "enclin") else "second" + for gi, (H, W) in enumerate([(8, 12), (12, 8), (16, 16)]): + for parity in (0, 1): + p = O.random_block_params(rng, C, HEADS, MLP) + off = T.block_offset(T.TABLE, tkind, parity) + special = is_nan or (gi + parity + seed) % 2 == 1 + scale = 60.0 if special else 1.5 + ex = {} + if kind == "enclin": # q (FP8, 20 values + 12 zero bytes), the linears at the oracle's scales + lp = O.random_params(rng)["enc0"] + qv = np.zeros((H + 4, W + 4, 32)) + qv[..., :20] = q8(rng.normal(0, 30.0 if special else 1.5, (H + 4, W + 4, 20))) + qb = fp8_encode(qv).astype(np.uint8) + if special: + pick = rng.random((H + 4, W + 4, 20)) < 0.05 + sub = qb[..., :20] + sub[pick] = rng.choice(sp8, int(pick.sum())) + qb[..., :20] = sub + ex = lin_extra(qb, lp) + xin, nbytes, skip = ex.pop("x1"), H * W * C, None + elif kind == "deccoef": + cp = O.random_params(rng)["dec0"] + ex = {"coef": cp, "rw": W - (gi & 1), "rh": H - parity} + if kind == "enc": + xin = rng.normal(0, scale, (H + 4, W + 4, C)).astype(np.float16) + if special: + pick = rng.random(xin.shape) < 0.05 + xin[pick] = rng.choice(sp16, int(pick.sum())) + nbytes, skip = H * W * C, None + elif kind == "enclin": + pass + elif kind == "blk": + xin = fp8_encode(q8(rng.normal(0, scale, (H, W, C)))).astype(np.uint8) + if special: + pick = rng.random(xin.shape) < 0.05 + xin[pick] = rng.choice(sp8, int(pick.sum())) + nbytes, skip = H * W * C, None + else: + xin = fp8_encode(q8(rng.normal(0, scale, (H, W, C)))).astype(np.uint8) + skip = fp8_encode(q8(rng.normal(0, scale, (H, W, C)))).astype(np.uint8) + if special: + pick = rng.random(xin.shape) < 0.05 + xin[pick] = rng.choice(sp8, int(pick.sum())) + pick = rng.random(skip.shape) < 0.05 + skip[pick] = rng.choice(sp8, int(pick.sum())) + nbytes = H * W * C * (2 if kind == "dec" else 1) + ox, oy = off + nwin = -(-(W + ox) // 4) * -(-(H + oy) // 4) + lines += case_lines(cdir, n, kernel, kind, H, W, ox, oy, nwin, xin, skip, nbytes, p, ex) + meta.append((n, spec, H, W, off, nwin, p, xin, skip, ex)) + if is_nan: + nan_leg.add(n) + n += 1 + if a.real_inputs: # the real cases replace the random ones + a.real = True + lines, meta, n, nan_leg = [], [], 0, set() + for fn in sorted(f for f in os.listdir(a.real_inputs) if f.endswith(".npz")): + z = np.load(os.path.join(a.real_inputs, fn), allow_pickle=True) + parity = int(z["parity"]) + x3 = fp8_encode(q8(z["x3"].astype(np.float64))).astype(np.uint8) + for kernel, spec in kernels.items(): + kind = spec[0] + configure(*spec[1:]) + tkind = "first" if kind in ("enc", "enclin") else "second" + p = z["enc0_block" if kind in ("enc", "enclin") else "dec0_block"][0] + off = T.block_offset(T.TABLE, tkind, parity) + H, W = x3.shape[:2] + ex = {} + if kind == "enc": + xin, skip, nbytes = z["x2"].astype(np.float16), None, H * W * C + elif kind == "enclin": # the real q halo map (20 values) and the real linears + qv = np.zeros(z["q"].shape[:2] + (32,)) + qv[..., :20] = q8(z["q"].astype(np.float64)) + ex = lin_extra(fp8_encode(qv).astype(np.uint8), z["enc0"][0]) + xin, skip, nbytes = ex.pop("x1"), None, H * W * C + else: + xin, skip, nbytes = fp8_encode(q8(z["y"].astype(np.float64))).astype(np.uint8), x3, H * W * C * 2 + if kind == "deccoef": + ex = {"coef": z["dec0"][0], "rw": W, "rh": H} + ox, oy = off + nwin = -(-(W + ox) // 4) * -(-(H + oy) // 4) + lines += case_lines(cdir, n, kernel, kind, H, W, ox, oy, nwin, xin, skip, nbytes, p, ex) + meta.append((n, spec, H, W, off, nwin, p, xin, skip, ex)) + n += 1 + with open(os.path.join(cdir, "cases.txt"), "w") as f: + f.write("\n".join(lines)) + if not a.no_replay: + r = subprocess.run([os.path.join(a.build, "kernel_replay.exe"), a.build, os.path.join(cdir, "cases.txt")], + env=dict(os.environ, WG_L_FORM="hybrid"), capture_output=True, text=True, timeout=150, + creationflags=0x08000000) + if r.returncode: + raise SystemExit(f"kernel_replay failed: {r.stdout[-800:]}{r.stderr[-400:]}") + # the default leg (standing rule): no form variable set == WG_L_FORM=hybrid, every output and dump byte + dl = LD.replay_default_leg(a.build, os.path.join(cdir, "cases.txt"), dict(os.environ, WG_L_FORM="hybrid")) + with open(os.path.join(cdir, "default_leg.json"), "w") as f: + json.dump(dl, f) + print("default leg", "PASS" if dl["pass"] else "FAIL", dl["identical"], "/", dl["outputs"]) + if a.replay_only: # the GPU part only (inside a GPU slot); the analysis runs afterwards with --no-replay + print("replayed", len(meta), "cases into", cdir) + return 0 + tally = Tally() + # the NaN-input leg (oversight decision on the NaN finding, permanent until the 0.6.6 fix): its cases go to + # tally_nan, listed in the record, neither pass nor fail; every random-leg case (finite codes) must pass + tally_nan, nan_cases = Tally(), {} + for n, spec, H, W, off, nwin, p, xin, skip, ex in meta: + kind = spec[0] + configure(*spec[1:]) + dump = open(os.path.join(cdir, f"d{n}.bin"), "rb").read() + if kind == "deccoef": + out, y2 = None, np.full((H, W, C), np.nan) + else: + outb = open(os.path.join(cdir, f"o{n}.bin"), "rb").read() + out = (np.frombuffer(outb, np.float16).reshape(H, W, C).astype(np.float64) if kind == "dec" + else np.frombuffer(outb, np.uint8).reshape(H, W, C)) + for win in range(nwin): + xt, toks = gather(kind, xin, H, W, off, win, skip) + d = stages(dump, win) + lin = (xt, ex["w2"], ex["b2"]) if kind == "enclin" else None + tw_ = tally_nan if n in nan_leg else tally + if n in nan_leg and np.isnan(np.asarray(xt, np.float64)).any(): + nan_cases.setdefault(n, {"kernel": spec[0], "H": H, "W": W, "windows": 0})["windows"] += 1 + if kind == "deccoef": + for t, rc in toks.items(): + y2[rc] = d["OUT"][t] + check_window(tw_, d, p, xt, {}, kind, a.mutant) + else: + check_window(tw_, d, p, xt, {t: out[rc] for t, rc in toks.items()}, kind, a.mutant, lin=lin) + if kind == "deccoef": + coef_check(tally_nan if n in nan_leg else tally, cdir, n, y2, ex) + if n in nan_leg: + nan_cases.setdefault(n, {"kernel": spec[0], "H": H, "W": W, "windows": 0})["pack12_pixels_nan"] = \ + int(np.isnan(y2[:ex["rh"], :ex["rw"]]).any(axis=-1).sum()) + res = {"stages": tally.s} + if nan_leg: # reported, neither pass nor fail + for t in tally_nan.s.values(): + t["differ_rate"] = t["differ"] / max(1, t["outputs"]) + res["nan_input"] = {"rule": "oversight decision on the NaN finding (until the 0.6.6 fix): the dedicated NaN-input " + "leg, listed, neither pass nor fail", "leg_cases": len(nan_leg), + "cases_with_nan": {str(k): v for k, v in sorted(nan_cases.items())}, "stages": tally_nan.s} + dlp = os.path.join(cdir, "default_leg.json") + res["default_leg"] = json.load(open(dlp)) if os.path.exists(dlp) else {"pass": False, "missing": True} + for st, t in tally.s.items(): + t["near_share"] = t["near"] / max(1, t["outputs"]) + for t in tally.s.values(): + t["differ_rate"] = t["differ"] / max(1, t["outputs"]) + # oversight 2026-11-15 ~18:20Z: the allowance share is reported only; in the --real run (local files, read at run + # time) every FP8 chain stage had 0 actual differences (P.V, the one chain whose float32 sum is not exact, keeps the + # 1e-4 rate) + fp8_chains = ("qk", "v", "scores", "wp", "w1", "w2") + fp8_chains = fp8_chains + ("lin2",) + res["pass"] = all(t["violations"] == 0 and t["differ_outside"] == 0 + and (st.startswith("coef12_") or t["differ_rate"] <= DIFF_MAX) + and not (a.real and st in fp8_chains and t["differ"] > 0) for st, t in tally.s.items()) + res["pass"] = res["pass"] and res["default_leg"]["pass"] # the default leg (standing rule) + res["rules"] = {"bound_exceedance": 0, "differ_rate_max": DIFF_MAX, "allowance_share": "reported only", + "random_legs": "finite codes only; one NaN-input leg reported separately", + "fp8_chain_differences": 0 if a.real else "not required (random weights)"} + print(json.dumps(res, indent=1)) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + return 0 if res["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_hyb_stage_pack.py b/clean/l/runner/l_hyb_stage_pack.py new file mode 100644 index 00000000..62645e0a --- /dev/null +++ b/clean/l/runner/l_hyb_stage_pack.py @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: MIT +"""Model L's HYBRID form on the fast kernels (WG_L_FORM=hybrid, D3D12): every block stage's parameters in the values +the hybrid kernels read, packed by the unchanged M-form packers (clean/runner/packing.py pack_stage for the ten shared +stages, clean/l/runner/l_pack.py pack_model_l for Model L's own launches). Reference of the runtime's C++ +(clean/l/runtime/l_hyb_pack.h, byte for byte: clean/l/runtime/tests/test_l_runtime_hyb_pack.py). + +Values (as clean/l/reference/oracle.py uses them in the FP16 form): +- the Swin block section: l_hyb_pack.hyb_block_values (gamma1 / gamma2 -> RN16(float32(gamma) sqrtf(C)); bp, b1, b2, + B -> FP16 values; Wqk, Wv, Wp, W1, W2 -> FP8 values); +- the merge (enc1, enc2: Wm, bm) and the expansion (dec2, dec1: We, be), as run_stage_q7 reads them: the weights q8, + the biases f16. +Stages other than these blocks (enc0, dec0: Model L's linears, projections, coefficient scalars) keep their values. +""" +import numpy as np + +from clean.l.reference import oracle as O +from clean.l.runner.l_hyb_pack import hyb_block_values +from clean.reference.fp8 import q8 + +BLOCK_STAGES = ("enc0_block", "dec0_block", "enc1", "enc2", "b0", "b1", "b2", "b3", "b4", "b5", "dec2", "dec1") +LIN_W = ("Wm", "We") +LIN_B = ("bm", "be") + + +def hyb_stage_values(p): + """One block stage's tensors in the hybrid values (same keys; float32 arrays).""" + out = hyb_block_values(p) + for n in LIN_W: + if n in p: + out[n] = q8(np.asarray(p[n], np.float64)).astype(np.float32) + for n in LIN_B: + if n in p: + out[n] = O.f16(p[n]).astype(np.float32) + return out + + +def hyb_params(params): + """The whole model's stage map with every block stage in the hybrid values.""" + return {s: (hyb_stage_values(v) if s in BLOCK_STAGES else v) for s, v in params.items()} diff --git a/clean/l/runner/l_mm8_f32_check.py b/clean/l/runner/l_mm8_f32_check.py new file mode 100644 index 00000000..af16ebf5 --- /dev/null +++ b/clean/l/runner/l_mm8_f32_check.py @@ -0,0 +1,430 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): the FP32-sequential arithmetic of one FP8 matrix op (what the FP16-form kernels compute: +every product exact in float32, added one at a time in float32 in ascending k with round-to-nearest-even, the FP16 +start value added last in float32, one conversion to FP16) against the oracle's exact model (clean/l/reference/oracle.py +MM8_MODELS "exact": the exact sum rounded once to FP16). + +Adversarial families (each a set of single 32-term ops): +- typical: random activations and weights at typical network scales, any start; +- cancellation: pairs of large products of opposite sign with small terms between them; +- dynamic_range: products spread over the whole FP8 x FP8 range (2^-18 ... 448^2); +- near_tie: the exact sum placed on an FP16 rounding tie, plus or minus one small product; +- subnormal: FP8 subnormal operands (2^-9 ... 7 x 2^-9) with normal ones. +Every op where the two differ is recorded (the 32 FP8 codes of each operand and the start's FP16 bits) with the +difference in FP16 steps. With --frames , the same comparison runs on every FP8 op of Model L's own stages for +the recorded run's frames (l_frame_check), and each op's margin is reported: the distance of its exact sum from the +nearest FP16 rounding boundary divided by the bound on the FP32-sequential error (the two can differ only below 1). + + python clean/l/runner/l_mm8_f32_check.py --json out.json [--frames ] [--cases 20000] + python clean/l/runner/l_mm8_f32_check.py --wmma-prepare [--frames ] --json x # CPU + python clean/l/runner/l_mm8_f32_check.py --wmma-run --json out.json # GPU (a GPU slot) +""" +import argparse +import copy +import json +import os +import pickle +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import oracle as L # noqa: E402 +from clean.reference.fp8 import f16, fp8_encode, q8 # noqa: E402 + +FP8_VALUES = np.unique(np.abs(np.nan_to_num(np.asarray(__import__("clean.reference.fp8", fromlist=["DECODE"]).DECODE), + nan=0.0))) + + +def op_f32seq(start, a8, W8): + """start [..., n] FP16 values, a8 [..., k], W8 [n, k] (k <= 32) -> FP16 [..., n]: products in float32 (exact), + summed in float32 in ascending k, + start in float32, one conversion to FP16.""" + a8 = np.asarray(a8, np.float32) + W8 = np.asarray(W8, np.float32) + acc = np.zeros(np.broadcast_shapes(a8.shape[:-1] + (W8.shape[0],), np.shape(start)), np.float32) + with np.errstate(over="ignore", invalid="ignore"): + for k in range(a8.shape[-1]): + acc = (acc + a8[..., k:k + 1] * W8[:, k]).astype(np.float32) + acc = (acc + np.asarray(start, np.float32)).astype(np.float32) + return f16(acc.astype(np.float64)) + + +ABS_FAST = np.float32(63.0) # fast path while the float32 sum of |products| stays below this (true sum < 2^6) + + +def _two_sum(a, b): + """Error-free float32 sum: a + b = s + e exactly (e exact in float32).""" + s = (a + b).astype(np.float32) + bb = (s - a).astype(np.float32) + e = ((a - (s - bb).astype(np.float32)).astype(np.float32) + (b - bb).astype(np.float32)).astype(np.float32) + return s, e + + +def op_kernel(start, a8, W8): + """The FP16-form kernels' arithmetic of one FP8 op (exact): products in float32 (exact); their float32 sum in + ascending k with the float32 sum of |products| alongside. While that stays below 63 the sum is exact (every + partial sum a multiple of 2^-18 below 2^6); otherwise the products are summed again as a TwoSum chain (s, c), c + the exact sum of the errors (multiples of 2^-18 below 8). The FP16 start is added with error-free sums into a + head z and a tail t (z = fl(exact total), |t| <= half an ulp of z); RN16(z), and where z is exactly an FP16 + midpoint the tail's sign picks the side (no tail: ties to even).""" + a8 = np.asarray(a8, np.float32) + W8 = np.asarray(W8, np.float32) + shape = np.broadcast_shapes(a8.shape[:-1] + (W8.shape[0],), np.shape(start)) + st = np.broadcast_to(np.asarray(start, np.float32), shape) + with np.errstate(over="ignore", invalid="ignore"): + acc = np.zeros(shape, np.float32) + absacc = np.zeros(shape, np.float32) + prods = [] + for k in range(a8.shape[-1]): + p = (a8[..., k:k + 1] * W8[:, k]).astype(np.float32) + prods.append(p) + acc = (acc + p).astype(np.float32) + absacc = (absacc + np.abs(p)).astype(np.float32) + s, c = acc, np.zeros(shape, np.float32) + slow = ~(absacc < ABS_FAST) + if slow.any(): + s2, c2 = np.zeros(shape, np.float32), np.zeros(shape, np.float32) + for p in prods: + s2, e = _two_sum(s2, p) + c2 = (c2 + e).astype(np.float32) + s, c = np.where(slow, s2, s), np.where(slow, c2, c) + u, e1 = _two_sum(s, st) + v, e2 = _two_sum(e1, c) + z, e3 = _two_sum(u, v) + t = (e3 + e2).astype(np.float32) + z, t = _two_sum(z, t) + r16 = z.astype(np.float16) + r = r16.astype(np.float64) + zd = z.astype(np.float64) + fin = np.isfinite(zd) & np.isfinite(t) + # the FP16 neighbour of r on z's side (beyond 65504: the overflow boundary 65536 stands for infinity) + toward = np.where(zd > r, np.float16(np.inf), np.float16(-np.inf)).astype(np.float16) + nb = np.nextafter(np.where(np.isinf(r16), np.copysign(np.float16(65504), r16), r16), toward).astype(np.float64) + rr = np.where(np.isinf(r), np.copysign(65536.0, r), r) + nb = np.where(np.isinf(nb), np.copysign(65536.0, nb), nb) + nb = np.where(np.isinf(r), np.copysign(65504.0, r), nb) # r = inf: z on the boundary 65520, below it 65504 + mid = fin & (zd != r) & (zd == (rr + nb) / 2.0) & (t != 0) + lo, hi = np.minimum(rr, nb), np.maximum(rr, nb) + cand = np.where(t > 0, hi, lo) + cand = np.where(np.abs(cand) >= 65536.0, np.copysign(np.inf, cand), cand) + out = np.where(mid, cand, r) + plain = (acc + st).astype(np.float32).astype(np.float16).astype(np.float64) + return np.where(np.isfinite(acc) & np.isfinite(st), out, plain) + + +def op_exact(start, a8, W8): + return L._op8(np.asarray(start, np.float64), np.asarray(a8, np.float64), np.asarray(W8, np.float64), "exact") + + +def f16_steps(a, b): + ia = np.asarray(a, np.float64).astype(np.float16).view(np.int16).astype(np.int64) + ib = np.asarray(b, np.float64).astype(np.float16).view(np.int16).astype(np.int64) + ia = np.where(ia < 0, -(ia & 0x7FFF), ia) + ib = np.where(ib < 0, -(ib & 0x7FFF), ib) + return np.abs(ia - ib) + + +def margin(start, a8, W8): + """Per output: |exact sum - nearest FP16 rounding boundary| / (the FP32-sequential error bound: 32 additions, each + off by at most half an ulp of the running sum's magnitude bound in float32). Below 1 the two may differ.""" + a8 = np.asarray(a8, np.float64) + W8 = np.asarray(W8, np.float64) + prods = a8[..., None, :] * W8 # [..., n, k] + s = prods.sum(-1) + np.asarray(start, np.float64) + bound_mag = np.abs(prods).sum(-1) + np.abs(np.asarray(start, np.float64)) + err = (prods.shape[-1] + 1) * bound_mag * 2.0 ** -24 + r = f16(s) + with np.errstate(over="ignore", invalid="ignore"): + up = np.nextafter(r.astype(np.float16), np.float16(np.inf)).astype(np.float64) + dn = np.nextafter(r.astype(np.float16), np.float16(-np.inf)).astype(np.float64) + dist = np.minimum(np.abs(s - (r + up) / 2), np.abs(s - (r + dn) / 2)) + return np.where(err > 0, dist / np.maximum(err, 1e-300), np.inf) + + +# ---- adversarial families --------------------------------------------------------------------------------------------- + +def _fp8(rng, shape, lo_exp, hi_exp): + """FP8 values log-uniform in magnitude between 2^lo_exp and 2^hi_exp, random sign.""" + mag = 2.0 ** rng.uniform(lo_exp, hi_exp, shape) + return q8(mag * rng.choice([-1.0, 1.0], shape)) + + +def family(name, rng, n): + """n ops: (start [n], a [n, 32], w [n, 32]) of the named family (one output per op).""" + if name == "typical": + a = q8(rng.normal(0, 1.5, (n, 32))) + w = q8(rng.normal(0, 0.25, (n, 32))) + st = f16(rng.normal(0, 4, n)) + elif name == "cancellation": + a = np.zeros((n, 32)) + w = np.zeros((n, 32)) + big = _fp8(rng, (n, 8), 4, 8.8) + a[:, 0:8], w[:, 0:8] = big, _fp8(rng, (n, 8), 4, 8.8) + a[:, 8:16], w[:, 8:16] = -a[:, 0:8], w[:, 0:8] # the same products, opposite sign + a[:, 16:], w[:, 16:] = _fp8(rng, (n, 16), -9, 0), _fp8(rng, (n, 16), -9, 0) + perm = np.argsort(rng.random((n, 32)), axis=1) + a, w = np.take_along_axis(a, perm, 1), np.take_along_axis(w, perm, 1) + st = f16(_fp8(rng, n, -12, 2)) + elif name == "dynamic_range": + a = _fp8(rng, (n, 32), -9, 8.8) + w = _fp8(rng, (n, 32), -9, 8.8) + st = f16(rng.normal(0, 1, n) * 2.0 ** rng.uniform(-14, 15, n)) + elif name == "near_tie": + a = q8(rng.normal(0, 1.5, (n, 32))) + w = q8(rng.normal(0, 0.25, (n, 32))) + a[:, 31], w[:, 31] = 0.0, 0.0 + s = (a * w).sum(1) + r = f16(s + 8.0) # a target FP16 value near the sum + ulp = np.abs(np.nextafter(r.astype(np.float16), np.float16(np.inf)).astype(np.float64) - r) + tie = r + ulp / 2 + st = f16(tie - s) # start so that the sum sits near a tie + a[:, 31] = _fp8(rng, n, -9, -3) + w[:, 31] = _fp8(rng, n, -9, -3) + elif name == "start_midpoint": + # a large FP16 start and products summing to half its FP16 ulp plus or minus one tiny product: the float32 + # addition of the start lands on the FP16 midpoint (random synthetic operands) + e = rng.integers(8, 15, n) # start in [2^e, 2^(e+1)) + st = f16(rng.uniform(1, 2, n) * 2.0 ** e * rng.choice([-1.0, 1.0], n)) + half = 2.0 ** (e - 11) # half an FP16 ulp there + a = np.zeros((n, 32)) + w = np.zeros((n, 32)) + a[:, 0], w[:, 0] = q8(half * np.sign(st)), 1.0 + tiny = 2.0 ** rng.integers(-18, -12, n) + a[:, 1] = q8(np.sqrt(tiny)) + w[:, 1] = q8(np.sqrt(tiny) * rng.choice([-1.0, 1.0], n)) + a[:, 2:8], w[:, 2:8] = q8(rng.normal(0, 2.0 ** -6, (n, 6))), 0.0 + elif name == "subnormal": + a = q8(rng.integers(-7, 8, (n, 32)) * 2.0 ** -9) + w = np.where(rng.random((n, 32)) < 0.5, q8(rng.integers(-7, 8, (n, 32)) * 2.0 ** -9), + q8(rng.normal(0, 2, (n, 32)))) + st = f16(rng.normal(0, 2.0 ** -12, n)) + else: + raise ValueError(name) + return st, a, w + + +FAMILIES = ("typical", "cancellation", "dynamic_range", "near_tie", "start_midpoint", "subnormal") + + +def compare_family(name, rng, n, keep=50, op=None): + op = op or op_f32seq + st, a, w = family(name, rng, n) + got = np.array([op(st[i:i + 1], a[i], w[i:i + 1])[0] for i in range(n)]) + ref = np.array([op_exact(st[i:i + 1], a[i], w[i:i + 1])[0] for i in range(n)]) + steps = f16_steps(got, ref) + nan_same = np.isnan(got) & np.isnan(ref) + diff = (steps > 0) & ~nan_same + rows = [] + for i in np.flatnonzero(diff)[:keep]: + rows.append({"start_f16_bits": int(np.array([st[i]], np.float16).view(np.uint16)[0]), + "a_fp8_codes": fp8_encode(a[i]).tolist(), "w_fp8_codes": fp8_encode(w[i]).tolist(), + "exact": float(ref[i]), "f32seq": float(got[i]), "f16_steps": int(steps[i])}) + return {"cases": int(n), "differ": int(diff.sum()), "differ_over_1": int(((steps > 1) & ~nan_same).sum()), + "max_f16_steps": int(steps[~nan_same].max()) if (~nan_same).any() else 0, + "examples": rows} + + +# ---- a local recorded run (read at run time; nothing of it is in the repository) ----------------------------------- + +def frames_study(build): + """Every FP8 op of Model L's own stages (both blocks, input linears, projections) on the recorded run's frames: + FP32-sequential vs exact, and the smallest margin.""" + from clean.l.reference import table as T + from clean.l.runner import l_frame_check as F + rec = pickle.load(open(F.rec_path(build, "L", "stages"), "rb")) + params = pickle.load(open(os.path.join(F.model_dir(build), "params.pkl"), "rb"))["L"] + stats = {"ops": 0, "results": 0, "differ": 0, "kernel_differ": 0, "min_margin": float("inf"), "margin_below_1": 0, + "margin_below_10": 0} + exact = L._op8 + + def spy(start, a8, W8, model): + out = exact(start, a8, W8, model) + k = a8.shape[-1] + a = a8.reshape(-1, k) + st = np.broadcast_to(np.asarray(start, np.float64), a8.shape[:-1] + (W8.shape[0],)).reshape(-1, W8.shape[0]) + ref = out.reshape(-1, W8.shape[0]) + d = f16_steps(op_f32seq(st, a, W8), ref) + dk = f16_steps(op_kernel(st, a, W8), ref) + m = margin(st, a, W8) + stats["ops"] += 1 + stats["results"] += int(d.size) + stats["differ"] += int(np.sum(d > 0)) + stats["kernel_differ"] += int(np.sum(dk > 0)) + fin = np.isfinite(m) + if fin.any(): + stats["min_margin"] = min(stats["min_margin"], float(m[fin].min())) + stats["margin_below_1"] += int(np.sum(m < 1)) + stats["margin_below_10"] += int(np.sum(m < 10)) + return out + + t = copy.deepcopy(T.TABLE) + L._op8 = spy + try: + F.compare_model(rec, params, table=t) + finally: + L._op8 = exact + stats["identity"] = rec.get("identity") + return stats + + +# ---- the matrix instruction (GPU probe, a side measurement) ------------------------------------------------------------ + +def wmma_prepare(build, n, seed=7, real_build=None, real_max=200000): + """CPU: the probe's inputs (every family, and a sample of a local recorded run's 32-term FP8 ops) with the exact sums and + the exact model's FP16 results, saved to /l_wmma_cases.pkl.""" + sets = {} + for name in FAMILIES: + st, a, w = family(name, np.random.default_rng([seed, FAMILIES.index(name)]), n) + sets[name] = (st, a, w) + identity = None + if real_build: + from clean.l.reference import table as T + from clean.l.runner import l_frame_check as F + rec = pickle.load(open(F.rec_path(real_build, "L", "stages"), "rb")) + params = pickle.load(open(os.path.join(F.model_dir(real_build), "params.pkl"), "rb"))["L"] + identity = rec.get("identity") + sample = {"st": [], "a": [], "w": []} + exact = L._op8 + + def spy(start, a8, W8, model): + out = exact(start, a8, W8, model) + if a8.shape[-1] == 32 and len(sample["st"]) < real_max: + aa = a8.reshape(-1, 32) + st_ = np.broadcast_to(np.asarray(start, np.float64), a8.shape[:-1] + (W8.shape[0],)).reshape( + -1, W8.shape[0]) + for i in range(min(64, aa.shape[0])): + j = i % W8.shape[0] + sample["st"].append(st_[i, j]) + sample["a"].append(aa[i]) + sample["w"].append(W8[j]) + return out + + L._op8 = spy + try: + F.compare_model(rec, params, table=copy.deepcopy(T.TABLE)) + finally: + L._op8 = exact + sets["real_frames_sample"] = tuple(np.array(sample[k][:real_max]) for k in ("st", "a", "w")) + cases = {} + for name, (st, a, w) in sets.items(): + exact_sum = (np.asarray(a, np.float64) * np.asarray(w, np.float64)).sum(1) + np.asarray(st, np.float64) + ref16 = np.array([op_exact(st[i:i + 1], a[i], w[i:i + 1])[0] for i in range(len(st))]) + cases[name] = {"st": st, "a": a, "w": w, "exact_sum": exact_sum, "ref16": ref16} + print("prepared", name, len(st), flush=True) + with open(os.path.join(build, "l_wmma_cases.pkl"), "wb") as f: + pickle.dump({"cases": cases, "identity": identity}, f) + + +def wmma_run(build, rocm): + """GPU: the probe kernel (clean/l/kernels/l_wmma_probe.hip, /l_wmma_probe.co) on every prepared set: per op, + whether the instruction's float32 result equals the exact sum rounded once to float32, and whether its FP16 rounding + equals the exact model's result.""" + import ctypes + from clean.runner.hip import Hip, clean_process_environment + clean_process_environment(rocm) + with open(os.path.join(build, "l_wmma_cases.pkl"), "rb") as f: + prep = pickle.load(f) + h = Hip(rocm) + with open(os.path.join(build, "l_wmma_probe.co"), "rb") as f: + fn = h.get_function(h.load_module(f.read()), "l_wmma_probe") + res = {} + for name, c in prep["cases"].items(): + st, a, w = c["st"], c["a"], c["w"] + m = len(st) + ab, bb = fp8_encode(a).tobytes(), fp8_encode(w).tobytes() + sb = np.asarray(st, np.float32).tobytes() + pa, pb, ps, po = h.malloc(len(ab)), h.malloc(len(bb)), h.malloc(len(sb)), h.malloc(4 * m) + h.upload(pa, ab) + h.upload(pb, bb) + h.upload(ps, sb) + h.launch(fn, -(-m // 16), 32, [ctypes.c_void_p(pa), ctypes.c_void_p(pb), ctypes.c_void_p(ps), + ctypes.c_void_p(po), ctypes.c_int(m)]) + h.sync() + got = np.frombuffer(h.download(po, 4 * m), np.float32).astype(np.float64) + # S2a's use: the accumulator starts at 0 (the FP16 start is added exactly afterwards), so the instruction + # computes the product sum alone; compare that with the exact product sum + h.upload(ps, bytes(len(sb))) + h.launch(fn, -(-m // 16), 32, [ctypes.c_void_p(pa), ctypes.c_void_p(pb), ctypes.c_void_p(ps), + ctypes.c_void_p(po), ctypes.c_int(m)]) + h.sync() + got0 = np.frombuffer(h.download(po, 4 * m), np.float32).astype(np.float64) + for ptr in (pa, pb, ps, po): + h.free(ptr) + fin = np.isfinite(c["exact_sum"]) + psum = c["exact_sum"] - np.asarray(st, np.float64) # exact in float64 + p_exact = got0 == psum.astype(np.float32).astype(np.float64) + aa0, ww0 = np.abs(np.asarray(a, np.float64)), np.abs(np.asarray(w, np.float64)) + bt0, bp0 = (aa0 * ww0).sum(1), ww0.sum(1) * aa0.max(1) + zero_start = {"ops": int(fin.sum()), "f32_not_exact_product_sum": int(np.sum(fin & ~p_exact)), + "bounded_true_63": int(np.sum(fin & (bt0 < 63))), + "bounded_true_63_not_exact": int(np.sum(fin & (bt0 < 63) & ~p_exact)), + "bounded_pre_63": int(np.sum(fin & (bp0 < 63))), + "bounded_pre_63_not_exact": int(np.sum(fin & (bp0 < 63) & ~p_exact))} + f32_exact = got == c["exact_sum"].astype(np.float32).astype(np.float64) + d16 = f16_steps(f16(got), c["ref16"]) + rows = [{"start_f16_bits": int(np.array([st[i]], np.float16).view(np.uint16)[0]), + "a_fp8_codes": fp8_encode(a[i]).tolist(), "w_fp8_codes": fp8_encode(w[i]).tolist(), + "exact_sum": float(c["exact_sum"][i]), "instruction_f32": float(got[i])} + for i in np.flatnonzero(fin & (d16 > 0))[:20]] + # S2a's exactness argument: while the bound stays below 63, every partial sum is a multiple of 2^-18 below 2^6, + # exact in float32 in any order, so the instruction must give the exact sum. Two bounds: the true sum |a w| and + # the one the kernels can afford (precomputed sum |w| of the row, times the op's max |a|). + aa, ww = np.abs(np.asarray(a, np.float64)), np.abs(np.asarray(w, np.float64)) + b_true = (aa * ww).sum(1) + np.abs(np.asarray(st, np.float64)) + b_pre = ww.sum(1) * aa.max(1) + res[name] = {"ops": int(fin.sum()), "f32_not_exact_sum": int(np.sum(fin & ~f32_exact)), + "fp16_differs_from_exact_model": int(np.sum(fin & (d16 > 0))), + "bounded_true_63": int(np.sum(fin & (b_true < 63))), + "bounded_true_63_not_exact": int(np.sum(fin & (b_true < 63) & ~f32_exact)), + "bounded_pre_63": int(np.sum(fin & (b_pre < 63))), + "bounded_pre_63_not_exact": int(np.sum(fin & (b_pre < 63) & ~f32_exact)), + "zero_start": zero_start, "examples": rows} + print(f"wmma {name:18s}", {k: v for k, v in res[name].items() if k != "examples"}, flush=True) + if prep["identity"]: + res["real_frames_identity"] = prep["identity"] + from clean.l.runner.l_q7_check import run_identity + res["identity"] = run_identity(build, ["l_wmma_probe.co", "l_wmma_cases.pkl"]) + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--json", required=True) + ap.add_argument("--cases", type=int, default=20000) + ap.add_argument("--seed", type=int, default=7) + ap.add_argument("--frames", help="a build with a recorded l_frame_check run (lframe_L_stages.pkl)") + ap.add_argument("--wmma-prepare", metavar="BUILD", help="CPU: the matrix-instruction probe's inputs into BUILD") + ap.add_argument("--wmma-run", metavar="BUILD", help="GPU: the probe (BUILD/l_wmma_probe.co) on the prepared inputs") + ap.add_argument("--rocm", default=r"C:\Program Files\AMD\ROCm\7.2") + a = ap.parse_args(argv) + if a.wmma_prepare: + wmma_prepare(a.wmma_prepare, a.cases, a.seed, a.frames) + return 0 + if a.wmma_run: + from clean.l.runner import l_record as LR + res = {"what": "RDNA 4 FP8 matrix instruction (two 16-term steps, float32 accumulator from the start) against " + "the exact model, a side measurement", "fp8_op_model": "exact", "probe": wmma_run(a.wmma_run, a.rocm)} + LR.write_l_record(res, a.json) + return 0 + rng = np.random.default_rng(a.seed) + res = {"what": "FP32-sequential FP8 op (the kernels' arithmetic) against the exact model", "seed": a.seed, + "families": {}} + for name in FAMILIES: + for opname, op in (("f32seq", op_f32seq), ("kernel", op_kernel)): + r = compare_family(name, np.random.default_rng([a.seed, FAMILIES.index(name)]), a.cases, op=op) + res["families"].setdefault(name, {})[opname] = r + print(f"{name:14s} {opname:7s} cases {r['cases']} differ {r['differ']} more than 1 step " + f"{r['differ_over_1']} max {r['max_f16_steps']}", flush=True) + if a.frames: + res["frames"] = frames_study(a.frames) + print("local recorded run:", json.dumps({k: v for k, v in res["frames"].items() if k != "identity"})) + from clean.l.runner import l_record as LR + LR.write_l_record(res, a.json) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_pack.py b/clean/l/runner/l_pack.py index 28de34f1..6f8db34d 100644 --- a/clean/l/runner/l_pack.py +++ b/clean/l/runner/l_pack.py @@ -85,6 +85,102 @@ def pack_block(tensors, C, heads, mlp, d_row="gfx12"): return blob + bytes((-len(blob)) % ALIGN) +# ---- the FP16-form block kernels' blob ("q7", clean/l/kernels/l_q7.hip) ------------------------------------------------- +# +# Natural order, no fragment layout: FP16 vectors g1' [C], g2' [C], bp [C], b1 [mlp], b2 [C], B [heads][16][16] +# (g' = RN16(float32(gamma) x sqrtf(C)), oracle.gamma_q7), then the FP8 codes of Wqk [heads][32][C], Wv [heads][32][C], +# Wp [C][32 heads], W1 [mlp][C], W2 [C][mlp], each row-major; padded to ALIGN. + +def q7_block_offsets(C, heads, mlp): + """Byte offsets of the q7 blob's parts and its unpadded length.""" + o, at = {}, 0 + for name, n in (("g1", C), ("g2", C), ("bp", C), ("b1", mlp), ("b2", C), ("B", heads * 256)): + o[name], at = at, at + 2 * n + mats = (("Wqk", heads * 32 * C), ("Wv", heads * 32 * C), ("Wp", C * 32 * heads), ("W1", mlp * C), ("W2", C * mlp)) + for name, n in mats: + o[name], at = at, at + n + o["end8"] = at # a multiple of 16 for every width + for name, n in mats: # the same matrices as FP16 (exact: every FP8 value is an FP16 value), read by + o[name + "16"], at = at, at + 2 * n # the scalar kernels without an FP8 decode (Q7F-1) + o["end16"] = at + for name, n in mats: # sum |w| per 32-wide chunk and row, float32 [K / 32][rows] (exact), the bound of + o[name + "_wsum"], at = at, at + 4 * (n // 32) # the matrix-instruction path (Q7F-2 / S2a, wsum_table) + o["end"] = at + return o + + +def wsum_table(W): + """sum |w| per row and 32-wide chunk of an FP8 matrix W [rows, K] (K a multiple of 32), float32, CHUNK-major + [K / 32][rows] (a lane's 8 consecutive rows of one chunk are one 32-byte load): exact (at most 32 FP8 magnitudes, + multiples of 2^-9 below 2^14). A NaN weight gives NaN, and wsum x amax < 63 is then false: the exact fallback.""" + from clean.reference.fp8 import q8 + a = np.abs(q8(np.asarray(W, np.float64))) + rows, K = a.shape + s = a.reshape(rows, K // 32, 32).sum(-1).T + assert np.array_equal(s.astype(np.float32).astype(np.float64), s, equal_nan=True) + return np.ascontiguousarray(s.astype(np.float32)) + + +def pack_q7_block(tensors, C, heads, mlp): + """One block's q7 blob from its tensors (gamma1, gamma2, Wqk, Wv, B, Wp, bp, W1, b1, W2, b2).""" + from clean.l.reference import oracle as L + from clean.reference.fp8 import fp8_encode, f16, q8 + t = {k: np.asarray(v, np.float64) for k, v in tensors.items()} + shapes = _table.block_shapes(C, heads, mlp) + for name, shape in shapes.items(): + if t[name].shape != tuple(shape): + raise ValueError(f"q7 block: {name} has shape {t[name].shape}, expected {tuple(shape)}") + parts = [L.gamma_q7(t["gamma1"], C), L.gamma_q7(t["gamma2"], C), f16(t["bp"]), f16(t["b1"]), f16(t["b2"]), + f16(t["B"]).reshape(-1)] + blob = b"".join(np.asarray(v, np.float64).astype(np.float16).tobytes() for v in parts) + for name in ("Wqk", "Wv", "Wp", "W1", "W2"): + blob += fp8_encode(q8(t[name])).reshape(-1).tobytes() + for name in ("Wqk", "Wv", "Wp", "W1", "W2"): + blob += q8(t[name]).astype(np.float16).reshape(-1).tobytes() + for name in ("Wqk", "Wv", "Wp", "W1", "W2"): + blob += wsum_table(t[name].reshape(-1, t[name].shape[-1])).tobytes() + assert len(blob) == q7_block_offsets(C, heads, mlp)["end"] + return blob + bytes((-len(blob)) % ALIGN) + + +def q7_linear_offsets(n, K): + """Byte offsets of a q7 linear blob (a merge or an expansion, l_q7w.hip): FP16 bias [n] at 0, FP8 rows [n][K] at + the next multiple of 16, then the same rows as FP16 (Q7F-1), then sum |w| [K / 32][n] float32 (Q7F-2).""" + rows = (2 * n + 15) // 16 * 16 + return {"b": 0, "W": rows, "W16": rows + n * K, "W_wsum": rows + 3 * n * K, + "end": rows + 3 * n * K + 4 * n * (K // 32)} + + +def pack_q7_linear(W, b): + """A merge's (Wm, bm) or an expansion's (We, be) q7 linear blob: W [n, K] (FP8 values), b [n] (FP16 values).""" + from clean.reference.fp8 import fp8_encode, f16, q8 + W = np.asarray(W, np.float64) + b = np.asarray(b, np.float64) + n, K = W.shape + if b.shape != (n,) or K % 32: + raise ValueError(f"q7 linear: W {W.shape}, b {b.shape}") + o = q7_linear_offsets(n, K) + head = f16(b).astype(np.float16).tobytes() + blob = (head + bytes(o["W"] - len(head)) + fp8_encode(q8(W)).reshape(-1).tobytes() + + q8(W).astype(np.float16).reshape(-1).tobytes() + wsum_table(W).tobytes()) + assert len(blob) == o["end"] + return blob + bytes((-len(blob)) % ALIGN) + + +def q7_tables(): + """The two FP16 lookup tables of the q7 kernels, indexed by an FP16 bit pattern (65536 entries each, FP16): + rsqrt[b] = RN16(float32(1 / float32(sqrt(float32(x))))) (the norm's r of S' = x) and rcp[b] = RN16(float32(1 / + float32(x))) (the attention weights' normaliser of a row sum x), as the oracle computes them (oracle._rsqrt32, + oracle._rcp32). Returns the bytes rsqrt || rcp.""" + from clean.l.reference import oracle as L + from clean.reference.fp8 import f16 + x = np.arange(65536, dtype=np.uint32).astype(np.uint16).view(np.float16).astype(np.float64) + with np.errstate(all="ignore"): + rs = f16(L._rsqrt32(x)).astype(np.float16) + rc = f16(L._rcp32(x)).astype(np.float16) + return rs.tobytes() + rc.tobytes() + + def pack_values(tensors, d_row="gfx12"): """Model M's enc0 blob with zero projection: only the dither table and the two scalars are used.""" t = {"W": np.zeros((64, 80), np.float32), "b": np.zeros(64, np.float32), diff --git a/clean/l/runner/l_post_c12_check.py b/clean/l/runner/l_post_c12_check.py new file mode 100644 index 00000000..36ea83e6 --- /dev/null +++ b/clean/l/runner/l_post_c12_check.py @@ -0,0 +1,194 @@ +# SPDX-License-Identifier: MIT +"""Q11 part C: bit-exact probe of the reconstruction's fused-rounding forms C1 and C2 (handoff +preset-M/from-spec/PRESET-M-C1C2-machine-code.md) as l_post__morig.co compiles them, on adversarial inputs. + + python clean/l/runner/l_post_c12_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --json out.json + +/l_post_c12_probe.co (WG_LP_C1 = 1, WG_CR_C2 = 1) and /l_post_c12_probe__mut.co (WG_LP_C1 = 0: item 3's +exponent; WG_CR_C2 = 2: w1 with 2.5 f2 fused) come from clean/runner/build.py. The GPU part runs in a GPU slot. + +C1: eps = fused_half(h(u v), C, fused_half(h(v v), B, h(h(u u) A))), FP16 in and out. C2 (float32, one rounding per +operation): f2 = f f, f3 = f f2, w0 = fused(-(f + f3), 1/2, f2), w1 = fused(f3, 1.5, -RN(2.5 f2)) + 1, +w3 = RN(RN(f3 - f2) 1/2), w2 = ((1 - w0) - w1) - w3. +The NumPy model rounds each fused operation once, exactly: the product of two FP16 (or float32) values is exact in +float64; the float64 sum's own rounding error (TwoSum) moves the sum one float64 step towards the exact value before +the single rounding to FP16 (float32), which gives the correctly rounded result (every FP16 / float32 rounding boundary +is a float64 number). NaNs agree when both are NaN (any payload). +Pass: the probe build equals the model in every value; the mutant build differs from the model in both C1 and C2. +Inputs: C1 offsets u, v as the kernel forms them (h(d + i), d in (-1, 1), i = -1 ... 2) plus random FP16 values in +(-2.5, 2.5), coefficients in spec 06's ranges (A, B <= 0, C of either sign) and wide random FP16 bit patterns; C2 +fractions: 0, uniform [0, 1), the kernel's smallest nonzero fractions (2^-25 ...) and the values just below 1. +""" +import argparse +import ctypes +import json +import os +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + + +def _round_once(a, b, out_dtype): + """RN_out(a + b) with one rounding (a, b float64 arrays whose values are exact).""" + s = a + b + bb = s - a + err = (a - (s - bb)) + (b - bb) # TwoSum: s + err = a + b exactly + with np.errstate(invalid="ignore"): + s2 = np.where((err != 0) & np.isfinite(s), np.nextafter(s, s + np.sign(err) * np.inf), s) + with np.errstate(over="ignore", invalid="ignore"): + return s2.astype(out_dtype).astype(np.float64) + + +def _h(v): + with np.errstate(over="ignore", invalid="ignore"): + return np.asarray(v, np.float64).astype(np.float16).astype(np.float64) + + +def eps_model(u, v, kA, kB, kC): + """C1, the machine-code form (float64 in, FP16 values out).""" + with np.errstate(over="ignore", invalid="ignore"): + uu, vv, uv = _h(u * u), _h(v * v), _h(u * v) + ta = _h(uu * kA) + s = _round_once(vv * kB, ta, np.float16) + return _round_once(uv * kC, s, np.float16) + + +def eps_item3(u, v, kA, kB, kC): + """item 3's form (Model L's kernel): for the report only.""" + with np.errstate(over="ignore", invalid="ignore"): + uu, vv, uv = _h(u * u), _h(v * v), _h(u * v) + s = _h(_h(uu * kA) + _h(vv * kB)) + return _round_once(uv * kC, s, np.float16) + + +def cr_model(f): + """C2 (float32 values in [0, 1), float32 values out as float64) -> [n, 4].""" + F = np.float32 + f = np.asarray(f, F) + f2 = (f * f).astype(F) + f3 = (f * f2).astype(F) + t = (-(f + f3).astype(F)).astype(np.float64) + w0 = _round_once(t * 0.5, f2.astype(np.float64), F).astype(F) + r = (F(2.5) * f2).astype(F) + w1 = (_round_once(f3.astype(np.float64) * 1.5, -r.astype(np.float64), F).astype(F) + F(1.0)).astype(F) + w3 = ((f3 - f2).astype(F) * F(0.5)).astype(F) + w2 = (((F(1.0) - w0).astype(F) - w1).astype(F) - w3).astype(F) + return np.stack([w0, w1, w2, w3], -1).astype(np.float64) + + +def eps_inputs(rng, n): + """[n, 5] FP16 values (u, v, A, B, C): four structured groups of n / 5 and n / 5 wide random bit patterns.""" + k = n // 5 + + def kern(m): # offsets as the kernel forms them: h(h(d) + i), d in (-1, 1), i = -1 ... 2 + return _h(_h(rng.uniform(-1.0, 1.0, size=m)) + rng.integers(-1, 3, size=m)) + + def coef(m): # spec 06 ranges: A, B <= 0, C of either sign + return (_h(_h(rng.uniform(0.0, 40.0, size=m)) * -0.721325), _h(_h(rng.uniform(0.0, 40.0, size=m)) * -0.721325), + _h(_h(rng.uniform(-40.0, 40.0, size=m)) * -1.44265)) + + rnd = lambda m: _h(rng.uniform(-2.5, 2.5, size=m)) # noqa: E731 + u = np.concatenate([kern(k), rnd(k), kern(k), rnd(k)]) + v = np.concatenate([kern(k), rnd(k), rnd(k), kern(k)]) + cA, cB, cC = (np.asarray(t) for t in coef(4 * k)) + m = n - 4 * k + bits = rng.integers(0, 1 << 16, size=(5, m), dtype=np.uint64).astype(np.uint16) + wide = bits.view(np.float16).astype(np.float64) + wide[:2] = np.where(np.abs(wide[:2]) < 4.0, wide[:2], _h(rng.uniform(-2.5, 2.5, size=wide[:2].shape))) + x = np.concatenate([np.stack([u, v, cA, cB, cC], -1), wide.T]) + edge = np.array([[0.0, 0.0, -0.0, 0.0, 0.0], [2.5, -2.5, -65504.0, -65504.0, 65504.0], + [6.1e-5, 6.1e-5, -1.0, -1.0, 1.0], [1.0, 1.0, -2.0 ** -24, -2.0 ** -24, 2.0 ** -24], + [1.0, 2.0, -np.inf, -1.0, 1.0], [2.0, 1.0, -1.0, -1.0, np.inf], [1.0, 1.0, np.nan, -1.0, 1.0], + [2.0, 2.0, -40000.0, -40000.0, 0.0], [0.5, -0.5, -0.0, -0.0, -0.0]]) + return _h(np.concatenate([edge, x])[:n]) + + +def cr_inputs(rng, n): + F = np.float32 + k = n // 3 + a = rng.uniform(0.0, 1.0, size=k).astype(F) + small = (2.0 ** rng.uniform(-25.0, -1.0, size=k)).astype(F) + top = (F(1.0) - (rng.integers(1, 1 << 20, size=n - 2 * k) * F(2.0 ** -24))).astype(F) + fixed = np.array([0.0, 2.0 ** -25, 0.5, 0.25, 0.75, 1.0 - 2.0 ** -24], F) + return np.concatenate([fixed, a, small, top])[:n].astype(F) + + +def same(got, ref): + g, r = np.asarray(got, np.float64), np.asarray(ref, np.float64) + eq = (g == r) & (np.signbit(g) == np.signbit(r)) + return eq | (np.isnan(g) & np.isnan(r)) + + +def run_probe(hip, mod, ein, fin): + n = ein.shape[0] + xb = ein.astype(np.float16).view(np.uint16).tobytes() + pi, po = hip.malloc(len(xb)), hip.malloc(2 * n) + hip.upload(pi, xb) + hip.launch(hip.get_function(mod, "l_probe_eps"), -(-n // 256), 256, + [ctypes.c_void_p(pi), ctypes.c_void_p(po), ctypes.c_int(n)]) + hip.sync() + eps = np.frombuffer(hip.download(po, 2 * n), np.uint16).view(np.float16).astype(np.float64) + hip.free(pi) + hip.free(po) + m = fin.size + fb = fin.astype(np.float32).tobytes() + pf, pw = hip.malloc(len(fb)), hip.malloc(16 * m) + hip.upload(pf, fb) + hip.launch(hip.get_function(mod, "l_probe_cr"), -(-m // 256), 256, + [ctypes.c_void_p(pf), ctypes.c_void_p(pw), ctypes.c_int(m)]) + hip.sync() + w = np.frombuffer(hip.download(pw, 16 * m), np.float32).reshape(m, 4).astype(np.float64) + hip.free(pf) + hip.free(pw) + return eps, w + + +def run(build, rocm, n_eps=4_000_000, n_cr=1_000_000, seed=11): + from clean.runner.hip import Hip + hip = Hip(rocm) + rng = np.random.default_rng(seed) + ein, fin = eps_inputs(rng, n_eps), cr_inputs(rng, n_cr) + e_ref = eps_model(*ein.T) + w_ref = cr_model(fin) + e_i3 = eps_item3(*ein.T) + res = {"inputs": {"eps": int(ein.shape[0]), "cr": int(fin.size), "seed": seed}, + "model_item3_vs_machine_differ_frac": float(np.mean(~same(e_i3, e_ref)))} + for tag, co in (("probe", "l_post_c12_probe.co"), ("mutant", "l_post_c12_probe__mut.co")): + with open(os.path.join(build, co), "rb") as f: + mod = hip.load_module(f.read()) + eps, w = run_probe(hip, mod, ein, fin) + de = ~same(eps, e_ref) + dw = ~same(w, w_ref).all(-1) + r = {"eps_differ": int(de.sum()), "cr_differ": int(dw.sum())} + if de.any(): + j = int(np.argmax(de)) + r["eps_first"] = {"in": ein[j].tolist(), "got": float(eps[j]), "model": float(e_ref[j])} + if dw.any(): + j = int(np.argmax(dw)) + r["cr_first"] = {"f": float(fin[j]), "got": w[j].tolist(), "model": w_ref[j].tolist()} + res[tag] = r + res["probe"]["pass"] = res["probe"]["eps_differ"] == 0 and res["probe"]["cr_differ"] == 0 + res["mutant"]["caught"] = res["mutant"]["eps_differ"] > 0 and res["mutant"]["cr_differ"] > 0 + res["pass"] = res["probe"]["pass"] and res["mutant"]["caught"] + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--json", required=True) + a = ap.parse_args(argv) + res = run(a.build, a.rocm) + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + print(json.dumps(res)) + return 0 if res["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_post_check.py b/clean/l/runner/l_post_check.py index ef68a443..9e394c9d 100644 --- a/clean/l/runner/l_post_check.py +++ b/clean/l/runner/l_post_check.py @@ -30,6 +30,11 @@ Known answers (C = (65504, 65504, 0), D = R, every target offset from a texel centre by the jitter): offset 0.1, total weight 0 everywhere: history -1.4404296875, display 0, reduced code 0; offset 0.05, total weight > 0 but every FP16 weight 0: the output is the candidate history (here the reset fallback: the linear colour times e). +--form morig (Q11 part C): l_post__morig.co (Model M's ORIGINAL reconstruction: 3 x 3 window, encoded-domain +combination and fallback, C1 machine form, C2) against the oracle table M_RECON_ORIG, the same criteria and negative +control; offset 0.05's known answer is M's fallback through one decode, h(E((D(E(e_prev c)) / e_prev) e)), within one +FP16 step. Also reported (not a criterion): the values that differ from the oracle with C1 in item 3's form, which +must be more than with the machine form (the compiled kernel uses C1; the bit-exact check is l_post_c12_check.py). """ import argparse import ctypes @@ -44,6 +49,7 @@ sys.path.insert(0, ROOT) from clean.l.reference import recon as R # noqa: E402 +from clean.l.reference import table as T # noqa: E402 from clean.l.runner import l_c32_check as C32 # noqa: E402 from clean.reference import post as MP # noqa: E402 from clean.runner import post as RP # noqa: E402 @@ -174,13 +180,31 @@ def m_rule(got, ref): return res -def check_case(hip, mod, x, kind, s=S, ldr=False): +FORMS = {"l": ("l_post.co", None), "morig": ("l_post__morig.co", {"recon": T.M_RECON_ORIG}), + # Model L's own reconstruction with C1 in the machine-code form and C2 (WG_L_POST=c12, Q11 part C) + "lc12": ("l_post__lc12.co", {"recon": dict(T.TABLE["recon"], c1="machine")})} + + +def _ref(x, s, ldr, table): + return R.reconstruct(x, s, ldr=ldr) if table is None else R.reconstruct(x, s, table, ldr=ldr) + + +def check_case(hip, mod, x, kind, s=S, ldr=False, table=None): raw, sizes, dims = run_kernel(hip, mod, x, s, "l_post_recon_ldr" if ldr else "l_post_recon") got = decode(raw, sizes, dims) - ref = R.reconstruct(x, s, ldr=ldr) + ref = _ref(x, s, ldr, table) + if table is not None and table["recon"].get("c1") == "machine": + alt = _ref(x, s, ldr, {"recon": dict(table["recon"], c1="item3")}) + fin = np.isfinite(got["history"]) + m_alt = {"history_differ_machine_c1": int(np.sum(fin & (got["history"] != ref["history"]))), + "history_differ_item3_c1": int(np.sum(fin & (got["history"] != alt["history"])))} + else: + m_alt = None m = m_rule(got, ref) m.update({"alpha_zero": got["alpha_zero"], "reduced_top_zero": got["reduced_top_zero"], "slack_untouched": got["slack_untouched"]}) + if m_alt is not None: + m["c1_report"] = m_alt ok = m["pass"] and got["alpha_zero"] and got["reduced_top_zero"] and got["slack_untouched"] W = ref["_lin"]["W"] m["weight_zero_pixels"] = int((W == 0).sum()) @@ -206,26 +230,35 @@ def check_case(hip, mod, x, kind, s=S, ldr=False): if kind == "offset_0.05": if ldr: # LDR: the linear colour as it is (no exposure, identity transfer) want = MP.f16(np.asarray(x["color"], np.float64)) + elif table is not None and table["recon"]["fallback"] != "linear_colour": # M: encoded fallback, one decode + c = np.asarray(x["color"], np.float64) + ep, ec = float(np.asarray(x["exposure_prev"])), float(np.asarray(x["exposure"])) + want = MP.f16(MP.enc(R.dec(MP.enc(ep * c)) / ep * ec)) else: want = MP.f16(MP.enc(np.asarray(x["color"], np.float64) * float(np.asarray(x["exposure"])))) - known = bool((W > 0).all()) and np.array_equal(got["history"], want, equal_nan=True) + if table is not None and table["recon"]["fallback"] != "linear_colour": + known = bool((W > 0).all()) and int(C32.f16_steps(got["history"], want).max()) <= 1 + else: + known = bool((W > 0).all()) and np.array_equal(got["history"], want, equal_nan=True) m["known_answer"] = bool(known) ok = ok and known m["pass"] = bool(ok) return m -def run(build, rocm, seeds=(89,), ldr=False): +def run(build, rocm, seeds=(89,), ldr=False, form="l"): from clean.runner.hip import Hip hip = Hip(rocm) - with open(os.path.join(build, "l_post.co"), "rb") as f: + co, table = FORMS[form] + with open(os.path.join(build, co), "rb") as f: mod = hip.load_module(f.read()) - res = {"cases": [], "kernel": "l_post_recon_ldr" if ldr else "l_post_recon", "ldr": ldr} + res = {"cases": [], "kernel": "l_post_recon_ldr" if ldr else "l_post_recon", "ldr": ldr, "form": form, + "code_object": co} for seed in seeds: rng = np.random.default_rng(seed) for rw, rh, dw, dh, reset, kind in CASES: x = make_case(rng, rw, rh, dw, dh, reset, kind) - m = check_case(hip, mod, x, kind, ldr=ldr) + m = check_case(hip, mod, x, kind, ldr=ldr, table=table) m["case"] = [rw, rh, dw, dh, reset, kind] res["cases"].append(m) # Negative control: the first random case with the jitter moved by half a render pixel must fail. @@ -233,11 +266,14 @@ def run(build, rocm, seeds=(89,), ldr=False): x = make_case(rng, *CASES[0]) raw, sizes, dims = run_kernel(hip, mod, dict(x, jitter=np.asarray(x["jitter"], np.float32) + 0.5), S, "l_post_recon_ldr" if ldr else "l_post_recon") - neg = m_rule(decode(raw, sizes, dims), R.reconstruct(x, S, ldr=ldr)) + neg = m_rule(decode(raw, sizes, dims), _ref(x, S, ldr, table)) res["negative_control_fails"] = not neg["pass"] res["run"] = {k: l_post_run([c[k] for c in res["cases"]]) for k in ("history", "display")} res["pass"] = (all(c["pass"] for c in res["cases"]) and res["negative_control_fails"] and all(r["pass"] for r in res["run"].values())) + rep = [c["c1_report"] for c in res["cases"] if "c1_report" in c] + if rep: + res["c1_report"] = {k: sum(r[k] for r in rep) for k in rep[0]} return res @@ -247,15 +283,21 @@ def main(argv=None): ap.add_argument("--rocm", required=True) ap.add_argument("--seeds", type=int, default=1, help="run the case list with this many seeds (89, 90, ...)") ap.add_argument("--ldr", action="store_true", help="the LDR form l_post_recon_ldr against the LDR oracle") + ap.add_argument("--form", choices=sorted(FORMS), default="l", + help="morig: l_post__morig.co (Model M's original reconstruction); lc12: l_post__lc12.co (Model L's with " + "C1 machine form + C2); Q11 part C") ap.add_argument("--json", required=True) a = ap.parse_args(argv) - res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds)), ldr=a.ldr) + if a.form == "morig" and a.ldr: + ap.error("--form morig: HDR only") + res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds)), ldr=a.ldr, form=a.form) with open(a.json, "w") as f: json.dump(res, f, indent=1) for c in res["cases"]: print(json.dumps({k: c[k] for k in c if k not in ("history", "display", "reduced")}), {k: (c[k]["differ"], c[k].get("max_steps", c[k].get("max"))) for k in ("history", "display", "reduced")}) - print({"run": res["run"], "negative_control_fails": res["negative_control_fails"], "pass": res["pass"]}) + print({"run": res["run"], "negative_control_fails": res["negative_control_fails"], "pass": res["pass"], + "c1_report": res.get("c1_report")}) return 0 if res["pass"] else 1 diff --git a/clean/l/runner/l_q7_check.py b/clean/l/runner/l_q7_check.py new file mode 100644 index 00000000..854ff1df --- /dev/null +++ b/clean/l/runner/l_q7_check.py @@ -0,0 +1,454 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): GPU check of the FP16-form width-32 block kernels (clean/l/kernels/l_q7.hip) against the oracle +(clean/l/reference/oracle.py swin_block with the decoder / encoder block constants, form "q7", FP8 op model "exact"), +byte for byte. + + python clean/l/runner/l_q7_check.py --build --compile # CPU: /l_q7.co + python clean/l/runner/l_q7_check.py --build --prepare [--real ] # CPU + python clean/l/runner/l_q7_check.py --build --prepare-d3d12 # CPU: D3D12 ports + kernel_replay.exe + python clean/l/runner/l_q7_check.py --build --run [--d3d12] --json out.json # GPU (inside a GPU slot) + +Cases: grid sizes (multiples of 4) x both window offsets of the block's frame parities x input scales; synthetic block +weights (several seeds) and, with --real, the real model's enc0_block / dec0_block. Encoder kernel: input the FP16 halo +map [H + 4][W + 4][32], output the FP8 map. Decoder kernel: input the FP8 projection map and the FP8 skip map, output +the FP16 map. Every output buffer is filled with the sentinel 0xA5 and has 64 slack bytes after it, which must keep the +sentinel; the output must equal the oracle's bytes exactly. Special values: inputs with the largest FP16 / FP8 +magnitudes, zeros and a NaN member. Negative control: the same launch with half the workgroups must differ. +With --d3d12 the same cases run on the native D3D12 device (the ports in clean/dx12/kernels/l_q7.hlsl, kernel_replay, +output buffers filled with the D3D12 sentinel 0x5A): raw bytes identical to the HIP run's and to the oracle's, slack +kept, and its own half-grid control. +Output: counts and digests only. +--form hyb / hybmut (Q11 part C, Model L's HYBRID evaluation form; HIP only): l_q7__hyb.co (l_q7.hip Q7_HYB: float32 +chains, one FP16 rounding each) against the oracle with FP8 op model "f32chain", raw bytes, the same cases (synthetic +weights) and half-grid control; negative control: the exact (q7) oracle's bytes must differ in some case. hybmut: +l_q7__hybmut.co (W2 rounded after every slice) must differ from the f32chain oracle in some case (the mutant caught). +Prepare once with --form hyb (cases file l_q7_cases_hyb.pkl, used by both). +""" +import copy +import argparse +import ctypes +import hashlib +import json +import os +import pickle +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import oracle as L # noqa: E402 +from clean.l.reference import table as T # noqa: E402 +from clean.l.runner import l_pack as LP # noqa: E402 +from clean.l.runner import l_record as LR # noqa: E402 +from clean.reference.fp8 import f16, fp8_encode, q8 # noqa: E402 + +SENTINEL, D3D12_SENTINEL, SLACK = 0xA5, 0x5A, 64 +KERNELS = ("l_q7_block_c32_enc", "l_q7_block_c32_dec") +GRIDS = [(8, 12), (16, 8), (12, 20), (32, 32)] +SCALES = (0.5, 3.0, 40.0) + + +def _sha256(path): + with open(path, "rb") as f: + return hashlib.sha256(f.read()).hexdigest() + + +def run_identity(build, files): + """At run time: this checkout's git commit (git rev-parse HEAD, run here), whether its tracked files differ from + it, and the SHA-256 of the given files in build.""" + import subprocess + + def git(*a): + r = subprocess.run(["git", "-C", ROOT] + list(a), capture_output=True, text=True, + creationflags=0x08000000 if os.name == "nt" else 0) + if r.returncode != 0 or (a[0] == "rev-parse" and not r.stdout.strip()): + raise RuntimeError(f"run identity: git {a[0]} failed ({r.returncode}): {r.stderr.strip()}") + return r.stdout.strip() + ident = {"git_commit": git("rev-parse", "HEAD"), + "git_tracked_changes": bool(git("status", "--porcelain", "--untracked-files=no"))} + for f in files: + ident[f.replace(".", "_") + "_sha256"] = _sha256(os.path.join(build, f)) + return ident + + +def compile_kernel(build, rocm): + from clean.runner import build as B + from clean.runner.hip import clean_process_environment + clean_process_environment(rocm) + clang = B.find_clang(rocm) + os.makedirs(build, exist_ok=True) + B.run(B.device_command(clang, rocm, os.path.join(ROOT, "clean", "l", "kernels", "l_q7.hip"), + os.path.join(build, "l_q7.co")) + ["-I", os.path.join(ROOT, "clean", "kernels")]) + + +class Gpu: + def __init__(self, build, rocm, co="l_q7.co"): + from clean.runner.hip import Hip + self.h = Hip(rocm) + with open(os.path.join(build, co), "rb") as f: + self.mod = self.h.load_module(f.read()) + tab = LP.q7_tables() + self.tab = self.h.malloc(len(tab)) + self.h.upload(self.tab, tab) + + def launch(self, kernel, xb, sb, out_bytes, blob, H, W, off, half=False): + h = self.h + bufs = [h.malloc(len(xb)), h.malloc(max(len(sb), 1)), h.malloc(out_bytes + SLACK), h.malloc(len(blob))] + px, ps, po, pw = bufs + h.upload(px, xb) + if sb: + h.upload(ps, sb) + h.upload(pw, blob) + h.memset(po, SENTINEL, out_bytes + SLACK) + ox, oy = off + nwin = -(-(W + ox) // 4) * -(-(H + oy) // 4) + args = [ctypes.c_void_p(px), ctypes.c_void_p(ps if sb else 0), ctypes.c_void_p(po), ctypes.c_void_p(pw), + ctypes.c_void_p(self.tab), ctypes.c_int(H), ctypes.c_int(W), ctypes.c_int(ox), ctypes.c_int(oy)] + h.launch(h.get_function(self.mod, kernel), max(1, nwin // 2) if half else nwin, 32, args) + h.sync() + out = h.download(po, out_bytes + SLACK) + for p in bufs: + h.free(p) + return out + + +def out8(v): + """The kernels' output bytes of FP8 values: a NaN as the positive quiet NaN code 0x7f.""" + b = fp8_encode(q8(v)) + return np.where((b & 0x7F) == 0x7F, np.uint8(0x7F), b).astype(np.uint8).tobytes() + + +def out16(v): + """The kernels' output bytes of FP16 values: a NaN as the positive quiet NaN 0x7e00.""" + h = f16(v).astype(np.float16).view(np.uint16) + return np.where(np.isnan(f16(v)), np.uint16(0x7E00), h).astype(np.uint16).tobytes() + + +FORM_CO = {"q7": "l_q7.co", "hyb": "l_q7__hyb.co", "hybmut": "l_q7__hybmut.co"} + + +def form_table(form): + t = copy.deepcopy(T.TABLE) + if form in ("hyb", "hybmut"): + t["mm8"] = "f32chain" + for name in t["consts"]: + t["consts"][name] = dict(t["consts"][name]) + return t + + +def cases_file(build, form): + return os.path.join(build, "l_q7_cases.pkl" if form == "q7" else "l_q7_cases_hyb.pkl") + + +def _k(t, name): + k = T.block_consts(t, name) + k["mm8"] = t.get("mm8", "exact") + return k + + +def enc_case(p, x_halo, off, t=T.TABLE): + """Oracle: q8 of the encoder block on the halo map (border "input").""" + k = _k(t, "enc0_block") + y = L.swin_block(x_halo, p, off, 2, k, "input", 2) + return out8(y) + + +def dec_case(p, P, S, off, t=T.TABLE): + """Oracle: the decoder block on the projection map (border "coarse") with the skip, FP16 output.""" + k = _k(t, "dec0_block") + y = L.swin_block(P, p, off, 2, k, "coarse", skip=S) + return out16(y) + + +def inputs(rng, H, W, scale, special): + x = f16(rng.normal(0, scale, (H + 4, W + 4, 32))) + P = q8(rng.normal(0, scale, (H, W, 32))) + S = q8(rng.normal(0, scale, (H, W, 32))) + if special: + x[0, 0, :4] = [65504.0, -65504.0, 0.0, -0.0] + x[3, 5, 7] = np.nan + P[1, 1, :3] = [448.0, -448.0, 0.0] + S[1, 2, :2] = [448.0, 448.0] + P[2, 3, 4] = np.nan + return x, P, S + + +def mask_nan(body, kind): + """Output bytes with only a NaN's sign bit cleared (the accepted port exception: the D3D12 driver generates NaNs + with the sign set, HIP without; a canonical NaN in the kernels would cost VALU per store). kind "fp8" or "fp16".""" + if kind == "fp8": + b = np.frombuffer(body, np.uint8) + return np.where((b & 0x7F) == 0x7F, np.uint8(0x7F), b).astype(np.uint8).tobytes() + h = np.frombuffer(body, np.uint16) + nan = ((h & 0x7C00) == 0x7C00) & ((h & 0x3FF) != 0) + return np.where(nan, h & 0x7FFF, h).astype(np.uint16).tobytes() + + +def mask_control(): + """Proof that mask_nan clears ONLY a NaN's sign bit: a NaN with the sign flipped compares equal; a non-NaN value + with the sign flipped, a NaN payload change and a quiet-bit change (FP16) still differ. Every check records it.""" + f8 = lambda a, b: mask_nan(bytes(a), "fp8") == mask_nan(bytes(b), "fp8") # noqa: E731 + h16 = lambda a, b: (mask_nan(np.array(a, np.uint16).tobytes(), "fp16") # noqa: E731 + == mask_nan(np.array(b, np.uint16).tobytes(), "fp16")) + r = {"fp8_nan_sign_equal": f8([0x7F], [0xFF]), + "fp8_nonnan_sign_differs": all(not f8([c], [c ^ 0x80]) for c in range(256) if (c & 0x7F) != 0x7F), + "fp16_nan_sign_equal": h16([0x7E00, 0x7C01], [0xFE00, 0xFC01]), + "fp16_nonnan_sign_differs": all(not h16([h], [h ^ 0x8000]) for h in range(0x8000) + if not ((h & 0x7C00) == 0x7C00 and (h & 0x3FF))), + "fp16_nan_payload_differs": not h16([0x7E00], [0x7E01]) and not h16([0x7E01], [0xFE02]), + "fp16_nan_quiet_bit_differs": not h16([0x7E01], [0x7C01])} + r["pass"] = all(r.values()) + return r + + +def control_blobs(blob): + """(FP8 rows zeroed, every sum |w| table +inf, +inf tables and FP16 rows zeroed) variants of a C32 q7 blob.""" + o = LP.q7_block_offsets(32, 2, 128) + z8 = blob[:o["Wqk"]] + bytes(o["end8"] - o["Wqk"]) + blob[o["end8"]:] + inf = np.full((o["end"] - o["end16"]) // 4, np.inf, np.float32).tobytes() + allfb = blob[:o["end16"]] + inf + blob[o["end"]:] + allfb_z16 = blob[:o["end8"]] + bytes(o["end16"] - o["end8"]) + inf + blob[o["end"]:] + return z8, allfb, allfb_z16 + + +def section_controls(g, c, H, W): + z8, allfb, allfb_z16 = control_blobs(c["blob"]) + run = lambda b: mask_nan(g.launch("l_q7_block_c32_enc", c["xb"], c["sb"], c["nbytes"], b, H, W, # noqa: E731 + tuple(c["offset"]))[:c["nbytes"]], "fp8") + return {"fp8_section_zeroed_differs": run(z8) != c["want"], + "all_fallback_identical": run(allfb) == c["want"], + "all_fallback_fp16_zeroed_differs": run(allfb_z16) != c["want"]} + + +def compare(got, want, out_bytes, kind="fp8"): + body, slack = mask_nan(got[:out_bytes], kind), got[out_bytes:] + return {"identical": body == want, "slack_kept": slack == bytes([SENTINEL]) * SLACK, + "bytes_differ": sum(a != b for a, b in zip(body, want))} + + +def prepare(build, real=None, form="q7"): + """CPU: every case's inputs and the oracle's bytes, saved to /l_q7_cases.pkl (the GPU run only launches).""" + blocks = [("synthetic_%d" % s, {k: np.asarray(v, np.float64) for k, v in + L.random_block_params(np.random.default_rng(s), 32, 2, 128).items()}) + for s in (89, 90)] + real_sha = None + if real: + real_sha = _sha256(os.path.join(real, "lframe_model", "params.pkl")) + params = pickle.load(open(os.path.join(real, "lframe_model", "params.pkl"), "rb"))["L"] + blocks += [("real_enc0_block", params["enc0_block"]), ("real_dec0_block", params["dec0_block"])] + rng = np.random.default_rng(7) + cases = [] + for bname, p in blocks: + blob = LP.pack_q7_block(p, 32, 2, 128) + for (H, W) in GRIDS: + for scale in SCALES: + special = scale == SCALES[1] and (H, W) == GRIDS[0] + x, P, S = inputs(rng, H, W, scale, special) + for kind, offs in (("enc", T.TABLE["window_offsets"]["first"]), + ("dec", T.TABLE["window_offsets"]["second"])): + for off in offs: + tf = form_table(form) + if kind == "enc": + xb, sb, nbytes = x.astype(np.float16).tobytes(), b"", H * W * 32 + want = enc_case(p, x, off, tf) + want_exact = enc_case(p, x, off) if form != "q7" else None + else: + xb, sb, nbytes = fp8_encode(P).tobytes(), fp8_encode(S).tobytes(), H * W * 64 + want = dec_case(p, P, S, off, tf) + want_exact = dec_case(p, P, S, off) if form != "q7" else None + cases.append({"block": bname, "kernel": kind, "grid": [H, W], "offset": list(off), + "scale": scale, "special": special, "xb": xb, "sb": sb, "nbytes": nbytes, + "blob": blob, "want": want, "want_exact": want_exact}) + print(bname, H, W, scale, "prepared", flush=True) + with open(cases_file(build, form), "wb") as f: + pickle.dump({"cases": cases, "real_params_sha256": real_sha, "form": form}, f) + return len(cases) + + +def run(build, rocm, d3d12=False, form="q7"): + """GPU: launch every prepared case and compare with the oracle's bytes (with d3d12, also the D3D12 ports).""" + with open(cases_file(build, form), "rb") as f: + prep = pickle.load(f) + cases = prep["cases"] + co = FORM_CO[form] + g = Gpu(build, rocm, co) + res = {"what": "FP16-form width-32 block kernels against the oracle, raw bytes", "cases": [], "form": form, + "fp8_op_model": "exact" if form == "q7" else "f32chain", + "identity": run_identity(build, [co, os.path.basename(cases_file(build, form))]), + "real_params_sha256": prep["real_params_sha256"], + "nan_sign_masked": "outputs compared and hashed with only a NaN's sign bit cleared (accepted port exception; " + "a canonical NaN store costs about 3 VALU per stored dword in the RGA ISA)", + "nan_mask_control": mask_control()} + for c in cases: + H, W = c["grid"] + got = g.launch("l_q7_block_c32_" + c["kernel"], c["xb"], c["sb"], c["nbytes"], c["blob"], H, W, + tuple(c["offset"])) + kind = "fp8" if c["kernel"] == "enc" else "fp16" + r = compare(got, c["want"], c["nbytes"], kind) + r.update({k: c[k] for k in ("block", "kernel", "grid", "offset", "scale", "special")}) + r["output_sha256"] = hashlib.sha256(mask_nan(got[:c["nbytes"]], kind)).hexdigest() + if c.get("want_exact") is not None: + r["differs_from_exact_oracle"] = mask_nan(got[:c["nbytes"]], kind) != c["want_exact"] + res["cases"].append(r) + if "negative_control" not in res and c["kernel"] == "enc" and tuple(c["grid"]) == GRIDS[2]: + half = g.launch("l_q7_block_c32_enc", c["xb"], c["sb"], c["nbytes"], c["blob"], H, W, + tuple(c["offset"]), half=True) + res["negative_control"] = {"case": "%s enc %dx%d" % (c["block"], H, W), + "half_grid_differs": half[:c["nbytes"]] != c["want"]} + # Q7F-2 section controls: the matrix-instruction path reads the FP8 rows (zeroed: the output differs); with + # every sum |w| table set to +inf each output takes the exact fallback (FP16 rows), and the output must stay + # identical; with the FP16 rows zeroed as well that forced fallback must differ + if form == "q7": # the matrix-instruction sections exist in the q7 build only + res["section_control"] = section_controls(g, c, H, W) + res["cases_identical"] = sum(r["identical"] and r["slack_kept"] for r in res["cases"]) + if form == "hybmut": # the mutant must be caught: some case differs from the f32chain oracle + res["mutant_cases_differing"] = len(res["cases"]) - res["cases_identical"] + res["pass"] = res["mutant_cases_differing"] > 0 + return res + res["pass"] = (res["cases_identical"] == len(res["cases"]) and res["negative_control"]["half_grid_differs"] + and res["nan_mask_control"]["pass"] and all(res.get("section_control", {}).values())) + if form == "hyb": # negative control: the exact (q7) oracle's bytes differ somewhere + res["cases_differing_from_exact_oracle"] = sum(bool(r["differs_from_exact_oracle"]) for r in res["cases"]) + res["pass"] = res["pass"] and res["cases_differing_from_exact_oracle"] > 0 + if d3d12: + res["d3d12"] = run_d3d12(build, cases, res) + res["pass"] = res["pass"] and res["d3d12"]["pass"] + return res + + +def prepare_d3d12(build, rocm): + """CPU: the D3D12 ports (clean/dx12/kernels/l_q7.hlsl, build_dx12.py --only) and kernel_replay.exe into build.""" + from clean.l.runner.l_d3d12_check import prepare_replay + return prepare_replay(build, rocm, list(KERNELS)) + + +def run_d3d12(build, cases, res): + """GPU: every prepared case on the native D3D12 device (kernel_replay, a child process), output buffers filled with + the D3D12 sentinel 0x5A; raw bytes against the HIP run's and the oracle's, and the half-grid control.""" + import subprocess + import time + cdir = os.path.join(build, "replay-q7-" + time.strftime("%Y%m%d-%H%M%S")) + os.makedirs(cdir) + with open(os.path.join(cdir, "tab.bin"), "wb") as f: + f.write(LP.q7_tables()) + blobs, lines, neg = {}, [], None + for i, c in enumerate(cases): + name = "c%03d" % i + if c["block"] not in blobs: + blobs[c["block"]] = "blob_%s.bin" % c["block"] + with open(os.path.join(cdir, blobs[c["block"]]), "wb") as f: + f.write(c["blob"]) + with open(os.path.join(cdir, name + ".x"), "wb") as f: + f.write(c["xb"]) + if c["sb"]: + with open(os.path.join(cdir, name + ".s"), "wb") as f: + f.write(c["sb"]) + H, W = c["grid"] + ox, oy = c["offset"] + nwin = -(-(W + ox) // 4) * -(-(H + oy) // 4) + runs = [(name, nwin, blobs[c["block"]])] + if neg is None and c["kernel"] == "enc" and tuple(c["grid"]) == GRIDS[2]: + neg = i + runs.append(("negative_" + name, max(1, nwin // 2), blobs[c["block"]])) + for tag, vb in zip(("z8", "allfb", "allfbz16"), control_blobs(c["blob"])): + with open(os.path.join(cdir, "blob_%s.bin" % tag), "wb") as f: + f.write(vb) + runs.append(("%s_%s" % (tag, name), nwin, "blob_%s.bin" % tag)) + for cname, grid, bname in runs: + lines += ["case " + cname, "module l_q7.co", "kernel l_q7_block_c32_%s %d 32" % (c["kernel"], grid), + "arg buf %s.x" % name, ("arg buf %s.s" % name) if c["sb"] else "arg null", + "arg out %d %s.o %d" % (c["nbytes"] + SLACK, cname, D3D12_SENTINEL), + "arg buf " + bname, "arg buf tab.bin", + "arg u32 %d" % H, "arg u32 %d" % W, "arg u32 %d" % ox, "arg u32 %d" % oy, ""] + with open(os.path.join(cdir, "cases.txt"), "w", newline="\n") as f: + f.write("\n".join(lines)) + env = {k: v for k, v in os.environ.items() + if not any(s in k.upper() for s in ("HIP", "LLVM", "ROCM", "HSA")) or k.upper() == "HIP_PLATFORM"} + env["WG_D3D12_ARENA_MB"] = "256" + r = subprocess.run([os.path.join(build, "kernel_replay.exe"), build, os.path.join(cdir, "cases.txt")], env=env, + capture_output=True, text=True, timeout=100, creationflags=0x08000000) + + def read(n): + p = os.path.join(cdir, n + ".o") + return open(p, "rb").read() if os.path.exists(p) else b"" + + out = {"replay_exit": r.returncode, "replay_tail": (r.stdout[-300:] + r.stderr[-300:]).strip(), + "identity": run_identity(build, ["kernel_replay.exe", "dx12/l_q7_block_c32_enc.dxil", + "dx12/l_q7_block_c32_dec.dxil"])} + same_hip = same_oracle = 0 + for i, (c, rc) in enumerate(zip(cases, res["cases"])): + got = read("c%03d" % i) + body = mask_nan(got[:c["nbytes"]], "fp8" if c["kernel"] == "enc" else "fp16") if got else b"" + slack = got[c["nbytes"]:] + ok_slack = slack == bytes([D3D12_SENTINEL]) * SLACK + rc["d3d12_identical_to_oracle"] = body == c["want"] and ok_slack + rc["d3d12_output_sha256"] = hashlib.sha256(body).hexdigest() + rc["d3d12_bytes_differ_from_oracle"] = sum(a != b for a, b in zip(body, c["want"])) if body else -1 + same_oracle += rc["d3d12_identical_to_oracle"] + same_hip += rc["d3d12_output_sha256"] == rc["output_sha256"] and ok_slack + c = cases[neg] + half = read("negative_c%03d" % neg)[:c["nbytes"]] + ctl = {t: mask_nan(read("%s_c%03d" % (t, neg))[:c["nbytes"]], "fp8") for t in ("z8", "allfb", "allfbz16")} + out.update({"cases": len(cases), "cases_identical_to_hip": same_hip, "cases_identical_to_oracle": same_oracle, + "negative_control": {"case": "%s enc %dx%d" % (c["block"], *c["grid"]), + "half_grid_differs": half != c["want"]}, + "section_control": {"fp8_section_zeroed_differs": ctl["z8"] != c["want"], + "all_fallback_identical": ctl["allfb"] == c["want"], + "all_fallback_fp16_zeroed_differs": ctl["allfbz16"] != c["want"]}}) + out["pass"] = (r.returncode == 0 and same_hip == len(cases) and same_oracle == len(cases) + and out["negative_control"]["half_grid_differs"] and all(out["section_control"].values())) + return out + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", default=r"C:\Program Files\AMD\ROCm\7.2") + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument("--compile", action="store_true") + g.add_argument("--prepare", action="store_true") + g.add_argument("--prepare-d3d12", action="store_true") + g.add_argument("--run", action="store_true") + ap.add_argument("--d3d12", action="store_true", help="with --run: also the D3D12 ports through kernel_replay") + ap.add_argument("--real") + ap.add_argument("--form", choices=sorted(FORM_CO), default="q7") + ap.add_argument("--json") + a = ap.parse_args(argv) + if a.compile: + compile_kernel(a.build, a.rocm) + return 0 + if a.prepare_d3d12: + print("built", prepare_d3d12(a.build, a.rocm)) + return 0 + if a.prepare: + if a.form == "hybmut": + ap.error("prepare with --form hyb (its cases serve hybmut too)") + print("cases", prepare(a.build, a.real, a.form)) + return 0 + from clean.runner.hip import clean_process_environment + clean_process_environment(a.rocm) + if a.form != "q7" and a.d3d12: + ap.error("--form hyb / hybmut: HIP only") + res = run(a.build, a.rocm, a.d3d12, a.form) + print("form", a.form, "| HIP cases identical", res["cases_identical"], "of", len(res["cases"]), + "| negative control caught", res["negative_control"]["half_grid_differs"], + "| differing from exact oracle", res.get("cases_differing_from_exact_oracle"), + "| mutant cases differing", res.get("mutant_cases_differing")) + if a.d3d12: + d = res["d3d12"] + print("D3D12 replay exit", d["replay_exit"], "| identical to HIP", d["cases_identical_to_hip"], + "| to the oracle", d["cases_identical_to_oracle"], "of", d["cases"], "| negative control caught", + d["negative_control"]["half_grid_differs"]) + if d["replay_exit"]: + print(d["replay_tail"]) + print("PASS" if res["pass"] else "FAIL") + if a.json: + LR.write_l_record(res, a.json) + return 0 if res["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_q7_negative_control.py b/clean/l/runner/l_q7_negative_control.py index 3787647c..e7023749 100644 --- a/clean/l/runner/l_q7_negative_control.py +++ b/clean/l/runner/l_q7_negative_control.py @@ -46,6 +46,7 @@ def model_m_form_table(): t = copy.deepcopy(T.TABLE) t["consts"]["default"]["form"] = "m" t["consts"]["default"]["gelu"] = "sigmoid" + t["shared_form"] = "m" # the shared stages on Model M's oracle, as before their FP16 form for op in t["dec_ops"]: op.pop("skip", None) return t diff --git a/clean/l/runner/l_q7_s2a_bounds.py b/clean/l/runner/l_q7_s2a_bounds.py new file mode 100644 index 00000000..cdb16f33 --- /dev/null +++ b/clean/l/runner/l_q7_s2a_bounds.py @@ -0,0 +1,225 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): candidate S2a acceptance tests on a local recorded run (design study, CPU only; only counts are output). + +The matrix instruction's float32 sum of a 32-wide FP8 chunk is exact when every partial sum is representable. With +every product a multiple of g (the chunk's granularity) and sum |a w| < 2^24 g, every partial sum in any order is a +multiple of g below 2^24 g: exact. The shipped test takes g = 2^-18 (the smallest FP8 x FP8 granularity) and the bound +wsum x amax < 63 (63 = 2^24 x 2^-18 x 63 / 64). Candidates, each an upper bound of sum |a w| (B) against a limit (L): + B: cur = wsum x amax (shipped) h2 = sum over the two instructions' 16-K halves of wsum_h x amax_h + q4 = sum over the four 8-K groups (lane fragments) of wsum_q x amax_q + dual = asum x wmax (asum = sum |a| of the token's chunk, wmax = max |w| of the row's chunk) + min2 = min(h2, dual) min4 = min(q4, dual) true = sum |a w| itself (the best any magnitude bound can do) + L: 63 (g = 2^-18), or "g": 63 x 2^(18 + la + lw), la / lw = the lowest set bit's exponent over the token's / the row's + nonzero codes of the chunk (every product a w is a multiple of 2^(la + lw)). +For every candidate: outputs and wave steps (32 outputs that run together in a block kernel) that would take the +exact fallback. Scores: a = the query, w = the key (l_q7.hip). The study only counts; each test that is adopted gets +its own adversarial probe (l_q7_s2a_check.py) before any kernel uses it. + + python clean/l/runner/l_q7_s2a_bounds.py --build --json out.json +""" +import argparse +import copy +import os +import pickle +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import oracle as L # noqa: E402 +from clean.l.runner import l_record as LR # noqa: E402 +from clean.l.runner.l_q7_s2a_fallbacks import _amax # noqa: E402 + +BOUNDS = ("cur", "h2", "q4", "dual", "min2", "min4", "true") +LIMITS = ("63", "g") + + +def lsb_exp(v): + """Exponent of the lowest set significand bit of FP8 values (exact), +99 for 0, -99 for NaN.""" + x = np.abs(np.asarray(v, np.float64)) + m, e = np.frexp(np.where(np.isfinite(x) & (x > 0), x, 1.0)) + n = np.rint(m * 16).astype(np.int64) # 8 ... 15: 4 significand bits (FP8 values are exact) + tz = np.where(n % 2 == 1, 0, np.where(n % 4 == 2, 1, np.where(n % 8 == 4, 2, 3))) + out = (e - 4 + tz).astype(np.int64) + out = np.where(x == 0, 99, out) + return np.where(np.isnan(x), -99, out) + + +def group_max(x, groups): + return np.stack([np.where(np.isnan(x[..., g]), np.inf, np.abs(x[..., g])).max(-1) for g in groups], -1) + + +H2 = [slice(0, 16), slice(16, 32)] +Q4 = [slice(0, 8), slice(8, 16), slice(16, 24), slice(24, 32)] + + +def criteria(a, W, chunk=2048): + """a [..., 32] (token chunks), W [n, 32] (rows) -> {name: ok [..., n]} (in chunks of tokens: memory).""" + lead = a.shape[:-1] + flat = a.reshape(-1, a.shape[-1]) + parts = [_criteria(flat[i:i + chunk], W) for i in range(0, max(1, flat.shape[0]), chunk)] + return {k: np.concatenate([p[k] for p in parts]).reshape(lead + (W.shape[0],)) for k in parts[0]} + + +def _criteria(a, W): + with np.errstate(invalid="ignore", over="ignore"): + aa, ww = np.abs(a), np.abs(W) + f32 = np.float32 + am = _amax(a) # [...] + ws = ww.sum(-1).astype(f32) # [n] + b = {"cur": am[..., None] * ws} + for name, gr in (("h2", H2), ("q4", Q4)): + amg = group_max(a, gr).astype(f32) # [..., G] + wsg = np.stack([ww[:, g].sum(-1) for g in gr], -1).astype(f32) # [n, G] + b[name] = (amg[..., None, :] * wsg).sum(-1) # float64 here; the kernels' float32 rounding + asum = np.where(np.isnan(a), np.inf, aa).sum(-1).astype(f32) # is covered by the 63 / 64 margin + wmax = np.where(np.isnan(W), np.inf, ww).max(-1).astype(f32) + b["dual"] = asum[..., None] * wmax + b["min2"] = np.minimum(b["h2"], b["dual"]) + b["min4"] = np.minimum(b["q4"], b["dual"]) + b["true"] = np.where(np.isnan(a[..., None, :] * W), np.inf, np.abs(a[..., None, :] * W)).sum(-1) + la = lsb_exp(a).min(-1) # [...] + lw = lsb_exp(W).min(-1) # [n] + lim_g = 63.0 * np.exp2((18 + la[..., None] + lw).clip(-200, 200).astype(np.float64)) + nan = np.isnan(a).any(-1)[..., None] | np.isnan(W).any(-1) + out = {} + for k, v in b.items(): + out[k + "@63"] = (v < 63.0) & ~nan + out[k + "@g"] = (v < lim_g) & ~nan + return out + + +def wave_steps(bad): + if bad.ndim < 2 or bad.shape[-2] != 16 or bad.shape[-1] % 16: + return None + return bad.reshape(bad.shape[:-2] + (16, bad.shape[-1] // 16, 2, 8)).any(axis=(-4, -2)) + + +def study(build): + from clean.l.reference import table as T + from clean.l.runner import l_frame_check as F + rec = pickle.load(open(F.rec_path(build, "L", "stages"), "rb")) + params = pickle.load(open(os.path.join(F.model_dir(build), "params.pkl"), "rb"))["L"] + stats, label = {}, ["?"] + + def add(key, oks, steps_fn): + s = stats.setdefault(label[0] + ":" + key, {}) + for name, ok in oks.items(): + r = s.setdefault(name, [0, 0, 0, 0]) + r[0] += ok.size + r[1] += int((~ok).sum()) + st = steps_fn(~ok) + if st is not None: + r[2] += st.size + r[3] += int(st.sum()) + + op8, score, stage_q7, enc, dec, blk = L._op8, L._score_op, L.run_stage_q7, L.encode, L.decode, L._block_tokens_q7 + + def spy_op8(start, a8, W8, model): + if label[0].endswith("/block") or not label[0].startswith(("enc0", "dec0")): + add("rows%d" % W8.shape[0], criteria(np.asarray(a8, np.float64), np.asarray(W8, np.float64)), wave_steps) + return op8(start, a8, W8, model) + + def score_steps(bad): + if bad.shape[-1] != 16 or bad.shape[-2] != 16: + return None + return bad.reshape(bad.shape[:-1] + (2, 8)).any(axis=(-3, -2)) + + def spy_score(B, u, model): + uu = np.asarray(u, np.float64) # [..., h, T, 32] + oks = criteria_scores(uu) + add("scores", oks, score_steps) + return score(B, u, model) + + def wrap(name, f): + def g(*a, **k): + old = label[0] + label[0] = name(old, *a) if callable(name) else name + try: + return f(*a, **k) + finally: + label[0] = old + return g + + L._op8, L._score_op = spy_op8, spy_score + L.run_stage_q7 = wrap(lambda old, stage, *a: stage, stage_q7) + L.encode, L.decode = wrap("enc0", enc), wrap("dec0", dec) + L._block_tokens_q7 = wrap(lambda old, *a: old + "/block" if old in ("enc0", "dec0") else old, blk) + try: + F.compare_model(rec, params, table=copy.deepcopy(T.TABLE)) + finally: + L._op8, L._score_op, L.run_stage_q7, L.encode, L.decode, L._block_tokens_q7 = \ + op8, score, stage_q7, enc, dec, blk + out = {} + for site, crit in stats.items(): + out[site] = {name: {"outputs": n, "fallback_fraction": f / n if n else 0.0, + "wave_step_fallback_fraction": fs / ns if ns else None} + for name, (n, f, ns, fs) in crit.items()} + return {"sites": out, "frames": len(rec["frames"]), "identity": rec.get("identity")} + + +def criteria_scores(u, chunk=256): + """Scores: for query i and key j, a = u_i (the query's chunk), w = u_j (the key's): {name: ok [..., h, i, j]} + (in chunks of windows x heads: memory).""" + lead = u.shape[:-2] + flat = u.reshape((-1,) + u.shape[-2:]) + parts = [_criteria_scores(flat[i:i + chunk]) for i in range(0, max(1, flat.shape[0]), chunk)] + return {k: np.concatenate([p[k] for p in parts]).reshape(lead + u.shape[-2:-1] * 2) for k in parts[0]} + + +def _criteria_scores(u): + res = {} + a = u[..., :, None, :] # [..., h, T, 1, 32] + W = u[..., None, :, :] # [..., h, 1, T, 32] + with np.errstate(invalid="ignore", over="ignore"): + f32 = np.float32 + nan = np.isnan(a).any(-1) | np.isnan(W).any(-1) + absf = lambda x: np.where(np.isnan(x), np.inf, np.abs(x)) # noqa: E731 + am = absf(a).max(-1).astype(f32) + ws = absf(W).sum(-1).astype(f32) + b = {"cur": am * ws} + for name, gr in (("h2", H2), ("q4", Q4)): + b[name] = sum(absf(a[..., g]).max(-1).astype(f32).astype(np.float64) * + absf(W[..., g]).sum(-1).astype(f32) for g in gr) + b["dual"] = absf(a).sum(-1).astype(f32) * absf(W).max(-1).astype(f32) + b["min2"] = np.minimum(b["h2"], b["dual"]) + b["min4"] = np.minimum(b["q4"], b["dual"]) + b["true"] = absf(a * W).sum(-1) + la, lw = lsb_exp(a).min(-1), lsb_exp(W).min(-1) + lim_g = 63.0 * np.exp2((18 + la + lw).clip(-200, 200).astype(np.float64)) + for k, v in b.items(): + res[k + "@63"] = (v < 63.0) & ~nan + res[k + "@g"] = (v < lim_g) & ~nan + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--json") + ap.add_argument("--ldr", action="store_true", help="the build's LDR run (lframe_L_stages_ldr.pkl)") + a = ap.parse_args(argv) + if a.ldr: + from clean.l.runner import l_frame_check as F + F.LDR = True + res = {"what": "S2a acceptance-test candidates on a local recorded run: outputs / wave steps taking the exact fallback " + "(CPU replay of a recorded l_frame_check run; a design study)", "fp8_op_model": "exact", + "study": study(a.build)} + names = [b + "@" + lim for lim in LIMITS for b in BOUNDS] + print("site".ljust(22) + "".join(n.rjust(10) for n in names), " (wave steps with a fallback, %)") + for site, crit in res["study"]["sites"].items(): + row = [] + for n in names: + w = crit[n]["wave_step_fallback_fraction"] + row.append(("%.2f" % (100 * w)) if w is not None else ("o%.2f" % (100 * crit[n]["fallback_fraction"]))) + print(site.ljust(22) + "".join(x.rjust(10) for x in row)) + if a.json: + LR.write_l_record(res, a.json) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_q7_s2a_check.py b/clean/l/runner/l_q7_s2a_check.py new file mode 100644 index 00000000..adc9d35c --- /dev/null +++ b/clean/l/runner/l_q7_s2a_check.py @@ -0,0 +1,423 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): adversarial probe of the S2a primitive (clean/l/kernels/l_q7_op8.h and its +D3D12 twin clean/dx12/hlsl/l_q7_op8.hlsli): one 32-wide FP8 chunk summed by two matrix instructions started from 0, +used only while wsum x amax < 63 (wsum = sum |w| of the row's chunk, as the packers store it; amax = max |a| over the +SAME 32 values, from the codes), otherwise the exact scalar op8; the FP16 start joined afterwards. + +Every op's result must equal the exact model (oracle MM8 "exact": the exact sum plus the start, rounded once to FP16) +bit for bit (a NaN's sign bit masked: the accepted port exception), its path (bounded / fallback) must equal the bound +computed here in float32, and HIP and D3D12 must agree. Sets: the adversarial families of l_mm8_f32_check (typical, +cancellation, dynamic_range, near_tie, start_midpoint, subnormal) and S2a's own: +- bound_at_63: wsum x amax exactly 63 (must take the fallback); +- bound_below_63: the largest wsum x amax below 63 that FP8 operands can reach (bounded; every such product is a + multiple of 2^-18, so the largest float32 below 63, 63 - 2^-18, is not reachable: recorded); +- unbounded_inexact: wsum x amax in [64, 128) with a sum the float32 result cannot hold (64 + 2^-18), the start + cancelling the 64: wrong if the bound let it through; +- far_half: the chunk's largest |a| in the half of K the output's lane does not load (amax must cover both lanes); +- big_inexact: a product sum of 50176 + 2^-18 with the start -50176; +- nan: NaN activation codes (amax NaN) and NaN weights (wsum NaN), both must take the fallback; +- zeros: signed-zero products, exact cancellations, +-0 starts. +Mutants (probe builds only), each must be caught: 1 the bound limit 128; 2 the first instruction's sum fed as the +second's accumulator; 3 amax from the lane's own 16 values; 4 the start fed into the accumulator. + + python clean/l/runner/l_q7_s2a_check.py --build --compile # HIP probe + mutants (compile slot) + python clean/l/runner/l_q7_s2a_check.py --build --prepare # CPU + python clean/l/runner/l_q7_s2a_check.py --build --prepare-d3d12 # D3D12 probe ports + kernel_replay.exe + python clean/l/runner/l_q7_s2a_check.py --build --run [--d3d12] --json out.json # GPU slot +""" +import argparse +import ctypes +import hashlib +import os +import pickle +import subprocess +import sys +import time + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.runner import l_mm8_f32_check as M # noqa: E402 +from clean.l.runner import l_record as LR # noqa: E402 +from clean.l.runner.l_q7_check import mask_control, run_identity # noqa: E402 +from clean.reference.fp8 import DECODE, f16, fp8_encode, q8 # noqa: E402 + +LIMIT = np.float32(63.0) +MUTANTS = (1, 2, 3, 4) +MUTANT_WHAT = {1: "bound limit 128 instead of 63", 2: "first instruction's sum fed as the second's accumulator", + 3: "amax from the lane's own 16 values only", 4: "start fed into the first instruction's accumulator"} +FP8_POS = np.unique(np.abs(np.asarray(DECODE, np.float64)[np.isfinite(np.asarray(DECODE, np.float64))])) +FP8_POS = FP8_POS[FP8_POS > 0] + + +def _signs(rng, shape): + return rng.choice([-1.0, 1.0], shape) + + +def _pow2_row(target, rng): + """32 FP8 magnitudes summing exactly to target (a multiple of 2^-9, at most 511): its binary digits, padded with + zeros, shuffled.""" + k = int(round(target * 512)) + assert k / 512 == target and 0 < k < 512 * 512 + vals = [2.0 ** (b - 9) for b in range(18) if k >> b & 1] + assert len(vals) <= 32 + row = np.array(vals + [0.0] * (32 - len(vals))) + return row[rng.permutation(32)] + + +def _row_with_max(amax, rng): + """32 FP8 values with max |a| exactly amax, the rest smaller, random signs.""" + small = FP8_POS[FP8_POS <= amax] + row = rng.choice(small, 32) * _signs(rng, 32) + row[rng.integers(32)] = amax * rng.choice([-1.0, 1.0]) + return q8(row) + + +def largest_below_63(): + """The largest wsum x amax below 63 reachable with FP8 operands (wsum a sum of 32 FP8 magnitudes: a multiple of + 2^-9 up to 32 x 448; amax an FP8 value), as (product, wsum, amax), and whether 63 - 2^-18 is reachable.""" + best = (0.0, 0.0, 0.0) + for A in FP8_POS: + k = int(np.floor((63.0 - 2.0 ** -30) / A * 512)) # wsum = k / 512 with k A < 63 x 512 strictly + while k > 0 and (k / 512) * A >= 63.0: + k -= 1 + if k <= 0 or k > 32 * 448 * 512: + continue + p = (k / 512) * A + if p > best[0]: + best = (p, k / 512, float(A)) + return best, best[0] == 63.0 - 2.0 ** -18 + + +def s2a_family(name, rng, n): + st = np.zeros(n) + a = np.zeros((n, 32)) + w = np.zeros((n, 32)) + if name in ("bound_at_63", "bound_below_63"): + if name == "bound_at_63": + pairs = [(A, 63.0 / A) for A in (0.125, 0.25, 0.5, 1.0, 2.0, 4.0, 8.0, 0.75, 1.5, 3.0, 0.875, 1.75, 3.5)] + else: + (p, ws, A), _ = largest_below_63() + pairs = [(A, ws)] + for i in range(n): + A, ws = pairs[i % len(pairs)] + if name == "bound_below_63": + w[i] = _row_any_sum(ws, rng) + else: + w[i] = _pow2_row(ws, rng) * _signs(rng, 32) + a[i] = _row_with_max(A, rng) + st[i] = f16(rng.normal(0, 8)) + elif name == "unbounded_inexact": + for i in range(n): # 8 w + -2^-18: a float32 tie above 64 + k1, k2 = rng.choice(32, 2, replace=False) + a[i, k1], w[i, k1] = 8.0, rng.choice([8.0, 10.0, 12.0, 14.0]) * rng.choice([-1.0, 1.0]) + a[i, k2], w[i, k2] = 2.0 ** -9, 2.0 ** -9 * rng.choice([-1.0, 1.0]) + st[i] = -a[i, k1] * w[i, k1] + elif name == "far_half": + for i in range(n): + hf = (i % 16) >> 3 # the diagonal lane's half (l_q7_s2a_probe) + far = [k for k in range(32) if ((k % 16) >> 3) != hf] + near = [k for k in range(32) if ((k % 16) >> 3) == hf] + kb, kt = rng.choice(far), rng.choice(near) + a[i, kb], w[i, kb] = 8.0, 8.0 + a[i, kt], w[i, kt] = 2.0 ** -9, 2.0 ** -9 + st[i] = -64.0 + elif name == "big_inexact": + for i in range(n): + k1, k2 = rng.choice(32, 2, replace=False) + a[i, k1], w[i, k1] = 224.0, -224.0 if i & 1 else 224.0 + a[i, k2], w[i, k2] = 2.0 ** -9, 2.0 ** -9 + st[i] = -a[i, k1] * w[i, k1] + elif name == "nan": + a = q8(rng.normal(0, 1.5, (n, 32))) + w = q8(rng.normal(0, 0.25, (n, 32))) + st = f16(rng.normal(0, 4, n)) + for i in range(n): + (a if i % 2 == 0 else w)[i, rng.integers(32)] = np.nan + elif name == "zeros": + for i in range(n): + kind = i % 4 + if kind == 0: # all products signed zeros + a[i] = q8(rng.normal(0, 1, 32)) * (rng.random(32) < 0.5) + w[i] = np.where(a[i] == 0, q8(rng.normal(0, 1, 32)), 0.0) * _signs(rng, 32) + a[i] = np.where(a[i] == 0, 0.0 * _signs(rng, 32), a[i]) + elif kind == 1: # exact cancellation in pairs + v = q8(rng.normal(0, 1, 16)) + a[i, :16], w[i, :16] = v, 1.0 + a[i, 16:], w[i, 16:] = -v, 1.0 + else: # -0 products only + a[i] = -q8(np.abs(rng.normal(0, 1, 32))) + w[i] = 0.0 + st[i] = rng.choice([0.0, -0.0, 2.0 ** -24, -1.0]) + else: + raise ValueError(name) + return st, a, w + + +def _row_any_sum(target, rng): + """32 FP8 magnitudes summing exactly to target (any multiple of 2^-9 up to 32 x 448), random signs.""" + k = int(round(target * 512)) + assert k / 512 == target + vals = [] + for _ in range(31): + if k <= 0: + break + cand = FP8_POS[FP8_POS * 512 <= k] + v = cand[-1] + vals.append(v) + k -= int(round(v * 512)) + assert k >= 0 + if k: + last = k / 512 + assert last in FP8_POS, "target not reachable with 32 FP8 values" + vals.append(last) + row = np.array(vals + [0.0] * (32 - len(vals))) + return row[rng.permutation(32)] * _signs(rng, 32) + + +S2A_FAMILIES = ("bound_at_63", "bound_below_63", "unbounded_inexact", "far_half", "big_inexact", "nan", "zeros") + + +def bound_f32(a, w): + """(wsum, amax, bounded) per op as the kernels compute them: wsum the packers' exact float32 sum |w|, amax the + decoded max code (a NaN code 0x7f is the largest), bounded = float32(wsum x amax) < 63.""" + codes = fp8_encode(q8(a)) + mc = (codes & 0x7F).max(1) + amax = np.asarray(DECODE, np.float64)[mc].astype(np.float32) + with np.errstate(invalid="ignore"): + wsum = np.abs(q8(w)).sum(1) + wsum32 = wsum.astype(np.float32) + assert np.array_equal(wsum32.astype(np.float64), wsum, equal_nan=True) + with np.errstate(invalid="ignore", over="ignore"): + prod = (wsum32 * amax).astype(np.float32) + bounded = prod < LIMIT + return wsum32, amax, bounded, prod + + +def prepare(build, n, seed=11): + sets = {} + for name in M.FAMILIES: + sets[name] = M.family(name, np.random.default_rng([seed, M.FAMILIES.index(name)]), n) + for j, name in enumerate(S2A_FAMILIES): + sets[name] = s2a_family(name, np.random.default_rng([seed, 100 + j]), max(64, n // 8)) + (p, ws, A), reach = largest_below_63() + st_all, a_all, w_all, tags = [], [], [], [] + for name, (st, a, w) in sets.items(): + m = len(st) + pad = (-m) % 16 + st_all.append(np.concatenate([f16(np.asarray(st, np.float64)), np.zeros(pad)])) + a_all.append(np.concatenate([q8(np.asarray(a, np.float64)), np.zeros((pad, 32))])) + w_all.append(np.concatenate([q8(np.asarray(w, np.float64)), np.zeros((pad, 32))])) + tags += [name] * m + ["pad"] * pad + st, a, w = np.concatenate(st_all), np.concatenate(a_all), np.concatenate(w_all) + wsum, amax, bounded, prod = bound_f32(a, w) + ref = np.array([M.op_exact(st[i:i + 1], a[i], w[i:i + 1])[0] for i in range(len(st))]) + kern = np.array([M.op_kernel(st[i:i + 1], a[i], w[i:i + 1])[0] for i in range(len(st))]) + with np.errstate(invalid="ignore"): + true_abs = np.abs(a * w).sum(1) + exact_bound = wsum.astype(np.float64) * amax.astype(np.float64) + cpu = {"ops": int(np.sum(np.array(tags) != "pad")), + "bound_covers_true_abs_sum": bool(np.all(~np.isfinite(true_abs) | (true_abs <= exact_bound))), + "bounded_with_true_abs_sum_ge_64": int(np.sum(bounded & (true_abs >= 64))), + "op8_cpu_model_differs_from_exact": int(np.sum(_bits16(kern) != _bits16(ref))), + "largest_reachable_bound_below_63": {"product": float(p), "wsum": float(ws), "amax": float(A), + "largest_float32_below_63_reachable": bool(reach)}} + for name in S2A_FAMILIES[:2]: + sel = np.array(tags) == name + cpu[name + "_products"] = sorted(set(float(x) for x in prod[sel]))[:8] + cpu[name + "_bounded"] = int(bounded[sel].sum()) + prep = {"st": st, "a": a, "w": w, "tags": tags, "wsum": wsum, "amax": amax, "bounded": bounded, "ref": ref, + "cpu": cpu, "a_codes": fp8_encode(a), "w_codes": fp8_encode(w), + "w16": q8(w).astype(np.float16).view(np.uint16)} + with open(os.path.join(build, "l_q7_s2a_cases.pkl"), "wb") as f: + pickle.dump(prep, f) + print("prepared", len(st), "ops", cpu, flush=True) + return cpu + + +def _bits16(v): + """FP16 bits of values, a NaN's sign bit cleared (the accepted port exception).""" + h = f16(np.asarray(v, np.float64)).astype(np.float16).view(np.uint16) + nan = ((h & 0x7C00) == 0x7C00) & ((h & 0x3FF) != 0) + return np.where(nan, h & 0x7FFF, h) + + +def co_name(mut): + return "l_q7_s2a_probe%s.co" % ("" if not mut else "_m%d" % mut) + + +def compile_probe(build, rocm): + from clean.runner import build as B + from clean.runner.hip import clean_process_environment + clean_process_environment(rocm) + clang = B.find_clang(rocm) + os.makedirs(build, exist_ok=True) + src = os.path.join(ROOT, "clean", "l", "kernels", "l_q7_s2a_probe.hip") + for mut in (0,) + MUTANTS: + cmd = B.device_command(clang, rocm, src, os.path.join(build, co_name(mut))) + [ + "-I", os.path.join(ROOT, "clean", "kernels")] + (["-DQ7_S2A_MUTANT=%d" % mut] if mut else []) + B.run(cmd) + + +def kernel_names(): + return ["l_q7_s2a_probe"] + ["l_q7_s2a_probe_m%d" % m for m in MUTANTS] + + +def prepare_d3d12(build, rocm): + from clean.l.runner.l_d3d12_check import prepare_replay + return prepare_replay(build, rocm, kernel_names()) + + +def _inputs(prep): + return {"a": prep["a_codes"].astype(np.uint8).tobytes(), "w": prep["w_codes"].astype(np.uint8).tobytes(), + "w16": prep["w16"].astype(np.uint16).tobytes(), "st": prep["st"].astype(np.float32).tobytes(), + "wsum": prep["wsum"].astype(np.float32).tobytes()} + + +def run_hip(build, rocm, prep): + from clean.runner.hip import Hip + h = Hip(rocm) + ins = _inputs(prep) + n = len(prep["st"]) + bufs = {k: h.malloc(len(v)) for k, v in ins.items()} + for k, v in ins.items(): + h.upload(bufs[k], v) + po, pp = h.malloc(4 * n), h.malloc(4 * n) + out = {} + for mut in (0,) + MUTANTS: + with open(os.path.join(build, co_name(mut)), "rb") as f: + fn = h.get_function(h.load_module(f.read()), "l_q7_s2a_probe") + h.memset(po, 0xA5, 4 * n) + h.memset(pp, 0xA5, 4 * n) + h.launch(fn, n // 16, 32, [ctypes.c_void_p(bufs[k]) for k in ("a", "w", "w16", "st", "wsum")] + + [ctypes.c_void_p(po), ctypes.c_void_p(pp)]) + h.sync() + out[mut] = (np.frombuffer(h.download(po, 4 * n), np.float32).astype(np.float64), + np.frombuffer(h.download(pp, 4 * n), np.uint32)) + for p in list(bufs.values()) + [po, pp]: + h.free(p) + return out + + +def run_d3d12(build, prep): + cdir = os.path.join(build, "replay-s2a-" + time.strftime("%Y%m%d-%H%M%S")) + os.makedirs(cdir) + ins = _inputs(prep) + n = len(prep["st"]) + for k, v in ins.items(): + with open(os.path.join(cdir, k + ".bin"), "wb") as f: + f.write(v) + lines = [] + for mut, kname in zip((0,) + MUTANTS, kernel_names()): + lines += ["case m%d" % mut, "module l_q7_s2a_probe.co", "kernel %s %d 32" % (kname, n // 16)] + lines += ["arg buf %s.bin" % k for k in ("a", "w", "w16", "st", "wsum")] + lines += ["arg out %d m%d.o 90" % (4 * n, mut), "arg out %d m%d.p 90" % (4 * n, mut), ""] + with open(os.path.join(cdir, "cases.txt"), "w", newline="\n") as f: + f.write("\n".join(lines)) + env = {k: v for k, v in os.environ.items() + if not any(s in k.upper() for s in ("HIP", "LLVM", "ROCM", "HSA")) or k.upper() == "HIP_PLATFORM"} + env["WG_D3D12_ARENA_MB"] = "256" + r = subprocess.run([os.path.join(build, "kernel_replay.exe"), build, os.path.join(cdir, "cases.txt")], env=env, + capture_output=True, text=True, timeout=100, creationflags=0x08000000) + out = {} + for mut in (0,) + MUTANTS: + po, pp = os.path.join(cdir, "m%d.o" % mut), os.path.join(cdir, "m%d.p" % mut) + if os.path.exists(po) and os.path.exists(pp): + out[mut] = (np.frombuffer(open(po, "rb").read()[:4 * n], np.float32).astype(np.float64), + np.frombuffer(open(pp, "rb").read()[:4 * n], np.uint32)) + return out, {"replay_exit": r.returncode, "replay_tail": (r.stdout[-300:] + r.stderr[-300:]).strip(), + "identity": run_identity(build, ["kernel_replay.exe"] + ["dx12/%s.dxil" % k for k in kernel_names()])} + + +def score(prep, got): + """Per set: value mismatches against the exact model (FP16 bits, NaN sign masked) and path mismatches.""" + tags = np.array(prep["tags"]) + real = tags != "pad" + want, path = _bits16(prep["ref"]), prep["bounded"].astype(np.uint32) + res = {} + for mut, (y, p) in got.items(): + bad_v = (_bits16(y) != want) & real + bad_p = (p != path) & real + per = {} + for name in dict.fromkeys(tags[real]): + sel = tags == name + per[name] = {"ops": int(sel.sum()), "value_mismatch": int(bad_v[sel].sum()), + "path_mismatch": int(bad_p[sel].sum()), "bounded": int((p[sel] == 1).sum())} + res[mut] = {"value_mismatch": int(bad_v.sum()), "path_mismatch": int(bad_p.sum()), "sets": per, + "output_sha256": hashlib.sha256(_bits16(y).tobytes() + p.tobytes()).hexdigest()} + return res + + +def run(build, rocm, d3d12): + with open(os.path.join(build, "l_q7_s2a_cases.pkl"), "rb") as f: + prep = pickle.load(f) + res = {"what": "S2a primitive (two FP8 matrix instructions from 0 under wsum x amax < 63, exact op8 fallback, " + "FP16 start joined afterwards) against the exact model, adversarial sets, mutants", + "fp8_op_model": "exact", "cpu": prep["cpu"], "nan_sign_masked": True, "nan_mask_control": mask_control(), + "identity": run_identity(build, [co_name(m) for m in (0,) + MUTANTS] + ["l_q7_s2a_cases.pkl"]), + "mutants": MUTANT_WHAT} + hip = run_hip(build, rocm, prep) + res["hip"] = score(prep, hip) + ok = (res["hip"][0]["value_mismatch"] == 0 and res["hip"][0]["path_mismatch"] == 0 and res["cpu"] + ["bound_covers_true_abs_sum"] and res["cpu"]["bounded_with_true_abs_sum_ge_64"] == 0 + and res["nan_mask_control"]["pass"]) + res["hip_mutants_caught"] = {m: res["hip"][m]["value_mismatch"] + res["hip"][m]["path_mismatch"] > 0 + for m in MUTANTS} + if d3d12: + dx, info = run_d3d12(build, prep) + res["d3d12_run"] = info + res["d3d12"] = score(prep, dx) if dx else {} + res["d3d12_identical_to_hip"] = {m: (m in res["d3d12"] and + res["d3d12"][m]["output_sha256"] == res["hip"][m]["output_sha256"]) + for m in (0,) + MUTANTS} + res["d3d12_mutants_caught"] = {m: m in res["d3d12"] and res["d3d12"][m]["value_mismatch"] + + res["d3d12"][m]["path_mismatch"] > 0 for m in MUTANTS} + ok = ok and info["replay_exit"] == 0 and 0 in res["d3d12"] and res["d3d12"][0]["value_mismatch"] == 0 \ + and res["d3d12"][0]["path_mismatch"] == 0 and res["d3d12_identical_to_hip"][0] + res["pass"] = bool(ok) + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", default=r"C:\Program Files\AMD\ROCm\7.2") + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument("--compile", action="store_true") + g.add_argument("--prepare", action="store_true") + g.add_argument("--prepare-d3d12", action="store_true") + g.add_argument("--run", action="store_true") + ap.add_argument("--d3d12", action="store_true") + ap.add_argument("--cases", type=int, default=4096) + ap.add_argument("--json") + a = ap.parse_args(argv) + if a.compile: + compile_probe(a.build, a.rocm) + return 0 + if a.prepare: + os.makedirs(a.build, exist_ok=True) + prepare(a.build, a.cases) + return 0 + if a.prepare_d3d12: + print("built", prepare_d3d12(a.build, a.rocm)) + return 0 + from clean.runner.hip import clean_process_environment + clean_process_environment(a.rocm) + res = run(a.build, a.rocm, a.d3d12) + for be in ("hip", "d3d12"): + if be in res and res[be]: + for m, r in sorted(res[be].items()): + print(be, "mutant" if m else "probe", m, "value mismatch", r["value_mismatch"], "path mismatch", + r["path_mismatch"], flush=True) + print("D3D12 identical to HIP", res.get("d3d12_identical_to_hip")) + print("PASS" if res["pass"] else "FAIL") + if a.json: + LR.write_l_record(res, a.json) + return 0 if res["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_q7_s2a_fallbacks.py b/clean/l/runner/l_q7_s2a_fallbacks.py new file mode 100644 index 00000000..32fb6633 --- /dev/null +++ b/clean/l/runner/l_q7_s2a_fallbacks.py @@ -0,0 +1,141 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): how often the S2a bound sends an FP8 op to the exact fallback on a local recorded run, per layer. +CPU only: the recorded run of an l_frame_check build is replayed through the oracle (as the frame +check does), and every 32-wide FP8 op chunk is classified with the kernels' own bound: bounded when +float32(wsum) x float32(amax) < 63, wsum = sum |w| of the weight row's chunk (the packers' table), amax = max |a| of +the token's chunk (from the codes: a NaN code counts as the largest, so NaN fails). Scores: wsum = sum |u_key|, +amax = max |u_query| (l_q7.hip). Labels: the stage (enc0_block / dec0_block for Model L's own blocks, else the shared +stage), then the op kind (the weight rows of the call, or "scores"). + + python clean/l/runner/l_q7_s2a_fallbacks.py --build --json out.json +""" +import argparse +import copy +import os +import pickle +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import oracle as L # noqa: E402 +from clean.l.runner import l_record as LR # noqa: E402 + +LIMIT = np.float32(63.0) + + +def _amax(a): + """max |a| over the last axis, a NaN as +inf (the code 0x7f is the largest).""" + x = np.abs(np.asarray(a, np.float64)) + return np.where(np.isnan(x), np.inf, x).max(-1).astype(np.float32) + + +def count(build): + from clean.l.reference import table as T + from clean.l.runner import l_frame_check as F + rec = pickle.load(open(F.rec_path(build, "L", "stages"), "rb")) + params = pickle.load(open(os.path.join(F.model_dir(build), "params.pkl"), "rb"))["L"] + stats, label = {}, ["?"] + + def add(key, outputs, fails, steps=0, step_fails=0): + s = stats.setdefault(label[0], {}).setdefault(key, [0, 0, 0, 0]) + s[0] += int(outputs) + s[1] += int(fails) + s[2] += int(steps) + s[3] += int(step_fails) + + def wave_steps(bad): + """The block kernels' wave steps of a window's op [..., 16 tokens, n rows]: one step = the 16 tokens x rows + r0 + j and r0 + 8 + j of a 16-row tile (l_q7.hip / l_q7w.hip: lane = token + 16 x half); a step pays the + scalar fallback if any of its 32 outputs fails. (0, 0) for other shapes.""" + if bad.ndim < 2 or bad.shape[-2] != 16 or bad.shape[-1] % 16: + return 0, 0 + b = bad.reshape(bad.shape[:-2] + (16, bad.shape[-1] // 16, 2, 8)).any(axis=(-4, -2)) + return b.size, int(b.sum()) + + op8, score, stage_q7, enc, dec = L._op8, L._score_op, L.run_stage_q7, L.encode, L.decode + + def spy_op8(start, a8, W8, model): + a = np.asarray(a8, np.float64) + W = np.asarray(W8, np.float64) + with np.errstate(invalid="ignore"): + ws = np.abs(W).sum(-1).astype(np.float32) # [n] + am = _amax(a) # [...] + ok = (am[..., None] * ws).astype(np.float32) < LIMIT + add("rows%d" % W.shape[0], ok.size, np.sum(~ok), *wave_steps(~ok)) + return op8(start, a8, W8, model) + + def spy_score(B, u, model): + uu = np.asarray(u, np.float64) # [..., h, T, 32] + with np.errstate(invalid="ignore"): + su = np.abs(uu).sum(-1).astype(np.float32) # key sums [..., h, T] + am = _amax(uu) # query max [..., h, T] + ok = (am[..., :, None] * su[..., None, :]).astype(np.float32) < LIMIT # [..., h, query, key] + bad = ~ok + steps = (0, 0) + if bad.shape[-1] == 16 and bad.shape[-2] == 16: # lane = query + 16 x half, step jj = keys 8 half + jj + b = bad.reshape(bad.shape[:-1] + (2, 8)).any(axis=(-3, -2)) + steps = (b.size, int(b.sum())) + add("scores", ok.size, np.sum(bad), *steps) + return score(B, u, model) + + def wrap(name, f): + def g(*a, **k): + old = label[0] + label[0] = name if isinstance(name, str) else name(*a) + try: + return f(*a, **k) + finally: + label[0] = old + return g + + L._op8, L._score_op = spy_op8, spy_score + L.run_stage_q7 = wrap(lambda stage, *a: stage, stage_q7) + L.encode, L.decode = wrap("enc0_block", enc), wrap("dec0_block", dec) + try: + F.compare_model(rec, params, table=copy.deepcopy(T.TABLE)) + finally: + L._op8, L._score_op, L.run_stage_q7, L.encode, L.decode = op8, score, stage_q7, enc, dec + out, tot = {}, [0, 0, 0, 0] + for st, kinds in stats.items(): + out[st] = {} + for k, (n, f, ns, fs) in sorted(kinds.items()): + out[st][k] = {"outputs": n, "fallbacks": f, "fallback_fraction": f / n if n else 0.0, + "wave_steps": ns, "wave_steps_with_fallback": fs, + "wave_step_fallback_fraction": fs / ns if ns else None} + for i, v in enumerate((n, f, ns, fs)): + tot[i] += v + return {"layers": out, "outputs": tot[0], "fallbacks": tot[1], + "fallback_fraction": tot[1] / tot[0] if tot[0] else 0.0, "wave_steps": tot[2], + "wave_steps_with_fallback": tot[3], "wave_step_fallback_fraction": tot[3] / tot[2] if tot[2] else 0.0, + "frames": len(rec["frames"]), "identity": rec.get("identity")} + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--json") + a = ap.parse_args(argv) + res = {"what": "S2a bound on a local recorded run: FP8 op outputs per layer that take the exact fallback (CPU replay of a " + "recorded l_frame_check run, the kernels' bound in float32)", "fp8_op_model": "exact", + "counts": count(a.build)} + c = res["counts"] + for st, kinds in c["layers"].items(): + for k, v in kinds.items(): + ws = v["wave_step_fallback_fraction"] + print(f"{st:12s} {k:8s} outputs {v['outputs']:>12d} fallbacks {v['fallbacks']:>9d} " + f"({100 * v['fallback_fraction']:.3f} %) wave steps with a fallback " + + ("-" if ws is None else f"{100 * ws:.2f} %")) + print("total", c["outputs"], "fallbacks", c["fallbacks"], f"({100 * c['fallback_fraction']:.3f} %)", + "wave steps", c["wave_steps"], "with a fallback", c["wave_steps_with_fallback"], + f"({100 * c['wave_step_fallback_fraction']:.2f} %)") + if a.json: + LR.write_l_record(res, a.json) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_q7w_check.py b/clean/l/runner/l_q7w_check.py new file mode 100644 index 00000000..fea42cc7 --- /dev/null +++ b/clean/l/runner/l_q7w_check.py @@ -0,0 +1,348 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): GPU check of the shared stages' FP16-form kernels (clean/l/kernels/l_q7w.hip; D3D12 ports +clean/dx12/kernels/l_q7w.hlsl) against the oracle (clean/l/reference/oracle.py run_stage_q7's parts, FP8 op model +"exact"), raw bytes. + + python clean/l/runner/l_q7w_check.py --build --compile # CPU: /l_q7w.co + python clean/l/runner/l_q7w_check.py --build --prepare [--real ] # CPU + python clean/l/runner/l_q7w_check.py --build --prepare-d3d12 # CPU: D3D12 ports + kernel_replay.exe + python clean/l/runner/l_q7w_check.py --build --run [--d3d12] --json out.json # GPU (inside a GPU slot) + +Kernels and their oracle: l_q7_block_c64 / _c96 / _c128 = FP8(swin_block(x, border "mirror")) on an FP8 map; +l_q7_block_c64_dec / _c96_dec = FP8(swin_block(E, border "coarse", skip S)); l_q7_merge_c64 / _c96 = FP8(clamp(the +Wm chain over space_to_depth(s) from bm, +-6.28125)); l_q7_expand_c128 / _c96 = depth_to_space(FP8(the We chain from +be)). Weights: Model M's synthetic stage parameters (two seeds) and, with --real, Model L's real stage weights (enc1, +enc2, b0, b5, dec2, dec1). Grids of several sizes, both window offsets of each stage, input scales, special values +(+-448, zeros, a NaN code). Every output buffer is filled with the back end's sentinel (HIP 0xA5, D3D12 0x5A) and +has 64 slack bytes after it that must keep it; the output must equal the oracle's bytes exactly. Negative control +per kernel: its first case with 4 or more workgroups launched with half of them must differ. Counts and digests only. +""" +import argparse +import ctypes +import hashlib +import os +import pickle +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import oracle as L # noqa: E402 +from clean.l.reference import table as T # noqa: E402 +from clean.l.runner import l_pack as LP # noqa: E402 +from clean.l.runner import l_record as LR # noqa: E402 +from clean.l.runner.l_q7_check import mask_control, mask_nan, out8, run_identity # noqa: E402 +from clean.reference import swin as MS # noqa: E402 +from clean.reference.fp8 import f16, fp8_encode, q8 # noqa: E402 + +HIP_SENTINEL, D3D12_SENTINEL, SLACK = 0xA5, 0x5A, 64 +CO = "l_q7w.co" +BLOCKS = {"l_q7_block_c64": ("enc1", False), "l_q7_block_c96": ("enc2", False), "l_q7_block_c128": ("b0", False), + "l_q7_block_c64_dec": ("dec1", True), "l_q7_block_c96_dec": ("dec2", True)} +MERGES = {"l_q7_merge_c64": "enc1", "l_q7_merge_c96": "enc2"} +EXPANDS = {"l_q7_expand_c96": "dec1", "l_q7_expand_c128": "dec2"} +KERNELS = tuple(BLOCKS) + tuple(MERGES) + tuple(EXPANDS) +GRIDS = [(8, 8), (12, 16), (16, 12)] +SCALES = (0.5, 3.0) + + +def compile_kernel(build, rocm): + from clean.runner import build as B + from clean.runner.hip import clean_process_environment + clean_process_environment(rocm) + os.makedirs(build, exist_ok=True) + B.run(B.device_command(B.find_clang(rocm), rocm, os.path.join(ROOT, "clean", "l", "kernels", "l_q7w.hip"), + os.path.join(build, CO)) + ["-I", os.path.join(ROOT, "clean", "kernels")]) + + +def _consts(stage): + k = T.block_consts(T.TABLE, stage) + k["mm8"] = "exact" + return k + + +def _fp8_map(rng, shape, scale, special): + v = q8(rng.normal(0, scale, shape)) + if special: + v[0, 0, :3] = [448.0, -448.0, 0.0] + v[1, 1, 2] = np.nan + return v + + +def _b(a): + return fp8_encode(a).tobytes() + + +def make_cases(params_by_set, seed=7): + """[(kernel, label, grid, block, args, defined bytes, oracle bytes)] for every kernel and parameter set.""" + rng = np.random.default_rng(seed) + tab = LP.q7_tables() + cases = [] + for sname, params in params_by_set: + for kern, (stage, dec) in BLOCKS.items(): + cfg = MS.STAGES[stage] + p = params[stage if stage in params else stage] + C, heads = cfg["C"], cfg["heads"] + blob = LP.pack_q7_block(p, C, heads, 4 * C) + for (H, W) in GRIDS: + for scale in SCALES: + special = scale == SCALES[1] and (H, W) == GRIDS[0] + for parity in (0, 1): + off = MS.window_offset(stage, parity) + x = _fp8_map(rng, (H, W, C), scale, special) + if dec: + S = _fp8_map(rng, (H, W, C), scale, special) + want = L.swin_block(x, p, off, heads, _consts(stage), "coarse", skip=S) + args = [("buf", _b(x)), ("buf", _b(S))] + else: + want = L.swin_block(x, p, off, heads, _consts(stage), "mirror") + args = [("buf", _b(x)), ("null",)] + n = H * W * C + args += [("out", n), ("buf", blob), ("buf", tab), ("u32", H), ("u32", W), ("u32", off[0]), + ("u32", off[1])] + grid = -(-(W + off[0]) // 4) * -(-(H + off[1]) // 4) + cases.append({"kernel": kern, "set": sname, "label": f"{H}x{W} s{scale} off{off}" + + (" special" if special else ""), "grid": grid, "args": args, + "defined": n, "want": out8(want)}) + for kern, stage in MERGES.items(): + p = params[stage] + C, Co = MS.STAGES[stage]["C"], MS.STAGES[stage]["C_out"] + blob = LP.pack_q7_linear(p["Wm"], p["bm"]) + for (H, W) in GRIDS: + for scale in SCALES: + special = scale == SCALES[1] and (H, W) == GRIDS[0] + s = _fp8_map(rng, (H, W, C), scale, special) + m = L.mm8(np.asarray(p["bm"], np.float64), L.space_to_depth(s), p["Wm"]) + want = q8(np.minimum(np.maximum(m, -T.MERGE_CLAMP), T.MERGE_CLAMP)) + n = (H // 2) * (W // 2) * Co + cases.append({"kernel": kern, "set": sname, "label": f"{H}x{W} s{scale}" + + (" special" if special else ""), "grid": (H // 2) * (W // 2), + "args": [("buf", _b(s)), ("out", n), ("buf", blob), ("u32", H), ("u32", W)], + "defined": n, "want": out8(want)}) + for kern, stage in EXPANDS.items(): + p = params[stage] + C, Cl = MS.STAGES[stage]["C"], MS.STAGES[stage]["C_low"] + blob = LP.pack_q7_linear(p["We"], p["be"]) + for (H, W) in GRIDS: + for scale in SCALES: + special = scale == SCALES[1] and (H, W) == GRIDS[0] + h, w = H // 2, W // 2 + xl = _fp8_map(rng, (h, w, Cl), scale, special) + want = L.depth_to_space(q8(L.mm8(np.asarray(p["be"], np.float64), xl, p["We"]))) + n = H * W * C + cases.append({"kernel": kern, "set": sname, "label": f"{h}x{w} s{scale}" + + (" special" if special else ""), "grid": h * w, + "args": [("buf", _b(xl)), ("out", n), ("buf", blob), ("u32", h), ("u32", w)], + "defined": n, "want": out8(want)}) + return cases + + +def storage(p): + """Synthetic parameters in the formats the device stores (FP8 matrices, FP16 vectors and B).""" + return {k: (q8(v) if np.ndim(v) >= 2 and k != "B" else f16(v)) for k, v in + ((k, np.asarray(v, np.float64)) for k, v in p.items())} + + +def prepare(build, real=None): + sets = [("synthetic_%d" % s, {st: storage(MS.random_params(st, np.random.default_rng(s))) + for st in ("enc1", "enc2", "b0", "dec2", "dec1")}) for s in (89, 90)] + real_sha = None + if real: + path = os.path.join(real, "lframe_model", "params.pkl") + with open(path, "rb") as f: + real_sha = hashlib.sha256(f.read()).hexdigest() + params = pickle.load(open(path, "rb"))["L"] + sets.append(("real", {st: params[st] for st in ("enc1", "enc2", "b0", "dec2", "dec1")})) + sets.append(("real_b5", {"b0": params["b5"], **{st: params[st] for st in ("enc1", "enc2", "dec2", "dec1")}})) + cases = make_cases(sets) + with open(os.path.join(build, "l_q7w_cases.pkl"), "wb") as f: + pickle.dump({"cases": cases, "real_params_sha256": real_sha}, f) + return len(cases) + + +def hip_run(h, mod, case, grid=None): + args, owned, outs = [], [], [] + for a in case["args"]: + if a[0] == "buf": + p = h.malloc(max(len(a[1]), 4)) + h.upload(p, a[1]) + owned.append(p) + args.append(ctypes.c_void_p(p)) + elif a[0] == "out": + p = h.malloc(a[1] + SLACK) + h.memset(p, HIP_SENTINEL, a[1] + SLACK) + owned.append(p) + outs.append((p, a[1] + SLACK)) + args.append(ctypes.c_void_p(p)) + elif a[0] == "null": + args.append(ctypes.c_void_p(0)) + else: + args.append(ctypes.c_int(a[1])) + try: + h.launch(h.get_function(mod, case["kernel"]), grid or case["grid"], 32, args) + h.sync() + return bytes(h.download(*outs[0])) + finally: + for p in owned: + h.free(p) + + +def _check(got, case, sentinel): + body, slack = mask_nan(got[:case["defined"]], "fp8") if got else b"", got[case["defined"]:] + return {"identical": body == case["want"] and slack == bytes([sentinel]) * SLACK, + "bytes_differ": sum(a != b for a, b in zip(body, case["want"])) if body else -1, + "sha256": hashlib.sha256(body).hexdigest()} + + +def run(build, rocm, d3d12=False): + from clean.runner.hip import Hip, clean_process_environment + clean_process_environment(rocm) + with open(os.path.join(build, "l_q7w_cases.pkl"), "rb") as f: + prep = pickle.load(f) + cases = prep["cases"] + h = Hip(rocm) + with open(os.path.join(build, CO), "rb") as f: + mod = h.load_module(f.read()) + res = {"what": "the shared stages' FP16-form kernels against the oracle, raw bytes", "fp8_op_model": "exact", + "identity": run_identity(build, [CO, "l_q7w_cases.pkl"]), "real_params_sha256": prep["real_params_sha256"], + "nan_sign_masked": "outputs compared and hashed with only a NaN's sign bit cleared (accepted port exception; " + "a canonical NaN store costs about 3 VALU per stored dword in the RGA ISA)", + "nan_mask_control": mask_control(), + "shared_dec_border": T.TABLE["shared_dec_border"], + "shared_dec_border_source": "spec Q10b (PRESET-L-P3-Q10b-decoder-border-rule.md): E by the coarse mirror, " + "S by the fine mirror (FACT); the member's own sub-pixel kept (INFERENCE, " + "UNCONFIRMED U2)", + "cases": [], "negative_controls": {}} + negs = {} + for i, c in enumerate(cases): + r = _check(hip_run(h, mod, c), c, HIP_SENTINEL) + r.update({"kernel": c["kernel"], "set": c["set"], "label": c["label"]}) + res["cases"].append(r) + if c["kernel"] not in negs and c["grid"] >= 4: + negs[c["kernel"]] = i + half = hip_run(h, mod, c, grid=c["grid"] // 2) + res["negative_controls"][c["kernel"]] = {"case": c["label"], "set": c["set"], + "caught": not _check(half, c, HIP_SENTINEL)["identical"]} + per = {} + for r in res["cases"]: + k = per.setdefault(r["kernel"], [0, 0]) + k[0] += 1 + k[1] += r["identical"] + res["per_kernel"] = {k: {"cases": v[0], "identical": v[1]} for k, v in per.items()} + res["cases_identical"] = sum(r["identical"] for r in res["cases"]) + res["pass"] = (res["cases_identical"] == len(cases) and set(res["negative_controls"]) == set(KERNELS) + and all(v["caught"] for v in res["negative_controls"].values()) and res["nan_mask_control"]["pass"]) + if d3d12: + res["d3d12"] = run_d3d12(build, cases, res, negs) + res["pass"] = res["pass"] and res["d3d12"]["pass"] + return res + + +def prepare_d3d12(build, rocm): + from clean.l.runner.l_d3d12_check import prepare_replay + return prepare_replay(build, rocm, list(KERNELS)) + + +def run_d3d12(build, cases, res, negs): + import subprocess + import time + cdir = os.path.join(build, "replay-q7w-" + time.strftime("%Y%m%d-%H%M%S")) + os.makedirs(cdir) + files, lines = {}, [] + + def buf(data): + key = hashlib.sha256(data).hexdigest()[:20] + if key not in files: + files[key] = "b_%s.bin" % key + with open(os.path.join(cdir, files[key]), "wb") as f: + f.write(data) + return files[key] + + for i, c in enumerate(cases): + runs = [("c%04d" % i, c["grid"])] + if negs.get(c["kernel"]) == i: + runs.append(("negative_c%04d" % i, c["grid"] // 2)) + for name, grid in runs: + lines += ["case " + name, "module " + CO, "kernel %s %d 32" % (c["kernel"], grid)] + for a in c["args"]: + if a[0] == "buf": + lines.append("arg buf " + buf(a[1])) + elif a[0] == "out": + lines.append("arg out %d %s.o %d" % (a[1] + SLACK, name, D3D12_SENTINEL)) + elif a[0] == "null": + lines.append("arg null") + else: + lines.append("arg u32 %d" % a[1]) + lines.append("") + with open(os.path.join(cdir, "cases.txt"), "w", newline="\n") as f: + f.write("\n".join(lines)) + env = {k: v for k, v in os.environ.items() + if not any(s in k.upper() for s in ("HIP", "LLVM", "ROCM", "HSA")) or k.upper() == "HIP_PLATFORM"} + env["WG_D3D12_ARENA_MB"] = "256" + r = subprocess.run([os.path.join(build, "kernel_replay.exe"), build, os.path.join(cdir, "cases.txt")], env=env, + capture_output=True, text=True, timeout=110, creationflags=0x08000000) + + def read(n): + p = os.path.join(cdir, n + ".o") + return open(p, "rb").read() if os.path.exists(p) else b"" + + out = {"replay_exit": r.returncode, "replay_tail": (r.stdout[-300:] + r.stderr[-300:]).strip(), + "identity": run_identity(build, ["kernel_replay.exe"] + ["dx12/%s.dxil" % k for k in KERNELS]), + "negative_controls": {}} + same_hip = same_oracle = 0 + for i, (c, rc) in enumerate(zip(cases, res["cases"])): + d = _check(read("c%04d" % i), c, D3D12_SENTINEL) + rc["d3d12_identical_to_oracle"] = d["identical"] + rc["d3d12_sha256"] = d["sha256"] + same_oracle += d["identical"] + same_hip += d["identical"] and d["sha256"] == rc["sha256"] + for k, i in negs.items(): + out["negative_controls"][k] = {"caught": not _check(read("negative_c%04d" % i), cases[i], + D3D12_SENTINEL)["identical"]} + out.update({"cases": len(cases), "cases_identical_to_hip": same_hip, "cases_identical_to_oracle": same_oracle}) + out["pass"] = (r.returncode == 0 and same_hip == len(cases) and same_oracle == len(cases) + and all(v["caught"] for v in out["negative_controls"].values())) + return out + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", default=r"C:\Program Files\AMD\ROCm\7.2") + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument("--compile", action="store_true") + g.add_argument("--prepare", action="store_true") + g.add_argument("--prepare-d3d12", action="store_true") + g.add_argument("--run", action="store_true") + ap.add_argument("--d3d12", action="store_true") + ap.add_argument("--real") + ap.add_argument("--json") + a = ap.parse_args(argv) + if a.compile: + compile_kernel(a.build, a.rocm) + return 0 + if a.prepare: + print("cases", prepare(a.build, a.real)) + return 0 + if a.prepare_d3d12: + print("built", prepare_d3d12(a.build, a.rocm)) + return 0 + res = run(a.build, a.rocm, a.d3d12) + for k, v in res["per_kernel"].items(): + print(f"{k:22s} HIP identical {v['identical']} of {v['cases']} negative control caught " + f"{res['negative_controls'].get(k, {}).get('caught')}") + if a.d3d12: + d = res["d3d12"] + print("D3D12 replay exit", d["replay_exit"], "| identical to HIP", d["cases_identical_to_hip"], + "| to the oracle", d["cases_identical_to_oracle"], "of", d["cases"]) + print("PASS" if res["pass"] else "FAIL") + if a.json: + LR.write_l_record(res, a.json) + return 0 if res["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runtime/l_hyb_pack.h b/clean/l/runtime/l_hyb_pack.h new file mode 100644 index 00000000..69b60a27 --- /dev/null +++ b/clean/l/runtime/l_hyb_pack.h @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: MIT +// Model L's HYBRID form on the fast kernels (runtime WG_L_FORM=hybrid, native D3D12): the block stages' parameters in +// the values the hybrid kernels read. The runtime packs them through the unchanged M-form packers +// (pack_stage_blob, l::pack_model), under the same blob names. This is a C++ port of +// clean/l/runner/l_hyb_stage_pack.py hyb_stage_values (reference: clean/l/runner/l_hyb_pack.py hyb_block_values), +// byte for byte; test: clean/l/runtime/tests/test_l_runtime_hyb_pack.py. +// gamma1, gamma2 -> RN16(float32(gamma) * sqrtf(C)), C = the gamma's length (oracle.gamma_q7) +// bp, b1, b2, B, bm, be -> RN16(value) (FP16 values, stored as float32: exact) +// Wqk, Wv, Wp, W1, W2, Wm, We -> q8(value) (FP8 E4M3FN values, saturating at +-448) +// Every other tensor keeps its value. One rounding per operation (a product, then RN16; no fused forms). +#pragma once + +#include +#include + +#include "../../runtime/fpconv.h" +#include "../../runtime/model.h" + +namespace wgrt { +namespace l { + +// The block stages whose tensors take the hybrid values (enc0 / dec0 keep theirs). +inline bool hyb_block_stage(const std::string& s) { + static const char* const kS[] = {"enc0_block", "dec0_block", "enc1", "enc2", "b0", "b1", + "b2", "b3", "b4", "b5", "dec2", "dec1"}; + for (const char* x : kS) + if (s == x) return true; + return false; +} + +inline StageTensors hyb_stage_values(const StageTensors& t) { + StageTensors o = t; + for (const char* n : {"gamma1", "gamma2"}) { + const auto it = o.find(n); + if (it == o.end()) continue; + const float r = std::sqrt((float)it->second.f.size()); // sqrtf(C): correctly rounded + for (float& v : it->second.f) { + const float g = v * r; // float32 product, one rounding + v = rn16(g); + } + } + for (const char* n : {"bp", "b1", "b2", "B", "bm", "be"}) { + const auto it = o.find(n); + if (it == o.end()) continue; + for (float& v : it->second.f) v = rn16(v); + } + for (const char* n : {"Wqk", "Wv", "Wp", "W1", "W2", "Wm", "We"}) { + const auto it = o.find(n); + if (it == o.end()) continue; + for (float& v : it->second.f) v = (float)q8((double)v); + } + return o; +} + +} // namespace l +} // namespace wgrt diff --git a/clean/l/runtime/l_pack.cpp b/clean/l/runtime/l_pack.cpp index 243692b7..ee69d00c 100644 --- a/clean/l/runtime/l_pack.cpp +++ b/clean/l/runtime/l_pack.cpp @@ -1,6 +1,7 @@ // SPDX-License-Identifier: MIT // Model L weight packing for the runtime; see l_pack.h. #include "l_pack.h" +#include "l_q7_pack.h" namespace wgrt { namespace l { @@ -132,6 +133,57 @@ bool pack_dec_project(const std::vector& W, const std::vector& b, return true; } +bool pack_q7_model(const std::map& params, std::map>* out, + std::string* err) { + struct S { + const char* stage; + const char* blob; + int C, heads, lin_n, lin_k; // lin: merge (n C_out, K 4 C) or expansion (n 4 C, K C_low); 0 = none + const char *lw, *lb; + }; + static const S kS[] = {{"enc0_block", "enc0_block.q7", 32, 2, 0, 0, nullptr, nullptr}, + {"dec0_block", "dec0_block.q7", 32, 2, 0, 0, nullptr, nullptr}, + {"enc1", "enc1.q7", 64, 2, 96, 256, "Wm", "bm"}, + {"enc2", "enc2.q7", 96, 4, 128, 384, "Wm", "bm"}, + {"b0", "b0.q7", 128, 4, 0, 0, nullptr, nullptr}, + {"b1", "b1.q7", 128, 4, 0, 0, nullptr, nullptr}, + {"b2", "b2.q7", 128, 4, 0, 0, nullptr, nullptr}, + {"b3", "b3.q7", 128, 4, 0, 0, nullptr, nullptr}, + {"b4", "b4.q7", 128, 4, 0, 0, nullptr, nullptr}, + {"b5", "b5.q7", 128, 4, 0, 0, nullptr, nullptr}, + {"dec2", "dec2.q7", 96, 4, 384, 128, "We", "be"}, + {"dec1", "dec1.q7", 64, 2, 256, 96, "We", "be"}}; + for (const S& s : kS) { + const StageTensors* t = stage(params, s.stage, err); + if (!t) return false; + const size_t C = (size_t)s.C, h = (size_t)s.heads, m = 4 * C; + const Tensor *g1, *g2, *bp, *b1, *b2, *B, *Wqk, *Wv, *Wp, *W1, *W2; + if (!need(*t, s.stage, "gamma1", C, &g1, err) || !need(*t, s.stage, "gamma2", C, &g2, err) || + !need(*t, s.stage, "bp", C, &bp, err) || !need(*t, s.stage, "b1", m, &b1, err) || + !need(*t, s.stage, "b2", C, &b2, err) || !need(*t, s.stage, "B", h * 256, &B, err) || + !need(*t, s.stage, "Wqk", h * 32 * C, &Wqk, err) || !need(*t, s.stage, "Wv", h * 32 * C, &Wv, err) || + !need(*t, s.stage, "Wp", C * 32 * h, &Wp, err) || !need(*t, s.stage, "W1", m * C, &W1, err) || + !need(*t, s.stage, "W2", C * m, &W2, err)) + return false; + std::vector blob; + if (!q7pack::pack_block(g1->f, g2->f, bp->f, b1->f, b2->f, B->f, Wqk->f, Wv->f, Wp->f, W1->f, W2->f, s.C, s.heads, + (int)m, &blob, err)) + return *err = std::string(s.stage) + ": " + *err, false; + (*out)[s.blob] = std::move(blob); + if (s.lin_n) { + const Tensor *lw, *lb; + if (!need(*t, s.stage, s.lw, (size_t)s.lin_n * s.lin_k, &lw, err) || + !need(*t, s.stage, s.lb, (size_t)s.lin_n, &lb, err)) + return false; + if (!q7pack::pack_linear(lw->f, lb->f, s.lin_n, s.lin_k, &blob, err)) + return *err = std::string(s.stage) + ": " + *err, false; + (*out)[std::string(s.stage) + ".q7lin"] = std::move(blob); + } + } + (*out)["q7.tab"] = q7pack::q7_tables(); + return true; +} + bool pack_model(const std::map& params, const DRow& d, std::map>* out, std::string* err) { out->clear(); diff --git a/clean/l/runtime/l_pack.h b/clean/l/runtime/l_pack.h index 79497715..b6f15ec9 100644 --- a/clean/l/runtime/l_pack.h +++ b/clean/l/runtime/l_pack.h @@ -63,5 +63,11 @@ bool pack_dec_project(const std::vector& W, const std::vector& b, bool pack_model(const std::map& params, const DRow& d, std::map>* out, std::string* err); +// The FP16-form (q7) kernels' blobs (clean/l/runtime/l_q7_pack.h, byte for byte l_pack.py's): "enc0_block.q7", +// "dec0_block.q7"; per shared stage s (enc1, enc2, b0 ... b5, dec2, dec1) ".q7" (its block) and, for the encoder / +// decoder stages, ".q7lin" (the merge Wm / bm, the expansion We / be); "q7.tab" (the rsqrt / rcp tables). +bool pack_q7_model(const std::map& params, std::map>* out, + std::string* err); + } // namespace l } // namespace wgrt diff --git a/clean/l/runtime/l_q7_pack.h b/clean/l/runtime/l_q7_pack.h new file mode 100644 index 00000000..d7f2c1c1 --- /dev/null +++ b/clean/l/runtime/l_q7_pack.h @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: MIT +// Model L (issue #89): the weight blobs and lookup tables of the FP16-form (q7) kernels, header-only, byte for byte +// the Python packer's (clean/l/runner/l_pack.py pack_q7_block, pack_q7_linear, q7_tables). +// block blob: FP16 g1' g2' bp b1 b2 B, then FP8 codes Wqk Wv Wp W1 W2 row-major (g' = RN16(float32(gamma) x +// sqrtf(C)), the norm's gain with sqrt(C) folded in), padded to 256 bytes +// linear blob: FP16 bias [n], FP8 rows [n][K] at the next multiple of 16 (merges Wm / bm, expansions We / be) +// tables: rsqrt[65536] || rcp[65536], FP16, indexed by an FP16 bit pattern x: RN16(1 / sqrtf(x)) and RN16(1 / x) +// in float32 (a NaN keeps its payload, as numpy's conversion does) +#pragma once + +#include +#include +#include +#include +#include + +#include "../../runtime/fpconv.h" + +namespace wgrt { +namespace l { +namespace q7pack { + +inline void put16(std::vector* out, uint16_t h) { + out->push_back((uint8_t)(h & 0xffu)); + out->push_back((uint8_t)(h >> 8)); +} + +// binary32 -> binary16 as numpy converts it: round to nearest even; a NaN keeps its sign and top 10 payload bits +inline uint16_t f16_np(float f) { + const uint32_t a = f32_bits(f) & 0x7fffffffu; + if (a > 0x7f800000u) { + const uint16_t m = (uint16_t)((a >> 13) & 0x3ffu); + return (uint16_t)(((f32_bits(f) >> 16) & 0x8000u) | 0x7c00u | (m ? m : 0x200u)); + } + return f32_to_f16(f); +} + +// A weight's FP16 copy (Q7F-1): its FP8 value, exact in FP16; a NaN as 0x7e00 (as numpy writes it) +inline uint16_t f16_weight(float x) { + const double v = q8((double)x); + return std::isnan(v) ? (uint16_t)0x7e00u : f32_to_f16((float)v); +} + +// sum |w| per 32-wide chunk and row of an FP8 matrix W [rows][K], float32 little-endian, chunk-major [K / 32][rows] +// (exact: at most 32 FP8 magnitudes, multiples of 2^-9 below 2^14; a NaN weight gives NaN, which sends its outputs to +// the exact fallback) +inline void put_wsum(std::vector* out, const std::vector& W, int K) { + for (int c0 = 0; c0 < K; c0 += 32) + for (size_t r = 0; r < W.size() / (size_t)K; ++r) { + double s = 0.0; + for (int k = 0; k < 32; ++k) s += std::fabs(q8((double)W[r * K + c0 + k])); + const uint32_t b = f32_bits((float)s); + for (int i = 0; i < 4; ++i) out->push_back((uint8_t)(b >> (8 * i))); + } +} + +inline std::vector q7_tables() { + std::vector out; + out.reserve(4 * 65536); + std::vector rs(65536), rc(65536); + for (uint32_t b = 0; b < 65536; ++b) { + const float x = f16_to_f32((uint16_t)b); + rs[b] = f16_np(1.0f / std::sqrt(x)); + rc[b] = f16_np(1.0f / x); + } + for (uint16_t h : rs) put16(&out, h); + for (uint16_t h : rc) put16(&out, h); + return out; +} + +// One block's blob (C, heads, mlp = Model L's / Model M's block shapes); tensors as float32 row-major vectors. +inline bool pack_block(const std::vector& gamma1, const std::vector& gamma2, const std::vector& bp, + const std::vector& b1, const std::vector& b2, const std::vector& B, + const std::vector& Wqk, const std::vector& Wv, const std::vector& Wp, + const std::vector& W1, const std::vector& W2, int C, int heads, int mlp, + std::vector* out, std::string* err) { + const size_t c = (size_t)C, h = (size_t)heads, m = (size_t)mlp; + if (gamma1.size() != c || gamma2.size() != c || bp.size() != c || b1.size() != m || b2.size() != c || + B.size() != h * 256 || Wqk.size() != h * 32 * c || Wv.size() != h * 32 * c || Wp.size() != c * 32 * h || + W1.size() != m * c || W2.size() != c * m) + return *err = "q7 block: tensor sizes do not match C " + std::to_string(C) + ", heads " + std::to_string(heads) + + ", MLP " + std::to_string(mlp), + false; + out->clear(); + const float sc = std::sqrt((float)C); + for (const std::vector* g : {&gamma1, &gamma2}) + for (float v : *g) put16(out, f32_to_f16(v * sc)); + for (const std::vector* v : {&bp, &b1, &b2, &B}) + for (float x : *v) put16(out, f32_to_f16(x)); + for (const std::vector* w : {&Wqk, &Wv, &Wp, &W1, &W2}) + for (float x : *w) out->push_back(fp8_encode((double)x)); + // the same matrices as FP16 (exact for FP8 values), read by the scalar kernels without an FP8 decode (Q7F-1) + for (const std::vector* w : {&Wqk, &Wv, &Wp, &W1, &W2}) + for (float x : *w) put16(out, f16_weight(x)); + // per row and 32-wide chunk sum |w| (float32, exact), the matrix-instruction path's bound (Q7F-2) + put_wsum(out, Wqk, C); + put_wsum(out, Wv, C); + put_wsum(out, Wp, 32 * heads); + put_wsum(out, W1, C); + put_wsum(out, W2, mlp); + out->resize((out->size() + 255) / 256 * 256, 0); + return true; +} + +// A merge's or an expansion's blob: W [n][K] (K a multiple of 32), b [n]. +inline bool pack_linear(const std::vector& W, const std::vector& b, int n, int K, + std::vector* out, std::string* err) { + if (n <= 0 || K <= 0 || K % 32 || b.size() != (size_t)n || W.size() != (size_t)n * K) + return *err = "q7 linear: tensor sizes do not match", false; + out->clear(); + for (float x : b) put16(out, f32_to_f16(x)); + out->resize(((size_t)2 * n + 15) / 16 * 16, 0); + for (float x : W) out->push_back(fp8_encode((double)x)); + for (float x : W) put16(out, f16_weight(x)); // FP16 rows (Q7F-1) + put_wsum(out, W, K); // sum |w| per row chunk (Q7F-2) + out->resize((out->size() + 255) / 256 * 256, 0); + return true; +} + +} // namespace q7pack +} // namespace l +} // namespace wgrt diff --git a/clean/l/runtime/tests/l_hyb_pack_dump.cpp b/clean/l/runtime/tests/l_hyb_pack_dump.cpp new file mode 100644 index 00000000..f085bdba --- /dev/null +++ b/clean/l/runtime/tests/l_hyb_pack_dump.cpp @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: MIT +// Model L's HYBRID weight values (clean/l/runtime/l_hyb_pack.h) through the runtime's packers, on the CPU, for +// test_l_runtime_hyb_pack.py: +// +// l_hyb_pack_dump hyb [mutant] +// reads //.f32 (and enc0/T.u8) for every stage folder, applies hyb_stage_values to the +// block stages, packs as the runtime does (pack_stage_blob for the ten shared stages, l::pack_model for Model L's +// own launches) and writes /.bin; prints "OK " or "ERR ". "mutant": the gammas +// without the sqrtf(C) fold (the test's negative control: the gamma-bearing blobs must differ). +// l_hyb_pack_dump wgcm +// wgrt::l::load_wgcm_model, writes every tensor to /tensors//.f32 (enc0 T as .u8) and the +// hybrid blobs to /blobs/; prints "OK " or "ERR " (no values printed: model data). +#include +#include +#include +#include +#include +#include + +#include "../l_hyb_pack.h" +#include "../l_model.h" +#include "../l_pack.h" + +namespace fs = std::filesystem; + +namespace { + +const char* const kShared[] = {"enc1", "enc2", "b0", "b1", "b2", "b3", "b4", "b5", "dec2", "dec1"}; + +bool read_stage(const fs::path& dir, wgrt::StageTensors* t) { + for (const auto& e : fs::directory_iterator(dir)) { + const fs::path p = e.path(); + std::ifstream f(p, std::ios::binary | std::ios::ate); + const std::streamsize n = f.tellg(); + f.seekg(0); + wgrt::Tensor x; + if (p.extension() == ".f32") { + x.f.resize((size_t)n / 4); + f.read(reinterpret_cast(x.f.data()), n); + } else if (p.extension() == ".u8") { + x.u.resize((size_t)n); + f.read(reinterpret_cast(x.u.data()), n); + } else { + continue; + } + (*t)[p.stem().string()] = x; + } + return true; +} + +// The runtime's packing of a Model L model in the hybrid values (runtime.cpp, WG_L_FORM=hybrid on native D3D12). +bool pack_hyb(const std::map& model, bool mutant, + std::map>* blobs, std::string* err) { + std::map h; + for (const auto& [s, t] : model) { + if (!wgrt::l::hyb_block_stage(s)) { + h[s] = t; + continue; + } + h[s] = wgrt::l::hyb_stage_values(t); + if (mutant) + for (const char* n : {"gamma1", "gamma2"}) { + auto it = h[s].find(n); + const auto src = t.find(n); + if (it != h[s].end() && src != t.end()) + for (size_t i = 0; i < it->second.f.size(); ++i) it->second.f[i] = wgrt::rn16(src->second.f[i]); + } + } + for (const char* s : kShared) { + const auto it = h.find(s); + if (it == h.end()) return *err = std::string("missing stage ") + s, false; + if (!wgrt::pack_stage_blob(s, it->second, wgrt::drow_gfx12(), &(*blobs)[s], err)) return false; + } + std::map> lb; + if (!wgrt::l::pack_model(h, wgrt::drow_gfx12(), &lb, err)) return false; + for (auto& kv : lb) (*blobs)[kv.first] = std::move(kv.second); + return true; +} + +void write_blobs(const fs::path& dir, const std::map>& blobs) { + fs::create_directories(dir); + for (const auto& kv : blobs) { + std::ofstream f(dir / (kv.first + ".bin"), std::ios::binary); + f.write(reinterpret_cast(kv.second.data()), (std::streamsize)kv.second.size()); + } +} + +} // namespace + +int main(int argc, char** argv) { + const std::string mode = argc > 1 ? argv[1] : ""; + std::map model; + std::map> blobs; + std::string err; + if (mode == "hyb" && (argc == 4 || argc == 5)) { + for (const auto& e : fs::directory_iterator(argv[2])) + if (e.is_directory() && e.path().filename() != "out") read_stage(e.path(), &model[e.path().filename().string()]); + if (!pack_hyb(model, argc == 5 && std::string(argv[4]) == "mutant", &blobs, &err)) + return std::printf("ERR %s\n", err.c_str()), 0; + write_blobs(argv[3], blobs); + std::printf("OK %zu\n", blobs.size()); + return 0; + } + if (mode == "wgcm" && argc == 5) { + if (!wgrt::l::load_wgcm_model(argv[2], argv[3], &model, &err)) return std::printf("ERR %s\n", err.c_str()), 0; + const fs::path out = argv[4]; + for (const auto& [s, t] : model) { + const fs::path d = out / "tensors" / s; + fs::create_directories(d); + for (const auto& [n, x] : t) { + std::ofstream f(d / (n + (x.u.empty() ? ".f32" : ".u8")), std::ios::binary); + if (x.u.empty()) + f.write(reinterpret_cast(x.f.data()), (std::streamsize)(x.f.size() * 4)); + else + f.write(reinterpret_cast(x.u.data()), (std::streamsize)x.u.size()); + } + } + if (!pack_hyb(model, false, &blobs, &err)) return std::printf("ERR %s\n", err.c_str()), 0; + write_blobs(out / "blobs", blobs); + std::printf("OK %zu\n", blobs.size()); + return 0; + } + std::printf("ERR usage\n"); + return 2; +} diff --git a/clean/l/runtime/tests/test_l_runtime_frame.py b/clean/l/runtime/tests/test_l_runtime_frame.py index 193a0a5b..7b61650d 100644 --- a/clean/l/runtime/tests/test_l_runtime_frame.py +++ b/clean/l/runtime/tests/test_l_runtime_frame.py @@ -3,23 +3,32 @@ on the recording device of clean/tests/runtime_frame_trace.cpp (launches recorded with their arguments, not executed; no GPU, no HIP). The interface is clean/l/FORMATS.md. -- Launch for launch: Model L's frame is Model M's frame with step 2 (encoder) and step 4 (decoder) replaced: the - values into the halo map, the two per-pixel linears, the encoder block and the gather projection in place of - input_encoder; the decoder projection, the decoder block and l_coef in place of coef_decoder. The grids come from - FORMATS.md, the window offsets from the default table (clean/l/reference/table.py), the clamp-free block kernel - (_nc) from bit 0 of the Python packer's block blobs. With an empty build folder and with a full build's optional - code objects: Model L runs without pack12 (l_coef writes the base coefficient format). +- The 0.6.5 release routes (WG_L_FORM unset = hybrid_full): on HIP the scalar reference (the FP16-form kernels with + float32 chains, l_q7__hyb.co / l_q7w__hyb.co, for Model L's two blocks and the ten shared stages) and the C1/C2 + reconstruction (lc12/); on native D3D12 the fast route (every Swin / merge / expand kernel the manifest's "_hyb" + twin on the unfused M-form layout, the pack12 store, lc12/ reconstruction). Both decoder blocks read the encoder + block's map as their skip. The M-form L route is refused (ReleaseDefault); no test here runs it. +- Launch for launch (HIP): Model L's frame is Model M's frame with steps 2 to 5 replaced: the values into the halo + map, the two per-pixel linears, the encoder block and the gather projection in place of input_encoder; the ten + FP16-form shared stages in place of Model M's; the decoder projection, the decoder block and l_coef in place of + coef_decoder; the C1/C2 reconstruction in place of post_recon. The grids come from FORMATS.md and the kernels, the + window offsets from the default tables (clean/l/reference/table.py, clean/reference/swin.py). With an empty build + folder and with a full build's optional code objects: Model L on HIP runs without pack12 (l_coef writes the base + coefficient format). - Arguments: every map goes from the launch that writes it to the one that reads it (q -> x1 -> x2 -> x3 -> enc0's features -> enc1; dec1's output -> y -> y2 -> l_coef), no launch writes what it reads, every buffer holds its map (allocation sizes), the extents (npix, tw, th, H, W), the window offsets of both parities, the render size, padded grid and reset in the encoder's and decoder's parameters, and the history (the latent l_coef writes is what the next frame's encoder reads). Sizes aligned, non-aligned and changing from frame to frame, and a game reset. - A failing launch stops its frame; the next frame runs whole. -- Model L's code objects are loaded (l_enc_values.co under the tag lev, l_ops.co, l_swin.co), Model M's encoder and - decoder not. Model M runs unchanged in the build with Model L. +- Model L's code objects are loaded (l_enc_values.co under the tag lev, l_ops.co, l_swin.co, l_post.co, the hybrid + builds l_q7__hyb.co / l_q7w__hyb.co, l_post__lc12.co under the tag lc12), Model M's encoder and decoder not. Model M runs unchanged in the build with Model L. - Model data: only the converter's model.wgcm, checked against the preset-L map (a synthetic file and map here). A raw folder is refused, so is a file whose shared stages are Model M's (their digests are not the map's), and a missing file. The paths that keep dec1's map on chip, need Model M's encoder or the pack12 decoder are refused. +- Native D3D12: the shared stages, the pack12 store and the C1/C2 reconstruction, every argument by its buffer or + blob (the twins' blobs: Model M's packer on the hybrid values); the route stays unfused whatever the fusion option + or the manifest offers; each port the route needs refused by name when missing. Built with the host C++ compiler (clean/tests/host_cxx.py; skipped without one). @@ -43,7 +52,10 @@ import test_l_runtime_load as LL from clean.l.reference import oracle as L from clean.l.reference import table as T +from clean.l.runner import l_hyb_stage_pack as HSP from clean.l.runner import l_pack as LP +from clean.reference import swin as SW +from clean.runner import packing as P from clean.runner import synthetic_model as SM from clean.tests import host_cxx from clean.tests import test_runtime_frame_trace as FT @@ -113,42 +125,106 @@ def run(options="", frames=4, build_files=(), env=None, model=None, exe=None, fl return r.stdout.splitlines() -def blob_digests(): - """{blob name: SHA-256 hex} of Model L's blobs packed in Python (clean/l/runner/l_pack.py) from the test model's - parameters: the independent names of the weight buffers (the recorder logs what create uploads, with its hash).""" - blobs = LP.pack_model_l(L.random_params(np.random.default_rng(SEED))) +def blob_digests(route="hip"): + """{blob name: SHA-256 hex} of the blobs Model L's launches read on `route`, packed in Python from the test model's + parameters: the independent names of the weight buffers (the recorder logs what create uploads, with its hash). + hip (the scalar hybrid_full reference): Model L's own blobs (clean/l/runner/l_pack.py pack_model_l) with the + FP16-form blobs in place of the two block blobs (pack_q7_block: "enc0_block.q7", "dec0_block.q7"), the ten shared + stages' ".q7" / ".q7lin" and the tables "q7.tab"; d3d12 (the fast hybrid route): pack_model_l of the + parameters in the hybrid values (clean/l/runner/l_hyb_stage_pack.py hyb_params), the ten shared stages' blobs + packed from them by Model M's packer (clean/runner/packing.py pack_stage).""" + params = L.random_params(np.random.default_rng(SEED)) + if route == "d3d12": + hyb = HSP.hyb_params(params) + blobs = LP.pack_model_l(hyb) + blobs.update({s: P.pack_stage(s, hyb[s], "gfx12", model="M") for s in SHARED}) # the twins' stage blobs + else: + blobs = LP.pack_model_l(params) + del blobs["enc0_block"], blobs["dec0_block"] + for s, C, heads, lin in Q7_STAGES: + blobs[s + ".q7"] = LP.pack_q7_block(params[s], C, heads, 4 * C) + if lin: + blobs[s + ".q7lin"] = LP.pack_q7_linear(params[s][lin[0]], params[s][lin[1]]) + blobs["q7.tab"] = LP.q7_tables() # the post blob: s as float32, zero padding to 16 bytes (Model M's layout; the test model's s, LL.raw_model) blobs["post"] = struct.pack(" x1 "l_linear_32_32_f16": [n["l.narrow"], n["l.wide"], w["enc0.Win2"], npix], # x1 -> x2 - LIN: [n["l.narrow"], n["l.wide"], w["enc0.Win1"], w["enc0.Win2"], npix], # q -> x2 (x1 in registers) - "l_swin_block_c32_enc": [n["l.wide"], "0", n["l.x3"], w["enc0_block"], str(fh), str(fw)] + eo, # x2 -> x3 - # the block with the linears as its prologue: q -> x3, then the two linear blobs - ENCLIN_BASE: [n["l.narrow"], "0", n["l.x3"], w["enc0_block"], str(fh), str(fw)] + eo + - [w["enc0.Win1"], w["enc0.Win2"]], + # x2 -> x3 (no skip) + "l_enc_block": block_args(route, n["l.wide"], "0", n["l.x3"], w["enc0_block.q7" if hip else "enc0_block"], + tab, fh, fw, eo), "l_project_gather": [n["l.x3"], n["enc0.features"], w["enc0.project"], tw, th], # x3 -> enc1's input "l_dec_project": [n["dec1.out"], n["l.narrow"], w["dec0.project"], tw, th], # dec1's output -> y - "l_swin_block_c32_dec": [n["l.narrow"], "0", n["l.wide"], w["dec0_block"], str(fh), str(fw)] + do, # -> y2 + # y with the skip S = x3 -> y2 + "l_dec_block": block_args(route, n["l.narrow"], n["l.x3"], n["l.wide"], + w["dec0_block.q7" if hip else "dec0_block"], tab, fh, fw, do), + # the native hybrid route (speed lever 3): q in the narrow map's 20-byte form (lev/input_encoder_q20: the same + # arguments, q -> l.narrow), the encoder block with the linears as its prologue (x2 neither written nor read), + # the decoder block with the pack12 store as its epilogue (y2 not stored; A / C its pack12 images) + **({"l_enc_block_lin": [n["l.narrow"], "0", n["l.x3"], w["enc0_block"], str(fh), str(fw)] + eo + + [w["enc0.Win1"], w["enc0.Win2"]], + "l_dec_block_coef": [n["l.narrow"], n["l.x3"], "0", w["dec0_block"], str(fh), str(fw)] + do + + [n["dec0.A"], n["dec0.C"], n["dec0.latent"], w["dec0.coef"], str(info["rw"]), + str(info["rh"])]} if fused else {}), + # (l_coef_pack12: the same arguments; A and C its pack12 images) "l_coef": [n["l.wide"], n["dec0.A"], n["dec0.B"], n["dec0.C"], n["dec0.latent"], w["dec0.coef"], None], - # the folded form: the block's arguments (no map out), then the pack12 store's images, latent, blob, rectangle - FOLD_BASE: [n["l.narrow"], "0", "0", w["dec0_block"], str(fh), str(fw)] + do + - [n["dec0.A"], n["dec0.C"], n["dec0.latent"], w["dec0.coef"], str(info["rw"]), str(info["rh"])], # the reconstruction (Model M's post_recon arguments): colour, A, B, C, the previous history, this frame's # motion surfaces, the post blob, the new history, the reduced history, the display image, P, exposure pair "l_post_recon": [n["input.color"], n["dec0.A"], n["dec0.B"], n["dec0.C"], n["state.history_prev"], @@ -191,13 +275,22 @@ def expected_args(t, f, digests): def base(kernel): - """A Model L launch's kernel without its clamp-free suffix; the pack12 coefficient store (native D3D12) as l_coef, - whose arguments it shares.""" - k = kernel.replace("_nc", "") - post = ("l_post_recon_p12", "l_post_recon_ldr", "l_post_recon_p12_ldr") # pack12 and LDR forms - if k.startswith("lev/input_encoder") and k.endswith("_q20"): # the q20 forms (fusion le): the same arguments - return k[:-len("_q20")] - return "l_coef" if k == "l_coef_pack12" else "l_post_recon" if k in post else k + """A Model L launch's role: the blocks of both routes as l_enc_block / l_dec_block, the pack12 coefficient store + as l_coef (whose arguments it shares), every form of the reconstruction (C1/C2 lc12/, pack12, LDR) as + l_post_recon.""" + if kernel == ENC_LIN_HYB: + return "l_enc_block_lin" + if kernel == DEC_COEF_HYB: + return "l_dec_block_coef" + if kernel == "lev/input_encoder_q20": + return "lev/input_encoder" + if kernel in ENC_BLOCK.values(): + return "l_enc_block" + if kernel in DEC_BLOCK.values(): + return "l_dec_block" + k = kernel[len("lc12/"):] if kernel.startswith("lc12/") else kernel + post = ("l_post_recon", "l_post_recon_p12", "l_post_recon_ldr", "l_post_recon_p12_ldr") + return "l_coef" if k == "l_coef_pack12" else "l_post_recon" if k in post else kernel def model_m_post(names): @@ -207,12 +300,15 @@ def model_m_post(names): or n.startswith("pmo/") or n.endswith("_band")] -def check_roles(t, f, digests, l_post=True): - """Raises AssertionError naming every Model L launch argument that is not the expected buffer or blob.""" - exp = expected_args(t, f, digests) +def check_roles(t, f, digests, route, l_post=True): + """Raises AssertionError naming every Model L launch argument that is not the expected buffer or blob, every role + not launched, and any M-form L block or fusion launched.""" + exp = expected_args(t, f, digests, route) seen, bad = set(), [] for ln in f["launches"]: k = base(ln.kernel) + if ln.kernel in M_FORM_L: + bad.append("%s launched (the M-form L route is not offered)" % ln.kernel) if k not in exp: continue seen.add(k) @@ -222,23 +318,102 @@ def check_roles(t, f, digests, l_post=True): continue bad += ["%s argument %d: %s, expected %s" % (k, i, a, e) for i, (a, e) in enumerate(zip(ln.args, want)) if e is not None and a != e] - fold, lin, el = FOLD_BASE in seen, LIN in seen, ENCLIN_BASE in seen - pair_d, pair_l = ("l_swin_block_c32_dec", "l_coef"), ("l_linear_32_32_relu", "l_linear_32_32_f16") - pair_e = pair_l + ("l_swin_block_c32_enc",) - need = [k for k in L_KERNELS if not (fold and k in pair_d) and not (lin and k in pair_l) and - not (el and k in pair_e)] + ([FOLD_BASE] if fold else []) + ([LIN] if lin else []) + \ - ([ENCLIN_BASE] if el else []) - if el and (lin or set(pair_e) & seen): - bad.append("the encoder block with the linears and the linears or the plain block in one frame") - if fold and (set(pair_d) & seen): - bad.append("the folded decoder block and the pair in one frame") - if lin and (set(pair_l) & seen): - bad.append("the fused linears and the pair in one frame") + names = [ln.kernel for ln in f["launches"]] + need = L_KERNELS + if ENC_LIN_HYB in names: # the native hybrid route (lever 3): no separate linears; the fold on pack12 frames + need = tuple(k for k in L_KERNELS if k not in ("l_linear_32_32_relu", "l_linear_32_32_f16", "l_enc_block")) + need += ("l_enc_block_lin",) + if DEC_COEF_HYB in names: + need = tuple(k for k in need if k not in ("l_dec_block", "l_coef")) + ("l_dec_block_coef",) missing = [k for k in need if k not in seen and (l_post or k != "l_post_recon")] if bad or missing: raise AssertionError("; ".join(bad + ["not launched: " + k for k in missing])) +def shared_stages(t, f, route, digests=None): + """The ten shared stages between l_project_gather and l_dec_project, argument for argument (AssertionError): + HIP the FP16-form kernels (l_q7_stages: each block one 4 x 4 window per workgroup of 32 lanes, its ".q7" blob + and the tables; the merges and expansions their ".q7lin" blob), D3D12 the twins on the unfused layout (Model M's + block arguments and grids). The maps by the runtime's names; the offsets from the reference table + (clean/reference/swin.py window_offset); with digests, every FP16-form blob by its bytes.""" + ls, n, info = f["launches"], f["names"], f["info"] + fw, fh = cdiv(info["rw"], 32) * 32, cdiv(info["rh"], 32) * 32 + H2, W2, H4, W4, H8, W8 = fh // 2, fw // 2, fh // 4, fw // 4, fh // 8, fw // 8 + names = [ln.kernel for ln in ls] + a, b = names.index("l_project_gather"), names.index("l_dec_project") + got = ls[a + 1:b] + par = info["parity"] + bouts = [n["b%d.out" % i] for i in range(6)] + + def off(stage): + return [str(v) for v in SW.window_offset(stage, par)] + + w = (lambda blob: t.uploaded(digests[blob])) if digests else (lambda blob: None) + if route == "hip": + tab = w("q7.tab") + + def blk(k, x, skip, out, stage, H, W): + ox, oy = SW.window_offset(stage, par) + return (k, cdiv(W + ox, 4) * cdiv(H + oy, 4), 32, [x, skip, out, w(stage + ".q7"), tab, str(H), str(W)] + + off(stage)) + + def lin(k, x, out, stage, h, w_, grid): + return (k, grid, 32, [x, out, w(stage + ".q7lin"), str(h), str(w_)]) + + e2 = got[10].args[1] if len(got) > 10 and got[10].args else None # the expansions' maps (no name) + e1 = got[12].args[1] if len(got) > 12 and got[12].args else None + want = [blk("l_q7_block_c64", n["enc0.features"], "0", n["enc1.skip"], "enc1", H2, W2), + lin("l_q7_merge_c64", n["enc1.skip"], n["enc1.merged"], "enc1", H2, W2, (H2 // 2) * (W2 // 2)), + blk("l_q7_block_c96", n["enc1.merged"], "0", n["enc2.skip"], "enc2", H4, W4), + lin("l_q7_merge_c96", n["enc2.skip"], n["enc2.merged"], "enc2", H4, W4, (H4 // 2) * (W4 // 2))] + x = n["enc2.merged"] + for i in range(6): + want.append(blk("l_q7_block_c128", x, "0", bouts[i], "b%d" % i, H8, W8)) + x = bouts[i] + want += [lin("l_q7_expand_c128", bouts[5], e2, "dec2", H8, W8, H8 * W8), + blk("l_q7_block_c96_dec", e2, n["enc2.skip"], n["dec2.out"], "dec2", H4, W4), + lin("l_q7_expand_c96", n["dec2.out"], e1, "dec1", H4, W4, H4 * W4), + blk("l_q7_block_c64_dec", e1, n["enc1.skip"], n["dec1.out"], "dec1", H2, W2)] + if e2 in (None, "0") or e1 in (None, "0") or e1 == e2 or {e1, e2} & set(n.values()): + raise AssertionError("the expansions' maps: %s, %s" % (e2, e1)) + else: + def blk(k, x, skip, out, stage, H, W): + ox, oy = SW.window_offset(stage, par) + return (k, cdiv(cdiv(W + ox, 4) * cdiv(H + oy, 4), 8), 256, [x, skip, out, w(stage), str(H), str(W)] + + off(stage)) + + want = [blk("swin_block_c64_hyb", n["enc0.features"], "0", n["enc1.skip"], "enc1", H2, W2), + ("patch_merge_c64_hyb", None, None, None), + blk("swin_block_c96_hyb", n["enc1.merged"], "0", n["enc2.skip"], "enc2", H4, W4), + ("patch_merge_c96_hyb", None, None, None)] + # b0 ... b5: one launch (speed lever 4) of 128 workgroups; enc2's merged map in, b5.out out (tmp and the + # sync words: the runtime's scratch, not named), the six blobs, the even (b0) and odd (b1) blocks' offsets + want.append((BNECK_HYB, 128, 256, [n["enc2.merged"], bouts[5], None, None] + [w("b%d" % i) for i in range(6)] + + [str(H8), str(W8)] + off("b0") + off("b1") + ["128"])) + want += [("expand_c128_hyb", None, None, None), + blk("swin_block_c96_add_hyb", None, n["enc2.skip"], n["dec2.out"], "dec2", H4, W4), + ("expand_c96_hyb", None, None, None), + blk("swin_block_c64_add_hyb", None, n["enc1.skip"], n["dec1.out"], "dec1", H2, W2)] + bad = [] + if [ln.kernel for ln in got] != [k for k, _, _, _ in want]: + raise AssertionError("shared stages: %s, expected %s" % ([ln.kernel for ln in got], [k for k, *_ in want])) + for i, (ln, (k, grid, block, args)) in enumerate(zip(got, want)): + if grid is not None and (ln.grid, ln.block) != (grid, block): + bad.append("%s (launch %d): grid %s x %s, expected %s x %s" % (k, i, ln.grid, ln.block, grid, block)) + if args is not None: + if len(ln.args) != len(args): + bad.append("%s (launch %d): %d arguments, expected %d" % (k, i, len(ln.args), len(args))) + continue + bad += ["%s (launch %d) argument %d: %s, expected %s" % (k, i, j, a_, e) for j, (a_, e) in + enumerate(zip(ln.args, args)) if e is not None and a_ != e] + if route == "d3d12": # the maps the twin expansions write (no name, arguments not recorded): new maps + e2, e1 = got[6].args[0], got[8].args[0] + if e2 == e1 or {e2, e1} & ({"0"} | set(n.values())): + bad.append("the expanded maps: %s, %s" % (e2, e1)) + if bad: + raise AssertionError("; ".join(bad)) + + def declared_ports(): """[(kernel, code object)] of every D3D12 port declared in clean/dx12/kernels (the "@wg-kernel" lines that clean/dx12/build_dx12.py builds into the manifest): what a full build's manifest names.""" @@ -251,11 +426,15 @@ def declared_ports(): return out -# Model L's D3D12 kernels the runtime checks at create (runtime.cpp l_native_missing), with their code objects +# The hybrid route's D3D12 kernels the runtime checks at create besides the twins and the reconstruction +# (runtime.cpp l_native_missing), with their code objects L_D3D12 = [("lev/input_encoder", "l_enc_values.co"), ("l_linear_32_32_relu", "l_ops.co"), - ("l_linear_32_32_f16", "l_ops.co"), ("l_swin_block_c32_enc_nc", "l_swin.co"), ("l_project_gather", "l_ops.co"), - ("swin_block_c64_add_ex_nc", "swin.co"), ("l_dec_project", "l_ops.co"), ("l_swin_block_c32_dec_nc", "l_swin.co"), + ("l_linear_32_32_f16", "l_ops.co"), ("l_project_gather", "l_ops.co"), ("l_dec_project", "l_ops.co"), ("l_coef_pack12", "l_ops.co")] +HYB_TWINS = ("l_swin_block_c32_enc_hyb", "l_swin_block_c32_dec_hyb", "swin_block_c64_hyb", "patch_merge_c64_hyb", + "swin_block_c96_hyb", "patch_merge_c96_hyb", "swin_block_c128_hyb", "expand_c128_hyb", + "swin_block_c96_add_hyb", "expand_c96_hyb", "swin_block_c64_add_hyb") +LC12 = ("lc12/l_post_recon", "lc12/l_post_recon_p12", "lc12/l_post_recon_ldr", "lc12/l_post_recon_p12_ldr") def full_ports(): @@ -274,44 +453,65 @@ def test_analytic_post_declared(self): for k in ("w64/post_tf_f32_p12_diet_vsit", "w64/post_tf4_dl_f32_p12_diet_vsit"): self.assertIn((k, "post_w64.co"), have) + def test_release_route_declared(self): + """Every port of the release route (the hybrid twins, the C1/C2 reconstruction, Model L's own kernels and its + filters) is declared: a full build's manifest has them all.""" + have = [k for k, _ in declared_ports()] + need = [k for k, _ in L_D3D12] + list(HYB_TWINS) + list(LC12) + ["l_coef", "w64/post_filter_c1_s4", + "l_post_filter_ldr"] + self.assertEqual([k for k in need if k not in have], []) + def cdiv(a, b): return -(-a // b) -def nc_flags(): - blobs = LP.pack_model_l(L.random_params(np.random.default_rng(SEED))) - return [bool(int.from_bytes(blobs[n][:4], "little") & 1) for n in ("enc0_block", "dec0_block")] - - def expected_l(m_frame, parity, fh, fw): - """Model M's launches of one frame (on the unbanded post_recon route) with Model L's steps 2, 4 and 5 - (clean/l/FORMATS.md, "Kernels" and "Launch order of one frame"): kernel, grid, block.""" - nc = nc_flags() + """Model M's launches of one frame (on the unbanded post_recon route) with Model L's steps 2 to 5 on the HIP + reference route (clean/l/FORMATS.md, "Kernels" and "Launch order of one frame"; the FP16-form kernels): kernel, + grid, block. Model M's input encoder -> Model L's values, linears, encoder block and gather; Model M's ten shared + stages -> the FP16-form stages (l_q7_stages); its decoder -> Model L's projection, decoder block and l_coef; its + post_recon -> the C1/C2 reconstruction (same arguments and grid).""" npix, tokens = (fh + 4) * (fw + 4), (fh // 2) * (fw // 2) + H2, W2, H4, W4, H8, W8 = fh // 2, fw // 2, fh // 4, fw // 4, fh // 8, fw // 8 - def block(kind, offsets, i): - ox, oy = T.block_offset(T.TABLE, offsets, parity) - nwin = cdiv(fw + ox, 4) * cdiv(fh + oy, 4) - return ("l_swin_block_c32_%s%s" % (kind, "_nc" if nc[i] else ""), cdiv(nwin, 8), 256) + def nwin(H, W, ox, oy): + return cdiv(W + ox, 4) * cdiv(H + oy, 4) - out = [] + def lblock(k, offsets): + return (k, nwin(fh, fw, *T.block_offset(T.TABLE, offsets, parity)), 32) + + def sblock(k, stage, H, W): + return (k, nwin(H, W, *SW.window_offset(stage, parity)), 32) + + stages = [sblock("l_q7_block_c64", "enc1", H2, W2), ("l_q7_merge_c64", (H2 // 2) * (W2 // 2), 32), + sblock("l_q7_block_c96", "enc2", H4, W4), ("l_q7_merge_c96", (H4 // 2) * (W4 // 2), 32)] + \ + [sblock("l_q7_block_c128", "b%d" % i, H8, W8) for i in range(6)] + \ + [("l_q7_expand_c128", H8 * W8, 32), sblock("l_q7_block_c96_dec", "dec2", H4, W4), + ("l_q7_expand_c96", H4 * W4, 32), sblock("l_q7_block_c64_dec", "dec1", H2, W2)] + out, in_stages = [], False for k, g, b in m_frame: if k == "input_encoder": out += [("lev/input_encoder", cdiv(((fh + 4) // 2) * ((fw + 4) // 2), 128), 256), ("l_linear_32_32_relu", cdiv(npix, 128), 256), ("l_linear_32_32_f16", cdiv(npix, 128), 256), - block("enc", "first", 0), ("l_project_gather", cdiv(tokens, 128), 256)] + lblock("l_q7_block_c32_enc", "first"), ("l_project_gather", cdiv(tokens, 128), 256)] + stages + in_stages = True # Model M's shared stages follow, up to its decoder elif k == "coef_decoder": - out += [("l_dec_project", cdiv(tokens, 128), 256), block("dec", "second", 1), + out += [("l_dec_project", cdiv(tokens, 128), 256), lblock("l_q7_block_c32_dec", "second"), ("l_coef", cdiv(tokens, 128), 256)] - elif k.split("/")[-1] == "post_recon": # Model L's reconstruction, same arguments and grid - out.append(("l_post_recon", g, b)) + in_stages = False + elif in_stages: + continue + elif k.split("/")[-1] == "post_recon": # Model L's C1/C2 reconstruction, same arguments and grid + out.append(("lc12/l_post_recon", g, b)) else: out.append((k, g, b)) return out class Frame(unittest.TestCase): + """The HIP release route (the scalar hybrid_full reference) launch for launch against Model M's frame.""" + def check(self, build_files=(), m_env=None): # Model M on the route Model L's reconstruction takes: the unbanded post_recon, then the unbanded filter fh, fw = cdiv(RH, 32) * 32, cdiv(RW, 32) * 32 @@ -326,11 +526,14 @@ def check(self, build_files=(), m_env=None): self.assertIn("coef_decoder", names) # Model M's base decoder: the one Model L's l_coef stands in for self.assertEqual(sum(k.split("/")[-1] == "post_recon" for k in names), 1) self.assertEqual(l_, expected_l(m, i & 1, fh, fw), "frame %d" % i) - l_mods = ["l_enc_values.co as lev", "l_ops.co", "l_swin.co", "l_post.co"] + l_mods = ["l_enc_values.co as lev", "l_ops.co", "l_swin.co", "l_post.co", "l_q7__hyb.co", "l_q7w__hyb.co", + "l_post__lc12.co as lc12"] self.assertEqual([x for x in ll if x not in l_mods], [x for x in lm if x not in (km["enc0"], km["dec0"])]) for want in l_mods: self.assertIn(want, ll) self.assertEqual(kl.get("model"), "L") + self.assertEqual(kl.get("l_form"), "hybrid") + self.assertEqual(kl.get("l_post_c12"), "on") self.assertEqual(kl.get("post_pack"), "none") self.assertEqual(kl.get("model_map"), "same") # read from model.wgcm, checked against its map return fl @@ -339,8 +542,8 @@ def test_empty_build(self): self.check() def test_shipping_build(self): - # l_coef writes Model M's base coefficient format: Model L runs without pack12 (and so without the fused pack12 - # dec1 + dec0), compared with Model M without it (WG_POST_PACK=none) + # l_coef writes Model M's base coefficient format: Model L on HIP runs without pack12 (and so without the fused + # pack12 dec1 + dec0), compared with Model M without it (WG_POST_PACK=none) fl = self.check(FT.SHIPPING, {"WG_POST_PACK": "none"}) names = [k for k, _, _ in fl[0]] self.assertTrue(all(not n.startswith(("d0p12/", "d1p12/")) and "_p12" not in n for n in names), names) @@ -353,75 +556,127 @@ def test_model_m_unchanged(self): class Arguments(unittest.TestCase): - """The recorder's view of every Model L launch (clean/l/FORMATS.md, "Maps" and "Kernels").""" + """The recorder's view of every Model L launch (clean/l/FORMATS.md, "Maps" and "Kernels") on the release routes.""" - def check_frame(self, t, f, roles=True, l_post=True): + def check_frame_fused(self, t, f, roles=True, l_post=True): + """check_frame for the native hybrid route with its fused twins (speed lever 3): q (20-byte form) -> the encoder + block with the linears -> x3; y with the skip x3 -> the decoder block with the pack12 store (pack12 frames) or + the decoder block -> y2 -> l_coef (other frames). Returns (lev, coef) with coef the launch holding A / C / + latent at the l_coef positions 1 / 3 / 4 (a stand-in object for the folded store).""" info, ls = f["info"], f["launches"] rw, rh = info["rw"], info["rh"] fw, fh = cdiv(rw, 32) * 32, cdiv(rh, 32) * 32 npix, tw, th = (fh + 4) * (fw + 4), fw // 2, fh // 2 + names = [ln.kernel for ln in ls] + self.assertEqual([n for n in names if n in M_FORM_L], []) + self.assertEqual([n for n in names if n in ("l_linear_32_32_relu", "l_linear_32_32_f16", LIN)], []) k = {base(ln.kernel): (i, ln) for i, ln in enumerate(ls)} _, lev = k["lev/input_encoder"] - el = k[ENCLIN_BASE][1] if ENCLIN_BASE in k else None # the block with the linears (fusion le) - lin = k[LIN][1] if LIN in k else None - if not lin and not el: - _, relu = k["l_linear_32_32_relu"] - _, f16 = k["l_linear_32_32_f16"] - enc = el if el else k["l_swin_block_c32_enc"][1] + self.assertEqual(lev.kernel, "lev/input_encoder_q20") + _, enc = k["l_enc_block_lin"] gi, gather = k["l_project_gather"] di, dproj = k["l_dec_project"] - fold = FOLD_BASE in k - if fold: # one launch: the block with the coefficient store as its epilogue (no y2 map) - _, dec = k[FOLD_BASE] - coef = dec + q, x3, feat = lev.args[6], enc.args[2], gather.args[1] + self.assertEqual(enc.args[0], q) # the block reads q itself + self.assertEqual(enc.args[1], "0") + self.assertEqual(gather.args[0], x3) + self.assertEqual(ls[gi + 1].kernel, "swin_block_c64_hyb") + self.assertEqual(ls[gi + 1].args[0], feat) + dec1 = ls[di - 1] + self.assertEqual(dec1.kernel, "swin_block_c64_add_hyb") + y = dproj.args[1] + self.assertEqual(dproj.args[0], dec1.args[2]) + fold = "l_dec_block_coef" in k + if fold: + _, dec = k["l_dec_block_coef"] + self.assertEqual((dec.args[0], dec.args[1], dec.args[2]), (y, x3, "0")) + A, C, L = dec.args[8], dec.args[9], dec.args[10] + self.assertEqual(dec.args[12:14], [str(rw), str(rh)]) + + class _Coef: + args = [None, A, None, C, L] + coef = _Coef() + self.assertGreaterEqual(t.extent(A), 8 * rw * rh) + self.assertGreaterEqual(t.extent(C), 4 * rw * rh) + self.assertGreaterEqual(t.extent(L), 4 * rw * rh) else: - _, dec = k["l_swin_block_c32_dec"] + _, dec = k["l_dec_block"] _, coef = k["l_coef"] + self.assertEqual((dec.args[0], dec.args[1]), (y, x3)) + self.assertEqual(coef.args[0], dec.args[2]) + for a_, b_ in ((q, x3), (x3, feat), (dec1.args[2], y), (x3, y)): + self.assertNotEqual(a_, b_) + self.assertGreaterEqual(t.extent(q), 20 * npix) # the q20 map: 16 + 4 bytes a pixel (two planes) + self.assertGreaterEqual(t.extent(x3), 32 * fh * fw) + self.assertEqual(gather.args[3:5], [str(tw), str(th)]) + self.assertEqual(dproj.args[3:5], [str(tw), str(th)]) + for blk, kind in ((enc, "first"), (dec, "second")): + ox, oy = T.block_offset(T.TABLE, kind, info["parity"]) + self.assertEqual(blk.args[4:8], [str(fh), str(fw), str(ox), str(oy)]) + nwin = cdiv(fw + ox, 4) * cdiv(fh + oy, 4) + self.assertEqual((blk.grid, blk.block), (cdiv(nwin, 8), 256)) + e = lev.params(12) + self.assertEqual(e[:4], [rw, rh, fw, fh]) + self.assertEqual(e[5], info["reset"]) + post = [ln for ln in ls if base(ln.kernel) == "l_post_recon"] + self.assertEqual(len(post), 1 if l_post else 0) + if l_post: + self.assertTrue(post[0].kernel.startswith("lc12/"), post[0].kernel) + digests = blob_digests("d3d12") if roles else None + shared_stages(t, f, "d3d12", digests) + if roles: + check_roles(t, f, digests, "d3d12", l_post) + return lev, coef + + def check_frame(self, t, f, roles=True, l_post=True): + route = route_of(t) + if route == "d3d12" and any(ln.kernel == ENC_LIN_HYB for ln in f["launches"]): + return Arguments.check_frame_fused(self, t, f, roles, l_post) + info, ls = f["info"], f["launches"] + rw, rh = info["rw"], info["rh"] + fw, fh = cdiv(rw, 32) * 32, cdiv(rh, 32) * 32 + npix, tw, th = (fh + 4) * (fw + 4), fw // 2, fh // 2 + names = [ln.kernel for ln in ls] + self.assertEqual([n for n in names if n in M_FORM_L], []) # no M-form L block, no fusion on it + k = {base(ln.kernel): (i, ln) for i, ln in enumerate(ls)} + _, lev = k["lev/input_encoder"] + _, relu = k["l_linear_32_32_relu"] + _, f16 = k["l_linear_32_32_f16"] + _, enc = k["l_enc_block"] + gi, gather = k["l_project_gather"] + di, dproj = k["l_dec_project"] + _, dec = k["l_dec_block"] + _, coef = k["l_coef"] + self.assertEqual((enc.kernel, dec.kernel), (ENC_BLOCK[route], DEC_BLOCK[route])) + h = 5 if route == "hip" else 4 # the block's H (the FP16-form blocks take the tables after w) q, x3, feat = lev.args[6], enc.args[2], gather.args[1] # the chain of maps (no launch reads what it writes) - if el: # one launch for the linears and the block: no x1 or x2 map - x1 = x2 = None - self.assertEqual(el.args[0], q) - elif lin: # one launch, x1 in registers: no map - x1, x2 = None, lin.args[1] - self.assertEqual(lin.args[0], q) - self.assertEqual(lin.args[4], str(npix)) - else: - x1, x2 = relu.args[1], f16.args[1] - self.assertEqual(relu.args[0], q) - self.assertEqual(f16.args[0], x1) - self.assertEqual(relu.args[3], str(npix)) - self.assertEqual(f16.args[3], str(npix)) - if not el: - self.assertEqual(enc.args[0], x2) - self.assertEqual(enc.args[1], "0") # xadd: none + x1, x2 = relu.args[1], f16.args[1] + self.assertEqual(relu.args[0], q) + self.assertEqual(f16.args[0], x1) + self.assertEqual(relu.args[3], str(npix)) + self.assertEqual(f16.args[3], str(npix)) + self.assertEqual(enc.args[0], x2) + self.assertEqual(enc.args[1], "0") # the encoder block: no skip self.assertEqual(gather.args[0], x3) - self.assertTrue(ls[gi + 1].kernel.startswith("swin_block_c64")) + self.assertEqual(ls[gi + 1].kernel, "l_q7_block_c64" if route == "hip" else "swin_block_c64_hyb") self.assertEqual(ls[gi + 1].args[0], feat) # enc1 reads Model L's features - dec1 = [ln for ln in ls[:di] if ln.kernel.startswith("swin_block_c64")][-1] + dec1 = ls[di - 1] # dec1's block, right before the decoder + self.assertEqual(dec1.kernel, "l_q7_block_c64_dec" if route == "hip" else "swin_block_c64_add_hyb") y, y2 = dproj.args[1], dec.args[2] self.assertEqual(dproj.args[0], dec1.args[2]) # the decoder reads dec1's output self.assertEqual(dec.args[0], y) - self.assertEqual(dec.args[1], "0") - if fold: - self.assertEqual(y2, "0") # the map is not written - else: - self.assertEqual(coef.args[0], y2) - for a, b in ((q, x1), (x1, x2), (q, x2 if lin else None), (x2, x3), (q, x3), (x3, feat), (dec1.args[2], y), - (y, y2)): - if a is not None and b is not None: - self.assertNotEqual(a, b) - if not lin and not el: - self.assertEqual(q, x2) # the pair: x2 over q (q is dead once x1 is written) + self.assertEqual(dec.args[1], x3) # the decoder block's skip S: the encoder block's map + self.assertEqual(coef.args[0], y2) + for a, b in ((q, x1), (x1, x2), (x2, x3), (x3, feat), (dec1.args[2], y), (y, y2), (x3, y), (x3, y2)): + self.assertNotEqual(a, b) + self.assertEqual(q, x2) # x2 over q (q is dead once x1 is written) # every buffer holds its map (FORMATS.md "Maps") - maps = ((q, 32 * npix), (x3, 32 * fh * fw), (feat, 64 * tw * th), (y, 32 * fh * fw)) + \ - (() if el else ((x2, 64 * npix),)) + (() if fold else ((y2, 64 * fh * fw),)) + \ - (() if lin or el else ((x1, 32 * npix),)) - if fold: # pack12 image A (8 bytes a pixel), image C (4), the new latent - coefs = ((coef.args[8], 8 * rw * rh), (coef.args[9], 4 * rw * rh), (coef.args[10], 4 * rw * rh)) - else: - coefs = ((coef.args[1], 4 * rw * rh), (coef.args[2], 4 * rw * rh), (coef.args[3], 8 * rw * rh), - (coef.args[4], 4 * rw * rh)) # A, B, C, the new latent: Model M's formats + maps = ((q, 32 * npix), (x1, 32 * npix), (x2, 64 * npix), (x3, 32 * fh * fw), (feat, 64 * tw * th), + (y, 32 * fh * fw), (y2, 64 * fh * fw)) + p12 = coef.kernel == "l_coef_pack12" # pack12 image A: 8 bytes a pixel, C 4; Model M's base: A 4, C 8 + coefs = ((coef.args[1], (8 if p12 else 4) * rw * rh), (coef.args[2], 4 * rw * rh), + (coef.args[3], (4 if p12 else 8) * rw * rh), (coef.args[4], 4 * rw * rh)) for buf, need in maps + coefs: self.assertGreaterEqual(t.extent(buf), need, buf) # extents, offsets, parameters @@ -429,30 +684,36 @@ def check_frame(self, t, f, roles=True, l_post=True): self.assertEqual(dproj.args[3:5], [str(tw), str(th)]) for blk, kind in ((enc, "first"), (dec, "second")): ox, oy = T.block_offset(T.TABLE, kind, info["parity"]) - self.assertEqual(blk.args[4:8], [str(fh), str(fw), str(ox), str(oy)]) + self.assertEqual(blk.args[h:h + 4], [str(fh), str(fw), str(ox), str(oy)]) + nwin = cdiv(fw + ox, 4) * cdiv(fh + oy, 4) + self.assertEqual((blk.grid, blk.block), (nwin, 32) if route == "hip" else (cdiv(nwin, 8), 256)) + if route == "hip": + self.assertEqual(enc.args[4], dec.args[4]) # the FP16-form tables + self.assertTrue(enc.args[4].startswith("b")) e = lev.params(12) self.assertEqual(e[:4], [rw, rh, fw, fh]) self.assertEqual(e[5], info["reset"]) - if fold: - self.assertEqual(coef.args[12:14], [str(rw), str(rh)]) - else: - self.assertEqual(coef.params(6), [rw, rh, fw, fh, info["reset"]]) + self.assertEqual(coef.params(6), [rw, rh, fw, fh, info["reset"]]) post = [ln for ln in ls if base(ln.kernel) == "l_post_recon"] self.assertEqual(len(post), 1 if l_post else 0) if l_post: + self.assertTrue(post[0].kernel.startswith("lc12/"), post[0].kernel) # the C1/C2 reconstruction self.assertEqual(post[0].params(12), [rw, rh, (3 * 120 + 1) // 2, (3 * 72 + 1) // 2, info["reset"]]) self.assertEqual(post[0].grid, cdiv((3 * 120 + 1) // 2, 32) * cdiv(cdiv((3 * 72 + 1) // 2, 2), 8)) nxt = ls[ls.index(post[0]) + 1].kernel # the output filter next (LDR: Model L's LDR output stage) self.assertTrue(nxt.endswith("post_filter_c1_s4") or nxt == "l_post_filter_ldr", nxt) self.assertTrue(lev.args[11].startswith("b")) # the values blob: an allocation + digests = blob_digests(route) if roles else None + shared_stages(t, f, route, digests) if roles: # every argument against the named buffers and blobs - check_roles(t, f, blob_digests(), l_post) + check_roles(t, f, digests, route, l_post) return lev, coef def test_sizes_reset_history(self): sizes = "--sizes=96x96,96x96,80x48,80x48,80x48,64x64" # aligned, non-aligned, a smaller aligned size t = FT.Trace(run("model=L", frames=6, rw=96, rh=96, flags=(sizes, "--reset=4"))) self.assertEqual(t.create, "create ok model L") + self.assertEqual(t.unknown(), []) # first frame, a size change, the game's reset, a size change self.assertEqual([f["info"]["reset"] for f in t.frames], [1, 0, 1, 0, 1, 1]) prev = None @@ -477,24 +738,29 @@ def test_steady_frames(self): prev = (lev, coef) def test_failed_launch_stops_the_frame(self): - t = FT.Trace(run("model=L", frames=3, flags=("--fail=l_coef@1",))) - ok0, bad, ok2 = t.frames - self.assertEqual(bad["failed"], "l_coef") - self.assertIn("injected launch failure: l_coef", bad["error"]) - names = [ln.kernel for ln in bad["launches"]] - self.assertEqual(names, [ln.kernel for ln in ok0["launches"]][:len(names)]) - self.assertTrue(names[-1].startswith("l_swin_block_c32_dec")) # stopped right before l_coef - self.assertIsNone(ok2["error"]) - self.check_frame(t, ok2) + for kernel, before in (("l_coef", "l_q7_block_c32_dec"), ("l_q7_block_c128", "l_q7_merge_c96"), + ("lc12/l_post_recon", "l_coef")): + with self.subTest(kernel=kernel): + t = FT.Trace(run("model=L", frames=3, flags=("--fail=%s@1" % kernel,))) + ok0, bad, ok2 = t.frames + self.assertEqual(bad["failed"], kernel) + self.assertIn("injected launch failure: " + kernel, bad["error"]) + names = [ln.kernel for ln in bad["launches"]] + self.assertEqual(names, [ln.kernel for ln in ok0["launches"]][:len(names)]) + self.assertEqual(names[-1], before) # stopped right before the failing launch + self.assertIsNone(ok2["error"]) + self.check_frame(t, ok2) class Roles(unittest.TestCase): """Buffer identities: each Model L launch's weight blob, inputs and outputs are the named ones, and a mix-up of two - of them fails even where the map chain, sizes and argument kinds stay as they were.""" + of them fails even where the map chain, sizes and argument kinds stay as they were; on both release routes.""" - MIXUPS = ( # (kernel, argument, kernel, argument) swapped in a recorded frame + MIXUPS = ( # (kernel role, argument, kernel role, argument) swapped in a recorded frame ("l_linear_32_32_relu", 2, "l_linear_32_32_f16", 2), # enc0.Win1 <-> enc0.Win2 - ("l_swin_block_c32_enc", 3, "l_swin_block_c32_dec", 3), # enc0_block <-> dec0_block + ("l_enc_block", 3, "l_dec_block", 3), # the encoder's block blob <-> the decoder's + ("l_enc_block", 1, "l_dec_block", 1), # no skip <-> the decoder's skip S (x3) + ("l_dec_block", 0, "l_dec_block", 1), # y <-> the skip S ("l_project_gather", 2, "l_dec_project", 2), # enc0.project <-> dec0.project ("lev/input_encoder", 11, "l_coef", 5), # enc0.values <-> dec0.coef ("l_coef", 1, "l_coef", 2), # A <-> B @@ -507,31 +773,86 @@ class Roles(unittest.TestCase): ("l_post_recon", 8, "l_coef", 5), # post blob <-> dec0.coef ("l_post_recon", 1, "l_post_recon", 3), # A <-> C ) + Q7_MIXUPS = (("l_enc_block", 4, "l_dec_block", 3),) # the tables <-> the decoder's block blob + # the native route's fused twins (speed lever 3): the same mix-ups on their arguments (the encoder block with the + # linears: q, -, x3, block blob, H, W, ox, oy, Win1, Win2; the decoder block with the store: y, S, -, block blob, + # H, W, ox, oy, A, C, latent, dec0.coef, rw, rh) + FUSED_MIXUPS = ( + ("l_enc_block_lin", 8, "l_enc_block_lin", 9), # enc0.Win1 <-> enc0.Win2 + ("l_enc_block_lin", 3, "l_dec_block_coef", 3), # the encoder's block blob <-> the decoder's + ("l_enc_block_lin", 1, "l_dec_block_coef", 1), # no skip <-> the decoder's skip S (x3) + ("l_dec_block_coef", 0, "l_dec_block_coef", 1), # y <-> the skip S + ("l_project_gather", 2, "l_dec_project", 2), # enc0.project <-> dec0.project + ("lev/input_encoder", 11, "l_dec_block_coef", 11), # enc0.values <-> dec0.coef + ("l_dec_block_coef", 8, "l_dec_block_coef", 9), # A <-> C + ("l_dec_block_coef", 9, "l_dec_block_coef", 10), # C <-> the new latent + ("lev/input_encoder", 0, "lev/input_encoder", 1), # colour <-> depth + ("lev/input_encoder", 4, "lev/input_encoder", 5), # previous latent <-> motion history + ("lev/input_encoder", 7, "lev/input_encoder", 8), # m_near <-> m_far + ("l_enc_block_lin", 0, "l_enc_block_lin", 2), # q's map <-> x3's map + ("l_post_recon", 4, "l_post_recon", 9), # history read <-> history written + ("l_post_recon", 8, "l_dec_block_coef", 11), # post blob <-> dec0.coef + ("l_post_recon", 1, "l_post_recon", 3), # A <-> C + ) @classmethod def setUpClass(cls): - cls.trace = FT.Trace(run("model=L", frames=3, rw=96, rh=64, flags=("--sizes=96x64,80x48,80x48",))) + sizes = ("--sizes=96x64,80x48,80x48",) + cls.traces = {"hip": FT.Trace(run("model=L", frames=3, rw=96, rh=64, flags=sizes)), + "d3d12": FT.Trace(run("model=L", frames=3, rw=96, rh=64, build_files=FT.SHIPPING, + flags=("--backend=3",) + sizes, manifest=full_ports()))} def test_every_argument_named(self): - t, digests = self.trace, blob_digests() - addrs = [t.uploaded(d) for d in digests.values()] - self.assertNotIn(None, addrs) - self.assertEqual(len(set(addrs)), len(addrs)) # nine distinct blobs, each in its own allocation - for f in t.frames: - self.assertEqual(len(f["names"]) >= 30, True, sorted(f["names"])) - check_roles(t, f, digests) + for route, t in self.traces.items(): + with self.subTest(route=route): + self.assertEqual(route_of(t), route) + self.assertEqual(t.unknown(), []) + digests = blob_digests(route) + addrs = [t.uploaded(d) for d in digests.values()] + self.assertNotIn(None, addrs) + self.assertEqual(len(set(addrs)), len(addrs)) # distinct blobs, each in its own allocation + for f in t.frames: + self.assertEqual(len(f["names"]) >= 30, True, sorted(f["names"])) + check_roles(t, f, digests, route) + shared_stages(t, f, route, digests) def test_mixups_fail(self): - digests = blob_digests() - for ka, ia, kb, ib in self.MIXUPS: - with self.subTest(mixup=(ka, ia, kb, ib)): - t = copy.deepcopy(self.trace) - f = t.frames[1] - ls = {ln.kernel.replace("_nc", ""): ln for ln in f["launches"]} - a, b = ls[ka], ls[kb] - a.args[ia], b.args[ib] = b.args[ib], a.args[ia] - with self.assertRaises(AssertionError): - check_roles(t, f, digests) + for route, t0 in self.traces.items(): + digests = blob_digests(route) + mixups = self.FUSED_MIXUPS if route == "d3d12" else self.MIXUPS + self.Q7_MIXUPS + for ka, ia, kb, ib in mixups: + with self.subTest(route=route, mixup=(ka, ia, kb, ib)): + t = copy.deepcopy(t0) + f = t.frames[1] + ls = {base(ln.kernel): ln for ln in f["launches"]} + a, b = ls[ka], ls[kb] + a.args[ia], b.args[ib] = b.args[ib], a.args[ia] + with self.assertRaises(AssertionError): + check_roles(t, f, digests, route) + + def test_shared_stage_mixups_fail(self): + """The shared stages' check catches a swapped map or blob (HIP: the FP16-form blobs by their bytes).""" + for route, t0 in self.traces.items(): + digests = blob_digests(route) + # (launch, argument) pairs after l_project_gather; the d3d12 route runs b0 ... b5 as one launch (speed lever + # 4: positions 0 c64, 2 c96, 4 the bottleneck, 6 c96_add, 8 c64_add), swapped inside it too: its input and + # output maps, b0's and b5's blobs + swaps = (((0, 2, 2, 2), (4, 0, 5, 0), (11, 1, 13, 1), (4, 3, 5, 3), (0, 3, 13, 3)) if route != "d3d12" else + ((0, 2, 2, 2), (4, 0, 4, 1), (6, 1, 8, 1), (4, 4, 4, 9), (0, 3, 8, 3), (2, 2, 4, 1))) + for i, ia, j, ib in swaps: + with self.subTest(route=route, swap=(i, ia, j, ib)): + t = copy.deepcopy(t0) + f = t.frames[1] + names = [ln.kernel for ln in f["launches"]] + st = f["launches"][names.index("l_project_gather") + 1:] + a, b = st[i], st[j] + if not a.args or not b.args: # a twin merge / expand: arguments not recorded + continue + a.args[ia], b.args[ib] = b.args[ib], a.args[ia] + if a.args == b.args or a.args[ia] == b.args[ib]: + continue + with self.assertRaises(AssertionError): + shared_stages(t, f, route, digests) def test_swapped_weights_in_the_runtime(self): """The review's example in the runtime itself: enc0.Win1 and enc0.Win2 swapped in its source. The map chain, @@ -549,35 +870,38 @@ def test_swapped_weights_in_the_runtime(self): for s in FT.SOURCES] + L_SOURCES exe = host_cxx.build("runtime_frame_trace_lmut", sources, ["WG_NO_MODEL_K", "WG_MODEL_L"], include_dirs=[os.path.join(host_cxx.ROOT, "clean", "runtime")]) - t = FT.Trace(run("model=L", frames=2, exe=exe)) - digests = blob_digests() - for f in t.frames: - Arguments.check_frame(self, t, f, roles=False) - with self.assertRaises(AssertionError) as cm: - check_roles(t, f, digests) - msg = str(cm.exception) - self.assertIn("l_linear_32_32_relu argument 2", msg) - self.assertIn("l_linear_32_32_f16 argument 2", msg) - self.assertEqual(msg.count("argument"), 2, msg) + for route, kw in (("hip", {}), ("d3d12", dict(build_files=FT.SHIPPING, flags=("--backend=3",), + manifest=full_ports()))): + with self.subTest(route=route): + t = FT.Trace(run("model=L", frames=2, exe=exe, **kw)) + self.assertEqual(route_of(t), route) + digests = blob_digests(route) + for f in t.frames: + Arguments.check_frame(self, t, f, roles=False) + with self.assertRaises(AssertionError) as cm: + check_roles(t, f, digests, route) + msg = str(cm.exception) + if route == "d3d12": # the native route's encoder block takes both blobs (speed lever 3) + self.assertIn("l_enc_block_lin argument 8", msg) + self.assertIn("l_enc_block_lin argument 9", msg) + else: + self.assertIn("l_linear_32_32_relu argument 2", msg) + self.assertIn("l_linear_32_32_f16 argument 2", msg) + self.assertEqual(msg.count("argument"), 2, msg) class D3D12(unittest.TestCase): - """Model L on native D3D12 (backend 3): Model M's native frame with Model L's encoder, decoder, pack12 coefficient - store and its own reconstruction (no Model M post route); each missing D3D12 port refused by name, never - replaced.""" + """Model L on native D3D12 (backend 3), the release route: Model M's native frame layout, unfused, with every Swin / + merge / expand kernel its hybrid twin, Model L's encoder and decoder, the pack12 coefficient store and the C1/C2 + reconstruction (no Model M post route); each missing D3D12 port refused by name, never replaced.""" def l_run(self, ports, frames=3, flags=(), **kw): return run("model=L", frames=frames, build_files=FT.SHIPPING, flags=("--backend=3",) + tuple(flags), manifest=ports, **kw) - def test_missing_ports_named(self): - have = [k for k, _ in declared_ports()] - missing = [k for k, _ in L_D3D12 if k not in have] + def test_declared_build_creates(self): got = self.l_run(declared_ports(), frames=1) - if not missing: - self.assertEqual(got[0], "create ok model L") - return - self.assertEqual(got, ["ERR create: Model L on native D3D12: not yet ported to D3D12: " + ", ".join(missing)]) + self.assertEqual(got[0], "create ok model L") def test_each_port_named(self): for k, _ in L_D3D12: @@ -585,94 +909,79 @@ def test_each_port_named(self): got = self.l_run([kc for kc in full_ports() if kc[0] != k], frames=1) self.assertEqual(got, ["ERR create: Model L on native D3D12: not yet ported to D3D12: " + k]) - def check_d3d12_frame(self, t, f): - """The #111 checks (map chain, extents, offsets, parameters, every argument's buffer or blob, Model L's - reconstruction and the filter after it), then what the native frame adds: the pack12 store's images, dec1's own - block, l_post_recon_p12 right after the store, and no Model M post kernel.""" + def test_m_form_ports_not_needed(self): + """The M-form L blocks, dec1's M-form block and the fused L kernels are not the release route's: a manifest + without them creates and traces the same frames.""" + drop = ("l_swin_block_c32_enc_nc", "l_swin_block_c32_dec_nc", "swin_block_c64_add_ex_nc", FOLD, LIN, ENCLIN) + a = same_frames(self.l_run(full_ports(), frames=2, rw=96, rh=64)) + b = same_frames(self.l_run([kc for kc in full_ports() if kc[0] not in drop], frames=2, rw=96, rh=64)) + self.assertEqual(a[0], "create ok model L") + self.assertEqual(b, a) + + def check_d3d12_frame(self, t, f, sfx=""): + """The Arguments checks (map chain, extents, offsets, parameters, every argument's buffer or blob, the shared + stages, the C1/C2 reconstruction and the filter after it), then what the native frame adds: the twins only (no + M-form Swin, merge or expand kernel), the pack12 store's images and lc12/l_post_recon_p12 right after it (a + frame the pack12 kernels do not take: l_coef and lc12/l_post_recon), no Model M post kernel.""" info = f["info"] rw, rh = info["rw"], info["rh"] names = [ln.kernel for ln in f["launches"]] - fold = FOLD in names - if fold: - self.assertNotIn("l_coef_pack12", names) - self.assertNotIn("l_swin_block_c32_dec_nc", names) - self.assertEqual(names[names.index(FOLD) - 1], "l_dec_project") - else: - self.assertIn("l_coef_pack12", names) - if ENCLIN in names: - self.assertNotIn(LIN, names) - self.assertNotIn("l_linear_32_32_relu", names) - self.assertNotIn("l_swin_block_c32_enc_nc", names) - self.assertTrue(names[names.index(ENCLIN) - 1].startswith("lev/input_encoder")) - elif LIN in names: - self.assertNotIn("l_linear_32_32_relu", names) - self.assertNotIn("l_linear_32_32_f16", names) - self.assertTrue(names[names.index(LIN) - 1].startswith("lev/input_encoder")) - else: - self.assertIn("l_linear_32_32_relu", names) - self.assertNotIn("l_coef", names) + fold = DEC_COEF_HYB in names # pack12 frames: the decoder block with the store (speed lever 3) + p12 = fold or "l_coef_pack12" in names + self.assertEqual(int(fold) + int("l_coef_pack12" in names) + int("l_coef" in names), 1) # one store + self.assertNotIn("l_coef_pack12", names) # (the native route folds every pack12 store) self.assertTrue(all(not n.startswith(("d1p12/", "d0p12/")) and n != "input_encoder" for n in names), names) - dec1 = [ln for ln in f["launches"] if ln.kernel.startswith("swin_block_c64_add")] - self.assertEqual([ln.kernel for ln in dec1], ["swin_block_c64_add_ex_nc"]) # dec1's own block, not fused + stages = [n for n in names if n.split("/")[-1].startswith(("swin_", "l_swin_", "patch_merge", "expand"))] + self.assertEqual(stages, [ENC_LIN_HYB] + list(TWIN_STAGES) + [DEC_COEF_HYB if fold else "l_swin_block_c32_dec_hyb"]) + self.assertEqual([n for n in names if "l_q7" in n], []) lev, coef = Arguments.check_frame(self, t, f) - a, c = (coef.args[8], coef.args[9]) if fold else (coef.args[1], coef.args[3]) - self.assertGreaterEqual(t.extent(a), 8 * rw * rh) # pack12 image A: 8 bytes per pixel - self.assertGreaterEqual(t.extent(c), 4 * rw * rh) # pack12 image C: 4 bytes per pixel - i = names.index(FOLD if fold else "l_coef_pack12") - self.assertEqual(names[i + 1], "l_post_recon_p12", names[i + 1:]) + if p12: + self.assertGreaterEqual(t.extent(coef.args[1]), 8 * rw * rh) # pack12 image A: 8 bytes per pixel + self.assertGreaterEqual(t.extent(coef.args[3]), 4 * rw * rh) # pack12 image C: 4 bytes per pixel + i = names.index(DEC_COEF_HYB if fold else "l_coef") + self.assertEqual(names[i + 1], ("lc12/l_post_recon_p12" if p12 else "lc12/l_post_recon") + sfx, names[i + 1:]) self.assertEqual(model_m_post(names), []) return lev, coef def test_frame(self): - """Sizes aligned, non-aligned and changing, a game reset; history handed from frame to frame.""" + """Sizes aligned, non-aligned and changing, a game reset; history handed from frame to frame. The route is the + unfused one whatever the fusion option asks or the manifest offers.""" # upscaling sizes the pack12 post path takes (D = 180 x 108): aligned, non-aligned, a change of size sizes = "--sizes=96x64,96x64,80x48,80x48,80x48,77x45" - t = FT.Trace(self.l_run(full_ports(), frames=6, rw=96, rh=64, flags=(sizes, "--reset=4"))) + lines = self.l_run(full_ports(), frames=6, rw=96, rh=64, flags=(sizes, "--reset=4")) + t = FT.Trace(lines) self.assertEqual(t.create, "create ok model L") self.assertEqual(t.unknown(), []) + self.assertEqual(t.kernels.get("swin_fused"), "none") self.assertEqual([f["info"]["reset"] for f in t.frames], [1, 0, 1, 0, 1, 1]) self.assertEqual(len({f["info"]["parity"] for f in t.frames}), 2) + prev = None + for f in t.frames: + self.assertIsNone(f["error"]) + self.assertIn(DEC_COEF_HYB, [ln.kernel for ln in f["launches"]]) + lev, coef = self.check_d3d12_frame(t, f) + if prev: + self.assertEqual(lev.args[4], latent_out(prev[1])) # the latent the last frame's store wrote + self.assertEqual(lev.args[5], prev[0].args[7]) # its near motion + prev = (lev, coef) without = lambda *ks: [kc for kc in full_ports() if kc[0] not in ks] # noqa: E731 - for env, ports, fold, lin, el in ((None, full_ports(), True, False, True), - (NO_FOLD, full_ports(), False, False, False), - (None, without(FOLD), False, False, True), - (NO_LIN, full_ports(), True, False, False), - (NO_ENCLIN, full_ports(), True, True, False), - (None, without(ENCLIN), True, True, False), - (None, without(ENCLIN, LIN), True, False, False), - (None, without(LIN), True, False, True), - (None, without("lev/input_encoder_q20"), True, True, False)): - with self.subTest(fusion=env, fold_in_manifest=any(kc[0] == FOLD for kc in ports), - lin_in_manifest=any(kc[0] == LIN for kc in ports), - enclin_in_manifest=any(kc[0] == ENCLIN for kc in ports)): - # the fusions (lc, ll, le: the default; le over ll) and their fallbacks: without the fusion, or without - # the kernel - t = FT.Trace(self.l_run(ports, frames=6, rw=96, rh=64, flags=(sizes, "--reset=4"), env=env)) - self.assertEqual(t.create, "create ok model L") - self.assertEqual(t.unknown(), []) - self.assertEqual([f["info"]["reset"] for f in t.frames], [1, 0, 1, 0, 1, 1]) - self.assertEqual(len({f["info"]["parity"] for f in t.frames}), 2) - prev = None - for f in t.frames: - self.assertIsNone(f["error"]) - self.assertEqual(FOLD in [ln.kernel for ln in f["launches"]], fold) - self.assertEqual(LIN in [ln.kernel for ln in f["launches"]], lin) - self.assertEqual(ENCLIN in [ln.kernel for ln in f["launches"]], el) - lev, coef = self.check_d3d12_frame(t, f) - if prev: - self.assertEqual(lev.args[4], latent_out(prev[1])) # the latent the last frame's store wrote - self.assertEqual(lev.args[5], prev[0].args[7]) # its near motion - prev = (lev, coef) + for env, ports in ((NO_FOLD, full_ports()), (NO_LIN, full_ports()), (NO_ENCLIN, full_ports()), + ({"WG_D3D12_FUSION": "none"}, full_ports()), + (None, without(FOLD, LIN, ENCLIN))): # the M-form fusions' kernels: not needed + with self.subTest(fusion=env, ports=len(ports)): + got = self.l_run(ports, frames=6, rw=96, rh=64, flags=(sizes, "--reset=4"), env=env) + self.assertEqual(same_frames(got), same_frames(lines)) def test_failed_launch_stops_the_frame(self): - for env, kernel, before in ((None, FOLD, "l_dec_project"), (NO_FOLD, "l_coef_pack12", "l_swin_block_c32_dec")): + for kernel, before in ((DEC_COEF_HYB, "l_dec_project"), (BNECK_HYB, "patch_merge_c96_hyb"), + ("lc12/l_post_recon_p12", DEC_COEF_HYB)): with self.subTest(kernel=kernel): - t = FT.Trace(self.l_run(full_ports(), frames=3, flags=("--fail=%s@1" % kernel,), env=env)) + t = FT.Trace(self.l_run(full_ports(), frames=3, flags=("--fail=%s@1" % kernel,))) ok0, bad, ok2 = t.frames self.assertEqual(bad["failed"], kernel) names = [ln.kernel for ln in bad["launches"]] self.assertEqual(names, [ln.kernel for ln in ok0["launches"]][:len(names)]) - self.assertTrue(names[-1].startswith(before)) + self.assertEqual(names[-1], before) self.assertIsNone(ok2["error"]) self.check_d3d12_frame(t, ok2) @@ -689,21 +998,21 @@ def test_hip_unchanged(self): self.assertTrue(run("model=L", build_files=FT.SHIPPING, env={"WG_POST_PACK": "p12"})[0].startswith( "ERR create: Model L: post_pack=p12")) - # Model L's reconstruction on native D3D12, HDR and LDR colour: its D3D12 ports, then the output filter - OWN = {1: [("l_post_recon_p12", "l_post.co"), ("l_post_recon", "l_post.co"), ("l_coef", "l_ops.co"), - ("w64/post_filter_c1_s4", "post_w64.co")], - 0: [("l_post_recon_p12_ldr", "l_post.co"), ("l_post_recon_ldr", "l_post.co"), ("l_coef", "l_ops.co"), - ("l_post_filter_ldr", "l_post.co")]} + # Model L's C1/C2 reconstruction on native D3D12, HDR and LDR colour: its D3D12 ports, then the output filter + OWN = {1: [("lc12/l_post_recon_p12", "l_post__lc12.co"), ("lc12/l_post_recon", "l_post__lc12.co"), + ("l_coef", "l_ops.co"), ("w64/post_filter_c1_s4", "post_w64.co")], + 0: [("lc12/l_post_recon_p12_ldr", "l_post__lc12.co"), ("lc12/l_post_recon_ldr", "l_post__lc12.co"), + ("l_coef", "l_ops.co"), ("l_post_filter_ldr", "l_post.co")]} def own_ports(self): have = full_ports() return have + [kc for hdr in self.OWN for kc in self.OWN[hdr] if kc not in have] def test_own_post(self): - """HDR and LDR colour: l_post_recon_p12[_ldr] on the frames l_coef_pack12 stores, with post_recon's arguments - (the #111 role checks), then the unbanded filter (HDR w64/post_filter_c1_s4 with the exposure pair; LDR + """HDR and LDR colour: lc12/l_post_recon_p12[_ldr] on the frames l_coef_pack12 stores, with post_recon's + arguments (the role checks), then the unbanded filter (HDR w64/post_filter_c1_s4 with the exposure pair; LDR l_post_filter_ldr) from its display image into the output; no Model M post kernel. A frame the pack12 kernels - do not take (render 180 x 108 = D) runs l_coef and l_post_recon[_ldr] (the fallback route).""" + do not take (render 180 x 108 = D) runs l_coef and lc12/l_post_recon[_ldr] (the fallback route).""" sizes = "--sizes=96x64,96x64,180x108,80x48,77x45,180x108" for hdr in (1, 0): sfx = "" if hdr else "_ldr" @@ -713,18 +1022,21 @@ def test_own_post(self): self.assertEqual(t.create, "create ok model L") self.assertEqual(t.unknown(), []) self.assertEqual(t.kernels.get("l_post"), "l_post_recon" + sfx) + self.assertEqual(t.kernels.get("l_post_c12"), "on") prev = None for f in t.frames: self.assertIsNone(f["error"]) info, ls = f["info"], f["launches"] names = [ln.kernel for ln in ls] - p12 = (info["rw"], info["rh"]) != (180, 108) # pack12 frames: the fold (fusion lc, the default) - self.assertIn(FOLD if p12 else "l_coef", names) - self.assertFalse({"l_coef_pack12", "l_coef" if p12 else FOLD} & set(names), names) + p12 = (info["rw"], info["rh"]) != (180, 108) # pack12 frames + self.assertIn(DEC_COEF_HYB if p12 else "l_coef", names) # (lever 3: the store folded) + self.assertNotIn("l_coef" if p12 else DEC_COEF_HYB, names) + self.assertNotIn("l_coef_pack12", names) self.assertEqual(model_m_post(names), []) - post = [ln for ln in ls if ln.kernel == ("l_post_recon_p12" if p12 else "l_post_recon") + sfx] + post = [ln for ln in ls if ln.kernel == ("lc12/l_post_recon_p12" if p12 else "lc12/l_post_recon") + + sfx] self.assertEqual(len(post), 1, names) - lev, coef = Arguments.check_frame(self, t, f) # every argument's role, the filter after the post + lev, coef = self.check_d3d12_frame(t, f, sfx) # every argument's role, the filter after the post flt = ls[ls.index(post[0]) + 1] self.assertEqual(flt.kernel, "w64/post_filter_c1_s4" if hdr else "l_post_filter_ldr") self.assertEqual((flt.grid, flt.block), (cdiv(120, 40) * cdiv(cdiv(72, 2), 8), 64)) @@ -732,9 +1044,9 @@ def test_own_post(self): self.assertEqual(flt.args[2:6], [str((3 * 120 + 1) // 2), str((3 * 72 + 1) // 2), "120", "72"]) if hdr: self.assertEqual(flt.args[7], f["names"]["exposure.pair"]) - if p12: # pack12 image A (the fold's argument 8) is what the reconstruction reads - self.assertEqual(coef.args[8], post[0].args[1]) - self.assertGreaterEqual(t.extent(coef.args[8]), 8 * info["rw"] * info["rh"]) + if p12: # pack12 image A (the store's argument 1) is what the reconstruction reads + self.assertEqual(coef.args[1], post[0].args[1]) + self.assertGreaterEqual(t.extent(coef.args[1]), 8 * info["rw"] * info["rh"]) if prev: self.assertEqual(lev.args[4], latent_out(prev[1])) # the latent the last frame's store wrote prev = (lev, coef) @@ -753,9 +1065,9 @@ def test_color_early_option(self): def test_no_model_m_post_route(self): """Model L never runs Model M's post: every route traced (HDR and LDR colour; HIP and native D3D12; pack12 and - fallback frames) launches Model L's reconstruction and no Model M post kernel; l_post=m is refused; where Model - L's own route cannot run (a port missing, another post / filter route asked for) creation is refused with the - reason, never a different algorithm.""" + fallback frames) launches Model L's C1/C2 reconstruction and no Model M post kernel; l_post=m is refused; where + Model L's own route cannot run (a port missing, another post / filter route asked for) creation is refused with + the reason, never a different algorithm.""" def first(options, **kw): return run(options, frames=1, build_files=FT.SHIPPING, **kw)[0] d3d12 = dict(flags=("--backend=3",), manifest=self.own_ports()) @@ -766,21 +1078,26 @@ def first(options, **kw): "ERR create: l_post must be own (Model L's reconstruction; there is no other)") self.assertEqual(first("l_post=own", **d3d12), "ERR create: l_post=own is a Model L option (its own " "reconstruction)") - # a missing port of Model L's route: refused by name (HDR and LDR), never another post + # a missing port of Model L's route: refused by name (HDR and LDR), never another post (the C1/C2 ports: all + # four checked first, by the hybrid_full form) for hdr, kcs in self.OWN.items(): for k, _ in kcs: with self.subTest(hdr=hdr, port=k): got = first("model=L", hdr=hdr, flags=("--backend=3",), manifest=[kc for kc in self.own_ports() if kc[0] != k]) - self.assertEqual(got, "ERR create: Model L on native D3D12 (%s): not yet ported to D3D12: %s; " - "Model L has no other reconstruction (no Model M post route)" - % ("HDR" if hdr else "LDR", k)) + if k.startswith("lc12/"): + self.assertEqual(got, "ERR create: WG_L_POST=c12 on native D3D12: not in the kernel manifest: " + + k) + else: + self.assertEqual(got, "ERR create: Model L on native D3D12 (%s): not yet ported to D3D12: %s; " + "Model L has no other reconstruction (no Model M post route)" + % ("HDR" if hdr else "LDR", k)) # another of Model M's filter routes asked for: refused (the RGBA16F filter output) got = first("model=L", env={"WG_FILTER_KERNEL": "post_filter_c1_s4_o16"}, **d3d12) self.assertTrue(got.startswith("ERR create: "), got) self.assertNotIn("create ok", got) - # every route: Model L's reconstruction, no Model M post kernel (HIP: base coefficients and l_post_recon[_ldr]; - # native D3D12: pack12 frames and the fallback frame) + # every route: Model L's C1/C2 reconstruction, no Model M post kernel (HIP: base coefficients and + # lc12/l_post_recon[_ldr]; native D3D12: pack12 frames and the fallback frame) sizes = "--sizes=96x64,180x108,80x48" for hdr in (1, 0): sfx = "" if hdr else "_ldr" @@ -797,7 +1114,7 @@ def first(options, **kw): self.assertEqual(model_m_post(names), [], names) own = [n for n in names if base(n) == "l_post_recon"] self.assertEqual(len(own), 1, names) - self.assertTrue(own[0].endswith(sfx), own) + self.assertTrue(own[0].startswith("lc12/") and own[0].endswith(sfx), own) if not hdr: # LDR: Model L's LDR output stage, no exposure-decoding filter self.assertIn("l_post_filter_ldr", names) self.assertFalse([n for n in names if n.endswith("post_filter_c1_s4")], names) @@ -891,5 +1208,212 @@ def test_model_data(self): shutil.rmtree(d, ignore_errors=True) +class ReleaseDefault(unittest.TestCase): + """0.6.5 (oversight 2026-11-15 18:36Z): Model L = hybrid_full when WG_L_FORM is unset. On native D3D12 the fast + hybrid route (the manifest's "_hyb" twins on the unfused layout, the C1/C2 reconstruction lc12/*); on HIP the scalar + reference route (l_q7__hyb.co, l_q7w__hyb.co, l_post__lc12.co). q7 is an opt-in reference; the M-form is refused. + A hybrid route that cannot be taken is refused at create, by name, in one line: no fallback to the q7 reference, + the M-form or HIP. (The game-side rule, no move to the HIP executor when the D3D12 executor refuses a context at + create, is #225's for every model: clean/runtime/interop/executor_select.h dx12_refused_line.)""" + + def d3d12(self, ports, env=None, frames=1, **kw): + return run("model=L", frames=frames, build_files=FT.SHIPPING, flags=("--backend=3",), manifest=ports, env=env, + **kw) + + @staticmethod + def launched(lines): + return {ln.split()[1] for ln in lines if ln.startswith("launch ")} + + @staticmethod + def kernels_text(lines): + return next(ln for ln in lines if ln.startswith("kernels ")) + + def test_default_is_hybrid_full_on_d3d12(self): + for env in (None, {"WG_L_FORM": "hybrid"}): + with self.subTest(env=env): + got = self.d3d12(full_ports(), env=env) + self.assertEqual(got[0], "create ok model L") + text = self.kernels_text(got) + ntwins = sum(k.endswith("_hyb") for k, _ in full_ports()) + for want in ("l_form=hybrid", "l_hyb_d3d12=on", "l_post_c12=on", "hyb_twins=%d" % ntwins, + "swin_fused=none", "hyb_tables=on"): + self.assertIn(want, text) + names = self.launched(got) + used = set(HYB_TWINS) - {"l_swin_block_c32_enc_hyb", "l_swin_block_c32_dec_hyb", + "swin_block_c128_hyb"} | {ENC_LIN_HYB, DEC_COEF_HYB, BNECK_HYB} + self.assertTrue(used <= names, sorted(used - names)) + self.assertIn("lc12/l_post_recon_p12", names) + blocks = [n for n in names if n.split("/")[-1].startswith(("swin_", "l_swin_", "patch_merge", "expand"))] + self.assertEqual([n for n in blocks if not n.endswith("_hyb")], []) # no M-form block, merge, expand + self.assertEqual([n for n in names if "l_q7" in n], []) # no q7 reference kernel + self.assertEqual([n for n in names if n.startswith("l_post_recon")], []) # only the C1/C2 post + + def test_hybrid_route_refused_never_falls_back(self): + """Each twin and each C1/C2 port missing: create refused in one line naming it; nothing created, nothing + launched (no fallback to the q7 reference, the M-form or HIP).""" + for missing in HYB_TWINS + LC12 + FUSED_REQUIRED: + for env in (None, {"WG_L_FORM": "hybrid"}): + with self.subTest(missing=missing, env=env): + got = self.d3d12([kc for kc in full_ports() if kc[0] != missing], env=env, frames=2) + self.assertEqual(len(got), 1, got[:3]) + self.assertTrue(got[0].startswith("ERR create: ") and missing in got[0], got[0]) + # the form as resolved: nothing set = the release default (review finding) + form = "Model L hybrid (default)" if env is None else "WG_L_FORM=hybrid" + self.assertTrue(form in got[0] or "WG_L_POST=c12" in got[0], got[0]) + + def test_hybrid_geometry_overrides_refused(self): + """A Swin / GEMM launch-geometry override is refused at create for every hybrid context: the HIP route and the + native-D3D12 route without direct I/O (review P1: nt=4 sized the merge / expand grids for 512 tokens per workgroup + while the shaders kept 256; wpg 16 halved the block grid), explicit and default form, before any allocation.""" + for geo in ("nt=4", "gemm_threads=128", "swin_threads=128", "wpw64=2", "wpw96=2", "wpw128=2", "wpg0=16", + "wpg4=16"): + for backend, kw in ((0, {}), (3, dict(build_files=FT.SHIPPING, manifest=full_ports()))): + for env in (None, {"WG_L_FORM": "hybrid"}): + with self.subTest(geo=geo, backend=backend, env=env): + got = run("model=L", frames=2, flags=("--backend=%d" % backend, "--geo=" + geo), env=env, **kw) + self.assertEqual(len(got), 1, got[:3]) + self.assertTrue(got[0].startswith("ERR create: ") and "launch-geometry override" in got[0], got[0]) + # the base geometry spelled out is not an override + got = run("model=L", frames=1, flags=("--backend=3", "--geo=nt=2", "--geo=wpg4=8"), build_files=FT.SHIPPING, + manifest=full_ports()) + self.assertTrue(got[0].startswith("create ok model L"), got[0]) + + def test_default_route_loads_only_deployed_objects(self): + """Every Model L code object the default (hybrid) route loads on the HIP backend is built into a release runtime + library (clean/runner/build.py L_EMBED_FILES): a deployed library alone creates the default (review P2: l_q7__hyb, + l_q7w__hyb and l_post__lc12 were read from the build folder only).""" + from clean.runner import build as B + B.set_model_l(True) + deployed = set(B.EMBED_FILES) + for env in (None, {"WG_L_FORM": "hybrid"}): + with self.subTest(env=env): + got = run("model=L", frames=1, env=env) + self.assertTrue(got[0].startswith("create ok model L"), got[0]) + loads = [x.split()[1] for x in got if x.startswith("load ")] + self.assertTrue([n for n in loads if n.startswith("l_q7")], loads) + self.assertEqual([n for n in loads if n.startswith("l_") and n not in deployed], []) + self.assertFalse([n for n in deployed if B.is_check_object(n)]) + + def test_release_build_refuses_check_routes(self): + """The check routes (the mutant, the HIP twins) are not offered by a release build (the recorder is built + without WG_DEV_DIAGNOSTICS): refused at create in one line, nothing created.""" + for env in ({"WG_L_FORM": "hybrid_mutant"}, {"WG_L_FORM": "hybrid_twin"}): + with self.subTest(env=env): + got = run("model=L", frames=1, env=env) + self.assertEqual(len(got), 1, got[:3]) + self.assertTrue(got[0].startswith("ERR create: ") and "developer builds only" in got[0], got[0]) + got = run("", frames=1, env={"WG_M_FORM": "hybrid_mutant"}) + self.assertTrue(got[0].startswith("ERR create: ") and "developer builds only" in got[0], got[0]) + + def test_release_hybrid_route_ignores_the_diagnostic_hooks(self): + """A release build's hybrid route is never switched by WG_AB_FUSION / WG_AB_BLOCK (or the barrier / kernel + substitution hooks): the frames trace as without them, and one note says the keys were ignored.""" + def l_run(**kw): + return run("model=L", frames=4, rw=96, rh=64, build_files=FT.SHIPPING, flags=("--backend=3",), + manifest=full_ports(), **kw) + base = l_run() + for env in ({"WG_AB_FUSION": "none", "WG_AB_BLOCK": "1"}, {"WG_D3D12_NOBAR": "swin_bneck6_hyb"}, + {"WG_D3D12_KSUB": "swin_bneck6_hyb=swin_block_c128_hyb"}): + with self.subTest(env=env): + got = l_run(env=env) + self.assertTrue(any(x.startswith("kernels ") and " diag_hooks=ignored" in x for x in got)) + self.assertEqual(same_frames([x.replace(" diag_hooks=ignored", "") for x in got]), same_frames(base)) + + def test_hybrid_tables_refused_never_falls_back(self): + """The device refuses the hybrid tables (the recorder's --fail-hyb-tables, as native D3D12 when their arena + range is held): create refused in one line naming the resolved form, nothing launched; with the tables the + same build creates (the CPU recorder implements Device::hyb_reserve: review finding P1).""" + for env in (None, {"WG_L_FORM": "hybrid"}): + with self.subTest(env=env): + got = run("model=L", frames=2, build_files=FT.SHIPPING, flags=("--backend=3", "--fail-hyb-tables"), + manifest=full_ports(), env=env) + form = "Model L hybrid (default)" if env is None else "WG_L_FORM=hybrid" + self.assertEqual(len(got), 1, got[:3]) + self.assertTrue(got[0].startswith("ERR create: " + form + " on native D3D12: no hybrid tables"), got[0]) + ok = self.d3d12(full_ports(), env=env) + self.assertEqual(ok[0], "create ok model L") + self.assertIn("hyb_tables=on", self.kernels_text(ok)) + # the HIP reference route and Model L's q7 reference never ask for the tables + for flags, env in (((), None), (("--backend=3",), {"WG_L_FORM": "q7"})): + with self.subTest(flags=flags, env=env): + got = run("model=L", frames=1, build_files=FT.SHIPPING, flags=flags + ("--fail-hyb-tables",), + manifest=full_ports() if flags else None, env=env) + self.assertEqual(got[0], "create ok model L") + + def test_hybrid_bneck_timeout_refuses_and_writes_no_history(self): + """The bottleneck's bounded wait (swin_bneck6_hyb) REFUSES a frame and lets nothing of it through (review P1): + the whole hybrid frame runs under the device's frame guard, read again right after the bottleneck. In the frame + whose wait times out (the recorder's --guard-timeout), every launch after the bottleneck is skipped (the post + that writes the history and the output among them); the next frame is refused in one line, launching nothing; + the frame after it starts over (reset 1) with the abort word cleared first. Control: without the timeout, + nothing is skipped or refused. Default form and explicit form.""" + for env in (None, {"WG_L_FORM": "hybrid"}): + for timeout in (None, 1): + with self.subTest(env=env, timeout=timeout): + flags = ("--backend=3",) + (("--guard-timeout=%d" % timeout,) if timeout is not None else ()) + got = run("model=L", frames=5, rw=96, rh=64, build_files=FT.SHIPPING, flags=flags, + manifest=full_ports(), env=env) + self.assertEqual(got[0], "create ok model L") + blocks, cur = [], None + for x in got: + if x.startswith("frame "): + cur = [x] + blocks.append(cur) + elif cur is not None and not x.startswith(("name ", "kernels ")): + cur.append(x) + self.assertEqual(len(blocks), 5) + dev = [[x for x in b[1:] if x.startswith(("launch ", "skipped ", "guard_", "ERR "))] for b in blocks] + for f, d in enumerate(dev): + refused = timeout is not None and f == timeout + 1 + if refused: + self.assertEqual(len(d), 1, d[:4]) + self.assertTrue(d[0].startswith("ERR frame %d: " % f) and "timed out" in d[0] + and "refused" in d[0] and "starts over" in d[0], d[0]) + continue + clear = timeout is not None and f == timeout + 2 + if clear: + self.assertEqual(d[0], "guard_clear", d[:2]) + g = d[1:] if clear else d + self.assertNotIn("guard_clear", g) + self.assertTrue(g[0].startswith("guard_begin ") and "(skipping)" not in g[0], g[:2]) + self.assertEqual(g[-1], "guard_end") + i = [x.startswith("launch swin_bneck6_hyb ") for x in g].index(True) + self.assertEqual(g[i + 1].split(" (")[0], g[0]) # the guard read again, same word + after = g[i + 2:-1] + self.assertTrue([x for x in after if "post" in x], after) + if timeout is not None and f == timeout: + self.assertTrue(g[i + 1].endswith("(skipping)"), g[i + 1]) + self.assertEqual([x for x in after if not x.startswith("skipped ")], []) + else: + self.assertFalse([x for x in g if x.startswith("skipped ") or "(skipping)" in x]) + reset = int(blocks[f][0].split(" reset ")[1].split()[0]) + self.assertEqual(reset, 1 if f == 0 or clear else 0, blocks[f][0]) + + def test_m_form_refused(self): + for flags in ((), ("--backend=3",)): + with self.subTest(flags=flags): + got = run("model=L", frames=1, build_files=FT.SHIPPING, flags=flags, + manifest=full_ports() if flags else None, env={"WG_L_FORM": "m"}) + self.assertEqual(got, ["ERR create: WG_L_FORM=m refused: the M-form is not offered for Model L " + "(default hybrid_full; q7 = reference)"]) + + def test_q7_is_opt_in_reference(self): + got = self.d3d12(full_ports(), env={"WG_L_FORM": "q7"}) + self.assertEqual(got[0], "create ok model L") + names = self.launched(got) + self.assertTrue(any(n.startswith("l_q7_") for n in names), sorted(names)) + self.assertEqual([n for n in names if n.endswith("_hyb") or n.startswith("lc12/")], []) + + def test_hip_default_is_the_reference_hybrid(self): + got = run("model=L", frames=1, build_files=FT.SHIPPING) + self.assertEqual(got[0], "create ok model L") + self.assertIn("load l_q7__hyb.co", got) + self.assertIn("load l_q7w__hyb.co", got) + text = self.kernels_text(got) + self.assertIn("l_form=hybrid", text) + self.assertIn("l_post_c12=on", text) + self.assertIn("lc12/l_post_recon", self.launched(got)) + + if __name__ == "__main__": unittest.main() diff --git a/clean/l/runtime/tests/test_l_runtime_hyb_pack.py b/clean/l/runtime/tests/test_l_runtime_hyb_pack.py new file mode 100644 index 00000000..2d8381ef --- /dev/null +++ b/clean/l/runtime/tests/test_l_runtime_hyb_pack.py @@ -0,0 +1,183 @@ +# SPDX-License-Identifier: MIT +"""Model L's HYBRID weight values in the runtime (clean/l/runtime/l_hyb_pack.h, through pack_stage_blob and +l::pack_model) against clean/l/runner/l_hyb_stage_pack.py + the Python packers, byte for byte: + +- every blob (the ten shared stages, packing.pack_stage; Model L's own launches, l_pack.pack_model_l) for random + parameters (clean/l/reference/oracle.random_params), three seeds, plus one seed with the weights and biases scaled so + that the largest weight is 1000 (FP8 saturation at +-448, FP16 rounding of large biases); +- negative control: the C++ "mutant" (the gammas without the sqrtf(C) fold) must differ in every gamma-bearing blob; +- with WG_TEST_L_WGCM= and WG_TEST_L_MAP=: a local model file (never in the repository) loaded by the + runtime's loader, its tensors exported by the tool and the Python reference built from them; only counts and a hash + are printed, nothing of the model. + +No GPU: clean/l/runtime/tests/l_hyb_pack_dump.cpp is built with the runtime's sources by the host C++ compiler (as +test_l_runtime_pack.py; skipped without one). + +Run (repository root): + python -m unittest discover -s clean/l/runtime/tests -t clean/l/runtime/tests -p test_l_runtime_hyb_pack.py +""" +import hashlib +import os +import tempfile +import unittest + +import numpy as np + +import test_l_runtime_pack as TP +from clean.l.reference import oracle as L +from clean.l.runner import l_hyb_stage_pack as HS +from clean.l.runner import l_pack as LP +from clean.runner import packing as P + +ROOT = TP.ROOT +SOURCES = [os.path.join("clean", "l", "runtime", "tests", "l_hyb_pack_dump.cpp"), + os.path.join("clean", "l", "runtime", "l_pack.cpp"), os.path.join("clean", "l", "runtime", "l_model.cpp"), + os.path.join("clean", "runtime", "model.cpp"), os.path.join("clean", "runtime", "embedded.cpp"), + os.path.join("clean", "converter", "src", "sha256.cpp"), + os.path.join("clean", "converter", "src", "json.cpp")] +SHARED = ("enc1", "enc2", "b0", "b1", "b2", "b3", "b4", "b5", "dec2", "dec1") +GAMMA_BLOBS = SHARED + ("enc0_block", "dec0_block") +_EXE = None + + +def dump_exe(): + global _EXE + if _EXE is None: + saved, saved_exe = TP.SOURCES, TP._EXE + try: + TP.SOURCES, TP._EXE = SOURCES, None + _EXE = TP.dump_exe() + finally: + TP.SOURCES, TP._EXE = saved, saved_exe + return _EXE + + +def run(*args): + import subprocess + return subprocess.run([dump_exe(), *args], capture_output=True, text=True).stdout.strip() + + +def write_params(root, params): + for stage, tensors in params.items(): + d = os.path.join(root, stage) + os.makedirs(d) + for name, a in tensors.items(): + if stage == "enc0" and name == "T": + np.asarray(a).astype(np.uint8).tofile(os.path.join(d, "T.u8")) + else: + np.ascontiguousarray(a, " 448).any(), "the scaled seed must reach FP8 saturation") + self.check(p, "scaled seed 5") + + def test_mutant_differs(self): + params = L.random_params(np.random.default_rng(0)) + want = reference(params) + res, got = pack_cpp(params, mutant=True) + self.assertTrue(res.startswith("OK"), res) + for name in GAMMA_BLOBS: + self.assertNotEqual(got[name], want[name], "mutant not caught in " + name) + + def test_hybrid_values_differ_from_today(self): + # the hybrid blobs are not the M-form blobs (values changed in every block stage) + params = L.random_params(np.random.default_rng(1)) + want = reference(params) + today = {s: P.pack_stage(s, params[s], "gfx12", model="M") for s in SHARED} + today.update(LP.pack_model_l(params)) + for name in GAMMA_BLOBS: + self.assertNotEqual(today[name], want[name], name) + + @unittest.skipUnless(os.environ.get("WG_TEST_L_WGCM") and os.environ.get("WG_TEST_L_MAP"), + "WG_TEST_L_WGCM / WG_TEST_L_MAP not set (a local model file)") + def test_local_model_file(self): + d = tempfile.mkdtemp(prefix="wg_hyb_local_") + res = run("wgcm", os.environ["WG_TEST_L_WGCM"], os.environ["WG_TEST_L_MAP"], d) + self.assertTrue(res.startswith("OK"), res) + like = L.random_params(np.random.default_rng(0)) + params = shaped(read_params(os.path.join(d, "tensors")), like) + want = reference(params) + got = read_dir(os.path.join(d, "blobs")) + self.assertEqual(sorted(got), sorted(want)) + bad = [n for n in want if got[n] != want[n]] + digest = hashlib.sha256(b"".join(want[n] for n in sorted(want))).hexdigest() + print("local model file: %d hybrid blobs, %d differ, digest %s" % (len(want), len(bad), digest)) + self.assertEqual(bad, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/l/tests/test_hyb_tables.py b/clean/l/tests/test_hyb_tables.py new file mode 100644 index 00000000..e837e364 --- /dev/null +++ b/clean/l/tests/test_hyb_tables.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: MIT +"""The exhaustive facts behind the fast hybrid kernels' element-wise steps (clean/l/reference/hyb_tables.py), each over +every possible FP16 input, plus mutants each check catches.""" +import unittest + +import numpy as np + +from clean.l.reference import hyb_tables as T +from clean.l.reference import oracle as O + + +class Facts(unittest.TestCase): + def test_all_facts(self): + for name, ok in T.facts().items(): + self.assertTrue(ok, name) + + def test_gelu_table_is_the_oracle(self): + g = T.gelu16_q8() + self.assertEqual(g.shape, (65536,)) + # spot values: GELU16(2) = 2 (FP8 0x40), GELU16(-2) = -0 or 0, GELU16(0) = 0 + self.assertEqual(int(g[0x4000]), 0x40) + self.assertIn(int(g[0xC000]), (0x00, 0x80)) + self.assertEqual(int(g[0x0000]), 0x00) + + +class Mutants(unittest.TestCase): + """Each exhaustive comparison must fail for a deliberately wrong form.""" + + def test_softmax_two_roundings_inside_fma_is_caught(self): + u = T.clamped_u() + wrong_t = T.f16(T.f16(u * -u) + O.Q7_POLY_T) # product rounded to FP16 first: not FMA16 + p = T.f16(u * wrong_t + O.Q7_POLY_P) + e = ((p.astype(np.float16).view(np.uint16) & 0x03FF) << 5).astype(np.uint16) + self.assertFalse(np.array_equal(e, T.softmax_e_ref(u))) + + def test_rsq_table_off_by_one_is_caught(self): + tab = T.rsq16_table().copy() + tab[0, 512] += 1 + sb = T.ALL16[(T.H >= 2.0 ** -13) & np.isfinite(T.H)] + ref = T._rsq_ref(sb.view(np.float16).astype(np.float64)).astype(np.float16).view(np.uint16) + self.assertFalse(np.array_equal(T.rsq16_lookup(sb, tab), ref)) + + def test_gelu_fused_mutant_is_caught(self): + h = T.H[np.isfinite(T.H)] + y = np.fmin(np.fmax(h, -2.0), 2.0) + fused = T.f16(h * T.f16(0.5 + T.f16(y * (O.Q7_GELU_A - O.Q7_GELU_B * np.abs(y))))) # inner mul+sub fused + self.assertFalse(np.array_equal(T.f16(fused), O.gelu_q7(h))) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/l/tests/test_hyb_tables_cpp.py b/clean/l/tests/test_hyb_tables_cpp.py new file mode 100644 index 00000000..c26d78d5 --- /dev/null +++ b/clean/l/tests/test_hyb_tables_cpp.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: MIT +"""runtime/hyb_tables.h (the arena tables the D3D12 hybrid kernels read) gives exactly the bytes of +clean/l/reference/hyb_tables.py: GELU16 -> FP8 for all 65,536 inputs, the rsqrt and rcp tables, and the GELU16 table +with 16-bit entries.""" +import os +import subprocess +import tempfile +import unittest + +import numpy as np + +from clean.l.reference import hyb_tables as T +from clean.tests import host_cxx + + +def python_bytes(): + out = np.zeros(65536 + 4096 + 2048 + 131072, np.uint8) + out[:65536] = T.gelu16_q8() + out[65536:69632] = T.rsq16_table().astype(" 300.25, +# float32-sequential -> the midpoint 300.375 -> ties to even -> 300.5. +# -1500 (spacing 1 in [1024, 2048)) - 0.5 - 2^-14: exact -> -1501, float32-sequential -> -1500.5 -> -1500. +MIDPOINT_CASES = ( + (300.25, ((0.125, 1.0), (2.0 ** -8, -(2.0 ** -8))), 300.25), + (-1500.0, ((-0.5, 1.0), (2.0 ** -7, -(2.0 ** -7))), -1501.0), +) + + +class KernelOpTest(unittest.TestCase): + N = 2000 + + def _family(self, name, op): + return M.compare_family(name, np.random.default_rng([11, M.FAMILIES.index(name)]), self.N, op=op) + + def test_kernel_equals_exact_on_every_family(self): + for name in M.FAMILIES: + with self.subTest(family=name): + r = self._family(name, M.op_kernel) + self.assertEqual(r["differ"], 0, r["examples"][:3]) + + def test_plain_float32_fails_where_the_corrections_are_needed(self): + self.assertGreater(self._family("cancellation", M.op_f32seq)["differ"], 0) + self.assertGreater(self._family("start_midpoint", M.op_f32seq)["differ"], 0) + + def test_constructed_midpoint_cases(self): + for start, prods, want in MIDPOINT_CASES: + st, a, w = _op(start, prods) + self.assertEqual(M.op_exact(st, a, w)[0], want) + self.assertEqual(M.op_kernel(st, a, w)[0], want) + self.assertNotEqual(M.op_f32seq(st, a, w)[0], want) + + def test_without_the_slow_path_cancellation_fails(self): + old = M.ABS_FAST + M.ABS_FAST = np.float32(1e30) + try: + self.assertGreater(self._family("cancellation", M.op_kernel)["differ"], 0) + finally: + M.ABS_FAST = old + + def test_without_the_midpoint_fix_the_start_cases_fail(self): + def no_fix(start, a8, W8): + # the start added with one float32 addition and converted (the float32 product sum is exact here) + return M.op_f32seq(start, a8, W8) + self.assertGreater(self._family("start_midpoint", no_fix)["differ"], 0) + for start, prods, want in MIDPOINT_CASES: + st, a, w = _op(start, prods) + self.assertNotEqual(no_fix(st, a, w)[0], want) + + def test_overflow_boundary_and_specials(self): + cases = [(65504.0, ((4.0, 4.0),)), (65504.0, ((4.0, 4.0), (2.0 ** -9, -(2.0 ** -9)))), + (-65504.0, ((-4.0, 4.0),)), (65504.0, ((2.0, 4.0),)), (0.0, ((448.0, 448.0), (448.0, 448.0))), + (np.inf, ((1.0, 1.0),)), (1.0, ((np.nan, 1.0),)), (-np.inf, ((448.0, 448.0),))] + for start, prods in cases: + st, a, w = _op(start, prods) + want, got = M.op_exact(st, a, w)[0], M.op_kernel(st, a, w)[0] + with self.subTest(start=start, prods=prods): + self.assertTrue(want == got or (np.isnan(want) and np.isnan(got)), (want, got)) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/l/tests/test_l_q7_known.py b/clean/l/tests/test_l_q7_known.py index ff448e19..ba342e0c 100644 --- a/clean/l/tests/test_l_q7_known.py +++ b/clean/l/tests/test_l_q7_known.py @@ -304,9 +304,10 @@ def norm_case(): PER_PRODUCT = (" return f16(start + a8 @ W8.T)", " acc = start\n for i in range(a8.shape[-1]):\n" " acc = f16(acc + a8[..., i:i + 1] @ W8[:, i:i + 1].T)\n return acc") -W2_SLICE = ('acc = mm8(acc, q8(gelu_q7(hs)), W["W2"][:, r], model)', - 'acc = mm8(acc, q8(gelu_q7(hs)), W["W2"][:, slice(32 * ((sl + 1) % 4), 32 * ((sl + 1) % 4) + 32)], model)') -MLP_REVERSED = ('for sl in range(W["W1"].shape[0] // 32):', 'for sl in reversed(range(W["W1"].shape[0] // 32)):') +W2_SLICE = ('acc = mm8(acc, q8(gelu_q7(hs)), W["W2"][:, r], model, final=(model != "f32chain" or sl == nsl - 1))', + 'acc = mm8(acc, q8(gelu_q7(hs)), W["W2"][:, slice(32 * ((sl + 1) % 4), 32 * ((sl + 1) % 4) + 32)], model, ' + 'final=(model != "f32chain" or sl == nsl - 1))') +MLP_REVERSED = ('for sl in range(nsl):', 'for sl in reversed(range(nsl)):') class Q7KnownAnswerTest(unittest.TestCase): diff --git a/clean/l/tests/test_l_q7_mask.py b/clean/l/tests/test_l_q7_mask.py new file mode 100644 index 00000000..5270feb4 --- /dev/null +++ b/clean/l/tests/test_l_q7_mask.py @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: MIT +"""Model L: the q7 checks' NaN-sign exception (clean/l/runner/l_q7_check.py mask_nan) masks ONLY a NaN's sign bit. + +Run: python -m unittest discover -s clean/l/tests -t . +""" +import unittest + +import numpy as np + +from clean.l.runner.l_q7_check import mask_control, mask_nan + + +class NanSignMask(unittest.TestCase): + def test_control_passes(self): + r = mask_control() + self.assertTrue(r.pop("pass")) + self.assertTrue(all(r.values()), r) + + def test_non_nan_bytes_unchanged(self): + b8 = bytes(c for c in range(256) if (c & 0x7F) != 0x7F) + self.assertEqual(mask_nan(b8, "fp8"), b8) + h = np.array([v for v in range(65536) if not ((v & 0x7C00) == 0x7C00 and (v & 0x3FF))], np.uint16).tobytes() + self.assertEqual(mask_nan(h, "fp16"), h) + + def test_nan_only_sign_cleared(self): + self.assertEqual(mask_nan(bytes([0xFF, 0x7F]), "fp8"), bytes([0x7F, 0x7F])) + h = np.array([0xFE01, 0xFC01, 0x7E02], np.uint16).tobytes() + self.assertEqual(np.frombuffer(mask_nan(h, "fp16"), np.uint16).tolist(), [0x7E01, 0x7C01, 0x7E02]) + + def test_mutant_full_mask_is_caught(self): + """A mask that also cleared non-NaN signs would make the control fail.""" + import clean.l.runner.l_q7_check as Q + real = Q.mask_nan + Q.mask_nan = lambda body, kind: (bytes(b & 0x7F for b in body) if kind == "fp8" else + (np.frombuffer(body, np.uint16) & 0x7FFF).astype(np.uint16).tobytes()) + try: + self.assertFalse(Q.mask_control()["pass"]) + finally: + Q.mask_nan = real + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/l/tests/test_l_q7_pack.py b/clean/l/tests/test_l_q7_pack.py new file mode 100644 index 00000000..1006a057 --- /dev/null +++ b/clean/l/tests/test_l_q7_pack.py @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: MIT +"""The FP16-form block kernels' blob and tables (clean/l/runner/l_pack.py pack_q7_block, q7_tables): layout, values +and the agreement of the tables with the oracle's functions for every FP16 input. + + python -m unittest clean.l.tests.test_l_q7_pack +""" +import unittest + +import numpy as np + +from clean.l.reference import oracle as L +from clean.l.runner import l_pack as LP +from clean.reference.fp8 import f16, fp8_decode, q8 + + +class Q7PackTest(unittest.TestCase): + def test_blob_layout_and_values(self): + rng = np.random.default_rng(3) + p = {k: np.asarray(v, np.float64) for k, v in L.random_block_params(rng, 32, 2, 128).items()} + blob = LP.pack_q7_block(p, 32, 2, 128) + o = LP.q7_block_offsets(32, 2, 128) + self.assertEqual(len(blob) % LP.ALIGN, 0) + self.assertEqual((o["end8"], o["end16"], o["end"]), (15872, 44544, 46336)) + for name in ("Wqk", "Wv", "Wp", "W1", "W2"): # the sum |w| tables (Q7F-2), exact + W = p[name].reshape(-1, p[name].shape[-1]) + got = np.frombuffer(blob[o[name + "_wsum"]:o[name + "_wsum"] + 4 * W.size // 32], np.float32) + want = np.abs(q8(W)).reshape(W.shape[0], -1, 32).sum(-1).T.reshape(-1) # chunk-major [K/32][rows] + np.testing.assert_array_equal(got.astype(np.float64), want) + self.assertEqual((o["Wqk"], o["W2"]), (1536, 11776)) # the constants of clean/l/kernels/l_q7.hip + self.assertEqual((o["Wqk16"], o["Wv16"], o["Wp16"], o["W116"], o["W216"]), + (15872, 19968, 24064, 28160, 36352)) # and of its FP16 section (Q7F-1) + + def f16s(name, n): + return np.frombuffer(blob[o[name]:o[name] + 2 * n], np.float16).astype(np.float64) + + def fp8s(name, shape): + n = int(np.prod(shape)) + return fp8_decode(np.frombuffer(blob[o[name]:o[name] + n], np.uint8)).astype(np.float64).reshape(shape) + + np.testing.assert_array_equal(f16s("g1", 32), L.gamma_q7(p["gamma1"], 32)) + np.testing.assert_array_equal(f16s("g2", 32), L.gamma_q7(p["gamma2"], 32)) + np.testing.assert_array_equal(f16s("bp", 32), f16(p["bp"])) + np.testing.assert_array_equal(f16s("b1", 128), f16(p["b1"])) + np.testing.assert_array_equal(f16s("B", 512).reshape(2, 16, 16), f16(p["B"])) + for name in ("Wqk", "Wv", "Wp", "W1", "W2"): + np.testing.assert_array_equal(fp8s(name, p[name].shape), q8(p[name])) + n = p[name].size + h = np.frombuffer(blob[o[name + "16"]:o[name + "16"] + 2 * n], np.float16).astype(np.float64) + np.testing.assert_array_equal(h.reshape(p[name].shape), q8(p[name])) # the FP16 copy is exact + lin = LP.pack_q7_linear(p["W1"], p["b1"]) + lo = LP.q7_linear_offsets(128, 32) + np.testing.assert_array_equal(np.frombuffer(lin[lo["W16"]:lo["W16"] + 2 * 128 * 32], np.float16) + .astype(np.float64).reshape(128, 32), q8(p["W1"])) + with self.assertRaises(ValueError): + LP.pack_q7_block(dict(p, W1=p["W1"][:64]), 32, 2, 128) + + @staticmethod + def _fp16_copy_matches(copy_bits, codes): + """Adversarial probe of the FP16 weight copy (Q7F-1): for every FP8 E4M3 code, the copy's bits are the FP16 + bits of the code's exact value (+-0 with its sign, the subnormal codes, the largest 448), and the two NaN codes + give NaN. Model L has E4M3 weights only (no E5M2 anywhere in the network).""" + vals = fp8_decode(codes).astype(np.float64) + want = vals.astype(np.float16).view(np.uint16) + h = np.asarray(copy_bits, np.uint16) + nan = np.isnan(vals) + hnan = ((h & 0x7C00) == 0x7C00) & ((h & 0x3FF) != 0) + return bool(np.all(h[~nan] == want[~nan]) and np.all(hnan[nan])) + + def test_fp16_copy_exact_for_every_fp8_code(self): + codes = np.arange(256, dtype=np.uint8) + W = fp8_decode(codes).astype(np.float64).reshape(8, 32) + lin = LP.pack_q7_linear(W, np.zeros(8)) + o = LP.q7_linear_offsets(8, 32) + h = np.frombuffer(lin[o["W16"]:o["W16"] + 2 * 256], np.uint16) + self.assertTrue(self._fp16_copy_matches(h, codes)) + self.assertEqual((int(h[0x80]), int(h[0x01]), int(h[0x7E])), (0x8000, 0x1800, 0x5F00)) # -0, 2^-9, 448 + # mutant the probe must catch: a bit-shift conversion (rebias the exponent, shift the mantissa) that is right + # for normal codes but not for the subnormal ones + c = codes.astype(np.uint32) + mutant = (((c & 0x80) << 8) | (((c & 0x7F) << 7) + (8 << 10))).astype(np.uint16) + self.assertFalse(self._fp16_copy_matches(mutant, codes)) + + def test_tables_equal_the_oracle_for_every_fp16_input(self): + tab = np.frombuffer(LP.q7_tables(), np.float16).astype(np.float64) + self.assertEqual(tab.size, 2 * 65536) + x = np.arange(65536, dtype=np.uint32).astype(np.uint16).view(np.float16).astype(np.float64) + with np.errstate(all="ignore"): + rs, rc = f16(L._rsqrt32(x)), f16(L._rcp32(x)) + for got, want in ((tab[:65536], rs), (tab[65536:], rc)): + same = (got == want) | (np.isnan(got) & np.isnan(want)) + self.assertTrue(same.all()) + self.assertEqual(tab[0x5000], f16(L._rsqrt32(np.array([32.0])))[0]) # S' = 32 -> RN16(1 / sqrt 32) + self.assertEqual(tab[65536 + 0x4000], 0.5) # 1 / 2 + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/l/tests/test_l_recon.py b/clean/l/tests/test_l_recon.py index 81461bfc..8b847bd4 100644 --- a/clean/l/tests/test_l_recon.py +++ b/clean/l/tests/test_l_recon.py @@ -302,7 +302,7 @@ def test_oracle_entry_uses_the_post_scale(self): x = _inputs(rng) params = {"post": {"s": np.float32(2.9796233)}} self.assertTrue(_same(L.reconstruct(x, params)["display"], R.reconstruct(x, 2.9796233)["display"])) - self.assertEqual(set(T.PROVISIONAL), {"S8"}) # only the shared stages' block form is pending + self.assertEqual(set(T.PROVISIONAL), set()) # nothing pending (S8 ended with the q7 shared-stage kernels) if __name__ == "__main__": diff --git a/clean/l/tests/test_l_reference.py b/clean/l/tests/test_l_reference.py index da08dc26..826256e4 100644 --- a/clean/l/tests/test_l_reference.py +++ b/clean/l/tests/test_l_reference.py @@ -349,7 +349,7 @@ def test_reduces_to_model_m_chain(self): """With Model M's encoder and decoder op lists, the L network is Model M's oracle chain.""" rng = np.random.default_rng(13) xe, xd = self._inputs(rng, 40, 24) - t = T.table(enc_ops=M_ENC_OPS, dec_ops=M_DEC_OPS, dec_pixel="m") + t = T.table(enc_ops=M_ENC_OPS, dec_ops=M_DEC_OPS, dec_pixel="m", shared_form="m") p = L.random_params(rng, t) p["enc0"] = ME.random_params(rng) pm_dec = MD.random_params(rng) @@ -504,6 +504,18 @@ def test_both_blocks_use_the_fp16_form(self): self.assertFalse(np.array_equal(L.f16(m), q)) self.assertTrue(np.array_equal(L.f16(q), q)) # the FP16 form's output is FP16 + def test_nan_member_stays_in_its_window(self): + """A NaN input member gives NaN outputs in its own window only (the coverage check does not use NaN).""" + rng = np.random.default_rng(6) + p = L.random_block_params(rng, 32, 2, 128) + k = T.block_consts(T.TABLE, "dec0_block") + x = q8(rng.normal(0, 2, (8, 8, 32))) + x[1, 2, 3] = np.nan + y = L.swin_block(x, p, (0, 0), 2, k, "coarse") + bad = np.isnan(y).any(-1) + self.assertTrue(bad[:4, :4].all()) + self.assertFalse(bad[4:, :].any() or bad[:, 4:].any()) + if __name__ == "__main__": unittest.main() diff --git a/clean/l/tests/test_l_shared_q7.py b/clean/l/tests/test_l_shared_q7.py new file mode 100644 index 00000000..22d4e69c --- /dev/null +++ b/clean/l/tests/test_l_shared_q7.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: MIT +"""Model L: the shared stages in their FP16 form (oracle.run_stage_q7, table shared_form "q7"; Q8 Part B, Q10). + +Run: python -m unittest discover -s clean/l/tests -t . +""" +import unittest + +import numpy as np + +from clean.l.reference import oracle as L +from clean.l.reference import table as T +from clean.reference import swin as MS +from clean.reference.fp8 import f16, q8 + + +def _params(stage, seed): + return {k: np.asarray(v, np.float64) for k, v in MS.random_params(stage, np.random.default_rng(seed)).items()} + + +def _k(stage): + k = T.block_consts(T.TABLE, stage) + k["mm8"] = "exact" + return k + + +class SharedStagesQ7(unittest.TestCase): + def test_encoder_stage_merge_by_hand(self): + """enc1: skip = FP8(block); merged = FP8(clamp(chain of 8 ops over the 2 x 2 patch from bm)), checked against + a per-pixel loop of single ops.""" + p = _params("enc1", 1) + rng = np.random.default_rng(2) + x = q8(rng.normal(0, 2, (8, 8, 64))) + o = L.run_stage_q7("enc1", p, x, (2, 2)) + y = q8(L.swin_block(x, p, (2, 2), 2, _k("enc1"), "mirror")) + self.assertTrue(np.array_equal(o["skip"], y)) + want = np.empty((4, 4, 96)) + Wm = q8(p["Wm"]) + for r in range(4): + for c in range(4): + z = np.concatenate([y[2 * r, 2 * c], y[2 * r, 2 * c + 1], y[2 * r + 1, 2 * c], y[2 * r + 1, 2 * c + 1]]) + acc = f16(p["bm"]) # the FP16 bias the device holds + for c0 in range(0, 256, 32): + acc = f16(acc + Wm[:, c0:c0 + 32] @ z[c0:c0 + 32]) + want[r, c] = q8(np.clip(acc, -6.28125, 6.28125)) + self.assertTrue(np.array_equal(o["merged"], want)) + + def test_merge_clamp(self): + p = _params("enc2", 3) + p["Wm"] = np.zeros_like(p["Wm"]) + p["bm"] = np.where(np.arange(128) % 2 == 0, 100.0, -100.0) + x = q8(np.random.default_rng(4).normal(0, 1, (8, 8, 96))) + m = L.run_stage_q7("enc2", p, x, (0, 2))["merged"] + self.assertTrue(np.all(m[..., 0::2] == 6.5) and np.all(m[..., 1::2] == -6.5)) # FP8(6.28125) = 6.5 + + def test_decoder_stage_zero_expansion_is_the_skip_block(self): + """dec1 with We = 0, be = 0: E = 0, the block input is S everywhere (fine mirror outside the grid), so the stage + equals the block on S with the mirror border, whatever shared_dec_border says.""" + p = _params("dec1", 5) + p["We"] = np.zeros_like(p["We"]) + p["be"] = np.zeros_like(p["be"]) + rng = np.random.default_rng(6) + S = q8(rng.normal(0, 2, (8, 12, 64))) + xl = q8(rng.normal(0, 2, (4, 6, 96))) + want = q8(L.swin_block(S, p, (2, 0), 2, _k("dec1"), "mirror")) + for border in ("coarse", "mirror"): + t = T.table(shared_dec_border=border) + self.assertTrue(np.array_equal(L.run_stage_q7("dec1", p, xl, (2, 0), S, t)["out"], want)) + + def test_decoder_stage_expansion_layout_and_border(self): + """dec2 with S = 0: inside the grid the block input is E at the member's own sub-pixel (rows 32 s + c of the + coarse pixel, s = dx + 2 dy); the two border rules differ only in windows that reach outside the grid.""" + p = _params("dec2", 7) + rng = np.random.default_rng(8) + xl = q8(rng.normal(0, 2, (4, 4, 128))) + S = np.zeros((8, 8, 96)) + e = q8(L.mm8(p["be"], xl, p["We"])) + E = L.depth_to_space(e) + self.assertTrue(np.array_equal(E[3, 2], e[1, 1, 96 * 2:96 * 3])) # dy 1, dx 0: s = dx + 2 dy = 2 + self.assertTrue(np.array_equal(E[2, 3], e[1, 1, 96 * 1:96 * 2])) # dy 0, dx 1: s = 1 + a = L.run_stage_q7("dec2", p, xl, (0, 0), S, T.table(shared_dec_border="coarse"))["out"] + b = L.run_stage_q7("dec2", p, xl, (0, 0), S, T.table(shared_dec_border="mirror"))["out"] + self.assertTrue(np.array_equal(a, b)) # offset (0, 0): no member outside + a = L.run_stage_q7("dec2", p, xl, (2, 0), S, T.table(shared_dec_border="coarse"))["out"] + b = L.run_stage_q7("dec2", p, xl, (2, 0), S, T.table(shared_dec_border="mirror"))["out"] + self.assertFalse(np.array_equal(a, b)) + self.assertTrue(np.array_equal(a[:, 2:6], b[:, 2:6])) # windows fully inside agree + + def test_bottleneck_stage(self): + p = _params("b3", 9) + x = q8(np.random.default_rng(10).normal(0, 2, (4, 8, 128))) + o = L.run_stage_q7("b3", p, x, (0, 0))["out"] + self.assertTrue(np.array_equal(o, q8(L.swin_block(x, p, (0, 0), 4, _k("b3"), "mirror")))) + + def test_form_switch(self): + self.assertEqual(T.TABLE["shared_form"], "q7") # the default since the q7 kernels run them + p = _params("enc1", 11) + x = q8(np.random.default_rng(12).normal(0, 2, (8, 8, 64))) + m = L.run_shared_stage("enc1", p, x, (2, 2), t=T.table(shared_form="m")) + q = L.run_shared_stage("enc1", p, x, (2, 2)) + self.assertTrue(np.array_equal(m["skip"], MS.run_stage("enc1", p, x, (2, 2))["skip"])) + self.assertFalse(np.array_equal(m["skip"], q["skip"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/runner/build.py b/clean/runner/build.py index b4d9d827..b4e8f941 100644 --- a/clean/runner/build.py +++ b/clean/runner/build.py @@ -86,7 +86,10 @@ L_RUNTIME_SOURCES = [os.path.join("..", "l", "runtime", "l_model.cpp"), os.path.join("..", "l", "runtime", "l_pack.cpp")] L_MAP_FILE = os.path.join(os.path.dirname(os.path.dirname(HERE)), "data", "preset-L.v1.map.json") L_KERNEL_DIR = os.path.join(os.path.dirname(HERE), "l", "kernels") -L_CODE_OBJECTS = ["l_swin.co", "l_ops.co", "l_post.co", "l_enc_values.co"] +# (the hybrid default's own objects: l_q7__hyb / l_q7w__hyb, the C32 and shared-stage blocks of HIP's hybrid route, and +# l_post__lc12, the C1/C2 reconstruction: built in, so that a deployed library alone creates the default) +L_CODE_OBJECTS = ["l_swin.co", "l_ops.co", "l_post.co", "l_enc_values.co", "l_q7.co", "l_q7w.co", "l_q7__hyb.co", + "l_q7w__hyb.co", "l_post__lc12.co"] L_EMBED_FILES = L_CODE_OBJECTS + [os.path.basename(L_MAP_FILE)] @@ -104,6 +107,12 @@ def set_model_l(on): EMBED_FILES[:] = [n for n in EMBED_FILES if n not in L_EMBED_FILES] +def is_check_object(name): + """A check-only code object (a mutant, a probe, a HIP twin of the D3D12 hybrid ports): developer builds only.""" + n = os.path.basename(name) + return "mut" in n or "probe" in n or n in ("l_swin_hyb.co", "swin_hyb.co") + + def build_device_l(clang, rocm, out_dir, device_libs=True, report=False, arch=None): """Model L's code objects into out_dir (as clean/l/runner/l_c32_check.py compile_kernels builds them), for `arch` (default ARCH).""" @@ -112,9 +121,37 @@ def build_device_l(clang, rocm, out_dir, device_libs=True, report=False, arch=No ARCH = arch or ARCH try: os.makedirs(out_dir, exist_ok=True) - for name in ("l_swin", "l_ops", "l_post"): + for name in ("l_swin", "l_ops", "l_post", "l_q7", "l_q7w"): # l_q7*: the FP16-form blocks (default form) run(device_command(clang, rocm, os.path.join(L_KERNEL_DIR, name + ".hip"), os.path.join(out_dir, name + ".co"), device_libs=device_libs, report=report, extra=["-I", KERNEL_DIR])) + # Q11 part C evaluation code objects. Model M's HYBRID form (runtime WG_M_FORM=hybrid) and its check mutant: + # l_q7w with float32 chains (l_q7w.hip Q7_HYB). Model M's ORIGINAL reconstruction (Q11 part C, runtime WG_M_POST=orig): l_post.hip with M's steps D1-D3, the + # machine-form exponent C1 and the Catmull-Rom weights C2; the C1/C2 probe and its mutant + # (clean/l/runner/l_post_c12_check.py). Not embedded, read from the build folder. + evals = (("l_q7w__hyb", "l_q7w.hip", {"Q7_HYB": 1}), + ("l_q7w__hybmut", "l_q7w.hip", {"Q7_HYB": 1, "Q7_HYB_MUTANT": 1}), + ("l_post__morig", "l_post.hip", {"WG_LP_M": 1, "WG_LP_C1": 1, "WG_CR_C2": 1}), + ("l_post_c12_probe", "l_post_c12_probe.hip", {"WG_LP_C1": 1, "WG_CR_C2": 1}), + ("l_post_c12_probe__mut", "l_post_c12_probe.hip", {"WG_LP_C1": 0, "WG_CR_C2": 2}), + # Model L's evaluation forms: C32 blocks with float32 chains (runtime WG_L_FORM=hybrid) + mutant; + # L's own reconstruction with C1 (machine form) and C2 (runtime WG_L_POST=c12) + ("l_q7__hyb", "l_q7.hip", {"Q7_HYB": 1}), + ("l_q7__hybmut", "l_q7.hip", {"Q7_HYB": 1, "Q7_HYB_MUTANT": 1}), + ("l_post__lc12", "l_post.hip", {"WG_LP_C1": 1, "WG_CR_C2": 1}), + # the fast hybrid form's HIP twins (the D3D12 *_hyb ports' byte twins; runtime WG_L_FORM=hybrid_twin, + # the network-level byte leg): Model L's C32 blocks and the shared stages' blocks / merges / expands + ("l_swin_hyb", "l_swin_hyb.hip", {}), + ("swin_hyb", os.path.join("..", "..", "kernels", "swin_hyb.hip"), {})) + # mutants, probes and the HIP twins are check objects: developer builds only (a release build neither contains + # nor can select them; clean/tests/check_no_dev_diagnostics.py PAYLOAD_CHECK_NAMES) + if not DEV_DIAGNOSTICS: + evals = tuple(e for e in evals if not is_check_object(e[0] + ".co")) + for tag, src, defs in evals: + run(device_command(clang, rocm, os.path.join(L_KERNEL_DIR, src), os.path.join(out_dir, tag + ".co"), + device_libs=device_libs, report=report, extra=["-I", KERNEL_DIR], defines=defs)) + # the HIP loader reads "." (or "/"); an extra arch builds into out_dir = / + if os.path.basename(os.path.normpath(out_dir)) != ARCH: + shutil.copyfile(os.path.join(out_dir, tag + ".co"), os.path.join(out_dir, f"{ARCH}.{tag}.co")) run(device_command(clang, rocm, os.path.join(KERNEL_DIR, "encoder.hip"), os.path.join(out_dir, "l_enc_values.co"), device_libs=device_libs, report=report, defines={"WG_ENC_VALUES_OUT": 1})) finally: diff --git a/clean/runner/m_q7_check.py b/clean/runner/m_q7_check.py new file mode 100644 index 00000000..bfa1d4e8 --- /dev/null +++ b/clean/runner/m_q7_check.py @@ -0,0 +1,279 @@ +# SPDX-License-Identifier: MIT +"""Model M on Model L's FP16-form shared-stage kernels (runtime WG_M_FORM=q7): the runtime's Model M frame on HIP +with its ten shared stages (enc1, enc2, b0 ... b5, dec2, dec1) on the q7 kernels and Model M's weights, checked stage by +stage against the FP16-form oracle (clean/l/reference/oracle.py run_stage_q7) on the runtime's own stage inputs. +Correctness only: no timing (the q7 kernels are correctness kernels until the Q7-FAST work lands). + + python clean/runner/m_q7_check.py --build --rocm --run # GPU, inside a GPU slot + python clean/runner/m_q7_check.py --build --compare --json out.json # CPU + python clean/runner/m_q7_check.py --build --rocm --run-d3d12 # GPU: native D3D12 + HIP pack12 + +: a build prepared as clean/l/runner/l_frame_check.py --prepare does (Model M's and Model L's code objects incl. +l_q7.co / l_q7w.co, the runtime library with WG_MODEL_L, /lframe_model with the synthetic Model M in raw/ and its +tensors in params.pkl). The frames, sentinels and per-stage metrics are l_frame_check's (SEQUENCE, frame_inputs, +fp8_stage). --run runs the sequence twice for Model M: WG_M_FORM=q7 (the route under test) and the default form (its +own blocks, the reference point for the deliverable's "functional M"). --compare checks, per frame and shared stage: + - q7 route vs the FP16-form oracle with Model M's weights: every output (skip / merged / out) byte-identical + (0 differing values; the kernels reproduce the oracle exactly as Model L's do); + - negative control 1: the same q7 outputs vs Model M's functional reference (clean/reference/swin.py run_stage, the + form Model M's own blocks implement) must DIFFER (the switch really changed the arithmetic); + - negative control 2: the oracle with the stage's output bias b2[0] moved by +1.0 must DIFFER from the q7 outputs + (the comparison can fail). +The default-form run is checked against the functional reference under Model M's per-stage rule (the existing +contract) so that both sides of the deliverable are on record. +Native D3D12 (--run-d3d12: form "q7d", backend 3, WG_M_FORM=q7 and WG_D3D12_FUSION=none so that every stage output is +written; /dx12 holds the q7 ports l_q7_*): --compare then checks per frame and shared stage + - q7d vs the HIP q7 run: every output byte-identical (the D3D12 ports reproduce their HIP kernels); + - q7d vs the FP16-form oracle on its own stage inputs: byte-identical; + - negative control: q7d vs the HIP default-form run (Model M's own blocks) must DIFFER; the D3D12 run's sentinel + (0x3C) differs from the HIP runs' (0xC3 / 0xA5), so an output the D3D12 route never wrote cannot match; + - every raw output of the frame (stage maps, dec0's pack12 images, post, state) of q7d against form q7h12 (HIP, + WG_M_FORM=q7, default kernels: dec0 by d0p12/coef_decoder, the decoder the D3D12 port d1p12/dec0_q7 reproduces): + byte-identical; unwritten only the previous-frame state of frame 0 and the display image under the fused pack12 + post (both sides keep their sentinels there).""" +import argparse +import copy +import json +import os +import pickle +import sys + +import numpy as np + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +sys.path.insert(0, ROOT) +from clean.l.runner import l_frame_check as LF # noqa: E402 + +FORMS = ("q7", "m") +SENT = {"q7": 0xC3, "m": 0xA5, "q7d": 0x3C, "q7h12": 0x96, "hyb": 0x69, "hybmut": 0x5C} +# Model M's HYBRID evaluation form (Q11 part C): form "hyb" = WG_M_FORM=hybrid (l_q7w__hyb.co, float32 chains), +# checked against the oracle's FP8 op model "f32chain" (byte-identical per stage), negative control: the exact q7 +# oracle must DIFFER; form "hybmut" = WG_M_FORM=hybrid_mutant (the Wqk chains keep an FP16 rounding per chunk): it must +# DIFFER from the f32chain oracle (the check catches the mutant). +HYB_ENV = {"hyb": "hybrid", "hybmut": "hybrid_mutant"} + + +def rec_path(build, form): + return os.path.join(build, "mq7_M_%s.pkl" % form) + + +def run_form(build, rocm, form): + """one form in this process (the runtime library reads WG_M_FORM once, when this process first loads it)""" + if form != "q7d": + from clean.runner.hip import clean_process_environment + clean_process_environment(rocm) + from clean.runner.pipeline import Pipeline + d = LF.model_dir(build) + flags = {"depth_inverted": False, "auto_exposure": True} + # six bottleneck launches on the default form so that b0 ... b4 are written there too + if form == "q7d": # native D3D12 (backend 3), D3D12 fusions off (WG_D3D12_FUSION=none, set by run_d3d12) + p = Pipeline(build, "", d, LF.MAX_RENDER, LF.OUTPUT, flags, backend="d3d12", check_packing=True, kernels={}) + elif form == "q7h12": # HIP with the default kernels: pack12 frames decoded by d0p12/coef_decoder + p = Pipeline(build, rocm, d, LF.MAX_RENDER, LF.OUTPUT, flags, check_packing=True, kernels={}) + else: + p = Pipeline(build, rocm, d, LF.MAX_RENDER, LF.OUTPUT, flags, check_packing=True, + kernels={"pack": "none", "bneck": "none"}) + rng = np.random.default_rng(LF.SEED) + frames = [] + for i, ((rw, rh), reset) in enumerate(LF.SEQUENCE): + fr = LF.frame_inputs(rng, rw, rh, reset, True) + LF.fill_sentinels(p, SENT[form], i == 0) + out = p.frame(fr, debug=True) + frames.append({"inputs": fr, "out": out, "raw": LF.raw_outputs(p) if form in ("q7d", "q7h12") else {}}) + rec = {"model": "M", "form": form, "sentinel": SENT[form], "kernels": p.kernels(), "notes": p.notes(), "model_source": p.model_source, + "frames": frames, "identity": LF.capture_identity(build)} + with open(rec_path(build, form), "wb") as f: + pickle.dump(rec, f) + print(form, "frames", len(frames), "m_form", (rec["kernels"] or {}).get("m_form", "-")) + + +def run(build, rocm): + """both forms, each in its own process with its own environment (WG_M_FORM=q7 / unset)""" + import subprocess + for form in FORMS: + env = dict(os.environ) + env.pop("WG_M_FORM", None) + if form == "q7": + env["WG_M_FORM"] = "q7" + r = subprocess.run([sys.executable, os.path.abspath(__file__), "--build", build, "--rocm", rocm, + "--run-form", form], env=env) + if r.returncode != 0: + raise SystemExit("form %s: exit %d" % (form, r.returncode)) + + +def run_hybrid(build, rocm): + """forms hyb and hybmut (HIP), each in its own process with its WG_M_FORM""" + import subprocess + for form, mf in HYB_ENV.items(): + env = dict(os.environ) + env["WG_M_FORM"] = mf + r = subprocess.run([sys.executable, os.path.abspath(__file__), "--build", build, "--rocm", rocm, "--run-form", + form], env=env) + if r.returncode != 0: + raise SystemExit("form %s: exit %d" % (form, r.returncode)) + + +def run_d3d12(build, rocm): + """form q7d (native D3D12, WG_D3D12_FUSION=none) and form q7h12 (HIP, default kernels: the pack12 decoder the D3D12 + route's dec0 port reproduces), each in its own process with WG_M_FORM=q7 (read when the runtime library loads)""" + import subprocess + for form in ("q7d", "q7h12"): + env = dict(os.environ) + env["WG_M_FORM"] = "q7" + if form == "q7d": + env["WG_D3D12_FUSION"] = "none" + r = subprocess.run([sys.executable, os.path.abspath(__file__), "--build", build, "--rocm", rocm, "--run-form", + form], env=env) + if r.returncode != 0: + raise SystemExit("form %s: exit %d" % (form, r.returncode)) + + +def _perturbed(p, stage): + """the stage's output bias b2 of channel 0 moved by +1.0 (a change every token's channel 0 must show unless + clamped away; a one-FP8-step change of a single W1 weight was too weak: in b3 / b5 it moved no output code)""" + q = {k: np.array(v, copy=True) for k, v in p.items()} + q["b2"].reshape(-1)[0] += np.float32(1.0) + return q + + +def compare(build, json_path): + from clean.l.reference import oracle as L + from clean.l.reference import table as T + from clean.reference import swin as S + with open(os.path.join(LF.model_dir(build), "params.pkl"), "rb") as f: + params = pickle.load(f)["M"] + table = copy.deepcopy(T.TABLE) + table["consts"]["default"]["gelu"] = "sigmoid" # as l_frame_check (the kernels' GELU for the M block form) + table["shared_form"] = "q7" + table_h = copy.deepcopy(table) + table_h["mm8"] = "f32chain" + res = {"forms": {}} + recs = {} + for form in FORMS + ("q7d", "q7h12", "hyb", "hybmut"): + if form in ("q7d", "q7h12", "hyb", "hybmut") and not os.path.exists(rec_path(build, form)): + continue + if form == "q7h12": # raw outputs only (compared below) + with open(rec_path(build, form), "rb") as f: + recs[form] = pickle.load(f) + continue + with open(rec_path(build, form), "rb") as f: + rec = recs[form] = pickle.load(f) + frames = [] + for k, fd in enumerate(rec["frames"]): + out = fd["out"] + info = out["state"]["info"] + f = int(info["parity"]) + x, skips = out["enc0"]["features"], {} + m = {"frame": k, "stages": {}} + for s in T.SHARED_STAGES: + off = S.window_offset(s, f) + kind = S.STAGES[s]["kind"] + skip = skips.get("enc2" if s == "dec2" else "enc1") if kind == "dec" else None + ent = {} + if form == "q7": + ref = L.run_stage_q7(s, params[s], x, off, skip, table) + ent["vs_fp16_oracle"] = {key: LF.same_values(out[s][key], ref[key]) for key in out[s]} + fun = S.run_stage(s, params[s], x, off, skip, gelu="sigmoid") + ent["neg_vs_functional"] = {key: LF.same_values(out[s][key], fun[key]) for key in out[s]} + bad = L.run_stage_q7(s, _perturbed(params[s], s), x, off, skip, table) + ent["neg_perturbed_weight"] = {key: LF.same_values(out[s][key], bad[key]) for key in out[s]} + ent["pass"] = (all(v["differ"] == 0 for v in ent["vs_fp16_oracle"].values()) and + not all(v["differ"] == 0 for v in ent["neg_vs_functional"].values()) and + not all(v["differ"] == 0 for v in ent["neg_perturbed_weight"].values())) + elif form in ("hyb", "hybmut"): + refh = L.run_stage_q7(s, params[s], x, off, skip, table_h) + ent["vs_f32chain_oracle"] = {key: LF.same_values(out[s][key], refh[key]) for key in out[s]} + if form == "hyb": + ref = L.run_stage_q7(s, params[s], x, off, skip, table) + ent["neg_vs_exact_q7_oracle"] = {key: LF.same_values(out[s][key], ref[key]) for key in out[s]} + ent["pass"] = (all(v["differ"] == 0 for v in ent["vs_f32chain_oracle"].values()) and + not all(v["differ"] == 0 for v in ent["neg_vs_exact_q7_oracle"].values())) + else: # the mutant: judged over the whole run below (it must differ somewhere) + ent["pass"] = True + elif form == "q7d": + hq = recs["q7"]["frames"][k]["out"][s] + hm = recs["m"]["frames"][k]["out"][s] + ent["vs_hip_q7"] = {key: LF.same_values(out[s][key], hq[key]) for key in out[s]} + ref = L.run_stage_q7(s, params[s], x, off, skip, table) + ent["vs_fp16_oracle"] = {key: LF.same_values(out[s][key], ref[key]) for key in out[s]} + ent["neg_vs_hip_m_form"] = {key: LF.same_values(out[s][key], hm[key]) for key in out[s]} + ent["pass"] = (all(v["differ"] == 0 for v in ent["vs_hip_q7"].values()) and + all(v["differ"] == 0 for v in ent["vs_fp16_oracle"].values()) and + not all(v["differ"] == 0 for v in ent["neg_vs_hip_m_form"].values())) + else: + fun = S.run_stage(s, params[s], x, off, skip, gelu="sigmoid") + ent["vs_functional_rule"] = {key: LF.fp8_stage(out[s][key], fun[key]) for key in out[s]} + ent["pass"] = all(v["pass"] for v in ent["vs_functional_rule"].values()) + m["stages"][s] = ent + if kind == "enc": + skips[s] = out[s]["skip"] + x = out[s]["merged"] + else: + x = out[s]["out"] + m["pass"] = all(e["pass"] for e in m["stages"].values()) + frames.append(m) + res["forms"][form] = {"kernels": rec["kernels"], "model_source": rec["model_source"], + "identity": rec.get("identity"), "frames": frames, + "pass": all(m["pass"] for m in frames)} + if form == "hybmut": + nd = sum(v["differ"] for fr in frames for e in fr["stages"].values() for v in e["vs_f32chain_oracle"].values()) + res["forms"][form]["mutant_differing_values"] = nd + res["forms"][form]["pass"] = nd > 0 # caught + if form in ("hyb", "hybmut"): + res["forms"][form]["route_seen"] = (rec["kernels"] or {}).get("m_form") == HYB_ENV[form] + res["forms"][form]["pass"] = res["forms"][form]["pass"] and res["forms"][form]["route_seen"] + if form in ("q7", "q7d"): + res["forms"][form]["route_seen"] = (rec["kernels"] or {}).get("m_form") == "q7" + res["forms"][form]["pass"] = res["forms"][form]["pass"] and res["forms"][form]["route_seen"] + print(form, "PASS" if res["forms"][form]["pass"] else "FAIL", "frames", len(frames)) + for m in frames: + if not m["pass"]: + print(" frame", m["frame"], "failing:", [s for s, e in m["stages"].items() if not e["pass"]]) + if "q7d" in recs and "q7h12" in recs: + # every raw output of the frame (exposure ... dec0's pack12 images, post, state), D3D12 against HIP, both on q7 + # with pack12 dec0 (D3D12: d1p12/dec0_q7; HIP: d0p12/coef_decoder); the q7 route writes every stage map, so + # only the previous-frame state of frame 0 is unwritten (both sides keep their sentinels there) + # ... and the display image when both run the fused pack12 post (post_tf: it writes none). Both sides decode to + # pack12, so the images are compared in that layout (no conversion). + no_disp = all("post_tf" in str(recs[f]["kernels"].get("post_last", "")) for f in ("q7h12", "q7d")) + unw = lambda k, n: ("both" if (k == 0 and n in LF.UNWRITTEN_ON_FIRST) or (n == "post.display" and no_disp) + else None) + raw = LF.raw_compare(recs["q7h12"], recs["q7d"], SENT["q7h12"], SENT["q7d"], unw, (), convert_b=False) + res["d3d12_vs_hip_raw"] = {"summary": LF.raw_summary(raw), "frames": raw, + "kernels_hip": recs["q7h12"]["kernels"], "kernels_d3d12": recs["q7d"]["kernels"]} + ok = all(f["identical"] for f in raw) + print("q7d vs q7h12 raw:", "PASS" if ok else "FAIL", LF.raw_summary(raw)) + res["forms"]["q7d"]["pass"] = res["forms"]["q7d"]["pass"] and ok + res["pass"] = all(v["pass"] for v in res["forms"].values()) + with open(json_path, "w") as f: + json.dump(res, f, indent=1, default=lambda o: o.item() if hasattr(o, "item") else str(o)) + return 0 if res["pass"] else 1 + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", default="") + ap.add_argument("--run", action="store_true") + ap.add_argument("--run-form", choices=FORMS + ("q7d", "q7h12", "hyb", "hybmut"), help=argparse.SUPPRESS) + ap.add_argument("--run-hybrid", action="store_true") + ap.add_argument("--run-d3d12", action="store_true") + ap.add_argument("--compare", action="store_true") + ap.add_argument("--json", default="m_q7_check.json") + a = ap.parse_args(argv) + if a.run_form: + run_form(a.build, a.rocm, a.run_form) + return 0 + if a.run: + run(a.build, a.rocm) + if a.run_hybrid: + run_hybrid(a.build, a.rocm) + if a.run_d3d12: + run_d3d12(a.build, a.rocm) + if a.compare: + return compare(a.build, a.json) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/runtime/device.h b/clean/runtime/device.h index e3fc0517..47356dc6 100644 --- a/clean/runtime/device.h +++ b/clean/runtime/device.h @@ -101,7 +101,30 @@ class Device { return false; } virtual bool has_wait_value32() const { return false; } + // The hybrid block form's arena tables (hyb_tables.h: GELU16 -> FP8, rsqrt, rcp): hyb_reserve() reserves their fixed + // arena range and fills it (native D3D12; called by the runtime's create for a Model L context on the hybrid route, + // before its first allocation); false with err() when the range is taken or tables are off. hyb_tables(): reserved + // and filled. Other models never call it, so their arena layout is unchanged. + virtual bool hyb_reserve() { err_ = "this device has no hybrid tables (native D3D12 only)"; return false; } + virtual bool hyb_tables() const { return false; } virtual bool write_value32(DevPtr ptr, uint32_t value) { (void)ptr, (void)value; return false; } + // Frame guard (native D3D12; the hybrid Model L bottleneck's bounded wait, hyb_bneck.hlsl). guard_begin(word): from + // here on, every launch and copy of this recording is skipped on the GPU while the 8 bytes at `word` (8-aligned) are + // nonzero, read as they are at this point of the stream (a copy into a predicate buffer; D3D12 predication). Called + // again, it reads the word again. guard_end(): the guard is lifted (nothing after it is skipped) and the 16 bytes at + // `word` are copied into host-visible memory; guard_read(w) gives the newest copy the GPU has written (zeros before + // the first), without waiting. A device without the guard returns false (has_guard() false). + virtual bool has_guard() const { return false; } + virtual bool guard_begin(DevPtr word) { + (void)word; + err_ = "this device has no frame guard (native D3D12 only)"; + return false; + } + virtual bool guard_end() { return true; } + virtual bool guard_read(uint32_t w[4]) { + w[0] = w[1] = w[2] = w[3] = 0; + return false; + } // Optional (HIP graphs; runtime option graph=1): the launches between begin_capture and end_capture_launch are // recorded into a graph (stream capture) and launched as one; the instantiated graph is kept and updated in place // when the next capture has the same topology (only kernel arguments differ), else instantiated again. diff --git a/clean/runtime/device_d3d12.cpp b/clean/runtime/device_d3d12.cpp index 42cd6a7a..8bed5eab 100644 --- a/clean/runtime/device_d3d12.cpp +++ b/clean/runtime/device_d3d12.cpp @@ -15,6 +15,7 @@ #include "../converter/src/json.h" #include "../converter/src/sha256.h" #include "gelu_lut_table.h" +#include "hyb_tables.h" #include "gelu_t5_table.h" #include "native_direct_pi.h" #include "embedded.h" @@ -176,6 +177,8 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { direct_constants_map_ = nullptr; if (marker_map_ && marker_) marker_->Unmap(0, nullptr); marker_map_ = nullptr; marker_.reset(); + if (guard_map_ && guard_copy_) guard_copy_->Unmap(0, nullptr); + guard_map_ = nullptr; guard_copy_.reset(); guard_pred_.reset(); direct_constants_.reset(); direct_heap_.reset(); direct_root_.reset(); arena_.reset(); channel_.reset(); heap_.reset(); root_.reset(); gelu_t5_.reset(); queue_.reset(); fence_.reset(); @@ -219,6 +222,57 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { return reserve_events(64) && (!borrowed_ || driver_probe()) && gelu_lut_upload() && gelu_t5_upload(); } + // The hybrid block form's tables (hlsl/wg_hyb.hlsli, hyb_tables.h) at their fixed arena range right after the GELU + // table, for a Model L context on the hybrid route only: the runtime's create calls hyb_reserve() (WG_L_FORM unset = + // the 0.6.5 release default hybrid_full, or "hybrid") before its first allocation. Device init reserves nothing, so + // Model M / K contexts keep their arena layout. The range is carved out of the free list; if anything holds part of + // it, the reservation fails and the runtime refuses (never a run on other bytes). Once reserved, the tables stay + // for the device's lifetime. WG_HYB_TABLES=off is a developer diagnostic (WG_DEV_DIAGNOSTICS builds only, as #225's + // keys): it refuses, to prove the runtime's refusal on the GPU; a release build ignores it with one warning line, so a + // stray variable never refuses Model L for a player (the CPU recorder tests the refusal in every build). + bool hyb_reserve() override { + if (hyb_) return true; + const char* off = std::getenv("WG_HYB_TABLES"); +#ifdef WG_DEV_DIAGNOSTICS + if (off && std::string(off) == "off") return fail("diagnostic: hybrid tables off (WG_HYB_TABLES=off)"); +#else + if (off && *off) log(true, "WG_HYB_TABLES ignored: diagnostic key, release build; the hybrid tables are reserved"); +#endif + const uint64_t lo = wg::kHybBase, hi = (wg::kHybBase + wg::kHybBytes + kNativeAlign - 1) & ~(kNativeAlign - 1); + if (hi > arena_bytes_) return fail("hybrid tables: arena too small"); + auto i = free_.upper_bound(lo); + if (i == free_.begin()) return fail("hybrid tables: their arena range is in use"); + --i; + if (i->first > lo || i->first + i->second < hi) return fail("hybrid tables: their arena range is in use"); + const uint64_t start = i->first, size = i->second; + free_.erase(i); + if (lo > start) free_[start] = lo - start; + if (start + size > hi) free_[hi] = start + size - hi; + high_ = std::max(high_, hi); + if (!hyb_upload()) { // the context is refused; the range goes back to the free list (the device stays whole) + free_range(lo, hi - lo); + return false; + } + hyb_ = true; + return true; + } + bool hyb_upload() { + std::vector table(wg::kHybBytes); + wg::hyb_fill(table.data()); + if (!transient_room(wg::kHybBytes)) return false; + NativeCom staging; + if (!buffer(wg::kHybBytes, D3D12_HEAP_TYPE_UPLOAD, D3D12_RESOURCE_STATE_GENERIC_READ, false, staging)) return false; + void* mapped = nullptr; + D3D12_RANGE no_read{0, 0}; + if (!check(staging->Map(0, &no_read, &mapped), "Map(hybrid tables)")) return false; + std::memcpy(mapped, table.data(), wg::kHybBytes); + D3D12_RANGE written{0, wg::kHybBytes}; staging->Unmap(0, &written); + if (!open_list()) return false; + transition_arena(D3D12_RESOURCE_STATE_COPY_DEST); + list_->CopyBufferRegion(arena_.get(), wg::kHybBase, staging.get(), 0, wg::kHybBytes); + keep(std::move(staging), wg::kHybBytes); return true; + } + // The MLP kernels' exact GELU + FP8 table (hlsl/wg_gelu_lut.hlsli) at its fixed arena offset, reserved in // create_objects; recorded before any launch on this queue. bool gelu_lut_upload() { @@ -406,6 +460,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { // Create-time work on our own queue must have finished before the caller's list can run (normally a no-op). if (!finish_queue() || !create_direct_objects() || !create_marker()) return extension_result(false, error); ext_list_ = list; external_ = true; + guard_state_ = D3D12_RESOURCE_STATE_COMMON; // The arena is a buffer: it starts each ExecuteCommandLists in COMMON and is promoted implicitly to UNORDERED_ACCESS // by the first dispatch; only dispatches touch it while recording into the caller's list. arena_state_ = D3D12_RESOURCE_STATE_COMMON; @@ -414,6 +469,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { } bool end_external(std::string* error) override { if (!external_ || inside_pass_) return extension_result(fail("native D3D12 end_external: not recording"), error); + const bool guard_lifted = guard_end(); // never leave the caller's list predicated need_uav_barrier_ = true; uav_barrier(); // the caller's passes after ours read our results ++ext_recorded_; bool ok = true; @@ -425,12 +481,14 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { } else if (marker_map_) { ok = fail("native D3D12 end_external: the caller's list has no ID3D12GraphicsCommandList2 (completion marker)"); } + ok = ok && guard_lifted; ext_list_ = nullptr; external_ = false; arena_state_ = D3D12_RESOURCE_STATE_COMMON; native_binding_dirty_ = true; direct_bound_ = false; return extension_result(ok, error); } uint64_t arena_high_water() const override { return high_; } + bool hyb_tables() const override { return hyb_; } uint64_t arena_capacity() const override { return arena_bytes_; } uint64_t external_recorded() const override { return ext_recorded_; } uint64_t external_done() const override { @@ -489,6 +547,10 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { drain_debug_messages(); uint64_t offset = i->first, size = i->second.reserved; in_use_ -= size; used_.erase(i); + free_range(offset, size); + } + // [offset, offset + size) back to the free list, merged with its free neighbours + void free_range(uint64_t offset, uint64_t size) { auto next = free_.lower_bound(offset); if (next != free_.end() && offset + size == next->first) { size += next->second; next = free_.erase(next); } if (next != free_.begin()) { @@ -601,6 +663,41 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { profile_end(stamp); return true; } + bool has_guard() const override { return true; } + bool guard_begin(DevPtr word) override { + if (inside_pass_) return fail("native D3D12 guard_begin: not from a record_pass callback"); + if ((word & 7u) || !range(word, 16, "guard word")) return fail("native D3D12 guard_begin: invalid guard word"); + if (!guard_buffers() || !open_list()) return false; + if (guard_on_) cl()->SetPredication(nullptr, 0, D3D12_PREDICATION_OP_EQUAL_ZERO); + // the word as the stream has it here: earlier UAV writes complete (the arena transition), then 8 bytes copied + need_uav_barrier_ = true; uav_barrier(); + transition_arena(D3D12_RESOURCE_STATE_COPY_SOURCE); + guard_transition(D3D12_RESOURCE_STATE_COPY_DEST); + cl()->CopyBufferRegion(guard_pred_.get(), 0, arena_.get(), word, 8); + guard_word_ = word; guard_on_ = true; + guard_predicate(); + transition_arena(D3D12_RESOURCE_STATE_UNORDERED_ACCESS); + return true; + } + bool guard_end() override { + if (!guard_on_) return true; + guard_on_ = false; + if (!open_list()) return false; + cl()->SetPredication(nullptr, 0, D3D12_PREDICATION_OP_EQUAL_ZERO); + guard_transition(D3D12_RESOURCE_STATE_COMMON); + need_uav_barrier_ = true; uav_barrier(); + transition_arena(D3D12_RESOURCE_STATE_COPY_SOURCE); + cl()->CopyBufferRegion(guard_copy_.get(), 0, arena_.get(), guard_word_, 16); + // the arena back in COMMON (explicitly): what begin_external assumes for a next frame recorded into the same list + transition(arena_.get(), D3D12_RESOURCE_STATE_COPY_SOURCE, D3D12_RESOURCE_STATE_COMMON); + arena_state_ = D3D12_RESOURCE_STATE_COMMON; + return true; + } + bool guard_read(uint32_t w[4]) override { + for (int i = 0; i < 4; ++i) w[i] = guard_map_ ? guard_map_[i] : 0u; + return guard_map_ != nullptr; + } + bool load_module(const std::string& path) override { const std::string co = native_basename(path); if (!valid_co(co)) return fail("native D3D12: invalid code-object identity " + co); @@ -1006,6 +1103,29 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { auto i = tagged_cos_.find(k.name.substr(0, slash)); return i != tagged_cos_.end() && i->second == k.co; } + // The frame guard's buffers: the predicate (8 bytes used) and its host copy (16 bytes; READBACK, mapped for the + // device's life and read without a wait, as the completion marker: a stale copy is an older frame's). + bool guard_buffers() { + if (guard_pred_ && guard_copy_) return true; + if (!buffer(256, D3D12_HEAP_TYPE_DEFAULT, D3D12_RESOURCE_STATE_COMMON, false, guard_pred_) || + !buffer(256, D3D12_HEAP_TYPE_READBACK, D3D12_RESOURCE_STATE_COPY_DEST, false, guard_copy_)) + return false; + void* mapped = nullptr; D3D12_RANGE read{0, 256}; + if (!check(guard_copy_->Map(0, &read, &mapped), "Map(frame guard)")) return false; + guard_map_ = static_cast(mapped); + for (int i = 0; i < 4; ++i) guard_map_[i] = 0; + guard_state_ = D3D12_RESOURCE_STATE_COMMON; + return true; + } + void guard_transition(D3D12_RESOURCE_STATES target) { + if (guard_state_ != target) transition(guard_pred_.get(), guard_state_, target); + guard_state_ = target; + } + // predication of the current list from the predicate buffer: launches and copies are skipped while it is nonzero + void guard_predicate() { + guard_transition(D3D12_RESOURCE_STATE_PREDICATION); + cl()->SetPredication(guard_pred_.get(), 0, D3D12_PREDICATION_OP_NOT_EQUAL_ZERO); + } // The completion marker of caller-list frames: a CPU-visible buffer that WriteBufferImmediate writes (COPY_DEST). bool create_marker() { if (marker_) return true; @@ -1104,6 +1224,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (!check(list_->Reset(slot.allocator.get(), nullptr), "CommandList::Reset")) return false; current_slot_ = next; list_open_ = true; dispatches_ = 0; bind_native(); + if (guard_on_) guard_predicate(); return true; } void bind_native(bool direct = false) { @@ -1136,6 +1257,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { // Buffer resources decay to COMMON at the end of each ExecuteCommandLists call. The next list is separately // submitted on this ordered queue and can implicitly promote its first arena access without a transition. arena_state_ = D3D12_RESOURCE_STATE_COMMON; + guard_state_ = D3D12_RESOURCE_STATE_COMMON; } const uint64_t value = ++next_fence_; if (!check(queue_->Signal(fence_.get(), value), "CommandQueue::Signal")) return false; @@ -1158,7 +1280,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { channel_.p = nullptr; heap_.p = nullptr; root_.p = nullptr; gelu_t5_.p = nullptr; queue_.p = nullptr; fence_.p = nullptr; for (auto& set : direct_textures_) for (auto& texture : set) texture.p = nullptr; - marker_.p = nullptr; + marker_.p = nullptr; guard_copy_.p = nullptr; guard_pred_.p = nullptr; direct_constants_.p = nullptr; direct_heap_.p = nullptr; direct_root_.p = nullptr; info_queue_.p = nullptr; device_.p = nullptr; // Keep system DLL references and the completion-event handle too: an outstanding registration can still use it. @@ -1615,7 +1737,12 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (!arena_buffer() || !buffer(256, D3D12_HEAP_TYPE_DEFAULT, D3D12_RESOURCE_STATE_UNORDERED_ACCESS, true, channel_)) return false; static_assert(wg::kGeluLutBase == kNativeAlign, "the GELU table is the first arena range after the null header"); - free_[kNativeAlign + wg::kGeluLutBytes] = arena_bytes_ - kNativeAlign - wg::kGeluLutBytes; + static_assert(wg::kHybBase == kNativeAlign + wg::kGeluLutBytes, "the hybrid tables follow the GELU table"); + static_assert(wg::kHybBase % kNativeAlign == 0, "the hybrid tables start on an allocation boundary"); + { + const uint64_t reserved = wg::kGeluLutBytes; // the hybrid tables: hyb_reserve(), Model L hybrid contexts only + free_[kNativeAlign + reserved] = arena_bytes_ - kNativeAlign - reserved; + } D3D12_DESCRIPTOR_HEAP_DESC heap{}; heap.Type = D3D12_DESCRIPTOR_HEAP_TYPE_CBV_SRV_UAV; heap.NumDescriptors = kGeluT5Slot + 1; heap.Flags = D3D12_DESCRIPTOR_HEAP_FLAG_SHADER_VISIBLE; if (!check(device_->CreateDescriptorHeap(&heap, __uuidof(ID3D12DescriptorHeap), heap_.pv()), "CreateDescriptorHeap")) return false; @@ -1689,6 +1816,12 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { uint64_t ext_recorded_ = 0; NativeCom marker_; volatile uint32_t* marker_map_ = nullptr; + // the frame guard (Device::guard_begin): the predicate buffer (DEFAULT, COMMON between recordings) and the host copy + NativeCom guard_pred_, guard_copy_; + volatile uint32_t* guard_map_ = nullptr; + D3D12_RESOURCE_STATES guard_state_ = D3D12_RESOURCE_STATE_COMMON; + bool guard_on_ = false; + DevPtr guard_word_ = 0; NativeCom query_heap_; PFN_D3D12_SERIALIZE_ROOT_SIGNATURE serialize_ = nullptr; HMODULE dxgi_dll_ = nullptr, d3d12_dll_ = nullptr; @@ -1700,6 +1833,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { std::string profile_session_, profile_failure_; bool profile_enabled_ = false; bool borrowed_ = false, native_binding_dirty_ = false, inside_pass_ = false; + bool hyb_ = false; // hybrid tables reserved and filled (hyb_reserve) bool direct_bound_ = false, direct_root_bound_ = false; D3D12_COMMAND_LIST_TYPE queue_type_ = D3D12_COMMAND_LIST_TYPE_COMPUTE; uint64_t debug_cursor_ = 0, debug_denied_ = 0; diff --git a/clean/runtime/hyb_tables.h b/clean/runtime/hyb_tables.h new file mode 100644 index 00000000..6f1f52ad --- /dev/null +++ b/clean/runtime/hyb_tables.h @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: MIT +// The arena tables of the HYBRID block form (hlsl/wg_hyb.hlsli): devices that run SW_HYB kernels reserve arena bytes +// [kHybBase, kHybBase + kHybBytes) and fill them with hyb_fill(). The same values as clean/l/reference/hyb_tables.py +// (checked by clean/l/tests/test_hyb_tables_cpp.py through clean/tests/hyb_dump.cpp: identical bytes): +// [0, 65536) q8(GELU16(h)) by the FP16 bits of h (oracle gelu_q7 + the saturating FP8 rounding) +// [65536, 69632) r = RN16(1 / sqrtf(S')) for S' = 1.m x 2^p, p = 0 / 1 (u16, [p][m]) +// [69632, 71680) w = RN16(1 / x) for x = 1.m (u16, [m]) +// [71680, 202752) the GELU16 table again with 16-bit entries (u16 = the FP8 byte, high byte 0): the D3D12 +// kernels load one u16 per value at h << 1 (no byte extraction; hlsl/wg_hyb.hlsli hyb_gelu_q8) +// FP16 steps are float operations rounded once to FP16 (exact: float has 24 >= 2 x 11 + 2 bits); sqrtf and the +// division are IEEE correctly rounded, as numpy's float32 in the oracle. +#pragma once +#include +#include +#include +#include + +#include "gelu_lut_table.h" + +namespace wg { + +constexpr uint64_t kHybBase = kGeluLutBase + kGeluLutBytes; // = WG_HYB_BASE +constexpr size_t kHybBytes = 65536 + 4096 + 2048 + 131072; + +inline uint16_t hyb_f2h(float f) { // round to nearest even, IEEE half (subnormals, overflow to inf, NaN quiet) + uint32_t x; + std::memcpy(&x, &f, 4); + const uint32_t sign = (x >> 16) & 0x8000u; + const uint32_t ax = x & 0x7fffffffu; + if (ax > 0x7f800000u) return (uint16_t)(sign | 0x7e00u); + if (ax >= 0x477ff000u) return (uint16_t)(sign | 0x7c00u); // >= 65520: inf (65504 + half ulp rounds up) + if (ax < 0x38800000u) { // below 2^-14: subnormal half + const float a = std::fabs(f) * 16777216.0f; // x 2^24: exact scaling + const float r = std::nearbyint(a); // ties to even (default rounding mode) + return (uint16_t)(sign | (uint32_t)r); + } + const uint32_t m = ax & 0x7fffffu; + uint32_t h = ((ax >> 23) - 112u) << 10 | (m >> 13); + const uint32_t rest = m & 0x1fffu; + if (rest > 0x1000u || (rest == 0x1000u && (h & 1u))) ++h; + return (uint16_t)(sign | h); +} + +inline float hyb_h2f(uint16_t h) { + const uint32_t s = (uint32_t)(h & 0x8000u) << 16, e = (h >> 10) & 31u, m = h & 1023u; + uint32_t x; + if (e == 0) { + const float v = (float)m * (1.0f / 16777216.0f); + std::memcpy(&x, &v, 4); + x |= s; + } else if (e == 31) { + x = s | 0x7f800000u | (m << 13); + } else { + x = s | ((e + 112u) << 23) | (m << 13); + } + float f; + std::memcpy(&f, &x, 4); + return f; +} + +inline float hyb_rn16(float f) { return hyb_h2f(hyb_f2h(f)); } + +// FP8 E4M3FN, round to nearest even, saturating at +-448 (clean/reference/fp8.py q8 + encode); NaN -> 0xff, the +// hardware conversion's NaN on HIP and D3D12. +inline uint8_t hyb_fp8(float v) { + if (std::isnan(v)) return 0xffu; + const uint8_t s = std::signbit(v) ? 0x80u : 0u; + float a = std::fabs(v); + if (a >= 448.0f) return (uint8_t)(s | 0x7eu); + int e; + std::frexp(a, &e); // a = m 2^e, m in [0.5, 1) + int ex = e - 1; + if (ex < -6) ex = -6; + const float spacing = std::ldexp(1.0f, ex - 3); + const float r = std::nearbyint(a / spacing) * spacing; // exact quotient; ties to even + if (r >= 448.0f) return (uint8_t)(s | 0x7eu); + if (r == 0.0f) return s; + int er; + const float mr = std::frexp(r, &er); + int field = er - 1 + 7; + if (field <= 0) return (uint8_t)(s | (uint8_t)(r / std::ldexp(1.0f, -9))); // subnormal: r = k 2^-9 + const int mant = (int)((mr * 2.0f - 1.0f) * 8.0f); + return (uint8_t)(s | (uint8_t)(field << 3) | (uint8_t)mant); +} + +inline float hyb_gelu16(float h) { // oracle gelu_q7: every step its own FP16 rounding, no fused operation + const float a = hyb_h2f(0x3698u), b = hyb_h2f(0x2d30u); + const float y = std::fmin(std::fmax(h, -2.0f), 2.0f); + const float inner = hyb_rn16(a - hyb_rn16(b * std::fabs(y))); + return hyb_rn16(h * hyb_rn16(0.5f + hyb_rn16(y * inner))); +} + +// The values above assume round-to-nearest-even float arithmetic (nearbyint, the float steps, sqrtf and the division +// all follow the thread's rounding mode); hyb_fill() sets that mode for the generation and restores the caller's. +inline void hyb_fill_nearest(uint8_t* dst) { + for (uint32_t i = 0; i < 65536; ++i) dst[i] = hyb_fp8(hyb_gelu16(hyb_h2f((uint16_t)i))); + uint8_t* g16 = dst + 71680; // little-endian u16 entries: byte, 0 + for (uint32_t i = 0; i < 65536; ++i) { g16[2 * i] = dst[i]; g16[2 * i + 1] = 0; } + uint16_t* rsq = (uint16_t*)(dst + 65536); + uint16_t* rcp = (uint16_t*)(dst + 69632); + for (uint32_t m = 0; m < 1024; ++m) { + const float base = 1.0f + (float)m / 1024.0f; + rsq[m] = hyb_f2h(1.0f / std::sqrt(base)); + rsq[1024 + m] = hyb_f2h(1.0f / std::sqrt(base * 2.0f)); + rcp[m] = hyb_f2h(1.0f / base); + } +} + +// The tables with round-to-nearest-even arithmetic whatever the host thread's rounding mode; the caller's mode is +// restored (test_hyb_tables_cpp.py: identical bytes under FE_DOWNWARD, mode restored). +inline void hyb_fill(uint8_t* dst) { + const int mode = std::fegetround(); + std::fesetround(FE_TONEAREST); + hyb_fill_nearest(dst); + std::fesetround(mode); +} +} // namespace wg diff --git a/clean/runtime/m_q7_pack.h b/clean/runtime/m_q7_pack.h new file mode 100644 index 00000000..6df605cb --- /dev/null +++ b/clean/runtime/m_q7_pack.h @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: MIT +// Model M on the FP16-form (q7) kernels of Model L's shared stages (WG_M_FORM=q7, runtime.cpp): the blobs of Model M's +// ten shared stages for l_q7.co / l_q7w.co, packed from Model M's own tensors with the q7 packers of Model L +// (clean/l/runtime/l_q7_pack.h, byte for byte clean/l/runner/l_pack.py pack_q7_block / pack_q7_linear / q7_tables). +// Model M's stage tensors carry the same names and shapes as Model L's shared stages (spec 08 section 7.2): the block +// tensors gamma1, gamma2, bp, b1, b2, B, Wqk, Wv, Wp, W1, W2, the merges Wm / bm (enc1, enc2) and the expansions +// We / be (dec2, dec1). Blob names as l::pack_q7_model's: ".q7", ".q7lin", "q7.tab". +#pragma once + +#include +#include +#include + +#include "../l/runtime/l_q7_pack.h" +#include "model.h" + +namespace wgrt { +namespace m_q7 { + +inline const Tensor* tensor(const StageTensors& t, const char* stage, const char* name, size_t n, std::string* err) { + const auto it = t.find(name); + if (it == t.end()) return *err = std::string("Model M on q7: stage ") + stage + " has no tensor " + name, nullptr; + if (it->second.f.size() != n) + return *err = std::string("Model M on q7: ") + stage + "." + name + " has " + std::to_string(it->second.f.size()) + + " values, the q7 block needs " + std::to_string(n), + nullptr; + return &it->second; +} + +// The ten shared stages' q7 blobs and the tables. False with *err when a stage or tensor is missing or mis-sized. +inline bool pack(const std::map& model, std::map>* out, + std::string* err) { + struct S { + const char* stage; + int C, heads, lin_n, lin_k; // lin: merge (n = C_out, K = 4 C) or expansion (n = 4 C, K = C_low); 0 = none + const char *lw, *lb; + }; + static const S kS[] = {{"enc1", 64, 2, 96, 256, "Wm", "bm"}, {"enc2", 96, 4, 128, 384, "Wm", "bm"}, + {"b0", 128, 4, 0, 0, nullptr, nullptr}, {"b1", 128, 4, 0, 0, nullptr, nullptr}, + {"b2", 128, 4, 0, 0, nullptr, nullptr}, {"b3", 128, 4, 0, 0, nullptr, nullptr}, + {"b4", 128, 4, 0, 0, nullptr, nullptr}, {"b5", 128, 4, 0, 0, nullptr, nullptr}, + {"dec2", 96, 4, 384, 128, "We", "be"}, {"dec1", 64, 2, 256, 96, "We", "be"}}; + for (const S& s : kS) { + const auto st = model.find(s.stage); + if (st == model.end()) return *err = std::string("Model M on q7: the model has no stage ") + s.stage, false; + const StageTensors& t = st->second; + const size_t C = (size_t)s.C, h = (size_t)s.heads, m = 4 * C; + const Tensor* g1 = tensor(t, s.stage, "gamma1", C, err); + const Tensor* g2 = g1 ? tensor(t, s.stage, "gamma2", C, err) : nullptr; + const Tensor* bp = g2 ? tensor(t, s.stage, "bp", C, err) : nullptr; + const Tensor* b1 = bp ? tensor(t, s.stage, "b1", m, err) : nullptr; + const Tensor* b2 = b1 ? tensor(t, s.stage, "b2", C, err) : nullptr; + const Tensor* B = b2 ? tensor(t, s.stage, "B", h * 256, err) : nullptr; + const Tensor* Wqk = B ? tensor(t, s.stage, "Wqk", h * 32 * C, err) : nullptr; + const Tensor* Wv = Wqk ? tensor(t, s.stage, "Wv", h * 32 * C, err) : nullptr; + const Tensor* Wp = Wv ? tensor(t, s.stage, "Wp", C * 32 * h, err) : nullptr; + const Tensor* W1 = Wp ? tensor(t, s.stage, "W1", m * C, err) : nullptr; + const Tensor* W2 = W1 ? tensor(t, s.stage, "W2", C * m, err) : nullptr; + if (!W2) return false; + std::vector blob; + if (!l::q7pack::pack_block(g1->f, g2->f, bp->f, b1->f, b2->f, B->f, Wqk->f, Wv->f, Wp->f, W1->f, W2->f, s.C, + s.heads, (int)m, &blob, err)) + return *err = std::string("Model M on q7: ") + s.stage + ": " + *err, false; + (*out)[std::string(s.stage) + ".q7"] = std::move(blob); + if (s.lin_n) { + const Tensor* lw = tensor(t, s.stage, s.lw, (size_t)s.lin_n * s.lin_k, err); + const Tensor* lb = lw ? tensor(t, s.stage, s.lb, (size_t)s.lin_n, err) : nullptr; + if (!lb) return false; + if (!l::q7pack::pack_linear(lw->f, lb->f, s.lin_n, s.lin_k, &blob, err)) + return *err = std::string("Model M on q7: ") + s.stage + ": " + *err, false; + (*out)[std::string(s.stage) + ".q7lin"] = std::move(blob); + } + } + (*out)["q7.tab"] = l::q7pack::q7_tables(); + return true; +} + +} // namespace m_q7 +} // namespace wgrt diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index 87b4d2d0..0e91f8db 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -18,7 +18,9 @@ #endif #ifdef WG_MODEL_L // Model L (#89; model_select.h model_l_built): its model data and weight packer (clean/l/runtime) #include "../l/runtime/l_model.h" +#include "../l/runtime/l_hyb_pack.h" #include "../l/runtime/l_pack.h" +#include "m_q7_pack.h" #endif #include "device.h" #include "embedded.h" @@ -267,6 +269,53 @@ struct Runtime { DevPtr l_wide = 0, l_narrow = 0, l_x3 = 0; bool l_nc[2] = {false, false}; // enc0_block, dec0_block: clamp-free block kernel (bit 0 of the blob's first word) #endif + // Model L's blocks in their FP16 form (the q7 kernels l_q7.co / l_q7w.co, clean/l/reference/oracle.py form "q7" + // and shared_form "q7"): its own two blocks and the ten shared stages (L's own variants; Model M's kernels unchanged): + // WG_L_FORM=q7 (reference) and the HIP hybrid_full reference route (l_q7__hyb.co). Declared in every build (set only + // with WG_MODEL_L), so that the paths reading it compile without Model L. + bool l_q7 = false; + // Model M on Model L's FP16-form shared-stage kernels (WG_M_FORM=q7, default off: Model M's own blocks; set only in + // builds with WG_MODEL_L): its ten shared stages run l_q7_stages with Model M's weights (m_q7_pack.h), Model M's + // encoder / decoder / post unchanged, its fusions off on that route (dec1 not fused into dec0: the decoder reads + // out1). For Q11 part C (A11). + bool m_q7 = false; + // WG_M_FORM=hybrid (Q11 part C, HIP only, evaluation): the q7 route with l_q7w__hyb.co (float32 chains, l_q7w.hip + // Q7_HYB) in place of l_q7w.co; hybrid_mutant: l_q7w__hybmut.co (the check's mutant). m_form names the value. + std::string m_form_name; + // WG_M_POST=orig (Q11 part C, HIP only, evaluation): Model M reconstructs with its ORIGINAL form (handoff + // preset-M/from-spec/Q11C-M-POST-ITEMS.md): l_post_recon of l_post__morig.co (l_post.hip with WG_LP_M, WG_LP_C1, + // WG_CR_C2), loaded with tag "morig", with post_recon's arguments and Model M's scale s; the unbanded post_recon route + // (8-byte history, base coefficient format, no pack12) as Model L's, then Model M's output filter. HDR only. + bool m_post_orig = false; + // Model L's Q11 part C evaluation forms (HIP only): WG_L_FORM=hybrid = the q7 route with l_q7__hyb.co (C32 blocks) + // and l_q7w__hyb.co (shared stages), float32 chains (Q7_HYB); hybrid_mutant = l_q7__hybmut.co (the check's mutant) + // with l_q7w__hyb.co. WG_L_POST=c12 (HIP and native D3D12, HDR and LDR; implied by WG_L_FORM=hybrid on native D3D12): + // l_post_recon[_p12][_ldr] of l_post__lc12.co (l_post.hip with WG_LP_C1, WG_CR_C2: + // Model L's own reconstruction with the machine-form exponent C1 and the Catmull-Rom weights C2), tag "lc12". + std::string l_form_name; + std::string l_form_label = "Model L"; // the resolved form in refusals ("Model L hybrid (default)", "WG_L_FORM=hybrid") + bool l_post_c12 = false; + // WG_L_FORM=hybrid on native D3D12 (hyb-fast, 0.6.5 L release form): Model L's M-form route (l_q7 off) with every + // block stage packed from the hybrid values (clean/l/runtime/l_hyb_pack.h, the same blob names and layout) and every + // Swin kernel launched as its hybrid twin from the kernel manifest (hyb-fast's names: the M-form route's name with + // "_nc" replaced by "_hyb", or "_hyb" for a name without "_nc"; untagged). The route is the UNFUSED one (no + // D3D12 fusions, no fused merge / expand, no bottleneck / mid megakernels, no L lc / le fusions: hyb-fast has no fused + // hybrid kernels), and every block takes the "_nc" name (the hybrid kernels always saturate at +-448 and never read + // the blob's safe flag). A Swin / merge / expand kernel without a twin refuses the frame: never a silent mix with the + // M-form kernels. The C1/C2 reconstruction (WG_L_POST=c12) is implied. + bool l_hyb_d3d12 = false; + // native D3D12 hybrid route (speed lever 4): b0 ... b5 in one launch of swin_bneck6_hyb (kernels/hyb_bneck.hlsl, + // bneck.hlsl's persistent queue around swin_block_c128_hyb's body: b5's bytes of the six launches; b0 ... b4's + // outputs not kept), on bneck_tmp / bneck_sync; the HIP twins (hybrid_twin) keep the six launches + bool hyb_mega = false; + int hyb_mega_groups = 128; + // swin_bneck6_hyb's frame guard (hyb_bneck.hlsl): the wait-timeout count (sync word 15) last seen in the device's + // copy, and whether the abort word (sync word 12) is to be cleared before the next frame + uint32_t bneck_seen = 0; + bool bneck_clear = false; + int dev_bneck_abort = -1; // WG_DEV_BNECK_ABORT (developer builds): the frame whose bottleneck is made to time out + bool diag_hooks_ignored = false; // a release build ignored the diagnostic launch hooks on the hybrid route + std::vector> hyb_map; // M-form route name -> its hybrid twin DRow drow; int block_threads[5] = {256, 256, 256, 256, 256}, gemm_threads = 256, nt = 2, wpw[3] = {1, 1, 1}; // = variants.DEFAULT_GEOMETRY int block_wpg[5] = {0, 0, 0, 0, 0}; // windows per workgroup per block kernel; 0 = threads / 32 * wpw @@ -559,6 +608,76 @@ struct Runtime { if (model_id == ModelId::K) return fail("Model K is not part of this build"); // (model_from_options says so first) #endif model_l = model_id == ModelId::L; // (model_from_options accepts it only in builds with WG_MODEL_L) +#ifdef WG_MODEL_L + if (model_l) { + // Model L's form (WG_L_FORM; 0.6.5 release default: hybrid_full, oversight 2026-11-15 18:36Z): + // unset or "hybrid": hybrid_full = the FP16 form with float32 chains plus the C1/C2 reconstruction. On native + // D3D12 the fast route (l_hyb_d3d12: the manifest's "_hyb" twins on the unfused M-form layout, lc12 post); + // on HIP the scalar reference route (l_q7__hyb.co + l_q7w__hyb.co, lc12 post), the accuracy reference. + // "q7": the exact FP16 form, an opt-in REFERENCE only (HIP: l_q7.co / l_q7w.co; D3D12: the q7 ports). + // "hybrid_mutant": HIP only, the check's mutant. + // "m": REFUSED (the M-form is not offered for Model L). Any other value: refused. + // A route that cannot be taken is refused at create with its reason: never a fallback to another form, the q7 + // reference or HIP (and in the game, no move to the HIP executor: executor_select.h dx12_refusal_action). + const char* lform = std::getenv("WG_L_FORM"); + const std::string lf = lform ? lform : ""; + if (lf == "m") + return fail("WG_L_FORM=m refused: the M-form is not offered for Model L (default hybrid_full; q7 = reference)"); +#ifndef WG_DEV_DIAGNOSTICS + // release builds: the check routes (the deliberately altered mutant, the HIP twins) are not offered + if (lf == "hybrid_mutant" || lf == "hybrid_twin") + return fail("WG_L_FORM=" + lf + " refused: a check route of developer builds only (unset = hybrid_full)"); +#endif + if (!lf.empty() && lf != "q7" && lf != "hybrid" && lf != "hybrid_mutant" && lf != "hybrid_twin") + return fail("WG_L_FORM: unknown form \"" + lf + "\" (hybrid = the default, q7 = reference)"); + // hybrid_twin (HIP backend, check only): the fast hybrid route of native D3D12 on the HIP twins of its *_hyb + // ports (l_swin_hyb.co, swin_hyb.co): the network-level byte leg (twins vs port) + if (lf == "hybrid_twin" && c.backend != 0) return fail("WG_L_FORM=hybrid_twin runs on the HIP backend only"); + const bool hyb = lf.empty() || lf == "hybrid" || lf == "hybrid_twin"; + l_q7 = true; + l_form_name = hyb ? std::string("hybrid") : (lf == "hybrid_mutant" ? lf : std::string()); + // the hybrid refusals name the form as resolved: nothing set is the release default, not a WG_L_FORM value + l_form_label = lf.empty() ? std::string("Model L hybrid (default)") : "WG_L_FORM=" + lf; + l_hyb_d3d12 = (hyb && c.backend == 3) || lf == "hybrid_twin"; // the fast hybrid route (twins: on HIP) + if (l_hyb_d3d12) l_q7 = false; // the fast kernels' M-form layout (the HIP reference route keeps l_q7__hyb) + if (lf == "hybrid_mutant" && c.backend != 0) return fail("WG_L_FORM=hybrid_mutant: the HIP backend only"); + const char* lpost = std::getenv("WG_L_POST"); + const std::string lp = lpost ? lpost : ""; + if (!lp.empty() && lp != "c12") return fail("WG_L_POST: unknown value \"" + lp + "\" (c12)"); + l_post_c12 = lp == "c12" || hyb; // hybrid_full: the hybrid network AND the C1/C2 reconstruction + if (l_post_c12 && c.backend != 0 && c.backend != 3) + return fail("WG_L_POST=c12: HIP (backend 0) and native D3D12 (backend 3) only"); + } +#endif +#ifdef WG_MODEL_L + if (model_id == ModelId::M) { + const char* mform = std::getenv("WG_M_FORM"); + const std::string mf = mform ? mform : ""; +#ifndef WG_DEV_DIAGNOSTICS + if (mf == "hybrid_mutant") + return fail("WG_M_FORM=hybrid_mutant refused: a check route of developer builds only"); +#endif + m_q7 = mf == "q7" || mf == "hybrid" || mf == "hybrid_mutant"; + m_form_name = m_q7 ? mf : ""; + if (!mf.empty() && !m_q7) return fail("WG_M_FORM: unknown form \"" + mf + "\" (q7, hybrid)"); + const char* mpost = std::getenv("WG_M_POST"); + const std::string mp = mpost ? mpost : ""; + m_post_orig = mp == "orig"; + if (!mp.empty() && !m_post_orig) return fail("WG_M_POST: unknown value \"" + mp + "\" (orig)"); + if (m_post_orig && c.backend != 0) return fail("WG_M_POST=orig runs on the HIP backend only"); + if (m_post_orig && !c.hdr) return fail("WG_M_POST=orig: HDR colour only"); + if (m_q7 && mf != "q7" && c.backend != 0) return fail("WG_M_FORM=" + mf + " runs on the HIP backend only"); + if (m_q7 && (c.backend == 1 || c.backend == 2)) + return fail("WG_M_FORM=q7 runs on the HIP backend (backend 0) and native D3D12 (backend 3) only"); + if (m_q7 && c.backend == 3) { // the q7 shared-stage ports (clean/dx12/kernels/l_q7.hlsl, l_q7w.hlsl) + const std::string missing = + manifest_missing({"l_q7_block_c64", "l_q7_merge_c64", "l_q7_block_c96", "l_q7_merge_c96", "l_q7_block_c128", + "l_q7_expand_c128", "l_q7_block_c96_dec", "l_q7_expand_c96", "l_q7_block_c64_dec", + "d1p12/dec0_q7"}); + if (!missing.empty()) return fail("WG_M_FORM=q7 on native D3D12: not in the kernel manifest: " + missing); + } + } +#endif if (model_l && (c.backend == 1 || c.backend == 2)) return fail("Model L runs on the HIP backend (backend 0) and native D3D12 (backend 3) only"); #ifdef WG_MODEL_L @@ -571,6 +690,30 @@ struct Runtime { l_le_built = std::find(have.begin(), have.end(), "l_swin_block_c32_enc_lin_nc") != have.end(); l_q20_built = std::find(have.begin(), have.end(), "lev/input_encoder_q20") != have.end(); l_q20d_built = std::find(have.begin(), have.end(), "lev/input_encoder_direct_q20") != have.end(); + if (l_post_c12) { + const std::string miss = manifest_missing({"lc12/l_post_recon", "lc12/l_post_recon_p12", "lc12/l_post_recon_ldr", + "lc12/l_post_recon_p12_ldr"}); + if (!miss.empty()) return fail("WG_L_POST=c12 on native D3D12: not in the kernel manifest: " + miss); + } + if (l_hyb_d3d12) { + for (const std::string& h : have) + if (h.size() > 4 && h.compare(h.size() - 4, 4, "_hyb") == 0) { + const std::string base = h.substr(0, h.size() - 4); + hyb_map.push_back({base + "_nc", h}); + hyb_map.push_back({base, h}); + } + // the hybrid twins of the fusions le / lc (speed lever 3) are part of the route: ONE native-D3D12 hybrid route + // (the encoder block with the linears from the q20 map, the decoder block with the pack12 store on pack12 + // frames), each kernel it needs refused by name when missing (no second, untested route) + l_fold_built = l_le_built = true; + hyb_mega = true; // and the bottleneck in one launch (speed lever 4), refused by name when missing + if (c.swin_force_clamp) return fail(l_form_label + " on native D3D12: no forced clamped Swin kernels"); + const std::string miss = manifest_missing( + {"l_swin_block_c32_enc_hyb", "l_swin_block_c32_dec_hyb", "l_swin_block_c32_enc_lin_hyb", + "l_swin_block_c32_dec_coef12_hyb", "lev/input_encoder_q20", "lev/input_encoder_direct_q20", + "swin_bneck6_hyb"}); + if (!miss.empty()) return fail(l_form_label + " on native D3D12: hybrid kernels not in the manifest: " + miss); + } } #endif if (c.max_rw <= 0 || c.max_rh <= 0 || c.ow <= 0 || c.oh <= 0) return fail("invalid render or output size"); @@ -598,6 +741,17 @@ struct Runtime { if (c.wpw96) wpw[1] = c.wpw96; if (c.wpw128) wpw[2] = c.wpw128; if (!choose_kernels(c)) return false; + // Model L's hybrid form (every backend, direct I/O or not): its kernels have one launch geometry (256-lane GEMM + // workgroups of NT 2 x 16 tokens per wave, one window per wave, eight windows per 256-lane block workgroup). A + // geometry override would size the grids for another shape while the shaders keep theirs (work silently dropped), + // so it is refused here, before anything is allocated. + if (model_l && (l_hyb_d3d12 || l_form_name == "hybrid")) { + bool bad = gemm_threads != 256 || nt != 2 || wpw[0] != 1 || wpw[1] != 1 || wpw[2] != 1; + for (int k = 0; k < 5; ++k) bad = bad || block_threads[k] != 256 || (block_wpg[k] != 0 && block_wpg[k] != 8); + if (bad) + return fail(l_form_label + ": a Swin / GEMM launch-geometry override (threads, wpg, wpw, nt) is refused: the " + "hybrid kernels have one geometry (256 lanes, eight windows per workgroup, nt 2)"); + } if (model_l && recon_output) return fail("Model L: recon=output (D = O) is Model M's experimental mode"); // (Model L's reconstruction has HDR and LDR forms: l_post_recon / l_post_recon_ldr; the frame chooses by hdr.) if (model_l && enc0_co != "encoder.co") @@ -651,6 +805,11 @@ struct Runtime { model_source = use_wgcm ? "model.wgcm" : "raw"; if (c.swin_force_clamp) force_clamp = true; if (c.swin_fuse) fuse = true; +#ifdef WG_MODEL_L + if (l_hyb_d3d12) // every block stage in the hybrid values, packed below by the unchanged packers + for (auto& kv : model) + if (l::hyb_block_stage(kv.first)) kv.second = l::hyb_stage_values(kv.second); +#endif for (const std::string& s : model_stages()) { if (s == "post" || (model_l && (s == "enc0" || s == "dec0"))) continue; // Model L's: below if (!pack_stage_blob(s, model[s], drow, &blobs[s], &e)) return fail(e); @@ -660,22 +819,32 @@ struct Runtime { std::map> lb; if (!l::pack_model(model, drow, &lb, &e)) return fail(e); for (auto& kv : lb) blobs[kv.first] = std::move(kv.second); - for (int i = 0; i < 2; ++i) { + if (l_q7) { + std::map> qb; + if (!l::pack_q7_model(model, &qb, &e)) return fail(e); + for (auto& kv : qb) blobs[kv.first] = std::move(kv.second); + } + for (int i = 0; i < 2 && !l_q7; ++i) { uint32_t flags = 0; std::memcpy(&flags, blobs[i ? "dec0_block" : "enc0_block"].data(), 4); - l_nc[i] = (flags & 1u) != 0; + l_nc[i] = (flags & 1u) != 0 || l_hyb_d3d12; // hybrid: the twins of the "_nc" names (always saturating) if (!l_nc[i] && c.backend == 3) return fail(std::string("Model L on native D3D12: ") + (i ? "dec0_block" : "enc0_block") + "'s weights need the clamped block, which is not ported to D3D12 (only the _nc forms are)"); } } + if (m_q7) { // Model M's ten shared stages for the q7 kernels (its own stage blobs above stay; unused on that route) + std::map> qb; + if (!wgrt::m_q7::pack(model, &qb, &e)) return fail(e); + for (auto& kv : qb) blobs[kv.first] = std::move(kv.second); + } #endif // kernels/swin.hip: the first word of a Swin stage's packed weights says whether the clamp-free block kernel // (`_nc`, identical results) may be used. for (const char* s : kSwinOrder) { uint32_t flags = 0; std::memcpy(&flags, blobs[s].data(), 4); - swin_nc[s] = (flags & 1u) != 0; + swin_nc[s] = (flags & 1u) != 0 || l_hyb_d3d12; // hybrid: the "_nc" names, whose twins always saturate } post_s = model["post"]["s"].f[0]; blobs["post"].assign(16, 0); // runner/post.py pack: s as f32, zero padding to 16 bytes @@ -720,6 +889,47 @@ struct Runtime { !dev->load_module(build_dir + "/l_ops.co") || !dev->load_module(build_dir + "/l_swin.co") || !dev->load_module(build_dir + "/l_post.co"))) return fail(dev->err() + " (Model L's code objects l_enc_values.co, l_ops.co, l_swin.co, l_post.co)"); + if (model_l && l_q7 && + (!dev->load_module(build_dir + (l_form_name == "hybrid" ? "/l_q7__hyb.co" +#ifdef WG_DEV_DIAGNOSTICS + : l_form_name == "hybrid_mutant" ? "/l_q7__hybmut.co" +#endif + : "/l_q7.co")) || + !dev->load_module(build_dir + (l_form_name.empty() ? "/l_q7w.co" : "/l_q7w__hyb.co")))) + return fail(dev->err() + " (Model L's FP16-form code objects l_q7.co, l_q7w.co; WG_L_FORM=m for the old form)"); + // The hybrid route reads the arena tables (hyb_tables.h), reserved here for Model L hybrid contexts only (before the + // first allocation; other models keep their arena layout). Without them it would read whatever buffer sits there, + // so it is refused with one line, never run and never replaced by another route (no-fallback rule). + if (model_l && l_hyb_d3d12 && c.backend == 3 && !dev->hyb_reserve()) + return fail(l_form_label + " on native D3D12: no hybrid tables (" + dev->err() + + "); refused, no other route"); +#ifdef WG_DEV_DIAGNOSTICS // the twin route exists only in developer builds + if (model_l && l_hyb_d3d12 && c.backend == 0) { // hybrid_twin: the HIP twins under the D3D12 port names + if (!dev->load_module(build_dir + "/l_swin_hyb.co") || !dev->load_module(build_dir + "/swin_hyb.co")) + return fail(dev->err() + " (WG_L_FORM=hybrid_twin: l_swin_hyb.co, swin_hyb.co)"); + for (const char* h : {"l_swin_block_c32_enc_hyb", "l_swin_block_c32_dec_hyb", "swin_block_c64_hyb", + "swin_block_c64_add_hyb", "swin_block_c96_hyb", "swin_block_c96_add_hyb", "swin_block_c128_hyb", + "patch_merge_c64_hyb", "patch_merge_c96_hyb", "expand_c96_hyb", "expand_c128_hyb"}) { + const std::string hs = h, base = hs.substr(0, hs.size() - 4); + hyb_map.push_back({base + "_nc", hs}); + hyb_map.push_back({base, hs}); + } + } +#endif + if (model_l && l_post_c12 && !dev->load_module_tagged(build_dir + "/l_post__lc12.co", "lc12")) + return fail(dev->err() + " (WG_L_POST=c12: l_post__lc12.co)"); +#ifdef WG_MODEL_L + if (m_q7 && // every backend (native D3D12 maps its l_q7_* ports by these code-object names) + (!dev->load_module(build_dir + "/l_q7.co") || + !dev->load_module(build_dir + (m_form_name == "hybrid" ? "/l_q7w__hyb.co" +#ifdef WG_DEV_DIAGNOSTICS + : m_form_name == "hybrid_mutant" ? "/l_q7w__hybmut.co" +#endif + : "/l_q7w.co")))) + return fail(dev->err() + " (WG_M_FORM=q7: the FP16-form code objects l_q7.co, l_q7w.co)"); + if (m_post_orig && !dev->load_module_tagged(build_dir + "/l_post__morig.co", "morig")) + return fail(dev->err() + " (WG_M_POST=orig: l_post__morig.co)"); +#endif if (!stage_variants(c)) return false; if (!runtime_options(c)) return false; @@ -740,6 +950,43 @@ struct Runtime { if (l_post_own_given && !model_l) return fail("l_post=own is a Model L option (its own reconstruction)"); if (!mid_setup(c)) return false; if (!bneck_setup(c)) return false; +#ifdef WG_MODEL_L + if (l_hyb_d3d12) { // the unfused route (see l_hyb_d3d12), after every option and variant geometry is read + // dec1 (and dec2) run as the standalone expand + the block (swin_stage's unfused decoder path), enc1 / enc2 as + // the block + the standalone merge, all on the base geometry (the twins take the base route's grid, lanes and + // arguments): no per-stage variant geometries, no weight-resident kernels, no megakernels + fuse = false; + fuse_stages.clear(); + stage_geo.clear(); + swin_resident = 0; + bneck_variant.clear(); + mid_variant.clear(); + // every twin this route launches, checked at create (the frame-time guard stays as the backstop) + const std::string miss = manifest_missing( + {"l_swin_block_c32_enc_hyb", "l_swin_block_c32_dec_hyb", "swin_block_c64_hyb", "patch_merge_c64_hyb", + "swin_block_c96_hyb", "patch_merge_c96_hyb", "swin_block_c128_hyb", "expand_c128_hyb", + "swin_block_c96_add_hyb", "expand_c96_hyb", "swin_block_c64_add_hyb"}); + if (!miss.empty()) + return fail(l_form_label + " on native D3D12: the unfused route's hybrid kernels are not in the manifest: " + miss); + if (hyb_mega) { // swin_bneck6_hyb's scratch map and zeroed sync words (bneck_setup's sizes; it leaves them zero) + // its bounded wait refuses a frame through the device's frame guard (no guard: no route) + if (!dev->has_guard()) + return fail(l_form_label + " on native D3D12: the device has no frame guard (the bottleneck's wait timeout)"); +#ifdef WG_DEV_DIAGNOSTICS + if (const char* e = std::getenv("WG_DEV_BNECK_ABORT")) { + if (external_io || native_direct_io) + return fail("WG_DEV_BNECK_ABORT: a check of the runtime's own frames (not interop / direct I/O)"); + dev_bneck_abort = std::atoi(e); + } +#endif + const size_t FH = ceil32(max_rh), FW = ceil32(max_rw), H8 = FH / 8, W8 = FW / 8; + const size_t nsync = 16 + 7 * ((H8 + 6) / 4); + if (!(bneck_tmp || alloc(&bneck_tmp, H8 * W8 * 128))) return false; + if (!bneck_sync && !(alloc(&bneck_sync, 4 * nsync) && (dev->memset(bneck_sync, 0, 4 * nsync) || dfail()))) + return false; + } + } +#endif // D = O (fast mode, recon=output) runs without pack12, so dec1 + dec0 fused is the default there (bit-identical; // measured -0.027 ms per 1440p-Q frame); the D = 1.5 x path keeps pack12 and the separate kernels. if (choose(c.dec1_dec0, "WG_DEC1_DEC0", recon_output && !p12 ? "fused" : "separate") == "fused") { @@ -769,6 +1016,14 @@ struct Runtime { if (!dev->load_module_tagged(d1p, "d1p12")) return dfail(); d1d0_p12 = true; } +#ifdef WG_MODEL_L + // Model M on q7, native D3D12: dec0 runs as d1p12/dec0_q7, a port under this code object (the fused kernel's + // epilogue on dec1's stored map), also when the fused d1p12 path itself is not taken + if (m_q7 && c.backend == 3 && !d1d0_p12) { + if (!dec_div.empty()) return fail("WG_M_FORM=q7 on native D3D12: dec0 with the default divisions only"); + if (!dev->load_module_tagged(build_dir + "/dec1_dec0__pack12.co", "d1p12")) return dfail(); + } +#endif } if (output_original) { // "original" needs its own build of the post kernels (wave64, HIP). Asked for by name: an error without it; @@ -823,7 +1078,12 @@ struct Runtime { } } // dec1 and dec2 (dec2 measured 2026-10-07: -0.012 ms per 1440p-Q frame, bit-identical) - fuse_stages_text = choose(c.swin_fuse_stages, "WG_SWIN_FUSE_STAGES", swin_co == "swin.co" ? "dec1,dec2" : "none"); + // (WG_L_FORM=hybrid on native D3D12: the unfused route, default none; the setup above cleared the list, and this + // later read must not bring the fused decoder stages back: no hybrid twin of swin_block_c96_add_ex / _c64_add_ex) + fuse_stages_text = choose(c.swin_fuse_stages, "WG_SWIN_FUSE_STAGES", + swin_co == "swin.co" && !l_hyb_d3d12 ? "dec1,dec2" : "none"); + if (l_hyb_d3d12 && fuse_stages_text != "none") + return fail(l_form_label + " on native D3D12: the unfused route; swin fuse stages must be none"); if (fuse_stages_text != "none") { for (size_t a = 0; a <= fuse_stages_text.size();) { size_t b = fuse_stages_text.find_first_of(",/", a); @@ -838,7 +1098,11 @@ struct Runtime { } } if (c.backend == 3) { - fz_text = choose(nullptr, "WG_D3D12_FUSION", kDefaultD3d12Fusion); + // WG_L_FORM=hybrid: none of the Swin fusions (they replace blocks with M-form kernels that have no hybrid twin); + // Model L's ll (the two input linears in one kernel, no block arithmetic), le and lc (their hybrid twins + // l_swin_block_c32_enc_lin_hyb / _dec_coef12_hyb, used only when built): each the same bytes as the launches it + // replaces; the network byte leg against the HIP twins' unfused route checks them + fz_text = l_hyb_d3d12 ? std::string("ll,lc,le") : choose(nullptr, "WG_D3D12_FUSION", kDefaultD3d12Fusion); if (!parse_fusion(fz_text, &fz_set[0])) return false; fz_set[1] = fz_set[0]; // Diagnostics (A/B in one process): WG_AB_FUSION / WG_AB_NOBAR apply on the frames of every odd block of @@ -853,6 +1117,22 @@ struct Runtime { nobar_ab = split_list(abn ? abn : ""); if (!(abn && *abn)) nobar_ab = nobar_list; } +#ifndef WG_DEV_DIAGNOSTICS + if (l_hyb_d3d12) { // release builds: the hybrid route is never switched or altered by the diagnostic hooks + const bool any = ab_block || !nobar_list.empty() || !nobar_ab.empty() || !ksub_list.empty() || !ksub_ab.empty(); + ab_block = 0; + fz_set[1] = fz_set[0]; + nobar_list.clear(); + nobar_ab.clear(); + ksub_list.clear(); + ksub_ab.clear(); + if (any) { + notes_text += std::string(notes_text.empty() ? "" : "\n") + + "WG_AB_* / WG_D3D12_NOBAR / WG_D3D12_KSUB ignored: diagnostic keys, release build (Model L hybrid)"; + diag_hooks_ignored = true; + } + } +#endif fz_set[0].apply(this); fz_any_xd = fz_set[0].xd || fz_set[1].xd; } @@ -963,6 +1243,15 @@ struct Runtime { "coefficient format"); post_kernel = "post_recon"; } + if (m_post_orig) { // Model M's original reconstruction: Model L's route (see m_post_orig) + if ((c.post_kernel && *c.post_kernel) || choose(nullptr, "WG_POST_KERNEL", "") != "") + return fail("WG_M_POST=orig: the reconstruction is morig/l_post_recon; no post kernel (post_kernel / " + "WG_POST_KERNEL) can be chosen"); + if (choose(c.post_pack, "WG_POST_PACK", "none") == "p12") + return fail("WG_M_POST=orig: reads the base coefficient format (no post_pack=p12)"); + if (recon_output) return fail("WG_M_POST=orig: not with recon=output"); + post_kernel = "post_recon"; + } #endif // post_pack / WG_POST_PACK=p12: the _p12 form of the post kernel (post_recon2_r[_hp][_band] -> ..._p12). Default // p12 (measured in game with the banded path: -0.04 ms, exact) when nothing else is chosen for post / dec0, not @@ -972,7 +1261,8 @@ struct Runtime { { const bool post_given = (c.post_kernel && *c.post_kernel) || choose(nullptr, "WG_POST_KERNEL", "") != ""; const bool dec0_given = (c.dec0_variant && *c.dec0_variant) || choose(nullptr, "WG_DEC0_VARIANT", "") != ""; - p12_default = (!model_l || c.backend == 3) && module_available(p12_path) && !post_given && !dec0_given && + p12_default = (!model_l || c.backend == 3) && !m_post_orig && module_available(p12_path) && !post_given && + !dec0_given && !recon_output && choose(c.dec1_dec0, "WG_DEC1_DEC0", "separate") != "fused"; } @@ -1220,8 +1510,14 @@ struct Runtime { !post_tf || !post_diet || !tf_vdiet || !post_sel || !post_idx || !post_trim)) return fail("native direct resources support only post_tf_f32_p12_diet_vsit (Model L: or l_post=own); " "alternate post/TF routes are unsupported"); - if ((!d1d0_p12 && !model_l) || !dec_div.empty() || dec1_dec0 || bneck_variant != "bmegahg" || - bneck_threads != 256 || bneck_groups != 128 || !mid_variant.empty() || swin_resident || force_clamp || fuse) + // WG_L_FORM=hybrid on native D3D12 (l_hyb_d3d12): the UNFUSED Swin route of the hybrid twins (six b0..b5 block + // launches, standalone merges and expands, no fused dec1 / dec2), set up in create; its Swin launches read and write + // arena maps only, independent of the direct resources, so it is accepted here in place of the bmegahg / fused-dec2 + // requirement (without this, a native-direct frame fell back to the HIP executor's scalar reference kernels) + const bool hyb_route = l_hyb_d3d12 && bneck_variant.empty() && mid_variant.empty() && fuse_stages.empty() && !fuse; + if ((!d1d0_p12 && !model_l) || !dec_div.empty() || dec1_dec0 || + (!hyb_route && (bneck_variant != "bmegahg" || bneck_threads != 256 || bneck_groups != 128)) || + !mid_variant.empty() || swin_resident || force_clamp || fuse) return fail("native direct resources require default pack12 decoder, bmegahg bottleneck and nonresident unclamped Swin routes"); if (gemm_threads != 256 || nt != 2 || wpw[0] != 1 || wpw[1] != 1 || wpw[2] != 1) return fail("native direct resources require the ported Swin/GEMM geometry"); @@ -1230,7 +1526,7 @@ struct Runtime { return fail("native direct resources require 256-lane, eight-window Swin blocks"); for (const char* stage : kSwinOrder) if (!swin_nc[stage]) return fail("native direct resources do not support clamped Swin stage " + std::string(stage)); - if (std::find(fuse_stages.begin(), fuse_stages.end(), "dec2") == fuse_stages.end()) + if (!hyb_route && std::find(fuse_stages.begin(), fuse_stages.end(), "dec2") == fuse_stages.end()) return fail("native direct resources require the fused dec2 expand/block port"); for (const std::string& stage : fuse_stages) if (stage != "dec1" && stage != "dec2") @@ -1433,7 +1729,8 @@ struct Runtime { if (on && si->kind != 'b') fz += (fz.empty() ? "" : ",") + std::string(st); } kernels_text += " swin_fused=" + (fz.empty() ? std::string("none") : fz); - kernels_text += " swin_bneck=" + (bneck_variant.empty() ? std::string("none") : bneck_variant); + kernels_text += " swin_bneck=" + (bneck_variant.empty() ? std::string(hyb_mega ? "hyb6" : "none") : bneck_variant); + if (diag_hooks_ignored) kernels_text += " diag_hooks=ignored"; kernels_text += " swin_resident=" + std::to_string(swin_resident); kernels_text += std::string(" d1d0_p12=") + (d1d0_p12 ? "1" : "0"); kernels_text += std::string(" dec_rdiv=") + (dec_div.find('r') == std::string::npos ? "1" : "0"); @@ -1453,8 +1750,25 @@ struct Runtime { // Model L: its encoder and decoder (enc0= / dec0= above name Model M's, not loaded for Model L) // (l_post: Model L's own reconstruction, HDR or LDR, on every backend) if (model_l) - kernels_text += std::string(" model=L l_kernels=l_enc_values.co,l_ops.co,l_swin.co,l_post.co l_post=") + + kernels_text += std::string(" model=L l_kernels=l_enc_values.co,l_ops.co,l_swin.co,l_post.co") + + (l_q7 ? (l_form_name.empty() ? std::string(",l_q7.co,l_q7w.co l_form=q7") + : ",l_q7" + std::string(l_form_name == "hybrid" ? "__hyb" : "__hybmut") + + ".co,l_q7w__hyb.co l_form=" + l_form_name) + : std::string(l_hyb_d3d12 ? " l_form=hybrid" : " l_form=m")) + + (l_post_c12 ? " l_post_c12=on" : "") + " l_post=" + (l_ldr ? "l_post_recon_ldr" : "l_post_recon"); + if (m_q7) // Model M's shared stages on the q7 kernels (hybrid: the float32-chain build) + kernels_text += " m_form=" + m_form_name + " m_shared=l_q7.co," + + (m_form_name == "hybrid" ? "l_q7w__hyb.co" +#ifdef WG_DEV_DIAGNOSTICS + : m_form_name == "hybrid_mutant" ? "l_q7w__hybmut.co" +#endif + : "l_q7w.co"); + if (m_post_orig) kernels_text += " m_post=orig m_post_kernel=l_post__morig.co:l_post_recon"; + if (l_hyb_d3d12) // l_dec_skip: the decoder block's skip S = the encoder block's map l_x3 (RN16(E + S)) + kernels_text += " l_hyb_d3d12=on hyb_twins=" + std::to_string(hyb_map.size() / 2) + " l_dec_skip=l_x3" + + // the arena tables (native D3D12; "off" on the HIP twins' route, whose device has none) + std::string(" hyb_tables=") + (dev->hyb_tables() ? "on" : "off"); if (l_fold_built) kernels_text += std::string(" l_dec_fold=") + (fz_lc ? "on" : "off"); // (native D3D12 only) if (l_lin_built) kernels_text += std::string(" l_lin_fused=") + (fz_ll ? "on" : "off"); if (model_l && l_color_early) kernels_text += " l_color_early=on"; @@ -1583,6 +1897,24 @@ struct Runtime { if (n == k) dev->unordered_next(); for (const auto& m : ((ab_block && ksub_b) ? ksub_ab : ksub_list)) if (m.first == k) k = m.second.c_str(); + std::string hyb_k; + if (l_hyb_d3d12) { + const std::string kk = k; + const auto it = std::find_if(hyb_map.begin(), hyb_map.end(), [&](const auto& m) { return m.first == kk; }); + if (it != hyb_map.end()) { + hyb_k = it->second; + k = hyb_k.c_str(); + } else if (kk.size() > 4 && kk.compare(kk.size() - 4, 4, "_hyb") == 0) { + // launched by its own hybrid name (swin_bneck6_hyb, speed lever 4: no M-form launch it replaces one for one; + // required by name at create) + } else { + const std::string base = kk.substr(kk.rfind('/') == std::string::npos ? 0 : kk.rfind('/') + 1); + for (const char* pre : {"swin_", "l_swin_", "patch_merge", "expand"}) + if (base.compare(0, std::strlen(pre), pre) == 0) + return fail(l_form_label + " on native D3D12: no hybrid twin of " + kk + + " in the kernel manifest; refusing the frame (no mix with the M-form kernels)"); + } + } if (pending_mark >= 0) { const int e = pending_mark; pending_mark = -1; @@ -1605,8 +1937,9 @@ struct Runtime { // One width-32 Swin block on the fh x fw grid (l_swin.co): "_enc" reads the FP16 halo map x2 and writes FP8, "_dec" // reads FP8 and writes FP16. - bool l_block(const char* kind, DevPtr x, DevPtr out, DevPtr w, bool nc, int H, int W, bool decoder, int f) { - DevPtr none = 0; + bool l_block(const char* kind, DevPtr x, DevPtr out, DevPtr w, bool nc, int H, int W, bool decoder, int f, + DevPtr skip = 0) { + DevPtr none = skip; // the hybrid decoder's skip S (l_x3); the M-form blocks take none int ox = 0, oy = 0; l_offsets(decoder, f, &ox, &oy); const int nwin = ((W + ox + 3) / 4) * ((H + oy + 3) / 4); @@ -1614,6 +1947,50 @@ struct Runtime { return launch(k.c_str(), cdiv(nwin, 8), 256, {&x, &none, &out, &w, &H, &W, &ox, &oy}); } + // Model L's own two blocks in the FP16 form (l_q7.co): the encoder's (FP16 halo map x2 in, FP8 out = the decoder's + // skip) or the decoder's (FP8 projection map in with the skip, FP16 out), the offsets of l_offsets. + bool l_q7_block(bool decoder, DevPtr x, DevPtr skip, DevPtr out, int H, int W, int f) { + int ox = 0, oy = 0; + l_offsets(decoder, f, &ox, &oy); + return l_q7_launch_block(decoder ? "l_q7_block_c32_dec" : "l_q7_block_c32_enc", x, skip, out, + wdev[decoder ? "dec0_block.q7" : "enc0_block.q7"], H, W, ox, oy); + } + + // One FP16-form block (l_q7.co / l_q7w.co): one 4 x 4 window per workgroup of 32 lanes. x, skip, out as the kernel + // (clean/l/kernels/l_q7.hip, l_q7w.hip); w = its q7 blob, the rsqrt / rcp tables from "q7.tab". + bool l_q7_launch_block(const char* k, DevPtr x, DevPtr skip, DevPtr out, DevPtr w, int H, int W, int ox, int oy) { + DevPtr tab = wdev["q7.tab"]; + const int nwin = ((W + ox + 3) / 4) * ((H + oy + 3) / 4); + return launch(k, nwin, 32, {&x, &skip, &out, &w, &tab, &H, &W, &ox, &oy}); + } + + // Step 3 for Model L in the FP16 form: the ten shared stages with Model L's own q7 kernels on Model M's stage maps + // (feat0 -> enc1 -> skip1 / merged1 -> enc2 -> skip2 / merged2 -> b0 ... b5 -> bout[i] -> dec2 (expansion e2, skip2) + // -> out2 -> dec1 (expansion e1, skip1) -> out1); every map plain [h][w][C] FP8 (oracle run_stage_q7). + bool l_q7_stages(const int off[10][2], int H2, int W2, int H4, int W4, int H8, int W8) { + DevPtr none = 0; + auto lin = [&](const char* k, DevPtr in, DevPtr out, const char* blob, int h, int w, int grid) { + DevPtr wl = wdev[blob]; + return launch(k, grid, 32, {&in, &out, &wl, &h, &w}); + }; + if (!l_q7_launch_block("l_q7_block_c64", feat0, none, skip1, wdev["enc1.q7"], H2, W2, off[0][0], off[0][1]) || + !lin("l_q7_merge_c64", skip1, merged1, "enc1.q7lin", H2, W2, (H2 / 2) * (W2 / 2)) || + !l_q7_launch_block("l_q7_block_c96", merged1, none, skip2, wdev["enc2.q7"], H4, W4, off[1][0], off[1][1]) || + !lin("l_q7_merge_c96", skip2, merged2, "enc2.q7lin", H4, W4, (H4 / 2) * (W4 / 2))) + return false; + DevPtr x = merged2; + for (int i = 0; i < 6; ++i) { + const std::string b = "b" + std::to_string(i) + ".q7"; + if (!l_q7_launch_block("l_q7_block_c128", x, none, bout[i], wdev[b], H8, W8, off[2 + i][0], off[2 + i][1])) + return false; + x = bout[i]; + } + return lin("l_q7_expand_c128", bout[5], e2, "dec2.q7lin", H8, W8, H8 * W8) && + l_q7_launch_block("l_q7_block_c96_dec", e2, skip2, out2, wdev["dec2.q7"], H4, W4, off[8][0], off[8][1]) && + lin("l_q7_expand_c96", out2, e1, "dec1.q7lin", H4, W4, H4 * W4) && + l_q7_launch_block("l_q7_block_c64_dec", e1, skip1, out1, wdev["dec1.q7"], H2, W2, off[9][0], off[9][1]); + } + // Step 2 for Model L: the 20 values and their dithered FP8 for every pixel of the halo map (Model M's input encoder // arguments, q in place of the features; the motion surfaces as Model M), the two per-pixel linears, the encoder // block, and the gather of each token's 2 x 2 pixels into the 64 features enc1 reads (feat0). @@ -1631,7 +2008,8 @@ struct Runtime { int npix = (fh + 4) * (fw + 4), tw = fw / 2, th = fh / 2; const bool fused = fz_ll && l_lin_built; // (native D3D12 only: l_lin_built) // fusion le: the linears in the block (its _nc form), q in the 20-byte form from the q20 input encoder - const bool in_block = fz_le && l_le_built && l_nc[0] && (native_direct_io ? l_q20d_built : l_q20_built); + const bool in_block = + !l_q7 && fz_le && l_le_built && l_nc[0] && (native_direct_io ? l_q20d_built : l_q20_built); DevPtr& q = fused || in_block ? l_narrow : l_wide; // q's map (fused / in the block: not under x2) if (in_block) { DevPtr none = 0, wb = wdev["enc0_block"]; @@ -1655,7 +2033,8 @@ struct Runtime { (fused ? launch("l_linear_32_32_fused", cdiv(npix, 128), 256, {&l_narrow, &l_wide, &w1, &w2, &npix}) : launch("l_linear_32_32_relu", cdiv(npix, 128), 256, {&l_wide, &l_narrow, &w1, &npix}) && launch("l_linear_32_32_f16", cdiv(npix, 128), 256, {&l_narrow, &l_wide, &w2, &npix})) && - l_block("enc", l_wide, l_x3, wdev["enc0_block"], l_nc[0], fh, fw, false, f) && + (l_q7 ? l_q7_block(false, l_wide, 0, l_x3, fh, fw, f) + : l_block("enc", l_wide, l_x3, wdev["enc0_block"], l_nc[0], fh, fw, false, f)) && launch("l_project_gather", cdiv(tw * th, 128), 256, {&l_x3, &feat0, &wp, &tw, &th}); } @@ -1666,17 +2045,20 @@ struct Runtime { bool l_decoder(DecParams& p, int nxt, int fh, int fw, int f) { DevPtr wd = wdev["dec0.project"], wc = wdev["dec0.coef"]; int tw = fw / 2, th = fh / 2; - if (fz_lc && l_fold_built && p12_frame && l_nc[1]) { + if (!l_q7 && fz_lc && l_fold_built && p12_frame && l_nc[1]) { DevPtr none = 0, wb = wdev["dec0_block"]; + DevPtr skip = l_hyb_d3d12 ? l_x3 : 0; // hybrid: the skip S = the encoder block's map (as l_block's dec) int ox = 0, oy = 0, H = fh, W = fw, rw = p.rw, rh = p.rh; l_offsets(true, f, &ox, &oy); const int nwin = ((W + ox + 3) / 4) * ((H + oy + 3) / 4); return launch("l_dec_project", cdiv(tw * th, 128), 256, {&out1, &l_narrow, &wd, &tw, &th}) && launch("l_swin_block_c32_dec_coef12_nc", cdiv(nwin, 8), 256, - {&l_narrow, &none, &none, &wb, &H, &W, &ox, &oy, &A, &C, &latent[nxt], &wc, &rw, &rh}); + {&l_narrow, &skip, &none, &wb, &H, &W, &ox, &oy, &A, &C, &latent[nxt], &wc, &rw, &rh}); } return launch("l_dec_project", cdiv(tw * th, 128), 256, {&out1, &l_narrow, &wd, &tw, &th}) && - l_block("dec", l_narrow, l_wide, wdev["dec0_block"], l_nc[1], fh, fw, true, f) && + (l_q7 ? l_q7_block(true, l_narrow, l_x3, l_wide, fh, fw, f) // skip S = the encoder block's map l_x3 + : l_block("dec", l_narrow, l_wide, wdev["dec0_block"], l_nc[1], fh, fw, true, f, + l_hyb_d3d12 ? l_x3 : 0)) && // hybrid: the skip S = the encoder block's map launch(p12_frame ? "l_coef_pack12" : "l_coef", cdiv(tw * th, 128), 256, {&l_wide, &A, &B, &C, &latent[nxt], &wc, &p}); } @@ -1702,6 +2084,16 @@ struct Runtime { } std::string l_native_missing() const { + if (l_q7) // the FP16-form blocks (clean/dx12/kernels/l_q7.hlsl, l_q7w.hlsl) in place of the l_swin / Model M blocks + return manifest_missing({native_direct_io ? "lev/input_encoder_direct" : "lev/input_encoder", + "l_linear_32_32_relu", "l_linear_32_32_f16", "l_q7_block_c32_enc", "l_project_gather", + "l_q7_block_c64", "l_q7_merge_c64", "l_q7_block_c96", "l_q7_merge_c96", + "l_q7_block_c128", "l_q7_expand_c128", "l_q7_block_c96_dec", "l_q7_expand_c96", + "l_q7_block_c64_dec", "l_dec_project", "l_q7_block_c32_dec", "l_coef_pack12"}); + if (l_hyb_d3d12) // the hybrid route: its Swin / merge / expand kernels are the "_hyb" twins, checked by name + return manifest_missing({native_direct_io ? "lev/input_encoder_direct" : "lev/input_encoder", + "l_linear_32_32_relu", "l_linear_32_32_f16", "l_project_gather", "l_dec_project", + "l_coef_pack12"}); return manifest_missing({native_direct_io ? "lev/input_encoder_direct" : "lev/input_encoder", "l_linear_32_32_relu", "l_linear_32_32_f16", "l_swin_block_c32_enc_nc", "l_project_gather", @@ -1743,10 +2135,10 @@ struct Runtime { // w64/post_filter_c1_s4, the analytic build, as Model L's file has no output-filter weights; LDR: l_post_filter_ldr). // The kernels Model L's route needs on native D3D12 (HDR or LDR colour; the direct route's two more). std::vector l_own_post_kernels(bool ldr) const { - std::vector need = ldr ? std::vector{"l_post_recon_p12_ldr", "l_post_recon_ldr", "l_coef", + const std::string rc = l_post_c12 ? "lc12/l_post_recon" : "l_post_recon"; // C1/C2: the lc12 ports in their place + std::vector need = ldr ? std::vector{rc + "_p12_ldr", rc + "_ldr", "l_coef", "l_post_filter_ldr"} - : std::vector{"l_post_recon_p12", "l_post_recon", "l_coef", - "w64/post_filter_c1_s4"}; + : std::vector{rc + "_p12", rc, "l_coef", "w64/post_filter_c1_s4"}; if (native_direct_io) { need.push_back("l_direct_color"); need.push_back(ldr ? "l_post_filter_direct_ldr" : "l_post_filter_direct"); @@ -1943,6 +2335,13 @@ struct Runtime { // (native D3D12: l_post_recon_p12 on the frames whose coefficients l_coef_pack12 stored; LDR colour: the _ldr // form) post_last = std::string(p12_frame ? "l_post_recon_p12" : "l_post_recon") + (l_ldr ? "_ldr" : ""); + if (l_post_c12) post_last = "lc12/" + post_last; // l_post__lc12.co (C1 machine form, C2), tag lc12 + return launch(post_last.c_str(), grid, 256, + {&cnet, &A, &B, &C, &hist[cur], &mnear[nxt], &mcenter, &mfar, &w, &hist[nxt], &reduced, &display, + &p, &expo}); + } + if (m_post_orig) { // Model M's original reconstruction (tagged module; Model M's s in w[0]) + post_last = "morig/l_post_recon"; return launch(post_last.c_str(), grid, 256, {&cnet, &A, &B, &C, &hist[cur], &mnear[nxt], &mcenter, &mfar, &w, &hist[nxt], &reduced, &display, &p, &expo}); @@ -2011,6 +2410,34 @@ struct Runtime { if (rw <= 0 || rh <= 0 || rw > max_rw || rh > max_rh) return fail("render size larger than the maximum"); const int fw = ceil32(rw), fh = ceil32(rh); if (fw > 2 * rw - 1 || fh > 2 * rh - 1) return fail("render size too small for its padding (spec 03)"); + // The hybrid bottleneck's bounded wait (hyb_bneck.hlsl). A wait that timed out set the abort word, and the frame + // guard skipped the rest of that frame, and every frame recorded after it, on the GPU (no output, no history). When + // the device's copy shows the timeout count moved, this frame is REFUSED, the next one starts over (reset: it reads + // no history) and the abort word is cleared before it. Otherwise the whole frame runs under the guard (lifted at + // every exit of this function), read again right after the bottleneck. + struct GuardScope { + Device* d; + bool on; + ~GuardScope() { + if (on) d->guard_end(); + } + } guard{dev.get(), false}; + if (hyb_mega) { + uint32_t g[4]; + dev->guard_read(g); + if (g[3] != bneck_seen) { + bneck_seen = g[3]; + bneck_clear = true; + frames = 0; + return fail(l_form_label + ": the bottleneck's wait timed out on the GPU (swin_bneck6_hyb, " + + std::to_string(g[3]) + " in all): that frame and the frames queued after it were skipped (no output, " + "no history); this frame is refused, the next one starts over"); + } + if (bneck_clear && !(dev->memset(bneck_sync + 48, 0, 4) || dfail())) return false; + bneck_clear = false; + if (!dev->guard_begin(bneck_sync + 48)) return dfail(); + guard.on = true; + } const int reset = (fr.reset || frames == 0 || rw != last_rw || rh != last_rh) ? 1 : 0; // timing marks (FrameIo.timing_base >= 0): events base + 0 start of enc0 (first kernel after the input wait), 1 start // of the Swin stages, 2 start of dec0, 3 start of post, 4 end of the last kernel @@ -2235,8 +2662,14 @@ struct Runtime { xd_sync != 0 && (xd_max_pixels <= 0 || (long long)rw * rh <= xd_max_pixels); for (int i = 0; i < 2; ++i) xo_off[0][i] = off[2][i], xo_off[1][i] = off[3][i]; } - if (!swin_stage("enc1", feat0, 0, skip1, merged1, 0, H2, W2, off[0][0], off[0][1])) return false; - if (!mid_variant.empty()) { +#ifdef WG_MODEL_L + if ((model_l && l_q7) || m_q7) { + if (!l_q7_stages(off, H2, W2, H4, W4, H8, W8)) return false; + } else +#endif + if (!swin_stage("enc1", feat0, 0, skip1, merged1, 0, H2, W2, off[0][0], off[0][1])) { + return false; + } else if (!mid_variant.empty()) { // enc2 + merge, b0 ... b5, expand + dec2 in one launch (swin_mid; scratch maps b5.out, which holds b5's output // afterwards, and bneck_tmp); enc2.out (skip2) and dec2.out kept, merged2, b0 ... b4 outputs and e2 not written; // dec1 runs as usual (own launch, fused expand / dec1_dec0 as chosen) @@ -2255,8 +2688,9 @@ struct Runtime { } else if (!swin_stage("enc2", merged1, 0, skip2, merged2, 0, H4, W4, off[1][0], off[1][1])) { return false; } + const bool q7_done = (model_l && l_q7) || m_q7; // all ten stages ran in l_q7_stages DevPtr x = merged2; - if (!mid_variant.empty()) { + if (q7_done || !mid_variant.empty()) { // done above } else if (!bneck_variant.empty()) { // b0 ... b5 in one launch; b0 ... b4 outputs are not kept (only b5.out) bool safe = true; @@ -2279,17 +2713,36 @@ struct Runtime { return false; } x = bout[5]; + } else if (hyb_mega) { // the hybrid route: b0 ... b5 in one launch (only b5.out kept) + DevPtr wb[6]; + for (int i = 0; i < 6; ++i) wb[i] = wdev["b" + std::to_string(i)]; + int groups = hyb_mega_groups, oxe = off[2][0], oye = off[2][1], oxo = off[3][0], oyo = off[3][1]; + int h8 = H8, w8 = W8; + if (!launch("swin_bneck6_hyb", groups, 256, + {&merged2, &bout[5], &bneck_tmp, &bneck_sync, &wb[0], &wb[1], &wb[2], &wb[3], &wb[4], &wb[5], &h8, &w8, + &oxe, &oye, &oxo, &oyo, &groups})) + return false; + x = bout[5]; +#ifdef WG_DEV_DIAGNOSTICS + if (dev_bneck_abort >= 0 && dev_bneck_abort == frames) { // the check's forced timeout: the words it would write + dev_bneck_abort = -1; + const uint32_t wv[4] = {1u, 0u, 0u, bneck_seen + 1u}; + if (!dev->upload(bneck_sync + 48, wv, sizeof wv)) return dfail(); + } +#endif + if (!dev->guard_begin(bneck_sync + 48)) return dfail(); // this frame's bottleneck may have refused it } - for (int i = 0; bneck_variant.empty() && mid_variant.empty() && i < 6; ++i) { + for (int i = 0; !q7_done && bneck_variant.empty() && mid_variant.empty() && !hyb_mega && i < 6; ++i) { const std::string s = "b" + std::to_string(i); if (!swin_stage(s, x, 0, bout[i], 0, 0, H8, W8, off[2 + i][0], off[2 + i][1])) return false; x = bout[i]; } // pack12 frame with the fused dec1 + dec0: dec1's expand only here, the block runs in step 4 const bool fused_p12 = d1d0_p12 && p12 && recon2_supported(rw, rh, dw, dh); - if ((mid_variant.empty() && !swin_stage("dec2", bout[5], skip2, out2, 0, e2, H4, W4, off[8][0], off[8][1])) || + if (!q7_done && + ((mid_variant.empty() && !swin_stage("dec2", bout[5], skip2, out2, 0, e2, H4, W4, off[8][0], off[8][1])) || (!fz_xe_now && // xe: dec1's expand ran in dec2's block - !swin_stage("dec1", out2, skip1, out1, 0, e1, H2, W2, off[9][0], off[9][1], dec1_dec0 || fused_p12))) + !swin_stage("dec1", out2, skip1, out1, 0, e1, H2, W2, off[9][0], off[9][1], dec1_dec0 || fused_p12)))) return false; // 4. coefficient decoder (spec 05) @@ -2315,7 +2768,7 @@ struct Runtime { #ifdef WG_MODEL_L if (!l_decoder(p, nxt, fh, fw, f)) return false; #endif - } else if (p12_frame && fused_p12) { // dec1's block with dec0 (pack12) as its epilogue + } else if (p12_frame && fused_p12 && !m_q7) { // dec1's block with dec0 (pack12) as its epilogue DevPtr w1 = wdev["dec1"]; int ox = off[9][0], oy = off[9][1], h2 = H2, w2 = W2; const int nwin = ((W2 + ox + 3) / 4) * ((H2 + oy + 3) / 4); @@ -2332,11 +2785,18 @@ struct Runtime { &w, &p})) { return false; } + } else if (p12_frame && m_q7 && d3d12_native) { // native D3D12 has no standalone coefficient decoder: d1p12's + // dec0 epilogue fed from dec1's stored map (out1, written by l_q7_block_c64_dec), d1p12's windows unshifted + int ox = 0, oy = 0, h2 = H2, w2 = W2; + const int nwin = ((W2 + 3) / 4) * ((H2 + 3) / 4); + if (!launch("d1p12/dec0_q7", cdiv(nwin, 8), 256, + {&out1, &out1, &w, &h2, &w2, &ox, &oy, &mnear[nxt], &latent[cur], &A, &B, &C, &latent[nxt], &w, &p})) + return false; } else if (p12_frame) { // pack12 coefficients for the _p12 post kernels if (!launch("d0p12/coef_decoder", cdiv((fh / 2) * (fw / 2), 128), 256, {&out1, &mnear[nxt], &latent[cur], &A, &B, &C, &latent[nxt], &w, &p})) return false; - } else if (dec1_dec0) { // dec1's block with dec0 as its epilogue (dec1's map stays on chip; dec1.out not kept) + } else if (dec1_dec0 && !m_q7) { // dec1's block with dec0 as its epilogue (dec1's map stays on chip) DevPtr w1 = wdev["dec1"]; int ox = off[9][0], oy = off[9][1], h2 = H2, w2 = W2; const int nwin = ((W2 + ox + 3) / 4) * ((H2 + oy + 3) / 4); diff --git a/clean/tests/check_no_dev_diagnostics.py b/clean/tests/check_no_dev_diagnostics.py index c71252c4..b072ad59 100644 --- a/clean/tests/check_no_dev_diagnostics.py +++ b/clean/tests/check_no_dev_diagnostics.py @@ -37,8 +37,25 @@ b"diagnostic: executor=hip selects the HIP executor", b"diagnostic: frame capture forces the HIP executor", b"frame capture armed: create ", + b"diagnostic: hybrid tables off", + # the check routes of Model L / M (runtime.cpp, behind WG_DEV_DIAGNOSTICS): the mutant and twin code objects + b"/l_q7__hybmut.co", + b"/l_q7w__hybmut.co", + b"(WG_L_FORM=hybrid_twin: l_swin_hyb.co, swin_hyb.co)", b"WG_DEV_BNECK_ABORT", ) +# Check-only code objects (mutants, probes, the HIP twins of the D3D12 hybrid ports, the fused-square scan's mutants): never +# in a release payload, a package or a runtime library's built-in files. +# (The HIP self-test's probe.co and the D3D12 device's native_driver_probe are not check objects.) +PAYLOAD_CHECK_NAMES = re.compile(r"(mut\b|mut\.co|_mut\.dxil|_tmo\.dxil|_scanmut|_probe(__mut)?\.co|(^|[\\/])l_swin_hyb\.co|(^|[\\/])swin_hyb\.co)", + re.I) + + +def check_object_names_in(data): + """Built-in check-object names in a runtime library (embedded files are listed by name).""" + return [t.decode() for t in (b"l_q7__hybmut.co", b"l_q7w__hybmut.co", b"l_post_c12_probe", b"l_swin_hyb.co", + b"swin_hyb.co", b"_scanmut") if t in data] + def dev_only_strings_in(data): return [t.decode("ascii") for t in DEV_ONLY_STRINGS if t in data] @@ -59,6 +76,9 @@ def check_bytes(data): carried = dev_only_strings_in(data) if carried: return False, "says dev-diagnostics=0 but carries developer-diagnostic code (%r): a forged marker" % carried[0] + objs = check_object_names_in(data) + if objs: + return False, "carries check-only code objects (%r)" % objs[0] return True, "dev-diagnostics=0" @@ -96,6 +116,8 @@ def check_artifact(name, data): if is_runtime_name(name): ok, text = check_bytes(data) return ok, text, ok + if PAYLOAD_CHECK_NAMES.search(os.path.basename(name)): + return False, "a check-only code object (mutant, probe or HIP twin) in a release payload", False if any(v == b"1" for v in found): return False, "carries a developer runtime (dev-diagnostics=1)", False carried = dev_only_strings_in(data) @@ -113,6 +135,8 @@ def artifacts(path): low = n.lower() if is_runtime_name(n) or low.endswith((".zip", ".exe", ".asi")): out += artifacts(os.path.join(dirpath, n)) + elif PAYLOAD_CHECK_NAMES.search(n): # a check-only object anywhere in the folder fails the check + out.append((os.path.join(dirpath, n), n, b"")) return out if path.lower().endswith(".zip"): with zipfile.ZipFile(path) as z: diff --git a/clean/tests/hyb_dump.cpp b/clean/tests/hyb_dump.cpp new file mode 100644 index 00000000..c2edf944 --- /dev/null +++ b/clean/tests/hyb_dump.cpp @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: MIT +// Writes runtime/hyb_tables.h's hyb_fill() bytes to the file given (test_hyb_tables_cpp.py compares them with +// clean/l/reference/hyb_tables.py). With a second argument "downward" the generation runs with the thread in FE_DOWNWARD +// (the bytes must not change) and the program fails (exit 3) unless that mode is restored afterwards. +#include +#include +#include +#include + +#include "hyb_tables.h" + +int main(int argc, char** argv) { + if (argc < 2) return 2; + std::vector t(wg::kHybBytes); + const bool down = argc > 2 && std::strcmp(argv[2], "downward") == 0; + if (down && std::fesetround(FE_DOWNWARD) != 0) return 4; + wg::hyb_fill(t.data()); + if (down) { + const bool restored = std::fegetround() == FE_DOWNWARD; + std::fesetround(FE_TONEAREST); + if (!restored) return 3; + } + FILE* f = std::fopen(argv[1], "wb"); + if (!f) return 1; + std::fwrite(t.data(), 1, t.size(), f); + std::fclose(f); + return 0; +} diff --git a/clean/tests/runtime_frame_trace.cpp b/clean/tests/runtime_frame_trace.cpp index 5a3e98a8..bf28c222 100644 --- a/clean/tests/runtime_frame_trace.cpp +++ b/clean/tests/runtime_frame_trace.cpp @@ -12,7 +12,19 @@ // --sizes=WxH,... the render size of frame 0, 1, ... (repeated; default: the maximum every frame) // --reset=f,... frames the game asks for a reset (WgRtFrame.reset = 1) // --fail=@ that kernel's launch fails in that frame (the device returns false) +// --geo== a Swin / GEMM launch-geometry override in WgRtConfig (nt, gemm_threads, swin_threads, wpw64, +// wpw96, wpw128, wpg0 .. wpg4 = block_wpg[k]); repeatable // --hdr=<0|1> WgRtConfig.hdr (default 0) +// --guard-timeout= the frame guard (Device::guard_begin, the hybrid bottleneck's wait timeout): in that frame +// the bottleneck "times out" (at the guard's second reading of that frame, the words a timed-out wait +// writes are set first: sync words 12 and 15 + 1). The device models the guard: "guard_begin +// " / "guard_end" lines, and while the word read by the last guard_begin is nonzero, every +// launch, copy, upload and memset is printed as "skipped " and not done; guard_end copies the +// 16 bytes at the word for guard_read; a memset / upload of the word prints "guard_clear" / +// "guard_upload " +// --fail-hyb-tables the device refuses the hybrid tables (Device::hyb_reserve returns false: create must refuse a +// Model L hybrid context on the native-D3D12 stand-in, --backend=3); without it the reservation +// succeeds (silently: the trace lines stay those of every other device) // // Prints the create result and the allocations ("alloc b "), what create uploads ("upload b+ // ": the weight blobs, so a test can tell which blob an address holds), the code objects @@ -74,11 +86,19 @@ const Signature kSignatures[] = { {"input_encoder_q20", "ppppppppppppE"}, // Model L's values, q in its 20-byte form (native D3D12, fusion le) {"coef_decoder", "ppppppppD"}, // Model M's decoder (also its pack12 form) {"swin_block_*", "ppppiiii"}, // Model M's blocks: x, skip, out, w, H, W, ox, oy + {"swin_bneck6_hyb", "ppppppppppiiiiiii"}, // the hybrid bottleneck in one launch (speed lever 4): x, out, tmp, + // sync, b0 ... b5's blobs, H, W, even / odd offsets, workgroups {"l_swin_block_c32_dec_coef12_nc", "ppppiiiippppii"}, // the decoder block with l_coef_pack12 folded in: then // A, C, latent, dec0.coef, rw, rh (native D3D12, fusion lc) {"l_swin_block_c32_enc_lin_nc", "ppppiiiipp"}, // the encoder block with the linears as its prologue (fusion // le): x = q, then enc0.Win1, enc0.Win2 + {"l_swin_block_c32_enc_lin_hyb", "ppppiiiipp"}, // its hybrid twin (speed lever 3; the same arguments) + {"l_swin_block_c32_dec_coef12_hyb", "ppppiiiippppii"}, // its hybrid twin (speed lever 3; the same arguments) {"l_swin_block_c32*", "ppppiiii"}, // Model L's blocks (clean/l/FORMATS.md) + {"l_q7_block_*", "pppppiiii"}, // the FP16-form blocks (l_q7.co / l_q7w.co, their hybrid builds): x, skip, out, + // w, the tables, H, W, ox, oy + {"l_q7_merge_*", "pppii"}, // the FP16-form merge and expansion: in, out, w, h, w + {"l_q7_expand_*", "pppii"}, {"l_linear_32_32_relu", "pppi"}, {"l_linear_32_32_f16", "pppi"}, {"l_linear_32_32_fused", "ppppi"}, // the two in one: q, x2, Win1, Win2, npix (native D3D12, fusion ll) @@ -115,9 +135,60 @@ class RecordingDevice : public Device { std::map index; // allocation number of each block int allocs = 0, frame = -1, fail_frame = -1; std::string fail_kernel; + bool fail_hyb = false, hyb = false; DevPtr next = 0x10000; + // the frame guard (see --guard-timeout) + int guard_timeout = -1, guard_reads = 0; + bool guard_on = false; + uint64_t guard_pred = 0; + DevPtr guard_word = 0; + uint32_t guard_copy[4] = {0, 0, 0, 0}; + bool skipped(const std::string& what) { + if (!guard_on || !guard_pred) return false; + log->push_back("skipped " + what); + return true; + } + bool has_guard() const override { return true; } + bool guard_begin(DevPtr word) override { + uint8_t* w = at(word, 16); + if (!w || (word & 7u)) return err_ = "recording device: invalid guard word", false; + ++guard_reads; + if (frame == guard_timeout && guard_on) { // the bottleneck's timed-out wait (the second reading of the frame) + uint32_t v[4]; + std::memcpy(v, w, 16); + v[0] += 1; + v[3] += 1; + std::memcpy(w, v, 16); + } + std::memcpy(&guard_pred, w, 8); + guard_on = true; + guard_word = word; + log->push_back("guard_begin " + addr(word) + (guard_pred ? " (skipping)" : "")); + return true; + } + bool guard_end() override { + if (!guard_on) return true; + guard_on = false; + std::memcpy(guard_copy, at(guard_word, 16), 16); + log->push_back("guard_end"); + return true; + } + bool guard_read(uint32_t w[4]) override { + std::memcpy(w, guard_copy, 16); + return true; + } explicit RecordingDevice(std::vector* l) : log(l) {} std::string name() const override { return "recording device (CPU test)"; } + // the hybrid tables (native D3D12's fixed arena range; nothing to hold here): reserved unless --fail-hyb-tables + bool hyb_reserve() override { + if (fail_hyb) { + err_ = "recording device: hybrid tables refused (--fail-hyb-tables)"; + return false; + } + hyb = true; + return true; + } + bool hyb_tables() const override { return hyb; } bool alloc(DevPtr* p, size_t bytes) override { *p = next; mem[next].assign(bytes ? bytes : 1, 0); @@ -146,6 +217,12 @@ class RecordingDevice : public Device { return "b" + std::to_string(index[it->first]) + "+" + std::to_string(p - it->first); } bool upload(DevPtr dst, const void* src, size_t n) override { + if (frame >= 0 && skipped("upload " + addr(dst))) return true; + if (guard_word && dst == guard_word && n >= 16) { + uint32_t v[4]; + std::memcpy(v, src, 16); + log->push_back("guard_upload " + std::to_string(v[0]) + " " + std::to_string(v[3])); + } uint8_t* d = at(dst, n); if (d && n) std::memcpy(d, src, n); if (frame < 0 && d) // create: the weight blobs (and the runtime's tables) @@ -159,11 +236,14 @@ class RecordingDevice : public Device { return s != nullptr; } bool copy(DevPtr dst, DevPtr src, size_t n) override { + if (frame >= 0 && skipped("copy " + addr(dst))) return true; uint8_t *d = at(dst, n), *s = at(src, n); if (d && s && n) std::memmove(d, s, n); return d && s; } bool memset(DevPtr dst, int v, size_t n) override { + if (frame >= 0 && skipped("memset " + addr(dst))) return true; + if (guard_word && dst == guard_word) log->push_back("guard_clear"); uint8_t* d = at(dst, n); if (d && n) std::memset(d, v, n); return d != nullptr; @@ -184,6 +264,7 @@ class RecordingDevice : public Device { log->push_back("failed " + kernel); return false; } + if (skipped("launch " + kernel)) return true; std::string line = "launch " + kernel + " " + std::to_string(grid) + " " + std::to_string(block); if (const char* kinds = kinds_of(kernel)) { line += " |"; @@ -274,6 +355,8 @@ int main(int argc, char** argv) { cfg.auto_exposure = 0; const int frames = std::atoi(argv[7]); std::string options, map_path, fail; + bool fail_hyb = false; + int guard_timeout = -1; std::vector> sizes; std::set resets; for (int a = 8; a < argc; ++a) { @@ -296,6 +379,23 @@ int main(int argc, char** argv) { cfg.hdr = std::atoi(s.c_str() + 6); } else if (s.compare(0, 7, "--fail=") == 0) { fail = s.substr(7); + } else if (s == "--fail-hyb-tables") { + fail_hyb = true; + } else if (s.compare(0, 16, "--guard-timeout=") == 0) { + guard_timeout = std::atoi(s.c_str() + 16); + } else if (s.compare(0, 6, "--geo=") == 0) { + const size_t q = s.find('=', 6); + if (q == std::string::npos) return std::printf("ERR bad --geo %s\n", s.c_str()), 2; + const std::string k = s.substr(6, q - 6); + const int v = std::atoi(s.c_str() + q + 1); + if (k == "nt") cfg.nt = v; + else if (k == "gemm_threads") cfg.gemm_threads = v; + else if (k == "swin_threads") cfg.swin_threads = v; + else if (k == "wpw64") cfg.wpw64 = v; + else if (k == "wpw96") cfg.wpw96 = v; + else if (k == "wpw128") cfg.wpw128 = v; + else if (k.size() == 4 && k.compare(0, 3, "wpg") == 0 && k[3] >= '0' && k[3] <= '4') cfg.block_wpg[k[3] - '0'] = v; + else return std::printf("ERR bad --geo key %s\n", k.c_str()), 2; } else if (s.compare(0, 2, "--") == 0) { return std::printf("ERR unknown flag %s\n", s.c_str()), 2; } else { @@ -306,6 +406,8 @@ int main(int argc, char** argv) { cfg.map_path = map_path.empty() ? nullptr : map_path.c_str(); auto dev = std::make_unique(&log); RecordingDevice* rec = dev.get(); + rec->fail_hyb = fail_hyb; + rec->guard_timeout = guard_timeout; if (!fail.empty()) { const size_t at = fail.find('@'); rec->fail_kernel = fail.substr(0, at); diff --git a/clean/tests/test_dev_diagnostics_off.py b/clean/tests/test_dev_diagnostics_off.py index b3c64743..6ff0a612 100644 --- a/clean/tests/test_dev_diagnostics_off.py +++ b/clean/tests/test_dev_diagnostics_off.py @@ -233,6 +233,31 @@ def test_every_runtime_library_and_every_file_is_checked(self): r, _ = self._pack({"wronggpu_runtime.dll": good, "sub/other.dll": b"plain"}, mf) self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + def test_check_routes_and_check_objects_are_refused(self): + """The check routes' selector texts (the mutant / twin code objects, the forced bottleneck timeout) and the + check-only code objects (mutants, probes, the timeout control, the HIP twins, the scan mutants) are never in a + release library or payload; the default route's objects are.""" + for t in chk.DEV_ONLY_STRINGS: + ok, text = chk.check_bytes(b"MZ" + MARK0 + b" " + t) + self.assertFalse(ok, t) + self.assertIn("forged marker", text) + for t in (b"l_q7__hybmut.co", b"l_q7w__hybmut.co", b"l_swin_hyb.co", b"swin_hyb.co", b"x_scanmut", + b"l_post_c12_probe"): + ok, text = chk.check_bytes(b"MZ" + MARK0 + b" " + t) + self.assertFalse(ok, t) + self.assertIn("check-only code objects", text) + for n in ("l_q7__hybmut.co", "l_q7w__hybmut.co", "swin_hyb.co", "sub/l_swin_hyb.co", "l_post_c12_probe.co", + "l_post_c12_probe__mut.co", "l_hyb_probe.co", + "dx12/scanmut/swin_block_c128_hyb_scanmut.dxil", "swin_bneck6_hyb_mut.dxil", "swin_bneck6_hyb_tmo.dxil"): + ok, text, _ = chk.check_artifact(n, b"plain") + self.assertFalse(ok, n) + for n in ("l_q7__hyb.co", "l_q7w__hyb.co", "l_post__lc12.co", "swin_bneck6_hyb.dxil", "swin_block_c128_hyb.dxil", + "manifest.json", "probe.co", "native_driver_probe.dxil"): + ok, text, _ = chk.check_artifact(n, b"plain") + self.assertTrue(ok, (n, text)) + ok, text = chk.check_bytes(b"MZ" + MARK0 + b" /l_q7__hyb.co /l_q7w__hyb.co /l_post__lc12.co") + self.assertTrue(ok, text) + def test_the_final_package_check(self): with tempfile.TemporaryDirectory() as d: good_zip, bad_zip = os.path.join(d, "good.zip"), os.path.join(d, "bad.zip") diff --git a/clean/tests/test_runtime_frame_trace.py b/clean/tests/test_runtime_frame_trace.py index 062004fa..34db0858 100644 --- a/clean/tests/test_runtime_frame_trace.py +++ b/clean/tests/test_runtime_frame_trace.py @@ -77,7 +77,7 @@ def run_trace(rw=64, rh=64, ow=96, oh=96, frames=3, options="", build_files=(), class Trace: """A parsed run: create (the first line), allocs {k: bytes}, uploads {address: (bytes, SHA-256 hex)} (create's), loads, frames [{"info": {...}, "launches": [Launch], "failed": kernel or None, "error": message or None, "names": - {buffer name: address}}], kernels {key: value}.""" + {buffer name: address}, "guard": [frame guard lines], "skipped": [what the guard skipped]}], kernels {key: value}.""" def __init__(self, lines): self.create, self.allocs, self.uploads, self.loads, self.frames, self.kernels = lines[0], {}, {}, [], [], {} @@ -96,9 +96,14 @@ def __init__(self, lines): elif ln.startswith("frame "): f = ln.split(" ") info = {f[i]: int(f[i + 1]) for i in range(2, len(f) - 1, 2)} - self.frames.append({"info": info, "launches": [], "failed": None, "error": None, "names": {}}) + self.frames.append({"info": info, "launches": [], "failed": None, "error": None, "names": {}, + "guard": [], "skipped": []}) elif ln.startswith("launch "): self.frames[-1]["launches"].append(Launch(ln[7:])) + elif ln.startswith("guard_"): # the frame guard (the hybrid bottleneck's wait timeout; native D3D12) + self.frames[-1]["guard"].append(ln) + elif ln.startswith("skipped "): # what the frame guard skipped (not done) + self.frames[-1]["skipped"].append(ln[8:]) elif ln.startswith("failed "): self.frames[-1]["failed"] = ln[7:] elif ln.startswith("ERR frame "): From 1de8481fd070418d46dcf30d8dc38915b2699745 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Mon, 16 Nov 2026 00:51:38 -0500 Subject: [PATCH 2/5] Model L hybrid: the bottleneck timeout through the game path, release check routes, ISA scalar sources, private real runs - Bottleneck wait timeout in a game: the device keeps the frame guard's predicate on a caller's list until the end of our recording, so the executor's output pass (proxy copy or output draw) is skipped with a timed-out or queued frame; the refused frame travels as its own result ("refused, keep previous output": one log line, no fallback, WG_OK, the context kept) and the frame after it is a reset. The developer hook writes its words with fills (any list). Interop test (test_interop_d3d12 bnecktimeout): timed-out, refused and reset frames through the game interface, synchronous and pipelined. - Release builds: the A/B hooks are not read on the hybrid route; the check routes' names are not in a release build; the ISA rules' mutants are built into a temporary folder, never into the build output; build.py refuses to embed a check kernel; the release gate refuses each selector text and architecture-prefixed check names. - Deployed library check (l_hyb_deployed_check.py): the default Model L route from the built library alone. - ISA rule 2: scalar registers are constants only when a literal was moved in on every path. - l_mm8_f32_check.py: synthetic by default; --real writes intermediates only outside the repository and records only aggregates (test_l_mm8_private.py). Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/build_dx12.py | 19 +- clean/dx12/hyb_isa_scan.py | 51 +++- clean/dx12/tests/test_hyb_isa_scan.py | 27 +- .../l/records/l_time_1440q_hybrid_d3d12.json | 244 +++++++++--------- clean/l/runner/l_hyb_deployed_check.py | 104 ++++++++ clean/l/runner/l_mm8_f32_check.py | 112 ++++++-- clean/l/runtime/tests/test_l_runtime_frame.py | 4 +- clean/l/tests/test_l_mm8_private.py | 71 +++++ clean/runner/build.py | 24 +- clean/runtime/device.h | 5 +- clean/runtime/device_d3d12.cpp | 29 ++- clean/runtime/interop/backend.cpp | 8 + clean/runtime/interop/executor_d3d12.cpp | 11 +- clean/runtime/interop/executor_select.h | 11 + .../interop/tests/test_interop_d3d12.cpp | 161 ++++++++++++ clean/runtime/runtime.cpp | 79 ++++-- clean/tests/check_no_dev_diagnostics.py | 7 +- clean/tests/test_dev_diagnostics_off.py | 51 +++- 18 files changed, 819 insertions(+), 199 deletions(-) create mode 100644 clean/l/runner/l_hyb_deployed_check.py create mode 100644 clean/l/tests/test_l_mm8_private.py diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index 7777762d..9e939702 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -41,7 +41,8 @@ into the FP8 pack). A finding fails the build. No skip switch (WG_DX12_VGPR_CHECK=0 does not skip them; RGA is required when hybrid kernels are built). Negative controls, fail-closed: every model hybrid kernel K built (probe=0) needs its mutant K_scanmut (the same kernel without the contraction barrier and without the RN16 before the FP8 pack, -HYB_SCAN_MUTANT=1, probe=1), built beside every build of hybrid kernels into /dx12/scanmut (never in the manifest); +HYB_SCAN_MUTANT=1, probe=1), built beside every build of hybrid kernels into a temporary folder of their own (never +into , its manifest or a payload; removed after the scan); the build fails when it is not declared, gives no ISA, or PASSES rule 2, or rule 1 (unless K has no norm tree: HYB_NO_NORM). """ @@ -496,14 +497,16 @@ def main(argv=None): raise SystemExit(f"kernel {d['name']} ({d['co']}): DXIL file {os.path.basename(out)} already used " "(a second port of a name needs file=)") jobs.append((src, d, out, k_source)) - # the fused-square scan's mutants beside every build of hybrid kernels (outside the manifest, /dx12/scanmut) + # the ISA rules' mutants beside every build of hybrid kernels: into a temporary folder of their own, never into the + # build's output (a release folder, its manifest and a payload hold no check object); removed after the scan + scan_dir = None if any(hyb_scanned(j[1]["name"]) for j in jobs): have = {j[1]["name"] for j in jobs} for src, k_source in source_files(False): for d in kernel_decls(src): if scan_mutant(d["name"]) and d["name"] not in have: - os.makedirs(os.path.join(dst, "scanmut"), exist_ok=True) - jobs.append((src, d, os.path.join(dst, "scanmut", output_name(d["name"])), k_source)) + scan_dir = scan_dir or tempfile.mkdtemp(prefix="wg_scanmut_") + jobs.append((src, d, os.path.join(scan_dir, output_name(d["name"])), k_source)) def build_one(job): src, d, out, k_source = job @@ -525,6 +528,8 @@ def build_one(job): r["gscale"] = d["gscale"] if info.get("ext_doubles"): r["fp64"] = "ext" + if d["probe"]: + r["probe"] = True # a check kernel (built only when --only names it): clean/runner/build.py never embeds it return r # Every kernel is an independent chain of child processes (dxc -> dxil_asm -> dxv [-> rga]) with its own temporary @@ -550,7 +555,11 @@ def build_one(job): names = {n for n, _ in kernels} | {n for n, _ in built} if only and not (only <= names): raise SystemExit(f"not declared in clean/dx12/kernels or clean/k/dx12/kernels: {sorted(only - names)}") - vgpr_check(built, {d["name"] for _, d, _, _ in jobs if hyb_scanned(d["name"]) and not d["probe"]}) + try: + vgpr_check(built, {d["name"] for _, d, _, _ in jobs if hyb_scanned(d["name"]) and not d["probe"]}) + finally: + if scan_dir: + shutil.rmtree(scan_dir, ignore_errors=True) with open(man_path, "w") as f: json.dump({"format": FORMAT, "dxc": os.path.basename(os.path.dirname(os.path.dirname(DXC_DIR))), "kernels": [kernels[k] for k in sorted(kernels)]}, f, indent=1) diff --git a/clean/dx12/hyb_isa_scan.py b/clean/dx12/hyb_isa_scan.py index 4772b168..e91b2c90 100644 --- a/clean/dx12/hyb_isa_scan.py +++ b/clean/dx12/hyb_isa_scan.py @@ -23,7 +23,8 @@ by v_cvt_f32_f16 (the exact widening after the RN16), or by the saturating pack's steps on such values only (v_mov_b32, v_cndmask_b32, v_max / v_min / v_max3 / v_min3 / v_maxmin / v_minmax / v_med3 (_num)_f32 with constants: the ReLU and the clamps; the NaN quieting v_or_b32 with 0x400000), on every path to the conversion (a dataflow over - the basic blocks). Anything else + the basic blocks). A constant is a literal, or a scalar register that s_mov_b32 / s_movk_i32 set to a literal on + every path (any other write makes it unknown); an unknown scalar register is not an FP16 value. Anything else (a WMMA result, an FMA, a load, an instruction the rule does not know) fails: a driver that drops the RN16 or folds it into the FP8 conversion leaves the float32 value as the source. @@ -51,6 +52,12 @@ QUIET_NAN = "0x400000" NO_DEST = re.compile(r"^(s_|v_cmp_|v_cmpx_|v_readlane|v_readfirstlane|.*_store|ds_write|exp\b|v_nop)") VREG = re.compile(r"^v(\d+)(?:\.(l|h))?$") +SREG = re.compile(r"^s(\d+)$") +SRANGE = re.compile(r"^s\[(\d+):(\d+)\]$") +LITERAL = re.compile(r"^-?(0x[0-9a-fA-F]+|\d+(\.\d+)?(e[-+]?\d+)?)$") +# scalar instructions whose first operand is read, not written +S_NO_DEST = re.compile(r"^(s_cmp|s_bitcmp|s_cbranch|s_branch|s_wait|s_delay_alu|s_nop|s_barrier|s_endpgm|s_sendmsg|" + r"s_setprio|s_sleep|s_clause|s_set_inst_prefetch|s_trap|s_setpc|.*_store)") VRANGE = re.compile(r"^v\[(\d+):(\d+)\]$") MOD = re.compile(r"\b[a-z_]+:(?:\[[^\]]*\]|[^\s,]+)") LABEL = re.compile(r"^\s*([A-Za-z_][\w.$]*):\s*(?://.*)?$") @@ -284,14 +291,49 @@ def fused_squares(text): ALL = None # the meet's top: every register an FP16 value (blocks not reached yet) +def _sdest(ins): + """Scalar registers an instruction writes ("s" names): the first operand of a scalar instruction (but compares, + branches, waits, stores) or of a vector instruction that writes a scalar (compares, readlane).""" + if not ins.ops or (ins.op.startswith("s_") and S_NO_DEST.match(ins.op)): + return [] + if not (ins.op.startswith("s_") or ins.op.startswith(("v_cmp_", "v_readlane", "v_readfirstlane"))): + return [] + t = _reg(ins.ops[0]) + m = SREG.match(t) + if m: + return ["s" + m.group(1)] + m = SRANGE.match(t) + return ["s%d" % i for i in range(int(m.group(1)), int(m.group(2)) + 1)] if m else [] + + +def _value_ok(tok, ex): + """A source that is an FP16 value: a VGPR the dataflow says is one, a literal, or a constant scalar register.""" + t = _reg(tok) + rs = _vregs(t) + if rs: + return all(r in ex for r in rs) + if LITERAL.match(t): + return True + m = SREG.match(t) + return bool(m) and ("s" + m.group(1)) in ex + + def _exact_transfer(ex, ins, bad): srcs = ins.ops[1:] if bad is not None and ins.op in FP8_PACK: for s in srcs[:2]: - rs = _vregs(s) - if rs and not all(r in ex for r in rs): + if not _value_ok(s, ex): bad.append((ins.line, ins.text)) break + sd = _sdest(ins) + if sd: # scalar provenance: a constant only from a literal (or a constant) moved in on every path + ex = set(ex) + const = ins.op in ("s_mov_b32", "s_movk_i32") and len(sd) == 1 and len(srcs) == 1 and _value_ok(srcs[0], ex) \ + and not _vregs(srcs[0]) + ex.difference_update(sd) + if const: + ex.add(sd[0]) + ex = frozenset(ex) dregs = _dest(ins) if not dregs: return ex @@ -299,7 +341,8 @@ def _exact_transfer(ex, ins, bad): if ins.op in WIDEN: keep = True elif ins.op in KEEP: - keep = all(r in ex for r in vsrc) and not ("v_cndmask" in ins.op and ins.mods and "neg" in ins.mods) + vals = srcs[:2] if ins.op == "v_cndmask_b32" else srcs # (cndmask's third operand: the lane mask) + keep = all(_value_ok(s, ex) for s in vals) and not ("v_cndmask" in ins.op and ins.mods and "neg" in ins.mods) elif ins.op == "v_or_b32": lits = [_reg(s) for s in srcs if not _vregs(s)] keep = len(vsrc) == 1 and lits == [QUIET_NAN] and vsrc[0] in ex diff --git a/clean/dx12/tests/test_hyb_isa_scan.py b/clean/dx12/tests/test_hyb_isa_scan.py index b0599249..53afc463 100644 --- a/clean/dx12/tests/test_hyb_isa_scan.py +++ b/clean/dx12/tests/test_hyb_isa_scan.py @@ -67,7 +67,8 @@ def bad(self, *lines): def test_fp16_sources(self): self.assertEqual(self.bad(*self.WIDEN, "v_cvt_pk_fp8_f32 v5.l, v3, v4"), 0) # the saturating pack's steps on FP16 values: clamp, NaN quieting, select - self.assertEqual(self.bad(*self.WIDEN, "v_maxmin_num_f32 v6, s6, v3, 0x43e00000", "v_or_b32_e32 v7, 0x400000, v3", + self.assertEqual(self.bad(*self.WIDEN, "s_mov_b32 s6, 0xc3e00000", "v_maxmin_num_f32 v6, s6, v3, 0x43e00000", + "v_or_b32_e32 v7, 0x400000, v3", "v_cndmask_b32_e64 v8, v6, v7, s4", "v_max3_num_f32 v9, 0, v4, 0xc3e00000", "v_cvt_pk_fp8_f32 v5.h, v8, v9 op_sel:[0,0,1]"), 0) self.assertEqual(self.bad("v_mov_b32_e32 v3, 0", "v_cvt_pk_fp8_f32 v5.l, v3, 0"), 0) # constants @@ -87,6 +88,30 @@ def test_removed_widening(self): # loads are not FP16 values self.assertEqual(self.bad("global_load_b32 v3, v1, s[0:1]", "v_cvt_pk_fp8_f32 v5.l, v3, v3"), 1) + def test_scalar_sources(self): + """The negative control the re-check asked for: a scalar register is a constant only when a literal was moved + into it on every path; an unknown one (a load, an arithmetic result, a kernel argument) is not an FP16 value.""" + self.assertEqual(self.bad("v_mov_b32_e32 v3, s0", "v_cvt_pk_fp8_f32 v5.l, v3, v3"), 1) + self.assertEqual(self.bad("s_load_b32 s0, s[2:3], 0x0", "v_mov_b32_e32 v3, s0", "v_cvt_pk_fp8_f32 v5.l, v3, v3"), 1) + self.assertEqual(self.bad(*self.WIDEN, "s_add_f32 s6, s7, s8", "v_maxmin_num_f32 v6, s6, v3, 0x43e00000", + "v_cvt_pk_fp8_f32 v5.l, v6, v4"), 1) + self.assertEqual(self.bad("v_cvt_pk_fp8_f32 v5.l, s0, s0"), 1) + # a literal moved in (also through a second scalar), then overwritten + self.assertEqual(self.bad("s_mov_b32 s0, 0x3f800000", "v_mov_b32_e32 v3, s0", "v_cvt_pk_fp8_f32 v5.l, v3, v3"), 0) + self.assertEqual(self.bad(*self.WIDEN, "s_mov_b32 s6, 0xc3e00000", "s_mov_b32 s7, s6", + "v_maxmin_num_f32 v6, s7, v3, 0x43e00000", "v_cvt_pk_fp8_f32 v5.l, v6, v4"), 0) + self.assertEqual(self.bad("s_mov_b32 s0, 0x3f800000", "s_mul_i32 s0, s0, s1", "v_mov_b32_e32 v3, s0", + "v_cvt_pk_fp8_f32 v5.l, v3, v3"), 1) + # a compare reads its scalars (no write), a vector compare writes its scalar destination + self.assertEqual(self.bad("s_mov_b32 s0, 0x3f800000", "s_cmp_eq_u32 s0, 0", "v_mov_b32_e32 v3, s0", + "v_cvt_pk_fp8_f32 v5.l, v3, v3"), 0) + self.assertEqual(self.bad("s_mov_b32 s0, 0x3f800000", "v_cmp_eq_f32_e64 s0, v1, v2", "v_mov_b32_e32 v3, s0", + "v_cvt_pk_fp8_f32 v5.l, v3, v3"), 1) + # on one path only + one = isa("s_cbranch_scc0 L1", "s_mov_b32 s0, 0x3f800000", "L1:", "v_mov_b32_e32 v3, s0", + "v_cvt_pk_fp8_f32 v5.l, v3, v3") + self.assertEqual(len(S.fp8_sources_not_fp16(one)), 1) + def test_paths(self): both = isa("s_cbranch_scc0 L1", *self.WIDEN, "s_branch L2", "L1:", *self.WIDEN, "L2:", "v_cvt_pk_fp8_f32 v5.l, v3, v4") diff --git a/clean/l/records/l_time_1440q_hybrid_d3d12.json b/clean/l/records/l_time_1440q_hybrid_d3d12.json index 02c77478..34cebf40 100644 --- a/clean/l/records/l_time_1440q_hybrid_d3d12.json +++ b/clean/l/records/l_time_1440q_hybrid_d3d12.json @@ -23,57 +23,57 @@ "dx12" ], "total_ms": [ - 3.822, - 3.88, - 3.8825, - 3.886, - 3.891 + 3.8125, + 3.883, + 3.887, + 3.8905, + 3.893 ], "network_ms": [ - 3.8155, - 3.874, + 3.806, 3.876, - 3.8795, - 3.8845 + 3.8805, + 3.8845, + 3.8865 ], "parts_ms": [ { - "enc0": 0.815, - "Swin": 1.228, - "dec0": 0.6475, - "post": 1.124 - }, - { - "enc0": 0.8255, - "Swin": 1.249, - "dec0": 0.66, - "post": 1.1395 + "enc0": 0.8145, + "Swin": 1.225, + "dec0": 0.6465, + "post": 1.1195 }, { "enc0": 0.826, - "Swin": 1.251, + "Swin": 1.25, "dec0": 0.6605, - "post": 1.139 + "post": 1.14 }, { - "enc0": 0.8265, + "enc0": 0.827, "Swin": 1.2515, "dec0": 0.661, - "post": 1.1405 + "post": 1.14 }, { - "enc0": 0.827, + "enc0": 0.828, "Swin": 1.252, "dec0": 0.662, + "post": 1.142 + }, + { + "enc0": 0.828, + "Swin": 1.2525, + "dec0": 0.662, "post": 1.143 } ], - "total_median_ms": 3.882, + "total_median_ms": 3.887, "parts_median_ms": { - "enc0": 0.826, - "Swin": 1.251, + "enc0": 0.827, + "Swin": 1.252, "dec0": 0.661, - "post": 1.139 + "post": 1.14 } }, "hyb": { @@ -89,56 +89,56 @@ "dx12" ], "total_ms": [ - 4.8085, - 4.843, + 4.8135, + 4.846, + 4.8565, 4.856, - 4.854, - 4.8505 + 4.855 ], "network_ms": [ - 4.7985, - 4.8325, + 4.8035, + 4.8355, 4.846, - 4.844, - 4.8405 + 4.8455, + 4.845 ], "parts_ms": [ { - "enc0": 1.0235, - "Swin": 1.8065, - "dec0": 0.894, - "post": 1.075 + "enc0": 1.025, + "Swin": 1.8075, + "dec0": 0.895, + "post": 1.0755 + }, + { + "enc0": 1.031, + "Swin": 1.821, + "dec0": 0.901, + "post": 1.083 }, { - "enc0": 1.032, - "Swin": 1.8195, - "dec0": 0.9, - "post": 1.081 + "enc0": 1.0325, + "Swin": 1.826, + "dec0": 0.9035, + "post": 1.0845 }, { "enc0": 1.032, - "Swin": 1.827, - "dec0": 0.9025, + "Swin": 1.8265, + "dec0": 0.903, "post": 1.084 }, { - "enc0": 1.032, + "enc0": 1.033, "Swin": 1.826, "dec0": 0.902, "post": 1.0835 - }, - { - "enc0": 1.0325, - "Swin": 1.8245, - "dec0": 0.901, - "post": 1.0825 } ], - "total_median_ms": 4.851, + "total_median_ms": 4.855, "parts_median_ms": { "enc0": 1.032, - "Swin": 1.825, - "dec0": 0.901, + "Swin": 1.826, + "dec0": 0.902, "post": 1.083 } }, @@ -153,49 +153,49 @@ "dx12" ], "total_ms": [ - 4.82, - 4.837, - 4.857, + 4.822, + 4.8395, + 4.858, 4.855, - 4.8595 + 4.856 ], "network_ms": [ - 4.81, - 4.827, - 4.847, - 4.845, - 4.8495 + 4.812, + 4.8295, + 4.848, + 4.8445, + 4.8455 ], "parts_ms": [ { - "enc0": 1.025, + "enc0": 1.027, "Swin": 1.812, - "dec0": 0.8965, - "post": 1.076 + "dec0": 0.8975, + "post": 1.0765 }, { - "enc0": 1.0305, - "Swin": 1.818, - "dec0": 0.8995, - "post": 1.08 + "enc0": 1.03, + "Swin": 1.8185, + "dec0": 0.9, + "post": 1.0815 }, { - "enc0": 1.0325, + "enc0": 1.033, "Swin": 1.827, - "dec0": 0.9035, - "post": 1.084 + "dec0": 0.903, + "post": 1.0845 }, { - "enc0": 1.033, - "Swin": 1.826, - "dec0": 0.903, + "enc0": 1.0325, + "Swin": 1.8265, + "dec0": 0.9025, "post": 1.0835 }, { - "enc0": 1.0325, - "Swin": 1.8285, - "dec0": 0.903, - "post": 1.0845 + "enc0": 1.0335, + "Swin": 1.826, + "dec0": 0.9025, + "post": 1.084 } ], "total_median_ms": 4.855, @@ -217,55 +217,55 @@ "dx12" ], "total_ms": [ - 4.817, - 4.8235, - 4.857, - 4.85, - 4.8625 + 4.822, + 4.826, + 4.852, + 4.8515, + 4.851 ], "network_ms": [ - 4.8105, - 4.817, - 4.8505, - 4.8435, - 4.8565 + 4.815, + 4.82, + 4.8455, + 4.8445, + 4.844 ], "parts_ms": [ { - "enc0": 1.0265, - "Swin": 1.8085, - "dec0": 0.8975, + "enc0": 1.0295, + "Swin": 1.8095, + "dec0": 0.8985, "post": 1.078 }, { - "enc0": 1.028, + "enc0": 1.029, "Swin": 1.811, - "dec0": 0.8985, - "post": 1.0795 + "dec0": 0.899, + "post": 1.0805 }, { - "enc0": 1.034, - "Swin": 1.8265, - "dec0": 0.905, - "post": 1.0855 + "enc0": 1.033, + "Swin": 1.825, + "dec0": 0.9035, + "post": 1.0845 }, { - "enc0": 1.033, - "Swin": 1.823, + "enc0": 1.032, + "Swin": 1.8245, "dec0": 0.9035, "post": 1.084 }, { - "enc0": 1.034, - "Swin": 1.8285, - "dec0": 0.906, - "post": 1.0865 + "enc0": 1.033, + "Swin": 1.8235, + "dec0": 0.9035, + "post": 1.085 } ], - "total_median_ms": 4.85, + "total_median_ms": 4.851, "parts_median_ms": { - "enc0": 1.033, - "Swin": 1.823, + "enc0": 1.032, + "Swin": 1.824, "dec0": 0.903, "post": 1.084 } @@ -274,20 +274,20 @@ "frame_guard_cost": { "what": "the bottleneck wait timeout's frame guard (D3D12 predication, read at frame start and after the bottleneck, lifted at the end, and the kernel's abort-word check): the release default minus the same build with the guard compiled out, per interleaved round", "total_ms_per_round": [ - 0.003, - 0.0135, 0.0, - 0.005, - -0.003 + 0.0135, + 0.006, + 0.0035, + 0.005 ], - "total_ms_median": 0.003, + "total_ms_median": 0.005, "network_ms_per_round": [ - -0.0005, - 0.01, - -0.0035, - 0.0015, - -0.007 + -0.003, + 0.0095, + 0.0025, + 0.0, + 0.0015 ], - "network_ms_median": -0.0005 + "network_ms_median": 0.0015 } } diff --git a/clean/l/runner/l_hyb_deployed_check.py b/clean/l/runner/l_hyb_deployed_check.py new file mode 100644 index 00000000..8607dd5e --- /dev/null +++ b/clean/l/runner/l_hyb_deployed_check.py @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: MIT +"""Model L's default route from a deployed runtime library alone (review P2: the embedded objects): the built +wronggpu_runtime.dll copied into an empty folder (no code object, no DXIL, no manifest next to it), and a Model L +context created there with no form variable set, then two frames: + d3d12: native D3D12 (the game's executor): the hybrid route (kernels text l_form=hybrid, l_hyb_d3d12=on, + swin_bneck=hyb6, hyb_tables=on) from the library's built-in manifest and DXIL; + hip: the HIP backend: the hybrid reference route (l_q7__hyb.co, l_q7w__hyb.co and the lc12 reconstruction + l_post__lc12.co, which the library carries: build.py L_CODE_OBJECTS). +The folder is checked to hold only the library before and after (nothing read from or written next to it). Each leg +runs in its own process (the library reads its environment at load). + + python clean/l/runner/l_hyb_deployed_check.py --build --models --work + [--rocm ] [--json out.json] (GPU: inside a GPU slot) +""" +import argparse +import json +import os +import shutil +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.runner.l_d3d12_check import FORM_VARS # noqa: E402 + +RW, RH, OW, OH = 128, 72, 192, 108 +LIB = "wronggpu_runtime.dll" + + +def child(folder, models, backend, rocm): + from clean.runner.pipeline import Pipeline + p = Pipeline(folder, rocm if backend == "hip" else None, models, (RW, RH), (OW, OH), {"hdr": True}, + backend=backend, map_path=os.path.join(ROOT, "data", "preset-L.v1.map.json"), check_packing=False, + kernels={"options": "model=L"}) + k = p.kernels() or {} + rng = np.random.default_rng(3) + finite = True + for f in range(2): + x = {"color": rng.uniform(0, 1, (RH, RW, 3)).astype(np.float32), + "depth": rng.uniform(0, 1, (RH, RW)).astype(np.float32), + "motion": np.zeros((RH, RW, 2), np.float32), "reset": f == 0, "mv_scale": (1.0, 1.0), "jitter": (0.0, 0.0)} + out = p.frame(x) + finite = finite and all(np.isfinite(np.asarray(v, np.float64)).all() for v in out.values() + if isinstance(v, np.ndarray)) + return {"kernels": {key: k.get(key) for key in ("model", "l_form", "l_hyb_d3d12", "swin_bneck", "hyb_tables", + "l_post_c12", "l_kernels")}, "finite": bool(finite)} + + +def listing(folder): + return sorted(os.listdir(folder)) + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--models", required=True) + ap.add_argument("--work", required=True) + ap.add_argument("--rocm", default=r"C:\Program Files\AMD\ROCm\7.2") + ap.add_argument("--json") + ap.add_argument("--child", choices=("d3d12", "hip"), help=argparse.SUPPRESS) + a = ap.parse_args(argv) + if a.child: + print(json.dumps(child(a.work, a.models, a.child, a.rocm))) + return 0 + res = {"legs": []} + for backend in ("d3d12", "hip"): + folder = os.path.join(a.work, "deployed-" + backend) + if os.path.exists(folder): + raise SystemExit(f"{folder} exists: use an empty --work") + os.makedirs(folder) + shutil.copyfile(os.path.join(a.build, LIB), os.path.join(folder, LIB)) + before = listing(folder) + env = {k: v for k, v in os.environ.items() if k.upper() not in FORM_VARS} + cp = subprocess.run([sys.executable, "-I", os.path.abspath(__file__), "--child", backend, "--build", a.build, + "--models", a.models, "--work", folder, "--rocm", a.rocm], env=env, capture_output=True, + text=True, timeout=150, creationflags=0x08000000) + try: + r = json.loads(cp.stdout.strip().splitlines()[-1]) + except (ValueError, IndexError): + r = {"child_failed": cp.stdout[-400:] + cp.stderr[-800:]} + after = listing(folder) + k = r.get("kernels", {}) + good = (before == [LIB] and after == [LIB] and r.get("finite") and k.get("model") == "L" + and k.get("l_form") == "hybrid" and k.get("l_post_c12") == "on") + if backend == "d3d12": + good = good and k.get("l_hyb_d3d12") == "on" and k.get("swin_bneck") == "hyb6" and k.get("hyb_tables") == "on" + else: + good = good and "l_q7__hyb.co" in (k.get("l_kernels") or "") + str(k) + res["legs"].append({"backend": backend, "folder_before": before, "folder_after": after, "result": r, + "pass": bool(good)}) + print(backend, "PASS" if good else "FAIL", json.dumps(r)[:400]) + res["pass"] = all(leg["pass"] for leg in res["legs"]) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + return 0 if res["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_mm8_f32_check.py b/clean/l/runner/l_mm8_f32_check.py index af16ebf5..e7cfc1d9 100644 --- a/clean/l/runner/l_mm8_f32_check.py +++ b/clean/l/runner/l_mm8_f32_check.py @@ -10,14 +10,22 @@ - dynamic_range: products spread over the whole FP8 x FP8 range (2^-18 ... 448^2); - near_tie: the exact sum placed on an FP16 rounding tie, plus or minus one small product; - subnormal: FP8 subnormal operands (2^-9 ... 7 x 2^-9) with normal ones. -Every op where the two differ is recorded (the 32 FP8 codes of each operand and the start's FP16 bits) with the -difference in FP16 steps. With --frames , the same comparison runs on every FP8 op of Model L's own stages for -the recorded run's frames (l_frame_check), and each op's margin is reported: the distance of its exact sum from the -nearest FP16 rounding boundary divided by the bound on the FP32-sequential error (the two can differ only below 1). - - python clean/l/runner/l_mm8_f32_check.py --json out.json [--frames ] [--cases 20000] - python clean/l/runner/l_mm8_f32_check.py --wmma-prepare [--frames ] --json x # CPU - python clean/l/runner/l_mm8_f32_check.py --wmma-run --json out.json # GPU (a GPU slot) +Every synthetic op where the two differ is recorded (the 32 FP8 codes of each operand and the start's FP16 bits) with +the difference in FP16 steps. + +Synthetic values only, by default. --real (with --frames : a local recorded run of the real model) adds the same +comparison on every FP8 op of Model L's own stages for that run's frames, and each op's margin: the distance of its +exact sum from the nearest FP16 rounding boundary divided by the bound on the FP32-sequential error (the two can differ +only below 1). Real-model values never reach the repository (the synthetic-values rule): +- a real run's intermediates (the probe's sample of real ops) go only to --private-out, a folder OUTSIDE the + repository (a path inside it is refused); +- every record a real run writes (--json) is aggregate-only: counts, rates, margins and hashes, never an operand, a + start value or a result of a real op (assert_aggregate_only refuses one that is not). + + python clean/l/runner/l_mm8_f32_check.py --json out.json [--cases 20000] # synthetic + python clean/l/runner/l_mm8_f32_check.py --json out.json --real --frames --private-out + python clean/l/runner/l_mm8_f32_check.py --wmma-prepare --json x [--real --frames --private-out ] + python clean/l/runner/l_mm8_f32_check.py --wmma-run --json out.json [--cases-dir ] # GPU (a GPU slot) """ import argparse import copy @@ -35,6 +43,44 @@ from clean.l.reference import oracle as L # noqa: E402 from clean.reference.fp8 import f16, fp8_encode, q8 # noqa: E402 +# Operand-level fields of an op: allowed in a record only for synthetic sets (assert_aggregate_only) +OP_FIELDS = ("start_f16_bits", "a_fp8_codes", "w_fp8_codes", "exact", "f32seq", "exact_sum", "instruction_f32") +REAL_SETS = ("real_frames_sample",) + + +def private_dir(path): + """A real run's private output folder: refused when it is inside the repository (the synthetic-values rule).""" + p = os.path.realpath(os.path.abspath(path)) + root = os.path.realpath(ROOT) + try: + inside = os.path.commonpath([p, root]) == root + except ValueError: # another drive + inside = False + if inside: + raise SystemExit(f"--private-out {path}: inside the repository ({root}); a real run's intermediates go only " + "to a folder outside it") + os.makedirs(p, exist_ok=True) + return p + + +def assert_aggregate_only(res, path="record"): + """A real run's record: no operand-level field under a real set (or anywhere under a "frames" study).""" + if isinstance(res, dict): + for k, v in res.items(): + real = k in REAL_SETS or k == "frames" + if real and _has_op_fields(v): + raise SystemExit(f"{path}.{k}: holds operand-level values of real ops; a record is aggregate-only") + assert_aggregate_only(v, path + "." + str(k)) + + +def _has_op_fields(v): + if isinstance(v, dict): + return any(k in OP_FIELDS for k in v) or any(_has_op_fields(x) for x in v.values()) + if isinstance(v, list): + return any(_has_op_fields(x) for x in v) + return False + + FP8_VALUES = np.unique(np.abs(np.nan_to_num(np.asarray(__import__("clean.reference.fp8", fromlist=["DECODE"]).DECODE), nan=0.0))) @@ -272,9 +318,14 @@ def spy(start, a8, W8, model): # ---- the matrix instruction (GPU probe, a side measurement) ------------------------------------------------------------ -def wmma_prepare(build, n, seed=7, real_build=None, real_max=200000): - """CPU: the probe's inputs (every family, and a sample of a local recorded run's 32-term FP8 ops) with the exact sums and - the exact model's FP16 results, saved to /l_wmma_cases.pkl.""" +def wmma_prepare(build, n, seed=7, real_build=None, real_max=200000, private_out=None): + """CPU: the probe's inputs (every family, and with real_build a sample of a local recorded run's 32-term FP8 ops) + with the exact sums and the exact model's FP16 results, saved to /l_wmma_cases.pkl; with a real sample, to + /l_wmma_cases.pkl only (a folder outside the repository: private_dir).""" + if real_build: + if not private_out: + raise SystemExit("--real: the probe's real sample needs --private-out (a folder outside the repository)") + build = private_dir(private_out) sets = {} for name in FAMILIES: st, a, w = family(name, np.random.default_rng([seed, FAMILIES.index(name)]), n) @@ -318,14 +369,15 @@ def spy(start, a8, W8, model): pickle.dump({"cases": cases, "identity": identity}, f) -def wmma_run(build, rocm): +def wmma_run(build, rocm, cases_dir=None): """GPU: the probe kernel (clean/l/kernels/l_wmma_probe.hip, /l_wmma_probe.co) on every prepared set: per op, whether the instruction's float32 result equals the exact sum rounded once to float32, and whether its FP16 rounding equals the exact model's result.""" import ctypes from clean.runner.hip import Hip, clean_process_environment clean_process_environment(rocm) - with open(os.path.join(build, "l_wmma_cases.pkl"), "rb") as f: + cases_pkl = os.path.join(cases_dir or build, "l_wmma_cases.pkl") + with open(cases_pkl, "rb") as f: prep = pickle.load(f) h = Hip(rocm) with open(os.path.join(build, "l_wmma_probe.co"), "rb") as f: @@ -365,10 +417,11 @@ def wmma_run(build, rocm): "bounded_pre_63_not_exact": int(np.sum(fin & (bp0 < 63) & ~p_exact))} f32_exact = got == c["exact_sum"].astype(np.float32).astype(np.float64) d16 = f16_steps(f16(got), c["ref16"]) - rows = [{"start_f16_bits": int(np.array([st[i]], np.float16).view(np.uint16)[0]), - "a_fp8_codes": fp8_encode(a[i]).tolist(), "w_fp8_codes": fp8_encode(w[i]).tolist(), - "exact_sum": float(c["exact_sum"][i]), "instruction_f32": float(got[i])} - for i in np.flatnonzero(fin & (d16 > 0))[:20]] + rows = [] if name in REAL_SETS else [ # a real op's values never go into a record + {"start_f16_bits": int(np.array([st[i]], np.float16).view(np.uint16)[0]), + "a_fp8_codes": fp8_encode(a[i]).tolist(), "w_fp8_codes": fp8_encode(w[i]).tolist(), + "exact_sum": float(c["exact_sum"][i]), "instruction_f32": float(got[i])} + for i in np.flatnonzero(fin & (d16 > 0))[:20]] # S2a's exactness argument: while the bound stays below 63, every partial sum is a multiple of 2^-18 below 2^6, # exact in float32 in any order, so the instruction must give the exact sum. Two bounds: the true sum |a w| and # the one the kernels can afford (precomputed sum |w| of the row, times the op's max |a|). @@ -386,7 +439,10 @@ def wmma_run(build, rocm): if prep["identity"]: res["real_frames_identity"] = prep["identity"] from clean.l.runner.l_q7_check import run_identity - res["identity"] = run_identity(build, ["l_wmma_probe.co", "l_wmma_cases.pkl"]) + res["identity"] = run_identity(build, ["l_wmma_probe.co"]) + import hashlib + with open(cases_pkl, "rb") as f: + res["identity"]["l_wmma_cases_pkl_sha256"] = hashlib.sha256(f.read()).hexdigest() return res @@ -395,18 +451,29 @@ def main(argv=None): ap.add_argument("--json", required=True) ap.add_argument("--cases", type=int, default=20000) ap.add_argument("--seed", type=int, default=7) - ap.add_argument("--frames", help="a build with a recorded l_frame_check run (lframe_L_stages.pkl)") + ap.add_argument("--frames", help="with --real: a build with a local recorded l_frame_check run of the real model") + ap.add_argument("--real", action="store_true", help="also the local recorded run (--frames; needs --private-out)") + ap.add_argument("--private-out", help="--real: the folder (outside the repository) for a real run's intermediates") + ap.add_argument("--cases-dir", help="--wmma-run: the folder of l_wmma_cases.pkl (default the build folder)") ap.add_argument("--wmma-prepare", metavar="BUILD", help="CPU: the matrix-instruction probe's inputs into BUILD") ap.add_argument("--wmma-run", metavar="BUILD", help="GPU: the probe (BUILD/l_wmma_probe.co) on the prepared inputs") ap.add_argument("--rocm", default=r"C:\Program Files\AMD\ROCm\7.2") a = ap.parse_args(argv) + if a.frames and not a.real: + raise SystemExit("--frames reads a local recorded run of the real model: say --real (with --private-out)") + if a.real and not (a.frames and a.private_out): + raise SystemExit("--real needs --frames and --private-out ") + if a.real: + private_dir(a.private_out) if a.wmma_prepare: - wmma_prepare(a.wmma_prepare, a.cases, a.seed, a.frames) + wmma_prepare(a.wmma_prepare, a.cases, a.seed, a.frames if a.real else None, private_out=a.private_out) return 0 if a.wmma_run: from clean.l.runner import l_record as LR res = {"what": "RDNA 4 FP8 matrix instruction (two 16-term steps, float32 accumulator from the start) against " - "the exact model, a side measurement", "fp8_op_model": "exact", "probe": wmma_run(a.wmma_run, a.rocm)} + "the exact model, a side measurement", "fp8_op_model": "exact", + "probe": wmma_run(a.wmma_run, a.rocm, a.cases_dir)} + assert_aggregate_only(res) LR.write_l_record(res, a.json) return 0 rng = np.random.default_rng(a.seed) @@ -418,10 +485,11 @@ def main(argv=None): res["families"].setdefault(name, {})[opname] = r print(f"{name:14s} {opname:7s} cases {r['cases']} differ {r['differ']} more than 1 step " f"{r['differ_over_1']} max {r['max_f16_steps']}", flush=True) - if a.frames: + if a.real: res["frames"] = frames_study(a.frames) print("local recorded run:", json.dumps({k: v for k, v in res["frames"].items() if k != "identity"})) from clean.l.runner import l_record as LR + assert_aggregate_only(res) LR.write_l_record(res, a.json) return 0 diff --git a/clean/l/runtime/tests/test_l_runtime_frame.py b/clean/l/runtime/tests/test_l_runtime_frame.py index 7b61650d..6a82d789 100644 --- a/clean/l/runtime/tests/test_l_runtime_frame.py +++ b/clean/l/runtime/tests/test_l_runtime_frame.py @@ -1313,7 +1313,9 @@ def l_run(**kw): manifest=full_ports(), **kw) base = l_run() for env in ({"WG_AB_FUSION": "none", "WG_AB_BLOCK": "1"}, {"WG_D3D12_NOBAR": "swin_bneck6_hyb"}, - {"WG_D3D12_KSUB": "swin_bneck6_hyb=swin_block_c128_hyb"}): + {"WG_D3D12_KSUB": "swin_bneck6_hyb=swin_block_c128_hyb"}, + # an invalid value is not even read: it cannot refuse the release hybrid create (review P2) + {"WG_AB_FUSION": "no-such-fusion", "WG_AB_BLOCK": "x"}, {"WG_AB_KSUB": "=", "WG_AB_NOBAR": ","}): with self.subTest(env=env): got = l_run(env=env) self.assertTrue(any(x.startswith("kernels ") and " diag_hooks=ignored" in x for x in got)) diff --git a/clean/l/tests/test_l_mm8_private.py b/clean/l/tests/test_l_mm8_private.py new file mode 100644 index 00000000..dfdf5db2 --- /dev/null +++ b/clean/l/tests/test_l_mm8_private.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: MIT +"""The synthetic-values rule for clean/l/runner/l_mm8_f32_check.py (its --real mode reads a local recorded run): that +mode's intermediates go only to a folder outside the repository, and every record it writes is aggregate-only. +Regression checks of both; no GPU, no model, no recorded data (every value here is constructed). +Run: python clean/l/tests/test_l_mm8_private.py +""" +import os +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.runner import l_mm8_f32_check as M # noqa: E402 + + +class PrivateOutput(unittest.TestCase): + def test_a_folder_inside_the_repository_is_refused(self): + for p in (ROOT, os.path.join(ROOT, "clean", "l", "records"), os.path.join(ROOT, "out", "real-run")): + with self.subTest(p=p), self.assertRaises(SystemExit) as cm: + M.private_dir(p) + self.assertIn("inside the repository", str(cm.exception)) + + def test_a_folder_outside_it_is_taken(self): + with tempfile.TemporaryDirectory() as d: + try: + inside = os.path.commonpath([os.path.realpath(d), os.path.realpath(ROOT)]) == os.path.realpath(ROOT) + except ValueError: # another drive + inside = False + if inside: + self.skipTest("the temporary folder is inside the repository on this machine") + self.assertEqual(M.private_dir(os.path.join(d, "x")), os.path.realpath(os.path.join(d, "x"))) + + def test_the_real_mode_is_explicit_and_private(self): + """A recorded run is read only with --real, and --real only with a private folder outside the repository: the + command refuses before it reads anything (the paths below do not exist).""" + with tempfile.TemporaryDirectory() as d: + j = os.path.join(d, "r.json") + cases = (["--json", j, "--frames", os.path.join(d, "nobuild")], + ["--json", j, "--real", "--frames", os.path.join(d, "nobuild")], + ["--json", j, "--real", "--frames", os.path.join(d, "nobuild"), "--private-out", + os.path.join(ROOT, "out", "real-run")], + ["--json", j, "--wmma-prepare", d, "--real", "--frames", os.path.join(d, "nobuild"), + "--private-out", os.path.join(ROOT, "clean")]) + for argv in cases: + with self.subTest(argv=argv[2:]), self.assertRaises(SystemExit): + M.main(argv) + self.assertFalse(os.path.exists(j)) + + +class AggregateOnly(unittest.TestCase): + ROW = {"start_f16_bits": 15360, "a_fp8_codes": [56] * 32, "w_fp8_codes": [48] * 32, "exact_sum": 1.0, + "instruction_f32": 1.0} # constructed values (the shape of an example row) + + def test_operand_values_of_a_real_set_are_refused(self): + for rec in ({"probe": {"real_frames_sample": {"ops": 10, "examples": [self.ROW]}}}, + {"frames": {"ops": 3, "examples": [self.ROW]}}, + {"probe": {"real_frames_sample": {"zero_start": {"a_fp8_codes": [1, 2]}}}}): + with self.subTest(rec=list(rec)), self.assertRaises(SystemExit): + M.assert_aggregate_only(rec) + + def test_aggregates_and_synthetic_examples_pass(self): + M.assert_aggregate_only({"probe": {"real_frames_sample": {"ops": 10, "f32_not_exact_sum": 0, "examples": []}, + "near_tie": {"ops": 5, "examples": [self.ROW]}}, + "frames": {"ops": 3, "differ": 0, "min_margin": 2.5}}) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/runner/build.py b/clean/runner/build.py index b4e8f941..87ff9fbd 100644 --- a/clean/runner/build.py +++ b/clean/runner/build.py @@ -107,6 +107,15 @@ def set_model_l(on): EMBED_FILES[:] = [n for n in EMBED_FILES if n not in L_EMBED_FILES] +CHECK_KERNEL = re.compile(r"(_mut|_mutdump|_dump|_scanmut|_tmo)$|(^|/)l_hyb_probe$|_probe__mut$") + + +def is_check_kernel(entry): + """A D3D12 manifest entry of a check kernel (mutant, dump, scan mutant, timeout control, probe): never embedded in a + runtime library, whatever build put it into the manifest (a partial --only build of check kernels included).""" + return bool(entry.get("probe")) or bool(CHECK_KERNEL.search(entry.get("name", ""))) + + def is_check_object(name): """A check-only code object (a mutant, a probe, a HIP twin of the D3D12 hybrid ports): developer builds only.""" n = os.path.basename(name) @@ -312,11 +321,16 @@ def write_embedded(out_dir): if os.path.isfile(man): src["manifest.json"] = man with open(man) as f: - for k in json.load(f)["kernels"]: - p = os.path.join(out_dir, "dx12", k["file"]) - if not os.path.isfile(p): - raise SystemExit(f"missing D3D12 kernel file {p} (clean/dx12/build_dx12.py)") - src[k["file"]] = p + entries = json.load(f)["kernels"] + bad = [k.get("name") for k in entries if is_check_kernel(k)] + if bad: + raise SystemExit(f"{man} lists check kernels ({', '.join(bad[:6])}{' ...' if len(bad) > 6 else ''}): a runtime " + "library never embeds them; build the check kernels into a separate folder") + for k in entries: + p = os.path.join(out_dir, "dx12", k["file"]) + if not os.path.isfile(p): + raise SystemExit(f"missing D3D12 kernel file {p} (clean/dx12/build_dx12.py)") + src[k["file"]] = p files = list(src) lines = ["// Generated by clean/runner/build.py: files built into the runtime library (clean/runtime/embedded.h).", "#include ", "namespace wg_embedded {", diff --git a/clean/runtime/device.h b/clean/runtime/device.h index 47356dc6..feaeccc9 100644 --- a/clean/runtime/device.h +++ b/clean/runtime/device.h @@ -113,7 +113,10 @@ class Device { // nonzero, read as they are at this point of the stream (a copy into a predicate buffer; D3D12 predication). Called // again, it reads the word again. guard_end(): the guard is lifted (nothing after it is skipped) and the 16 bytes at // `word` are copied into host-visible memory; guard_read(w) gives the newest copy the GPU has written (zeros before - // the first), without waiting. A device without the guard returns false (has_guard() false). + // the first), without waiting. Recording into a caller's list (native D3D12 begin_external), the predicate stays on + // after guard_end until end_external, so the caller's own pass after the network (e.g. a copy into the game's output) + // is skipped with the frame; the caller's commands after end_external never are. A device without the guard returns + // false (has_guard() false). virtual bool has_guard() const { return false; } virtual bool guard_begin(DevPtr word) { (void)word; diff --git a/clean/runtime/device_d3d12.cpp b/clean/runtime/device_d3d12.cpp index 8bed5eab..8a79d941 100644 --- a/clean/runtime/device_d3d12.cpp +++ b/clean/runtime/device_d3d12.cpp @@ -469,7 +469,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { } bool end_external(std::string* error) override { if (!external_ || inside_pass_) return extension_result(fail("native D3D12 end_external: not recording"), error); - const bool guard_lifted = guard_end(); // never leave the caller's list predicated + const bool guard_lifted = guard_lift(); // never leave the caller's list predicated need_uav_barrier_ = true; uav_barrier(); // the caller's passes after ours read our results ++ext_recorded_; bool ok = true; @@ -668,7 +668,8 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (inside_pass_) return fail("native D3D12 guard_begin: not from a record_pass callback"); if ((word & 7u) || !range(word, 16, "guard word")) return fail("native D3D12 guard_begin: invalid guard word"); if (!guard_buffers() || !open_list()) return false; - if (guard_on_) cl()->SetPredication(nullptr, 0, D3D12_PREDICATION_OP_EQUAL_ZERO); + if (guard_on_ || guard_held_) cl()->SetPredication(nullptr, 0, D3D12_PREDICATION_OP_EQUAL_ZERO); + guard_held_ = false; // the word as the stream has it here: earlier UAV writes complete (the arena transition), then 8 bytes copied need_uav_barrier_ = true; uav_barrier(); transition_arena(D3D12_RESOURCE_STATE_COPY_SOURCE); @@ -683,16 +684,33 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (!guard_on_) return true; guard_on_ = false; if (!open_list()) return false; - cl()->SetPredication(nullptr, 0, D3D12_PREDICATION_OP_EQUAL_ZERO); - guard_transition(D3D12_RESOURCE_STATE_COMMON); + cl()->SetPredication(nullptr, 0, D3D12_PREDICATION_OP_EQUAL_ZERO); // the copy below is never skipped + const bool hold = external_ && !guard_ending_; + if (!hold) guard_transition(D3D12_RESOURCE_STATE_COMMON); need_uav_barrier_ = true; uav_barrier(); transition_arena(D3D12_RESOURCE_STATE_COPY_SOURCE); cl()->CopyBufferRegion(guard_copy_.get(), 0, arena_.get(), guard_word_, 16); // the arena back in COMMON (explicitly): what begin_external assumes for a next frame recorded into the same list transition(arena_.get(), D3D12_RESOURCE_STATE_COPY_SOURCE, D3D12_RESOURCE_STATE_COMMON); arena_state_ = D3D12_RESOURCE_STATE_COMMON; + if (hold) { // caller-list frame: predicated again until end_external (the executor's output pass) + cl()->SetPredication(guard_pred_.get(), 0, D3D12_PREDICATION_OP_NOT_EQUAL_ZERO); + guard_held_ = true; + } return true; } + // end_external: the guard ended (if the runtime did not) and the held predicate lifted + bool guard_lift() { + guard_ending_ = true; + const bool ok = guard_end(); + guard_ending_ = false; + if (guard_held_) { + cl()->SetPredication(nullptr, 0, D3D12_PREDICATION_OP_EQUAL_ZERO); + guard_transition(D3D12_RESOURCE_STATE_COMMON); + guard_held_ = false; + } + return ok; + } bool guard_read(uint32_t w[4]) override { for (int i = 0; i < 4; ++i) w[i] = guard_map_ ? guard_map_[i] : 0u; return guard_map_ != nullptr; @@ -1821,6 +1839,9 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { volatile uint32_t* guard_map_ = nullptr; D3D12_RESOURCE_STATES guard_state_ = D3D12_RESOURCE_STATE_COMMON; bool guard_on_ = false; + // a caller-list recording keeps the predicate after guard_end (the executor's output pass after the network, e.g. a + // proxy copy into the game's output, is skipped with the frame) until end_external lifts it + bool guard_held_ = false, guard_ending_ = false; DevPtr guard_word_ = 0; NativeCom query_heap_; PFN_D3D12_SERIALIZE_ROOT_SIGNATURE serialize_ = nullptr; diff --git a/clean/runtime/interop/backend.cpp b/clean/runtime/interop/backend.cpp index 43aff736..dae5f604 100644 --- a/clean/runtime/interop/backend.cpp +++ b/clean/runtime/interop/backend.cpp @@ -1136,6 +1136,14 @@ class Context { } return WG_ERROR_UNSUPPORTED; // one frame's answer: the context stays usable (failed_ not set) } + // Model L hybrid, the frame after a bottleneck wait timeout: "refused, keep previous output" (executor_select.h + // frame_timeout_refused). One line, no fallback written into the game's output, WG_OK, the context kept; the next + // frame starts over. + if (!ok && frame_timeout_refused(err)) { + say(WG_LOG_WARN, "frame " + std::to_string(v) + ": " + err + "; the game keeps its previous output for this frame"); + force_reset_ = true; + return WG_OK; + } note_path(&input_path_, "input: read directly by the network's D3D12 kernels (executor dx12, no input pass)"); if ((flags_ & WG_CREATE_MOTION_OUTPUT_RES) && model_k_) note_path(&motion_path_, "motion: output resolution, the texel under each render pixel's centre picked by the " diff --git a/clean/runtime/interop/executor_d3d12.cpp b/clean/runtime/interop/executor_d3d12.cpp index e28d8a41..0deb05d4 100644 --- a/clean/runtime/interop/executor_d3d12.cpp +++ b/clean/runtime/interop/executor_d3d12.cpp @@ -44,6 +44,7 @@ #include "../runtime_core.h" #include "convert.h" #include "executor.h" +#include "executor_select.h" #include "texture_shape.h" namespace wgi { @@ -324,7 +325,13 @@ class D3d12Executor : public Executor { }, &e, "dx12_input_states"); } - if (ok && !wgrt::runtime_frame(rt_, job.frame, &io)) ok = false, e = "network frame failed: " + wgrt::runtime_error(rt_); + bool timeout = false; // the bottleneck wait timeout's refusal: passed on as it is (executor_select.h) + if (ok && !wgrt::runtime_frame(rt_, job.frame, &io)) { + ok = false; + e = wgrt::runtime_error(rt_); + timeout = frame_timeout_refused(e); + if (!timeout) e = "network frame failed: " + e; + } if (restore) { // also after a failed frame: the game's textures go back to the states it reported std::string e2; const bool r = ext_->record_pass( @@ -355,7 +362,7 @@ class D3d12Executor : public Executor { } serials_.push_back({ext_->external_recorded(), job.value}); while (serials_.size() > 64) serials_.pop_front(); - if (!ok) return *err = "D3D12 executor: " + e, false; + if (!ok) return *err = timeout ? e : "D3D12 executor: " + e, false; *output_written = true; if (tb >= 0) tagged_[job.value % kRing] = job.value; return true; diff --git a/clean/runtime/interop/executor_select.h b/clean/runtime/interop/executor_select.h index 4a089dbf..e20cd8f3 100644 --- a/clean/runtime/interop/executor_select.h +++ b/clean/runtime/interop/executor_select.h @@ -241,6 +241,17 @@ inline std::string drop_hip_only_filter(const std::string& options, std::string* // no output texture, a render size outside the configured maximum, an output texture without unordered access, ...). inline bool frame_refused(const std::string& err) { return err.rfind("frame not taken", 0) == 0; } +// Model L hybrid: the frame after one whose bottleneck wait timed out on the GPU (runtime.cpp kBneckTimeoutRefusal; +// hyb_bneck.hlsl). The documented result: "refused, keep previous output". The timed-out frame and the frames queued +// after it were skipped on the GPU, their output pass included (the device's frame guard predicates the network and the +// executor's output pass alike): the game's output keeps what it held. This frame records nothing that writes (only the +// game textures' state changes, restored), the backend writes no fallback into the output, logs exactly one line, +// answers WG_OK and keeps the context (never failed_); the next frame is a reset (the runtime starts over, and the +// backend asks for the reset too). +inline bool frame_timeout_refused(const std::string& err) { + return err.rfind("frame refused (bottleneck wait timeout): ", 0) == 0; +} + inline std::string k_refusal_line(uint64_t frame, const std::string& why) { return "frame " + std::to_string(frame) + ": model K: the D3D12 executor does not take this frame (" + why + "); the frame goes to the front end's own upscaler (AMD FSR, else its stand-in), never to the HIP executor"; diff --git a/clean/runtime/interop/tests/test_interop_d3d12.cpp b/clean/runtime/interop/tests/test_interop_d3d12.cpp index f3d102e7..c29622d7 100644 --- a/clean/runtime/interop/tests/test_interop_d3d12.cpp +++ b/clean/runtime/interop/tests/test_interop_d3d12.cpp @@ -23,6 +23,9 @@ // test_interop_d3d12.exe matrix "" [output heights, default 1080,1440,2160] // R1 / H2: every usual upscaler-mode ratio x output size x declared maximum x HDR / LDR gets a default-executor context and a // frame: each must be taken by the D3D12 executor (no create refusal, no HIP line); see case_matrix. +// test_interop_d3d12.exe bnecktimeout "" +// Model L hybrid, the bottleneck wait timeout through the game path (WG_DEV_BNECK_ABORT= in the environment: the +// developer runtime makes accepted frame n's bottleneck time out); see case_bneck_timeout. Synchronous, then pipelined. // test_interop_d3d12.exe hipswitch "" "" // Creates, dispatches, destroys M -> K -> M in one process and verifies the HIP DLL stays loaded between them. // test_interop_d3d12.exe hipleak "" ["" | -] [cycles] [frames] [MiB per cycle] [hold] [thread] @@ -2528,6 +2531,157 @@ bool case_async_sim(Gpu& g, const char* dll, const std::string& common, const st return ok; } +// Model L hybrid: the bottleneck wait timeout through the game path (the backend's dispatch on the game's list, the +// executor's caller-list recording; developer runtime with WG_DEV_BNECK_ABORT=, read at create). Three frames: +// 1. the timed-out frame (WG_OK): the game's output keeps the previous frame's bytes (the network and the executor's +// output pass are skipped under the device's frame guard), and the game's own copy of the output after our +// dispatch in the same list still runs (the readback below is that copy: the guard is lifted at our end); +// 2. the refused frame: WG_OK ("refused, keep previous output", executor_select.h frame_timeout_refused), exactly one +// captured log line names the refusal, no failure or fallback line, the output still the previous frame's; +// 3. the reset frame (WG_OK): the output is written again, byte for byte a fresh context's first frame on the same +// inputs (a reset reads no history), and the frames after it run. +// pipelined: up to 4 frames in flight, static colour, cyclic jitter; the refusal comes once the timeout count is read +// back (1 .. 4 frames later); every frame from the timed-out one to the refused one keeps the previous output. +bool case_bneck_timeout(Gpu& g, const char* dll, const std::string& options, bool pipelined) { + const char* ab = std::getenv("WG_DEV_BNECK_ABORT"); + if (!ab) { std::printf(" WG_DEV_BNECK_ABORT is not set (a developer runtime's check)\n"); CHECK(false); return false; } + const int A = std::atoi(ab) + 1; // the runtime counts accepted frames from 0, this test's frames from 1 + CHECK(A >= 3); + Scene s; + s.tw = 1707, s.th = 960, s.rx = 0, s.ry = 0, s.rw = 1707, s.rh = 960, s.ow = 2560, s.oh = 1440, s.mw = 1707, + s.mh = 960; + apply_test_dimensions(&s); + s.make(g); + s.set_color(g, 1); + Ctx c; + c.g = &g; + if (!c.be.load(dll)) return false; + const WgResult cr = c.create(s, WG_CREATE_HDR | WG_CREATE_DEPTH_INVERTED, options.c_str()); + CHECK(cr == WG_OK); + if (cr != WG_OK) return false; + const int T = A + 6; + std::vector> out(T + 1); + std::vector res(T + 1, WG_OK); + std::vector> lines(T + 1); + const float jit[4][2] = {{0, -1.0f / 6}, {-0.25f, 1.0f / 6}, {0.25f, -7.0f / 18}, {-0.375f, -1.0f / 18}}; + auto frame_of = [&](int t) { + WgFrame f = s.frame(g, t); + f.motion_scale[0] = f.motion_scale[1] = 0.0f; + f.motion_offset[0] = f.motion_offset[1] = 0.0f; + f.jitter[0] = jit[(t - 1) % 4][0]; + f.jitter[1] = jit[(t - 1) % 4][1]; + return f; + }; + if (!pipelined) { + for (int t = 1; t <= T; ++t) { + s.set_color(g, t); + const size_t before = g_log.size(); + out[t] = c.run(s, frame_of(t), &res[t]); + lines[t].assign(g_log.begin() + (long)before, g_log.end()); + } + } else { + std::vector rb(T + 1, nullptr); + std::vector pitch(T + 1, 0); + for (int t = 1; t <= T; ++t) { + const int k = t % 4; + if (!g.begin_ring(k)) { std::printf(" HANG before frame %d\n", t); CHECK(false); return false; } + WgFrame f = frame_of(t); + f.command_list = g.list; + const size_t before = g_log.size(); + res[t] = c.be.b->dispatch(c.u, &f); + lines[t].assign(g_log.begin() + (long)before, g_log.end()); + rb[t] = record_readback(g, s.out, s.ow, s.oh, s.outState, &pitch[t]); // the game's pass after ours + g.ringv[k] = c.fe.submit_nowait(); + } + g.wait_idle(); + CHECK(g.fence->GetCompletedValue() >= g.fv); + g.split_alloc = nullptr; + for (int t = 1; t <= T; ++t) { + out[t] = read_rgb(rb[t], s.ow, s.oh, pitch[t]); + rb[t]->Release(); + } + } + int R = -1, refusals = 0, bad = 0; + for (int t = 1; t <= T; ++t) { + CHECK(res[t] == WG_OK); + for (const std::string& m : lines[t]) { + if (m.find("frame refused (bottleneck wait timeout): ") != std::string::npos) ++refusals, R = t; + if (m.find("network frame failed") != std::string::npos || m.find("fallback") != std::string::npos) ++bad; + } + } + std::printf(" %s: timed-out frame %d, refused frame %d, refusal lines %d, failure / fallback lines %d\n", + pipelined ? "pipelined" : "synchronous", A, R, refusals, bad); + CHECK(refusals == 1); + CHECK(bad == 0); + if (refusals != 1) return false; + CHECK(pipelined ? (R > A && R <= A + 4) : R == A + 1); + CHECK(R + 2 <= T); + if (R + 2 > T) return false; + const std::vector& prev = out[A - 1]; + bool lit = false; + for (float v : prev) lit = lit || v != 0.0f; + CHECK(lit); // the previous output is a real image (a skipped copy would leave the fresh readback at zero) + for (int t = A; t <= R; ++t) { + const bool same = out[t] == prev; + if (!same) std::printf(" frame %d: the game's output changed under the timeout (must keep frame %d's)\n", t, A - 1); + CHECK(same); + } + CHECK(out[R + 1] != prev); + size_t nonfinite = 0; + for (int t = R + 1; t <= T; ++t) + for (float v : out[t]) nonfinite += !std::isfinite(v); + CHECK(nonfinite == 0); + if (!pipelined) { // the reset frame against a fresh context's first frame on the same inputs + Ctx d; + d.g = &g; + if (!d.be.load(dll)) return false; + CHECK(d.create(s, WG_CREATE_HDR | WG_CREATE_DEPTH_INVERTED, options.c_str()) == WG_OK); + s.set_color(g, R + 1); + WgResult r2 = WG_ERROR; + const std::vector fresh = d.run(s, frame_of(R + 1), &r2); + CHECK(r2 == WG_OK); + auto diff = [](const std::vector& x, const std::vector& y, double* worst) { + size_t n = 0; + *worst = 0.0; + for (size_t i = 0; i < x.size() && i < y.size(); ++i) + if (!(x[i] == y[i])) { + ++n; + const double d = std::fabs((double)x[i] - (double)y[i]); + *worst = d > *worst ? d : *worst; + } + return n; + }; + double w1 = 0.0; + const size_t n1 = diff(fresh, out[R + 1], &w1); + std::printf(" reset frame %d vs a fresh context's first frame: %zu of %zu values differ (max %.6g)\n", R + 1, n1, + fresh.size(), w1); + // control: the same context history (frames 1 .. A - 1), then the game's own reset at frame R + 1 + { + Ctx e; + e.g = &g; + if (!e.be.load(dll)) return false; + CHECK(e.create(s, WG_CREATE_HDR | WG_CREATE_DEPTH_INVERTED, options.c_str()) == WG_OK); + std::vector oe; + for (int t = 1; t < A; ++t) { + s.set_color(g, t); + WgResult r3 = WG_ERROR; + oe = e.run(s, frame_of(t), &r3); + } + s.set_color(g, R + 1); + WgFrame fr = frame_of(R + 1); + fr.reset = 1; + WgResult r4 = WG_ERROR; + oe = e.run(s, fr, &r4); + double w2 = 0.0, w3 = 0.0; + const size_t n2 = diff(oe, fresh, &w2), n3 = diff(oe, out[R + 1], &w3); + std::printf(" control (game reset after frames 1..%d): vs fresh %zu differ (max %.6g); vs the refusal's reset %zu " + "differ (max %.6g)\n", A - 1, n2, w2, n3, w3); + } + CHECK(n1 == 0); + } + return true; +} + int main(int argc, char** argv) { if (argc < 2) { std::printf("usage: test_interop_d3d12 [case]\n"); @@ -2621,6 +2775,13 @@ int main(int argc, char** argv) { std::puts(ok && !g_fail ? "PASS" : "FAIL"); return ok && !g_fail ? 0 : 1; } + if (only == "bnecktimeout") { + const std::string opt = argc > 3 ? argv[3] : ""; + const bool a = case_bneck_timeout(g, dll, opt, false); + const bool b = case_bneck_timeout(g, dll, opt, true); + std::puts(a && b && !g_fail ? "PASS" : "FAIL"); + return a && b && !g_fail ? 0 : 1; + } if (only == "hipbench") { const int frames = argc > 4 ? std::atoi(argv[4]) : 1500; if (frames < 600) { std::puts("hipbench requires >=600 frames (300 warm-up)"); return 1; } diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index 0e91f8db..2f8c81e0 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -232,6 +232,10 @@ int json_int(const wgc::Json* j, int def) { } // namespace +// The bottleneck wait timeout's refusal (hyb_bneck.hlsl, frame_body): the message of the refused frame starts with this +// text; the interop path recognises it (interop/executor_select.h frame_timeout_refused) as one recoverable frame. +constexpr const char* kBneckTimeoutRefusal = "frame refused (bottleneck wait timeout): "; + struct Runtime { WgRtConfig cfg = {}; std::string build_dir, err; @@ -624,23 +628,27 @@ struct Runtime { if (lf == "m") return fail("WG_L_FORM=m refused: the M-form is not offered for Model L (default hybrid_full; q7 = reference)"); #ifndef WG_DEV_DIAGNOSTICS - // release builds: the check routes (the deliberately altered mutant, the HIP twins) are not offered - if (lf == "hybrid_mutant" || lf == "hybrid_twin") - return fail("WG_L_FORM=" + lf + " refused: a check route of developer builds only (unset = hybrid_full)"); -#endif - if (!lf.empty() && lf != "q7" && lf != "hybrid" && lf != "hybrid_mutant" && lf != "hybrid_twin") + // release builds know no check route (the deliberately altered mutant, the HIP twins): not even their names + const bool l_twin = false, l_mut = false; + if (!lf.empty() && lf != "q7" && lf != "hybrid") + return fail("WG_L_FORM=" + lf + " refused: not a form of this build (unset = hybrid_full, q7 = reference; the " + "check routes exist in developer builds only)"); +#else + const bool l_twin = lf == "hybrid_twin", l_mut = lf == "hybrid_mutant"; + if (!lf.empty() && lf != "q7" && lf != "hybrid" && !l_mut && !l_twin) return fail("WG_L_FORM: unknown form \"" + lf + "\" (hybrid = the default, q7 = reference)"); // hybrid_twin (HIP backend, check only): the fast hybrid route of native D3D12 on the HIP twins of its *_hyb // ports (l_swin_hyb.co, swin_hyb.co): the network-level byte leg (twins vs port) - if (lf == "hybrid_twin" && c.backend != 0) return fail("WG_L_FORM=hybrid_twin runs on the HIP backend only"); - const bool hyb = lf.empty() || lf == "hybrid" || lf == "hybrid_twin"; + if (l_twin && c.backend != 0) return fail("WG_L_FORM=hybrid_twin runs on the HIP backend only"); + if (l_mut && c.backend != 0) return fail("WG_L_FORM=hybrid_mutant: the HIP backend only"); +#endif + const bool hyb = lf.empty() || lf == "hybrid" || l_twin; l_q7 = true; - l_form_name = hyb ? std::string("hybrid") : (lf == "hybrid_mutant" ? lf : std::string()); + l_form_name = hyb ? std::string("hybrid") : (l_mut ? lf : std::string()); // the hybrid refusals name the form as resolved: nothing set is the release default, not a WG_L_FORM value l_form_label = lf.empty() ? std::string("Model L hybrid (default)") : "WG_L_FORM=" + lf; - l_hyb_d3d12 = (hyb && c.backend == 3) || lf == "hybrid_twin"; // the fast hybrid route (twins: on HIP) + l_hyb_d3d12 = (hyb && c.backend == 3) || l_twin; // the fast hybrid route (twins: on HIP) if (l_hyb_d3d12) l_q7 = false; // the fast kernels' M-form layout (the HIP reference route keeps l_q7__hyb) - if (lf == "hybrid_mutant" && c.backend != 0) return fail("WG_L_FORM=hybrid_mutant: the HIP backend only"); const char* lpost = std::getenv("WG_L_POST"); const std::string lp = lpost ? lpost : ""; if (!lp.empty() && lp != "c12") return fail("WG_L_POST: unknown value \"" + lp + "\" (c12)"); @@ -654,10 +662,14 @@ struct Runtime { const char* mform = std::getenv("WG_M_FORM"); const std::string mf = mform ? mform : ""; #ifndef WG_DEV_DIAGNOSTICS - if (mf == "hybrid_mutant") - return fail("WG_M_FORM=hybrid_mutant refused: a check route of developer builds only"); -#endif + // release builds know no check route (not even its name) + if (!mf.empty() && mf != "q7" && mf != "hybrid") + return fail("WG_M_FORM=" + mf + " refused: not a form of this build (the check routes exist in developer " + "builds only)"); + m_q7 = mf == "q7" || mf == "hybrid"; +#else m_q7 = mf == "q7" || mf == "hybrid" || mf == "hybrid_mutant"; +#endif m_form_name = m_q7 ? mf : ""; if (!mf.empty() && !m_q7) return fail("WG_M_FORM: unknown form \"" + mf + "\" (q7, hybrid)"); const char* mpost = std::getenv("WG_M_POST"); @@ -973,11 +985,7 @@ struct Runtime { if (!dev->has_guard()) return fail(l_form_label + " on native D3D12: the device has no frame guard (the bottleneck's wait timeout)"); #ifdef WG_DEV_DIAGNOSTICS - if (const char* e = std::getenv("WG_DEV_BNECK_ABORT")) { - if (external_io || native_direct_io) - return fail("WG_DEV_BNECK_ABORT: a check of the runtime's own frames (not interop / direct I/O)"); - dev_bneck_abort = std::atoi(e); - } + if (const char* e = std::getenv("WG_DEV_BNECK_ABORT")) dev_bneck_abort = std::atoi(e); #endif const size_t FH = ceil32(max_rh), FW = ceil32(max_rw), H8 = FH / 8, W8 = FW / 8; const size_t nsync = 16 + 7 * ((H8 + 6) / 4); @@ -1111,7 +1119,13 @@ struct Runtime { const char* abn = std::getenv("WG_AB_NOBAR"); const char* abb = std::getenv("WG_AB_BLOCK"); const char* abk = std::getenv("WG_AB_KSUB"); - if ((abf && *abf) || (abn && *abn) || (abk && *abk)) { +#ifndef WG_DEV_DIAGNOSTICS + // release builds: the hybrid route never reads the A/B values (an invalid one cannot refuse its create) + const bool ab_read = !l_hyb_d3d12; +#else + const bool ab_read = true; +#endif + if (ab_read && ((abf && *abf) || (abn && *abn) || (abk && *abk))) { ab_block = abb && *abb ? std::max(1, std::atoi(abb)) : 300; if (abf && *abf && !parse_fusion(abf, &fz_set[1])) return false; nobar_ab = split_list(abn ? abn : ""); @@ -1119,7 +1133,8 @@ struct Runtime { } #ifndef WG_DEV_DIAGNOSTICS if (l_hyb_d3d12) { // release builds: the hybrid route is never switched or altered by the diagnostic hooks - const bool any = ab_block || !nobar_list.empty() || !nobar_ab.empty() || !ksub_list.empty() || !ksub_ab.empty(); + const bool any = (abf && *abf) || (abn && *abn) || (abk && *abk) || (abb && *abb) || !nobar_list.empty() || + !ksub_list.empty(); ab_block = 0; fz_set[1] = fz_set[0]; nobar_list.clear(); @@ -1749,10 +1764,15 @@ struct Runtime { if (native_direct_io) kernels_text += " native_io=direct_textures"; // Model L: its encoder and decoder (enc0= / dec0= above name Model M's, not loaded for Model L) // (l_post: Model L's own reconstruction, HDR or LDR, on every backend) +#ifdef WG_DEV_DIAGNOSTICS + const char* l_q7_tag = l_form_name == "hybrid" ? "__hyb" : "__hybmut"; +#else + const char* l_q7_tag = "__hyb"; // (a release build has no other hybrid form) +#endif if (model_l) kernels_text += std::string(" model=L l_kernels=l_enc_values.co,l_ops.co,l_swin.co,l_post.co") + (l_q7 ? (l_form_name.empty() ? std::string(",l_q7.co,l_q7w.co l_form=q7") - : ",l_q7" + std::string(l_form_name == "hybrid" ? "__hyb" : "__hybmut") + + : ",l_q7" + std::string(l_q7_tag) + ".co,l_q7w__hyb.co l_form=" + l_form_name) : std::string(l_hyb_d3d12 ? " l_form=hybrid" : " l_form=m")) + (l_post_c12 ? " l_post_c12=on" : "") + " l_post=" + @@ -2428,10 +2448,13 @@ struct Runtime { if (g[3] != bneck_seen) { bneck_seen = g[3]; bneck_clear = true; - frames = 0; - return fail(l_form_label + ": the bottleneck's wait timed out on the GPU (swin_bneck6_hyb, " + - std::to_string(g[3]) + " in all): that frame and the frames queued after it were skipped (no output, " - "no history); this frame is refused, the next one starts over"); + // the next frame starts over as a newly created runtime's first frame (reset = 2 above: every host state a + // frame leaves for the next one), so nothing of the skipped frames reaches it + frames = n = f = last_rw = last_rh = 0; + jprev[0] = jprev[1] = 0.0f; + return fail(std::string(kBneckTimeoutRefusal) + l_form_label + ": the bottleneck's wait timed out on the GPU " + "(swin_bneck6_hyb, " + std::to_string(g[3]) + " in all): that frame and the frames queued after it " + "were skipped (no output, no history); this frame is refused, the next one starts over"); } if (bneck_clear && !(dev->memset(bneck_sync + 48, 0, 4) || dfail())) return false; bneck_clear = false; @@ -2725,9 +2748,9 @@ struct Runtime { x = bout[5]; #ifdef WG_DEV_DIAGNOSTICS if (dev_bneck_abort >= 0 && dev_bneck_abort == frames) { // the check's forced timeout: the words it would write - dev_bneck_abort = -1; - const uint32_t wv[4] = {1u, 0u, 0u, bneck_seen + 1u}; - if (!dev->upload(bneck_sync + 48, wv, sizeof wv)) return dfail(); + dev_bneck_abort = -1; // (fills: launches, so they record into a caller's list as well as into our own) + const int b = bneck_seen == 0x5A5A5A5Au ? 0x5B : 0x5A; // a count other than the one last seen + if (!(dev->memset(bneck_sync + 48, 1, 4) && dev->memset(bneck_sync + 60, b, 4))) return dfail(); } #endif if (!dev->guard_begin(bneck_sync + 48)) return dfail(); // this frame's bottleneck may have refused it diff --git a/clean/tests/check_no_dev_diagnostics.py b/clean/tests/check_no_dev_diagnostics.py index b072ad59..fec7272f 100644 --- a/clean/tests/check_no_dev_diagnostics.py +++ b/clean/tests/check_no_dev_diagnostics.py @@ -42,19 +42,22 @@ b"/l_q7__hybmut.co", b"/l_q7w__hybmut.co", b"(WG_L_FORM=hybrid_twin: l_swin_hyb.co, swin_hyb.co)", b"WG_DEV_BNECK_ABORT", + # the check routes' selector values (WG_L_FORM / WG_M_FORM): a release build does not know their names + b"hybrid_mutant", b"hybrid_twin", b"__hybmut", ) # Check-only code objects (mutants, probes, the HIP twins of the D3D12 hybrid ports, the fused-square scan's mutants): never # in a release payload, a package or a runtime library's built-in files. # (The HIP self-test's probe.co and the D3D12 device's native_driver_probe are not check objects.) -PAYLOAD_CHECK_NAMES = re.compile(r"(mut\b|mut\.co|_mut\.dxil|_tmo\.dxil|_scanmut|_probe(__mut)?\.co|(^|[\\/])l_swin_hyb\.co|(^|[\\/])swin_hyb\.co)", +PAYLOAD_CHECK_NAMES = re.compile(r"(mut\b|mut\.co|_mut\.dxil|_mutdump\.dxil|_dump\.dxil|_tmo\.dxil|_scanmut|_probe(__mut)?\.co|l_hyb_probe|(^|[\\/]|gfx\w+\.)l?_?swin_hyb\.co)", re.I) def check_object_names_in(data): """Built-in check-object names in a runtime library (embedded files are listed by name).""" return [t.decode() for t in (b"l_q7__hybmut.co", b"l_q7w__hybmut.co", b"l_post_c12_probe", b"l_swin_hyb.co", - b"swin_hyb.co", b"_scanmut") if t in data] + b"swin_hyb.co", b"_scanmut", b"_mut.dxil", b"_mutdump.dxil", b"_dump.dxil", + b"_tmo.dxil", b"l_hyb_probe", b'_hyb_mut"', b'_hyb_tmo"', b'_hyb_dump"') if t in data] def dev_only_strings_in(data): diff --git a/clean/tests/test_dev_diagnostics_off.py b/clean/tests/test_dev_diagnostics_off.py index 6ff0a612..fbfcab56 100644 --- a/clean/tests/test_dev_diagnostics_off.py +++ b/clean/tests/test_dev_diagnostics_off.py @@ -245,9 +245,11 @@ def test_check_routes_and_check_objects_are_refused(self): b"l_post_c12_probe"): ok, text = chk.check_bytes(b"MZ" + MARK0 + b" " + t) self.assertFalse(ok, t) - self.assertIn("check-only code objects", text) + # the mutant objects' names hold a selector text too ("__hybmut"): either refusal + self.assertTrue("check-only code objects" in text or "forged marker" in text, text) for n in ("l_q7__hybmut.co", "l_q7w__hybmut.co", "swin_hyb.co", "sub/l_swin_hyb.co", "l_post_c12_probe.co", - "l_post_c12_probe__mut.co", "l_hyb_probe.co", + "l_post_c12_probe__mut.co", "l_hyb_probe.co", "gfx1201.swin_hyb.co", "gfx1201.l_swin_hyb.co", + "gfx1201.l_q7__hybmut.co", "swin_bneck6_hyb_mutdump.dxil", "l_swin_block_c32_enc_hyb_dump.dxil", "dx12/scanmut/swin_block_c128_hyb_scanmut.dxil", "swin_bneck6_hyb_mut.dxil", "swin_bneck6_hyb_tmo.dxil"): ok, text, _ = chk.check_artifact(n, b"plain") self.assertFalse(ok, n) @@ -258,6 +260,51 @@ def test_check_routes_and_check_objects_are_refused(self): ok, text = chk.check_bytes(b"MZ" + MARK0 + b" /l_q7__hyb.co /l_q7w__hyb.co /l_post__lc12.co") self.assertTrue(ok, text) + def test_each_selector_text_is_refused_alone(self): + """Every check route's selector value is refused on its own in a library that says dev-diagnostics=0 (review: + one negative test per selector).""" + for t in (b"hybrid_mutant", b"hybrid_twin", b"__hybmut", b"WG_DEV_BNECK_ABORT", b"/l_q7__hybmut.co", + b"/l_q7w__hybmut.co"): + with self.subTest(t=t): + self.assertIn(t, chk.DEV_ONLY_STRINGS) + ok, text = chk.check_bytes(b"MZ" + MARK0 + b" ... " + t + b" ...") + self.assertFalse(ok, t) + for t in (b"swin_bneck6_hyb_mut.dxil", b"swin_bneck6_hyb_tmo.dxil", b'"swin_bneck6_hyb_tmo"', + b'"l_swin_block_c32_dec_hyb_mut"', b"gfx1201.swin_hyb.co", b"l_hyb_probe"): + with self.subTest(t=t): + ok, text = chk.check_bytes(b"MZ" + MARK0 + b" ... " + t + b" ...") + self.assertFalse(ok, t) + self.assertIn("check-only code objects", text) + + def test_build_never_embeds_a_check_kernel(self): + """clean/runner/build.py write_embedded refuses a D3D12 manifest that lists a check kernel (marked probe by + build_dx12.py, or named as one: mutant, dump, scan mutant, timeout control, probe), e.g. after a partial --only + build of check kernels into a release folder; the same folder without it embeds.""" + B = load_build() + self.assertTrue(B.is_check_kernel({"name": "swin_bneck6_hyb_tmo"})) + self.assertTrue(B.is_check_kernel({"name": "swin_block_c64_hyb_scanmut"})) + self.assertTrue(B.is_check_kernel({"name": "l_swin_block_c32_enc_hyb_mutdump"})) + self.assertTrue(B.is_check_kernel({"name": "anything", "probe": True})) + self.assertFalse(B.is_check_kernel({"name": "swin_bneck6_hyb"})) + self.assertFalse(B.is_check_kernel({"name": "native_driver_probe"})) + for bad in ({"name": "swin_bneck6_hyb_mut", "file": "swin_bneck6_hyb_mut.dxil"}, + {"name": "x", "file": "x.dxil", "probe": True}): + with self.subTest(bad=bad["name"]), tempfile.TemporaryDirectory() as d: + for n in B.EMBED_FILES: + open(os.path.join(d, n), "wb").close() + os.makedirs(os.path.join(d, "dx12")) + good = {"name": "swin_bneck6_hyb", "file": "swin_bneck6_hyb.dxil"} + for k in (good, bad): + open(os.path.join(d, "dx12", k["file"]), "wb").close() + with open(os.path.join(d, "dx12", "manifest.json"), "w") as f: + json.dump({"format": 1, "kernels": [good, bad]}, f) + with self.assertRaises(SystemExit) as cm: + B.write_embedded(d) + self.assertIn("check kernels", str(cm.exception)) + with open(os.path.join(d, "dx12", "manifest.json"), "w") as f: + json.dump({"format": 1, "kernels": [good]}, f) + B.write_embedded(d) # without it: embedded + def test_the_final_package_check(self): with tempfile.TemporaryDirectory() as d: good_zip, bad_zip = os.path.join(d, "good.zip"), os.path.join(d, "bad.zip") From 94bd2358e1ebd18ab8e7f9d565417c5065c44d60 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Mon, 16 Nov 2026 02:12:46 -0500 Subject: [PATCH 3/5] Model L hybrid: a stuck frame's own upscale, the refused frame to the stand-in, the game's predicate kept, private real-data folders - Bottleneck wait timeout in a game: under the inverse predication on the frame guard's word, the executor writes a bilinear upscale of each timed-out or queued frame's colour into that frame's output texture (every output route; no CPU wait; the very first frame included). The refused frame takes the path of a frame not taken (WG_ERROR_UNSUPPORTED: the front end's stand-in renders it); the context is kept and the next frame is a reset. - The game's predicate on its list: WgFrontEndServices::command_list_predication (appended; the front end answers from the state its hooks track). A guarded frame lifts it while it records and sets it again exactly at the end; a Model L hybrid frame whose predicate the front end cannot report goes to the stand-in. - Interop tests: bnecktimeout with a different output texture on each frame (sentinel before the dispatch), the timeout on the very first frame, an output without unordered access, and the reset frame against a fresh context made in a child process; callerpred (predicate false, true, not known). - Real-data modes of l_hyb_stage_check, l_q7_check, l_q7w_check and l_mm8_f32_check write only to --private-out, a resolved folder outside the repository (l_private.py; test_l_private_out.py). - Speed record re-timed on the harness built from this branch (frame guard with its fallback pass: +0.0045 ms). Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/fsr/include/wronggpu_upscaler.h | 8 + clean/fsr/src/d3d12_hooks.cpp | 35 +- .../l/records/l_time_1440q_hybrid_d3d12.json | 234 ++++----- clean/l/runner/l_hyb_stage_check.py | 20 +- clean/l/runner/l_mm8_f32_check.py | 13 +- clean/l/runner/l_private.py | 32 ++ clean/l/runner/l_q7_check.py | 63 ++- clean/l/runner/l_q7w_check.py | 35 +- clean/l/tests/test_l_private_out.py | 129 +++++ clean/runtime/device_d3d12.cpp | 20 +- clean/runtime/device_d3d12.h | 11 + clean/runtime/interop/backend.cpp | 24 +- clean/runtime/interop/convert.cpp | 15 +- clean/runtime/interop/convert.h | 7 + clean/runtime/interop/executor.h | 8 + clean/runtime/interop/executor_d3d12.cpp | 31 ++ clean/runtime/interop/executor_select.h | 15 +- .../interop/tests/test_interop_d3d12.cpp | 485 +++++++++++++----- clean/runtime/runtime.cpp | 1 + clean/runtime/runtime_core.h | 2 + 20 files changed, 899 insertions(+), 289 deletions(-) create mode 100644 clean/l/runner/l_private.py create mode 100644 clean/l/tests/test_l_private_out.py diff --git a/clean/fsr/include/wronggpu_upscaler.h b/clean/fsr/include/wronggpu_upscaler.h index 966b9190..495286a7 100644 --- a/clean/fsr/include/wronggpu_upscaler.h +++ b/clean/fsr/include/wronggpu_upscaler.h @@ -192,6 +192,14 @@ typedef struct WgFrontEndServices { void* context; /* first argument of every service */ /* Only from within dispatch, with the frame passed to dispatch. */ WgResult (*split_command_list)(void* context, const WgFrame* frame, const WgSplitRequest* request); + /* Appended (0.6.5; check struct_size): the predicate the game set on frame->command_list + * (ID3D12GraphicsCommandList::SetPredication) as the front end tracked it since the list's last Reset. WG_OK with + * *buffer (ID3D12Resource*, null: none set), *offset and *op (D3D12_PREDICATION_OP); WG_ERROR_NOT_READY when the + * front end has not seen every call the list recorded since its Reset; WG_ERROR_PARAMETER for a null argument. + * Only from within dispatch, with the frame passed to dispatch. A backend that replaces the predicate while it + * records (the D3D12 executor's frame guard) sets it again with these values before it returns. */ + WgResult (*command_list_predication)(void* context, const WgFrame* frame, void** buffer, uint64_t* offset, + uint32_t* op); } WgFrontEndServices; typedef struct WgUpscaler WgUpscaler; /* opaque, owned by the implementation */ diff --git a/clean/fsr/src/d3d12_hooks.cpp b/clean/fsr/src/d3d12_hooks.cpp index 68106139..92434297 100644 --- a/clean/fsr/src/d3d12_hooks.cpp +++ b/clean/fsr/src/d3d12_hooks.cpp @@ -2064,7 +2064,40 @@ WgResult WGAPI_Split(void*, const WgFrame* frame, const WgSplitRequest* request) } } -const WgFrontEndServices g_services = {sizeof(WgFrontEndServices), nullptr, &WGAPI_Split}; +// WgFrontEndServices::command_list_predication: the list's predicate as tracked (HPredication; a Reset clears it), +// when every call it recorded since its Reset reached the hooks (a patched class table, or its own table tracked since +// that Reset). +WgResult PredicationImpl(const WgFrame* frame, void** buffer, uint64_t* offset, uint32_t* op) { + if (!frame || !frame->command_list || !buffer || !offset || !op) return WG_ERROR_PARAMETER; + *buffer = nullptr; + *offset = 0; + *op = 0; + if (!g_installed.load()) return WG_ERROR_NOT_READY; + std::lock_guard once(g_patchMutex); + auto* list = static_cast(frame->command_list); + void** vt = *reinterpret_cast(list); + ListState* s = FindState(list); + if (!s || !s->known) return WG_ERROR_NOT_READY; + const bool tracked = IsPatched(g_listVt, vt) || (s->own && s->own->table == vt && s->own->tracked && OwnIntact(*s->own)); + if (!tracked) return WG_ERROR_NOT_READY; + if (s->b.predicationSet) { + *buffer = s->b.predicationBuffer; + *offset = s->b.predicationOffset; + *op = static_cast(s->b.predicationOp); + } + return WG_OK; +} + +WgResult WGAPI_Predication(void*, const WgFrame* frame, void** buffer, uint64_t* offset, uint32_t* op) { + try { + return PredicationImpl(frame, buffer, offset, op); + } catch (...) { + HLog("ERROR: exception in command_list_predication"); + return WG_ERROR; + } +} + +const WgFrontEndServices g_services = {sizeof(WgFrontEndServices), nullptr, &WGAPI_Split, &WGAPI_Predication}; void NoteImpl(void* listPtr, WgFrame* frame) { if (!listPtr) return; diff --git a/clean/l/records/l_time_1440q_hybrid_d3d12.json b/clean/l/records/l_time_1440q_hybrid_d3d12.json index 34cebf40..27bbd201 100644 --- a/clean/l/records/l_time_1440q_hybrid_d3d12.json +++ b/clean/l/records/l_time_1440q_hybrid_d3d12.json @@ -1,5 +1,5 @@ { - "what": "Model L, 1440p Quality, harness, D3D12: frame GPU time per configuration, 5 interleaved rounds (median of each round's frames), the direct-route benchmark (clean/runner/direct_bench.py) on the D3D12 interop test harness, executor dx12", + "what": "Model L, 1440p Quality, harness, D3D12: frame GPU time per configuration, 5 interleaved rounds (median of each round's frames), the direct-route benchmark (clean/runner/direct_bench.py) on the D3D12 interop test harness built from this branch (every configuration), executor dx12", "label": "Model L, 1440p Quality, harness, D3D12", "shape": "1440p Quality", "render": [ @@ -23,57 +23,57 @@ "dx12" ], "total_ms": [ - 3.8125, - 3.883, - 3.887, - 3.8905, - 3.893 + 3.812, + 3.8855, + 3.889, + 3.8865, + 3.889 ], "network_ms": [ - 3.806, - 3.876, - 3.8805, - 3.8845, - 3.8865 + 3.8055, + 3.8785, + 3.883, + 3.8795, + 3.8825 ], "parts_ms": [ { - "enc0": 0.8145, + "enc0": 0.814, "Swin": 1.225, - "dec0": 0.6465, - "post": 1.1195 + "dec0": 0.6475, + "post": 1.119 }, { "enc0": 0.826, - "Swin": 1.25, - "dec0": 0.6605, - "post": 1.14 - }, - { - "enc0": 0.827, - "Swin": 1.2515, + "Swin": 1.2505, "dec0": 0.661, - "post": 1.14 + "post": 1.1415 }, { - "enc0": 0.828, + "enc0": 0.8265, "Swin": 1.252, "dec0": 0.662, - "post": 1.142 + "post": 1.1425 + }, + { + "enc0": 0.826, + "Swin": 1.252, + "dec0": 0.6615, + "post": 1.1405 }, { - "enc0": 0.828, - "Swin": 1.2525, + "enc0": 0.826, + "Swin": 1.252, "dec0": 0.662, - "post": 1.143 + "post": 1.142 } ], - "total_median_ms": 3.887, + "total_median_ms": 3.886, "parts_median_ms": { - "enc0": 0.827, + "enc0": 0.826, "Swin": 1.252, "dec0": 0.661, - "post": 1.14 + "post": 1.141 } }, "hyb": { @@ -89,57 +89,57 @@ "dx12" ], "total_ms": [ - 4.8135, - 4.846, - 4.8565, - 4.856, - 4.855 + 4.815, + 4.8445, + 4.857, + 4.8545, + 4.853 ], "network_ms": [ - 4.8035, - 4.8355, - 4.846, + 4.804, + 4.8335, 4.8455, - 4.845 + 4.843, + 4.8415 ], "parts_ms": [ { - "enc0": 1.025, - "Swin": 1.8075, + "enc0": 1.0255, + "Swin": 1.8085, "dec0": 0.895, - "post": 1.0755 + "post": 1.074 }, { "enc0": 1.031, - "Swin": 1.821, + "Swin": 1.8215, "dec0": 0.901, - "post": 1.083 + "post": 1.08 }, { "enc0": 1.0325, - "Swin": 1.826, - "dec0": 0.9035, - "post": 1.0845 + "Swin": 1.8275, + "dec0": 0.903, + "post": 1.0825 }, { - "enc0": 1.032, - "Swin": 1.8265, + "enc0": 1.0325, + "Swin": 1.826, "dec0": 0.903, - "post": 1.084 + "post": 1.082 }, { - "enc0": 1.033, - "Swin": 1.826, - "dec0": 0.902, - "post": 1.0835 + "enc0": 1.032, + "Swin": 1.8255, + "dec0": 0.9025, + "post": 1.0815 } ], - "total_median_ms": 4.855, + "total_median_ms": 4.853, "parts_median_ms": { "enc0": 1.032, - "Swin": 1.826, - "dec0": 0.902, - "post": 1.083 + "Swin": 1.825, + "dec0": 0.903, + "post": 1.082 } }, "hybdef": { @@ -154,56 +154,56 @@ ], "total_ms": [ 4.822, - 4.8395, - 4.858, - 4.855, - 4.856 + 4.839, + 4.8585, + 4.8535, + 4.8515 ], "network_ms": [ - 4.812, - 4.8295, - 4.848, - 4.8445, - 4.8455 + 4.811, + 4.8275, + 4.8475, + 4.8425, + 4.8405 ], "parts_ms": [ { "enc0": 1.027, - "Swin": 1.812, + "Swin": 1.8115, "dec0": 0.8975, - "post": 1.0765 + "post": 1.075 }, { - "enc0": 1.03, - "Swin": 1.8185, + "enc0": 1.0305, + "Swin": 1.8195, "dec0": 0.9, - "post": 1.0815 + "post": 1.0785 }, { "enc0": 1.033, - "Swin": 1.827, + "Swin": 1.8285, "dec0": 0.903, - "post": 1.0845 + "post": 1.082 }, { - "enc0": 1.0325, - "Swin": 1.8265, + "enc0": 1.032, + "Swin": 1.826, "dec0": 0.9025, - "post": 1.0835 + "post": 1.082 }, { - "enc0": 1.0335, - "Swin": 1.826, + "enc0": 1.032, + "Swin": 1.8245, "dec0": 0.9025, - "post": 1.084 + "post": 1.081 } ], - "total_median_ms": 4.855, + "total_median_ms": 4.851, "parts_median_ms": { "enc0": 1.032, - "Swin": 1.826, + "Swin": 1.825, "dec0": 0.903, - "post": 1.083 + "post": 1.081 } }, "ng": { @@ -217,55 +217,55 @@ "dx12" ], "total_ms": [ - 4.822, + 4.8225, 4.826, - 4.852, - 4.8515, - 4.851 + 4.851, + 4.849, + 4.849 ], "network_ms": [ - 4.815, + 4.816, 4.82, - 4.8455, - 4.8445, - 4.844 + 4.845, + 4.8425, + 4.8425 ], "parts_ms": [ { - "enc0": 1.0295, - "Swin": 1.8095, + "enc0": 1.0285, + "Swin": 1.812, "dec0": 0.8985, - "post": 1.078 + "post": 1.0775 }, { - "enc0": 1.029, - "Swin": 1.811, + "enc0": 1.0295, + "Swin": 1.8115, "dec0": 0.899, - "post": 1.0805 + "post": 1.0795 }, { "enc0": 1.033, - "Swin": 1.825, - "dec0": 0.9035, - "post": 1.0845 + "Swin": 1.8235, + "dec0": 0.904, + "post": 1.084 }, { "enc0": 1.032, - "Swin": 1.8245, + "Swin": 1.823, "dec0": 0.9035, "post": 1.084 }, { - "enc0": 1.033, - "Swin": 1.8235, - "dec0": 0.9035, - "post": 1.085 + "enc0": 1.0325, + "Swin": 1.823, + "dec0": 0.904, + "post": 1.084 } ], - "total_median_ms": 4.851, + "total_median_ms": 4.849, "parts_median_ms": { "enc0": 1.032, - "Swin": 1.824, + "Swin": 1.823, "dec0": 0.903, "post": 1.084 } @@ -274,20 +274,20 @@ "frame_guard_cost": { "what": "the bottleneck wait timeout's frame guard (D3D12 predication, read at frame start and after the bottleneck, lifted at the end, and the kernel's abort-word check): the release default minus the same build with the guard compiled out, per interleaved round", "total_ms_per_round": [ - 0.0, - 0.0135, - 0.006, - 0.0035, - 0.005 + -0.0005, + 0.013, + 0.0075, + 0.0045, + 0.0025 ], - "total_ms_median": 0.005, + "total_ms_median": 0.0045, "network_ms_per_round": [ - -0.003, - 0.0095, + -0.005, + 0.0075, 0.0025, 0.0, - 0.0015 + -0.002 ], - "network_ms_median": 0.0015 + "network_ms_median": 0.0 } } diff --git a/clean/l/runner/l_hyb_stage_check.py b/clean/l/runner/l_hyb_stage_check.py index 8a00ca70..3a0351f9 100644 --- a/clean/l/runner/l_hyb_stage_check.py +++ b/clean/l/runner/l_hyb_stage_check.py @@ -362,13 +362,24 @@ def main(argv=None): ap.add_argument("--no-replay", action="store_true", help="analyse the outputs of an earlier --replay-only (CPU)") ap.add_argument("--real", action="store_true", help="real L weights / recorded frames: the allowance share counts") ap.add_argument("--real-inputs", help="a directory of -.npz (real L weights and recorded-frame block " - "inputs: x2, x3, y, parity, enc0_block, dec0_block; private, never in the repository); implies --real") + "inputs: x2, x3, y, parity, enc0_block, dec0_block; private, never in the repository); implies --real; " + "needs --private-out") + ap.add_argument("--private-out", help="--real-inputs: the folder (outside the repository) for the real cases' files " + "and replay outputs (refused inside the repository)") ap.add_argument("--set", choices=("c32", "s3", "fused"), default="c32", help="c32: Model L's two blocks; s3: the " "shared stages' hybrid blocks (C64, C96 with and without the skip, C128); fused: the twins of the " "fusions le / lc (encoder with the linears, decoder with the pack12 epilogue)") ap.add_argument("--mutant", action="store_true", help="the order mutant's dump kernel (SW_HYB_MUT_ORDER=2, encoder): the tolerance leg must PASS it") a = ap.parse_args(argv) + # the synthetic-values rule: a real run's case files and replay outputs only in a private folder outside the + # repository, checked before anything is read or built + cdir = None + if a.real_inputs: + from clean.l.runner.l_private import private_dir + cdir = private_dir(a.private_out, what="--real-inputs: the real cases' files and replay outputs") + elif a.private_out: + raise SystemExit("--private-out: only with --real-inputs") if a.set == "fused": kernels = FUSED_MUT if a.mutant else FUSED_KERNELS else: @@ -379,12 +390,13 @@ def main(argv=None): if not a.run: return 0 rng = np.random.default_rng(13) - cdir = os.path.join(a.build, "hyb_stage") - os.makedirs(cdir, exist_ok=True) + if cdir is None: + cdir = os.path.join(a.build, "hyb_stage") + os.makedirs(cdir, exist_ok=True) lines, meta, nan_leg = [], [], set() n = 0 # the random legs (finite codes), then the one NaN-input leg (seed a.seeds, every case special, NaN codes included) - for seed in range(a.seeds + 1): + for seed in (range(a.seeds + 1) if not a.real_inputs else ()): # (the real cases replace them) is_nan = seed == a.seeds sp8 = LD.SPECIAL_FP8 if is_nan else FINITE_FP8 sp16 = LD.SPECIAL_F16 if is_nan else FINITE_F16 diff --git a/clean/l/runner/l_mm8_f32_check.py b/clean/l/runner/l_mm8_f32_check.py index e7cfc1d9..5cf6c5d7 100644 --- a/clean/l/runner/l_mm8_f32_check.py +++ b/clean/l/runner/l_mm8_f32_check.py @@ -50,17 +50,8 @@ def private_dir(path): """A real run's private output folder: refused when it is inside the repository (the synthetic-values rule).""" - p = os.path.realpath(os.path.abspath(path)) - root = os.path.realpath(ROOT) - try: - inside = os.path.commonpath([p, root]) == root - except ValueError: # another drive - inside = False - if inside: - raise SystemExit(f"--private-out {path}: inside the repository ({root}); a real run's intermediates go only " - "to a folder outside it") - os.makedirs(p, exist_ok=True) - return p + from clean.l.runner.l_private import private_dir as shared + return shared(path) def assert_aggregate_only(res, path="record"): diff --git a/clean/l/runner/l_private.py b/clean/l/runner/l_private.py new file mode 100644 index 00000000..ca588f4f --- /dev/null +++ b/clean/l/runner/l_private.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: MIT +"""The synthetic-values rule for the Model L checkers' real modes (l_hyb_stage_check --real-inputs, l_q7_check / +l_q7w_check --real, l_mm8_f32_check --real): every intermediate made from real weights or recorded frames (case files, +replay inputs and outputs) goes only to a private folder OUTSIDE the repository, resolved (real path) before anything is +read; a folder inside the repository is refused. The records these checkers write stay counts and digests. +""" +import os + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) + + +def inside_repository(path): + """Whether `path` (resolved) is the repository or inside it.""" + p = os.path.realpath(os.path.abspath(path)) + root = os.path.realpath(ROOT) + try: + return os.path.commonpath([p, root]) == root + except ValueError: # another drive + return False + + +def private_dir(path, flag="--private-out", what="a real run's intermediates"): + """The resolved private folder (made if missing); SystemExit when it is not given or is inside the repository.""" + if not path: + raise SystemExit(f"{flag} is required: {what} go only to a folder outside the repository") + if inside_repository(path): + raise SystemExit(f"{flag} {path}: inside the repository ({os.path.realpath(ROOT)}); {what} go only to a " + "folder outside it") + p = os.path.realpath(os.path.abspath(path)) + os.makedirs(p, exist_ok=True) + return p diff --git a/clean/l/runner/l_q7_check.py b/clean/l/runner/l_q7_check.py index 854ff1df..cc2da11b 100644 --- a/clean/l/runner/l_q7_check.py +++ b/clean/l/runner/l_q7_check.py @@ -4,7 +4,8 @@ byte for byte. python clean/l/runner/l_q7_check.py --build --compile # CPU: /l_q7.co - python clean/l/runner/l_q7_check.py --build --prepare [--real ] # CPU + python clean/l/runner/l_q7_check.py --build --prepare [--real + --private-out ] # CPU python clean/l/runner/l_q7_check.py --build --prepare-d3d12 # CPU: D3D12 ports + kernel_replay.exe python clean/l/runner/l_q7_check.py --build --run [--d3d12] --json out.json # GPU (inside a GPU slot) @@ -17,7 +18,9 @@ With --d3d12 the same cases run on the native D3D12 device (the ports in clean/dx12/kernels/l_q7.hlsl, kernel_replay, output buffers filled with the D3D12 sentinel 0x5A): raw bytes identical to the HIP run's and to the oracle's, slack kept, and its own half-grid control. -Output: counts and digests only. +Output: counts and digests only. --real (the synthetic-values rule): the cases file (real weights, the oracle's bytes on +them) and the D3D12 replay folder go only to --private-out, a folder outside the repository (refused inside it); give +the same --private-out to --run. --form hyb / hybmut (Q11 part C, Model L's HYBRID evaluation form; HIP only): l_q7__hyb.co (l_q7.hip Q7_HYB: float32 chains, one FP16 rounding each) against the oracle with FP8 op model "f32chain", raw bytes, the same cases (synthetic weights) and half-grid control; negative control: the exact (q7) oracle's bytes must differ in some case. hybmut: @@ -69,8 +72,9 @@ def git(*a): return r.stdout.strip() ident = {"git_commit": git("rev-parse", "HEAD"), "git_tracked_changes": bool(git("status", "--porcelain", "--untracked-files=no"))} - for f in files: - ident[f.replace(".", "_") + "_sha256"] = _sha256(os.path.join(build, f)) + for f in files: # an absolute path (a private folder) is named by its file name only + key = os.path.basename(f) if os.path.isabs(f) else f + ident[key.replace(".", "_") + "_sha256"] = _sha256(os.path.join(build, f)) return ident @@ -229,8 +233,26 @@ def compare(got, want, out_bytes, kind="fp8"): "bytes_differ": sum(a != b for a, b in zip(body, want))} +def data_dir(build, private_out): + """Where the cases file and the replay folder are: --private-out (resolved, outside the repository) or the build.""" + if private_out: + from clean.l.runner.l_private import private_dir + return private_dir(private_out) + return build + + +def load_cases(path, private_out): + """The prepared cases; a file holding real weights is read only from a private folder (--private-out).""" + with open(path, "rb") as f: + prep = pickle.load(f) + if prep.get("real_params_sha256") and not private_out: + raise SystemExit(f"{path}: prepared with --real (real weights); run it from its --private-out folder") + return prep + + def prepare(build, real=None, form="q7"): - """CPU: every case's inputs and the oracle's bytes, saved to /l_q7_cases.pkl (the GPU run only launches).""" + """CPU: every case's inputs and the oracle's bytes, saved to /l_q7_cases.pkl (the GPU run only launches); + with real (real weights), `build` is the private folder (data_dir).""" blocks = [("synthetic_%d" % s, {k: np.asarray(v, np.float64) for k, v in L.random_block_params(np.random.default_rng(s), 32, 2, 128).items()}) for s in (89, 90)] @@ -268,16 +290,17 @@ def prepare(build, real=None, form="q7"): return len(cases) -def run(build, rocm, d3d12=False, form="q7"): +def run(build, rocm, d3d12=False, form="q7", private_out=None): """GPU: launch every prepared case and compare with the oracle's bytes (with d3d12, also the D3D12 ports).""" - with open(cases_file(build, form), "rb") as f: - prep = pickle.load(f) + ddir = data_dir(build, private_out) + prep = load_cases(cases_file(ddir, form), private_out) cases = prep["cases"] co = FORM_CO[form] g = Gpu(build, rocm, co) res = {"what": "FP16-form width-32 block kernels against the oracle, raw bytes", "cases": [], "form": form, "fp8_op_model": "exact" if form == "q7" else "f32chain", - "identity": run_identity(build, [co, os.path.basename(cases_file(build, form))]), + "identity": run_identity(build, [co, cases_file(ddir, form) if private_out + else os.path.basename(cases_file(build, form))]), "real_params_sha256": prep["real_params_sha256"], "nan_sign_masked": "outputs compared and hashed with only a NaN's sign bit cleared (accepted port exception; " "a canonical NaN store costs about 3 VALU per stored dword in the RGA ISA)", @@ -314,7 +337,7 @@ def run(build, rocm, d3d12=False, form="q7"): res["cases_differing_from_exact_oracle"] = sum(bool(r["differs_from_exact_oracle"]) for r in res["cases"]) res["pass"] = res["pass"] and res["cases_differing_from_exact_oracle"] > 0 if d3d12: - res["d3d12"] = run_d3d12(build, cases, res) + res["d3d12"] = run_d3d12(build, cases, res, ddir) res["pass"] = res["pass"] and res["d3d12"]["pass"] return res @@ -325,12 +348,13 @@ def prepare_d3d12(build, rocm): return prepare_replay(build, rocm, list(KERNELS)) -def run_d3d12(build, cases, res): +def run_d3d12(build, cases, res, ddir=None): """GPU: every prepared case on the native D3D12 device (kernel_replay, a child process), output buffers filled with - the D3D12 sentinel 0x5A; raw bytes against the HIP run's and the oracle's, and the half-grid control.""" + the D3D12 sentinel 0x5A; raw bytes against the HIP run's and the oracle's, and the half-grid control. The replay + folder is made in ddir (the cases file's folder: the private one for real weights).""" import subprocess import time - cdir = os.path.join(build, "replay-q7-" + time.strftime("%Y%m%d-%H%M%S")) + cdir = os.path.join(ddir or build, "replay-q7-" + time.strftime("%Y%m%d-%H%M%S")) os.makedirs(cdir) with open(os.path.join(cdir, "tab.bin"), "wb") as f: f.write(LP.q7_tables()) @@ -414,9 +438,18 @@ def main(argv=None): g.add_argument("--run", action="store_true") ap.add_argument("--d3d12", action="store_true", help="with --run: also the D3D12 ports through kernel_replay") ap.add_argument("--real") + ap.add_argument("--private-out", help="--real: the folder (outside the repository) for the cases file and the " + "replay folder; the same one for --run") ap.add_argument("--form", choices=sorted(FORM_CO), default="q7") ap.add_argument("--json") a = ap.parse_args(argv) + if a.real and not a.prepare: + ap.error("--real: with --prepare (give --run the same --private-out)") + if a.real and not a.private_out: + raise SystemExit("--real is required to have --private-out : the cases file " + "holds real weights and the oracle's bytes on them") + if a.private_out and (a.real or a.run): + data_dir(a.build, a.private_out) # refused inside the repository, before anything is read if a.compile: compile_kernel(a.build, a.rocm) return 0 @@ -426,13 +459,13 @@ def main(argv=None): if a.prepare: if a.form == "hybmut": ap.error("prepare with --form hyb (its cases serve hybmut too)") - print("cases", prepare(a.build, a.real, a.form)) + print("cases", prepare(data_dir(a.build, a.private_out) if a.real else a.build, a.real, a.form)) return 0 from clean.runner.hip import clean_process_environment clean_process_environment(a.rocm) if a.form != "q7" and a.d3d12: ap.error("--form hyb / hybmut: HIP only") - res = run(a.build, a.rocm, a.d3d12, a.form) + res = run(a.build, a.rocm, a.d3d12, a.form, a.private_out) print("form", a.form, "| HIP cases identical", res["cases_identical"], "of", len(res["cases"]), "| negative control caught", res["negative_control"]["half_grid_differs"], "| differing from exact oracle", res.get("cases_differing_from_exact_oracle"), diff --git a/clean/l/runner/l_q7w_check.py b/clean/l/runner/l_q7w_check.py index fea42cc7..8f3e2aba 100644 --- a/clean/l/runner/l_q7w_check.py +++ b/clean/l/runner/l_q7w_check.py @@ -4,7 +4,8 @@ "exact"), raw bytes. python clean/l/runner/l_q7w_check.py --build --compile # CPU: /l_q7w.co - python clean/l/runner/l_q7w_check.py --build --prepare [--real ] # CPU + python clean/l/runner/l_q7w_check.py --build --prepare [--real + --private-out ] # CPU python clean/l/runner/l_q7w_check.py --build --prepare-d3d12 # CPU: D3D12 ports + kernel_replay.exe python clean/l/runner/l_q7w_check.py --build --run [--d3d12] --json out.json # GPU (inside a GPU slot) @@ -16,6 +17,8 @@ (+-448, zeros, a NaN code). Every output buffer is filled with the back end's sentinel (HIP 0xA5, D3D12 0x5A) and has 64 slack bytes after it that must keep it; the output must equal the oracle's bytes exactly. Negative control per kernel: its first case with 4 or more workgroups launched with half of them must differ. Counts and digests only. +--real (the synthetic-values rule): the cases file and the D3D12 replay folder go only to --private-out, a folder +outside the repository (refused inside it); give the same --private-out to --run. """ import argparse import ctypes @@ -34,7 +37,7 @@ from clean.l.reference import table as T # noqa: E402 from clean.l.runner import l_pack as LP # noqa: E402 from clean.l.runner import l_record as LR # noqa: E402 -from clean.l.runner.l_q7_check import mask_control, mask_nan, out8, run_identity # noqa: E402 +from clean.l.runner.l_q7_check import data_dir, load_cases, mask_control, mask_nan, out8, run_identity # noqa: E402 from clean.reference import swin as MS # noqa: E402 from clean.reference.fp8 import f16, fp8_encode, q8 # noqa: E402 @@ -197,17 +200,18 @@ def _check(got, case, sentinel): "sha256": hashlib.sha256(body).hexdigest()} -def run(build, rocm, d3d12=False): +def run(build, rocm, d3d12=False, private_out=None): from clean.runner.hip import Hip, clean_process_environment + ddir = data_dir(build, private_out) + prep = load_cases(os.path.join(ddir, "l_q7w_cases.pkl"), private_out) clean_process_environment(rocm) - with open(os.path.join(build, "l_q7w_cases.pkl"), "rb") as f: - prep = pickle.load(f) cases = prep["cases"] h = Hip(rocm) with open(os.path.join(build, CO), "rb") as f: mod = h.load_module(f.read()) res = {"what": "the shared stages' FP16-form kernels against the oracle, raw bytes", "fp8_op_model": "exact", - "identity": run_identity(build, [CO, "l_q7w_cases.pkl"]), "real_params_sha256": prep["real_params_sha256"], + "identity": run_identity(build, [CO, os.path.join(ddir, "l_q7w_cases.pkl") if private_out + else "l_q7w_cases.pkl"]), "real_params_sha256": prep["real_params_sha256"], "nan_sign_masked": "outputs compared and hashed with only a NaN's sign bit cleared (accepted port exception; " "a canonical NaN store costs about 3 VALU per stored dword in the RGA ISA)", "nan_mask_control": mask_control(), @@ -236,7 +240,7 @@ def run(build, rocm, d3d12=False): res["pass"] = (res["cases_identical"] == len(cases) and set(res["negative_controls"]) == set(KERNELS) and all(v["caught"] for v in res["negative_controls"].values()) and res["nan_mask_control"]["pass"]) if d3d12: - res["d3d12"] = run_d3d12(build, cases, res, negs) + res["d3d12"] = run_d3d12(build, cases, res, negs, ddir) res["pass"] = res["pass"] and res["d3d12"]["pass"] return res @@ -246,10 +250,10 @@ def prepare_d3d12(build, rocm): return prepare_replay(build, rocm, list(KERNELS)) -def run_d3d12(build, cases, res, negs): +def run_d3d12(build, cases, res, negs, ddir=None): import subprocess import time - cdir = os.path.join(build, "replay-q7w-" + time.strftime("%Y%m%d-%H%M%S")) + cdir = os.path.join(ddir or build, "replay-q7w-" + time.strftime("%Y%m%d-%H%M%S")) # the cases file's folder os.makedirs(cdir) files, lines = {}, [] @@ -319,18 +323,27 @@ def main(argv=None): g.add_argument("--run", action="store_true") ap.add_argument("--d3d12", action="store_true") ap.add_argument("--real") + ap.add_argument("--private-out", help="--real: the folder (outside the repository) for the cases file and the " + "replay folder; the same one for --run") ap.add_argument("--json") a = ap.parse_args(argv) + if a.real and not a.prepare: + ap.error("--real: with --prepare (give --run the same --private-out)") + if a.real and not a.private_out: + raise SystemExit("--real is required to have --private-out : the cases file " + "holds real weights and the oracle's bytes on them") + if a.private_out and (a.real or a.run): + data_dir(a.build, a.private_out) # refused inside the repository, before anything is read if a.compile: compile_kernel(a.build, a.rocm) return 0 if a.prepare: - print("cases", prepare(a.build, a.real)) + print("cases", prepare(data_dir(a.build, a.private_out) if a.real else a.build, a.real)) return 0 if a.prepare_d3d12: print("built", prepare_d3d12(a.build, a.rocm)) return 0 - res = run(a.build, a.rocm, a.d3d12) + res = run(a.build, a.rocm, a.d3d12, a.private_out) for k, v in res["per_kernel"].items(): print(f"{k:22s} HIP identical {v['identical']} of {v['cases']} negative control caught " f"{res['negative_controls'].get(k, {}).get('caught')}") diff --git a/clean/l/tests/test_l_private_out.py b/clean/l/tests/test_l_private_out.py new file mode 100644 index 00000000..28b0a25f --- /dev/null +++ b/clean/l/tests/test_l_private_out.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: MIT +"""The synthetic-values rule for the Model L checkers' real modes: l_hyb_stage_check --real-inputs, l_q7_check --real +and l_q7w_check --real write their real-data intermediates (case files, replay inputs and outputs) only to --private-out, +a resolved folder outside the repository. Each real-input path refuses a missing folder and a folder inside the +repository before it reads or writes anything; a cases file prepared with --real is read only from such a folder. +No GPU, no model, no recorded data (every value here is constructed; the paths given as real inputs do not exist). +Run: python clean/l/tests/test_l_private_out.py +""" +import os +import pickle +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.runner import l_hyb_stage_check as HS # noqa: E402 +from clean.l.runner import l_private as P # noqa: E402 +from clean.l.runner import l_q7_check as Q7 # noqa: E402 +from clean.l.runner import l_q7w_check as Q7W # noqa: E402 + +IN_REPO = (ROOT, os.path.join(ROOT, "clean", "l", "records"), os.path.join(ROOT, "out", "real-run")) + + +def outside_tmp(test): + d = tempfile.mkdtemp(prefix="wg_priv_") + test.addCleanup(lambda: __import__("shutil").rmtree(d, ignore_errors=True)) + if P.inside_repository(d): + test.skipTest("the temporary folder is inside the repository on this machine") + return d + + +def listing(d): + return sorted(os.path.relpath(os.path.join(r, f), d) for r, _, fs in os.walk(d) for f in fs) + \ + sorted(os.path.relpath(os.path.join(r, x), d) for r, xs, _ in os.walk(d) for x in xs) + + +class Helper(unittest.TestCase): + def test_inside_is_refused_outside_is_taken(self): + for p in IN_REPO: + with self.subTest(p=p), self.assertRaises(SystemExit) as cm: + P.private_dir(p) + self.assertIn("inside the repository", str(cm.exception)) + with self.assertRaises(SystemExit): + P.private_dir(None) + d = outside_tmp(self) + self.assertEqual(P.private_dir(os.path.join(d, "x")), os.path.realpath(os.path.join(d, "x"))) + + +class HybStageRealInputs(unittest.TestCase): + def test_refused_before_anything_is_written(self): + d = outside_tmp(self) + build = os.path.join(d, "build") + os.makedirs(build) + nothing = os.path.join(d, "no-real-inputs") # never read: the refusal comes first + cases = ([], ["--private-out", os.path.join(ROOT, "out", "real-run")], + ["--private-out", os.path.join(ROOT, "clean")]) + for extra in cases: + for mode in (["--run"], ["--run", "--replay-only"], ["--run", "--no-replay"]): + argv = ["--build", build, "--real-inputs", nothing] + mode + extra + with self.subTest(argv=argv[4:]), self.assertRaises(SystemExit): + HS.main(argv) + self.assertEqual(listing(build), []) + self.assertFalse(os.path.exists(os.path.join(ROOT, "out", "real-run"))) + + def test_private_out_alone_is_refused(self): + d = outside_tmp(self) + with self.assertRaises(SystemExit): + HS.main(["--build", d, "--run", "--private-out", os.path.join(d, "p")]) + + +class Q7Real(unittest.TestCase): + MODS = (("l_q7_check", Q7, "l_q7_cases.pkl"), ("l_q7w_check", Q7W, "l_q7w_cases.pkl")) + + def test_prepare_real_needs_a_private_folder_outside(self): + for name, mod, _ in self.MODS: + d = outside_tmp(self) + build = os.path.join(d, "build") + os.makedirs(build) + real = os.path.join(d, "no-real-build") # never read: the refusal comes first + for extra in ([], ["--private-out", os.path.join(ROOT, "out", "real-run")], + ["--private-out", os.path.join(ROOT, "clean", "l")]): + argv = ["--build", build, "--prepare", "--real", real] + extra + with self.subTest(checker=name, argv=argv[4:]), self.assertRaises(SystemExit): + mod.main(argv) + self.assertEqual(listing(build), []) + self.assertFalse(os.path.exists(os.path.join(ROOT, "out", "real-run"))) + + def test_run_with_a_private_folder_inside_is_refused(self): + for name, mod, _ in self.MODS: + d = outside_tmp(self) + with self.subTest(checker=name), self.assertRaises(SystemExit): + mod.main(["--build", d, "--run", "--d3d12", "--private-out", os.path.join(ROOT, "out", "real-run")]) + self.assertFalse(os.path.exists(os.path.join(ROOT, "out", "real-run"))) + + def test_a_real_cases_file_is_read_only_from_its_private_folder(self): + """A cases file marked as prepared with --real (constructed: no cases, a made-up digest) in the build folder + is refused by --run without --private-out, before any GPU work.""" + for name, mod, fn in self.MODS: + d = outside_tmp(self) + p = os.path.join(d, fn) + with open(p, "wb") as f: + pickle.dump({"cases": [], "real_params_sha256": "0" * 64, "form": "q7"}, f) + with self.subTest(checker=name), self.assertRaises(SystemExit) as cm: + mod.main(["--build", d, "--run"]) + self.assertIn("--private-out", str(cm.exception)) + + def test_real_only_with_prepare(self): + for name, mod, _ in self.MODS: + d = outside_tmp(self) + with self.subTest(checker=name), self.assertRaises(SystemExit): + mod.main(["--build", d, "--run", "--real", os.path.join(d, "r"), "--private-out", os.path.join(d, "p")]) + + def test_the_private_folder_holds_the_cases_file(self): + """data_dir resolves the private folder (outside the repository); load_cases reads a real file from it.""" + d = outside_tmp(self) + priv = Q7.data_dir(os.path.join(d, "build"), os.path.join(d, "priv")) + self.assertEqual(priv, os.path.realpath(os.path.join(d, "priv"))) + p = os.path.join(priv, "l_q7_cases.pkl") + with open(p, "wb") as f: + pickle.dump({"cases": [], "real_params_sha256": "0" * 64, "form": "q7"}, f) + self.assertEqual(Q7.load_cases(p, priv)["cases"], []) + self.assertEqual(Q7.data_dir(os.path.join(d, "build"), None), os.path.join(d, "build")) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/runtime/device_d3d12.cpp b/clean/runtime/device_d3d12.cpp index 8a79d941..9ba27176 100644 --- a/clean/runtime/device_d3d12.cpp +++ b/clean/runtime/device_d3d12.cpp @@ -461,6 +461,8 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (!finish_queue() || !create_direct_objects() || !create_marker()) return extension_result(false, error); ext_list_ = list; external_ = true; guard_state_ = D3D12_RESOURCE_STATE_COMMON; + // the caller's predicate lifted: our recording runs unconditionally (set again at end_external) + if (caller_given_ && caller_pred_) ext_list_->SetPredication(nullptr, 0, D3D12_PREDICATION_OP_EQUAL_ZERO); // The arena is a buffer: it starts each ExecuteCommandLists in COMMON and is promoted implicitly to UNORDERED_ACCESS // by the first dispatch; only dispatches touch it while recording into the caller's list. arena_state_ = D3D12_RESOURCE_STATE_COMMON; @@ -482,6 +484,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { ok = fail("native D3D12 end_external: the caller's list has no ID3D12GraphicsCommandList2 (completion marker)"); } ok = ok && guard_lifted; + caller_given_ = false; caller_pred_ = nullptr; // (given again before the next recording) ext_list_ = nullptr; external_ = false; arena_state_ = D3D12_RESOURCE_STATE_COMMON; native_binding_dirty_ = true; direct_bound_ = false; @@ -668,7 +671,8 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (inside_pass_) return fail("native D3D12 guard_begin: not from a record_pass callback"); if ((word & 7u) || !range(word, 16, "guard word")) return fail("native D3D12 guard_begin: invalid guard word"); if (!guard_buffers() || !open_list()) return false; - if (guard_on_ || guard_held_) cl()->SetPredication(nullptr, 0, D3D12_PREDICATION_OP_EQUAL_ZERO); + // ours, or a caller's that was not given (set_caller_predicate): lifted for the copy below, never skipped + if (guard_on_ || guard_held_ || external_) cl()->SetPredication(nullptr, 0, D3D12_PREDICATION_OP_EQUAL_ZERO); guard_held_ = false; // the word as the stream has it here: earlier UAV writes complete (the arena transition), then 8 bytes copied need_uav_barrier_ = true; uav_barrier(); @@ -709,8 +713,17 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { guard_transition(D3D12_RESOURCE_STATE_COMMON); guard_held_ = false; } + // the caller's predicate as it was before our recording (its conditional work after ours stays conditional) + if (external_ && caller_given_ && caller_pred_) cl()->SetPredication(caller_pred_, caller_pred_offset_, caller_pred_op_); return ok; } + void set_caller_predicate(ID3D12Resource* buffer, uint64_t offset, D3D12_PREDICATION_OP op) override { + caller_given_ = true; + caller_pred_ = buffer; + caller_pred_offset_ = offset; + caller_pred_op_ = op; + } + ID3D12Resource* guard_held_predicate() const override { return guard_held_ ? guard_pred_.get() : nullptr; } bool guard_read(uint32_t w[4]) override { for (int i = 0; i < 4; ++i) w[i] = guard_map_ ? guard_map_[i] : 0u; return guard_map_ != nullptr; @@ -1842,6 +1855,11 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { // a caller-list recording keeps the predicate after guard_end (the executor's output pass after the network, e.g. a // proxy copy into the game's output, is skipped with the frame) until end_external lifts it bool guard_held_ = false, guard_ending_ = false; + // the caller's predicate for the next / current caller-list recording (set_caller_predicate) + bool caller_given_ = false; + ID3D12Resource* caller_pred_ = nullptr; + uint64_t caller_pred_offset_ = 0; + D3D12_PREDICATION_OP caller_pred_op_ = D3D12_PREDICATION_OP_EQUAL_ZERO; DevPtr guard_word_ = 0; NativeCom query_heap_; PFN_D3D12_SERIALIZE_ROOT_SIGNATURE serialize_ = nullptr; diff --git a/clean/runtime/device_d3d12.h b/clean/runtime/device_d3d12.h index 41477db3..4f70627c 100644 --- a/clean/runtime/device_d3d12.h +++ b/clean/runtime/device_d3d12.h @@ -87,6 +87,17 @@ class NativeD3D12Ext { // completion marker (ID3D12GraphicsCommandList2::WriteBufferImmediate) behind the recorded work. virtual bool begin_external(ID3D12GraphicsCommandList* list, std::string* err) = 0; virtual bool end_external(std::string* err) = 0; + // The caller's predicate on that list before our recording (ID3D12GraphicsCommandList::SetPredication as the front + // end tracked it; buffer null: none), given before begin_external for a recording that uses the frame guard + // (Device::guard_begin, Model L hybrid), whose predication replaces the caller's on that list: begin_external lifts + // it (the whole recording runs unconditionally) and end_external sets it again exactly as given, so the caller's + // conditional work after ours stays conditional. Not given: the predication is left as the recording leaves it. + virtual void set_caller_predicate(ID3D12Resource* buffer, uint64_t offset, D3D12_PREDICATION_OP op) = 0; + // The frame guard's predicate buffer while it is held on the caller's list (after the network's frame, until + // end_external), else null: the executor records its output-pass fallback under the INVERSE predication on it + // (D3D12_PREDICATION_OP_EQUAL_ZERO: runs only when the frame was skipped) and sets the guard's + // (D3D12_PREDICATION_OP_NOT_EQUAL_ZERO) again after it. Only from within a record_pass callback of that recording. + virtual ID3D12Resource* guard_held_predicate() const = 0; // Frames recorded so far / known complete (the marker, read without waiting). Without ID3D12GraphicsCommandList2 // there is no marker and every recorded frame counts as complete (external_markers() false). virtual uint64_t external_recorded() const = 0; diff --git a/clean/runtime/interop/backend.cpp b/clean/runtime/interop/backend.cpp index dae5f604..cd9737c2 100644 --- a/clean/runtime/interop/backend.cpp +++ b/clean/runtime/interop/backend.cpp @@ -1117,6 +1117,19 @@ class Context { job.game = &f; job.output_copy = output_copy_; job.conv = &conv_; + // the game's predicate on its list (the executor's frame guard replaces it while it records, then sets it again) + if (services_ && services_->struct_size >= offsetof(WgFrontEndServices, command_list_predication) + sizeof(void*) && + services_->command_list_predication) { + void* pb = nullptr; + uint64_t po = 0; + uint32_t pop = 0; + if (services_->command_list_predication(services_->context, &f, &pb, &po, &pop) == WG_OK) { + job.caller_pred_known = true; + job.caller_pred = pb; + job.caller_pred_offset = po; + job.caller_pred_op = pop; + } + } if (timing_) timing_->stamp(list, v, 0); if (timing_) timing_->stamp(list, v, 1); bool wrote = false; @@ -1136,13 +1149,14 @@ class Context { } return WG_ERROR_UNSUPPORTED; // one frame's answer: the context stays usable (failed_ not set) } - // Model L hybrid, the frame after a bottleneck wait timeout: "refused, keep previous output" (executor_select.h - // frame_timeout_refused). One line, no fallback written into the game's output, WG_OK, the context kept; the next - // frame starts over. + // Model L hybrid, the frame after a bottleneck wait timeout (executor_select.h frame_timeout_refused): nothing of + // it written, the path of a frame not taken: one line, WG_ERROR_UNSUPPORTED (the front end's stand-in renders it), + // the context kept; the next frame starts over. if (!ok && frame_timeout_refused(err)) { - say(WG_LOG_WARN, "frame " + std::to_string(v) + ": " + err + "; the game keeps its previous output for this frame"); + say(WG_LOG_WARN, "frame " + std::to_string(v) + ": " + err + "; this frame goes back to the front end (AMD FSR, " + "else its stand-in), the context stays"); force_reset_ = true; - return WG_OK; + return WG_ERROR_UNSUPPORTED; // one frame's answer: the context stays usable (failed_ not set) } note_path(&input_path_, "input: read directly by the network's D3D12 kernels (executor dx12, no input pass)"); if ((flags_ & WG_CREATE_MOTION_OUTPUT_RES) && model_k_) diff --git a/clean/runtime/interop/convert.cpp b/clean/runtime/interop/convert.cpp index e4f63a88..7b57250a 100644 --- a/clean/runtime/interop/convert.cpp +++ b/clean/runtime/interop/convert.cpp @@ -540,6 +540,17 @@ void Converter::record_inputs_pass(ID3D12GraphicsCommandList* list, const WgFram } void Converter::record_output(ID3D12GraphicsCommandList* list, const WgFrame& f, const BufferRange& range) { + output_pass(list, f, range, static_cast(f.output.resource), (DXGI_FORMAT)f.output.format, + (D3D12_RESOURCE_STATES)f.output.state); +} + +void Converter::record_fallback_into(ID3D12GraphicsCommandList* list, const WgFrame& f, ID3D12Resource* dst, + DXGI_FORMAT fmt) { + output_pass(list, f, BufferRange(), dst, fmt, D3D12_RESOURCE_STATE_UNORDERED_ACCESS); +} + +void Converter::output_pass(ID3D12GraphicsCommandList* list, const WgFrame& f, const BufferRange& range, + ID3D12Resource* out, DXGI_FORMAT ofmt, D3D12_RESOURCE_STATES ostate) { ID3D12Resource* const result = range.resource; const UINT set = next_set(); D3D12_CPU_DESCRIPTOR_HANDLE cpu = heap_->GetCPUDescriptorHandleForHeapStart(); @@ -547,8 +558,6 @@ void Converter::record_output(ID3D12GraphicsCommandList* list, const WgFrame& f, D3D12_CPU_DESCRIPTOR_HANDLE t0 = cpu, u0 = cpu, u1 = cpu; u0.ptr += (SIZE_T)4 * increment_; u1.ptr += (SIZE_T)5 * increment_; - ID3D12Resource* out = static_cast(f.output.resource); - const DXGI_FORMAT ofmt = (DXGI_FORMAT)f.output.format; // bug hunt D: never an unordered access view of an output without unordered access (the backend refuses such an output // unless the RGBA16F copy route writes it; a fallback frame of that route leaves the output as it is) if (!out || !(out->GetDesc().Flags & D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS)) return; @@ -582,7 +591,7 @@ void Converter::record_output(ID3D12GraphicsCommandList* list, const WgFrame& f, c.srgb = is_srgb(ofmt) ? 1 : 0; c.flags = model_k_ ? kOutputAlpha : 0; Barriers bar; - bar.need(out, f.output.state, D3D12_RESOURCE_STATE_UNORDERED_ACCESS); + bar.need(out, ostate, D3D12_RESOURCE_STATE_UNORDERED_ACCESS); if (!result) bar.need(static_cast(f.color.resource), f.color.state, D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE); diff --git a/clean/runtime/interop/convert.h b/clean/runtime/interop/convert.h index 52a173d1..f002dfdb 100644 --- a/clean/runtime/interop/convert.h +++ b/clean/runtime/interop/convert.h @@ -70,6 +70,10 @@ class Converter { // Records the output write: from the result buffer (R32_FLOAT elements, 3 per pixel; model K: 4 per pixel, RGBA) or, // when result is null, the bilinear fallback from the game's color texture (result.resource null). void record_output(ID3D12GraphicsCommandList* list, const WgFrame& f, const BufferRange& result); + // That bilinear fallback into `dst` instead of the game's output (executor=dx12, an output route through a texture + // of ours): `dst` has unordered access, is in UNORDERED_ACCESS before and after, and is viewed as `fmt` (a linear + // float format: no sRGB encoding here, the route's own pass after it does that). + void record_fallback_into(ID3D12GraphicsCommandList* list, const WgFrame& f, ID3D12Resource* dst, DXGI_FORMAT fmt); // The output pass from an RGBA32F texture holding the result (executor=dx12; the texture in UNORDERED_ACCESS before // and after): the operations of record_output's pass on the same FP32 values. False (*err) if its shader fails. bool record_output_tex(ID3D12GraphicsCommandList* list, const WgFrame& f, ID3D12Resource* proxy, std::string* err); @@ -107,6 +111,9 @@ class Converter { uint32_t result_pixel_bytes() const { return model_k_ ? 16u : 12u; } private: + // record_output / record_fallback_into: the output pass into `out` (in `ostate` before and after), viewed as `ofmt` + void output_pass(ID3D12GraphicsCommandList* list, const WgFrame& f, const BufferRange& result, ID3D12Resource* out, + DXGI_FORMAT ofmt, D3D12_RESOURCE_STATES ostate); bool model_k_ = false; struct Constants { uint32_t rx, ry, rw, rh; diff --git a/clean/runtime/interop/executor.h b/clean/runtime/interop/executor.h index c9a0d881..3c585368 100644 --- a/clean/runtime/interop/executor.h +++ b/clean/runtime/interop/executor.h @@ -114,6 +114,14 @@ struct Job { // output pass for formats the network's post kernel cannot store). bool output_copy = true; class Converter* conv = nullptr; + // executor=dx12: the game's predicate on its list before this frame (WgFrontEndServices::command_list_predication; + // caller_pred_known false: the front end could not tell). A frame under the device's frame guard (Model L hybrid) + // replaces it while it records and sets it again after (NativeD3D12Ext::set_caller_predicate); such a frame is not + // taken when it is not known. + bool caller_pred_known = false; + void* caller_pred = nullptr; // ID3D12Resource*, null: none set + uint64_t caller_pred_offset = 0; + uint32_t caller_pred_op = 0; // D3D12_PREDICATION_OP }; // Motion at output resolution (model M, both executors): motion_scale converts the game's raw motion to render pixels diff --git a/clean/runtime/interop/executor_d3d12.cpp b/clean/runtime/interop/executor_d3d12.cpp index 0deb05d4..637e68e0 100644 --- a/clean/runtime/interop/executor_d3d12.cpp +++ b/clean/runtime/interop/executor_d3d12.cpp @@ -299,6 +299,17 @@ class D3d12Executor : public Executor { df.params.padding[4] = (Converter::motion_fp16(f) ? 1u : 0u) | ((cw - 1u) << 1) | ((ch - 1u) << 16); } + // Model L hybrid: the device's frame guard predicates this frame on the game's list, in place of the game's own + // predicate, which is lifted while we record and set again after (its conditional work after ours stays + // conditional); a frame whose game predicate the front end cannot tell is not taken. + const bool guarded = wgrt::runtime_frame_guard(rt_); + if (guarded && !job.caller_pred_known) + return *err = "frame not taken (nothing recorded): the front end does not know the predicate set on the game's " + "command list, which the Model L hybrid frame guard would replace", false; + if (guarded) + ext_->set_caller_predicate(static_cast(job.caller_pred), job.caller_pred_offset, + (D3D12_PREDICATION_OP)job.caller_pred_op); + std::string e; if (!ext_->begin_external(list, &e)) return *err = "D3D12 executor: " + e, false; bool ok = ext_->bind_direct_frame(df, &e); @@ -351,6 +362,26 @@ class D3d12Executor : public Executor { if (!ok) return true; if (route == kProxy16) copy16(l, f); if (route == kProxy32 && job.conv && !job.conv->record_output_tex(l, f, proxy32_, &m)) return false; + // The frame guard's fallback: everything above (the network, and this output pass) is skipped on the GPU + // when the frame's bottleneck wait timed out or an earlier one did (the abort word set); under the INVERSE + // predication on the same word, a simple upscale of this frame's colour goes into this frame's output by the + // same route instead. No CPU wait; when the word is clear, these commands are skipped (barriers still run). + if (ID3D12Resource* gp = ext_->guard_held_predicate()) { + if (!job.conv) return m = "no Converter for the frame guard's fallback", false; + l->SetPredication(gp, 0, D3D12_PREDICATION_OP_EQUAL_ZERO); + bool fb = true; + if (route == kDirect) job.conv->record_output(l, f, BufferRange()); + if (route == kProxy16) { + job.conv->record_fallback_into(l, f, proxy16_, DXGI_FORMAT_R16G16B16A16_FLOAT); + copy16(l, f); + } + if (route == kProxy32) { + job.conv->record_fallback_into(l, f, proxy32_, DXGI_FORMAT_R32G32B32A32_FLOAT); + fb = job.conv->record_output_tex(l, f, proxy32_, &m); + } + l->SetPredication(gp, 0, D3D12_PREDICATION_OP_NOT_EQUAL_ZERO); // the guard's own, held until end_external + if (!fb) return false; + } return true; }, &e2, "dx12_output"); diff --git a/clean/runtime/interop/executor_select.h b/clean/runtime/interop/executor_select.h index e20cd8f3..7c17db01 100644 --- a/clean/runtime/interop/executor_select.h +++ b/clean/runtime/interop/executor_select.h @@ -242,12 +242,15 @@ inline std::string drop_hip_only_filter(const std::string& options, std::string* inline bool frame_refused(const std::string& err) { return err.rfind("frame not taken", 0) == 0; } // Model L hybrid: the frame after one whose bottleneck wait timed out on the GPU (runtime.cpp kBneckTimeoutRefusal; -// hyb_bneck.hlsl). The documented result: "refused, keep previous output". The timed-out frame and the frames queued -// after it were skipped on the GPU, their output pass included (the device's frame guard predicates the network and the -// executor's output pass alike): the game's output keeps what it held. This frame records nothing that writes (only the -// game textures' state changes, restored), the backend writes no fallback into the output, logs exactly one line, -// answers WG_OK and keeps the context (never failed_); the next frame is a reset (the runtime starts over, and the -// backend asks for the reset too). +// hyb_bneck.hlsl). The timed-out frame and the frames queued after it were skipped on the GPU (the device's frame guard +// predicates the network and the executor's output pass alike), and each of them got the executor's fallback instead: +// under the INVERSE predication on the same word, a simple upscale of that frame's colour into that frame's output +// texture (Converter::record_output with no result, or through the route's texture of ours), so every such frame, +// the first one of a context included, shows an image whatever output texture the game passed. This frame (the one +// whose start sees the timeout count move) records nothing that writes (only the game textures' state changes, +// restored) and takes the path of a frame the executor did not take: the backend logs one line and answers +// WG_ERROR_UNSUPPORTED, so the front end's stand-in (AMD FSR, else its own upscale) renders it; the context is kept +// (never failed_), and the next frame is a reset (the runtime starts over, and the backend asks for the reset too). inline bool frame_timeout_refused(const std::string& err) { return err.rfind("frame refused (bottleneck wait timeout): ", 0) == 0; } diff --git a/clean/runtime/interop/tests/test_interop_d3d12.cpp b/clean/runtime/interop/tests/test_interop_d3d12.cpp index c29622d7..adb0f180 100644 --- a/clean/runtime/interop/tests/test_interop_d3d12.cpp +++ b/clean/runtime/interop/tests/test_interop_d3d12.cpp @@ -25,7 +25,11 @@ // frame: each must be taken by the D3D12 executor (no create refusal, no HIP line); see case_matrix. // test_interop_d3d12.exe bnecktimeout "" // Model L hybrid, the bottleneck wait timeout through the game path (WG_DEV_BNECK_ABORT= in the environment: the -// developer runtime makes accepted frame n's bottleneck time out); see case_bneck_timeout. Synchronous, then pipelined. +// developer runtime makes accepted frame n's bottleneck time out; 0: the very first frame); see case_bneck_timeout. +// Synchronous, pipelined, then synchronous with an output texture without unordered access. +// test_interop_d3d12.exe callerpred "" +// Model L hybrid: the game's predicate on its list set again after our frame (false and true, and not known to the +// front end); see case_caller_predicate. // test_interop_d3d12.exe hipswitch "" "" // Creates, dispatches, destroys M -> K -> M in one process and verifies the HIP DLL stays loaded between them. // test_interop_d3d12.exe hipleak "" ["" | -] [cycles] [frames] [MiB per cycle] [hold] [thread] @@ -321,6 +325,21 @@ struct FrontEnd { uint64_t resumeValue = 0; int splits = 0; WgResult answer = WG_OK; // what split_command_list returns (WG_ERROR_NOT_READY to test that path) + // command_list_predication: what the front end tracked on the game's list (the test sets it as it records) + WgResult predAnswer = WG_OK; + ID3D12Resource* predBuffer = nullptr; + uint64_t predOffset = 0; + uint32_t predOp = 0; + int predQueries = 0; + static WgResult predication(void* ctx, const WgFrame*, void** b, uint64_t* o, uint32_t* op) { + FrontEnd* fe = (FrontEnd*)ctx; + fe->predQueries++; + if (!b || !o || !op) return WG_ERROR_PARAMETER; + *b = nullptr, *o = 0, *op = 0; + if (fe->predAnswer != WG_OK) return fe->predAnswer; + *b = fe->predBuffer, *o = fe->predOffset, *op = fe->predOp; + return WG_OK; + } static WgResult split(void* ctx, const WgFrame* f, const WgSplitRequest* r) { FrontEnd* fe = (FrontEnd*)ctx; @@ -542,6 +561,7 @@ struct Ctx { services.struct_size = sizeof services; services.context = &fe; services.split_command_list = &FrontEnd::split; + services.command_list_predication = &FrontEnd::predication; WgCreateInfo ci = {}; ci.struct_size = sizeof ci; ci.interface_version = WG_UPSCALER_INTERFACE_VERSION; @@ -2531,28 +2551,154 @@ bool case_async_sim(Gpu& g, const char* dll, const std::string& common, const st return ok; } +// Model L hybrid scene for the frame guard's cases: 1707 x 960 -> 2560 x 1440 (apply_test_dimensions may change it). +static void guard_scene(Gpu& g, Scene& s, bool proxy) { + s.tw = 1707, s.th = 960, s.rx = 0, s.ry = 0, s.rw = 1707, s.rh = 960, s.ow = 2560, s.oh = 1440, s.mw = 1707, + s.mh = 960; + apply_test_dimensions(&s); + s.outUav = !proxy; // proxy: an RGBA16F output without unordered access (the executor's RGBA16F texture + copy) + if (proxy) // (a shader-resource state: no render-target flag on this texture) + s.outState = test_compute_list() ? D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE + : D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE; + s.make(g); + s.set_color(g, 1); +} + +// A texture of the output's shape filled with -7 (FP16) in every channel, in COPY_SOURCE: copied into a frame's output +// texture before the dispatch, it shows what nothing of ours wrote. +static ID3D12Resource* sentinel_texture(Gpu& g, const Scene& s) { + ID3D12Resource* t = g.texture(s.ow, s.oh, s.outRes, false, D3D12_RESOURCE_STATE_COPY_DEST); + std::vector v((size_t)s.ow * s.oh * 4, to_half(-7.0f)); + g.upload(t, s.ow, s.oh, 8, v.data(), D3D12_RESOURCE_STATE_COPY_SOURCE); + return t; +} +static void record_sentinel(Gpu& g, const Scene& s, ID3D12Resource* out, ID3D12Resource* sentinel) { + g.barrier(out, s.outState, D3D12_RESOURCE_STATE_COPY_DEST); + g.list->CopyResource(out, sentinel); + g.barrier(out, D3D12_RESOURCE_STATE_COPY_DEST, s.outState); +} +static bool all_sentinel(const std::vector& o) { + for (float v : o) + if (v != -7.0f) return false; + return !o.empty(); +} +static size_t count_sentinel(const std::vector& o) { + size_t n = 0; + for (float v : o) n += v == -7.0f; + return n; +} +// The executor's fallback for a skipped frame (convert.cpp PASS_FALLBACK): the bilinear upscale of frame t's colour. +static float fallback_err(const Scene& s, const std::vector& o, int t) { + return max_err(o, [&](UINT x, UINT y, int k) { + const double fx = (x + 0.5) * s.rw / s.ow - 0.5, fy = (y + 0.5) * s.rh / s.oh - 0.5; + const int ax = (int)std::floor(fx), ay = (int)std::floor(fy); + const double wx = fx - ax, wy = fy - ay; + auto at = [&](int px, int py) { + px = std::min(std::max(px, 0), (int)s.rw - 1); + py = std::min(std::max(py, 0), (int)s.rh - 1); + return (double)Scene::cval(t, s.rx + px, s.ry + py, k); + }; + const double top = at(ax, ay) * (1 - wx) + at(ax + 1, ay) * wx; + const double bot = at(ax, ay + 1) * (1 - wx) + at(ax + 1, ay + 1) * wx; + return (float)(top * (1 - wy) + bot * wy); + }, s.ow, s.oh); +} + +// Frame t of the frame guard's cases: cyclic jitter, no motion, the given output texture. +static WgFrame guard_frame(Gpu& g, const Scene& s, int t, ID3D12Resource* out) { + static const float jit[4][2] = {{0, -1.0f / 6}, {-0.25f, 1.0f / 6}, {0.25f, -7.0f / 18}, {-0.375f, -1.0f / 18}}; + WgFrame f = s.frame(g, t); + f.motion_scale[0] = f.motion_scale[1] = 0.0f; + f.motion_offset[0] = f.motion_offset[1] = 0.0f; + f.jitter[0] = jit[(t - 1) % 4][0]; + f.jitter[1] = jit[(t - 1) % 4][1]; + f.output.resource = out; + return f; +} + +// "bneckfresh" (a child process of case_bneck_timeout): a fresh context's first frame on frame t's inputs, its output +// written to `path` as raw floats. A child, started without WG_DEV_BNECK_ABORT: the runtime reads that hook at create +// from its own C runtime's copy of the environment, so a context made in the parent process would get the hook too +// (with the hook on frame 0, its first frame would time out). +static bool case_bneck_fresh(Gpu& g, const char* dll, const std::string& options, int t, bool proxy, const char* path) { + Scene s; + guard_scene(g, s, proxy); + s.set_color(g, t); + Ctx d; + d.g = &g; + if (!d.be.load(dll)) return false; + CHECK(d.create(s, WG_CREATE_HDR | WG_CREATE_DEPTH_INVERTED, options.c_str()) == WG_OK); + if (!d.u) return false; + WgResult r = WG_ERROR; + const std::vector o = d.run(s, guard_frame(g, s, t, s.out), &r); + CHECK(r == WG_OK); + FILE* f = std::fopen(path, "wb"); + if (!f) return false; + const bool ok = std::fwrite(o.data(), sizeof(float), o.size(), f) == o.size(); + std::fclose(f); + return ok && r == WG_OK; +} + +static bool fresh_in_child(const char* dll, const std::string& options, int t, bool proxy, std::vector* out) { + char exe[MAX_PATH] = {}, tmp[MAX_PATH] = {}; + GetModuleFileNameA(nullptr, exe, MAX_PATH); + GetTempPathA(MAX_PATH, tmp); + const std::string path = std::string(tmp) + "wg_bneck_fresh_" + std::to_string(GetCurrentProcessId()) + ".bin"; + std::string cmd = std::string("\"") + exe + "\" \"" + dll + "\" bneckfresh \"" + options + "\" " + std::to_string(t) + + (proxy ? " 1" : " 0") + " \"" + path + "\""; + char saved[64] = {}; + const DWORD had = GetEnvironmentVariableA("WG_DEV_BNECK_ABORT", saved, sizeof saved); + SetEnvironmentVariableA("WG_DEV_BNECK_ABORT", nullptr); + STARTUPINFOA si = {}; + si.cb = sizeof si; + PROCESS_INFORMATION pi = {}; + std::vector line(cmd.begin(), cmd.end()); + line.push_back(0); + const BOOL started = CreateProcessA(nullptr, line.data(), nullptr, nullptr, FALSE, 0, nullptr, nullptr, &si, &pi); + if (had && had < sizeof saved) SetEnvironmentVariableA("WG_DEV_BNECK_ABORT", saved); + if (!started) return std::printf(" cannot start the fresh-context child\n"), false; + const DWORD w = WaitForSingleObject(pi.hProcess, 120000); + DWORD code = 1; + GetExitCodeProcess(pi.hProcess, &code); + CloseHandle(pi.hThread); + CloseHandle(pi.hProcess); + if (w != WAIT_OBJECT_0 || code != 0) return std::printf(" fresh-context child failed (%lu)\n", code), false; + FILE* f = std::fopen(path.c_str(), "rb"); + if (!f) return false; + std::fseek(f, 0, SEEK_END); + const long n = std::ftell(f); + std::fseek(f, 0, SEEK_SET); + out->assign((size_t)n / sizeof(float), 0.0f); + const bool ok = std::fread(out->data(), sizeof(float), out->size(), f) == out->size(); + std::fclose(f); + DeleteFileA(path.c_str()); + return ok; +} + // Model L hybrid: the bottleneck wait timeout through the game path (the backend's dispatch on the game's list, the -// executor's caller-list recording; developer runtime with WG_DEV_BNECK_ABORT=, read at create). Three frames: -// 1. the timed-out frame (WG_OK): the game's output keeps the previous frame's bytes (the network and the executor's -// output pass are skipped under the device's frame guard), and the game's own copy of the output after our -// dispatch in the same list still runs (the readback below is that copy: the guard is lifted at our end); -// 2. the refused frame: WG_OK ("refused, keep previous output", executor_select.h frame_timeout_refused), exactly one -// captured log line names the refusal, no failure or fallback line, the output still the previous frame's; -// 3. the reset frame (WG_OK): the output is written again, byte for byte a fresh context's first frame on the same +// executor's caller-list recording; developer runtime with WG_DEV_BNECK_ABORT=, read at create; n = 0: the very +// first frame). The game passes a different output texture on each frame (two, alternating) and copies a sentinel +// into it before our dispatch, so each frame's output shows exactly what that frame wrote: +// 1. the timed-out frame (WG_OK), and in pipelined mode every frame queued after it until the refusal: the network +// and the executor's output pass are skipped under the device's frame guard, and the fallback under the inverse +// predication writes the bilinear upscale of THAT frame's colour into THAT frame's output texture; the game's own +// copy of the output after our dispatch in the same list still runs (the readback below is that copy); +// 2. the refused frame: WG_ERROR_UNSUPPORTED (the path of a frame not taken: the front end's stand-in renders it; in +// this harness nothing does), exactly one captured log line names the refusal, no failure or fallback line, +// nothing of ours in its output (the sentinel), the context kept; +// 3. the reset frame (WG_OK): the network's output again, byte for byte a fresh context's first frame on the same // inputs (a reset reads no history), and the frames after it run. // pipelined: up to 4 frames in flight, static colour, cyclic jitter; the refusal comes once the timeout count is read -// back (1 .. 4 frames later); every frame from the timed-out one to the refused one keeps the previous output. -bool case_bneck_timeout(Gpu& g, const char* dll, const std::string& options, bool pipelined) { +// back (1 .. 4 frames later). proxy: an output without unordered access (the RGBA16F texture + copy route). +bool case_bneck_timeout(Gpu& g, const char* dll, const std::string& options, bool pipelined, bool proxy = false) { const char* ab = std::getenv("WG_DEV_BNECK_ABORT"); if (!ab) { std::printf(" WG_DEV_BNECK_ABORT is not set (a developer runtime's check)\n"); CHECK(false); return false; } const int A = std::atoi(ab) + 1; // the runtime counts accepted frames from 0, this test's frames from 1 - CHECK(A >= 3); + CHECK(A >= 1); Scene s; - s.tw = 1707, s.th = 960, s.rx = 0, s.ry = 0, s.rw = 1707, s.rh = 960, s.ow = 2560, s.oh = 1440, s.mw = 1707, - s.mh = 960; - apply_test_dimensions(&s); - s.make(g); - s.set_color(g, 1); + guard_scene(g, s, proxy); + ID3D12Resource* outs[2] = {s.out, g.texture(s.ow, s.oh, s.outRes, s.outUav, s.outState)}; + ID3D12Resource* sentinel = sentinel_texture(g, s); Ctx c; c.g = &g; if (!c.be.load(dll)) return false; @@ -2563,122 +2709,219 @@ bool case_bneck_timeout(Gpu& g, const char* dll, const std::string& options, boo std::vector> out(T + 1); std::vector res(T + 1, WG_OK); std::vector> lines(T + 1); - const float jit[4][2] = {{0, -1.0f / 6}, {-0.25f, 1.0f / 6}, {0.25f, -7.0f / 18}, {-0.375f, -1.0f / 18}}; - auto frame_of = [&](int t) { - WgFrame f = s.frame(g, t); - f.motion_scale[0] = f.motion_scale[1] = 0.0f; - f.motion_offset[0] = f.motion_offset[1] = 0.0f; - f.jitter[0] = jit[(t - 1) % 4][0]; - f.jitter[1] = jit[(t - 1) % 4][1]; - return f; - }; - if (!pipelined) { - for (int t = 1; t <= T; ++t) { + auto frame_of = [&](int t) { return guard_frame(g, s, t, outs[t % 2]); }; + std::vector rb(T + 1, nullptr); + std::vector pitch(T + 1, 0); + for (int t = 1; t <= T; ++t) { + const int k = t % 4; + if (!pipelined) { s.set_color(g, t); - const size_t before = g_log.size(); - out[t] = c.run(s, frame_of(t), &res[t]); - lines[t].assign(g_log.begin() + (long)before, g_log.end()); - } - } else { - std::vector rb(T + 1, nullptr); - std::vector pitch(T + 1, 0); - for (int t = 1; t <= T; ++t) { - const int k = t % 4; - if (!g.begin_ring(k)) { std::printf(" HANG before frame %d\n", t); CHECK(false); return false; } - WgFrame f = frame_of(t); - f.command_list = g.list; - const size_t before = g_log.size(); - res[t] = c.be.b->dispatch(c.u, &f); - lines[t].assign(g_log.begin() + (long)before, g_log.end()); - rb[t] = record_readback(g, s.out, s.ow, s.oh, s.outState, &pitch[t]); // the game's pass after ours - g.ringv[k] = c.fe.submit_nowait(); - } - g.wait_idle(); - CHECK(g.fence->GetCompletedValue() >= g.fv); - g.split_alloc = nullptr; - for (int t = 1; t <= T; ++t) { - out[t] = read_rgb(rb[t], s.ow, s.oh, pitch[t]); - rb[t]->Release(); + g.begin(); + } else if (!g.begin_ring(k)) { + std::printf(" HANG before frame %d\n", t); + CHECK(false); + return false; } + record_sentinel(g, s, outs[t % 2], sentinel); + WgFrame f = frame_of(t); + f.command_list = g.list; + const size_t before = g_log.size(); + res[t] = c.be.b->dispatch(c.u, &f); + lines[t].assign(g_log.begin() + (long)before, g_log.end()); + rb[t] = record_readback(g, outs[t % 2], s.ow, s.oh, s.outState, &pitch[t]); // the game's pass after ours + if (pipelined) g.ringv[k] = c.fe.submit_nowait(); + else c.fe.submit(); } - int R = -1, refusals = 0, bad = 0; + g.wait_idle(); + CHECK(g.fence->GetCompletedValue() >= g.fv); + g.split_alloc = nullptr; for (int t = 1; t <= T; ++t) { - CHECK(res[t] == WG_OK); + out[t] = read_rgb(rb[t], s.ow, s.oh, pitch[t]); + rb[t]->Release(); + } + int R = -1, refusals = 0, bad = 0; + for (int t = 1; t <= T; ++t) for (const std::string& m : lines[t]) { if (m.find("frame refused (bottleneck wait timeout): ") != std::string::npos) ++refusals, R = t; if (m.find("network frame failed") != std::string::npos || m.find("fallback") != std::string::npos) ++bad; } - } - std::printf(" %s: timed-out frame %d, refused frame %d, refusal lines %d, failure / fallback lines %d\n", - pipelined ? "pipelined" : "synchronous", A, R, refusals, bad); + std::printf(" %s%s: timed-out frame %d, refused frame %d, refusal lines %d, failure / fallback lines %d\n", + pipelined ? "pipelined" : "synchronous", proxy ? " (output without unordered access)" : "", A, R, + refusals, bad); CHECK(refusals == 1); CHECK(bad == 0); if (refusals != 1) return false; CHECK(pipelined ? (R > A && R <= A + 4) : R == A + 1); CHECK(R + 2 <= T); if (R + 2 > T) return false; - const std::vector& prev = out[A - 1]; - bool lit = false; - for (float v : prev) lit = lit || v != 0.0f; - CHECK(lit); // the previous output is a real image (a skipped copy would leave the fresh readback at zero) - for (int t = A; t <= R; ++t) { - const bool same = out[t] == prev; - if (!same) std::printf(" frame %d: the game's output changed under the timeout (must keep frame %d's)\n", t, A - 1); - CHECK(same); + for (int t = 1; t <= T; ++t) { + const WgResult want = t == R ? WG_ERROR_UNSUPPORTED : WG_OK; + if (res[t] != want) std::printf(" frame %d: result %d (want %d)\n", t, (int)res[t], (int)want); + CHECK(res[t] == want); + } + // the skipped frames: this frame's colour, upscaled, in this frame's output texture + for (int t = A; t < R; ++t) { + const int ct = pipelined ? 1 : t; + const float e = fallback_err(s, out[t], ct); + const size_t sn = count_sentinel(out[t]); + std::printf(" frame %d (output %d): fallback max error vs the upscale of frame %d's colour %g, sentinel values %zu\n", + t, t % 2, ct, e, sn); + CHECK(e <= kTol); + CHECK(sn == 0); + if (!pipelined && t > 1) { + const float ep = fallback_err(s, out[t], t - 1); + CHECK(ep > 10 * kTol); // not the previous frame's colour + } } - CHECK(out[R + 1] != prev); - size_t nonfinite = 0; - for (int t = R + 1; t <= T; ++t) + // the refused frame: nothing of ours written (the front end's stand-in would render it) + { + const bool none = all_sentinel(out[R]); + std::printf(" refused frame %d (output %d): %s\n", R, R % 2, + none ? "nothing written (the sentinel)" : "WRITTEN (must be nothing)"); + CHECK(none); + } + // the network's frames before and after: written, finite + size_t nonfinite = 0, sentinels = 0; + for (int t = 1; t <= T; ++t) { + if (t >= A && t <= R) continue; for (float v : out[t]) nonfinite += !std::isfinite(v); + sentinels += count_sentinel(out[t]); + } CHECK(nonfinite == 0); - if (!pipelined) { // the reset frame against a fresh context's first frame on the same inputs - Ctx d; - d.g = &g; - if (!d.be.load(dll)) return false; - CHECK(d.create(s, WG_CREATE_HDR | WG_CREATE_DEPTH_INVERTED, options.c_str()) == WG_OK); - s.set_color(g, R + 1); - WgResult r2 = WG_ERROR; - const std::vector fresh = d.run(s, frame_of(R + 1), &r2); - CHECK(r2 == WG_OK); - auto diff = [](const std::vector& x, const std::vector& y, double* worst) { - size_t n = 0; - *worst = 0.0; - for (size_t i = 0; i < x.size() && i < y.size(); ++i) - if (!(x[i] == y[i])) { - ++n; - const double d = std::fabs((double)x[i] - (double)y[i]); - *worst = d > *worst ? d : *worst; - } - return n; - }; + CHECK(sentinels == 0); + if (!pipelined) { // the reset frame against a fresh context's first frame on the same inputs (a child process) + std::vector fresh; + const bool got = fresh_in_child(dll, options, R + 1, proxy, &fresh); + CHECK(got); + CHECK(fresh.size() == out[R + 1].size()); + size_t n1 = 0; double w1 = 0.0; - const size_t n1 = diff(fresh, out[R + 1], &w1); + for (size_t i = 0; i < fresh.size() && i < out[R + 1].size(); ++i) + if (!(fresh[i] == out[R + 1][i])) { + ++n1; + w1 = std::max(w1, std::fabs((double)fresh[i] - (double)out[R + 1][i])); + } std::printf(" reset frame %d vs a fresh context's first frame: %zu of %zu values differ (max %.6g)\n", R + 1, n1, fresh.size(), w1); - // control: the same context history (frames 1 .. A - 1), then the game's own reset at frame R + 1 - { - Ctx e; - e.g = &g; - if (!e.be.load(dll)) return false; - CHECK(e.create(s, WG_CREATE_HDR | WG_CREATE_DEPTH_INVERTED, options.c_str()) == WG_OK); - std::vector oe; - for (int t = 1; t < A; ++t) { - s.set_color(g, t); - WgResult r3 = WG_ERROR; - oe = e.run(s, frame_of(t), &r3); - } - s.set_color(g, R + 1); - WgFrame fr = frame_of(R + 1); - fr.reset = 1; - WgResult r4 = WG_ERROR; - oe = e.run(s, fr, &r4); - double w2 = 0.0, w3 = 0.0; - const size_t n2 = diff(oe, fresh, &w2), n3 = diff(oe, out[R + 1], &w3); - std::printf(" control (game reset after frames 1..%d): vs fresh %zu differ (max %.6g); vs the refusal's reset %zu " - "differ (max %.6g)\n", A - 1, n2, w2, n3, w3); - } CHECK(n1 == 0); } + outs[1]->Release(); + sentinel->Release(); + return true; +} + +// Model L hybrid, the game's predicate on its list (WgFrontEndServices::command_list_predication): the frame guard +// replaces it while our frame records and sets it again after. The game sets SetPredication(buffer, 8, NOT_EQUAL_ZERO) +// before our dispatch (the value at offset 8; offset 0 holds the opposite value, so a predicate set again at another +// offset shows), and records conditional work after it: a copy of a marker into a readback buffer. +// frame 1, predicate false (0: the game's conditional work runs): our output written, the marker copied; +// frame 2, predicate true (1: skipped): our output written all the same (our frame records unconditionally), the +// marker NOT copied (the game's predicate is in effect again after our dispatch); +// frame 3, the front end cannot tell the predicate (WG_ERROR_NOT_READY): WG_ERROR_UNSUPPORTED, nothing of ours +// written, the game's predicate untouched (true: the marker not copied); +// frame 4, predicate false again: the context kept, our output written, the marker copied. +bool case_caller_predicate(Gpu& g, const char* dll, const std::string& options) { + Scene s; + guard_scene(g, s, false); + ID3D12Resource* sentinel = sentinel_texture(g, s); + ID3D12Resource* pred = g.buffer(256, D3D12_HEAP_TYPE_DEFAULT); // COPY_DEST + ID3D12Resource* marker = g.buffer(256, D3D12_HEAP_TYPE_DEFAULT); + ID3D12Resource* up = g.buffer(512, D3D12_HEAP_TYPE_UPLOAD); + { + uint8_t* p = nullptr; + up->Map(0, nullptr, (void**)&p); + for (int i = 0; i < 256; ++i) p[256 + i] = (uint8_t)(0xA5 ^ i); // the marker + up->Unmap(0, nullptr); + g.begin(); + g.list->CopyBufferRegion(marker, 0, up, 256, 256); + g.barrier(marker, D3D12_RESOURCE_STATE_COPY_DEST, D3D12_RESOURCE_STATE_COPY_SOURCE); + g.barrier(pred, D3D12_RESOURCE_STATE_COPY_DEST, D3D12_RESOURCE_STATE_PREDICATION); + g.submit(); + } + auto set_pred = [&](uint64_t v) { + uint8_t* p = nullptr; + up->Map(0, nullptr, (void**)&p); + const uint64_t other = v ? 0 : 1; + std::memcpy(p, &other, 8); + std::memcpy(p + 8, &v, 8); + up->Unmap(0, nullptr); + g.begin(); + g.barrier(pred, D3D12_RESOURCE_STATE_PREDICATION, D3D12_RESOURCE_STATE_COPY_DEST); + g.list->CopyBufferRegion(pred, 0, up, 0, 16); + g.barrier(pred, D3D12_RESOURCE_STATE_COPY_DEST, D3D12_RESOURCE_STATE_PREDICATION); + g.submit(); + }; + Ctx c; + c.g = &g; + if (!c.be.load(dll)) return false; + const WgResult cr = c.create(s, WG_CREATE_HDR | WG_CREATE_DEPTH_INVERTED, options.c_str()); + CHECK(cr == WG_OK); + if (cr != WG_OK) return false; + c.fe.predBuffer = pred; + c.fe.predOffset = 8; + c.fe.predOp = (uint32_t)D3D12_PREDICATION_OP_NOT_EQUAL_ZERO; + struct Step { + uint64_t value; + WgResult answer, want; + bool written, copied; + } steps[4] = {{0, WG_OK, WG_OK, true, true}, + {1, WG_OK, WG_OK, true, false}, + {1, WG_ERROR_NOT_READY, WG_ERROR_UNSUPPORTED, false, false}, + {0, WG_OK, WG_OK, true, true}}; + for (int t = 1; t <= 4; ++t) { + const Step& st = steps[t - 1]; + set_pred(st.value); + s.set_color(g, t); + c.fe.predAnswer = st.answer; + ID3D12Resource* mrb = g.buffer(256, D3D12_HEAP_TYPE_READBACK); + { + uint8_t* p = nullptr; + D3D12_RANGE none = {0, 0}; + mrb->Map(0, &none, (void**)&p); + std::memset(p, 0, 256); + mrb->Unmap(0, nullptr); + } + g.begin(); + record_sentinel(g, s, s.out, sentinel); + g.list->SetPredication(pred, 8, D3D12_PREDICATION_OP_NOT_EQUAL_ZERO); + WgFrame f = s.frame(g, t); + f.command_list = g.list; + const WgResult r = c.be.b->dispatch(c.u, &f); + g.list->CopyBufferRegion(mrb, 0, marker, 0, 256); // the game's conditional work after ours + g.list->SetPredication(nullptr, 0, D3D12_PREDICATION_OP_EQUAL_ZERO); + UINT pitch = 0; + ID3D12Resource* rb = record_readback(g, s.out, s.ow, s.oh, s.outState, &pitch); + c.fe.submit(); + const std::vector o = read_rgb(rb, s.ow, s.oh, pitch); + rb->Release(); + bool copied = true, untouched = true; + { + uint8_t* p = nullptr; + mrb->Map(0, nullptr, (void**)&p); + for (int i = 0; i < 256; ++i) copied &= p[i] == (uint8_t)(0xA5 ^ i), untouched &= p[i] == 0; + mrb->Unmap(0, nullptr); + } + mrb->Release(); + size_t nonfinite = 0; + for (float v : o) nonfinite += !std::isfinite(v); + const size_t sn = count_sentinel(o); + const bool written = sn == 0 && nonfinite == 0; + std::printf(" frame %d: game predicate %s%s, result %d, our output %s, the game's conditional copy after ours %s\n", t, + st.value ? "true (skip)" : "false (run)", st.answer == WG_OK ? "" : " (front end: not known)", (int)r, + written ? "written" : sn == o.size() ? "not written" : "PARTLY written", + copied ? "ran" : untouched ? "skipped" : "PARTLY ran"); + CHECK(r == st.want); + CHECK(written == st.written); + if (!st.written) CHECK(sn == o.size()); + CHECK(copied == st.copied); + CHECK(copied || untouched); + } + CHECK(c.fe.predQueries >= 4); + c.fe.predAnswer = WG_OK; + pred->Release(); + marker->Release(); + up->Release(); + sentinel->Release(); return true; } @@ -2775,12 +3018,24 @@ int main(int argc, char** argv) { std::puts(ok && !g_fail ? "PASS" : "FAIL"); return ok && !g_fail ? 0 : 1; } - if (only == "bnecktimeout") { + if (only == "bnecktimeout") { // synchronous, pipelined, synchronous with an output without unordered access const std::string opt = argc > 3 ? argv[3] : ""; - const bool a = case_bneck_timeout(g, dll, opt, false); - const bool b = case_bneck_timeout(g, dll, opt, true); - std::puts(a && b && !g_fail ? "PASS" : "FAIL"); - return a && b && !g_fail ? 0 : 1; + const std::string which = argc > 4 ? argv[4] : ""; // "proxy": that variant alone + const bool a = which == "proxy" || case_bneck_timeout(g, dll, opt, false); + const bool b = which == "proxy" || case_bneck_timeout(g, dll, opt, true); + const bool p = case_bneck_timeout(g, dll, opt, false, true); + std::puts(a && b && p && !g_fail ? "PASS" : "FAIL"); + return a && b && p && !g_fail ? 0 : 1; + } + if (only == "bneckfresh") { // (case_bneck_timeout's child) + if (argc < 7) return 2; + const bool ok = case_bneck_fresh(g, dll, argv[3], std::atoi(argv[4]), std::atoi(argv[5]) != 0, argv[6]); + return ok && !g_fail ? 0 : 1; + } + if (only == "callerpred") { // Model L hybrid options (release or developer runtime) + const bool ok = case_caller_predicate(g, dll, argc > 3 ? argv[3] : ""); + std::puts(ok && !g_fail ? "PASS" : "FAIL"); + return ok && !g_fail ? 0 : 1; } if (only == "hipbench") { const int frames = argc > 4 ? std::atoi(argv[4]) : 1500; diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index 2f8c81e0..de1b9bb2 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -3188,6 +3188,7 @@ const std::string& runtime_error(Runtime* rt) { return rt->err; } Device* runtime_device(Runtime* rt) { return rt->dev.get(); } float runtime_effective_pre_exposure(Runtime* rt, float value) { return rt->effective_pre_exposure(value); } const char* runtime_kernels(Runtime* rt) { return rt->kernels(); } +bool runtime_frame_guard(Runtime* rt) { return rt->hyb_mega; } const char* runtime_notes(Runtime* rt) { return rt->notes_text.c_str(); } uint64_t runtime_memory_bytes(Runtime* rt) { return rt->owned_bytes; } #ifdef WG_NO_MODEL_K diff --git a/clean/runtime/runtime_core.h b/clean/runtime/runtime_core.h index 73e0da32..86781a89 100644 --- a/clean/runtime/runtime_core.h +++ b/clean/runtime/runtime_core.h @@ -96,6 +96,8 @@ float runtime_effective_pre_exposure(Runtime* rt, float pre_exposure); // The kernels in use, one line (as wg_rt_kernels): for the front end's log at create, so that every run documents its // kernel choice. Valid until the next call. const char* runtime_kernels(Runtime* rt); +// Whether the frames use the device's frame guard (Model L hybrid on native D3D12: the bottleneck wait timeout). +bool runtime_frame_guard(Runtime* rt); // Notes of create for the front end's log (as wg_rt_notes): one per line, or empty. Valid while the runtime lives. const char* runtime_notes(Runtime* rt); // Device memory the runtime allocated itself (model data, state, intermediates), in bytes. From 80e3e1337a99d22aaee87a60a4a77d493f85f8c3 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Mon, 16 Nov 2026 02:38:48 -0500 Subject: [PATCH 4/5] Model L hybrid: one-frame refusals (WG_ERROR_NOT_READY), an older services table taken, real cases checked before the read - The frame after a bottleneck wait timeout, and a Model L hybrid frame whose game predicate the front end cannot report, answer WG_ERROR_NOT_READY, the interface's one-frame result: the front end's stand-in renders that frame, no takeover, and the next frame goes to the network again (the reset). WG_ERROR_UNSUPPORTED keeps its permanent handoff. Front end: a WG_ERROR_NOT_READY the stand-in cannot take is one frame's matter (AMD's FSR for that frame), never a failed upscaler. - WgFrontEndServices: an older version 2 table is checked up to split_command_list by offset (the appended predicate service keeps its own size check at dispatch). Interop test oldservices. - Real-data cases (l_q7_check, l_q7w_check, l_mm8_f32_check): a provenance marker next to every prepared real cases file, checked with its private folder outside the repository before the file is opened; a real run's --json is refused inside the repository (test_l_private_out.py: the loader fails the test if called). - test_dll_d3d12 bneck_real: the one-frame refusals through the real front end with the developer runtime (stuck, stand-in, network; unknown predicate, stand-in, network; no AMD FSR takeover). Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/fsr/include/wronggpu_upscaler.h | 7 +- clean/fsr/src/frontend.cpp | 6 +- clean/fsr/tests/test_dll_d3d12.cpp | 120 +++++++++++++++++- clean/l/runner/l_hyb_stage_check.py | 3 +- clean/l/runner/l_mm8_f32_check.py | 20 ++- clean/l/runner/l_private.py | 37 +++++- clean/l/runner/l_q7_check.py | 13 +- clean/l/runner/l_q7w_check.py | 8 ++ clean/l/tests/test_l_private_out.py | 76 ++++++++++- clean/runtime/interop/backend.cpp | 26 +++- clean/runtime/interop/executor.h | 4 +- clean/runtime/interop/executor_d3d12.cpp | 4 +- clean/runtime/interop/executor_select.h | 12 +- .../interop/tests/test_interop_d3d12.cpp | 41 +++++- 14 files changed, 343 insertions(+), 34 deletions(-) diff --git a/clean/fsr/include/wronggpu_upscaler.h b/clean/fsr/include/wronggpu_upscaler.h index 495286a7..2ced0d92 100644 --- a/clean/fsr/include/wronggpu_upscaler.h +++ b/clean/fsr/include/wronggpu_upscaler.h @@ -40,8 +40,11 @@ typedef enum WgResult { WG_ERROR_MEMORY = 4, WG_ERROR_PARAMETER = 5, /* a required input is missing or invalid */ WG_ERROR_NOT_READY = 6, /* v2: not possible this frame, try again next frame (e.g. split of a command list the - front end has not tracked since its last Reset); from dispatch it makes the front end - use AMD's FSR (or the stand-in) for this frame only, the context stays the backend's */ + front end has not tracked since its last Reset; the network runtime's frame after a + Model L bottleneck wait timeout, or a Model L frame whose command-list predicate the + front end cannot report); from dispatch it makes the front end use its stand-in (else + AMD's FSR) for this frame only, never a takeover: the context stays the backend's and + its next frame goes to the backend again */ WG_ERROR_INPUT = 7 /* v2 (0.6.0 follow-up): the game's inputs can never be taken (a texture's shape, format or flags the upscaler refuses), the same on every frame: from dispatch the context is failed as for WG_ERROR, and in OptiScaler's custom slot the front end remembers it for diff --git a/clean/fsr/src/frontend.cpp b/clean/fsr/src/frontend.cpp index 79c2df9a..bbef29bb 100644 --- a/clean/fsr/src/frontend.cpp +++ b/clean/fsr/src/frontend.cpp @@ -1882,8 +1882,10 @@ ffxReturnCode_t FrontEnd::DispatchUpscale(UpscaleContext& u, const ffxDispatchDe } } // Inputs the upscaler cannot take (the mapping's checks, or the upscaler's own: WG_ERROR_PARAMETER, - // WG_ERROR_UNSUPPORTED) are one frame's matter (BadFrame); every other result means the upscaler is broken. - if (!mapped || r == WG_ERROR_PARAMETER || r == WG_ERROR_UNSUPPORTED) { + // WG_ERROR_UNSUPPORTED) are one frame's matter (BadFrame), and so is WG_ERROR_NOT_READY that the stand-in could + // not take above (none, or it failed): AMD's FSR for this frame only, the context stays the upscaler's. Every + // other result means the upscaler is broken. + if (!mapped || r == WG_ERROR_PARAMETER || r == WG_ERROR_UNSUPPORTED || r == WG_ERROR_NOT_READY) { frame.reset = resetAsked ? 1u : 0u; // (for the stand-in: as the game gave it) return BadFrame(u, d, call, frameIndex, problem, &frame, false, mapped ? r : WG_ERROR_PARAMETER); } diff --git a/clean/fsr/tests/test_dll_d3d12.cpp b/clean/fsr/tests/test_dll_d3d12.cpp index 72869757..3198e455 100644 --- a/clean/fsr/tests/test_dll_d3d12.cpp +++ b/clean/fsr/tests/test_dll_d3d12.cpp @@ -42,7 +42,8 @@ // network_cyberpunk_fail Cyberpunk's FSR 3.0 path: the runtime fails on frame 1, AMD's upscaler takes over // By hand, with the real runtime (not ctests): park_real (a game's dynamic resolution), switch_real (a player // switching upscalers in an upscaler mod's menu; GPU memory after every step; see RunSwitchReal), resize_real (a game -// changing its output size: the upscalers kept until the context is destroyed, GPU memory per step; see RunResizeReal). +// changing its output size: the upscalers kept until the context is destroyed, GPU memory per step; see RunResizeReal), +// bneck_real (Model L hybrid's one-frame refusals through the real front end, developer runtime; see RunBneckReal). // With MSVC, the D3D12 debug layer is on when installed (WG_TEST_D3D12_DEBUG=0 turns it off); its errors fail the test. #define WIN32_LEAN_AND_MEAN #define NOMINMAX @@ -2131,6 +2132,97 @@ void RunResizeReal(Api& api, Gpu& gpu) { removed("at the end"); } +// bneck_real (run by hand, not a ctest; needs a DEVELOPER wronggpu_runtime.dll with Model L next to the test's DLLs, +// WG_TEST_RUNTIME_OPTIONS naming its model and kernels, and WG_DEV_BNECK_ABORT=2): the Model L hybrid one-frame +// refusals through the real front end, with AMD's FSR (the test double) available, so a takeover would show. +// frames 0 .. 5 on the game's list: frame 2's bottleneck wait times out (the network skips it, its fallback upscale is +// the output), frame 3 is refused (WG_ERROR_NOT_READY: the stand-in renders it, no takeover), frame 4 runs the +// network again (the reset), frame 5 too; +// frame 6 on a second list the game reset before the front end installed its hooks and did not reset since: its +// predicate cannot be reported (WG_ERROR_NOT_READY: the stand-in, no takeover); +// frame 7 on the game's list again: the network. +// The checks are on the log (main): two stand-in frames, no AMD FSR, every frame OK, the refusal and the unknown +// predicate each named. +void RunBneckReal(Api& api, Gpu& gpu) { + const UINT W = 2560, H = 1440, RW = 1706, RH = 960; + ID3D12Resource* color = gpu.Texture(DXGI_FORMAT_R16G16B16A16_FLOAT, W, H, D3D12_RESOURCE_FLAG_NONE, D3D12_RESOURCE_STATE_COPY_DEST); + ID3D12Resource* depth = gpu.Texture(DXGI_FORMAT_R32_FLOAT, W, H, D3D12_RESOURCE_FLAG_NONE, D3D12_RESOURCE_STATE_COPY_DEST); + ID3D12Resource* motion = gpu.Texture(DXGI_FORMAT_R16G16_FLOAT, W, H, D3D12_RESOURCE_FLAG_NONE, D3D12_RESOURCE_STATE_COPY_DEST); + ID3D12Resource* out = gpu.Texture(DXGI_FORMAT_R16G16B16A16_FLOAT, W, H, D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS, + D3D12_RESOURCE_STATE_UNORDERED_ACCESS); + CHECK(color && depth && motion && out); + if (!color || !depth || !motion || !out) return; + { + std::vector px(size_t(W) * H * 4); + for (UINT y = 0; y < H; ++y) + for (UINT x = 0; x < W; ++x) { + uint16_t* p = &px[(size_t(y) * W + x) * 4]; + p[0] = HalfOf(((x / 16 + y / 16) & 1) ? 0.6f : 0.15f); + p[1] = HalfOf(0.1f + 0.8f * float(y % 193) / 193.0f); + p[2] = HalfOf(0.05f + 1.5f * float((x + y) % 61) / 61.0f); + p[3] = HalfOf(1.0f); + } + std::vector z(size_t(W) * H); + for (UINT y = 0; y < H; ++y) + for (UINT x = 0; x < W; ++x) z[size_t(y) * W + x] = 0.02f + 0.9f * float((x / 64 + y / 64) % 11) / 11.0f; + std::vector mv(size_t(W) * H * 2, 0); + gpu.Begin(); + gpu.Upload(color, px.data(), 8, D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE); + gpu.Upload(depth, z.data(), 4, D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE); + gpu.Upload(motion, mv.data(), 4, D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE); + CHECK(gpu.SubmitAndWait()); + } + // the second list: reset now, before the context (and the front end's hooks) exist, and not reset again + gpu.allocatorB->Reset(); + gpu.listB->Reset(gpu.allocatorB, nullptr); + ffxCreateBackendDX12Desc backend{}; + backend.header.type = FFX_API_CREATE_CONTEXT_DESC_TYPE_BACKEND_DX12; + backend.device = gpu.device; + ffxCreateContextDescUpscale c{}; + c.header.type = FFX_API_CREATE_CONTEXT_DESC_TYPE_UPSCALE; + c.header.pNext = &backend.header; + c.flags = FFX_UPSCALE_ENABLE_HIGH_DYNAMIC_RANGE | FFX_UPSCALE_ENABLE_DEPTH_INVERTED; + c.maxRenderSize = {RW, RH}; // (a game's Quality mode) + c.maxUpscaleSize = {W, H}; + ffxContext ctx = nullptr; + CHECK(api.create(&ctx, &c.header, nullptr) == FFX_API_RETURN_OK && ctx); + if (!ctx) return; + static const float kJitter[4][2] = {{0.0f, -1.0f / 6}, {-0.25f, 1.0f / 6}, {0.25f, -7.0f / 18}, {-0.375f, -1.0f / 18}}; + for (int i = 0; i < 8; ++i) { + const bool second = i == 6; + ID3D12GraphicsCommandList* list = second ? gpu.listB : gpu.list; + if (!second) gpu.Begin(); + ffxDispatchDescUpscale d{}; + d.header.type = FFX_API_DISPATCH_DESC_TYPE_UPSCALE; + d.commandList = list; + d.color = Resource(color, FFX_API_SURFACE_FORMAT_R16G16B16A16_FLOAT, W, H, FFX_API_RESOURCE_STATE_COMPUTE_READ); + d.depth = Resource(depth, FFX_API_SURFACE_FORMAT_R32_FLOAT, W, H, FFX_API_RESOURCE_STATE_COMPUTE_READ); + d.motionVectors = Resource(motion, FFX_API_SURFACE_FORMAT_R16G16_FLOAT, W, H, FFX_API_RESOURCE_STATE_COMPUTE_READ); + d.output = Resource(out, FFX_API_SURFACE_FORMAT_R16G16B16A16_FLOAT, W, H, FFX_API_RESOURCE_STATE_UNORDERED_ACCESS); + d.jitterOffset = {kJitter[i & 3][0], kJitter[i & 3][1]}; + d.motionVectorScale = {float(RW), float(RH)}; + d.renderSize = {RW, RH}; + d.upscaleSize = {W, H}; + d.preExposure = 1.0f; + d.frameTimeDelta = 16.7f; + d.cameraNear = 100000.0f; + d.cameraFar = 0.1f; + d.cameraFovAngleVertical = 1.0472f; + const ffxReturnCode_t rc = api.dispatch(&ctx, &d.header); + std::printf("bneck_real: frame %d (%s list) -> %d\n", i, second ? "second" : "game's", int(rc)); + CHECK(rc == FFX_API_RETURN_OK); + if (second) { + CHECK(SUCCEEDED(gpu.listB->Close())); + ID3D12CommandList* lists[] = {gpu.listB}; + gpu.queue->ExecuteCommandLists(1, lists); + CHECK(gpu.Wait()); + } else { + CHECK(gpu.SubmitAndWait()); + } + } + CHECK(api.destroy(&ctx, nullptr) == FFX_API_RETURN_OK); +} + int main(int argc, char** argv) { const std::string mode = argc > 1 ? argv[1] : "upscale"; if (mode == "switch_real") g_dllName = L"wronggpu_ffx_dx12.dll"; @@ -2156,6 +2248,12 @@ int main(int argc, char** argv) { SetEnvironmentVariableA("WG_FAKE_RUNTIME", fake); if (mode == "network_park") settings += "context_park_ms = 300\n"; } + if (mode == "bneck_real") { + char options[2048] = ""; + GetEnvironmentVariableA("WG_TEST_RUNTIME_OPTIONS", options, sizeof options); + settings = "upscaler = network\ncreate_wait_ms = 60000\n"; // (the first frame waits for the real create) + if (options[0]) settings += std::string("runtime_options = ") + options + "\n"; + } if (mode == "park_real" || mode == "resize_real") { char park[32] = "3000", options[2048] = ""; GetEnvironmentVariableA("WG_TEST_PARK_MS", park, sizeof park); @@ -2212,6 +2310,7 @@ int main(int argc, char** argv) { else if (mode == "network_park") RunNetworkPark(api, gpu); else if (mode == "park_real") RunParkReal(api, gpu); else if (mode == "resize_real") RunResizeReal(api, gpu); + else if (mode == "bneck_real") RunBneckReal(api, gpu); else if (mode == "switch_real") RunSwitchReal(api, gpu, argc > 2 ? std::atoi(argv[2]) : 6, argc > 3 ? std::atoi(argv[3]) : 20, argc > 4 ? std::atoi(argv[4]) : 3500); else if (mode.rfind("network_", 0) == 0) RunNetwork(api, gpu, mode); @@ -2284,6 +2383,21 @@ int main(int argc, char** argv) { std::printf("park_real: log: %s\n", log.substr(a + 1, b - a - 1).c_str()); } } + if (mode == "bneck_real") { + const size_t standIn = Count(log, "stand-in for this frame"), refused = Count(log, "frame refused (bottleneck wait timeout)"), + unknown = Count(log, "does not know the predicate"), ok = Count(log, "-> OK (WrongGPU)"), + amd = Count(log, "AMD FSR upscales") + Count(log, "forwarded to AMD") + Count(log, "for the rest of its life"); + std::printf("bneck_real: log: %zu frames OK, %zu of them the stand-in's, %zu bottleneck refusal(s), %zu unknown-predicate line(s), " + "%zu AMD FSR line(s), %zu error line(s)\n", ok, standIn, refused, unknown, amd, + Count(log, "ERROR:") + Count(log, " ERROR U") + Count(log, "upscaler ERROR")); + CHECK(ok == 8); + CHECK(standIn == 2); + CHECK(refused == 1); + CHECK(unknown >= 1); + CHECK(amd == 0); + CHECK(Count(log, "ERROR:") == 0 && Count(log, " ERROR U") == 0 && Count(log, "upscaler ERROR") == 0); + CHECK(Count(log, "network runtime ready") == 1); // one network context throughout + } if (mode == "resize_real") { std::printf("resize_real: log: %zu upscalers created, %zu created again for a new output size, %zu old upscalers kept until the context is destroyed, " "%zu not created again after the limit, %zu errors\n", @@ -2306,7 +2420,9 @@ int main(int argc, char** argv) { Count(log, "kept arena of"), Count(log, "the host chose AMD's upscaler version"), Count(log, "ERROR")); } // Same-frame split and submission trace (interface version 2). - if (mode.rfind("split_", 0) != 0 && !trace && mode != "network_split" && mode.rfind("network_split_stats",0)!=0 && !parks) CHECK(Count(log, "d3d12 hooks:") == 0); // auto: no hooks without a split backend + if (mode.rfind("split_", 0) != 0 && !trace && mode != "network_split" && mode.rfind("network_split_stats",0)!=0 && !parks && + mode != "bneck_real") // (the real runtime sets WG_BACKEND_NEEDS_SPLIT) + CHECK(Count(log, "d3d12 hooks:") == 0); // auto: no hooks without a split backend // upscaler = network: the runtime from the FSR DLLs' folder, its errors in words, AMD after a failure. if (mode.rfind("network_", 0) == 0) { const bool loads = mode != "network_missing"; diff --git a/clean/l/runner/l_hyb_stage_check.py b/clean/l/runner/l_hyb_stage_check.py index 3a0351f9..ecdb5ba6 100644 --- a/clean/l/runner/l_hyb_stage_check.py +++ b/clean/l/runner/l_hyb_stage_check.py @@ -376,7 +376,8 @@ def main(argv=None): # repository, checked before anything is read or built cdir = None if a.real_inputs: - from clean.l.runner.l_private import private_dir + from clean.l.runner.l_private import private_dir, private_json + private_json(a.json) # the record of a real run outside the repository cdir = private_dir(a.private_out, what="--real-inputs: the real cases' files and replay outputs") elif a.private_out: raise SystemExit("--private-out: only with --real-inputs") diff --git a/clean/l/runner/l_mm8_f32_check.py b/clean/l/runner/l_mm8_f32_check.py index 5cf6c5d7..3e93b378 100644 --- a/clean/l/runner/l_mm8_f32_check.py +++ b/clean/l/runner/l_mm8_f32_check.py @@ -356,18 +356,23 @@ def spy(start, a8, W8, model): ref16 = np.array([op_exact(st[i:i + 1], a[i], w[i:i + 1])[0] for i in range(len(st))]) cases[name] = {"st": st, "a": a, "w": w, "exact_sum": exact_sum, "ref16": ref16} print("prepared", name, len(st), flush=True) + if real_build: + from clean.l.runner.l_private import mark_real + mark_real(os.path.join(build, "l_wmma_cases.pkl"), "l_mm8_f32_check --real") with open(os.path.join(build, "l_wmma_cases.pkl"), "wb") as f: pickle.dump({"cases": cases, "identity": identity}, f) -def wmma_run(build, rocm, cases_dir=None): +def wmma_run(build, rocm, cases_dir=None, private_out=None): """GPU: the probe kernel (clean/l/kernels/l_wmma_probe.hip, /l_wmma_probe.co) on every prepared set: per op, whether the instruction's float32 result equals the exact sum rounded once to float32, and whether its FP16 rounding equals the exact model's result.""" import ctypes + from clean.l.runner.l_private import guard_cases from clean.runner.hip import Hip, clean_process_environment - clean_process_environment(rocm) cases_pkl = os.path.join(cases_dir or build, "l_wmma_cases.pkl") + guard_cases(cases_pkl, private_out) # a real set (marker) only from its private folder, checked before the read + clean_process_environment(rocm) with open(cases_pkl, "rb") as f: prep = pickle.load(f) h = Hip(rocm) @@ -444,7 +449,8 @@ def main(argv=None): ap.add_argument("--seed", type=int, default=7) ap.add_argument("--frames", help="with --real: a build with a local recorded l_frame_check run of the real model") ap.add_argument("--real", action="store_true", help="also the local recorded run (--frames; needs --private-out)") - ap.add_argument("--private-out", help="--real: the folder (outside the repository) for a real run's intermediates") + ap.add_argument("--private-out", help="--real: the folder (outside the repository) for a real run's intermediates; " + "--wmma-run: the folder of a real prepared set (its --cases-dir)") ap.add_argument("--cases-dir", help="--wmma-run: the folder of l_wmma_cases.pkl (default the build folder)") ap.add_argument("--wmma-prepare", metavar="BUILD", help="CPU: the matrix-instruction probe's inputs into BUILD") ap.add_argument("--wmma-run", metavar="BUILD", help="GPU: the probe (BUILD/l_wmma_probe.co) on the prepared inputs") @@ -456,6 +462,12 @@ def main(argv=None): raise SystemExit("--real needs --frames and --private-out ") if a.real: private_dir(a.private_out) + if a.real or a.private_out: # a real run: its record outside the repository too + from clean.l.runner.l_private import private_json + private_json(a.json) + if a.wmma_run: # a real prepared set (marker) only from its private folder, before anything is read + from clean.l.runner.l_private import guard_cases + guard_cases(os.path.join(a.cases_dir or a.wmma_run, "l_wmma_cases.pkl"), a.private_out) if a.wmma_prepare: wmma_prepare(a.wmma_prepare, a.cases, a.seed, a.frames if a.real else None, private_out=a.private_out) return 0 @@ -463,7 +475,7 @@ def main(argv=None): from clean.l.runner import l_record as LR res = {"what": "RDNA 4 FP8 matrix instruction (two 16-term steps, float32 accumulator from the start) against " "the exact model, a side measurement", "fp8_op_model": "exact", - "probe": wmma_run(a.wmma_run, a.rocm, a.cases_dir)} + "probe": wmma_run(a.wmma_run, a.rocm, a.cases_dir, a.private_out)} assert_aggregate_only(res) LR.write_l_record(res, a.json) return 0 diff --git a/clean/l/runner/l_private.py b/clean/l/runner/l_private.py index ca588f4f..0b604f01 100644 --- a/clean/l/runner/l_private.py +++ b/clean/l/runner/l_private.py @@ -2,7 +2,10 @@ """The synthetic-values rule for the Model L checkers' real modes (l_hyb_stage_check --real-inputs, l_q7_check / l_q7w_check --real, l_mm8_f32_check --real): every intermediate made from real weights or recorded frames (case files, replay inputs and outputs) goes only to a private folder OUTSIDE the repository, resolved (real path) before anything is -read; a folder inside the repository is refused. The records these checkers write stay counts and digests. +read; a folder inside the repository is refused. The records these checkers write stay counts and digests, and a +real run's --json goes outside the repository too (private_json). A prepared cases file made from real data gets a +provenance marker next to it (.real, mark_real); every run path checks the marker and the private folder BEFORE +it opens the file (guard_cases). """ import os @@ -30,3 +33,35 @@ def private_dir(path, flag="--private-out", what="a real run's intermediates"): p = os.path.realpath(os.path.abspath(path)) os.makedirs(p, exist_ok=True) return p + + +MARKER = ".real" + + +def mark_real(cases_path, what): + """The provenance marker of a prepared cases file made from real data (written before the file itself).""" + with open(cases_path + MARKER, "w", encoding="utf-8", newline="\n") as f: + f.write("prepared from real data (%s): read only from its private folder outside the repository " + "(--private-out)\n" % what) + + +def guard_cases(cases_path, private_out): + """Before a prepared cases file is opened: a marked (real) one only from the given private folder, which is + outside the repository; SystemExit otherwise. Unmarked files (synthetic) pass.""" + folder = os.path.dirname(os.path.realpath(os.path.abspath(cases_path))) + if not os.path.exists(cases_path + MARKER): + return + if inside_repository(folder): + raise SystemExit(f"{cases_path}: prepared from real data, inside the repository ({os.path.realpath(ROOT)}); " + "refused (it belongs in a private folder outside it)") + if not private_out: + raise SystemExit(f"{cases_path}: prepared from real data; read only with --private-out naming its folder") + if os.path.realpath(os.path.abspath(private_out)) != folder: + raise SystemExit(f"{cases_path}: prepared from real data; --private-out {private_out} is not its folder") + + +def private_json(path, flag="--json"): + """A real run's record: SystemExit when it would be written inside the repository.""" + if path and inside_repository(os.path.dirname(os.path.abspath(path)) or "."): + raise SystemExit(f"{flag} {path}: a real run's record goes outside the repository (or into the private " + "folder), never inside it") diff --git a/clean/l/runner/l_q7_check.py b/clean/l/runner/l_q7_check.py index cc2da11b..c64753bc 100644 --- a/clean/l/runner/l_q7_check.py +++ b/clean/l/runner/l_q7_check.py @@ -242,7 +242,10 @@ def data_dir(build, private_out): def load_cases(path, private_out): - """The prepared cases; a file holding real weights is read only from a private folder (--private-out).""" + """The prepared cases; a file holding real weights is read only from a private folder (--private-out): its marker + and the folder are checked before the file is opened (l_private.guard_cases).""" + from clean.l.runner.l_private import guard_cases + guard_cases(path, private_out) with open(path, "rb") as f: prep = pickle.load(f) if prep.get("real_params_sha256") and not private_out: @@ -285,6 +288,9 @@ def prepare(build, real=None, form="q7"): "scale": scale, "special": special, "xb": xb, "sb": sb, "nbytes": nbytes, "blob": blob, "want": want, "want_exact": want_exact}) print(bname, H, W, scale, "prepared", flush=True) + if real: + from clean.l.runner.l_private import mark_real + mark_real(cases_file(build, form), "l_q7_check --real") with open(cases_file(build, form), "wb") as f: pickle.dump({"cases": cases, "real_params_sha256": real_sha, "form": form}, f) return len(cases) @@ -449,7 +455,12 @@ def main(argv=None): raise SystemExit("--real is required to have --private-out : the cases file " "holds real weights and the oracle's bytes on them") if a.private_out and (a.real or a.run): + from clean.l.runner.l_private import private_json + private_json(a.json) # a real run's record outside the repository data_dir(a.build, a.private_out) # refused inside the repository, before anything is read + if a.run: # a real prepared file (marker) only from its private folder: checked before anything else + from clean.l.runner.l_private import guard_cases + guard_cases(cases_file(data_dir(a.build, a.private_out), a.form), a.private_out) if a.compile: compile_kernel(a.build, a.rocm) return 0 diff --git a/clean/l/runner/l_q7w_check.py b/clean/l/runner/l_q7w_check.py index 8f3e2aba..ba5ba4fa 100644 --- a/clean/l/runner/l_q7w_check.py +++ b/clean/l/runner/l_q7w_check.py @@ -161,6 +161,9 @@ def prepare(build, real=None): sets.append(("real", {st: params[st] for st in ("enc1", "enc2", "b0", "dec2", "dec1")})) sets.append(("real_b5", {"b0": params["b5"], **{st: params[st] for st in ("enc1", "enc2", "dec2", "dec1")}})) cases = make_cases(sets) + if real: + from clean.l.runner.l_private import mark_real + mark_real(os.path.join(build, "l_q7w_cases.pkl"), "l_q7w_check --real") with open(os.path.join(build, "l_q7w_cases.pkl"), "wb") as f: pickle.dump({"cases": cases, "real_params_sha256": real_sha}, f) return len(cases) @@ -333,7 +336,12 @@ def main(argv=None): raise SystemExit("--real is required to have --private-out : the cases file " "holds real weights and the oracle's bytes on them") if a.private_out and (a.real or a.run): + from clean.l.runner.l_private import private_json + private_json(a.json) # a real run's record outside the repository data_dir(a.build, a.private_out) # refused inside the repository, before anything is read + if a.run: # a real prepared file (marker) only from its private folder: checked before anything else + from clean.l.runner.l_private import guard_cases + guard_cases(os.path.join(data_dir(a.build, a.private_out), "l_q7w_cases.pkl"), a.private_out) if a.compile: compile_kernel(a.build, a.rocm) return 0 diff --git a/clean/l/tests/test_l_private_out.py b/clean/l/tests/test_l_private_out.py index 28b0a25f..dde9bd68 100644 --- a/clean/l/tests/test_l_private_out.py +++ b/clean/l/tests/test_l_private_out.py @@ -2,7 +2,8 @@ """The synthetic-values rule for the Model L checkers' real modes: l_hyb_stage_check --real-inputs, l_q7_check --real and l_q7w_check --real write their real-data intermediates (case files, replay inputs and outputs) only to --private-out, a resolved folder outside the repository. Each real-input path refuses a missing folder and a folder inside the -repository before it reads or writes anything; a cases file prepared with --real is read only from such a folder. +repository before it reads or writes anything; a cases file prepared with --real carries a marker and is read only +from such a folder, checked before the file is opened; a real run's --json goes outside the repository. No GPU, no model, no recorded data (every value here is constructed; the paths given as real inputs do not exist). Run: python clean/l/tests/test_l_private_out.py """ @@ -11,12 +12,14 @@ import sys import tempfile import unittest +from unittest import mock HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) sys.path.insert(0, ROOT) from clean.l.runner import l_hyb_stage_check as HS # noqa: E402 +from clean.l.runner import l_mm8_f32_check as MM8 # noqa: E402 from clean.l.runner import l_private as P # noqa: E402 from clean.l.runner import l_q7_check as Q7 # noqa: E402 from clean.l.runner import l_q7w_check as Q7W # noqa: E402 @@ -125,5 +128,76 @@ def test_the_private_folder_holds_the_cases_file(self): self.assertEqual(Q7.data_dir(os.path.join(d, "build"), None), os.path.join(d, "build")) + +class MarkedCases(unittest.TestCase): + """A prepared cases file made from real data carries a marker (.real); every run path checks the marker and + the private folder BEFORE it opens the file: pickle.load is replaced by a function that fails the test if called. + The files here are constructed (a marker and bytes that are no pickle at all).""" + + def marked(self, folder, name): + os.makedirs(folder, exist_ok=True) + path = os.path.join(folder, name) + with open(path, "wb") as f: + f.write(b"not a pickle: never to be opened") + P.mark_real(path, "test") + return path + + def no_read(self, mod): + return mock.patch.object(mod.pickle, "load", side_effect=AssertionError("the cases file was read")) + + def test_q7_and_q7w_refuse_before_the_read(self): + for name, mod, fn in Q7Real.MODS: + d = outside_tmp(self) + self.marked(d, fn) + with self.subTest(checker=name), self.no_read(mod), self.assertRaises(SystemExit) as cm: + mod.main(["--build", d, "--run"]) + self.assertIn("prepared from real data", str(cm.exception)) + + def test_mm8_wmma_run_refuses_before_the_read(self): + d = outside_tmp(self) + cases = os.path.join(d, "cases") + self.marked(cases, "l_wmma_cases.pkl") + j = os.path.join(d, "r.json") + for extra in ([], ["--private-out", os.path.join(d, "elsewhere")]): + with self.subTest(extra=extra), self.no_read(MM8), self.assertRaises(SystemExit) as cm: + MM8.main(["--json", j, "--wmma-run", d, "--cases-dir", cases] + extra) + self.assertIn("prepared from real data", str(cm.exception)) + self.assertFalse(os.path.exists(j)) + + def test_guard_cases(self): + d = outside_tmp(self) + path = self.marked(d, "x.pkl") + with self.assertRaises(SystemExit): + P.guard_cases(path, None) # no private folder + with self.assertRaises(SystemExit): + P.guard_cases(path, os.path.join(d, "other")) # not its folder + P.guard_cases(path, d) # its own private folder outside the repository + with mock.patch.object(P, "inside_repository", return_value=True), self.assertRaises(SystemExit) as cm: + P.guard_cases(path, d) # a real cases folder inside the repository + self.assertIn("inside the repository", str(cm.exception)) + plain = os.path.join(d, "plain.pkl") # unmarked (synthetic): passes + P.guard_cases(plain, None) + + +class RealRecords(unittest.TestCase): + """A real run's --json goes outside the repository (refused inside it before anything is read or written).""" + J = os.path.join(ROOT, "clean", "l", "records", "real-run-test.json") + + def test_in_repo_json_refused(self): + d = outside_tmp(self) + priv = os.path.join(d, "priv") + runs = (("l_q7_check", lambda: Q7.main(["--build", d, "--run", "--private-out", priv, "--json", self.J])), + ("l_q7w_check", lambda: Q7W.main(["--build", d, "--run", "--private-out", priv, "--json", self.J])), + ("l_mm8_f32_check", lambda: MM8.main(["--json", self.J, "--wmma-run", d, "--private-out", priv])), + ("l_hyb_stage_check", lambda: HS.main(["--build", d, "--run", "--real-inputs", os.path.join(d, "none"), + "--private-out", priv, "--json", self.J]))) + for name, run in runs: + with self.subTest(checker=name), self.assertRaises(SystemExit) as cm: + run() + self.assertIn("outside the repository", str(cm.exception)) + self.assertFalse(os.path.exists(self.J)) + self.assertFalse(os.path.exists(priv)) # refused before the private folder was even made + + if __name__ == "__main__": unittest.main() diff --git a/clean/runtime/interop/backend.cpp b/clean/runtime/interop/backend.cpp index cd9737c2..ac65392d 100644 --- a/clean/runtime/interop/backend.cpp +++ b/clean/runtime/interop/backend.cpp @@ -635,7 +635,10 @@ class Context { } } k_default_ = model_k_default(o); - const bool split = services_ && services_->struct_size >= sizeof(WgFrontEndServices) && services_->split_command_list; + // (a version 2 table up to split_command_list: fields appended later have their own size checks where they are used) + const bool split = services_ && + services_->struct_size >= offsetof(WgFrontEndServices, split_command_list) + sizeof(void*) && + services_->split_command_list; if (!split && o.executor != "dx12") return fail(WG_ERROR_UNSUPPORTED, "the front end offers no command-list split (interface version 2)"); if (!o.unknown.empty()) say(WG_LOG_INFO, "options passed on to the network runtime: " + o.unknown); @@ -1149,14 +1152,22 @@ class Context { } return WG_ERROR_UNSUPPORTED; // one frame's answer: the context stays usable (failed_ not set) } - // Model L hybrid, the frame after a bottleneck wait timeout (executor_select.h frame_timeout_refused): nothing of - // it written, the path of a frame not taken: one line, WG_ERROR_UNSUPPORTED (the front end's stand-in renders it), - // the context kept; the next frame starts over. + // Model L hybrid, the frame after a bottleneck wait timeout (executor_select.h frame_timeout_refused), and a + // guarded frame whose game predicate the front end cannot report (frame_not_ready): nothing of it recorded, THIS + // frame only: WG_ERROR_NOT_READY (the front end's stand-in renders it, no takeover; the context stays this + // backend's), and the next frame runs the network again, as a reset. if (!ok && frame_timeout_refused(err)) { - say(WG_LOG_WARN, "frame " + std::to_string(v) + ": " + err + "; this frame goes back to the front end (AMD FSR, " - "else its stand-in), the context stays"); + say(WG_LOG_WARN, "frame " + std::to_string(v) + ": " + err + "; the front end's stand-in renders this frame " + "(WG_ERROR_NOT_READY), the next one runs the network again"); force_reset_ = true; - return WG_ERROR_UNSUPPORTED; // one frame's answer: the context stays usable (failed_ not set) + return WG_ERROR_NOT_READY; // one frame's answer: the context stays this backend's (failed_ not set) + } + if (!ok && frame_not_ready(err)) { + if (pred_unknown_++ == 0 || pred_unknown_ % 600 == 0) + say(WG_LOG_WARN, "frame " + std::to_string(v) + ": " + err + "; the front end's stand-in renders this frame " + "(WG_ERROR_NOT_READY; " + std::to_string(pred_unknown_) + " such frame(s) so far)"); + force_reset_ = true; + return WG_ERROR_NOT_READY; } note_path(&input_path_, "input: read directly by the network's D3D12 kernels (executor dx12, no input pass)"); if ((flags_ & WG_CREATE_MOTION_OUTPUT_RES) && model_k_) @@ -2029,6 +2040,7 @@ class Context { Core log_; const WgFrontEndServices* services_ = nullptr; + uint64_t pred_unknown_ = 0; // guarded frames whose game predicate the front end could not report ID3D12Device* device_ = nullptr; ID3D12Device* underlying_ = nullptr; // what the given device's objects report as their device (sharing path 2) ID3D12Device* own_ = nullptr; // our own device on the same adapter (sharing path 3) diff --git a/clean/runtime/interop/executor.h b/clean/runtime/interop/executor.h index 3c585368..b2372bac 100644 --- a/clean/runtime/interop/executor.h +++ b/clean/runtime/interop/executor.h @@ -116,8 +116,8 @@ struct Job { class Converter* conv = nullptr; // executor=dx12: the game's predicate on its list before this frame (WgFrontEndServices::command_list_predication; // caller_pred_known false: the front end could not tell). A frame under the device's frame guard (Model L hybrid) - // replaces it while it records and sets it again after (NativeD3D12Ext::set_caller_predicate); such a frame is not - // taken when it is not known. + // replaces it while it records and sets it again after (NativeD3D12Ext::set_caller_predicate); when it is not + // known, such a frame is not taken this time (executor_select.h frame_not_ready: WG_ERROR_NOT_READY). bool caller_pred_known = false; void* caller_pred = nullptr; // ID3D12Resource*, null: none set uint64_t caller_pred_offset = 0; diff --git a/clean/runtime/interop/executor_d3d12.cpp b/clean/runtime/interop/executor_d3d12.cpp index 637e68e0..0c89f668 100644 --- a/clean/runtime/interop/executor_d3d12.cpp +++ b/clean/runtime/interop/executor_d3d12.cpp @@ -301,10 +301,10 @@ class D3d12Executor : public Executor { // Model L hybrid: the device's frame guard predicates this frame on the game's list, in place of the game's own // predicate, which is lifted while we record and set again after (its conditional work after ours stays - // conditional); a frame whose game predicate the front end cannot tell is not taken. + // conditional); a frame whose game predicate the front end cannot tell is not taken THIS time (frame_not_ready). const bool guarded = wgrt::runtime_frame_guard(rt_); if (guarded && !job.caller_pred_known) - return *err = "frame not taken (nothing recorded): the front end does not know the predicate set on the game's " + return *err = "frame not ready (nothing recorded): the front end does not know the predicate set on the game's " "command list, which the Model L hybrid frame guard would replace", false; if (guarded) ext_->set_caller_predicate(static_cast(job.caller_pred), job.caller_pred_offset, diff --git a/clean/runtime/interop/executor_select.h b/clean/runtime/interop/executor_select.h index 7c17db01..4c4fa0ca 100644 --- a/clean/runtime/interop/executor_select.h +++ b/clean/runtime/interop/executor_select.h @@ -241,6 +241,12 @@ inline std::string drop_hip_only_filter(const std::string& options, std::string* // no output texture, a render size outside the configured maximum, an output texture without unordered access, ...). inline bool frame_refused(const std::string& err) { return err.rfind("frame not taken", 0) == 0; } +// A frame the executor cannot take THIS time only (nothing recorded), the next one may run: Model L hybrid when the +// front end cannot report the game's predicate on its list (WgFrontEndServices::command_list_predication; the frame +// guard would replace it). The backend answers WG_ERROR_NOT_READY: the front end's stand-in renders that frame, no +// takeover, and the context stays the backend's. +inline bool frame_not_ready(const std::string& err) { return err.rfind("frame not ready", 0) == 0; } + // Model L hybrid: the frame after one whose bottleneck wait timed out on the GPU (runtime.cpp kBneckTimeoutRefusal; // hyb_bneck.hlsl). The timed-out frame and the frames queued after it were skipped on the GPU (the device's frame guard // predicates the network and the executor's output pass alike), and each of them got the executor's fallback instead: @@ -248,9 +254,9 @@ inline bool frame_refused(const std::string& err) { return err.rfind("frame not // texture (Converter::record_output with no result, or through the route's texture of ours), so every such frame, // the first one of a context included, shows an image whatever output texture the game passed. This frame (the one // whose start sees the timeout count move) records nothing that writes (only the game textures' state changes, -// restored) and takes the path of a frame the executor did not take: the backend logs one line and answers -// WG_ERROR_UNSUPPORTED, so the front end's stand-in (AMD FSR, else its own upscale) renders it; the context is kept -// (never failed_), and the next frame is a reset (the runtime starts over, and the backend asks for the reset too). +// restored): the backend logs one line and answers WG_ERROR_NOT_READY, the documented one-frame result, so the front +// end's stand-in renders that frame without taking the context over; the context stays the backend's (never failed_), +// and the next frame runs the network as a reset (the runtime starts over, and the backend asks for the reset too). inline bool frame_timeout_refused(const std::string& err) { return err.rfind("frame refused (bottleneck wait timeout): ", 0) == 0; } diff --git a/clean/runtime/interop/tests/test_interop_d3d12.cpp b/clean/runtime/interop/tests/test_interop_d3d12.cpp index adb0f180..17d23ad7 100644 --- a/clean/runtime/interop/tests/test_interop_d3d12.cpp +++ b/clean/runtime/interop/tests/test_interop_d3d12.cpp @@ -38,7 +38,7 @@ // (e.g. "" and "model=K"); fails when the process's GPU memory grows by more than the tolerance per cycle. // // Cases: inline, pipelined, depth, motion, motion_outres, exposure, not_ready, pipelined_not_ready, nohip, -// noservices, timing (timing_log=1 summary), ready_flag (ready_wait=memory flag pass with timing marks), arena, arena_minsize (wrapper refusing shared objects below 100 MB), arena_keep (arena pool), wrapped, wrapped_noopen, wrapped_own +// noservices, oldservices (an older version 2 services table), timing (timing_log=1 summary), ready_flag (ready_wait=memory flag pass with timing marks), arena, arena_minsize (wrapper refusing shared objects below 100 MB), arena_keep (arena pool), wrapped, wrapped_noopen, wrapped_own // (device wrapper whose CreateSharedHandle fails); default all. // "hip" (only when named; Windows with an RX 9000 GPU, the kernels next to the DLL and an installed model): the real // network through the same split, at 1707x960 -> 2560x1440, inline then pipelined; checks finite output, that the @@ -59,6 +59,7 @@ #include #include +#include #include #include #include @@ -556,9 +557,10 @@ struct Ctx { FrontEnd fe; WgFrontEndServices services = {}; WgUpscaler* u = nullptr; + uint32_t servicesSize = 0; // 0: sizeof(WgFrontEndServices); else an older front end's table size WgResult create(const Scene& s, uint32_t flags, const char* options, bool withServices = true) { fe.g = g; - services.struct_size = sizeof services; + services.struct_size = servicesSize ? servicesSize : (uint32_t)sizeof services; services.context = &fe; services.split_command_list = &FrontEnd::split; services.command_list_predication = &FrontEnd::predication; @@ -2683,7 +2685,7 @@ static bool fresh_in_child(const char* dll, const std::string& options, int t, b // and the executor's output pass are skipped under the device's frame guard, and the fallback under the inverse // predication writes the bilinear upscale of THAT frame's colour into THAT frame's output texture; the game's own // copy of the output after our dispatch in the same list still runs (the readback below is that copy); -// 2. the refused frame: WG_ERROR_UNSUPPORTED (the path of a frame not taken: the front end's stand-in renders it; in +// 2. the refused frame: WG_ERROR_NOT_READY (this frame only: the front end's stand-in renders it, no takeover; in // this harness nothing does), exactly one captured log line names the refusal, no failure or fallback line, // nothing of ours in its output (the sentinel), the context kept; // 3. the reset frame (WG_OK): the network's output again, byte for byte a fresh context's first frame on the same @@ -2755,7 +2757,7 @@ bool case_bneck_timeout(Gpu& g, const char* dll, const std::string& options, boo CHECK(R + 2 <= T); if (R + 2 > T) return false; for (int t = 1; t <= T; ++t) { - const WgResult want = t == R ? WG_ERROR_UNSUPPORTED : WG_OK; + const WgResult want = t == R ? WG_ERROR_NOT_READY : WG_OK; if (res[t] != want) std::printf(" frame %d: result %d (want %d)\n", t, (int)res[t], (int)want); CHECK(res[t] == want); } @@ -2810,6 +2812,32 @@ bool case_bneck_timeout(Gpu& g, const char* dll, const std::string& options, boo return true; } +// An older version 2 front end: its WgFrontEndServices ends after split_command_list (no command_list_predication). +// The split executor's create must take it (the table is checked up to the split service), and its frames split; the +// appended predicate service is never called through the short table. +bool case_old_services(Gpu& g, const char* dll) { + Ctx c; + c.g = &g; + if (!c.be.load(dll)) return false; + Scene s; + s.make(g); + c.servicesSize = (uint32_t)offsetof(WgFrontEndServices, command_list_predication); + const WgResult cr = c.create(s, WG_CREATE_HDR | WG_CREATE_DEPTH_INVERTED, "executor=d3d12-test"); + std::printf(" version 2 table of %u bytes (current %u): create %d\n", c.servicesSize, (unsigned)sizeof(WgFrontEndServices), + (int)cr); + CHECK(cr == WG_OK); + if (!c.u) return false; + for (int t = 1; t <= 2; ++t) { + s.set_color(g, t); + WgResult r = WG_ERROR; + c.run(s, s.frame(g, t), &r); + CHECK(r == WG_OK); + } + CHECK(c.fe.splits == 2); + CHECK(c.fe.predQueries == 0); + return true; +} + // Model L hybrid, the game's predicate on its list (WgFrontEndServices::command_list_predication): the frame guard // replaces it while our frame records and sets it again after. The game sets SetPredication(buffer, 8, NOT_EQUAL_ZERO) // before our dispatch (the value at offset 8; offset 0 holds the opposite value, so a predicate set again at another @@ -2817,7 +2845,7 @@ bool case_bneck_timeout(Gpu& g, const char* dll, const std::string& options, boo // frame 1, predicate false (0: the game's conditional work runs): our output written, the marker copied; // frame 2, predicate true (1: skipped): our output written all the same (our frame records unconditionally), the // marker NOT copied (the game's predicate is in effect again after our dispatch); -// frame 3, the front end cannot tell the predicate (WG_ERROR_NOT_READY): WG_ERROR_UNSUPPORTED, nothing of ours +// frame 3, the front end cannot tell the predicate (WG_ERROR_NOT_READY): WG_ERROR_NOT_READY, nothing of ours // written, the game's predicate untouched (true: the marker not copied); // frame 4, predicate false again: the context kept, our output written, the marker copied. bool case_caller_predicate(Gpu& g, const char* dll, const std::string& options) { @@ -2866,7 +2894,7 @@ bool case_caller_predicate(Gpu& g, const char* dll, const std::string& options) bool written, copied; } steps[4] = {{0, WG_OK, WG_OK, true, true}, {1, WG_OK, WG_OK, true, false}, - {1, WG_ERROR_NOT_READY, WG_ERROR_UNSUPPORTED, false, false}, + {1, WG_ERROR_NOT_READY, WG_ERROR_NOT_READY, false, false}, {0, WG_OK, WG_OK, true, true}}; for (int t = 1; t <= 4; ++t) { const Step& st = steps[t - 1]; @@ -2954,6 +2982,7 @@ int main(int argc, char** argv) { {"pipelined_not_ready", [&] { return case_pipelined_not_ready(g, dll); }}, {"nohip", [&] { return case_create_fails(g, dll, true); }}, {"noservices", [&] { return case_create_fails(g, dll, false); }}, + {"oldservices", [&] { return case_old_services(g, dll); }}, {"timing", [&] { return case_timing(g, dll); }}, {"copies", [&] { return case_sync(g, dll, true, "executor=d3d12-test;input_copy=1;output_copy=1") && copied(); }}, {"copies_pipelined", From aa4a41e1fd8535dc3dbb413eb6c78b08c51ce80d Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Mon, 16 Nov 2026 02:58:20 -0500 Subject: [PATCH 5/5] Front end: a failed stand-in never turns a one-frame refusal into a takeover; the stage checker marks its real cases - DispatchWithFallback keeps WG_ERROR_NOT_READY when the stand-in cannot be created or fails its frame (never the stand-in's own error). The front end then gives that frame to AMD's FSR only (takeOver = false), else skips it; no Fail, no broken mark, no bad-frame count, and the next frame goes to the upscaler again with a reset (test_frontend TestNotReadyStandInFails: a stand-in that fails to create and one that fails to dispatch, with and without AMD's FSR). - l_hyb_stage_check: a --real-inputs run writes the provenance marker next to its private cases.txt before any real file; every run checks the marker and its private folder before anything in the case folder is read (test_l_private_out.py StageCheckMarker). Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/fsr/src/frontend.cpp | 24 +++++++++++++++++ clean/fsr/src/param_map.cpp | 4 +-- clean/fsr/src/param_map.h | 3 ++- clean/fsr/tests/test_frontend.cpp | 41 ++++++++++++++++++++++++++++- clean/l/runner/l_hyb_stage_check.py | 7 ++++- clean/l/tests/test_l_private_out.py | 29 ++++++++++++++++++++ 6 files changed, 103 insertions(+), 5 deletions(-) diff --git a/clean/fsr/src/frontend.cpp b/clean/fsr/src/frontend.cpp index bbef29bb..6afab443 100644 --- a/clean/fsr/src/frontend.cpp +++ b/clean/fsr/src/frontend.cpp @@ -1881,6 +1881,30 @@ ffxReturnCode_t FrontEnd::DispatchUpscale(UpscaleContext& u, const ffxDispatchDe Log(tag + " note U" + std::to_string(u.id) + ": AMD FSR is not available for this context; the stand-in upscales the frames the upscaler does not take"); } } + // WG_ERROR_NOT_READY whose stand-in failed (create or dispatch; DispatchWithFallback keeps the refusal's code): + // still THIS frame only, never a takeover: AMD's FSR for this frame (its context from the saved parameters, + // takeOver = false); when that fails too, the frame is skipped (the game's output keeps what it held). No Fail, no + // broken mark, no bad-frame count; the upscaler's next frame starts without history. + if (mapped && r == WG_ERROR_NOT_READY && u.fallbackBackend && u.fallbackBackend != u.backend) { + u.resetNext = true; + const std::string what = "frame " + std::to_string(frameIndex) + ": upscaler not ready and its stand-in failed"; + if (config_.fallbackToAmd && CreateAmdFromSaved(u, call, false) == FFX_API_RETURN_OK) { + ffxDispatchDescUpscale amd = d; + if (!u.amdLast) amd.reset = true; // AMD's history is old, or never started + u.amdLast = true; + const ffxReturnCode_t rc = u.amdApi->dispatch(&u.amd, &amd.header); + Log(tag + " note U" + std::to_string(u.id) + ": " + what + ": AMD FSR upscales this frame only -> " + RcName(rc) + + "; the context stays WrongGPU's"); + if (rc == FFX_API_RETURN_OK) { + Report(u, UpscaleState::Amd, nullptr); + return rc; + } + } + Log(tag + " note U" + std::to_string(u.id) + ": " + what + ": this frame is skipped (the game's output keeps what it held); " + "the context stays WrongGPU's"); + Report(u, UpscaleState::Failed, nullptr, what + "; skipped"); + return FFX_API_RETURN_OK; + } // Inputs the upscaler cannot take (the mapping's checks, or the upscaler's own: WG_ERROR_PARAMETER, // WG_ERROR_UNSUPPORTED) are one frame's matter (BadFrame), and so is WG_ERROR_NOT_READY that the stand-in could // not take above (none, or it failed): AMD's FSR for this frame only, the context stays the upscaler's. Every diff --git a/clean/fsr/src/param_map.cpp b/clean/fsr/src/param_map.cpp index 6154071e..fc781395 100644 --- a/clean/fsr/src/param_map.cpp +++ b/clean/fsr/src/param_map.cpp @@ -245,7 +245,7 @@ WgResult DispatchWithFallback(const WgUpscalerBackend* backend, WgUpscaler* upsc if (c != WG_OK || !*fallback) { *fallback = nullptr; note = "upscaler not ready this frame; the stand-in could not be created: " + WgResultText(c); - return c == WG_OK ? WG_ERROR : c; + return WG_ERROR_NOT_READY; // still this frame's answer: never the stand-in's error (a refusal stays one) } } AdaptFrame(standIn, frame); @@ -253,7 +253,7 @@ WgResult DispatchWithFallback(const WgUpscalerBackend* backend, WgUpscaler* upsc AdaptFrame(backend, frame); note = s == WG_OK ? "upscaler not ready this frame (WG_ERROR_NOT_READY): stand-in for this frame" : "upscaler not ready this frame; the stand-in failed too: " + WgResultText(s); - return s; + return s == WG_OK ? WG_OK : WG_ERROR_NOT_READY; // a failed stand-in never turns the refusal into its own error } bool MapDispatch(const ffxDispatchDescUpscale& d, const UpscaleCreateParams& create, const Config& config, diff --git a/clean/fsr/src/param_map.h b/clean/fsr/src/param_map.h index 04f97180..c5fc84c4 100644 --- a/clean/fsr/src/param_map.h +++ b/clean/fsr/src/param_map.h @@ -60,7 +60,8 @@ void AdaptFrame(const WgUpscalerBackend* backend, WgFrame& frame); // Dispatches `frame` on `backend`. When it answers WG_ERROR_NOT_READY (version 2: e.g. the command list cannot be // split this frame), the stand-in upscales this one frame instead: it is created on first need from `info` (the // context's create info; services are not passed on) and kept in *fallback (destroy it with the context). `note` -// then says so. Returns the backend's result, or the stand-in's after WG_ERROR_NOT_READY. +// then says so. Returns the backend's result; after WG_ERROR_NOT_READY, WG_OK when the stand-in took the frame, else +// WG_ERROR_NOT_READY still (the stand-in's create or dispatch failed: `note` says which; never the stand-in's error). WgResult DispatchWithFallback(const WgUpscalerBackend* backend, WgUpscaler* upscaler, WgFrame& frame, const WgUpscalerBackend* standIn, const WgCreateInfo& info, WgUpscaler** fallback, std::string& note); diff --git a/clean/fsr/tests/test_frontend.cpp b/clean/fsr/tests/test_frontend.cpp index f15b109f..34683e32 100644 --- a/clean/fsr/tests/test_frontend.cpp +++ b/clean/fsr/tests/test_frontend.cpp @@ -212,14 +212,17 @@ static const WgUpscalerBackend* g_useBackend = &g_backend; static std::vector g_standInFrames; static WgCreateInfo g_standInCreate{}; static int g_standInDestroyed = 0; +static WgResult g_standInCreateResult = WG_OK; // StandInCreate answers this (a stand-in that cannot be made) +static WgResult g_standInDispatchResult = WG_OK; // StandInDispatch answers this (a stand-in that fails its frame) static WgResult StandInCreate(const WgCreateInfo* info, WgUpscaler** out) { g_standInCreate = *info; + if (g_standInCreateResult != WG_OK) return g_standInCreateResult; *out = reinterpret_cast(new FakeUpscaler{2}); return WG_OK; } static WgResult StandInDispatch(WgUpscaler*, const WgFrame* frame) { g_standInFrames.push_back(*frame); - return WG_OK; + return g_standInDispatchResult; } static void StandInDestroy(WgUpscaler* u) { ++g_standInDestroyed; @@ -393,6 +396,7 @@ static void Reset() { g_standInFrames.clear(); g_standInCreate = WgCreateInfo{}; g_standInDestroyed = 0; + g_standInCreateResult = g_standInDispatchResult = WG_OK; g_servicesAsked.clear(); g_notes.clear(); g_amdCreateResult = FFX_API_RETURN_OK; @@ -1759,6 +1763,40 @@ static void TestInterfaceVersions() { CHECK(fe2->DestroyContext(&ctx, nullptr) == FFX_API_RETURN_OK); } +// WG_ERROR_NOT_READY (one frame only) whose stand-in FAILS never becomes a takeover: the stand-in's error is not +// passed on, AMD's FSR takes that frame only (takeOver = false), and without AMD's FSR the frame is skipped (FFX OK, the +// output as it was); no Fail, no broken mark, and the next frame goes to the upscaler again, with a reset. The stand-in +// fails its dispatch with WG_ERROR_UNSUPPORTED (an error that would otherwise reach the permanent handoff), or cannot be +// created (WG_ERROR_MEMORY: an error that would otherwise mark the upscaler broken). +static void TestNotReadyStandInFails() { + for (int amd = 1; amd >= 0; --amd) + for (int failCreate = 0; failCreate < 2; ++failCreate) { + Reset(); + g_originalAvailable = amd != 0; + if (failCreate) g_standInCreateResult = WG_ERROR_MEMORY; + else g_standInDispatchResult = WG_ERROR_UNSUPPORTED; + auto fe = MakeFrontEnd(Config{}); + UpscaleCreate c(0); + ffxContext ctx = nullptr; + CHECK(fe->CreateContext(&ctx, &c.upscale.header, nullptr) == FFX_API_RETURN_OK); + g_notReadyUntil = 2; // frames 0 and 1 refused (one frame each) + ffxDispatchDescUpscale d = MakeDispatch(); + for (int i = 0; i < 4; ++i) CHECK(fe->Dispatch(&ctx, &d.header) == FFX_API_RETURN_OK); + // every frame reached the upscaler: the context stayed WrongGPU's + CHECK(g_frames.size() == 4); + CHECK(g_frames.size() == 4 && g_frames[2].reset != 0 && g_frames[3].reset == 0); + size_t amdDispatches = 0; + for (const AmdCall& a : g_amd) amdDispatches += a.fn == "dispatch"; + CHECK(amdDispatches == (amd ? 2u : 0u)); // AMD's FSR for the two refused frames only + CHECK(CountLog("its stand-in failed") == 2); + CHECK(CountLog(amd ? "AMD FSR upscales this frame only" : "this frame is skipped") == 2); + CHECK(CountLog("AMD FSR upscales this context") == 0 && CountLog("for the rest of its life") == 0); + CHECK(CountLog(" ERROR U") == 0 && CountLog("ERROR:") == 0); + CHECK(g_destroyed == 0); + CHECK(fe->DestroyContext(&ctx, nullptr) == FFX_API_RETURN_OK); + } +} + // Non-linear (perceptual) colour goes to AMD's FSR (ANSWERS.md, spec 20 item 5): at create (context flag) and per // frame (dispatch flags), logged as a note, not an error. static void TestNonLinearColour() { @@ -4259,6 +4297,7 @@ int main() { TestWorkerLaunchFailureGivesTheDeviceBack(); TestDeviceHoldAllocationFailure(); TestInterfaceVersions(); + TestNotReadyStandInFails(); TestAmdMode(); TestNonLinearColour(); TestMappingDetails(); diff --git a/clean/l/runner/l_hyb_stage_check.py b/clean/l/runner/l_hyb_stage_check.py index ecdb5ba6..c671d690 100644 --- a/clean/l/runner/l_hyb_stage_check.py +++ b/clean/l/runner/l_hyb_stage_check.py @@ -374,13 +374,18 @@ def main(argv=None): a = ap.parse_args(argv) # the synthetic-values rule: a real run's case files and replay outputs only in a private folder outside the # repository, checked before anything is read or built + from clean.l.runner.l_private import guard_cases, mark_real, private_dir, private_json cdir = None if a.real_inputs: - from clean.l.runner.l_private import private_dir, private_json private_json(a.json) # the record of a real run outside the repository cdir = private_dir(a.private_out, what="--real-inputs: the real cases' files and replay outputs") elif a.private_out: raise SystemExit("--private-out: only with --real-inputs") + # a case folder holding real cases (the marker next to cases.txt) is read or written only as the private folder of + # a --real-inputs run, checked before anything there is read + guard_cases(os.path.join(cdir or os.path.join(a.build, "hyb_stage"), "cases.txt"), a.private_out) + if a.real_inputs and a.run: + mark_real(os.path.join(cdir, "cases.txt"), "l_hyb_stage_check --real-inputs") # before any real file if a.set == "fused": kernels = FUSED_MUT if a.mutant else FUSED_KERNELS else: diff --git a/clean/l/tests/test_l_private_out.py b/clean/l/tests/test_l_private_out.py index dde9bd68..fe94122b 100644 --- a/clean/l/tests/test_l_private_out.py +++ b/clean/l/tests/test_l_private_out.py @@ -179,6 +179,35 @@ def test_guard_cases(self): P.guard_cases(plain, None) +class StageCheckMarker(unittest.TestCase): + """l_hyb_stage_check: a --real-inputs run marks its private case folder (cases.txt.real) before it writes any real + file; a run on a marked case folder that is not its --private-out refuses before anything there is read (the + module's open and numpy's load fail the test if called).""" + + def test_real_run_marks_its_folder(self): + d = outside_tmp(self) + inputs, priv = os.path.join(d, "inputs"), os.path.join(d, "priv") + os.makedirs(inputs) # no real input files: nothing real is read; the run only lays out its folder + HS.main(["--build", d, "--run", "--no-replay", "--real-inputs", inputs, "--private-out", priv]) + self.assertTrue(os.path.exists(os.path.join(priv, "cases.txt.real"))) + + def test_marked_build_folder_refused_before_the_read(self): + d = outside_tmp(self) + cdir = os.path.join(d, "hyb_stage") + os.makedirs(cdir) + with open(os.path.join(cdir, "cases.txt"), "w") as f: + f.write("constructed") + P.mark_real(os.path.join(cdir, "cases.txt"), "test") + fail = AssertionError("a file of the real case folder was read") + with mock.patch.object(HS, "open", create=True, side_effect=fail), \ + mock.patch.object(HS.np, "load", side_effect=fail), \ + mock.patch.object(HS.np, "fromfile", side_effect=fail): + for mode in (["--run", "--no-replay"], ["--run", "--replay-only"], ["--run"]): + with self.subTest(mode=mode), self.assertRaises(SystemExit) as cm: + HS.main(["--build", d] + mode) + self.assertIn("prepared from real data", str(cm.exception)) + + class RealRecords(unittest.TestCase): """A real run's --json goes outside the repository (refused inside it before anything is read or written).""" J = os.path.join(ROOT, "clean", "l", "records", "real-run-test.json")