diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index 9e939702..3781dcd4 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -45,6 +45,20 @@ into , its manifest or a payload; removed after the scan); the build fails when it is not declared, gives no ISA, or PASSES rule 2, or rule 1 (unless K has no norm tree: HYB_NO_NORM). + +ISA check: every kernel built that clean/dx12/isa_checks.json lists is run through RGA (gfx1201) and the number of +instructions whose mnemonic matches each of its patterns must equal the listed count (Model K's softmax weights: one +v_fma_mixlo/hi_f16 per weight, the driver's fold that makes the mix form exact; none in the __rodd ports). A different +count fails the build. WG_DX12_ISA_CHECK=0 skips it. + +Provenance: every manifest entry records "source_digest" (source_digest(): its declaration, the text of its source file and +#include closure, and the build's extra defines) and "defines" (the extra defines, {} for a normal build). A build with +--define, or with --test-only, writes a TEST-ONLY manifest ("test_only": true; a partial build keeps the flag, only a full +build without them clears it). Model K's softmax self-check reference must belong to the sources of a self-check kernel +built (smx_digest_check); WG_DX12_SMX_DIGEST=0 lets a negative-control build through only together with --test-only, and +marks those entries "smx_bypass". clean/runner/build.py never embeds a test-only manifest, a bypassed, define-built or +unprovenanced entry, or one whose source digest differs from this tree's (dx12_manifest_problems); the payload packager +refuses a library whose build manifest does not record that check (clean/tests/check_no_dev_diagnostics.py). """ import argparse import concurrent.futures @@ -351,6 +365,129 @@ def scan_mutant(name): return name.endswith("_scanmut") +ISA_CHECKS = os.path.join(HERE, "isa_checks.json") + + +def rga_mnemonics(dxil): + """The mnemonics of one kernel's RGA (gfx1201) ISA, in order; None when RGA gives no ISA.""" + with tempfile.TemporaryDirectory(prefix="wgrga_") as tmp: + subprocess.run([RGA, "-s", "dx12", "-c", "gfx1201", "--cs-blob", dxil, "--isa", os.path.join(tmp, "x.txt")], + capture_output=True, text=True) + for f in os.listdir(tmp): + if f.endswith("_x_comp.txt"): + out = [] + with open(os.path.join(tmp, f), errors="replace") as fh: + for line in fh: + t = line.split("//")[0].strip() + if re.match(r"^[sv]_|^(buffer|global|ds|image|scratch|flat)_", t): + out.append(t.split()[0]) + return out + return None + + +def smx_digest_check(names, defines, test_only=False): + """Model K's softmax self-check reference (clean/k/runtime/k_smx_selfcheck.h) must belong to the sources and defines of + this build when it builds one of the self-check kernels (clean/k/dx12/smx_selfcheck_digest.py); a stale header would + let the runtime compare its stage shaders with values from other sources. WG_DX12_SMX_DIGEST=0 lets a negative- + control build through only into a test-only output (test_only: --test-only); returns the self-check kernels built + past a stale header that way (their entries say "smx_bypass"), else an empty set.""" + root = os.path.dirname(os.path.dirname(HERE)) + if root not in sys.path: + sys.path.insert(0, root) + from clean.k.dx12 import smx_selfcheck_digest as D + checked = set(names) & set(D.KERNELS) + why = D.stale(defines=defines) if checked else None + if not why: + return set() + if os.environ.get("WG_DX12_SMX_DIGEST", "1") != "0": + raise SystemExit("softmax self-check reference: " + why) + if not test_only: + raise SystemExit("softmax self-check reference: WG_DX12_SMX_DIGEST=0 builds past a stale header only into a " + "test-only output: pass --test-only (the manifest then says test_only, and clean/runner/build.py " + "and the payload packager refuse it)") + return checked + + +_INCLUDE = re.compile(r'^\s*#\s*include\s+"([^"]+)"', re.M) + + +def source_digest(src, decl, defines, gen_dir=None, _cache=None): + """SHA-256 (hex) of what one shader is compiled from: its declaration (kernel_decls), the text of its source file and + of its #include closure (looked up as DXC does: the including file's folder, clean/dx12/hlsl, and for a Model K + source clean/k/dx12/hlsl and the build's generated folder gen_dir), and the extra defines ({name: value}). Line ends + are normalised; paths enter relative to the repository (a generated include as gen/), so the digest does not + depend on where the clone or the build folder is.""" + root = os.path.dirname(os.path.dirname(HERE)) + k_source = os.path.normcase(os.path.abspath(src)).startswith(os.path.normcase(os.path.abspath(K_KERNEL_DIR))) + dirs = [HLSL_DIR] + ([K_HLSL_DIR] + ([gen_dir] if gen_dir else []) if k_source else []) + cache = {} if _cache is None else _cache + + def text(path): + if path not in cache: + with open(path, "rb") as f: + cache[path] = f.read().replace(b"\r\n", b"\n") + return cache[path] + + def label(path): + if gen_dir and os.path.normcase(os.path.abspath(path)).startswith(os.path.normcase(os.path.abspath(gen_dir))): + return "gen/" + os.path.relpath(path, gen_dir).replace(os.sep, "/") + return os.path.relpath(path, root).replace(os.sep, "/") + + seen, order, missing = set(), [], set() + + def walk(path): + key = os.path.normcase(os.path.abspath(path)) + if key in seen: + return + seen.add(key) + order.append(path) + for inc in _INCLUDE.findall(text(path).decode("utf-8", "replace")): + for d in [os.path.dirname(path)] + dirs: + cand = os.path.normpath(os.path.join(d, inc)) + if os.path.isfile(cand): + walk(cand) + break + else: + missing.add(inc) + + walk(src) + h = hashlib.sha256() + h.update(b"decl\n" + json.dumps(decl, sort_keys=True).encode() + b"\n") + for path in sorted(order, key=label): + h.update(b"file " + label(path).encode() + b"\n" + text(path) + b"\n") + h.update(b"missing " + ",".join(sorted(missing)).encode() + b"\n") + h.update(b"defines\n" + "\n".join("%s=%s" % kv for kv in sorted(defines.items())).encode() + b"\n") + return h.hexdigest() + + +def isa_check(built): + """built: [(kernel name, dxil path)] of this build. Fails when a kernel listed in isa_checks.json has another number + of instructions matching one of its patterns.""" + if os.environ.get("WG_DX12_ISA_CHECK", "1") == "0": + return + with open(ISA_CHECKS, encoding="utf-8") as f: + checks = {k: v for k, v in json.load(f).items() if not k.startswith("_")} + todo = [(n, p) for n, p in built if n in checks] + if not todo: + return + if not os.path.exists(RGA): + raise SystemExit(f"ISA check: RGA not found at {RGA} (WG_RGA, or WG_DX12_ISA_CHECK=0 to skip)") + with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, min(6, int(os.environ.get("WG_DX12_JOBS", 6))))) as ex: + isa = dict(zip([n for n, _ in todo], ex.map(rga_mnemonics, [p for _, p in todo]))) + bad = [] + for n in sorted(isa): + if isa[n] is None: + bad.append(f"{n}: RGA gave no ISA") + continue + for pat, want in checks[n].items(): + got = sum(1 for m in isa[n] if re.match(pat, m)) + print(f"ISA {n:40s} {pat} {got} / {want}", flush=True) + if got != want: + bad.append(f"{n}: {got} instructions match {pat}, {want} expected ({os.path.basename(ISA_CHECKS)})") + if bad: + raise SystemExit("ISA check failed:\n " + "\n ".join(bad)) + + def vgpr_check(built, required=()): """built: [(kernel name, dxil path)] of this build. Fails when a Swin / bottleneck kernel exceeds its bound, when a hybrid kernel breaks an ISA rule, and when a kernel of `required` (the model hybrid kernels built) has no built @@ -441,6 +578,9 @@ def main(argv=None): ap.add_argument("--define", action="append", default=[], metavar="NAME=VALUE", help="an extra preprocessor define for every kernel built (test variants: negative-control mutants, " "test counters; never for a shipped build)") + ap.add_argument("--test-only", action="store_true", + help="mark the output test-only (the manifest says test_only: clean/runner/build.py never embeds it and " + "the payload packager refuses it); required for WG_DX12_SMX_DIGEST=0; implied by --define") ap.add_argument("--list", action="store_true", help="print the kernels a build with these options compiles (name and code object, JSON) and exit: " "the cheap check before a build (kernel_sets.json, tests/test_kernel_sets.py)") @@ -461,9 +601,12 @@ def main(argv=None): only = set(a.only.split(",")) if a.only else None man_path = os.path.join(dst, MANIFEST) old = {} - if only and os.path.exists(man_path): # partial build: keep the other kernels' entries + old_test_only = False + if only and os.path.exists(man_path): # partial build: keep the other kernels' entries (and a test-only flag) with open(man_path) as f: - old = {(k["name"], k.get("co", "")): k for k in json.load(f)["kernels"]} + old_doc = json.load(f) + old = {(k["name"], k.get("co", "")): k for k in old_doc["kernels"]} + old_test_only = old_doc.get("test_only") is True kernels = dict(old) # (name, co) -> manifest entry built = [] # generated kernels (kernels/fused_*_gen.py: the fused forms of swin_block.hlsl / bneck.hlsl) are regenerated from @@ -497,6 +640,9 @@ def main(argv=None): raise SystemExit(f"kernel {d['name']} ({d['co']}): DXIL file {os.path.basename(out)} already used " "(a second port of a name needs file=)") jobs.append((src, d, out, k_source)) + bypassed = smx_digest_check({j[1]["name"] for j in jobs}, a.define, a.test_only) + test_only = bool(a.test_only or extra_defs or bypassed or old_test_only) + digest_cache = {} # the ISA rules' mutants beside every build of hybrid kernels: into a temporary folder of their own, never into the # build's output (a release folder, its manifest and a payload hold no check object); removed after the scan scan_dir = None @@ -522,7 +668,12 @@ def build_one(job): rga_isa(out, os.path.join(dst, "isa"), os.path.basename(out)[:-5]) r = {"name": d["name"], "file": os.path.basename(out), "co": d["co"], "entry": d["entry"], "block": d["block"], "wave": d["wave"], "args": d["args"], "arg_dwords": d["arg_dwords"], - "source": d["source"], "sha256": sha, "rs": d["rs"]} + "source": d["source"], "sha256": sha, "rs": d["rs"], + # provenance (module doc): what it was compiled from, and the extra defines of this build + "source_digest": source_digest(src, d, extra_defs, k_gen if k_source else None, digest_cache), + "defines": dict(sorted(extra_defs.items()))} + if d["name"] in bypassed: + r["smx_bypass"] = True # built past a stale self-check reference (WG_DX12_SMX_DIGEST=0, --test-only) if d["group"] != d["block"]: r["group"] = d["group"] r["gscale"] = d["gscale"] @@ -557,13 +708,14 @@ def build_one(job): raise SystemExit(f"not declared in clean/dx12/kernels or clean/k/dx12/kernels: {sorted(only - names)}") try: vgpr_check(built, {d["name"] for _, d, _, _ in jobs if hyb_scanned(d["name"]) and not d["probe"]}) + isa_check(built) finally: if scan_dir: shutil.rmtree(scan_dir, ignore_errors=True) with open(man_path, "w") as f: json.dump({"format": FORMAT, "dxc": os.path.basename(os.path.dirname(os.path.dirname(DXC_DIR))), - "kernels": [kernels[k] for k in sorted(kernels)]}, f, indent=1) - print(f"{len(kernels)} kernel(s) in {man_path}") + "test_only": test_only, "kernels": [kernels[k] for k in sorted(kernels)]}, f, indent=1) + print(f"{len(kernels)} kernel(s) in {man_path}" + (" (TEST-ONLY: never embedded or packaged)" if test_only else "")) return 0 diff --git a/clean/dx12/isa_checks.json b/clean/dx12/isa_checks.json new file mode 100644 index 00000000..c9c4a79e --- /dev/null +++ b/clean/dx12/isa_checks.json @@ -0,0 +1,25 @@ +{ + "_doc": "build_dx12.py ISA check: for each kernel listed, the RGA (gfx1201) ISA must contain exactly the given number of instructions whose mnemonic matches each pattern (a regular expression, matched at the start of the mnemonic). Model K's softmax weights (clean/k/dx12/hlsl/k_smx.hlsli) are exact only when the driver folds the mix form p + e into one v_fma_mixlo/hi_f16 rounding per weight: 32 weights per lane per softmax round, i.e. 32 x heads x windows per workgroup for the narrow stages (e2 / d2: 2 heads, 1 window; d1: 2 heads, 2 windows) and 32 x heads / NG for the wide stages (e3 / d3 and the NG = 2 e4 / d4: 2 rounds; NG = 4 e4 / d4: 1; m: 8 heads / NG 4 = 2). A missing fold leaves v_cvt_f16_f32 after an FP32 add (a second rounding) and fails here. The __rodd ports (round to odd) must have none. The native D3D12 runtime checks the same fold on the player's driver at create (k_mix_probe). WG_DX12_ISA_CHECK=0 skips the check.", + "k_stage_e2": {"v_fma_mix(lo|hi)_f16": 64}, + "k_stage_d2": {"v_fma_mix(lo|hi)_f16": 64}, + "k_stage2_d1": {"v_fma_mix(lo|hi)_f16": 128}, + "k_stagew2_e3": {"v_fma_mix(lo|hi)_f16": 64}, + "k_stagew4_e4": {"v_fma_mix(lo|hi)_f16": 32}, + "k_stagew2_e4": {"v_fma_mix(lo|hi)_f16": 64}, + "k_stagew4_m": {"v_fma_mix(lo|hi)_f16": 64}, + "k_stagew4_d4": {"v_fma_mix(lo|hi)_f16": 32}, + "k_stagew2_d4": {"v_fma_mix(lo|hi)_f16": 64}, + "k_stagew2_d3": {"v_fma_mix(lo|hi)_f16": 64}, + "k_mix_probe": {"v_fma_mix(lo|hi)_f16": 8}, + "k_stage_e2__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stage_d2__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stage2_d1__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stagew2_e3__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stagew4_e4__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stagew2_e4__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stagew4_m__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stagew4_d4__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stagew2_d4__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stagew2_d3__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_mix_probe__rodd": {"v_fma_mix(lo|hi)_f16": 0} +} diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index 881354c2..db66536c 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -416,6 +416,14 @@ "k_expo_scalars_direct", "k_io.co" ], + [ + "k_mix_probe", + "k_block.co" + ], + [ + "k_mix_probe__rodd", + "k_block.co" + ], [ "k_out_color", "k_io.co" @@ -452,6 +460,10 @@ "k_stage2_d1", "k_block.co" ], + [ + "k_stage2_d1__rodd", + "k_block.co" + ], [ "k_stage2_e0", "k_block.co" @@ -468,6 +480,10 @@ "k_stage_d2", "k_block.co" ], + [ + "k_stage_d2__rodd", + "k_block.co" + ], [ "k_stage_e0", "k_block.co" @@ -480,34 +496,66 @@ "k_stage_e2", "k_block.co" ], + [ + "k_stage_e2__rodd", + "k_block.co" + ], [ "k_stagew2_d3", "k_block_wide.co" ], + [ + "k_stagew2_d3__rodd", + "k_block_wide.co" + ], [ "k_stagew2_d4", "k_block_wide.co" ], + [ + "k_stagew2_d4__rodd", + "k_block_wide.co" + ], [ "k_stagew2_e3", "k_block_wide.co" ], + [ + "k_stagew2_e3__rodd", + "k_block_wide.co" + ], [ "k_stagew2_e4", "k_block_wide.co" ], + [ + "k_stagew2_e4__rodd", + "k_block_wide.co" + ], [ "k_stagew4_d4", "k_block_wide.co" ], + [ + "k_stagew4_d4__rodd", + "k_block_wide.co" + ], [ "k_stagew4_e4", "k_block_wide.co" ], + [ + "k_stagew4_e4__rodd", + "k_block_wide.co" + ], [ "k_stagew4_m", "k_block_wide.co" ], + [ + "k_stagew4_m__rodd", + "k_block_wide.co" + ], [ "k_warm", "k_block_wide.co" diff --git a/clean/installer/tests/run_installer_tests.py b/clean/installer/tests/run_installer_tests.py index 8576309d..ba274ef6 100644 --- a/clean/installer/tests/run_installer_tests.py +++ b/clean/installer/tests/run_installer_tests.py @@ -332,7 +332,7 @@ def stand_in_runtime(folder, data, name="wronggpu_runtime.dll"): data = data if data.endswith(RELEASE_MARKER) else data + RELEASE_MARKER write(os.path.join(folder, name), data) manifest = {"format": 1, "file": name, "sha256": hashlib.sha256(data).hexdigest(), "size": len(data), "dev_diagnostics": False, - "stand_in": True} + "dx12_kernels": None, "stand_in": True} write(os.path.join(folder, os.path.splitext(name)[0] + ".build.json"), json.dumps(manifest, indent=1).encode()) diff --git a/clean/k/dx12/README.md b/clean/k/dx12/README.md index ec514023..07cc0f4e 100644 --- a/clean/k/dx12/README.md +++ b/clean/k/dx12/README.md @@ -59,6 +59,50 @@ real textures (RGBA16F colour, R32F depth, RG16F motion, exposure, an RGBA16F ou mode and compares the output texture and the buffers with the HIP kernels' run on the raw buffer route (`KIO_C16` / `KIO_M16`, with the motion scale and offset folded as the interop passes them). +`mixprobe` (D3D12 only, no HIP kernel: checked against the exact rounding): the softmax weight rounding of every +softmax stage port, A = f16(w inv) rounded once from the exact product (`hlsl/k_smx.hlsli`). The ports write it as the +FP32 product plus its exact residual fed into the driver's round-to-nearest-even fragment conversion, which the gfx1201 +compiler folds into one `v_fma_mixlo/hi_f16` (the mix form); the `__rodd` ports round the product to odd first (the +form the ports had before; the same bits). `kernels/k_mix_probe.hlsl` runs the same helper on the records of +`k_mix_cases.py`: products whose FP32 rounding is an FP16 midpoint with a residual of either sign (normal and subnormal +FP16), exact midpoints (ties to even) and ordinary products; every result must equal the exact single rounding +(Fraction arithmetic), for `k_mix_probe` and `k_mix_probe__rodd`, and the records must tell a second rounding through +FP32 apart (the residual-dropped control `K_SMX_MUT=1` fails this family). The build's ISA check +(`clean/dx12/isa_checks.json`: one `v_fma_mix*_f16` per weight in every mix-form port, none in the `__rodd` ports) keeps +the fold in the shipped DXIL as RGA compiles it. + +The player's driver compiles every stage shader on its own, so neither form is taken on trust there: at create the native +D3D12 runtime checks the shaders that will run (`clean/k/runtime/runtime_k.cpp`, "create-time self-check"). It runs +`k_mix_probe` and `k_mix_probe__rodd` on the records of `clean/k/runtime/k_mix_probe.h`, and every softmax stage kernel it +can select, in both forms, at the window offsets this model's frames use: `k_stage_e2`, `k_stagew2_e3`, `k_stagew2_d3`, +`k_stage_d2`, `k_stage2_d1` at (4, 4) and `k_stagew4_e4`, `k_stagew4_d4` (8 data sets) at a 512 x 384 base map; +`k_stagew2_e4`, `k_stagew2_d4` and `k_stagew4_m` at 1536 x 1280; e4 / d4 and m at the offsets the model's window table +gives them (`k_stage_offsets` over its eight entries). The cases run on the runtime's own synthetic inputs and weights, +and their outputs' hashes must equal the HIP reference's (`clean/k/runtime/k_smx_selfcheck.h`, written by +`gen_k_smx_selfcheck.py` for every offset any window table can give). The mix form is used when its probe and all its +cases match, else the `__rodd` ports when theirs all do (kernels line `smx=rodd(...)` and one note of create, +`wg_rt_notes`, which the executor logs), else create fails and names the first case that differs. + +Before that, native create prepares every kernel the context can select (`Device::prepare`: the shader loaded, its +SHA-256 checked against the manifest, its pipeline compiled), the fixed-weight stages e0, e1 and d0 included: a missing +or corrupt shader fails create in one line naming it, not the first frame. + +The reference is bound to its sources: `k_smx_selfcheck.h` records `kSourceDigest`, the SHA-256 of the self-check +kernels' `@wg-kernel` lines, their sources' include closure, the runtime regions marked `SMX-SELFCHECK-SOURCE` (the cases, +the synthetic data, the block parameters, the kernel-name and window-offset rules) and a build's extra defines +(`smx_selfcheck_digest.py`). `build_dx12.py` (building a self-check kernel) and `build.py` (the runtime with model K) +refuse a header whose digest differs; `WG_DX12_SMX_DIGEST=0` lets a negative-control DXIL build through only with +`--test-only`, into a test-only output (the manifest says `test_only` and the entries `smx_bypass`; +`clean/k/tests/test_k_smx_selfcheck_digest.py`). Every manifest entry records its `source_digest` and extra `defines`; +`build.py` never embeds a test-only, bypassed, define-built, unprovenanced or stale entry, and the payload packager +refuses a library whose build manifest does not record that check (`clean/tests/test_dev_diagnostics_off.py`). + +The test hook `k_smx_test` (`mix_fail`, `both_fail`, `rodd_unchecked`, `report`) exists only in runtime libraries built +with `build.py --dev-diagnostics`; a release build refuses it (`test_k_runtime_d3d12.py`: +`test_native_softmax_switch_is_logged`, `test_native_softmax_refusals`, `test_native_fallback_frames_*`). Negative +controls, never shipped: `K_SMX_MUT=1` (the mix form's residual dropped), `K_SMX_MUT4=1` (dropped only at window offset +4), `K_SMX_RODD_MUT=1` (the `__rodd` ports' inexact bit dropped). + NaN: IEEE 754 does not define the sign of a NaN that a stage generates or propagates, and where the input holds NaN the two devices can differ in it (the decoder stages). For the cases with NaN in the input the check compares the raw bytes and, where they differ, accepts only the elements that are NaN on both sides and differ in the sign bit alone (the diff --git a/clean/k/dx12/gen_k_mix_probe.py b/clean/k/dx12/gen_k_mix_probe.py new file mode 100644 index 00000000..73796125 --- /dev/null +++ b/clean/k/dx12/gen_k_mix_probe.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: MIT +"""Writes clean/k/runtime/k_mix_probe.h: the records and exact expected FP16 results of the native D3D12 runtime's +softmax rounding probe (kernels/k_mix_probe.hlsl, run at create; k_mix_cases.py, seed 7, 64 lanes = two workgroups). +WrongGPU's own code. clean/k/tests/test_k_mix_probe_table.py checks the committed header against this output. + + python clean/k/dx12/gen_k_mix_probe.py [out.h] + python clean/k/dx12/gen_k_mix_probe.py --check (exit 1 unless the committed header equals this output) +""" +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) +from clean.k.dx12 import k_mix_cases # noqa: E402 + +SEED, LANES = 7, 64 +DEFAULT_OUT = os.path.join(ROOT, "clean", "k", "runtime", "k_mix_probe.h") + + +def header(): + rec, want, kinds = k_mix_cases.cases(SEED, LANES) + dbl = k_mix_cases.double_rounding_bits(rec) + words = rec.view("", + "", + "namespace k_mix_probe {", + "", + "constexpr int kLanes = %d;" % LANES, + "constexpr uint32_t kRecords[kLanes * 12] = {"] + for i in range(0, len(words), 8): + out.append(" " + ", ".join("0x%08xu" % w for w in words[i:i + 8]) + ",") + out.append("};") + out.append("constexpr uint16_t kExpected[kLanes * 8] = {") + for i in range(0, len(exp), 8): + out.append(" " + ", ".join("0x%04x" % w for w in exp[i:i + 8]) + ",") + out += ["};", "", "} // namespace k_mix_probe", ""] + return "\n".join(out) + + +if __name__ == "__main__": + if sys.argv[1:] == ["--check"]: + with open(DEFAULT_OUT, encoding="utf-8", newline="") as f: + same = f.read() == header() + print(DEFAULT_OUT, "matches the generator" if same else "DIFFERS from the generator") + sys.exit(0 if same else 1) + path = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_OUT + with open(path, "w", newline="\n") as f: + f.write(header()) + print(path) diff --git a/clean/k/dx12/gen_k_smx_selfcheck.py b/clean/k/dx12/gen_k_smx_selfcheck.py new file mode 100644 index 00000000..d5bd82aa --- /dev/null +++ b/clean/k/dx12/gen_k_smx_selfcheck.py @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: MIT +"""Writes clean/k/runtime/k_smx_selfcheck.h: the reference hashes of model K's softmax self-check (runtime_k.cpp +"create-time self-check"; KRuntime::smx_select) and the source digest they belong to. Each case runs one softmax stage +kernel at one window offset on the runtime's own synthetic inputs and weights; the header holds the SHA-256 (first 8 +bytes) of its outputs on the HIP reference, for every offset any model's window table can give. The native D3D12 +device compares both of its forms (mix and __rodd) at its model's offsets with these values at create. kSourceDigest +(smx_selfcheck_digest.py) binds the values to the shader sources, definitions and runtime regions they came from; the +builds refuse a header whose digest differs. WrongGPU's own code. + + python clean/k/dx12/gen_k_smx_selfcheck.py --build (GPU: a runtime library built from this tree with + clean/runner/build.py --dev-diagnostics, HIP code objects, ROCM_PATH; runs the HIP backend with the developer + hook k_smx_test=report and writes the header) + python clean/k/dx12/gen_k_smx_selfcheck.py --bootstrap (a placeholder header: lets a first developer build + exist; a native D3D12 create refuses with it) +""" +import argparse +import ctypes +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) +from clean.k.dx12 import smx_selfcheck_digest as D # noqa: E402 + +OUT = os.path.join(ROOT, D.HEADER) +TAG = "model K self-check reference:" + + +def header(entries, source_digest): + out = ["// SPDX-License-Identifier: MIT", + "// Generated by clean/k/dx12/gen_k_smx_selfcheck.py; do not edit. The reference hashes (SHA-256 of the outputs,", + "// first 8 bytes) of model K's softmax self-check cases at every possible window offset (runtime_k.cpp kSmxCases,", + "// KRuntime::smx_select) on the HIP reference, and the source digest they belong to (smx_selfcheck_digest.py).", + "#pragma once", + "", + "#include ", + "", + "namespace k_smx_selfcheck {", + "", + 'constexpr char kSourceDigest[] = "%s";' % source_digest, + "constexpr int kCount = %d;" % len(entries), + "constexpr const char* kNames[kCount] = {"] + out += [' "%s",' % n for n, _ in entries] + out += ["};", "constexpr uint64_t kExpected[kCount] = {"] + out += [" 0x%016xull," % v for _, v in entries] + out += ["};", "", "} // namespace k_smx_selfcheck", ""] + return "\n".join(out) + + +def reference(build): + import tempfile + + from clean.k.reference.network import random_network_params + from clean.k.tests.test_k_runtime_gpu import KRuntimeExt, write_k_model + mdir = tempfile.mkdtemp(prefix="k_smx_ref_") + _, map_path = write_k_model(mdir, random_network_params(11)) + rt = KRuntimeExt(build, mdir, map_path, (200, 120), (130, 78), True, False, False, "k_smx_test=report", backend=0) + rt.lib.wg_rt_notes.argtypes = [ctypes.c_void_p] + rt.lib.wg_rt_notes.restype = ctypes.c_char_p + notes = rt.lib.wg_rt_notes(rt.rt).decode().split("\n") + line = [n for n in notes if n.startswith(TAG)] + if len(line) != 1: + raise SystemExit("no reference line in the notes (a --dev-diagnostics build?): %r" % notes) + entries = [(kv.split("=", 1)[0], int(kv.split("=", 1)[1], 16)) for kv in line[0][len(TAG):].split()] + if len({n for n, _ in entries}) != len(entries): + raise SystemExit("duplicate case names in the reference") + return entries + + +def main(): + ap = argparse.ArgumentParser() + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument("--build") + g.add_argument("--bootstrap", action="store_true") + ap.add_argument("--out", default=OUT) + a = ap.parse_args() + entries = [("bootstrap", 0)] if a.bootstrap else reference(a.build) + text = header(entries, D.digest()) + with open(a.out, "w", encoding="utf-8", newline="\n") as f: + f.write(text) + print(a.out, len(entries), "case(s), source digest", D.digest()) + + +if __name__ == "__main__": + main() diff --git a/clean/k/dx12/hlsl/k_common.hlsli b/clean/k/dx12/hlsl/k_common.hlsli index 187a0f0d..eacbe45b 100644 --- a/clean/k/dx12/hlsl/k_common.hlsli +++ b/clean/k/dx12/hlsl/k_common.hlsli @@ -25,6 +25,8 @@ // * Driver channel calls (WMMA, conversion) only in wave-uniform control flow, with branchless neighbours. #pragma once +#include "k_smx.hlsli" + // ---- the opaque zero and half2 views --------------------------------------------------------------------------------- // wgk_zero() is 0 at run time (grids < 2^31) and unknown to the compiler; wgk_opq xors it into BOTH halves of a packed // pair (x 0x10001): with the plain 0 / 1 value the driver proved the high half's xor zero and dropped it (then fused @@ -195,53 +197,20 @@ uint4 wgk_act8n(uint4 x) // ---- softmax weights (HIP wgk_softmax_weights: FP32 weights, the network's default) --------------------------- // gg: one query's 4 fragments of 8 keys (lane = query, the clamped polynomial g of spec 41 applied). w = 2^g in FP32 -// (v_exp_f32), the sum over 64 keys = 32 per lane in fragment / element order plus one lane exchange, inv = v_rcp_f32, -// A = f16(w inv) rounded ONCE from the exact product (HIP: v_fma_mixlo_f16 / v_fma_mixhi_f16 w * inv + 0). Here: the -// product's FP32 rounding p and its exact residual e = fma(w, inv, -p) give the round-to-odd FP32 value (13 bits more -// than FP16: rounding it to FP16 is the single rounding of the exact product), then the driver's RNE conversion. -// w >= 2^-14.43 and inv > 0 are finite and normal, so no zero / sign / FTZ case arises. -// Round to odd for a positive normal FP32 product p = RNE(x) with exact residual e = x - p (wg_math.hlsli -// wg_round_odd, specialised: no zero / infinity / sign cases): truncation of x is p, or p - 1 ulp when e < 0; the -// inexact bit e != 0 makes it odd. Gives the same bits as wg_round_odd for every p > 0 finite. -uint wgk_round_odd_pos(float p, float e) -{ - return (asuint(p) - (e < 0.0f ? 1u : 0u)) | (e != 0.0f ? 1u : 0u); -} -#ifndef KB_SMX -#define KB_SMX 1 -#endif +// (v_exp_f32, every 2^g first, then the sum in fragment / element order), the sum over 64 keys = 32 per lane plus one +// lane exchange, inv = v_rcp_f32, A = f16(w inv) rounded ONCE from the exact product (HIP: v_fma_mixlo_f16 / +// v_fma_mixhi_f16 w * inv + 0): k_smx.hlsli (the mix form, or round to odd in the __rodd ports). void wgk_softmax_weights(uint4 gg[4], out uint4 pa[4]) { float w[4][8]; float sum = 0.0f; -#if KB_SMX - // every 2^g first (independent), then the sum in the same element order [unroll] for (uint st = 0; st < 4u; ++st) [unroll] for (uint j = 0; j < 8u; ++j) w[st][j] = wg_exp2(wgk_elf(gg[st], j)); [unroll] for (uint s1 = 0; s1 < 4u; ++s1) [unroll] for (uint j = 0; j < 8u; ++j) sum = sum + w[s1][j]; -#else - [unroll] for (uint st = 0; st < 4u; ++st) - [unroll] for (uint j = 0; j < 8u; ++j) { - w[st][j] = wg_exp2(wgk_elf(gg[st], j)); - sum = sum + w[st][j]; - } -#endif sum = sum + wgk_xor16(sum); const float inv = wg_rcp(sum); - [unroll] for (uint s2 = 0; s2 < 4u; ++s2) { - v8f ro; - [unroll] for (uint j = 0; j < 8u; ++j) { - const float p = w[s2][j] * inv; - const float e = mad(w[s2][j], inv, -p); -#if KB_SMX - ro.v[j] = asfloat(wgk_round_odd_pos(p, e)); -#else - ro.v[j] = asfloat(wg_round_odd(p, e)); -#endif - } - pa[s2] = wgk_round8(ro); - } + [unroll] for (uint s2 = 0; s2 < 4u; ++s2) pa[s2] = k_smx_weights8(w[s2], inv); } // ss + the sum of f16(x_c^2) over the 8 channels of x, in channel order, each addition in FP32 (HIP wgk_sumsq8: packed diff --git a/clean/k/dx12/hlsl/k_smx.hlsli b/clean/k/dx12/hlsl/k_smx.hlsli new file mode 100644 index 00000000..47d4b6ea --- /dev/null +++ b/clean/k/dx12/hlsl/k_smx.hlsli @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: MIT +// Model K's softmax weight rounding, shared by the narrow (k_common.hlsli wgk_softmax_weights) and wide +// (k_block_wide.hlsl) stage ports and by the probe kernel k_mix_probe (kernels/k_mix_probe.hlsl). WrongGPU's own code. +// +// HIP (k_block.hip / k_block_wide.hip) rounds each weight A = f16(w inv) ONCE from the exact product +// (v_fma_mixlo_f16 / v_fma_mixhi_f16 w, inv, 0). With p = RN32(w inv) and its exact residual e = fma(w, inv, -p), +// w inv = p + e exactly (w and inv are positive, finite and normal in the softmax: w = 2^g with |g| <= ~14.4, inv the +// reciprocal of a sum of such w, so p >= ~3e-11 and e is exact; no zero, sign or FTZ case arises). +// K_SMX 2 (default, the mix form): the sum p + e goes into the driver's round-to-nearest-even fragment conversion +// (wg_to_f16_ags); the gfx1201 D3D12 compiler folds that addition into v_fma_mixlo_f16 / v_fma_mixhi_f16 p, 1.0, e, +// whose single rounding of the exact p + e is RN16(w inv), HIP's bits (3 VALU per weight instead of 8). The fold +// is the driver's choice: without it the FP32 sum is p and the conversion a second rounding (one FP16 ulp off +// where p is an FP16 midpoint). Guards: build_dx12.py --isa checks every mix-form port's v_fma_mix*_f16 count +// (clean/dx12/isa_checks.json); the native D3D12 runtime runs k_mix_probe on FP16-midpoint products at create and +// takes the K_SMX 1 ports (suffix __rodd) when it fails (clean/k/runtime/runtime_k.cpp, k_mix_probe.h). +// K_SMX 1 (the __rodd ports): p and e give the round-to-odd FP32 value (13 bits more than FP16, so rounding it to +// FP16 is the single rounding of the exact product), then the same conversion. Truncation of w inv is p, or p - 1 +// ulp when e < 0; the inexact bit e != 0 makes it odd (the same bits as wg_round_odd for every finite p > 0). +// K_SMX_MUT 1: negative control of the checks (the residual dropped: a second rounding); never shipped. +// K_SMX_MUT4 1: negative control of the offsets (a second, residual-dropped conversion whose result is taken only +// when the window offset is 4; the exact folded conversion is unchanged elsewhere; needs the stage kernels' P_ox); +// never shipped. +// K_SMX_RODD_MUT 1: negative control of the __rodd ports (the inexact bit dropped: truncation, then the conversion's +// rounding, wrong where the product is an FP16 midpoint); never shipped. +#pragma once + +#ifndef K_SMX +#define K_SMX 2 +#endif +#ifndef K_SMX_MUT +#define K_SMX_MUT 0 +#endif +#ifndef K_SMX_MUT4 +#define K_SMX_MUT4 0 +#endif +#ifndef K_SMX_RODD_MUT +#define K_SMX_RODD_MUT 0 +#endif + +uint k_smx_round_odd_pos(float p, float e) +{ +#if K_SMX_RODD_MUT + return asuint(p) - (e < 0.0f ? 1u : 0u); +#else + return (asuint(p) - (e < 0.0f ? 1u : 0u)) | (e != 0.0f ? 1u : 0u); +#endif +} + +// A = f16(w[j] inv) for one fragment's eight weights, packed two per dword (element 2 i in the low half of dword i). +uint4 k_smx_weights8(float w[8], float inv) +{ + v8f ro; +#if K_SMX >= 2 && K_SMX_MUT4 + v8f rb; // negative control only: the residual-dropped conversion, taken at window offset 4 alone +#endif + [unroll] for (uint j = 0; j < 8u; ++j) { + const float p = w[j] * inv; + const float e = mad(w[j], inv, -p); // exact: w inv = p + e +#if K_SMX >= 2 && K_SMX_MUT + ro.v[j] = p; +#elif K_SMX >= 2 + ro.v[j] = p + e; +#if K_SMX_MUT4 + rb.v[j] = p; +#endif +#else + ro.v[j] = asfloat(k_smx_round_odd_pos(p, e)); +#endif + } +#if K_SMX >= 2 && K_SMX_MUT4 + // two conversions, the choice made on their results: the folded exact one stays as it is at every other offset + const uint4 good = wg_to_f16_ags(ro), bad = wg_to_f16_ags(rb); + return P_ox == 4 ? bad : good; +#else + return wg_to_f16_ags(ro); +#endif +} diff --git a/clean/k/dx12/hlsl/k_wide.hlsli b/clean/k/dx12/hlsl/k_wide.hlsli index b197ef0f..3b8b0b3a 100644 --- a/clean/k/dx12/hlsl/k_wide.hlsli +++ b/clean/k/dx12/hlsl/k_wide.hlsli @@ -15,8 +15,9 @@ // RN16(v_rsq_f32(s)) (wg_f2h: RNE; the plain f32tof16 rounds toward zero on this driver); // - softmax weights: w = v_exp_f32(g), FP32 sum in key order plus the lane ^ 16 half, inv = v_rcp_f32(sum), and // A = f16(w inv) rounded ONCE from the exact product, as HIP's compiled form (v_fma_mixlo/hi_f16 w * inv + 0, seen in -// the k_block_wide.co ISA): the FP32 product rounded to odd with its exact residual (wg_round_odd), then the RNE -// conversion (FP32 has 13 more bits than FP16: no double rounding); +// the k_block_wide.co ISA): k_smx.hlsli, the FP32 product and its exact residual summed into the RNE conversion +// (folded into v_fma_mix*_f16 by the driver), or in the __rodd ports the product rounded to odd, then the conversion +// (FP32 has 13 more bits than FP16: no double rounding); // - mirrored window members, ownership of the output tiles (a group without a last tile computes tile KT - 1 too and // does not store it), merge / expand index maps: as HIP. // Deviations (no effect on any result): weights and LDS operands of each product group are loaded before its multiplies @@ -25,8 +26,7 @@ // k_block_wide.hlsl), each loaded through a per-lane base address plus a constant (the load's immediate offset); // independent accumulators are interleaved (the scores' four key tiles, z's two channel tiles, W1's MM hidden tiles, // the expand's tiles), each still summing in its own order; the LDS indices are masked to the array's power of two -// (kw_lr / kw_lw); the residual x of the owned tiles is reloaded at step 3 for map inputs; the softmax round-to-odd uses -// the positive-product form (kw_round_odd_pos, same bits); NG = 4 uses another LDS layout that fits D3D12's 32 KB (HIP: 56 KB): K / VT per group unpadded with an xor +// (kw_lr / kw_lw); the residual x of the owned tiles is reloaded at step 3 for map inputs; NG = 4 uses another LDS layout that fits D3D12's 32 KB (HIP: 56 KB): K / VT per group unpadded with an xor // swizzle of the 16-byte chunks, and the round's Z, the MLP's A and its H buffers all start at 0, with extra barriers // where the phases now overlap (K / VT reads done before Z is written, Z reads done before A / the next round's K / VT, // A reads done before H, H reads done before the merge's x2). The arithmetic is the same in both layouts. @@ -131,16 +131,6 @@ uint4 kw_softpoly8(uint4 s) return r; } -// wg_round_odd (wg_math.hlsli) for a product p = RN32(w inv) that is positive, finite and normal, with its exact residual -// e: the softmax weights (w = v_exp_f32 of a clamped FP16 polynomial value in about [-14.4, 14.4], inv = v_rcp_f32 of a -// sum of 64 such w, so p >= ~3e-11). There wg_round_odd's finite / nonzero test always holds and "one ulp toward x" is -// +1 for e > 0 and -1 for e < 0 (p's sign bit is 0): the same bits in fewer instructions. -uint kw_round_odd_pos(float p, float e) -{ - const uint ur = asuint(p); - return ur + ((e != 0.0f && (ur & 1u) == 0u) ? (uint)((asint(e) >> 31) | 1) : 0u); -} - v8f kw_widen8(uint4 a) { v8f r; diff --git a/clean/k/dx12/k_mix_cases.py b/clean/k/dx12/k_mix_cases.py new file mode 100644 index 00000000..65a91f64 --- /dev/null +++ b/clean/k/dx12/k_mix_cases.py @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: MIT +"""Adversarial cases for Model K's softmax weight rounding (hlsl/k_smx.hlsli, kernels/k_mix_probe.hlsl). WrongGPU's own +code. + +A softmax weight is A = f16(w inv), ONE round-to-nearest-even rounding of the exact product of two positive FP32 values. +A form that rounds twice (to FP32 first, then to FP16) differs exactly where p = RN32(w inv) is an FP16 rounding +midpoint and the residual e = w inv - p is nonzero: the exact product lies on one side of the midpoint, the double +rounding sees a tie. These cases put such products in every lane, with both residual signs, in the FP16 normal and +subnormal ranges, plus exact midpoints (e = 0: ties to even) and ordinary products. + +One record per lane (k_mix_probe's layout): 12 FP32 values, inv, w0 .. w7, three zeros; the expected output is 8 FP16 +patterns, A_j = RN16(w_j inv) computed exactly here (Fraction arithmetic: no floating-point rounding in the oracle). +Products lie in [2^-26, 1] (a softmax weight is at most 1; below 2^-25 the FP16 result is 0 or the smallest subnormal). + + records, expected, kinds = cases(seed, lanes) +""" +from fractions import Fraction + +import numpy as np + +KINDS = ("mid+", "mid-", "tie", "sub+", "sub-", "plain") + + +def rn16_bits(x): + """RN16 of a positive exact value (Fraction or int/float converted exactly), as the FP16 bit pattern.""" + x = Fraction(x) + if x <= 0: + raise ValueError("positive values only") + # exponent of x: 2^e <= x < 2^(e+1) + e = x.numerator.bit_length() - x.denominator.bit_length() + if Fraction(2) ** e > x: + e -= 1 + q_exp = max(e, -14) - 10 # quantum: 2^(e-10) for normal FP16, 2^-24 below 2^-14 + q = x / (Fraction(2) ** q_exp) + n = q.numerator // q.denominator + r = q - n + if r > Fraction(1, 2) or (r == Fraction(1, 2) and n % 2 == 1): + n += 1 + if e < -14: # subnormal (n may reach 1024 = the smallest normal) + return int(n) + if n == 2048: + n, e = 1024, e + 1 + if e + 15 >= 31: + return 0x7C00 + return ((e + 15) << 10) | (int(n) - 1024) + + +def _midpoint(p): + """float32 array p > 0: True where p is an FP16 rounding midpoint (normal: low 13 significand bits 0x1000; + subnormal FP16 range: p / 2^-24 has the fraction 1/2).""" + b = p.view(np.uint32) + normal = p >= np.float32(2.0 ** -14) + mid_n = (b & np.uint32(0x1FFF)) == np.uint32(0x1000) + s = p.astype(np.float64) * 2.0 ** 24 + mid_s = (s - np.floor(s)) == 0.5 + return np.where(normal, mid_n, mid_s), normal + + +def cases(seed, lanes): + rng = np.random.default_rng(seed) + records = np.zeros((lanes, 12), np.float32) + kinds = [] + searched = [k for k in KINDS if k != "tie"] + for lane in range(lanes): + # every fourth lane holds exact midpoints (its inv is a power of two, chosen with them); the others the searched + # kinds for their own inv + want = ["tie"] * 8 if lane % 4 == 3 else [searched[(lane + j) % len(searched)] for j in range(8)] + # inv: wide range; for subnormal-result lanes small enough that products below 2^-14 are common + inv = np.float32(2.0 ** rng.uniform(-30.0, 0.0)) + got = {} + for _ in range(64): + if all(k in got and len(got[k]) >= want.count(k) for k in set(want)): + break + w = (2.0 ** rng.uniform(-14.4, 14.4, 1 << 16)).astype(np.float32) + p = w * inv # RN32 product (numpy float32) + x = w.astype(np.float64) * np.float64(inv) # exact (48-bit product) + ok = (x <= 1.0) & (x >= 2.0 ** -26) + mid, normal = _midpoint(p) + e = x - p.astype(np.float64) # exact residual + sel = {"mid+": ok & mid & normal & (e > 0), "mid-": ok & mid & normal & (e < 0), + "sub+": ok & mid & ~normal & (e > 0), "sub-": ok & mid & ~normal & (e < 0), + "plain": ok & ~mid} + for k, m in sel.items(): + idx = np.nonzero(m)[0] + got.setdefault(k, []).extend(w[idx[:8]].tolist()) + ws = [] + for j, k in enumerate(want): + if k == "tie": + # an exact FP16 midpoint as the product: w = (2n + 1) 2^(t - 11) with 2^t in the normal FP16 range, + # times inv' = a power of two (the lane's inv is replaced by it when the lane has a tie: e = 0) + ws.append(None) + continue + pool = got.get(k) or got.get("plain") + ws.append(np.float32(pool[j % len(pool)])) + if any(v is None for v in ws): + inv = np.float32(2.0 ** -int(rng.integers(0, 10))) + for j, v in enumerate(ws): + if v is None: + t = int(rng.integers(-12, 0)) # product in [2^-12, 1) + n = int(rng.integers(0, 1024)) + prod = Fraction(2 * (1024 + n) + 1, 2 ** 11) * Fraction(2) ** t + ws[j] = np.float32(float(prod / Fraction(float(inv)))) + elif not (2.0 ** -26 <= float(v) * float(inv) <= 1.0): + ws[j] = np.float32(0.5 / float(inv)) # keep the product in range + records[lane, 0] = inv + records[lane, 1:9] = ws + kinds.append(want) + expected = np.zeros((lanes, 8), np.uint16) + for lane in range(lanes): + inv = Fraction(float(records[lane, 0])) + for j in range(8): + expected[lane, j] = rn16_bits(Fraction(float(records[lane, 1 + j])) * inv) + return records, expected, kinds + + +def double_rounding_bits(records): + """What a second rounding through FP32 gives (the K_SMX_MUT control): RN16(RN32(w inv)) per element.""" + out = np.zeros((records.shape[0], 8), np.uint16) + for lane in range(records.shape[0]): + for j in range(8): + p = np.float32(records[lane, 1 + j]) * np.float32(records[lane, 0]) + out[lane, j] = rn16_bits(Fraction(float(p))) + return out diff --git a/clean/k/dx12/k_port_check.py b/clean/k/dx12/k_port_check.py index 4f0aae70..a18578a7 100644 --- a/clean/k/dx12/k_port_check.py +++ b/clean/k/dx12/k_port_check.py @@ -24,7 +24,9 @@ low-resolution motion, inverted depth, reset), odd and tiny base grids, exposure scalars from a device buffer, special values in the colour, depth, motion and histories (the same three levels as the stage families); wide = the wide network stages (k_block_wide.co: k_stagew_e3 .. _d3), as narrow; narrow = the narrow network stages (k_block.co: k_stage_e2 .. k_stage2_d0_hidden; the stage runner's launch plans, clean/k/dx12/k_plan_bridge.py), on random stage weights and inputs with special values (NaN, infinity, -+-65504, zero, -0, subnormal) in every other case; expo = k_expo_scalars against k_io.co (HIP); warm = k_warm (D3D12 only, no HIP kernel: it is checked ++-65504, zero, -0, subnormal) in every other case; expo = k_expo_scalars against k_io.co (HIP); mixprobe = k_mix_probe / k_mix_probe__rodd (D3D12 only: the +softmax weight rounding of hlsl/k_smx.hlsli on k_mix_cases.py's FP16-midpoint records, against the exact single +rounding; README.md); warm = k_warm (D3D12 only, no HIP kernel: it is checked against its own definition, the dummy word is written exactly when the key equals a lane's loaded value). One known limit, shared with Model M's ports: D3D12 flushes FP32 denormals (inputs and results) to zero, HIP keeps them. k_expo_scalars only meets one with a pre-exposure above about 8.5e37 (kappa = RN(1 / pi) * E), so the inputs here keep @@ -53,6 +55,7 @@ NARROW_KERNELS = ["k_stage_e2", "k_stage_e0", "k_stage_e1", "k_stage_d2", "k_stage_d1", "k_stage2_e0", "k_stage2_e1", "k_stage2_d1", "k_stage2_d0_hp", "k_stage2_d0_hidden"] FAMILY_KERNELS = {"expo": (["k_io.co"], ["k_expo_scalars"]), "warm": ([], ["k_warm"]), + "mixprobe": ([], ["k_mix_probe", "k_mix_probe__rodd"]), "narrow": (["k_block.co"], NARROW_KERNELS), "input": (["k_io_exact.co"], ["kx/k_in_tiled"]), "legacy": (["k_io_exact.co"], ["kx/k_in_color", "kx/k_in_reset", "kx/k_in_base"]), @@ -301,6 +304,54 @@ def run_warm(build, rocm, seeds, cdir, hip): return items, lines +# ---- family mixprobe: the softmax weight rounding (hlsl/k_smx.hlsli) ----------------------------------------------- + +MIXPROBE_LANES = 256 + + +def run_mixprobe(build, rocm, seeds, cdir, hip): + """D3D12 only: k_mix_probe (the mix form of every softmax port) and k_mix_probe__rodd (the __rodd ports' form) on + the adversarial records of k_mix_cases.py, checked against the exact single rounding; the records must also tell a + second rounding through FP32 apart (else they could not catch the residual-dropped control).""" + from clean.k.dx12 import k_mix_cases + lines, items = [], [] + for si, seed in enumerate(seeds): + rec, want, kinds = k_mix_cases.cases(seed + 2000, MIXPROBE_LANES) + dbl = k_mix_cases.double_rounding_bits(rec) + name = f"s{si}_mixprobe" + write(os.path.join(cdir, name + ".in"), rec.tobytes()) + for kern in FAMILY_KERNELS["mixprobe"][1]: + tag = f"{name}_{kern}" + nbytes = MIXPROBE_LANES * 16 + lines += [f"chain {tag}", "module k_block.co", f"buf in {rec.nbytes} {name}.in 0", + f"buf out {nbytes + SLACK} - {D3D12_SENTINEL}", + f"step {kern} {MIXPROBE_LANES // 32} 32 @in @out", f"save out {tag}.out", ""] + items.append({"case": tag, "kernel": kern, "want": want, "double": dbl, "kinds": kinds}) + return items, lines + + +def check_mixprobe(items, read): + out = {} + good = True + for it in items: + got = read(it["case"] + ".out") + n = MIXPROBE_LANES * 16 + if len(got) < n: + r = {"written": False} + good = False + else: + g = np.frombuffer(got[:n], np.uint16).reshape(MIXPROBE_LANES, 8) + bad = g != it["want"] + kinds = np.array(it["kinds"]) + r = {"elements": int(g.size), "differ": int(bad.sum()), + "differ_by_kind": {k: int((bad & (kinds == k)).sum()) for k in sorted(set(kinds.ravel()))}, + "double_rounding_would_differ": int((it["double"] != it["want"]).sum()), + "slack_ok": got[n:] == bytes([D3D12_SENTINEL]) * (len(got) - n)} + good = good and r["differ"] == 0 and r["slack_ok"] and r["double_rounding_would_differ"] > 0 + out[it["case"]] = r + return {"cases": len(items), "results": out}, good + + # ---- family narrow: the narrow network stages (k_block.co) --------------------------------------------------------- # kernel -> (stage, stage-runner suffix, d0 form); the d0 kernels are the network's own forms (hidden history, compact @@ -909,6 +960,9 @@ def run(build, rocm, families, seeds, json_out): if "warm" in families: state["warm"] = run_warm(build, rocm, seeds, cdir, hip) lines += state["warm"][1] + if "mixprobe" in families: + state["mixprobe"] = run_mixprobe(build, rocm, seeds, cdir, hip) + lines += state["mixprobe"][1] if "narrow" in families: state["narrow"] = run_stages(NARROW, "k_block.co", "nar", build, rocm, seeds, cdir, hip) lines += state["narrow"][1] @@ -982,6 +1036,9 @@ def read(n): fails.append(it["case"]) res["results"]["warm"] = {"cases": len(state["warm"][0]), "as_defined": good, "failures": fails[:5]} ok = ok and good == len(state["warm"][0]) and good > 0 + if "mixprobe" in state: + res["results"]["mixprobe"], good = check_mixprobe(state["mixprobe"][0], read) + ok = ok and good for fam in ("narrow", "wide", "input", "legacy", "output", "direct", "directleg"): if fam in state: res["results"][fam], good = check_narrow(state[fam], read) diff --git a/clean/k/dx12/kernels/k_block.hlsl b/clean/k/dx12/kernels/k_block.hlsl index 158cdbc4..1ce5ac9d 100644 --- a/clean/k/dx12/kernels/k_block.hlsl +++ b/clean/k/dx12/kernels/k_block.hlsl @@ -30,6 +30,11 @@ // @wg-kernel k_stage_d1 co=k_block.co block=128 wave=32 args=s184:p19.b32 // @wg-kernel k_stage2_d0_hp co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_HIDDEN_ARG=1 // @wg-kernel k_stage2_d0_hidden co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_HIDDEN_ARG=1 +// The __rodd forms: the same stages with the round-to-odd softmax weights (hlsl/k_smx.hlsli K_SMX 1), launched by the +// native D3D12 runtime when the mix-form probe (kernels/k_mix_probe.hlsl) fails on the adapter's driver. +// @wg-kernel k_stage_e2__rodd co=k_block.co entry=k_stage_e2 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1 +// @wg-kernel k_stage_d2__rodd co=k_block.co entry=k_stage_d2 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1 +// @wg-kernel k_stage2_d1__rodd co=k_block.co entry=k_stage2_d1 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1 #include "wg_platform.hlsli" #include "k_blkparams.hlsli" diff --git a/clean/k/dx12/kernels/k_block_wide.hlsl b/clean/k/dx12/kernels/k_block_wide.hlsl index 347d8df0..0303fd57 100644 --- a/clean/k/dx12/kernels/k_block_wide.hlsl +++ b/clean/k/dx12/kernels/k_block_wide.hlsl @@ -11,6 +11,15 @@ // @wg-kernel k_stagew4_d4 co=k_block_wide.co block=512 wave=32 args=s184:p19.b32 entry=kw_stage defines=KW_C=128,KW_HEADS=4,KW_IN=1,KW_CIN=160,KW_CNEXT=0,KW_NG=4,KW_RD=12 // @wg-kernel k_stagew2_d4 co=k_block_wide.co block=256 wave=32 args=s184:p19.b32 entry=kw_stage defines=KW_C=128,KW_HEADS=4,KW_IN=1,KW_CIN=160,KW_CNEXT=0,KW_NG=2 // @wg-kernel k_stagew2_d3 co=k_block_wide.co block=256 wave=32 args=s184:p19.b32 entry=kw_stage defines=KW_C=96,KW_HEADS=4,KW_IN=1,KW_CIN=128,KW_CNEXT=0,KW_NG=2,KW_RD=4 +// The __rodd forms: the same stages with the round-to-odd softmax weights (hlsl/k_smx.hlsli K_SMX 1), launched by the +// native D3D12 runtime when the mix-form probe (kernels/k_mix_probe.hlsl) fails on the adapter's driver. +// @wg-kernel k_stagew2_e3__rodd co=k_block_wide.co block=256 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=96,KW_HEADS=4,KW_IN=0,KW_CIN=96,KW_CNEXT=128,KW_NG=2,KW_RD=4 +// @wg-kernel k_stagew4_e4__rodd co=k_block_wide.co block=512 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=128,KW_HEADS=4,KW_IN=0,KW_CIN=128,KW_CNEXT=160,KW_NG=4,KW_RD=12 +// @wg-kernel k_stagew2_e4__rodd co=k_block_wide.co block=256 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=128,KW_HEADS=4,KW_IN=0,KW_CIN=128,KW_CNEXT=160,KW_NG=2 +// @wg-kernel k_stagew4_m__rodd co=k_block_wide.co block=512 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4,KW_RD=4 +// @wg-kernel k_stagew4_d4__rodd co=k_block_wide.co block=512 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=128,KW_HEADS=4,KW_IN=1,KW_CIN=160,KW_CNEXT=0,KW_NG=4,KW_RD=12 +// @wg-kernel k_stagew2_d4__rodd co=k_block_wide.co block=256 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=128,KW_HEADS=4,KW_IN=1,KW_CIN=160,KW_CNEXT=0,KW_NG=2 +// @wg-kernel k_stagew2_d3__rodd co=k_block_wide.co block=256 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=96,KW_HEADS=4,KW_IN=1,KW_CIN=128,KW_CNEXT=0,KW_NG=2,KW_RD=4 #include "wg_platform.hlsli" WG_ARGS_BEGIN @@ -20,6 +29,7 @@ WG_ARGS_BEGIN WG_ARGS_END #include "k_wide.hlsli" +#include "k_smx.hlsli" // ---- the weight stream ---------------------------------------------------------------------------------------------- // Every global weight fragment of the kernel's matrix products (A operands; v^T: the B operand), in product order: @@ -256,15 +266,9 @@ WG_KERNEL(kw_stage) } sum += wg_xor16(sum); const float inv = rcp(sum); // v_rcp_f32 - [unroll] for (int st4 = 0; st4 < 4; ++st4) { - v8f ro; - [unroll] for (int j = 0; j < 8; ++j) { - const float p = w[st4][j] * inv; - const float e = mad(w[st4][j], inv, -p); // exact: w inv = p + e - ro.v[j] = asfloat(kw_round_odd_pos(p, e)); - } - pa[st4] = kw_out(ro); // f16(w inv), one rounding of the exact product - } + // A = f16(w inv), one rounding of the exact product (k_smx.hlsli: the mix form, or round to odd in the + // __rodd ports) + [unroll] for (int st4 = 0; st4 < 4; ++st4) pa[st4] = k_smx_weights8(w[st4], inv); } // z_h^T = VT A^T (the two channel tiles' chains interleaved, each over st) uint4 zz[2]; diff --git a/clean/k/dx12/kernels/k_mix_probe.hlsl b/clean/k/dx12/kernels/k_mix_probe.hlsl new file mode 100644 index 00000000..5d0a244b --- /dev/null +++ b/clean/k/dx12/kernels/k_mix_probe.hlsl @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: MIT +// Probe of Model K's softmax weight rounding on D3D12 (hlsl/k_smx.hlsli). WrongGPU's own code; D3D12 only (no HIP +// kernel: it is checked against the exact rounding of its inputs, clean/k/dx12/k_port_check.py family mixprobe, and the +// native D3D12 runtime runs it at create on the cases of clean/k/runtime/k_mix_probe.h). +// Lane i of the grid reads one record of 12 floats at byte 48 i: inv, w0 .. w7 (and 3 unused) and writes +// k_smx_weights8(w, inv) = f16(w_j inv) for j = 0..7, two per dword, at byte 16 i: the same helper, the same operations +// and the same driver conversion as every softmax port. The records are FP16-midpoint products with a nonzero residual +// of either sign, exact midpoints (ties to even), FP16-subnormal and ordinary products, so a second rounding through +// FP32 (the mix form without the driver's fold, or the K_SMX_MUT control) cannot pass. +// k_mix_probe: the default (mix) form; k_mix_probe__rodd: the round-to-odd form of the __rodd stage ports. +// Control flow is uniform (the driver channel's rule): every lane of every launched workgroup has a record. +// @wg-kernel k_mix_probe co=k_block.co block=32 wave=32 args=p,p +// @wg-kernel k_mix_probe__rodd co=k_block.co entry=k_mix_probe block=32 wave=32 args=p,p defines=K_SMX=1 +#include "wg_platform.hlsli" + +WG_ARGS_BEGIN + uint P_in, P_out; +WG_ARGS_END + +#include "k_smx.hlsli" + +WG_KERNEL(k_mix_probe) +{ + WG_SETUP(); + const uint i = wg_block_id() * 32u + wg_lane_id(); + const uint4 a = wg_ld128(P_in, i * 48u); // inv, w0, w1, w2 + const uint4 b = wg_ld128(P_in, i * 48u + 16u); // w3 .. w6 + const uint4 c = wg_ld128(P_in, i * 48u + 32u); // w7, unused + float w[8]; + w[0] = asfloat(a.y); + w[1] = asfloat(a.z); + w[2] = asfloat(a.w); + w[3] = asfloat(b.x); + w[4] = asfloat(b.y); + w[5] = asfloat(b.z); + w[6] = asfloat(b.w); + w[7] = asfloat(c.x); + wg_st128(P_out, i * 16u, k_smx_weights8(w, asfloat(a.x))); +} diff --git a/clean/k/dx12/smx_selfcheck_digest.py b/clean/k/dx12/smx_selfcheck_digest.py new file mode 100644 index 00000000..00d743af --- /dev/null +++ b/clean/k/dx12/smx_selfcheck_digest.py @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: MIT +"""The source digest of model K's softmax self-check reference (clean/k/runtime/k_smx_selfcheck.h). WrongGPU's own code. + +The reference hashes in k_smx_selfcheck.h are valid only for the shaders and the synthetic data they were generated from. +The digest is a SHA-256 over: +- the @wg-kernel definition lines of every self-check kernel (both forms: the names, code objects and defines); +- the text of every source file those kernels compile from (their .hlsl file and its #include closure in + clean/k/dx12/hlsl and clean/dx12/hlsl; an include generated into the build folder is covered by its generator, + clean/k/dx12/gen_k_fp_tables.py); +- the runtime regions marked "SMX-SELFCHECK-SOURCE BEGIN" / "END" in clean/k/runtime/runtime_k.cpp and model_k.cpp (the + cases, the synthetic data, the block parameters, the kernel-name rule and the window-offset rule); +- the extra preprocessor defines of a build (build_dx12.py --define). +gen_k_smx_selfcheck.py writes it into the header (kSourceDigest); build_dx12.py and build.py refuse to build when the +current digest differs from the header's (WG_DX12_SMX_DIGEST=0 lets a negative-control build through only into a +test-only output, build_dx12.py --test-only, which build.py never embeds); clean/k/tests/test_k_smx_selfcheck_digest.py checks the committed header and that a changed byte is caught. +""" +import hashlib +import os +import re + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +HEADER = os.path.join("clean", "k", "runtime", "k_smx_selfcheck.h") +KERNEL_DIR = os.path.join("clean", "k", "dx12", "kernels") +INCLUDE_DIRS = [os.path.join("clean", "k", "dx12", "hlsl"), os.path.join("clean", "dx12", "hlsl")] +GENERATED_BY = os.path.join("clean", "k", "dx12", "gen_k_fp_tables.py") +RUNTIME_FILES = [os.path.join("clean", "k", "runtime", "runtime_k.cpp"), os.path.join("clean", "k", "runtime", "model_k.cpp")] +# the softmax stage kernels the self-check runs (runtime_k.cpp kSmxCases), both forms +BASE = ["k_stage_e2", "k_stagew2_e3", "k_stagew2_d3", "k_stage_d2", "k_stage2_d1", "k_stagew4_e4", "k_stagew4_d4", + "k_stagew2_e4", "k_stagew2_d4", "k_stagew4_m"] +KERNELS = sorted(BASE + [k + "__rodd" for k in BASE]) +DECL = re.compile(r"^//\s*@wg-kernel\s+(\S+)\s.*$", re.M) +INCLUDE = re.compile(r'^\s*#\s*include\s+"([^"]+)"', re.M) +REGION = re.compile(r"// SMX-SELFCHECK-SOURCE BEGIN.*?// SMX-SELFCHECK-SOURCE END", re.S) + + +def _text(root, rel): + with open(os.path.join(root, rel), "rb") as f: + return f.read().replace(b"\r\n", b"\n") + + +def _closure(root, rel, seen, missing): + if rel in seen: + return + seen.add(rel) + for inc in INCLUDE.findall(_text(root, rel).decode("utf-8", "replace")): + for d in [os.path.dirname(rel)] + INCLUDE_DIRS: + cand = os.path.normpath(os.path.join(d, inc)) + if os.path.isfile(os.path.join(root, cand)): + _closure(root, cand, seen, missing) + break + else: + missing.add(inc) + + +def digest(root=ROOT, defines=()): + """The digest (64 hex digits) of the self-check's shader sources, definitions, runtime regions and extra defines.""" + lines, files, missing = {}, set(), set() + kdir = os.path.join(root, KERNEL_DIR) + for fn in sorted(os.listdir(kdir)): + if not fn.endswith(".hlsl"): + continue + rel = os.path.join(KERNEL_DIR, fn) + for m in DECL.finditer(_text(root, rel).decode("utf-8", "replace")): + if m.group(1) in KERNELS: + lines[m.group(1)] = m.group(0).strip() + _closure(root, rel, files, missing) + absent = sorted(set(KERNELS) - set(lines)) + if absent: + raise ValueError("self-check kernels not declared: %s" % absent) + h = hashlib.sha256() + h.update(b"kernels\n") + for k in sorted(lines): + h.update(lines[k].encode() + b"\n") + h.update(b"sources\n") + for rel in sorted(files, key=lambda r: r.replace(os.sep, "/")): + h.update(rel.replace(os.sep, "/").encode() + b"\n" + _text(root, rel) + b"\n") + if missing: # includes generated into the build folder: their generator + h.update(b"generated " + ",".join(sorted(missing)).encode() + b"\n" + _text(root, GENERATED_BY) + b"\n") + h.update(b"runtime\n") + for rel in RUNTIME_FILES: + for r in REGION.findall(_text(root, rel).decode("utf-8", "replace")): + h.update(r.encode() + b"\n") + h.update(b"defines\n" + "\n".join(sorted(defines)).encode() + b"\n") + return h.hexdigest() + + +def header_digest(root=ROOT): + """kSourceDigest of the header, or None.""" + p = os.path.join(root, HEADER) + if not os.path.isfile(p): + return None + m = re.search(r'kSourceDigest\[\]\s*=\s*"([0-9a-f]{64})"', _text(root, HEADER).decode("utf-8", "replace")) + return m.group(1) if m else None + + +def stale(root=ROOT, defines=()): + """None when the header matches the current sources and defines, else the reason (one line).""" + want, got = header_digest(root), digest(root, defines) + if want == got: + return None + return ("clean/k/runtime/k_smx_selfcheck.h was generated from other shader sources or defines (header %s, this build " + "%s): regenerate it with clean/k/dx12/gen_k_smx_selfcheck.py" % (want, got)) diff --git a/clean/k/runtime/k_mix_probe.h b/clean/k/runtime/k_mix_probe.h new file mode 100644 index 00000000..9d4de095 --- /dev/null +++ b/clean/k/runtime/k_mix_probe.h @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: MIT +// Generated by clean/k/dx12/gen_k_mix_probe.py (k_mix_cases.py seed 7, 64 lanes); do not edit. +// The native D3D12 runtime's softmax rounding probe (clean/k/dx12/kernels/k_mix_probe.hlsl): per lane 12 FP32 +// words (inv, w0 .. w7, 0, 0, 0) and the 8 exact FP16 results RN16(w_j inv). A second rounding through FP32 differs +// from them in 152 of the 512 results. +#pragma once + +#include + +namespace k_mix_probe { + +constexpr int kLanes = 64; +constexpr uint32_t kRecords[kLanes * 12] = { + 0x39d7b27fu, 0x3f204c7bu, 0x3f0f465fu, 0x3e094db6u, 0x3df79b37u, 0x385783b3u, 0x3fb91300u, 0x401347cfu, + 0x3dfcf277u, 0x00000000u, 0x00000000u, 0x00000000u, 0x36bbcc73u, 0x45b716f8u, 0x407458cau, 0x40c8e562u, + 0x408e53beu, 0x42922814u, 0x438e6869u, 0x4124afe2u, 0x411af979u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3dd702c4u, 0x39412e56u, 0x39a5f1f5u, 0x3c103180u, 0x3ee46115u, 0x3dc6e933u, 0x39d1b37bu, 0x393d9becu, + 0x38f86329u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c800000u, 0x3dfed000u, 0x3cf8b000u, 0x3deeb000u, + 0x3e957000u, 0x40b3d000u, 0x3dacd000u, 0x41f4f000u, 0x401c5000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3f3a32e6u, 0x3d0072d1u, 0x3ebe298eu, 0x3f3c9d97u, 0x3879b62au, 0x38a35a2eu, 0x3ce10db0u, 0x3dac96f7u, + 0x3f258b21u, 0x00000000u, 0x00000000u, 0x00000000u, 0x356ae9dfu, 0x432735e0u, 0x4532ece6u, 0x4071ed63u, + 0x404ab222u, 0x40c95021u, 0x467a4e0au, 0x450863feu, 0x421d55a0u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x31b5c3adu, 0x46918d14u, 0x45febf57u, 0x45aa9831u, 0x440dd21bu, 0x467a6f24u, 0x4652d289u, 0x45febf57u, + 0x46177e2bu, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x3be5d000u, 0x3f71b000u, 0x3f3e7000u, + 0x3b869000u, 0x3b185000u, 0x3d4bf000u, 0x3d42d000u, 0x3d45d000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3d5c7659u, 0x3a2c0090u, 0x3871bc10u, 0x3bc1d8e3u, 0x41710553u, 0x3a1b2a34u, 0x39a2db97u, 0x3f7d7e87u, + 0x3b355d46u, 0x00000000u, 0x00000000u, 0x00000000u, 0x344807adu, 0x3eb55e05u, 0x43cf3fbau, 0x449693d2u, + 0x437071a7u, 0x438fe5ffu, 0x458d0d08u, 0x461ddb39u, 0x43b64aecu, 0x00000000u, 0x00000000u, 0x00000000u, + 0x37bd3ae6u, 0x4696b74cu, 0x4626b251u, 0x3f6e709eu, 0x3f61967bu, 0x3b080bcdu, 0x4696b74cu, 0x462b99c3u, + 0x4024a000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x3a763000u, 0x3f417000u, 0x3dec9000u, + 0x3fbed000u, 0x3a49f000u, 0x3c039000u, 0x3b0f5000u, 0x3b5ff000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x32fd8d40u, 0x44862874u, 0x447210a2u, 0x446f7410u, 0x45541740u, 0x46105153u, 0x4424d389u, 0x43d68d46u, + 0x44ee3be7u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3e1f7225u, 0x39a5125au, 0x3f749bd8u, 0x3e8e0abcu, + 0x3cb129dfu, 0x39131c07u, 0x399907adu, 0x3fc13b88u, 0x3f7357c0u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x314d9726u, 0x41bf3cfbu, 0x430bbf65u, 0x4392ee81u, 0x469af2e8u, 0x462c9d73u, 0x41cd1b8du, 0x45ce0d79u, + 0x414852c0u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x3fd8b000u, 0x3b1f7000u, 0x3a0d9000u, + 0x3e195000u, 0x3ae99000u, 0x3b8d1000u, 0x3eda3000u, 0x3b6b5000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3abf11edu, 0x405da82eu, 0x3ca54cf8u, 0x3cb965e3u, 0x44140e9bu, 0x41254240u, 0x3d3c2950u, 0x3cd48788u, + 0x3bc448b8u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c46de96u, 0x3b3e08c1u, 0x3adb28a8u, 0x4106057eu, + 0x3ba7e8bdu, 0x3d801fceu, 0x3b4e726du, 0x3adb28a8u, 0x3c0299c2u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3aae66f4u, 0x3bc1066fu, 0x3e94a619u, 0x4118e355u, 0x3e20afa3u, 0x3d1487c0u, 0x3c47437au, 0x41d89d28u, + 0x3e8fb6bdu, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3a0a9000u, 0x3d2a9000u, 0x39aad000u, + 0x3e05b000u, 0x3c971000u, 0x3cdff000u, 0x3a1fd000u, 0x399c5000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3e92025cu, 0x3a6088a6u, 0x3a000bf5u, 0x3906e546u, 0x38fd227cu, 0x3ff7e50du, 0x3c23fd5au, 0x3b68dcacu, + 0x391d4017u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c8f5796u, 0x40659ac1u, 0x3af1c56eu, 0x3b1a5f36u, + 0x3a205212u, 0x40255dd9u, 0x3cfd7dcdu, 0x39aa8e96u, 0x3aa48789u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x37eff3fau, 0x3e6bc78cu, 0x3fd53de2u, 0x4575349eu, 0x40e771ffu, 0x4026e63au, 0x3fbcf867u, 0x3f42f8b3u, + 0x417ab287u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3e800000u, 0x3f67b000u, 0x3e9eb000u, 0x3b213000u, + 0x3bf5d000u, 0x3f28f000u, 0x3e67d000u, 0x3c0a5000u, 0x3af85000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3e9a65b0u, 0x3c65dc72u, 0x3c8e7014u, 0x4024ecddu, 0x38dcbab8u, 0x392cf4dcu, 0x38ccadb0u, 0x3eb8ecf5u, + 0x3aebf67au, 0x00000000u, 0x00000000u, 0x00000000u, 0x39fdf808u, 0x40090651u, 0x42def4dau, 0x3d53ee76u, + 0x3d18f6b0u, 0x3c161ec5u, 0x443784c0u, 0x42819737u, 0x3d04cdbeu, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3461e365u, 0x44b24868u, 0x4226076cu, 0x40cbfea2u, 0x466a1b50u, 0x43de32c8u, 0x45890604u, 0x438d4f0eu, + 0x438a0cf1u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3dcc9000u, 0x3baf9000u, 0x3f3d7000u, + 0x3dcfb000u, 0x3d2cb000u, 0x3cf91000u, 0x3b96d000u, 0x3e89f000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3e889a76u, 0x38c0caafu, 0x3c7ff4e2u, 0x3df5d9b8u, 0x3db95aa9u, 0x39541e26u, 0x38953868u, 0x3fa341f9u, + 0x3fcc3630u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3b7bf115u, 0x41f84d29u, 0x422a01f0u, 0x40546e19u, + 0x3c2b1652u, 0x3b896db9u, 0x3a092d40u, 0x3fcaa669u, 0x3e3dd259u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x35a27e45u, 0x463b871eu, 0x442591e9u, 0x419c8f74u, 0x4216f2a4u, 0x44be5f6bu, 0x43d4fb2eu, 0x45f18944u, + 0x41a3418bu, 0x00000000u, 0x00000000u, 0x00000000u, 0x3e000000u, 0x40aa5000u, 0x3b1fd000u, 0x3d3ef000u, + 0x40779000u, 0x3dbb7000u, 0x3cf8d000u, 0x3ea37000u, 0x3c881000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3519a651u, 0x4239fb45u, 0x40b8f0b0u, 0x404d4f0fu, 0x45085cb6u, 0x459970a1u, 0x42022a8fu, 0x40f4ebb9u, + 0x453b35b1u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3b7efffbu, 0x3c0d6d70u, 0x3a6d9466u, 0x4367575cu, + 0x40a9d9ddu, 0x3c036366u, 0x3c705055u, 0x3b851907u, 0x430adadeu, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3288e07eu, 0x44315c67u, 0x459a01c9u, 0x46081f4eu, 0x4522218du, 0x439c2b64u, 0x45eb293fu, 0x461cede7u, + 0x45b64b8fu, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x3c455000u, 0x3f5bd000u, 0x3e9f7000u, + 0x3ffb7000u, 0x3a8a5000u, 0x3ff65000u, 0x3a0ff000u, 0x3a625000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3d0d325eu, 0x41075b73u, 0x3a50bae3u, 0x3a270783u, 0x3d089b76u, 0x3e0b6fc8u, 0x4021df98u, 0x3a2ebc23u, + 0x3985b6d1u, 0x00000000u, 0x00000000u, 0x00000000u, 0x36ae3c71u, 0x3e95dd99u, 0x412a28f0u, 0x45d3bf6cu, + 0x41a88d8bu, 0x4588231au, 0x4137c02cu, 0x405859a1u, 0x466b0b21u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x32a94b9eu, 0x4397f898u, 0x45832c8bu, 0x46478261u, 0x468bbb6bu, 0x4405727eu, 0x45266e56u, 0x45cb8992u, + 0x46a5704cu, 0x00000000u, 0x00000000u, 0x00000000u, 0x3d000000u, 0x3d2cf000u, 0x41265000u, 0x3c705000u, + 0x3ec17000u, 0x3ea55000u, 0x3c357000u, 0x3e421000u, 0x41c17000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x364c553au, 0x43071ce3u, 0x44e3c8feu, 0x40ffe568u, 0x4117203cu, 0x3f6ab4aau, 0x43854fd6u, 0x42b9804eu, + 0x40a972efu, 0x00000000u, 0x00000000u, 0x00000000u, 0x36faa53du, 0x455f3b4du, 0x405ee998u, 0x40681ad0u, + 0x3b697d6au, 0x42fcfa96u, 0x4586a0dfu, 0x3ef72b7bu, 0x40b6f39fu, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3c3ba6e3u, 0x3a6e0ecdu, 0x3b09f523u, 0x3ce39934u, 0x3cf34249u, 0x3bb19afeu, 0x3babfa5fu, 0x3b71a38fu, + 0x38a96fe1u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3c4e1000u, 0x3ac01000u, 0x3e6ab000u, + 0x3c62f000u, 0x3d8a1000u, 0x3f5ad000u, 0x3e021000u, 0x3f46d000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3ca14080u, 0x3ef64322u, 0x3b719c07u, 0x4115c72du, 0x3a4541acu, 0x3a51f507u, 0x3f2cfb49u, 0x3be0bdd1u, + 0x3c8a11d4u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3366ab82u, 0x4663dba8u, 0x4511745fu, 0x44834df3u, + 0x443a4f3au, 0x4630dcf4u, 0x45d54ab1u, 0x44cbb83au, 0x445ac57eu, 0x00000000u, 0x00000000u, 0x00000000u, + 0x36995651u, 0x4376fc30u, 0x408986e9u, 0x4024dd8du, 0x42c6469fu, 0x44d06df8u, 0x45b81dfcu, 0x40d93f2au, + 0x40afed10u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3bd3b000u, 0x3a885000u, 0x3e65f000u, + 0x3a175000u, 0x3a537000u, 0x3a62d000u, 0x3b02b000u, 0x3e561000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x32d77930u, 0x4505e1c8u, 0x402a4d59u, 0x45ee22dau, 0x46832bf1u, 0x43a5bcc7u, 0x443d33b7u, 0x44ed549eu, + 0x4654bdabu, 0x00000000u, 0x00000000u, 0x00000000u, 0x3bf342fau, 0x42011847u, 0x40b555ecu, 0x423d5c23u, + 0x3be5055bu, 0x3bfa1176u, 0x40901553u, 0x401deba7u, 0x40f2e623u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3c182e02u, 0x41924cfbu, 0x3f4bb4deu, 0x3953f5bdu, 0x3baed88du, 0x3e03b51au, 0x3d6ac66au, 0x42b1a1d0u, + 0x3a22559au, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3c84d000u, 0x39f2f000u, 0x3afe5000u, + 0x3c9b1000u, 0x3b54b000u, 0x3ac79000u, 0x3c139000u, 0x3ab09000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x39064412u, 0x3cf5f5afu, 0x3e2fa6c5u, 0x3f888ddfu, 0x4297e0b3u, 0x42be9b5au, 0x3e9274a6u, 0x3eba04d9u, + 0x414ed334u, 0x00000000u, 0x00000000u, 0x00000000u, 0x32ca37a1u, 0x44322f3fu, 0x40caf233u, 0x46101f48u, + 0x460c8fc9u, 0x448fd863u, 0x44d15be3u, 0x41bf95ddu, 0x46186e1au, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3094426fu, 0x428d3d3du, 0x41e27d3cu, 0x42328e35u, 0x455ebe9du, 0x4690d3bfu, 0x42b59041u, 0x4182ccfeu, + 0x42aafbb8u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3d4d9000u, 0x3ab87000u, 0x3e9dd000u, + 0x3c92f000u, 0x3a027000u, 0x3a07d000u, 0x3bf3f000u, 0x3a429000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3d486b0du, 0x419afef5u, 0x3a23fa27u, 0x3a57b580u, 0x3958a501u, 0x3d788538u, 0x3c5b760du, 0x38fe3044u, + 0x3a5cd17cu, 0x00000000u, 0x00000000u, 0x00000000u, 0x38824b1du, 0x3f34a372u, 0x3e968d71u, 0x3b5ac47fu, + 0x4191f291u, 0x441e988cu, 0x3f72c66fu, 0x3f1a5bfbu, 0x45ac9d08u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3f7cee9cu, 0x38546bb1u, 0x38c20ef7u, 0x3b050822u, 0x3bac60d8u, 0x3842f621u, 0x385cc54fu, 0x3a819effu, + 0x38dcb51eu, 0x00000000u, 0x00000000u, 0x00000000u, 0x3d800000u, 0x3ecb9000u, 0x3d6cf000u, 0x400bd000u, + 0x3fc7d000u, 0x3ed93000u, 0x3cb23000u, 0x3e2d1000u, 0x4059f000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3b605644u, 0x3da0afd5u, 0x3dab3e0du, 0x3b272d55u, 0x3bf7e9abu, 0x3d5ab024u, 0x430a419du, 0x3ee0e041u, + 0x3c23e574u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3b99ac50u, 0x3f5b2aa2u, 0x3bc1087cu, 0x3af40cefu, + 0x406655d4u, 0x42b1c7adu, 0x3f14a08fu, 0x3c135363u, 0x3c4b573au, 0x00000000u, 0x00000000u, 0x00000000u, + 0x350a75b3u, 0x420bd2f3u, 0x41d9b59fu, 0x4359163eu, 0x454601cau, 0x43fbcc7du, 0x427c9b92u, 0x41d9b59fu, + 0x3ece82bfu, 0x00000000u, 0x00000000u, 0x00000000u, 0x3b800000u, 0x4074b000u, 0x3f715000u, 0x3ed27000u, + 0x41613000u, 0x4294d000u, 0x400bb000u, 0x40013000u, 0x3eda1000u, 0x00000000u, 0x00000000u, 0x00000000u, +}; +constexpr uint16_t kExpected[kLanes * 8] = { + 0x0c39, 0x0b8b, 0x039e, 0x0342, 0x0000, 0x10e0, 0x13c1, 0x0355, + 0x2832, 0x0167, 0x024d, 0x01a2, 0x0eb4, 0x1687, 0x03c7, 0x038d, + 0x0145, 0x022d, 0x1392, 0x29ff, 0x2138, 0x02c1, 0x013e, 0x00d1, + 0x17f6, 0x0fc6, 0x1776, 0x1cac, 0x2d9e, 0x1566, 0x37a8, 0x28e2, + 0x25d7, 0x3453, 0x3849, 0x02d7, 0x03b6, 0x251e, 0x2bd9, 0x3786, + 0x08cc, 0x1921, 0x0038, 0x002e, 0x005c, 0x232e, 0x17d2, 0x0242, + 0x0675, 0x02d4, 0x01e4, 0x0032, 0x058f, 0x04ad, 0x02d4, 0x035c, + 0x1b2e, 0x378e, 0x35f4, 0x1834, 0x14c2, 0x2660, 0x2616, 0x262e, + 0x0250, 0x0034, 0x0d38, 0x3a7c, 0x0217, 0x0118, 0x2ad2, 0x08e2, + 0x0001, 0x0510, 0x0b5a, 0x02f0, 0x0383, 0x12e3, 0x17b6, 0x0473, + 0x36f7, 0x33b3, 0x0161, 0x014d, 0x0001, 0x36f7, 0x33ed, 0x03ce, + 0x0fb2, 0x360c, 0x2b64, 0x39f6, 0x0e50, 0x1c1c, 0x147a, 0x1700, + 0x0214, 0x01df, 0x01da, 0x0691, 0x0c77, 0x0147, 0x00d4, 0x03b0, + 0x0336, 0x30c3, 0x2988, 0x1ae5, 0x016f, 0x02fa, 0x3386, 0x30bd, + 0x0001, 0x0007, 0x000f, 0x03e4, 0x022a, 0x0001, 0x014b, 0x0001, + 0x3ac6, 0x14fc, 0x0c6c, 0x2cca, 0x134c, 0x1868, 0x32d2, 0x175a, + 0x1d2b, 0x01ee, 0x0229, 0x3ae8, 0x23b6, 0x0463, 0x027b, 0x0092, + 0x024f, 0x0154, 0x2e82, 0x0414, 0x1238, 0x0282, 0x0154, 0x0657, + 0x0083, 0x0e54, 0x2283, 0x0ad7, 0x032a, 0x010f, 0x289d, 0x0e1f, + 0x1054, 0x2954, 0x0d56, 0x302e, 0x24b8, 0x2700, 0x10fe, 0x0ce2, + 0x0c01, 0x0890, 0x0268, 0x0241, 0x386b, 0x19d9, 0x1426, 0x02ce, + 0x2c04, 0x021e, 0x02b3, 0x00b4, 0x29ca, 0x106f, 0x0060, 0x0170, + 0x006f, 0x031f, 0x2f2f, 0x0ac8, 0x04e3, 0x02c5, 0x016d, 0x0f58, + 0x333e, 0x2cf6, 0x110a, 0x17ae, 0x3148, 0x2b3e, 0x1852, 0x0fc2, + 0x1c55, 0x1d5f, 0x3a37, 0x0215, 0x0342, 0x01ee, 0x2ef9, 0x1072, + 0x1440, 0x2ae9, 0x01a5, 0x012f, 0x004a, 0x35b1, 0x2804, 0x0108, + 0x0cea, 0x0093, 0x0016, 0x1a75, 0x0621, 0x138e, 0x03e6, 0x03ce, + 0x2e64, 0x1d7c, 0x39ec, 0x2e7e, 0x2966, 0x27c8, 0x1cb6, 0x3450, + 0x019b, 0x1c45, 0x281a, 0x262e, 0x038a, 0x013e, 0x3572, 0x36d0, + 0x2fa3, 0x313b, 0x2288, 0x02a2, 0x010e, 0x0022, 0x1e3c, 0x11d6, + 0x2371, 0x1291, 0x018e, 0x02fe, 0x178d, 0x103a, 0x20ca, 0x019f, + 0x3952, 0x0cfe, 0x1df8, 0x37bc, 0x21dc, 0x1bc6, 0x291c, 0x1840, + 0x01bf, 0x0037, 0x001f, 0x151e, 0x19c1, 0x0139, 0x0049, 0x1706, + 0x0233, 0x003b, 0x3b34, 0x2549, 0x020c, 0x03bd, 0x0109, 0x3853, + 0x00be, 0x0526, 0x088c, 0x02b6, 0x0053, 0x07dc, 0x093f, 0x0617, + 0x1e2a, 0x36de, 0x30fc, 0x3bdc, 0x1052, 0x3bb2, 0x0c80, 0x0f12, + 0x34aa, 0x01cd, 0x0170, 0x14b6, 0x1ccf, 0x2d94, 0x0182, 0x0093, + 0x001a, 0x039e, 0x2881, 0x072c, 0x25ca, 0x03e9, 0x0126, 0x2d00, + 0x0064, 0x056c, 0x0c20, 0x0dc6, 0x00b1, 0x0370, 0x0835, 0x0ed7, + 0x1568, 0x3532, 0x0f82, 0x220c, 0x212a, 0x0dac, 0x1e10, 0x3a0c, + 0x0ebe, 0x1dae, 0x0199, 0x01e2, 0x002f, 0x12a7, 0x0ca0, 0x010f, + 0x26d4, 0x01b5, 0x01c6, 0x0000, 0x13be, 0x281e, 0x003d, 0x02cc, + 0x00af, 0x0194, 0x0d37, 0x0d93, 0x0411, 0x03f1, 0x02c4, 0x0010, + 0x2270, 0x1600, 0x3356, 0x2318, 0x2c50, 0x3ad6, 0x3010, 0x3a36, + 0x20d9, 0x04c2, 0x31e5, 0x00f9, 0x0108, 0x22cf, 0x086d, 0x0d6f, + 0x126b, 0x0818, 0x03b3, 0x029f, 0x10fb, 0x0e02, 0x05bc, 0x0315, + 0x149f, 0x014a, 0x00c5, 0x0f6c, 0x1fce, 0x26e4, 0x0209, 0x01a5, + 0x1e9e, 0x1442, 0x3330, 0x10ba, 0x129c, 0x1316, 0x1816, 0x32b0, + 0x0385, 0x0001, 0x0a44, 0x0ee6, 0x008c, 0x013e, 0x031f, 0x0d99, + 0x33ab, 0x2963, 0x359f, 0x0367, 0x03b6, 0x2847, 0x24b1, 0x2b36, + 0x3170, 0x1f91, 0x0020, 0x033f, 0x14e5, 0x105d, 0x3a99, 0x0061, + 0x2426, 0x0f98, 0x17f2, 0x24d8, 0x1aa6, 0x163c, 0x209c, 0x1584, + 0x0041, 0x0170, 0x087a, 0x20fb, 0x223f, 0x0267, 0x030c, 0x16c8, + 0x0119, 0x0003, 0x0b1e, 0x0af0, 0x01c7, 0x0295, 0x0009, 0x0b87, + 0x0001, 0x0001, 0x0001, 0x0041, 0x014f, 0x0002, 0x0000, 0x0002, + 0x2a6c, 0x15c4, 0x34ee, 0x2498, 0x1014, 0x103e, 0x1fa0, 0x1214, + 0x3b95, 0x0202, 0x02a3, 0x00aa, 0x1a15, 0x115e, 0x0064, 0x02b3, + 0x02e0, 0x0132, 0x0003, 0x14a5, 0x290b, 0x03dd, 0x0274, 0x357e, + 0x0347, 0x05fe, 0x181c, 0x1d52, 0x0303, 0x0368, 0x1401, 0x06d1, + 0x265c, 0x1b68, 0x305e, 0x2e3e, 0x26ca, 0x1592, 0x2168, 0x32d0, + 0x0c67, 0x0cb0, 0x0093, 0x01b2, 0x09fd, 0x3793, 0x1628, 0x023f, + 0x1c1c, 0x01d0, 0x0092, 0x2452, 0x36ac, 0x1993, 0x02c4, 0x03d0, + 0x012f, 0x00eb, 0x0757, 0x16b2, 0x0c41, 0x0223, 0x00eb, 0x0003, + 0x23a6, 0x1b8a, 0x1694, 0x2b0a, 0x34a6, 0x205e, 0x200a, 0x16d0, +}; + +} // namespace k_mix_probe diff --git a/clean/k/runtime/k_smx_selfcheck.h b/clean/k/runtime/k_smx_selfcheck.h new file mode 100644 index 00000000..c4fed657 --- /dev/null +++ b/clean/k/runtime/k_smx_selfcheck.h @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: MIT +// Generated by clean/k/dx12/gen_k_smx_selfcheck.py; do not edit. The reference hashes (SHA-256 of the outputs, +// first 8 bytes) of model K's softmax self-check cases at every possible window offset (runtime_k.cpp kSmxCases, +// KRuntime::smx_select) on the HIP reference, and the source digest they belong to (smx_selfcheck_digest.py). +#pragma once + +#include + +namespace k_smx_selfcheck { + +constexpr char kSourceDigest[] = "2ad91491cb5edf84114baa30bc0fe79a4859ac281a90d8989bc57130f8d05d69"; +constexpr int kCount = 133; +constexpr const char* kNames[kCount] = { + "k_stage_e2@512x384/o4,4", + "k_stagew2_e3@512x384/o4,4", + "k_stagew2_d3@512x384/o4,4", + "k_stage_d2@512x384/o4,4", + "k_stage2_d1@512x384/o4,4", + "k_stagew4_e4@512x384*8/o0,0", + "k_stagew4_e4@512x384*8/o0,2", + "k_stagew4_e4@512x384*8/o0,4", + "k_stagew4_e4@512x384*8/o0,6", + "k_stagew4_e4@512x384*8/o2,0", + "k_stagew4_e4@512x384*8/o2,2", + "k_stagew4_e4@512x384*8/o2,4", + "k_stagew4_e4@512x384*8/o2,6", + "k_stagew4_e4@512x384*8/o4,0", + "k_stagew4_e4@512x384*8/o4,2", + "k_stagew4_e4@512x384*8/o4,4", + "k_stagew4_e4@512x384*8/o4,6", + "k_stagew4_e4@512x384*8/o6,0", + "k_stagew4_e4@512x384*8/o6,2", + "k_stagew4_e4@512x384*8/o6,4", + "k_stagew4_e4@512x384*8/o6,6", + "k_stagew4_d4@512x384*8/o0,0", + "k_stagew4_d4@512x384*8/o0,2", + "k_stagew4_d4@512x384*8/o0,4", + "k_stagew4_d4@512x384*8/o0,6", + "k_stagew4_d4@512x384*8/o2,0", + "k_stagew4_d4@512x384*8/o2,2", + "k_stagew4_d4@512x384*8/o2,4", + "k_stagew4_d4@512x384*8/o2,6", + "k_stagew4_d4@512x384*8/o4,0", + "k_stagew4_d4@512x384*8/o4,2", + "k_stagew4_d4@512x384*8/o4,4", + "k_stagew4_d4@512x384*8/o4,6", + "k_stagew4_d4@512x384*8/o6,0", + "k_stagew4_d4@512x384*8/o6,2", + "k_stagew4_d4@512x384*8/o6,4", + "k_stagew4_d4@512x384*8/o6,6", + "k_stagew2_e4@1536x1280/o0,0", + "k_stagew2_e4@1536x1280/o0,2", + "k_stagew2_e4@1536x1280/o0,4", + "k_stagew2_e4@1536x1280/o0,6", + "k_stagew2_e4@1536x1280/o2,0", + "k_stagew2_e4@1536x1280/o2,2", + "k_stagew2_e4@1536x1280/o2,4", + "k_stagew2_e4@1536x1280/o2,6", + "k_stagew2_e4@1536x1280/o4,0", + "k_stagew2_e4@1536x1280/o4,2", + "k_stagew2_e4@1536x1280/o4,4", + "k_stagew2_e4@1536x1280/o4,6", + "k_stagew2_e4@1536x1280/o6,0", + "k_stagew2_e4@1536x1280/o6,2", + "k_stagew2_e4@1536x1280/o6,4", + "k_stagew2_e4@1536x1280/o6,6", + "k_stagew2_d4@1536x1280/o0,0", + "k_stagew2_d4@1536x1280/o0,2", + "k_stagew2_d4@1536x1280/o0,4", + "k_stagew2_d4@1536x1280/o0,6", + "k_stagew2_d4@1536x1280/o2,0", + "k_stagew2_d4@1536x1280/o2,2", + "k_stagew2_d4@1536x1280/o2,4", + "k_stagew2_d4@1536x1280/o2,6", + "k_stagew2_d4@1536x1280/o4,0", + "k_stagew2_d4@1536x1280/o4,2", + "k_stagew2_d4@1536x1280/o4,4", + "k_stagew2_d4@1536x1280/o4,6", + "k_stagew2_d4@1536x1280/o6,0", + "k_stagew2_d4@1536x1280/o6,2", + "k_stagew2_d4@1536x1280/o6,4", + "k_stagew2_d4@1536x1280/o6,6", + "k_stagew4_m@1536x1280/o0,0", + "k_stagew4_m@1536x1280/o0,1", + "k_stagew4_m@1536x1280/o0,2", + "k_stagew4_m@1536x1280/o0,3", + "k_stagew4_m@1536x1280/o0,4", + "k_stagew4_m@1536x1280/o0,5", + "k_stagew4_m@1536x1280/o0,6", + "k_stagew4_m@1536x1280/o0,7", + "k_stagew4_m@1536x1280/o1,0", + "k_stagew4_m@1536x1280/o1,1", + "k_stagew4_m@1536x1280/o1,2", + "k_stagew4_m@1536x1280/o1,3", + "k_stagew4_m@1536x1280/o1,4", + "k_stagew4_m@1536x1280/o1,5", + "k_stagew4_m@1536x1280/o1,6", + "k_stagew4_m@1536x1280/o1,7", + "k_stagew4_m@1536x1280/o2,0", + "k_stagew4_m@1536x1280/o2,1", + "k_stagew4_m@1536x1280/o2,2", + "k_stagew4_m@1536x1280/o2,3", + "k_stagew4_m@1536x1280/o2,4", + "k_stagew4_m@1536x1280/o2,5", + "k_stagew4_m@1536x1280/o2,6", + "k_stagew4_m@1536x1280/o2,7", + "k_stagew4_m@1536x1280/o3,0", + "k_stagew4_m@1536x1280/o3,1", + "k_stagew4_m@1536x1280/o3,2", + "k_stagew4_m@1536x1280/o3,3", + "k_stagew4_m@1536x1280/o3,4", + "k_stagew4_m@1536x1280/o3,5", + "k_stagew4_m@1536x1280/o3,6", + "k_stagew4_m@1536x1280/o3,7", + "k_stagew4_m@1536x1280/o4,0", + "k_stagew4_m@1536x1280/o4,1", + "k_stagew4_m@1536x1280/o4,2", + "k_stagew4_m@1536x1280/o4,3", + "k_stagew4_m@1536x1280/o4,4", + "k_stagew4_m@1536x1280/o4,5", + "k_stagew4_m@1536x1280/o4,6", + "k_stagew4_m@1536x1280/o4,7", + "k_stagew4_m@1536x1280/o5,0", + "k_stagew4_m@1536x1280/o5,1", + "k_stagew4_m@1536x1280/o5,2", + "k_stagew4_m@1536x1280/o5,3", + "k_stagew4_m@1536x1280/o5,4", + "k_stagew4_m@1536x1280/o5,5", + "k_stagew4_m@1536x1280/o5,6", + "k_stagew4_m@1536x1280/o5,7", + "k_stagew4_m@1536x1280/o6,0", + "k_stagew4_m@1536x1280/o6,1", + "k_stagew4_m@1536x1280/o6,2", + "k_stagew4_m@1536x1280/o6,3", + "k_stagew4_m@1536x1280/o6,4", + "k_stagew4_m@1536x1280/o6,5", + "k_stagew4_m@1536x1280/o6,6", + "k_stagew4_m@1536x1280/o6,7", + "k_stagew4_m@1536x1280/o7,0", + "k_stagew4_m@1536x1280/o7,1", + "k_stagew4_m@1536x1280/o7,2", + "k_stagew4_m@1536x1280/o7,3", + "k_stagew4_m@1536x1280/o7,4", + "k_stagew4_m@1536x1280/o7,5", + "k_stagew4_m@1536x1280/o7,6", + "k_stagew4_m@1536x1280/o7,7", +}; +constexpr uint64_t kExpected[kCount] = { + 0x3ac5935c84145d35ull, + 0x6ef5737586c29dddull, + 0xd7d64d479e3f8bb1ull, + 0xe5dc2941fa454a8bull, + 0x7ddd98c1b73d00faull, + 0x6ae8f4b16fa37f3full, + 0x7f9327becac9ed3full, + 0xf0162bd28c3e631full, + 0xb06c8431a1a639a6ull, + 0x482838e4db437cb5ull, + 0xa1db88d469e36077ull, + 0x35ce94197af4fbcdull, + 0x7678ed860409f5d5ull, + 0x1577f775f869f317ull, + 0xc5016a510c102b30ull, + 0xd14fab0163c1cd7cull, + 0x3de4637f62ade642ull, + 0x5cac84dabd6bd5a7ull, + 0x917660586ee12ad3ull, + 0x9dbc8b23ef04ae4eull, + 0x0eb0372fb3322adbull, + 0x008af3b9d84fe6e3ull, + 0xe73616cac05a875bull, + 0xe592583ce9369841ull, + 0x12324f069155f22bull, + 0x192125a21ed0b31aull, + 0x5a91b982d550a3f0ull, + 0x9b5d7000fc43ce62ull, + 0xc3261e2e8d007192ull, + 0xf3acd774b672e4f1ull, + 0x8aecb720a21db37full, + 0x4166511c6134d38full, + 0x6f8307cbc1bd127full, + 0x7e3f358472f039fbull, + 0x0277d2d83eb7c59cull, + 0xf6ae26af33f39836ull, + 0xb7799d4fc39f8c4eull, + 0x48ce1666e34b5c96ull, + 0x54f81218604b34b9ull, + 0x585134f7f38ce2d6ull, + 0xd2163fac3ea4470full, + 0x6060aabdf29d9d8aull, + 0xb01a01dbe45082afull, + 0xaf7dc39770db9372ull, + 0xab4172074ef7ede6ull, + 0x4217dc5fa7820181ull, + 0xaa8a15b977446402ull, + 0x11acbf0580a373ecull, + 0x60428a8be11b088aull, + 0x587f27ecf0ce8fb8ull, + 0xf2df04cc3f2156ceull, + 0xcc1effdd45455c77ull, + 0xcfcda6515c8ba5a6ull, + 0x120cde9d351abb2full, + 0xf23b325643410618ull, + 0x901640cbe4264441ull, + 0x8aa7bac63af2fe1full, + 0xc72839b6b0f3d735ull, + 0x5458bd7f3a32989aull, + 0xced4f5ff1bf38b85ull, + 0x3ba4300a3694abf6ull, + 0x49ef3180db4133deull, + 0x8b9fd5d3a9395ab1ull, + 0x4eca00d04a7f385dull, + 0x886e5bf3d1854711ull, + 0xcae27e427d9a8b96ull, + 0x08061437a11f8b96ull, + 0x65341a0e84624b87ull, + 0x5a0eba0d3084986bull, + 0xa437162423dd9208ull, + 0x892f815625d1281bull, + 0xf111e96e309a4558ull, + 0x815b897364b67cc5ull, + 0x5b9070114791fad9ull, + 0x6ead93a55e47303dull, + 0x0d72071e8506a97bull, + 0x2f97369cb21b1175ull, + 0xc5ed9a094441a609ull, + 0xdba41a2dfdafa4eeull, + 0x6eb17f0b69bcff7aull, + 0x3895d67b96f2f7f5ull, + 0x9b0a7a6a4c740628ull, + 0xd8d3695164893e50ull, + 0xba532ef3362bccfdull, + 0x9210b90067ea4a28ull, + 0xa7c0e106d9479fabull, + 0x36def1e9cf8dea52ull, + 0x8ab307a6b6d53a65ull, + 0x5e8b315d7c05ef49ull, + 0x00b8cd81ed7f48aeull, + 0x1ed1487ffe336269ull, + 0x89fd3882cc0bda27ull, + 0xdb709dd88525bb4cull, + 0x2f744f875e6bce6aull, + 0x1b035adbf577c34eull, + 0xaf6c1d12eb2969e5ull, + 0x8226cb13382fc4d4ull, + 0xcb089d9ba82f020dull, + 0x54ae711628930f40ull, + 0xa42e9768331f7923ull, + 0x8aa805bb64c7ff16ull, + 0xcf0026e5ad169971ull, + 0xf49888f55fa6e67cull, + 0x359f52d28161bf22ull, + 0xa1b29747d78981bbull, + 0xcb891e663621b0adull, + 0xa8b1cf9a23bfa82dull, + 0xe886e1530f9b6ed1ull, + 0xefe55524a320037full, + 0x55cf4ef7679771e8ull, + 0x138a1ea0840ccf3eull, + 0x0e39b9296d65faa4ull, + 0x0e92c4d29c16554eull, + 0x0f0c478a0c59e279ull, + 0x9d59340de6d71fffull, + 0xc390a236c05fd994ull, + 0xacdd3c223b6dbb6aull, + 0x1cc7ba2debb6cc1aull, + 0xaf3277b1b0eced43ull, + 0xe0cf34d804185042ull, + 0x146a6c8c19c0f9e2ull, + 0x74ffc1144a641646ull, + 0x9803563998e3b3b0ull, + 0x990f0e6f43b826deull, + 0x1372dab09bd490beull, + 0x79ab35463d9b255aull, + 0x047b1f2a51f35bb4ull, + 0x4dae2f839e01d124ull, + 0x5d441a52e4199db8ull, + 0xda83d1fa3095b59bull, + 0x8e538b24d75a9667ull, + 0xf8fa3111a724e021ull, + 0x5abea3dc416160f5ull, +}; + +} // namespace k_smx_selfcheck diff --git a/clean/k/runtime/model_k.cpp b/clean/k/runtime/model_k.cpp index 83621515..c7ec0962 100644 --- a/clean/k/runtime/model_k.cpp +++ b/clean/k/runtime/model_k.cpp @@ -57,6 +57,7 @@ void k_base_size(int ow, int oh, double s, int* wb, int* hb) { *hb = one(oh); } +// SMX-SELFCHECK-SOURCE BEGIN (level sizes, window offsets and counts: k_smx_selfcheck.h's digest covers this text) void k_level_size(int wb, int hb, int level, int* w, int* h) { *w = cdiv(wb, 1 << level); *h = cdiv(hb, 1 << level); @@ -75,6 +76,7 @@ void k_stage_offsets(int f, const int table[8][2], int off[11][2]) { } int k_window_count(int w, int h, int ox, int oy) { return cdiv(w + ox, 8) * cdiv(h + oy, 8); } +// SMX-SELFCHECK-SOURCE END std::vector k_buffer_plan(int wb, int hb) { auto tokens = [&](int level) { diff --git a/clean/k/runtime/runtime_k.cpp b/clean/k/runtime/runtime_k.cpp index f01d01c4..3b72a621 100644 --- a/clean/k/runtime/runtime_k.cpp +++ b/clean/k/runtime/runtime_k.cpp @@ -3,14 +3,19 @@ // variant "fused"), k_launch.py (stage_fused_launch) and k_io.py (frame_in, out_uniforms, input / output launches). #include "runtime_k.h" #include "k_input_uniforms.h" +#include "k_mix_probe.h" +#include "k_smx_selfcheck.h" #include #include +#include #include #include #include +#include #include "../../converter/src/json.h" +#include "../../converter/src/sha256.h" #include "../../runtime/embedded.h" #include "../../runtime/fpconv.h" #include "../../runtime/runtime_core.h" @@ -262,6 +267,7 @@ const char* const kColorAtDefault = "input"; const char* const kWarmAtDefault = "e0"; const char* const kWarmToDefault = "d0"; +// SMX-SELFCHECK-SOURCE BEGIN (wide_ng, block_params, stage_kernel_base: the self-check reference depends on them) int wide_ng(const std::string& s, bool f16, int nwin) { if (f16) return s == "e4" || s == "m" || s == "d4" ? 4 : 0; if (s == "m") return 4; @@ -270,6 +276,70 @@ int wide_ng(const std::string& s, bool f16, int nwin) { return 0; } +// The block parameters of stage st (KBlkParams): its weight blob, its maps and the window grid of a level map of lw x lh +// with window offset (ox, oy); iw x ih is a decoder's input map. network() and the softmax self-check (smx_check). +KBlkParams block_params(const KStageInfo& st, DevPtr blob, DevPtr x, DevPtr y, DevPtr skip, DevPtr merged, DevPtr head, + int lw, int lh, int iw, int ih, int ox, int oy) { + const StageSec sec = stage_sections(st); + const BlockSec bs = block_sections(st); + const DevPtr blk = blob + sec.block; + KBlkParams p = {}; + p.x = x; + p.y = y; + p.g1 = blk + bs.g1; + p.g2 = blk + bs.g2; + p.wqkv = blk + bs.qkv; + p.mix = blk + bs.mix; + p.wo = blk + bs.wo; + p.bo = blk + bs.bo; + p.w1 = blk + bs.w1; + p.b1 = blk + bs.b1; + p.w2 = blk + bs.w2; + p.b2 = blk + bs.b2; + p.w = lw; + p.h = lh; + p.ox = ox; + p.oy = oy; + p.nwx = cdiv(lw + ox, 8); // layout.py window_grid + p.nwy = cdiv(lh + oy, 8); + if (st.embed) { + p.wa = blob + sec.embed; + p.ba = blob + sec.embed + linear_bias(32, 16); + } + if (st.kind == 'd') { + p.wa = blob + sec.expand; + p.ba = blob + sec.expand + linear_bias(4 * st.C, st.C_prev); + p.skip = skip; + p.cw = iw; + p.ch = ih; + } + if (st.kind == 'e') { + p.wz = blob + sec.merge; + p.bz = blob + sec.merge + linear_bias(st.C_next, 4 * st.C); + p.merged = merged; + } + if (st.head) { + p.wz = blob + sec.head; + p.bz = blob + sec.head + linear_bias(40, st.C); + p.head = head; + } + return p; +} + +// The stage kernel of a non-head stage as network() launches it (before a __rodd suffix): k_stagew_ for the +// wide stages, k_stage2_ (two windows per workgroup) for e0, e1, d1 with its block output map and d0, else +// k_stage_; *ng = NG (0: not wide), *w2 = two windows per workgroup. +std::string stage_kernel_base(const KStageInfo& st, bool f16, bool has_y, int nwin, int* ng, bool* w2) { + *ng = wide_ng(st.name, f16, nwin); + // e0, e1, d1, d0: two windows per workgroup (k_stage2_*, k_launch.py WINDOWS2_STAGES; k_net.py WINDOWS2_VARIANTS; + // d1's needs its block output map p.y, which the network always gives it) + *w2 = !*ng && (std::strcmp(st.name, "e0") == 0 || std::strcmp(st.name, "e1") == 0 || + (std::strcmp(st.name, "d1") == 0 && has_y) || std::strcmp(st.name, "d0") == 0); + const std::string h = f16 ? "h" : ""; + return (*ng ? "k_stagew" + std::to_string(*ng) : std::string(*w2 ? "k_stage2" : "k_stage")) + h + "_" + st.name; +} +// SMX-SELFCHECK-SOURCE END + // Runtime options of model K (runtime_options, then WG_RUNTIME_OPTIONS whose keys win): model_k_dir, model_k_map, // network (fused, the default; fused16: FP16 accumulation, the spec 49 speed approximation, opt-in), k_diag (timing // diagnostics, same output: "input2" runs the input stage once more before its timing mark, so the "input" part is a @@ -445,6 +515,15 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e if (end == kfill.c_str() || *end || v < 0 || v > 255) return fail("model K: k_fill must be 0..255"); fill_ = (int)v; } + // k_smx_test: the softmax self-check's test hook (smx_select), developer builds only + smx_test_ = option(c, "k_smx_test"); +#ifdef WG_DEV_DIAGNOSTICS + if (!smx_test_.empty() && smx_test_ != "mix_fail" && smx_test_ != "both_fail" && smx_test_ != "rodd_unchecked" && + smx_test_ != "report") + return fail("model K: k_smx_test must be mix_fail, both_fail, rodd_unchecked or report"); +#else + if (!smx_test_.empty()) return fail("model K: k_smx_test is a test hook of developer builds only"); +#endif // The native D3D12 device (backend 3: the D3D12 executor, or k_bench.py's own-buffer timing) runs the exact default // path only, and only when every kernel this context can launch has a port in /dx12/manifest.json; both are // checked here, before the model is loaded, so a refused context costs nothing more (for model K the context ends and @@ -542,7 +621,275 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e std::vector tab((size_t)8 * kFilterEntries * kFilterValues); for (int l = 0; l < 8; ++l) k_filter_table(l, 1, tab.data() + (size_t)l * kFilterEntries * kFilterValues); if (!dev_->upload(tables_, tab.data(), tab.size() * 2)) return dfail(); - return dev_->sync() || dfail(); + if (!dev_->sync()) return dfail(); + // the native device prepares (loads, verifies and compiles) every kernel this context can select now: a missing or + // corrupt shader fails create here, not at the first frame + if (c.backend == 3) + for (const std::string& k : d3d12_kernels()) + if (!dev_->prepare(k)) return dfail(); + return smx_select(c.backend, smx_test_); +} + +// ---- the softmax weight rounding: create-time self-check (native D3D12 device) --------------------------------------- +// The softmax stages' weight rounding A = f16(w inv) (clean/k/dx12/hlsl/k_smx.hlsli) has two forms: the mix form (the +// exact sum p + e into the driver's round-to-nearest conversion, exact when the driver folds it into one +// v_fma_mix*_f16) and the round-to-odd ports (suffix __rodd, exact when the driver keeps the round-to-odd integer +// operations and converts round-to-nearest). Both depend on how the player's driver compiles each stage shader, so +// neither is taken on trust. At create the native D3D12 device checks the shaders that will run: +// - k_mix_probe and k_mix_probe__rodd on k_mix_probe.h's records (FP16-midpoint products with residuals of both signs, +// exact ties, FP16 subnormals) against the exact results; +// - every softmax stage kernel this runtime can select, in both forms, at the window offsets this model's frames use +// (k_stage_offsets over the eight entries of its window table: e2, e3, d3, d2, d1 at (4, 4); e4 and d4 at +// (2 t - 4) mod 8; m at the table's entries t), on the runtime's own synthetic inputs and weights (kSmxCases); each +// case's outputs are hashed and compared with k_smx_selfcheck.h, the HIP reference's hashes of every offset any window +// table can give (clean/k/dx12/gen_k_smx_selfcheck.py, bound to the shader sources by smx_selfcheck_digest.py). +// The mix form is used when its probe and all its cases match; else the round-to-odd ports when theirs all match (one +// note of create); else create fails, naming the first case that differs. Every kernel the context can select is +// prepared at create (KRuntime::create, Device::prepare): a missing or corrupt shader fails create. +// Test hook (developer builds only, WG_DEV_DIAGNOSTICS): runtime option k_smx_test = mix_fail (the mix form counts as +// failed), both_fail (both forms count as failed), rodd_unchecked (the round-to-odd ports without the check; for the +// frame test's negative control), report (adds the computed hashes as a note; on any device, every possible offset). +namespace { +// SMX-SELFCHECK-SOURCE BEGIN (the cases, the synthetic data, the offsets and the check: k_smx_selfcheck.h's digest covers this text) +struct SmxCase { + const char* stage; + int wb, hb; // base map size + int sets; // independent synthetic data sets, the outputs of all hashed together +}; +// e2, e3, d3, d2, d1 (k_stage_e2, k_stagew2_e3, k_stagew2_d3, k_stage_d2, k_stage2_d1) and e4 / d4 with NG = 4 (at most 24 +// windows: 20 at 512 x 384; on 8 data sets) at a 512 x 384 base map; e4 / d4 with NG = 2 (above 24 windows) and m +// (k_stagew4_m, 42 or more windows) at 1536 x 1280. Sized so that a one-stage wrong-rounding mutant of either form +// changes its case's outputs (the mutant runs in the #254 evidence). +const SmxCase kSmxCases[] = {{"e2", 512, 384, 1}, {"e3", 512, 384, 1}, {"d3", 512, 384, 1}, {"d2", 512, 384, 1}, + {"d1", 512, 384, 1}, {"e4", 512, 384, 8}, {"d4", 512, 384, 8}, {"e4", 1536, 1280, 1}, + {"d4", 1536, 1280, 1}, {"m", 1536, 1280, 1}}; +constexpr uint32_t kSmxSeed = 0x6b5eu; // synthetic data (xorshift32) +// FP16 words of uniform values in [-scale, scale): also finite FP32 values where two words are read as one +void smx_fill(std::vector* v, uint32_t* state, float scale) { + for (uint16_t& h : *v) { + uint32_t x = *state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + *state = x; + h = f32_to_f16(((float)(x >> 8) * (1.0f / 8388608.0f) - 1.0f) * scale); + } +} +constexpr int kSmxCount = (int)(sizeof kSmxCases / sizeof kSmxCases[0]); + +const KStageInfo& stage_info(const char* name) { + for (const KStageInfo& s : k_stage_table()) + if (std::strcmp(s.name, name) == 0) return s; + return k_stage_table()[0]; +} +int stage_index(const char* name) { + const auto& t = k_stage_table(); + for (size_t i = 0; i < t.size(); ++i) + if (std::strcmp(t[i].name, name) == 0) return (int)i; + return 0; +} +size_t level_tokens(int wb, int hb, int level) { + int w, h; + k_level_size(wb, hb, level, &w, &h); + return (size_t)w * h; +} +// The window offsets of stage `stage` in frames (k_stage_offsets, the network's rule): over the eight entries of +// `table`, or (table == nullptr) over every table entry 0 .. 7 x 0 .. 7 any model can hold. Distinct, sorted. +std::vector> smx_offsets(const char* stage, const int (*table)[2]) { + std::vector> v; + const int si = stage_index(stage); + for (int e = 0; e < (table ? 8 : 64); ++e) { + int t[8][2], off[11][2]; + for (int k = 0; k < 8; ++k) { + t[k][0] = table ? table[k][0] : e % 8; + t[k][1] = table ? table[k][1] : e / 8; + } + k_stage_offsets(table ? e : 0, t, off); + v.emplace_back(off[si][0], off[si][1]); + } + std::sort(v.begin(), v.end()); + v.erase(std::unique(v.begin(), v.end()), v.end()); + return v; +} +} // namespace + +std::string KRuntime::smx_case_name(int i, int ox, int oy, bool rodd) const { + const SmxCase& c = kSmxCases[i]; + const KStageInfo& st = stage_info(c.stage); + int lw, lh, ng = 0; + bool w2 = false; + k_level_size(c.wb, c.hb, st.level, &lw, &lh); + const std::string k = stage_kernel_base(st, false, true, cdiv(lw + ox, 8) * cdiv(lh + oy, 8), &ng, &w2); + return k + (rodd ? "__rodd" : "") + "@" + std::to_string(c.wb) + "x" + std::to_string(c.hb) + + (c.sets > 1 ? "*" + std::to_string(c.sets) : std::string()) + "/o" + std::to_string(ox) + "," + std::to_string(oy); +} + +// One form of every case at its offsets (this model's: all = false; every possible one: all = true): (case name of the +// mix form, the outputs' SHA-256, first 8 bytes big-endian) into *out. +bool KRuntime::smx_check(bool rodd, bool all, std::vector>* out) { + size_t in_bytes = 0, y_bytes = 0, m_bytes = 0, w_bytes = 0; + for (const SmxCase& c : kSmxCases) { + const KStageInfo& st = stage_info(c.stage); + const size_t own = level_tokens(c.wb, c.hb, st.level) * st.C * 2; + in_bytes = std::max({in_bytes, own, st.kind == 'd' ? level_tokens(c.wb, c.hb, st.level + 1) * st.C_prev * 2 : 0}); + y_bytes = std::max(y_bytes, own); + if (st.kind == 'e') m_bytes = std::max(m_bytes, level_tokens(c.wb, c.hb, st.level + 1) * st.C_next * 2); + w_bytes = std::max(w_bytes, blobs_.at(c.stage).size()); + } + // inputs (the input map and a decoder's skip map read the same synthetic data) and one weight blob for every stage + std::vector in(in_bytes / 2), w(w_bytes / 2); + DevPtr din = 0, dw = 0, dy = 0, dm = 0; + int loaded = -1; // the data set in din / dw + auto load = [&](int set) { + if (set == loaded) return true; + uint32_t state = kSmxSeed + 0x9e3779b9u * (uint32_t)set; + if (!state) state = 1; + smx_fill(&in, &state, 1.0f); + smx_fill(&w, &state, 0.25f); + loaded = set; + return dev_->upload(din, in.data(), in_bytes) && dev_->upload(dw, w.data(), w_bytes); + }; + bool ok = dev_->alloc(&din, in_bytes) && dev_->alloc(&dw, w_bytes) && dev_->alloc(&dy, y_bytes) && + dev_->alloc(&dm, std::max(m_bytes, 256)); + std::vector buf; + for (int i = 0; ok && i < kSmxCount; ++i) { + const SmxCase& c = kSmxCases[i]; + const KStageInfo& st = stage_info(c.stage); + int lw, lh, iw, ih; + k_level_size(c.wb, c.hb, st.level, &lw, &lh); + k_level_size(c.wb, c.hb, st.kind == 'd' ? st.level + 1 : st.level, &iw, &ih); + const size_t yb = level_tokens(c.wb, c.hb, st.level) * st.C * 2; + const size_t mb = st.kind == 'e' ? level_tokens(c.wb, c.hb, st.level + 1) * st.C_next * 2 : 0; + buf.resize(yb + mb); + for (const auto& o : smx_offsets(c.stage, all ? nullptr : offsets_table_)) { + KBlkParams p = block_params(st, dw, din, dy, din, dm, 0, lw, lh, iw, ih, o.first, o.second); + int ng = 0; + bool w2 = false; + const std::string k = stage_kernel_base(st, false, true, p.nwx * p.nwy, &ng, &w2) + (rodd ? "__rodd" : ""); + wgc::Sha256 sha; + for (int set = 0; ok && set < c.sets; ++set) { + ok = load(set) && dev_->memset(dy, 0, y_bytes) && dev_->memset(dm, 0, std::max(m_bytes, 256)) && + launch(k.c_str(), (unsigned)(w2 ? cdiv(p.nwx * p.nwy, 2) : p.nwx * p.nwy), + (unsigned)(kThreads * (ng ? ng : 1)), &p) && + dev_->download(buf.data(), dy, yb) && (!mb || dev_->download(buf.data() + yb, dm, mb)); + if (ok) sha.update(buf.data(), buf.size()); + } + if (!ok) break; + const wgc::Digest d = sha.finish(); + uint64_t v = 0; + for (int b = 0; b < 8; ++b) v = (v << 8) | d[b]; + out->emplace_back(smx_case_name(i, o.first, o.second, false), v); + } + } + for (DevPtr p : {din, dw, dy, dm}) + if (p) dev_->free(p); + return ok || dfail(); +} + +// SMX-SELFCHECK-SOURCE END + +// The mix / round-to-odd probe: the number of results that differ from the exact ones. +bool KRuntime::smx_probe(bool rodd, int* bad) { + DevPtr in = 0, out = 0; + const size_t out_bytes = (size_t)k_mix_probe::kLanes * 16; + if (!dev_->alloc(&in, sizeof k_mix_probe::kRecords)) return dfail(); + if (!dev_->alloc(&out, out_bytes)) { + dev_->free(in); + return dfail(); + } + std::vector got((size_t)k_mix_probe::kLanes * 8); + const bool ok = dev_->upload(in, k_mix_probe::kRecords, sizeof k_mix_probe::kRecords) && + dev_->memset(out, 0xA5, out_bytes) && + launch(rodd ? "k_mix_probe__rodd" : "k_mix_probe", (unsigned)(k_mix_probe::kLanes / 32), 32, &in, &out) && + dev_->download(got.data(), out, out_bytes); + dev_->free(in); + dev_->free(out); + if (!ok) return dfail(); + *bad = 0; + for (size_t i = 0; i < got.size(); ++i) *bad += got[i] != k_mix_probe::kExpected[i]; + return true; +} + +// smx_select (create, last step): the softmax form of this runtime, from the checks above (native D3D12 device), or only +// the report hook's hashes (other devices: the HIP reference for clean/k/dx12/gen_k_smx_selfcheck.py). +bool KRuntime::smx_select(int backend, const std::string& hook) { + auto hex = [](uint64_t v) { + char b[17]; + std::snprintf(b, sizeof b, "%016llx", (unsigned long long)v); + return std::string(b); + }; + auto report = [&](const char* form, const std::vector>& h, int probe_bad) { + std::string r = std::string("model K self-check ") + form + ":" + + (probe_bad >= 0 ? " probe=" + std::to_string(probe_bad) : std::string()); + for (const auto& e : h) r += " " + e.first + "=" + hex(e.second); + return r; + }; + auto add_note = [&](const std::string& line) { notes_ += (notes_.empty() ? "" : "\n") + line; }; + if (backend != 3) { // the report hook only: the reference's hashes at every possible offset + if (hook != "report") return true; + std::vector> h; + if (!smx_check(false, true, &h)) return false; + add_note(report("reference", h, -1)); + return true; + } + if (hook == "rodd_unchecked") { + smx_rodd_ = true; + smx_note_ = " smx=rodd(unchecked)"; + add_note("model K: test hook: the round-to-odd softmax ports without the create-time check"); + return true; + } + int bad[2] = {0, 0}; + std::vector> h[2]; + const bool all = hook == "report"; + if (!smx_probe(false, &bad[0]) || !smx_probe(true, &bad[1]) || !smx_check(false, all, &h[0]) || + !smx_check(true, all, &h[1])) + return false; + // the first check that differs, per form ("" = none); a case without a reference refuses create + std::string first[2]; + for (int f = 0; f < 2; ++f) { + if (bad[f]) first[f] = std::string(f ? "k_mix_probe__rodd" : "k_mix_probe") + " " + std::to_string(bad[f]) + " of " + + std::to_string(k_mix_probe::kLanes * 8); + for (const auto& e : h[f]) { + int at = -1; + for (int j = 0; j < k_smx_selfcheck::kCount && at < 0; ++j) + if (e.first == k_smx_selfcheck::kNames[j]) at = j; + if (at < 0) return fail("model K: k_smx_selfcheck.h has no reference for " + e.first + " (regenerate it)"); + if (first[f].empty() && e.second != k_smx_selfcheck::kExpected[at]) { + const size_t a = e.first.find('@'); + first[f] = f ? e.first.substr(0, a) + "__rodd" + e.first.substr(a) : e.first; + } + } + } + if (hook == "mix_fail" || hook == "both_fail") first[0] = "test hook"; + if (hook == "both_fail") first[1] = "test hook"; + if (hook == "report") { + add_note(report("mix", h[0], bad[0])); + add_note(report("rodd", h[1], bad[1])); + } + if (first[0].empty()) { + smx_rodd_ = false; + smx_note_ = " smx=mix"; + if (!first[1].empty()) + add_note("model K: the round-to-odd softmax ports failed the create-time check on this driver (" + first[1] + + "); the mix form passed it and is used"); + return true; + } + if (first[1].empty()) { + smx_rodd_ = true; + std::string token = first[0] == "test hook" ? std::string("hook") : first[0].substr(0, first[0].find(' ')); + smx_note_ = " smx=rodd(" + token + ")"; + add_note("model K: softmax weight rounding takes the round-to-odd ports (the mix form failed the create-time check: " + + first[0] + "); they passed it, output unchanged"); + return true; + } + return fail("model K: the D3D12 softmax stages are not exact on this driver (create-time check: mix form " + first[0] + + ", round-to-odd form " + first[1] + "); not started"); +} + +std::string KRuntime::stage_kernel(const std::string& k, const char* stage) const { + // e0, e1 and d0 mix with fixed weights (no softmax): no __rodd form + const bool softmax = std::strcmp(stage, "e0") != 0 && std::strcmp(stage, "e1") != 0 && std::strcmp(stage, "d0") != 0; + return smx_rodd_ && softmax ? k + "__rodd" : k; } bool KRuntime::launch(const char* kernel, unsigned grid, unsigned block, void* a0, void* a1, void* a2, void* a3, @@ -589,58 +936,12 @@ bool KRuntime::network(int f, const std::function& after) { const int ox = off[i][0], oy = off[i][1]; if (st.kind != 'm' && (ox % 2 || oy % 2)) return fail(std::string("model K: odd window offset at ") + st.name); const DevPtr blob = wdev_[st.name]; - const StageSec sec = stage_sections(st); - const BlockSec bs = block_sections(st); - const DevPtr blk = blob + sec.block; - KBlkParams p = {}; - p.x = net_[io.in]; - p.y = io.block ? net_[io.block] : 0; - p.g1 = blk + bs.g1; - p.g2 = blk + bs.g2; - p.wqkv = blk + bs.qkv; - p.mix = blk + bs.mix; - p.wo = blk + bs.wo; - p.bo = blk + bs.bo; - p.w1 = blk + bs.w1; - p.b1 = blk + bs.b1; - p.w2 = blk + bs.w2; - p.b2 = blk + bs.b2; - p.w = lw; - p.h = lh; - p.ox = ox; - p.oy = oy; - p.nwx = cdiv(lw + ox, 8); // layout.py window_grid - p.nwy = cdiv(lh + oy, 8); - if (st.embed) { - p.wa = blob + sec.embed; - p.ba = blob + sec.embed + linear_bias(32, 16); - } - if (st.kind == 'd') { - p.wa = blob + sec.expand; - p.ba = blob + sec.expand + linear_bias(4 * st.C, st.C_prev); - p.skip = net_[io.skip]; - p.cw = iw; - p.ch = ih; - } - if (st.kind == 'e') { - p.wz = blob + sec.merge; - p.bz = blob + sec.merge + linear_bias(st.C_next, 4 * st.C); - p.merged = net_[io.merged]; - } - if (st.head) { - p.wz = blob + sec.head; - p.bz = blob + sec.head + linear_bias(40, st.C); - p.head = net_[io.head]; - } - const int ng = wide_ng(st.name, f16_, p.nwx * p.nwy); + KBlkParams p = block_params(st, blob, net_[io.in], io.block ? net_[io.block] : 0, io.skip ? net_[io.skip] : 0, + io.merged ? net_[io.merged] : 0, io.head ? net_[io.head] : 0, lw, lh, iw, ih, ox, oy); + int ng = 0; + bool w2 = false; + const std::string k = stage_kernel_base(st, f16_, p.y != 0, p.nwx * p.nwy, &ng, &w2); if (st.kind == 'm' && !ng) return fail("model K: stage m needs its wide kernel"); - const std::string h = f16_ ? "h" : ""; - // e0, e1, d1, d0: two windows per workgroup (k_stage2_*, k_launch.py WINDOWS2_STAGES; k_net.py WINDOWS2_VARIANTS; - // d1's needs its block output map p.y, which the network always gives it) - const bool w2 = !ng && (std::strcmp(st.name, "e0") == 0 || std::strcmp(st.name, "e1") == 0 || - (std::strcmp(st.name, "d1") == 0 && p.y) || std::strcmp(st.name, "d0") == 0); - const std::string k = - (ng ? "k_stagew" + std::to_string(ng) : std::string(w2 ? "k_stage2" : "k_stage")) + h + "_" + st.name; const unsigned grid = (unsigned)(w2 ? cdiv(p.nwx * p.nwy, 2) : p.nwx * p.nwy); auto f8 = f8data_.find(st.name); if (f8_ && w2 && f8 != f8data_.end()) { // network=fused8: k_stage2q_* (k_block8.h) @@ -657,7 +958,7 @@ bool KRuntime::network(int f, const std::function& after) { const char* k0 = compact_head_ ? (f16_ ? "k_stage2h_d0_hp" : "k_stage2_d0_hp") : (f16_ ? "k_stage2h_d0_hidden" : "k_stage2_d0_hidden"); if (!launch(k0, grid, kThreads, &p, &hidden)) return false; - } else if (!launch(k.c_str(), grid, (unsigned)(kThreads * (ng ? ng : 1)), &p)) return false; + } else if (!launch(stage_kernel(k, st.name).c_str(), grid, (unsigned)(kThreads * (ng ? ng : 1)), &p)) return false; if (after && !after(st.name)) return false; if (i < 10) { info_.offsets[i][0] = ox; // WgRtFrameInfo has ten rows: e0 ... d1 (d0's offset is (4, 4)) @@ -1057,8 +1358,13 @@ std::vector KRuntime::d3d12_kernels() const { else if (ng) k = "k_stagew" + std::to_string(ng) + "_" + n; else k = (n == "e0" || n == "e1" || n == "d1" ? "k_stage2_" : "k_stage_") + n; if (std::find(v.begin(), v.end(), k) == v.end()) v.push_back(k); + // the round-to-odd form the create-time self-check can select (KRuntime::smx_select) + if (!st.head && n != "e0" && n != "e1" && std::find(v.begin(), v.end(), k + "__rodd") == v.end()) + v.push_back(k + "__rodd"); } } + v.push_back("k_mix_probe"); + v.push_back("k_mix_probe__rodd"); v.push_back((compact_head_ ? "k_out_pixel_c" : "k_out_pixel") + d); if (!warm_at_.empty()) v.push_back("k_warm"); // k_warm (option k_warm, backend 3) v.push_back("timing_barrier"); // the timing marks (timing_log=1) @@ -1124,7 +1430,7 @@ const char* KRuntime::kernels() { " legacy_color_launches=" + std::to_string(legacy_color_launches_) + " legacy_base_launches=" + std::to_string(legacy_base_launches_) + (fast_ ? " k_fast=1" : "") + (expo_v051_ ? " k_expo_rule=v051" : "") + - " hidden=fused_d0" + std::string(" head=") + (compact_head_ ? "compact" : "full") + + smx_note_ + " hidden=fused_d0" + std::string(" head=") + (compact_head_ ? "compact" : "full") + " base=" + std::to_string(Bw_) + "x" + std::to_string(Bh_) + " hdr=" + (hdr_ ? "1" : "0") + " mv=render" + " filter_level=" + std::to_string(level_last_) + diff --git a/clean/k/runtime/runtime_k.h b/clean/k/runtime/runtime_k.h index a3dc3fc5..6f5df54b 100644 --- a/clean/k/runtime/runtime_k.h +++ b/clean/k/runtime/runtime_k.h @@ -81,6 +81,8 @@ class KRuntime { const WgRtFrameInfo& info() const { return info_; } const std::map>& blobs() const { return blobs_; } const std::string& err() const { return err_; } + // notes of create for the caller's log (wg_rt_notes), one per line, or empty: the round-to-odd softmax ports + const std::string& notes() const { return notes_; } const std::string& model_path() const { return model_path_; } // What the timing marks 0..4 bracket (FrameIo.timing_base): "exposure", "input", "network", "output". static const char* const kTimingLabels[4]; @@ -102,6 +104,16 @@ class KRuntime { // the tiled input stage's guard (frame and direct_preflight): sizes, the 0.68 ratio, finite jitter in [-0.5, 0.5] bool tiled_guard(int rw, int rh, float jx, float jy) const; bool options_ok_direct(); + // the softmax weight rounding (runtime_k.cpp "create-time self-check"): smx_select picks the mix form or the __rodd + // ports after checking both on this device (native D3D12), or refuses; smx_check hashes one form's stage outputs at + // this model's window offsets (all: every possible offset), smx_probe counts one form's probe mismatches, + // smx_case_name names a self-check case + bool smx_select(int backend, const std::string& hook); + bool smx_check(bool rodd, bool all, std::vector>* out); + bool smx_probe(bool rodd, int* bad); + std::string smx_case_name(int i, int ox, int oy, bool rodd) const; + // the stage kernel network() launches: the __rodd form of a softmax stage after a failed probe + std::string stage_kernel(const std::string& k, const char* stage) const; int fill_ = -1; // option k_fill (test hook): the byte every frame-written buffer and the histories start with WgRtConfig cfg_ = {}; @@ -148,6 +160,11 @@ class KRuntime { // D3D12 latency options (create: k_color_at, k_warm, k_warm_line, k_warm_groups): the stage after which k_out_color // runs unordered (empty: before the input stage, HIP's order), the stage after which k_warm runs (empty: off) std::string color_at_, warm_at_, warm_to_; + // the softmax stages run their round-to-odd (__rodd) ports: the mix-form probe failed (native D3D12 device only) + bool smx_rodd_ = false; + std::string smx_note_; + std::string smx_test_; // option k_smx_test (developer builds only) + std::string notes_; // notes of create for the caller's log (notes()), one per line, or empty unsigned warm_line_ = 128, warm_groups_ = 64; DevPtr warm_dummy_ = 0; WgRtFrameInfo info_ = {}; diff --git a/clean/k/tests/test_k_mix_probe_table.py b/clean/k/tests/test_k_mix_probe_table.py new file mode 100644 index 00000000..42b4818f --- /dev/null +++ b/clean/k/tests/test_k_mix_probe_table.py @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: MIT +"""Model K's softmax rounding probe cases (clean/k/dx12/k_mix_cases.py) and the runtime's table (clean/k/runtime/ +k_mix_probe.h): CPU only. +- the committed header is exactly gen_k_mix_probe.py's output; +- the exact oracle (Fraction arithmetic) agrees with an independent conversion: numpy's binary64 -> binary16 rounding of + the products, which are exact in binary64 (two 24-bit significands); +- the cases cover every kind (FP16-midpoint products with residuals of both signs, normal and subnormal FP16, exact + ties, ordinary products) and a second rounding through FP32 differs from the exact result in many of them, so the + probe catches a missing fold and the residual-dropped control. +Run: python -m unittest clean.k.tests.test_k_mix_probe_table +""" +import os +import unittest + +import numpy as np + +from clean.k.dx12 import gen_k_mix_probe, k_mix_cases + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + + +class TestKMixProbeTable(unittest.TestCase): + def test_header_is_generated(self): + with open(os.path.join(ROOT, "clean", "k", "runtime", "k_mix_probe.h"), newline="") as f: + self.assertEqual(f.read(), gen_k_mix_probe.header(), "regenerate: python clean/k/dx12/gen_k_mix_probe.py") + + def test_oracle_matches_numpy(self): + for seed in (7, 2000, 2001): + rec, want, _ = k_mix_cases.cases(seed, 64) + prod = rec[:, 1:9].astype(np.float64) * rec[:, :1].astype(np.float64) + self.assertTrue(np.array_equal(prod.astype(np.float16).view(np.uint16), want), seed) + rng = np.random.default_rng(5) + x = (2.0 ** rng.uniform(-26, 0, 4000)).astype(np.float32).astype(np.float64) + got = np.array([k_mix_cases.rn16_bits(float(v)) for v in x], np.uint16) + self.assertTrue(np.array_equal(x.astype(np.float16).view(np.uint16), got)) + + def test_rn16_corners(self): + from fractions import Fraction as F + self.assertEqual(k_mix_cases.rn16_bits(1), 0x3C00) + self.assertEqual(k_mix_cases.rn16_bits(F(1, 2 ** 24)), 0x0001) # smallest subnormal + self.assertEqual(k_mix_cases.rn16_bits(F(1, 2 ** 25)), 0x0000 + 0) # tie between 0 and 2^-24: even (0) + self.assertEqual(k_mix_cases.rn16_bits(F(3, 2 ** 25)), 0x0002) # tie 1.5 ulp: even (2) + self.assertEqual(k_mix_cases.rn16_bits(F(2047, 2 ** 25)), 0x0400) # rounds up into the normal range + self.assertEqual(k_mix_cases.rn16_bits(F(2049, 2 ** 11)), 0x3C00) # 1 + 2^-11: tie, even (1.0) + self.assertEqual(k_mix_cases.rn16_bits(F(2051, 2 ** 11)), 0x3C02) # 1 + 3 2^-11: tie, even + self.assertEqual(k_mix_cases.rn16_bits(65504), 0x7BFF) + self.assertEqual(k_mix_cases.rn16_bits(65520), 0x7C00) + + def test_cases_separate_single_and_double_rounding(self): + rec, want, kinds = k_mix_cases.cases(gen_k_mix_probe.SEED, gen_k_mix_probe.LANES) + kinds = np.array(kinds) + dbl = k_mix_cases.double_rounding_bits(rec) + for k in k_mix_cases.KINDS: + self.assertGreater(int((kinds == k).sum()), 40, k) + for k in ("mid+", "mid-", "sub+", "sub-"): + self.assertGreater(int(((dbl != want) & (kinds == k)).sum()), 10, k) + self.assertEqual(int(((dbl != want) & np.isin(kinds, ("tie", "plain"))).sum()), 0) + prod = rec[:, 1:9].astype(np.float64) * rec[:, :1].astype(np.float64) + self.assertTrue(((prod >= 2.0 ** -26) & (prod <= 1.0)).all()) + self.assertTrue((rec[:, 9:] == 0).all()) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/k/tests/test_k_runtime_d3d12.py b/clean/k/tests/test_k_runtime_d3d12.py index 1ca88c09..a77397dd 100644 --- a/clean/k/tests/test_k_runtime_d3d12.py +++ b/clean/k/tests/test_k_runtime_d3d12.py @@ -23,13 +23,23 @@ value, with the game's exposure texel, and with auto-exposure, and LDR; output rows RGBA16F and dense RGB; render size equal to the output (native AA, DLAA 1.0: the legacy input kernels), a jitter outside [-0.5, 0.5], and input_tiled=0. Refusals (one line, create or frame): an approximation option on the native device, an input-copy frame (raw_inputs). +Softmax weight rounding (native device; runtime_k.cpp "create-time self-check"): with nothing set the mix form passes +the check (smx=mix, no note); with the developer hook k_smx_test=mix_fail the round-to-odd ports are taken (smx=rodd, +exactly one note of create, which the executor logs) and the frames match HIP with every softmax stage on its __rodd port, +in both e4 / d4 shapes (1440p Quality: NG = 4; 4K Performance: NG = 2); with both forms failed, or with a corrupt stage +shader (a softmax stage's __rodd port, the fixed-weight stage e0), create refuses in one line. Each create leg runs in its own process. The hook exists only in a runtime library +built with clean/runner/build.py --dev-diagnostics: these tests skip on another build. GPU only: WG_K_GPU_BUILD=, ROCM_PATH; run each test under the machine's GPU lock. Run: python -m unittest clean.k.tests.test_k_runtime_d3d12 """ +import ctypes +import json import os +import subprocess +import sys import unittest import numpy as np @@ -127,7 +137,8 @@ def sequence(self, backend, hdr, mode, half, frames, options="", seed=0, out=OUT for ln in lines: parts = ln.split() if parts: - tgt = on_dx if parts[-1] == "dx12" else on_hip + # where: "dx12" (hybrid device) or "d3d12" (native device), else HIP + tgt = on_dx if parts[-1] in ("dx12", "d3d12") else on_hip tgt[parts[0]] = tgt.get(parts[0], 0) + 1 want, got = out_bytes(hip, half), out_bytes(dx, half) compare_raw(f"frame {i}: output", want, got) @@ -268,6 +279,83 @@ def test_native_refuses_approximation_options(self): self.assertNotIn("\n", msg.strip()) self.assertIn("HIP", msg) + # ---- softmax weight rounding: the create-time self-check and its round-to-odd ports ---------------------------- + def leg(self, options, build=None): + """One native create in its own process: {"ok", "smx", "notes", "error"}.""" + r = subprocess.run([sys.executable, "-m", "clean.k.tests.test_k_runtime_d3d12", "--softmax-leg", self.mdir, + self.map, options, build or self.build], capture_output=True, text=True, timeout=300) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + return json.loads(r.stdout.strip().splitlines()[-1]) + + def need_dev_build(self): + got = self.leg("k_smx_test=report") + if not got["ok"] and "developer builds only" in got["error"]: + self.skipTest("the softmax test hook needs a runtime built with --dev-diagnostics") + self.assertTrue(got["ok"], got) + + def test_native_softmax_switch_is_logged(self): + """The round-to-odd softmax ports are exact too, but taking them is never silent: one note of create.""" + self.need_dev_build() + plain = self.leg("") + self.assertTrue(plain["ok"], plain) + self.assertEqual((plain["smx"], plain["notes"]), ("mix", [])) + fb = self.leg("k_smx_test=mix_fail") + self.assertTrue(fb["ok"], fb) + self.assertEqual(fb["smx"], "rodd(hook)") + self.assertEqual(len(fb["notes"]), 1, fb) + self.assertIn("round-to-odd", fb["notes"][0]) + self.assertIn("test hook", fb["notes"][0]) + + def test_native_softmax_refusals(self): + """Neither form shown exact, or a corrupt stage shader (a softmax stage's __rodd port, a fixed-weight stage): create + refuses in one line naming it (the game keeps its own upscaler).""" + import shutil + import tempfile + self.need_dev_build() + both = self.leg("k_smx_test=both_fail") + self.assertFalse(both["ok"], both) + self.assertIn("not exact on this driver", both["error"]) + self.assertNotIn("\n", both["error"].strip()) + # a copy of the build (its top-level files and dx12/) with one flipped byte in one stage shader: a softmax stage's + # __rodd port and a fixed-weight stage (e0); create prepares every selectable kernel, so it refuses in one line + for victim in ("k_stagew2_e3__rodd", "k_stage2_e0"): + with self.subTest(victim=victim): + tmp = tempfile.mkdtemp(prefix="k_smx_corrupt_") + try: + for f in os.listdir(self.build): + src = os.path.join(self.build, f) + if os.path.isfile(src): + shutil.copy2(src, tmp) + shutil.copytree(os.path.join(self.build, "dx12"), os.path.join(tmp, "dx12")) + path = os.path.join(tmp, "dx12", victim + ".dxil") + data = bytearray(open(path, "rb").read()) + data[len(data) // 2] ^= 0x40 + open(path, "wb").write(bytes(data)) + bad = self.leg("", build=tmp) + self.assertFalse(bad["ok"], bad) + self.assertIn(victim, bad["error"]) + self.assertNotIn("\n", bad["error"].strip()) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + def check_fallback_frames(self, frames, big, **size): + """Forced fallback: the frames match HIP (raw outputs and intermediates) and every softmax stage ran its __rodd + port on D3D12, none its mix form.""" + self.need_dev_build() + on_dx, _ = self.sequence(3, True, 0, True, frames, "k_smx_test=mix_fail", **size) + ng = "k_stagew2_" if big else "k_stagew4_" + want = ["k_stage_e2", "k_stagew2_e3", ng + "e4", "k_stagew4_m", ng + "d4", "k_stagew2_d3", "k_stage_d2", + "k_stage2_d1"] + self.assertEqual([k for k in want if not on_dx.get(k + "__rodd")], [], on_dx) + self.assertEqual([k for k in want if on_dx.get(k)], [], on_dx) + + def test_native_fallback_frames_1440p_quality(self): + self.check_fallback_frames(self.FRAMES_Q, False, **self.QUALITY) + + def test_native_fallback_frames_4k_performance(self): + self.check_fallback_frames([(1920, 1080, True, 1.0, 1.0), (1920, 1080, False, 1.0, 1.0)], True, + out=(3840, 2160), max_render=(1920, 1080)) + def test_input_copy_frames_refused_on_d3d12(self): """raw_inputs (the interop's HIP input copies) has no checked D3D12 port: refused, not run unchecked.""" for backend in (2, 3): @@ -288,5 +376,23 @@ def test_input_copy_frames_refused_on_d3d12(self): del rt +def softmax_leg(mdir, map_path, options, build): + """Child of the softmax tests: one native create; prints ok, the smx field, the notes and the error as JSON.""" + try: + rt = KRuntimeExt(build, mdir, map_path, OUT, MAX_RENDER, True, False, False, options, backend=3) + except RuntimeError as e: + print(json.dumps({"ok": False, "smx": "", "notes": [], "error": str(e)})) + return + rt.lib.wg_rt_notes.argtypes = [ctypes.c_void_p] + rt.lib.wg_rt_notes.restype = ctypes.c_char_p + kern = rt.lib.wg_rt_kernels(rt.rt).decode().split() + smx = [kv.split("=", 1)[1] for kv in kern if kv.startswith("smx=")] + notes = [n for n in rt.lib.wg_rt_notes(rt.rt).decode().split("\n") if n] + print(json.dumps({"ok": True, "smx": smx[0] if smx else "", "notes": notes, "error": ""})) + + if __name__ == "__main__": - unittest.main() + if len(sys.argv) == 6 and sys.argv[1] == "--softmax-leg": + softmax_leg(*sys.argv[2:6]) + else: + unittest.main() diff --git a/clean/k/tests/test_k_smx_selfcheck_digest.py b/clean/k/tests/test_k_smx_selfcheck_digest.py new file mode 100644 index 00000000..f92bf6dc --- /dev/null +++ b/clean/k/tests/test_k_smx_selfcheck_digest.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: MIT +"""Model K's softmax self-check reference is bound to its sources (host checks, no GPU). + +clean/k/runtime/k_smx_selfcheck.h holds the HIP reference's hashes of the self-check cases and kSourceDigest, the digest +of the shader sources, @wg-kernel definitions, runtime regions and extra defines they were generated from +(clean/k/dx12/smx_selfcheck_digest.py). Checked here: +- the committed header belongs to the committed sources (a source change without a regenerated header fails here); +- a copy of the sources with one changed byte in a shader source, a definition line or a runtime region, or with an + extra define, gives another digest and is reported stale; +- clean/dx12/build_dx12.py refuses to build a self-check kernel with a stale header (WG_DX12_SMX_DIGEST=0 lets a + negative-control build through only into a test-only output, --test-only), and ignores builds without one. + +Run: python -m unittest clean.k.tests.test_k_smx_selfcheck_digest +""" +import os +import shutil +import tempfile +import unittest +from unittest import mock + +from clean.k.dx12 import smx_selfcheck_digest as D + +ROOT = D.ROOT + + +def copy_sources(dst): + """The files the digest reads, at the same relative paths under dst.""" + for d in [D.KERNEL_DIR] + D.INCLUDE_DIRS: + shutil.copytree(os.path.join(ROOT, d), os.path.join(dst, d)) + for rel in D.RUNTIME_FILES + [D.GENERATED_BY, D.HEADER]: + os.makedirs(os.path.dirname(os.path.join(dst, rel)), exist_ok=True) + shutil.copy2(os.path.join(ROOT, rel), os.path.join(dst, rel)) + + +def flip(path, needle): + """Change one byte of the first occurrence of `needle` in the file.""" + with open(path, "rb") as f: + data = bytearray(f.read()) + at = data.find(needle.encode()) + assert at >= 0, (path, needle) + data[at] = ord("#") if data[at] != ord("#") else ord("!") + with open(path, "wb") as f: + f.write(bytes(data)) + + +class TestSmxSelfcheckDigest(unittest.TestCase): + def test_committed_header_is_current(self): + self.assertIsNotNone(D.header_digest(), "k_smx_selfcheck.h has no kSourceDigest") + self.assertIsNone(D.stale(), "regenerate clean/k/runtime/k_smx_selfcheck.h (gen_k_smx_selfcheck.py)") + + def test_changes_are_caught(self): + cases = [(os.path.join("clean", "k", "dx12", "hlsl", "k_smx.hlsli"), "ro.v[j] = p + e;"), + (os.path.join("clean", "k", "dx12", "hlsl", "k_wide.hlsli"), "kw_softpoly8"), + (os.path.join("clean", "dx12", "hlsl", "wg_math.hlsli"), "float"), + (os.path.join("clean", "k", "dx12", "kernels", "k_block_wide.hlsl"), "KW_RD=12"), + (os.path.join("clean", "k", "runtime", "runtime_k.cpp"), "kSmxSeed = 0x6b5eu"), + (os.path.join("clean", "k", "runtime", "model_k.cpp"), "(2 * oxm - 4)")] + for rel, needle in cases: + with self.subTest(file=rel): + tmp = tempfile.mkdtemp(prefix="smx_digest_") + try: + copy_sources(tmp) + self.assertIsNone(D.stale(tmp), "an unchanged copy must match") + flip(os.path.join(tmp, rel), needle) + why = D.stale(tmp) + self.assertIsNotNone(why, "a changed byte in %s was not caught" % rel) + self.assertIn("regenerate", why) + finally: + shutil.rmtree(tmp, ignore_errors=True) + self.assertIsNotNone(D.stale(defines=["K_SMX_MUT=1"]), "an extra define was not caught") + + def test_build_refuses_a_stale_header(self): + from clean.dx12 import build_dx12 as B + with mock.patch.dict(os.environ, {"WG_DX12_SMX_DIGEST": "1"}): + with self.assertRaises(SystemExit) as cm: + B.smx_digest_check({"k_stage_e2"}, ["K_SMX_MUT=1"]) + self.assertIn("regenerate", str(cm.exception)) + B.smx_digest_check({"k_stage_e2"}, []) # current header, no extra define: builds + B.smx_digest_check({"k_out_color"}, ["K_SMX_MUT=1"]) # no self-check kernel: not concerned + with mock.patch.dict(os.environ, {"WG_DX12_SMX_DIGEST": "0"}): + with self.assertRaises(SystemExit) as cm: # the bypass alone: refused (never into a normal output) + B.smx_digest_check({"k_stage_e2"}, ["K_SMX_MUT=1"]) + self.assertIn("--test-only", str(cm.exception)) + # a negative-control build into a test-only output: through, and the bypassed kernels are reported + self.assertEqual(B.smx_digest_check({"k_stage_e2", "k_out_color"}, ["K_SMX_MUT=1"], test_only=True), + {"k_stage_e2"}) + self.assertEqual(B.smx_digest_check({"k_stage_e2"}, [], test_only=True), set()) # current: no bypass + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/k/tests/test_k_wide_ng.py b/clean/k/tests/test_k_wide_ng.py index a27b5dde..81b63135 100644 --- a/clean/k/tests/test_k_wide_ng.py +++ b/clean/k/tests/test_k_wide_ng.py @@ -2,7 +2,8 @@ """Model K: the NG = 4 forms of the wide stages e3 and d3 (k_stagew4_e3, k_stagew4_d3) cannot be reached (host checks). The runtime launches a wide stage as k_stagew_ with NG from wide_ng (clean/k/runtime/runtime_k.cpp), in one -place for the frame (network()) and one for the list of kernels a context can launch (d3d12_kernels()). For the FP32 +place for the frame and the softmax self-check (stage_kernel_base, which network() and smx_check / smx_case_name call) +and one for the list of kernels a context can launch (d3d12_kernels()). For the FP32 (exact) network, the only one the D3D12 devices run, wide_ng gives e3 and d3 NG = 2 whatever the window count, the render size, the output size or any option; the FP16 networks (network=fused16 / fused8) take other kernels and are refused on the native device. So the NG = 4 forms of e3 and d3 are not part of any context's kernel set and are not @@ -52,13 +53,15 @@ def test_wide_ng_source(self): def test_wide_names_come_from_wide_ng(self): src = source() - # the two sites that build a wide kernel name (the frame and the kernel list) use wide_ng's result only - sites = re.findall(r'"k_stagew" \+ std::to_string\((\w+)\)', src) - self.assertEqual(sorted(sites), ["ng", "ng"]) - calls = re.findall(r"const int ng = wide_ng\(([^;]*)\);", src) - self.assertEqual(len(calls), 2, calls) - self.assertIn("st.name, f16_, p.nwx * p.nwy", calls) - self.assertIn("n, false, k_window_count(lw, lh, o, o)", calls) + # the two sites that build a wide kernel name use wide_ng's result only: stage_kernel_base (the frame and the + # softmax self-check) and the kernel list + sites = re.findall(r'"k_stagew" \+ std::to_string\((\*?\w+)\)', src) + self.assertEqual(sorted(sites), ["*ng", "ng"]) + self.assertEqual(re.findall(r"\*ng = wide_ng\(([^;]*)\);", src), ["st.name, f16, nwin"]) + self.assertEqual(re.findall(r"const int ng = wide_ng\(([^;]*)\);", src), ["n, false, k_window_count(lw, lh, o, o)"]) + # network() names its stage through stage_kernel_base with the frame's window count; the self-check with FP32 + self.assertIn("stage_kernel_base(st, f16_, p.y != 0, p.nwx * p.nwy, &ng, &w2)", src) + self.assertEqual(len(re.findall(r"stage_kernel_base\(st, false, true,", src)), 2) self.assertNotIn("k_stagew4_e3", src) self.assertNotIn("k_stagew4_d3", src) diff --git a/clean/runner/build.py b/clean/runner/build.py index 87ff9fbd..8e27ae2a 100644 --- a/clean/runner/build.py +++ b/clean/runner/build.py @@ -402,7 +402,69 @@ def runtime_command(cxx, out, embedded=None): RR_OBJECTS = None # --with-rr: the RR runtime's object files (clean/rr_f/runtime/tools/build.py /obj) +def dx12_manifest_problems(out_dir, doc): + """Why the D3D12 kernel manifest `doc` (/dx12/manifest.json) must not be built into a runtime library: a + test-only manifest (clean/dx12/build_dx12.py --define / --test-only / the self-check bypass), an entry built past a + stale self-check reference, with extra defines, without provenance (a build older than the provenance fields), not + declared in this tree's sources, or whose source digest differs from this tree's sources (a stale or foreign DXIL). + Empty when every entry is a normal build of this tree.""" + from clean.dx12 import build_dx12 as BD + out = [] + if doc.get("test_only") is not False: + out.append("the manifest is test-only or older than the test-only flag (a build with --define, --test-only or " + "WG_DX12_SMX_DIGEST=0, or before provenance was recorded)") + decls = {} + for src, k_source in BD.source_files(model_k=True): + for d in BD.kernel_decls(src): + decls[(d["name"], d["co"])] = (src, d, k_source) + gen, cache = os.path.join(out_dir, "dx12", "gen"), {} + for e in doc.get("kernels", []): + n = e.get("name") + hit = decls.get((n, e.get("co", ""))) + if e.get("smx_bypass"): + out.append(f"{n}: built past a stale softmax self-check reference (WG_DX12_SMX_DIGEST=0)") + elif e.get("defines"): + out.append(f"{n}: built with extra defines {e['defines']}") + elif "source_digest" not in e or "defines" not in e: + out.append(f"{n}: no provenance (source_digest / defines)") + elif hit is None: + out.append(f"{n}: not declared in this tree's sources") + elif BD.source_digest(hit[0], hit[1], {}, gen if hit[2] else None, cache) != e["source_digest"]: + out.append(f"{n}: built from other sources than this tree's") + return out + + +def check_dx12_kernels(out_dir): + """Before a runtime library embeds /dx12: refuses any problem of dx12_manifest_problems (failing closed). + Returns the record for the build manifest ("dx12_kernels"): None without a D3D12 manifest, else its SHA-256, its + entry count and provenance "checked".""" + man = os.path.join(out_dir, "dx12", "manifest.json") + if not os.path.isfile(man): + return None + with open(man, "rb") as f: + raw = f.read() + doc = json.loads(raw.decode("utf-8")) + bad = dx12_manifest_problems(out_dir, doc) + if bad: + raise SystemExit(f"{man}: a runtime library embeds only a normal D3D12 kernel build of this tree; rebuild them " + "(clean/dx12/build_dx12.py, without --define / --test-only, into this folder) or build test " + "kernels into a folder of their own:\n " + "\n ".join(bad[:12]) + + (f"\n ... ({len(bad)} problems)" if len(bad) > 12 else "")) + return {"manifest_sha256": hashlib.sha256(raw).hexdigest(), "entries": len(doc.get("kernels", [])), + "provenance": "checked"} + + def build_runtime(cxx, out_dir, embed=True): + # the D3D12 kernels a library embeds: a normal build of this tree only (never a test-only or stale one) + dx12 = check_dx12_kernels(out_dir) if embed else None + if MODELS_BUILT.get("K"): # model K's softmax self-check reference must belong to these sources (no bypass here) + root = os.path.dirname(os.path.dirname(HERE)) + if root not in sys.path: + sys.path.insert(0, root) + from clean.k.dx12 import smx_selfcheck_digest as D + why = D.stale() + if why: + raise SystemExit("softmax self-check reference: " + why) for m in [MAP_FILE] + [o["map"] for name, o in OPTIONAL_MODELS.items() if MODELS_BUILT[name]] + \ ([L_MAP_FILE] if MODEL_L else []): shutil.copyfile(m, os.path.join(out_dir, os.path.basename(m))) @@ -410,13 +472,15 @@ def build_runtime(cxx, out_dir, embed=True): cmd = runtime_command(cxx, os.path.join(out_dir, runtime_library_name()), embedded) run(cmd) if os.path.isfile(os.path.join(out_dir, runtime_library_name())): # (a build that failed raised in run) - write_build_manifest(out_dir, cmd, embedded is not None) + write_build_manifest(out_dir, cmd, embedded is not None, dx12) -def write_build_manifest(out_dir, cmd, embedded): +def write_build_manifest(out_dir, cmd, embedded, dx12=None): """.build.json next to the library: what this build is (R1). The payload packager accepts a runtime library only when its SHA-256 matches a manifest that says dev_diagnostics = false (make_payload.py, failing closed); - any later change of the library (stripping, signing, a copy from another build) breaks the match.""" + any later change of the library (stripping, signing, a copy from another build) breaks the match. dx12_kernels: the + embedded D3D12 kernels' check (check_dx12_kernels), None when the library embeds none; the packager refuses a manifest + without it.""" dll = os.path.join(out_dir, runtime_library_name()) with open(dll, "rb") as f: data = f.read() @@ -430,6 +494,7 @@ def write_build_manifest(out_dir, cmd, embedded): "models_left_out": sorted(n for n in OPTIONAL_MODELS if not MODELS_BUILT[n]), "embedded_files": bool(embedded), "defines": sorted(a for a in cmd if a.startswith("-D")), + "dx12_kernels": dx12 if embedded else None, } path = os.path.splitext(dll)[0] + ".build.json" with open(path, "w", encoding="utf-8", newline="\n") as f: diff --git a/clean/runtime/device.h b/clean/runtime/device.h index feaeccc9..3cf5c021 100644 --- a/clean/runtime/device.h +++ b/clean/runtime/device.h @@ -49,6 +49,9 @@ class Device { } // args: pointers to the argument values (hipModuleLaunchKernel convention). virtual bool launch(const std::string& kernel, unsigned grid, unsigned block, void** args) = 0; + // Loads, verifies and compiles one kernel now (its first launch then compiles nothing); false with err() when its + // shader is missing or corrupt. Devices that prepare nothing ahead: true. + virtual bool prepare(const std::string& kernel) { (void)kernel; return true; } // The next launch may start before the previous one has finished (native D3D12: no UAV barrier between the two; the // kernels synchronize themselves). Devices that always order launches ignore it. virtual void unordered_next() {} diff --git a/clean/runtime/device_d3d12.cpp b/clean/runtime/device_d3d12.cpp index 9ba27176..31bc79d2 100644 --- a/clean/runtime/device_d3d12.cpp +++ b/clean/runtime/device_d3d12.cpp @@ -756,6 +756,13 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (variant_matches(i->second)) return &i->second; return range.first == range.second ? nullptr : &range.first->second; } + bool prepare(const std::string& name) override { + NativeKernel* port = find_port(name); + if (!port) return fail("native D3D12 unsupported kernel: " + name + "; no HIP fallback"); + NativeKernel& k = *port; + if (k.pso || !variant_matches(k) || (k.rs != "buffer" && k.rs != "direct")) return true; + return create_pso(k); + } bool launch(const std::string& name, unsigned grid, unsigned block, void** args) override { NativeKernel* port = find_port(name); if (!port) return fail("native D3D12 unsupported kernel: " + name + "; no HIP fallback"); diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index de1b9bb2..8131f852 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -1193,6 +1193,7 @@ struct Runtime { if (!k->create(c, dev.get(), [this](DevPtr* p, size_t n) { return alloc(p, n); }, external_io, native_direct_io)) return fail(k->err()); blobs = k->blobs(); + notes_text = k->notes(); // one line when the softmax stages take their round-to-odd ports (no silent switch) model_source = "model.wgcm"; return true; } diff --git a/clean/runtime/wg_runtime.h b/clean/runtime/wg_runtime.h index 7c424d02..353e3077 100644 --- a/clean/runtime/wg_runtime.h +++ b/clean/runtime/wg_runtime.h @@ -159,9 +159,12 @@ WG_RT_API const char* wg_rt_model_source(void* rt); /* The kernels in use, as "swin= enc0= dec0= exposure= * post= filter= post_last=". */ WG_RT_API const char* wg_rt_kernels(void* rt); -/* Notes of create for the caller's log, one per line, or an empty string. At present one: the output filter is - * "analytic" because the model file has no output filter weights (spec 06 draft 4), or because this build cannot - * run them. The filter in use is also in wg_rt_kernels() ("output_filter=original|analytic|quality5|quality9 +/* Notes of create for the caller's log, one per line, or an empty string. At present two. Model M / L: the output + * filter is "analytic" because the model file has no output filter weights (spec 06 draft 4), or because this build + * cannot run them. Model K on the native D3D12 device: its softmax stages take their round-to-odd ports because the mix + * form failed the create-time self-check (output unchanged; also "smx=rodd(...)" in wg_rt_kernels()), or the + * round-to-odd ports failed it while the mix form passed. The filter in use + * is also in wg_rt_kernels() ("output_filter=original|analytic|quality5|quality9 * output_taps=model|none"); runtime option output_filter=original|analytic|quality5|quality9 selects it (default: * original when the model file has the weights, else analytic; original without them fails create). */ WG_RT_API const char* wg_rt_notes(void* rt); diff --git a/clean/tests/check_no_dev_diagnostics.py b/clean/tests/check_no_dev_diagnostics.py index fec7272f..e3219b76 100644 --- a/clean/tests/check_no_dev_diagnostics.py +++ b/clean/tests/check_no_dev_diagnostics.py @@ -12,7 +12,10 @@ found in it (DEV_ONLY_STRINGS: a developer build whose marker was forged to =0 still carries them); (2) the payload packager also demands the library's BUILD MANIFEST (clean/runner/build.py writes .build.json: the build's configuration and the SHA-256 of the library) and accepts a library only when its SHA-256 matches a manifest that says dev_diagnostics = false -(check_runtime_with_manifest). +and records that the D3D12 kernels it embeds were checked against their sources ("dx12_kernels": null, or provenance +"checked": clean/runner/build.py refuses test-only, bypassed, define-built or stale kernels before embedding them) +(check_runtime_with_manifest). A loose D3D12 kernel manifest (dx12/manifest.json) in a payload or package fails when it is +test-only or holds a bypassed, define-built or unprovenanced entry (dx12_manifest_problem). A path is a runtime library (wronggpu_runtime*.dll), a folder (searched for libraries, zip files and executables), a zip file (every member), or an executable (the launcher embeds the payload's files as they are, so its bytes hold the marker). @@ -108,9 +111,36 @@ def check_runtime_with_manifest(name, data, manifest_bytes): return False, "the library does not match its build manifest (SHA-256): changed after its build, or another build's file" if m.get("dev_diagnostics") is not False: return False, "the build manifest says dev_diagnostics is not false" + if "dx12_kernels" not in m: + return False, ("the build manifest does not record the check of the D3D12 kernels the library embeds (a build " + "older than that check): rebuild it with clean/runner/build.py") + dk = m["dx12_kernels"] + if dk is not None and not (isinstance(dk, dict) and dk.get("provenance") == "checked"): + return False, "the build manifest says the library's D3D12 kernels were not checked against their sources" return True, "dev-diagnostics=0, matches its build manifest" +def dx12_manifest_problem(name, data): + """None, or why a file named manifest.json that is a D3D12 kernel manifest (clean/dx12/build_dx12.py: format, kernels) + must not ship: test-only (or older than the flag), or an entry built past the softmax self-check reference, with + extra defines, or without provenance.""" + if os.path.basename(name).lower() != "manifest.json": + return None + try: + doc = json.loads(data.decode("utf-8")) + except Exception: + return None + if not (isinstance(doc, dict) and isinstance(doc.get("kernels"), list) and "format" in doc): + return None + if doc.get("test_only") is not False: + return "a test-only D3D12 kernel manifest (or one older than the test-only flag)" + for e in doc["kernels"]: + if not isinstance(e, dict) or e.get("smx_bypass") or e.get("defines") or "source_digest" not in e or "defines" not in e: + return "a D3D12 kernel manifest with a bypassed, define-built or unprovenanced entry (%s)" % ( + e.get("name") if isinstance(e, dict) else e) + return None + + def check_artifact(name, data): """(ok, text, evidence) for one file by its name: a runtime library must carry the =0 marker; any other file only must not say =1. evidence: the file says dev-diagnostics=0 (so a package check knows it looked at a runtime).""" @@ -121,6 +151,9 @@ def check_artifact(name, data): return ok, text, ok if PAYLOAD_CHECK_NAMES.search(os.path.basename(name)): return False, "a check-only code object (mutant, probe or HIP twin) in a release payload", False + why = dx12_manifest_problem(name, data) + if why: + return False, why, False if any(v == b"1" for v in found): return False, "carries a developer runtime (dev-diagnostics=1)", False carried = dev_only_strings_in(data) @@ -136,7 +169,7 @@ def artifacts(path): for dirpath, _, names in os.walk(path): for n in sorted(names): low = n.lower() - if is_runtime_name(n) or low.endswith((".zip", ".exe", ".asi")): + if is_runtime_name(n) or low.endswith((".zip", ".exe", ".asi")) or low == "manifest.json": out += artifacts(os.path.join(dirpath, n)) elif PAYLOAD_CHECK_NAMES.search(n): # a check-only object anywhere in the folder fails the check out.append((os.path.join(dirpath, n), n, b"")) diff --git a/clean/tests/test_dev_diagnostics_off.py b/clean/tests/test_dev_diagnostics_off.py index fbfcab56..7dbf2b9a 100644 --- a/clean/tests/test_dev_diagnostics_off.py +++ b/clean/tests/test_dev_diagnostics_off.py @@ -12,6 +12,7 @@ import sys import tempfile import unittest +from unittest import mock import zipfile HERE = os.path.dirname(os.path.abspath(__file__)) @@ -92,11 +93,17 @@ def test_the_library_marker_and_its_check(self): def manifest_for(name, data, dev=False, **over): - m = {"format": 1, "file": name, "sha256": hashlib.sha256(data).hexdigest(), "size": len(data), "dev_diagnostics": dev} + m = {"format": 1, "file": name, "sha256": hashlib.sha256(data).hexdigest(), "size": len(data), "dev_diagnostics": dev, + "dx12_kernels": None} m.update(over) + for k in [k for k, v in m.items() if v is DROP]: + del m[k] return json.dumps(m).encode() +DROP = object() # manifest_for(..., key=DROP): the key left out + + class BuildManifest(unittest.TestCase): def test_build_py_writes_the_manifest_of_what_it_built(self): b = load_build() @@ -174,6 +181,55 @@ def test_the_test_suites_stand_in_only_in_a_testing_build(self): r, _ = self._pack({"wronggpu_runtime.dll": lib}, extra_args=("--testing",)) self.assertNotEqual(r.returncode, 0, "a testing build still needs a manifest") + def test_unchecked_d3d12_kernels_refuse_the_payload(self): + """#254 re-check P1: a library is packaged only when its build manifest records that the D3D12 kernels it embeds + were checked against their sources (clean/runner/build.py check_dx12_kernels); an older manifest or an unchecked + record refuses the payload, and so does a test-only D3D12 kernel manifest among the payload's files.""" + lib = self._release() + checked = {"manifest_sha256": "0" * 64, "entries": 3, "provenance": "checked"} + r, _ = self._pack({"wronggpu_runtime.dll": lib}, {"wronggpu_runtime.dll": manifest_for("wronggpu_runtime.dll", lib, dx12_kernels=checked)}) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) # control: a checked record is packaged + for bad, why in ((DROP, "does not record the check"), ({"provenance": "bypass"}, "not checked"), ("checked", "not checked")): + with self.subTest(dx12_kernels=bad): + r, out = self._pack({"wronggpu_runtime.dll": lib}, {"wronggpu_runtime.dll": manifest_for("wronggpu_runtime.dll", lib, dx12_kernels=bad)}) + self.assertNotEqual(r.returncode, 0) + self.assertIn(why, r.stderr + r.stdout) + self.assertFalse(os.path.isfile(os.path.join(out, "payload_table.h"))) + entry = {"name": "k_stage_e2", "file": "k_stage_e2.dxil", "source_digest": "0" * 64, "defines": {}} + for doc in ({"format": 1, "test_only": True, "kernels": [entry]}, + {"format": 1, "kernels": [entry]}, # older than the flag + {"format": 1, "test_only": False, "kernels": [dict(entry, smx_bypass=True)]}, + {"format": 1, "test_only": False, "kernels": [dict(entry, defines={"K_SMX_MUT4": "1"})]}, + {"format": 1, "test_only": False, "kernels": [{"name": "k_stage_e2", "file": "k_stage_e2.dxil"}]}): + with self.subTest(doc=doc): + r, _ = self._pack({"wronggpu_runtime.dll": lib, "dx12/manifest.json": json.dumps(doc).encode()}, + {"wronggpu_runtime.dll": manifest_for("wronggpu_runtime.dll", lib)}) + self.assertNotEqual(r.returncode, 0) + self.assertIn("D3D12 kernel manifest", r.stderr + r.stdout) + good = {"format": 1, "test_only": False, "kernels": [entry]} + r, _ = self._pack({"wronggpu_runtime.dll": lib, "dx12/manifest.json": json.dumps(good).encode()}, + {"wronggpu_runtime.dll": manifest_for("wronggpu_runtime.dll", lib)}) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) # control: a normal manifest passes + + def test_the_package_check_refuses_a_test_only_d3d12_manifest(self): + bad = {"format": 1, "test_only": True, "kernels": [{"name": "k_stage_e2", "file": "k_stage_e2.dxil", + "source_digest": "0" * 64, "defines": {"K_SMX_MUT4": "1"}, + "smx_bypass": True}]} + with tempfile.TemporaryDirectory() as d: + z = os.path.join(d, "package.zip") + with zipfile.ZipFile(z, "w") as f: + f.writestr("WrongGPU/wronggpu_runtime.dll", b"MZ" + MARK0) + f.writestr("WrongGPU/dx12/manifest.json", json.dumps(bad)) + res = chk.check_path(z) + self.assertTrue(any(not ok and "test-only D3D12 kernel manifest" in text for _, ok, text, _ in res), res) + os.makedirs(os.path.join(d, "folder", "dx12")) + with open(os.path.join(d, "folder", "wronggpu_runtime.dll"), "wb") as f: + f.write(b"MZ" + MARK0) + with open(os.path.join(d, "folder", "dx12", "manifest.json"), "w") as f: + json.dump(bad, f) + res = chk.check_path(os.path.join(d, "folder")) + self.assertTrue(any(not ok for _, ok, _, _ in res), res) + def test_no_manifest_refuses_the_payload(self): r, out = self._pack({"wronggpu_runtime.dll": self._release()}) self.assertNotEqual(r.returncode, 0) @@ -337,5 +393,111 @@ def run(path): self.assertEqual(run(d), 1, "a folder holding any developer package fails") +class Dx12KernelProvenance(unittest.TestCase): + """#254 re-check P1: a negative-control D3D12 build (WG_DX12_SMX_DIGEST=0) goes only into a test-only output, and a + runtime library never embeds a test-only, bypassed, define-built, unprovenanced or stale kernel. build_dx12.main runs + for real with DXC replaced by a writer of stand-in bytes (no compiler, no GPU).""" + + def setUp(self): + sys.path.insert(0, ROOT) + from clean.dx12 import build_dx12 as BD + self.BD = BD + self.b = load_build() + self.env = mock.patch.dict(os.environ, {"WG_DX12_ISA_CHECK": "0", "WG_DX12_VGPR_CHECK": "0"}) + self.env.start() + self.addCleanup(self.env.stop) + + def dx12(self, out, *args, bypass=False, only="k_stage_e2,k_out_color"): + def fake_compile(src, entry, out_dxil, defines=None, **kw): + with open(out_dxil, "wb") as f: + f.write(b"DXBC stand-in " + json.dumps([src, entry, sorted((defines or {}).items())]).encode()) + with mock.patch.object(self.BD, "compile_hlsl", fake_compile), mock.patch.object(self.BD, "build_asm_tool", lambda o: None), \ + mock.patch.dict(os.environ, {"WG_DX12_SMX_DIGEST": "0" if bypass else "1"}): + return self.BD.main(["--out", out, "--only", only, "-j", "1", *args]) + + def manifest(self, out): + with open(os.path.join(out, "dx12", "manifest.json"), encoding="utf-8") as f: + return json.load(f) + + def test_normal_build_is_embedded(self): + with tempfile.TemporaryDirectory() as d: + self.dx12(d) + doc = self.manifest(d) + self.assertIs(doc["test_only"], False) + self.assertTrue(all(len(e["source_digest"]) == 64 and e["defines"] == {} for e in doc["kernels"]), doc) + rec = self.b.check_dx12_kernels(d) # control: a normal build of this tree passes + self.assertEqual((rec["provenance"], rec["entries"]), ("checked", 2)) + + def test_bypass_build_then_runtime_build_is_refused(self): + with tempfile.TemporaryDirectory() as d: + self.dx12(d) + with self.assertRaises(SystemExit) as cm: # the bypass without --test-only: refused, the folder unchanged + self.dx12(d, "--define", "K_SMX_MUT4=1", bypass=True) + self.assertIn("--test-only", str(cm.exception)) + self.assertIs(self.manifest(d)["test_only"], False) + self.dx12(d, "--define", "K_SMX_MUT4=1", "--test-only", bypass=True) # the negative-control build + doc = self.manifest(d) + self.assertIs(doc["test_only"], True) + e2 = next(e for e in doc["kernels"] if e["name"] == "k_stage_e2") + self.assertEqual((e2.get("smx_bypass"), e2["defines"]), (True, {"K_SMX_MUT4": "1"})) + for call in (lambda: self.b.check_dx12_kernels(d), lambda: self.b.build_runtime("no-compiler-needed", d)): + with self.assertRaises(SystemExit) as cm: # the ordinary runtime build after it: refused before compiling + call() + self.assertIn("test-only", str(cm.exception)) + self.assertIn("k_stage_e2: built past a stale softmax self-check reference", str(cm.exception)) + self.dx12(d) # a normal partial build into that folder keeps it test-only (other entries may be test builds) + self.assertIs(self.manifest(d)["test_only"], True) + with self.assertRaises(SystemExit): + self.b.check_dx12_kernels(d) + + def test_define_build_is_test_only(self): + with tempfile.TemporaryDirectory() as d: + self.dx12(d, "--define", "WG_SOME_TEST=1", only="k_out_color") # (a self-check kernel: refused already) + self.assertIs(self.manifest(d)["test_only"], True) + with self.assertRaises(SystemExit) as cm: + self.b.check_dx12_kernels(d) + self.assertIn("built with extra defines", str(cm.exception)) + + def test_stale_or_unprovenanced_entries_are_refused(self): + with tempfile.TemporaryDirectory() as d: + self.dx12(d) + path = os.path.join(d, "dx12", "manifest.json") + base = self.manifest(d) + cases = [("other sources", lambda e: e.update(source_digest="0" * 64)), + ("no provenance", lambda e: e.pop("source_digest")), + ("no provenance", lambda e: e.pop("defines")), + ("not declared", lambda e: e.update(name="k_stage_e2_gone"))] + for why, change in cases: + with self.subTest(why=why): + doc = json.loads(json.dumps(base)) + change(doc["kernels"][0]) + with open(path, "w", encoding="utf-8") as f: + json.dump(doc, f) + with self.assertRaises(SystemExit) as cm: + self.b.check_dx12_kernels(d) + self.assertIn(why, str(cm.exception)) + doc = json.loads(json.dumps(base)) + del doc["test_only"] # a manifest older than the flag + with open(path, "w", encoding="utf-8") as f: + json.dump(doc, f) + with self.assertRaises(SystemExit): + self.b.check_dx12_kernels(d) + + def test_build_manifest_records_the_check(self): + with tempfile.TemporaryDirectory() as d: + dll = os.path.join(d, self.b.runtime_library_name()) + with open(dll, "wb") as f: + f.write(b"MZ library " + MARK0) + rec = {"manifest_sha256": "1" * 64, "entries": 2, "provenance": "checked"} + self.b.write_build_manifest(d, ["clang++"], True, rec) + m = json.load(open(os.path.splitext(dll)[0] + ".build.json", encoding="utf-8")) + self.assertEqual(m["dx12_kernels"], rec) + ok, text = chk.check_runtime_with_manifest(dll, open(dll, "rb").read(), json.dumps(m).encode()) + self.assertTrue(ok, text) + self.b.write_build_manifest(d, ["clang++"], False, rec) # nothing embedded: no kernels to vouch for + m = json.load(open(os.path.splitext(dll)[0] + ".build.json", encoding="utf-8")) + self.assertIsNone(m["dx12_kernels"]) + + if __name__ == "__main__": unittest.main()