From 2839112cf7d0fd7bfab5e05fe9bb7e4e925be9c8 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:26:37 -0500 Subject: [PATCH 1/5] K softmax weight rounding in one v_fma_mix*_f16 (exact), with a create-time probe and a logged round-to-odd fallback The softmax stages round each weight A = f16(w * inv) exactly once. They now form p = w * inv and its exact residual e = fma(w, inv, -p) and convert p + e through the driver's round-to-nearest fragment conversion, which the driver folds into v_fma_mixlo/hi_f16: a single rounding of the exact product, the same bits as the HIP reference, fewer instructions than the round-to-odd sequence it replaces. One helper (clean/k/dx12/hlsl/k_smx.hlsli) serves the narrow and wide stages. Guards: - build: clean/dx12/isa_checks.json and build_dx12.py isa_check() require the exact v_fma_mix*_f16 count in every mix-form port and none in the round-to-odd ports (WG_DX12_ISA_CHECK=0 skips it); - port check: family mixprobe (k_mix_probe / k_mix_probe__rodd) on adversarial records (FP16 midpoints with residuals of both signs, exact ties, FP16 subnormals) against the exact single rounding; the residual-dropped control K_SMX_MUT=1 must fail it; - runtime: the native D3D12 device runs k_mix_probe at create; if any result differs, every softmax stage takes its round-to-odd port (suffix __rodd, the same bits by another sequence). The switch is reported in the kernels line (smx=mix | smx=rodd(...)) and adds one note of create (wg_rt_notes, logged by the executor). WG_TEST_K_MIX_PROBE_FAIL=1 is the test hook; test_native_softmax_switch_is_logged checks both legs, each in its own process. Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/build_dx12.py | 54 +++++++ clean/dx12/isa_checks.json | 25 ++++ clean/dx12/kernel_sets.json | 48 +++++++ clean/k/dx12/README.md | 16 +++ clean/k/dx12/gen_k_mix_probe.py | 58 ++++++++ clean/k/dx12/hlsl/k_common.hlsli | 43 +----- clean/k/dx12/hlsl/k_smx.hlsli | 50 +++++++ clean/k/dx12/hlsl/k_wide.hlsli | 18 +-- clean/k/dx12/k_mix_cases.py | 123 ++++++++++++++++ clean/k/dx12/k_port_check.py | 59 +++++++- clean/k/dx12/kernels/k_block.hlsl | 5 + clean/k/dx12/kernels/k_block_wide.hlsl | 22 +-- clean/k/dx12/kernels/k_mix_probe.hlsl | 39 ++++++ clean/k/runtime/k_mix_probe.h | 178 ++++++++++++++++++++++++ clean/k/runtime/runtime_k.cpp | 51 ++++++- clean/k/runtime/runtime_k.h | 11 ++ clean/k/tests/test_k_mix_probe_table.py | 64 +++++++++ clean/k/tests/test_k_runtime_d3d12.py | 41 +++++- clean/runtime/runtime.cpp | 1 + clean/runtime/wg_runtime.h | 8 +- 20 files changed, 846 insertions(+), 68 deletions(-) create mode 100644 clean/dx12/isa_checks.json create mode 100644 clean/k/dx12/gen_k_mix_probe.py create mode 100644 clean/k/dx12/hlsl/k_smx.hlsli create mode 100644 clean/k/dx12/k_mix_cases.py create mode 100644 clean/k/dx12/kernels/k_mix_probe.hlsl create mode 100644 clean/k/runtime/k_mix_probe.h create mode 100644 clean/k/tests/test_k_mix_probe_table.py diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index 9e939702..da8a9b0f 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -45,6 +45,11 @@ into , its manifest or a payload; removed after the scan); the build fails when it is not declared, gives no ISA, or PASSES rule 2, or rule 1 (unless K has no norm tree: HYB_NO_NORM). + +ISA check: every kernel built that clean/dx12/isa_checks.json lists is run through RGA (gfx1201) and the number of +instructions whose mnemonic matches each of its patterns must equal the listed count (Model K's softmax weights: one +v_fma_mixlo/hi_f16 per weight, the driver's fold that makes the mix form exact; none in the __rodd ports). A different +count fails the build. WG_DX12_ISA_CHECK=0 skips it. """ import argparse import concurrent.futures @@ -351,6 +356,54 @@ def scan_mutant(name): return name.endswith("_scanmut") +ISA_CHECKS = os.path.join(HERE, "isa_checks.json") + + +def rga_mnemonics(dxil): + """The mnemonics of one kernel's RGA (gfx1201) ISA, in order; None when RGA gives no ISA.""" + with tempfile.TemporaryDirectory(prefix="wgrga_") as tmp: + subprocess.run([RGA, "-s", "dx12", "-c", "gfx1201", "--cs-blob", dxil, "--isa", os.path.join(tmp, "x.txt")], + capture_output=True, text=True) + for f in os.listdir(tmp): + if f.endswith("_x_comp.txt"): + out = [] + with open(os.path.join(tmp, f), errors="replace") as fh: + for line in fh: + t = line.split("//")[0].strip() + if re.match(r"^[sv]_|^(buffer|global|ds|image|scratch|flat)_", t): + out.append(t.split()[0]) + return out + return None + + +def isa_check(built): + """built: [(kernel name, dxil path)] of this build. Fails when a kernel listed in isa_checks.json has another number + of instructions matching one of its patterns.""" + if os.environ.get("WG_DX12_ISA_CHECK", "1") == "0": + return + with open(ISA_CHECKS, encoding="utf-8") as f: + checks = {k: v for k, v in json.load(f).items() if not k.startswith("_")} + todo = [(n, p) for n, p in built if n in checks] + if not todo: + return + if not os.path.exists(RGA): + raise SystemExit(f"ISA check: RGA not found at {RGA} (WG_RGA, or WG_DX12_ISA_CHECK=0 to skip)") + with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, min(6, int(os.environ.get("WG_DX12_JOBS", 6))))) as ex: + isa = dict(zip([n for n, _ in todo], ex.map(rga_mnemonics, [p for _, p in todo]))) + bad = [] + for n in sorted(isa): + if isa[n] is None: + bad.append(f"{n}: RGA gave no ISA") + continue + for pat, want in checks[n].items(): + got = sum(1 for m in isa[n] if re.match(pat, m)) + print(f"ISA {n:40s} {pat} {got} / {want}", flush=True) + if got != want: + bad.append(f"{n}: {got} instructions match {pat}, {want} expected ({os.path.basename(ISA_CHECKS)})") + if bad: + raise SystemExit("ISA check failed:\n " + "\n ".join(bad)) + + def vgpr_check(built, required=()): """built: [(kernel name, dxil path)] of this build. Fails when a Swin / bottleneck kernel exceeds its bound, when a hybrid kernel breaks an ISA rule, and when a kernel of `required` (the model hybrid kernels built) has no built @@ -557,6 +610,7 @@ def build_one(job): raise SystemExit(f"not declared in clean/dx12/kernels or clean/k/dx12/kernels: {sorted(only - names)}") try: vgpr_check(built, {d["name"] for _, d, _, _ in jobs if hyb_scanned(d["name"]) and not d["probe"]}) + isa_check(built) finally: if scan_dir: shutil.rmtree(scan_dir, ignore_errors=True) diff --git a/clean/dx12/isa_checks.json b/clean/dx12/isa_checks.json new file mode 100644 index 00000000..c9c4a79e --- /dev/null +++ b/clean/dx12/isa_checks.json @@ -0,0 +1,25 @@ +{ + "_doc": "build_dx12.py ISA check: for each kernel listed, the RGA (gfx1201) ISA must contain exactly the given number of instructions whose mnemonic matches each pattern (a regular expression, matched at the start of the mnemonic). Model K's softmax weights (clean/k/dx12/hlsl/k_smx.hlsli) are exact only when the driver folds the mix form p + e into one v_fma_mixlo/hi_f16 rounding per weight: 32 weights per lane per softmax round, i.e. 32 x heads x windows per workgroup for the narrow stages (e2 / d2: 2 heads, 1 window; d1: 2 heads, 2 windows) and 32 x heads / NG for the wide stages (e3 / d3 and the NG = 2 e4 / d4: 2 rounds; NG = 4 e4 / d4: 1; m: 8 heads / NG 4 = 2). A missing fold leaves v_cvt_f16_f32 after an FP32 add (a second rounding) and fails here. The __rodd ports (round to odd) must have none. The native D3D12 runtime checks the same fold on the player's driver at create (k_mix_probe). WG_DX12_ISA_CHECK=0 skips the check.", + "k_stage_e2": {"v_fma_mix(lo|hi)_f16": 64}, + "k_stage_d2": {"v_fma_mix(lo|hi)_f16": 64}, + "k_stage2_d1": {"v_fma_mix(lo|hi)_f16": 128}, + "k_stagew2_e3": {"v_fma_mix(lo|hi)_f16": 64}, + "k_stagew4_e4": {"v_fma_mix(lo|hi)_f16": 32}, + "k_stagew2_e4": {"v_fma_mix(lo|hi)_f16": 64}, + "k_stagew4_m": {"v_fma_mix(lo|hi)_f16": 64}, + "k_stagew4_d4": {"v_fma_mix(lo|hi)_f16": 32}, + "k_stagew2_d4": {"v_fma_mix(lo|hi)_f16": 64}, + "k_stagew2_d3": {"v_fma_mix(lo|hi)_f16": 64}, + "k_mix_probe": {"v_fma_mix(lo|hi)_f16": 8}, + "k_stage_e2__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stage_d2__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stage2_d1__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stagew2_e3__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stagew4_e4__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stagew2_e4__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stagew4_m__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stagew4_d4__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stagew2_d4__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_stagew2_d3__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_mix_probe__rodd": {"v_fma_mix(lo|hi)_f16": 0} +} diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index 881354c2..db66536c 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -416,6 +416,14 @@ "k_expo_scalars_direct", "k_io.co" ], + [ + "k_mix_probe", + "k_block.co" + ], + [ + "k_mix_probe__rodd", + "k_block.co" + ], [ "k_out_color", "k_io.co" @@ -452,6 +460,10 @@ "k_stage2_d1", "k_block.co" ], + [ + "k_stage2_d1__rodd", + "k_block.co" + ], [ "k_stage2_e0", "k_block.co" @@ -468,6 +480,10 @@ "k_stage_d2", "k_block.co" ], + [ + "k_stage_d2__rodd", + "k_block.co" + ], [ "k_stage_e0", "k_block.co" @@ -480,34 +496,66 @@ "k_stage_e2", "k_block.co" ], + [ + "k_stage_e2__rodd", + "k_block.co" + ], [ "k_stagew2_d3", "k_block_wide.co" ], + [ + "k_stagew2_d3__rodd", + "k_block_wide.co" + ], [ "k_stagew2_d4", "k_block_wide.co" ], + [ + "k_stagew2_d4__rodd", + "k_block_wide.co" + ], [ "k_stagew2_e3", "k_block_wide.co" ], + [ + "k_stagew2_e3__rodd", + "k_block_wide.co" + ], [ "k_stagew2_e4", "k_block_wide.co" ], + [ + "k_stagew2_e4__rodd", + "k_block_wide.co" + ], [ "k_stagew4_d4", "k_block_wide.co" ], + [ + "k_stagew4_d4__rodd", + "k_block_wide.co" + ], [ "k_stagew4_e4", "k_block_wide.co" ], + [ + "k_stagew4_e4__rodd", + "k_block_wide.co" + ], [ "k_stagew4_m", "k_block_wide.co" ], + [ + "k_stagew4_m__rodd", + "k_block_wide.co" + ], [ "k_warm", "k_block_wide.co" diff --git a/clean/k/dx12/README.md b/clean/k/dx12/README.md index ec514023..22c05172 100644 --- a/clean/k/dx12/README.md +++ b/clean/k/dx12/README.md @@ -59,6 +59,22 @@ real textures (RGBA16F colour, R32F depth, RG16F motion, exposure, an RGBA16F ou mode and compares the output texture and the buffers with the HIP kernels' run on the raw buffer route (`KIO_C16` / `KIO_M16`, with the motion scale and offset folded as the interop passes them). +`mixprobe` (D3D12 only, no HIP kernel: checked against the exact rounding): the softmax weight rounding of every +softmax stage port, A = f16(w inv) rounded once from the exact product (`hlsl/k_smx.hlsli`). The ports write it as the +FP32 product plus its exact residual fed into the driver's round-to-nearest-even fragment conversion, which the gfx1201 +compiler folds into one `v_fma_mixlo/hi_f16` (the mix form); the `__rodd` ports round the product to odd first (the +form the ports had before; the same bits). `kernels/k_mix_probe.hlsl` runs the same helper on the records of +`k_mix_cases.py`: products whose FP32 rounding is an FP16 midpoint with a residual of either sign (normal and subnormal +FP16), exact midpoints (ties to even) and ordinary products; every result must equal the exact single rounding +(Fraction arithmetic), for `k_mix_probe` and `k_mix_probe__rodd`, and the records must tell a second rounding through +FP32 apart (the residual-dropped control `K_SMX_MUT=1` fails this family). Two more guards keep the fold: the build's +ISA check (`clean/dx12/isa_checks.json`: one `v_fma_mix*_f16` per weight in every mix-form port, none in the `__rodd` +ports) and the native D3D12 runtime, which runs `k_mix_probe` on the records of `clean/k/runtime/k_mix_probe.h` at +create and launches the `__rodd` stage ports when the result is not exact (kernels line: `smx=mix` or `smx=rodd(...)`; +test hook `WG_TEST_K_MIX_PROBE_FAIL=1`). The `__rodd` ports give the same bits, but the switch is never silent: it adds +one note of create (`wg_rt_notes`, which the executor logs; `test_k_runtime_d3d12.py` +`test_native_softmax_switch_is_logged`). + NaN: IEEE 754 does not define the sign of a NaN that a stage generates or propagates, and where the input holds NaN the two devices can differ in it (the decoder stages). For the cases with NaN in the input the check compares the raw bytes and, where they differ, accepts only the elements that are NaN on both sides and differ in the sign bit alone (the diff --git a/clean/k/dx12/gen_k_mix_probe.py b/clean/k/dx12/gen_k_mix_probe.py new file mode 100644 index 00000000..73796125 --- /dev/null +++ b/clean/k/dx12/gen_k_mix_probe.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: MIT +"""Writes clean/k/runtime/k_mix_probe.h: the records and exact expected FP16 results of the native D3D12 runtime's +softmax rounding probe (kernels/k_mix_probe.hlsl, run at create; k_mix_cases.py, seed 7, 64 lanes = two workgroups). +WrongGPU's own code. clean/k/tests/test_k_mix_probe_table.py checks the committed header against this output. + + python clean/k/dx12/gen_k_mix_probe.py [out.h] + python clean/k/dx12/gen_k_mix_probe.py --check (exit 1 unless the committed header equals this output) +""" +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) +from clean.k.dx12 import k_mix_cases # noqa: E402 + +SEED, LANES = 7, 64 +DEFAULT_OUT = os.path.join(ROOT, "clean", "k", "runtime", "k_mix_probe.h") + + +def header(): + rec, want, kinds = k_mix_cases.cases(SEED, LANES) + dbl = k_mix_cases.double_rounding_bits(rec) + words = rec.view("", + "", + "namespace k_mix_probe {", + "", + "constexpr int kLanes = %d;" % LANES, + "constexpr uint32_t kRecords[kLanes * 12] = {"] + for i in range(0, len(words), 8): + out.append(" " + ", ".join("0x%08xu" % w for w in words[i:i + 8]) + ",") + out.append("};") + out.append("constexpr uint16_t kExpected[kLanes * 8] = {") + for i in range(0, len(exp), 8): + out.append(" " + ", ".join("0x%04x" % w for w in exp[i:i + 8]) + ",") + out += ["};", "", "} // namespace k_mix_probe", ""] + return "\n".join(out) + + +if __name__ == "__main__": + if sys.argv[1:] == ["--check"]: + with open(DEFAULT_OUT, encoding="utf-8", newline="") as f: + same = f.read() == header() + print(DEFAULT_OUT, "matches the generator" if same else "DIFFERS from the generator") + sys.exit(0 if same else 1) + path = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_OUT + with open(path, "w", newline="\n") as f: + f.write(header()) + print(path) diff --git a/clean/k/dx12/hlsl/k_common.hlsli b/clean/k/dx12/hlsl/k_common.hlsli index 187a0f0d..eacbe45b 100644 --- a/clean/k/dx12/hlsl/k_common.hlsli +++ b/clean/k/dx12/hlsl/k_common.hlsli @@ -25,6 +25,8 @@ // * Driver channel calls (WMMA, conversion) only in wave-uniform control flow, with branchless neighbours. #pragma once +#include "k_smx.hlsli" + // ---- the opaque zero and half2 views --------------------------------------------------------------------------------- // wgk_zero() is 0 at run time (grids < 2^31) and unknown to the compiler; wgk_opq xors it into BOTH halves of a packed // pair (x 0x10001): with the plain 0 / 1 value the driver proved the high half's xor zero and dropped it (then fused @@ -195,53 +197,20 @@ uint4 wgk_act8n(uint4 x) // ---- softmax weights (HIP wgk_softmax_weights: FP32 weights, the network's default) --------------------------- // gg: one query's 4 fragments of 8 keys (lane = query, the clamped polynomial g of spec 41 applied). w = 2^g in FP32 -// (v_exp_f32), the sum over 64 keys = 32 per lane in fragment / element order plus one lane exchange, inv = v_rcp_f32, -// A = f16(w inv) rounded ONCE from the exact product (HIP: v_fma_mixlo_f16 / v_fma_mixhi_f16 w * inv + 0). Here: the -// product's FP32 rounding p and its exact residual e = fma(w, inv, -p) give the round-to-odd FP32 value (13 bits more -// than FP16: rounding it to FP16 is the single rounding of the exact product), then the driver's RNE conversion. -// w >= 2^-14.43 and inv > 0 are finite and normal, so no zero / sign / FTZ case arises. -// Round to odd for a positive normal FP32 product p = RNE(x) with exact residual e = x - p (wg_math.hlsli -// wg_round_odd, specialised: no zero / infinity / sign cases): truncation of x is p, or p - 1 ulp when e < 0; the -// inexact bit e != 0 makes it odd. Gives the same bits as wg_round_odd for every p > 0 finite. -uint wgk_round_odd_pos(float p, float e) -{ - return (asuint(p) - (e < 0.0f ? 1u : 0u)) | (e != 0.0f ? 1u : 0u); -} -#ifndef KB_SMX -#define KB_SMX 1 -#endif +// (v_exp_f32, every 2^g first, then the sum in fragment / element order), the sum over 64 keys = 32 per lane plus one +// lane exchange, inv = v_rcp_f32, A = f16(w inv) rounded ONCE from the exact product (HIP: v_fma_mixlo_f16 / +// v_fma_mixhi_f16 w * inv + 0): k_smx.hlsli (the mix form, or round to odd in the __rodd ports). void wgk_softmax_weights(uint4 gg[4], out uint4 pa[4]) { float w[4][8]; float sum = 0.0f; -#if KB_SMX - // every 2^g first (independent), then the sum in the same element order [unroll] for (uint st = 0; st < 4u; ++st) [unroll] for (uint j = 0; j < 8u; ++j) w[st][j] = wg_exp2(wgk_elf(gg[st], j)); [unroll] for (uint s1 = 0; s1 < 4u; ++s1) [unroll] for (uint j = 0; j < 8u; ++j) sum = sum + w[s1][j]; -#else - [unroll] for (uint st = 0; st < 4u; ++st) - [unroll] for (uint j = 0; j < 8u; ++j) { - w[st][j] = wg_exp2(wgk_elf(gg[st], j)); - sum = sum + w[st][j]; - } -#endif sum = sum + wgk_xor16(sum); const float inv = wg_rcp(sum); - [unroll] for (uint s2 = 0; s2 < 4u; ++s2) { - v8f ro; - [unroll] for (uint j = 0; j < 8u; ++j) { - const float p = w[s2][j] * inv; - const float e = mad(w[s2][j], inv, -p); -#if KB_SMX - ro.v[j] = asfloat(wgk_round_odd_pos(p, e)); -#else - ro.v[j] = asfloat(wg_round_odd(p, e)); -#endif - } - pa[s2] = wgk_round8(ro); - } + [unroll] for (uint s2 = 0; s2 < 4u; ++s2) pa[s2] = k_smx_weights8(w[s2], inv); } // ss + the sum of f16(x_c^2) over the 8 channels of x, in channel order, each addition in FP32 (HIP wgk_sumsq8: packed diff --git a/clean/k/dx12/hlsl/k_smx.hlsli b/clean/k/dx12/hlsl/k_smx.hlsli new file mode 100644 index 00000000..1187b61e --- /dev/null +++ b/clean/k/dx12/hlsl/k_smx.hlsli @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: MIT +// Model K's softmax weight rounding, shared by the narrow (k_common.hlsli wgk_softmax_weights) and wide +// (k_block_wide.hlsl) stage ports and by the probe kernel k_mix_probe (kernels/k_mix_probe.hlsl). WrongGPU's own code. +// +// HIP (k_block.hip / k_block_wide.hip) rounds each weight A = f16(w inv) ONCE from the exact product +// (v_fma_mixlo_f16 / v_fma_mixhi_f16 w, inv, 0). With p = RN32(w inv) and its exact residual e = fma(w, inv, -p), +// w inv = p + e exactly (w and inv are positive, finite and normal in the softmax: w = 2^g with |g| <= ~14.4, inv the +// reciprocal of a sum of such w, so p >= ~3e-11 and e is exact; no zero, sign or FTZ case arises). +// K_SMX 2 (default, the mix form): the sum p + e goes into the driver's round-to-nearest-even fragment conversion +// (wg_to_f16_ags); the gfx1201 D3D12 compiler folds that addition into v_fma_mixlo_f16 / v_fma_mixhi_f16 p, 1.0, e, +// whose single rounding of the exact p + e is RN16(w inv), HIP's bits (3 VALU per weight instead of 8). The fold +// is the driver's choice: without it the FP32 sum is p and the conversion a second rounding (one FP16 ulp off +// where p is an FP16 midpoint). Guards: build_dx12.py --isa checks every mix-form port's v_fma_mix*_f16 count +// (clean/dx12/isa_checks.json); the native D3D12 runtime runs k_mix_probe on FP16-midpoint products at create and +// takes the K_SMX 1 ports (suffix __rodd) when it fails (clean/k/runtime/runtime_k.cpp, k_mix_probe.h). +// K_SMX 1 (the __rodd ports): p and e give the round-to-odd FP32 value (13 bits more than FP16, so rounding it to +// FP16 is the single rounding of the exact product), then the same conversion. Truncation of w inv is p, or p - 1 +// ulp when e < 0; the inexact bit e != 0 makes it odd (the same bits as wg_round_odd for every finite p > 0). +// K_SMX_MUT 1: negative control of the checks (the residual dropped: a second rounding); never shipped. +#pragma once + +#ifndef K_SMX +#define K_SMX 2 +#endif +#ifndef K_SMX_MUT +#define K_SMX_MUT 0 +#endif + +uint k_smx_round_odd_pos(float p, float e) +{ + return (asuint(p) - (e < 0.0f ? 1u : 0u)) | (e != 0.0f ? 1u : 0u); +} + +// A = f16(w[j] inv) for one fragment's eight weights, packed two per dword (element 2 i in the low half of dword i). +uint4 k_smx_weights8(float w[8], float inv) +{ + v8f ro; + [unroll] for (uint j = 0; j < 8u; ++j) { + const float p = w[j] * inv; + const float e = mad(w[j], inv, -p); // exact: w inv = p + e +#if K_SMX >= 2 && K_SMX_MUT + ro.v[j] = p; +#elif K_SMX >= 2 + ro.v[j] = p + e; +#else + ro.v[j] = asfloat(k_smx_round_odd_pos(p, e)); +#endif + } + return wg_to_f16_ags(ro); +} diff --git a/clean/k/dx12/hlsl/k_wide.hlsli b/clean/k/dx12/hlsl/k_wide.hlsli index b197ef0f..3b8b0b3a 100644 --- a/clean/k/dx12/hlsl/k_wide.hlsli +++ b/clean/k/dx12/hlsl/k_wide.hlsli @@ -15,8 +15,9 @@ // RN16(v_rsq_f32(s)) (wg_f2h: RNE; the plain f32tof16 rounds toward zero on this driver); // - softmax weights: w = v_exp_f32(g), FP32 sum in key order plus the lane ^ 16 half, inv = v_rcp_f32(sum), and // A = f16(w inv) rounded ONCE from the exact product, as HIP's compiled form (v_fma_mixlo/hi_f16 w * inv + 0, seen in -// the k_block_wide.co ISA): the FP32 product rounded to odd with its exact residual (wg_round_odd), then the RNE -// conversion (FP32 has 13 more bits than FP16: no double rounding); +// the k_block_wide.co ISA): k_smx.hlsli, the FP32 product and its exact residual summed into the RNE conversion +// (folded into v_fma_mix*_f16 by the driver), or in the __rodd ports the product rounded to odd, then the conversion +// (FP32 has 13 more bits than FP16: no double rounding); // - mirrored window members, ownership of the output tiles (a group without a last tile computes tile KT - 1 too and // does not store it), merge / expand index maps: as HIP. // Deviations (no effect on any result): weights and LDS operands of each product group are loaded before its multiplies @@ -25,8 +26,7 @@ // k_block_wide.hlsl), each loaded through a per-lane base address plus a constant (the load's immediate offset); // independent accumulators are interleaved (the scores' four key tiles, z's two channel tiles, W1's MM hidden tiles, // the expand's tiles), each still summing in its own order; the LDS indices are masked to the array's power of two -// (kw_lr / kw_lw); the residual x of the owned tiles is reloaded at step 3 for map inputs; the softmax round-to-odd uses -// the positive-product form (kw_round_odd_pos, same bits); NG = 4 uses another LDS layout that fits D3D12's 32 KB (HIP: 56 KB): K / VT per group unpadded with an xor +// (kw_lr / kw_lw); the residual x of the owned tiles is reloaded at step 3 for map inputs; NG = 4 uses another LDS layout that fits D3D12's 32 KB (HIP: 56 KB): K / VT per group unpadded with an xor // swizzle of the 16-byte chunks, and the round's Z, the MLP's A and its H buffers all start at 0, with extra barriers // where the phases now overlap (K / VT reads done before Z is written, Z reads done before A / the next round's K / VT, // A reads done before H, H reads done before the merge's x2). The arithmetic is the same in both layouts. @@ -131,16 +131,6 @@ uint4 kw_softpoly8(uint4 s) return r; } -// wg_round_odd (wg_math.hlsli) for a product p = RN32(w inv) that is positive, finite and normal, with its exact residual -// e: the softmax weights (w = v_exp_f32 of a clamped FP16 polynomial value in about [-14.4, 14.4], inv = v_rcp_f32 of a -// sum of 64 such w, so p >= ~3e-11). There wg_round_odd's finite / nonzero test always holds and "one ulp toward x" is -// +1 for e > 0 and -1 for e < 0 (p's sign bit is 0): the same bits in fewer instructions. -uint kw_round_odd_pos(float p, float e) -{ - const uint ur = asuint(p); - return ur + ((e != 0.0f && (ur & 1u) == 0u) ? (uint)((asint(e) >> 31) | 1) : 0u); -} - v8f kw_widen8(uint4 a) { v8f r; diff --git a/clean/k/dx12/k_mix_cases.py b/clean/k/dx12/k_mix_cases.py new file mode 100644 index 00000000..65a91f64 --- /dev/null +++ b/clean/k/dx12/k_mix_cases.py @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: MIT +"""Adversarial cases for Model K's softmax weight rounding (hlsl/k_smx.hlsli, kernels/k_mix_probe.hlsl). WrongGPU's own +code. + +A softmax weight is A = f16(w inv), ONE round-to-nearest-even rounding of the exact product of two positive FP32 values. +A form that rounds twice (to FP32 first, then to FP16) differs exactly where p = RN32(w inv) is an FP16 rounding +midpoint and the residual e = w inv - p is nonzero: the exact product lies on one side of the midpoint, the double +rounding sees a tie. These cases put such products in every lane, with both residual signs, in the FP16 normal and +subnormal ranges, plus exact midpoints (e = 0: ties to even) and ordinary products. + +One record per lane (k_mix_probe's layout): 12 FP32 values, inv, w0 .. w7, three zeros; the expected output is 8 FP16 +patterns, A_j = RN16(w_j inv) computed exactly here (Fraction arithmetic: no floating-point rounding in the oracle). +Products lie in [2^-26, 1] (a softmax weight is at most 1; below 2^-25 the FP16 result is 0 or the smallest subnormal). + + records, expected, kinds = cases(seed, lanes) +""" +from fractions import Fraction + +import numpy as np + +KINDS = ("mid+", "mid-", "tie", "sub+", "sub-", "plain") + + +def rn16_bits(x): + """RN16 of a positive exact value (Fraction or int/float converted exactly), as the FP16 bit pattern.""" + x = Fraction(x) + if x <= 0: + raise ValueError("positive values only") + # exponent of x: 2^e <= x < 2^(e+1) + e = x.numerator.bit_length() - x.denominator.bit_length() + if Fraction(2) ** e > x: + e -= 1 + q_exp = max(e, -14) - 10 # quantum: 2^(e-10) for normal FP16, 2^-24 below 2^-14 + q = x / (Fraction(2) ** q_exp) + n = q.numerator // q.denominator + r = q - n + if r > Fraction(1, 2) or (r == Fraction(1, 2) and n % 2 == 1): + n += 1 + if e < -14: # subnormal (n may reach 1024 = the smallest normal) + return int(n) + if n == 2048: + n, e = 1024, e + 1 + if e + 15 >= 31: + return 0x7C00 + return ((e + 15) << 10) | (int(n) - 1024) + + +def _midpoint(p): + """float32 array p > 0: True where p is an FP16 rounding midpoint (normal: low 13 significand bits 0x1000; + subnormal FP16 range: p / 2^-24 has the fraction 1/2).""" + b = p.view(np.uint32) + normal = p >= np.float32(2.0 ** -14) + mid_n = (b & np.uint32(0x1FFF)) == np.uint32(0x1000) + s = p.astype(np.float64) * 2.0 ** 24 + mid_s = (s - np.floor(s)) == 0.5 + return np.where(normal, mid_n, mid_s), normal + + +def cases(seed, lanes): + rng = np.random.default_rng(seed) + records = np.zeros((lanes, 12), np.float32) + kinds = [] + searched = [k for k in KINDS if k != "tie"] + for lane in range(lanes): + # every fourth lane holds exact midpoints (its inv is a power of two, chosen with them); the others the searched + # kinds for their own inv + want = ["tie"] * 8 if lane % 4 == 3 else [searched[(lane + j) % len(searched)] for j in range(8)] + # inv: wide range; for subnormal-result lanes small enough that products below 2^-14 are common + inv = np.float32(2.0 ** rng.uniform(-30.0, 0.0)) + got = {} + for _ in range(64): + if all(k in got and len(got[k]) >= want.count(k) for k in set(want)): + break + w = (2.0 ** rng.uniform(-14.4, 14.4, 1 << 16)).astype(np.float32) + p = w * inv # RN32 product (numpy float32) + x = w.astype(np.float64) * np.float64(inv) # exact (48-bit product) + ok = (x <= 1.0) & (x >= 2.0 ** -26) + mid, normal = _midpoint(p) + e = x - p.astype(np.float64) # exact residual + sel = {"mid+": ok & mid & normal & (e > 0), "mid-": ok & mid & normal & (e < 0), + "sub+": ok & mid & ~normal & (e > 0), "sub-": ok & mid & ~normal & (e < 0), + "plain": ok & ~mid} + for k, m in sel.items(): + idx = np.nonzero(m)[0] + got.setdefault(k, []).extend(w[idx[:8]].tolist()) + ws = [] + for j, k in enumerate(want): + if k == "tie": + # an exact FP16 midpoint as the product: w = (2n + 1) 2^(t - 11) with 2^t in the normal FP16 range, + # times inv' = a power of two (the lane's inv is replaced by it when the lane has a tie: e = 0) + ws.append(None) + continue + pool = got.get(k) or got.get("plain") + ws.append(np.float32(pool[j % len(pool)])) + if any(v is None for v in ws): + inv = np.float32(2.0 ** -int(rng.integers(0, 10))) + for j, v in enumerate(ws): + if v is None: + t = int(rng.integers(-12, 0)) # product in [2^-12, 1) + n = int(rng.integers(0, 1024)) + prod = Fraction(2 * (1024 + n) + 1, 2 ** 11) * Fraction(2) ** t + ws[j] = np.float32(float(prod / Fraction(float(inv)))) + elif not (2.0 ** -26 <= float(v) * float(inv) <= 1.0): + ws[j] = np.float32(0.5 / float(inv)) # keep the product in range + records[lane, 0] = inv + records[lane, 1:9] = ws + kinds.append(want) + expected = np.zeros((lanes, 8), np.uint16) + for lane in range(lanes): + inv = Fraction(float(records[lane, 0])) + for j in range(8): + expected[lane, j] = rn16_bits(Fraction(float(records[lane, 1 + j])) * inv) + return records, expected, kinds + + +def double_rounding_bits(records): + """What a second rounding through FP32 gives (the K_SMX_MUT control): RN16(RN32(w inv)) per element.""" + out = np.zeros((records.shape[0], 8), np.uint16) + for lane in range(records.shape[0]): + for j in range(8): + p = np.float32(records[lane, 1 + j]) * np.float32(records[lane, 0]) + out[lane, j] = rn16_bits(Fraction(float(p))) + return out diff --git a/clean/k/dx12/k_port_check.py b/clean/k/dx12/k_port_check.py index 4f0aae70..a18578a7 100644 --- a/clean/k/dx12/k_port_check.py +++ b/clean/k/dx12/k_port_check.py @@ -24,7 +24,9 @@ low-resolution motion, inverted depth, reset), odd and tiny base grids, exposure scalars from a device buffer, special values in the colour, depth, motion and histories (the same three levels as the stage families); wide = the wide network stages (k_block_wide.co: k_stagew_e3 .. _d3), as narrow; narrow = the narrow network stages (k_block.co: k_stage_e2 .. k_stage2_d0_hidden; the stage runner's launch plans, clean/k/dx12/k_plan_bridge.py), on random stage weights and inputs with special values (NaN, infinity, -+-65504, zero, -0, subnormal) in every other case; expo = k_expo_scalars against k_io.co (HIP); warm = k_warm (D3D12 only, no HIP kernel: it is checked ++-65504, zero, -0, subnormal) in every other case; expo = k_expo_scalars against k_io.co (HIP); mixprobe = k_mix_probe / k_mix_probe__rodd (D3D12 only: the +softmax weight rounding of hlsl/k_smx.hlsli on k_mix_cases.py's FP16-midpoint records, against the exact single +rounding; README.md); warm = k_warm (D3D12 only, no HIP kernel: it is checked against its own definition, the dummy word is written exactly when the key equals a lane's loaded value). One known limit, shared with Model M's ports: D3D12 flushes FP32 denormals (inputs and results) to zero, HIP keeps them. k_expo_scalars only meets one with a pre-exposure above about 8.5e37 (kappa = RN(1 / pi) * E), so the inputs here keep @@ -53,6 +55,7 @@ NARROW_KERNELS = ["k_stage_e2", "k_stage_e0", "k_stage_e1", "k_stage_d2", "k_stage_d1", "k_stage2_e0", "k_stage2_e1", "k_stage2_d1", "k_stage2_d0_hp", "k_stage2_d0_hidden"] FAMILY_KERNELS = {"expo": (["k_io.co"], ["k_expo_scalars"]), "warm": ([], ["k_warm"]), + "mixprobe": ([], ["k_mix_probe", "k_mix_probe__rodd"]), "narrow": (["k_block.co"], NARROW_KERNELS), "input": (["k_io_exact.co"], ["kx/k_in_tiled"]), "legacy": (["k_io_exact.co"], ["kx/k_in_color", "kx/k_in_reset", "kx/k_in_base"]), @@ -301,6 +304,54 @@ def run_warm(build, rocm, seeds, cdir, hip): return items, lines +# ---- family mixprobe: the softmax weight rounding (hlsl/k_smx.hlsli) ----------------------------------------------- + +MIXPROBE_LANES = 256 + + +def run_mixprobe(build, rocm, seeds, cdir, hip): + """D3D12 only: k_mix_probe (the mix form of every softmax port) and k_mix_probe__rodd (the __rodd ports' form) on + the adversarial records of k_mix_cases.py, checked against the exact single rounding; the records must also tell a + second rounding through FP32 apart (else they could not catch the residual-dropped control).""" + from clean.k.dx12 import k_mix_cases + lines, items = [], [] + for si, seed in enumerate(seeds): + rec, want, kinds = k_mix_cases.cases(seed + 2000, MIXPROBE_LANES) + dbl = k_mix_cases.double_rounding_bits(rec) + name = f"s{si}_mixprobe" + write(os.path.join(cdir, name + ".in"), rec.tobytes()) + for kern in FAMILY_KERNELS["mixprobe"][1]: + tag = f"{name}_{kern}" + nbytes = MIXPROBE_LANES * 16 + lines += [f"chain {tag}", "module k_block.co", f"buf in {rec.nbytes} {name}.in 0", + f"buf out {nbytes + SLACK} - {D3D12_SENTINEL}", + f"step {kern} {MIXPROBE_LANES // 32} 32 @in @out", f"save out {tag}.out", ""] + items.append({"case": tag, "kernel": kern, "want": want, "double": dbl, "kinds": kinds}) + return items, lines + + +def check_mixprobe(items, read): + out = {} + good = True + for it in items: + got = read(it["case"] + ".out") + n = MIXPROBE_LANES * 16 + if len(got) < n: + r = {"written": False} + good = False + else: + g = np.frombuffer(got[:n], np.uint16).reshape(MIXPROBE_LANES, 8) + bad = g != it["want"] + kinds = np.array(it["kinds"]) + r = {"elements": int(g.size), "differ": int(bad.sum()), + "differ_by_kind": {k: int((bad & (kinds == k)).sum()) for k in sorted(set(kinds.ravel()))}, + "double_rounding_would_differ": int((it["double"] != it["want"]).sum()), + "slack_ok": got[n:] == bytes([D3D12_SENTINEL]) * (len(got) - n)} + good = good and r["differ"] == 0 and r["slack_ok"] and r["double_rounding_would_differ"] > 0 + out[it["case"]] = r + return {"cases": len(items), "results": out}, good + + # ---- family narrow: the narrow network stages (k_block.co) --------------------------------------------------------- # kernel -> (stage, stage-runner suffix, d0 form); the d0 kernels are the network's own forms (hidden history, compact @@ -909,6 +960,9 @@ def run(build, rocm, families, seeds, json_out): if "warm" in families: state["warm"] = run_warm(build, rocm, seeds, cdir, hip) lines += state["warm"][1] + if "mixprobe" in families: + state["mixprobe"] = run_mixprobe(build, rocm, seeds, cdir, hip) + lines += state["mixprobe"][1] if "narrow" in families: state["narrow"] = run_stages(NARROW, "k_block.co", "nar", build, rocm, seeds, cdir, hip) lines += state["narrow"][1] @@ -982,6 +1036,9 @@ def read(n): fails.append(it["case"]) res["results"]["warm"] = {"cases": len(state["warm"][0]), "as_defined": good, "failures": fails[:5]} ok = ok and good == len(state["warm"][0]) and good > 0 + if "mixprobe" in state: + res["results"]["mixprobe"], good = check_mixprobe(state["mixprobe"][0], read) + ok = ok and good for fam in ("narrow", "wide", "input", "legacy", "output", "direct", "directleg"): if fam in state: res["results"][fam], good = check_narrow(state[fam], read) diff --git a/clean/k/dx12/kernels/k_block.hlsl b/clean/k/dx12/kernels/k_block.hlsl index 158cdbc4..1ce5ac9d 100644 --- a/clean/k/dx12/kernels/k_block.hlsl +++ b/clean/k/dx12/kernels/k_block.hlsl @@ -30,6 +30,11 @@ // @wg-kernel k_stage_d1 co=k_block.co block=128 wave=32 args=s184:p19.b32 // @wg-kernel k_stage2_d0_hp co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_HIDDEN_ARG=1 // @wg-kernel k_stage2_d0_hidden co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_HIDDEN_ARG=1 +// The __rodd forms: the same stages with the round-to-odd softmax weights (hlsl/k_smx.hlsli K_SMX 1), launched by the +// native D3D12 runtime when the mix-form probe (kernels/k_mix_probe.hlsl) fails on the adapter's driver. +// @wg-kernel k_stage_e2__rodd co=k_block.co entry=k_stage_e2 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1 +// @wg-kernel k_stage_d2__rodd co=k_block.co entry=k_stage_d2 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1 +// @wg-kernel k_stage2_d1__rodd co=k_block.co entry=k_stage2_d1 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1 #include "wg_platform.hlsli" #include "k_blkparams.hlsli" diff --git a/clean/k/dx12/kernels/k_block_wide.hlsl b/clean/k/dx12/kernels/k_block_wide.hlsl index 347d8df0..0303fd57 100644 --- a/clean/k/dx12/kernels/k_block_wide.hlsl +++ b/clean/k/dx12/kernels/k_block_wide.hlsl @@ -11,6 +11,15 @@ // @wg-kernel k_stagew4_d4 co=k_block_wide.co block=512 wave=32 args=s184:p19.b32 entry=kw_stage defines=KW_C=128,KW_HEADS=4,KW_IN=1,KW_CIN=160,KW_CNEXT=0,KW_NG=4,KW_RD=12 // @wg-kernel k_stagew2_d4 co=k_block_wide.co block=256 wave=32 args=s184:p19.b32 entry=kw_stage defines=KW_C=128,KW_HEADS=4,KW_IN=1,KW_CIN=160,KW_CNEXT=0,KW_NG=2 // @wg-kernel k_stagew2_d3 co=k_block_wide.co block=256 wave=32 args=s184:p19.b32 entry=kw_stage defines=KW_C=96,KW_HEADS=4,KW_IN=1,KW_CIN=128,KW_CNEXT=0,KW_NG=2,KW_RD=4 +// The __rodd forms: the same stages with the round-to-odd softmax weights (hlsl/k_smx.hlsli K_SMX 1), launched by the +// native D3D12 runtime when the mix-form probe (kernels/k_mix_probe.hlsl) fails on the adapter's driver. +// @wg-kernel k_stagew2_e3__rodd co=k_block_wide.co block=256 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=96,KW_HEADS=4,KW_IN=0,KW_CIN=96,KW_CNEXT=128,KW_NG=2,KW_RD=4 +// @wg-kernel k_stagew4_e4__rodd co=k_block_wide.co block=512 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=128,KW_HEADS=4,KW_IN=0,KW_CIN=128,KW_CNEXT=160,KW_NG=4,KW_RD=12 +// @wg-kernel k_stagew2_e4__rodd co=k_block_wide.co block=256 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=128,KW_HEADS=4,KW_IN=0,KW_CIN=128,KW_CNEXT=160,KW_NG=2 +// @wg-kernel k_stagew4_m__rodd co=k_block_wide.co block=512 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4,KW_RD=4 +// @wg-kernel k_stagew4_d4__rodd co=k_block_wide.co block=512 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=128,KW_HEADS=4,KW_IN=1,KW_CIN=160,KW_CNEXT=0,KW_NG=4,KW_RD=12 +// @wg-kernel k_stagew2_d4__rodd co=k_block_wide.co block=256 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=128,KW_HEADS=4,KW_IN=1,KW_CIN=160,KW_CNEXT=0,KW_NG=2 +// @wg-kernel k_stagew2_d3__rodd co=k_block_wide.co block=256 wave=32 args=s184:p19.b32 entry=kw_stage defines=K_SMX=1,KW_C=96,KW_HEADS=4,KW_IN=1,KW_CIN=128,KW_CNEXT=0,KW_NG=2,KW_RD=4 #include "wg_platform.hlsli" WG_ARGS_BEGIN @@ -20,6 +29,7 @@ WG_ARGS_BEGIN WG_ARGS_END #include "k_wide.hlsli" +#include "k_smx.hlsli" // ---- the weight stream ---------------------------------------------------------------------------------------------- // Every global weight fragment of the kernel's matrix products (A operands; v^T: the B operand), in product order: @@ -256,15 +266,9 @@ WG_KERNEL(kw_stage) } sum += wg_xor16(sum); const float inv = rcp(sum); // v_rcp_f32 - [unroll] for (int st4 = 0; st4 < 4; ++st4) { - v8f ro; - [unroll] for (int j = 0; j < 8; ++j) { - const float p = w[st4][j] * inv; - const float e = mad(w[st4][j], inv, -p); // exact: w inv = p + e - ro.v[j] = asfloat(kw_round_odd_pos(p, e)); - } - pa[st4] = kw_out(ro); // f16(w inv), one rounding of the exact product - } + // A = f16(w inv), one rounding of the exact product (k_smx.hlsli: the mix form, or round to odd in the + // __rodd ports) + [unroll] for (int st4 = 0; st4 < 4; ++st4) pa[st4] = k_smx_weights8(w[st4], inv); } // z_h^T = VT A^T (the two channel tiles' chains interleaved, each over st) uint4 zz[2]; diff --git a/clean/k/dx12/kernels/k_mix_probe.hlsl b/clean/k/dx12/kernels/k_mix_probe.hlsl new file mode 100644 index 00000000..5d0a244b --- /dev/null +++ b/clean/k/dx12/kernels/k_mix_probe.hlsl @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: MIT +// Probe of Model K's softmax weight rounding on D3D12 (hlsl/k_smx.hlsli). WrongGPU's own code; D3D12 only (no HIP +// kernel: it is checked against the exact rounding of its inputs, clean/k/dx12/k_port_check.py family mixprobe, and the +// native D3D12 runtime runs it at create on the cases of clean/k/runtime/k_mix_probe.h). +// Lane i of the grid reads one record of 12 floats at byte 48 i: inv, w0 .. w7 (and 3 unused) and writes +// k_smx_weights8(w, inv) = f16(w_j inv) for j = 0..7, two per dword, at byte 16 i: the same helper, the same operations +// and the same driver conversion as every softmax port. The records are FP16-midpoint products with a nonzero residual +// of either sign, exact midpoints (ties to even), FP16-subnormal and ordinary products, so a second rounding through +// FP32 (the mix form without the driver's fold, or the K_SMX_MUT control) cannot pass. +// k_mix_probe: the default (mix) form; k_mix_probe__rodd: the round-to-odd form of the __rodd stage ports. +// Control flow is uniform (the driver channel's rule): every lane of every launched workgroup has a record. +// @wg-kernel k_mix_probe co=k_block.co block=32 wave=32 args=p,p +// @wg-kernel k_mix_probe__rodd co=k_block.co entry=k_mix_probe block=32 wave=32 args=p,p defines=K_SMX=1 +#include "wg_platform.hlsli" + +WG_ARGS_BEGIN + uint P_in, P_out; +WG_ARGS_END + +#include "k_smx.hlsli" + +WG_KERNEL(k_mix_probe) +{ + WG_SETUP(); + const uint i = wg_block_id() * 32u + wg_lane_id(); + const uint4 a = wg_ld128(P_in, i * 48u); // inv, w0, w1, w2 + const uint4 b = wg_ld128(P_in, i * 48u + 16u); // w3 .. w6 + const uint4 c = wg_ld128(P_in, i * 48u + 32u); // w7, unused + float w[8]; + w[0] = asfloat(a.y); + w[1] = asfloat(a.z); + w[2] = asfloat(a.w); + w[3] = asfloat(b.x); + w[4] = asfloat(b.y); + w[5] = asfloat(b.z); + w[6] = asfloat(b.w); + w[7] = asfloat(c.x); + wg_st128(P_out, i * 16u, k_smx_weights8(w, asfloat(a.x))); +} diff --git a/clean/k/runtime/k_mix_probe.h b/clean/k/runtime/k_mix_probe.h new file mode 100644 index 00000000..9d4de095 --- /dev/null +++ b/clean/k/runtime/k_mix_probe.h @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: MIT +// Generated by clean/k/dx12/gen_k_mix_probe.py (k_mix_cases.py seed 7, 64 lanes); do not edit. +// The native D3D12 runtime's softmax rounding probe (clean/k/dx12/kernels/k_mix_probe.hlsl): per lane 12 FP32 +// words (inv, w0 .. w7, 0, 0, 0) and the 8 exact FP16 results RN16(w_j inv). A second rounding through FP32 differs +// from them in 152 of the 512 results. +#pragma once + +#include + +namespace k_mix_probe { + +constexpr int kLanes = 64; +constexpr uint32_t kRecords[kLanes * 12] = { + 0x39d7b27fu, 0x3f204c7bu, 0x3f0f465fu, 0x3e094db6u, 0x3df79b37u, 0x385783b3u, 0x3fb91300u, 0x401347cfu, + 0x3dfcf277u, 0x00000000u, 0x00000000u, 0x00000000u, 0x36bbcc73u, 0x45b716f8u, 0x407458cau, 0x40c8e562u, + 0x408e53beu, 0x42922814u, 0x438e6869u, 0x4124afe2u, 0x411af979u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3dd702c4u, 0x39412e56u, 0x39a5f1f5u, 0x3c103180u, 0x3ee46115u, 0x3dc6e933u, 0x39d1b37bu, 0x393d9becu, + 0x38f86329u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c800000u, 0x3dfed000u, 0x3cf8b000u, 0x3deeb000u, + 0x3e957000u, 0x40b3d000u, 0x3dacd000u, 0x41f4f000u, 0x401c5000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3f3a32e6u, 0x3d0072d1u, 0x3ebe298eu, 0x3f3c9d97u, 0x3879b62au, 0x38a35a2eu, 0x3ce10db0u, 0x3dac96f7u, + 0x3f258b21u, 0x00000000u, 0x00000000u, 0x00000000u, 0x356ae9dfu, 0x432735e0u, 0x4532ece6u, 0x4071ed63u, + 0x404ab222u, 0x40c95021u, 0x467a4e0au, 0x450863feu, 0x421d55a0u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x31b5c3adu, 0x46918d14u, 0x45febf57u, 0x45aa9831u, 0x440dd21bu, 0x467a6f24u, 0x4652d289u, 0x45febf57u, + 0x46177e2bu, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x3be5d000u, 0x3f71b000u, 0x3f3e7000u, + 0x3b869000u, 0x3b185000u, 0x3d4bf000u, 0x3d42d000u, 0x3d45d000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3d5c7659u, 0x3a2c0090u, 0x3871bc10u, 0x3bc1d8e3u, 0x41710553u, 0x3a1b2a34u, 0x39a2db97u, 0x3f7d7e87u, + 0x3b355d46u, 0x00000000u, 0x00000000u, 0x00000000u, 0x344807adu, 0x3eb55e05u, 0x43cf3fbau, 0x449693d2u, + 0x437071a7u, 0x438fe5ffu, 0x458d0d08u, 0x461ddb39u, 0x43b64aecu, 0x00000000u, 0x00000000u, 0x00000000u, + 0x37bd3ae6u, 0x4696b74cu, 0x4626b251u, 0x3f6e709eu, 0x3f61967bu, 0x3b080bcdu, 0x4696b74cu, 0x462b99c3u, + 0x4024a000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x3a763000u, 0x3f417000u, 0x3dec9000u, + 0x3fbed000u, 0x3a49f000u, 0x3c039000u, 0x3b0f5000u, 0x3b5ff000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x32fd8d40u, 0x44862874u, 0x447210a2u, 0x446f7410u, 0x45541740u, 0x46105153u, 0x4424d389u, 0x43d68d46u, + 0x44ee3be7u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3e1f7225u, 0x39a5125au, 0x3f749bd8u, 0x3e8e0abcu, + 0x3cb129dfu, 0x39131c07u, 0x399907adu, 0x3fc13b88u, 0x3f7357c0u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x314d9726u, 0x41bf3cfbu, 0x430bbf65u, 0x4392ee81u, 0x469af2e8u, 0x462c9d73u, 0x41cd1b8du, 0x45ce0d79u, + 0x414852c0u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x3fd8b000u, 0x3b1f7000u, 0x3a0d9000u, + 0x3e195000u, 0x3ae99000u, 0x3b8d1000u, 0x3eda3000u, 0x3b6b5000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3abf11edu, 0x405da82eu, 0x3ca54cf8u, 0x3cb965e3u, 0x44140e9bu, 0x41254240u, 0x3d3c2950u, 0x3cd48788u, + 0x3bc448b8u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c46de96u, 0x3b3e08c1u, 0x3adb28a8u, 0x4106057eu, + 0x3ba7e8bdu, 0x3d801fceu, 0x3b4e726du, 0x3adb28a8u, 0x3c0299c2u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3aae66f4u, 0x3bc1066fu, 0x3e94a619u, 0x4118e355u, 0x3e20afa3u, 0x3d1487c0u, 0x3c47437au, 0x41d89d28u, + 0x3e8fb6bdu, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3a0a9000u, 0x3d2a9000u, 0x39aad000u, + 0x3e05b000u, 0x3c971000u, 0x3cdff000u, 0x3a1fd000u, 0x399c5000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3e92025cu, 0x3a6088a6u, 0x3a000bf5u, 0x3906e546u, 0x38fd227cu, 0x3ff7e50du, 0x3c23fd5au, 0x3b68dcacu, + 0x391d4017u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c8f5796u, 0x40659ac1u, 0x3af1c56eu, 0x3b1a5f36u, + 0x3a205212u, 0x40255dd9u, 0x3cfd7dcdu, 0x39aa8e96u, 0x3aa48789u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x37eff3fau, 0x3e6bc78cu, 0x3fd53de2u, 0x4575349eu, 0x40e771ffu, 0x4026e63au, 0x3fbcf867u, 0x3f42f8b3u, + 0x417ab287u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3e800000u, 0x3f67b000u, 0x3e9eb000u, 0x3b213000u, + 0x3bf5d000u, 0x3f28f000u, 0x3e67d000u, 0x3c0a5000u, 0x3af85000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3e9a65b0u, 0x3c65dc72u, 0x3c8e7014u, 0x4024ecddu, 0x38dcbab8u, 0x392cf4dcu, 0x38ccadb0u, 0x3eb8ecf5u, + 0x3aebf67au, 0x00000000u, 0x00000000u, 0x00000000u, 0x39fdf808u, 0x40090651u, 0x42def4dau, 0x3d53ee76u, + 0x3d18f6b0u, 0x3c161ec5u, 0x443784c0u, 0x42819737u, 0x3d04cdbeu, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3461e365u, 0x44b24868u, 0x4226076cu, 0x40cbfea2u, 0x466a1b50u, 0x43de32c8u, 0x45890604u, 0x438d4f0eu, + 0x438a0cf1u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3dcc9000u, 0x3baf9000u, 0x3f3d7000u, + 0x3dcfb000u, 0x3d2cb000u, 0x3cf91000u, 0x3b96d000u, 0x3e89f000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3e889a76u, 0x38c0caafu, 0x3c7ff4e2u, 0x3df5d9b8u, 0x3db95aa9u, 0x39541e26u, 0x38953868u, 0x3fa341f9u, + 0x3fcc3630u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3b7bf115u, 0x41f84d29u, 0x422a01f0u, 0x40546e19u, + 0x3c2b1652u, 0x3b896db9u, 0x3a092d40u, 0x3fcaa669u, 0x3e3dd259u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x35a27e45u, 0x463b871eu, 0x442591e9u, 0x419c8f74u, 0x4216f2a4u, 0x44be5f6bu, 0x43d4fb2eu, 0x45f18944u, + 0x41a3418bu, 0x00000000u, 0x00000000u, 0x00000000u, 0x3e000000u, 0x40aa5000u, 0x3b1fd000u, 0x3d3ef000u, + 0x40779000u, 0x3dbb7000u, 0x3cf8d000u, 0x3ea37000u, 0x3c881000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3519a651u, 0x4239fb45u, 0x40b8f0b0u, 0x404d4f0fu, 0x45085cb6u, 0x459970a1u, 0x42022a8fu, 0x40f4ebb9u, + 0x453b35b1u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3b7efffbu, 0x3c0d6d70u, 0x3a6d9466u, 0x4367575cu, + 0x40a9d9ddu, 0x3c036366u, 0x3c705055u, 0x3b851907u, 0x430adadeu, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3288e07eu, 0x44315c67u, 0x459a01c9u, 0x46081f4eu, 0x4522218du, 0x439c2b64u, 0x45eb293fu, 0x461cede7u, + 0x45b64b8fu, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x3c455000u, 0x3f5bd000u, 0x3e9f7000u, + 0x3ffb7000u, 0x3a8a5000u, 0x3ff65000u, 0x3a0ff000u, 0x3a625000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3d0d325eu, 0x41075b73u, 0x3a50bae3u, 0x3a270783u, 0x3d089b76u, 0x3e0b6fc8u, 0x4021df98u, 0x3a2ebc23u, + 0x3985b6d1u, 0x00000000u, 0x00000000u, 0x00000000u, 0x36ae3c71u, 0x3e95dd99u, 0x412a28f0u, 0x45d3bf6cu, + 0x41a88d8bu, 0x4588231au, 0x4137c02cu, 0x405859a1u, 0x466b0b21u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x32a94b9eu, 0x4397f898u, 0x45832c8bu, 0x46478261u, 0x468bbb6bu, 0x4405727eu, 0x45266e56u, 0x45cb8992u, + 0x46a5704cu, 0x00000000u, 0x00000000u, 0x00000000u, 0x3d000000u, 0x3d2cf000u, 0x41265000u, 0x3c705000u, + 0x3ec17000u, 0x3ea55000u, 0x3c357000u, 0x3e421000u, 0x41c17000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x364c553au, 0x43071ce3u, 0x44e3c8feu, 0x40ffe568u, 0x4117203cu, 0x3f6ab4aau, 0x43854fd6u, 0x42b9804eu, + 0x40a972efu, 0x00000000u, 0x00000000u, 0x00000000u, 0x36faa53du, 0x455f3b4du, 0x405ee998u, 0x40681ad0u, + 0x3b697d6au, 0x42fcfa96u, 0x4586a0dfu, 0x3ef72b7bu, 0x40b6f39fu, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3c3ba6e3u, 0x3a6e0ecdu, 0x3b09f523u, 0x3ce39934u, 0x3cf34249u, 0x3bb19afeu, 0x3babfa5fu, 0x3b71a38fu, + 0x38a96fe1u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3c4e1000u, 0x3ac01000u, 0x3e6ab000u, + 0x3c62f000u, 0x3d8a1000u, 0x3f5ad000u, 0x3e021000u, 0x3f46d000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3ca14080u, 0x3ef64322u, 0x3b719c07u, 0x4115c72du, 0x3a4541acu, 0x3a51f507u, 0x3f2cfb49u, 0x3be0bdd1u, + 0x3c8a11d4u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3366ab82u, 0x4663dba8u, 0x4511745fu, 0x44834df3u, + 0x443a4f3au, 0x4630dcf4u, 0x45d54ab1u, 0x44cbb83au, 0x445ac57eu, 0x00000000u, 0x00000000u, 0x00000000u, + 0x36995651u, 0x4376fc30u, 0x408986e9u, 0x4024dd8du, 0x42c6469fu, 0x44d06df8u, 0x45b81dfcu, 0x40d93f2au, + 0x40afed10u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3bd3b000u, 0x3a885000u, 0x3e65f000u, + 0x3a175000u, 0x3a537000u, 0x3a62d000u, 0x3b02b000u, 0x3e561000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x32d77930u, 0x4505e1c8u, 0x402a4d59u, 0x45ee22dau, 0x46832bf1u, 0x43a5bcc7u, 0x443d33b7u, 0x44ed549eu, + 0x4654bdabu, 0x00000000u, 0x00000000u, 0x00000000u, 0x3bf342fau, 0x42011847u, 0x40b555ecu, 0x423d5c23u, + 0x3be5055bu, 0x3bfa1176u, 0x40901553u, 0x401deba7u, 0x40f2e623u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3c182e02u, 0x41924cfbu, 0x3f4bb4deu, 0x3953f5bdu, 0x3baed88du, 0x3e03b51au, 0x3d6ac66au, 0x42b1a1d0u, + 0x3a22559au, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3c84d000u, 0x39f2f000u, 0x3afe5000u, + 0x3c9b1000u, 0x3b54b000u, 0x3ac79000u, 0x3c139000u, 0x3ab09000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x39064412u, 0x3cf5f5afu, 0x3e2fa6c5u, 0x3f888ddfu, 0x4297e0b3u, 0x42be9b5au, 0x3e9274a6u, 0x3eba04d9u, + 0x414ed334u, 0x00000000u, 0x00000000u, 0x00000000u, 0x32ca37a1u, 0x44322f3fu, 0x40caf233u, 0x46101f48u, + 0x460c8fc9u, 0x448fd863u, 0x44d15be3u, 0x41bf95ddu, 0x46186e1au, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3094426fu, 0x428d3d3du, 0x41e27d3cu, 0x42328e35u, 0x455ebe9du, 0x4690d3bfu, 0x42b59041u, 0x4182ccfeu, + 0x42aafbb8u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3d4d9000u, 0x3ab87000u, 0x3e9dd000u, + 0x3c92f000u, 0x3a027000u, 0x3a07d000u, 0x3bf3f000u, 0x3a429000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3d486b0du, 0x419afef5u, 0x3a23fa27u, 0x3a57b580u, 0x3958a501u, 0x3d788538u, 0x3c5b760du, 0x38fe3044u, + 0x3a5cd17cu, 0x00000000u, 0x00000000u, 0x00000000u, 0x38824b1du, 0x3f34a372u, 0x3e968d71u, 0x3b5ac47fu, + 0x4191f291u, 0x441e988cu, 0x3f72c66fu, 0x3f1a5bfbu, 0x45ac9d08u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3f7cee9cu, 0x38546bb1u, 0x38c20ef7u, 0x3b050822u, 0x3bac60d8u, 0x3842f621u, 0x385cc54fu, 0x3a819effu, + 0x38dcb51eu, 0x00000000u, 0x00000000u, 0x00000000u, 0x3d800000u, 0x3ecb9000u, 0x3d6cf000u, 0x400bd000u, + 0x3fc7d000u, 0x3ed93000u, 0x3cb23000u, 0x3e2d1000u, 0x4059f000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x3b605644u, 0x3da0afd5u, 0x3dab3e0du, 0x3b272d55u, 0x3bf7e9abu, 0x3d5ab024u, 0x430a419du, 0x3ee0e041u, + 0x3c23e574u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3b99ac50u, 0x3f5b2aa2u, 0x3bc1087cu, 0x3af40cefu, + 0x406655d4u, 0x42b1c7adu, 0x3f14a08fu, 0x3c135363u, 0x3c4b573au, 0x00000000u, 0x00000000u, 0x00000000u, + 0x350a75b3u, 0x420bd2f3u, 0x41d9b59fu, 0x4359163eu, 0x454601cau, 0x43fbcc7du, 0x427c9b92u, 0x41d9b59fu, + 0x3ece82bfu, 0x00000000u, 0x00000000u, 0x00000000u, 0x3b800000u, 0x4074b000u, 0x3f715000u, 0x3ed27000u, + 0x41613000u, 0x4294d000u, 0x400bb000u, 0x40013000u, 0x3eda1000u, 0x00000000u, 0x00000000u, 0x00000000u, +}; +constexpr uint16_t kExpected[kLanes * 8] = { + 0x0c39, 0x0b8b, 0x039e, 0x0342, 0x0000, 0x10e0, 0x13c1, 0x0355, + 0x2832, 0x0167, 0x024d, 0x01a2, 0x0eb4, 0x1687, 0x03c7, 0x038d, + 0x0145, 0x022d, 0x1392, 0x29ff, 0x2138, 0x02c1, 0x013e, 0x00d1, + 0x17f6, 0x0fc6, 0x1776, 0x1cac, 0x2d9e, 0x1566, 0x37a8, 0x28e2, + 0x25d7, 0x3453, 0x3849, 0x02d7, 0x03b6, 0x251e, 0x2bd9, 0x3786, + 0x08cc, 0x1921, 0x0038, 0x002e, 0x005c, 0x232e, 0x17d2, 0x0242, + 0x0675, 0x02d4, 0x01e4, 0x0032, 0x058f, 0x04ad, 0x02d4, 0x035c, + 0x1b2e, 0x378e, 0x35f4, 0x1834, 0x14c2, 0x2660, 0x2616, 0x262e, + 0x0250, 0x0034, 0x0d38, 0x3a7c, 0x0217, 0x0118, 0x2ad2, 0x08e2, + 0x0001, 0x0510, 0x0b5a, 0x02f0, 0x0383, 0x12e3, 0x17b6, 0x0473, + 0x36f7, 0x33b3, 0x0161, 0x014d, 0x0001, 0x36f7, 0x33ed, 0x03ce, + 0x0fb2, 0x360c, 0x2b64, 0x39f6, 0x0e50, 0x1c1c, 0x147a, 0x1700, + 0x0214, 0x01df, 0x01da, 0x0691, 0x0c77, 0x0147, 0x00d4, 0x03b0, + 0x0336, 0x30c3, 0x2988, 0x1ae5, 0x016f, 0x02fa, 0x3386, 0x30bd, + 0x0001, 0x0007, 0x000f, 0x03e4, 0x022a, 0x0001, 0x014b, 0x0001, + 0x3ac6, 0x14fc, 0x0c6c, 0x2cca, 0x134c, 0x1868, 0x32d2, 0x175a, + 0x1d2b, 0x01ee, 0x0229, 0x3ae8, 0x23b6, 0x0463, 0x027b, 0x0092, + 0x024f, 0x0154, 0x2e82, 0x0414, 0x1238, 0x0282, 0x0154, 0x0657, + 0x0083, 0x0e54, 0x2283, 0x0ad7, 0x032a, 0x010f, 0x289d, 0x0e1f, + 0x1054, 0x2954, 0x0d56, 0x302e, 0x24b8, 0x2700, 0x10fe, 0x0ce2, + 0x0c01, 0x0890, 0x0268, 0x0241, 0x386b, 0x19d9, 0x1426, 0x02ce, + 0x2c04, 0x021e, 0x02b3, 0x00b4, 0x29ca, 0x106f, 0x0060, 0x0170, + 0x006f, 0x031f, 0x2f2f, 0x0ac8, 0x04e3, 0x02c5, 0x016d, 0x0f58, + 0x333e, 0x2cf6, 0x110a, 0x17ae, 0x3148, 0x2b3e, 0x1852, 0x0fc2, + 0x1c55, 0x1d5f, 0x3a37, 0x0215, 0x0342, 0x01ee, 0x2ef9, 0x1072, + 0x1440, 0x2ae9, 0x01a5, 0x012f, 0x004a, 0x35b1, 0x2804, 0x0108, + 0x0cea, 0x0093, 0x0016, 0x1a75, 0x0621, 0x138e, 0x03e6, 0x03ce, + 0x2e64, 0x1d7c, 0x39ec, 0x2e7e, 0x2966, 0x27c8, 0x1cb6, 0x3450, + 0x019b, 0x1c45, 0x281a, 0x262e, 0x038a, 0x013e, 0x3572, 0x36d0, + 0x2fa3, 0x313b, 0x2288, 0x02a2, 0x010e, 0x0022, 0x1e3c, 0x11d6, + 0x2371, 0x1291, 0x018e, 0x02fe, 0x178d, 0x103a, 0x20ca, 0x019f, + 0x3952, 0x0cfe, 0x1df8, 0x37bc, 0x21dc, 0x1bc6, 0x291c, 0x1840, + 0x01bf, 0x0037, 0x001f, 0x151e, 0x19c1, 0x0139, 0x0049, 0x1706, + 0x0233, 0x003b, 0x3b34, 0x2549, 0x020c, 0x03bd, 0x0109, 0x3853, + 0x00be, 0x0526, 0x088c, 0x02b6, 0x0053, 0x07dc, 0x093f, 0x0617, + 0x1e2a, 0x36de, 0x30fc, 0x3bdc, 0x1052, 0x3bb2, 0x0c80, 0x0f12, + 0x34aa, 0x01cd, 0x0170, 0x14b6, 0x1ccf, 0x2d94, 0x0182, 0x0093, + 0x001a, 0x039e, 0x2881, 0x072c, 0x25ca, 0x03e9, 0x0126, 0x2d00, + 0x0064, 0x056c, 0x0c20, 0x0dc6, 0x00b1, 0x0370, 0x0835, 0x0ed7, + 0x1568, 0x3532, 0x0f82, 0x220c, 0x212a, 0x0dac, 0x1e10, 0x3a0c, + 0x0ebe, 0x1dae, 0x0199, 0x01e2, 0x002f, 0x12a7, 0x0ca0, 0x010f, + 0x26d4, 0x01b5, 0x01c6, 0x0000, 0x13be, 0x281e, 0x003d, 0x02cc, + 0x00af, 0x0194, 0x0d37, 0x0d93, 0x0411, 0x03f1, 0x02c4, 0x0010, + 0x2270, 0x1600, 0x3356, 0x2318, 0x2c50, 0x3ad6, 0x3010, 0x3a36, + 0x20d9, 0x04c2, 0x31e5, 0x00f9, 0x0108, 0x22cf, 0x086d, 0x0d6f, + 0x126b, 0x0818, 0x03b3, 0x029f, 0x10fb, 0x0e02, 0x05bc, 0x0315, + 0x149f, 0x014a, 0x00c5, 0x0f6c, 0x1fce, 0x26e4, 0x0209, 0x01a5, + 0x1e9e, 0x1442, 0x3330, 0x10ba, 0x129c, 0x1316, 0x1816, 0x32b0, + 0x0385, 0x0001, 0x0a44, 0x0ee6, 0x008c, 0x013e, 0x031f, 0x0d99, + 0x33ab, 0x2963, 0x359f, 0x0367, 0x03b6, 0x2847, 0x24b1, 0x2b36, + 0x3170, 0x1f91, 0x0020, 0x033f, 0x14e5, 0x105d, 0x3a99, 0x0061, + 0x2426, 0x0f98, 0x17f2, 0x24d8, 0x1aa6, 0x163c, 0x209c, 0x1584, + 0x0041, 0x0170, 0x087a, 0x20fb, 0x223f, 0x0267, 0x030c, 0x16c8, + 0x0119, 0x0003, 0x0b1e, 0x0af0, 0x01c7, 0x0295, 0x0009, 0x0b87, + 0x0001, 0x0001, 0x0001, 0x0041, 0x014f, 0x0002, 0x0000, 0x0002, + 0x2a6c, 0x15c4, 0x34ee, 0x2498, 0x1014, 0x103e, 0x1fa0, 0x1214, + 0x3b95, 0x0202, 0x02a3, 0x00aa, 0x1a15, 0x115e, 0x0064, 0x02b3, + 0x02e0, 0x0132, 0x0003, 0x14a5, 0x290b, 0x03dd, 0x0274, 0x357e, + 0x0347, 0x05fe, 0x181c, 0x1d52, 0x0303, 0x0368, 0x1401, 0x06d1, + 0x265c, 0x1b68, 0x305e, 0x2e3e, 0x26ca, 0x1592, 0x2168, 0x32d0, + 0x0c67, 0x0cb0, 0x0093, 0x01b2, 0x09fd, 0x3793, 0x1628, 0x023f, + 0x1c1c, 0x01d0, 0x0092, 0x2452, 0x36ac, 0x1993, 0x02c4, 0x03d0, + 0x012f, 0x00eb, 0x0757, 0x16b2, 0x0c41, 0x0223, 0x00eb, 0x0003, + 0x23a6, 0x1b8a, 0x1694, 0x2b0a, 0x34a6, 0x205e, 0x200a, 0x16d0, +}; + +} // namespace k_mix_probe diff --git a/clean/k/runtime/runtime_k.cpp b/clean/k/runtime/runtime_k.cpp index f01d01c4..1fbf40fa 100644 --- a/clean/k/runtime/runtime_k.cpp +++ b/clean/k/runtime/runtime_k.cpp @@ -3,6 +3,7 @@ // variant "fused"), k_launch.py (stage_fused_launch) and k_io.py (frame_in, out_uniforms, input / output launches). #include "runtime_k.h" #include "k_input_uniforms.h" +#include "k_mix_probe.h" #include #include @@ -542,7 +543,47 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e std::vector tab((size_t)8 * kFilterEntries * kFilterValues); for (int l = 0; l < 8; ++l) k_filter_table(l, 1, tab.data() + (size_t)l * kFilterEntries * kFilterValues); if (!dev_->upload(tables_, tab.data(), tab.size() * 2)) return dfail(); - return dev_->sync() || dfail(); + if (!dev_->sync()) return dfail(); + return c.backend != 3 || mix_probe(); +} + +// The softmax stages' weight rounding A = f16(w inv) (clean/k/dx12/hlsl/k_smx.hlsli) relies on the driver folding the +// exact sum p + e into one v_fma_mix*_f16 rounding. The native D3D12 device checks that fold on this adapter once, at +// create: k_mix_probe runs the same helper on k_mix_probe.h's records (FP16-midpoint products with residuals of both +// signs, exact ties, FP16 subnormals) and must return the exact results; otherwise every softmax stage takes its +// round-to-odd port (suffix __rodd, the same bits by another sequence). The result goes into the kernels line +// (smx=mix | smx=rodd), and taking the round-to-odd ports adds one line to the notes of create (wg_rt_notes; the +// executor logs it), so the switch is never silent. WG_TEST_K_MIX_PROBE_FAIL=1 (test hook) treats the probe as failed. +bool KRuntime::mix_probe() { + DevPtr in = 0, out = 0; + const size_t out_bytes = (size_t)k_mix_probe::kLanes * 16; + if (!alloc(&in, sizeof k_mix_probe::kRecords) || !alloc(&out, out_bytes)) return false; + if (!dev_->upload(in, k_mix_probe::kRecords, sizeof k_mix_probe::kRecords) || !dev_->memset(out, 0xA5, out_bytes)) + return dfail(); + if (!launch("k_mix_probe", (unsigned)(k_mix_probe::kLanes / 32), 32, &in, &out)) return false; + std::vector got((size_t)k_mix_probe::kLanes * 8); + if (!dev_->download(got.data(), out, out_bytes)) return dfail(); + int bad = 0; + for (size_t i = 0; i < got.size(); ++i) bad += got[i] != k_mix_probe::kExpected[i]; + const char* hook = std::getenv("WG_TEST_K_MIX_PROBE_FAIL"); + const bool forced = hook && std::strcmp(hook, "1") == 0; + smx_rodd_ = bad != 0 || forced; + // one token (the kernels line is split at spaces into key=value pairs); the details are in the note + smx_note_ = smx_rodd_ ? std::string(" smx=rodd(") + (forced ? std::string("hook") : std::to_string(bad) + "/" + + std::to_string(got.size())) + ")" + : std::string(" smx=mix"); + notes_ = smx_rodd_ ? std::string("model K: softmax weight rounding takes the round-to-odd ports (") + + (forced ? "probe failed by test hook" : "mix-form probe: " + std::to_string(bad) + " of " + + std::to_string(got.size()) + " differ") + + "); output unchanged, both forms are exact" + : std::string(); + return true; +} + +std::string KRuntime::stage_kernel(const std::string& k, const char* stage) const { + // e0, e1 and d0 mix with fixed weights (no softmax): no __rodd form + const bool softmax = std::strcmp(stage, "e0") != 0 && std::strcmp(stage, "e1") != 0 && std::strcmp(stage, "d0") != 0; + return smx_rodd_ && softmax ? k + "__rodd" : k; } bool KRuntime::launch(const char* kernel, unsigned grid, unsigned block, void* a0, void* a1, void* a2, void* a3, @@ -657,7 +698,7 @@ bool KRuntime::network(int f, const std::function& after) { const char* k0 = compact_head_ ? (f16_ ? "k_stage2h_d0_hp" : "k_stage2_d0_hp") : (f16_ ? "k_stage2h_d0_hidden" : "k_stage2_d0_hidden"); if (!launch(k0, grid, kThreads, &p, &hidden)) return false; - } else if (!launch(k.c_str(), grid, (unsigned)(kThreads * (ng ? ng : 1)), &p)) return false; + } else if (!launch(stage_kernel(k, st.name).c_str(), grid, (unsigned)(kThreads * (ng ? ng : 1)), &p)) return false; if (after && !after(st.name)) return false; if (i < 10) { info_.offsets[i][0] = ox; // WgRtFrameInfo has ten rows: e0 ... d1 (d0's offset is (4, 4)) @@ -1057,8 +1098,12 @@ std::vector KRuntime::d3d12_kernels() const { else if (ng) k = "k_stagew" + std::to_string(ng) + "_" + n; else k = (n == "e0" || n == "e1" || n == "d1" ? "k_stage2_" : "k_stage_") + n; if (std::find(v.begin(), v.end(), k) == v.end()) v.push_back(k); + // the round-to-odd form a failed mix-form probe switches to (KRuntime::mix_probe) + if (!st.head && n != "e0" && n != "e1" && std::find(v.begin(), v.end(), k + "__rodd") == v.end()) + v.push_back(k + "__rodd"); } } + v.push_back("k_mix_probe"); v.push_back((compact_head_ ? "k_out_pixel_c" : "k_out_pixel") + d); if (!warm_at_.empty()) v.push_back("k_warm"); // k_warm (option k_warm, backend 3) v.push_back("timing_barrier"); // the timing marks (timing_log=1) @@ -1124,7 +1169,7 @@ const char* KRuntime::kernels() { " legacy_color_launches=" + std::to_string(legacy_color_launches_) + " legacy_base_launches=" + std::to_string(legacy_base_launches_) + (fast_ ? " k_fast=1" : "") + (expo_v051_ ? " k_expo_rule=v051" : "") + - " hidden=fused_d0" + std::string(" head=") + (compact_head_ ? "compact" : "full") + + smx_note_ + " hidden=fused_d0" + std::string(" head=") + (compact_head_ ? "compact" : "full") + " base=" + std::to_string(Bw_) + "x" + std::to_string(Bh_) + " hdr=" + (hdr_ ? "1" : "0") + " mv=render" + " filter_level=" + std::to_string(level_last_) + diff --git a/clean/k/runtime/runtime_k.h b/clean/k/runtime/runtime_k.h index a3dc3fc5..93489bf6 100644 --- a/clean/k/runtime/runtime_k.h +++ b/clean/k/runtime/runtime_k.h @@ -81,6 +81,8 @@ class KRuntime { const WgRtFrameInfo& info() const { return info_; } const std::map>& blobs() const { return blobs_; } const std::string& err() const { return err_; } + // notes of create for the caller's log (wg_rt_notes), one per line, or empty: the round-to-odd softmax ports + const std::string& notes() const { return notes_; } const std::string& model_path() const { return model_path_; } // What the timing marks 0..4 bracket (FrameIo.timing_base): "exposure", "input", "network", "output". static const char* const kTimingLabels[4]; @@ -102,6 +104,11 @@ class KRuntime { // the tiled input stage's guard (frame and direct_preflight): sizes, the 0.68 ratio, finite jitter in [-0.5, 0.5] bool tiled_guard(int rw, int rh, float jx, float jy) const; bool options_ok_direct(); + // the native D3D12 device's softmax rounding probe (k_mix_probe at create): sets smx_rodd_ when the driver's fold of + // the mix form (clean/k/dx12/hlsl/k_smx.hlsli) does not give the exact single rounding on this adapter + bool mix_probe(); + // the stage kernel network() launches: the __rodd form of a softmax stage after a failed probe + std::string stage_kernel(const std::string& k, const char* stage) const; int fill_ = -1; // option k_fill (test hook): the byte every frame-written buffer and the histories start with WgRtConfig cfg_ = {}; @@ -148,6 +155,10 @@ class KRuntime { // D3D12 latency options (create: k_color_at, k_warm, k_warm_line, k_warm_groups): the stage after which k_out_color // runs unordered (empty: before the input stage, HIP's order), the stage after which k_warm runs (empty: off) std::string color_at_, warm_at_, warm_to_; + // the softmax stages run their round-to-odd (__rodd) ports: the mix-form probe failed (native D3D12 device only) + bool smx_rodd_ = false; + std::string smx_note_; + std::string notes_; // notes of create for the caller's log (notes()), one per line, or empty unsigned warm_line_ = 128, warm_groups_ = 64; DevPtr warm_dummy_ = 0; WgRtFrameInfo info_ = {}; diff --git a/clean/k/tests/test_k_mix_probe_table.py b/clean/k/tests/test_k_mix_probe_table.py new file mode 100644 index 00000000..42b4818f --- /dev/null +++ b/clean/k/tests/test_k_mix_probe_table.py @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: MIT +"""Model K's softmax rounding probe cases (clean/k/dx12/k_mix_cases.py) and the runtime's table (clean/k/runtime/ +k_mix_probe.h): CPU only. +- the committed header is exactly gen_k_mix_probe.py's output; +- the exact oracle (Fraction arithmetic) agrees with an independent conversion: numpy's binary64 -> binary16 rounding of + the products, which are exact in binary64 (two 24-bit significands); +- the cases cover every kind (FP16-midpoint products with residuals of both signs, normal and subnormal FP16, exact + ties, ordinary products) and a second rounding through FP32 differs from the exact result in many of them, so the + probe catches a missing fold and the residual-dropped control. +Run: python -m unittest clean.k.tests.test_k_mix_probe_table +""" +import os +import unittest + +import numpy as np + +from clean.k.dx12 import gen_k_mix_probe, k_mix_cases + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + + +class TestKMixProbeTable(unittest.TestCase): + def test_header_is_generated(self): + with open(os.path.join(ROOT, "clean", "k", "runtime", "k_mix_probe.h"), newline="") as f: + self.assertEqual(f.read(), gen_k_mix_probe.header(), "regenerate: python clean/k/dx12/gen_k_mix_probe.py") + + def test_oracle_matches_numpy(self): + for seed in (7, 2000, 2001): + rec, want, _ = k_mix_cases.cases(seed, 64) + prod = rec[:, 1:9].astype(np.float64) * rec[:, :1].astype(np.float64) + self.assertTrue(np.array_equal(prod.astype(np.float16).view(np.uint16), want), seed) + rng = np.random.default_rng(5) + x = (2.0 ** rng.uniform(-26, 0, 4000)).astype(np.float32).astype(np.float64) + got = np.array([k_mix_cases.rn16_bits(float(v)) for v in x], np.uint16) + self.assertTrue(np.array_equal(x.astype(np.float16).view(np.uint16), got)) + + def test_rn16_corners(self): + from fractions import Fraction as F + self.assertEqual(k_mix_cases.rn16_bits(1), 0x3C00) + self.assertEqual(k_mix_cases.rn16_bits(F(1, 2 ** 24)), 0x0001) # smallest subnormal + self.assertEqual(k_mix_cases.rn16_bits(F(1, 2 ** 25)), 0x0000 + 0) # tie between 0 and 2^-24: even (0) + self.assertEqual(k_mix_cases.rn16_bits(F(3, 2 ** 25)), 0x0002) # tie 1.5 ulp: even (2) + self.assertEqual(k_mix_cases.rn16_bits(F(2047, 2 ** 25)), 0x0400) # rounds up into the normal range + self.assertEqual(k_mix_cases.rn16_bits(F(2049, 2 ** 11)), 0x3C00) # 1 + 2^-11: tie, even (1.0) + self.assertEqual(k_mix_cases.rn16_bits(F(2051, 2 ** 11)), 0x3C02) # 1 + 3 2^-11: tie, even + self.assertEqual(k_mix_cases.rn16_bits(65504), 0x7BFF) + self.assertEqual(k_mix_cases.rn16_bits(65520), 0x7C00) + + def test_cases_separate_single_and_double_rounding(self): + rec, want, kinds = k_mix_cases.cases(gen_k_mix_probe.SEED, gen_k_mix_probe.LANES) + kinds = np.array(kinds) + dbl = k_mix_cases.double_rounding_bits(rec) + for k in k_mix_cases.KINDS: + self.assertGreater(int((kinds == k).sum()), 40, k) + for k in ("mid+", "mid-", "sub+", "sub-"): + self.assertGreater(int(((dbl != want) & (kinds == k)).sum()), 10, k) + self.assertEqual(int(((dbl != want) & np.isin(kinds, ("tie", "plain"))).sum()), 0) + prod = rec[:, 1:9].astype(np.float64) * rec[:, :1].astype(np.float64) + self.assertTrue(((prod >= 2.0 ** -26) & (prod <= 1.0)).all()) + self.assertTrue((rec[:, 9:] == 0).all()) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/k/tests/test_k_runtime_d3d12.py b/clean/k/tests/test_k_runtime_d3d12.py index 1ca88c09..08cdd2a3 100644 --- a/clean/k/tests/test_k_runtime_d3d12.py +++ b/clean/k/tests/test_k_runtime_d3d12.py @@ -23,13 +23,20 @@ value, with the game's exposure texel, and with auto-exposure, and LDR; output rows RGBA16F and dense RGB; render size equal to the output (native AA, DLAA 1.0: the legacy input kernels), a jitter outside [-0.5, 0.5], and input_tiled=0. Refusals (one line, create or frame): an approximation option on the native device, an input-copy frame (raw_inputs). +Softmax rounding switch (native device): with the create-time probe failed (test hook WG_TEST_K_MIX_PROBE_FAIL=1) the +runtime reports smx=rodd and exactly one note of create (wg_rt_notes, logged by the executor); with nothing set, +smx=mix and no note. Each leg runs in its own process. GPU only: WG_K_GPU_BUILD=, ROCM_PATH; run each test under the machine's GPU lock. Run: python -m unittest clean.k.tests.test_k_runtime_d3d12 """ +import ctypes +import json import os +import subprocess +import sys import unittest import numpy as np @@ -268,6 +275,24 @@ def test_native_refuses_approximation_options(self): self.assertNotIn("\n", msg.strip()) self.assertIn("HIP", msg) + def test_native_softmax_switch_is_logged(self): + """The round-to-odd softmax ports are exact too, but taking them is never silent: one note of create.""" + legs = {} + for hook in ("", "1"): + env = {k: v for k, v in os.environ.items() if k != "WG_TEST_K_MIX_PROBE_FAIL"} + if hook: + env["WG_TEST_K_MIX_PROBE_FAIL"] = hook + r = subprocess.run([sys.executable, "-m", "clean.k.tests.test_k_runtime_d3d12", "--softmax-leg", self.mdir, + self.map], env=env, capture_output=True, text=True, timeout=300) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + legs[hook] = json.loads(r.stdout.strip().splitlines()[-1]) + self.assertEqual(legs[""]["smx"], "mix") + self.assertEqual(legs[""]["notes"], []) + self.assertTrue(legs["1"]["smx"].startswith("rodd"), legs["1"]) + self.assertEqual(len(legs["1"]["notes"]), 1, legs["1"]) + self.assertIn("round-to-odd", legs["1"]["notes"][0]) + self.assertIn("test hook", legs["1"]["notes"][0]) + def test_input_copy_frames_refused_on_d3d12(self): """raw_inputs (the interop's HIP input copies) has no checked D3D12 port: refused, not run unchecked.""" for backend in (2, 3): @@ -288,5 +313,19 @@ def test_input_copy_frames_refused_on_d3d12(self): del rt +def softmax_leg(mdir, map_path): + """Child of test_native_softmax_switch_is_logged: create on the native device, print the smx field and the notes.""" + rt = KRuntimeExt(gpu_build(), mdir, map_path, OUT, MAX_RENDER, True, False, False, "", backend=3) + rt.lib.wg_rt_notes.argtypes = [ctypes.c_void_p] + rt.lib.wg_rt_notes.restype = ctypes.c_char_p + kern = rt.lib.wg_rt_kernels(rt.rt).decode().split() + smx = [kv.split("=", 1)[1] for kv in kern if kv.startswith("smx=")] + notes = [n for n in rt.lib.wg_rt_notes(rt.rt).decode().split("\n") if n] + print(json.dumps({"smx": smx[0] if smx else "", "notes": notes})) + + if __name__ == "__main__": - unittest.main() + if len(sys.argv) == 4 and sys.argv[1] == "--softmax-leg": + softmax_leg(sys.argv[2], sys.argv[3]) + else: + unittest.main() diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index de1b9bb2..8131f852 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -1193,6 +1193,7 @@ struct Runtime { if (!k->create(c, dev.get(), [this](DevPtr* p, size_t n) { return alloc(p, n); }, external_io, native_direct_io)) return fail(k->err()); blobs = k->blobs(); + notes_text = k->notes(); // one line when the softmax stages take their round-to-odd ports (no silent switch) model_source = "model.wgcm"; return true; } diff --git a/clean/runtime/wg_runtime.h b/clean/runtime/wg_runtime.h index 7c424d02..46a2a666 100644 --- a/clean/runtime/wg_runtime.h +++ b/clean/runtime/wg_runtime.h @@ -159,9 +159,11 @@ WG_RT_API const char* wg_rt_model_source(void* rt); /* The kernels in use, as "swin= enc0= dec0= exposure= * post= filter= post_last=". */ WG_RT_API const char* wg_rt_kernels(void* rt); -/* Notes of create for the caller's log, one per line, or an empty string. At present one: the output filter is - * "analytic" because the model file has no output filter weights (spec 06 draft 4), or because this build cannot - * run them. The filter in use is also in wg_rt_kernels() ("output_filter=original|analytic|quality5|quality9 +/* Notes of create for the caller's log, one per line, or an empty string. At present two. Model M / L: the output + * filter is "analytic" because the model file has no output filter weights (spec 06 draft 4), or because this build + * cannot run them. Model K on the native D3D12 device: its softmax stages take their round-to-odd ports because the + * create-time rounding probe failed (output unchanged; also "smx=rodd(...)" in wg_rt_kernels()). The filter in use + * is also in wg_rt_kernels() ("output_filter=original|analytic|quality5|quality9 * output_taps=model|none"); runtime option output_filter=original|analytic|quality5|quality9 selects it (default: * original when the model file has the weights, else analytic; original without them fails create). */ WG_RT_API const char* wg_rt_notes(void* rt); From a81a6fdf8af531b6de83a9a865b1bb686bb40610 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Mon, 16 Nov 2026 00:05:50 -0500 Subject: [PATCH 2/5] K softmax: create-time self-check of both rounding forms on every selectable stage shader The player's driver compiles each stage shader on its own, so neither the mix form (exact when the driver folds p + e into one v_fma_mix*_f16) nor the round-to-odd ports are taken on trust. At create the native D3D12 device now checks the shaders that will run: - k_mix_probe and k_mix_probe__rodd against the exact results of k_mix_probe.h; - every softmax stage kernel the runtime can select, in both forms (k_stage_e2, k_stagew2_e3, k_stagew4_e4, k_stagew4_m, k_stagew4_d4, k_stagew2_d3, k_stage_d2, k_stage2_d1 at a 512 x 384 base map, e4 / d4 on 8 data sets; k_stagew2_e4, k_stagew2_d4 and k_stagew4_m at 1536 x 1280), on the runtime's own synthetic inputs and weights; the outputs' hashes must equal the HIP reference's (clean/k/runtime/k_smx_selfcheck.h, written by clean/k/dx12/gen_k_smx_selfcheck.py). The mix form is used when its probe and all its stages match, else the round-to-odd ports when theirs do (one note of create), else create fails in one line naming the first stage that differs. Launching every stage kernel of both forms at create also loads, verifies and prepares their shaders, so a missing or corrupt one fails create. network() and the self-check share the block parameters (block_params) and the kernel-name rule (stage_kernel_base). The test hook is now the runtime option k_smx_test (mix_fail, both_fail, rodd_unchecked, report), compiled only into developer builds (WG_DEV_DIAGNOSTICS); a release build refuses it. The environment variable hook is gone. K_SMX_RODD_MUT=1 is the round-to-odd ports' negative control (never shipped). Tests (test_k_runtime_d3d12.py, each create leg in its own process): the switch and its note; refusals (both forms failed, a corrupt __rodd shader); the forced-fallback frames against HIP at 1440p Quality (NG = 4) and 4K Performance (NG = 2) with every softmax stage on its __rodd port. Native launches are counted by their trace tag "d3d12". Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/k/dx12/README.md | 25 +- clean/k/dx12/gen_k_smx_selfcheck.py | 85 ++++++ clean/k/dx12/hlsl/k_smx.hlsli | 9 + clean/k/runtime/k_smx_selfcheck.h | 39 +++ clean/k/runtime/runtime_k.cpp | 379 ++++++++++++++++++++------ clean/k/runtime/runtime_k.h | 11 +- clean/k/tests/test_k_runtime_d3d12.py | 114 ++++++-- clean/runtime/wg_runtime.h | 5 +- 8 files changed, 550 insertions(+), 117 deletions(-) create mode 100644 clean/k/dx12/gen_k_smx_selfcheck.py create mode 100644 clean/k/runtime/k_smx_selfcheck.h diff --git a/clean/k/dx12/README.md b/clean/k/dx12/README.md index 22c05172..5493707a 100644 --- a/clean/k/dx12/README.md +++ b/clean/k/dx12/README.md @@ -67,13 +67,24 @@ form the ports had before; the same bits). `kernels/k_mix_probe.hlsl` runs the s `k_mix_cases.py`: products whose FP32 rounding is an FP16 midpoint with a residual of either sign (normal and subnormal FP16), exact midpoints (ties to even) and ordinary products; every result must equal the exact single rounding (Fraction arithmetic), for `k_mix_probe` and `k_mix_probe__rodd`, and the records must tell a second rounding through -FP32 apart (the residual-dropped control `K_SMX_MUT=1` fails this family). Two more guards keep the fold: the build's -ISA check (`clean/dx12/isa_checks.json`: one `v_fma_mix*_f16` per weight in every mix-form port, none in the `__rodd` -ports) and the native D3D12 runtime, which runs `k_mix_probe` on the records of `clean/k/runtime/k_mix_probe.h` at -create and launches the `__rodd` stage ports when the result is not exact (kernels line: `smx=mix` or `smx=rodd(...)`; -test hook `WG_TEST_K_MIX_PROBE_FAIL=1`). The `__rodd` ports give the same bits, but the switch is never silent: it adds -one note of create (`wg_rt_notes`, which the executor logs; `test_k_runtime_d3d12.py` -`test_native_softmax_switch_is_logged`). +FP32 apart (the residual-dropped control `K_SMX_MUT=1` fails this family). The build's ISA check +(`clean/dx12/isa_checks.json`: one `v_fma_mix*_f16` per weight in every mix-form port, none in the `__rodd` ports) keeps +the fold in the shipped DXIL as RGA compiles it. + +The player's driver compiles every stage shader on its own, so neither form is taken on trust there: at create the native +D3D12 runtime checks the shaders that will run (`clean/k/runtime/runtime_k.cpp`, "create-time self-check"). It runs +`k_mix_probe` and `k_mix_probe__rodd` on the records of `clean/k/runtime/k_mix_probe.h`, and every softmax stage kernel it +can select, in both forms (`k_stage_e2`, `k_stagew2_e3`, `k_stagew4_e4`, `k_stagew4_m`, `k_stagew4_d4`, `k_stagew2_d3`, +`k_stage_d2`, `k_stage2_d1` at a 512 x 384 base map; `k_stagew2_e4`, `k_stagew2_d4`, `k_stagew4_m` at 1536 x 1280), on the runtime's own +synthetic inputs and weights; the outputs' hashes must equal the HIP reference's (`clean/k/runtime/k_smx_selfcheck.h`, +written by `gen_k_smx_selfcheck.py`). The mix form is used when its probe and all its stages match, else the `__rodd` +ports when theirs all do (kernels line `smx=rodd(...)` and one note of create, `wg_rt_notes`, which the executor logs), +else create fails and names the first stage that differs. Launching every stage kernel of both forms at create also loads, +verifies and prepares their shaders: a missing or corrupt one fails create. The test hook `k_smx_test` (`mix_fail`, +`both_fail`, `rodd_unchecked`, `report`) exists only in runtime libraries built with `build.py --dev-diagnostics`; a +release build refuses it (`test_k_runtime_d3d12.py`: `test_native_softmax_switch_is_logged`, +`test_native_softmax_refusals`, `test_native_fallback_frames_*`). `K_SMX_RODD_MUT=1` is the `__rodd` ports' negative +control (the inexact bit dropped), never shipped. NaN: IEEE 754 does not define the sign of a NaN that a stage generates or propagates, and where the input holds NaN the two devices can differ in it (the decoder stages). For the cases with NaN in the input the check compares the raw bytes diff --git a/clean/k/dx12/gen_k_smx_selfcheck.py b/clean/k/dx12/gen_k_smx_selfcheck.py new file mode 100644 index 00000000..56ca44ea --- /dev/null +++ b/clean/k/dx12/gen_k_smx_selfcheck.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: MIT +"""Writes clean/k/runtime/k_smx_selfcheck.h: the reference hashes of model K's softmax self-check (runtime_k.cpp +"create-time self-check"; KRuntime::smx_select). Each case runs one softmax stage kernel on the runtime's own synthetic +inputs and weights; the header holds the SHA-256 (first 8 bytes) of its outputs on the HIP reference. The native D3D12 +device compares both of its forms (mix and __rodd) with these values at create. WrongGPU's own code. + + python clean/k/dx12/gen_k_smx_selfcheck.py --build (GPU: a runtime library built with + clean/runner/build.py --dev-diagnostics, HIP code objects, ROCM_PATH; runs the HIP backend with the developer + hook k_smx_test=report and writes the header) + python clean/k/dx12/gen_k_smx_selfcheck.py --bootstrap (the case names with zero hashes: lets a first + developer build exist; a native D3D12 create refuses with it) +""" +import argparse +import ctypes +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) +OUT = os.path.join(ROOT, "clean", "k", "runtime", "k_smx_selfcheck.h") +# KRuntime::smx_case_name for runtime_k.cpp kSmxCases, in order (the runtime checks the names at create) +NAMES = ["k_stage_e2@512x384", "k_stagew2_e3@512x384", "k_stagew4_e4@512x384*8", "k_stagew4_m@512x384", + "k_stagew4_d4@512x384*8", "k_stagew2_d3@512x384", "k_stage_d2@512x384", "k_stage2_d1@512x384", + "k_stagew2_e4@1536x1280", "k_stagew2_d4@1536x1280", "k_stagew4_m@1536x1280"] +TAG = "model K self-check reference:" + + +def header(values): + out = ["// SPDX-License-Identifier: MIT", + "// Generated by clean/k/dx12/gen_k_smx_selfcheck.py; do not edit. The reference hashes (SHA-256 of the outputs,", + "// first 8 bytes) of model K's softmax self-check cases (runtime_k.cpp kSmxCases, KRuntime::smx_select) on the", + "// HIP reference." + (" BOOTSTRAP: zero hashes (a native D3D12 create refuses)." if not any(values) else ""), + "#pragma once", + "", + "#include ", + "", + "namespace k_smx_selfcheck {", + "", + "constexpr int kCount = %d;" % len(NAMES), + "constexpr const char* kNames[kCount] = {"] + out += [' "%s",' % n for n in NAMES] + out += ["};", "constexpr uint64_t kExpected[kCount] = {"] + out += [" 0x%016xull, // %s" % (v, n) for v, n in zip(values, NAMES)] + out += ["};", "", "} // namespace k_smx_selfcheck", ""] + return "\n".join(out) + + +def reference(build): + import tempfile + + from clean.k.reference.network import random_network_params + from clean.k.tests.test_k_runtime_gpu import KRuntimeExt, write_k_model + mdir = tempfile.mkdtemp(prefix="k_smx_ref_") + _, map_path = write_k_model(mdir, random_network_params(11)) + rt = KRuntimeExt(build, mdir, map_path, (200, 120), (130, 78), True, False, False, "k_smx_test=report", backend=0) + rt.lib.wg_rt_notes.argtypes = [ctypes.c_void_p] + rt.lib.wg_rt_notes.restype = ctypes.c_char_p + notes = rt.lib.wg_rt_notes(rt.rt).decode().split("\n") + line = [n for n in notes if n.startswith(TAG)] + if len(line) != 1: + raise SystemExit("no reference line in the notes (a --dev-diagnostics build?): %r" % notes) + got = dict(kv.split("=", 1) for kv in line[0][len(TAG):].split()) + if list(got) != NAMES: + raise SystemExit("case names differ from the runtime's: %r" % list(got)) + return [int(got[n], 16) for n in NAMES] + + +def main(): + ap = argparse.ArgumentParser() + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument("--build") + g.add_argument("--bootstrap", action="store_true") + ap.add_argument("--out", default=OUT) + a = ap.parse_args() + values = [0] * len(NAMES) if a.bootstrap else reference(a.build) + with open(a.out, "w", encoding="utf-8", newline="\n") as f: + f.write(header(values)) + print(a.out) + for n, v in zip(NAMES, values): + print(" %016x %s" % (v, n)) + + +if __name__ == "__main__": + main() diff --git a/clean/k/dx12/hlsl/k_smx.hlsli b/clean/k/dx12/hlsl/k_smx.hlsli index 1187b61e..ea20a79d 100644 --- a/clean/k/dx12/hlsl/k_smx.hlsli +++ b/clean/k/dx12/hlsl/k_smx.hlsli @@ -17,6 +17,8 @@ // FP16 is the single rounding of the exact product), then the same conversion. Truncation of w inv is p, or p - 1 // ulp when e < 0; the inexact bit e != 0 makes it odd (the same bits as wg_round_odd for every finite p > 0). // K_SMX_MUT 1: negative control of the checks (the residual dropped: a second rounding); never shipped. +// K_SMX_RODD_MUT 1: negative control of the __rodd ports (the inexact bit dropped: truncation, then the conversion's +// rounding, wrong where the product is an FP16 midpoint); never shipped. #pragma once #ifndef K_SMX @@ -25,10 +27,17 @@ #ifndef K_SMX_MUT #define K_SMX_MUT 0 #endif +#ifndef K_SMX_RODD_MUT +#define K_SMX_RODD_MUT 0 +#endif uint k_smx_round_odd_pos(float p, float e) { +#if K_SMX_RODD_MUT + return asuint(p) - (e < 0.0f ? 1u : 0u); +#else return (asuint(p) - (e < 0.0f ? 1u : 0u)) | (e != 0.0f ? 1u : 0u); +#endif } // A = f16(w[j] inv) for one fragment's eight weights, packed two per dword (element 2 i in the low half of dword i). diff --git a/clean/k/runtime/k_smx_selfcheck.h b/clean/k/runtime/k_smx_selfcheck.h new file mode 100644 index 00000000..e47c8a57 --- /dev/null +++ b/clean/k/runtime/k_smx_selfcheck.h @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: MIT +// Generated by clean/k/dx12/gen_k_smx_selfcheck.py; do not edit. The reference hashes (SHA-256 of the outputs, +// first 8 bytes) of model K's softmax self-check cases (runtime_k.cpp kSmxCases, KRuntime::smx_select) on the +// HIP reference. +#pragma once + +#include + +namespace k_smx_selfcheck { + +constexpr int kCount = 11; +constexpr const char* kNames[kCount] = { + "k_stage_e2@512x384", + "k_stagew2_e3@512x384", + "k_stagew4_e4@512x384*8", + "k_stagew4_m@512x384", + "k_stagew4_d4@512x384*8", + "k_stagew2_d3@512x384", + "k_stage_d2@512x384", + "k_stage2_d1@512x384", + "k_stagew2_e4@1536x1280", + "k_stagew2_d4@1536x1280", + "k_stagew4_m@1536x1280", +}; +constexpr uint64_t kExpected[kCount] = { + 0x276cf4e74f3e48adull, // k_stage_e2@512x384 + 0x87cb930d0c75bc14ull, // k_stagew2_e3@512x384 + 0xa1db88d469e36077ull, // k_stagew4_e4@512x384*8 + 0xd9a25f0ab4e5189bull, // k_stagew4_m@512x384 + 0x5a91b982d550a3f0ull, // k_stagew4_d4@512x384*8 + 0xcda38b7fb94b7fffull, // k_stagew2_d3@512x384 + 0x6cbdcdb82b666f9cull, // k_stage_d2@512x384 + 0xcf11492061147f2cull, // k_stage2_d1@512x384 + 0xb01a01dbe45082afull, // k_stagew2_e4@1536x1280 + 0x5458bd7f3a32989aull, // k_stagew2_d4@1536x1280 + 0x8ab307a6b6d53a65ull, // k_stagew4_m@1536x1280 +}; + +} // namespace k_smx_selfcheck diff --git a/clean/k/runtime/runtime_k.cpp b/clean/k/runtime/runtime_k.cpp index 1fbf40fa..ba68728d 100644 --- a/clean/k/runtime/runtime_k.cpp +++ b/clean/k/runtime/runtime_k.cpp @@ -4,14 +4,17 @@ #include "runtime_k.h" #include "k_input_uniforms.h" #include "k_mix_probe.h" +#include "k_smx_selfcheck.h" #include #include +#include #include #include #include #include "../../converter/src/json.h" +#include "../../converter/src/sha256.h" #include "../../runtime/embedded.h" #include "../../runtime/fpconv.h" #include "../../runtime/runtime_core.h" @@ -271,6 +274,69 @@ int wide_ng(const std::string& s, bool f16, int nwin) { return 0; } +// The block parameters of stage st (KBlkParams): its weight blob, its maps and the window grid of a level map of lw x lh +// with window offset (ox, oy); iw x ih is a decoder's input map. network() and the softmax self-check (smx_check). +KBlkParams block_params(const KStageInfo& st, DevPtr blob, DevPtr x, DevPtr y, DevPtr skip, DevPtr merged, DevPtr head, + int lw, int lh, int iw, int ih, int ox, int oy) { + const StageSec sec = stage_sections(st); + const BlockSec bs = block_sections(st); + const DevPtr blk = blob + sec.block; + KBlkParams p = {}; + p.x = x; + p.y = y; + p.g1 = blk + bs.g1; + p.g2 = blk + bs.g2; + p.wqkv = blk + bs.qkv; + p.mix = blk + bs.mix; + p.wo = blk + bs.wo; + p.bo = blk + bs.bo; + p.w1 = blk + bs.w1; + p.b1 = blk + bs.b1; + p.w2 = blk + bs.w2; + p.b2 = blk + bs.b2; + p.w = lw; + p.h = lh; + p.ox = ox; + p.oy = oy; + p.nwx = cdiv(lw + ox, 8); // layout.py window_grid + p.nwy = cdiv(lh + oy, 8); + if (st.embed) { + p.wa = blob + sec.embed; + p.ba = blob + sec.embed + linear_bias(32, 16); + } + if (st.kind == 'd') { + p.wa = blob + sec.expand; + p.ba = blob + sec.expand + linear_bias(4 * st.C, st.C_prev); + p.skip = skip; + p.cw = iw; + p.ch = ih; + } + if (st.kind == 'e') { + p.wz = blob + sec.merge; + p.bz = blob + sec.merge + linear_bias(st.C_next, 4 * st.C); + p.merged = merged; + } + if (st.head) { + p.wz = blob + sec.head; + p.bz = blob + sec.head + linear_bias(40, st.C); + p.head = head; + } + return p; +} + +// The stage kernel of a non-head stage as network() launches it (before a __rodd suffix): k_stagew_ for the +// wide stages, k_stage2_ (two windows per workgroup) for e0, e1, d1 with its block output map and d0, else +// k_stage_; *ng = NG (0: not wide), *w2 = two windows per workgroup. +std::string stage_kernel_base(const KStageInfo& st, bool f16, bool has_y, int nwin, int* ng, bool* w2) { + *ng = wide_ng(st.name, f16, nwin); + // e0, e1, d1, d0: two windows per workgroup (k_stage2_*, k_launch.py WINDOWS2_STAGES; k_net.py WINDOWS2_VARIANTS; + // d1's needs its block output map p.y, which the network always gives it) + *w2 = !*ng && (std::strcmp(st.name, "e0") == 0 || std::strcmp(st.name, "e1") == 0 || + (std::strcmp(st.name, "d1") == 0 && has_y) || std::strcmp(st.name, "d0") == 0); + const std::string h = f16 ? "h" : ""; + return (*ng ? "k_stagew" + std::to_string(*ng) : std::string(*w2 ? "k_stage2" : "k_stage")) + h + "_" + st.name; +} + // Runtime options of model K (runtime_options, then WG_RUNTIME_OPTIONS whose keys win): model_k_dir, model_k_map, // network (fused, the default; fused16: FP16 accumulation, the spec 49 speed approximation, opt-in), k_diag (timing // diagnostics, same output: "input2" runs the input stage once more before its timing mark, so the "input" part is a @@ -446,6 +512,15 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e if (end == kfill.c_str() || *end || v < 0 || v > 255) return fail("model K: k_fill must be 0..255"); fill_ = (int)v; } + // k_smx_test: the softmax self-check's test hook (smx_select), developer builds only + smx_test_ = option(c, "k_smx_test"); +#ifdef WG_DEV_DIAGNOSTICS + if (!smx_test_.empty() && smx_test_ != "mix_fail" && smx_test_ != "both_fail" && smx_test_ != "rodd_unchecked" && + smx_test_ != "report") + return fail("model K: k_smx_test must be mix_fail, both_fail, rodd_unchecked or report"); +#else + if (!smx_test_.empty()) return fail("model K: k_smx_test is a test hook of developer builds only"); +#endif // The native D3D12 device (backend 3: the D3D12 executor, or k_bench.py's own-buffer timing) runs the exact default // path only, and only when every kernel this context can launch has a port in /dx12/manifest.json; both are // checked here, before the model is loaded, so a refused context costs nothing more (for model K the context ends and @@ -544,42 +619,231 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e for (int l = 0; l < 8; ++l) k_filter_table(l, 1, tab.data() + (size_t)l * kFilterEntries * kFilterValues); if (!dev_->upload(tables_, tab.data(), tab.size() * 2)) return dfail(); if (!dev_->sync()) return dfail(); - return c.backend != 3 || mix_probe(); + return smx_select(c.backend, smx_test_); } -// The softmax stages' weight rounding A = f16(w inv) (clean/k/dx12/hlsl/k_smx.hlsli) relies on the driver folding the -// exact sum p + e into one v_fma_mix*_f16 rounding. The native D3D12 device checks that fold on this adapter once, at -// create: k_mix_probe runs the same helper on k_mix_probe.h's records (FP16-midpoint products with residuals of both -// signs, exact ties, FP16 subnormals) and must return the exact results; otherwise every softmax stage takes its -// round-to-odd port (suffix __rodd, the same bits by another sequence). The result goes into the kernels line -// (smx=mix | smx=rodd), and taking the round-to-odd ports adds one line to the notes of create (wg_rt_notes; the -// executor logs it), so the switch is never silent. WG_TEST_K_MIX_PROBE_FAIL=1 (test hook) treats the probe as failed. -bool KRuntime::mix_probe() { +// ---- the softmax weight rounding: create-time self-check (native D3D12 device) --------------------------------------- +// The softmax stages' weight rounding A = f16(w inv) (clean/k/dx12/hlsl/k_smx.hlsli) has two forms: the mix form (the +// exact sum p + e into the driver's round-to-nearest conversion, exact when the driver folds it into one +// v_fma_mix*_f16) and the round-to-odd ports (suffix __rodd, exact when the driver keeps the round-to-odd integer +// operations and converts round-to-nearest). Both depend on how the player's driver compiles each stage shader, so +// neither is taken on trust. At create the native D3D12 device checks the shaders that will run: +// - k_mix_probe and k_mix_probe__rodd on k_mix_probe.h's records (FP16-midpoint products with residuals of both signs, +// exact ties, FP16 subnormals) against the exact results; +// - every softmax stage kernel this runtime can select, in both forms, on the same synthetic inputs and weights +// (kSmxCases: each stage at a 512 x 384 base map; e4 / d4 (NG = 2) and m again at 1536 x 1280), its +// outputs hashed and compared with k_smx_selfcheck.h, the hashes of the HIP reference. +// The mix form is used when its probe and all its stages match; else the round-to-odd ports when theirs all match (one +// note of create); else create fails, naming the first stage that differs. Launching each stage kernel also loads, +// verifies and prepares its shader at create: a missing or corrupt stage shader of either form fails create there. +// Test hook (developer builds only, WG_DEV_DIAGNOSTICS): runtime option k_smx_test = mix_fail (the mix form counts as +// failed), both_fail (both forms count as failed), rodd_unchecked (the round-to-odd ports without the check; for the +// frame test's negative control), report (adds the computed hashes as a note; on any device). +namespace { +struct SmxCase { + const char* stage; + int wb, hb; // base map size + int sets = 1; // independent synthetic data sets, the outputs of all hashed together +}; +// the softmax stages (spec 41: e2 ... d1; e0, e1 and d0 mix with fixed weights) at a 512 x 384 base map, which selects +// k_stage_e2, k_stagew2_e3, k_stagew4_e4, k_stagew4_m, k_stagew4_d4, k_stagew2_d3, k_stage_d2 and k_stage2_d1 (e4 / d4: +// 20 windows at offset 2; NG = 4 takes at most 24, so these two run on 8 data sets); at 1536 x 1280 e4 and d4 select +// their NG = 2 kernels (156 windows) and m runs again with 42 windows. Sized so that a one-stage wrong-rounding mutant +// of either form changes its case's outputs (the mutant runs in the #254 evidence). +const SmxCase kSmxCases[] = {{"e2", 512, 384}, {"e3", 512, 384}, {"e4", 512, 384, 8}, {"m", 512, 384}, + {"d4", 512, 384, 8}, {"d3", 512, 384}, {"d2", 512, 384}, {"d1", 512, 384}, + {"e4", 1536, 1280}, {"d4", 1536, 1280}, {"m", 1536, 1280}}; +constexpr int kSmxOffset = 2; // window offset of every case (even, as the network's) +constexpr uint32_t kSmxSeed = 0x6b5eu; // synthetic data (xorshift32) +static_assert(sizeof kSmxCases / sizeof kSmxCases[0] == k_smx_selfcheck::kCount, "k_smx_selfcheck.h case count"); + +const KStageInfo& stage_info(const char* name) { + for (const KStageInfo& s : k_stage_table()) + if (std::strcmp(s.name, name) == 0) return s; + return k_stage_table()[0]; +} +size_t level_tokens(int wb, int hb, int level) { + int w, h; + k_level_size(wb, hb, level, &w, &h); + return (size_t)w * h; +} +// FP16 words of uniform values in [-scale, scale): also finite FP32 values where two words are read as one +void smx_fill(std::vector* v, uint32_t* state, float scale) { + for (uint16_t& h : *v) { + uint32_t x = *state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + *state = x; + h = f32_to_f16(((float)(x >> 8) * (1.0f / 8388608.0f) - 1.0f) * scale); + } +} +} // namespace + +std::string KRuntime::smx_case_name(int i, bool rodd) const { + const SmxCase& c = kSmxCases[i]; + const KStageInfo& st = stage_info(c.stage); + int lw, lh, ng = 0; + bool w2 = false; + k_level_size(c.wb, c.hb, st.level, &lw, &lh); + const std::string k = stage_kernel_base(st, false, true, cdiv(lw + kSmxOffset, 8) * cdiv(lh + kSmxOffset, 8), + &ng, &w2); + return k + (rodd ? "__rodd" : "") + "@" + std::to_string(c.wb) + "x" + std::to_string(c.hb) + + (c.sets > 1 ? "*" + std::to_string(c.sets) : std::string()); +} + +// One form of every case: the outputs' SHA-256 (first 8 bytes, big-endian) per case into *hashes. +bool KRuntime::smx_check(bool rodd, std::vector* hashes) { + const int n = (int)(sizeof kSmxCases / sizeof kSmxCases[0]); + size_t in_bytes = 0, y_bytes = 0, m_bytes = 0, w_bytes = 0; + for (const SmxCase& c : kSmxCases) { + const KStageInfo& st = stage_info(c.stage); + const size_t own = level_tokens(c.wb, c.hb, st.level) * st.C * 2; + in_bytes = std::max({in_bytes, own, st.kind == 'd' ? level_tokens(c.wb, c.hb, st.level + 1) * st.C_prev * 2 : 0}); + y_bytes = std::max(y_bytes, own); + if (st.kind == 'e') m_bytes = std::max(m_bytes, level_tokens(c.wb, c.hb, st.level + 1) * st.C_next * 2); + w_bytes = std::max(w_bytes, blobs_.at(c.stage).size()); + } + // inputs (the input map and a decoder's skip map read the same synthetic data) and one weight blob for every stage + std::vector in(in_bytes / 2), w(w_bytes / 2); + DevPtr din = 0, dw = 0, dy = 0, dm = 0; + int loaded = -1; // the data set in din / dw + auto load = [&](int set) { + if (set == loaded) return true; + uint32_t state = kSmxSeed + 0x9e3779b9u * (uint32_t)set; + if (!state) state = 1; + smx_fill(&in, &state, 1.0f); + smx_fill(&w, &state, 0.25f); + loaded = set; + return dev_->upload(din, in.data(), in_bytes) && dev_->upload(dw, w.data(), w_bytes); + }; + auto release = [&]() { + for (DevPtr p : {din, dw, dy, dm}) + if (p) dev_->free(p); + }; + bool ok = dev_->alloc(&din, in_bytes) && dev_->alloc(&dw, w_bytes) && dev_->alloc(&dy, y_bytes) && + dev_->alloc(&dm, std::max(m_bytes, 256)); + std::vector out; + for (int i = 0; ok && i < n; ++i) { + const SmxCase& c = kSmxCases[i]; + const KStageInfo& st = stage_info(c.stage); + int lw, lh, iw, ih; + k_level_size(c.wb, c.hb, st.level, &lw, &lh); + k_level_size(c.wb, c.hb, st.kind == 'd' ? st.level + 1 : st.level, &iw, &ih); + KBlkParams p = block_params(st, dw, din, dy, din, dm, 0, lw, lh, iw, ih, kSmxOffset, kSmxOffset); + int ng = 0; + bool w2 = false; + const std::string k = stage_kernel_base(st, false, true, p.nwx * p.nwy, &ng, &w2) + (rodd ? "__rodd" : ""); + const size_t yb = level_tokens(c.wb, c.hb, st.level) * st.C * 2; + const size_t mb = st.kind == 'e' ? level_tokens(c.wb, c.hb, st.level + 1) * st.C_next * 2 : 0; + wgc::Sha256 sha; + out.resize(yb + mb); + for (int set = 0; ok && set < c.sets; ++set) { + ok = load(set) && dev_->memset(dy, 0, y_bytes) && dev_->memset(dm, 0, std::max(m_bytes, 256)) && + launch(k.c_str(), (unsigned)(w2 ? cdiv(p.nwx * p.nwy, 2) : p.nwx * p.nwy), (unsigned)(kThreads * (ng ? ng : 1)), + &p) && + dev_->download(out.data(), dy, yb) && (!mb || dev_->download(out.data() + yb, dm, mb)); + if (ok) sha.update(out.data(), out.size()); + } + if (ok) { + const wgc::Digest d = sha.finish(); + uint64_t v = 0; + for (int b = 0; b < 8; ++b) v = (v << 8) | d[b]; + hashes->push_back(v); + } + } + release(); + return ok || dfail(); +} + +// The mix / round-to-odd probe: the number of results that differ from the exact ones. +bool KRuntime::smx_probe(bool rodd, int* bad) { DevPtr in = 0, out = 0; const size_t out_bytes = (size_t)k_mix_probe::kLanes * 16; - if (!alloc(&in, sizeof k_mix_probe::kRecords) || !alloc(&out, out_bytes)) return false; - if (!dev_->upload(in, k_mix_probe::kRecords, sizeof k_mix_probe::kRecords) || !dev_->memset(out, 0xA5, out_bytes)) - return dfail(); - if (!launch("k_mix_probe", (unsigned)(k_mix_probe::kLanes / 32), 32, &in, &out)) return false; + if (!dev_->alloc(&in, sizeof k_mix_probe::kRecords) || !dev_->alloc(&out, out_bytes)) return dfail(); std::vector got((size_t)k_mix_probe::kLanes * 8); - if (!dev_->download(got.data(), out, out_bytes)) return dfail(); - int bad = 0; - for (size_t i = 0; i < got.size(); ++i) bad += got[i] != k_mix_probe::kExpected[i]; - const char* hook = std::getenv("WG_TEST_K_MIX_PROBE_FAIL"); - const bool forced = hook && std::strcmp(hook, "1") == 0; - smx_rodd_ = bad != 0 || forced; - // one token (the kernels line is split at spaces into key=value pairs); the details are in the note - smx_note_ = smx_rodd_ ? std::string(" smx=rodd(") + (forced ? std::string("hook") : std::to_string(bad) + "/" + - std::to_string(got.size())) + ")" - : std::string(" smx=mix"); - notes_ = smx_rodd_ ? std::string("model K: softmax weight rounding takes the round-to-odd ports (") + - (forced ? "probe failed by test hook" : "mix-form probe: " + std::to_string(bad) + " of " + - std::to_string(got.size()) + " differ") + - "); output unchanged, both forms are exact" - : std::string(); + const bool ok = dev_->upload(in, k_mix_probe::kRecords, sizeof k_mix_probe::kRecords) && + dev_->memset(out, 0xA5, out_bytes) && + launch(rodd ? "k_mix_probe__rodd" : "k_mix_probe", (unsigned)(k_mix_probe::kLanes / 32), 32, &in, &out) && + dev_->download(got.data(), out, out_bytes); + dev_->free(in); + dev_->free(out); + if (!ok) return dfail(); + *bad = 0; + for (size_t i = 0; i < got.size(); ++i) *bad += got[i] != k_mix_probe::kExpected[i]; return true; } +// smx_select (create, last step): the softmax form of this runtime, from the checks above (native D3D12 device), or only +// the report hook's hashes (other devices: the HIP reference for clean/k/dx12/gen_k_smx_selfcheck.py). +bool KRuntime::smx_select(int backend, const std::string& hook) { + const int n = (int)(sizeof kSmxCases / sizeof kSmxCases[0]); + for (int i = 0; i < n; ++i) + if (smx_case_name(i, false) != k_smx_selfcheck::kNames[i]) + return fail("model K: k_smx_selfcheck.h does not match the self-check cases (" + smx_case_name(i, false) + ")"); + auto hex = [](uint64_t v) { + char b[17]; + std::snprintf(b, sizeof b, "%016llx", (unsigned long long)v); + return std::string(b); + }; + auto report = [&](const char* form, const std::vector& h, int probe_bad) { + std::string r = std::string("model K self-check ") + form + ":" + + (probe_bad >= 0 ? " probe=" + std::to_string(probe_bad) : std::string()); + for (int i = 0; i < (int)h.size(); ++i) r += " " + smx_case_name(i, false) + "=" + hex(h[i]); + return r; + }; + auto add_note = [&](const std::string& line) { notes_ += (notes_.empty() ? "" : "\n") + line; }; + if (backend != 3) { // the report hook only: the reference's hashes + if (hook != "report") return true; + std::vector h; + if (!smx_check(false, &h)) return false; + add_note(report("reference", h, -1)); + return true; + } + if (hook == "rodd_unchecked") { + smx_rodd_ = true; + smx_note_ = " smx=rodd(unchecked)"; + add_note("model K: test hook: the round-to-odd softmax ports without the create-time check"); + return true; + } + int bad[2] = {0, 0}; + std::vector h[2]; + if (!smx_probe(false, &bad[0]) || !smx_probe(true, &bad[1]) || !smx_check(false, &h[0]) || !smx_check(true, &h[1])) + return false; + // the first check that differs, per form ("" = none) + std::string first[2]; + for (int f = 0; f < 2; ++f) { + if (bad[f]) first[f] = std::string(f ? "k_mix_probe__rodd" : "k_mix_probe") + " " + std::to_string(bad[f]) + " of " + + std::to_string(k_mix_probe::kLanes * 8); + for (int i = 0; first[f].empty() && i < n; ++i) + if (h[f][i] != k_smx_selfcheck::kExpected[i]) first[f] = smx_case_name(i, f == 1); + } + if (hook == "mix_fail" || hook == "both_fail") first[0] = "test hook"; + if (hook == "both_fail") first[1] = "test hook"; + if (hook == "report") { + add_note(report("mix", h[0], bad[0])); + add_note(report("rodd", h[1], bad[1])); + } + if (first[0].empty()) { + smx_rodd_ = false; + smx_note_ = " smx=mix"; + if (!first[1].empty()) + add_note("model K: the round-to-odd softmax ports failed the create-time check on this driver (" + first[1] + + "); the mix form passed it and is used"); + return true; + } + if (first[1].empty()) { + smx_rodd_ = true; + std::string token = first[0] == "test hook" ? std::string("hook") : first[0].substr(0, first[0].find(' ')); + smx_note_ = " smx=rodd(" + token + ")"; + add_note("model K: softmax weight rounding takes the round-to-odd ports (the mix form failed the create-time check: " + + first[0] + "); they passed it, output unchanged"); + return true; + } + return fail("model K: the D3D12 softmax stages are not exact on this driver (create-time check: mix form " + first[0] + + ", round-to-odd form " + first[1] + "); not started"); +} + std::string KRuntime::stage_kernel(const std::string& k, const char* stage) const { // e0, e1 and d0 mix with fixed weights (no softmax): no __rodd form const bool softmax = std::strcmp(stage, "e0") != 0 && std::strcmp(stage, "e1") != 0 && std::strcmp(stage, "d0") != 0; @@ -630,58 +894,12 @@ bool KRuntime::network(int f, const std::function& after) { const int ox = off[i][0], oy = off[i][1]; if (st.kind != 'm' && (ox % 2 || oy % 2)) return fail(std::string("model K: odd window offset at ") + st.name); const DevPtr blob = wdev_[st.name]; - const StageSec sec = stage_sections(st); - const BlockSec bs = block_sections(st); - const DevPtr blk = blob + sec.block; - KBlkParams p = {}; - p.x = net_[io.in]; - p.y = io.block ? net_[io.block] : 0; - p.g1 = blk + bs.g1; - p.g2 = blk + bs.g2; - p.wqkv = blk + bs.qkv; - p.mix = blk + bs.mix; - p.wo = blk + bs.wo; - p.bo = blk + bs.bo; - p.w1 = blk + bs.w1; - p.b1 = blk + bs.b1; - p.w2 = blk + bs.w2; - p.b2 = blk + bs.b2; - p.w = lw; - p.h = lh; - p.ox = ox; - p.oy = oy; - p.nwx = cdiv(lw + ox, 8); // layout.py window_grid - p.nwy = cdiv(lh + oy, 8); - if (st.embed) { - p.wa = blob + sec.embed; - p.ba = blob + sec.embed + linear_bias(32, 16); - } - if (st.kind == 'd') { - p.wa = blob + sec.expand; - p.ba = blob + sec.expand + linear_bias(4 * st.C, st.C_prev); - p.skip = net_[io.skip]; - p.cw = iw; - p.ch = ih; - } - if (st.kind == 'e') { - p.wz = blob + sec.merge; - p.bz = blob + sec.merge + linear_bias(st.C_next, 4 * st.C); - p.merged = net_[io.merged]; - } - if (st.head) { - p.wz = blob + sec.head; - p.bz = blob + sec.head + linear_bias(40, st.C); - p.head = net_[io.head]; - } - const int ng = wide_ng(st.name, f16_, p.nwx * p.nwy); + KBlkParams p = block_params(st, blob, net_[io.in], io.block ? net_[io.block] : 0, io.skip ? net_[io.skip] : 0, + io.merged ? net_[io.merged] : 0, io.head ? net_[io.head] : 0, lw, lh, iw, ih, ox, oy); + int ng = 0; + bool w2 = false; + const std::string k = stage_kernel_base(st, f16_, p.y != 0, p.nwx * p.nwy, &ng, &w2); if (st.kind == 'm' && !ng) return fail("model K: stage m needs its wide kernel"); - const std::string h = f16_ ? "h" : ""; - // e0, e1, d1, d0: two windows per workgroup (k_stage2_*, k_launch.py WINDOWS2_STAGES; k_net.py WINDOWS2_VARIANTS; - // d1's needs its block output map p.y, which the network always gives it) - const bool w2 = !ng && (std::strcmp(st.name, "e0") == 0 || std::strcmp(st.name, "e1") == 0 || - (std::strcmp(st.name, "d1") == 0 && p.y) || std::strcmp(st.name, "d0") == 0); - const std::string k = - (ng ? "k_stagew" + std::to_string(ng) : std::string(w2 ? "k_stage2" : "k_stage")) + h + "_" + st.name; const unsigned grid = (unsigned)(w2 ? cdiv(p.nwx * p.nwy, 2) : p.nwx * p.nwy); auto f8 = f8data_.find(st.name); if (f8_ && w2 && f8 != f8data_.end()) { // network=fused8: k_stage2q_* (k_block8.h) @@ -1098,12 +1316,13 @@ std::vector KRuntime::d3d12_kernels() const { else if (ng) k = "k_stagew" + std::to_string(ng) + "_" + n; else k = (n == "e0" || n == "e1" || n == "d1" ? "k_stage2_" : "k_stage_") + n; if (std::find(v.begin(), v.end(), k) == v.end()) v.push_back(k); - // the round-to-odd form a failed mix-form probe switches to (KRuntime::mix_probe) + // the round-to-odd form the create-time self-check can select (KRuntime::smx_select) if (!st.head && n != "e0" && n != "e1" && std::find(v.begin(), v.end(), k + "__rodd") == v.end()) v.push_back(k + "__rodd"); } } v.push_back("k_mix_probe"); + v.push_back("k_mix_probe__rodd"); v.push_back((compact_head_ ? "k_out_pixel_c" : "k_out_pixel") + d); if (!warm_at_.empty()) v.push_back("k_warm"); // k_warm (option k_warm, backend 3) v.push_back("timing_barrier"); // the timing marks (timing_log=1) diff --git a/clean/k/runtime/runtime_k.h b/clean/k/runtime/runtime_k.h index 93489bf6..5315a903 100644 --- a/clean/k/runtime/runtime_k.h +++ b/clean/k/runtime/runtime_k.h @@ -104,9 +104,13 @@ class KRuntime { // the tiled input stage's guard (frame and direct_preflight): sizes, the 0.68 ratio, finite jitter in [-0.5, 0.5] bool tiled_guard(int rw, int rh, float jx, float jy) const; bool options_ok_direct(); - // the native D3D12 device's softmax rounding probe (k_mix_probe at create): sets smx_rodd_ when the driver's fold of - // the mix form (clean/k/dx12/hlsl/k_smx.hlsli) does not give the exact single rounding on this adapter - bool mix_probe(); + // the softmax weight rounding (runtime_k.cpp "create-time self-check"): smx_select picks the mix form or the __rodd + // ports after checking both on this device (native D3D12), or refuses; smx_check hashes one form's stage outputs, + // smx_probe counts one form's probe mismatches, smx_case_name names a self-check case + bool smx_select(int backend, const std::string& hook); + bool smx_check(bool rodd, std::vector* hashes); + bool smx_probe(bool rodd, int* bad); + std::string smx_case_name(int i, bool rodd) const; // the stage kernel network() launches: the __rodd form of a softmax stage after a failed probe std::string stage_kernel(const std::string& k, const char* stage) const; int fill_ = -1; // option k_fill (test hook): the byte every frame-written buffer and the histories start with @@ -158,6 +162,7 @@ class KRuntime { // the softmax stages run their round-to-odd (__rodd) ports: the mix-form probe failed (native D3D12 device only) bool smx_rodd_ = false; std::string smx_note_; + std::string smx_test_; // option k_smx_test (developer builds only) std::string notes_; // notes of create for the caller's log (notes()), one per line, or empty unsigned warm_line_ = 128, warm_groups_ = 64; DevPtr warm_dummy_ = 0; diff --git a/clean/k/tests/test_k_runtime_d3d12.py b/clean/k/tests/test_k_runtime_d3d12.py index 08cdd2a3..198c02e0 100644 --- a/clean/k/tests/test_k_runtime_d3d12.py +++ b/clean/k/tests/test_k_runtime_d3d12.py @@ -23,9 +23,12 @@ value, with the game's exposure texel, and with auto-exposure, and LDR; output rows RGBA16F and dense RGB; render size equal to the output (native AA, DLAA 1.0: the legacy input kernels), a jitter outside [-0.5, 0.5], and input_tiled=0. Refusals (one line, create or frame): an approximation option on the native device, an input-copy frame (raw_inputs). -Softmax rounding switch (native device): with the create-time probe failed (test hook WG_TEST_K_MIX_PROBE_FAIL=1) the -runtime reports smx=rodd and exactly one note of create (wg_rt_notes, logged by the executor); with nothing set, -smx=mix and no note. Each leg runs in its own process. +Softmax weight rounding (native device; runtime_k.cpp "create-time self-check"): with nothing set the mix form passes +the check (smx=mix, no note); with the developer hook k_smx_test=mix_fail the round-to-odd ports are taken (smx=rodd, +exactly one note of create, which the executor logs) and the frames match HIP with every softmax stage on its __rodd port, +in both e4 / d4 shapes (1440p Quality: NG = 4; 4K Performance: NG = 2); with both forms failed, or with a corrupt __rodd +shader, create refuses in one line. Each create leg runs in its own process. The hook exists only in a runtime library +built with clean/runner/build.py --dev-diagnostics: these tests skip on another build. GPU only: WG_K_GPU_BUILD=, ROCM_PATH; run each test under the @@ -134,7 +137,8 @@ def sequence(self, backend, hdr, mode, half, frames, options="", seed=0, out=OUT for ln in lines: parts = ln.split() if parts: - tgt = on_dx if parts[-1] == "dx12" else on_hip + # where: "dx12" (hybrid device) or "d3d12" (native device), else HIP + tgt = on_dx if parts[-1] in ("dx12", "d3d12") else on_hip tgt[parts[0]] = tgt.get(parts[0], 0) + 1 want, got = out_bytes(hip, half), out_bytes(dx, half) compare_raw(f"frame {i}: output", want, got) @@ -275,23 +279,79 @@ def test_native_refuses_approximation_options(self): self.assertNotIn("\n", msg.strip()) self.assertIn("HIP", msg) + # ---- softmax weight rounding: the create-time self-check and its round-to-odd ports ---------------------------- + def leg(self, options, build=None): + """One native create in its own process: {"ok", "smx", "notes", "error"}.""" + r = subprocess.run([sys.executable, "-m", "clean.k.tests.test_k_runtime_d3d12", "--softmax-leg", self.mdir, + self.map, options, build or self.build], capture_output=True, text=True, timeout=300) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + return json.loads(r.stdout.strip().splitlines()[-1]) + + def need_dev_build(self): + got = self.leg("k_smx_test=report") + if not got["ok"] and "developer builds only" in got["error"]: + self.skipTest("the softmax test hook needs a runtime built with --dev-diagnostics") + self.assertTrue(got["ok"], got) + def test_native_softmax_switch_is_logged(self): """The round-to-odd softmax ports are exact too, but taking them is never silent: one note of create.""" - legs = {} - for hook in ("", "1"): - env = {k: v for k, v in os.environ.items() if k != "WG_TEST_K_MIX_PROBE_FAIL"} - if hook: - env["WG_TEST_K_MIX_PROBE_FAIL"] = hook - r = subprocess.run([sys.executable, "-m", "clean.k.tests.test_k_runtime_d3d12", "--softmax-leg", self.mdir, - self.map], env=env, capture_output=True, text=True, timeout=300) - self.assertEqual(r.returncode, 0, r.stdout + r.stderr) - legs[hook] = json.loads(r.stdout.strip().splitlines()[-1]) - self.assertEqual(legs[""]["smx"], "mix") - self.assertEqual(legs[""]["notes"], []) - self.assertTrue(legs["1"]["smx"].startswith("rodd"), legs["1"]) - self.assertEqual(len(legs["1"]["notes"]), 1, legs["1"]) - self.assertIn("round-to-odd", legs["1"]["notes"][0]) - self.assertIn("test hook", legs["1"]["notes"][0]) + self.need_dev_build() + plain = self.leg("") + self.assertTrue(plain["ok"], plain) + self.assertEqual((plain["smx"], plain["notes"]), ("mix", [])) + fb = self.leg("k_smx_test=mix_fail") + self.assertTrue(fb["ok"], fb) + self.assertEqual(fb["smx"], "rodd(hook)") + self.assertEqual(len(fb["notes"]), 1, fb) + self.assertIn("round-to-odd", fb["notes"][0]) + self.assertIn("test hook", fb["notes"][0]) + + def test_native_softmax_refusals(self): + """Neither form shown exact, or a corrupt fallback shader: create refuses in one line (the game keeps its own + upscaler).""" + import shutil + import tempfile + self.need_dev_build() + both = self.leg("k_smx_test=both_fail") + self.assertFalse(both["ok"], both) + self.assertIn("not exact on this driver", both["error"]) + self.assertNotIn("\n", both["error"].strip()) + # a copy of the build (its top-level files and dx12/) with one flipped byte in a __rodd stage shader + tmp = tempfile.mkdtemp(prefix="k_smx_corrupt_") + try: + for f in os.listdir(self.build): + src = os.path.join(self.build, f) + if os.path.isfile(src): + shutil.copy2(src, tmp) + shutil.copytree(os.path.join(self.build, "dx12"), os.path.join(tmp, "dx12")) + victim = os.path.join(tmp, "dx12", "k_stagew2_e3__rodd.dxil") + data = bytearray(open(victim, "rb").read()) + data[len(data) // 2] ^= 0x40 + open(victim, "wb").write(bytes(data)) + bad = self.leg("", build=tmp) + self.assertFalse(bad["ok"], bad) + self.assertIn("k_stagew2_e3__rodd", bad["error"]) + self.assertNotIn("\n", bad["error"].strip()) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + def check_fallback_frames(self, frames, big, **size): + """Forced fallback: the frames match HIP (raw outputs and intermediates) and every softmax stage ran its __rodd + port on D3D12, none its mix form.""" + self.need_dev_build() + on_dx, _ = self.sequence(3, True, 0, True, frames, "k_smx_test=mix_fail", **size) + ng = "k_stagew2_" if big else "k_stagew4_" + want = ["k_stage_e2", "k_stagew2_e3", ng + "e4", "k_stagew4_m", ng + "d4", "k_stagew2_d3", "k_stage_d2", + "k_stage2_d1"] + self.assertEqual([k for k in want if not on_dx.get(k + "__rodd")], [], on_dx) + self.assertEqual([k for k in want if on_dx.get(k)], [], on_dx) + + def test_native_fallback_frames_1440p_quality(self): + self.check_fallback_frames(self.FRAMES_Q, False, **self.QUALITY) + + def test_native_fallback_frames_4k_performance(self): + self.check_fallback_frames([(1920, 1080, True, 1.0, 1.0), (1920, 1080, False, 1.0, 1.0)], True, + out=(3840, 2160), max_render=(1920, 1080)) def test_input_copy_frames_refused_on_d3d12(self): """raw_inputs (the interop's HIP input copies) has no checked D3D12 port: refused, not run unchecked.""" @@ -313,19 +373,23 @@ def test_input_copy_frames_refused_on_d3d12(self): del rt -def softmax_leg(mdir, map_path): - """Child of test_native_softmax_switch_is_logged: create on the native device, print the smx field and the notes.""" - rt = KRuntimeExt(gpu_build(), mdir, map_path, OUT, MAX_RENDER, True, False, False, "", backend=3) +def softmax_leg(mdir, map_path, options, build): + """Child of the softmax tests: one native create; prints ok, the smx field, the notes and the error as JSON.""" + try: + rt = KRuntimeExt(build, mdir, map_path, OUT, MAX_RENDER, True, False, False, options, backend=3) + except RuntimeError as e: + print(json.dumps({"ok": False, "smx": "", "notes": [], "error": str(e)})) + return rt.lib.wg_rt_notes.argtypes = [ctypes.c_void_p] rt.lib.wg_rt_notes.restype = ctypes.c_char_p kern = rt.lib.wg_rt_kernels(rt.rt).decode().split() smx = [kv.split("=", 1)[1] for kv in kern if kv.startswith("smx=")] notes = [n for n in rt.lib.wg_rt_notes(rt.rt).decode().split("\n") if n] - print(json.dumps({"smx": smx[0] if smx else "", "notes": notes})) + print(json.dumps({"ok": True, "smx": smx[0] if smx else "", "notes": notes, "error": ""})) if __name__ == "__main__": - if len(sys.argv) == 4 and sys.argv[1] == "--softmax-leg": - softmax_leg(sys.argv[2], sys.argv[3]) + if len(sys.argv) == 6 and sys.argv[1] == "--softmax-leg": + softmax_leg(*sys.argv[2:6]) else: unittest.main() diff --git a/clean/runtime/wg_runtime.h b/clean/runtime/wg_runtime.h index 46a2a666..353e3077 100644 --- a/clean/runtime/wg_runtime.h +++ b/clean/runtime/wg_runtime.h @@ -161,8 +161,9 @@ WG_RT_API const char* wg_rt_model_source(void* rt); WG_RT_API const char* wg_rt_kernels(void* rt); /* Notes of create for the caller's log, one per line, or an empty string. At present two. Model M / L: the output * filter is "analytic" because the model file has no output filter weights (spec 06 draft 4), or because this build - * cannot run them. Model K on the native D3D12 device: its softmax stages take their round-to-odd ports because the - * create-time rounding probe failed (output unchanged; also "smx=rodd(...)" in wg_rt_kernels()). The filter in use + * cannot run them. Model K on the native D3D12 device: its softmax stages take their round-to-odd ports because the mix + * form failed the create-time self-check (output unchanged; also "smx=rodd(...)" in wg_rt_kernels()), or the + * round-to-odd ports failed it while the mix form passed. The filter in use * is also in wg_rt_kernels() ("output_filter=original|analytic|quality5|quality9 * output_taps=model|none"); runtime option output_filter=original|analytic|quality5|quality9 selects it (default: * original when the model file has the weights, else analytic; original without them fails create). */ From e865d8cacb10bd5fc63dfad2b0be6a98ef37139e Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Mon, 16 Nov 2026 00:30:47 -0500 Subject: [PATCH 3/5] test_k_wide_ng: wide kernel names now come from stage_kernel_base (frame and self-check) and the kernel list Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/k/tests/test_k_wide_ng.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/clean/k/tests/test_k_wide_ng.py b/clean/k/tests/test_k_wide_ng.py index a27b5dde..81b63135 100644 --- a/clean/k/tests/test_k_wide_ng.py +++ b/clean/k/tests/test_k_wide_ng.py @@ -2,7 +2,8 @@ """Model K: the NG = 4 forms of the wide stages e3 and d3 (k_stagew4_e3, k_stagew4_d3) cannot be reached (host checks). The runtime launches a wide stage as k_stagew_ with NG from wide_ng (clean/k/runtime/runtime_k.cpp), in one -place for the frame (network()) and one for the list of kernels a context can launch (d3d12_kernels()). For the FP32 +place for the frame and the softmax self-check (stage_kernel_base, which network() and smx_check / smx_case_name call) +and one for the list of kernels a context can launch (d3d12_kernels()). For the FP32 (exact) network, the only one the D3D12 devices run, wide_ng gives e3 and d3 NG = 2 whatever the window count, the render size, the output size or any option; the FP16 networks (network=fused16 / fused8) take other kernels and are refused on the native device. So the NG = 4 forms of e3 and d3 are not part of any context's kernel set and are not @@ -52,13 +53,15 @@ def test_wide_ng_source(self): def test_wide_names_come_from_wide_ng(self): src = source() - # the two sites that build a wide kernel name (the frame and the kernel list) use wide_ng's result only - sites = re.findall(r'"k_stagew" \+ std::to_string\((\w+)\)', src) - self.assertEqual(sorted(sites), ["ng", "ng"]) - calls = re.findall(r"const int ng = wide_ng\(([^;]*)\);", src) - self.assertEqual(len(calls), 2, calls) - self.assertIn("st.name, f16_, p.nwx * p.nwy", calls) - self.assertIn("n, false, k_window_count(lw, lh, o, o)", calls) + # the two sites that build a wide kernel name use wide_ng's result only: stage_kernel_base (the frame and the + # softmax self-check) and the kernel list + sites = re.findall(r'"k_stagew" \+ std::to_string\((\*?\w+)\)', src) + self.assertEqual(sorted(sites), ["*ng", "ng"]) + self.assertEqual(re.findall(r"\*ng = wide_ng\(([^;]*)\);", src), ["st.name, f16, nwin"]) + self.assertEqual(re.findall(r"const int ng = wide_ng\(([^;]*)\);", src), ["n, false, k_window_count(lw, lh, o, o)"]) + # network() names its stage through stage_kernel_base with the frame's window count; the self-check with FP32 + self.assertIn("stage_kernel_base(st, f16_, p.y != 0, p.nwx * p.nwy, &ng, &w2)", src) + self.assertEqual(len(re.findall(r"stage_kernel_base\(st, false, true,", src)), 2) self.assertNotIn("k_stagew4_e3", src) self.assertNotIn("k_stagew4_d3", src) From 7d8206dbb2433ba9944bbcc39e086939e6f86d03 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Mon, 16 Nov 2026 02:21:55 -0500 Subject: [PATCH 4/5] K softmax self-check: the frames' window offsets, every selectable kernel prepared at create, reference bound to its sources Offsets: the self-check cases now run at the window offsets this model's frames use, computed with the frames' own rule (k_stage_offsets over the eight entries of the model's window table): e2, e3, d3, d2 and d1 at (4, 4), e4 and d4 at (2 t - 4) mod 8, m at the table's entries. k_smx_selfcheck.h holds the HIP reference for every offset any table can give (133 cases), so any model is covered; a case without a reference refuses create. K_SMX_MUT4 is a negative control that drops the residual only at window offset 4. Prepare: native create now prepares (loads, verifies against the manifest SHA-256, compiles) every kernel the context can select, through the new Device::prepare: the fixed-weight stages e0, e1 and d0 included, so a missing or corrupt stage shader refuses create in one line naming it instead of failing the first frame. Stale reference: k_smx_selfcheck.h records kSourceDigest, a SHA-256 over the self-check kernels' @wg-kernel lines, their sources' include closure, the runtime regions marked SMX-SELFCHECK-SOURCE (the cases, the synthetic data, the block parameters, the kernel-name and window-offset rules) and a build's extra defines (clean/k/dx12/smx_selfcheck_digest.py). build_dx12.py (when it builds a self-check kernel) and build.py (the runtime with model K) refuse a header whose digest differs; WG_DX12_SMX_DIGEST=0 lets a negative-control DXIL build through. gen_k_smx_selfcheck.py writes the digest and takes the case names from the reference. Tests: test_k_smx_selfcheck_digest.py (host: the committed header is current; a changed byte in a shader source, a definition line or a runtime region, and an extra define, are caught; build_dx12 refuses a stale header); test_native_softmax_refusals also corrupts the fixed-weight stage k_stage2_e0. smx_probe frees its first buffer when the second allocation fails. Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/build_dx12.py | 17 ++ clean/k/dx12/README.md | 36 ++- clean/k/dx12/gen_k_smx_selfcheck.py | 53 ++-- clean/k/dx12/hlsl/k_smx.hlsli | 18 ++ clean/k/dx12/smx_selfcheck_digest.py | 103 +++++++ clean/k/runtime/k_smx_selfcheck.h | 295 +++++++++++++++++-- clean/k/runtime/model_k.cpp | 2 + clean/k/runtime/runtime_k.cpp | 186 +++++++----- clean/k/runtime/runtime_k.h | 9 +- clean/k/tests/test_k_runtime_d3d12.py | 47 +-- clean/k/tests/test_k_smx_selfcheck_digest.py | 85 ++++++ clean/runner/build.py | 8 + clean/runtime/device.h | 3 + clean/runtime/device_d3d12.cpp | 7 + 14 files changed, 709 insertions(+), 160 deletions(-) create mode 100644 clean/k/dx12/smx_selfcheck_digest.py create mode 100644 clean/k/tests/test_k_smx_selfcheck_digest.py diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index da8a9b0f..b7dfa8e4 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -376,6 +376,22 @@ def rga_mnemonics(dxil): return None +def smx_digest_check(names, defines): + """Model K's softmax self-check reference (clean/k/runtime/k_smx_selfcheck.h) must belong to the sources and defines of + this build when it builds one of the self-check kernels (clean/k/dx12/smx_selfcheck_digest.py); a stale header would + let the runtime compare its stage shaders with values from other sources. WG_DX12_SMX_DIGEST=0 lets a negative- + control build through (test variants with --define), never a shipped build.""" + root = os.path.dirname(os.path.dirname(HERE)) + if root not in sys.path: + sys.path.insert(0, root) + from clean.k.dx12 import smx_selfcheck_digest as D + if not set(names) & set(D.KERNELS) or os.environ.get("WG_DX12_SMX_DIGEST", "1") == "0": + return + why = D.stale(defines=defines) + if why: + raise SystemExit("softmax self-check reference: " + why) + + def isa_check(built): """built: [(kernel name, dxil path)] of this build. Fails when a kernel listed in isa_checks.json has another number of instructions matching one of its patterns.""" @@ -550,6 +566,7 @@ def main(argv=None): raise SystemExit(f"kernel {d['name']} ({d['co']}): DXIL file {os.path.basename(out)} already used " "(a second port of a name needs file=)") jobs.append((src, d, out, k_source)) + smx_digest_check({j[1]["name"] for j in jobs}, a.define) # the ISA rules' mutants beside every build of hybrid kernels: into a temporary folder of their own, never into the # build's output (a release folder, its manifest and a payload hold no check object); removed after the scan scan_dir = None diff --git a/clean/k/dx12/README.md b/clean/k/dx12/README.md index 5493707a..9f7380ff 100644 --- a/clean/k/dx12/README.md +++ b/clean/k/dx12/README.md @@ -74,17 +74,31 @@ the fold in the shipped DXIL as RGA compiles it. The player's driver compiles every stage shader on its own, so neither form is taken on trust there: at create the native D3D12 runtime checks the shaders that will run (`clean/k/runtime/runtime_k.cpp`, "create-time self-check"). It runs `k_mix_probe` and `k_mix_probe__rodd` on the records of `clean/k/runtime/k_mix_probe.h`, and every softmax stage kernel it -can select, in both forms (`k_stage_e2`, `k_stagew2_e3`, `k_stagew4_e4`, `k_stagew4_m`, `k_stagew4_d4`, `k_stagew2_d3`, -`k_stage_d2`, `k_stage2_d1` at a 512 x 384 base map; `k_stagew2_e4`, `k_stagew2_d4`, `k_stagew4_m` at 1536 x 1280), on the runtime's own -synthetic inputs and weights; the outputs' hashes must equal the HIP reference's (`clean/k/runtime/k_smx_selfcheck.h`, -written by `gen_k_smx_selfcheck.py`). The mix form is used when its probe and all its stages match, else the `__rodd` -ports when theirs all do (kernels line `smx=rodd(...)` and one note of create, `wg_rt_notes`, which the executor logs), -else create fails and names the first stage that differs. Launching every stage kernel of both forms at create also loads, -verifies and prepares their shaders: a missing or corrupt one fails create. The test hook `k_smx_test` (`mix_fail`, -`both_fail`, `rodd_unchecked`, `report`) exists only in runtime libraries built with `build.py --dev-diagnostics`; a -release build refuses it (`test_k_runtime_d3d12.py`: `test_native_softmax_switch_is_logged`, -`test_native_softmax_refusals`, `test_native_fallback_frames_*`). `K_SMX_RODD_MUT=1` is the `__rodd` ports' negative -control (the inexact bit dropped), never shipped. +can select, in both forms, at the window offsets this model's frames use: `k_stage_e2`, `k_stagew2_e3`, `k_stagew2_d3`, +`k_stage_d2`, `k_stage2_d1` at (4, 4) and `k_stagew4_e4`, `k_stagew4_d4` (8 data sets) at a 512 x 384 base map; +`k_stagew2_e4`, `k_stagew2_d4` and `k_stagew4_m` at 1536 x 1280; e4 / d4 and m at the offsets the model's window table +gives them (`k_stage_offsets` over its eight entries). The cases run on the runtime's own synthetic inputs and weights, +and their outputs' hashes must equal the HIP reference's (`clean/k/runtime/k_smx_selfcheck.h`, written by +`gen_k_smx_selfcheck.py` for every offset any window table can give). The mix form is used when its probe and all its +cases match, else the `__rodd` ports when theirs all do (kernels line `smx=rodd(...)` and one note of create, +`wg_rt_notes`, which the executor logs), else create fails and names the first case that differs. + +Before that, native create prepares every kernel the context can select (`Device::prepare`: the shader loaded, its +SHA-256 checked against the manifest, its pipeline compiled), the fixed-weight stages e0, e1 and d0 included: a missing +or corrupt shader fails create in one line naming it, not the first frame. + +The reference is bound to its sources: `k_smx_selfcheck.h` records `kSourceDigest`, the SHA-256 of the self-check +kernels' `@wg-kernel` lines, their sources' include closure, the runtime regions marked `SMX-SELFCHECK-SOURCE` (the cases, +the synthetic data, the block parameters, the kernel-name and window-offset rules) and a build's extra defines +(`smx_selfcheck_digest.py`). `build_dx12.py` (building a self-check kernel) and `build.py` (the runtime with model K) +refuse a header whose digest differs; `WG_DX12_SMX_DIGEST=0` lets a negative-control DXIL build through, never a shipped +one (`clean/k/tests/test_k_smx_selfcheck_digest.py`). + +The test hook `k_smx_test` (`mix_fail`, `both_fail`, `rodd_unchecked`, `report`) exists only in runtime libraries built +with `build.py --dev-diagnostics`; a release build refuses it (`test_k_runtime_d3d12.py`: +`test_native_softmax_switch_is_logged`, `test_native_softmax_refusals`, `test_native_fallback_frames_*`). Negative +controls, never shipped: `K_SMX_MUT=1` (the mix form's residual dropped), `K_SMX_MUT4=1` (dropped only at window offset +4), `K_SMX_RODD_MUT=1` (the `__rodd` ports' inexact bit dropped). NaN: IEEE 754 does not define the sign of a NaN that a stage generates or propagates, and where the input holds NaN the two devices can differ in it (the decoder stages). For the cases with NaN in the input the check compares the raw bytes diff --git a/clean/k/dx12/gen_k_smx_selfcheck.py b/clean/k/dx12/gen_k_smx_selfcheck.py index 56ca44ea..d5bd82aa 100644 --- a/clean/k/dx12/gen_k_smx_selfcheck.py +++ b/clean/k/dx12/gen_k_smx_selfcheck.py @@ -1,14 +1,17 @@ # SPDX-License-Identifier: MIT """Writes clean/k/runtime/k_smx_selfcheck.h: the reference hashes of model K's softmax self-check (runtime_k.cpp -"create-time self-check"; KRuntime::smx_select). Each case runs one softmax stage kernel on the runtime's own synthetic -inputs and weights; the header holds the SHA-256 (first 8 bytes) of its outputs on the HIP reference. The native D3D12 -device compares both of its forms (mix and __rodd) with these values at create. WrongGPU's own code. +"create-time self-check"; KRuntime::smx_select) and the source digest they belong to. Each case runs one softmax stage +kernel at one window offset on the runtime's own synthetic inputs and weights; the header holds the SHA-256 (first 8 +bytes) of its outputs on the HIP reference, for every offset any model's window table can give. The native D3D12 +device compares both of its forms (mix and __rodd) at its model's offsets with these values at create. kSourceDigest +(smx_selfcheck_digest.py) binds the values to the shader sources, definitions and runtime regions they came from; the +builds refuse a header whose digest differs. WrongGPU's own code. - python clean/k/dx12/gen_k_smx_selfcheck.py --build (GPU: a runtime library built with + python clean/k/dx12/gen_k_smx_selfcheck.py --build (GPU: a runtime library built from this tree with clean/runner/build.py --dev-diagnostics, HIP code objects, ROCM_PATH; runs the HIP backend with the developer hook k_smx_test=report and writes the header) - python clean/k/dx12/gen_k_smx_selfcheck.py --bootstrap (the case names with zero hashes: lets a first - developer build exist; a native D3D12 create refuses with it) + python clean/k/dx12/gen_k_smx_selfcheck.py --bootstrap (a placeholder header: lets a first developer build + exist; a native D3D12 create refuses with it) """ import argparse import ctypes @@ -18,30 +21,29 @@ HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) sys.path.insert(0, ROOT) -OUT = os.path.join(ROOT, "clean", "k", "runtime", "k_smx_selfcheck.h") -# KRuntime::smx_case_name for runtime_k.cpp kSmxCases, in order (the runtime checks the names at create) -NAMES = ["k_stage_e2@512x384", "k_stagew2_e3@512x384", "k_stagew4_e4@512x384*8", "k_stagew4_m@512x384", - "k_stagew4_d4@512x384*8", "k_stagew2_d3@512x384", "k_stage_d2@512x384", "k_stage2_d1@512x384", - "k_stagew2_e4@1536x1280", "k_stagew2_d4@1536x1280", "k_stagew4_m@1536x1280"] +from clean.k.dx12 import smx_selfcheck_digest as D # noqa: E402 + +OUT = os.path.join(ROOT, D.HEADER) TAG = "model K self-check reference:" -def header(values): +def header(entries, source_digest): out = ["// SPDX-License-Identifier: MIT", "// Generated by clean/k/dx12/gen_k_smx_selfcheck.py; do not edit. The reference hashes (SHA-256 of the outputs,", - "// first 8 bytes) of model K's softmax self-check cases (runtime_k.cpp kSmxCases, KRuntime::smx_select) on the", - "// HIP reference." + (" BOOTSTRAP: zero hashes (a native D3D12 create refuses)." if not any(values) else ""), + "// first 8 bytes) of model K's softmax self-check cases at every possible window offset (runtime_k.cpp kSmxCases,", + "// KRuntime::smx_select) on the HIP reference, and the source digest they belong to (smx_selfcheck_digest.py).", "#pragma once", "", "#include ", "", "namespace k_smx_selfcheck {", "", - "constexpr int kCount = %d;" % len(NAMES), + 'constexpr char kSourceDigest[] = "%s";' % source_digest, + "constexpr int kCount = %d;" % len(entries), "constexpr const char* kNames[kCount] = {"] - out += [' "%s",' % n for n in NAMES] + out += [' "%s",' % n for n, _ in entries] out += ["};", "constexpr uint64_t kExpected[kCount] = {"] - out += [" 0x%016xull, // %s" % (v, n) for v, n in zip(values, NAMES)] + out += [" 0x%016xull," % v for _, v in entries] out += ["};", "", "} // namespace k_smx_selfcheck", ""] return "\n".join(out) @@ -60,10 +62,10 @@ def reference(build): line = [n for n in notes if n.startswith(TAG)] if len(line) != 1: raise SystemExit("no reference line in the notes (a --dev-diagnostics build?): %r" % notes) - got = dict(kv.split("=", 1) for kv in line[0][len(TAG):].split()) - if list(got) != NAMES: - raise SystemExit("case names differ from the runtime's: %r" % list(got)) - return [int(got[n], 16) for n in NAMES] + entries = [(kv.split("=", 1)[0], int(kv.split("=", 1)[1], 16)) for kv in line[0][len(TAG):].split()] + if len({n for n, _ in entries}) != len(entries): + raise SystemExit("duplicate case names in the reference") + return entries def main(): @@ -73,12 +75,11 @@ def main(): g.add_argument("--bootstrap", action="store_true") ap.add_argument("--out", default=OUT) a = ap.parse_args() - values = [0] * len(NAMES) if a.bootstrap else reference(a.build) + entries = [("bootstrap", 0)] if a.bootstrap else reference(a.build) + text = header(entries, D.digest()) with open(a.out, "w", encoding="utf-8", newline="\n") as f: - f.write(header(values)) - print(a.out) - for n, v in zip(NAMES, values): - print(" %016x %s" % (v, n)) + f.write(text) + print(a.out, len(entries), "case(s), source digest", D.digest()) if __name__ == "__main__": diff --git a/clean/k/dx12/hlsl/k_smx.hlsli b/clean/k/dx12/hlsl/k_smx.hlsli index ea20a79d..47d4b6ea 100644 --- a/clean/k/dx12/hlsl/k_smx.hlsli +++ b/clean/k/dx12/hlsl/k_smx.hlsli @@ -17,6 +17,9 @@ // FP16 is the single rounding of the exact product), then the same conversion. Truncation of w inv is p, or p - 1 // ulp when e < 0; the inexact bit e != 0 makes it odd (the same bits as wg_round_odd for every finite p > 0). // K_SMX_MUT 1: negative control of the checks (the residual dropped: a second rounding); never shipped. +// K_SMX_MUT4 1: negative control of the offsets (a second, residual-dropped conversion whose result is taken only +// when the window offset is 4; the exact folded conversion is unchanged elsewhere; needs the stage kernels' P_ox); +// never shipped. // K_SMX_RODD_MUT 1: negative control of the __rodd ports (the inexact bit dropped: truncation, then the conversion's // rounding, wrong where the product is an FP16 midpoint); never shipped. #pragma once @@ -27,6 +30,9 @@ #ifndef K_SMX_MUT #define K_SMX_MUT 0 #endif +#ifndef K_SMX_MUT4 +#define K_SMX_MUT4 0 +#endif #ifndef K_SMX_RODD_MUT #define K_SMX_RODD_MUT 0 #endif @@ -44,6 +50,9 @@ uint k_smx_round_odd_pos(float p, float e) uint4 k_smx_weights8(float w[8], float inv) { v8f ro; +#if K_SMX >= 2 && K_SMX_MUT4 + v8f rb; // negative control only: the residual-dropped conversion, taken at window offset 4 alone +#endif [unroll] for (uint j = 0; j < 8u; ++j) { const float p = w[j] * inv; const float e = mad(w[j], inv, -p); // exact: w inv = p + e @@ -51,9 +60,18 @@ uint4 k_smx_weights8(float w[8], float inv) ro.v[j] = p; #elif K_SMX >= 2 ro.v[j] = p + e; +#if K_SMX_MUT4 + rb.v[j] = p; +#endif #else ro.v[j] = asfloat(k_smx_round_odd_pos(p, e)); #endif } +#if K_SMX >= 2 && K_SMX_MUT4 + // two conversions, the choice made on their results: the folded exact one stays as it is at every other offset + const uint4 good = wg_to_f16_ags(ro), bad = wg_to_f16_ags(rb); + return P_ox == 4 ? bad : good; +#else return wg_to_f16_ags(ro); +#endif } diff --git a/clean/k/dx12/smx_selfcheck_digest.py b/clean/k/dx12/smx_selfcheck_digest.py new file mode 100644 index 00000000..0b723ee5 --- /dev/null +++ b/clean/k/dx12/smx_selfcheck_digest.py @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: MIT +"""The source digest of model K's softmax self-check reference (clean/k/runtime/k_smx_selfcheck.h). WrongGPU's own code. + +The reference hashes in k_smx_selfcheck.h are valid only for the shaders and the synthetic data they were generated from. +The digest is a SHA-256 over: +- the @wg-kernel definition lines of every self-check kernel (both forms: the names, code objects and defines); +- the text of every source file those kernels compile from (their .hlsl file and its #include closure in + clean/k/dx12/hlsl and clean/dx12/hlsl; an include generated into the build folder is covered by its generator, + clean/k/dx12/gen_k_fp_tables.py); +- the runtime regions marked "SMX-SELFCHECK-SOURCE BEGIN" / "END" in clean/k/runtime/runtime_k.cpp and model_k.cpp (the + cases, the synthetic data, the block parameters, the kernel-name rule and the window-offset rule); +- the extra preprocessor defines of a build (build_dx12.py --define). +gen_k_smx_selfcheck.py writes it into the header (kSourceDigest); build_dx12.py and build.py refuse to build when the +current digest differs from the header's (WG_DX12_SMX_DIGEST=0 lets a negative-control build through, never a shipped +one); clean/k/tests/test_k_smx_selfcheck_digest.py checks the committed header and that a changed byte is caught. +""" +import hashlib +import os +import re + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +HEADER = os.path.join("clean", "k", "runtime", "k_smx_selfcheck.h") +KERNEL_DIR = os.path.join("clean", "k", "dx12", "kernels") +INCLUDE_DIRS = [os.path.join("clean", "k", "dx12", "hlsl"), os.path.join("clean", "dx12", "hlsl")] +GENERATED_BY = os.path.join("clean", "k", "dx12", "gen_k_fp_tables.py") +RUNTIME_FILES = [os.path.join("clean", "k", "runtime", "runtime_k.cpp"), os.path.join("clean", "k", "runtime", "model_k.cpp")] +# the softmax stage kernels the self-check runs (runtime_k.cpp kSmxCases), both forms +BASE = ["k_stage_e2", "k_stagew2_e3", "k_stagew2_d3", "k_stage_d2", "k_stage2_d1", "k_stagew4_e4", "k_stagew4_d4", + "k_stagew2_e4", "k_stagew2_d4", "k_stagew4_m"] +KERNELS = sorted(BASE + [k + "__rodd" for k in BASE]) +DECL = re.compile(r"^//\s*@wg-kernel\s+(\S+)\s.*$", re.M) +INCLUDE = re.compile(r'^\s*#\s*include\s+"([^"]+)"', re.M) +REGION = re.compile(r"// SMX-SELFCHECK-SOURCE BEGIN.*?// SMX-SELFCHECK-SOURCE END", re.S) + + +def _text(root, rel): + with open(os.path.join(root, rel), "rb") as f: + return f.read().replace(b"\r\n", b"\n") + + +def _closure(root, rel, seen, missing): + if rel in seen: + return + seen.add(rel) + for inc in INCLUDE.findall(_text(root, rel).decode("utf-8", "replace")): + for d in [os.path.dirname(rel)] + INCLUDE_DIRS: + cand = os.path.normpath(os.path.join(d, inc)) + if os.path.isfile(os.path.join(root, cand)): + _closure(root, cand, seen, missing) + break + else: + missing.add(inc) + + +def digest(root=ROOT, defines=()): + """The digest (64 hex digits) of the self-check's shader sources, definitions, runtime regions and extra defines.""" + lines, files, missing = {}, set(), set() + kdir = os.path.join(root, KERNEL_DIR) + for fn in sorted(os.listdir(kdir)): + if not fn.endswith(".hlsl"): + continue + rel = os.path.join(KERNEL_DIR, fn) + for m in DECL.finditer(_text(root, rel).decode("utf-8", "replace")): + if m.group(1) in KERNELS: + lines[m.group(1)] = m.group(0).strip() + _closure(root, rel, files, missing) + absent = sorted(set(KERNELS) - set(lines)) + if absent: + raise ValueError("self-check kernels not declared: %s" % absent) + h = hashlib.sha256() + h.update(b"kernels\n") + for k in sorted(lines): + h.update(lines[k].encode() + b"\n") + h.update(b"sources\n") + for rel in sorted(files, key=lambda r: r.replace(os.sep, "/")): + h.update(rel.replace(os.sep, "/").encode() + b"\n" + _text(root, rel) + b"\n") + if missing: # includes generated into the build folder: their generator + h.update(b"generated " + ",".join(sorted(missing)).encode() + b"\n" + _text(root, GENERATED_BY) + b"\n") + h.update(b"runtime\n") + for rel in RUNTIME_FILES: + for r in REGION.findall(_text(root, rel).decode("utf-8", "replace")): + h.update(r.encode() + b"\n") + h.update(b"defines\n" + "\n".join(sorted(defines)).encode() + b"\n") + return h.hexdigest() + + +def header_digest(root=ROOT): + """kSourceDigest of the header, or None.""" + p = os.path.join(root, HEADER) + if not os.path.isfile(p): + return None + m = re.search(r'kSourceDigest\[\]\s*=\s*"([0-9a-f]{64})"', _text(root, HEADER).decode("utf-8", "replace")) + return m.group(1) if m else None + + +def stale(root=ROOT, defines=()): + """None when the header matches the current sources and defines, else the reason (one line).""" + want, got = header_digest(root), digest(root, defines) + if want == got: + return None + return ("clean/k/runtime/k_smx_selfcheck.h was generated from other shader sources or defines (header %s, this build " + "%s): regenerate it with clean/k/dx12/gen_k_smx_selfcheck.py" % (want, got)) diff --git a/clean/k/runtime/k_smx_selfcheck.h b/clean/k/runtime/k_smx_selfcheck.h index e47c8a57..c4fed657 100644 --- a/clean/k/runtime/k_smx_selfcheck.h +++ b/clean/k/runtime/k_smx_selfcheck.h @@ -1,39 +1,284 @@ // SPDX-License-Identifier: MIT // Generated by clean/k/dx12/gen_k_smx_selfcheck.py; do not edit. The reference hashes (SHA-256 of the outputs, -// first 8 bytes) of model K's softmax self-check cases (runtime_k.cpp kSmxCases, KRuntime::smx_select) on the -// HIP reference. +// first 8 bytes) of model K's softmax self-check cases at every possible window offset (runtime_k.cpp kSmxCases, +// KRuntime::smx_select) on the HIP reference, and the source digest they belong to (smx_selfcheck_digest.py). #pragma once #include namespace k_smx_selfcheck { -constexpr int kCount = 11; +constexpr char kSourceDigest[] = "2ad91491cb5edf84114baa30bc0fe79a4859ac281a90d8989bc57130f8d05d69"; +constexpr int kCount = 133; constexpr const char* kNames[kCount] = { - "k_stage_e2@512x384", - "k_stagew2_e3@512x384", - "k_stagew4_e4@512x384*8", - "k_stagew4_m@512x384", - "k_stagew4_d4@512x384*8", - "k_stagew2_d3@512x384", - "k_stage_d2@512x384", - "k_stage2_d1@512x384", - "k_stagew2_e4@1536x1280", - "k_stagew2_d4@1536x1280", - "k_stagew4_m@1536x1280", + "k_stage_e2@512x384/o4,4", + "k_stagew2_e3@512x384/o4,4", + "k_stagew2_d3@512x384/o4,4", + "k_stage_d2@512x384/o4,4", + "k_stage2_d1@512x384/o4,4", + "k_stagew4_e4@512x384*8/o0,0", + "k_stagew4_e4@512x384*8/o0,2", + "k_stagew4_e4@512x384*8/o0,4", + "k_stagew4_e4@512x384*8/o0,6", + "k_stagew4_e4@512x384*8/o2,0", + "k_stagew4_e4@512x384*8/o2,2", + "k_stagew4_e4@512x384*8/o2,4", + "k_stagew4_e4@512x384*8/o2,6", + "k_stagew4_e4@512x384*8/o4,0", + "k_stagew4_e4@512x384*8/o4,2", + "k_stagew4_e4@512x384*8/o4,4", + "k_stagew4_e4@512x384*8/o4,6", + "k_stagew4_e4@512x384*8/o6,0", + "k_stagew4_e4@512x384*8/o6,2", + "k_stagew4_e4@512x384*8/o6,4", + "k_stagew4_e4@512x384*8/o6,6", + "k_stagew4_d4@512x384*8/o0,0", + "k_stagew4_d4@512x384*8/o0,2", + "k_stagew4_d4@512x384*8/o0,4", + "k_stagew4_d4@512x384*8/o0,6", + "k_stagew4_d4@512x384*8/o2,0", + "k_stagew4_d4@512x384*8/o2,2", + "k_stagew4_d4@512x384*8/o2,4", + "k_stagew4_d4@512x384*8/o2,6", + "k_stagew4_d4@512x384*8/o4,0", + "k_stagew4_d4@512x384*8/o4,2", + "k_stagew4_d4@512x384*8/o4,4", + "k_stagew4_d4@512x384*8/o4,6", + "k_stagew4_d4@512x384*8/o6,0", + "k_stagew4_d4@512x384*8/o6,2", + "k_stagew4_d4@512x384*8/o6,4", + "k_stagew4_d4@512x384*8/o6,6", + "k_stagew2_e4@1536x1280/o0,0", + "k_stagew2_e4@1536x1280/o0,2", + "k_stagew2_e4@1536x1280/o0,4", + "k_stagew2_e4@1536x1280/o0,6", + "k_stagew2_e4@1536x1280/o2,0", + "k_stagew2_e4@1536x1280/o2,2", + "k_stagew2_e4@1536x1280/o2,4", + "k_stagew2_e4@1536x1280/o2,6", + "k_stagew2_e4@1536x1280/o4,0", + "k_stagew2_e4@1536x1280/o4,2", + "k_stagew2_e4@1536x1280/o4,4", + "k_stagew2_e4@1536x1280/o4,6", + "k_stagew2_e4@1536x1280/o6,0", + "k_stagew2_e4@1536x1280/o6,2", + "k_stagew2_e4@1536x1280/o6,4", + "k_stagew2_e4@1536x1280/o6,6", + "k_stagew2_d4@1536x1280/o0,0", + "k_stagew2_d4@1536x1280/o0,2", + "k_stagew2_d4@1536x1280/o0,4", + "k_stagew2_d4@1536x1280/o0,6", + "k_stagew2_d4@1536x1280/o2,0", + "k_stagew2_d4@1536x1280/o2,2", + "k_stagew2_d4@1536x1280/o2,4", + "k_stagew2_d4@1536x1280/o2,6", + "k_stagew2_d4@1536x1280/o4,0", + "k_stagew2_d4@1536x1280/o4,2", + "k_stagew2_d4@1536x1280/o4,4", + "k_stagew2_d4@1536x1280/o4,6", + "k_stagew2_d4@1536x1280/o6,0", + "k_stagew2_d4@1536x1280/o6,2", + "k_stagew2_d4@1536x1280/o6,4", + "k_stagew2_d4@1536x1280/o6,6", + "k_stagew4_m@1536x1280/o0,0", + "k_stagew4_m@1536x1280/o0,1", + "k_stagew4_m@1536x1280/o0,2", + "k_stagew4_m@1536x1280/o0,3", + "k_stagew4_m@1536x1280/o0,4", + "k_stagew4_m@1536x1280/o0,5", + "k_stagew4_m@1536x1280/o0,6", + "k_stagew4_m@1536x1280/o0,7", + "k_stagew4_m@1536x1280/o1,0", + "k_stagew4_m@1536x1280/o1,1", + "k_stagew4_m@1536x1280/o1,2", + "k_stagew4_m@1536x1280/o1,3", + "k_stagew4_m@1536x1280/o1,4", + "k_stagew4_m@1536x1280/o1,5", + "k_stagew4_m@1536x1280/o1,6", + "k_stagew4_m@1536x1280/o1,7", + "k_stagew4_m@1536x1280/o2,0", + "k_stagew4_m@1536x1280/o2,1", + "k_stagew4_m@1536x1280/o2,2", + "k_stagew4_m@1536x1280/o2,3", + "k_stagew4_m@1536x1280/o2,4", + "k_stagew4_m@1536x1280/o2,5", + "k_stagew4_m@1536x1280/o2,6", + "k_stagew4_m@1536x1280/o2,7", + "k_stagew4_m@1536x1280/o3,0", + "k_stagew4_m@1536x1280/o3,1", + "k_stagew4_m@1536x1280/o3,2", + "k_stagew4_m@1536x1280/o3,3", + "k_stagew4_m@1536x1280/o3,4", + "k_stagew4_m@1536x1280/o3,5", + "k_stagew4_m@1536x1280/o3,6", + "k_stagew4_m@1536x1280/o3,7", + "k_stagew4_m@1536x1280/o4,0", + "k_stagew4_m@1536x1280/o4,1", + "k_stagew4_m@1536x1280/o4,2", + "k_stagew4_m@1536x1280/o4,3", + "k_stagew4_m@1536x1280/o4,4", + "k_stagew4_m@1536x1280/o4,5", + "k_stagew4_m@1536x1280/o4,6", + "k_stagew4_m@1536x1280/o4,7", + "k_stagew4_m@1536x1280/o5,0", + "k_stagew4_m@1536x1280/o5,1", + "k_stagew4_m@1536x1280/o5,2", + "k_stagew4_m@1536x1280/o5,3", + "k_stagew4_m@1536x1280/o5,4", + "k_stagew4_m@1536x1280/o5,5", + "k_stagew4_m@1536x1280/o5,6", + "k_stagew4_m@1536x1280/o5,7", + "k_stagew4_m@1536x1280/o6,0", + "k_stagew4_m@1536x1280/o6,1", + "k_stagew4_m@1536x1280/o6,2", + "k_stagew4_m@1536x1280/o6,3", + "k_stagew4_m@1536x1280/o6,4", + "k_stagew4_m@1536x1280/o6,5", + "k_stagew4_m@1536x1280/o6,6", + "k_stagew4_m@1536x1280/o6,7", + "k_stagew4_m@1536x1280/o7,0", + "k_stagew4_m@1536x1280/o7,1", + "k_stagew4_m@1536x1280/o7,2", + "k_stagew4_m@1536x1280/o7,3", + "k_stagew4_m@1536x1280/o7,4", + "k_stagew4_m@1536x1280/o7,5", + "k_stagew4_m@1536x1280/o7,6", + "k_stagew4_m@1536x1280/o7,7", }; constexpr uint64_t kExpected[kCount] = { - 0x276cf4e74f3e48adull, // k_stage_e2@512x384 - 0x87cb930d0c75bc14ull, // k_stagew2_e3@512x384 - 0xa1db88d469e36077ull, // k_stagew4_e4@512x384*8 - 0xd9a25f0ab4e5189bull, // k_stagew4_m@512x384 - 0x5a91b982d550a3f0ull, // k_stagew4_d4@512x384*8 - 0xcda38b7fb94b7fffull, // k_stagew2_d3@512x384 - 0x6cbdcdb82b666f9cull, // k_stage_d2@512x384 - 0xcf11492061147f2cull, // k_stage2_d1@512x384 - 0xb01a01dbe45082afull, // k_stagew2_e4@1536x1280 - 0x5458bd7f3a32989aull, // k_stagew2_d4@1536x1280 - 0x8ab307a6b6d53a65ull, // k_stagew4_m@1536x1280 + 0x3ac5935c84145d35ull, + 0x6ef5737586c29dddull, + 0xd7d64d479e3f8bb1ull, + 0xe5dc2941fa454a8bull, + 0x7ddd98c1b73d00faull, + 0x6ae8f4b16fa37f3full, + 0x7f9327becac9ed3full, + 0xf0162bd28c3e631full, + 0xb06c8431a1a639a6ull, + 0x482838e4db437cb5ull, + 0xa1db88d469e36077ull, + 0x35ce94197af4fbcdull, + 0x7678ed860409f5d5ull, + 0x1577f775f869f317ull, + 0xc5016a510c102b30ull, + 0xd14fab0163c1cd7cull, + 0x3de4637f62ade642ull, + 0x5cac84dabd6bd5a7ull, + 0x917660586ee12ad3ull, + 0x9dbc8b23ef04ae4eull, + 0x0eb0372fb3322adbull, + 0x008af3b9d84fe6e3ull, + 0xe73616cac05a875bull, + 0xe592583ce9369841ull, + 0x12324f069155f22bull, + 0x192125a21ed0b31aull, + 0x5a91b982d550a3f0ull, + 0x9b5d7000fc43ce62ull, + 0xc3261e2e8d007192ull, + 0xf3acd774b672e4f1ull, + 0x8aecb720a21db37full, + 0x4166511c6134d38full, + 0x6f8307cbc1bd127full, + 0x7e3f358472f039fbull, + 0x0277d2d83eb7c59cull, + 0xf6ae26af33f39836ull, + 0xb7799d4fc39f8c4eull, + 0x48ce1666e34b5c96ull, + 0x54f81218604b34b9ull, + 0x585134f7f38ce2d6ull, + 0xd2163fac3ea4470full, + 0x6060aabdf29d9d8aull, + 0xb01a01dbe45082afull, + 0xaf7dc39770db9372ull, + 0xab4172074ef7ede6ull, + 0x4217dc5fa7820181ull, + 0xaa8a15b977446402ull, + 0x11acbf0580a373ecull, + 0x60428a8be11b088aull, + 0x587f27ecf0ce8fb8ull, + 0xf2df04cc3f2156ceull, + 0xcc1effdd45455c77ull, + 0xcfcda6515c8ba5a6ull, + 0x120cde9d351abb2full, + 0xf23b325643410618ull, + 0x901640cbe4264441ull, + 0x8aa7bac63af2fe1full, + 0xc72839b6b0f3d735ull, + 0x5458bd7f3a32989aull, + 0xced4f5ff1bf38b85ull, + 0x3ba4300a3694abf6ull, + 0x49ef3180db4133deull, + 0x8b9fd5d3a9395ab1ull, + 0x4eca00d04a7f385dull, + 0x886e5bf3d1854711ull, + 0xcae27e427d9a8b96ull, + 0x08061437a11f8b96ull, + 0x65341a0e84624b87ull, + 0x5a0eba0d3084986bull, + 0xa437162423dd9208ull, + 0x892f815625d1281bull, + 0xf111e96e309a4558ull, + 0x815b897364b67cc5ull, + 0x5b9070114791fad9ull, + 0x6ead93a55e47303dull, + 0x0d72071e8506a97bull, + 0x2f97369cb21b1175ull, + 0xc5ed9a094441a609ull, + 0xdba41a2dfdafa4eeull, + 0x6eb17f0b69bcff7aull, + 0x3895d67b96f2f7f5ull, + 0x9b0a7a6a4c740628ull, + 0xd8d3695164893e50ull, + 0xba532ef3362bccfdull, + 0x9210b90067ea4a28ull, + 0xa7c0e106d9479fabull, + 0x36def1e9cf8dea52ull, + 0x8ab307a6b6d53a65ull, + 0x5e8b315d7c05ef49ull, + 0x00b8cd81ed7f48aeull, + 0x1ed1487ffe336269ull, + 0x89fd3882cc0bda27ull, + 0xdb709dd88525bb4cull, + 0x2f744f875e6bce6aull, + 0x1b035adbf577c34eull, + 0xaf6c1d12eb2969e5ull, + 0x8226cb13382fc4d4ull, + 0xcb089d9ba82f020dull, + 0x54ae711628930f40ull, + 0xa42e9768331f7923ull, + 0x8aa805bb64c7ff16ull, + 0xcf0026e5ad169971ull, + 0xf49888f55fa6e67cull, + 0x359f52d28161bf22ull, + 0xa1b29747d78981bbull, + 0xcb891e663621b0adull, + 0xa8b1cf9a23bfa82dull, + 0xe886e1530f9b6ed1ull, + 0xefe55524a320037full, + 0x55cf4ef7679771e8ull, + 0x138a1ea0840ccf3eull, + 0x0e39b9296d65faa4ull, + 0x0e92c4d29c16554eull, + 0x0f0c478a0c59e279ull, + 0x9d59340de6d71fffull, + 0xc390a236c05fd994ull, + 0xacdd3c223b6dbb6aull, + 0x1cc7ba2debb6cc1aull, + 0xaf3277b1b0eced43ull, + 0xe0cf34d804185042ull, + 0x146a6c8c19c0f9e2ull, + 0x74ffc1144a641646ull, + 0x9803563998e3b3b0ull, + 0x990f0e6f43b826deull, + 0x1372dab09bd490beull, + 0x79ab35463d9b255aull, + 0x047b1f2a51f35bb4ull, + 0x4dae2f839e01d124ull, + 0x5d441a52e4199db8ull, + 0xda83d1fa3095b59bull, + 0x8e538b24d75a9667ull, + 0xf8fa3111a724e021ull, + 0x5abea3dc416160f5ull, }; } // namespace k_smx_selfcheck diff --git a/clean/k/runtime/model_k.cpp b/clean/k/runtime/model_k.cpp index 83621515..c7ec0962 100644 --- a/clean/k/runtime/model_k.cpp +++ b/clean/k/runtime/model_k.cpp @@ -57,6 +57,7 @@ void k_base_size(int ow, int oh, double s, int* wb, int* hb) { *hb = one(oh); } +// SMX-SELFCHECK-SOURCE BEGIN (level sizes, window offsets and counts: k_smx_selfcheck.h's digest covers this text) void k_level_size(int wb, int hb, int level, int* w, int* h) { *w = cdiv(wb, 1 << level); *h = cdiv(hb, 1 << level); @@ -75,6 +76,7 @@ void k_stage_offsets(int f, const int table[8][2], int off[11][2]) { } int k_window_count(int w, int h, int ox, int oy) { return cdiv(w + ox, 8) * cdiv(h + oy, 8); } +// SMX-SELFCHECK-SOURCE END std::vector k_buffer_plan(int wb, int hb) { auto tokens = [&](int level) { diff --git a/clean/k/runtime/runtime_k.cpp b/clean/k/runtime/runtime_k.cpp index ba68728d..3b72a621 100644 --- a/clean/k/runtime/runtime_k.cpp +++ b/clean/k/runtime/runtime_k.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include "../../converter/src/json.h" #include "../../converter/src/sha256.h" @@ -266,6 +267,7 @@ const char* const kColorAtDefault = "input"; const char* const kWarmAtDefault = "e0"; const char* const kWarmToDefault = "d0"; +// SMX-SELFCHECK-SOURCE BEGIN (wide_ng, block_params, stage_kernel_base: the self-check reference depends on them) int wide_ng(const std::string& s, bool f16, int nwin) { if (f16) return s == "e4" || s == "m" || s == "d4" ? 4 : 0; if (s == "m") return 4; @@ -336,6 +338,7 @@ std::string stage_kernel_base(const KStageInfo& st, bool f16, bool has_y, int nw const std::string h = f16 ? "h" : ""; return (*ng ? "k_stagew" + std::to_string(*ng) : std::string(*w2 ? "k_stage2" : "k_stage")) + h + "_" + st.name; } +// SMX-SELFCHECK-SOURCE END // Runtime options of model K (runtime_options, then WG_RUNTIME_OPTIONS whose keys win): model_k_dir, model_k_map, // network (fused, the default; fused16: FP16 accumulation, the spec 49 speed approximation, opt-in), k_diag (timing @@ -619,6 +622,11 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e for (int l = 0; l < 8; ++l) k_filter_table(l, 1, tab.data() + (size_t)l * kFilterEntries * kFilterValues); if (!dev_->upload(tables_, tab.data(), tab.size() * 2)) return dfail(); if (!dev_->sync()) return dfail(); + // the native device prepares (loads, verifies and compiles) every kernel this context can select now: a missing or + // corrupt shader fails create here, not at the first frame + if (c.backend == 3) + for (const std::string& k : d3d12_kernels()) + if (!dev_->prepare(k)) return dfail(); return smx_select(c.backend, smx_test_); } @@ -630,71 +638,95 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e // neither is taken on trust. At create the native D3D12 device checks the shaders that will run: // - k_mix_probe and k_mix_probe__rodd on k_mix_probe.h's records (FP16-midpoint products with residuals of both signs, // exact ties, FP16 subnormals) against the exact results; -// - every softmax stage kernel this runtime can select, in both forms, on the same synthetic inputs and weights -// (kSmxCases: each stage at a 512 x 384 base map; e4 / d4 (NG = 2) and m again at 1536 x 1280), its -// outputs hashed and compared with k_smx_selfcheck.h, the hashes of the HIP reference. -// The mix form is used when its probe and all its stages match; else the round-to-odd ports when theirs all match (one -// note of create); else create fails, naming the first stage that differs. Launching each stage kernel also loads, -// verifies and prepares its shader at create: a missing or corrupt stage shader of either form fails create there. +// - every softmax stage kernel this runtime can select, in both forms, at the window offsets this model's frames use +// (k_stage_offsets over the eight entries of its window table: e2, e3, d3, d2, d1 at (4, 4); e4 and d4 at +// (2 t - 4) mod 8; m at the table's entries t), on the runtime's own synthetic inputs and weights (kSmxCases); each +// case's outputs are hashed and compared with k_smx_selfcheck.h, the HIP reference's hashes of every offset any window +// table can give (clean/k/dx12/gen_k_smx_selfcheck.py, bound to the shader sources by smx_selfcheck_digest.py). +// The mix form is used when its probe and all its cases match; else the round-to-odd ports when theirs all match (one +// note of create); else create fails, naming the first case that differs. Every kernel the context can select is +// prepared at create (KRuntime::create, Device::prepare): a missing or corrupt shader fails create. // Test hook (developer builds only, WG_DEV_DIAGNOSTICS): runtime option k_smx_test = mix_fail (the mix form counts as // failed), both_fail (both forms count as failed), rodd_unchecked (the round-to-odd ports without the check; for the -// frame test's negative control), report (adds the computed hashes as a note; on any device). +// frame test's negative control), report (adds the computed hashes as a note; on any device, every possible offset). namespace { +// SMX-SELFCHECK-SOURCE BEGIN (the cases, the synthetic data, the offsets and the check: k_smx_selfcheck.h's digest covers this text) struct SmxCase { const char* stage; - int wb, hb; // base map size - int sets = 1; // independent synthetic data sets, the outputs of all hashed together + int wb, hb; // base map size + int sets; // independent synthetic data sets, the outputs of all hashed together }; -// the softmax stages (spec 41: e2 ... d1; e0, e1 and d0 mix with fixed weights) at a 512 x 384 base map, which selects -// k_stage_e2, k_stagew2_e3, k_stagew4_e4, k_stagew4_m, k_stagew4_d4, k_stagew2_d3, k_stage_d2 and k_stage2_d1 (e4 / d4: -// 20 windows at offset 2; NG = 4 takes at most 24, so these two run on 8 data sets); at 1536 x 1280 e4 and d4 select -// their NG = 2 kernels (156 windows) and m runs again with 42 windows. Sized so that a one-stage wrong-rounding mutant -// of either form changes its case's outputs (the mutant runs in the #254 evidence). -const SmxCase kSmxCases[] = {{"e2", 512, 384}, {"e3", 512, 384}, {"e4", 512, 384, 8}, {"m", 512, 384}, - {"d4", 512, 384, 8}, {"d3", 512, 384}, {"d2", 512, 384}, {"d1", 512, 384}, - {"e4", 1536, 1280}, {"d4", 1536, 1280}, {"m", 1536, 1280}}; -constexpr int kSmxOffset = 2; // window offset of every case (even, as the network's) +// e2, e3, d3, d2, d1 (k_stage_e2, k_stagew2_e3, k_stagew2_d3, k_stage_d2, k_stage2_d1) and e4 / d4 with NG = 4 (at most 24 +// windows: 20 at 512 x 384; on 8 data sets) at a 512 x 384 base map; e4 / d4 with NG = 2 (above 24 windows) and m +// (k_stagew4_m, 42 or more windows) at 1536 x 1280. Sized so that a one-stage wrong-rounding mutant of either form +// changes its case's outputs (the mutant runs in the #254 evidence). +const SmxCase kSmxCases[] = {{"e2", 512, 384, 1}, {"e3", 512, 384, 1}, {"d3", 512, 384, 1}, {"d2", 512, 384, 1}, + {"d1", 512, 384, 1}, {"e4", 512, 384, 8}, {"d4", 512, 384, 8}, {"e4", 1536, 1280, 1}, + {"d4", 1536, 1280, 1}, {"m", 1536, 1280, 1}}; constexpr uint32_t kSmxSeed = 0x6b5eu; // synthetic data (xorshift32) -static_assert(sizeof kSmxCases / sizeof kSmxCases[0] == k_smx_selfcheck::kCount, "k_smx_selfcheck.h case count"); +// FP16 words of uniform values in [-scale, scale): also finite FP32 values where two words are read as one +void smx_fill(std::vector* v, uint32_t* state, float scale) { + for (uint16_t& h : *v) { + uint32_t x = *state; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + *state = x; + h = f32_to_f16(((float)(x >> 8) * (1.0f / 8388608.0f) - 1.0f) * scale); + } +} +constexpr int kSmxCount = (int)(sizeof kSmxCases / sizeof kSmxCases[0]); const KStageInfo& stage_info(const char* name) { for (const KStageInfo& s : k_stage_table()) if (std::strcmp(s.name, name) == 0) return s; return k_stage_table()[0]; } +int stage_index(const char* name) { + const auto& t = k_stage_table(); + for (size_t i = 0; i < t.size(); ++i) + if (std::strcmp(t[i].name, name) == 0) return (int)i; + return 0; +} size_t level_tokens(int wb, int hb, int level) { int w, h; k_level_size(wb, hb, level, &w, &h); return (size_t)w * h; } -// FP16 words of uniform values in [-scale, scale): also finite FP32 values where two words are read as one -void smx_fill(std::vector* v, uint32_t* state, float scale) { - for (uint16_t& h : *v) { - uint32_t x = *state; - x ^= x << 13; - x ^= x >> 17; - x ^= x << 5; - *state = x; - h = f32_to_f16(((float)(x >> 8) * (1.0f / 8388608.0f) - 1.0f) * scale); +// The window offsets of stage `stage` in frames (k_stage_offsets, the network's rule): over the eight entries of +// `table`, or (table == nullptr) over every table entry 0 .. 7 x 0 .. 7 any model can hold. Distinct, sorted. +std::vector> smx_offsets(const char* stage, const int (*table)[2]) { + std::vector> v; + const int si = stage_index(stage); + for (int e = 0; e < (table ? 8 : 64); ++e) { + int t[8][2], off[11][2]; + for (int k = 0; k < 8; ++k) { + t[k][0] = table ? table[k][0] : e % 8; + t[k][1] = table ? table[k][1] : e / 8; + } + k_stage_offsets(table ? e : 0, t, off); + v.emplace_back(off[si][0], off[si][1]); } + std::sort(v.begin(), v.end()); + v.erase(std::unique(v.begin(), v.end()), v.end()); + return v; } } // namespace -std::string KRuntime::smx_case_name(int i, bool rodd) const { +std::string KRuntime::smx_case_name(int i, int ox, int oy, bool rodd) const { const SmxCase& c = kSmxCases[i]; const KStageInfo& st = stage_info(c.stage); int lw, lh, ng = 0; bool w2 = false; k_level_size(c.wb, c.hb, st.level, &lw, &lh); - const std::string k = stage_kernel_base(st, false, true, cdiv(lw + kSmxOffset, 8) * cdiv(lh + kSmxOffset, 8), - &ng, &w2); + const std::string k = stage_kernel_base(st, false, true, cdiv(lw + ox, 8) * cdiv(lh + oy, 8), &ng, &w2); return k + (rodd ? "__rodd" : "") + "@" + std::to_string(c.wb) + "x" + std::to_string(c.hb) + - (c.sets > 1 ? "*" + std::to_string(c.sets) : std::string()); + (c.sets > 1 ? "*" + std::to_string(c.sets) : std::string()) + "/o" + std::to_string(ox) + "," + std::to_string(oy); } -// One form of every case: the outputs' SHA-256 (first 8 bytes, big-endian) per case into *hashes. -bool KRuntime::smx_check(bool rodd, std::vector* hashes) { - const int n = (int)(sizeof kSmxCases / sizeof kSmxCases[0]); +// One form of every case at its offsets (this model's: all = false; every possible one: all = true): (case name of the +// mix form, the outputs' SHA-256, first 8 bytes big-endian) into *out. +bool KRuntime::smx_check(bool rodd, bool all, std::vector>* out) { size_t in_bytes = 0, y_bytes = 0, m_bytes = 0, w_bytes = 0; for (const SmxCase& c : kSmxCases) { const KStageInfo& st = stage_info(c.stage); @@ -717,50 +749,54 @@ bool KRuntime::smx_check(bool rodd, std::vector* hashes) { loaded = set; return dev_->upload(din, in.data(), in_bytes) && dev_->upload(dw, w.data(), w_bytes); }; - auto release = [&]() { - for (DevPtr p : {din, dw, dy, dm}) - if (p) dev_->free(p); - }; bool ok = dev_->alloc(&din, in_bytes) && dev_->alloc(&dw, w_bytes) && dev_->alloc(&dy, y_bytes) && dev_->alloc(&dm, std::max(m_bytes, 256)); - std::vector out; - for (int i = 0; ok && i < n; ++i) { + std::vector buf; + for (int i = 0; ok && i < kSmxCount; ++i) { const SmxCase& c = kSmxCases[i]; const KStageInfo& st = stage_info(c.stage); int lw, lh, iw, ih; k_level_size(c.wb, c.hb, st.level, &lw, &lh); k_level_size(c.wb, c.hb, st.kind == 'd' ? st.level + 1 : st.level, &iw, &ih); - KBlkParams p = block_params(st, dw, din, dy, din, dm, 0, lw, lh, iw, ih, kSmxOffset, kSmxOffset); - int ng = 0; - bool w2 = false; - const std::string k = stage_kernel_base(st, false, true, p.nwx * p.nwy, &ng, &w2) + (rodd ? "__rodd" : ""); const size_t yb = level_tokens(c.wb, c.hb, st.level) * st.C * 2; const size_t mb = st.kind == 'e' ? level_tokens(c.wb, c.hb, st.level + 1) * st.C_next * 2 : 0; - wgc::Sha256 sha; - out.resize(yb + mb); - for (int set = 0; ok && set < c.sets; ++set) { - ok = load(set) && dev_->memset(dy, 0, y_bytes) && dev_->memset(dm, 0, std::max(m_bytes, 256)) && - launch(k.c_str(), (unsigned)(w2 ? cdiv(p.nwx * p.nwy, 2) : p.nwx * p.nwy), (unsigned)(kThreads * (ng ? ng : 1)), - &p) && - dev_->download(out.data(), dy, yb) && (!mb || dev_->download(out.data() + yb, dm, mb)); - if (ok) sha.update(out.data(), out.size()); - } - if (ok) { + buf.resize(yb + mb); + for (const auto& o : smx_offsets(c.stage, all ? nullptr : offsets_table_)) { + KBlkParams p = block_params(st, dw, din, dy, din, dm, 0, lw, lh, iw, ih, o.first, o.second); + int ng = 0; + bool w2 = false; + const std::string k = stage_kernel_base(st, false, true, p.nwx * p.nwy, &ng, &w2) + (rodd ? "__rodd" : ""); + wgc::Sha256 sha; + for (int set = 0; ok && set < c.sets; ++set) { + ok = load(set) && dev_->memset(dy, 0, y_bytes) && dev_->memset(dm, 0, std::max(m_bytes, 256)) && + launch(k.c_str(), (unsigned)(w2 ? cdiv(p.nwx * p.nwy, 2) : p.nwx * p.nwy), + (unsigned)(kThreads * (ng ? ng : 1)), &p) && + dev_->download(buf.data(), dy, yb) && (!mb || dev_->download(buf.data() + yb, dm, mb)); + if (ok) sha.update(buf.data(), buf.size()); + } + if (!ok) break; const wgc::Digest d = sha.finish(); uint64_t v = 0; for (int b = 0; b < 8; ++b) v = (v << 8) | d[b]; - hashes->push_back(v); + out->emplace_back(smx_case_name(i, o.first, o.second, false), v); } } - release(); + for (DevPtr p : {din, dw, dy, dm}) + if (p) dev_->free(p); return ok || dfail(); } +// SMX-SELFCHECK-SOURCE END + // The mix / round-to-odd probe: the number of results that differ from the exact ones. bool KRuntime::smx_probe(bool rodd, int* bad) { DevPtr in = 0, out = 0; const size_t out_bytes = (size_t)k_mix_probe::kLanes * 16; - if (!dev_->alloc(&in, sizeof k_mix_probe::kRecords) || !dev_->alloc(&out, out_bytes)) return dfail(); + if (!dev_->alloc(&in, sizeof k_mix_probe::kRecords)) return dfail(); + if (!dev_->alloc(&out, out_bytes)) { + dev_->free(in); + return dfail(); + } std::vector got((size_t)k_mix_probe::kLanes * 8); const bool ok = dev_->upload(in, k_mix_probe::kRecords, sizeof k_mix_probe::kRecords) && dev_->memset(out, 0xA5, out_bytes) && @@ -777,26 +813,22 @@ bool KRuntime::smx_probe(bool rodd, int* bad) { // smx_select (create, last step): the softmax form of this runtime, from the checks above (native D3D12 device), or only // the report hook's hashes (other devices: the HIP reference for clean/k/dx12/gen_k_smx_selfcheck.py). bool KRuntime::smx_select(int backend, const std::string& hook) { - const int n = (int)(sizeof kSmxCases / sizeof kSmxCases[0]); - for (int i = 0; i < n; ++i) - if (smx_case_name(i, false) != k_smx_selfcheck::kNames[i]) - return fail("model K: k_smx_selfcheck.h does not match the self-check cases (" + smx_case_name(i, false) + ")"); auto hex = [](uint64_t v) { char b[17]; std::snprintf(b, sizeof b, "%016llx", (unsigned long long)v); return std::string(b); }; - auto report = [&](const char* form, const std::vector& h, int probe_bad) { + auto report = [&](const char* form, const std::vector>& h, int probe_bad) { std::string r = std::string("model K self-check ") + form + ":" + (probe_bad >= 0 ? " probe=" + std::to_string(probe_bad) : std::string()); - for (int i = 0; i < (int)h.size(); ++i) r += " " + smx_case_name(i, false) + "=" + hex(h[i]); + for (const auto& e : h) r += " " + e.first + "=" + hex(e.second); return r; }; auto add_note = [&](const std::string& line) { notes_ += (notes_.empty() ? "" : "\n") + line; }; - if (backend != 3) { // the report hook only: the reference's hashes + if (backend != 3) { // the report hook only: the reference's hashes at every possible offset if (hook != "report") return true; - std::vector h; - if (!smx_check(false, &h)) return false; + std::vector> h; + if (!smx_check(false, true, &h)) return false; add_note(report("reference", h, -1)); return true; } @@ -807,16 +839,26 @@ bool KRuntime::smx_select(int backend, const std::string& hook) { return true; } int bad[2] = {0, 0}; - std::vector h[2]; - if (!smx_probe(false, &bad[0]) || !smx_probe(true, &bad[1]) || !smx_check(false, &h[0]) || !smx_check(true, &h[1])) + std::vector> h[2]; + const bool all = hook == "report"; + if (!smx_probe(false, &bad[0]) || !smx_probe(true, &bad[1]) || !smx_check(false, all, &h[0]) || + !smx_check(true, all, &h[1])) return false; - // the first check that differs, per form ("" = none) + // the first check that differs, per form ("" = none); a case without a reference refuses create std::string first[2]; for (int f = 0; f < 2; ++f) { if (bad[f]) first[f] = std::string(f ? "k_mix_probe__rodd" : "k_mix_probe") + " " + std::to_string(bad[f]) + " of " + std::to_string(k_mix_probe::kLanes * 8); - for (int i = 0; first[f].empty() && i < n; ++i) - if (h[f][i] != k_smx_selfcheck::kExpected[i]) first[f] = smx_case_name(i, f == 1); + for (const auto& e : h[f]) { + int at = -1; + for (int j = 0; j < k_smx_selfcheck::kCount && at < 0; ++j) + if (e.first == k_smx_selfcheck::kNames[j]) at = j; + if (at < 0) return fail("model K: k_smx_selfcheck.h has no reference for " + e.first + " (regenerate it)"); + if (first[f].empty() && e.second != k_smx_selfcheck::kExpected[at]) { + const size_t a = e.first.find('@'); + first[f] = f ? e.first.substr(0, a) + "__rodd" + e.first.substr(a) : e.first; + } + } } if (hook == "mix_fail" || hook == "both_fail") first[0] = "test hook"; if (hook == "both_fail") first[1] = "test hook"; diff --git a/clean/k/runtime/runtime_k.h b/clean/k/runtime/runtime_k.h index 5315a903..6f5df54b 100644 --- a/clean/k/runtime/runtime_k.h +++ b/clean/k/runtime/runtime_k.h @@ -105,12 +105,13 @@ class KRuntime { bool tiled_guard(int rw, int rh, float jx, float jy) const; bool options_ok_direct(); // the softmax weight rounding (runtime_k.cpp "create-time self-check"): smx_select picks the mix form or the __rodd - // ports after checking both on this device (native D3D12), or refuses; smx_check hashes one form's stage outputs, - // smx_probe counts one form's probe mismatches, smx_case_name names a self-check case + // ports after checking both on this device (native D3D12), or refuses; smx_check hashes one form's stage outputs at + // this model's window offsets (all: every possible offset), smx_probe counts one form's probe mismatches, + // smx_case_name names a self-check case bool smx_select(int backend, const std::string& hook); - bool smx_check(bool rodd, std::vector* hashes); + bool smx_check(bool rodd, bool all, std::vector>* out); bool smx_probe(bool rodd, int* bad); - std::string smx_case_name(int i, bool rodd) const; + std::string smx_case_name(int i, int ox, int oy, bool rodd) const; // the stage kernel network() launches: the __rodd form of a softmax stage after a failed probe std::string stage_kernel(const std::string& k, const char* stage) const; int fill_ = -1; // option k_fill (test hook): the byte every frame-written buffer and the histories start with diff --git a/clean/k/tests/test_k_runtime_d3d12.py b/clean/k/tests/test_k_runtime_d3d12.py index 198c02e0..a77397dd 100644 --- a/clean/k/tests/test_k_runtime_d3d12.py +++ b/clean/k/tests/test_k_runtime_d3d12.py @@ -26,8 +26,8 @@ Softmax weight rounding (native device; runtime_k.cpp "create-time self-check"): with nothing set the mix form passes the check (smx=mix, no note); with the developer hook k_smx_test=mix_fail the round-to-odd ports are taken (smx=rodd, exactly one note of create, which the executor logs) and the frames match HIP with every softmax stage on its __rodd port, -in both e4 / d4 shapes (1440p Quality: NG = 4; 4K Performance: NG = 2); with both forms failed, or with a corrupt __rodd -shader, create refuses in one line. Each create leg runs in its own process. The hook exists only in a runtime library +in both e4 / d4 shapes (1440p Quality: NG = 4; 4K Performance: NG = 2); with both forms failed, or with a corrupt stage +shader (a softmax stage's __rodd port, the fixed-weight stage e0), create refuses in one line. Each create leg runs in its own process. The hook exists only in a runtime library built with clean/runner/build.py --dev-diagnostics: these tests skip on another build. GPU only: WG_K_GPU_BUILD== 0, (path, needle) + data[at] = ord("#") if data[at] != ord("#") else ord("!") + with open(path, "wb") as f: + f.write(bytes(data)) + + +class TestSmxSelfcheckDigest(unittest.TestCase): + def test_committed_header_is_current(self): + self.assertIsNotNone(D.header_digest(), "k_smx_selfcheck.h has no kSourceDigest") + self.assertIsNone(D.stale(), "regenerate clean/k/runtime/k_smx_selfcheck.h (gen_k_smx_selfcheck.py)") + + def test_changes_are_caught(self): + cases = [(os.path.join("clean", "k", "dx12", "hlsl", "k_smx.hlsli"), "ro.v[j] = p + e;"), + (os.path.join("clean", "k", "dx12", "hlsl", "k_wide.hlsli"), "kw_softpoly8"), + (os.path.join("clean", "dx12", "hlsl", "wg_math.hlsli"), "float"), + (os.path.join("clean", "k", "dx12", "kernels", "k_block_wide.hlsl"), "KW_RD=12"), + (os.path.join("clean", "k", "runtime", "runtime_k.cpp"), "kSmxSeed = 0x6b5eu"), + (os.path.join("clean", "k", "runtime", "model_k.cpp"), "(2 * oxm - 4)")] + for rel, needle in cases: + with self.subTest(file=rel): + tmp = tempfile.mkdtemp(prefix="smx_digest_") + try: + copy_sources(tmp) + self.assertIsNone(D.stale(tmp), "an unchanged copy must match") + flip(os.path.join(tmp, rel), needle) + why = D.stale(tmp) + self.assertIsNotNone(why, "a changed byte in %s was not caught" % rel) + self.assertIn("regenerate", why) + finally: + shutil.rmtree(tmp, ignore_errors=True) + self.assertIsNotNone(D.stale(defines=["K_SMX_MUT=1"]), "an extra define was not caught") + + def test_build_refuses_a_stale_header(self): + from clean.dx12 import build_dx12 as B + with mock.patch.dict(os.environ, {"WG_DX12_SMX_DIGEST": "1"}): + with self.assertRaises(SystemExit) as cm: + B.smx_digest_check({"k_stage_e2"}, ["K_SMX_MUT=1"]) + self.assertIn("regenerate", str(cm.exception)) + B.smx_digest_check({"k_stage_e2"}, []) # current header, no extra define: builds + B.smx_digest_check({"k_out_color"}, ["K_SMX_MUT=1"]) # no self-check kernel: not concerned + with mock.patch.dict(os.environ, {"WG_DX12_SMX_DIGEST": "0"}): + B.smx_digest_check({"k_stage_e2"}, ["K_SMX_MUT=1"]) # negative-control build + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/runner/build.py b/clean/runner/build.py index 87ff9fbd..671c20a7 100644 --- a/clean/runner/build.py +++ b/clean/runner/build.py @@ -403,6 +403,14 @@ def runtime_command(cxx, out, embedded=None): def build_runtime(cxx, out_dir, embed=True): + if MODELS_BUILT.get("K"): # model K's softmax self-check reference must belong to these sources (no bypass here) + root = os.path.dirname(os.path.dirname(HERE)) + if root not in sys.path: + sys.path.insert(0, root) + from clean.k.dx12 import smx_selfcheck_digest as D + why = D.stale() + if why: + raise SystemExit("softmax self-check reference: " + why) for m in [MAP_FILE] + [o["map"] for name, o in OPTIONAL_MODELS.items() if MODELS_BUILT[name]] + \ ([L_MAP_FILE] if MODEL_L else []): shutil.copyfile(m, os.path.join(out_dir, os.path.basename(m))) diff --git a/clean/runtime/device.h b/clean/runtime/device.h index feaeccc9..3cf5c021 100644 --- a/clean/runtime/device.h +++ b/clean/runtime/device.h @@ -49,6 +49,9 @@ class Device { } // args: pointers to the argument values (hipModuleLaunchKernel convention). virtual bool launch(const std::string& kernel, unsigned grid, unsigned block, void** args) = 0; + // Loads, verifies and compiles one kernel now (its first launch then compiles nothing); false with err() when its + // shader is missing or corrupt. Devices that prepare nothing ahead: true. + virtual bool prepare(const std::string& kernel) { (void)kernel; return true; } // The next launch may start before the previous one has finished (native D3D12: no UAV barrier between the two; the // kernels synchronize themselves). Devices that always order launches ignore it. virtual void unordered_next() {} diff --git a/clean/runtime/device_d3d12.cpp b/clean/runtime/device_d3d12.cpp index 9ba27176..31bc79d2 100644 --- a/clean/runtime/device_d3d12.cpp +++ b/clean/runtime/device_d3d12.cpp @@ -756,6 +756,13 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (variant_matches(i->second)) return &i->second; return range.first == range.second ? nullptr : &range.first->second; } + bool prepare(const std::string& name) override { + NativeKernel* port = find_port(name); + if (!port) return fail("native D3D12 unsupported kernel: " + name + "; no HIP fallback"); + NativeKernel& k = *port; + if (k.pso || !variant_matches(k) || (k.rs != "buffer" && k.rs != "direct")) return true; + return create_pso(k); + } bool launch(const std::string& name, unsigned grid, unsigned block, void** args) override { NativeKernel* port = find_port(name); if (!port) return fail("native D3D12 unsupported kernel: " + name + "; no HIP fallback"); From b518f4e0f8e085dd3a5c1b91394d5d328142d7cd Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Mon, 16 Nov 2026 03:31:20 -0500 Subject: [PATCH 5/5] D3D12 kernels: the self-check bypass only into test-only outputs; provenance per entry, refused by build.py and the packager - build_dx12.py: WG_DX12_SMX_DIGEST=0 builds past a stale softmax self-check reference only with the new --test-only flag (else refused). The manifest says "test_only" (set by --test-only, any --define, or a bypass; a partial build keeps it, a full normal build clears it); bypassed entries say "smx_bypass". Every entry records "source_digest" (source_digest(): declaration, source file and #include closure, extra defines) and "defines". - clean/runner/build.py: before embedding the D3D12 kernels (check_dx12_kernels / dx12_manifest_problems) refuses a test-only manifest and any bypassed, define-built, unprovenanced, undeclared or stale entry (its source digest recomputed from this tree); the build manifest records "dx12_kernels" (manifest SHA-256, entries, provenance "checked"; null when nothing is embedded). - clean/tests/check_no_dev_diagnostics.py: the packager accepts a runtime library only when its build manifest records dx12_kernels (null or checked); a loose D3D12 kernel manifest that is test-only or holds a bypassed, define-built or unprovenanced entry fails the payload and the package check. - Tests (CPU): test_dev_diagnostics_off.py Dx12KernelProvenance (a real build_dx12 run with the compiler stubbed: the bypass without --test-only refused; a bypass build followed by an ordinary runtime build refused before compiling; define builds test-only; stale / unprovenanced / undeclared entries refused; a normal build passes) and packager cases (unchecked or missing dx12_kernels, test-only loose manifests refused; controls pass); test_k_smx_selfcheck_digest.py: the bypass needs test_only. The installer suite's stand-in manifest records dx12_kernels null. Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/build_dx12.py | 105 ++++++++++-- clean/installer/tests/run_installer_tests.py | 2 +- clean/k/dx12/README.md | 7 +- clean/k/dx12/smx_selfcheck_digest.py | 4 +- clean/k/tests/test_k_smx_selfcheck_digest.py | 10 +- clean/runner/build.py | 63 ++++++- clean/tests/check_no_dev_diagnostics.py | 37 ++++- clean/tests/test_dev_diagnostics_off.py | 164 ++++++++++++++++++- 8 files changed, 367 insertions(+), 25 deletions(-) diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index b7dfa8e4..3781dcd4 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -50,6 +50,15 @@ instructions whose mnemonic matches each of its patterns must equal the listed count (Model K's softmax weights: one v_fma_mixlo/hi_f16 per weight, the driver's fold that makes the mix form exact; none in the __rodd ports). A different count fails the build. WG_DX12_ISA_CHECK=0 skips it. + +Provenance: every manifest entry records "source_digest" (source_digest(): its declaration, the text of its source file and +#include closure, and the build's extra defines) and "defines" (the extra defines, {} for a normal build). A build with +--define, or with --test-only, writes a TEST-ONLY manifest ("test_only": true; a partial build keeps the flag, only a full +build without them clears it). Model K's softmax self-check reference must belong to the sources of a self-check kernel +built (smx_digest_check); WG_DX12_SMX_DIGEST=0 lets a negative-control build through only together with --test-only, and +marks those entries "smx_bypass". clean/runner/build.py never embeds a test-only manifest, a bypassed, define-built or +unprovenanced entry, or one whose source digest differs from this tree's (dx12_manifest_problems); the payload packager +refuses a library whose build manifest does not record that check (clean/tests/check_no_dev_diagnostics.py). """ import argparse import concurrent.futures @@ -376,20 +385,79 @@ def rga_mnemonics(dxil): return None -def smx_digest_check(names, defines): +def smx_digest_check(names, defines, test_only=False): """Model K's softmax self-check reference (clean/k/runtime/k_smx_selfcheck.h) must belong to the sources and defines of this build when it builds one of the self-check kernels (clean/k/dx12/smx_selfcheck_digest.py); a stale header would let the runtime compare its stage shaders with values from other sources. WG_DX12_SMX_DIGEST=0 lets a negative- - control build through (test variants with --define), never a shipped build.""" + control build through only into a test-only output (test_only: --test-only); returns the self-check kernels built + past a stale header that way (their entries say "smx_bypass"), else an empty set.""" root = os.path.dirname(os.path.dirname(HERE)) if root not in sys.path: sys.path.insert(0, root) from clean.k.dx12 import smx_selfcheck_digest as D - if not set(names) & set(D.KERNELS) or os.environ.get("WG_DX12_SMX_DIGEST", "1") == "0": - return - why = D.stale(defines=defines) - if why: + checked = set(names) & set(D.KERNELS) + why = D.stale(defines=defines) if checked else None + if not why: + return set() + if os.environ.get("WG_DX12_SMX_DIGEST", "1") != "0": raise SystemExit("softmax self-check reference: " + why) + if not test_only: + raise SystemExit("softmax self-check reference: WG_DX12_SMX_DIGEST=0 builds past a stale header only into a " + "test-only output: pass --test-only (the manifest then says test_only, and clean/runner/build.py " + "and the payload packager refuse it)") + return checked + + +_INCLUDE = re.compile(r'^\s*#\s*include\s+"([^"]+)"', re.M) + + +def source_digest(src, decl, defines, gen_dir=None, _cache=None): + """SHA-256 (hex) of what one shader is compiled from: its declaration (kernel_decls), the text of its source file and + of its #include closure (looked up as DXC does: the including file's folder, clean/dx12/hlsl, and for a Model K + source clean/k/dx12/hlsl and the build's generated folder gen_dir), and the extra defines ({name: value}). Line ends + are normalised; paths enter relative to the repository (a generated include as gen/), so the digest does not + depend on where the clone or the build folder is.""" + root = os.path.dirname(os.path.dirname(HERE)) + k_source = os.path.normcase(os.path.abspath(src)).startswith(os.path.normcase(os.path.abspath(K_KERNEL_DIR))) + dirs = [HLSL_DIR] + ([K_HLSL_DIR] + ([gen_dir] if gen_dir else []) if k_source else []) + cache = {} if _cache is None else _cache + + def text(path): + if path not in cache: + with open(path, "rb") as f: + cache[path] = f.read().replace(b"\r\n", b"\n") + return cache[path] + + def label(path): + if gen_dir and os.path.normcase(os.path.abspath(path)).startswith(os.path.normcase(os.path.abspath(gen_dir))): + return "gen/" + os.path.relpath(path, gen_dir).replace(os.sep, "/") + return os.path.relpath(path, root).replace(os.sep, "/") + + seen, order, missing = set(), [], set() + + def walk(path): + key = os.path.normcase(os.path.abspath(path)) + if key in seen: + return + seen.add(key) + order.append(path) + for inc in _INCLUDE.findall(text(path).decode("utf-8", "replace")): + for d in [os.path.dirname(path)] + dirs: + cand = os.path.normpath(os.path.join(d, inc)) + if os.path.isfile(cand): + walk(cand) + break + else: + missing.add(inc) + + walk(src) + h = hashlib.sha256() + h.update(b"decl\n" + json.dumps(decl, sort_keys=True).encode() + b"\n") + for path in sorted(order, key=label): + h.update(b"file " + label(path).encode() + b"\n" + text(path) + b"\n") + h.update(b"missing " + ",".join(sorted(missing)).encode() + b"\n") + h.update(b"defines\n" + "\n".join("%s=%s" % kv for kv in sorted(defines.items())).encode() + b"\n") + return h.hexdigest() def isa_check(built): @@ -510,6 +578,9 @@ def main(argv=None): ap.add_argument("--define", action="append", default=[], metavar="NAME=VALUE", help="an extra preprocessor define for every kernel built (test variants: negative-control mutants, " "test counters; never for a shipped build)") + ap.add_argument("--test-only", action="store_true", + help="mark the output test-only (the manifest says test_only: clean/runner/build.py never embeds it and " + "the payload packager refuses it); required for WG_DX12_SMX_DIGEST=0; implied by --define") ap.add_argument("--list", action="store_true", help="print the kernels a build with these options compiles (name and code object, JSON) and exit: " "the cheap check before a build (kernel_sets.json, tests/test_kernel_sets.py)") @@ -530,9 +601,12 @@ def main(argv=None): only = set(a.only.split(",")) if a.only else None man_path = os.path.join(dst, MANIFEST) old = {} - if only and os.path.exists(man_path): # partial build: keep the other kernels' entries + old_test_only = False + if only and os.path.exists(man_path): # partial build: keep the other kernels' entries (and a test-only flag) with open(man_path) as f: - old = {(k["name"], k.get("co", "")): k for k in json.load(f)["kernels"]} + old_doc = json.load(f) + old = {(k["name"], k.get("co", "")): k for k in old_doc["kernels"]} + old_test_only = old_doc.get("test_only") is True kernels = dict(old) # (name, co) -> manifest entry built = [] # generated kernels (kernels/fused_*_gen.py: the fused forms of swin_block.hlsl / bneck.hlsl) are regenerated from @@ -566,7 +640,9 @@ def main(argv=None): raise SystemExit(f"kernel {d['name']} ({d['co']}): DXIL file {os.path.basename(out)} already used " "(a second port of a name needs file=)") jobs.append((src, d, out, k_source)) - smx_digest_check({j[1]["name"] for j in jobs}, a.define) + bypassed = smx_digest_check({j[1]["name"] for j in jobs}, a.define, a.test_only) + test_only = bool(a.test_only or extra_defs or bypassed or old_test_only) + digest_cache = {} # the ISA rules' mutants beside every build of hybrid kernels: into a temporary folder of their own, never into the # build's output (a release folder, its manifest and a payload hold no check object); removed after the scan scan_dir = None @@ -592,7 +668,12 @@ def build_one(job): rga_isa(out, os.path.join(dst, "isa"), os.path.basename(out)[:-5]) r = {"name": d["name"], "file": os.path.basename(out), "co": d["co"], "entry": d["entry"], "block": d["block"], "wave": d["wave"], "args": d["args"], "arg_dwords": d["arg_dwords"], - "source": d["source"], "sha256": sha, "rs": d["rs"]} + "source": d["source"], "sha256": sha, "rs": d["rs"], + # provenance (module doc): what it was compiled from, and the extra defines of this build + "source_digest": source_digest(src, d, extra_defs, k_gen if k_source else None, digest_cache), + "defines": dict(sorted(extra_defs.items()))} + if d["name"] in bypassed: + r["smx_bypass"] = True # built past a stale self-check reference (WG_DX12_SMX_DIGEST=0, --test-only) if d["group"] != d["block"]: r["group"] = d["group"] r["gscale"] = d["gscale"] @@ -633,8 +714,8 @@ def build_one(job): shutil.rmtree(scan_dir, ignore_errors=True) with open(man_path, "w") as f: json.dump({"format": FORMAT, "dxc": os.path.basename(os.path.dirname(os.path.dirname(DXC_DIR))), - "kernels": [kernels[k] for k in sorted(kernels)]}, f, indent=1) - print(f"{len(kernels)} kernel(s) in {man_path}") + "test_only": test_only, "kernels": [kernels[k] for k in sorted(kernels)]}, f, indent=1) + print(f"{len(kernels)} kernel(s) in {man_path}" + (" (TEST-ONLY: never embedded or packaged)" if test_only else "")) return 0 diff --git a/clean/installer/tests/run_installer_tests.py b/clean/installer/tests/run_installer_tests.py index 8576309d..ba274ef6 100644 --- a/clean/installer/tests/run_installer_tests.py +++ b/clean/installer/tests/run_installer_tests.py @@ -332,7 +332,7 @@ def stand_in_runtime(folder, data, name="wronggpu_runtime.dll"): data = data if data.endswith(RELEASE_MARKER) else data + RELEASE_MARKER write(os.path.join(folder, name), data) manifest = {"format": 1, "file": name, "sha256": hashlib.sha256(data).hexdigest(), "size": len(data), "dev_diagnostics": False, - "stand_in": True} + "dx12_kernels": None, "stand_in": True} write(os.path.join(folder, os.path.splitext(name)[0] + ".build.json"), json.dumps(manifest, indent=1).encode()) diff --git a/clean/k/dx12/README.md b/clean/k/dx12/README.md index 9f7380ff..07cc0f4e 100644 --- a/clean/k/dx12/README.md +++ b/clean/k/dx12/README.md @@ -91,8 +91,11 @@ The reference is bound to its sources: `k_smx_selfcheck.h` records `kSourceDiges kernels' `@wg-kernel` lines, their sources' include closure, the runtime regions marked `SMX-SELFCHECK-SOURCE` (the cases, the synthetic data, the block parameters, the kernel-name and window-offset rules) and a build's extra defines (`smx_selfcheck_digest.py`). `build_dx12.py` (building a self-check kernel) and `build.py` (the runtime with model K) -refuse a header whose digest differs; `WG_DX12_SMX_DIGEST=0` lets a negative-control DXIL build through, never a shipped -one (`clean/k/tests/test_k_smx_selfcheck_digest.py`). +refuse a header whose digest differs; `WG_DX12_SMX_DIGEST=0` lets a negative-control DXIL build through only with +`--test-only`, into a test-only output (the manifest says `test_only` and the entries `smx_bypass`; +`clean/k/tests/test_k_smx_selfcheck_digest.py`). Every manifest entry records its `source_digest` and extra `defines`; +`build.py` never embeds a test-only, bypassed, define-built, unprovenanced or stale entry, and the payload packager +refuses a library whose build manifest does not record that check (`clean/tests/test_dev_diagnostics_off.py`). The test hook `k_smx_test` (`mix_fail`, `both_fail`, `rodd_unchecked`, `report`) exists only in runtime libraries built with `build.py --dev-diagnostics`; a release build refuses it (`test_k_runtime_d3d12.py`: diff --git a/clean/k/dx12/smx_selfcheck_digest.py b/clean/k/dx12/smx_selfcheck_digest.py index 0b723ee5..00d743af 100644 --- a/clean/k/dx12/smx_selfcheck_digest.py +++ b/clean/k/dx12/smx_selfcheck_digest.py @@ -11,8 +11,8 @@ cases, the synthetic data, the block parameters, the kernel-name rule and the window-offset rule); - the extra preprocessor defines of a build (build_dx12.py --define). gen_k_smx_selfcheck.py writes it into the header (kSourceDigest); build_dx12.py and build.py refuse to build when the -current digest differs from the header's (WG_DX12_SMX_DIGEST=0 lets a negative-control build through, never a shipped -one); clean/k/tests/test_k_smx_selfcheck_digest.py checks the committed header and that a changed byte is caught. +current digest differs from the header's (WG_DX12_SMX_DIGEST=0 lets a negative-control build through only into a +test-only output, build_dx12.py --test-only, which build.py never embeds); clean/k/tests/test_k_smx_selfcheck_digest.py checks the committed header and that a changed byte is caught. """ import hashlib import os diff --git a/clean/k/tests/test_k_smx_selfcheck_digest.py b/clean/k/tests/test_k_smx_selfcheck_digest.py index 92590d16..f92bf6dc 100644 --- a/clean/k/tests/test_k_smx_selfcheck_digest.py +++ b/clean/k/tests/test_k_smx_selfcheck_digest.py @@ -8,7 +8,7 @@ - a copy of the sources with one changed byte in a shader source, a definition line or a runtime region, or with an extra define, gives another digest and is reported stale; - clean/dx12/build_dx12.py refuses to build a self-check kernel with a stale header (WG_DX12_SMX_DIGEST=0 lets a - negative-control build through), and ignores builds without one. + negative-control build through only into a test-only output, --test-only), and ignores builds without one. Run: python -m unittest clean.k.tests.test_k_smx_selfcheck_digest """ @@ -78,7 +78,13 @@ def test_build_refuses_a_stale_header(self): B.smx_digest_check({"k_stage_e2"}, []) # current header, no extra define: builds B.smx_digest_check({"k_out_color"}, ["K_SMX_MUT=1"]) # no self-check kernel: not concerned with mock.patch.dict(os.environ, {"WG_DX12_SMX_DIGEST": "0"}): - B.smx_digest_check({"k_stage_e2"}, ["K_SMX_MUT=1"]) # negative-control build + with self.assertRaises(SystemExit) as cm: # the bypass alone: refused (never into a normal output) + B.smx_digest_check({"k_stage_e2"}, ["K_SMX_MUT=1"]) + self.assertIn("--test-only", str(cm.exception)) + # a negative-control build into a test-only output: through, and the bypassed kernels are reported + self.assertEqual(B.smx_digest_check({"k_stage_e2", "k_out_color"}, ["K_SMX_MUT=1"], test_only=True), + {"k_stage_e2"}) + self.assertEqual(B.smx_digest_check({"k_stage_e2"}, [], test_only=True), set()) # current: no bypass if __name__ == "__main__": diff --git a/clean/runner/build.py b/clean/runner/build.py index 671c20a7..8e27ae2a 100644 --- a/clean/runner/build.py +++ b/clean/runner/build.py @@ -402,7 +402,61 @@ def runtime_command(cxx, out, embedded=None): RR_OBJECTS = None # --with-rr: the RR runtime's object files (clean/rr_f/runtime/tools/build.py /obj) +def dx12_manifest_problems(out_dir, doc): + """Why the D3D12 kernel manifest `doc` (/dx12/manifest.json) must not be built into a runtime library: a + test-only manifest (clean/dx12/build_dx12.py --define / --test-only / the self-check bypass), an entry built past a + stale self-check reference, with extra defines, without provenance (a build older than the provenance fields), not + declared in this tree's sources, or whose source digest differs from this tree's sources (a stale or foreign DXIL). + Empty when every entry is a normal build of this tree.""" + from clean.dx12 import build_dx12 as BD + out = [] + if doc.get("test_only") is not False: + out.append("the manifest is test-only or older than the test-only flag (a build with --define, --test-only or " + "WG_DX12_SMX_DIGEST=0, or before provenance was recorded)") + decls = {} + for src, k_source in BD.source_files(model_k=True): + for d in BD.kernel_decls(src): + decls[(d["name"], d["co"])] = (src, d, k_source) + gen, cache = os.path.join(out_dir, "dx12", "gen"), {} + for e in doc.get("kernels", []): + n = e.get("name") + hit = decls.get((n, e.get("co", ""))) + if e.get("smx_bypass"): + out.append(f"{n}: built past a stale softmax self-check reference (WG_DX12_SMX_DIGEST=0)") + elif e.get("defines"): + out.append(f"{n}: built with extra defines {e['defines']}") + elif "source_digest" not in e or "defines" not in e: + out.append(f"{n}: no provenance (source_digest / defines)") + elif hit is None: + out.append(f"{n}: not declared in this tree's sources") + elif BD.source_digest(hit[0], hit[1], {}, gen if hit[2] else None, cache) != e["source_digest"]: + out.append(f"{n}: built from other sources than this tree's") + return out + + +def check_dx12_kernels(out_dir): + """Before a runtime library embeds /dx12: refuses any problem of dx12_manifest_problems (failing closed). + Returns the record for the build manifest ("dx12_kernels"): None without a D3D12 manifest, else its SHA-256, its + entry count and provenance "checked".""" + man = os.path.join(out_dir, "dx12", "manifest.json") + if not os.path.isfile(man): + return None + with open(man, "rb") as f: + raw = f.read() + doc = json.loads(raw.decode("utf-8")) + bad = dx12_manifest_problems(out_dir, doc) + if bad: + raise SystemExit(f"{man}: a runtime library embeds only a normal D3D12 kernel build of this tree; rebuild them " + "(clean/dx12/build_dx12.py, without --define / --test-only, into this folder) or build test " + "kernels into a folder of their own:\n " + "\n ".join(bad[:12]) + + (f"\n ... ({len(bad)} problems)" if len(bad) > 12 else "")) + return {"manifest_sha256": hashlib.sha256(raw).hexdigest(), "entries": len(doc.get("kernels", [])), + "provenance": "checked"} + + def build_runtime(cxx, out_dir, embed=True): + # the D3D12 kernels a library embeds: a normal build of this tree only (never a test-only or stale one) + dx12 = check_dx12_kernels(out_dir) if embed else None if MODELS_BUILT.get("K"): # model K's softmax self-check reference must belong to these sources (no bypass here) root = os.path.dirname(os.path.dirname(HERE)) if root not in sys.path: @@ -418,13 +472,15 @@ def build_runtime(cxx, out_dir, embed=True): cmd = runtime_command(cxx, os.path.join(out_dir, runtime_library_name()), embedded) run(cmd) if os.path.isfile(os.path.join(out_dir, runtime_library_name())): # (a build that failed raised in run) - write_build_manifest(out_dir, cmd, embedded is not None) + write_build_manifest(out_dir, cmd, embedded is not None, dx12) -def write_build_manifest(out_dir, cmd, embedded): +def write_build_manifest(out_dir, cmd, embedded, dx12=None): """.build.json next to the library: what this build is (R1). The payload packager accepts a runtime library only when its SHA-256 matches a manifest that says dev_diagnostics = false (make_payload.py, failing closed); - any later change of the library (stripping, signing, a copy from another build) breaks the match.""" + any later change of the library (stripping, signing, a copy from another build) breaks the match. dx12_kernels: the + embedded D3D12 kernels' check (check_dx12_kernels), None when the library embeds none; the packager refuses a manifest + without it.""" dll = os.path.join(out_dir, runtime_library_name()) with open(dll, "rb") as f: data = f.read() @@ -438,6 +494,7 @@ def write_build_manifest(out_dir, cmd, embedded): "models_left_out": sorted(n for n in OPTIONAL_MODELS if not MODELS_BUILT[n]), "embedded_files": bool(embedded), "defines": sorted(a for a in cmd if a.startswith("-D")), + "dx12_kernels": dx12 if embedded else None, } path = os.path.splitext(dll)[0] + ".build.json" with open(path, "w", encoding="utf-8", newline="\n") as f: diff --git a/clean/tests/check_no_dev_diagnostics.py b/clean/tests/check_no_dev_diagnostics.py index fec7272f..e3219b76 100644 --- a/clean/tests/check_no_dev_diagnostics.py +++ b/clean/tests/check_no_dev_diagnostics.py @@ -12,7 +12,10 @@ found in it (DEV_ONLY_STRINGS: a developer build whose marker was forged to =0 still carries them); (2) the payload packager also demands the library's BUILD MANIFEST (clean/runner/build.py writes .build.json: the build's configuration and the SHA-256 of the library) and accepts a library only when its SHA-256 matches a manifest that says dev_diagnostics = false -(check_runtime_with_manifest). +and records that the D3D12 kernels it embeds were checked against their sources ("dx12_kernels": null, or provenance +"checked": clean/runner/build.py refuses test-only, bypassed, define-built or stale kernels before embedding them) +(check_runtime_with_manifest). A loose D3D12 kernel manifest (dx12/manifest.json) in a payload or package fails when it is +test-only or holds a bypassed, define-built or unprovenanced entry (dx12_manifest_problem). A path is a runtime library (wronggpu_runtime*.dll), a folder (searched for libraries, zip files and executables), a zip file (every member), or an executable (the launcher embeds the payload's files as they are, so its bytes hold the marker). @@ -108,9 +111,36 @@ def check_runtime_with_manifest(name, data, manifest_bytes): return False, "the library does not match its build manifest (SHA-256): changed after its build, or another build's file" if m.get("dev_diagnostics") is not False: return False, "the build manifest says dev_diagnostics is not false" + if "dx12_kernels" not in m: + return False, ("the build manifest does not record the check of the D3D12 kernels the library embeds (a build " + "older than that check): rebuild it with clean/runner/build.py") + dk = m["dx12_kernels"] + if dk is not None and not (isinstance(dk, dict) and dk.get("provenance") == "checked"): + return False, "the build manifest says the library's D3D12 kernels were not checked against their sources" return True, "dev-diagnostics=0, matches its build manifest" +def dx12_manifest_problem(name, data): + """None, or why a file named manifest.json that is a D3D12 kernel manifest (clean/dx12/build_dx12.py: format, kernels) + must not ship: test-only (or older than the flag), or an entry built past the softmax self-check reference, with + extra defines, or without provenance.""" + if os.path.basename(name).lower() != "manifest.json": + return None + try: + doc = json.loads(data.decode("utf-8")) + except Exception: + return None + if not (isinstance(doc, dict) and isinstance(doc.get("kernels"), list) and "format" in doc): + return None + if doc.get("test_only") is not False: + return "a test-only D3D12 kernel manifest (or one older than the test-only flag)" + for e in doc["kernels"]: + if not isinstance(e, dict) or e.get("smx_bypass") or e.get("defines") or "source_digest" not in e or "defines" not in e: + return "a D3D12 kernel manifest with a bypassed, define-built or unprovenanced entry (%s)" % ( + e.get("name") if isinstance(e, dict) else e) + return None + + def check_artifact(name, data): """(ok, text, evidence) for one file by its name: a runtime library must carry the =0 marker; any other file only must not say =1. evidence: the file says dev-diagnostics=0 (so a package check knows it looked at a runtime).""" @@ -121,6 +151,9 @@ def check_artifact(name, data): return ok, text, ok if PAYLOAD_CHECK_NAMES.search(os.path.basename(name)): return False, "a check-only code object (mutant, probe or HIP twin) in a release payload", False + why = dx12_manifest_problem(name, data) + if why: + return False, why, False if any(v == b"1" for v in found): return False, "carries a developer runtime (dev-diagnostics=1)", False carried = dev_only_strings_in(data) @@ -136,7 +169,7 @@ def artifacts(path): for dirpath, _, names in os.walk(path): for n in sorted(names): low = n.lower() - if is_runtime_name(n) or low.endswith((".zip", ".exe", ".asi")): + if is_runtime_name(n) or low.endswith((".zip", ".exe", ".asi")) or low == "manifest.json": out += artifacts(os.path.join(dirpath, n)) elif PAYLOAD_CHECK_NAMES.search(n): # a check-only object anywhere in the folder fails the check out.append((os.path.join(dirpath, n), n, b"")) diff --git a/clean/tests/test_dev_diagnostics_off.py b/clean/tests/test_dev_diagnostics_off.py index fbfcab56..7dbf2b9a 100644 --- a/clean/tests/test_dev_diagnostics_off.py +++ b/clean/tests/test_dev_diagnostics_off.py @@ -12,6 +12,7 @@ import sys import tempfile import unittest +from unittest import mock import zipfile HERE = os.path.dirname(os.path.abspath(__file__)) @@ -92,11 +93,17 @@ def test_the_library_marker_and_its_check(self): def manifest_for(name, data, dev=False, **over): - m = {"format": 1, "file": name, "sha256": hashlib.sha256(data).hexdigest(), "size": len(data), "dev_diagnostics": dev} + m = {"format": 1, "file": name, "sha256": hashlib.sha256(data).hexdigest(), "size": len(data), "dev_diagnostics": dev, + "dx12_kernels": None} m.update(over) + for k in [k for k, v in m.items() if v is DROP]: + del m[k] return json.dumps(m).encode() +DROP = object() # manifest_for(..., key=DROP): the key left out + + class BuildManifest(unittest.TestCase): def test_build_py_writes_the_manifest_of_what_it_built(self): b = load_build() @@ -174,6 +181,55 @@ def test_the_test_suites_stand_in_only_in_a_testing_build(self): r, _ = self._pack({"wronggpu_runtime.dll": lib}, extra_args=("--testing",)) self.assertNotEqual(r.returncode, 0, "a testing build still needs a manifest") + def test_unchecked_d3d12_kernels_refuse_the_payload(self): + """#254 re-check P1: a library is packaged only when its build manifest records that the D3D12 kernels it embeds + were checked against their sources (clean/runner/build.py check_dx12_kernels); an older manifest or an unchecked + record refuses the payload, and so does a test-only D3D12 kernel manifest among the payload's files.""" + lib = self._release() + checked = {"manifest_sha256": "0" * 64, "entries": 3, "provenance": "checked"} + r, _ = self._pack({"wronggpu_runtime.dll": lib}, {"wronggpu_runtime.dll": manifest_for("wronggpu_runtime.dll", lib, dx12_kernels=checked)}) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) # control: a checked record is packaged + for bad, why in ((DROP, "does not record the check"), ({"provenance": "bypass"}, "not checked"), ("checked", "not checked")): + with self.subTest(dx12_kernels=bad): + r, out = self._pack({"wronggpu_runtime.dll": lib}, {"wronggpu_runtime.dll": manifest_for("wronggpu_runtime.dll", lib, dx12_kernels=bad)}) + self.assertNotEqual(r.returncode, 0) + self.assertIn(why, r.stderr + r.stdout) + self.assertFalse(os.path.isfile(os.path.join(out, "payload_table.h"))) + entry = {"name": "k_stage_e2", "file": "k_stage_e2.dxil", "source_digest": "0" * 64, "defines": {}} + for doc in ({"format": 1, "test_only": True, "kernels": [entry]}, + {"format": 1, "kernels": [entry]}, # older than the flag + {"format": 1, "test_only": False, "kernels": [dict(entry, smx_bypass=True)]}, + {"format": 1, "test_only": False, "kernels": [dict(entry, defines={"K_SMX_MUT4": "1"})]}, + {"format": 1, "test_only": False, "kernels": [{"name": "k_stage_e2", "file": "k_stage_e2.dxil"}]}): + with self.subTest(doc=doc): + r, _ = self._pack({"wronggpu_runtime.dll": lib, "dx12/manifest.json": json.dumps(doc).encode()}, + {"wronggpu_runtime.dll": manifest_for("wronggpu_runtime.dll", lib)}) + self.assertNotEqual(r.returncode, 0) + self.assertIn("D3D12 kernel manifest", r.stderr + r.stdout) + good = {"format": 1, "test_only": False, "kernels": [entry]} + r, _ = self._pack({"wronggpu_runtime.dll": lib, "dx12/manifest.json": json.dumps(good).encode()}, + {"wronggpu_runtime.dll": manifest_for("wronggpu_runtime.dll", lib)}) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) # control: a normal manifest passes + + def test_the_package_check_refuses_a_test_only_d3d12_manifest(self): + bad = {"format": 1, "test_only": True, "kernels": [{"name": "k_stage_e2", "file": "k_stage_e2.dxil", + "source_digest": "0" * 64, "defines": {"K_SMX_MUT4": "1"}, + "smx_bypass": True}]} + with tempfile.TemporaryDirectory() as d: + z = os.path.join(d, "package.zip") + with zipfile.ZipFile(z, "w") as f: + f.writestr("WrongGPU/wronggpu_runtime.dll", b"MZ" + MARK0) + f.writestr("WrongGPU/dx12/manifest.json", json.dumps(bad)) + res = chk.check_path(z) + self.assertTrue(any(not ok and "test-only D3D12 kernel manifest" in text for _, ok, text, _ in res), res) + os.makedirs(os.path.join(d, "folder", "dx12")) + with open(os.path.join(d, "folder", "wronggpu_runtime.dll"), "wb") as f: + f.write(b"MZ" + MARK0) + with open(os.path.join(d, "folder", "dx12", "manifest.json"), "w") as f: + json.dump(bad, f) + res = chk.check_path(os.path.join(d, "folder")) + self.assertTrue(any(not ok for _, ok, _, _ in res), res) + def test_no_manifest_refuses_the_payload(self): r, out = self._pack({"wronggpu_runtime.dll": self._release()}) self.assertNotEqual(r.returncode, 0) @@ -337,5 +393,111 @@ def run(path): self.assertEqual(run(d), 1, "a folder holding any developer package fails") +class Dx12KernelProvenance(unittest.TestCase): + """#254 re-check P1: a negative-control D3D12 build (WG_DX12_SMX_DIGEST=0) goes only into a test-only output, and a + runtime library never embeds a test-only, bypassed, define-built, unprovenanced or stale kernel. build_dx12.main runs + for real with DXC replaced by a writer of stand-in bytes (no compiler, no GPU).""" + + def setUp(self): + sys.path.insert(0, ROOT) + from clean.dx12 import build_dx12 as BD + self.BD = BD + self.b = load_build() + self.env = mock.patch.dict(os.environ, {"WG_DX12_ISA_CHECK": "0", "WG_DX12_VGPR_CHECK": "0"}) + self.env.start() + self.addCleanup(self.env.stop) + + def dx12(self, out, *args, bypass=False, only="k_stage_e2,k_out_color"): + def fake_compile(src, entry, out_dxil, defines=None, **kw): + with open(out_dxil, "wb") as f: + f.write(b"DXBC stand-in " + json.dumps([src, entry, sorted((defines or {}).items())]).encode()) + with mock.patch.object(self.BD, "compile_hlsl", fake_compile), mock.patch.object(self.BD, "build_asm_tool", lambda o: None), \ + mock.patch.dict(os.environ, {"WG_DX12_SMX_DIGEST": "0" if bypass else "1"}): + return self.BD.main(["--out", out, "--only", only, "-j", "1", *args]) + + def manifest(self, out): + with open(os.path.join(out, "dx12", "manifest.json"), encoding="utf-8") as f: + return json.load(f) + + def test_normal_build_is_embedded(self): + with tempfile.TemporaryDirectory() as d: + self.dx12(d) + doc = self.manifest(d) + self.assertIs(doc["test_only"], False) + self.assertTrue(all(len(e["source_digest"]) == 64 and e["defines"] == {} for e in doc["kernels"]), doc) + rec = self.b.check_dx12_kernels(d) # control: a normal build of this tree passes + self.assertEqual((rec["provenance"], rec["entries"]), ("checked", 2)) + + def test_bypass_build_then_runtime_build_is_refused(self): + with tempfile.TemporaryDirectory() as d: + self.dx12(d) + with self.assertRaises(SystemExit) as cm: # the bypass without --test-only: refused, the folder unchanged + self.dx12(d, "--define", "K_SMX_MUT4=1", bypass=True) + self.assertIn("--test-only", str(cm.exception)) + self.assertIs(self.manifest(d)["test_only"], False) + self.dx12(d, "--define", "K_SMX_MUT4=1", "--test-only", bypass=True) # the negative-control build + doc = self.manifest(d) + self.assertIs(doc["test_only"], True) + e2 = next(e for e in doc["kernels"] if e["name"] == "k_stage_e2") + self.assertEqual((e2.get("smx_bypass"), e2["defines"]), (True, {"K_SMX_MUT4": "1"})) + for call in (lambda: self.b.check_dx12_kernels(d), lambda: self.b.build_runtime("no-compiler-needed", d)): + with self.assertRaises(SystemExit) as cm: # the ordinary runtime build after it: refused before compiling + call() + self.assertIn("test-only", str(cm.exception)) + self.assertIn("k_stage_e2: built past a stale softmax self-check reference", str(cm.exception)) + self.dx12(d) # a normal partial build into that folder keeps it test-only (other entries may be test builds) + self.assertIs(self.manifest(d)["test_only"], True) + with self.assertRaises(SystemExit): + self.b.check_dx12_kernels(d) + + def test_define_build_is_test_only(self): + with tempfile.TemporaryDirectory() as d: + self.dx12(d, "--define", "WG_SOME_TEST=1", only="k_out_color") # (a self-check kernel: refused already) + self.assertIs(self.manifest(d)["test_only"], True) + with self.assertRaises(SystemExit) as cm: + self.b.check_dx12_kernels(d) + self.assertIn("built with extra defines", str(cm.exception)) + + def test_stale_or_unprovenanced_entries_are_refused(self): + with tempfile.TemporaryDirectory() as d: + self.dx12(d) + path = os.path.join(d, "dx12", "manifest.json") + base = self.manifest(d) + cases = [("other sources", lambda e: e.update(source_digest="0" * 64)), + ("no provenance", lambda e: e.pop("source_digest")), + ("no provenance", lambda e: e.pop("defines")), + ("not declared", lambda e: e.update(name="k_stage_e2_gone"))] + for why, change in cases: + with self.subTest(why=why): + doc = json.loads(json.dumps(base)) + change(doc["kernels"][0]) + with open(path, "w", encoding="utf-8") as f: + json.dump(doc, f) + with self.assertRaises(SystemExit) as cm: + self.b.check_dx12_kernels(d) + self.assertIn(why, str(cm.exception)) + doc = json.loads(json.dumps(base)) + del doc["test_only"] # a manifest older than the flag + with open(path, "w", encoding="utf-8") as f: + json.dump(doc, f) + with self.assertRaises(SystemExit): + self.b.check_dx12_kernels(d) + + def test_build_manifest_records_the_check(self): + with tempfile.TemporaryDirectory() as d: + dll = os.path.join(d, self.b.runtime_library_name()) + with open(dll, "wb") as f: + f.write(b"MZ library " + MARK0) + rec = {"manifest_sha256": "1" * 64, "entries": 2, "provenance": "checked"} + self.b.write_build_manifest(d, ["clang++"], True, rec) + m = json.load(open(os.path.splitext(dll)[0] + ".build.json", encoding="utf-8")) + self.assertEqual(m["dx12_kernels"], rec) + ok, text = chk.check_runtime_with_manifest(dll, open(dll, "rb").read(), json.dumps(m).encode()) + self.assertTrue(ok, text) + self.b.write_build_manifest(d, ["clang++"], False, rec) # nothing embedded: no kernels to vouch for + m = json.load(open(os.path.splitext(dll)[0] + ".build.json", encoding="utf-8")) + self.assertIsNone(m["dx12_kernels"]) + + if __name__ == "__main__": unittest.main()