From e5b4967b45f545075208a3e83e59972a76b85c76 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sat, 14 Nov 2026 23:51:02 -0500 Subject: [PATCH 1/6] L post recon (D3D12): FP16 products as fma with an opaque -0, staged LDS index without per-tap multiplies, history encode through the conversion fold Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/kernels/l_post.hlsl | 56 +++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 7 deletions(-) diff --git a/clean/dx12/kernels/l_post.hlsl b/clean/dx12/kernels/l_post.hlsl index 93451d89..553737b2 100644 --- a/clean/dx12/kernels/l_post.hlsl +++ b/clean/dx12/kernels/l_post.hlsl @@ -81,6 +81,25 @@ static float g_e, g_ep; // exposures (0 replaced by 1) static float g_rp; // 1 / e_prev (IEEE division, round to nearest even) float opq(float x) { return asfloat(asuint(x) ^ g_z); } half opqh(half x) { return asfloat16((uint16_t)(asuint16(x) ^ (uint16_t)g_z)); } +#ifndef LP_MUT +#define LP_MUT 0 // 1..3: negative controls of the byte checks (never shipped) +#endif +#ifndef LP_ENCFOLD +#define LP_ENCFOLD 1 +#endif +#ifndef LP_FMANZ +#define LP_FMANZ 1 +#endif +// h(a b) as its own rounding that the driver cannot fuse into a following FP16 add: fma(a, b, -0) with an opaque -0 +// (RN16(a b + -0) = RN16(a b) for every non-NaN pair, zeros included); LP_FMANZ 0: the product through the xor guard +half mulh(half a, half b) +{ +#if LP_FMANZ + return mad(a, b, asfloat16((uint16_t)(0x8000u | g_z))); +#else + return opqh(a * b); +#endif +} half h_bits(uint b) { return asfloat16((uint16_t)b); } uint bits_h(half h) { return (uint)asuint16(h); } @@ -216,7 +235,11 @@ LGather gather_zero() float tap_weight(uint t[kTexDw], half uu, half vv, half uv) { const half kA = h_bits(t[4] >> 16), kB = h_bits(t[5]), kC = h_bits(t[5] >> 16); - const half ta = opqh(uu * kA), tb = opqh(vv * kB); +#if LP_MUT == 1 + const half ta = mulh(uu, kB), tb = mulh(vv, kB); // mutant 1: wrong factor +#else + const half ta = mulh(uu, kA), tb = mulh(vv, kB); +#endif const half s = tb + ta; const half eps = mad(uv, kC, s); const float w = exp2((float)opqh(eps)); @@ -464,26 +487,33 @@ WG_KERNEL(l_post_recon) [unroll] for (int jj = 0; jj < kLWin; ++jj) { v0[jj] = dy0 + (half)(float)(jj - 1); v1[jj] = dy1 + (half)(float)(jj - 1); - vv0[jj] = opqh(v0[jj] * v0[jj]); - vv1[jj] = opqh(v1[jj] * v1[jj]); + vv0[jj] = mulh(v0[jj], v0[jj]); + vv1[jj] = mulh(v1[jj], v1[jj]); } // taps column by column (i outer, j inner, each gather in that order as before); per column the eight weights (four // rows, two pixels) are rounded to FP16 by one driver conversion LGather g0 = gather_zero(), g1 = gather_zero(); + // the staged texel's LDS index without per-tap multiplies: one base per lane, the row stride and the column + // step wave-uniform (integer identity: ((b + j - r0) ncol + (ax + i - c0)) kTexDw) + const int tb0 = ((b - 1 - r0) * ncol + (ax - 1 - c0)) * kTexDw, rstride = ncol * kTexDw; [loop] for (int i = -1; i <= kLWin - 2; ++i) { const half u = dx + (half)(float)i; - const half uu = opqh(u * u); + const half uu = mulh(u, u); uint t[kLWin][kTexDw]; v8f wv; [unroll] for (int j = -1; j <= kLWin - 2; ++j) { if (staged) { - const int base = ((b + j - r0) * ncol + (ax + i - c0)) * kTexDw; + const int base = tb0 + (j + 1) * rstride + (i + 1) * kTexDw +#if LP_MUT == 2 + + (j == 2 && i == 2 ? kTexDw : 0) // mutant 2: one tap reads the wrong texel +#endif + ; [unroll] for (int d = 0; d < kTexDw; ++d) t[j + 1][d] = sh_tex[base + d]; } else { load_texel_hw(clampi(b + j, 0, P_rh - 1) * P_rw + clampi(ax + i, 0, P_rw - 1), t[j + 1]); } - wv.v[2 * (j + 1)] = tap_weight(t[j + 1], uu, vv0[j + 1], opqh(u * v0[j + 1])); - wv.v[2 * (j + 1) + 1] = tap_weight(t[j + 1], uu, vv1[j + 1], opqh(u * v1[j + 1])); + wv.v[2 * (j + 1)] = tap_weight(t[j + 1], uu, vv0[j + 1], mulh(u, v0[j + 1])); + wv.v[2 * (j + 1) + 1] = tap_weight(t[j + 1], uu, vv1[j + 1], mulh(u, v1[j + 1])); } const uint4 hw = wg_to_f16_ags(wv); [unroll] for (int jj = 0; jj < kLWin; ++jj) { @@ -512,8 +542,16 @@ WG_KERNEL(l_post_recon) const float k = asfloat(ci == 2 ? 0x3a802008u : 0x3a001002u); cv.v[ci] = opq(hh[ci]); cv.v[3 + ci] = opq(floor(clampf(mad(oo[ci], n, 0.5f), 0.0f, n)) * k); +#else +#if LP_ENCFOLD +#if LP_MUT == 3 + cv.v[ci] = opq(kEncK * enc_log(hh[ci])); // mutant 3: two roundings (FP32 product, then FP16) +#else + cv.v[ci] = mad(enc_log(hh[ci]), kEncK, 0.0f); // folded by the driver: v_fma_mixlo_f16 .., kEncK, 0 (HIP's) +#endif #else cv.v[ci] = mul_odd(kEncK, enc_log(hh[ci])); +#endif cv.v[3 + ci] = opq(clampf(enc_e(oo[ci]), 0.0f, kDisplayMax)); #endif } @@ -524,10 +562,14 @@ WG_KERNEL(l_post_recon) if (active && (!second || has1)) { #if LP_LDR const uint q0 = hc.x & 0xffffu, q1 = hc.x >> 16, q2 = hc.y & 0xffffu; +#else +#if LP_ENCFOLD + const uint q0 = hc.x & 0xffffu, q1 = hc.x >> 16, q2 = hc.y & 0xffffu; // the fold's +0 addend gives +0 #else const uint q0 = mul_zero(kEncK, enc_log(hh[0])) ? 0u : (hc.x & 0xffffu); const uint q1 = mul_zero(kEncK, enc_log(hh[1])) ? 0u : (hc.x >> 16); const uint q2 = mul_zero(kEncK, enc_log(hh[2])) ? 0u : (hc.y & 0xffffu); +#endif #endif wg_st64(hist_out, (uint)(y * P_dw + x) * 8u, uint2(q0 | (q1 << 16), q2)); const int di = y * pitch + x; From 45501b163b4ebf086a467aa017954beff002d4dd Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sat, 14 Nov 2026 23:59:18 -0500 Subject: [PATCH 2/6] L post recon (D3D12): interior fast path of the history sample (span loads, same texels and sums) Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/kernels/l_post.hlsl | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/clean/dx12/kernels/l_post.hlsl b/clean/dx12/kernels/l_post.hlsl index 553737b2..56ca5ab2 100644 --- a/clean/dx12/kernels/l_post.hlsl +++ b/clean/dx12/kernels/l_post.hlsl @@ -82,7 +82,7 @@ static float g_rp; // 1 / e_prev (IEEE division, round to nearest even) float opq(float x) { return asfloat(asuint(x) ^ g_z); } half opqh(half x) { return asfloat16((uint16_t)(asuint16(x) ^ (uint16_t)g_z)); } #ifndef LP_MUT -#define LP_MUT 0 // 1..3: negative controls of the byte checks (never shipped) +#define LP_MUT 0 // 1..4: negative controls of the byte checks (never shipped) #endif #ifndef LP_ENCFOLD #define LP_ENCFOLD 1 @@ -274,6 +274,24 @@ float3 history_cr(float qx, float qy) const float4 wx = cr_weights(tx - fx0), wy = cr_weights(ty - fy0); const int ix = (int)fx0, iy = (int)fy0; const uint pitch = (uint)P_dw * 8u; + // interior fast path (Model M's history_cr_span): every active lane's 4 x 4 footprint inside the image, so the + // clamps are the identity; six 16-byte span loads from one base instead of twelve 8-byte loads, the same texels + // in the same sums (same mad order) + [branch] if (WaveActiveAllTrue(and((uint)(ix - 1) <= (uint)(P_dw - 4), (uint)(iy - 1) <= (uint)(P_dh - 4)))) { + const uint b0 = hist_prev + (uint)(ix - 1) * 8u + (uint)(iy - 1) * pitch; + const uint b1 = b0 + pitch, b2 = b0 + 2u * pitch, b3 = b0 + 3u * pitch; + const uint4 a0 = wg_mem.Load4(b0 + 8u), a10 = wg_mem.Load4(b1), a11 = wg_mem.Load4(b1 + 16u); + const uint4 a20 = wg_mem.Load4(b2), a21 = wg_mem.Load4(b2 + 16u), a3 = wg_mem.Load4(b3 + (LP_MUT == 4 ? 16u : 8u)); // mutant 4: wrong row span + const float3 zi = float3(0.0f, 0.0f, 0.0f); + const float3 q0 = mad(wx.z, h3(a0.zw), mad(wx.y, h3(a0.xy), zi)); + const float3 q1 = mad(wx.w, h3(a11.zw), mad(wx.z, h3(a11.xy), mad(wx.y, h3(a10.zw), mad(wx.x, h3(a10.xy), zi)))); + const float3 q2 = mad(wx.w, h3(a21.zw), mad(wx.z, h3(a21.xy), mad(wx.y, h3(a20.zw), mad(wx.x, h3(a20.xy), zi)))); + const float3 q3 = mad(wx.z, h3(a3.zw), mad(wx.y, h3(a3.xy), zi)); + const float3 acci = mad(wy.w, q3, mad(wy.z, q2, mad(wy.y, q1, mad(wy.x, q0, zi)))); + const float sxi = ((wx.x + wx.y) + wx.z) + wx.w; + const float normi = (wx.y + wx.z) * (wy.x + wy.w) + sxi * (wy.y + wy.z); + return acci * rcp(normi); + } uint ro[4], co[4]; [unroll] for (int k = 0; k < 4; ++k) { co[k] = hist_prev + (uint)clampi(ix - 1 + k, 0, P_dw - 1) * 8u; From cd561529a3a78280c06596ddee5942764b9868da Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sun, 15 Nov 2026 00:51:27 -0500 Subject: [PATCH 3/6] L post recon (D3D12): ml / sl / dp forms, the history encode through the driver's conversion fold with a start-up probe and a no-fold port, an ISA check in the build, mixed-lane frames in the L frame check Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/build_dx12.py | 67 +++++++- clean/dx12/hlsl/l_post_encode.hlsli | 39 +++++ clean/dx12/kernel_sets.json | 6 +- clean/dx12/kernels/l_fold_probe.hlsl | 28 ++++ clean/dx12/kernels/l_post.hlsl | 92 +++++++---- clean/dx12/tests/test_fold_probe.py | 56 +++++++ clean/l/runner/l_d3d12_own_post_check.py | 2 +- clean/l/runner/l_default_post_check.py | 2 +- clean/l/runner/l_frame_check.py | 17 ++- clean/runtime/device_d3d12.cpp | 62 +++++++- clean/runtime/device_dx12.cpp | 3 + clean/runtime/fold_probe.h | 86 +++++++++++ clean/runtime/gen_fold_probe.py | 186 +++++++++++++++++++++++ 13 files changed, 598 insertions(+), 48 deletions(-) create mode 100644 clean/dx12/hlsl/l_post_encode.hlsli create mode 100644 clean/dx12/kernels/l_fold_probe.hlsl create mode 100644 clean/dx12/tests/test_fold_probe.py create mode 100644 clean/runtime/fold_probe.h create mode 100644 clean/runtime/gen_fold_probe.py diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index 0ac9e2aa..161c7b60 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -191,6 +191,16 @@ def kernel_decls(path): if d["rs"] not in ("buffer", "tex", "direct"): raise SystemExit(f"{path}: {name}: unsupported root-signature kind {d['rs']}") d["arg_dwords"] = arg_dwords(d["args"], f"{path}: {name}") + # needs=fold | nofold: a port the device runs only when its start-up probe of the driver's conversion fold + # passed (fold) or failed (nofold); the two ports of one name and code object differ by this key + d["needs"] = kv.get("needs", "") + if d["needs"] not in ("", "fold", "nofold"): + raise SystemExit(f"{path}: {name}: needs={d['needs']} (fold or nofold)") + # isa=,: the port's RGA ISA must satisfy each rule, [+...]>= or == (counts of + # instruction mnemonics), checked after the build (isa_check; WG_DX12_ISA_CHECK=0 skips it) + d["isa"] = kv.get("isa", "") + if d["isa"]: + parse_isa_rules(d["isa"], f"{path}: {name}") decls.append(d) return decls @@ -298,6 +308,45 @@ def rga_isa(dxil, isa_dir, stem): shutil.copyfile(os.path.join(tmp, f), os.path.join(isa_dir, f"{stem}.{kind}")) +def parse_isa_rules(text, where): + """isa=,...: [(mnemonics, op, n)] with op '>=' or '=='.""" + rules = [] + for r in text.split(","): + m = re.fullmatch(r"([a-z0-9_+]+)(>=|==)(\d+)", r) + if not m: + raise SystemExit(f"{where}: isa rule {r!r} ([+]>= or ==)") + rules.append((m.group(1).split("+"), m.group(2), int(m.group(3)))) + return rules + + +def isa_check(built_isa): + """built_isa: [(kernel name, dxil path, rules text)] of this build's ports with isa=. Each port's RGA ISA must + satisfy its rules (e.g. a driver fold the port's exactness depends on: a v_fma_mixlo_f16 count). Fails the build + otherwise; WG_DX12_ISA_CHECK=0 skips it (no RGA).""" + if not built_isa or os.environ.get("WG_DX12_ISA_CHECK", "1") == "0": + return + if not os.path.exists(RGA) and not RGA.endswith(".cmd"): + raise SystemExit(f"ISA check: RGA not found at {RGA} (WG_RGA, or WG_DX12_ISA_CHECK=0 to skip)") + bad = [] + for name, dxil, text in built_isa: + with tempfile.TemporaryDirectory(prefix="wgisa_") as tmp: + rga_isa(dxil, tmp, "k") + path = os.path.join(tmp, "k.isa.txt") + if not os.path.exists(path): + bad.append(f"{name}: RGA gave no ISA") + continue + ops = [l.split("//")[0].split() for l in open(path, encoding="utf-8", errors="replace")] + ops = [o[0] for o in ops if o] + for mn, op, n in parse_isa_rules(text, name): + c = sum(ops.count(x) for x in mn) + ok = c >= n if op == ">=" else c == n + print(f"ISA {name:40s} {'+'.join(mn)} = {c} (rule {op} {n}) {'ok' if ok else 'FAILED'}", flush=True) + if not ok: + bad.append(f"{name} ({os.path.basename(dxil)}): {'+'.join(mn)} = {c}, rule {op} {n}") + if bad: + raise SystemExit("ISA check failed:\n " + "\n ".join(bad)) + + VGPR_LIMITS = os.path.join(HERE, "vgpr_limits.json") @@ -375,9 +424,12 @@ def main(argv=None): old = {} if only and os.path.exists(man_path): # partial build: keep the other kernels' entries with open(man_path) as f: - old = {(k["name"], k.get("co", "")): k for k in json.load(f)["kernels"]} - kernels = dict(old) # (name, co) -> manifest entry + # every port of a rebuilt name is replaced (a name may gain or lose ports, e.g. needs=fold / nofold) + old = {(k["name"], k.get("co", ""), k.get("needs", "")): k for k in json.load(f)["kernels"] + if k["name"] not in only} + kernels = dict(old) # (name, co, needs) -> manifest entry built = [] + built_isa = [] # (name, dxil, isa rules) of the ports with isa= # generated kernels (kernels/fused_*_gen.py: the fused forms of swin_block.hlsl / bneck.hlsl) are regenerated from # the live sources first, so that no build compiles a stale copy for fn in sorted(os.listdir(KERNEL_DIR)) if os.path.isdir(KERNEL_DIR) else []: @@ -400,7 +452,7 @@ def main(argv=None): continue if not only and is_frame_gen(d): continue - key = (d["name"], d["co"]) + key = (d["name"], d["co"], d["needs"]) if key in kernels and key not in old: raise SystemExit(f"kernel {d['name']} ({d['co']}) declared twice") kernels[key] = None # placeholder: the declared-twice check sees it before any compile starts @@ -425,6 +477,8 @@ def build_one(job): r = {"name": d["name"], "file": os.path.basename(out), "co": d["co"], "entry": d["entry"], "block": d["block"], "wave": d["wave"], "args": d["args"], "arg_dwords": d["arg_dwords"], "source": d["source"], "sha256": sha, "rs": d["rs"]} + if d["needs"]: + r["needs"] = d["needs"] if d["group"] != d["block"]: r["group"] = d["group"] r["gscale"] = d["gscale"] @@ -440,18 +494,21 @@ def build_one(job): futures = [pool.submit(build_one, j) for j in jobs] try: for (src, d, out, _k), fut in zip(jobs, futures): - kernels[(d["name"], d["co"])] = fut.result() + kernels[(d["name"], d["co"], d["needs"])] = fut.result() built.append((d["name"], out)) + if d["isa"]: + built_isa.append((d["name"], out, d["isa"])) print(f"{d['name']:40s} -> {os.path.basename(out)} block {d['block']} wave {d['wave']} " f"args {d['args']}", flush=True) except BaseException: for fut in futures: fut.cancel() # kernels not started yet; the running ones finish their current child process raise - names = {n for n, _ in kernels} + names = {k[0] for k in kernels} if only and not (only <= names): raise SystemExit(f"not declared in clean/dx12/kernels or clean/k/dx12/kernels: {sorted(only - names)}") vgpr_check(built) + isa_check(built_isa) with open(man_path, "w") as f: json.dump({"format": FORMAT, "dxc": os.path.basename(os.path.dirname(os.path.dirname(DXC_DIR))), "kernels": [kernels[k] for k in sorted(kernels)]}, f, indent=1) diff --git a/clean/dx12/hlsl/l_post_encode.hlsli b/clean/dx12/hlsl/l_post_encode.hlsli new file mode 100644 index 00000000..5974d3de --- /dev/null +++ b/clean/dx12/hlsl/l_post_encode.hlsli @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: MIT +// Model L post: the history texel's FP16 encode through the driver's conversion fold, shared by +// clean/dx12/kernels/l_post.hlsl (l_post_recon_p12) and its start-up probe clean/dx12/kernels/l_fold_probe.hlsl, so +// that both compile the same expression. WrongGPU's own code. Include after wg_platform.hlsli. +// +// q_i = (_Float16)(kEncK x_i), one rounding of the exact product, as HIP's v_fma_mixlo_f16 x, kEncK, 0 (+0 for a zero +// product). Written as mad(x, kEncK, 0) feeding the driver's round-to-nearest-even matrix conversion, which the gfx1201 +// D3D12 compiler folds into exactly that instruction (v_fma_mixlo_f16 / v_fma_mixhi_f16, as Model M's post). Without +// the fold the FP32 product would be rounded twice (a value whose FP32 product lands on an FP16 midpoint moves by one +// FP16 ulp). Guards: build_dx12.py's ISA check on the kernel (isa= in its declaration) and the device's start-up probe +// (l_fold_probe, clean/runtime/fold_probe.h): when the probe fails, the device runs the no-fold port of the kernel. +#pragma once + +#ifndef LP_MUT +#define LP_MUT 0 +#endif +static const float kEncK = asfloat(0x3dddf184u); // 0.36f * 0.30102999566398120f + +// the 8-value conversion of the history codes (x0 .. x2: the FP32 log2 arguments) and the three display values +// d0 .. d2 (already rounded to FP32, passed as opaque values by the caller); z = an opaque zero (mutant 3 only) +uint4 lp_encode_fold(float x0, float x1, float x2, float d0, float d1, float d2, uint z) +{ + v8f cv; +#if LP_MUT == 3 // negative control (never shipped): two roundings, the FP32 product made opaque before the conversion + cv.v[0] = asfloat(asuint(kEncK * x0) ^ z); + cv.v[1] = asfloat(asuint(kEncK * x1) ^ z); + cv.v[2] = asfloat(asuint(kEncK * x2) ^ z); +#else + cv.v[0] = mad(x0, kEncK, 0.0f); + cv.v[1] = mad(x1, kEncK, 0.0f); + cv.v[2] = mad(x2, kEncK, 0.0f); +#endif + cv.v[3] = d0; + cv.v[4] = d1; + cv.v[5] = d2; + cv.v[6] = 0.0f; + cv.v[7] = 0.0f; + return wg_to_f16_ags(cv); +} diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index 492c509f..dc37fbf2 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -202,6 +202,10 @@ "l_direct_color", "-" ], + [ + "l_fold_probe", + "-" + ], [ "l_linear_32_32_f16", "l_ops.co" @@ -425,4 +429,4 @@ "k_io_exact.co" ] ] -} +} diff --git a/clean/dx12/kernels/l_fold_probe.hlsl b/clean/dx12/kernels/l_fold_probe.hlsl new file mode 100644 index 00000000..37c2451b --- /dev/null +++ b/clean/dx12/kernels/l_fold_probe.hlsl @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: MIT +// Start-up probe of the driver's conversion fold used by Model L's l_post_recon_p12 (hlsl/l_post_encode.hlsli): the +// device runs it once (clean/runtime/fold_probe.h holds the inputs and the expected bytes, computed from the exact +// products) and takes the no-fold port of the kernel when any history code differs. WrongGPU's own code. +// Buffer: lane l reads six FP32 values at byte 24 l (x0 x1 x2 = log2 arguments of the history codes, d0 d1 d2 = display +// values) and writes the three converted dwords at byte 768 + 12 l (q0 | q1 << 16, q2 | h(d0) << 16, h(d1) | h(d2) << 16). +// @wg-kernel l_fold_probe co=- block=32 wave=32 args=p +#include "wg_platform.hlsli" +WG_ARGS_BEGIN + uint dst; +WG_ARGS_END +#include "l_post_encode.hlsli" + +WG_KERNEL(l_fold_probe) +{ + WG_SETUP(); + const uint lane = wg_lane_id(); + const uint z = wg_grid_ >> 31; // 0, opaque to the compiler + const uint in0 = lane * 24u; + const float x0 = wg_ldf(dst, in0), x1 = wg_ldf(dst, in0 + 4u), x2 = wg_ldf(dst, in0 + 8u); + const float d0 = asfloat(asuint(wg_ldf(dst, in0 + 12u)) ^ z), d1 = asfloat(asuint(wg_ldf(dst, in0 + 16u)) ^ z); + const float d2 = asfloat(asuint(wg_ldf(dst, in0 + 20u)) ^ z); + const uint4 hc = lp_encode_fold(x0, x1, x2, d0, d1, d2, z); + const uint o = 768u + lane * 12u; + wg_st32(dst, o, hc.x); + wg_st32(dst, o + 4u, hc.y); + wg_st32(dst, o + 8u, hc.z); +} diff --git a/clean/dx12/kernels/l_post.hlsl b/clean/dx12/kernels/l_post.hlsl index 56ca5ab2..807aaec7 100644 --- a/clean/dx12/kernels/l_post.hlsl +++ b/clean/dx12/kernels/l_post.hlsl @@ -31,7 +31,8 @@ // gives 0), the reduced codes rint(clamp(mean, 0, 1) 1023). // // @wg-kernel l_post_recon co=l_post.co block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p -// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1 +// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_ENCFOLD=1 needs=fold isa=v_fma_mixlo_f16+v_fma_mixhi_f16>=6,s_round_mode==0 +// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 file=l_post_recon_p12__nofold block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1 needs=nofold // @wg-kernel l_post_recon_ldr co=l_post.co entry=l_post_recon_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_LDR=1 // @wg-kernel l_post_recon_p12_ldr co=l_post.co entry=l_post_recon_p12_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_LDR=1 #include "wg_platform.hlsli" @@ -49,6 +50,7 @@ WG_ARGS_BEGIN uint expo; WG_ARGS_END +#include "l_post_encode.hlsli" // kEncK and the history encode through the driver's conversion fold #define kThreads 256 #define kTileX 32 #define kTilePairs 8 @@ -56,7 +58,6 @@ WG_ARGS_END #define kLWin 4 #define kTexDw 6 // staged texel: f0 f1 | f2 f3 | f4 f5 | f6 f7 | f8 kA | kB kC (FP16 bits) -static const float kEncK = asfloat(0x3dddf184u); // 0.36f * 0.30102999566398120f static const float kDecK = asfloat(0x4113a428u); // 3.32192809488736235f / 0.36f static const float kDecLn = asfloat(0x40ccac9au); // 2.30258509299404568f / 0.36f static const float kDecPolyMax = asfloat(0x3d60233du); // 0.35f / kDecLn @@ -81,11 +82,15 @@ static float g_e, g_ep; // exposures (0 replaced by 1) static float g_rp; // 1 / e_prev (IEEE division, round to nearest even) float opq(float x) { return asfloat(asuint(x) ^ g_z); } half opqh(half x) { return asfloat16((uint16_t)(asuint16(x) ^ (uint16_t)g_z)); } +float mul_nz(float a, float b) { return mad(a, b, asfloat(g_z | 0x80000000u)); } #ifndef LP_MUT -#define LP_MUT 0 // 1..4: negative controls of the byte checks (never shipped) +#define LP_MUT 0 // 1..7: negative controls of the byte checks (never shipped) +#endif +#ifndef LP_DP +#define LP_DP 1 #endif #ifndef LP_ENCFOLD -#define LP_ENCFOLD 1 +#define LP_ENCFOLD 0 // 1 (l_post_recon_p12's fold port): the history encode through the driver's conversion fold #endif #ifndef LP_FMANZ #define LP_FMANZ 1 @@ -128,8 +133,20 @@ float dec_m1_poly(float qc) float l_dec(float q) { const float qc = clampf(q, k1em4, 3.0f); +#if LP_DP + // Model M's wave-level skip (post.hlsl post_core_s): the exponential for every lane, the polynomial only when some + // active lane of the wave is below kDecPolyMax; per lane the same select as below, so the same value + float v = exp2(qc * kDecK) - 1.0f; + const bool sm = qc < kDecPolyMax; +#if LP_MUT == 6 + [branch] if (WaveActiveAllTrue(sm)) v = sm ? dec_m1_poly(qc) : v; // mutant 6: all lanes instead of any lane +#else + [branch] if (WaveActiveAnyTrue(sm)) v = sm ? dec_m1_poly(qc) : v; +#endif +#else const float v = qc < kDecPolyMax ? dec_m1_poly(qc) : exp2(qc * kDecK) - 1.0f; - return isnan(q) ? q : opq(v) * k0p01; +#endif + return isnan(q) ? q : mul_nz(v, k0p01); } // (_Float16)((float)k / 255.0f) for the code k = 0 ... 255 (an IEEE FP32 division, then round to nearest even: HIP's @@ -318,15 +335,22 @@ void l_post_core(LGather g, int x, int y, int fx, int fy, float mx[3], float my[ float fbar[9]; [unroll] for (int k = 0; k < 9; ++k) fbar[k] = inv * (float)opqh(g.F[k]); const float sig0 = fbar[3], sig1 = fbar[4], sig2 = fbar[5]; - int first = 0; - float best = sig0; - if (sig1 > best) { first = 1; best = sig1; } - if (sig2 > best) first = 2; - const int o1 = first == 0 ? 1 : 0, o2 = first == 2 ? 1 : 2; - const float so1 = o1 == 1 ? sig1 : sig0, so2 = o2 == 1 ? sig1 : sig2; - const int second = so2 > so1 ? o2 : o1; - const float s1 = first == 0 ? sig0 : (first == 1 ? sig1 : sig2); - const float s2 = second == 0 ? sig0 : (second == 1 ? sig1 : sig2); + const bool sb1 = sig1 > sig0; + const float best = sb1 ? sig1 : sig0; + const bool sb2 = sig2 > best; + const float s1 = sb2 ? sig2 : best; + const float m1x = sb2 ? mx[2] : (sb1 ? mx[1] : mx[0]), m1y = sb2 ? my[2] : (sb1 ? my[1] : my[0]); + const bool sf0 = !or(sb1, sb2); + const float so1 = sf0 ? sig1 : sig0, so2 = sb2 ? sig1 : sig2; + const float mo1x = sf0 ? mx[1] : mx[0], mo1y = sf0 ? my[1] : my[0]; +#if LP_MUT == 7 + const float mo2x = sb2 ? mx[2] : mx[1], mo2y = sb2 ? my[1] : my[2]; // mutant 7: wrong candidate motion +#else + const float mo2x = sb2 ? mx[1] : mx[2], mo2y = sb2 ? my[1] : my[2]; +#endif + const bool scc = so2 > so1; + const float s2 = scc ? so2 : so1; + const float m2x = scc ? mo2x : mo1x, m2y = scc ? mo2y : mo1y; const float q1 = clampf(k1p1 * s1 - k0p05, 0.0f, 1.0f); const float q2 = clampf(k1p1 * s2 - k0p05, 0.0f, 1.0f); const float rden = rcp(max(q1 + q2, k1em10)); @@ -343,10 +367,13 @@ void l_post_core(LGather g, int x, int y, int fx, int fy, float mx[3], float my[ [unroll] for (int n = 0; n < 2; ++n) { L[n][0] = 0.0f; L[n][1] = 0.0f; L[n][2] = 0.0f; if (n < ncand) { - const int k = n == 0 ? first : second; - const float qx = X + (k == 0 ? mx[0] : (k == 1 ? mx[1] : mx[2])); - const float qy = Y + (k == 0 ? my[0] : (k == 1 ? my[1] : my[2])); - const bool valid = P_reset == 0 && qx >= 0.0f && qx <= (float)P_dw && qy >= 0.0f && qy <= (float)P_dh; + const float qx = X + (n == 0 ? m1x : m2x); + const float qy = Y + (n == 0 ? m1y : m2y); +#if LP_MUT == 5 + const bool valid = and(and(and(and(P_reset == 0, qx >= 0.0f), qx <= (float)P_dh), qy >= 0.0f), qy <= (float)P_dh); // mutant 5 +#else + const bool valid = and(and(and(and(P_reset == 0, qx >= 0.0f), qx <= (float)P_dw), qy >= 0.0f), qy <= (float)P_dh); +#endif if (valid) { const float3 h = history_cr(qx, qy); #if LP_LDR @@ -370,9 +397,9 @@ void l_post_core(LGather g, int x, int y, int fx, int fy, float mx[3], float my[ #if LP_LDR Hl[i] = opq(comb); #else - Hl[i] = opq(comb) * g_e; + Hl[i] = mul_nz(comb, g_e); #endif - hl_inf = hl_inf || abs(Hl[i]) == asfloat(0x7f800000u); + hl_inf = or(hl_inf, abs(Hl[i]) == asfloat(0x7f800000u)); #if LP_LDR Cl[i] = fbar[i]; #else @@ -387,10 +414,10 @@ void l_post_core(LGather g, int x, int y, int fx, int fy, float mx[3], float my[ oh[i] = mad(Cl[i] - Hl[i], bo2, Hl[i]); } const float sh = hh[0] + hh[1] + hh[2], so = oh[0] + oh[1] + oh[2]; - if (isnan(sh) || hl_inf) { + if (or(isnan(sh), hl_inf)) { [unroll] for (int i = 0; i < 3; ++i) hh[i] = Cl[i]; } - if (isnan(so) || hl_inf) { + if (or(isnan(so), hl_inf)) { [unroll] for (int i = 0; i < 3; ++i) oh[i] = Cl[i]; } } @@ -482,7 +509,7 @@ WG_KERNEL(l_post_recon) } GroupMemoryBarrierWithGroupSync(); - const bool active = x0 + (lane & 31) < P_dw && k0 + (lane >> 5) < npairs; + const bool active = and(x0 + (lane & 31) < P_dw, k0 + (lane >> 5) < npairs); const int x = mini(x0 + (lane & 31), P_dw - 1); const int y0 = 2 * mini(k0 + (lane >> 5), npairs - 1); const bool has1 = y0 + 1 < P_dh; @@ -551,6 +578,14 @@ WG_KERNEL(l_post_recon) if (second) g = g1; float hh[3], oo[3]; finish_pixel(g, x, y0 + row, px, second ? py1 : py0, s, hh, oo); +#if !LP_LDR && LP_ENCFOLD + float xe[3], de[3]; + [unroll] for (int ce = 0; ce < 3; ++ce) { + xe[ce] = enc_log(hh[ce]); + de[ce] = opq(clampf(enc_e(oo[ce]), 0.0f, kDisplayMax)); + } + const uint4 hc = lp_encode_fold(xe[0], xe[1], xe[2], de[0], de[1], de[2], g_z); +#else v8f cv; [unroll] for (int ci = 0; ci < 3; ++ci) { #if LP_LDR @@ -560,24 +595,17 @@ WG_KERNEL(l_post_recon) const float k = asfloat(ci == 2 ? 0x3a802008u : 0x3a001002u); cv.v[ci] = opq(hh[ci]); cv.v[3 + ci] = opq(floor(clampf(mad(oo[ci], n, 0.5f), 0.0f, n)) * k); -#else -#if LP_ENCFOLD -#if LP_MUT == 3 - cv.v[ci] = opq(kEncK * enc_log(hh[ci])); // mutant 3: two roundings (FP32 product, then FP16) -#else - cv.v[ci] = mad(enc_log(hh[ci]), kEncK, 0.0f); // folded by the driver: v_fma_mixlo_f16 .., kEncK, 0 (HIP's) -#endif #else cv.v[ci] = mul_odd(kEncK, enc_log(hh[ci])); -#endif cv.v[3 + ci] = opq(clampf(enc_e(oo[ci]), 0.0f, kDisplayMax)); #endif } cv.v[6] = 0.0f; cv.v[7] = 0.0f; const uint4 hc = wg_to_f16_ags(cv); +#endif const int y = y0 + row; - if (active && (!second || has1)) { + if (and(active, or(!second, has1))) { #if LP_LDR const uint q0 = hc.x & 0xffffu, q1 = hc.x >> 16, q2 = hc.y & 0xffffu; #else diff --git a/clean/dx12/tests/test_fold_probe.py b/clean/dx12/tests/test_fold_probe.py new file mode 100644 index 00000000..6d52d86b --- /dev/null +++ b/clean/dx12/tests/test_fold_probe.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: MIT +"""CPU checks of the conversion-fold guard: clean/runtime/fold_probe.h is what clean/runtime/gen_fold_probe.py writes +(inputs and expected bytes from exact rationals), and build_dx12.py's needs= / isa= declarations parse (and refuse bad +values). The GPU side (the probe kernel against these bytes, its two-roundings mutant) is a GPU-slot check.""" +import os +import subprocess +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.dx12 import build_dx12 as B # noqa: E402 + + +class FoldProbeTest(unittest.TestCase): + def test_header_is_generated(self): + with tempfile.TemporaryDirectory() as tmp: + out = os.path.join(tmp, "fold_probe.h") + subprocess.run([sys.executable, os.path.join(ROOT, "clean", "runtime", "gen_fold_probe.py"), out], check=True, + capture_output=True) + with open(out, encoding="utf-8") as f: + want = f.read() + with open(os.path.join(ROOT, "clean", "runtime", "fold_probe.h"), encoding="utf-8") as f: + self.assertEqual(f.read(), want) + + def test_isa_rules(self): + self.assertEqual(B.parse_isa_rules("v_fma_mixlo_f16+v_fma_mixhi_f16>=6,s_round_mode==0", "t"), + [(["v_fma_mixlo_f16", "v_fma_mixhi_f16"], ">=", 6), (["s_round_mode"], "==", 0)]) + with self.assertRaises(SystemExit): + B.parse_isa_rules("v_fma_mixlo_f16>6", "t") + + def test_post_ports(self): + decls = [d for d in B.kernel_decls(os.path.join(ROOT, "clean", "dx12", "kernels", "l_post.hlsl")) + if d["name"] == "l_post_recon_p12"] + self.assertEqual(sorted(d["needs"] for d in decls), ["fold", "nofold"]) + fold = [d for d in decls if d["needs"] == "fold"][0] + self.assertEqual(fold["defines"].get("LP_ENCFOLD"), "1") + self.assertTrue(fold["isa"]) + nofold = [d for d in decls if d["needs"] == "nofold"][0] + self.assertNotIn("LP_ENCFOLD", nofold["defines"]) + self.assertEqual(nofold["file"], "l_post_recon_p12__nofold") + + def test_bad_needs(self): + with tempfile.TemporaryDirectory() as tmp: + p = os.path.join(tmp, "k.hlsl") + with open(p, "w") as f: + f.write("// @wg-kernel x co=- block=32 wave=32 args=p needs=sometimes\n") + with self.assertRaises(SystemExit): + B.kernel_decls(p) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/l/runner/l_d3d12_own_post_check.py b/clean/l/runner/l_d3d12_own_post_check.py index 6adf3b57..90bfb23d 100644 --- a/clean/l/runner/l_d3d12_own_post_check.py +++ b/clean/l/runner/l_d3d12_own_post_check.py @@ -61,7 +61,7 @@ def run(build, special=True, stages=False): rng = np.random.default_rng(F.SEED) frames = [] for i, ((rw, rh), reset) in enumerate(F.SEQUENCE): - fr = F.frame_inputs(rng, rw, rh, reset, special) + fr = F.frame_inputs(rng, rw, rh, reset, special, index=i) F.fill_sentinels(p, SENTINELS[config], i == 0) out = p.frame(fr, debug=True) frames.append({"inputs": fr, "out": out, "raw": F.raw_outputs(p), "post_last": p.kernels().get("post_last")}) diff --git a/clean/l/runner/l_default_post_check.py b/clean/l/runner/l_default_post_check.py index b63d7cde..be765ea7 100644 --- a/clean/l/runner/l_default_post_check.py +++ b/clean/l/runner/l_default_post_check.py @@ -54,7 +54,7 @@ def run_one(build, config): rng = np.random.default_rng(F.SEED) frames = [] for i, ((rw, rh), reset) in enumerate(F.SEQUENCE): - fr = F.frame_inputs(rng, rw, rh, reset, True) + fr = F.frame_inputs(rng, rw, rh, reset, True, index=i) F.fill_sentinels(p, sentinel, i == 0) out = p.frame(fr, debug=True) frames.append({"inputs": fr, "out": out, "raw": F.raw_outputs(p), "post_last": p.kernels().get("post_last")}) diff --git a/clean/l/runner/l_frame_check.py b/clean/l/runner/l_frame_check.py index b0215c84..5febcf53 100644 --- a/clean/l/runner/l_frame_check.py +++ b/clean/l/runner/l_frame_check.py @@ -92,7 +92,11 @@ # and on the first frame; both parities on every size SEQUENCE = [((96, 64), False), ((96, 64), False), ((96, 64), True), ((96, 64), False), ((96, 64), False), ((80, 48), False), ((80, 48), False), ((80, 48), False), ((77, 45), False), ((77, 45), True), - ((77, 45), False)] + ((77, 45), False), ((77, 45), False), ((77, 45), False)] +# frames whose colour alternates dark and bright blocks of 8 columns (the dark ones scaled by 2^-10; blocks, not single +# columns, so that the 4 x 4 filters keep both levels), so that one wave of the post kernels holds lanes on both sides +# of a per-lane branch (e.g. l_dec's polynomial / exponential split); same size as the frame before (no reset) +MIXED_FRAMES = (11, 12) PRE_EXPOSURE = 1.25 DEFAULT_L_MAP = os.path.join(ROOT, "data", "preset-L.v1.map.json") @@ -188,9 +192,14 @@ def prepare(build, rocm, model=None, map_path=None): # ---- --run (GPU) ---------------------------------------------------------------------------------------------------- -def frame_inputs(rng, rw, rh, reset, special=True): +def frame_inputs(rng, rw, rh, reset, special=True, index=None): from clean.reference import encoder as En x = En.random_inputs(rng, rw=rw, rh=rh, fw=-(-rw // 32) * 32, fh=-(-rh // 32) * 32, special=special) + if index in MIXED_FRAMES: # dark blocks of 8 columns (exact power-of-two scaling of the colour channels) + c = np.array(x["color"], copy=True) + dark = (np.arange(c.shape[1]) // 8) % 2 == 1 + c[:, dark, :3] = c[:, dark, :3] * c.dtype.type(2.0 ** -10) + x["color"] = c return {"color": x["color"], "depth": x["depth"], "motion": x["motion"], "mv_scale": x["mv_scale"], "jitter": x["jitter"], "exposure": 1.0, "pre_exposure": PRE_EXPOSURE, "reset": reset} @@ -348,7 +357,7 @@ def run_d3d12(build, special=True, stages=False): rng = np.random.default_rng(SEED) frames = [] for i, ((rw, rh), reset) in enumerate(SEQUENCE): - fr = frame_inputs(rng, rw, rh, reset, special) + fr = frame_inputs(rng, rw, rh, reset, special, index=i) fill_sentinels(p, SENTINELS[config], i == 0) out = p.frame(fr, debug=True) frames.append({"inputs": fr, "out": out, "raw": raw_outputs(p)}) @@ -553,7 +562,7 @@ def run(build, rocm, special=True): rng = np.random.default_rng(SEED) frames = [] for i, ((rw, rh), reset) in enumerate(SEQUENCE): - fr = frame_inputs(rng, rw, rh, reset, special) + fr = frame_inputs(rng, rw, rh, reset, special, index=i) fill_sentinels(p, SENTINELS[config], i == 0) out = p.frame(fr, debug=True) frames.append({"inputs": fr, "out": out, "raw": raw_outputs(p)}) diff --git a/clean/runtime/device_d3d12.cpp b/clean/runtime/device_d3d12.cpp index 9d738819..9cfdc4b3 100644 --- a/clean/runtime/device_d3d12.cpp +++ b/clean/runtime/device_d3d12.cpp @@ -20,6 +20,7 @@ #include "embedded.h" #include "rdna4.h" #include "wmma_probe.h" +#include "fold_probe.h" #include #include @@ -75,6 +76,9 @@ struct NativeKernel { // workgroups (manifest "group", build_dx12.py group=; 0 = block). The launch keeps the HIP geometry (grid x block). unsigned group = 0, gscale = 1; // gscale: D3D12 workgroups per launched workgroup (manifest "gscale", default block / group) bool ext_doubles = false; // manifest "fp64": "ext": binary64 fma / division (DoublePrecisionFloatShaderOps) + // manifest "needs": "fold" (run only when the start-up probe of the driver's conversion fold passed) or "nofold" + // (only when it failed): two ports of one kernel and code object whose exactness depends on that fold + std::string needs; std::vector args; wgc::Digest digest{}; NativeCom pso; @@ -215,7 +219,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { } if (!create_objects()) return false; if (!profile_init()) return false; - return reserve_events(64) && (!borrowed_ || driver_probe()) && gelu_lut_upload() && gelu_t5_upload(); + return reserve_events(64) && (!borrowed_ || driver_probe()) && fold_probe() && gelu_lut_upload() && gelu_t5_upload(); } // The MLP kernels' exact GELU + FP8 table (hlsl/wg_gelu_lut.hlsli) at its fixed arena offset, reserved in @@ -454,7 +458,8 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { std::string note() const override { return "native D3D12: arena " + std::to_string(arena_bytes_ >> 20) + " MiB; no HIP runtime or fallback; timestamp frequency " + std::to_string(timestamp_frequency_) + " Hz" + - (borrowed_ ? "; borrowed device/queue; exact FP16/FP8 driver-channel probe passed" : ""); + (borrowed_ ? "; borrowed device/queue; exact FP16/FP8 driver-channel probe passed" : "") + + (fold_note_.empty() ? "" : "; " + fold_note_); } bool adapter_luid(uint8_t out[8]) const override { if (!out) return false; @@ -998,7 +1003,9 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { return native_filename(co) && co.size() > 3 && co.substr(co.size() - 3) == ".co"; } // variant_loaded without an error: whether the runtime's loaded code objects select this port. + bool needs_met(const NativeKernel& k) const { return k.needs.empty() || (k.needs == "fold") == fold_ok_; } bool variant_matches(const NativeKernel& k) const { + if (!needs_met(k)) return false; if (k.co == "-") return true; const size_t slash = k.name.find('/'); if (slash == std::string::npos) return untagged_cos_.count(k.co) != 0; @@ -1028,6 +1035,9 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { } ID3D12GraphicsCommandList* cl() const { return external_ ? ext_list_ : list_.get(); } bool variant_loaded(const NativeKernel& k) { + if (!needs_met(k)) + return fail("native D3D12 " + k.name + ": the port for the conversion-fold probe's result (" + + std::string(fold_ok_ ? "fold" : "nofold") + ") is missing"); if (k.co == "-") return true; const size_t slash = k.name.find('/'); if (slash == std::string::npos) { @@ -1278,6 +1288,11 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (!fp->is_string() || fp->text != "ext") return fail("native D3D12 manifest: invalid fp64 for " + k.name); k.ext_doubles = true; } + if (const auto* nd = entry.find("needs")) { + if (!nd->is_string() || (nd->text != "fold" && nd->text != "nofold")) + return fail("native D3D12 manifest: invalid needs for " + k.name); + k.needs = nd->text; + } const auto* signature = entry.find("rs"); if (signature) { if (!signature->is_string() || (signature->text != "buffer" && signature->text != "tex" && signature->text != "direct")) @@ -1325,9 +1340,10 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { } if (words != k.dwords) return fail("native D3D12 manifest: argument DWORD count mismatch for " + k.name); k.file = directory + filename; const std::string key = k.name; - auto same = kernels_.equal_range(key); // one port per (name, code object) + auto same = kernels_.equal_range(key); // one port per (name, code object, needs) for (auto i = same.first; i != same.second; ++i) - if (i->second.co == k.co) return fail("native D3D12 manifest: duplicate kernel " + key + " (" + k.co + ")"); + if (i->second.co == k.co && i->second.needs == k.needs) + return fail("native D3D12 manifest: duplicate kernel " + key + " (" + k.co + ")"); kernels_.emplace(key, std::move(k)); } return true; @@ -1407,6 +1423,42 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (!ok) err_ = error; return ok; } + // Start-up probe of the driver's conversion fold (fold_probe.h, kernel l_fold_probe): only when a port needs it. + // Passed: the "fold" ports run; failed (or no probe kernel): the "nofold" ports. Logged once per process. A failing + // probe is not an error: the device keeps going on the exact no-fold ports. + bool fold_probe() { + bool wanted = false; + for (const auto& kv : kernels_) wanted = wanted || kv.second.needs == "fold"; + fold_ok_ = false; + if (!wanted) return true; + std::string note; + auto probe = kernels_.find("l_fold_probe"); + if (probe == kernels_.end() || probe->second.co != "-" || probe->second.block != 32 || probe->second.dwords != 1 || + probe->second.args.size() != 1 || probe->second.args[0].kind != 'p') { + note = "no l_fold_probe kernel"; + } else { + DevPtr buf = 0; + uint8_t bytes[fold_probe::kBytes]; + fold_probe::fill(bytes); + uint32_t out[96] = {}; + void* args[] = {&buf}; + const std::string before = err_; + bool ok = alloc(&buf, fold_probe::kBytes) && upload(buf, bytes, sizeof bytes) && launch("l_fold_probe", 1, 32, args) && + download(out, buf + fold_probe::kOutOffset, sizeof out); + if (buf) free(buf); + const char* fail_test = std::getenv("WG_TEST_FOLD_PROBE_FAIL"); // tests of the fallback + if (ok && fail_test && fail_test[0] == '1') out[0] ^= 1u; + const int bad = ok ? fold_probe::first_mismatch(out) : -1; + if (!ok) note = "probe run failed (" + err_ + ")"; + else if (bad >= 0) note = "mismatch at half " + std::to_string(bad); + else fold_ok_ = true; + err_ = before; + } + // one line per device: in note(), which the runtime logs once when the network runtime is ready + fold_note_ = std::string("conversion-fold probe ") + + (fold_ok_ ? "passed: the fold ports run" : "failed (" + note + "): the no-fold ports run"); + return true; + } bool create_device() { dxgi_dll_ = LoadLibraryExW(L"dxgi.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32); d3d12_dll_ = LoadLibraryExW(L"d3d12.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32); @@ -1695,6 +1747,8 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { std::string profile_session_, profile_failure_; bool profile_enabled_ = false; bool borrowed_ = false, native_binding_dirty_ = false, inside_pass_ = false; + bool fold_ok_ = false; // fold_probe's result + std::string fold_note_; // fold_probe's result as a note() clause (empty: no port needs the fold) bool direct_bound_ = false, direct_root_bound_ = false; D3D12_COMMAND_LIST_TYPE queue_type_ = D3D12_COMMAND_LIST_TYPE_COMPUTE; uint64_t debug_cursor_ = 0, debug_denied_ = 0; diff --git a/clean/runtime/device_dx12.cpp b/clean/runtime/device_dx12.cpp index d06e9c60..ec064742 100644 --- a/clean/runtime/device_dx12.cpp +++ b/clean/runtime/device_dx12.cpp @@ -524,6 +524,9 @@ class Dx12Device : public Device { } std::set seen; for (const wgc::Json& k : ks->items) { + // a port that needs the driver's conversion fold (manifest "needs": "fold") runs only on the native device after + // its start-up probe (device_d3d12.cpp fold_probe); this device has no probe and takes the exact "nofold" port + if (const wgc::Json* nd = k.find("needs"); nd && nd->is_string() && nd->text == "fold") continue; Kernel x; int64_t block = 0, wave = 0, dwords = 0; const wgc::Json *n = k.find("name"), *fl = k.find("file"), *ar = k.find("args"), *co = k.find("co"); diff --git a/clean/runtime/fold_probe.h b/clean/runtime/fold_probe.h new file mode 100644 index 00000000..70f08656 --- /dev/null +++ b/clean/runtime/fold_probe.h @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: MIT +// Generated by clean/runtime/gen_fold_probe.py - do not edit. +// Start-up probe of the driver's conversion fold (clean/dx12/kernels/l_fold_probe.hlsl): inputs (32 lanes x +// 6 FP32 bits) and expected output dwords (32 x 3); nan_mask bit 0 / 1 = the low / high FP16 half must be a +// NaN (any sign and payload: the accepted port exception), the other halves must match bit for bit. +#pragma once +#include +#include + +namespace fold_probe { +constexpr unsigned kInBytes = 768, kOutOffset = 768, kOutBytes = 384, kBytes = 1152; +constexpr uint32_t kInput[192] = { + 0x3de24abbu, 0x3d1f8941u, 0x3db62c2du, 0x3f2bd000u, 0x3f2bd001u, 0x39091000u, + 0x3c4ef8bfu, 0xc00113f4u, 0x3e59edeeu, 0x39091001u, 0x3a9f9000u, 0x3a9f9001u, + 0xbe68340du, 0xbcb24799u, 0xbdd67815u, 0x40391000u, 0x40391001u, 0x470db000u, + 0xbc580e19u, 0xbd14b8fbu, 0x414eaeedu, 0x470db001u, 0x3f7e3000u, 0x3f7e3001u, + 0x3b516c39u, 0xbb2b5be7u, 0x3ddfd741u, 0x44a05000u, 0x44a05001u, 0x4641d000u, + 0xc012ab03u, 0x3e00a539u, 0x3e7b3c36u, 0x4641d001u, 0x44d9f000u, 0x44d9f001u, + 0xbd31fdc6u, 0xba79a633u, 0x3b9c5d3bu, 0x41f03000u, 0x41f03001u, 0x3f1ef000u, + 0xc023b7a8u, 0xc1d224c6u, 0xc0829783u, 0x3f1ef001u, 0x40363000u, 0x40363001u, + 0x399ebe40u, 0x39a5164du, 0xbb7e8d26u, 0x3c7e7000u, 0x3c7e7001u, 0x3ff3f000u, + 0x3d7cad50u, 0x3e21442eu, 0xc048c59bu, 0x3ff3f001u, 0x3e30d000u, 0x3e30d001u, + 0xbfb14539u, 0xbc3a10c1u, 0x3a24def0u, 0x3c367000u, 0x3c367001u, 0x4765f000u, + 0x3bec3d8au, 0x3c5c6168u, 0xbfb3b8b3u, 0x4765f001u, 0x38ad5000u, 0x38ad5001u, + 0x40d730a3u, 0x3db0b195u, 0xc1f4e428u, 0x39f53000u, 0x39f53001u, 0x3b62d000u, + 0xbd45be7du, 0xbf9fd313u, 0x3f6a13e2u, 0x3b62d001u, 0x4708f000u, 0x4708f001u, + 0x40fffe41u, 0x3af87eeau, 0x40d83302u, 0x42e15000u, 0x42e15001u, 0x45ba3000u, + 0xbfb97d1du, 0x4200a539u, 0x409b7fc4u, 0x45ba3001u, 0x3b825000u, 0x3b825001u, + 0xbb4fd635u, 0xbd0a16d9u, 0xc12a0fb5u, 0x43507000u, 0x43507001u, 0x3e0af000u, + 0x3b90cb2du, 0xbff04707u, 0xbed52be4u, 0x3e0af001u, 0x3fd5d000u, 0x3fd5d001u, + 0xbfa6e3aeu, 0xbd5eaff9u, 0xc028e86du, 0x388e5000u, 0x388e5001u, 0x41c35000u, + 0xc0133ea7u, 0x3b67c552u, 0x3bec6273u, 0x41c35001u, 0x44f35000u, 0x44f35001u, + 0x00000000u, 0x80000000u, 0x7fc00000u, 0x42747000u, 0x42747001u, 0x3ad11000u, + 0xc1549a78u, 0x00000000u, 0x3f800000u, 0x3ad11001u, 0x4081b000u, 0x4081b001u, + 0xbf800000u, 0x41a00000u, 0x399d4952u, 0x3c839000u, 0x3c839001u, 0x46f91000u, + 0xb951b717u, 0x358637bdu, 0xb58637bdu, 0x46f91001u, 0x45b45000u, 0x45b45001u, + 0x4102409du, 0x418bcba4u, 0x4154e574u, 0x3b4b1000u, 0x3b4b1001u, 0x40259000u, + 0xc0b2699fu, 0xc0404665u, 0x41854d96u, 0x40259001u, 0x46989000u, 0x46989001u, + 0xc151e910u, 0x416de3e1u, 0x4160a1b7u, 0x4534f000u, 0x4534f001u, 0x00000000u, + 0x403002c4u, 0xc039fbb4u, 0xc0700006u, 0x80000000u, 0x7fc00000u, 0x477fef00u, + 0xc091daeau, 0x3ffba9bbu, 0x40803132u, 0x477ff000u, 0x49742400u, 0x33800000u, + 0x40b5eb30u, 0x41a6c3e9u, 0x415e367eu, 0x33000001u, 0x40a3c481u, 0x40bde4b3u, + 0x4100a6e7u, 0x41a4f87du, 0xc0bd46f5u, 0x3f3b620cu, 0x408a8867u, 0x4081fd5cu, + 0xc0f9c053u, 0x40f6b935u, 0xc13caf44u, 0x40df1019u, 0x4038f79au, 0x40992297u, +}; +constexpr uint32_t kExpect[96] = { + 0x1c522222u, 0x395e20f0u, 0x0848395fu, 0xb2fe159bu, 0x084925e7u, 0x14fd14fcu, + 0x98d5a64au, 0x41c8a1cfu, 0x786e41c9u, 0x9c0795dbu, 0x786e3d99u, 0x3bf23bf2u, + 0x8ca50dadu, 0x65022211u, 0x720e6503u, 0x22f9b3f3u, 0x720f26ceu, 0x66d066d0u, + 0x86c49cd2u, 0x4f82103du, 0x38f84f82u, 0xc1b1b470u, 0x38f8b714u, 0x41b241b2u, + 0x023c0227u, 0x23f48ee5u, 0x3fa023f4u, 0x245f1ed8u, 0x3fa0b571u, 0x31873186u, + 0x950bb0cdu, 0x21b40478u, 0x7b3021b4u, 0x15f91266u, 0x7b30b0dfu, 0x056b056au, + 0x20c939d5u, 0x0faac2a2u, 0x1b160faau, 0xb0549d5cu, 0x1b172e57u, 0x78487848u, + 0x0abb3af0u, 0x570a39dcu, 0x6dd2570bu, 0x42f9b107u, 0x6dd23836u, 0x1c131c12u, + 0x9b7c8da1u, 0x5a84bc9bu, 0x30585a84u, 0xb2820fd8u, 0x3058a9c7u, 0x3eaf3eaeu, + 0x9e09b085u, 0x0472b493u, 0x4e1a0473u, 0x0e48b3fbu, 0x4e1b1267u, 0x679b679au, + 0x00000000u, 0x53a47e00u, 0x168853a4u, 0x0000bdc3u, 0x16892ef0u, 0x440e440eu, + 0x4056aef0u, 0x241c0221u, 0x77c8241du, 0x0002816cu, 0x77c98002u, 0x6da36da2u, + 0x3f933b0fu, 0x1a583dc5u, 0x412c1a59u, 0xb536b8d5u, 0x412d3f39u, 0x74c574c4u, + 0x3e72bdb0u, 0x69a83e16u, 0x000069a8u, 0xb50a34c5u, 0x8000b681u, 0x7bff7e00u, + 0x32d1b7e7u, 0x7c0036f2u, 0x00017c00u, 0x408538eeu, 0x00013e05u, 0x45ef451eu, + 0x40783af9u, 0x39dbb921u, 0x44104454u, 0x3aafbac4u, 0x46f9bd1du, 0x44c941c8u, +}; +constexpr uint8_t kNanMask[96] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +inline void fill(uint8_t* buffer) { + std::memset(buffer, 0, kBytes); + std::memcpy(buffer, kInput, sizeof kInput); +} +inline bool half_ok(uint32_t got, uint32_t want, bool nan) { + if (nan) return (got & 0x7c00u) == 0x7c00u && (got & 0x03ffu) != 0u; + return got == want; +} +// index of the first differing output half (0 .. 191), or -1 when every half matches +inline int first_mismatch(const uint32_t* out) { + for (unsigned i = 0; i < 96; ++i) { + if (!half_ok(out[i] & 0xffffu, kExpect[i] & 0xffffu, (kNanMask[i] & 1u) != 0)) return (int)(2 * i); + if (!half_ok(out[i] >> 16, kExpect[i] >> 16, (kNanMask[i] & 2u) != 0)) return (int)(2 * i + 1); + } + return -1; +} +} // namespace fold_probe diff --git a/clean/runtime/gen_fold_probe.py b/clean/runtime/gen_fold_probe.py new file mode 100644 index 00000000..acf7d6ff --- /dev/null +++ b/clean/runtime/gen_fold_probe.py @@ -0,0 +1,186 @@ +# SPDX-License-Identifier: MIT +"""Writes clean/runtime/fold_probe.h: inputs and expected bytes of the start-up probe of the driver's conversion fold +(clean/dx12/kernels/l_fold_probe.hlsl, hlsl/l_post_encode.hlsli). WrongGPU's own code. + + python clean/runtime/gen_fold_probe.py [out.h] + +Lane l reads x0 x1 x2 (the FP32 log2 arguments of the three history codes) and d0 d1 d2 (display values) and writes +q_i = (_Float16)(kEncK x_i) with ONE rounding of the exact product (+0 for a zero product: the fold's +0 addend) and +h(d_i) (round to nearest even). Expected values are computed from exact rationals (fractions.Fraction), never from +FP arithmetic. The x inputs are adversarial: values whose FP32 product kEncK x rounds exactly onto an FP16 midpoint +while the exact product is not that midpoint (residual of both signs; a second rounding moves them by one FP16 ulp), +FP16-subnormal products, +-0, NaN, the encode's floor log2(1e-4) and ordinary values; the d inputs: FP16 midpoints and +their FP32 neighbours, +-0, NaN, values beyond the FP16 range. +""" +import os +import struct +import sys +from fractions import Fraction + +import numpy as np + +K_BITS = 0x3dddf184 # kEncK + + +def f32(b): + return struct.unpack("= Fraction(65520): + return s | 0x7c00 + # exponent: 2^e <= a < 2^(e+1) + e = a.numerator.bit_length() - a.denominator.bit_length() + if Fraction(2) ** e > a: + e -= 1 + e = max(e, -14) # subnormal range uses the exponent -14 grid + q = a / (Fraction(2) ** (e - 10)) # in units of the ulp + n = q.numerator // q.denominator + r = q - n + if r > Fraction(1, 2) or (r == Fraction(1, 2) and n % 2 == 1): + n += 1 + if n >= 2048: # carried into the next binade + n //= 2 + e += 1 + if e > 15: + return s | 0x7c00 + if n < 1024: # subnormal + return s | n + return s | ((e + 15) << 10) | (n - 1024) + + +def h16_value(b): + """The exact value of FP16 bits b (None for infinity / NaN).""" + if (b & 0x7c00) == 0x7c00: + return None + return Fraction(float(np.frombuffer(struct.pack("> 10) & 31) - 25) + m = v + (ulp / 2 if v >= 0 else -ulp / 2) + x0 = rn32(m / K) + for dx in (0, 1, -1, 2, -2): + x = f32((bits32(x0) + dx) & 0xffffffff) + p = Fraction(x) * K + if rn32(p) == np.float32(float(m)) and p != m and ((p > m) == want_up): + out.append(x) + want_up = not want_up + break + return out + + +def main(): + out = sys.argv[1] if len(sys.argv) > 1 else os.path.join(os.path.dirname(os.path.abspath(__file__)), "fold_probe.h") + K = Fraction(f32(K_BITS)) + xs = midpoint_xs(K, 60, 20261115) + nan = f32(0x7fc00000) + floor = float(np.log2(np.float32(1e-4)).astype(np.float32)) + xs += [0.0, -0.0, nan, floor, 0.0, 1.0, -1.0, 20.0, float(np.float32(3.0e-4)), float(np.float32(-2.0e-4)), + float(np.float32(1e-6)), -float(np.float32(1e-6))] + rng = np.random.default_rng(7) + while len(xs) < 96: + xs.append(float(np.float32(rng.uniform(-13.3, 21.0)))) + ds = [] + for h in rng.integers(0x0400, 0x7bff, 40): + v = h16_value(int(h)) + ulp = Fraction(2) ** ((((int(h) >> 10) & 31)) - 25) + m = v + ulp / 2 + ds.append(float(np.float32(float(m)))) # an exact FP16 midpoint (ties to even) + ds.append(f32(bits32(float(m)) + 1)) # just above + ds += [0.0, -0.0, nan, 65519.0, 65520.0, 1e6, f32(0x33800000), f32(0x33000001)] + while len(ds) < 96: + ds.append(float(np.float32(rng.uniform(0.0, 8.0)))) + xs, ds = xs[:96], ds[:96] + inputs, expect, nanmask = [], [], [] + for lane in range(32): + x = xs[3 * lane:3 * lane + 3] + d = ds[3 * lane:3 * lane + 3] + inputs += [bits32(v) for v in x + d] + q, nm = [], [] + for v in x: + if v != v: + q.append(0x7e00); nm.append(True) + else: + p = Fraction(v) * K + q.append(0 if p == 0 else rn16_bits(p)); nm.append(False) + for v in d: + if v != v: + q.append(0x7e00); nm.append(True) + elif v == 0: # a plain conversion keeps the sign of a zero + q.append(0x8000 if struct.pack("\n#include \n\nnamespace fold_probe {\n") + f.write("constexpr unsigned kInBytes = 768, kOutOffset = 768, kOutBytes = 384, kBytes = 1152;\n") + f.write("constexpr uint32_t kInput[192] = {\n") + for i in range(0, 192, 6): + f.write(" " + ", ".join(f"0x{v:08x}u" for v in inputs[i:i + 6]) + ",\n") + f.write("};\nconstexpr uint32_t kExpect[96] = {\n") + for i in range(0, 96, 6): + f.write(" " + ", ".join(f"0x{v:08x}u" for v in expect[i:i + 6]) + ",\n") + f.write("};\nconstexpr uint8_t kNanMask[96] = {\n") + for i in range(0, 96, 24): + f.write(" " + ", ".join(str(v) for v in nanmask[i:i + 24]) + ",\n") + f.write("};\n") + f.write("""inline void fill(uint8_t* buffer) { + std::memset(buffer, 0, kBytes); + std::memcpy(buffer, kInput, sizeof kInput); +} +inline bool half_ok(uint32_t got, uint32_t want, bool nan) { + if (nan) return (got & 0x7c00u) == 0x7c00u && (got & 0x03ffu) != 0u; + return got == want; +} +// index of the first differing output half (0 .. 191), or -1 when every half matches +inline int first_mismatch(const uint32_t* out) { + for (unsigned i = 0; i < 96; ++i) { + if (!half_ok(out[i] & 0xffffu, kExpect[i] & 0xffffu, (kNanMask[i] & 1u) != 0)) return (int)(2 * i); + if (!half_ok(out[i] >> 16, kExpect[i] >> 16, (kNanMask[i] & 2u) != 0)) return (int)(2 * i + 1); + } + return -1; +} +} // namespace fold_probe +""") + # self-check of rn16_bits against numpy's double -> half conversion (correctly rounded from a double; every value + # here is an exact double) + chk = np.random.default_rng(3) + for v in list(chk.uniform(-70000, 70000, 2000)) + list(chk.uniform(-1e-4, 1e-4, 2000)): + nb = int(np.array([v]).astype(np.float16).view(np.uint16)[0]) + assert rn16_bits(Fraction(float(v))) == nb or (v == 0), (v, nb) + print("wrote", out, "midpoint x:", sum(1 for v in xs[:60]), "lanes 32") + + +if __name__ == "__main__": + main() From 33158b79203bf944d952f54ce1666880f8d608e4 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sun, 15 Nov 2026 00:57:22 -0500 Subject: [PATCH 4/6] kernel_sets.json: LF line ending Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/kernel_sets.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index dc37fbf2..189ecbb2 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -429,4 +429,4 @@ "k_io_exact.co" ] ] -} +} From 2a50c83c455cbbe3109de0585c3f2f307096a7f0 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:41:31 -0500 Subject: [PATCH 5/6] L post recon (review fixes): differential self-test of the real fold / no-fold post ports, history fast path guarded for images under 4 x 4, edge-size check - device_d3d12: after l_fold_probe, the native device runs l_post_recon_p12's fold port and no-fold port on one crafted frame (clean/runtime/fold_selftest.h from gen_fold_selftest.py: FP32-product ties of both sides on each of the six encode sites, zero and FP16-subnormal products, the floor) and compares the raw output bytes; the fold ports run only when both steps pass (fail closed, also for a fold port without a self-test) - build_dx12: isa= terms @ count only the x * literal + 0 sites; the fold port's rule is the six kEncK sites; --define for test variants - l_post.hlsl: the interior predicate requires P_dw >= 4 and P_dh >= 4 before its unsigned tests; LP_MUT 8 / 9 mutants and the LP_HSCOUNT test counter - l_post_edge_check.py: the four post kernels and the no-fold port at 14 sizes (1 x 1 ... 97 x 35) and 12 motion kinds, D3D12 against HIP raw bytes with per-edge bands, the counter, the mutants and the oracle - kernel_replay prints the device note Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/build_dx12.py | 53 +++- clean/dx12/hlsl/l_post_encode.hlsli | 7 +- clean/dx12/kernels/l_post.hlsl | 27 +- clean/dx12/tests/kernel_replay.cpp | 1 + clean/dx12/tests/test_fold_probe.py | 58 +++- clean/dx12/tests/test_post_edge_cases.py | 92 ++++++ clean/l/runner/l_post_edge_check.py | 376 +++++++++++++++++++++++ clean/runtime/device_d3d12.cpp | 110 ++++++- clean/runtime/fold_selftest.h | 118 +++++++ clean/runtime/gen_fold_selftest.py | 255 +++++++++++++++ 10 files changed, 1061 insertions(+), 36 deletions(-) create mode 100644 clean/dx12/tests/test_post_edge_cases.py create mode 100644 clean/l/runner/l_post_edge_check.py create mode 100644 clean/runtime/fold_selftest.h create mode 100644 clean/runtime/gen_fold_selftest.py diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index 161c7b60..2bcfd2f9 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -197,7 +197,8 @@ def kernel_decls(path): if d["needs"] not in ("", "fold", "nofold"): raise SystemExit(f"{path}: {name}: needs={d['needs']} (fold or nofold)") # isa=,: the port's RGA ISA must satisfy each rule, [+...]>= or == (counts of - # instruction mnemonics), checked after the build (isa_check; WG_DX12_ISA_CHECK=0 skips it) + # instruction mnemonics; @: only its x * literal + 0 sites, parse_isa_rules), checked after the + # build (isa_check; WG_DX12_ISA_CHECK=0 skips it) d["isa"] = kv.get("isa", "") if d["isa"]: parse_isa_rules(d["isa"], f"{path}: {name}") @@ -309,16 +310,36 @@ def rga_isa(dxil, isa_dir, stem): def parse_isa_rules(text, where): - """isa=,...: [(mnemonics, op, n)] with op '>=' or '=='.""" + """isa=,...: [(terms, op, n)] with op '>=' or '=='; a term is (mnemonic, literal or None): counts every + instruction of that mnemonic, @<8 hex digits> only those whose literal dword is that value, whose last source + operand is 0 and that carry no op_sel modifier (an FP32 x * literal + 0: the sites of a conversion fold).""" rules = [] for r in text.split(","): - m = re.fullmatch(r"([a-z0-9_+]+)(>=|==)(\d+)", r) + m = re.fullmatch(r"([a-z0-9_]+(?:@[0-9a-f]{8})?(?:\+[a-z0-9_]+(?:@[0-9a-f]{8})?)*)(>=|==)(\d+)", r) if not m: - raise SystemExit(f"{where}: isa rule {r!r} ([+]>= or ==)") - rules.append((m.group(1).split("+"), m.group(2), int(m.group(3)))) + raise SystemExit(f"{where}: isa rule {r!r} ([@][+[@]]>= or ...==)") + terms = [(t.split("@")[0], t.split("@")[1] if "@" in t else None) for t in m.group(1).split("+")] + rules.append((terms, m.group(2), int(m.group(3)))) return rules +def isa_count(lines, mnemonic, literal): + """Instructions of `mnemonic` in RGA ISA lines (with literal: only the x * literal + 0 sites, parse_isa_rules).""" + c = 0 + for line in lines: + code, _, comment = line.partition("//") + f = code.split() + if not f or f[0] != mnemonic: + continue + if literal is not None: + words = comment.split(":")[-1].split() + srcs = [o.strip() for o in " ".join(f[1:]).split(",")] + if not words or words[-1].lower() != literal or srcs[-1] != "0" or "op_sel" in code: + continue + c += 1 + return c + + def isa_check(built_isa): """built_isa: [(kernel name, dxil path, rules text)] of this build's ports with isa=. Each port's RGA ISA must satisfy its rules (e.g. a driver fold the port's exactness depends on: a v_fma_mixlo_f16 count). Fails the build @@ -335,14 +356,14 @@ def isa_check(built_isa): if not os.path.exists(path): bad.append(f"{name}: RGA gave no ISA") continue - ops = [l.split("//")[0].split() for l in open(path, encoding="utf-8", errors="replace")] - ops = [o[0] for o in ops if o] - for mn, op, n in parse_isa_rules(text, name): - c = sum(ops.count(x) for x in mn) + lines = open(path, encoding="utf-8", errors="replace").read().splitlines() + for terms, op, n in parse_isa_rules(text, name): + c = sum(isa_count(lines, mn, lit) for mn, lit in terms) ok = c >= n if op == ">=" else c == n - print(f"ISA {name:40s} {'+'.join(mn)} = {c} (rule {op} {n}) {'ok' if ok else 'FAILED'}", flush=True) + label = "+".join(mn + (f"@{lit}" if lit else "") for mn, lit in terms) + print(f"ISA {name:40s} {label} = {c} (rule {op} {n}) {'ok' if ok else 'FAILED'}", flush=True) if not ok: - bad.append(f"{name} ({os.path.basename(dxil)}): {'+'.join(mn)} = {c}, rule {op} {n}") + bad.append(f"{name} ({os.path.basename(dxil)}): {label} = {c}, rule {op} {n}") if bad: raise SystemExit("ISA check failed:\n " + "\n ".join(bad)) @@ -407,6 +428,9 @@ def main(argv=None): ap.add_argument("--model-k", action="store_true", help="also build Model K's kernels (clean/k/dx12/kernels; off by default: Model M's and Model L's " "kernel sets unchanged)") + ap.add_argument("--define", action="append", default=[], metavar="NAME=VALUE", + help="an extra preprocessor define for every kernel built (test variants: negative-control mutants, " + "test counters; never for a shipped build)") ap.add_argument("--list", action="store_true", help="print the kernels a build with these options compiles (name and code object, JSON) and exit: " "the cheap check before a build (kernel_sets.json, tests/test_kernel_sets.py)") @@ -416,6 +440,11 @@ def main(argv=None): return 0 if not a.out: ap.error("--out is required") + extra_defs = {} + for kv in a.define: + if not re.fullmatch(r"[A-Z][A-Z0-9_]*=-?[0-9A-Za-z_]+", kv): + ap.error(f"--define {kv!r} (NAME=VALUE)") + extra_defs[kv.split("=", 1)[0]] = kv.split("=", 1)[1] dst = os.path.join(a.out, "dx12") os.makedirs(dst, exist_ok=True) asm = build_asm_tool(a.out) @@ -464,7 +493,7 @@ def main(argv=None): def build_one(job): src, d, out, k_source = job - defs = {"WG_BLOCK": d["group"], "WG_WAVE": d["wave"], **d["defines"]} + defs = {"WG_BLOCK": d["group"], "WG_WAVE": d["wave"], **d["defines"], **extra_defs} if d["group"] != d["block"]: defs["WG_LAUNCH_BLOCK"] = d["block"] info = {} diff --git a/clean/dx12/hlsl/l_post_encode.hlsli b/clean/dx12/hlsl/l_post_encode.hlsli index 5974d3de..f23b23e3 100644 --- a/clean/dx12/hlsl/l_post_encode.hlsli +++ b/clean/dx12/hlsl/l_post_encode.hlsli @@ -7,8 +7,11 @@ // product). Written as mad(x, kEncK, 0) feeding the driver's round-to-nearest-even matrix conversion, which the gfx1201 // D3D12 compiler folds into exactly that instruction (v_fma_mixlo_f16 / v_fma_mixhi_f16, as Model M's post). Without // the fold the FP32 product would be rounded twice (a value whose FP32 product lands on an FP16 midpoint moves by one -// FP16 ulp). Guards: build_dx12.py's ISA check on the kernel (isa= in its declaration) and the device's start-up probe -// (l_fold_probe, clean/runtime/fold_probe.h): when the probe fails, the device runs the no-fold port of the kernel. +// FP16 ulp). Guards: build_dx12.py's ISA check on the kernel (isa= in its declaration: the six sites, kEncK as the +// literal with a 0 addend) and two start-up steps of the native device, both of which must pass: the probe +// (l_fold_probe, clean/runtime/fold_probe.h: this expression on adversarial inputs) and the differential self-test of +// the real post shader (its fold port against its no-fold port on the crafted frame of clean/runtime/fold_selftest.h, +// raw bytes). Otherwise, or on any error, the device runs the no-fold port of the kernel. #pragma once #ifndef LP_MUT diff --git a/clean/dx12/kernels/l_post.hlsl b/clean/dx12/kernels/l_post.hlsl index 807aaec7..48abc660 100644 --- a/clean/dx12/kernels/l_post.hlsl +++ b/clean/dx12/kernels/l_post.hlsl @@ -31,7 +31,7 @@ // gives 0), the reduced codes rint(clamp(mean, 0, 1) 1023). // // @wg-kernel l_post_recon co=l_post.co block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p -// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_ENCFOLD=1 needs=fold isa=v_fma_mixlo_f16+v_fma_mixhi_f16>=6,s_round_mode==0 +// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_ENCFOLD=1 needs=fold isa=v_fma_mixlo_f16@3dddf184+v_fma_mixhi_f16@3dddf184==6,s_round_mode==0 // @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 file=l_post_recon_p12__nofold block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1 needs=nofold // @wg-kernel l_post_recon_ldr co=l_post.co entry=l_post_recon_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_LDR=1 // @wg-kernel l_post_recon_p12_ldr co=l_post.co entry=l_post_recon_p12_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_LDR=1 @@ -84,7 +84,7 @@ float opq(float x) { return asfloat(asuint(x) ^ g_z); } half opqh(half x) { return asfloat16((uint16_t)(asuint16(x) ^ (uint16_t)g_z)); } float mul_nz(float a, float b) { return mad(a, b, asfloat(g_z | 0x80000000u)); } #ifndef LP_MUT -#define LP_MUT 0 // 1..7: negative controls of the byte checks (never shipped) +#define LP_MUT 0 // 1..9: negative controls of the byte checks (never shipped) #endif #ifndef LP_DP #define LP_DP 1 @@ -95,6 +95,9 @@ float mul_nz(float a, float b) { return mad(a, b, asfloat(g_z | 0x80000000u)); } #ifndef LP_FMANZ #define LP_FMANZ 1 #endif +#ifndef LP_HSCOUNT +#define LP_HSCOUNT 0 // 1: test builds only (clean/l/runner/l_post_edge_check.py): the history fast path's wave counter +#endif // h(a b) as its own rounding that the driver cannot fuse into a following FP16 add: fma(a, b, -0) with an opaque -0 // (RN16(a b + -0) = RN16(a b) for every non-NaN pair, zeros included); LP_FMANZ 0: the product through the xor guard half mulh(half a, half b) @@ -291,10 +294,22 @@ float3 history_cr(float qx, float qy) const float4 wx = cr_weights(tx - fx0), wy = cr_weights(ty - fy0); const int ix = (int)fx0, iy = (int)fy0; const uint pitch = (uint)P_dw * 8u; - // interior fast path (Model M's history_cr_span): every active lane's 4 x 4 footprint inside the image, so the - // clamps are the identity; six 16-byte span loads from one base instead of twelve 8-byte loads, the same texels - // in the same sums (same mad order) - [branch] if (WaveActiveAllTrue(and((uint)(ix - 1) <= (uint)(P_dw - 4), (uint)(iy - 1) <= (uint)(P_dh - 4)))) { + // interior fast path (Model M's history_cr_span): every lane's 4 x 4 footprint inside the image, so the clamps are + // the identity; six 16-byte span loads from one base instead of twelve 8-byte loads, the same texels in the same + // sums (same mad order). Footprint inside: 1 <= ix <= P_dw - 3 and 1 <= iy <= P_dh - 3, the unsigned tests valid only + // for an image of at least 4 x 4 (below, P_d* - 4 would wrap to a huge bound): smaller images take the clamped path. +#if LP_MUT == 8 // mutant 8: the predicate always false (the fast path never taken; same bytes, found by the counter) + const bool interior = false; +#elif LP_MUT == 9 // mutant 9: without the size guard (the span reads past a history narrower or shorter than 4) + const bool interior = and((uint)(ix - 1) <= (uint)(P_dw - 4), (uint)(iy - 1) <= (uint)(P_dh - 4)); +#else + const bool interior = and(and(P_dw >= 4, P_dh >= 4), + and((uint)(ix - 1) <= (uint)(P_dw - 4), (uint)(iy - 1) <= (uint)(P_dh - 4))); +#endif + [branch] if (WaveActiveAllTrue(interior)) { +#if LP_HSCOUNT // test builds only: count the waves taking the fast path, in the first dword after the full history + if (WaveIsFirstLane()) wg_atomic_add(hist_out, (uint)(P_dw * P_dh) * 8u, 1u); +#endif const uint b0 = hist_prev + (uint)(ix - 1) * 8u + (uint)(iy - 1) * pitch; const uint b1 = b0 + pitch, b2 = b0 + 2u * pitch, b3 = b0 + 3u * pitch; const uint4 a0 = wg_mem.Load4(b0 + 8u), a10 = wg_mem.Load4(b1), a11 = wg_mem.Load4(b1 + 16u); diff --git a/clean/dx12/tests/kernel_replay.cpp b/clean/dx12/tests/kernel_replay.cpp index 96d123f2..97b916da 100644 --- a/clean/dx12/tests/kernel_replay.cpp +++ b/clean/dx12/tests/kernel_replay.cpp @@ -591,6 +591,7 @@ int main(int argc, char** argv) { auto d = make_d3d12_device(argv[1], &err); if (!d) { std::printf("device: %s\n", err.c_str()); return 1; } std::printf("device: %s\n", d->name().c_str()); + if (!d->note().empty()) std::printf("device note: %s\n", d->note().c_str()); // e.g. the conversion-fold guard std::ifstream f(argv[2]); if (!f) { std::printf("cannot read %s\n", argv[2]); return 2; } const std::string base = dir_of(argv[2]); diff --git a/clean/dx12/tests/test_fold_probe.py b/clean/dx12/tests/test_fold_probe.py index 6d52d86b..f6ad8094 100644 --- a/clean/dx12/tests/test_fold_probe.py +++ b/clean/dx12/tests/test_fold_probe.py @@ -1,7 +1,9 @@ # SPDX-License-Identifier: MIT -"""CPU checks of the conversion-fold guard: clean/runtime/fold_probe.h is what clean/runtime/gen_fold_probe.py writes -(inputs and expected bytes from exact rationals), and build_dx12.py's needs= / isa= declarations parse (and refuse bad -values). The GPU side (the probe kernel against these bytes, its two-roundings mutant) is a GPU-slot check.""" +"""CPU checks of the conversion-fold guard: clean/runtime/fold_probe.h and fold_selftest.h are what their generators +write (inputs and expected bytes from exact rationals), the self-test frame holds its value classes on every encode +site, and build_dx12.py's needs= / isa= declarations parse (and refuse bad values), the literal-tied ISA count included. +The GPU side (the probe kernel against these bytes, the self-test of the real post ports, the two-roundings mutant) is +a GPU-slot check.""" import os import subprocess import sys @@ -26,11 +28,57 @@ def test_header_is_generated(self): with open(os.path.join(ROOT, "clean", "runtime", "fold_probe.h"), encoding="utf-8") as f: self.assertEqual(f.read(), want) + def test_selftest_header_is_generated(self): + with tempfile.TemporaryDirectory() as tmp: + out = os.path.join(tmp, "fold_selftest.h") + subprocess.run([sys.executable, os.path.join(ROOT, "clean", "runtime", "gen_fold_selftest.py"), out], + check=True, capture_output=True) + with open(out, encoding="utf-8") as f: + want = f.read() + with open(os.path.join(ROOT, "clean", "runtime", "fold_selftest.h"), encoding="utf-8") as f: + self.assertEqual(f.read(), want) + + def test_selftest_frame_classes(self): + """Every one of the six encode sites (row of the pair, channel) holds FP32-product ties of both sides that a + second rounding gets wrong, the zero product and the floor.""" + sys.path.insert(0, os.path.join(ROOT, "clean", "runtime")) + import gen_fold_selftest as G + colour, _, _ = G.build() + floor = G.Fraction(G.f32(G.bits32(1e-4))) + for site in range(G.SITES): + row, ch = divmod(site, 3) + vals = [colour[3 * (y * G.W + x) + ch] for y in range(row, G.H, 2) for x in range(G.W)] + sides = {"up": 0, "down": 0} + for cb in vals: + r = G.classify(G.f32(cb)) + if r and r[2]: + sides[r[1]] += 1 + self.assertGreaterEqual(min(sides.values()), 40, (site, sides)) + ms = [G.m_of(G.f32(cb)) for cb in vals] + self.assertIn(G.Fraction(1), ms) # x = 0: the zero product + self.assertTrue(any(m < floor for m in ms) and any(floor <= m < 2 * floor for m in ms)) + def test_isa_rules(self): self.assertEqual(B.parse_isa_rules("v_fma_mixlo_f16+v_fma_mixhi_f16>=6,s_round_mode==0", "t"), - [(["v_fma_mixlo_f16", "v_fma_mixhi_f16"], ">=", 6), (["s_round_mode"], "==", 0)]) + [([("v_fma_mixlo_f16", None), ("v_fma_mixhi_f16", None)], ">=", 6), + ([("s_round_mode", None)], "==", 0)]) + self.assertEqual(B.parse_isa_rules("v_fma_mixlo_f16@3dddf184+v_fma_mixhi_f16@3dddf184==6", "t"), + [([("v_fma_mixlo_f16", "3dddf184"), ("v_fma_mixhi_f16", "3dddf184")], "==", 6)]) with self.assertRaises(SystemExit): B.parse_isa_rules("v_fma_mixlo_f16>6", "t") + with self.assertRaises(SystemExit): + B.parse_isa_rules("v_fma_mixlo_f16@3dddf18==6", "t") + + def test_isa_count(self): + lines = [" v_fma_mixlo_f16 v3, v3, 0xf184, 0 // 000000008224: CC210003 0201FF03 3DDDF184", + " v_fma_mixhi_f16 v10, v2, 0xf184, 0 // 000000008244: CC22000A 0201FF02 3DDDF184", + " v_fma_mixlo_f16 v3, v3, 0xf184, v4 // 000000008254: CC210003 0411FF03 3DDDF184", + " v_fma_mixlo_f16 v3, v3, 0xf184, 0 op_sel_hi:[1,0,0] // 000000008264: CC210003 0201FF03 3DDDF184", + " v_fma_mixlo_f16 v3, v3, 0xa8c1, 0 // 000000008274: CC210003 0201FF03 BF38A8C1", + " v_mul_f32_e32 v1, 0x3dddf184, v1 // 0000000009DC: 100202FF 3DDDF184"] + self.assertEqual(B.isa_count(lines, "v_fma_mixlo_f16", None), 4) + self.assertEqual(B.isa_count(lines, "v_fma_mixlo_f16", "3dddf184"), 1) # no other addend, no op_sel, literal + self.assertEqual(B.isa_count(lines, "v_fma_mixhi_f16", "3dddf184"), 1) def test_post_ports(self): decls = [d for d in B.kernel_decls(os.path.join(ROOT, "clean", "dx12", "kernels", "l_post.hlsl")) @@ -38,7 +86,7 @@ def test_post_ports(self): self.assertEqual(sorted(d["needs"] for d in decls), ["fold", "nofold"]) fold = [d for d in decls if d["needs"] == "fold"][0] self.assertEqual(fold["defines"].get("LP_ENCFOLD"), "1") - self.assertTrue(fold["isa"]) + self.assertIn("@3dddf184", fold["isa"]) # the fold's sites: kEncK as the literal, addend 0 nofold = [d for d in decls if d["needs"] == "nofold"][0] self.assertNotIn("LP_ENCFOLD", nofold["defines"]) self.assertEqual(nofold["file"], "l_post_recon_p12__nofold") diff --git a/clean/dx12/tests/test_post_edge_cases.py b/clean/dx12/tests/test_post_edge_cases.py new file mode 100644 index 00000000..910dc017 --- /dev/null +++ b/clean/dx12/tests/test_post_edge_cases.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: MIT +"""CPU checks of Model L's post history fast path (clean/dx12/kernels/l_post.hlsl history_cr) and of the inputs of +its GPU edge check (clean/l/runner/l_post_edge_check.py): +- the fast path's predicate as written (unsigned tests with the size guard) holds exactly when the 4 x 4 footprint + lies inside the image, at every size from 1 x 1; without the guard (the old predicate, mutant LP_MUT 9) it also + holds outside an image narrower or shorter than 4; +- each edge-check kind puts the history positions where it says (inside the footprint, outside it, or both in one + wave), so the counter expectations of the GPU check follow.""" +import os +import sys +import unittest + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.runner import l_post_edge_check as E # noqa: E402 + + +def u32(v): + return np.int64(v) & 0xFFFFFFFF + + +def predicate(ix, iy, dw, dh, guard=True): + """l_post.hlsl's interior predicate of one lane (int32 arithmetic, unsigned compares).""" + p = u32(ix - 1) <= u32(dw - 4) and u32(iy - 1) <= u32(dh - 4) + return bool(p and (not guard or (dw >= 4 and dh >= 4))) + + +def inside(ix, iy, dw, dh): + """Rows iy - 1 ... iy + 2 and columns ix - 1 ... ix + 2 all in the image.""" + return ix - 1 >= 0 and ix + 2 <= dw - 1 and iy - 1 >= 0 and iy + 2 <= dh - 1 + + +class PredicateTest(unittest.TestCase): + def test_guarded_predicate_is_the_footprint(self): + for dw in range(1, 10): + for dh in range(1, 10): + for ix in range(-3, 12): + for iy in range(-3, 12): + self.assertEqual(predicate(ix, iy, dw, dh), inside(ix, iy, dw, dh), (dw, dh, ix, iy)) + + def test_old_predicate_reads_outside(self): + bad = [(dw, dh, ix, iy) for dw in range(1, 10) for dh in range(1, 10) for ix in range(-3, 12) + for iy in range(-3, 12) if predicate(ix, iy, dw, dh, guard=False) and not inside(ix, iy, dw, dh)] + self.assertIn((5, 3, 2, 1), bad) # height 3: the span reads row 3 + self.assertTrue(all(dw < 4 or dh < 4 for dw, dh, _, _ in bad)) + + +class EdgeCasesTest(unittest.TestCase): + def footprints(self, x, dw, dh): + X, Y = np.meshgrid(np.arange(dw) + 0.5, np.arange(dh) + 0.5) + qx = X + x["m_near"][..., 0].astype(np.float64) + qy = Y + x["m_near"][..., 1].astype(np.float64) + ix, iy = np.floor(qx - 0.5).astype(int), np.floor(qy - 0.5).astype(int) + return np.vectorize(lambda a, b: inside(a, b, dw, dh))(ix, iy) + + def test_kinds(self): + rng = np.random.default_rng(1) + mixed = set() + for dw, dh in E.SIZES: + for kind in E.KINDS: + if kind == "random": + continue + x = E.edge_case(rng, dw, dh, kind) + for k in ("m_center", "m_far"): + self.assertTrue(np.array_equal(x[k], x["m_near"])) + case = {"x": x, "dw": dw, "dh": dh, "kind": kind} + want = E.counter_expected(case) + rows = E.wave_rows(x, dw, dh) + if dw < 4 or dh < 4: + self.assertIs(want, False) + elif kind in ("center", "in_lo", "in_hi"): + self.assertTrue(want, (dw, dh, kind)) + self.assertTrue(self.footprints(x, dw, dh).all(), (dw, dh, kind)) + elif kind in ("left", "right", "out_lo", "out_hi"): + self.assertIs(want, False, (dw, dh, kind)) + self.assertFalse(self.footprints(x, dw, dh).any(), (dw, dh, kind)) + elif kind in ("split", "split_half"): + if any(i > 0 and o > 0 for i, o in rows): + mixed.add((dw, dh, kind)) + # the mixed waves (both inside and outside lanes) appear at every size of at least 4 x 4, for both split kinds + for dw, dh in E.SIZES: + if dw >= 4 and dh >= 4: + self.assertIn((dw, dh, "split"), mixed) + self.assertIn((dw, dh, "split_half"), mixed) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/l/runner/l_post_edge_check.py b/clean/l/runner/l_post_edge_check.py new file mode 100644 index 00000000..b11e6ec3 --- /dev/null +++ b/clean/l/runner/l_post_edge_check.py @@ -0,0 +1,376 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): the reconstruction kernels' history sample at the image edges and at small, odd image sizes, +D3D12 against HIP byte for byte, HIP against the oracle, with a counter that proves the D3D12 interior fast path runs. + + python clean/l/runner/l_post_edge_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --prepare + python clean/l/runner/l_post_edge_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --run --json out.json + +--prepare (CPU): l_post.co (HIP), kernel_replay.exe and four D3D12 builds of the post ports (clean/dx12/build_dx12.py +--define): /dx12 as shipped; /hscount (LP_HSCOUNT=1: each wave taking the history fast path adds 1 to the +first dword after the full history); /mut8 (counter + LP_MUT=8: the fast path's predicate always false) and +/mut9 (LP_MUT=9: the predicate without its size guard, the bug of images narrower or shorter than 4). +--run (GPU, one GPU slot). The kernels l_post_recon, l_post_recon_p12, l_post_recon_ldr and l_post_recon_p12_ldr (the +history sample, history_cr, is shared by all four), and l_post_recon_p12 again with the no-fold port +(WG_TEST_FOLD_PROBE_FAIL=1), on every size of SIZES (1 x 1 ... 97 x 35: below 4, odd, a tile's interior boundary in the +middle of a wave) and every kind of KINDS, with a valid history (no reset): +- random: Model M's random inputs (random motions around a random base); +- center / left / right / top / bottom: every motion points at one history position: the centre, or just inside + one edge (the clamped sample on every lane); +- in_lo / in_hi / out_lo / out_hi: the position at the fast path's boundary, ix (and iy) = 1 or size - 3 (inside: the + 4 x 4 footprint fits) and 0 or size - 2 (outside); +- split: even columns point inside, odd columns at an edge (every wave mixes both); split_half: in each 32-column + tile the first 16 columns (half the width when narrower) inside, the rest at an edge (the boundary in the middle of + a wave). +Checks: +- raw bytes, D3D12 == HIP, every output (full history, reduced, display; the l_d3d12_ops_check rules: each back end's + own sentinel, the pitch padding and 64 slack bytes untouched), and the full history per edge band (the two columns + or rows at each of the four edges) reported apart; +- the counter (hscount build, same bytes otherwise): 0 at every size below 4; otherwise, for the non-random kinds, > 0 + exactly when some wave's sampling lanes all have their footprint inside (wave_rows: e.g. center, in_lo, in_hi) and + 0 when none has (left, right, out_lo, out_hi, the mixed waves of split); +- negative controls: mut8 must break the counter expectation (the fast path gone is noticed), mut9 must differ from + HIP (the old predicate reads past a history of height or width 3), and the run's first multi-tile case with half its + workgroups must differ; +- the oracle: l_post_recon and l_post_recon_ldr (HIP) against Model L's reconstruction (clean/l/runner/l_post_check.py + check_case: Model L's post acceptance) on every case. +Output: counts only, no timing. +""" +import argparse +import json +import os +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.runner import l_c32_check as C32 # noqa: E402 +from clean.l.runner import l_d3d12_ops_check as OPS # noqa: E402 +from clean.l.runner.l_d3d12_check import D3D12_SENTINEL, HIP_SENTINEL, prepare_replay # noqa: E402 + +KERNELS = ["l_post_recon", "l_post_recon_p12", "l_post_recon_ldr", "l_post_recon_p12_ldr"] +SIZES = [(1, 1), (2, 3), (3, 2), (3, 3), (3, 5), (5, 3), (4, 4), (5, 4), (4, 5), (7, 7), (33, 9), (37, 6), (67, 13), + (97, 35)] +KINDS = ["random", "center", "left", "right", "top", "bottom", "in_lo", "in_hi", "out_lo", "out_hi", "split", + "split_half"] +VARIANTS = {"hscount": ["LP_HSCOUNT=1"], "mut8": ["LP_HSCOUNT=1", "LP_MUT=8"], "mut9": ["LP_MUT=9"]} +SEED = 231 + + +def _pos(kind, n, lo): + """History coordinate q (q - 1/2 = t, ix = floor(t)) of one axis of size n for a kind; lo: the low side.""" + if kind == "center": + return n // 2 + 0.25 + if kind in ("in_lo", "in_hi"): # ix = 1 or n - 3 + return 2.25 if (kind == "in_lo") == lo else n - 2.25 + if kind in ("out_lo", "out_hi"): # ix = 0 or n - 2 + return 1.25 if (kind == "out_lo") == lo else n - 1.25 + return n // 2 + 0.25 + + +def _point(kind, dw, dh): + if kind == "left": + return 0.75, dh // 2 + 0.25 + if kind == "right": + return dw - 0.25, dh // 2 + 0.25 + if kind == "top": + return dw // 2 + 0.25, 0.75 + if kind == "bottom": + return dw // 2 + 0.25, dh - 0.25 + return _pos(kind, dw, True), _pos(kind, dh, True) + + +def edge_case(rng, dw, dh, kind): + """Model M's random post inputs at render size = output size (ratio 1, jitter 0: render pixel x is output pixel + x, so every candidate motion is the pixel's own) with the motions of `kind` (FP16-exact).""" + from clean.l.runner import l_post_check as LPC + if kind == "random": + rw, rh = max(1, (2 * dw) // 3), max(1, (2 * dh) // 3) + return LPC.make_case(rng, rw, rh, dw, dh, False, "random") + x = LPC.make_case(rng, dw, dh, dw, dh, False, "random") + x["jitter"] = np.zeros(2) + X, Y = np.meshgrid(np.arange(dw) + 0.5, np.arange(dh) + 0.5) + if kind in ("split", "split_half"): + inside = (np.arange(dw) % 2 == 0) if kind == "split" else (np.arange(dw) % 32 < min(16, dw // 2)) + qi, qe = _point("center", dw, dh), _point("left", dw, dh) + qx = np.where(inside[None, :], qi[0], qe[0]) + 0 * X + qy = np.where(inside[None, :], qi[1], qe[1]) + 0 * Y + else: + q = _point(kind, dw, dh) + qx, qy = q[0] + 0 * X, q[1] + 0 * Y + m = np.stack([qx - X, qy - Y], -1) + assert np.array_equal(m.astype(np.float16).astype(np.float64), m), "motions must be FP16-exact" + for k in ("m_near", "m_center", "m_far"): + x[k] = m.astype(np.float16) + return x + + +def wave_rows(x, dw, dh): + """The kernel's history samples of the non-random kinds, per wave and row: [(inside, outside)] counts of the lanes + that sample the history (valid position), as l_post.hlsl's grid runs them: 32-column tiles, wave k = row pair + k0 + k (pairs past the end repeat the last), both rows (a second row past the image is computed too: its motion is + the last row's), lanes past the width repeat the last column. Render size = output size, jitter 0.""" + m = x["m_near"].astype(np.float64) + npairs = (dh + 1) // 2 + out = [] + for x0 in range(0, dw, 32): + for k0 in range(0, npairs, 8): + for k in range(8): + y0 = 2 * min(k0 + k, npairs - 1) + for row in (0, 1): + y = y0 + row + ins = outs = 0 + for lane in range(32): + xx = min(x0 + lane, dw - 1) + fy = min(max(y, 0), dh - 1) + qx, qy = xx + 0.5 + m[fy, xx, 0], y + 0.5 + m[fy, xx, 1] + if not (0.0 <= qx <= dw and 0.0 <= qy <= dh): + continue + ix, iy = int(np.floor(qx - 0.5)), int(np.floor(qy - 0.5)) + fits = ix - 1 >= 0 and ix + 2 <= dw - 1 and iy - 1 >= 0 and iy + 2 <= dh - 1 + ins, outs = ins + fits, outs + (not fits) + out.append((ins, outs)) + return out + + +def counter_expected(case): + """True: the fast path must run (counter > 0: some wave's sampling lanes all fit); False: it must not (0); None: + either (random motions and jitter, at least 4 x 4).""" + dw, dh = case["dw"], case["dh"] + if dw < 4 or dh < 4: + return False + if case["kind"] == "random": + return None + return any(i > 0 and o == 0 for i, o in wave_rows(case["x"], dw, dh)) + + +def cases(kernel, rng): + """The post cases of l_d3d12_ops_check (same argument layout, sentinels, slack) on the edge inputs.""" + from clean.runner import post as RP + from clean.l.runner import l_post_check as LPC + out = [] + for (dw, dh) in SIZES: + for kind in KINDS: + x = edge_case(rng, dw, dh, kind) + prm = RP.post_params(x) + data = RP.post_device_inputs(x) + n_hist, n_red = dw * dh * 8, max(4, (dw // 2) * (dh // 2) * 4) + n_disp, pitch = RP.display_bytes(dw, dh), RP.display_pitch(dw) + disp_def = np.zeros((3, dh, pitch, 2), bool) + disp_def[:, :, :dw, :] = True + red_def = np.zeros(n_red, bool) + red_def[:(dw // 2) * (dh // 2) * 4] = True + defined = [np.concatenate([np.ones(n_hist, bool), np.zeros(OPS.POST_SLACK, bool)]), + np.concatenate([red_def, np.zeros(OPS.POST_SLACK, bool)]), + np.concatenate([disp_def.reshape(-1), np.zeros(OPS.POST_SLACK, bool)])] + if kernel.startswith("l_post_recon_p12"): + data = dict(data, **OPS._pack12(data)) + args = [("null",) if data[k] == b"" else ("buf", data[k]) + for k in ("color", "A", "B", "C", "hist_prev", "m_near", "m_center", "m_far")] + args += [("buf", RP.pack({"s": np.array(LPC.S)})), ("out", n_hist + OPS.POST_SLACK), + ("out", n_red + OPS.POST_SLACK), ("out", n_disp + OPS.POST_SLACK), ("bytes", bytes(prm)), ("null",)] + out.append({"kernel": kernel, "co": "l_post.co", "grid": RP.recon_grid(dw, dh), "block": 256, "args": args, + "defined": defined, "x": x, "dw": dw, "dh": dh, "kind": kind, + "label": f"{dw}x{dh} {kind}"}) + return out + + +def prepare(build, rocm): + C32.compile_kernels(build, rocm) + names = KERNELS + ["l_fold_probe", "native_fill"] + prepare_replay(build, rocm, names) + from clean.runner.hip import hip_child_environment + env, _ = hip_child_environment(rocm_dir=rocm) + env = dict(env, WG_DX12_ISA_CHECK="0") # test variants: the ISA rules are the shipped ports' (checked above) + for tag, defs in VARIANTS.items(): + cmd = [sys.executable, os.path.join(ROOT, "clean", "dx12", "build_dx12.py"), "--out", os.path.join(build, tag), + "--only", ",".join(names), "-j", "1"] + for d in defs: + cmd += ["--define", d] + subprocess.check_call(cmd, env=env, cwd=ROOT) + + +def edge_bands(dw, dh): + """Pixel masks [dh][dw] of the two columns / rows at each edge.""" + ys, xs = np.mgrid[0:dh, 0:dw] + return {"left": xs < 2, "right": xs >= dw - 2, "top": ys < 2, "bottom": ys >= dh - 2} + + +def replay(build_dir, exe, cdir, lines, env_extra=None): + with open(os.path.join(cdir, "cases.txt"), "w", newline="\n") as f: + f.write("\n".join(lines)) + env = {k: v for k, v in os.environ.items() + if not any(s in k.upper() for s in ("HIP", "LLVM", "ROCM", "HSA")) or k.upper() in ("HIP_PLATFORM",)} + env["WG_D3D12_ARENA_MB"] = "512" + env.pop("WG_TEST_FOLD_PROBE_FAIL", None) + env.update(env_extra or {}) + r = subprocess.run([exe, build_dir, os.path.join(cdir, "cases.txt")], env=env, capture_output=True, text=True, + timeout=150, creationflags=0x08000000) + note = [ln for ln in r.stdout.splitlines() if ln.startswith("device note:")] + return {"exit": r.returncode, "note": note[0] if note else "", "tail": r.stdout[-400:] + r.stderr[-200:]} + + +def compare_case(cdir, name, case, wants, counter=False): + """Raw comparison of one case's three outputs (l_d3d12_ops_check rules); counter: the hscount build, whose first + dword after the full history holds the counter (excluded from the slack rule, returned).""" + dw, dh = case["dw"], case["dh"] + n_hist = dw * dh * 8 + res = {"case": name, "label": case["label"], "outputs": []} + count = None + for oi, want in enumerate(wants): + path = os.path.join(cdir, f"{name}.o{oi}") + got = open(path, "rb").read() if os.path.exists(path) else b"" + dmask = case["defined"][oi].copy() + if counter and oi == 0 and len(got) == len(want): + count = int(np.frombuffer(got[n_hist:n_hist + 4], np.uint32)[0]) - 0x5A5A5A5A + g, w = bytearray(got), bytearray(want) + g[n_hist:n_hist + 4] = bytes([D3D12_SENTINEL]) * 4 + w[n_hist:n_hist + 4] = bytes([HIP_SENTINEL]) * 4 + got, want = bytes(g), bytes(w) + a, b = np.frombuffer(got, np.uint8), np.frombuffer(want, np.uint8) + o = {"bytes": len(want)} + if a.size != b.size: + o.update(identical=False, differ=-1) + else: + o["differ"] = int(np.sum((a != b) & dmask)) + o["unwritten_bad"] = int(np.sum(~dmask & ((b != HIP_SENTINEL) | (a != D3D12_SENTINEL)))) + o["identical"] = o["differ"] == 0 and o["unwritten_bad"] == 0 + if oi == 0: # the full history per edge band + d = (a[:n_hist] != b[:n_hist]).reshape(dh, dw, 8).any(-1) + o["edges"] = {k: {"pixels": int(m.sum()), "differ": int((d & m).sum())} + for k, m in edge_bands(dw, dh).items()} + res["outputs"].append(o) + res["identical"] = all(o["identical"] for o in res["outputs"]) + if counter: + res["counter"] = count + return res + + +def run(build, rocm, part="all"): + """part: all, replay (D3D12 legs only) or oracle (the oracle leg only): two GPU slots when one is too short.""" + from clean.l.runner import l_post_check as LPC + g = C32.Gpu(build, rocm) + with open(os.path.join(build, "l_post.co"), "rb") as f: + mod = g.hip.load_module(f.read()) + exe = os.path.join(build, "kernel_replay.exe") + rng = np.random.default_rng(SEED) + allc, hip_out, lines = {}, {}, [] + neg = None + for kernel in KERNELS: + for ci, case in enumerate(cases(kernel, rng)): + name = f"{kernel}_{ci}" + allc[name] = case + if part != "oracle": + hip_out[name] = OPS.hip_run(g, mod, case) + res = {"sizes": SIZES, "kinds": KINDS, "part": part, "legs": {}} + cdir_base = os.path.join(build, "replay-edge") + for leg, bdir, env_extra, counter in () if part == "oracle" else (("d3d12", build, None, False), + ("d3d12_nofold", build, {"WG_TEST_FOLD_PROBE_FAIL": "1"}, False), + ("hscount", os.path.join(build, "hscount"), None, True), + ("mut8", os.path.join(build, "mut8"), None, True), + ("mut9", os.path.join(build, "mut9"), None, False)): + cdir = os.path.join(cdir_base, leg) + os.makedirs(cdir, exist_ok=True) + names = [n for n in allc if leg != "d3d12_nofold" or allc[n]["kernel"] == "l_post_recon_p12"] + lines = [] + for n in names: + lines += OPS.replay_lines(n, allc[n], cdir) + if leg == "d3d12" and neg is None and allc[n]["grid"] >= 4: + neg = n + lines += OPS.replay_lines(n, allc[n], cdir, grid=allc[n]["grid"] // 2, prefix="negative_") + r = replay(bdir, exe, cdir, lines, env_extra) + out = [compare_case(cdir, n, allc[n], hip_out[n], counter) for n in names] + leg_res = {"replay_exit": r["exit"], "device_note": r["note"], "replay_tail": r["tail"], "cases": len(out), + "identical": sum(c["identical"] for c in out), + "edges": {k: {"pixels": sum(c["outputs"][0].get("edges", {}).get(k, {}).get("pixels", 0) for c in out), + "differ": sum(c["outputs"][0].get("edges", {}).get(k, {}).get("differ", 0) for c in out)} + for k in ("left", "right", "top", "bottom")}, + "differing": [c["label"] + " " + c["case"] for c in out if not c["identical"]][:40]} + if counter: + bad, entered = [], 0 + for c in out: + case = allc[c["case"]] + want = counter_expected(case) + entered += c["counter"] > 0 if c["counter"] is not None else 0 + if c["counter"] is None or (want is True and c["counter"] <= 0) or (want is False and c["counter"] != 0): + bad.append(f"{c['case']} {c['label']}: counter {c['counter']}") + leg_res["counter"] = {"cases_entering_fast_path": entered, "expectation_failures": len(bad), + "first_failures": bad[:20], + "must_enter": sum(counter_expected(allc[c["case"]]) is True for c in out), + "must_not_enter": sum(counter_expected(allc[c["case"]]) is False for c in out)} + if leg == "d3d12" and neg: + wants = hip_out[neg] + got = [open(os.path.join(cdir, f"negative_{neg}.o{oi}"), "rb").read() + if os.path.exists(os.path.join(cdir, f"negative_{neg}.o{oi}")) else b"" for oi in range(len(wants))] + leg_res["negative_control"] = {"case": neg, "half_grid": True, "caught": got != list(wants)} + res["legs"][leg] = leg_res + # the oracle: HIP's l_post_recon / l_post_recon_ldr against Model L's reconstruction on every case's inputs + # (cases under 100 pixels: Model L's post acceptance over all of them together, where one value is not 1 %) + orc, small = [], {"history": [], "display": [], "reduced_max": 0} + for name, case in allc.items() if part != "replay" else (): + if case["kernel"] not in ("l_post_recon", "l_post_recon_ldr"): + continue + m = LPC.check_case(g.hip, mod, case["x"], "random", ldr=case["kernel"].endswith("_ldr")) + tiny = case["dw"] * case["dh"] < 100 + ok = m["pass"] + if tiny: + for k in ("history", "display"): + small[k].append(m[k]) + small["reduced_max"] = max(small["reduced_max"], m["reduced"]["max"]) + ok = m["slack_untouched"] and m["alpha_zero"] and m["reduced_top_zero"] + orc.append({"case": name, "label": case["label"], "kernel": case["kernel"], "pass": bool(ok), "tiny": tiny}) + pooled = {k: {f: sum(i[f] for i in small[k]) for f in ("values", "nonfinite", "over_1", "differ")} + for k in ("history", "display")} + for k, v in pooled.items(): + v["pass"] = LPC.l_post_rule(v["values"], v["nonfinite"], v["over_1"], v["differ"], + max((i["mean_rel_rms"] for i in small[k]), default=0.0)) + pooled["reduced_max"] = small["reduced_max"] + res["oracle"] = {"cases": len(orc), "passed": sum(o["pass"] for o in orc), "tiny_pooled": pooled, + "failing": [o["kernel"] + " " + o["label"] for o in orc if not o["pass"]][:40]} + L = res["legs"] + checks = {} + if part != "oracle": + checks.update({ + "d3d12_identical": L["d3d12"]["identical"] == L["d3d12"]["cases"] and L["d3d12"]["replay_exit"] == 0, + "fold_port_selected": "passed: the fold ports run" in L["d3d12"]["device_note"], + "nofold_identical": L["d3d12_nofold"]["identical"] == L["d3d12_nofold"]["cases"], + "nofold_port_selected": "the no-fold ports run" in L["d3d12_nofold"]["device_note"], + "hscount_identical": L["hscount"]["identical"] == L["hscount"]["cases"], + "hscount_counter_ok": L["hscount"]["counter"]["expectation_failures"] == 0, + "mut8_caught": L["mut8"]["counter"]["expectation_failures"] > 0, + "mut9_caught": L["mut9"]["identical"] < L["mut9"]["cases"], + "negative_control_caught": bool(L["d3d12"].get("negative_control", {}).get("caught"))}) + if part != "replay": + checks["oracle_pass"] = (res["oracle"]["passed"] == res["oracle"]["cases"] and pooled["history"]["pass"] and + pooled["display"]["pass"] and pooled["reduced_max"] <= 1) + res["checks"] = checks + res["pass"] = all(checks.values()) + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--prepare", action="store_true") + ap.add_argument("--run", action="store_true") + ap.add_argument("--part", choices=("all", "replay", "oracle"), default="all") + ap.add_argument("--json") + a = ap.parse_args(argv) + if a.prepare: + prepare(a.build, a.rocm) + if a.run: + res = run(a.build, a.rocm, a.part) + print(json.dumps(res["checks"], indent=1), "pass", res["pass"]) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + return 0 if res["pass"] else 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/runtime/device_d3d12.cpp b/clean/runtime/device_d3d12.cpp index 9cfdc4b3..705979c5 100644 --- a/clean/runtime/device_d3d12.cpp +++ b/clean/runtime/device_d3d12.cpp @@ -21,6 +21,7 @@ #include "rdna4.h" #include "wmma_probe.h" #include "fold_probe.h" +#include "fold_selftest.h" #include #include @@ -643,6 +644,11 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (direct && !direct_bound_) return fail("native D3D12 direct kernel requires a bound direct frame: " + name); // Check the variant BEFORE touching args: a 37-DWORD post variant cannot satisfy a 42-DWORD port. if (!variant_loaded(k)) return false; + return dispatch(k, name, grid, block, args); + } + // One launch of the port k (launch() after its port selection; the fold self-test runs both ports of a kernel). + bool dispatch(NativeKernel& k, const std::string& name, unsigned grid, unsigned block, void** args) { + const bool direct = k.rs == "direct"; if (block != k.block) return fail("native D3D12 kernel " + name + ": block must be " + std::to_string(k.block)); if (!grid || grid > 0x7fffffffu) return fail("native D3D12 kernel " + name + ": grid must be 1..2147483647"); const uint64_t groups = (uint64_t)grid * k.gscale; // D3D12 workgroups @@ -1423,18 +1429,31 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (!ok) err_ = error; return ok; } - // Start-up probe of the driver's conversion fold (fold_probe.h, kernel l_fold_probe): only when a port needs it. - // Passed: the "fold" ports run; failed (or no probe kernel): the "nofold" ports. Logged once per process. A failing - // probe is not an error: the device keeps going on the exact no-fold ports. + // Start-up guard of the driver's conversion fold: only when a port needs it. Two steps, both must pass, else (or on + // any error) the "nofold" ports run (fail closed): + // 1. probe: l_fold_probe (fold_probe.h) on adversarial inputs of the shared encode expression, against expected + // bytes from exact rationals; + // 2. differential self-test (fold_selftest): the REAL post shader, its fold port and its no-fold port (exact by + // construction), on one crafted frame (fold_selftest.h); the raw output bytes must be equal. This tests the + // driver's lowering of the shader that will run, at its six encode sites. + // Logged once per process (note()). A failing guard is not an error: the device keeps going on the no-fold ports. bool fold_probe() { bool wanted = false; - for (const auto& kv : kernels_) wanted = wanted || kv.second.needs == "fold"; + std::string uncovered; + for (const auto& kv : kernels_) { + if (kv.second.needs != "fold") continue; + wanted = true; + if (kv.first != "l_post_recon_p12") uncovered = kv.first; // a fold port with no self-test of its own + } fold_ok_ = false; if (!wanted) return true; std::string note; + const std::string before = err_; auto probe = kernels_.find("l_fold_probe"); - if (probe == kernels_.end() || probe->second.co != "-" || probe->second.block != 32 || probe->second.dwords != 1 || - probe->second.args.size() != 1 || probe->second.args[0].kind != 'p') { + if (!uncovered.empty()) { + note = "no self-test for " + uncovered; + } else if (probe == kernels_.end() || probe->second.co != "-" || probe->second.block != 32 || + probe->second.dwords != 1 || probe->second.args.size() != 1 || probe->second.args[0].kind != 'p') { note = "no l_fold_probe kernel"; } else { DevPtr buf = 0; @@ -1442,7 +1461,6 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { fold_probe::fill(bytes); uint32_t out[96] = {}; void* args[] = {&buf}; - const std::string before = err_; bool ok = alloc(&buf, fold_probe::kBytes) && upload(buf, bytes, sizeof bytes) && launch("l_fold_probe", 1, 32, args) && download(out, buf + fold_probe::kOutOffset, sizeof out); if (buf) free(buf); @@ -1450,15 +1468,85 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (ok && fail_test && fail_test[0] == '1') out[0] ^= 1u; const int bad = ok ? fold_probe::first_mismatch(out) : -1; if (!ok) note = "probe run failed (" + err_ + ")"; - else if (bad >= 0) note = "mismatch at half " + std::to_string(bad); - else fold_ok_ = true; - err_ = before; + else if (bad >= 0) note = "probe mismatch at half " + std::to_string(bad); + else fold_ok_ = fold_selftest(¬e); } + err_ = before; // one line per device: in note(), which the runtime logs once when the network runtime is ready - fold_note_ = std::string("conversion-fold probe ") + + fold_note_ = std::string("conversion-fold probe and self-test ") + (fold_ok_ ? "passed: the fold ports run" : "failed (" + note + "): the no-fold ports run"); return true; } + // The differential self-test of l_post_recon_p12 (fold_probe step 2). The frame (fold_selftest.h; render size = + // output size, jitter 0, reset, exposure 1, post scale 0, codes and kernel shape 0) makes each full-history value the + // render colour, so the history codes are the encode of crafted values (FP32-product ties on both sides, zero and + // FP16-subnormal products, the floor). Each port's outputs start filled with its own sentinel; every defined byte of + // the full history, the reduced history and the display image must be equal. True only when both ports ran. + bool fold_selftest(std::string* note) { + NativeKernel *fold = nullptr, *nofold = nullptr; + auto range = kernels_.equal_range("l_post_recon_p12"); + for (auto i = range.first; i != range.second; ++i) { + if (i->second.needs == "fold") fold = &i->second; + if (i->second.needs == "nofold") nofold = &i->second; + } + auto abi = [](const NativeKernel* k) { + if (!k || k->block != 256 || k->rs != "buffer" || k->args.size() != 14) return false; + for (int n = 0; n < 12; ++n) if (k->args[n].kind != 'p') return false; + return k->args[12].kind == 'b' && k->args[12].bytes == 64 && k->args[13].kind == 'p'; + }; + if (!abi(fold) || !abi(nofold)) { *note = "self-test: the fold / no-fold ports of l_post_recon_p12 are missing"; return false; } + constexpr int W = ::fold_selftest::kW, H = ::fold_selftest::kH, N = W * H; + constexpr int pitch = (W + 63) & ~63; + constexpr size_t hist = size_t(N) * 8, reduced = size_t(W / 2) * (H / 2) * 4, display = size_t(3) * pitch * H * 2; + constexpr size_t outs = hist + reduced + display; + // layout: colour | A8 | C4 | previous history | motions x 3 | post blob | outputs (fold) | outputs (no-fold) + const size_t o_col = 0, o_a = o_col + size_t(N) * 16, o_c = o_a + size_t(N) * 8, o_hp = o_c + size_t(N) * 4, + o_m = o_hp + hist, o_w = o_m + size_t(N) * 12, o_out = o_w + 256, total = o_out + 2 * outs; + std::vector host(total, 0); + for (int i = 0; i < N; ++i) std::memcpy(&host[o_col + size_t(i) * 16], &::fold_selftest::kColour[3 * i], 12); + std::memset(&host[o_out], 0xA5, outs); + std::memset(&host[o_out + outs], 0x5A, outs); + struct { int32_t rw, rh, dw, dh, reset, pad0, pad1, pad2; float rx, ry, jx, jy, e, e_prev, ratio, pad3; } prm = + {W, H, W, H, 1, 0, 0, 0, 1.0f, 1.0f, 0.0f, 0.0f, 1.0f, 1.0f, 1.0f, 0.0f}; + static_assert(sizeof prm == 64, "WgPostParams"); + DevPtr buf = 0; + std::vector got(2 * outs); + bool ok = alloc(&buf, total) && upload(buf, host.data(), total); + for (int port = 0; ok && port < 2; ++port) { + DevPtr p[12], none = 0; + const DevPtr out = buf + o_out + port * outs; + const DevPtr in[9] = {buf + o_col, buf + o_a, buf + o_a, buf + o_c, buf + o_hp, buf + o_m, buf + o_m + N * 4, + buf + o_m + N * 8, buf + o_w}; + for (int n = 0; n < 9; ++n) p[n] = in[n]; + p[9] = out; p[10] = out + hist; p[11] = out + hist + reduced; + void* args[14]; + for (int n = 0; n < 12; ++n) args[n] = &p[n]; + args[12] = &prm; args[13] = &none; + const unsigned grid = unsigned((W + 31) / 32) * unsigned(((H + 1) / 2 + 7) / 8); + ok = dispatch(port == 0 ? *fold : *nofold, "l_post_recon_p12", grid, 256, args); + } + ok = ok && download(got.data(), buf + o_out, 2 * outs); + const std::string error = err_; + if (buf) free(buf); + if (!ok) { *note = "self-test run failed (" + error + ")"; return false; } + if (const char* dump = std::getenv("WG_TEST_FOLD_SELFTEST_DUMP")) { // tests: both ports' outputs, raw + if (FILE* f = std::fopen(dump, "wb")) { std::fwrite(got.data(), 1, got.size(), f); std::fclose(f); } + } + size_t differ = 0, hist_differ = 0; + for (size_t i = 0; i < outs; ++i) { + if (i >= hist + reduced) { // display: only the pixels of each pitch row are defined + const size_t px = ((i - hist - reduced) / 2) % pitch; + if (px >= size_t(W)) continue; + } + if (got[i] != got[outs + i]) { ++differ; if (i < hist) ++hist_differ; } + } + if (differ) { + *note = "self-test: the fold port differs from the no-fold port in " + std::to_string(differ) + " bytes (" + + std::to_string(hist_differ) + " of the full history)"; + return false; + } + return true; + } bool create_device() { dxgi_dll_ = LoadLibraryExW(L"dxgi.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32); d3d12_dll_ = LoadLibraryExW(L"d3d12.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32); diff --git a/clean/runtime/fold_selftest.h b/clean/runtime/fold_selftest.h new file mode 100644 index 00000000..29d8ed70 --- /dev/null +++ b/clean/runtime/fold_selftest.h @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: MIT +// Generated by clean/runtime/gen_fold_selftest.py (do not edit): the render colour image of the native D3D12 +// device's differential self-test of Model L's post (fold port against no-fold port, device_d3d12.cpp). +// FP32 bits, slot 3 (y kW + x) + channel. Synthetic values from exact rational arithmetic; no model data. +// Per site (row of the pair, channel): 25 special values (floor, zero and FP16-subnormal products), then +// FP32-product ties in the image (exact product above / below the midpoint; non-detecting: the second rounding +// happens to go the right way): +// subnormal up 0, down 0, up_nondetecting 0, down_nondetecting 0 +// negative up 72, down 72, up_nondetecting 0, down_nondetecting 0 +// small up 72, down 72, up_nondetecting 0, down_nondetecting 0 +// medium up 72, down 72, up_nondetecting 6, down_nondetecting 12 +// large up 72, down 72, up_nondetecting 12, down_nondetecting 12 +#pragma once +#include + +namespace fold_selftest { + +constexpr int kW = 32, kH = 8; +constexpr uint32_t kColour[768] = { + 0xbc23d2d9u, 0xbc23d2d9u, 0xbc23d2d9u, 0xbc23d2dau, 0xbc23d2dau, 0xbc23d2dau, 0xbc23d2dbu, 0xbc23d2dbu, + 0xbc23d2dbu, 0xbc23d2d8u, 0xbc23d2d8u, 0xbc23d2d8u, 0xbc23d2d7u, 0xbc23d2d7u, 0xbc23d2d7u, 0xbc23d2d6u, + 0xbc23d2d6u, 0xbc23d2d6u, 0x00000000u, 0x00000000u, 0x00000000u, 0xbc23d70au, 0xbc23d70au, 0xbc23d70au, + 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xf149f2cau, 0xf149f2cau, 0xf149f2cau, 0x00000000u, 0x00000000u, + 0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x00000001u, 0x00000001u, 0x00000001u, 0x80000001u, + 0x80000001u, 0x80000001u, 0x0da24260u, 0x0da24260u, 0x0da24260u, 0x8da24260u, 0x8da24260u, 0x8da24260u, + 0x30a3d70au, 0x30a3d70au, 0x30a3d70au, 0x3123d70au, 0x3123d70au, 0x3123d70au, 0x320f5c29u, 0x320f5c29u, + 0x320f5c29u, 0x34000000u, 0x34000000u, 0x34000000u, 0x36700000u, 0x36700000u, 0x36700000u, 0xb023d70au, + 0xb023d70au, 0xb023d70au, 0xb0f5c28fu, 0xb0f5c28fu, 0xb0f5c28fu, 0xb3000000u, 0xb3000000u, 0xb3000000u, + 0xb6700000u, 0xb6700000u, 0xb6700000u, 0x464c98b1u, 0x46ba54b6u, 0x46f544dau, 0x494547a2u, 0x499aa7c7u, + 0x4b0119b5u, 0x4d1615d7u, 0x4d1615eeu, 0x4d7da1bau, 0x4ea808b0u, 0x4f20dfa6u, 0x4fe5b674u, 0x516a2e6bu, + 0x51c0fd9cu, 0x52023b29u, 0x56091904u, 0x5784f3d1u, 0x57f22a88u, 0x599e797bu, 0x59e0d63fu, 0x5a304241u, + 0xbc23d2d9u, 0xbc23d2d9u, 0xbc23d2d9u, 0xbc23d2dau, 0xbc23d2dau, 0xbc23d2dau, 0xbc23d2dbu, 0xbc23d2dbu, + 0xbc23d2dbu, 0xbc23d2d8u, 0xbc23d2d8u, 0xbc23d2d8u, 0xbc23d2d7u, 0xbc23d2d7u, 0xbc23d2d7u, 0xbc23d2d6u, + 0xbc23d2d6u, 0xbc23d2d6u, 0x00000000u, 0x00000000u, 0x00000000u, 0xbc23d70au, 0xbc23d70au, 0xbc23d70au, + 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xf149f2cau, 0xf149f2cau, 0xf149f2cau, 0x00000000u, 0x00000000u, + 0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x00000001u, 0x00000001u, 0x00000001u, 0x80000001u, + 0x80000001u, 0x80000001u, 0x0da24260u, 0x0da24260u, 0x0da24260u, 0x8da24260u, 0x8da24260u, 0x8da24260u, + 0x30a3d70au, 0x30a3d70au, 0x30a3d70au, 0x3123d70au, 0x3123d70au, 0x3123d70au, 0x320f5c29u, 0x320f5c29u, + 0x320f5c29u, 0x34000000u, 0x34000000u, 0x34000000u, 0x36700000u, 0x36700000u, 0x36700000u, 0xb023d70au, + 0xb023d70au, 0xb023d70au, 0xb0f5c28fu, 0xb0f5c28fu, 0xb0f5c28fu, 0xb3000000u, 0xb3000000u, 0xb3000000u, + 0xb6700000u, 0xb6700000u, 0xb6700000u, 0x47078611u, 0x47a69344u, 0x485cad4du, 0x4b9ac3c1u, 0x4bab07eeu, + 0x4c27e2b5u, 0x4d8c24feu, 0x4df8f788u, 0x4e062c35u, 0x500c44b8u, 0x510e14e4u, 0x5153e8c5u, 0x55254b3eu, + 0x552dc357u, 0x5596896fu, 0x58479238u, 0x5873b9ffu, 0x58bf1120u, 0x5c1bab74u, 0x5dd62e6eu, 0x5de127a7u, + 0x5e097c54u, 0x5efc094du, 0x5fc6dabfu, 0x60a4eee7u, 0x611bf1d6u, 0x611bf1d8u, 0x6303041au, 0x6310c920u, + 0x632000c6u, 0x65a539a9u, 0x66a75c6du, 0x683c8ceau, 0x6a2f0edbu, 0x6a828bb2u, 0x6b6f5036u, 0x48121fb1u, + 0x483b9917u, 0x48aad998u, 0x499c9984u, 0x4a4f7634u, 0x4a54b5dau, 0x4d1bd134u, 0x4e0ed21eu, 0x4ed26777u, + 0x50c35909u, 0x5111ad42u, 0x5131e866u, 0x520c5e2eu, 0x529c1ec5u, 0x5456cd08u, 0x565c63ffu, 0x5725692eu, + 0x5796a4cau, 0x5bd6079du, 0x5bd6079fu, 0x5d29f4afu, 0x5fe150afu, 0x5fecdc44u, 0x60180b36u, 0x617480c3u, + 0x6277aa31u, 0x637addb1u, 0x660186f3u, 0x660f23fbu, 0x660f240au, 0x69069d96u, 0x69069dadu, 0x6924666eu, + 0x6b999eabu, 0x6d535c44u, 0x6d810fcbu, 0x3e90e64du, 0x3e979abeu, 0x3ea9296eu, 0x3ec7e4a3u, 0x3ee790fau, + 0x3ee90ca3u, 0x3f42f26bu, 0x3f5251dcu, 0x3f6ea559u, 0x4059b3e8u, 0x405f3a05u, 0x405f3a06u, 0x40999a8eu, + 0x409b89e9u, 0x4117a47du, 0x415f6fb1u, 0x4176efdbu, 0x41802fc2u, 0x41ad07a7u, 0x41ce1d3du, 0x41e98d05u, + 0x42a2aa57u, 0x42a4b5dcu, 0x42b3c3a0u, 0x434f9617u, 0x436b3595u, 0x437742ffu, 0x43bd0eccu, 0x43e6e372u, + 0x43efb4b4u, 0x449add2eu, 0x449ccf92u, 0x44cbd960u, 0x456aa69fu, 0x4576ac8du, 0x458f48beu, 0x3e2847b6u, + 0x3e7a760du, 0x3ea02672u, 0x3ee3e629u, 0x3f1da81eu, 0x3f67f5e8u, 0x3fedb803u, 0x3ff0b925u, 0x4019bcecu, + 0x5fc6dadeu, 0x60953f37u, 0x60953f53u, 0x61527713u, 0x622e903fu, 0x630303efu, 0x63d95576u, 0x63d95577u, + 0x64a31dbeu, 0x68664438u, 0x69adefc3u, 0x6a2f0eccu, 0x6d06ce54u, 0x6e088c6eu, 0x6e088c7du, 0x4902a18fu, + 0x4905ef98u, 0x4998bc49u, 0x4b02b931u, 0x4bfbeb3fu, 0x4caf6b97u, 0x4fae834eu, 0x504c0ae0u, 0x509ff503u, + 0x5131e875u, 0x51aa53a7u, 0x51b30dd4u, 0x5456cd12u, 0x55a2412bu, 0x56234cb7u, 0x598bdd23u, 0x59db4a3eu, + 0x5b770d63u, 0x5e0617e9u, 0x5e1bc7c1u, 0x5f0ec979u, 0x60efec0bu, 0x61352a84u, 0x615d3ff4u, 0x643c48b0u, + 0x652c944bu, 0x6568ea52u, 0x662ecf81u, 0x67311203u, 0x68a35912u, 0x6a97a89fu, 0x6aa7992fu, 0x6b8b0263u, + 0x6d810fdbu, 0x6d8ea073u, 0x6dae2efau, 0x3eab5a49u, 0x3eb58eaeu, 0x3ec41659u, 0x3f155f4eu, 0x3f25573fu, + 0x3f3f47cbu, 0x3fb4dafcu, 0x40065685u, 0x40400fc0u, 0x4080e804u, 0x408359c3u, 0x40842cf4u, 0x411b7ba8u, + 0x414f4a20u, 0x415ca90du, 0x41836e7eu, 0x418daaf1u, 0x419140fcu, 0x4229e2e3u, 0x4242eaceu, 0x425fa2aau, + 0x42db8ab3u, 0x43046553u, 0x43107edeu, 0x437a5ebcu, 0x43916e82u, 0x43a6daeeu, 0x4421c16fu, 0x44524718u, + 0x4454ebd3u, 0x450d09f7u, 0x454329cfu, 0x454d29cfu, 0x45cb4947u, 0x460ca631u, 0x461f5d33u, 0x3ecdbcc5u, + 0x3ee272bbu, 0x3ee272bdu, 0x3f67f5ebu, 0x3f6de4cfu, 0x3f9364a6u, 0x4027d879u, 0x40871aa4u, 0x40a0fc6fu, + 0x40d1b345u, 0x40e7c62cu, 0x40f6bbbcu, 0x415e0b4au, 0x4160d660u, 0x41696df9u, 0x41b9578cu, 0x41dcc78fu, + 0x41dcc793u, 0x42e281d5u, 0x42eb2862u, 0x42eb2863u, 0x43846c98u, 0x43ca8132u, 0x43cd0ce0u, 0x4428fc31u, + 0x442d42afu, 0x446cc647u, 0x44a7f6c0u, 0x44d7a30fu, 0x44dd17b5u, 0x451ae460u, 0x451ae461u, 0x456d99d4u, + 0x45ba43c7u, 0x45e379f9u, 0x45e93b4bu, 0xb83f81e1u, 0xba0435c8u, 0xba9b1c16u, 0xbb4e7552u, 0xbb628ae3u, + 0xbb6f140fu, 0xbba0ac35u, 0xbba23ae8u, 0xbba2dfeeu, 0xbbb3d2feu, 0xbbb8495bu, 0xbbc88821u, 0xbbe47ca3u, + 0xbbe5b7e3u, 0xbbee0f96u, 0xbbf3395bu, 0xbbf4cab9u, 0xbbf913edu, 0xbc013558u, 0xbc0150fdu, 0xbc02535bu, + 0xbc05e6b8u, 0xbc06e98cu, 0xbc094066u, 0xbc112869u, 0xbc117283u, 0xbc13816cu, 0xbc167700u, 0xbc16d5e7u, + 0xbc18d1c3u, 0xbc1d1c99u, 0xbc1d4869u, 0xbc1dcb54u, 0xbc20bdbfu, 0xbc21c326u, 0xbc2230e7u, 0xba12dd5du, + 0xba3d01e6u, 0xba4d5525u, 0xbb88e48au, 0xbb9969aeu, 0xbb9aa212u, 0xbbc2f3e9u, 0xbbc73c10u, 0xbbc9d0d7u, + 0xbbdf52a8u, 0xbbe32957u, 0xbbe4b80cu, 0xbbf846a5u, 0xbbf8c537u, 0xbbfb8dddu, 0xbc08894au, 0xbc08e010u, + 0xbc09f2b6u, 0xbc0bf7a9u, 0xbc0cd875u, 0xbc0d9f40u, 0xbc12f606u, 0xbc14e589u, 0xbc1625dau, 0xbc1901cfu, + 0xbc1a3826u, 0xbc1a5e4fu, 0xbc1e08e8u, 0xbc1e1229u, 0xbc1eef8bu, 0xbc1ffefbu, 0xbc201d3du, 0xbc20460bu, + 0x4108598au, 0x4148e92eu, 0x4148e936u, 0x419d8c27u, 0x41a39122u, 0x41b28584u, 0x428eaa9bu, 0x42c7e7a3u, + 0x42c7e7a8u, 0x431eb070u, 0x4326d21eu, 0x43812816u, 0x43f1353au, 0x441523d7u, 0x4424d0beu, 0x446fc02cu, + 0x4475d102u, 0x44a1c916u, 0x44e2af9du, 0x44e5892du, 0x451711f6u, 0x4579c672u, 0x458a0387u, 0x45b5ab2du, + 0x45fb6289u, 0x46042201u, 0x461d62d3u, 0xbaac5932u, 0xbb05738au, 0xbb4e1b8fu, 0xbb855c1fu, 0xbb9a4b84u, + 0xbb9de96du, 0xbba4280eu, 0xbba4aa97u, 0xbba7714au, 0xbbcae463u, 0xbbd214dau, 0xbbe17ff2u, 0xbbef0900u, + 0xbbf0f3b9u, 0xbbf2d3d5u, 0xbbf971f8u, 0xbbfc3486u, 0xbbff1846u, 0xbc0260beu, 0xbc029615u, 0xbc05cebfu, + 0xbc0b045du, 0xbc0e4586u, 0xbc1090b0u, 0xbc143451u, 0xbc153269u, 0xbc16411du, 0xbc1bd9ceu, 0xbc1bf332u, + 0xbc1d06f2u, 0xbc1dd4f7u, 0xbc1f37e0u, 0xbc1fe93bu, 0xbc22d6ecu, 0xbc22e1e1u, 0xbc235148u, 0xba9a8bb0u, + 0xbacca7e8u, 0xbb2f3000u, 0xbba66026u, 0xbba6e0e9u, 0xbbaa1c24u, 0xbbcafd51u, 0xbbd16fddu, 0xbbd5c94eu, + 0xbbe93f7bu, 0xbbed0150u, 0xbbf5e282u, 0xbc025a0du, 0xbc03d26bu, 0xbc06af80u, 0xbc0b0e47u, 0xbc0baad6u, + 0xbc0bd15eu, 0xbc0f8bc0u, 0xbc1102ecu, 0xbc112fdfu, 0xbc16468au, 0xbc17c17cu, 0xbc188ee6u, 0xbc1b065fu, + 0xbc1b2263u, 0xbc1bc9c6u, 0xbc1f4a35u, 0xbc1f58abu, 0xbc1f6e06u, 0xbc20a205u, 0xbc20dd4fu, 0xbc211d17u, + 0xbc215386u, 0xbc218968u, 0xbc22a12cu, 0x381e585eu, 0x3972289eu, 0x3a853572u, 0x3b4c2fb3u, 0x3b5dd03cu, + 0x3b6e67c5u, 0x3bafa8f7u, 0x3bb4cb7fu, 0x3be4ff3eu, 0x3c20865du, 0x3c3014e8u, 0x3c566233u, 0x3c766070u, + 0x3c835a95u, 0x3c8c89beu, 0x3caa8854u, 0x3caa8855u, 0x3cb49533u, 0x3cd9d7dbu, 0x3cdb4030u, 0x3ce2636cu, + 0x3d177655u, 0x3d1ecf79u, 0x3d4486b5u, 0x3d64f7d9u, 0x3d66a8cfu, 0x3d6d8800u, 0x3db61313u, 0x3dbe77abu, + 0x3dc47b19u, 0x3dcf98ffu, 0x3dd90c53u, 0x3de3a9feu, 0x3df6fa6bu, 0x3e0ccf91u, 0x3e0d4885u, 0x387e1cd8u, + 0x38f2b795u, 0x39f089c7u, 0x3ae5b10cu, 0x3b0be672u, 0x3b16687au, 0x3be29c98u, 0x3c027a8bu, 0x3c05046fu, + 0x3c24fcbbu, 0x3c408611u, 0x3c516588u, 0x3c858794u, 0x3c9763b1u, 0x3c9c2c8bu, 0x3ccd3e3du, 0x3ceb6cc1u, + 0x3ceb6cc3u, 0x3cf91737u, 0x3d021006u, 0x3d036733u, 0x3d246cd0u, 0x3d251145u, 0x3d2cef6du, 0x3d407b54u, + 0x3d41f1c8u, 0x3d4a2790u, 0x3d6a80f5u, 0x3d7f04beu, 0x3d84ae7cu, 0x3da28630u, 0x3dbe236au, 0x3dc0c935u, + 0x3df0c94cu, 0x3dfac37au, 0x3e0fe893u, 0x4bde55a1u, 0x51701b37u, 0x5694b873u, 0x5f5fdf22u, 0x60280625u, + 0x61352a65u, 0x49b3aae1u, 0x4c805d5bu, 0x501374bdu, 0x5f65896cu, 0x6080786eu, 0x6357f138u, 0x3f51a7eeu, + 0x404bd79eu, 0x4072ec2cu, 0x4135c9a5u, 0x421151ceu, 0x4301f012u, 0x3e4f7021u, 0x3e92c915u, 0x3fda6459u, + 0xbc2321f4u, 0xbc235cddu, 0xbc239370u, 0x3acd1597u, 0x3b215577u, 0x3b2860d0u, 0x3b858f37u, 0x3b8b5b84u, + 0x3b957e6au, 0x3be95c45u, 0x3c12a5f1u, 0x3c1c3f7cu, 0x3c5e23d7u, 0x3c6f6400u, 0x3c7518fau, 0x3c906468u, + 0x3c99c503u, 0x3c9f5af6u, 0x3cb9f751u, 0x3ccac991u, 0x3cd532d7u, 0x3ceda9a3u, 0x3d0f3aeeu, 0x3d15f738u, + 0x3d46c2ffu, 0x3d5179c3u, 0x3d57cfbdu, 0x3d884fbbu, 0x3d894b3au, 0x3db38f56u, 0x3dc685eeu, 0x3dcc1055u, + 0x3dce2d6fu, 0x3deb8939u, 0x3dedf22bu, 0x3df55015u, 0x3e1311d6u, 0x3e180f63u, 0x3e189159u, 0x3a856b90u, + 0x3aa45de8u, 0x3ada0277u, 0x3b33dfa4u, 0x3b405b36u, 0x3bcdce66u, 0x3c0f2727u, 0x3c13fc00u, 0x3c1ae05cu, + 0x3c529067u, 0x3c7b3592u, 0x3c7bdb9eu, 0x3ca10e85u, 0x3cacb5c8u, 0x3cb59c10u, 0x3cefea8fu, 0x3cefea90u, + 0x3cf892fdu, 0x3d09fc3fu, 0x3d183702u, 0x3d23256cu, 0x3d2e4685u, 0x3d34693cu, 0x3d37e51eu, 0x3d5115bcu, + 0x3d5c4299u, 0x3d5d13c7u, 0x3d8b86e0u, 0x3d990fbdu, 0x3d9e911fu, 0x3dcff440u, 0x3dd0ab2cu, 0x3dd21abcu, + 0x3e13cef0u, 0x3e154bcdu, 0x3e15cb8du, 0x5715add1u, 0x5a47b634u, 0x5b07a290u, 0x66d6dcc7u, 0x6c50a93bu, + 0x6d003c56u, 0x56428261u, 0x59078a1du, 0x5d9eb2dau, 0x6391b828u, 0x680bb501u, 0x682a9e27u, 0x408d2818u, + 0x41085982u, 0x4114d311u, 0x43e57362u, 0x44eb57cbu, 0x4522d41eu, 0x4056fdf7u, 0x40fb680au, 0x4129b4ccu, +}; + +} // namespace fold_selftest diff --git a/clean/runtime/gen_fold_selftest.py b/clean/runtime/gen_fold_selftest.py new file mode 100644 index 00000000..7326e366 --- /dev/null +++ b/clean/runtime/gen_fold_selftest.py @@ -0,0 +1,255 @@ +# SPDX-License-Identifier: MIT +"""Writes clean/runtime/fold_selftest.h: the colour image of the native D3D12 device's differential self-test of Model +L's post (device_d3d12.cpp fold_selftest). WrongGPU's own code. + + python clean/runtime/gen_fold_selftest.py [out.h] + +The self-test runs the REAL post shader twice on one tiny frame: the fold port of l_post_recon_p12 (its history encode +through the driver's conversion fold, hlsl/l_post_encode.hlsli) and the no-fold port (round to odd, exact without the +fold), and compares the raw output bytes. Equal: the device runs the fold port; any difference or error: the no-fold +port. The frame (kW x kH, render size = output size, jitter 0, reset, exposure 1, post scale s = 0, every code and +kernel-shape value 0) makes the full history H^ of pixel (x, y) channel i exactly the render colour c_i, so the history +code is q = (_Float16)(kEncK x) with x = log2(max(fma(100, c, 1), 1e-4)) evaluated by the post shader itself +(enc_log): one wave of the shader's six encode sites (three channels, two rows of each row pair) sees each class: +- FP32-product ties: c whose x (predicted with the correctly rounded log2) makes the FP32 product RN32(kEncK x) land + exactly on an FP16 midpoint while the exact product does not, the exact product above and below the midpoint, most + of them on the side where the second rounding (to even) goes the wrong way, in four ranges of x and among the + FP16-subnormal products (the ones there are: x near 0 is sparse); +- exact-product ties do not exist: kEncK's significand has the odd factor 0x377c61 (22 bits) and an FP16 midpoint's + has at most 12 bits, so kEncK x is never exactly a midpoint unless it is 0; +- zero products: c = +-0, +-2^-149, +-1e-30 (fma(100, c, 1) = 1, x = +0, the code +0); +- FP16-subnormal products of both signs (fma(100, c, 1) = 1 +- a few ulps); the exhaustive search over every FP32 + value of fma(100, c, 1) in that range finds no FP32-product tie there; +- the floor: c whose fma(100, c, 1) is just below, at or near 1e-4 (the max picks 1e-4), and far below it (0, -1, -1e30). +Which x the GPU's log2 gives is not assumed: an x that differs from the prediction just tests another value. The +fold-removed mutant (LP_MUT 3, two roundings) must make the self-test fail (GPU record of the PR). +Every predicted value is checked with exact rationals (fractions.Fraction, decimal logarithms to 60 digits). +""" +import decimal +import os +import struct +import sys +from fractions import Fraction + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +from gen_fold_probe import K_BITS, bits32, f32, rn16_bits # noqa: E402 + +W, H = 32, 8 +SITES = 6 # (row of the pair, channel) +PER_SITE = W * H * 3 // SITES +SEED = 20261115 +K = Fraction(f32(K_BITS)) +DEC = decimal.Context(prec=60) + + +def rn(fr, p, emin): + """Round a Fraction to p significant bits, nearest even, exponents below emin on the emin grid (no overflow).""" + if fr == 0: + return Fraction(0) + s = -1 if fr < 0 else 1 + a = abs(fr) + e = a.numerator.bit_length() - a.denominator.bit_length() + if Fraction(2) ** e > a: + e -= 1 + e = max(e, emin) + q = Fraction(2) ** (e - p + 1) + n = a / q + f = n.numerator // n.denominator + r = n - f + if r > Fraction(1, 2) or (r == Fraction(1, 2) and f % 2 == 1): + f += 1 + return s * f * q + + +def rn32(fr): + return rn(fr, 24, -126) + + +def m_of(c): + """RN32(fma(100, c, 1)) for the FP32 value c (exact).""" + return rn32(100 * Fraction(c) + 1) + + +def log2_exact(m): + """(the correctly rounded FP32 log2 of the FP32 value m > 0, the distance of the exact log2 from an FP32 midpoint in + units of that FP32 ulp) from 60-digit decimal logarithms.""" + lm = DEC.divide(DEC.ln(decimal.Decimal(m.numerator) / decimal.Decimal(m.denominator)), DEC.ln(decimal.Decimal(2))) + v = Fraction(lm) + x = rn32(v) + if x == 0: + return x, None + a = abs(v) + e = a.numerator.bit_length() - a.denominator.bit_length() + if Fraction(2) ** e > a: + e -= 1 + ulp = Fraction(2) ** (e - 23) + frac = (a / ulp) - (a / ulp).numerator // (a / ulp).denominator + return x, abs(frac - Fraction(1, 2)) + + +def fp16_midpoint(p): + """Whether the FP32 value p lies exactly halfway between two adjacent FP16 values (finite range).""" + a = abs(p) + if a == 0 or a >= 65504: + return False + e = a.numerator.bit_length() - a.denominator.bit_length() + if Fraction(2) ** e > a: + e -= 1 + ulp = Fraction(2) ** (max(e, -14) - 10) + n = a / ulp + return n.denominator == 2 + + +def classify(c): + """(x, product class) of the colour c, checked exactly: None unless kEncK x is an FP32-product tie; else + ('up' / 'down' = the exact product's side, detecting = the second rounding goes to the other FP16 value).""" + m = m_of(c) + if m < Fraction(f32(bits32(1e-4))): + return None + x, margin = log2_exact(m) + if x == 0 or margin is None or margin < Fraction(1, 10**12): + return None + pe = x * K + p32 = rn32(pe) + if not fp16_midpoint(p32) or pe == p32: + return None + detecting = rn16_bits(pe) != rn16_bits(p32) + return x, ("up" if pe > p32 else "down"), detecting + + +def search(rng, lo, hi, want, sub=False): + """want ties (half the exact products above the midpoint, half below; detecting, plus a sixth non-detecting) from + c whose fma(100, c, 1) lies in [lo, hi], log-uniform; sub: every m = 1 +- k ulp in the range (exhaustive).""" + k32 = np.float64(f32(K_BITS)) + if sub: + ks = np.arange(1, int((hi - 1) * 2**23) + 1) + mt = np.concatenate([1.0 + ks * 2.0**-23, 1.0 - np.arange(1, int((1 - lo) * 2**24) + 1) * 2.0**-24]) + rounds = [mt] + else: + rounds = (np.exp2(rng.uniform(np.log2(lo), np.log2(hi), 1 << 20)) for _ in range(400)) + picks = {("up", True): [], ("down", True): [], ("up", False): [], ("down", False): []} + quota = {("up", True): want // 2, ("down", True): want // 2, ("up", False): want // 12, ("down", False): want // 12} + seen = set() + for mt in rounds: + c = ((mt - 1.0) / 100.0).astype(np.float32) + m = (100.0 * c.astype(np.float64) + 1.0).astype(np.float32) + ok = (m >= lo) & (m <= hi) + x = np.log2(m.astype(np.float64)).astype(np.float32) + pe = x.astype(np.float64) * k32 + p32 = pe.astype(np.float32).astype(np.float64) + a = np.abs(p32) + normal = a >= 2.0**-14 + pb = np.abs(p32.astype(np.float32)).view(np.uint32) + tie_n = normal & ((pb & 0x1fff) == 0x1000) + t = a * 2.0**25 + tie_s = ~normal & (t == np.round(t)) & (np.fmod(t, 2.0) == 1.0) & (a > 0) + for i in np.nonzero(ok & (tie_n | tie_s) & (pe != p32))[0]: + cb = int(c[i].view(np.uint32)) + if cb in seen: + continue + seen.add(cb) + r = classify(f32(cb)) + if r is None: + continue + key = (r[1], r[2]) + if len(picks[key]) < quota[key]: + picks[key].append(cb) + if all(len(picks[k]) >= quota[k] for k in picks): + break + return picks + + +def floor_values(): + """c around the floor: fma(100, c, 1) the nearest reachable values below / at / above 1e-4, and far below.""" + one_em4 = Fraction(f32(bits32(1e-4))) + c0 = bits32(np.float32((1e-4 - 1.0) / 100.0)) + below, above = [], [] + for d in range(-40, 41): + cb = c0 + d # negative c: a larger bit pattern is a more negative value + m = m_of(f32(cb)) + (below if m < one_em4 else above).append((m, cb)) + below = [cb for _, cb in sorted(below, reverse=True)[:3]] + above = [cb for _, cb in sorted(above)[:3]] + far = [bits32(0.0) | 0, bits32(-0.01), bits32(-1.0), bits32(-1e30)] + return below + above + far + + +def zero_values(): + return [0x00000000, 0x80000000, 0x00000001, 0x80000001, bits32(1e-30), bits32(-1e-30)] + + +def subnormal_values(): + """c with fma(100, c, 1) = 1 + k 2^-23 or 1 - k 2^-24: FP16-subnormal products of both signs (no tie among them).""" + out = [] + for m in [1.0 + k * 2.0**-23 for k in (1, 2, 7, 100, 3000)] + [1.0 - k * 2.0**-24 for k in (1, 3, 50, 6000)]: + cb = bits32(np.float32((m - 1.0) / 100.0)) + assert m_of(f32(cb)) == Fraction(m) + out.append(cb) + return out + + +def build(): + rng = np.random.default_rng(SEED) + sub_lo, sub_hi = 1.0 - 3.9e-4, 1.0 + 3.9e-4 # |kEncK x| < 2^-14: FP16-subnormal products + groups = {"subnormal": search(rng, sub_lo, sub_hi, 64, sub=True), + "negative": search(rng, 1.0e-4, sub_lo, 144), + "small": search(rng, sub_hi, 16.0, 144), + "medium": search(rng, 16.0, 2.0**20, 144), + "large": search(rng, 2.0**20, 2.0**100, 144)} + specials = floor_values() + zero_values() + subnormal_values() + ties = [] + for g, picks in groups.items(): + for key, lst in picks.items(): + ties += [(g, key, cb) for cb in lst] + # each site (row of the pair, channel): every special value, then the ties dealt round-robin (each group and side + # spread over the six sites; the non-detecting ones last, so that they are the ones a full site drops) + order = sorted(ties, key=lambda t: (not t[1][1], t[0], t[1][0], t[2])) + per_site = [list(specials) for _ in range(SITES)] + for i, t in enumerate(order): + per_site[i % SITES].append(t[2]) + fill = [bits32(1.0 + 0.25 * k) for k in range(PER_SITE)] # ordinary values for any slot left + for s in per_site: + del s[PER_SITE:] + s += fill[:PER_SITE - len(s)] + colour = [0] * (W * H * 3) + for site in range(SITES): + row, ch = divmod(site, 3) + pixels = [(x, 2 * j + row) for j in range(H // 2) for x in range(W)] + for (x, y), cb in zip(pixels, per_site[site]): + colour[3 * (y * W + x) + ch] = cb + tag = {cb: (g, k) for g, k, cb in ties} + counts = {g: {f"{k[0]}{'' if k[1] else '_nondetecting'}": 0 for k in picks} for g, picks in groups.items()} + for cb in colour: + if cb in tag: + g, k = tag[cb] + counts[g][f"{k[0]}{'' if k[1] else '_nondetecting'}"] += 1 + return colour, counts, len(specials) + + +def write(path): + colour, counts, nspec = build() + lines = ["// SPDX-License-Identifier: MIT", + "// Generated by clean/runtime/gen_fold_selftest.py (do not edit): the render colour image of the native D3D12", + "// device's differential self-test of Model L's post (fold port against no-fold port, device_d3d12.cpp).", + "// FP32 bits, slot 3 (y kW + x) + channel. Synthetic values from exact rational arithmetic; no model data.", + "// Per site (row of the pair, channel): %d special values (floor, zero and FP16-subnormal products), then" % nspec, + "// FP32-product ties in the image (exact product above / below the midpoint; non-detecting: the second rounding", + "// happens to go the right way):"] + for g, c in counts.items(): + lines.append("// %-9s %s" % (g, ", ".join(f"{k} {v}" for k, v in c.items()))) + lines += ["#pragma once", "#include ", "", "namespace fold_selftest {", "", + f"constexpr int kW = {W}, kH = {H};", f"constexpr uint32_t kColour[{W * H * 3}] = {{"] + for i in range(0, len(colour), 8): + lines.append(" " + ", ".join(f"0x{v:08x}u" for v in colour[i:i + 8]) + ",") + lines += ["};", "", "} // namespace fold_selftest", ""] + with open(path, "w", encoding="utf-8", newline="\n") as f: + f.write("\n".join(lines)) + return counts + + +if __name__ == "__main__": + out = sys.argv[1] if len(sys.argv) > 1 else os.path.join(HERE, "fold_selftest.h") + print(write(out)) From 7b943098129c29c8edbece1d1abaf90a8c5f61b6 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sun, 15 Nov 2026 02:12:06 -0500 Subject: [PATCH 6/6] L post recon: split off the history-encode fold (follow-up PR); the no-fold port is the only port; edge check legs must be complete The fold, its start-up probe and self-test, the needs= / isa= port selection and their tests move to a follow-up branch. l_post_recon_p12 is one port again: the round-to-odd history encode, exact on any driver. The edge check drops its fold legs, empties each leg's output folder before the run, and counts a leg (mutants included) only when kernel_replay exits 0 and every case wrote every output at full size. Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/build_dx12.py | 88 +-------- clean/dx12/hlsl/l_post_encode.hlsli | 42 ----- clean/dx12/kernel_sets.json | 4 - clean/dx12/kernels/l_fold_probe.hlsl | 28 --- clean/dx12/kernels/l_post.hlsl | 23 +-- clean/dx12/tests/kernel_replay.cpp | 1 - clean/dx12/tests/test_fold_probe.py | 104 ----------- clean/l/runner/l_post_edge_check.py | 52 +++--- clean/runtime/device_d3d12.cpp | 150 +--------------- clean/runtime/device_dx12.cpp | 3 - clean/runtime/fold_probe.h | 86 --------- clean/runtime/fold_selftest.h | 118 ------------- clean/runtime/gen_fold_probe.py | 186 ------------------- clean/runtime/gen_fold_selftest.py | 255 --------------------------- 14 files changed, 43 insertions(+), 1097 deletions(-) delete mode 100644 clean/dx12/hlsl/l_post_encode.hlsli delete mode 100644 clean/dx12/kernels/l_fold_probe.hlsl delete mode 100644 clean/dx12/tests/test_fold_probe.py delete mode 100644 clean/runtime/fold_probe.h delete mode 100644 clean/runtime/fold_selftest.h delete mode 100644 clean/runtime/gen_fold_probe.py delete mode 100644 clean/runtime/gen_fold_selftest.py diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index 2bcfd2f9..0c5bbccb 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -191,17 +191,6 @@ def kernel_decls(path): if d["rs"] not in ("buffer", "tex", "direct"): raise SystemExit(f"{path}: {name}: unsupported root-signature kind {d['rs']}") d["arg_dwords"] = arg_dwords(d["args"], f"{path}: {name}") - # needs=fold | nofold: a port the device runs only when its start-up probe of the driver's conversion fold - # passed (fold) or failed (nofold); the two ports of one name and code object differ by this key - d["needs"] = kv.get("needs", "") - if d["needs"] not in ("", "fold", "nofold"): - raise SystemExit(f"{path}: {name}: needs={d['needs']} (fold or nofold)") - # isa=,: the port's RGA ISA must satisfy each rule, [+...]>= or == (counts of - # instruction mnemonics; @: only its x * literal + 0 sites, parse_isa_rules), checked after the - # build (isa_check; WG_DX12_ISA_CHECK=0 skips it) - d["isa"] = kv.get("isa", "") - if d["isa"]: - parse_isa_rules(d["isa"], f"{path}: {name}") decls.append(d) return decls @@ -309,65 +298,6 @@ def rga_isa(dxil, isa_dir, stem): shutil.copyfile(os.path.join(tmp, f), os.path.join(isa_dir, f"{stem}.{kind}")) -def parse_isa_rules(text, where): - """isa=,...: [(terms, op, n)] with op '>=' or '=='; a term is (mnemonic, literal or None): counts every - instruction of that mnemonic, @<8 hex digits> only those whose literal dword is that value, whose last source - operand is 0 and that carry no op_sel modifier (an FP32 x * literal + 0: the sites of a conversion fold).""" - rules = [] - for r in text.split(","): - m = re.fullmatch(r"([a-z0-9_]+(?:@[0-9a-f]{8})?(?:\+[a-z0-9_]+(?:@[0-9a-f]{8})?)*)(>=|==)(\d+)", r) - if not m: - raise SystemExit(f"{where}: isa rule {r!r} ([@][+[@]]>= or ...==)") - terms = [(t.split("@")[0], t.split("@")[1] if "@" in t else None) for t in m.group(1).split("+")] - rules.append((terms, m.group(2), int(m.group(3)))) - return rules - - -def isa_count(lines, mnemonic, literal): - """Instructions of `mnemonic` in RGA ISA lines (with literal: only the x * literal + 0 sites, parse_isa_rules).""" - c = 0 - for line in lines: - code, _, comment = line.partition("//") - f = code.split() - if not f or f[0] != mnemonic: - continue - if literal is not None: - words = comment.split(":")[-1].split() - srcs = [o.strip() for o in " ".join(f[1:]).split(",")] - if not words or words[-1].lower() != literal or srcs[-1] != "0" or "op_sel" in code: - continue - c += 1 - return c - - -def isa_check(built_isa): - """built_isa: [(kernel name, dxil path, rules text)] of this build's ports with isa=. Each port's RGA ISA must - satisfy its rules (e.g. a driver fold the port's exactness depends on: a v_fma_mixlo_f16 count). Fails the build - otherwise; WG_DX12_ISA_CHECK=0 skips it (no RGA).""" - if not built_isa or os.environ.get("WG_DX12_ISA_CHECK", "1") == "0": - return - if not os.path.exists(RGA) and not RGA.endswith(".cmd"): - raise SystemExit(f"ISA check: RGA not found at {RGA} (WG_RGA, or WG_DX12_ISA_CHECK=0 to skip)") - bad = [] - for name, dxil, text in built_isa: - with tempfile.TemporaryDirectory(prefix="wgisa_") as tmp: - rga_isa(dxil, tmp, "k") - path = os.path.join(tmp, "k.isa.txt") - if not os.path.exists(path): - bad.append(f"{name}: RGA gave no ISA") - continue - lines = open(path, encoding="utf-8", errors="replace").read().splitlines() - for terms, op, n in parse_isa_rules(text, name): - c = sum(isa_count(lines, mn, lit) for mn, lit in terms) - ok = c >= n if op == ">=" else c == n - label = "+".join(mn + (f"@{lit}" if lit else "") for mn, lit in terms) - print(f"ISA {name:40s} {label} = {c} (rule {op} {n}) {'ok' if ok else 'FAILED'}", flush=True) - if not ok: - bad.append(f"{name} ({os.path.basename(dxil)}): {label} = {c}, rule {op} {n}") - if bad: - raise SystemExit("ISA check failed:\n " + "\n ".join(bad)) - - VGPR_LIMITS = os.path.join(HERE, "vgpr_limits.json") @@ -453,12 +383,9 @@ def main(argv=None): old = {} if only and os.path.exists(man_path): # partial build: keep the other kernels' entries with open(man_path) as f: - # every port of a rebuilt name is replaced (a name may gain or lose ports, e.g. needs=fold / nofold) - old = {(k["name"], k.get("co", ""), k.get("needs", "")): k for k in json.load(f)["kernels"] - if k["name"] not in only} - kernels = dict(old) # (name, co, needs) -> manifest entry + old = {(k["name"], k.get("co", "")): k for k in json.load(f)["kernels"]} + kernels = dict(old) # (name, co) -> manifest entry built = [] - built_isa = [] # (name, dxil, isa rules) of the ports with isa= # generated kernels (kernels/fused_*_gen.py: the fused forms of swin_block.hlsl / bneck.hlsl) are regenerated from # the live sources first, so that no build compiles a stale copy for fn in sorted(os.listdir(KERNEL_DIR)) if os.path.isdir(KERNEL_DIR) else []: @@ -481,7 +408,7 @@ def main(argv=None): continue if not only and is_frame_gen(d): continue - key = (d["name"], d["co"], d["needs"]) + key = (d["name"], d["co"]) if key in kernels and key not in old: raise SystemExit(f"kernel {d['name']} ({d['co']}) declared twice") kernels[key] = None # placeholder: the declared-twice check sees it before any compile starts @@ -506,8 +433,6 @@ def build_one(job): r = {"name": d["name"], "file": os.path.basename(out), "co": d["co"], "entry": d["entry"], "block": d["block"], "wave": d["wave"], "args": d["args"], "arg_dwords": d["arg_dwords"], "source": d["source"], "sha256": sha, "rs": d["rs"]} - if d["needs"]: - r["needs"] = d["needs"] if d["group"] != d["block"]: r["group"] = d["group"] r["gscale"] = d["gscale"] @@ -523,21 +448,18 @@ def build_one(job): futures = [pool.submit(build_one, j) for j in jobs] try: for (src, d, out, _k), fut in zip(jobs, futures): - kernels[(d["name"], d["co"], d["needs"])] = fut.result() + kernels[(d["name"], d["co"])] = fut.result() built.append((d["name"], out)) - if d["isa"]: - built_isa.append((d["name"], out, d["isa"])) print(f"{d['name']:40s} -> {os.path.basename(out)} block {d['block']} wave {d['wave']} " f"args {d['args']}", flush=True) except BaseException: for fut in futures: fut.cancel() # kernels not started yet; the running ones finish their current child process raise - names = {k[0] for k in kernels} + names = {n for n, _ in kernels} if only and not (only <= names): raise SystemExit(f"not declared in clean/dx12/kernels or clean/k/dx12/kernels: {sorted(only - names)}") vgpr_check(built) - isa_check(built_isa) with open(man_path, "w") as f: json.dump({"format": FORMAT, "dxc": os.path.basename(os.path.dirname(os.path.dirname(DXC_DIR))), "kernels": [kernels[k] for k in sorted(kernels)]}, f, indent=1) diff --git a/clean/dx12/hlsl/l_post_encode.hlsli b/clean/dx12/hlsl/l_post_encode.hlsli deleted file mode 100644 index f23b23e3..00000000 --- a/clean/dx12/hlsl/l_post_encode.hlsli +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: MIT -// Model L post: the history texel's FP16 encode through the driver's conversion fold, shared by -// clean/dx12/kernels/l_post.hlsl (l_post_recon_p12) and its start-up probe clean/dx12/kernels/l_fold_probe.hlsl, so -// that both compile the same expression. WrongGPU's own code. Include after wg_platform.hlsli. -// -// q_i = (_Float16)(kEncK x_i), one rounding of the exact product, as HIP's v_fma_mixlo_f16 x, kEncK, 0 (+0 for a zero -// product). Written as mad(x, kEncK, 0) feeding the driver's round-to-nearest-even matrix conversion, which the gfx1201 -// D3D12 compiler folds into exactly that instruction (v_fma_mixlo_f16 / v_fma_mixhi_f16, as Model M's post). Without -// the fold the FP32 product would be rounded twice (a value whose FP32 product lands on an FP16 midpoint moves by one -// FP16 ulp). Guards: build_dx12.py's ISA check on the kernel (isa= in its declaration: the six sites, kEncK as the -// literal with a 0 addend) and two start-up steps of the native device, both of which must pass: the probe -// (l_fold_probe, clean/runtime/fold_probe.h: this expression on adversarial inputs) and the differential self-test of -// the real post shader (its fold port against its no-fold port on the crafted frame of clean/runtime/fold_selftest.h, -// raw bytes). Otherwise, or on any error, the device runs the no-fold port of the kernel. -#pragma once - -#ifndef LP_MUT -#define LP_MUT 0 -#endif -static const float kEncK = asfloat(0x3dddf184u); // 0.36f * 0.30102999566398120f - -// the 8-value conversion of the history codes (x0 .. x2: the FP32 log2 arguments) and the three display values -// d0 .. d2 (already rounded to FP32, passed as opaque values by the caller); z = an opaque zero (mutant 3 only) -uint4 lp_encode_fold(float x0, float x1, float x2, float d0, float d1, float d2, uint z) -{ - v8f cv; -#if LP_MUT == 3 // negative control (never shipped): two roundings, the FP32 product made opaque before the conversion - cv.v[0] = asfloat(asuint(kEncK * x0) ^ z); - cv.v[1] = asfloat(asuint(kEncK * x1) ^ z); - cv.v[2] = asfloat(asuint(kEncK * x2) ^ z); -#else - cv.v[0] = mad(x0, kEncK, 0.0f); - cv.v[1] = mad(x1, kEncK, 0.0f); - cv.v[2] = mad(x2, kEncK, 0.0f); -#endif - cv.v[3] = d0; - cv.v[4] = d1; - cv.v[5] = d2; - cv.v[6] = 0.0f; - cv.v[7] = 0.0f; - return wg_to_f16_ags(cv); -} diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index 189ecbb2..492c509f 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -202,10 +202,6 @@ "l_direct_color", "-" ], - [ - "l_fold_probe", - "-" - ], [ "l_linear_32_32_f16", "l_ops.co" diff --git a/clean/dx12/kernels/l_fold_probe.hlsl b/clean/dx12/kernels/l_fold_probe.hlsl deleted file mode 100644 index 37c2451b..00000000 --- a/clean/dx12/kernels/l_fold_probe.hlsl +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-License-Identifier: MIT -// Start-up probe of the driver's conversion fold used by Model L's l_post_recon_p12 (hlsl/l_post_encode.hlsli): the -// device runs it once (clean/runtime/fold_probe.h holds the inputs and the expected bytes, computed from the exact -// products) and takes the no-fold port of the kernel when any history code differs. WrongGPU's own code. -// Buffer: lane l reads six FP32 values at byte 24 l (x0 x1 x2 = log2 arguments of the history codes, d0 d1 d2 = display -// values) and writes the three converted dwords at byte 768 + 12 l (q0 | q1 << 16, q2 | h(d0) << 16, h(d1) | h(d2) << 16). -// @wg-kernel l_fold_probe co=- block=32 wave=32 args=p -#include "wg_platform.hlsli" -WG_ARGS_BEGIN - uint dst; -WG_ARGS_END -#include "l_post_encode.hlsli" - -WG_KERNEL(l_fold_probe) -{ - WG_SETUP(); - const uint lane = wg_lane_id(); - const uint z = wg_grid_ >> 31; // 0, opaque to the compiler - const uint in0 = lane * 24u; - const float x0 = wg_ldf(dst, in0), x1 = wg_ldf(dst, in0 + 4u), x2 = wg_ldf(dst, in0 + 8u); - const float d0 = asfloat(asuint(wg_ldf(dst, in0 + 12u)) ^ z), d1 = asfloat(asuint(wg_ldf(dst, in0 + 16u)) ^ z); - const float d2 = asfloat(asuint(wg_ldf(dst, in0 + 20u)) ^ z); - const uint4 hc = lp_encode_fold(x0, x1, x2, d0, d1, d2, z); - const uint o = 768u + lane * 12u; - wg_st32(dst, o, hc.x); - wg_st32(dst, o + 4u, hc.y); - wg_st32(dst, o + 8u, hc.z); -} diff --git a/clean/dx12/kernels/l_post.hlsl b/clean/dx12/kernels/l_post.hlsl index 48abc660..d214e775 100644 --- a/clean/dx12/kernels/l_post.hlsl +++ b/clean/dx12/kernels/l_post.hlsl @@ -31,8 +31,7 @@ // gives 0), the reduced codes rint(clamp(mean, 0, 1) 1023). // // @wg-kernel l_post_recon co=l_post.co block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p -// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_ENCFOLD=1 needs=fold isa=v_fma_mixlo_f16@3dddf184+v_fma_mixhi_f16@3dddf184==6,s_round_mode==0 -// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 file=l_post_recon_p12__nofold block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1 needs=nofold +// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1 // @wg-kernel l_post_recon_ldr co=l_post.co entry=l_post_recon_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_LDR=1 // @wg-kernel l_post_recon_p12_ldr co=l_post.co entry=l_post_recon_p12_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_LDR=1 #include "wg_platform.hlsli" @@ -50,7 +49,6 @@ WG_ARGS_BEGIN uint expo; WG_ARGS_END -#include "l_post_encode.hlsli" // kEncK and the history encode through the driver's conversion fold #define kThreads 256 #define kTileX 32 #define kTilePairs 8 @@ -58,6 +56,7 @@ WG_ARGS_END #define kLWin 4 #define kTexDw 6 // staged texel: f0 f1 | f2 f3 | f4 f5 | f6 f7 | f8 kA | kB kC (FP16 bits) +static const float kEncK = asfloat(0x3dddf184u); // 0.36f * 0.30102999566398120f static const float kDecK = asfloat(0x4113a428u); // 3.32192809488736235f / 0.36f static const float kDecLn = asfloat(0x40ccac9au); // 2.30258509299404568f / 0.36f static const float kDecPolyMax = asfloat(0x3d60233du); // 0.35f / kDecLn @@ -84,14 +83,11 @@ float opq(float x) { return asfloat(asuint(x) ^ g_z); } half opqh(half x) { return asfloat16((uint16_t)(asuint16(x) ^ (uint16_t)g_z)); } float mul_nz(float a, float b) { return mad(a, b, asfloat(g_z | 0x80000000u)); } #ifndef LP_MUT -#define LP_MUT 0 // 1..9: negative controls of the byte checks (never shipped) +#define LP_MUT 0 // 1, 2, 4 ... 9: negative controls of the byte checks (never shipped) #endif #ifndef LP_DP #define LP_DP 1 #endif -#ifndef LP_ENCFOLD -#define LP_ENCFOLD 0 // 1 (l_post_recon_p12's fold port): the history encode through the driver's conversion fold -#endif #ifndef LP_FMANZ #define LP_FMANZ 1 #endif @@ -593,14 +589,6 @@ WG_KERNEL(l_post_recon) if (second) g = g1; float hh[3], oo[3]; finish_pixel(g, x, y0 + row, px, second ? py1 : py0, s, hh, oo); -#if !LP_LDR && LP_ENCFOLD - float xe[3], de[3]; - [unroll] for (int ce = 0; ce < 3; ++ce) { - xe[ce] = enc_log(hh[ce]); - de[ce] = opq(clampf(enc_e(oo[ce]), 0.0f, kDisplayMax)); - } - const uint4 hc = lp_encode_fold(xe[0], xe[1], xe[2], de[0], de[1], de[2], g_z); -#else v8f cv; [unroll] for (int ci = 0; ci < 3; ++ci) { #if LP_LDR @@ -618,19 +606,14 @@ WG_KERNEL(l_post_recon) cv.v[6] = 0.0f; cv.v[7] = 0.0f; const uint4 hc = wg_to_f16_ags(cv); -#endif const int y = y0 + row; if (and(active, or(!second, has1))) { #if LP_LDR const uint q0 = hc.x & 0xffffu, q1 = hc.x >> 16, q2 = hc.y & 0xffffu; -#else -#if LP_ENCFOLD - const uint q0 = hc.x & 0xffffu, q1 = hc.x >> 16, q2 = hc.y & 0xffffu; // the fold's +0 addend gives +0 #else const uint q0 = mul_zero(kEncK, enc_log(hh[0])) ? 0u : (hc.x & 0xffffu); const uint q1 = mul_zero(kEncK, enc_log(hh[1])) ? 0u : (hc.x >> 16); const uint q2 = mul_zero(kEncK, enc_log(hh[2])) ? 0u : (hc.y & 0xffffu); -#endif #endif wg_st64(hist_out, (uint)(y * P_dw + x) * 8u, uint2(q0 | (q1 << 16), q2)); const int di = y * pitch + x; diff --git a/clean/dx12/tests/kernel_replay.cpp b/clean/dx12/tests/kernel_replay.cpp index 97b916da..96d123f2 100644 --- a/clean/dx12/tests/kernel_replay.cpp +++ b/clean/dx12/tests/kernel_replay.cpp @@ -591,7 +591,6 @@ int main(int argc, char** argv) { auto d = make_d3d12_device(argv[1], &err); if (!d) { std::printf("device: %s\n", err.c_str()); return 1; } std::printf("device: %s\n", d->name().c_str()); - if (!d->note().empty()) std::printf("device note: %s\n", d->note().c_str()); // e.g. the conversion-fold guard std::ifstream f(argv[2]); if (!f) { std::printf("cannot read %s\n", argv[2]); return 2; } const std::string base = dir_of(argv[2]); diff --git a/clean/dx12/tests/test_fold_probe.py b/clean/dx12/tests/test_fold_probe.py deleted file mode 100644 index f6ad8094..00000000 --- a/clean/dx12/tests/test_fold_probe.py +++ /dev/null @@ -1,104 +0,0 @@ -# SPDX-License-Identifier: MIT -"""CPU checks of the conversion-fold guard: clean/runtime/fold_probe.h and fold_selftest.h are what their generators -write (inputs and expected bytes from exact rationals), the self-test frame holds its value classes on every encode -site, and build_dx12.py's needs= / isa= declarations parse (and refuse bad values), the literal-tied ISA count included. -The GPU side (the probe kernel against these bytes, the self-test of the real post ports, the two-roundings mutant) is -a GPU-slot check.""" -import os -import subprocess -import sys -import tempfile -import unittest - -HERE = os.path.dirname(os.path.abspath(__file__)) -ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) -sys.path.insert(0, ROOT) - -from clean.dx12 import build_dx12 as B # noqa: E402 - - -class FoldProbeTest(unittest.TestCase): - def test_header_is_generated(self): - with tempfile.TemporaryDirectory() as tmp: - out = os.path.join(tmp, "fold_probe.h") - subprocess.run([sys.executable, os.path.join(ROOT, "clean", "runtime", "gen_fold_probe.py"), out], check=True, - capture_output=True) - with open(out, encoding="utf-8") as f: - want = f.read() - with open(os.path.join(ROOT, "clean", "runtime", "fold_probe.h"), encoding="utf-8") as f: - self.assertEqual(f.read(), want) - - def test_selftest_header_is_generated(self): - with tempfile.TemporaryDirectory() as tmp: - out = os.path.join(tmp, "fold_selftest.h") - subprocess.run([sys.executable, os.path.join(ROOT, "clean", "runtime", "gen_fold_selftest.py"), out], - check=True, capture_output=True) - with open(out, encoding="utf-8") as f: - want = f.read() - with open(os.path.join(ROOT, "clean", "runtime", "fold_selftest.h"), encoding="utf-8") as f: - self.assertEqual(f.read(), want) - - def test_selftest_frame_classes(self): - """Every one of the six encode sites (row of the pair, channel) holds FP32-product ties of both sides that a - second rounding gets wrong, the zero product and the floor.""" - sys.path.insert(0, os.path.join(ROOT, "clean", "runtime")) - import gen_fold_selftest as G - colour, _, _ = G.build() - floor = G.Fraction(G.f32(G.bits32(1e-4))) - for site in range(G.SITES): - row, ch = divmod(site, 3) - vals = [colour[3 * (y * G.W + x) + ch] for y in range(row, G.H, 2) for x in range(G.W)] - sides = {"up": 0, "down": 0} - for cb in vals: - r = G.classify(G.f32(cb)) - if r and r[2]: - sides[r[1]] += 1 - self.assertGreaterEqual(min(sides.values()), 40, (site, sides)) - ms = [G.m_of(G.f32(cb)) for cb in vals] - self.assertIn(G.Fraction(1), ms) # x = 0: the zero product - self.assertTrue(any(m < floor for m in ms) and any(floor <= m < 2 * floor for m in ms)) - - def test_isa_rules(self): - self.assertEqual(B.parse_isa_rules("v_fma_mixlo_f16+v_fma_mixhi_f16>=6,s_round_mode==0", "t"), - [([("v_fma_mixlo_f16", None), ("v_fma_mixhi_f16", None)], ">=", 6), - ([("s_round_mode", None)], "==", 0)]) - self.assertEqual(B.parse_isa_rules("v_fma_mixlo_f16@3dddf184+v_fma_mixhi_f16@3dddf184==6", "t"), - [([("v_fma_mixlo_f16", "3dddf184"), ("v_fma_mixhi_f16", "3dddf184")], "==", 6)]) - with self.assertRaises(SystemExit): - B.parse_isa_rules("v_fma_mixlo_f16>6", "t") - with self.assertRaises(SystemExit): - B.parse_isa_rules("v_fma_mixlo_f16@3dddf18==6", "t") - - def test_isa_count(self): - lines = [" v_fma_mixlo_f16 v3, v3, 0xf184, 0 // 000000008224: CC210003 0201FF03 3DDDF184", - " v_fma_mixhi_f16 v10, v2, 0xf184, 0 // 000000008244: CC22000A 0201FF02 3DDDF184", - " v_fma_mixlo_f16 v3, v3, 0xf184, v4 // 000000008254: CC210003 0411FF03 3DDDF184", - " v_fma_mixlo_f16 v3, v3, 0xf184, 0 op_sel_hi:[1,0,0] // 000000008264: CC210003 0201FF03 3DDDF184", - " v_fma_mixlo_f16 v3, v3, 0xa8c1, 0 // 000000008274: CC210003 0201FF03 BF38A8C1", - " v_mul_f32_e32 v1, 0x3dddf184, v1 // 0000000009DC: 100202FF 3DDDF184"] - self.assertEqual(B.isa_count(lines, "v_fma_mixlo_f16", None), 4) - self.assertEqual(B.isa_count(lines, "v_fma_mixlo_f16", "3dddf184"), 1) # no other addend, no op_sel, literal - self.assertEqual(B.isa_count(lines, "v_fma_mixhi_f16", "3dddf184"), 1) - - def test_post_ports(self): - decls = [d for d in B.kernel_decls(os.path.join(ROOT, "clean", "dx12", "kernels", "l_post.hlsl")) - if d["name"] == "l_post_recon_p12"] - self.assertEqual(sorted(d["needs"] for d in decls), ["fold", "nofold"]) - fold = [d for d in decls if d["needs"] == "fold"][0] - self.assertEqual(fold["defines"].get("LP_ENCFOLD"), "1") - self.assertIn("@3dddf184", fold["isa"]) # the fold's sites: kEncK as the literal, addend 0 - nofold = [d for d in decls if d["needs"] == "nofold"][0] - self.assertNotIn("LP_ENCFOLD", nofold["defines"]) - self.assertEqual(nofold["file"], "l_post_recon_p12__nofold") - - def test_bad_needs(self): - with tempfile.TemporaryDirectory() as tmp: - p = os.path.join(tmp, "k.hlsl") - with open(p, "w") as f: - f.write("// @wg-kernel x co=- block=32 wave=32 args=p needs=sometimes\n") - with self.assertRaises(SystemExit): - B.kernel_decls(p) - - -if __name__ == "__main__": - unittest.main() diff --git a/clean/l/runner/l_post_edge_check.py b/clean/l/runner/l_post_edge_check.py index b11e6ec3..35d989f0 100644 --- a/clean/l/runner/l_post_edge_check.py +++ b/clean/l/runner/l_post_edge_check.py @@ -10,8 +10,7 @@ first dword after the full history); /mut8 (counter + LP_MUT=8: the fast path's predicate always false) and /mut9 (LP_MUT=9: the predicate without its size guard, the bug of images narrower or shorter than 4). --run (GPU, one GPU slot). The kernels l_post_recon, l_post_recon_p12, l_post_recon_ldr and l_post_recon_p12_ldr (the -history sample, history_cr, is shared by all four), and l_post_recon_p12 again with the no-fold port -(WG_TEST_FOLD_PROBE_FAIL=1), on every size of SIZES (1 x 1 ... 97 x 35: below 4, odd, a tile's interior boundary in the +history sample, history_cr, is shared by all four), on every size of SIZES (1 x 1 ... 97 x 35: below 4, odd, a tile's interior boundary in the middle of a wave) and every kind of KINDS, with a valid history (no reset): - random: Model M's random inputs (random motions around a random base); - center / left / right / top / bottom: every motion points at one history position: the centre, or just inside @@ -21,6 +20,9 @@ - split: even columns point inside, odd columns at an edge (every wave mixes both); split_half: in each 32-column tile the first 16 columns (half the width when narrower) inside, the rest at an edge (the boundary in the middle of a wave). +Every replay leg (the shipped ports, hscount, mut8, mut9) starts from an emptied output folder and counts only when +kernel_replay exits 0 and every case wrote every output at its full size; otherwise the leg is invalid and every check +that uses it fails (a failed mutant run is never taken as caught). Checks: - raw bytes, D3D12 == HIP, every output (full history, reduced, display; the l_d3d12_ops_check rules: each back end's own sentinel, the pitch padding and 64 slack bytes untouched), and the full history per edge band (the two columns @@ -180,11 +182,10 @@ def cases(kernel, rng): def prepare(build, rocm): C32.compile_kernels(build, rocm) - names = KERNELS + ["l_fold_probe", "native_fill"] + names = KERNELS + ["native_fill"] prepare_replay(build, rocm, names) from clean.runner.hip import hip_child_environment env, _ = hip_child_environment(rocm_dir=rocm) - env = dict(env, WG_DX12_ISA_CHECK="0") # test variants: the ISA rules are the shipped ports' (checked above) for tag, defs in VARIANTS.items(): cmd = [sys.executable, os.path.join(ROOT, "clean", "dx12", "build_dx12.py"), "--out", os.path.join(build, tag), "--only", ",".join(names), "-j", "1"] @@ -205,12 +206,19 @@ def replay(build_dir, exe, cdir, lines, env_extra=None): env = {k: v for k, v in os.environ.items() if not any(s in k.upper() for s in ("HIP", "LLVM", "ROCM", "HSA")) or k.upper() in ("HIP_PLATFORM",)} env["WG_D3D12_ARENA_MB"] = "512" - env.pop("WG_TEST_FOLD_PROBE_FAIL", None) env.update(env_extra or {}) r = subprocess.run([exe, build_dir, os.path.join(cdir, "cases.txt")], env=env, capture_output=True, text=True, timeout=150, creationflags=0x08000000) - note = [ln for ln in r.stdout.splitlines() if ln.startswith("device note:")] - return {"exit": r.returncode, "note": note[0] if note else "", "tail": r.stdout[-400:] + r.stderr[-200:]} + return {"exit": r.returncode, "tail": r.stdout[-400:] + r.stderr[-200:]} + + +def empty_dir(d): + """The leg's output folder, emptied of the files of an earlier run (files only, this folder only).""" + os.makedirs(d, exist_ok=True) + for f in os.listdir(d): + p = os.path.join(d, f) + if os.path.isfile(p): + os.remove(p) def compare_case(cdir, name, case, wants, counter=False): @@ -218,11 +226,12 @@ def compare_case(cdir, name, case, wants, counter=False): dword after the full history holds the counter (excluded from the slack rule, returned).""" dw, dh = case["dw"], case["dh"] n_hist = dw * dh * 8 - res = {"case": name, "label": case["label"], "outputs": []} + res = {"case": name, "label": case["label"], "outputs": [], "complete": True} count = None for oi, want in enumerate(wants): path = os.path.join(cdir, f"{name}.o{oi}") got = open(path, "rb").read() if os.path.exists(path) else b"" + res["complete"] = res["complete"] and len(got) == len(want) dmask = case["defined"][oi].copy() if counter and oi == 0 and len(got) == len(want): count = int(np.frombuffer(got[n_hist:n_hist + 4], np.uint32)[0]) - 0x5A5A5A5A @@ -268,13 +277,12 @@ def run(build, rocm, part="all"): res = {"sizes": SIZES, "kinds": KINDS, "part": part, "legs": {}} cdir_base = os.path.join(build, "replay-edge") for leg, bdir, env_extra, counter in () if part == "oracle" else (("d3d12", build, None, False), - ("d3d12_nofold", build, {"WG_TEST_FOLD_PROBE_FAIL": "1"}, False), ("hscount", os.path.join(build, "hscount"), None, True), ("mut8", os.path.join(build, "mut8"), None, True), ("mut9", os.path.join(build, "mut9"), None, False)): cdir = os.path.join(cdir_base, leg) - os.makedirs(cdir, exist_ok=True) - names = [n for n in allc if leg != "d3d12_nofold" or allc[n]["kernel"] == "l_post_recon_p12"] + empty_dir(cdir) + names = list(allc) lines = [] for n in names: lines += OPS.replay_lines(n, allc[n], cdir) @@ -283,7 +291,9 @@ def run(build, rocm, part="all"): lines += OPS.replay_lines(n, allc[n], cdir, grid=allc[n]["grid"] // 2, prefix="negative_") r = replay(bdir, exe, cdir, lines, env_extra) out = [compare_case(cdir, n, allc[n], hip_out[n], counter) for n in names] - leg_res = {"replay_exit": r["exit"], "device_note": r["note"], "replay_tail": r["tail"], "cases": len(out), + complete = sum(c["complete"] for c in out) + leg_res = {"replay_exit": r["exit"], "complete_cases": complete, "valid": r["exit"] == 0 and complete == len(out), + "replay_tail": r["tail"], "cases": len(out), "identical": sum(c["identical"] for c in out), "edges": {k: {"pixels": sum(c["outputs"][0].get("edges", {}).get(k, {}).get("pixels", 0) for c in out), "differ": sum(c["outputs"][0].get("edges", {}).get(k, {}).get("differ", 0) for c in out)} @@ -305,7 +315,9 @@ def run(build, rocm, part="all"): wants = hip_out[neg] got = [open(os.path.join(cdir, f"negative_{neg}.o{oi}"), "rb").read() if os.path.exists(os.path.join(cdir, f"negative_{neg}.o{oi}")) else b"" for oi in range(len(wants))] - leg_res["negative_control"] = {"case": neg, "half_grid": True, "caught": got != list(wants)} + full = all(len(gv) == len(wv) for gv, wv in zip(got, wants)) + leg_res["negative_control"] = {"case": neg, "half_grid": True, "complete": full, + "caught": full and got != list(wants)} res["legs"][leg] = leg_res # the oracle: HIP's l_post_recon / l_post_recon_ldr against Model L's reconstruction on every case's inputs # (cases under 100 pixels: Model L's post acceptance over all of them together, where one value is not 1 %) @@ -334,14 +346,12 @@ def run(build, rocm, part="all"): checks = {} if part != "oracle": checks.update({ - "d3d12_identical": L["d3d12"]["identical"] == L["d3d12"]["cases"] and L["d3d12"]["replay_exit"] == 0, - "fold_port_selected": "passed: the fold ports run" in L["d3d12"]["device_note"], - "nofold_identical": L["d3d12_nofold"]["identical"] == L["d3d12_nofold"]["cases"], - "nofold_port_selected": "the no-fold ports run" in L["d3d12_nofold"]["device_note"], - "hscount_identical": L["hscount"]["identical"] == L["hscount"]["cases"], - "hscount_counter_ok": L["hscount"]["counter"]["expectation_failures"] == 0, - "mut8_caught": L["mut8"]["counter"]["expectation_failures"] > 0, - "mut9_caught": L["mut9"]["identical"] < L["mut9"]["cases"], + "legs_valid": all(v["valid"] for v in L.values()), + "d3d12_identical": L["d3d12"]["valid"] and L["d3d12"]["identical"] == L["d3d12"]["cases"], + "hscount_identical": L["hscount"]["valid"] and L["hscount"]["identical"] == L["hscount"]["cases"], + "hscount_counter_ok": L["hscount"]["valid"] and L["hscount"]["counter"]["expectation_failures"] == 0, + "mut8_caught": L["mut8"]["valid"] and L["mut8"]["counter"]["expectation_failures"] > 0, + "mut9_caught": L["mut9"]["valid"] and L["mut9"]["identical"] < L["mut9"]["cases"], "negative_control_caught": bool(L["d3d12"].get("negative_control", {}).get("caught"))}) if part != "replay": checks["oracle_pass"] = (res["oracle"]["passed"] == res["oracle"]["cases"] and pooled["history"]["pass"] and diff --git a/clean/runtime/device_d3d12.cpp b/clean/runtime/device_d3d12.cpp index 705979c5..9d738819 100644 --- a/clean/runtime/device_d3d12.cpp +++ b/clean/runtime/device_d3d12.cpp @@ -20,8 +20,6 @@ #include "embedded.h" #include "rdna4.h" #include "wmma_probe.h" -#include "fold_probe.h" -#include "fold_selftest.h" #include #include @@ -77,9 +75,6 @@ struct NativeKernel { // workgroups (manifest "group", build_dx12.py group=; 0 = block). The launch keeps the HIP geometry (grid x block). unsigned group = 0, gscale = 1; // gscale: D3D12 workgroups per launched workgroup (manifest "gscale", default block / group) bool ext_doubles = false; // manifest "fp64": "ext": binary64 fma / division (DoublePrecisionFloatShaderOps) - // manifest "needs": "fold" (run only when the start-up probe of the driver's conversion fold passed) or "nofold" - // (only when it failed): two ports of one kernel and code object whose exactness depends on that fold - std::string needs; std::vector args; wgc::Digest digest{}; NativeCom pso; @@ -220,7 +215,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { } if (!create_objects()) return false; if (!profile_init()) return false; - return reserve_events(64) && (!borrowed_ || driver_probe()) && fold_probe() && gelu_lut_upload() && gelu_t5_upload(); + return reserve_events(64) && (!borrowed_ || driver_probe()) && gelu_lut_upload() && gelu_t5_upload(); } // The MLP kernels' exact GELU + FP8 table (hlsl/wg_gelu_lut.hlsli) at its fixed arena offset, reserved in @@ -459,8 +454,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { std::string note() const override { return "native D3D12: arena " + std::to_string(arena_bytes_ >> 20) + " MiB; no HIP runtime or fallback; timestamp frequency " + std::to_string(timestamp_frequency_) + " Hz" + - (borrowed_ ? "; borrowed device/queue; exact FP16/FP8 driver-channel probe passed" : "") + - (fold_note_.empty() ? "" : "; " + fold_note_); + (borrowed_ ? "; borrowed device/queue; exact FP16/FP8 driver-channel probe passed" : ""); } bool adapter_luid(uint8_t out[8]) const override { if (!out) return false; @@ -644,11 +638,6 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (direct && !direct_bound_) return fail("native D3D12 direct kernel requires a bound direct frame: " + name); // Check the variant BEFORE touching args: a 37-DWORD post variant cannot satisfy a 42-DWORD port. if (!variant_loaded(k)) return false; - return dispatch(k, name, grid, block, args); - } - // One launch of the port k (launch() after its port selection; the fold self-test runs both ports of a kernel). - bool dispatch(NativeKernel& k, const std::string& name, unsigned grid, unsigned block, void** args) { - const bool direct = k.rs == "direct"; if (block != k.block) return fail("native D3D12 kernel " + name + ": block must be " + std::to_string(k.block)); if (!grid || grid > 0x7fffffffu) return fail("native D3D12 kernel " + name + ": grid must be 1..2147483647"); const uint64_t groups = (uint64_t)grid * k.gscale; // D3D12 workgroups @@ -1009,9 +998,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { return native_filename(co) && co.size() > 3 && co.substr(co.size() - 3) == ".co"; } // variant_loaded without an error: whether the runtime's loaded code objects select this port. - bool needs_met(const NativeKernel& k) const { return k.needs.empty() || (k.needs == "fold") == fold_ok_; } bool variant_matches(const NativeKernel& k) const { - if (!needs_met(k)) return false; if (k.co == "-") return true; const size_t slash = k.name.find('/'); if (slash == std::string::npos) return untagged_cos_.count(k.co) != 0; @@ -1041,9 +1028,6 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { } ID3D12GraphicsCommandList* cl() const { return external_ ? ext_list_ : list_.get(); } bool variant_loaded(const NativeKernel& k) { - if (!needs_met(k)) - return fail("native D3D12 " + k.name + ": the port for the conversion-fold probe's result (" + - std::string(fold_ok_ ? "fold" : "nofold") + ") is missing"); if (k.co == "-") return true; const size_t slash = k.name.find('/'); if (slash == std::string::npos) { @@ -1294,11 +1278,6 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (!fp->is_string() || fp->text != "ext") return fail("native D3D12 manifest: invalid fp64 for " + k.name); k.ext_doubles = true; } - if (const auto* nd = entry.find("needs")) { - if (!nd->is_string() || (nd->text != "fold" && nd->text != "nofold")) - return fail("native D3D12 manifest: invalid needs for " + k.name); - k.needs = nd->text; - } const auto* signature = entry.find("rs"); if (signature) { if (!signature->is_string() || (signature->text != "buffer" && signature->text != "tex" && signature->text != "direct")) @@ -1346,10 +1325,9 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { } if (words != k.dwords) return fail("native D3D12 manifest: argument DWORD count mismatch for " + k.name); k.file = directory + filename; const std::string key = k.name; - auto same = kernels_.equal_range(key); // one port per (name, code object, needs) + auto same = kernels_.equal_range(key); // one port per (name, code object) for (auto i = same.first; i != same.second; ++i) - if (i->second.co == k.co && i->second.needs == k.needs) - return fail("native D3D12 manifest: duplicate kernel " + key + " (" + k.co + ")"); + if (i->second.co == k.co) return fail("native D3D12 manifest: duplicate kernel " + key + " (" + k.co + ")"); kernels_.emplace(key, std::move(k)); } return true; @@ -1429,124 +1407,6 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (!ok) err_ = error; return ok; } - // Start-up guard of the driver's conversion fold: only when a port needs it. Two steps, both must pass, else (or on - // any error) the "nofold" ports run (fail closed): - // 1. probe: l_fold_probe (fold_probe.h) on adversarial inputs of the shared encode expression, against expected - // bytes from exact rationals; - // 2. differential self-test (fold_selftest): the REAL post shader, its fold port and its no-fold port (exact by - // construction), on one crafted frame (fold_selftest.h); the raw output bytes must be equal. This tests the - // driver's lowering of the shader that will run, at its six encode sites. - // Logged once per process (note()). A failing guard is not an error: the device keeps going on the no-fold ports. - bool fold_probe() { - bool wanted = false; - std::string uncovered; - for (const auto& kv : kernels_) { - if (kv.second.needs != "fold") continue; - wanted = true; - if (kv.first != "l_post_recon_p12") uncovered = kv.first; // a fold port with no self-test of its own - } - fold_ok_ = false; - if (!wanted) return true; - std::string note; - const std::string before = err_; - auto probe = kernels_.find("l_fold_probe"); - if (!uncovered.empty()) { - note = "no self-test for " + uncovered; - } else if (probe == kernels_.end() || probe->second.co != "-" || probe->second.block != 32 || - probe->second.dwords != 1 || probe->second.args.size() != 1 || probe->second.args[0].kind != 'p') { - note = "no l_fold_probe kernel"; - } else { - DevPtr buf = 0; - uint8_t bytes[fold_probe::kBytes]; - fold_probe::fill(bytes); - uint32_t out[96] = {}; - void* args[] = {&buf}; - bool ok = alloc(&buf, fold_probe::kBytes) && upload(buf, bytes, sizeof bytes) && launch("l_fold_probe", 1, 32, args) && - download(out, buf + fold_probe::kOutOffset, sizeof out); - if (buf) free(buf); - const char* fail_test = std::getenv("WG_TEST_FOLD_PROBE_FAIL"); // tests of the fallback - if (ok && fail_test && fail_test[0] == '1') out[0] ^= 1u; - const int bad = ok ? fold_probe::first_mismatch(out) : -1; - if (!ok) note = "probe run failed (" + err_ + ")"; - else if (bad >= 0) note = "probe mismatch at half " + std::to_string(bad); - else fold_ok_ = fold_selftest(¬e); - } - err_ = before; - // one line per device: in note(), which the runtime logs once when the network runtime is ready - fold_note_ = std::string("conversion-fold probe and self-test ") + - (fold_ok_ ? "passed: the fold ports run" : "failed (" + note + "): the no-fold ports run"); - return true; - } - // The differential self-test of l_post_recon_p12 (fold_probe step 2). The frame (fold_selftest.h; render size = - // output size, jitter 0, reset, exposure 1, post scale 0, codes and kernel shape 0) makes each full-history value the - // render colour, so the history codes are the encode of crafted values (FP32-product ties on both sides, zero and - // FP16-subnormal products, the floor). Each port's outputs start filled with its own sentinel; every defined byte of - // the full history, the reduced history and the display image must be equal. True only when both ports ran. - bool fold_selftest(std::string* note) { - NativeKernel *fold = nullptr, *nofold = nullptr; - auto range = kernels_.equal_range("l_post_recon_p12"); - for (auto i = range.first; i != range.second; ++i) { - if (i->second.needs == "fold") fold = &i->second; - if (i->second.needs == "nofold") nofold = &i->second; - } - auto abi = [](const NativeKernel* k) { - if (!k || k->block != 256 || k->rs != "buffer" || k->args.size() != 14) return false; - for (int n = 0; n < 12; ++n) if (k->args[n].kind != 'p') return false; - return k->args[12].kind == 'b' && k->args[12].bytes == 64 && k->args[13].kind == 'p'; - }; - if (!abi(fold) || !abi(nofold)) { *note = "self-test: the fold / no-fold ports of l_post_recon_p12 are missing"; return false; } - constexpr int W = ::fold_selftest::kW, H = ::fold_selftest::kH, N = W * H; - constexpr int pitch = (W + 63) & ~63; - constexpr size_t hist = size_t(N) * 8, reduced = size_t(W / 2) * (H / 2) * 4, display = size_t(3) * pitch * H * 2; - constexpr size_t outs = hist + reduced + display; - // layout: colour | A8 | C4 | previous history | motions x 3 | post blob | outputs (fold) | outputs (no-fold) - const size_t o_col = 0, o_a = o_col + size_t(N) * 16, o_c = o_a + size_t(N) * 8, o_hp = o_c + size_t(N) * 4, - o_m = o_hp + hist, o_w = o_m + size_t(N) * 12, o_out = o_w + 256, total = o_out + 2 * outs; - std::vector host(total, 0); - for (int i = 0; i < N; ++i) std::memcpy(&host[o_col + size_t(i) * 16], &::fold_selftest::kColour[3 * i], 12); - std::memset(&host[o_out], 0xA5, outs); - std::memset(&host[o_out + outs], 0x5A, outs); - struct { int32_t rw, rh, dw, dh, reset, pad0, pad1, pad2; float rx, ry, jx, jy, e, e_prev, ratio, pad3; } prm = - {W, H, W, H, 1, 0, 0, 0, 1.0f, 1.0f, 0.0f, 0.0f, 1.0f, 1.0f, 1.0f, 0.0f}; - static_assert(sizeof prm == 64, "WgPostParams"); - DevPtr buf = 0; - std::vector got(2 * outs); - bool ok = alloc(&buf, total) && upload(buf, host.data(), total); - for (int port = 0; ok && port < 2; ++port) { - DevPtr p[12], none = 0; - const DevPtr out = buf + o_out + port * outs; - const DevPtr in[9] = {buf + o_col, buf + o_a, buf + o_a, buf + o_c, buf + o_hp, buf + o_m, buf + o_m + N * 4, - buf + o_m + N * 8, buf + o_w}; - for (int n = 0; n < 9; ++n) p[n] = in[n]; - p[9] = out; p[10] = out + hist; p[11] = out + hist + reduced; - void* args[14]; - for (int n = 0; n < 12; ++n) args[n] = &p[n]; - args[12] = &prm; args[13] = &none; - const unsigned grid = unsigned((W + 31) / 32) * unsigned(((H + 1) / 2 + 7) / 8); - ok = dispatch(port == 0 ? *fold : *nofold, "l_post_recon_p12", grid, 256, args); - } - ok = ok && download(got.data(), buf + o_out, 2 * outs); - const std::string error = err_; - if (buf) free(buf); - if (!ok) { *note = "self-test run failed (" + error + ")"; return false; } - if (const char* dump = std::getenv("WG_TEST_FOLD_SELFTEST_DUMP")) { // tests: both ports' outputs, raw - if (FILE* f = std::fopen(dump, "wb")) { std::fwrite(got.data(), 1, got.size(), f); std::fclose(f); } - } - size_t differ = 0, hist_differ = 0; - for (size_t i = 0; i < outs; ++i) { - if (i >= hist + reduced) { // display: only the pixels of each pitch row are defined - const size_t px = ((i - hist - reduced) / 2) % pitch; - if (px >= size_t(W)) continue; - } - if (got[i] != got[outs + i]) { ++differ; if (i < hist) ++hist_differ; } - } - if (differ) { - *note = "self-test: the fold port differs from the no-fold port in " + std::to_string(differ) + " bytes (" + - std::to_string(hist_differ) + " of the full history)"; - return false; - } - return true; - } bool create_device() { dxgi_dll_ = LoadLibraryExW(L"dxgi.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32); d3d12_dll_ = LoadLibraryExW(L"d3d12.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32); @@ -1835,8 +1695,6 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { std::string profile_session_, profile_failure_; bool profile_enabled_ = false; bool borrowed_ = false, native_binding_dirty_ = false, inside_pass_ = false; - bool fold_ok_ = false; // fold_probe's result - std::string fold_note_; // fold_probe's result as a note() clause (empty: no port needs the fold) bool direct_bound_ = false, direct_root_bound_ = false; D3D12_COMMAND_LIST_TYPE queue_type_ = D3D12_COMMAND_LIST_TYPE_COMPUTE; uint64_t debug_cursor_ = 0, debug_denied_ = 0; diff --git a/clean/runtime/device_dx12.cpp b/clean/runtime/device_dx12.cpp index ec064742..d06e9c60 100644 --- a/clean/runtime/device_dx12.cpp +++ b/clean/runtime/device_dx12.cpp @@ -524,9 +524,6 @@ class Dx12Device : public Device { } std::set seen; for (const wgc::Json& k : ks->items) { - // a port that needs the driver's conversion fold (manifest "needs": "fold") runs only on the native device after - // its start-up probe (device_d3d12.cpp fold_probe); this device has no probe and takes the exact "nofold" port - if (const wgc::Json* nd = k.find("needs"); nd && nd->is_string() && nd->text == "fold") continue; Kernel x; int64_t block = 0, wave = 0, dwords = 0; const wgc::Json *n = k.find("name"), *fl = k.find("file"), *ar = k.find("args"), *co = k.find("co"); diff --git a/clean/runtime/fold_probe.h b/clean/runtime/fold_probe.h deleted file mode 100644 index 70f08656..00000000 --- a/clean/runtime/fold_probe.h +++ /dev/null @@ -1,86 +0,0 @@ -// SPDX-License-Identifier: MIT -// Generated by clean/runtime/gen_fold_probe.py - do not edit. -// Start-up probe of the driver's conversion fold (clean/dx12/kernels/l_fold_probe.hlsl): inputs (32 lanes x -// 6 FP32 bits) and expected output dwords (32 x 3); nan_mask bit 0 / 1 = the low / high FP16 half must be a -// NaN (any sign and payload: the accepted port exception), the other halves must match bit for bit. -#pragma once -#include -#include - -namespace fold_probe { -constexpr unsigned kInBytes = 768, kOutOffset = 768, kOutBytes = 384, kBytes = 1152; -constexpr uint32_t kInput[192] = { - 0x3de24abbu, 0x3d1f8941u, 0x3db62c2du, 0x3f2bd000u, 0x3f2bd001u, 0x39091000u, - 0x3c4ef8bfu, 0xc00113f4u, 0x3e59edeeu, 0x39091001u, 0x3a9f9000u, 0x3a9f9001u, - 0xbe68340du, 0xbcb24799u, 0xbdd67815u, 0x40391000u, 0x40391001u, 0x470db000u, - 0xbc580e19u, 0xbd14b8fbu, 0x414eaeedu, 0x470db001u, 0x3f7e3000u, 0x3f7e3001u, - 0x3b516c39u, 0xbb2b5be7u, 0x3ddfd741u, 0x44a05000u, 0x44a05001u, 0x4641d000u, - 0xc012ab03u, 0x3e00a539u, 0x3e7b3c36u, 0x4641d001u, 0x44d9f000u, 0x44d9f001u, - 0xbd31fdc6u, 0xba79a633u, 0x3b9c5d3bu, 0x41f03000u, 0x41f03001u, 0x3f1ef000u, - 0xc023b7a8u, 0xc1d224c6u, 0xc0829783u, 0x3f1ef001u, 0x40363000u, 0x40363001u, - 0x399ebe40u, 0x39a5164du, 0xbb7e8d26u, 0x3c7e7000u, 0x3c7e7001u, 0x3ff3f000u, - 0x3d7cad50u, 0x3e21442eu, 0xc048c59bu, 0x3ff3f001u, 0x3e30d000u, 0x3e30d001u, - 0xbfb14539u, 0xbc3a10c1u, 0x3a24def0u, 0x3c367000u, 0x3c367001u, 0x4765f000u, - 0x3bec3d8au, 0x3c5c6168u, 0xbfb3b8b3u, 0x4765f001u, 0x38ad5000u, 0x38ad5001u, - 0x40d730a3u, 0x3db0b195u, 0xc1f4e428u, 0x39f53000u, 0x39f53001u, 0x3b62d000u, - 0xbd45be7du, 0xbf9fd313u, 0x3f6a13e2u, 0x3b62d001u, 0x4708f000u, 0x4708f001u, - 0x40fffe41u, 0x3af87eeau, 0x40d83302u, 0x42e15000u, 0x42e15001u, 0x45ba3000u, - 0xbfb97d1du, 0x4200a539u, 0x409b7fc4u, 0x45ba3001u, 0x3b825000u, 0x3b825001u, - 0xbb4fd635u, 0xbd0a16d9u, 0xc12a0fb5u, 0x43507000u, 0x43507001u, 0x3e0af000u, - 0x3b90cb2du, 0xbff04707u, 0xbed52be4u, 0x3e0af001u, 0x3fd5d000u, 0x3fd5d001u, - 0xbfa6e3aeu, 0xbd5eaff9u, 0xc028e86du, 0x388e5000u, 0x388e5001u, 0x41c35000u, - 0xc0133ea7u, 0x3b67c552u, 0x3bec6273u, 0x41c35001u, 0x44f35000u, 0x44f35001u, - 0x00000000u, 0x80000000u, 0x7fc00000u, 0x42747000u, 0x42747001u, 0x3ad11000u, - 0xc1549a78u, 0x00000000u, 0x3f800000u, 0x3ad11001u, 0x4081b000u, 0x4081b001u, - 0xbf800000u, 0x41a00000u, 0x399d4952u, 0x3c839000u, 0x3c839001u, 0x46f91000u, - 0xb951b717u, 0x358637bdu, 0xb58637bdu, 0x46f91001u, 0x45b45000u, 0x45b45001u, - 0x4102409du, 0x418bcba4u, 0x4154e574u, 0x3b4b1000u, 0x3b4b1001u, 0x40259000u, - 0xc0b2699fu, 0xc0404665u, 0x41854d96u, 0x40259001u, 0x46989000u, 0x46989001u, - 0xc151e910u, 0x416de3e1u, 0x4160a1b7u, 0x4534f000u, 0x4534f001u, 0x00000000u, - 0x403002c4u, 0xc039fbb4u, 0xc0700006u, 0x80000000u, 0x7fc00000u, 0x477fef00u, - 0xc091daeau, 0x3ffba9bbu, 0x40803132u, 0x477ff000u, 0x49742400u, 0x33800000u, - 0x40b5eb30u, 0x41a6c3e9u, 0x415e367eu, 0x33000001u, 0x40a3c481u, 0x40bde4b3u, - 0x4100a6e7u, 0x41a4f87du, 0xc0bd46f5u, 0x3f3b620cu, 0x408a8867u, 0x4081fd5cu, - 0xc0f9c053u, 0x40f6b935u, 0xc13caf44u, 0x40df1019u, 0x4038f79au, 0x40992297u, -}; -constexpr uint32_t kExpect[96] = { - 0x1c522222u, 0x395e20f0u, 0x0848395fu, 0xb2fe159bu, 0x084925e7u, 0x14fd14fcu, - 0x98d5a64au, 0x41c8a1cfu, 0x786e41c9u, 0x9c0795dbu, 0x786e3d99u, 0x3bf23bf2u, - 0x8ca50dadu, 0x65022211u, 0x720e6503u, 0x22f9b3f3u, 0x720f26ceu, 0x66d066d0u, - 0x86c49cd2u, 0x4f82103du, 0x38f84f82u, 0xc1b1b470u, 0x38f8b714u, 0x41b241b2u, - 0x023c0227u, 0x23f48ee5u, 0x3fa023f4u, 0x245f1ed8u, 0x3fa0b571u, 0x31873186u, - 0x950bb0cdu, 0x21b40478u, 0x7b3021b4u, 0x15f91266u, 0x7b30b0dfu, 0x056b056au, - 0x20c939d5u, 0x0faac2a2u, 0x1b160faau, 0xb0549d5cu, 0x1b172e57u, 0x78487848u, - 0x0abb3af0u, 0x570a39dcu, 0x6dd2570bu, 0x42f9b107u, 0x6dd23836u, 0x1c131c12u, - 0x9b7c8da1u, 0x5a84bc9bu, 0x30585a84u, 0xb2820fd8u, 0x3058a9c7u, 0x3eaf3eaeu, - 0x9e09b085u, 0x0472b493u, 0x4e1a0473u, 0x0e48b3fbu, 0x4e1b1267u, 0x679b679au, - 0x00000000u, 0x53a47e00u, 0x168853a4u, 0x0000bdc3u, 0x16892ef0u, 0x440e440eu, - 0x4056aef0u, 0x241c0221u, 0x77c8241du, 0x0002816cu, 0x77c98002u, 0x6da36da2u, - 0x3f933b0fu, 0x1a583dc5u, 0x412c1a59u, 0xb536b8d5u, 0x412d3f39u, 0x74c574c4u, - 0x3e72bdb0u, 0x69a83e16u, 0x000069a8u, 0xb50a34c5u, 0x8000b681u, 0x7bff7e00u, - 0x32d1b7e7u, 0x7c0036f2u, 0x00017c00u, 0x408538eeu, 0x00013e05u, 0x45ef451eu, - 0x40783af9u, 0x39dbb921u, 0x44104454u, 0x3aafbac4u, 0x46f9bd1du, 0x44c941c8u, -}; -constexpr uint8_t kNanMask[96] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; -inline void fill(uint8_t* buffer) { - std::memset(buffer, 0, kBytes); - std::memcpy(buffer, kInput, sizeof kInput); -} -inline bool half_ok(uint32_t got, uint32_t want, bool nan) { - if (nan) return (got & 0x7c00u) == 0x7c00u && (got & 0x03ffu) != 0u; - return got == want; -} -// index of the first differing output half (0 .. 191), or -1 when every half matches -inline int first_mismatch(const uint32_t* out) { - for (unsigned i = 0; i < 96; ++i) { - if (!half_ok(out[i] & 0xffffu, kExpect[i] & 0xffffu, (kNanMask[i] & 1u) != 0)) return (int)(2 * i); - if (!half_ok(out[i] >> 16, kExpect[i] >> 16, (kNanMask[i] & 2u) != 0)) return (int)(2 * i + 1); - } - return -1; -} -} // namespace fold_probe diff --git a/clean/runtime/fold_selftest.h b/clean/runtime/fold_selftest.h deleted file mode 100644 index 29d8ed70..00000000 --- a/clean/runtime/fold_selftest.h +++ /dev/null @@ -1,118 +0,0 @@ -// SPDX-License-Identifier: MIT -// Generated by clean/runtime/gen_fold_selftest.py (do not edit): the render colour image of the native D3D12 -// device's differential self-test of Model L's post (fold port against no-fold port, device_d3d12.cpp). -// FP32 bits, slot 3 (y kW + x) + channel. Synthetic values from exact rational arithmetic; no model data. -// Per site (row of the pair, channel): 25 special values (floor, zero and FP16-subnormal products), then -// FP32-product ties in the image (exact product above / below the midpoint; non-detecting: the second rounding -// happens to go the right way): -// subnormal up 0, down 0, up_nondetecting 0, down_nondetecting 0 -// negative up 72, down 72, up_nondetecting 0, down_nondetecting 0 -// small up 72, down 72, up_nondetecting 0, down_nondetecting 0 -// medium up 72, down 72, up_nondetecting 6, down_nondetecting 12 -// large up 72, down 72, up_nondetecting 12, down_nondetecting 12 -#pragma once -#include - -namespace fold_selftest { - -constexpr int kW = 32, kH = 8; -constexpr uint32_t kColour[768] = { - 0xbc23d2d9u, 0xbc23d2d9u, 0xbc23d2d9u, 0xbc23d2dau, 0xbc23d2dau, 0xbc23d2dau, 0xbc23d2dbu, 0xbc23d2dbu, - 0xbc23d2dbu, 0xbc23d2d8u, 0xbc23d2d8u, 0xbc23d2d8u, 0xbc23d2d7u, 0xbc23d2d7u, 0xbc23d2d7u, 0xbc23d2d6u, - 0xbc23d2d6u, 0xbc23d2d6u, 0x00000000u, 0x00000000u, 0x00000000u, 0xbc23d70au, 0xbc23d70au, 0xbc23d70au, - 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xf149f2cau, 0xf149f2cau, 0xf149f2cau, 0x00000000u, 0x00000000u, - 0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x00000001u, 0x00000001u, 0x00000001u, 0x80000001u, - 0x80000001u, 0x80000001u, 0x0da24260u, 0x0da24260u, 0x0da24260u, 0x8da24260u, 0x8da24260u, 0x8da24260u, - 0x30a3d70au, 0x30a3d70au, 0x30a3d70au, 0x3123d70au, 0x3123d70au, 0x3123d70au, 0x320f5c29u, 0x320f5c29u, - 0x320f5c29u, 0x34000000u, 0x34000000u, 0x34000000u, 0x36700000u, 0x36700000u, 0x36700000u, 0xb023d70au, - 0xb023d70au, 0xb023d70au, 0xb0f5c28fu, 0xb0f5c28fu, 0xb0f5c28fu, 0xb3000000u, 0xb3000000u, 0xb3000000u, - 0xb6700000u, 0xb6700000u, 0xb6700000u, 0x464c98b1u, 0x46ba54b6u, 0x46f544dau, 0x494547a2u, 0x499aa7c7u, - 0x4b0119b5u, 0x4d1615d7u, 0x4d1615eeu, 0x4d7da1bau, 0x4ea808b0u, 0x4f20dfa6u, 0x4fe5b674u, 0x516a2e6bu, - 0x51c0fd9cu, 0x52023b29u, 0x56091904u, 0x5784f3d1u, 0x57f22a88u, 0x599e797bu, 0x59e0d63fu, 0x5a304241u, - 0xbc23d2d9u, 0xbc23d2d9u, 0xbc23d2d9u, 0xbc23d2dau, 0xbc23d2dau, 0xbc23d2dau, 0xbc23d2dbu, 0xbc23d2dbu, - 0xbc23d2dbu, 0xbc23d2d8u, 0xbc23d2d8u, 0xbc23d2d8u, 0xbc23d2d7u, 0xbc23d2d7u, 0xbc23d2d7u, 0xbc23d2d6u, - 0xbc23d2d6u, 0xbc23d2d6u, 0x00000000u, 0x00000000u, 0x00000000u, 0xbc23d70au, 0xbc23d70au, 0xbc23d70au, - 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xf149f2cau, 0xf149f2cau, 0xf149f2cau, 0x00000000u, 0x00000000u, - 0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x00000001u, 0x00000001u, 0x00000001u, 0x80000001u, - 0x80000001u, 0x80000001u, 0x0da24260u, 0x0da24260u, 0x0da24260u, 0x8da24260u, 0x8da24260u, 0x8da24260u, - 0x30a3d70au, 0x30a3d70au, 0x30a3d70au, 0x3123d70au, 0x3123d70au, 0x3123d70au, 0x320f5c29u, 0x320f5c29u, - 0x320f5c29u, 0x34000000u, 0x34000000u, 0x34000000u, 0x36700000u, 0x36700000u, 0x36700000u, 0xb023d70au, - 0xb023d70au, 0xb023d70au, 0xb0f5c28fu, 0xb0f5c28fu, 0xb0f5c28fu, 0xb3000000u, 0xb3000000u, 0xb3000000u, - 0xb6700000u, 0xb6700000u, 0xb6700000u, 0x47078611u, 0x47a69344u, 0x485cad4du, 0x4b9ac3c1u, 0x4bab07eeu, - 0x4c27e2b5u, 0x4d8c24feu, 0x4df8f788u, 0x4e062c35u, 0x500c44b8u, 0x510e14e4u, 0x5153e8c5u, 0x55254b3eu, - 0x552dc357u, 0x5596896fu, 0x58479238u, 0x5873b9ffu, 0x58bf1120u, 0x5c1bab74u, 0x5dd62e6eu, 0x5de127a7u, - 0x5e097c54u, 0x5efc094du, 0x5fc6dabfu, 0x60a4eee7u, 0x611bf1d6u, 0x611bf1d8u, 0x6303041au, 0x6310c920u, - 0x632000c6u, 0x65a539a9u, 0x66a75c6du, 0x683c8ceau, 0x6a2f0edbu, 0x6a828bb2u, 0x6b6f5036u, 0x48121fb1u, - 0x483b9917u, 0x48aad998u, 0x499c9984u, 0x4a4f7634u, 0x4a54b5dau, 0x4d1bd134u, 0x4e0ed21eu, 0x4ed26777u, - 0x50c35909u, 0x5111ad42u, 0x5131e866u, 0x520c5e2eu, 0x529c1ec5u, 0x5456cd08u, 0x565c63ffu, 0x5725692eu, - 0x5796a4cau, 0x5bd6079du, 0x5bd6079fu, 0x5d29f4afu, 0x5fe150afu, 0x5fecdc44u, 0x60180b36u, 0x617480c3u, - 0x6277aa31u, 0x637addb1u, 0x660186f3u, 0x660f23fbu, 0x660f240au, 0x69069d96u, 0x69069dadu, 0x6924666eu, - 0x6b999eabu, 0x6d535c44u, 0x6d810fcbu, 0x3e90e64du, 0x3e979abeu, 0x3ea9296eu, 0x3ec7e4a3u, 0x3ee790fau, - 0x3ee90ca3u, 0x3f42f26bu, 0x3f5251dcu, 0x3f6ea559u, 0x4059b3e8u, 0x405f3a05u, 0x405f3a06u, 0x40999a8eu, - 0x409b89e9u, 0x4117a47du, 0x415f6fb1u, 0x4176efdbu, 0x41802fc2u, 0x41ad07a7u, 0x41ce1d3du, 0x41e98d05u, - 0x42a2aa57u, 0x42a4b5dcu, 0x42b3c3a0u, 0x434f9617u, 0x436b3595u, 0x437742ffu, 0x43bd0eccu, 0x43e6e372u, - 0x43efb4b4u, 0x449add2eu, 0x449ccf92u, 0x44cbd960u, 0x456aa69fu, 0x4576ac8du, 0x458f48beu, 0x3e2847b6u, - 0x3e7a760du, 0x3ea02672u, 0x3ee3e629u, 0x3f1da81eu, 0x3f67f5e8u, 0x3fedb803u, 0x3ff0b925u, 0x4019bcecu, - 0x5fc6dadeu, 0x60953f37u, 0x60953f53u, 0x61527713u, 0x622e903fu, 0x630303efu, 0x63d95576u, 0x63d95577u, - 0x64a31dbeu, 0x68664438u, 0x69adefc3u, 0x6a2f0eccu, 0x6d06ce54u, 0x6e088c6eu, 0x6e088c7du, 0x4902a18fu, - 0x4905ef98u, 0x4998bc49u, 0x4b02b931u, 0x4bfbeb3fu, 0x4caf6b97u, 0x4fae834eu, 0x504c0ae0u, 0x509ff503u, - 0x5131e875u, 0x51aa53a7u, 0x51b30dd4u, 0x5456cd12u, 0x55a2412bu, 0x56234cb7u, 0x598bdd23u, 0x59db4a3eu, - 0x5b770d63u, 0x5e0617e9u, 0x5e1bc7c1u, 0x5f0ec979u, 0x60efec0bu, 0x61352a84u, 0x615d3ff4u, 0x643c48b0u, - 0x652c944bu, 0x6568ea52u, 0x662ecf81u, 0x67311203u, 0x68a35912u, 0x6a97a89fu, 0x6aa7992fu, 0x6b8b0263u, - 0x6d810fdbu, 0x6d8ea073u, 0x6dae2efau, 0x3eab5a49u, 0x3eb58eaeu, 0x3ec41659u, 0x3f155f4eu, 0x3f25573fu, - 0x3f3f47cbu, 0x3fb4dafcu, 0x40065685u, 0x40400fc0u, 0x4080e804u, 0x408359c3u, 0x40842cf4u, 0x411b7ba8u, - 0x414f4a20u, 0x415ca90du, 0x41836e7eu, 0x418daaf1u, 0x419140fcu, 0x4229e2e3u, 0x4242eaceu, 0x425fa2aau, - 0x42db8ab3u, 0x43046553u, 0x43107edeu, 0x437a5ebcu, 0x43916e82u, 0x43a6daeeu, 0x4421c16fu, 0x44524718u, - 0x4454ebd3u, 0x450d09f7u, 0x454329cfu, 0x454d29cfu, 0x45cb4947u, 0x460ca631u, 0x461f5d33u, 0x3ecdbcc5u, - 0x3ee272bbu, 0x3ee272bdu, 0x3f67f5ebu, 0x3f6de4cfu, 0x3f9364a6u, 0x4027d879u, 0x40871aa4u, 0x40a0fc6fu, - 0x40d1b345u, 0x40e7c62cu, 0x40f6bbbcu, 0x415e0b4au, 0x4160d660u, 0x41696df9u, 0x41b9578cu, 0x41dcc78fu, - 0x41dcc793u, 0x42e281d5u, 0x42eb2862u, 0x42eb2863u, 0x43846c98u, 0x43ca8132u, 0x43cd0ce0u, 0x4428fc31u, - 0x442d42afu, 0x446cc647u, 0x44a7f6c0u, 0x44d7a30fu, 0x44dd17b5u, 0x451ae460u, 0x451ae461u, 0x456d99d4u, - 0x45ba43c7u, 0x45e379f9u, 0x45e93b4bu, 0xb83f81e1u, 0xba0435c8u, 0xba9b1c16u, 0xbb4e7552u, 0xbb628ae3u, - 0xbb6f140fu, 0xbba0ac35u, 0xbba23ae8u, 0xbba2dfeeu, 0xbbb3d2feu, 0xbbb8495bu, 0xbbc88821u, 0xbbe47ca3u, - 0xbbe5b7e3u, 0xbbee0f96u, 0xbbf3395bu, 0xbbf4cab9u, 0xbbf913edu, 0xbc013558u, 0xbc0150fdu, 0xbc02535bu, - 0xbc05e6b8u, 0xbc06e98cu, 0xbc094066u, 0xbc112869u, 0xbc117283u, 0xbc13816cu, 0xbc167700u, 0xbc16d5e7u, - 0xbc18d1c3u, 0xbc1d1c99u, 0xbc1d4869u, 0xbc1dcb54u, 0xbc20bdbfu, 0xbc21c326u, 0xbc2230e7u, 0xba12dd5du, - 0xba3d01e6u, 0xba4d5525u, 0xbb88e48au, 0xbb9969aeu, 0xbb9aa212u, 0xbbc2f3e9u, 0xbbc73c10u, 0xbbc9d0d7u, - 0xbbdf52a8u, 0xbbe32957u, 0xbbe4b80cu, 0xbbf846a5u, 0xbbf8c537u, 0xbbfb8dddu, 0xbc08894au, 0xbc08e010u, - 0xbc09f2b6u, 0xbc0bf7a9u, 0xbc0cd875u, 0xbc0d9f40u, 0xbc12f606u, 0xbc14e589u, 0xbc1625dau, 0xbc1901cfu, - 0xbc1a3826u, 0xbc1a5e4fu, 0xbc1e08e8u, 0xbc1e1229u, 0xbc1eef8bu, 0xbc1ffefbu, 0xbc201d3du, 0xbc20460bu, - 0x4108598au, 0x4148e92eu, 0x4148e936u, 0x419d8c27u, 0x41a39122u, 0x41b28584u, 0x428eaa9bu, 0x42c7e7a3u, - 0x42c7e7a8u, 0x431eb070u, 0x4326d21eu, 0x43812816u, 0x43f1353au, 0x441523d7u, 0x4424d0beu, 0x446fc02cu, - 0x4475d102u, 0x44a1c916u, 0x44e2af9du, 0x44e5892du, 0x451711f6u, 0x4579c672u, 0x458a0387u, 0x45b5ab2du, - 0x45fb6289u, 0x46042201u, 0x461d62d3u, 0xbaac5932u, 0xbb05738au, 0xbb4e1b8fu, 0xbb855c1fu, 0xbb9a4b84u, - 0xbb9de96du, 0xbba4280eu, 0xbba4aa97u, 0xbba7714au, 0xbbcae463u, 0xbbd214dau, 0xbbe17ff2u, 0xbbef0900u, - 0xbbf0f3b9u, 0xbbf2d3d5u, 0xbbf971f8u, 0xbbfc3486u, 0xbbff1846u, 0xbc0260beu, 0xbc029615u, 0xbc05cebfu, - 0xbc0b045du, 0xbc0e4586u, 0xbc1090b0u, 0xbc143451u, 0xbc153269u, 0xbc16411du, 0xbc1bd9ceu, 0xbc1bf332u, - 0xbc1d06f2u, 0xbc1dd4f7u, 0xbc1f37e0u, 0xbc1fe93bu, 0xbc22d6ecu, 0xbc22e1e1u, 0xbc235148u, 0xba9a8bb0u, - 0xbacca7e8u, 0xbb2f3000u, 0xbba66026u, 0xbba6e0e9u, 0xbbaa1c24u, 0xbbcafd51u, 0xbbd16fddu, 0xbbd5c94eu, - 0xbbe93f7bu, 0xbbed0150u, 0xbbf5e282u, 0xbc025a0du, 0xbc03d26bu, 0xbc06af80u, 0xbc0b0e47u, 0xbc0baad6u, - 0xbc0bd15eu, 0xbc0f8bc0u, 0xbc1102ecu, 0xbc112fdfu, 0xbc16468au, 0xbc17c17cu, 0xbc188ee6u, 0xbc1b065fu, - 0xbc1b2263u, 0xbc1bc9c6u, 0xbc1f4a35u, 0xbc1f58abu, 0xbc1f6e06u, 0xbc20a205u, 0xbc20dd4fu, 0xbc211d17u, - 0xbc215386u, 0xbc218968u, 0xbc22a12cu, 0x381e585eu, 0x3972289eu, 0x3a853572u, 0x3b4c2fb3u, 0x3b5dd03cu, - 0x3b6e67c5u, 0x3bafa8f7u, 0x3bb4cb7fu, 0x3be4ff3eu, 0x3c20865du, 0x3c3014e8u, 0x3c566233u, 0x3c766070u, - 0x3c835a95u, 0x3c8c89beu, 0x3caa8854u, 0x3caa8855u, 0x3cb49533u, 0x3cd9d7dbu, 0x3cdb4030u, 0x3ce2636cu, - 0x3d177655u, 0x3d1ecf79u, 0x3d4486b5u, 0x3d64f7d9u, 0x3d66a8cfu, 0x3d6d8800u, 0x3db61313u, 0x3dbe77abu, - 0x3dc47b19u, 0x3dcf98ffu, 0x3dd90c53u, 0x3de3a9feu, 0x3df6fa6bu, 0x3e0ccf91u, 0x3e0d4885u, 0x387e1cd8u, - 0x38f2b795u, 0x39f089c7u, 0x3ae5b10cu, 0x3b0be672u, 0x3b16687au, 0x3be29c98u, 0x3c027a8bu, 0x3c05046fu, - 0x3c24fcbbu, 0x3c408611u, 0x3c516588u, 0x3c858794u, 0x3c9763b1u, 0x3c9c2c8bu, 0x3ccd3e3du, 0x3ceb6cc1u, - 0x3ceb6cc3u, 0x3cf91737u, 0x3d021006u, 0x3d036733u, 0x3d246cd0u, 0x3d251145u, 0x3d2cef6du, 0x3d407b54u, - 0x3d41f1c8u, 0x3d4a2790u, 0x3d6a80f5u, 0x3d7f04beu, 0x3d84ae7cu, 0x3da28630u, 0x3dbe236au, 0x3dc0c935u, - 0x3df0c94cu, 0x3dfac37au, 0x3e0fe893u, 0x4bde55a1u, 0x51701b37u, 0x5694b873u, 0x5f5fdf22u, 0x60280625u, - 0x61352a65u, 0x49b3aae1u, 0x4c805d5bu, 0x501374bdu, 0x5f65896cu, 0x6080786eu, 0x6357f138u, 0x3f51a7eeu, - 0x404bd79eu, 0x4072ec2cu, 0x4135c9a5u, 0x421151ceu, 0x4301f012u, 0x3e4f7021u, 0x3e92c915u, 0x3fda6459u, - 0xbc2321f4u, 0xbc235cddu, 0xbc239370u, 0x3acd1597u, 0x3b215577u, 0x3b2860d0u, 0x3b858f37u, 0x3b8b5b84u, - 0x3b957e6au, 0x3be95c45u, 0x3c12a5f1u, 0x3c1c3f7cu, 0x3c5e23d7u, 0x3c6f6400u, 0x3c7518fau, 0x3c906468u, - 0x3c99c503u, 0x3c9f5af6u, 0x3cb9f751u, 0x3ccac991u, 0x3cd532d7u, 0x3ceda9a3u, 0x3d0f3aeeu, 0x3d15f738u, - 0x3d46c2ffu, 0x3d5179c3u, 0x3d57cfbdu, 0x3d884fbbu, 0x3d894b3au, 0x3db38f56u, 0x3dc685eeu, 0x3dcc1055u, - 0x3dce2d6fu, 0x3deb8939u, 0x3dedf22bu, 0x3df55015u, 0x3e1311d6u, 0x3e180f63u, 0x3e189159u, 0x3a856b90u, - 0x3aa45de8u, 0x3ada0277u, 0x3b33dfa4u, 0x3b405b36u, 0x3bcdce66u, 0x3c0f2727u, 0x3c13fc00u, 0x3c1ae05cu, - 0x3c529067u, 0x3c7b3592u, 0x3c7bdb9eu, 0x3ca10e85u, 0x3cacb5c8u, 0x3cb59c10u, 0x3cefea8fu, 0x3cefea90u, - 0x3cf892fdu, 0x3d09fc3fu, 0x3d183702u, 0x3d23256cu, 0x3d2e4685u, 0x3d34693cu, 0x3d37e51eu, 0x3d5115bcu, - 0x3d5c4299u, 0x3d5d13c7u, 0x3d8b86e0u, 0x3d990fbdu, 0x3d9e911fu, 0x3dcff440u, 0x3dd0ab2cu, 0x3dd21abcu, - 0x3e13cef0u, 0x3e154bcdu, 0x3e15cb8du, 0x5715add1u, 0x5a47b634u, 0x5b07a290u, 0x66d6dcc7u, 0x6c50a93bu, - 0x6d003c56u, 0x56428261u, 0x59078a1du, 0x5d9eb2dau, 0x6391b828u, 0x680bb501u, 0x682a9e27u, 0x408d2818u, - 0x41085982u, 0x4114d311u, 0x43e57362u, 0x44eb57cbu, 0x4522d41eu, 0x4056fdf7u, 0x40fb680au, 0x4129b4ccu, -}; - -} // namespace fold_selftest diff --git a/clean/runtime/gen_fold_probe.py b/clean/runtime/gen_fold_probe.py deleted file mode 100644 index acf7d6ff..00000000 --- a/clean/runtime/gen_fold_probe.py +++ /dev/null @@ -1,186 +0,0 @@ -# SPDX-License-Identifier: MIT -"""Writes clean/runtime/fold_probe.h: inputs and expected bytes of the start-up probe of the driver's conversion fold -(clean/dx12/kernels/l_fold_probe.hlsl, hlsl/l_post_encode.hlsli). WrongGPU's own code. - - python clean/runtime/gen_fold_probe.py [out.h] - -Lane l reads x0 x1 x2 (the FP32 log2 arguments of the three history codes) and d0 d1 d2 (display values) and writes -q_i = (_Float16)(kEncK x_i) with ONE rounding of the exact product (+0 for a zero product: the fold's +0 addend) and -h(d_i) (round to nearest even). Expected values are computed from exact rationals (fractions.Fraction), never from -FP arithmetic. The x inputs are adversarial: values whose FP32 product kEncK x rounds exactly onto an FP16 midpoint -while the exact product is not that midpoint (residual of both signs; a second rounding moves them by one FP16 ulp), -FP16-subnormal products, +-0, NaN, the encode's floor log2(1e-4) and ordinary values; the d inputs: FP16 midpoints and -their FP32 neighbours, +-0, NaN, values beyond the FP16 range. -""" -import os -import struct -import sys -from fractions import Fraction - -import numpy as np - -K_BITS = 0x3dddf184 # kEncK - - -def f32(b): - return struct.unpack("= Fraction(65520): - return s | 0x7c00 - # exponent: 2^e <= a < 2^(e+1) - e = a.numerator.bit_length() - a.denominator.bit_length() - if Fraction(2) ** e > a: - e -= 1 - e = max(e, -14) # subnormal range uses the exponent -14 grid - q = a / (Fraction(2) ** (e - 10)) # in units of the ulp - n = q.numerator // q.denominator - r = q - n - if r > Fraction(1, 2) or (r == Fraction(1, 2) and n % 2 == 1): - n += 1 - if n >= 2048: # carried into the next binade - n //= 2 - e += 1 - if e > 15: - return s | 0x7c00 - if n < 1024: # subnormal - return s | n - return s | ((e + 15) << 10) | (n - 1024) - - -def h16_value(b): - """The exact value of FP16 bits b (None for infinity / NaN).""" - if (b & 0x7c00) == 0x7c00: - return None - return Fraction(float(np.frombuffer(struct.pack("> 10) & 31) - 25) - m = v + (ulp / 2 if v >= 0 else -ulp / 2) - x0 = rn32(m / K) - for dx in (0, 1, -1, 2, -2): - x = f32((bits32(x0) + dx) & 0xffffffff) - p = Fraction(x) * K - if rn32(p) == np.float32(float(m)) and p != m and ((p > m) == want_up): - out.append(x) - want_up = not want_up - break - return out - - -def main(): - out = sys.argv[1] if len(sys.argv) > 1 else os.path.join(os.path.dirname(os.path.abspath(__file__)), "fold_probe.h") - K = Fraction(f32(K_BITS)) - xs = midpoint_xs(K, 60, 20261115) - nan = f32(0x7fc00000) - floor = float(np.log2(np.float32(1e-4)).astype(np.float32)) - xs += [0.0, -0.0, nan, floor, 0.0, 1.0, -1.0, 20.0, float(np.float32(3.0e-4)), float(np.float32(-2.0e-4)), - float(np.float32(1e-6)), -float(np.float32(1e-6))] - rng = np.random.default_rng(7) - while len(xs) < 96: - xs.append(float(np.float32(rng.uniform(-13.3, 21.0)))) - ds = [] - for h in rng.integers(0x0400, 0x7bff, 40): - v = h16_value(int(h)) - ulp = Fraction(2) ** ((((int(h) >> 10) & 31)) - 25) - m = v + ulp / 2 - ds.append(float(np.float32(float(m)))) # an exact FP16 midpoint (ties to even) - ds.append(f32(bits32(float(m)) + 1)) # just above - ds += [0.0, -0.0, nan, 65519.0, 65520.0, 1e6, f32(0x33800000), f32(0x33000001)] - while len(ds) < 96: - ds.append(float(np.float32(rng.uniform(0.0, 8.0)))) - xs, ds = xs[:96], ds[:96] - inputs, expect, nanmask = [], [], [] - for lane in range(32): - x = xs[3 * lane:3 * lane + 3] - d = ds[3 * lane:3 * lane + 3] - inputs += [bits32(v) for v in x + d] - q, nm = [], [] - for v in x: - if v != v: - q.append(0x7e00); nm.append(True) - else: - p = Fraction(v) * K - q.append(0 if p == 0 else rn16_bits(p)); nm.append(False) - for v in d: - if v != v: - q.append(0x7e00); nm.append(True) - elif v == 0: # a plain conversion keeps the sign of a zero - q.append(0x8000 if struct.pack("\n#include \n\nnamespace fold_probe {\n") - f.write("constexpr unsigned kInBytes = 768, kOutOffset = 768, kOutBytes = 384, kBytes = 1152;\n") - f.write("constexpr uint32_t kInput[192] = {\n") - for i in range(0, 192, 6): - f.write(" " + ", ".join(f"0x{v:08x}u" for v in inputs[i:i + 6]) + ",\n") - f.write("};\nconstexpr uint32_t kExpect[96] = {\n") - for i in range(0, 96, 6): - f.write(" " + ", ".join(f"0x{v:08x}u" for v in expect[i:i + 6]) + ",\n") - f.write("};\nconstexpr uint8_t kNanMask[96] = {\n") - for i in range(0, 96, 24): - f.write(" " + ", ".join(str(v) for v in nanmask[i:i + 24]) + ",\n") - f.write("};\n") - f.write("""inline void fill(uint8_t* buffer) { - std::memset(buffer, 0, kBytes); - std::memcpy(buffer, kInput, sizeof kInput); -} -inline bool half_ok(uint32_t got, uint32_t want, bool nan) { - if (nan) return (got & 0x7c00u) == 0x7c00u && (got & 0x03ffu) != 0u; - return got == want; -} -// index of the first differing output half (0 .. 191), or -1 when every half matches -inline int first_mismatch(const uint32_t* out) { - for (unsigned i = 0; i < 96; ++i) { - if (!half_ok(out[i] & 0xffffu, kExpect[i] & 0xffffu, (kNanMask[i] & 1u) != 0)) return (int)(2 * i); - if (!half_ok(out[i] >> 16, kExpect[i] >> 16, (kNanMask[i] & 2u) != 0)) return (int)(2 * i + 1); - } - return -1; -} -} // namespace fold_probe -""") - # self-check of rn16_bits against numpy's double -> half conversion (correctly rounded from a double; every value - # here is an exact double) - chk = np.random.default_rng(3) - for v in list(chk.uniform(-70000, 70000, 2000)) + list(chk.uniform(-1e-4, 1e-4, 2000)): - nb = int(np.array([v]).astype(np.float16).view(np.uint16)[0]) - assert rn16_bits(Fraction(float(v))) == nb or (v == 0), (v, nb) - print("wrote", out, "midpoint x:", sum(1 for v in xs[:60]), "lanes 32") - - -if __name__ == "__main__": - main() diff --git a/clean/runtime/gen_fold_selftest.py b/clean/runtime/gen_fold_selftest.py deleted file mode 100644 index 7326e366..00000000 --- a/clean/runtime/gen_fold_selftest.py +++ /dev/null @@ -1,255 +0,0 @@ -# SPDX-License-Identifier: MIT -"""Writes clean/runtime/fold_selftest.h: the colour image of the native D3D12 device's differential self-test of Model -L's post (device_d3d12.cpp fold_selftest). WrongGPU's own code. - - python clean/runtime/gen_fold_selftest.py [out.h] - -The self-test runs the REAL post shader twice on one tiny frame: the fold port of l_post_recon_p12 (its history encode -through the driver's conversion fold, hlsl/l_post_encode.hlsli) and the no-fold port (round to odd, exact without the -fold), and compares the raw output bytes. Equal: the device runs the fold port; any difference or error: the no-fold -port. The frame (kW x kH, render size = output size, jitter 0, reset, exposure 1, post scale s = 0, every code and -kernel-shape value 0) makes the full history H^ of pixel (x, y) channel i exactly the render colour c_i, so the history -code is q = (_Float16)(kEncK x) with x = log2(max(fma(100, c, 1), 1e-4)) evaluated by the post shader itself -(enc_log): one wave of the shader's six encode sites (three channels, two rows of each row pair) sees each class: -- FP32-product ties: c whose x (predicted with the correctly rounded log2) makes the FP32 product RN32(kEncK x) land - exactly on an FP16 midpoint while the exact product does not, the exact product above and below the midpoint, most - of them on the side where the second rounding (to even) goes the wrong way, in four ranges of x and among the - FP16-subnormal products (the ones there are: x near 0 is sparse); -- exact-product ties do not exist: kEncK's significand has the odd factor 0x377c61 (22 bits) and an FP16 midpoint's - has at most 12 bits, so kEncK x is never exactly a midpoint unless it is 0; -- zero products: c = +-0, +-2^-149, +-1e-30 (fma(100, c, 1) = 1, x = +0, the code +0); -- FP16-subnormal products of both signs (fma(100, c, 1) = 1 +- a few ulps); the exhaustive search over every FP32 - value of fma(100, c, 1) in that range finds no FP32-product tie there; -- the floor: c whose fma(100, c, 1) is just below, at or near 1e-4 (the max picks 1e-4), and far below it (0, -1, -1e30). -Which x the GPU's log2 gives is not assumed: an x that differs from the prediction just tests another value. The -fold-removed mutant (LP_MUT 3, two roundings) must make the self-test fail (GPU record of the PR). -Every predicted value is checked with exact rationals (fractions.Fraction, decimal logarithms to 60 digits). -""" -import decimal -import os -import struct -import sys -from fractions import Fraction - -import numpy as np - -HERE = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, HERE) -from gen_fold_probe import K_BITS, bits32, f32, rn16_bits # noqa: E402 - -W, H = 32, 8 -SITES = 6 # (row of the pair, channel) -PER_SITE = W * H * 3 // SITES -SEED = 20261115 -K = Fraction(f32(K_BITS)) -DEC = decimal.Context(prec=60) - - -def rn(fr, p, emin): - """Round a Fraction to p significant bits, nearest even, exponents below emin on the emin grid (no overflow).""" - if fr == 0: - return Fraction(0) - s = -1 if fr < 0 else 1 - a = abs(fr) - e = a.numerator.bit_length() - a.denominator.bit_length() - if Fraction(2) ** e > a: - e -= 1 - e = max(e, emin) - q = Fraction(2) ** (e - p + 1) - n = a / q - f = n.numerator // n.denominator - r = n - f - if r > Fraction(1, 2) or (r == Fraction(1, 2) and f % 2 == 1): - f += 1 - return s * f * q - - -def rn32(fr): - return rn(fr, 24, -126) - - -def m_of(c): - """RN32(fma(100, c, 1)) for the FP32 value c (exact).""" - return rn32(100 * Fraction(c) + 1) - - -def log2_exact(m): - """(the correctly rounded FP32 log2 of the FP32 value m > 0, the distance of the exact log2 from an FP32 midpoint in - units of that FP32 ulp) from 60-digit decimal logarithms.""" - lm = DEC.divide(DEC.ln(decimal.Decimal(m.numerator) / decimal.Decimal(m.denominator)), DEC.ln(decimal.Decimal(2))) - v = Fraction(lm) - x = rn32(v) - if x == 0: - return x, None - a = abs(v) - e = a.numerator.bit_length() - a.denominator.bit_length() - if Fraction(2) ** e > a: - e -= 1 - ulp = Fraction(2) ** (e - 23) - frac = (a / ulp) - (a / ulp).numerator // (a / ulp).denominator - return x, abs(frac - Fraction(1, 2)) - - -def fp16_midpoint(p): - """Whether the FP32 value p lies exactly halfway between two adjacent FP16 values (finite range).""" - a = abs(p) - if a == 0 or a >= 65504: - return False - e = a.numerator.bit_length() - a.denominator.bit_length() - if Fraction(2) ** e > a: - e -= 1 - ulp = Fraction(2) ** (max(e, -14) - 10) - n = a / ulp - return n.denominator == 2 - - -def classify(c): - """(x, product class) of the colour c, checked exactly: None unless kEncK x is an FP32-product tie; else - ('up' / 'down' = the exact product's side, detecting = the second rounding goes to the other FP16 value).""" - m = m_of(c) - if m < Fraction(f32(bits32(1e-4))): - return None - x, margin = log2_exact(m) - if x == 0 or margin is None or margin < Fraction(1, 10**12): - return None - pe = x * K - p32 = rn32(pe) - if not fp16_midpoint(p32) or pe == p32: - return None - detecting = rn16_bits(pe) != rn16_bits(p32) - return x, ("up" if pe > p32 else "down"), detecting - - -def search(rng, lo, hi, want, sub=False): - """want ties (half the exact products above the midpoint, half below; detecting, plus a sixth non-detecting) from - c whose fma(100, c, 1) lies in [lo, hi], log-uniform; sub: every m = 1 +- k ulp in the range (exhaustive).""" - k32 = np.float64(f32(K_BITS)) - if sub: - ks = np.arange(1, int((hi - 1) * 2**23) + 1) - mt = np.concatenate([1.0 + ks * 2.0**-23, 1.0 - np.arange(1, int((1 - lo) * 2**24) + 1) * 2.0**-24]) - rounds = [mt] - else: - rounds = (np.exp2(rng.uniform(np.log2(lo), np.log2(hi), 1 << 20)) for _ in range(400)) - picks = {("up", True): [], ("down", True): [], ("up", False): [], ("down", False): []} - quota = {("up", True): want // 2, ("down", True): want // 2, ("up", False): want // 12, ("down", False): want // 12} - seen = set() - for mt in rounds: - c = ((mt - 1.0) / 100.0).astype(np.float32) - m = (100.0 * c.astype(np.float64) + 1.0).astype(np.float32) - ok = (m >= lo) & (m <= hi) - x = np.log2(m.astype(np.float64)).astype(np.float32) - pe = x.astype(np.float64) * k32 - p32 = pe.astype(np.float32).astype(np.float64) - a = np.abs(p32) - normal = a >= 2.0**-14 - pb = np.abs(p32.astype(np.float32)).view(np.uint32) - tie_n = normal & ((pb & 0x1fff) == 0x1000) - t = a * 2.0**25 - tie_s = ~normal & (t == np.round(t)) & (np.fmod(t, 2.0) == 1.0) & (a > 0) - for i in np.nonzero(ok & (tie_n | tie_s) & (pe != p32))[0]: - cb = int(c[i].view(np.uint32)) - if cb in seen: - continue - seen.add(cb) - r = classify(f32(cb)) - if r is None: - continue - key = (r[1], r[2]) - if len(picks[key]) < quota[key]: - picks[key].append(cb) - if all(len(picks[k]) >= quota[k] for k in picks): - break - return picks - - -def floor_values(): - """c around the floor: fma(100, c, 1) the nearest reachable values below / at / above 1e-4, and far below.""" - one_em4 = Fraction(f32(bits32(1e-4))) - c0 = bits32(np.float32((1e-4 - 1.0) / 100.0)) - below, above = [], [] - for d in range(-40, 41): - cb = c0 + d # negative c: a larger bit pattern is a more negative value - m = m_of(f32(cb)) - (below if m < one_em4 else above).append((m, cb)) - below = [cb for _, cb in sorted(below, reverse=True)[:3]] - above = [cb for _, cb in sorted(above)[:3]] - far = [bits32(0.0) | 0, bits32(-0.01), bits32(-1.0), bits32(-1e30)] - return below + above + far - - -def zero_values(): - return [0x00000000, 0x80000000, 0x00000001, 0x80000001, bits32(1e-30), bits32(-1e-30)] - - -def subnormal_values(): - """c with fma(100, c, 1) = 1 + k 2^-23 or 1 - k 2^-24: FP16-subnormal products of both signs (no tie among them).""" - out = [] - for m in [1.0 + k * 2.0**-23 for k in (1, 2, 7, 100, 3000)] + [1.0 - k * 2.0**-24 for k in (1, 3, 50, 6000)]: - cb = bits32(np.float32((m - 1.0) / 100.0)) - assert m_of(f32(cb)) == Fraction(m) - out.append(cb) - return out - - -def build(): - rng = np.random.default_rng(SEED) - sub_lo, sub_hi = 1.0 - 3.9e-4, 1.0 + 3.9e-4 # |kEncK x| < 2^-14: FP16-subnormal products - groups = {"subnormal": search(rng, sub_lo, sub_hi, 64, sub=True), - "negative": search(rng, 1.0e-4, sub_lo, 144), - "small": search(rng, sub_hi, 16.0, 144), - "medium": search(rng, 16.0, 2.0**20, 144), - "large": search(rng, 2.0**20, 2.0**100, 144)} - specials = floor_values() + zero_values() + subnormal_values() - ties = [] - for g, picks in groups.items(): - for key, lst in picks.items(): - ties += [(g, key, cb) for cb in lst] - # each site (row of the pair, channel): every special value, then the ties dealt round-robin (each group and side - # spread over the six sites; the non-detecting ones last, so that they are the ones a full site drops) - order = sorted(ties, key=lambda t: (not t[1][1], t[0], t[1][0], t[2])) - per_site = [list(specials) for _ in range(SITES)] - for i, t in enumerate(order): - per_site[i % SITES].append(t[2]) - fill = [bits32(1.0 + 0.25 * k) for k in range(PER_SITE)] # ordinary values for any slot left - for s in per_site: - del s[PER_SITE:] - s += fill[:PER_SITE - len(s)] - colour = [0] * (W * H * 3) - for site in range(SITES): - row, ch = divmod(site, 3) - pixels = [(x, 2 * j + row) for j in range(H // 2) for x in range(W)] - for (x, y), cb in zip(pixels, per_site[site]): - colour[3 * (y * W + x) + ch] = cb - tag = {cb: (g, k) for g, k, cb in ties} - counts = {g: {f"{k[0]}{'' if k[1] else '_nondetecting'}": 0 for k in picks} for g, picks in groups.items()} - for cb in colour: - if cb in tag: - g, k = tag[cb] - counts[g][f"{k[0]}{'' if k[1] else '_nondetecting'}"] += 1 - return colour, counts, len(specials) - - -def write(path): - colour, counts, nspec = build() - lines = ["// SPDX-License-Identifier: MIT", - "// Generated by clean/runtime/gen_fold_selftest.py (do not edit): the render colour image of the native D3D12", - "// device's differential self-test of Model L's post (fold port against no-fold port, device_d3d12.cpp).", - "// FP32 bits, slot 3 (y kW + x) + channel. Synthetic values from exact rational arithmetic; no model data.", - "// Per site (row of the pair, channel): %d special values (floor, zero and FP16-subnormal products), then" % nspec, - "// FP32-product ties in the image (exact product above / below the midpoint; non-detecting: the second rounding", - "// happens to go the right way):"] - for g, c in counts.items(): - lines.append("// %-9s %s" % (g, ", ".join(f"{k} {v}" for k, v in c.items()))) - lines += ["#pragma once", "#include ", "", "namespace fold_selftest {", "", - f"constexpr int kW = {W}, kH = {H};", f"constexpr uint32_t kColour[{W * H * 3}] = {{"] - for i in range(0, len(colour), 8): - lines.append(" " + ", ".join(f"0x{v:08x}u" for v in colour[i:i + 8]) + ",") - lines += ["};", "", "} // namespace fold_selftest", ""] - with open(path, "w", encoding="utf-8", newline="\n") as f: - f.write("\n".join(lines)) - return counts - - -if __name__ == "__main__": - out = sys.argv[1] if len(sys.argv) > 1 else os.path.join(HERE, "fold_selftest.h") - print(write(out))