From a8a2dd61625f08be364d15e2aed84a03293e299c Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sun, 15 Nov 2026 02:36:27 -0500 Subject: [PATCH 1/2] L post recon (D3D12): history encode through the driver's conversion fold, guarded by a start-up probe, a differential self-test of the real ports and a per-site sensitivity control - l_post_recon_p12 gets three ports: needs=fold (the encode as mad(log2, kEncK, 0) into the RNE conversion, which the driver folds into v_fma_mixlo/mixhi_f16), needs=nofold (round to odd, exact on any driver) and needs=control (the encode rounded twice, never selected to run) - start-up guard (device_d3d12 fold_probe), all steps must pass, else the no-fold port runs: l_fold_probe against exact expected bytes; the fold and no-fold ports on a crafted frame (fold_selftest.h: per encode site 13 FP32-product ties a second rounding gets wrong, each at predicted-log2 offsets -8..8 ulp) with equal raw bytes; the control port differing from the no-fold port at each of the six sites; bounded queue waits (a timeout fails device creation) - build_dx12: needs=fold|nofold|control, isa= rules with literal-tied site counts (fold: six kEncK sites; control: none) - l_post_edge_check: fold and no-fold legs; test_fold_probe: generated headers, the -8..8 ulp shift test (N = 13 per site), ports and ISA rules Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/build_dx12.py | 89 +++++++- clean/dx12/hlsl/l_post_encode.hlsli | 46 ++++ clean/dx12/kernel_sets.json | 4 + clean/dx12/kernels/l_fold_probe.hlsl | 28 +++ clean/dx12/kernels/l_post.hlsl | 24 +- clean/dx12/tests/kernel_replay.cpp | 1 + clean/dx12/tests/test_fold_probe.py | 109 +++++++++ clean/l/runner/l_post_edge_check.py | 20 +- clean/runtime/device_d3d12.cpp | 209 +++++++++++++++++- clean/runtime/device_dx12.cpp | 5 + clean/runtime/fold_probe.h | 86 ++++++++ clean/runtime/fold_selftest.h | 209 ++++++++++++++++++ clean/runtime/gen_fold_probe.py | 186 ++++++++++++++++ clean/runtime/gen_fold_selftest.py | 315 +++++++++++++++++++++++++++ 14 files changed, 1312 insertions(+), 19 deletions(-) create mode 100644 clean/dx12/hlsl/l_post_encode.hlsli create mode 100644 clean/dx12/kernels/l_fold_probe.hlsl create mode 100644 clean/dx12/tests/test_fold_probe.py create mode 100644 clean/runtime/fold_probe.h create mode 100644 clean/runtime/fold_selftest.h create mode 100644 clean/runtime/gen_fold_probe.py create mode 100644 clean/runtime/gen_fold_selftest.py diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index 0c5bbccb..d25cb1ce 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -191,6 +191,18 @@ def kernel_decls(path): if d["rs"] not in ("buffer", "tex", "direct"): raise SystemExit(f"{path}: {name}: unsupported root-signature kind {d['rs']}") d["arg_dwords"] = arg_dwords(d["args"], f"{path}: {name}") + # needs=fold | nofold | control: a port the device runs only when its start-up guard of the driver's + # conversion fold passed (fold) or failed (nofold), or only inside that guard (control: the sensitivity + # control, never selected); the ports of one name and code object differ by this key + d["needs"] = kv.get("needs", "") + if d["needs"] not in ("", "fold", "nofold", "control"): + raise SystemExit(f"{path}: {name}: needs={d['needs']} (fold, nofold or control)") + # isa=,: the port's RGA ISA must satisfy each rule, [+...]>= or == (counts of + # instruction mnemonics; @: only its x * literal + 0 sites, parse_isa_rules), checked after the + # build (isa_check; WG_DX12_ISA_CHECK=0 skips it) + d["isa"] = kv.get("isa", "") + if d["isa"]: + parse_isa_rules(d["isa"], f"{path}: {name}") decls.append(d) return decls @@ -298,6 +310,65 @@ def rga_isa(dxil, isa_dir, stem): shutil.copyfile(os.path.join(tmp, f), os.path.join(isa_dir, f"{stem}.{kind}")) +def parse_isa_rules(text, where): + """isa=,...: [(terms, op, n)] with op '>=' or '=='; a term is (mnemonic, literal or None): counts every + instruction of that mnemonic, @<8 hex digits> only those whose literal dword is that value, whose last source + operand is 0 and that carry no op_sel modifier (an FP32 x * literal + 0: the sites of a conversion fold).""" + rules = [] + for r in text.split(","): + m = re.fullmatch(r"([a-z0-9_]+(?:@[0-9a-f]{8})?(?:\+[a-z0-9_]+(?:@[0-9a-f]{8})?)*)(>=|==)(\d+)", r) + if not m: + raise SystemExit(f"{where}: isa rule {r!r} ([@][+[@]]>= or ...==)") + terms = [(t.split("@")[0], t.split("@")[1] if "@" in t else None) for t in m.group(1).split("+")] + rules.append((terms, m.group(2), int(m.group(3)))) + return rules + + +def isa_count(lines, mnemonic, literal): + """Instructions of `mnemonic` in RGA ISA lines (with literal: only the x * literal + 0 sites, parse_isa_rules).""" + c = 0 + for line in lines: + code, _, comment = line.partition("//") + f = code.split() + if not f or f[0] != mnemonic: + continue + if literal is not None: + words = comment.split(":")[-1].split() + srcs = [o.strip() for o in " ".join(f[1:]).split(",")] + if not words or words[-1].lower() != literal or srcs[-1] != "0" or "op_sel" in code: + continue + c += 1 + return c + + +def isa_check(built_isa): + """built_isa: [(kernel name, dxil path, rules text)] of this build's ports with isa=. Each port's RGA ISA must + satisfy its rules (e.g. a driver fold the port's exactness depends on: a v_fma_mixlo_f16 count). Fails the build + otherwise; WG_DX12_ISA_CHECK=0 skips it (no RGA).""" + if not built_isa or os.environ.get("WG_DX12_ISA_CHECK", "1") == "0": + return + if not os.path.exists(RGA) and not RGA.endswith(".cmd"): + raise SystemExit(f"ISA check: RGA not found at {RGA} (WG_RGA, or WG_DX12_ISA_CHECK=0 to skip)") + bad = [] + for name, dxil, text in built_isa: + with tempfile.TemporaryDirectory(prefix="wgisa_") as tmp: + rga_isa(dxil, tmp, "k") + path = os.path.join(tmp, "k.isa.txt") + if not os.path.exists(path): + bad.append(f"{name}: RGA gave no ISA") + continue + lines = open(path, encoding="utf-8", errors="replace").read().splitlines() + for terms, op, n in parse_isa_rules(text, name): + c = sum(isa_count(lines, mn, lit) for mn, lit in terms) + ok = c >= n if op == ">=" else c == n + label = "+".join(mn + (f"@{lit}" if lit else "") for mn, lit in terms) + print(f"ISA {name:40s} {label} = {c} (rule {op} {n}) {'ok' if ok else 'FAILED'}", flush=True) + if not ok: + bad.append(f"{name} ({os.path.basename(dxil)}): {label} = {c}, rule {op} {n}") + if bad: + raise SystemExit("ISA check failed:\n " + "\n ".join(bad)) + + VGPR_LIMITS = os.path.join(HERE, "vgpr_limits.json") @@ -383,9 +454,12 @@ def main(argv=None): old = {} if only and os.path.exists(man_path): # partial build: keep the other kernels' entries with open(man_path) as f: - old = {(k["name"], k.get("co", "")): k for k in json.load(f)["kernels"]} - kernels = dict(old) # (name, co) -> manifest entry + # every port of a rebuilt name is replaced (a name may gain or lose ports, e.g. needs=fold / nofold) + old = {(k["name"], k.get("co", ""), k.get("needs", "")): k for k in json.load(f)["kernels"] + if k["name"] not in only} + kernels = dict(old) # (name, co, needs) -> manifest entry built = [] + built_isa = [] # (name, dxil, isa rules) of the ports with isa= # generated kernels (kernels/fused_*_gen.py: the fused forms of swin_block.hlsl / bneck.hlsl) are regenerated from # the live sources first, so that no build compiles a stale copy for fn in sorted(os.listdir(KERNEL_DIR)) if os.path.isdir(KERNEL_DIR) else []: @@ -408,7 +482,7 @@ def main(argv=None): continue if not only and is_frame_gen(d): continue - key = (d["name"], d["co"]) + key = (d["name"], d["co"], d["needs"]) if key in kernels and key not in old: raise SystemExit(f"kernel {d['name']} ({d['co']}) declared twice") kernels[key] = None # placeholder: the declared-twice check sees it before any compile starts @@ -433,6 +507,8 @@ def build_one(job): r = {"name": d["name"], "file": os.path.basename(out), "co": d["co"], "entry": d["entry"], "block": d["block"], "wave": d["wave"], "args": d["args"], "arg_dwords": d["arg_dwords"], "source": d["source"], "sha256": sha, "rs": d["rs"]} + if d["needs"]: + r["needs"] = d["needs"] if d["group"] != d["block"]: r["group"] = d["group"] r["gscale"] = d["gscale"] @@ -448,18 +524,21 @@ def build_one(job): futures = [pool.submit(build_one, j) for j in jobs] try: for (src, d, out, _k), fut in zip(jobs, futures): - kernels[(d["name"], d["co"])] = fut.result() + kernels[(d["name"], d["co"], d["needs"])] = fut.result() built.append((d["name"], out)) + if d["isa"]: + built_isa.append((d["name"], out, d["isa"])) print(f"{d['name']:40s} -> {os.path.basename(out)} block {d['block']} wave {d['wave']} " f"args {d['args']}", flush=True) except BaseException: for fut in futures: fut.cancel() # kernels not started yet; the running ones finish their current child process raise - names = {n for n, _ in kernels} + names = {k[0] for k in kernels} if only and not (only <= names): raise SystemExit(f"not declared in clean/dx12/kernels or clean/k/dx12/kernels: {sorted(only - names)}") vgpr_check(built) + isa_check(built_isa) with open(man_path, "w") as f: json.dump({"format": FORMAT, "dxc": os.path.basename(os.path.dirname(os.path.dirname(DXC_DIR))), "kernels": [kernels[k] for k in sorted(kernels)]}, f, indent=1) diff --git a/clean/dx12/hlsl/l_post_encode.hlsli b/clean/dx12/hlsl/l_post_encode.hlsli new file mode 100644 index 00000000..e627deb3 --- /dev/null +++ b/clean/dx12/hlsl/l_post_encode.hlsli @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: MIT +// Model L post: the history texel's FP16 encode through the driver's conversion fold, shared by +// clean/dx12/kernels/l_post.hlsl (l_post_recon_p12) and its start-up probe clean/dx12/kernels/l_fold_probe.hlsl, so +// that both compile the same expression. WrongGPU's own code. Include after wg_platform.hlsli. +// +// q_i = (_Float16)(kEncK x_i), one rounding of the exact product, as HIP's v_fma_mixlo_f16 x, kEncK, 0 (+0 for a zero +// product). Written as mad(x, kEncK, 0) feeding the driver's round-to-nearest-even matrix conversion, which the gfx1201 +// D3D12 compiler folds into exactly that instruction (v_fma_mixlo_f16 / v_fma_mixhi_f16, as Model M's post). Without +// the fold the FP32 product would be rounded twice (a value whose FP32 product lands on an FP16 midpoint moves by one +// FP16 ulp). Guards: build_dx12.py's ISA check on the kernel (isa= in its declaration: the six sites, kEncK as the +// literal with a 0 addend) and two start-up steps of the native device, both of which must pass: the probe +// (l_fold_probe, clean/runtime/fold_probe.h: this expression on adversarial inputs) and the differential self-test of +// the real post shader (its fold port against its no-fold port on the crafted frame of clean/runtime/fold_selftest.h, +// raw bytes). Otherwise, or on any error, the device runs the no-fold port of the kernel. +#pragma once + +#ifndef LP_MUT +#define LP_MUT 0 +#endif +#ifndef LP_ENCCONTROL +#define LP_ENCCONTROL 0 // 1: the device guard's sensitivity control port (needs=control): two roundings +#endif +static const float kEncK = asfloat(0x3dddf184u); // 0.36f * 0.30102999566398120f + +// the 8-value conversion of the history codes (x0 .. x2: the FP32 log2 arguments) and the three display values +// d0 .. d2 (already rounded to FP32, passed as opaque values by the caller); z = an opaque zero (mutant 3 only) +uint4 lp_encode_fold(float x0, float x1, float x2, float d0, float d1, float d2, uint z) +{ + v8f cv; +#if LP_MUT == 3 || LP_ENCCONTROL // two roundings, the FP32 product made opaque before the conversion: the mutant + // (LP_MUT 3, never shipped) and the guard's control port (never selected to run) + cv.v[0] = asfloat(asuint(kEncK * x0) ^ z); + cv.v[1] = asfloat(asuint(kEncK * x1) ^ z); + cv.v[2] = asfloat(asuint(kEncK * x2) ^ z); +#else + cv.v[0] = mad(x0, kEncK, 0.0f); + cv.v[1] = mad(x1, kEncK, 0.0f); + cv.v[2] = mad(x2, kEncK, 0.0f); +#endif + cv.v[3] = d0; + cv.v[4] = d1; + cv.v[5] = d2; + cv.v[6] = 0.0f; + cv.v[7] = 0.0f; + return wg_to_f16_ags(cv); +} diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index 492c509f..189ecbb2 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -202,6 +202,10 @@ "l_direct_color", "-" ], + [ + "l_fold_probe", + "-" + ], [ "l_linear_32_32_f16", "l_ops.co" diff --git a/clean/dx12/kernels/l_fold_probe.hlsl b/clean/dx12/kernels/l_fold_probe.hlsl new file mode 100644 index 00000000..37c2451b --- /dev/null +++ b/clean/dx12/kernels/l_fold_probe.hlsl @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: MIT +// Start-up probe of the driver's conversion fold used by Model L's l_post_recon_p12 (hlsl/l_post_encode.hlsli): the +// device runs it once (clean/runtime/fold_probe.h holds the inputs and the expected bytes, computed from the exact +// products) and takes the no-fold port of the kernel when any history code differs. WrongGPU's own code. +// Buffer: lane l reads six FP32 values at byte 24 l (x0 x1 x2 = log2 arguments of the history codes, d0 d1 d2 = display +// values) and writes the three converted dwords at byte 768 + 12 l (q0 | q1 << 16, q2 | h(d0) << 16, h(d1) | h(d2) << 16). +// @wg-kernel l_fold_probe co=- block=32 wave=32 args=p +#include "wg_platform.hlsli" +WG_ARGS_BEGIN + uint dst; +WG_ARGS_END +#include "l_post_encode.hlsli" + +WG_KERNEL(l_fold_probe) +{ + WG_SETUP(); + const uint lane = wg_lane_id(); + const uint z = wg_grid_ >> 31; // 0, opaque to the compiler + const uint in0 = lane * 24u; + const float x0 = wg_ldf(dst, in0), x1 = wg_ldf(dst, in0 + 4u), x2 = wg_ldf(dst, in0 + 8u); + const float d0 = asfloat(asuint(wg_ldf(dst, in0 + 12u)) ^ z), d1 = asfloat(asuint(wg_ldf(dst, in0 + 16u)) ^ z); + const float d2 = asfloat(asuint(wg_ldf(dst, in0 + 20u)) ^ z); + const uint4 hc = lp_encode_fold(x0, x1, x2, d0, d1, d2, z); + const uint o = 768u + lane * 12u; + wg_st32(dst, o, hc.x); + wg_st32(dst, o + 4u, hc.y); + wg_st32(dst, o + 8u, hc.z); +} diff --git a/clean/dx12/kernels/l_post.hlsl b/clean/dx12/kernels/l_post.hlsl index d214e775..eb70ae58 100644 --- a/clean/dx12/kernels/l_post.hlsl +++ b/clean/dx12/kernels/l_post.hlsl @@ -31,7 +31,9 @@ // gives 0), the reduced codes rint(clamp(mean, 0, 1) 1023). // // @wg-kernel l_post_recon co=l_post.co block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p -// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1 +// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_ENCFOLD=1 needs=fold isa=v_fma_mixlo_f16@3dddf184+v_fma_mixhi_f16@3dddf184==6,s_round_mode==0 +// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 file=l_post_recon_p12__nofold block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1 needs=nofold +// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 file=l_post_recon_p12__control block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_ENCFOLD=1,LP_ENCCONTROL=1 needs=control isa=v_fma_mixlo_f16@3dddf184+v_fma_mixhi_f16@3dddf184==0 // @wg-kernel l_post_recon_ldr co=l_post.co entry=l_post_recon_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_LDR=1 // @wg-kernel l_post_recon_p12_ldr co=l_post.co entry=l_post_recon_p12_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_LDR=1 #include "wg_platform.hlsli" @@ -49,6 +51,7 @@ WG_ARGS_BEGIN uint expo; WG_ARGS_END +#include "l_post_encode.hlsli" // kEncK and the history encode through the driver's conversion fold #define kThreads 256 #define kTileX 32 #define kTilePairs 8 @@ -56,7 +59,6 @@ WG_ARGS_END #define kLWin 4 #define kTexDw 6 // staged texel: f0 f1 | f2 f3 | f4 f5 | f6 f7 | f8 kA | kB kC (FP16 bits) -static const float kEncK = asfloat(0x3dddf184u); // 0.36f * 0.30102999566398120f static const float kDecK = asfloat(0x4113a428u); // 3.32192809488736235f / 0.36f static const float kDecLn = asfloat(0x40ccac9au); // 2.30258509299404568f / 0.36f static const float kDecPolyMax = asfloat(0x3d60233du); // 0.35f / kDecLn @@ -83,11 +85,14 @@ float opq(float x) { return asfloat(asuint(x) ^ g_z); } half opqh(half x) { return asfloat16((uint16_t)(asuint16(x) ^ (uint16_t)g_z)); } float mul_nz(float a, float b) { return mad(a, b, asfloat(g_z | 0x80000000u)); } #ifndef LP_MUT -#define LP_MUT 0 // 1, 2, 4 ... 9: negative controls of the byte checks (never shipped) +#define LP_MUT 0 // 1..9: negative controls of the byte checks (never shipped) #endif #ifndef LP_DP #define LP_DP 1 #endif +#ifndef LP_ENCFOLD +#define LP_ENCFOLD 0 // 1 (l_post_recon_p12's fold port): the history encode through the driver's conversion fold +#endif #ifndef LP_FMANZ #define LP_FMANZ 1 #endif @@ -589,6 +594,14 @@ WG_KERNEL(l_post_recon) if (second) g = g1; float hh[3], oo[3]; finish_pixel(g, x, y0 + row, px, second ? py1 : py0, s, hh, oo); +#if !LP_LDR && LP_ENCFOLD + float xe[3], de[3]; + [unroll] for (int ce = 0; ce < 3; ++ce) { + xe[ce] = enc_log(hh[ce]); + de[ce] = opq(clampf(enc_e(oo[ce]), 0.0f, kDisplayMax)); + } + const uint4 hc = lp_encode_fold(xe[0], xe[1], xe[2], de[0], de[1], de[2], g_z); +#else v8f cv; [unroll] for (int ci = 0; ci < 3; ++ci) { #if LP_LDR @@ -606,14 +619,19 @@ WG_KERNEL(l_post_recon) cv.v[6] = 0.0f; cv.v[7] = 0.0f; const uint4 hc = wg_to_f16_ags(cv); +#endif const int y = y0 + row; if (and(active, or(!second, has1))) { #if LP_LDR const uint q0 = hc.x & 0xffffu, q1 = hc.x >> 16, q2 = hc.y & 0xffffu; +#else +#if LP_ENCFOLD + const uint q0 = hc.x & 0xffffu, q1 = hc.x >> 16, q2 = hc.y & 0xffffu; // the fold's +0 addend gives +0 #else const uint q0 = mul_zero(kEncK, enc_log(hh[0])) ? 0u : (hc.x & 0xffffu); const uint q1 = mul_zero(kEncK, enc_log(hh[1])) ? 0u : (hc.x >> 16); const uint q2 = mul_zero(kEncK, enc_log(hh[2])) ? 0u : (hc.y & 0xffffu); +#endif #endif wg_st64(hist_out, (uint)(y * P_dw + x) * 8u, uint2(q0 | (q1 << 16), q2)); const int di = y * pitch + x; diff --git a/clean/dx12/tests/kernel_replay.cpp b/clean/dx12/tests/kernel_replay.cpp index 96d123f2..97b916da 100644 --- a/clean/dx12/tests/kernel_replay.cpp +++ b/clean/dx12/tests/kernel_replay.cpp @@ -591,6 +591,7 @@ int main(int argc, char** argv) { auto d = make_d3d12_device(argv[1], &err); if (!d) { std::printf("device: %s\n", err.c_str()); return 1; } std::printf("device: %s\n", d->name().c_str()); + if (!d->note().empty()) std::printf("device note: %s\n", d->note().c_str()); // e.g. the conversion-fold guard std::ifstream f(argv[2]); if (!f) { std::printf("cannot read %s\n", argv[2]); return 2; } const std::string base = dir_of(argv[2]); diff --git a/clean/dx12/tests/test_fold_probe.py b/clean/dx12/tests/test_fold_probe.py new file mode 100644 index 00000000..87ac3a4d --- /dev/null +++ b/clean/dx12/tests/test_fold_probe.py @@ -0,0 +1,109 @@ +# SPDX-License-Identifier: MIT +"""CPU checks of the conversion-fold guard: clean/runtime/fold_probe.h and fold_selftest.h are what their generators +write (inputs and expected bytes from exact rationals), the self-test frame holds its value classes on every encode +site, and build_dx12.py's needs= / isa= declarations parse (and refuse bad values), the literal-tied ISA count included. +The GPU side (the probe kernel against these bytes, the self-test of the real post ports, the two-roundings mutant) is +a GPU-slot check.""" +import os +import subprocess +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.dx12 import build_dx12 as B # noqa: E402 + + +class FoldProbeTest(unittest.TestCase): + def test_header_is_generated(self): + with tempfile.TemporaryDirectory() as tmp: + out = os.path.join(tmp, "fold_probe.h") + subprocess.run([sys.executable, os.path.join(ROOT, "clean", "runtime", "gen_fold_probe.py"), out], check=True, + capture_output=True) + with open(out, encoding="utf-8") as f: + want = f.read() + with open(os.path.join(ROOT, "clean", "runtime", "fold_probe.h"), encoding="utf-8") as f: + self.assertEqual(f.read(), want) + + def test_selftest_header_is_generated(self): + with tempfile.TemporaryDirectory() as tmp: + out = os.path.join(tmp, "fold_selftest.h") + subprocess.run([sys.executable, os.path.join(ROOT, "clean", "runtime", "gen_fold_selftest.py"), out], + check=True, capture_output=True) + with open(out, encoding="utf-8") as f: + want = f.read() + with open(os.path.join(ROOT, "clean", "runtime", "fold_selftest.h"), encoding="utf-8") as f: + self.assertEqual(f.read(), want) + + def test_selftest_frame_shift(self): + """The P1 guarantee of the self-test frame: whatever log2 the device computes, within 8 ulp of the correctly + rounded one, each of the six encode sites (row of the pair, channel) keeps N = 13 FP32-product ties a second + rounding gets wrong. Model: every predicted log2 moved by the same e ulp, for every e in -8 ... 8. Also the + zero product and the floor are on every site.""" + sys.path.insert(0, os.path.join(ROOT, "clean", "runtime")) + import gen_fold_selftest as G + with open(os.path.join(ROOT, "clean", "runtime", "fold_selftest.h"), encoding="utf-8") as f: + body = f.read().split("{", 2)[2].split("}")[0] + colour = [int(v.strip().rstrip("u"), 16) for v in "".join(body.split()).split(",") if v.strip()] + self.assertEqual(len(colour), G.W * G.H * 3) + N = 13 + for e in G.OFFSETS: + per = G.shifted_detecting(colour, e) + self.assertGreaterEqual(min(per), N, (e, per)) + floor = G.Fraction(G.f32(G.bits32(1e-4))) + for site in range(G.SITES): + row, ch = divmod(site, 3) + ms = [G.m_of(G.f32(colour[3 * (y * G.W + x) + ch])) for y in range(row, G.H, 2) for x in range(G.W)] + self.assertIn(G.Fraction(1), ms) # x = 0: the zero product + self.assertTrue(any(m < floor for m in ms) and any(floor <= m < 2 * floor for m in ms)) + + def test_isa_rules(self): + self.assertEqual(B.parse_isa_rules("v_fma_mixlo_f16+v_fma_mixhi_f16>=6,s_round_mode==0", "t"), + [([("v_fma_mixlo_f16", None), ("v_fma_mixhi_f16", None)], ">=", 6), + ([("s_round_mode", None)], "==", 0)]) + self.assertEqual(B.parse_isa_rules("v_fma_mixlo_f16@3dddf184+v_fma_mixhi_f16@3dddf184==6", "t"), + [([("v_fma_mixlo_f16", "3dddf184"), ("v_fma_mixhi_f16", "3dddf184")], "==", 6)]) + with self.assertRaises(SystemExit): + B.parse_isa_rules("v_fma_mixlo_f16>6", "t") + with self.assertRaises(SystemExit): + B.parse_isa_rules("v_fma_mixlo_f16@3dddf18==6", "t") + + def test_isa_count(self): + lines = [" v_fma_mixlo_f16 v3, v3, 0xf184, 0 // 000000008224: CC210003 0201FF03 3DDDF184", + " v_fma_mixhi_f16 v10, v2, 0xf184, 0 // 000000008244: CC22000A 0201FF02 3DDDF184", + " v_fma_mixlo_f16 v3, v3, 0xf184, v4 // 000000008254: CC210003 0411FF03 3DDDF184", + " v_fma_mixlo_f16 v3, v3, 0xf184, 0 op_sel_hi:[1,0,0] // 000000008264: CC210003 0201FF03 3DDDF184", + " v_fma_mixlo_f16 v3, v3, 0xa8c1, 0 // 000000008274: CC210003 0201FF03 BF38A8C1", + " v_mul_f32_e32 v1, 0x3dddf184, v1 // 0000000009DC: 100202FF 3DDDF184"] + self.assertEqual(B.isa_count(lines, "v_fma_mixlo_f16", None), 4) + self.assertEqual(B.isa_count(lines, "v_fma_mixlo_f16", "3dddf184"), 1) # no other addend, no op_sel, literal + self.assertEqual(B.isa_count(lines, "v_fma_mixhi_f16", "3dddf184"), 1) + + def test_post_ports(self): + decls = [d for d in B.kernel_decls(os.path.join(ROOT, "clean", "dx12", "kernels", "l_post.hlsl")) + if d["name"] == "l_post_recon_p12"] + self.assertEqual(sorted(d["needs"] for d in decls), ["control", "fold", "nofold"]) + control = [d for d in decls if d["needs"] == "control"][0] + self.assertEqual(control["defines"].get("LP_ENCCONTROL"), "1") # two roundings + self.assertTrue(control["isa"].endswith("==0")) # and the build checks: no fold site + fold = [d for d in decls if d["needs"] == "fold"][0] + self.assertEqual(fold["defines"].get("LP_ENCFOLD"), "1") + self.assertIn("@3dddf184", fold["isa"]) # the fold's sites: kEncK as the literal, addend 0 + nofold = [d for d in decls if d["needs"] == "nofold"][0] + self.assertNotIn("LP_ENCFOLD", nofold["defines"]) + self.assertEqual(nofold["file"], "l_post_recon_p12__nofold") + + def test_bad_needs(self): + with tempfile.TemporaryDirectory() as tmp: + p = os.path.join(tmp, "k.hlsl") + with open(p, "w") as f: + f.write("// @wg-kernel x co=- block=32 wave=32 args=p needs=sometimes\n") + with self.assertRaises(SystemExit): + B.kernel_decls(p) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/l/runner/l_post_edge_check.py b/clean/l/runner/l_post_edge_check.py index 35d989f0..b5866d0e 100644 --- a/clean/l/runner/l_post_edge_check.py +++ b/clean/l/runner/l_post_edge_check.py @@ -10,7 +10,8 @@ first dword after the full history); /mut8 (counter + LP_MUT=8: the fast path's predicate always false) and /mut9 (LP_MUT=9: the predicate without its size guard, the bug of images narrower or shorter than 4). --run (GPU, one GPU slot). The kernels l_post_recon, l_post_recon_p12, l_post_recon_ldr and l_post_recon_p12_ldr (the -history sample, history_cr, is shared by all four), on every size of SIZES (1 x 1 ... 97 x 35: below 4, odd, a tile's interior boundary in the +history sample, history_cr, is shared by all four), and l_post_recon_p12 again on its no-fold port +(WG_TEST_FOLD_PROBE_FAIL=1; the shipped run must have selected the fold port), on every size of SIZES (1 x 1 ... 97 x 35: below 4, odd, a tile's interior boundary in the middle of a wave) and every kind of KINDS, with a valid history (no reset): - random: Model M's random inputs (random motions around a random base); - center / left / right / top / bottom: every motion points at one history position: the centre, or just inside @@ -20,7 +21,7 @@ - split: even columns point inside, odd columns at an edge (every wave mixes both); split_half: in each 32-column tile the first 16 columns (half the width when narrower) inside, the rest at an edge (the boundary in the middle of a wave). -Every replay leg (the shipped ports, hscount, mut8, mut9) starts from an emptied output folder and counts only when +Every replay leg (the shipped ports, the no-fold port, hscount, mut8, mut9) starts from an emptied output folder and counts only when kernel_replay exits 0 and every case wrote every output at its full size; otherwise the leg is invalid and every check that uses it fails (a failed mutant run is never taken as caught). Checks: @@ -182,10 +183,11 @@ def cases(kernel, rng): def prepare(build, rocm): C32.compile_kernels(build, rocm) - names = KERNELS + ["native_fill"] + names = KERNELS + ["l_fold_probe", "native_fill"] prepare_replay(build, rocm, names) from clean.runner.hip import hip_child_environment env, _ = hip_child_environment(rocm_dir=rocm) + env = dict(env, WG_DX12_ISA_CHECK="0") # test variants: the ISA rules are the shipped ports' (checked above) for tag, defs in VARIANTS.items(): cmd = [sys.executable, os.path.join(ROOT, "clean", "dx12", "build_dx12.py"), "--out", os.path.join(build, tag), "--only", ",".join(names), "-j", "1"] @@ -206,10 +208,12 @@ def replay(build_dir, exe, cdir, lines, env_extra=None): env = {k: v for k, v in os.environ.items() if not any(s in k.upper() for s in ("HIP", "LLVM", "ROCM", "HSA")) or k.upper() in ("HIP_PLATFORM",)} env["WG_D3D12_ARENA_MB"] = "512" + env.pop("WG_TEST_FOLD_PROBE_FAIL", None) env.update(env_extra or {}) r = subprocess.run([exe, build_dir, os.path.join(cdir, "cases.txt")], env=env, capture_output=True, text=True, timeout=150, creationflags=0x08000000) - return {"exit": r.returncode, "tail": r.stdout[-400:] + r.stderr[-200:]} + note = [ln for ln in r.stdout.splitlines() if ln.startswith("device note:")] + return {"exit": r.returncode, "note": note[0] if note else "", "tail": r.stdout[-400:] + r.stderr[-200:]} def empty_dir(d): @@ -277,12 +281,13 @@ def run(build, rocm, part="all"): res = {"sizes": SIZES, "kinds": KINDS, "part": part, "legs": {}} cdir_base = os.path.join(build, "replay-edge") for leg, bdir, env_extra, counter in () if part == "oracle" else (("d3d12", build, None, False), + ("d3d12_nofold", build, {"WG_TEST_FOLD_PROBE_FAIL": "1"}, False), ("hscount", os.path.join(build, "hscount"), None, True), ("mut8", os.path.join(build, "mut8"), None, True), ("mut9", os.path.join(build, "mut9"), None, False)): cdir = os.path.join(cdir_base, leg) empty_dir(cdir) - names = list(allc) + names = [n for n in allc if leg != "d3d12_nofold" or allc[n]["kernel"] == "l_post_recon_p12"] lines = [] for n in names: lines += OPS.replay_lines(n, allc[n], cdir) @@ -292,7 +297,7 @@ def run(build, rocm, part="all"): r = replay(bdir, exe, cdir, lines, env_extra) out = [compare_case(cdir, n, allc[n], hip_out[n], counter) for n in names] complete = sum(c["complete"] for c in out) - leg_res = {"replay_exit": r["exit"], "complete_cases": complete, "valid": r["exit"] == 0 and complete == len(out), + leg_res = {"replay_exit": r["exit"], "device_note": r["note"], "complete_cases": complete, "valid": r["exit"] == 0 and complete == len(out), "replay_tail": r["tail"], "cases": len(out), "identical": sum(c["identical"] for c in out), "edges": {k: {"pixels": sum(c["outputs"][0].get("edges", {}).get(k, {}).get("pixels", 0) for c in out), @@ -348,6 +353,9 @@ def run(build, rocm, part="all"): checks.update({ "legs_valid": all(v["valid"] for v in L.values()), "d3d12_identical": L["d3d12"]["valid"] and L["d3d12"]["identical"] == L["d3d12"]["cases"], + "fold_port_selected": "passed" in L["d3d12"]["device_note"] and "the fold ports run" in L["d3d12"]["device_note"], + "nofold_identical": L["d3d12_nofold"]["valid"] and L["d3d12_nofold"]["identical"] == L["d3d12_nofold"]["cases"], + "nofold_port_selected": "the no-fold ports run" in L["d3d12_nofold"]["device_note"], "hscount_identical": L["hscount"]["valid"] and L["hscount"]["identical"] == L["hscount"]["cases"], "hscount_counter_ok": L["hscount"]["valid"] and L["hscount"]["counter"]["expectation_failures"] == 0, "mut8_caught": L["mut8"]["valid"] and L["mut8"]["counter"]["expectation_failures"] > 0, diff --git a/clean/runtime/device_d3d12.cpp b/clean/runtime/device_d3d12.cpp index 9d738819..1681d6ca 100644 --- a/clean/runtime/device_d3d12.cpp +++ b/clean/runtime/device_d3d12.cpp @@ -20,6 +20,8 @@ #include "embedded.h" #include "rdna4.h" #include "wmma_probe.h" +#include "fold_probe.h" +#include "fold_selftest.h" #include #include @@ -75,6 +77,9 @@ struct NativeKernel { // workgroups (manifest "group", build_dx12.py group=; 0 = block). The launch keeps the HIP geometry (grid x block). unsigned group = 0, gscale = 1; // gscale: D3D12 workgroups per launched workgroup (manifest "gscale", default block / group) bool ext_doubles = false; // manifest "fp64": "ext": binary64 fma / division (DoublePrecisionFloatShaderOps) + // manifest "needs": "fold" (run only when the start-up probe of the driver's conversion fold passed) or "nofold" + // (only when it failed): two ports of one kernel and code object whose exactness depends on that fold + std::string needs; std::vector args; wgc::Digest digest{}; NativeCom pso; @@ -215,7 +220,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { } if (!create_objects()) return false; if (!profile_init()) return false; - return reserve_events(64) && (!borrowed_ || driver_probe()) && gelu_lut_upload() && gelu_t5_upload(); + return reserve_events(64) && (!borrowed_ || driver_probe()) && fold_probe() && gelu_lut_upload() && gelu_t5_upload(); } // The MLP kernels' exact GELU + FP8 table (hlsl/wg_gelu_lut.hlsli) at its fixed arena offset, reserved in @@ -454,7 +459,8 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { std::string note() const override { return "native D3D12: arena " + std::to_string(arena_bytes_ >> 20) + " MiB; no HIP runtime or fallback; timestamp frequency " + std::to_string(timestamp_frequency_) + " Hz" + - (borrowed_ ? "; borrowed device/queue; exact FP16/FP8 driver-channel probe passed" : ""); + (borrowed_ ? "; borrowed device/queue; exact FP16/FP8 driver-channel probe passed" : "") + + (fold_note_.empty() ? "" : "; " + fold_note_); } bool adapter_luid(uint8_t out[8]) const override { if (!out) return false; @@ -638,6 +644,11 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (direct && !direct_bound_) return fail("native D3D12 direct kernel requires a bound direct frame: " + name); // Check the variant BEFORE touching args: a 37-DWORD post variant cannot satisfy a 42-DWORD port. if (!variant_loaded(k)) return false; + return dispatch(k, name, grid, block, args); + } + // One launch of the port k (launch() after its port selection; the fold self-test runs both ports of a kernel). + bool dispatch(NativeKernel& k, const std::string& name, unsigned grid, unsigned block, void** args) { + const bool direct = k.rs == "direct"; if (block != k.block) return fail("native D3D12 kernel " + name + ": block must be " + std::to_string(k.block)); if (!grid || grid > 0x7fffffffu) return fail("native D3D12 kernel " + name + ": grid must be 1..2147483647"); const uint64_t groups = (uint64_t)grid * k.gscale; // D3D12 workgroups @@ -998,7 +1009,11 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { return native_filename(co) && co.size() > 3 && co.substr(co.size() - 3) == ".co"; } // variant_loaded without an error: whether the runtime's loaded code objects select this port. + bool needs_met(const NativeKernel& k) const { + return k.needs.empty() || (k.needs != "control" && (k.needs == "fold") == fold_ok_); // control: never selected + } bool variant_matches(const NativeKernel& k) const { + if (!needs_met(k)) return false; if (k.co == "-") return true; const size_t slash = k.name.find('/'); if (slash == std::string::npos) return untagged_cos_.count(k.co) != 0; @@ -1028,6 +1043,9 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { } ID3D12GraphicsCommandList* cl() const { return external_ ? ext_list_ : list_.get(); } bool variant_loaded(const NativeKernel& k) { + if (!needs_met(k)) + return fail("native D3D12 " + k.name + ": the port for the conversion-fold probe's result (" + + std::string(fold_ok_ ? "fold" : "nofold") + ") is missing"); if (k.co == "-") return true; const size_t slash = k.name.find('/'); if (slash == std::string::npos) { @@ -1084,7 +1102,11 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (completed >= value) return true; if (!check(fence_->SetEventOnCompletion(value, event_), "SetEventOnCompletion")) return false; ProfileCpu cpu(this, "fence_wait_block"); - const DWORD result = WaitForSingleObject(event_, INFINITE); + const DWORD result = WaitForSingleObject(event_, wait_ms_); + if (result == WAIT_TIMEOUT) { // only while wait_ms_ is bounded (the start-up fold guard) + stalled_ = true; + return fail("native D3D12: queue wait timed out after " + std::to_string(wait_ms_) + " ms"); + } if (result != WAIT_OBJECT_0) return fail("native D3D12: queue wait failed (Win32 " + std::to_string(GetLastError()) + ")"); completed = fence_->GetCompletedValue(); return completed != UINT64_MAX && completed >= value ? true : fail("native D3D12: queue wait ended without completion"); @@ -1278,6 +1300,11 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (!fp->is_string() || fp->text != "ext") return fail("native D3D12 manifest: invalid fp64 for " + k.name); k.ext_doubles = true; } + if (const auto* nd = entry.find("needs")) { + if (!nd->is_string() || (nd->text != "fold" && nd->text != "nofold" && nd->text != "control")) + return fail("native D3D12 manifest: invalid needs for " + k.name); + k.needs = nd->text; + } const auto* signature = entry.find("rs"); if (signature) { if (!signature->is_string() || (signature->text != "buffer" && signature->text != "tex" && signature->text != "direct")) @@ -1325,9 +1352,10 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { } if (words != k.dwords) return fail("native D3D12 manifest: argument DWORD count mismatch for " + k.name); k.file = directory + filename; const std::string key = k.name; - auto same = kernels_.equal_range(key); // one port per (name, code object) + auto same = kernels_.equal_range(key); // one port per (name, code object, needs) for (auto i = same.first; i != same.second; ++i) - if (i->second.co == k.co) return fail("native D3D12 manifest: duplicate kernel " + key + " (" + k.co + ")"); + if (i->second.co == k.co && i->second.needs == k.needs) + return fail("native D3D12 manifest: duplicate kernel " + key + " (" + k.co + ")"); kernels_.emplace(key, std::move(k)); } return true; @@ -1407,6 +1435,172 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (!ok) err_ = error; return ok; } + // Start-up guard of the driver's conversion fold: only when a port needs it. All three steps must pass, else (or on + // any error) the "nofold" ports run (fail closed): + // 1. probe: l_fold_probe (fold_probe.h) on adversarial inputs of the shared encode expression, against expected + // bytes from exact rationals; + // 2. differential self-test (fold_selftest): the REAL post shader, its fold port and its no-fold port (exact by + // construction), on one crafted frame (fold_selftest.h); the raw output bytes must be equal; + // 3. sensitivity control (in fold_selftest): the same frame through the "control" port (the post shader with the + // history encode rounded twice, as a driver without the fold would compile it) must differ from the no-fold port + // at each of the six encode sites, so the frame's values, as this device computes them, would show a missing fold + // at every site. + // Every queue wait of the guard is bounded (kGuardWaitMs). A timeout means a queue that did not complete: device + // creation fails (its allocations are kept, never freed under a running queue). Logged once per process (note()). + static constexpr DWORD kGuardWaitMs = 10000; + bool fold_probe() { + bool wanted = false; + std::string uncovered; + for (const auto& kv : kernels_) { + if (kv.second.needs != "fold") continue; + wanted = true; + if (kv.first != "l_post_recon_p12") uncovered = kv.first; // a fold port with no self-test of its own + } + fold_ok_ = false; + if (!wanted) return true; + std::string note; + const std::string before = err_; + wait_ms_ = kGuardWaitMs; + auto probe = kernels_.find("l_fold_probe"); + if (!uncovered.empty()) { + note = "no self-test for " + uncovered; + } else if (probe == kernels_.end() || probe->second.co != "-" || probe->second.block != 32 || + probe->second.dwords != 1 || probe->second.args.size() != 1 || probe->second.args[0].kind != 'p') { + note = "no l_fold_probe kernel"; + } else { + DevPtr buf = 0; + uint8_t bytes[fold_probe::kBytes]; + fold_probe::fill(bytes); + uint32_t out[96] = {}; + void* args[] = {&buf}; + bool ok = alloc(&buf, fold_probe::kBytes) && upload(buf, bytes, sizeof bytes) && launch("l_fold_probe", 1, 32, args) && + download(out, buf + fold_probe::kOutOffset, sizeof out); + if (buf && !stalled_) free(buf); // after a timeout the queue may still use it: kept + const char* fail_test = std::getenv("WG_TEST_FOLD_PROBE_FAIL"); // tests of the fallback + if (ok && fail_test && fail_test[0] == '1') out[0] ^= 1u; + const int bad = ok ? fold_probe::first_mismatch(out) : -1; + if (!ok) note = "probe run failed (" + err_ + ")"; + else if (bad >= 0) note = "probe mismatch at half " + std::to_string(bad); + else if (!stalled_) fold_ok_ = fold_selftest(¬e); + } + if (stalled_) { // the queue did not complete within the bound: no fallback can run on it + fold_ok_ = false; + return fail("native D3D12: the start-up fold guard's GPU work did not complete within " + + std::to_string(kGuardWaitMs) + " ms (" + note + ")"); + } + wait_ms_ = INFINITE; + err_ = before; + // one line per device: in note(), which the runtime logs once when the network runtime is ready + fold_note_ = std::string("conversion-fold probe and self-test ") + + (fold_ok_ ? "passed (" + sens_note_ + "): the fold ports run" : "failed (" + note + "): the no-fold ports run"); + return true; + } + // The differential self-test of l_post_recon_p12 with its sensitivity control (fold_probe steps 2 and 3). The frame + // (fold_selftest.h; render size = output size, jitter 0, reset, exposure 1, post scale 0, codes and kernel shape 0) + // makes each full-history value the render colour, so the history codes are the encode of crafted values: per site + // FP32-product ties at predicted-log2 offsets -8 ... 8 ulp (a log2 up to 8 ulp off still lands on ties), zero and + // FP16-subnormal products, the floor. Each port's outputs start filled with its own sentinel. The fold port's defined + // bytes (full history, reduced history, display image) must equal the no-fold port's; the control port's full + // history must differ from the no-fold port's in at least one value of each of the six encode sites (site = row of + // the pair x channel). True only when all three ports ran and both conditions hold. + bool fold_selftest(std::string* note) { + NativeKernel *fold = nullptr, *nofold = nullptr, *control = nullptr; + auto range = kernels_.equal_range("l_post_recon_p12"); + for (auto i = range.first; i != range.second; ++i) { + if (i->second.needs == "fold") fold = &i->second; + if (i->second.needs == "nofold") nofold = &i->second; + if (i->second.needs == "control") control = &i->second; + } + auto abi = [](const NativeKernel* k) { + if (!k || k->block != 256 || k->rs != "buffer" || k->args.size() != 14) return false; + for (int n = 0; n < 12; ++n) if (k->args[n].kind != 'p') return false; + return k->args[12].kind == 'b' && k->args[12].bytes == 64 && k->args[13].kind == 'p'; + }; + if (!abi(fold) || !abi(nofold) || !abi(control)) { + *note = "self-test: the fold / no-fold / control ports of l_post_recon_p12 are missing"; + return false; + } + constexpr int W = ::fold_selftest::kW, H = ::fold_selftest::kH, N = W * H; + constexpr int pitch = (W + 63) & ~63; + constexpr size_t hist = size_t(N) * 8, reduced = size_t(W / 2) * (H / 2) * 4, display = size_t(3) * pitch * H * 2; + constexpr size_t outs = hist + reduced + display; + constexpr int kPorts = 3; // fold, no-fold, control + const uint8_t sentinel[kPorts] = {0xA5, 0x5A, 0x3C}; + // layout: colour | A8 | C4 | previous history | motions x 3 | post blob | outputs (fold, no-fold, control) + const size_t o_col = 0, o_a = o_col + size_t(N) * 16, o_c = o_a + size_t(N) * 8, o_hp = o_c + size_t(N) * 4, + o_m = o_hp + hist, o_w = o_m + size_t(N) * 12, o_out = o_w + 256, total = o_out + kPorts * outs; + std::vector host(total, 0); + for (int i = 0; i < N; ++i) std::memcpy(&host[o_col + size_t(i) * 16], &::fold_selftest::kColour[3 * i], 12); + for (int port = 0; port < kPorts; ++port) std::memset(&host[o_out + port * outs], sentinel[port], outs); + struct { int32_t rw, rh, dw, dh, reset, pad0, pad1, pad2; float rx, ry, jx, jy, e, e_prev, ratio, pad3; } prm = + {W, H, W, H, 1, 0, 0, 0, 1.0f, 1.0f, 0.0f, 0.0f, 1.0f, 1.0f, 1.0f, 0.0f}; + static_assert(sizeof prm == 64, "WgPostParams"); + DevPtr buf = 0; + std::vector got(kPorts * outs); + bool ok = alloc(&buf, total) && upload(buf, host.data(), total); + NativeKernel* ports[kPorts] = {fold, nofold, control}; + for (int port = 0; ok && port < kPorts; ++port) { + DevPtr p[12], none = 0; + const DevPtr out = buf + o_out + port * outs; + const DevPtr in[9] = {buf + o_col, buf + o_a, buf + o_a, buf + o_c, buf + o_hp, buf + o_m, buf + o_m + N * 4, + buf + o_m + N * 8, buf + o_w}; + for (int n = 0; n < 9; ++n) p[n] = in[n]; + p[9] = out; p[10] = out + hist; p[11] = out + hist + reduced; + void* args[14]; + for (int n = 0; n < 12; ++n) args[n] = &p[n]; + args[12] = &prm; args[13] = &none; + const unsigned grid = unsigned((W + 31) / 32) * unsigned(((H + 1) / 2 + 7) / 8); + ok = dispatch(*ports[port], "l_post_recon_p12", grid, 256, args); + } + ok = ok && download(got.data(), buf + o_out, kPorts * outs); + const std::string error = err_; + if (buf && !stalled_) free(buf); // after a timeout the queue may still use it: kept + if (!ok) { *note = "self-test run failed (" + error + ")"; return false; } + if (const char* dump = std::getenv("WG_TEST_FOLD_SELFTEST_DUMP")) { // tests: the three ports' outputs, raw + if (FILE* f = std::fopen(dump, "wb")) { std::fwrite(got.data(), 1, got.size(), f); std::fclose(f); } + } + const uint8_t* fo = got.data(); + const uint8_t* nf = got.data() + outs; + const uint8_t* ct = got.data() + 2 * outs; + size_t differ = 0, hist_differ = 0; + for (size_t i = 0; i < outs; ++i) { + if (i >= hist + reduced) { // display: only the pixels of each pitch row are defined + const size_t px = ((i - hist - reduced) / 2) % pitch; + if (px >= size_t(W)) continue; + } + if (fo[i] != nf[i]) { ++differ; if (i < hist) ++hist_differ; } + } + if (differ) { + *note = "self-test: the fold port differs from the no-fold port in " + std::to_string(differ) + " bytes (" + + std::to_string(hist_differ) + " of the full history)"; + return false; + } + // sensitivity per site: history values (pixel (x, y), channel c: bytes 8 (y W + x) + 2 c) where the control port + // (two roundings) differs from the no-fold port; every control value must have been written (no sentinel left) + int sens[6] = {0, 0, 0, 0, 0, 0}; + bool written = true; + for (int y = 0; y < H; ++y) { + for (int x = 0; x < W; ++x) { + for (int c = 0; c < 3; ++c) { + const size_t o = size_t(8) * (size_t(y) * W + x) + 2 * c; + written = written && !(ct[o] == sentinel[2] && ct[o + 1] == sentinel[2]); + if (ct[o] != nf[o] || ct[o + 1] != nf[o + 1]) ++sens[3 * (y & 1) + c]; + } + } + } + int least = sens[0]; + for (int k = 1; k < 6; ++k) least = std::min(least, sens[k]); + if (!written || least < 1) { + std::string per; + for (int k = 0; k < 6; ++k) per += (k ? "/" : "") + std::to_string(sens[k]); + *note = std::string("sensitivity control: ") + (written ? "" : "control output incomplete; ") + + "the two-roundings port differs from the no-fold port at the six sites in " + per + + " values (each must be at least 1)"; + return false; + } + sens_note_ = "sensitivity control: at least " + std::to_string(least) + " values per site"; + return true; + } bool create_device() { dxgi_dll_ = LoadLibraryExW(L"dxgi.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32); d3d12_dll_ = LoadLibraryExW(L"d3d12.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32); @@ -1695,6 +1889,11 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { std::string profile_session_, profile_failure_; bool profile_enabled_ = false; bool borrowed_ = false, native_binding_dirty_ = false, inside_pass_ = false; + bool fold_ok_ = false; // fold_probe's result + DWORD wait_ms_ = INFINITE; // queue wait bound: kGuardWaitMs during the start-up fold guard, else none + bool stalled_ = false; // a bounded wait timed out: the queue never completed (device creation fails) + std::string sens_note_; // fold_selftest's sensitivity count (in the passed note) + std::string fold_note_; // fold_probe's result as a note() clause (empty: no port needs the fold) bool direct_bound_ = false, direct_root_bound_ = false; D3D12_COMMAND_LIST_TYPE queue_type_ = D3D12_COMMAND_LIST_TYPE_COMPUTE; uint64_t debug_cursor_ = 0, debug_denied_ = 0; diff --git a/clean/runtime/device_dx12.cpp b/clean/runtime/device_dx12.cpp index d06e9c60..1cfdfc59 100644 --- a/clean/runtime/device_dx12.cpp +++ b/clean/runtime/device_dx12.cpp @@ -524,6 +524,11 @@ class Dx12Device : public Device { } std::set seen; for (const wgc::Json& k : ks->items) { + // a port that needs the driver's conversion fold (manifest "needs": "fold") runs only on the native device after + // its start-up guard (device_d3d12.cpp fold_probe); this device has no guard and takes the exact "nofold" port + // (and never a guard's "control" port) + if (const wgc::Json* nd = k.find("needs"); nd && nd->is_string() && (nd->text == "fold" || nd->text == "control")) + continue; Kernel x; int64_t block = 0, wave = 0, dwords = 0; const wgc::Json *n = k.find("name"), *fl = k.find("file"), *ar = k.find("args"), *co = k.find("co"); diff --git a/clean/runtime/fold_probe.h b/clean/runtime/fold_probe.h new file mode 100644 index 00000000..70f08656 --- /dev/null +++ b/clean/runtime/fold_probe.h @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: MIT +// Generated by clean/runtime/gen_fold_probe.py - do not edit. +// Start-up probe of the driver's conversion fold (clean/dx12/kernels/l_fold_probe.hlsl): inputs (32 lanes x +// 6 FP32 bits) and expected output dwords (32 x 3); nan_mask bit 0 / 1 = the low / high FP16 half must be a +// NaN (any sign and payload: the accepted port exception), the other halves must match bit for bit. +#pragma once +#include +#include + +namespace fold_probe { +constexpr unsigned kInBytes = 768, kOutOffset = 768, kOutBytes = 384, kBytes = 1152; +constexpr uint32_t kInput[192] = { + 0x3de24abbu, 0x3d1f8941u, 0x3db62c2du, 0x3f2bd000u, 0x3f2bd001u, 0x39091000u, + 0x3c4ef8bfu, 0xc00113f4u, 0x3e59edeeu, 0x39091001u, 0x3a9f9000u, 0x3a9f9001u, + 0xbe68340du, 0xbcb24799u, 0xbdd67815u, 0x40391000u, 0x40391001u, 0x470db000u, + 0xbc580e19u, 0xbd14b8fbu, 0x414eaeedu, 0x470db001u, 0x3f7e3000u, 0x3f7e3001u, + 0x3b516c39u, 0xbb2b5be7u, 0x3ddfd741u, 0x44a05000u, 0x44a05001u, 0x4641d000u, + 0xc012ab03u, 0x3e00a539u, 0x3e7b3c36u, 0x4641d001u, 0x44d9f000u, 0x44d9f001u, + 0xbd31fdc6u, 0xba79a633u, 0x3b9c5d3bu, 0x41f03000u, 0x41f03001u, 0x3f1ef000u, + 0xc023b7a8u, 0xc1d224c6u, 0xc0829783u, 0x3f1ef001u, 0x40363000u, 0x40363001u, + 0x399ebe40u, 0x39a5164du, 0xbb7e8d26u, 0x3c7e7000u, 0x3c7e7001u, 0x3ff3f000u, + 0x3d7cad50u, 0x3e21442eu, 0xc048c59bu, 0x3ff3f001u, 0x3e30d000u, 0x3e30d001u, + 0xbfb14539u, 0xbc3a10c1u, 0x3a24def0u, 0x3c367000u, 0x3c367001u, 0x4765f000u, + 0x3bec3d8au, 0x3c5c6168u, 0xbfb3b8b3u, 0x4765f001u, 0x38ad5000u, 0x38ad5001u, + 0x40d730a3u, 0x3db0b195u, 0xc1f4e428u, 0x39f53000u, 0x39f53001u, 0x3b62d000u, + 0xbd45be7du, 0xbf9fd313u, 0x3f6a13e2u, 0x3b62d001u, 0x4708f000u, 0x4708f001u, + 0x40fffe41u, 0x3af87eeau, 0x40d83302u, 0x42e15000u, 0x42e15001u, 0x45ba3000u, + 0xbfb97d1du, 0x4200a539u, 0x409b7fc4u, 0x45ba3001u, 0x3b825000u, 0x3b825001u, + 0xbb4fd635u, 0xbd0a16d9u, 0xc12a0fb5u, 0x43507000u, 0x43507001u, 0x3e0af000u, + 0x3b90cb2du, 0xbff04707u, 0xbed52be4u, 0x3e0af001u, 0x3fd5d000u, 0x3fd5d001u, + 0xbfa6e3aeu, 0xbd5eaff9u, 0xc028e86du, 0x388e5000u, 0x388e5001u, 0x41c35000u, + 0xc0133ea7u, 0x3b67c552u, 0x3bec6273u, 0x41c35001u, 0x44f35000u, 0x44f35001u, + 0x00000000u, 0x80000000u, 0x7fc00000u, 0x42747000u, 0x42747001u, 0x3ad11000u, + 0xc1549a78u, 0x00000000u, 0x3f800000u, 0x3ad11001u, 0x4081b000u, 0x4081b001u, + 0xbf800000u, 0x41a00000u, 0x399d4952u, 0x3c839000u, 0x3c839001u, 0x46f91000u, + 0xb951b717u, 0x358637bdu, 0xb58637bdu, 0x46f91001u, 0x45b45000u, 0x45b45001u, + 0x4102409du, 0x418bcba4u, 0x4154e574u, 0x3b4b1000u, 0x3b4b1001u, 0x40259000u, + 0xc0b2699fu, 0xc0404665u, 0x41854d96u, 0x40259001u, 0x46989000u, 0x46989001u, + 0xc151e910u, 0x416de3e1u, 0x4160a1b7u, 0x4534f000u, 0x4534f001u, 0x00000000u, + 0x403002c4u, 0xc039fbb4u, 0xc0700006u, 0x80000000u, 0x7fc00000u, 0x477fef00u, + 0xc091daeau, 0x3ffba9bbu, 0x40803132u, 0x477ff000u, 0x49742400u, 0x33800000u, + 0x40b5eb30u, 0x41a6c3e9u, 0x415e367eu, 0x33000001u, 0x40a3c481u, 0x40bde4b3u, + 0x4100a6e7u, 0x41a4f87du, 0xc0bd46f5u, 0x3f3b620cu, 0x408a8867u, 0x4081fd5cu, + 0xc0f9c053u, 0x40f6b935u, 0xc13caf44u, 0x40df1019u, 0x4038f79au, 0x40992297u, +}; +constexpr uint32_t kExpect[96] = { + 0x1c522222u, 0x395e20f0u, 0x0848395fu, 0xb2fe159bu, 0x084925e7u, 0x14fd14fcu, + 0x98d5a64au, 0x41c8a1cfu, 0x786e41c9u, 0x9c0795dbu, 0x786e3d99u, 0x3bf23bf2u, + 0x8ca50dadu, 0x65022211u, 0x720e6503u, 0x22f9b3f3u, 0x720f26ceu, 0x66d066d0u, + 0x86c49cd2u, 0x4f82103du, 0x38f84f82u, 0xc1b1b470u, 0x38f8b714u, 0x41b241b2u, + 0x023c0227u, 0x23f48ee5u, 0x3fa023f4u, 0x245f1ed8u, 0x3fa0b571u, 0x31873186u, + 0x950bb0cdu, 0x21b40478u, 0x7b3021b4u, 0x15f91266u, 0x7b30b0dfu, 0x056b056au, + 0x20c939d5u, 0x0faac2a2u, 0x1b160faau, 0xb0549d5cu, 0x1b172e57u, 0x78487848u, + 0x0abb3af0u, 0x570a39dcu, 0x6dd2570bu, 0x42f9b107u, 0x6dd23836u, 0x1c131c12u, + 0x9b7c8da1u, 0x5a84bc9bu, 0x30585a84u, 0xb2820fd8u, 0x3058a9c7u, 0x3eaf3eaeu, + 0x9e09b085u, 0x0472b493u, 0x4e1a0473u, 0x0e48b3fbu, 0x4e1b1267u, 0x679b679au, + 0x00000000u, 0x53a47e00u, 0x168853a4u, 0x0000bdc3u, 0x16892ef0u, 0x440e440eu, + 0x4056aef0u, 0x241c0221u, 0x77c8241du, 0x0002816cu, 0x77c98002u, 0x6da36da2u, + 0x3f933b0fu, 0x1a583dc5u, 0x412c1a59u, 0xb536b8d5u, 0x412d3f39u, 0x74c574c4u, + 0x3e72bdb0u, 0x69a83e16u, 0x000069a8u, 0xb50a34c5u, 0x8000b681u, 0x7bff7e00u, + 0x32d1b7e7u, 0x7c0036f2u, 0x00017c00u, 0x408538eeu, 0x00013e05u, 0x45ef451eu, + 0x40783af9u, 0x39dbb921u, 0x44104454u, 0x3aafbac4u, 0x46f9bd1du, 0x44c941c8u, +}; +constexpr uint8_t kNanMask[96] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +inline void fill(uint8_t* buffer) { + std::memset(buffer, 0, kBytes); + std::memcpy(buffer, kInput, sizeof kInput); +} +inline bool half_ok(uint32_t got, uint32_t want, bool nan) { + if (nan) return (got & 0x7c00u) == 0x7c00u && (got & 0x03ffu) != 0u; + return got == want; +} +// index of the first differing output half (0 .. 191), or -1 when every half matches +inline int first_mismatch(const uint32_t* out) { + for (unsigned i = 0; i < 96; ++i) { + if (!half_ok(out[i] & 0xffffu, kExpect[i] & 0xffffu, (kNanMask[i] & 1u) != 0)) return (int)(2 * i); + if (!half_ok(out[i] >> 16, kExpect[i] >> 16, (kNanMask[i] & 2u) != 0)) return (int)(2 * i + 1); + } + return -1; +} +} // namespace fold_probe diff --git a/clean/runtime/fold_selftest.h b/clean/runtime/fold_selftest.h new file mode 100644 index 00000000..9269ef62 --- /dev/null +++ b/clean/runtime/fold_selftest.h @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: MIT +// Generated by clean/runtime/gen_fold_selftest.py (do not edit): the render colour image of the native D3D12 +// device's differential self-test of Model L's post (fold port against no-fold port, device_d3d12.cpp). +// FP32 bits, slot 3 (y kW + x) + channel. Synthetic values from exact rational arithmetic; no model data. +// Per site (row of the pair, channel): 25 special values (floor, zero and FP16-subnormal products), then +// per site 13 FP32-product ties a second rounding gets wrong, each at predicted-log2 offsets -8..8 ulp (39 exact +// products above the midpoint, 39 below), then 60 predicted ties of negative products over all sites. +#pragma once +#include + +namespace fold_selftest { + +constexpr int kW = 32, kH = 16; +constexpr uint32_t kColour[1536] = { + 0xbc23d2d9u, 0xbc23d2d9u, 0xbc23d2d9u, 0xbc23d2dau, 0xbc23d2dau, 0xbc23d2dau, 0xbc23d2dbu, 0xbc23d2dbu, + 0xbc23d2dbu, 0xbc23d2d8u, 0xbc23d2d8u, 0xbc23d2d8u, 0xbc23d2d7u, 0xbc23d2d7u, 0xbc23d2d7u, 0xbc23d2d6u, + 0xbc23d2d6u, 0xbc23d2d6u, 0x00000000u, 0x00000000u, 0x00000000u, 0xbc23d70au, 0xbc23d70au, 0xbc23d70au, + 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xf149f2cau, 0xf149f2cau, 0xf149f2cau, 0x00000000u, 0x00000000u, + 0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x00000001u, 0x00000001u, 0x00000001u, 0x80000001u, + 0x80000001u, 0x80000001u, 0x0da24260u, 0x0da24260u, 0x0da24260u, 0x8da24260u, 0x8da24260u, 0x8da24260u, + 0x30a3d70au, 0x30a3d70au, 0x30a3d70au, 0x3123d70au, 0x3123d70au, 0x3123d70au, 0x320f5c29u, 0x320f5c29u, + 0x320f5c29u, 0x34000000u, 0x34000000u, 0x34000000u, 0x36700000u, 0x36700000u, 0x36700000u, 0xb023d70au, + 0xb023d70au, 0xb023d70au, 0xb0f5c28fu, 0xb0f5c28fu, 0xb0f5c28fu, 0xb3000000u, 0xb3000000u, 0xb3000000u, + 0xb6700000u, 0xb6700000u, 0xb6700000u, 0x5ff901b8u, 0x621df7bbu, 0x6b8b03e2u, 0x5ff90163u, 0x621df784u, + 0x6b8b03b2u, 0x5ff9010du, 0x621df74eu, 0x6b8b0382u, 0x5ff900b6u, 0x621df717u, 0x6b8b0352u, 0x5ff90060u, + 0x621df6e0u, 0x6b8b0321u, 0x5ff90009u, 0x621df6a9u, 0x6b8b02f1u, 0x5ff8ffb3u, 0x621df673u, 0x6b8b02c1u, + 0xbc23d2d9u, 0xbc23d2d9u, 0xbc23d2d9u, 0xbc23d2dau, 0xbc23d2dau, 0xbc23d2dau, 0xbc23d2dbu, 0xbc23d2dbu, + 0xbc23d2dbu, 0xbc23d2d8u, 0xbc23d2d8u, 0xbc23d2d8u, 0xbc23d2d7u, 0xbc23d2d7u, 0xbc23d2d7u, 0xbc23d2d6u, + 0xbc23d2d6u, 0xbc23d2d6u, 0x00000000u, 0x00000000u, 0x00000000u, 0xbc23d70au, 0xbc23d70au, 0xbc23d70au, + 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xf149f2cau, 0xf149f2cau, 0xf149f2cau, 0x00000000u, 0x00000000u, + 0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x00000001u, 0x00000001u, 0x00000001u, 0x80000001u, + 0x80000001u, 0x80000001u, 0x0da24260u, 0x0da24260u, 0x0da24260u, 0x8da24260u, 0x8da24260u, 0x8da24260u, + 0x30a3d70au, 0x30a3d70au, 0x30a3d70au, 0x3123d70au, 0x3123d70au, 0x3123d70au, 0x320f5c29u, 0x320f5c29u, + 0x320f5c29u, 0x34000000u, 0x34000000u, 0x34000000u, 0x36700000u, 0x36700000u, 0x36700000u, 0xb023d70au, + 0xb023d70au, 0xb023d70au, 0xb0f5c28fu, 0xb0f5c28fu, 0xb0f5c28fu, 0xb3000000u, 0xb3000000u, 0xb3000000u, + 0xb6700000u, 0xb6700000u, 0xb6700000u, 0x6b589005u, 0x433f658fu, 0x54a43d32u, 0x6b588fbau, 0x433f6586u, + 0x54a43d16u, 0x6b588f6fu, 0x433f657fu, 0x54a43cfau, 0x6b588f24u, 0x433f6576u, 0x54a43cdcu, 0x6b588ed8u, + 0x433f656eu, 0x54a43cc0u, 0x6b588e8eu, 0x433f6565u, 0x54a43ca4u, 0x6b588e43u, 0x433f655du, 0x54a43c88u, + 0x5ff8ff5cu, 0x621df63cu, 0x6b8b0291u, 0x5ff8ff06u, 0x621df605u, 0x6b8b0261u, 0x5ff8feb1u, 0x621df5ceu, + 0x6b8b0231u, 0x5ff8fe5au, 0x621df597u, 0x6b8b0201u, 0x5ff8fe04u, 0x621df561u, 0x6b8b01d0u, 0x5ff8fdadu, + 0x621df52au, 0x6b8b01a0u, 0x5ff8fd57u, 0x621df4f3u, 0x6b8b0170u, 0x5ff8fd01u, 0x621df4bcu, 0x6b8b0140u, + 0x5ff8fcaau, 0x621df486u, 0x6b8b0110u, 0x5ff8fc54u, 0x621df44fu, 0x6b8b00e0u, 0x56acc6ddu, 0x5ac40b13u, + 0x546d61d2u, 0x56acc6c0u, 0x5ac40af1u, 0x546d61a9u, 0x56acc6a1u, 0x5ac40acfu, 0x546d617fu, 0x56acc684u, + 0x5ac40aadu, 0x546d6156u, 0x56acc665u, 0x5ac40a8cu, 0x546d612du, 0x56acc648u, 0x5ac40a69u, 0x546d6104u, + 0x56acc62au, 0x5ac40a48u, 0x546d60dbu, 0x56acc60cu, 0x5ac40a25u, 0x546d60b2u, 0x56acc5eeu, 0x5ac40a04u, + 0x546d6089u, 0x56acc5cfu, 0x5ac409e1u, 0x546d605fu, 0x56acc5b2u, 0x5ac409c0u, 0x546d6036u, 0x56acc594u, + 0x5ac4099du, 0x546d600du, 0x56acc576u, 0x5ac4097bu, 0x546d5fe4u, 0x56acc558u, 0x5ac4095au, 0x546d5fbbu, + 0x56acc53au, 0x5ac40937u, 0x546d5f92u, 0x56acc51cu, 0x5ac40916u, 0x546d5f69u, 0x56acc4ffu, 0x5ac408f3u, + 0x546d5f40u, 0x5f07d53au, 0x46d3205cu, 0x5b5f8f8fu, 0x5f07d50cu, 0x46d3204au, 0x5b5f8f69u, 0x5f07d4dcu, + 0x46d32037u, 0x5b5f8f43u, 0x5f07d4aeu, 0x46d32025u, 0x5b5f8f1bu, 0x5f07d47eu, 0x46d32013u, 0x5b5f8ef4u, + 0x6b588df7u, 0x433f6554u, 0x54a43c6au, 0x6b588dadu, 0x433f654du, 0x54a43c4eu, 0x6b588d61u, 0x433f6544u, + 0x54a43c32u, 0x6b588d17u, 0x433f653cu, 0x54a43c16u, 0x6b588cccu, 0x433f6533u, 0x54a43bfau, 0x6b588c80u, + 0x433f652cu, 0x54a43bdcu, 0x6b588c36u, 0x433f6523u, 0x54a43bc0u, 0x6b588beau, 0x433f651bu, 0x54a43ba4u, + 0x6b588b9fu, 0x433f6512u, 0x54a43b88u, 0x6b588b54u, 0x433f650au, 0x54a43b6au, 0x60b646bau, 0x5a680425u, + 0x5fc6dcf1u, 0x60b6467bu, 0x5a6803fdu, 0x5fc6dcacu, 0x60b6463bu, 0x5a6803d4u, 0x5fc6dc66u, 0x60b645fcu, + 0x5a6803adu, 0x5fc6dc23u, 0x60b645bcu, 0x5a680384u, 0x5fc6dbddu, 0x60b6457du, 0x5a68035cu, 0x5fc6db98u, + 0x60b6453fu, 0x5a680334u, 0x5fc6db53u, 0x60b644ffu, 0x5a68030cu, 0x5fc6db0eu, 0x60b644c0u, 0x5a6802e4u, + 0x5fc6dac9u, 0x60b64481u, 0x5a6802bbu, 0x5fc6da85u, 0x60b64441u, 0x5a680293u, 0x5fc6da40u, 0x60b64403u, + 0x5a68026au, 0x5fc6d9fbu, 0x60b643c4u, 0x5a680243u, 0x5fc6d9b6u, 0x60b64384u, 0x5a68021bu, 0x5fc6d971u, + 0x60b64345u, 0x5a6801f2u, 0x5fc6d92cu, 0x60b64306u, 0x5a6801cau, 0x5fc6d8e6u, 0x60b642c6u, 0x5a6801a1u, + 0x5fc6d8a3u, 0x4dff4574u, 0x5e34fb44u, 0x50137568u, 0x4dff4549u, 0x5e34fb05u, 0x5013754eu, 0x4dff451cu, + 0x5e34fac6u, 0x50137535u, 0x4dff44f1u, 0x5e34fa88u, 0x5013751bu, 0x4dff44c4u, 0x5e34fa49u, 0x50137501u, + 0x5f07d450u, 0x46d32000u, 0x5b5f8eceu, 0x5f07d420u, 0x46d31feeu, 0x5b5f8ea8u, 0x5f07d3f1u, 0x46d31fdcu, + 0x5b5f8e80u, 0x5f07d3c2u, 0x46d31fcau, 0x5b5f8e5au, 0x5f07d393u, 0x46d31fb7u, 0x5b5f8e33u, 0x5f07d364u, + 0x46d31fa5u, 0x5b5f8e0cu, 0x5f07d335u, 0x46d31f93u, 0x5b5f8de5u, 0x5f07d306u, 0x46d31f80u, 0x5b5f8dbfu, + 0x5f07d2d7u, 0x46d31f6eu, 0x5b5f8d98u, 0x5f07d2a8u, 0x46d31f5cu, 0x5b5f8d71u, 0x5f07d278u, 0x46d31f4au, + 0x5b5f8d4au, 0x5f07d24au, 0x46d31f37u, 0x5b5f8d24u, 0x462573b2u, 0x60b646bau, 0x590432a3u, 0x462573a3u, + 0x60b6467bu, 0x5904328du, 0x46257394u, 0x60b6463bu, 0x59043276u, 0x46257386u, 0x60b645fcu, 0x5904325fu, + 0x46257378u, 0x60b645bcu, 0x59043248u, 0x4625736au, 0x60b6457du, 0x59043231u, 0x4625735bu, 0x60b6453fu, + 0x5904321au, 0x4625734du, 0x60b644ffu, 0x59043203u, 0x4625733fu, 0x60b644c0u, 0x590431ecu, 0x46257331u, + 0x60b64481u, 0x590431d5u, 0x46257323u, 0x60b64441u, 0x590431beu, 0x46257313u, 0x60b64403u, 0x590431a8u, + 0x46257305u, 0x60b643c4u, 0x59043191u, 0x462572f7u, 0x60b64384u, 0x5904317au, 0x462572e9u, 0x60b64345u, + 0x59043163u, 0x462572dbu, 0x60b64306u, 0x5904314cu, 0x462572ccu, 0x60b642c6u, 0x59043135u, 0x5abf34d7u, + 0x515ec597u, 0x528663a8u, 0x5abf34b6u, 0x515ec571u, 0x52866391u, 0x5abf3494u, 0x515ec54au, 0x52866379u, + 0x4dff4497u, 0x5e34fa0au, 0x501374e8u, 0x4dff446cu, 0x5e34f9ccu, 0x501374ceu, 0x4dff443fu, 0x5e34f98du, + 0x501374b5u, 0x4dff4413u, 0x5e34f94eu, 0x5013749bu, 0x4dff43e6u, 0x5e34f90fu, 0x50137482u, 0x4dff43bbu, + 0x5e34f8d1u, 0x50137468u, 0x4dff438eu, 0x5e34f892u, 0x5013744fu, 0x4dff4363u, 0x5e34f853u, 0x50137435u, + 0x4dff4336u, 0x5e34f814u, 0x5013741cu, 0x4dff4309u, 0x5e34f7d6u, 0x50137402u, 0x4dff42ddu, 0x5e34f797u, + 0x501373e8u, 0x4dff42b1u, 0x5e34f758u, 0x501373cfu, 0x57db239du, 0x66c26feeu, 0x559e40f0u, 0x57db2378u, + 0x66c26facu, 0x559e40d4u, 0x57db2352u, 0x66c26f68u, 0x559e40b8u, 0x57db232cu, 0x66c26f25u, 0x559e409du, + 0x57db2306u, 0x66c26ee1u, 0x559e4082u, 0x57db22e0u, 0x66c26e9du, 0x559e4067u, 0x57db22bau, 0x66c26e5bu, + 0x559e404bu, 0x57db2294u, 0x66c26e17u, 0x559e4030u, 0x57db226eu, 0x66c26dd3u, 0x559e4014u, 0x57db2248u, + 0x66c26d91u, 0x559e3ff8u, 0x57db2223u, 0x66c26d4du, 0x559e3fddu, 0x57db21fcu, 0x66c26d09u, 0x559e3fc2u, + 0x57db21d6u, 0x66c26cc6u, 0x559e3fa7u, 0x57db21b1u, 0x66c26c83u, 0x559e3f8bu, 0x57db218au, 0x66c26c3fu, + 0x559e3f70u, 0x57db2164u, 0x66c26bfcu, 0x559e3f54u, 0x57db213fu, 0x66c26bb8u, 0x559e3f38u, 0x4c86f1dcu, + 0x48f887ceu, 0x5ece6265u, 0x4c86f1c5u, 0x48f887b8u, 0x5ece621du, 0x4c86f1adu, 0x48f887a4u, 0x5ece61d6u, + 0x5abf3473u, 0x515ec524u, 0x52866362u, 0x5abf3452u, 0x515ec4fdu, 0x5286634au, 0x5abf3431u, 0x515ec4d6u, + 0x52866333u, 0x5abf340fu, 0x515ec4afu, 0x5286631cu, 0x5abf33eeu, 0x515ec489u, 0x52866305u, 0x5abf33cdu, + 0x515ec463u, 0x528662eeu, 0x5abf33adu, 0x515ec43cu, 0x528662d6u, 0x5abf338cu, 0x515ec416u, 0x528662bfu, + 0x5abf336au, 0x515ec3efu, 0x528662a8u, 0x5abf3349u, 0x515ec3c8u, 0x52866290u, 0x5abf3328u, 0x515ec3a1u, + 0x52866279u, 0x5abf3306u, 0x515ec37bu, 0x52866261u, 0x5abf32e5u, 0x515ec354u, 0x5286624au, 0x5abf32c4u, + 0x515ec32eu, 0x52866233u, 0x6cd203f3u, 0x51d54773u, 0x553b4adau, 0x6cd203aau, 0x51d5474eu, 0x553b4abau, + 0x6cd20361u, 0x51d54729u, 0x553b4a98u, 0x6cd20318u, 0x51d54705u, 0x553b4a78u, 0x6cd202d1u, 0x51d546e0u, + 0x553b4a58u, 0x6cd20288u, 0x51d546bbu, 0x553b4a37u, 0x6cd2023fu, 0x51d54696u, 0x553b4a17u, 0x6cd201f6u, + 0x51d54671u, 0x553b49f7u, 0x6cd201adu, 0x51d5464cu, 0x553b49d6u, 0x6cd20164u, 0x51d54626u, 0x553b49b6u, + 0x6cd2011bu, 0x51d54601u, 0x553b4994u, 0x6cd200d2u, 0x51d545ddu, 0x553b4974u, 0x6cd2008au, 0x51d545b8u, + 0x553b4954u, 0x6cd20041u, 0x51d54593u, 0x553b4933u, 0x6cd1fff8u, 0x51d5456eu, 0x553b4913u, 0x6cd1ffafu, + 0x51d54549u, 0x553b48f3u, 0x6cd1ff66u, 0x51d54524u, 0x553b48d2u, 0x55e63476u, 0x4c1083f0u, 0x49304d88u, + 0x4c86f196u, 0x48f8878eu, 0x5ece618eu, 0x4c86f17eu, 0x48f88778u, 0x5ece6146u, 0x4c86f167u, 0x48f88762u, + 0x5ece60ffu, 0x4c86f150u, 0x48f8874cu, 0x5ece60b7u, 0x4c86f138u, 0x48f88738u, 0x5ece6071u, 0x4c86f121u, + 0x48f88722u, 0x5ece6029u, 0x4c86f10au, 0x48f8870cu, 0x5ece5fe1u, 0x4c86f0f2u, 0x48f886f7u, 0x5ece5f9au, + 0x4c86f0dau, 0x48f886e1u, 0x5ece5f52u, 0x4c86f0c3u, 0x48f886cbu, 0x5ece5f0au, 0x4c86f0acu, 0x48f886b7u, + 0x5ece5ec3u, 0x4c86f095u, 0x48f886a1u, 0x5ece5e7bu, 0x4c86f07du, 0x48f8868bu, 0x5ece5e34u, 0x4c86f066u, + 0x48f88675u, 0x5ece5dedu, 0x5dcbc00fu, 0x4f497148u, 0x60bf9da8u, 0x5dcbbfc9u, 0x4f497125u, 0x60bf9d65u, + 0x5dcbbf83u, 0x4f497101u, 0x60bf9d24u, 0x5dcbbf3cu, 0x4f4970dfu, 0x60bf9ce1u, 0x5dcbbef6u, 0x4f4970bcu, + 0x60bf9c9fu, 0x5dcbbeafu, 0x4f497098u, 0x60bf9c5cu, 0x5dcbbe68u, 0x4f497076u, 0x60bf9c1au, 0x5dcbbe21u, + 0x4f497053u, 0x60bf9bd7u, 0x5dcbbddbu, 0x4f497031u, 0x60bf9b94u, 0x5dcbbd94u, 0x4f49700du, 0x60bf9b52u, + 0x5dcbbd4eu, 0x4f496feau, 0x60bf9b11u, 0x5dcbbd08u, 0x4f496fc8u, 0x60bf9aceu, 0x5dcbbcc0u, 0x4f496fa5u, + 0x60bf9a8cu, 0x5dcbbc7au, 0x4f496f81u, 0x60bf9a49u, 0x5dcbbc33u, 0x4f496f5fu, 0x60bf9a06u, 0x5dcbbbedu, + 0x4f496f3cu, 0x60bf99c4u, 0x5dcbbba6u, 0x4f496f18u, 0x60bf9981u, 0x62d6919bu, 0x4ca2c17cu, 0x66cc661bu, + 0x55e6344eu, 0x4c1083e3u, 0x49304d78u, 0x55e63426u, 0x4c1083d7u, 0x49304d69u, 0x55e633ffu, 0x4c1083cau, + 0x49304d59u, 0x55e633d7u, 0x4c1083beu, 0x49304d4au, 0x55e633afu, 0x4c1083b1u, 0x49304d3bu, 0x55e63386u, + 0x4c1083a5u, 0x49304d2bu, 0x55e6335fu, 0x4c108398u, 0x49304d1du, 0x55e63337u, 0x4c10838cu, 0x49304d0eu, + 0x55e6330fu, 0x4c108380u, 0x49304cffu, 0x55e632e8u, 0x4c108373u, 0x49304cefu, 0x55e632c0u, 0x4c108366u, + 0x49304ce0u, 0x55e63297u, 0x4c10835au, 0x49304cd0u, 0x55e6326fu, 0x4c10834du, 0x49304cc1u, 0x55e63248u, + 0x4c108341u, 0x49304cb2u, 0x55e63220u, 0x4c108334u, 0x49304ca2u, 0x55e631f8u, 0x4c108328u, 0x49304c93u, + 0x5a13fb18u, 0x612c57aau, 0x5c396de4u, 0x5a13fafeu, 0x612c576eu, 0x5c396da4u, 0x5a13fae5u, 0x612c5733u, + 0x5c396d63u, 0x5a13facbu, 0x612c56f7u, 0x5c396d23u, 0x5a13fab1u, 0x612c56bbu, 0x5c396ce3u, 0x5a13fa98u, + 0x612c5680u, 0x5c396ca3u, 0x5a13fa7eu, 0x612c5644u, 0x5c396c63u, 0x5a13fa65u, 0x612c5608u, 0x5c396c21u, + 0x5a13fa4bu, 0x612c55cdu, 0x5c396be1u, 0x5a13fa31u, 0x612c5591u, 0x5c396ba1u, 0x5a13fa18u, 0x612c5554u, + 0x5c396b61u, 0x5a13f9feu, 0x612c551au, 0x5c396b20u, 0x5a13f9e5u, 0x612c54ddu, 0x5c396ae0u, 0x5a13f9cbu, + 0x612c54a1u, 0x5c396aa0u, 0x5a13f9b1u, 0x612c5466u, 0x5c396a60u, 0x5a13f997u, 0x612c542au, 0x5c396a20u, + 0x62d69151u, 0x4ca2c160u, 0x66cc65d4u, 0x62d69106u, 0x4ca2c143u, 0x66cc658du, 0x62d690bcu, 0x4ca2c127u, + 0x66cc6546u, 0x62d69072u, 0x4ca2c10bu, 0x66cc6500u, 0x62d69028u, 0x4ca2c0efu, 0x66cc64b8u, 0x62d68fddu, + 0x4ca2c0d3u, 0x66cc6472u, 0x62d68f93u, 0x4ca2c0b7u, 0x66cc642au, 0x62d68f49u, 0x4ca2c09au, 0x66cc63e4u, + 0x62d68efdu, 0x4ca2c07eu, 0x66cc639du, 0x62d68eb3u, 0x4ca2c062u, 0x66cc6356u, 0x62d68e69u, 0x4ca2c046u, + 0x66cc630fu, 0x62d68e1fu, 0x4ca2c02au, 0x66cc62c9u, 0x62d68dd4u, 0x4ca2c00eu, 0x66cc6281u, 0x62d68d8au, + 0x4ca2bff1u, 0x66cc623bu, 0x62d68d40u, 0x4ca2bfd5u, 0x66cc61f4u, 0x62d68cf6u, 0x4ca2bfb8u, 0x66cc61adu, + 0x6e16e827u, 0x4f24f293u, 0x4a8dc1e1u, 0x6e16e7f2u, 0x4f24f277u, 0x4a8dc1d5u, 0x6e16e7beu, 0x4f24f25bu, + 0x4a8dc1c8u, 0x6e16e78au, 0x4f24f23du, 0x4a8dc1bcu, 0x6e16e755u, 0x4f24f221u, 0x4a8dc1b0u, 0x6e16e721u, + 0x4f24f205u, 0x4a8dc1a3u, 0x6e16e6edu, 0x4f24f1e8u, 0x4a8dc197u, 0x6e16e6b8u, 0x4f24f1ccu, 0x4a8dc18bu, + 0x6e16e684u, 0x4f24f1afu, 0x4a8dc17fu, 0x6e16e650u, 0x4f24f192u, 0x4a8dc173u, 0x6e16e61cu, 0x4f24f176u, + 0x4a8dc166u, 0x6e16e5e7u, 0x4f24f15au, 0x4a8dc15au, 0x6e16e5b3u, 0x4f24f13cu, 0x4a8dc14eu, 0x6e16e57fu, + 0x4f24f120u, 0x4a8dc141u, 0x6e16e54au, 0x4f24f104u, 0x4a8dc135u, 0x6e16e516u, 0x4f24f0e6u, 0x4a8dc128u, + 0x5a13f97eu, 0x612c53eeu, 0x5c3969dfu, 0x4f53c39du, 0x5a42c9c6u, 0x55dafc06u, 0x4f53c37au, 0x5a42c9a5u, + 0x55dafbe0u, 0x4f53c354u, 0x5a42c983u, 0x55dafbbbu, 0x4f53c32fu, 0x5a42c961u, 0x55dafb94u, 0x4f53c30cu, + 0x5a42c93fu, 0x55dafb6eu, 0x4f53c2e6u, 0x5a42c91du, 0x55dafb49u, 0x4f53c2c1u, 0x5a42c8fcu, 0x55dafb23u, + 0x4f53c29du, 0x5a42c8dau, 0x55dafafcu, 0x4f53c278u, 0x5a42c8b8u, 0x55dafad7u, 0x4f53c253u, 0x5a42c897u, + 0x55dafab1u, 0x4f53c22fu, 0x5a42c874u, 0x55dafa8cu, 0x4f53c20au, 0x5a42c853u, 0x55dafa65u, 0x4f53c1e5u, + 0x5a42c831u, 0x55dafa3fu, 0x4f53c1c1u, 0x5a42c80fu, 0x55dafa1au, 0x4f53c19cu, 0x5a42c7eeu, 0x55daf9f3u, + 0x4f53c177u, 0x5a42c7ccu, 0x55daf9cdu, 0x4f53c153u, 0x5a42c7aau, 0x55daf9a8u, 0x5ae982dau, 0x52b9f625u, + 0x5f161bfcu, 0x5ae982b2u, 0x52b9f605u, 0x5f161bc8u, 0x5ae98289u, 0x52b9f5e5u, 0x5f161b94u, 0x5ae98261u, + 0x52b9f5c4u, 0x5f161b60u, 0x5ae98238u, 0x52b9f5a4u, 0x5f161b2cu, 0x5ae9820fu, 0x52b9f584u, 0x5f161af8u, + 0x5ae981e8u, 0x52b9f564u, 0x5f161ac4u, 0x5ae981bfu, 0x52b9f544u, 0x5f161a90u, 0x5ae98197u, 0x52b9f524u, + 0x5f161a5cu, 0x5ae9816eu, 0x52b9f503u, 0x5f161a28u, 0x5ae98145u, 0x52b9f4e3u, 0x5f1619f4u, 0x5ae9811du, + 0x52b9f4c3u, 0x5f1619c0u, 0x5ae980f4u, 0x52b9f4a3u, 0x5f16198cu, 0x5ae980ccu, 0x52b9f483u, 0x5f161958u, + 0x6e16e4e2u, 0x4f24f0cau, 0x4a8dc11cu, 0x4e02dd60u, 0x4cb3dca3u, 0x6b315411u, 0x4e02dd4au, 0x4cb3dc83u, + 0x6b3153d3u, 0x4e02dd33u, 0x4cb3dc64u, 0x6b315396u, 0x4e02dd1cu, 0x4cb3dc45u, 0x6b315357u, 0x4e02dd05u, + 0x4cb3dc25u, 0x6b31531au, 0x4e02dcefu, 0x4cb3dc06u, 0x6b3152dcu, 0x4e02dcd8u, 0x4cb3dbe8u, 0x6b31529fu, + 0x4e02dcc1u, 0x4cb3dbc8u, 0x6b315261u, 0x4e02dcabu, 0x4cb3dba9u, 0x6b315224u, 0x4e02dc94u, 0x4cb3db8au, + 0x6b3151e6u, 0x4e02dc7eu, 0x4cb3db6au, 0x6b3151a9u, 0x4e02dc67u, 0x4cb3db4cu, 0x6b31516cu, 0x4e02dc50u, + 0x4cb3db2du, 0x6b31512eu, 0x4e02dc39u, 0x4cb3db0du, 0x6b3150f1u, 0x4e02dc23u, 0x4cb3daeeu, 0x6b3150b3u, + 0x4e02dc0cu, 0x4cb3daceu, 0x6b315076u, 0x4e02dbf5u, 0x4cb3daafu, 0x6b315038u, 0x6b6f52e9u, 0x431eb0a6u, + 0x434f965bu, 0x6b6f5296u, 0x431eb09fu, 0x434f9652u, 0x6b6f5243u, 0x431eb098u, 0x434f9649u, 0x6b6f51efu, + 0x431eb091u, 0x434f9640u, 0x6b6f519du, 0x431eb08au, 0x434f9637u, 0x6b6f514au, 0x431eb083u, 0x434f962eu, + 0x6b6f50f7u, 0x431eb07du, 0x434f9625u, 0x6b6f50a4u, 0x431eb076u, 0x434f961cu, 0x6b6f5051u, 0x431eb06fu, + 0x434f9613u, 0x6b6f4fffu, 0x431eb068u, 0x434f960au, 0x6b6f4facu, 0x431eb061u, 0x434f9601u, 0x6b6f4f58u, + 0x431eb05au, 0x434f95f8u, 0x6b6f4f05u, 0x431eb053u, 0x434f95efu, 0x6b6f4eb2u, 0x431eb04cu, 0x434f95e6u, + 0x5ae980a4u, 0x52b9f461u, 0x5f161924u, 0x5ae9807bu, 0x52b9f441u, 0x5f1618f0u, 0x5ae98053u, 0x52b9f421u, + 0x5f1618bcu, 0x41bbac84u, 0x541b3bcfu, 0x4e8212b1u, 0x41bbac7cu, 0x541b3bb4u, 0x4e82129bu, 0x41bbac73u, + 0x541b3b99u, 0x4e821285u, 0x41bbac6cu, 0x541b3b7eu, 0x4e82126eu, 0x41bbac63u, 0x541b3b63u, 0x4e821258u, + 0x41bbac5bu, 0x541b3b48u, 0x4e821241u, 0x41bbac53u, 0x541b3b2eu, 0x4e82122au, 0x41bbac4au, 0x541b3b13u, + 0x4e821214u, 0x41bbac43u, 0x541b3af8u, 0x4e8211feu, 0x41bbac3bu, 0x541b3addu, 0x4e8211e7u, 0x41bbac32u, + 0x541b3ac2u, 0x4e8211d0u, 0x41bbac2au, 0x541b3aa7u, 0x4e8211bau, 0x41bbac23u, 0x541b3a8cu, 0x4e8211a3u, + 0x41bbac1au, 0x541b3a71u, 0x4e82118du, 0x41bbac12u, 0x541b3a57u, 0x4e821176u, 0x41bbac0au, 0x541b3a3cu, + 0x4e821160u, 0x41bbac01u, 0x541b3a21u, 0x4e821149u, 0x458a03e2u, 0x6c339ef3u, 0x66d6df83u, 0x458a03d7u, + 0x6c339eb4u, 0x66d6df38u, 0x458a03cau, 0x6c339e76u, 0x66d6deeeu, 0x458a03beu, 0x6c339e38u, 0x66d6dea4u, + 0x458a03b3u, 0x6c339dfau, 0x66d6de5au, 0x458a03a7u, 0x6c339dbcu, 0x66d6de0eu, 0x458a039au, 0x6c339d7du, + 0x66d6ddc4u, 0x458a038fu, 0x6c339d3fu, 0x66d6dd7au, 0x458a0383u, 0x6c339d01u, 0x66d6dd2fu, 0x458a0377u, + 0x6c339cc3u, 0x66d6dce5u, 0x458a036bu, 0x6c339c84u, 0x66d6dc9bu, 0x458a035fu, 0x6c339c46u, 0x66d6dc4fu, + 0x6b6f4e60u, 0x431eb046u, 0x434f95ddu, 0x6b6f4e0du, 0x431eb03fu, 0x434f95d4u, 0x6b6f4dbau, 0x431eb038u, + 0x434f95ccu, 0x53bc5d9bu, 0x640b8417u, 0x6c5b5cc1u, 0x53bc5d7au, 0x640b83e7u, 0x6c5b5c76u, 0x53bc5d5au, + 0x640b83b7u, 0x6c5b5c29u, 0x53bc5d38u, 0x640b8386u, 0x6c5b5bddu, 0x53bc5d18u, 0x640b8356u, 0x6c5b5b91u, + 0x53bc5cf7u, 0x640b8325u, 0x6c5b5b45u, 0x53bc5cd7u, 0x640b82f5u, 0x6c5b5afau, 0x53bc5cb6u, 0x640b82c5u, + 0x6c5b5aadu, 0x53bc5c96u, 0x640b8294u, 0x6c5b5a61u, 0x53bc5c74u, 0x640b8264u, 0x6c5b5a14u, 0x53bc5c54u, + 0x640b8233u, 0x6c5b59c9u, 0x53bc5c33u, 0x640b8203u, 0x6c5b597du, 0x53bc5c13u, 0x640b81d3u, 0x6c5b5931u, + 0x53bc5bf2u, 0x640b81a3u, 0x6c5b58e5u, 0x53bc5bd2u, 0x640b8172u, 0x6c5b5898u, 0x53bc5bb1u, 0x640b8142u, + 0x6c5b584du, 0x53bc5b91u, 0x640b8111u, 0x6c5b5801u, 0x4ab5fdfcu, 0x6568ecb8u, 0x450d0a55u, 0x4ab5fdedu, + 0x6568ec68u, 0x450d0a49u, 0x4ab5fddcu, 0x6568ec17u, 0x450d0a3du, 0x4ab5fdcdu, 0x6568ebc6u, 0x450d0a31u, + 0x4ab5fdbdu, 0x6568eb76u, 0x450d0a24u, 0x4ab5fdadu, 0x6568eb25u, 0x450d0a18u, 0x4ab5fd9du, 0x6568ead4u, + 0x450d0a0cu, 0x4ab5fd8eu, 0x6568ea84u, 0x450d0a00u, 0x4ab5fd7du, 0x6568ea33u, 0x450d09f3u, 0x4ab5fd6eu, + 0x6568e9e3u, 0x450d09e7u, 0x4ab5fd5fu, 0x6568e992u, 0x450d09dbu, 0x4ab5fd4fu, 0x6568e941u, 0x450d09cfu, + 0x458a0353u, 0x6c339c08u, 0x66d6dc05u, 0x458a0347u, 0x6c339bc9u, 0x66d6dbbbu, 0x458a033bu, 0x6c339b8cu, + 0x66d6db71u, 0x458a032fu, 0x6c339b4du, 0x66d6db26u, 0x458a0323u, 0x6c339b0fu, 0x66d6dadbu, 0x634d6d17u, + 0x64c736f1u, 0x42c2f908u, 0x634d6ccfu, 0x64c736acu, 0x42c2f8ffu, 0x634d6c89u, 0x64c73666u, 0x42c2f8f6u, + 0x634d6c41u, 0x64c73621u, 0x42c2f8eeu, 0x634d6bfau, 0x64c735dcu, 0x42c2f8e5u, 0x634d6bb3u, 0x64c73597u, + 0x42c2f8ddu, 0x634d6b6cu, 0x64c73552u, 0x42c2f8d4u, 0x634d6b25u, 0x64c7350du, 0x42c2f8ccu, 0x634d6addu, + 0x64c734c8u, 0x42c2f8c4u, 0x634d6a96u, 0x64c73484u, 0x42c2f8bbu, 0x634d6a4fu, 0x64c7343fu, 0x42c2f8b3u, + 0x634d6a08u, 0x64c733fau, 0x42c2f8aau, 0x634d69c0u, 0x64c733b4u, 0x42c2f8a1u, 0x634d697au, 0x64c7336fu, + 0x42c2f89au, 0x634d6932u, 0x64c7332au, 0x42c2f891u, 0x634d68ecu, 0x64c732e5u, 0x42c2f889u, 0x634d68a4u, + 0x64c732a0u, 0x42c2f880u, 0xbc2321f4u, 0xbc1102ecu, 0xbc112fdfu, 0xbc201d3du, 0xbba6e0e9u, 0xbc1f6e06u, + 0xbc239370u, 0xbbc73c10u, 0xbbe4b80cu, 0xbc1ffefbu, 0xbc20460bu, 0xbc0bf7a9u, 0xbc08e010u, 0xbc1b2263u, + 0xbc0d9f40u, 0xbc20dd4fu, 0xbc1f58abu, 0xbc03d26bu, 0xbc235148u, 0xbba7714au, 0xbc05e6b8u, 0xbc143451u, + 0xbba23ae8u, 0xbbe47ca3u, 0xbc21c326u, 0xbc153269u, 0xbc2230e7u, 0xbb628ae3u, 0xbc013558u, 0xbba0ac35u, + 0x4ab5fd3fu, 0x6568e8f1u, 0x450d09c3u, 0x4ab5fd2fu, 0x6568e8a0u, 0x450d09b7u, 0x4ab5fd20u, 0x6568e84fu, + 0x450d09aau, 0x4ab5fd0fu, 0x6568e7fdu, 0x450d099eu, 0x4ab5fd00u, 0x6568e7adu, 0x450d0991u, 0x6c2289aau, + 0x413cbca9u, 0x55254c1bu, 0x6c228972u, 0x413cbca0u, 0x55254bffu, 0x6c22893au, 0x413cbc98u, 0x55254be1u, + 0x6c228901u, 0x413cbc91u, 0x55254bc5u, 0x6c2288c9u, 0x413cbc88u, 0x55254ba9u, 0x6c228891u, 0x413cbc80u, + 0x55254b8cu, 0x6c228858u, 0x413cbc77u, 0x55254b6fu, 0x6c228820u, 0x413cbc6fu, 0x55254b52u, 0x6c2287e8u, + 0x413cbc68u, 0x55254b36u, 0x6c2287b0u, 0x413cbc5fu, 0x55254b1au, 0x6c228777u, 0x413cbc57u, 0x55254afcu, + 0x6c22873fu, 0x413cbc4eu, 0x55254ae0u, 0x6c228707u, 0x413cbc46u, 0x55254ac4u, 0x6c2286ceu, 0x413cbc3fu, + 0x55254aa6u, 0x6c228696u, 0x413cbc36u, 0x55254a8au, 0x6c22865eu, 0x413cbc2eu, 0x55254a6du, 0x6c228625u, + 0x413cbc25u, 0x55254a51u, 0xbc1a3826u, 0xbc1e1229u, 0xbc20a205u, 0xbc09f2b6u, 0xbc235cddu, 0xbc1e08e8u, + 0xbc218968u, 0xbbf846a5u, 0xbc211d17u, 0xbc17c17cu, 0xbc22a12cu, 0xbc06af80u, 0xbbd16fddu, 0xbc1bc9c6u, + 0xbc0cd875u, 0xbc0b0e47u, 0xbc16468au, 0xbbaa1c24u, 0xbc0260beu, 0xbba4aa97u, 0xbc22e1e1u, 0xbc22d6ecu, + 0xbbf4cab9u, 0xbbfc3486u, 0xbc1d1c99u, 0xbb855c1fu, 0xbba2dfeeu, 0xbc1fe93bu, 0xbc0150fdu, 0xbbb3d2feu, +}; + +} // namespace fold_selftest diff --git a/clean/runtime/gen_fold_probe.py b/clean/runtime/gen_fold_probe.py new file mode 100644 index 00000000..acf7d6ff --- /dev/null +++ b/clean/runtime/gen_fold_probe.py @@ -0,0 +1,186 @@ +# SPDX-License-Identifier: MIT +"""Writes clean/runtime/fold_probe.h: inputs and expected bytes of the start-up probe of the driver's conversion fold +(clean/dx12/kernels/l_fold_probe.hlsl, hlsl/l_post_encode.hlsli). WrongGPU's own code. + + python clean/runtime/gen_fold_probe.py [out.h] + +Lane l reads x0 x1 x2 (the FP32 log2 arguments of the three history codes) and d0 d1 d2 (display values) and writes +q_i = (_Float16)(kEncK x_i) with ONE rounding of the exact product (+0 for a zero product: the fold's +0 addend) and +h(d_i) (round to nearest even). Expected values are computed from exact rationals (fractions.Fraction), never from +FP arithmetic. The x inputs are adversarial: values whose FP32 product kEncK x rounds exactly onto an FP16 midpoint +while the exact product is not that midpoint (residual of both signs; a second rounding moves them by one FP16 ulp), +FP16-subnormal products, +-0, NaN, the encode's floor log2(1e-4) and ordinary values; the d inputs: FP16 midpoints and +their FP32 neighbours, +-0, NaN, values beyond the FP16 range. +""" +import os +import struct +import sys +from fractions import Fraction + +import numpy as np + +K_BITS = 0x3dddf184 # kEncK + + +def f32(b): + return struct.unpack("= Fraction(65520): + return s | 0x7c00 + # exponent: 2^e <= a < 2^(e+1) + e = a.numerator.bit_length() - a.denominator.bit_length() + if Fraction(2) ** e > a: + e -= 1 + e = max(e, -14) # subnormal range uses the exponent -14 grid + q = a / (Fraction(2) ** (e - 10)) # in units of the ulp + n = q.numerator // q.denominator + r = q - n + if r > Fraction(1, 2) or (r == Fraction(1, 2) and n % 2 == 1): + n += 1 + if n >= 2048: # carried into the next binade + n //= 2 + e += 1 + if e > 15: + return s | 0x7c00 + if n < 1024: # subnormal + return s | n + return s | ((e + 15) << 10) | (n - 1024) + + +def h16_value(b): + """The exact value of FP16 bits b (None for infinity / NaN).""" + if (b & 0x7c00) == 0x7c00: + return None + return Fraction(float(np.frombuffer(struct.pack("> 10) & 31) - 25) + m = v + (ulp / 2 if v >= 0 else -ulp / 2) + x0 = rn32(m / K) + for dx in (0, 1, -1, 2, -2): + x = f32((bits32(x0) + dx) & 0xffffffff) + p = Fraction(x) * K + if rn32(p) == np.float32(float(m)) and p != m and ((p > m) == want_up): + out.append(x) + want_up = not want_up + break + return out + + +def main(): + out = sys.argv[1] if len(sys.argv) > 1 else os.path.join(os.path.dirname(os.path.abspath(__file__)), "fold_probe.h") + K = Fraction(f32(K_BITS)) + xs = midpoint_xs(K, 60, 20261115) + nan = f32(0x7fc00000) + floor = float(np.log2(np.float32(1e-4)).astype(np.float32)) + xs += [0.0, -0.0, nan, floor, 0.0, 1.0, -1.0, 20.0, float(np.float32(3.0e-4)), float(np.float32(-2.0e-4)), + float(np.float32(1e-6)), -float(np.float32(1e-6))] + rng = np.random.default_rng(7) + while len(xs) < 96: + xs.append(float(np.float32(rng.uniform(-13.3, 21.0)))) + ds = [] + for h in rng.integers(0x0400, 0x7bff, 40): + v = h16_value(int(h)) + ulp = Fraction(2) ** ((((int(h) >> 10) & 31)) - 25) + m = v + ulp / 2 + ds.append(float(np.float32(float(m)))) # an exact FP16 midpoint (ties to even) + ds.append(f32(bits32(float(m)) + 1)) # just above + ds += [0.0, -0.0, nan, 65519.0, 65520.0, 1e6, f32(0x33800000), f32(0x33000001)] + while len(ds) < 96: + ds.append(float(np.float32(rng.uniform(0.0, 8.0)))) + xs, ds = xs[:96], ds[:96] + inputs, expect, nanmask = [], [], [] + for lane in range(32): + x = xs[3 * lane:3 * lane + 3] + d = ds[3 * lane:3 * lane + 3] + inputs += [bits32(v) for v in x + d] + q, nm = [], [] + for v in x: + if v != v: + q.append(0x7e00); nm.append(True) + else: + p = Fraction(v) * K + q.append(0 if p == 0 else rn16_bits(p)); nm.append(False) + for v in d: + if v != v: + q.append(0x7e00); nm.append(True) + elif v == 0: # a plain conversion keeps the sign of a zero + q.append(0x8000 if struct.pack("\n#include \n\nnamespace fold_probe {\n") + f.write("constexpr unsigned kInBytes = 768, kOutOffset = 768, kOutBytes = 384, kBytes = 1152;\n") + f.write("constexpr uint32_t kInput[192] = {\n") + for i in range(0, 192, 6): + f.write(" " + ", ".join(f"0x{v:08x}u" for v in inputs[i:i + 6]) + ",\n") + f.write("};\nconstexpr uint32_t kExpect[96] = {\n") + for i in range(0, 96, 6): + f.write(" " + ", ".join(f"0x{v:08x}u" for v in expect[i:i + 6]) + ",\n") + f.write("};\nconstexpr uint8_t kNanMask[96] = {\n") + for i in range(0, 96, 24): + f.write(" " + ", ".join(str(v) for v in nanmask[i:i + 24]) + ",\n") + f.write("};\n") + f.write("""inline void fill(uint8_t* buffer) { + std::memset(buffer, 0, kBytes); + std::memcpy(buffer, kInput, sizeof kInput); +} +inline bool half_ok(uint32_t got, uint32_t want, bool nan) { + if (nan) return (got & 0x7c00u) == 0x7c00u && (got & 0x03ffu) != 0u; + return got == want; +} +// index of the first differing output half (0 .. 191), or -1 when every half matches +inline int first_mismatch(const uint32_t* out) { + for (unsigned i = 0; i < 96; ++i) { + if (!half_ok(out[i] & 0xffffu, kExpect[i] & 0xffffu, (kNanMask[i] & 1u) != 0)) return (int)(2 * i); + if (!half_ok(out[i] >> 16, kExpect[i] >> 16, (kNanMask[i] & 2u) != 0)) return (int)(2 * i + 1); + } + return -1; +} +} // namespace fold_probe +""") + # self-check of rn16_bits against numpy's double -> half conversion (correctly rounded from a double; every value + # here is an exact double) + chk = np.random.default_rng(3) + for v in list(chk.uniform(-70000, 70000, 2000)) + list(chk.uniform(-1e-4, 1e-4, 2000)): + nb = int(np.array([v]).astype(np.float16).view(np.uint16)[0]) + assert rn16_bits(Fraction(float(v))) == nb or (v == 0), (v, nb) + print("wrote", out, "midpoint x:", sum(1 for v in xs[:60]), "lanes 32") + + +if __name__ == "__main__": + main() diff --git a/clean/runtime/gen_fold_selftest.py b/clean/runtime/gen_fold_selftest.py new file mode 100644 index 00000000..d668316d --- /dev/null +++ b/clean/runtime/gen_fold_selftest.py @@ -0,0 +1,315 @@ +# SPDX-License-Identifier: MIT +"""Writes clean/runtime/fold_selftest.h: the colour image of the native D3D12 device's differential self-test of Model +L's post (device_d3d12.cpp fold_selftest). WrongGPU's own code. + + python clean/runtime/gen_fold_selftest.py [out.h] + +The self-test runs the REAL post shader twice on one tiny frame: the fold port of l_post_recon_p12 (its history encode +through the driver's conversion fold, hlsl/l_post_encode.hlsli) and the no-fold port (round to odd, exact without the +fold), and compares the raw output bytes. Equal: the device runs the fold port; any difference or error: the no-fold +port. The frame (kW x kH, render size = output size, jitter 0, reset, exposure 1, post scale s = 0, every code and +kernel-shape value 0) makes the full history H^ of pixel (x, y) channel i exactly the render colour c_i, so the history +code is q = (_Float16)(kEncK x) with x = log2(max(fma(100, c, 1), 1e-4)) evaluated by the post shader itself +(enc_log): one wave of the shader's six encode sites (three channels, two rows of each row pair) sees each class: +- FP32-product ties: c whose x (predicted with the correctly rounded log2) makes the FP32 product RN32(kEncK x) land + exactly on an FP16 midpoint while the exact product does not, the exact product above and below the midpoint, most + of them on the side where the second rounding (to even) goes the wrong way, in four ranges of x and among the + FP16-subnormal products (the ones there are: x near 0 is sparse); +- exact-product ties do not exist: kEncK's significand has the odd factor 0x377c61 (22 bits) and an FP16 midpoint's + has at most 12 bits, so kEncK x is never exactly a midpoint unless it is 0; +- zero products: c = +-0, +-2^-149, +-1e-30 (fma(100, c, 1) = 1, x = +0, the code +0); +- FP16-subnormal products of both signs (fma(100, c, 1) = 1 +- a few ulps); the exhaustive search over every FP32 + value of fma(100, c, 1) in that range finds no FP32-product tie there; +- the floor: c whose fma(100, c, 1) is just below, at or near 1e-4 (the max picks 1e-4), and far below it (0, -1, -1e30). +Which x the GPU's log2 gives is not assumed: an x that differs from the prediction just tests another value. The +fold-removed mutant (LP_MUT 3, two roundings) must make the self-test fail (GPU record of the PR). +Every predicted value is checked with exact rationals (fractions.Fraction, decimal logarithms to 60 digits). +""" +import decimal +import os +import struct +import sys +from fractions import Fraction + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +from gen_fold_probe import K_BITS, bits32, f32, rn16_bits # noqa: E402 + +W, H = 32, 16 +SITES = 6 # (row of the pair, channel) +PER_SITE = W * H * 3 // SITES +SEED = 20261115 +OFFSETS = range(-8, 9) # the device's log2 may differ from the correctly rounded one by up to 8 ulp +TIES_PER_SITE = 13 # each with all 17 offsets: N = 13 detecting values per site for every uniform offset +K = Fraction(f32(K_BITS)) +DEC = decimal.Context(prec=60) + + +def rn(fr, p, emin): + """Round a Fraction to p significant bits, nearest even, exponents below emin on the emin grid (no overflow).""" + if fr == 0: + return Fraction(0) + s = -1 if fr < 0 else 1 + a = abs(fr) + e = a.numerator.bit_length() - a.denominator.bit_length() + if Fraction(2) ** e > a: + e -= 1 + e = max(e, emin) + q = Fraction(2) ** (e - p + 1) + n = a / q + f = n.numerator // n.denominator + r = n - f + if r > Fraction(1, 2) or (r == Fraction(1, 2) and f % 2 == 1): + f += 1 + return s * f * q + + +def rn32(fr): + return rn(fr, 24, -126) + + +def m_of(c): + """RN32(fma(100, c, 1)) for the FP32 value c (exact).""" + return rn32(100 * Fraction(c) + 1) + + +def log2_exact(m): + """(the correctly rounded FP32 log2 of the FP32 value m > 0, the distance of the exact log2 from an FP32 midpoint in + units of that FP32 ulp) from 60-digit decimal logarithms.""" + lm = DEC.divide(DEC.ln(decimal.Decimal(m.numerator) / decimal.Decimal(m.denominator)), DEC.ln(decimal.Decimal(2))) + v = Fraction(lm) + x = rn32(v) + if x == 0: + return x, None + a = abs(v) + e = a.numerator.bit_length() - a.denominator.bit_length() + if Fraction(2) ** e > a: + e -= 1 + ulp = Fraction(2) ** (e - 23) + frac = (a / ulp) - (a / ulp).numerator // (a / ulp).denominator + return x, abs(frac - Fraction(1, 2)) + + +def fp16_midpoint(p): + """Whether the FP32 value p lies exactly halfway between two adjacent FP16 values (finite range).""" + a = abs(p) + if a == 0 or a >= 65504: + return False + e = a.numerator.bit_length() - a.denominator.bit_length() + if Fraction(2) ** e > a: + e -= 1 + ulp = Fraction(2) ** (max(e, -14) - 10) + n = a / ulp + return n.denominator == 2 + + +def classify(c): + """(x, product class) of the colour c, checked exactly: None unless kEncK x is an FP32-product tie; else + ('up' / 'down' = the exact product's side, detecting = the second rounding goes to the other FP16 value).""" + m = m_of(c) + if m < Fraction(f32(bits32(1e-4))): + return None + x, margin = log2_exact(m) + if x == 0 or margin is None or margin < Fraction(1, 10**12): + return None + pe = x * K + p32 = rn32(pe) + if not fp16_midpoint(p32) or pe == p32: + return None + detecting = rn16_bits(pe) != rn16_bits(p32) + return x, ("up" if pe > p32 else "down"), detecting + + +def search(rng, lo, hi, want, sub=False): + """want ties (half the exact products above the midpoint, half below; detecting, plus a sixth non-detecting) from + c whose fma(100, c, 1) lies in [lo, hi], log-uniform; sub: every m = 1 +- k ulp in the range (exhaustive).""" + k32 = np.float64(f32(K_BITS)) + if sub: + ks = np.arange(1, int((hi - 1) * 2**23) + 1) + mt = np.concatenate([1.0 + ks * 2.0**-23, 1.0 - np.arange(1, int((1 - lo) * 2**24) + 1) * 2.0**-24]) + rounds = [mt] + else: + rounds = (np.exp2(rng.uniform(np.log2(lo), np.log2(hi), 1 << 20)) for _ in range(400)) + picks = {("up", True): [], ("down", True): [], ("up", False): [], ("down", False): []} + quota = {("up", True): want // 2, ("down", True): want // 2, ("up", False): want // 12, ("down", False): want // 12} + seen = set() + for mt in rounds: + c = ((mt - 1.0) / 100.0).astype(np.float32) + m = (100.0 * c.astype(np.float64) + 1.0).astype(np.float32) + ok = (m >= lo) & (m <= hi) + x = np.log2(m.astype(np.float64)).astype(np.float32) + pe = x.astype(np.float64) * k32 + p32 = pe.astype(np.float32).astype(np.float64) + a = np.abs(p32) + normal = a >= 2.0**-14 + pb = np.abs(p32.astype(np.float32)).view(np.uint32) + tie_n = normal & ((pb & 0x1fff) == 0x1000) + t = a * 2.0**25 + tie_s = ~normal & (t == np.round(t)) & (np.fmod(t, 2.0) == 1.0) & (a > 0) + for i in np.nonzero(ok & (tie_n | tie_s) & (pe != p32))[0]: + cb = int(c[i].view(np.uint32)) + if cb in seen: + continue + seen.add(cb) + r = classify(f32(cb)) + if r is None: + continue + key = (r[1], r[2]) + if len(picks[key]) < quota[key]: + picks[key].append(cb) + if all(len(picks[k]) >= quota[k] for k in picks): + break + return picks + + +def floor_values(): + """c around the floor: fma(100, c, 1) the nearest reachable values below / at / above 1e-4, and far below.""" + one_em4 = Fraction(f32(bits32(1e-4))) + c0 = bits32(np.float32((1e-4 - 1.0) / 100.0)) + below, above = [], [] + for d in range(-40, 41): + cb = c0 + d # negative c: a larger bit pattern is a more negative value + m = m_of(f32(cb)) + (below if m < one_em4 else above).append((m, cb)) + below = [cb for _, cb in sorted(below, reverse=True)[:3]] + above = [cb for _, cb in sorted(above)[:3]] + far = [bits32(0.0) | 0, bits32(-0.01), bits32(-1.0), bits32(-1e30)] + return below + above + far + + +def zero_values(): + return [0x00000000, 0x80000000, 0x00000001, 0x80000001, bits32(1e-30), bits32(-1e-30)] + + +def subnormal_values(): + """c with fma(100, c, 1) = 1 + k 2^-23 or 1 - k 2^-24: FP16-subnormal products of both signs (no tie among them).""" + out = [] + for m in [1.0 + k * 2.0**-23 for k in (1, 2, 7, 100, 3000)] + [1.0 - k * 2.0**-24 for k in (1, 3, 50, 6000)]: + cb = bits32(np.float32((m - 1.0) / 100.0)) + assert m_of(f32(cb)) == Fraction(m) + out.append(cb) + return out + + +def x_step(x, d): + """The FP32 value d ulps from the positive FP32 value x (same binade).""" + return Fraction(f32(bits32(float(x)) + d)) + + +def detecting_tie(x): + """(side) when kEncK x is an FP32-product tie that a second rounding gets wrong, else None (exact).""" + pe = x * K + p32 = rn32(pe) + if not fp16_midpoint(p32) or pe == p32 or rn16_bits(pe) == rn16_bits(p32): + return None + return "up" if pe > p32 else "down" + + +def c_for_log(xd): + """A colour c (FP32 bits) whose fma(100, c, 1) = m has the correctly rounded log2 xd (exact checks), or None.""" + mt = 2.0 ** float(xd) + m0 = bits32(np.float32(mt)) + for dm in (0, 1, -1, 2, -2, 3, -3, 4, -4): + m = Fraction(f32(m0 + dm)) + x, margin = log2_exact(m) + if x != xd or margin is None or margin < Fraction(1, 10**12): + continue + c0 = bits32(np.float32(float((m - 1) / 100))) + for dc in (0, 1, -1, 2, -2, 3, -3): + if m_of(f32(c0 + dc)) == m: + return c0 + dc + return None + + +def offset_ties(rng, count): + """count ties x in [8, 100) (exact products alternately above and below the midpoint), each with the colours whose + predicted log2 is x - d ulp for every d in OFFSETS: a device log2 that is uniformly d ulp off still lands one of + them on x. [(side, x, [c bits for d in OFFSETS])].""" + out, want_up = [], True + while len(out) < count: + x = Fraction(f32(bits32(np.float32(rng.uniform(8.0, 100.0))))) + side = detecting_tie(x) + if side is None or (side == "up") != want_up: + continue + if bits32(float(x)) >> 23 != (bits32(float(x)) + 8) >> 23 or bits32(float(x)) >> 23 != (bits32(float(x)) - 8) >> 23: + continue # all offsets in the same binade + cs = [c_for_log(x_step(x, -d)) for d in OFFSETS] + if any(c is None for c in cs): + continue + out.append((side, x, cs)) + want_up = not want_up + return out + + +def shifted_detecting(colour, e): + """Per site, the number of colours whose correctly rounded log2 moved by e ulp is a detecting tie (the CPU model + of a device log2 uniformly e ulp off).""" + per = [0] * SITES + for y in range(H): + for xx in range(W): + for ch in range(3): + m = m_of(f32(colour[3 * (y * W + xx) + ch])) + if m < Fraction(f32(bits32(1e-4))): + continue + x, _ = log2_exact(m) + if x == 0: + continue + xs = Fraction(f32(bits32(float(x)) + (e if x > 0 else -e))) + if detecting_tie(xs): + per[3 * (y % 2) + ch] += 1 + return per + + +def build(): + rng = np.random.default_rng(SEED) + specials = floor_values() + zero_values() + subnormal_values() + groups = offset_ties(rng, TIES_PER_SITE * SITES) + extra = search(rng, 1.0e-4, 1.0 - 3.9e-4, 72) # negative products (sparse x: no offsets), predicted ties + extra_cbs = extra[("up", True)] + extra[("down", True)] + per_site = [list(specials) for _ in range(SITES)] + for i, (side, x, cs) in enumerate(groups): + per_site[i % SITES] += cs + for i, cb in enumerate(extra_cbs): + if len(per_site[i % SITES]) < PER_SITE: + per_site[i % SITES].append(cb) + fill = [bits32(1.0 + 0.25 * k) for k in range(PER_SITE)] # ordinary values for any slot left + for s_ in per_site: + assert len(s_) <= PER_SITE + s_ += fill[:PER_SITE - len(s_)] + colour = [0] * (W * H * 3) + for site in range(SITES): + row, ch = divmod(site, 3) + pixels = [(x, 2 * j + row) for j in range(H // 2) for x in range(W)] + for (x, y), cb in zip(pixels, per_site[site]): + colour[3 * (y * W + x) + ch] = cb + counts = {"offset_ties_per_site": TIES_PER_SITE, "offsets": f"{OFFSETS.start}..{OFFSETS.stop - 1}", + "up": sum(g[0] == "up" for g in groups), "down": sum(g[0] == "down" for g in groups), + "negative_extra": sum(cb in set(extra_cbs) for cb in colour)} + return colour, counts, len(specials) + + +def write(path): + colour, counts, nspec = build() + lines = ["// SPDX-License-Identifier: MIT", + "// Generated by clean/runtime/gen_fold_selftest.py (do not edit): the render colour image of the native D3D12", + "// device's differential self-test of Model L's post (fold port against no-fold port, device_d3d12.cpp).", + "// FP32 bits, slot 3 (y kW + x) + channel. Synthetic values from exact rational arithmetic; no model data.", + "// Per site (row of the pair, channel): %d special values (floor, zero and FP16-subnormal products), then" % nspec, + ("// per site %d FP32-product ties a second rounding gets wrong, each at predicted-log2 offsets %s ulp (%d exact\n" + "// products above the midpoint, %d below), then %d predicted ties of negative products over all sites.") % ( + counts["offset_ties_per_site"], counts["offsets"], counts["up"], counts["down"], counts["negative_extra"])] + lines += ["#pragma once", "#include ", "", "namespace fold_selftest {", "", + f"constexpr int kW = {W}, kH = {H};", f"constexpr uint32_t kColour[{W * H * 3}] = {{"] + for i in range(0, len(colour), 8): + lines.append(" " + ", ".join(f"0x{v:08x}u" for v in colour[i:i + 8]) + ",") + lines += ["};", "", "} // namespace fold_selftest", ""] + with open(path, "w", encoding="utf-8", newline="\n") as f: + f.write("\n".join(lines)) + return counts + + +if __name__ == "__main__": + out = sys.argv[1] if len(sys.argv) > 1 else os.path.join(HERE, "fold_selftest.h") + print(write(out)) From 1e0399368f0857dcd978573403c74944fbb1e44a Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sun, 15 Nov 2026 05:09:07 -0500 Subject: [PATCH 2/2] L post fold guard: check the fold port's own values in the same PSO; single-site mutants (review fixes) - the fold port stores each pixel's three FP32 conversion inputs when the params' pad1 holds a side buffer (0 in every frame; the same compiled shader); RGA: the six v_fma_mixlo/hi_f16 .., 0x3dddf184, 0 sites keep their form, the side store writes exactly the registers they read, no s_round_mode - the start-up guard: fold and no-fold ports equal in raw bytes, and from the fold port's own stored inputs and codes, per site (row of the pair x channel) at least 8 FP32-product ties a second rounding gets wrong and every code the exact one-rounding RN16(xe kEncK) computed on the host; else the no-fold port - the separate control port is removed (needs=control gone from the build, the devices and the tests) - LP_MUTSITE 1..6: one site (3 row + channel) rounded twice, the other five folded (negative controls, never shipped) Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/build_dx12.py | 9 +- clean/dx12/hlsl/l_post_encode.hlsli | 43 +++++---- clean/dx12/kernels/l_post.hlsl | 13 ++- clean/dx12/tests/test_fold_probe.py | 5 +- clean/runtime/device_d3d12.cpp | 138 +++++++++++++++++----------- clean/runtime/device_dx12.cpp | 4 +- 6 files changed, 128 insertions(+), 84 deletions(-) diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index d25cb1ce..7a90d49f 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -191,12 +191,11 @@ def kernel_decls(path): if d["rs"] not in ("buffer", "tex", "direct"): raise SystemExit(f"{path}: {name}: unsupported root-signature kind {d['rs']}") d["arg_dwords"] = arg_dwords(d["args"], f"{path}: {name}") - # needs=fold | nofold | control: a port the device runs only when its start-up guard of the driver's - # conversion fold passed (fold) or failed (nofold), or only inside that guard (control: the sensitivity - # control, never selected); the ports of one name and code object differ by this key + # needs=fold | nofold: a port the device runs only when its start-up guard of the driver's conversion fold + # passed (fold) or failed (nofold); the two ports of one name and code object differ by this key d["needs"] = kv.get("needs", "") - if d["needs"] not in ("", "fold", "nofold", "control"): - raise SystemExit(f"{path}: {name}: needs={d['needs']} (fold, nofold or control)") + if d["needs"] not in ("", "fold", "nofold"): + raise SystemExit(f"{path}: {name}: needs={d['needs']} (fold or nofold)") # isa=,: the port's RGA ISA must satisfy each rule, [+...]>= or == (counts of # instruction mnemonics; @: only its x * literal + 0 sites, parse_isa_rules), checked after the # build (isa_check; WG_DX12_ISA_CHECK=0 skips it) diff --git a/clean/dx12/hlsl/l_post_encode.hlsli b/clean/dx12/hlsl/l_post_encode.hlsli index e627deb3..4212ec02 100644 --- a/clean/dx12/hlsl/l_post_encode.hlsli +++ b/clean/dx12/hlsl/l_post_encode.hlsli @@ -11,32 +11,31 @@ // literal with a 0 addend) and two start-up steps of the native device, both of which must pass: the probe // (l_fold_probe, clean/runtime/fold_probe.h: this expression on adversarial inputs) and the differential self-test of // the real post shader (its fold port against its no-fold port on the crafted frame of clean/runtime/fold_selftest.h, -// raw bytes). Otherwise, or on any error, the device runs the no-fold port of the kernel. +// raw bytes) with the check of the fold port's own values. Otherwise, or on any error, the device runs the no-fold +// port of the kernel. #pragma once #ifndef LP_MUT #define LP_MUT 0 #endif -#ifndef LP_ENCCONTROL -#define LP_ENCCONTROL 0 // 1: the device guard's sensitivity control port (needs=control): two roundings +#ifndef LP_MUTSITE +#define LP_MUTSITE 0 // 1 ... 6: negative control (never shipped): site LP_MUTSITE - 1 = 3 row + channel rounded twice #endif static const float kEncK = asfloat(0x3dddf184u); // 0.36f * 0.30102999566398120f -// the 8-value conversion of the history codes (x0 .. x2: the FP32 log2 arguments) and the three display values -// d0 .. d2 (already rounded to FP32, passed as opaque values by the caller); z = an opaque zero (mutant 3 only) -uint4 lp_encode_fold(float x0, float x1, float x2, float d0, float d1, float d2, uint z) +// one history code's conversion input: mad(x, kEncK, 0), which the driver folds into the conversion (one rounding), +// or (twice, a compile-time constant of the negative controls) the FP32 product made opaque first (two roundings) +float lp_enc_in(float x, uint z, bool twice) +{ + return twice ? asfloat(asuint(kEncK * x) ^ z) : mad(x, kEncK, 0.0f); +} + +uint4 lp_encode_conv(float x0, float x1, float x2, float d0, float d1, float d2, uint z, int twice_ch) { v8f cv; -#if LP_MUT == 3 || LP_ENCCONTROL // two roundings, the FP32 product made opaque before the conversion: the mutant - // (LP_MUT 3, never shipped) and the guard's control port (never selected to run) - cv.v[0] = asfloat(asuint(kEncK * x0) ^ z); - cv.v[1] = asfloat(asuint(kEncK * x1) ^ z); - cv.v[2] = asfloat(asuint(kEncK * x2) ^ z); -#else - cv.v[0] = mad(x0, kEncK, 0.0f); - cv.v[1] = mad(x1, kEncK, 0.0f); - cv.v[2] = mad(x2, kEncK, 0.0f); -#endif + cv.v[0] = lp_enc_in(x0, z, LP_MUT == 3 || twice_ch == 0); + cv.v[1] = lp_enc_in(x1, z, LP_MUT == 3 || twice_ch == 1); + cv.v[2] = lp_enc_in(x2, z, LP_MUT == 3 || twice_ch == 2); cv.v[3] = d0; cv.v[4] = d1; cv.v[5] = d2; @@ -44,3 +43,15 @@ uint4 lp_encode_fold(float x0, float x1, float x2, float d0, float d1, float d2, cv.v[7] = 0.0f; return wg_to_f16_ags(cv); } + +// the 8-value conversion of the history codes (x0 .. x2: the FP32 log2 arguments) and the three display values +// d0 .. d2 (already rounded to FP32, passed as opaque values by the caller); z = an opaque zero (the mutants only); +// row: the row of the pair (wave-uniform; the single-site mutants only) +uint4 lp_encode_fold(float x0, float x1, float x2, float d0, float d1, float d2, uint z, int row = 0) +{ +#if LP_MUTSITE + [branch] if (row == (LP_MUTSITE - 1) / 3) + return lp_encode_conv(x0, x1, x2, d0, d1, d2, z, (LP_MUTSITE - 1) % 3); +#endif + return lp_encode_conv(x0, x1, x2, d0, d1, d2, z, -1); +} diff --git a/clean/dx12/kernels/l_post.hlsl b/clean/dx12/kernels/l_post.hlsl index eb70ae58..0d312b5a 100644 --- a/clean/dx12/kernels/l_post.hlsl +++ b/clean/dx12/kernels/l_post.hlsl @@ -33,7 +33,6 @@ // @wg-kernel l_post_recon co=l_post.co block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p // @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_ENCFOLD=1 needs=fold isa=v_fma_mixlo_f16@3dddf184+v_fma_mixhi_f16@3dddf184==6,s_round_mode==0 // @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 file=l_post_recon_p12__nofold block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1 needs=nofold -// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 file=l_post_recon_p12__control block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_ENCFOLD=1,LP_ENCCONTROL=1 needs=control isa=v_fma_mixlo_f16@3dddf184+v_fma_mixhi_f16@3dddf184==0 // @wg-kernel l_post_recon_ldr co=l_post.co entry=l_post_recon_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_LDR=1 // @wg-kernel l_post_recon_p12_ldr co=l_post.co entry=l_post_recon_p12_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_LDR=1 #include "wg_platform.hlsli" @@ -600,7 +599,17 @@ WG_KERNEL(l_post_recon) xe[ce] = enc_log(hh[ce]); de[ce] = opq(clampf(enc_e(oo[ce]), 0.0f, kDisplayMax)); } - const uint4 hc = lp_encode_fold(xe[0], xe[1], xe[2], de[0], de[1], de[2], g_z); + const uint4 hc = lp_encode_fold(xe[0], xe[1], xe[2], de[0], de[1], de[2], g_z, row); + // the device's start-up guard (P_pad1 = its side buffer; 0 in every frame): this pixel's FP32 conversion inputs, + // so the guard checks each stored history code against the exact one-rounding encode of its own input + [branch] if (P_pad1 != 0) { + if (and(active, or(!second, has1))) { + const uint so = (uint)((y0 + row) * P_dw + x) * 12u; + wg_st32((uint)P_pad1, so, asuint(xe[0])); + wg_st32((uint)P_pad1, so + 4u, asuint(xe[1])); + wg_st32((uint)P_pad1, so + 8u, asuint(xe[2])); + } + } #else v8f cv; [unroll] for (int ci = 0; ci < 3; ++ci) { diff --git a/clean/dx12/tests/test_fold_probe.py b/clean/dx12/tests/test_fold_probe.py index 87ac3a4d..29f4759a 100644 --- a/clean/dx12/tests/test_fold_probe.py +++ b/clean/dx12/tests/test_fold_probe.py @@ -85,10 +85,7 @@ def test_isa_count(self): def test_post_ports(self): decls = [d for d in B.kernel_decls(os.path.join(ROOT, "clean", "dx12", "kernels", "l_post.hlsl")) if d["name"] == "l_post_recon_p12"] - self.assertEqual(sorted(d["needs"] for d in decls), ["control", "fold", "nofold"]) - control = [d for d in decls if d["needs"] == "control"][0] - self.assertEqual(control["defines"].get("LP_ENCCONTROL"), "1") # two roundings - self.assertTrue(control["isa"].endswith("==0")) # and the build checks: no fold site + self.assertEqual(sorted(d["needs"] for d in decls), ["fold", "nofold"]) fold = [d for d in decls if d["needs"] == "fold"][0] self.assertEqual(fold["defines"].get("LP_ENCFOLD"), "1") self.assertIn("@3dddf184", fold["isa"]) # the fold's sites: kEncK as the literal, addend 0 diff --git a/clean/runtime/device_d3d12.cpp b/clean/runtime/device_d3d12.cpp index 1681d6ca..506b89ab 100644 --- a/clean/runtime/device_d3d12.cpp +++ b/clean/runtime/device_d3d12.cpp @@ -24,6 +24,7 @@ #include "fold_selftest.h" #include +#include #include #include #include @@ -1009,9 +1010,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { return native_filename(co) && co.size() > 3 && co.substr(co.size() - 3) == ".co"; } // variant_loaded without an error: whether the runtime's loaded code objects select this port. - bool needs_met(const NativeKernel& k) const { - return k.needs.empty() || (k.needs != "control" && (k.needs == "fold") == fold_ok_); // control: never selected - } + bool needs_met(const NativeKernel& k) const { return k.needs.empty() || (k.needs == "fold") == fold_ok_; } bool variant_matches(const NativeKernel& k) const { if (!needs_met(k)) return false; if (k.co == "-") return true; @@ -1301,7 +1300,7 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { k.ext_doubles = true; } if (const auto* nd = entry.find("needs")) { - if (!nd->is_string() || (nd->text != "fold" && nd->text != "nofold" && nd->text != "control")) + if (!nd->is_string() || (nd->text != "fold" && nd->text != "nofold")) return fail("native D3D12 manifest: invalid needs for " + k.name); k.needs = nd->text; } @@ -1435,19 +1434,20 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { if (!ok) err_ = error; return ok; } - // Start-up guard of the driver's conversion fold: only when a port needs it. All three steps must pass, else (or on - // any error) the "nofold" ports run (fail closed): + // Start-up guard of the driver's conversion fold: only when a port needs it. All steps must pass, else (or on any + // error) the "nofold" ports run (fail closed): // 1. probe: l_fold_probe (fold_probe.h) on adversarial inputs of the shared encode expression, against expected // bytes from exact rationals; - // 2. differential self-test (fold_selftest): the REAL post shader, its fold port and its no-fold port (exact by - // construction), on one crafted frame (fold_selftest.h); the raw output bytes must be equal; - // 3. sensitivity control (in fold_selftest): the same frame through the "control" port (the post shader with the - // history encode rounded twice, as a driver without the fold would compile it) must differ from the no-fold port - // at each of the six encode sites, so the frame's values, as this device computes them, would show a missing fold - // at every site. + // 2. self-test of the REAL post shader (fold_selftest) on one crafted frame (fold_selftest.h): the fold port, with + // its side output on, and the no-fold port (exact by construction). The fold port's raw output bytes must equal + // the no-fold port's, and, from the fold port's OWN values in the same PSO (its stored FP32 conversion inputs xe + // and its stored history codes), at each of the six encode sites (row of the pair x channel): at least + // kGuardMinTies inputs are FP32-product ties a second rounding gets wrong, and every stored code is the exact + // one-rounding RN16(xe kEncK) computed here. // Every queue wait of the guard is bounded (kGuardWaitMs). A timeout means a queue that did not complete: device // creation fails (its allocations are kept, never freed under a running queue). Logged once per process (note()). static constexpr DWORD kGuardWaitMs = 10000; + static constexpr int kGuardMinTies = 8; bool fold_probe() { bool wanted = false; std::string uncovered; @@ -1495,51 +1495,70 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { (fold_ok_ ? "passed (" + sens_note_ + "): the fold ports run" : "failed (" + note + "): the no-fold ports run"); return true; } - // The differential self-test of l_post_recon_p12 with its sensitivity control (fold_probe steps 2 and 3). The frame - // (fold_selftest.h; render size = output size, jitter 0, reset, exposure 1, post scale 0, codes and kernel shape 0) - // makes each full-history value the render colour, so the history codes are the encode of crafted values: per site - // FP32-product ties at predicted-log2 offsets -8 ... 8 ulp (a log2 up to 8 ulp off still lands on ties), zero and - // FP16-subnormal products, the floor. Each port's outputs start filled with its own sentinel. The fold port's defined - // bytes (full history, reduced history, display image) must equal the no-fold port's; the control port's full - // history must differ from the no-fold port's in at least one value of each of the six encode sites (site = row of - // the pair x channel). True only when all three ports ran and both conditions hold. + // RN16 of the value v (exact: the product of two FP32 values fits a double), nearest even, FP16 subnormals kept, + // overflow to infinity; *tie: v lies exactly halfway between two FP16 values. + static uint16_t guard_rn16(double v, bool* tie = nullptr) { + if (tie) *tie = false; + const uint16_t sign = std::signbit(v) ? 0x8000u : 0u; + const double a = std::fabs(v); + if (a == 0.0) return sign; + if (a >= 65520.0) return sign | 0x7c00u; + int ex = 0; + std::frexp(a, &ex); // a = m 2^ex, m in [0.5, 1): exponent ex - 1 + const int e = std::max(ex - 1, -14); + const double q = std::ldexp(a, 10 - e); // in FP16 ulps (exact) + double n = std::floor(q); + const double r = q - n; + if (tie) *tie = r == 0.5; + if (r > 0.5 || (r == 0.5 && std::fmod(n, 2.0) == 1.0)) n += 1.0; + int ee = e; + if (n >= 2048.0) { n /= 2.0; ++ee; } + if (ee > 15) return sign | 0x7c00u; + const uint32_t ni = static_cast(n); + if (ni < 1024u) return static_cast(sign | ni); + return static_cast(sign | ((ee + 15) << 10) | (ni - 1024u)); + } + // The self-test of l_post_recon_p12 (fold_probe step 2). The frame (fold_selftest.h; render size = output size, + // jitter 0, reset, exposure 1, post scale 0, codes and kernel shape 0) makes each full-history value the render + // colour, so the history codes are the encode of crafted values: per site FP32-product ties at predicted-log2 offsets + // -8 ... 8 ulp, zero and FP16-subnormal products, the floor. The fold port runs with its side output on (the params' + // pad1 = the side buffer: each pixel's three FP32 conversion inputs xe); each port's outputs start filled with its own + // sentinel. True only when both ports ran, their defined bytes are equal, and the fold port's own values pass the + // per-site checks (fold_probe's comment). bool fold_selftest(std::string* note) { - NativeKernel *fold = nullptr, *nofold = nullptr, *control = nullptr; + NativeKernel *fold = nullptr, *nofold = nullptr; auto range = kernels_.equal_range("l_post_recon_p12"); for (auto i = range.first; i != range.second; ++i) { if (i->second.needs == "fold") fold = &i->second; if (i->second.needs == "nofold") nofold = &i->second; - if (i->second.needs == "control") control = &i->second; } auto abi = [](const NativeKernel* k) { if (!k || k->block != 256 || k->rs != "buffer" || k->args.size() != 14) return false; for (int n = 0; n < 12; ++n) if (k->args[n].kind != 'p') return false; return k->args[12].kind == 'b' && k->args[12].bytes == 64 && k->args[13].kind == 'p'; }; - if (!abi(fold) || !abi(nofold) || !abi(control)) { - *note = "self-test: the fold / no-fold / control ports of l_post_recon_p12 are missing"; - return false; - } + if (!abi(fold) || !abi(nofold)) { *note = "self-test: the fold / no-fold ports of l_post_recon_p12 are missing"; return false; } constexpr int W = ::fold_selftest::kW, H = ::fold_selftest::kH, N = W * H; constexpr int pitch = (W + 63) & ~63; constexpr size_t hist = size_t(N) * 8, reduced = size_t(W / 2) * (H / 2) * 4, display = size_t(3) * pitch * H * 2; - constexpr size_t outs = hist + reduced + display; - constexpr int kPorts = 3; // fold, no-fold, control - const uint8_t sentinel[kPorts] = {0xA5, 0x5A, 0x3C}; - // layout: colour | A8 | C4 | previous history | motions x 3 | post blob | outputs (fold, no-fold, control) + constexpr size_t outs = hist + reduced + display, side = size_t(N) * 12; + const uint8_t sentinel[2] = {0xA5, 0x5A}; + // layout: colour | A8 | C4 | previous history | motions x 3 | post blob | outputs (fold, no-fold) | side (fold) const size_t o_col = 0, o_a = o_col + size_t(N) * 16, o_c = o_a + size_t(N) * 8, o_hp = o_c + size_t(N) * 4, - o_m = o_hp + hist, o_w = o_m + size_t(N) * 12, o_out = o_w + 256, total = o_out + kPorts * outs; + o_m = o_hp + hist, o_w = o_m + size_t(N) * 12, o_out = o_w + 256, o_side = o_out + 2 * outs, + total = o_side + side; std::vector host(total, 0); for (int i = 0; i < N; ++i) std::memcpy(&host[o_col + size_t(i) * 16], &::fold_selftest::kColour[3 * i], 12); - for (int port = 0; port < kPorts; ++port) std::memset(&host[o_out + port * outs], sentinel[port], outs); + for (int port = 0; port < 2; ++port) std::memset(&host[o_out + port * outs], sentinel[port], outs); + std::memset(&host[o_side], 0xFF, side); // a NaN pattern: a value the kernel did not store fails the checks struct { int32_t rw, rh, dw, dh, reset, pad0, pad1, pad2; float rx, ry, jx, jy, e, e_prev, ratio, pad3; } prm = {W, H, W, H, 1, 0, 0, 0, 1.0f, 1.0f, 0.0f, 0.0f, 1.0f, 1.0f, 1.0f, 0.0f}; static_assert(sizeof prm == 64, "WgPostParams"); DevPtr buf = 0; - std::vector got(kPorts * outs); + std::vector got(2 * outs + side); bool ok = alloc(&buf, total) && upload(buf, host.data(), total); - NativeKernel* ports[kPorts] = {fold, nofold, control}; - for (int port = 0; ok && port < kPorts; ++port) { + NativeKernel* ports[2] = {fold, nofold}; + for (int port = 0; ok && port < 2; ++port) { DevPtr p[12], none = 0; const DevPtr out = buf + o_out + port * outs; const DevPtr in[9] = {buf + o_col, buf + o_a, buf + o_a, buf + o_c, buf + o_hp, buf + o_m, buf + o_m + N * 4, @@ -1548,20 +1567,21 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { p[9] = out; p[10] = out + hist; p[11] = out + hist + reduced; void* args[14]; for (int n = 0; n < 12; ++n) args[n] = &p[n]; + prm.pad1 = port == 0 ? static_cast(static_cast(buf + o_side)) : 0; // the fold port's side output args[12] = &prm; args[13] = &none; const unsigned grid = unsigned((W + 31) / 32) * unsigned(((H + 1) / 2 + 7) / 8); ok = dispatch(*ports[port], "l_post_recon_p12", grid, 256, args); } - ok = ok && download(got.data(), buf + o_out, kPorts * outs); + ok = ok && download(got.data(), buf + o_out, 2 * outs + side); const std::string error = err_; if (buf && !stalled_) free(buf); // after a timeout the queue may still use it: kept if (!ok) { *note = "self-test run failed (" + error + ")"; return false; } - if (const char* dump = std::getenv("WG_TEST_FOLD_SELFTEST_DUMP")) { // tests: the three ports' outputs, raw + if (const char* dump = std::getenv("WG_TEST_FOLD_SELFTEST_DUMP")) { // tests: both ports' outputs and the side output if (FILE* f = std::fopen(dump, "wb")) { std::fwrite(got.data(), 1, got.size(), f); std::fclose(f); } } const uint8_t* fo = got.data(); const uint8_t* nf = got.data() + outs; - const uint8_t* ct = got.data() + 2 * outs; + const uint8_t* sd = got.data() + 2 * outs; size_t differ = 0, hist_differ = 0; for (size_t i = 0; i < outs; ++i) { if (i >= hist + reduced) { // display: only the pixels of each pitch row are defined @@ -1575,30 +1595,40 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { std::to_string(hist_differ) + " of the full history)"; return false; } - // sensitivity per site: history values (pixel (x, y), channel c: bytes 8 (y W + x) + 2 c) where the control port - // (two roundings) differs from the no-fold port; every control value must have been written (no sentinel left) - int sens[6] = {0, 0, 0, 0, 0, 0}; - bool written = true; + // the fold port's own values: per site, detecting ties among its inputs and every stored code exact + const float k = [] { const uint32_t b = 0x3dddf184u; float f; std::memcpy(&f, &b, 4); return f; }(); + int ties[6] = {0, 0, 0, 0, 0, 0}, wrong[6] = {0, 0, 0, 0, 0, 0}; for (int y = 0; y < H; ++y) { for (int x = 0; x < W; ++x) { for (int c = 0; c < 3; ++c) { - const size_t o = size_t(8) * (size_t(y) * W + x) + 2 * c; - written = written && !(ct[o] == sentinel[2] && ct[o + 1] == sentinel[2]); - if (ct[o] != nf[o] || ct[o + 1] != nf[o + 1]) ++sens[3 * (y & 1) + c]; + const int s = 3 * (y & 1) + c; + float xe; + std::memcpy(&xe, sd + size_t(12) * (size_t(y) * W + x) + 4 * c, 4); + uint16_t q; + std::memcpy(&q, fo + size_t(8) * (size_t(y) * W + x) + 2 * c, 2); + if (!std::isfinite(xe)) { ++wrong[s]; continue; } // not stored, or not a value the encode takes + const double pe = double(xe) * double(k); // exact + const float p32 = static_cast(pe); // RN32 of the exact product + bool tie = false; + const uint16_t once = guard_rn16(pe), twice = guard_rn16(double(p32), &tie); + if (tie && pe != double(p32) && once != twice) ++ties[s]; + if (q != once) ++wrong[s]; } } } - int least = sens[0]; - for (int k = 1; k < 6; ++k) least = std::min(least, sens[k]); - if (!written || least < 1) { - std::string per; - for (int k = 0; k < 6; ++k) per += (k ? "/" : "") + std::to_string(sens[k]); - *note = std::string("sensitivity control: ") + (written ? "" : "control output incomplete; ") + - "the two-roundings port differs from the no-fold port at the six sites in " + per + - " values (each must be at least 1)"; + int least = ties[0], bad = 0; + std::string per; + for (int s = 0; s < 6; ++s) { + least = std::min(least, ties[s]); + bad += wrong[s]; + per += (s ? "/" : "") + std::to_string(ties[s]) + (wrong[s] ? "!" + std::to_string(wrong[s]) : ""); + } + if (bad || least < kGuardMinTies) { + *note = "own-value check: detecting ties per site " + per + " (each must be at least " + + std::to_string(kGuardMinTies) + "; !n: codes that are not the one-rounding encode of their input)"; return false; } - sens_note_ = "sensitivity control: at least " + std::to_string(least) + " values per site"; + sens_note_ = "detecting ties per site " + per + ", every code the one-rounding encode"; return true; } bool create_device() { diff --git a/clean/runtime/device_dx12.cpp b/clean/runtime/device_dx12.cpp index 1cfdfc59..9d7eb605 100644 --- a/clean/runtime/device_dx12.cpp +++ b/clean/runtime/device_dx12.cpp @@ -526,9 +526,7 @@ class Dx12Device : public Device { for (const wgc::Json& k : ks->items) { // a port that needs the driver's conversion fold (manifest "needs": "fold") runs only on the native device after // its start-up guard (device_d3d12.cpp fold_probe); this device has no guard and takes the exact "nofold" port - // (and never a guard's "control" port) - if (const wgc::Json* nd = k.find("needs"); nd && nd->is_string() && (nd->text == "fold" || nd->text == "control")) - continue; + if (const wgc::Json* nd = k.find("needs"); nd && nd->is_string() && nd->text == "fold") continue; Kernel x; int64_t block = 0, wave = 0, dwords = 0; const wgc::Json *n = k.find("name"), *fl = k.find("file"), *ar = k.find("args"), *co = k.find("co");