From 13e7a4fbb2c5900dc9442542ff9dc9f21beb0c30 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:50:18 -0500 Subject: [PATCH 1/2] D3D12: analytic-filter ports of the fused output kernels (post_w64.co), byte for byte against HIP post.hlsl WG_LZ_ANALYTIC: w64/post_tf_f32_p12_diet_vsit and w64/post_tf4_dl_f32_p12_diet_vsit of post_w64.co (the build the runtime loads when the model file has no output-filter weights, e.g. Model L): FP32 Lanczos constants, FP32 values between the passes, no weight arguments. The original's DXILs are unchanged. post_tf_d3d12_check.py: these ports against HIP post_w64.co with l_d3d12_ops_check.py's engine (raw bytes, sentinels, slack, negative control), the original port against post_w64__orig.co as the Model M control; the harness takes a kinds table and a HIP block. Refs #89 Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/kernels/post.hlsl | 59 ++++++++++ clean/dx12/tests/post_tf_d3d12_check.py | 138 ++++++++++++++++++++++++ clean/l/runner/l_d3d12_ops_check.py | 9 +- 3 files changed, 203 insertions(+), 3 deletions(-) create mode 100644 clean/dx12/tests/post_tf_d3d12_check.py diff --git a/clean/dx12/kernels/post.hlsl b/clean/dx12/kernels/post.hlsl index 21f0be3f..3f883734 100644 --- a/clean/dx12/kernels/post.hlsl +++ b/clean/dx12/kernels/post.hlsl @@ -50,7 +50,21 @@ // @wg-kernel w64/post_tf4_dl_f32_p12_diet_vsit_direct co=post_w64__orig.co rs=direct entry=post_tf_f32_p12_diet_vsit_direct block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u,u,u,u,u,u defines=WG_DIRECT_TEXTURES=1,POST_OUT=3,WG_TF_NW=4,WG_TF_CAP=448 // @wg-kernel pmo/post_tf4_dl_f32_p12_mo co=post_mo_w64__orig.co entry=post_tf_f32_p12_diet_vsit block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u,b40,u,u,u,u,u defines=WG_POST_MO=1,WG_TF_NW=4,WG_TF_CAP=448 // @wg-kernel pmo/post_tf4_dl_f32_p12_mo_direct co=post_mo_w64__orig.co rs=direct entry=post_tf_f32_p12_diet_vsit_direct block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u,b40,u,u,u,u,u defines=WG_DIRECT_TEXTURES=1,POST_OUT=3,WG_POST_MO=1,WG_TF_NW=4,WG_TF_CAP=448 +// +// Analytic output filter (WG_LZ_ANALYTIC = 1; HIP: post_w64.co, post.hip built with -mwavefrontsize64 and the default +// WG_LZ_PROFILE = 0, the build the runtime loads under "w64" when the model file has no output-filter weights, e.g. +// Model L's): the same kernels with spec 06's own Lanczos weights as FP32 constants (kLz0 ... kLz4): v = kLz0 t0, then +// one fused multiply-add per tap, clamp to the taps' range, every value FP32 (the horizontal results go to the ring as +// FP32, the vertical result straight into Dec). No weight arguments. Buffer forms 0 only (POST_OUT = 0). +// @wg-kernel w64/post_tf_f32_p12_diet_vsit co=post_w64.co file=w64__post_tf_f32_p12_diet_vsit__analytic entry=post_tf_f32_p12_diet_vsit block=512 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u defines=WG_LZ_ANALYTIC=1 +// @wg-kernel w64/post_tf4_dl_f32_p12_diet_vsit co=post_w64.co file=w64__post_tf4_dl_f32_p12_diet_vsit__analytic entry=post_tf_f32_p12_diet_vsit block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u defines=WG_TF_NW=4,WG_TF_CAP=448,WG_LZ_ANALYTIC=1 #include "wg_platform.hlsli" +#ifndef WG_LZ_ANALYTIC +#define WG_LZ_ANALYTIC 0 +#endif +#if WG_LZ_ANALYTIC && (POST_OUT != 0 || defined(WG_POST_MO)) +#error "analytic output filter: buffer form 0 only" +#endif #if defined(WG_DIRECT_TEXTURES) #include "native_direct.hlsli" #include "wg_f16lut.hlsli" @@ -85,7 +99,9 @@ WG_ARGS_BEGIN int mo_mw, mo_mh, mo_pitch, mo_wide; // WgMoParams (40 bytes) float mo_sx, mo_sy, mo_bx, mo_by, mo_idw, mo_idh; #endif +#if !WG_LZ_ANALYTIC uint lz_w0, lz_w1, lz_w2, lz_w3, lz_w4; // FP16 bit patterns of the filter weights of taps -2 ... +2 +#endif WG_ARGS_END // ---- constants (FP32 bit patterns of post.hip's constexpr values) --------------------------------------------------- @@ -697,6 +713,26 @@ void mo_core(Gather g, float2 mv, int x, int y, int fx, int fy, float s, out flo } #endif +#if WG_LZ_ANALYTIC +// ---- output filter, profile 0 ("analytic", post_w64.co): FP32 constant weights of taps -2 ... +2 for an even output +// index (post.hip kLz0 ... kLz4; mirrored for an odd one), v = w0 t0, one fma per further tap, clamp to the taps' range +static const float kLz0 = asfloat(0xbdba3b4bu), kLz1 = asfloat(0x3de71ca9u), kLz2 = asfloat(0x3f23b70fu); +static const float kLz3 = asfloat(0x3ed182b5u), kLz4 = asfloat(0xbd90a4acu); +float lz5a(float t0, float t1, float t2, float t3, float t4, float w0, float w1, float w3, float w4) +{ + float v = w0 * t0; + v = mad(w1, t1, v); + v = mad(kLz2, t2, v); + v = mad(w3, t3, v); + v = mad(w4, t4, v); + const float mid_lo = min(min(t1, t2), t3), mid_hi = max(max(t1, t2), t3); + const float lo = min(min(mid_lo, t0), t4), hi = max(max(mid_hi, t0), t4); + return min(max(v, lo), hi); // med3 (the taps are finite, lo <= hi) +} +// lz5 / lz5 on the FP32 taps t[o ... o + 4], as FP32 bits +uint lz5a_even(float t[10], int o) { return asuint(lz5a(t[o], t[o + 1], t[o + 2], t[o + 3], t[o + 4], kLz0, kLz1, kLz3, kLz4)); } +uint lz5a_odd(float t[10], int o) { return asuint(lz5a(t[o], t[o + 1], t[o + 2], t[o + 3], t[o + 4], kLz4, kLz3, kLz1, kLz0)); } +#else // ---- output filter, profile 2 ("original"): FP16 weights, one FP16 fma per tap, clamp to the taps' range ------------- // Values are "boxes": FP16 bits in the low half of a dword. half hbox(uint v) { return asfloat16((uint16_t)(v & 0xffffu)); } @@ -726,6 +762,7 @@ uint lz5_odd(uint t[10], int o) return lz5_original(t[o], t[o + 1], t[o + 2], t[o + 3], t[o + 4], hbox(lz_w4), hbox(lz_w3), hbox(lz_w2), hbox(lz_w1), hbox(lz_w0)); } +#endif // The reduced history's 10-bit code of one channel from its 2 x 2 sum (the chain of post.hip's reduced store) uint rh_code(float sum) @@ -1173,6 +1210,18 @@ WG_KERNEL(post_tf_p12_diet_vsit) #else const uint src = scr + (uint)hk * kRowDw + (uint)hc * 32u + (uint)hg * 3u; #endif +#if WG_LZ_ANALYTIC + float t[10]; // the FP16 display taps as FP32 values (exact); results FP32 bits + [unroll] for (int i = 0; i < 5; ++i) { + const uint q = lds[src + (uint)i]; + t[2 * i] = f16tof32(q); + t[2 * i + 1] = f16tof32(q >> 16); + } + hv[0] = lz5a_even(t, 0); + hv[1] = lz5a_odd(t, 2); + hv[2] = lz5a_even(t, 3); + hv[3] = lz5a_odd(t, 5); +#else uint t[10]; [unroll] for (int i = 0; i < 5; ++i) { const uint q = lds[src + (uint)i]; @@ -1183,6 +1232,7 @@ WG_KERNEL(post_tf_p12_diet_vsit) hv[1] = lz5_odd(t, 2); hv[2] = lz5_even(t, 3); hv[3] = lz5_odd(t, 5); +#endif } tf_row_sync(); if (hact) { @@ -1255,13 +1305,22 @@ WG_KERNEL(post_tf_p12_diet_vsit) } // emit (n = 1): weights of the lane's phase by selects (lz_wsel), lz5w, Dec, store 3 x f32 const bool ev = dy == 0; +#if WG_LZ_ANALYTIC + const float w0 = ev ? kLz0 : kLz4, w4 = ev ? kLz4 : kLz0, w1 = ev ? kLz1 : kLz3, w3 = ev ? kLz3 : kLz1; +#else const half w0 = hbox(ev ? lz_w0 : lz_w4), w4 = hbox(ev ? lz_w4 : lz_w0); const half w1 = hbox(ev ? lz_w1 : lz_w3), w3 = hbox(ev ? lz_w3 : lz_w1); +#endif float vo[3], qv[3]; bool vsmall = false; [unroll] for (int c = 0; c < 3; ++c) { +#if WG_LZ_ANALYTIC + qv[c] = clampf(lz5a(asfloat(t[0][c]), asfloat(t[1][c]), asfloat(t[2][c]), asfloat(t[3][c]), asfloat(t[4][c]), + w0, w1, w3, w4), k1em4, 3.0f); +#else const uint r = lz5_original(t[0][c], t[1][c], t[2][c], t[3][c], t[4][c], w0, w1, hbox(lz_w2), w3, w4); qv[c] = clampf(f16tof32(r), k1em4, 3.0f); +#endif vo[c] = exp2(qv[c] * kDecK) - 1.0f; vsmall = or(vsmall, qv[c] < kDecPolyMax); } diff --git a/clean/dx12/tests/post_tf_d3d12_check.py b/clean/dx12/tests/post_tf_d3d12_check.py new file mode 100644 index 00000000..462e5d50 --- /dev/null +++ b/clean/dx12/tests/post_tf_d3d12_check.py @@ -0,0 +1,138 @@ +# SPDX-License-Identifier: MIT +"""The analytic-filter D3D12 ports of Model M's fused output kernels (clean/dx12/kernels/post.hlsl, WG_LZ_ANALYTIC) +against HIP's post_w64.co on the same synthetic inputs, byte for byte; the original-filter port (post_w64__orig.co, +already proven on whole frames by clean/dx12/compare.py) as the Model M control on the same harness. + + python clean/dx12/tests/post_tf_d3d12_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --run + [--seeds N] --json out.json + + holds post_w64.co and post_w64__orig.co (clean/runner/build.py's builds of kernels/post.hip), the D3D12 ports +(clean/dx12/build_dx12.py --only w64/post_tf_f32_p12_diet_vsit,w64/post_tf4_dl_f32_p12_diet_vsit,native_fill: both +builds of each name) and kernel_replay.exe (clean/l/runner/l_d3d12_check.py prepare_replay). --run runs in one GPU slot. +The engine and rules are clean/l/runner/l_d3d12_ops_check.py's: the RAW downloaded bytes of every output (the full +history with its alpha, the reduced codes, the FP32 output image) are compared; each output buffer starts filled with +its back end's own sentinel (HIP 0xA5, D3D12 0x5A) and 64 slack bytes follow it, which no back end may write; a negative +control (one case again on D3D12 with half its workgroups) must be caught. Output: counts only, no timing. + +Kernels: +- w64/post_tf_f32_p12_diet_vsit (analytic port) vs HIP post_tf_f32_p12_diet_vsit of post_w64.co; +- w64/post_tf4_dl_f32_p12_diet_vsit (analytic port; D3D12 only, NW = 4 row pairs per step, 256 lanes) vs HIP + post_tf_f32_p12_diet_vsit (frames that its cap of 352 takes) or post_tf_dl_f32_p12 (native AA: D = 1.5 R), whose + every output pixel is the same function (the exposure pair holds the ratio and dec_scale that the non-diet kernel + computes itself); +- control: w64/post_tf_f32_p12_diet_vsit (original-filter port, five FP16 weights as arguments) vs post_w64__orig.co. + The same name as the first: the D3D12 device runs the port of the code object the case loads under "w64". +""" +import argparse +import json +import os +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.runner import l_d3d12_ops_check as OPS # noqa: E402 +from clean.reference import post as MP # noqa: E402 +from clean.runner import post as RP # noqa: E402 +from clean.runner import post_sizes as PZ # noqa: E402 + +SLACK = 64 +CPAIRS = 68 # runtime.cpp kTfCpairs +# (rw, rh, ow, oh, reset): Quality (D = 2.25 R), Performance (D = 3 R), odd sizes, reset, a wide frame of 64 strips whose +# chunk plan has k1 > 0 chunks of 68 row pairs and then chunks of 36 +TF_FRAMES = [(64, 36, 96, 54, False), (48, 27, 96, 54, False), (50, 29, 99, 57, True), (70, 41, 105, 61, False), + (1280, 187, 2560, 280, False)] +# post_tf4: Quality frames (HIP post_tf_f32_p12_diet_vsit takes them too) and native AA (D = 1.5 R: HIP post_tf_dl) +TF4_FRAMES = [(64, 36, 96, 54, False), (50, 29, 99, 57, True), (64, 36, 64, 36, False), (61, 33, 61, 33, False), + (96, 54, 96, 54, True)] +# FP16 bits of the original filter's five weights for the control (spec 06 draft 4 gives them from the model file; any +# FP16 set exercises the port) +ORIG_W = [np.float16(v).view(np.uint16) for v in (-0.0908, 0.1129, 0.6400, 0.4092, -0.0706)] + + +def _f32_bits(v): + return int(np.float32(v).view(np.uint32)) + + +def tf_case(kernel, rng, frame, co, hip_kernel, block, hip_block, taps, expo_pair): + rw, rh, ow, oh, reset = frame + dw, dh = MP.recon_size_for(ow), MP.recon_size_for(oh) + x = MP.random_inputs(rng, rw, rh, dw, dh, reset=reset) + prm = RP.post_params(x) + data = RP.pack12_inputs(RP.post_device_inputs(x)) + k1, c2, nch = PZ.tf_chunks(ow, oh, CPAIRS) + e, ep = prm.e, prm.e_prev + dec_scale = float(np.float32(0.01 / e)) + ratio = float(np.float32(np.float64(e) / np.float64(ep))) + n_hist, n_red, n_out = dw * dh * 8, max(4, (dw // 2) * (dh // 2) * 4), ow * oh * 12 + red_def = np.zeros(n_red, bool) + red_def[:(dw // 2) * (dh // 2) * 4] = True + slack = np.zeros(SLACK, bool) + defined = [np.concatenate([np.ones(n_hist, bool), slack]), np.concatenate([red_def, slack]), + np.concatenate([np.ones(n_out, bool), slack])] + args = [("buf", data[k]) for k in ("color", "A")] + [("null",)] + args += [("buf", data[k]) for k in ("C", "hist_prev", "m_near", "m_center", "m_far")] + args += [("buf", RP.pack({"s": np.array(rng.uniform(0.5, 4.0))})), ("out", n_hist + SLACK), ("out", n_red + SLACK), + ("out", n_out + SLACK), ("bytes", bytes(prm))] + args += [("buf", np.array([e, ep, 0.0, ratio, dec_scale], np.float32).tobytes())] if expo_pair else [("null",)] + args += [("u32", ow), ("u32", oh), ("u32", _f32_bits(dec_scale)), ("u32", _f32_bits(1.0)), ("u32", 12 * ow), + ("u32", CPAIRS), ("u32", k1), ("u32", c2)] + if taps: + args += [("u32", int(w)) for w in ORIG_W] + return {"kernel": kernel, "co": co, "tag": "w64", "hip_kernel": hip_kernel, "grid": PZ.PS._cdiv(ow, 40) * nch, + "block": block, "hip_block": hip_block, "args": args, "defined": defined, + "label": f"{rw}x{rh}->{ow}x{oh} (D {dw}x{dh})" + (" reset" if reset else "") + (" expo" if expo_pair else "")} + + +def tf_cases(kernel, rng): + return [tf_case(kernel, rng, f, "post_w64.co", "post_tf_f32_p12_diet_vsit", 512, 512, False, i % 2 == 0) + for i, f in enumerate(TF_FRAMES)] + + +def tf4_cases(kernel, rng): + out = [] + for i, f in enumerate(TF4_FRAMES): + rw, rh, ow, oh, _ = f + dw, dh = MP.recon_size_for(ow), MP.recon_size_for(oh) + hk = "post_tf_f32_p12_diet_vsit" if RP.tf_supported(rw, rh, dw, dh) else "post_tf_dl_f32_p12" + assert RP.tf_supported(rw, rh, dw, dh, 4, RP.TF4_DL_CAP), f + # the non-diet HIP kernel computes the ratio and takes dec_scale from its argument: the exposure pair is given + out.append(tf_case(kernel, rng, f, "post_w64.co", hk, 256, 512, False, True if hk == "post_tf_dl_f32_p12" + else i % 2 == 0)) + return out + + +def control_cases(kernel, rng): + return [dict(tf_case(kernel, rng, f, "post_w64__orig.co", "post_tf_f32_p12_diet_vsit", 512, 512, True, i % 2 == 0), + kernel="w64/post_tf_f32_p12_diet_vsit") for i, f in enumerate(TF_FRAMES[:4])] + + +KINDS = {"w64/post_tf_f32_p12_diet_vsit": tf_cases, "w64/post_tf4_dl_f32_p12_diet_vsit": tf4_cases, + "orig_w64/post_tf_f32_p12_diet_vsit": control_cases} + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--run", action="store_true") + ap.add_argument("--seeds", type=int, default=2) + ap.add_argument("--kernels", default=",".join(KINDS)) + ap.add_argument("--json") + a = ap.parse_args(argv) + if not a.run: + return 0 + kernels = [k for k in a.kernels.split(",") if k] + res = OPS.run(a.build, a.rocm, tuple(range(89, 89 + a.seeds)), kernels, kinds=KINDS) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + print({k: v for k, v in res.items() if k not in ("cases", "replay_tail")}) + return 0 if res["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_d3d12_ops_check.py b/clean/l/runner/l_d3d12_ops_check.py index 769a2ab2..f479bc35 100644 --- a/clean/l/runner/l_d3d12_ops_check.py +++ b/clean/l/runner/l_d3d12_ops_check.py @@ -281,7 +281,8 @@ def hip_run(g, mod, case): elif a[0] == "bytes": args.append((ctypes.c_ubyte * len(a[1])).from_buffer_copy(a[1])) try: - h.launch(h.get_function(mod, case.get("hip_kernel", case["kernel"])), case["grid"], case["block"], args) + h.launch(h.get_function(mod, case.get("hip_kernel", case["kernel"])), case["grid"], + case.get("hip_block", case["block"]), args) h.sync() return [bytes(h.download(p, n)) for p, n in outs] finally: @@ -310,7 +311,9 @@ def replay_lines(name, case, cdir, grid=None, prefix=""): return lines + [""] -def run(build, rocm, seeds, kernels): +def run(build, rocm, seeds, kernels, kinds=None): + """kinds: {kernel: case generator} (default this file's KINDS and CONTROL_KINDS). A case may name its HIP kernel + (hip_kernel) and HIP block size (hip_block) when the D3D12 port has none of its own.""" g = C32.Gpu(build, rocm) mods = {} @@ -320,7 +323,7 @@ def module(co): mods[co] = g.hip.load_module(f.read()) return mods[co] - kinds = dict(KINDS, **CONTROL_KINDS) + kinds = kinds or dict(KINDS, **CONTROL_KINDS) cdir = os.path.join(build, "replay-ops") os.makedirs(cdir, exist_ok=True) lines, hip_out, neg = [], {}, None From d0c92a5e196f570e49e44d122a2b98de9f7d7034 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:27:23 -0500 Subject: [PATCH 2/2] test_post_tf_geometry: the tf4 declarations by name set (one per ported build) The analytic-filter ports declare the same kernel names as the original filter's with co=post_w64.co; every declaration line is still checked for the 256 lanes and the cap. Refs #89 Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/tests/test_post_tf_geometry.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/clean/tests/test_post_tf_geometry.py b/clean/tests/test_post_tf_geometry.py index a7af950f..4f1b2355 100644 --- a/clean/tests/test_post_tf_geometry.py +++ b/clean/tests/test_post_tf_geometry.py @@ -161,7 +161,9 @@ def test_nw4_takes_every_ratio_from_1_to_1_5(self): def test_nw4_declarations_agree(self): """runtime.cpp's tf4_supported (cap, NW = 4 footprint), runner/post.py's TF4_DL_CAP and the four post_tf4_dl_* - kernels of dx12/kernels/post.hlsl (NW = 4, the same cap, 256 lanes) say the same.""" + kernels of dx12/kernels/post.hlsl (NW = 4, the same cap, 256 lanes) say the same. A name may be declared once per + build it ports (the original filter's post_w64__orig.co, the analytic filter's post_w64.co): every declaration + line is checked.""" import re clean = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) rt = open(os.path.join(clean, "runtime", "runtime.cpp"), encoding="utf-8").read() @@ -169,7 +171,7 @@ def test_nw4_declarations_agree(self): self.assertIn("((long long)(63.0 * rw / dw) + 4) * ((long long)(6.0 * rh / dh) + 5) <= kTf4Cap", rt) hlsl = open(os.path.join(clean, "dx12", "kernels", "post.hlsl"), encoding="utf-8").read() decl = re.findall(r"(?m)^// @wg-kernel (\S*post_tf4_dl\S*) .*$", hlsl) - self.assertEqual(sorted(decl), ["pmo/post_tf4_dl_f32_p12_mo", "pmo/post_tf4_dl_f32_p12_mo_direct", + self.assertEqual(sorted(set(decl)), ["pmo/post_tf4_dl_f32_p12_mo", "pmo/post_tf4_dl_f32_p12_mo_direct", "w64/post_tf4_dl_f32_p12_diet_vsit", "w64/post_tf4_dl_f32_p12_diet_vsit_direct"]) for line in re.findall(r"(?m)^// @wg-kernel \S*post_tf4_dl\S* .*$", hlsl): self.assertIn(" block=256 ", line)