diff --git a/clean/dx12/kernels/fused_swin.hlsl b/clean/dx12/kernels/fused_swin.hlsl index 0f24598b..0a0a605b 100644 --- a/clean/dx12/kernels/fused_swin.hlsl +++ b/clean/dx12/kernels/fused_swin.hlsl @@ -27,8 +27,8 @@ // 32 tokens per workgroup; ported to D3D12 they measured +41 / +0 / +8 us per frame: the per-workgroup weight traffic // and, for c64_mg, 134 VGPRs = one workgroup less per CU.) // -// @wg-kernel swin_block_c96_pm_nc co=swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_PM=1 -// @wg-kernel swin_block_c96_pm_rev_nc co=swin.co entry=swin_block_c96_pm_nc block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_PM=1,FZ_REV=1 +// @wg-kernel swin_block_c96_pm_nc co=swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_PM=1,SW_GELU_LATE0=3 +// @wg-kernel swin_block_c96_pm_rev_nc co=swin.co entry=swin_block_c96_pm_nc block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_PM=1,FZ_REV=1,SW_GELU_LATE0=3 // @wg-kernel swin_block_c96_add_xe_rev_nc co=swin.co entry=swin_block_c96_add_xe_nc block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=1,SW_SAFE=1,SW_ECL=128,SW_XE=1,FZ_REV=1,SW_HEAD_PAIRS=0 // @wg-kernel swin_block_c96_add_xe_nc co=swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=1,SW_SAFE=1,SW_ECL=128,SW_XE=1,SW_HEAD_PAIRS=0 // @wg-kernel swin_block_c96_add_xe_xo_nc co=swin.co entry=swin_block_c96_add_xe_nc block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p,p,p,u,u,u defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=1,SW_SAFE=1,SW_ECL=128,SW_XE=1,FZ_XO=1,SW_HEAD_PAIRS=0 diff --git a/clean/dx12/kernels/fused_swin_gen.py b/clean/dx12/kernels/fused_swin_gen.py index 202bde27..149b428a 100644 --- a/clean/dx12/kernels/fused_swin_gen.py +++ b/clean/dx12/kernels/fused_swin_gen.py @@ -22,6 +22,11 @@ # the pairs and the fused expand epilogue they took 215 VGPRs = one 256-lane workgroup per CU; without, 192 = two # (1440p Q -38 us per frame, 4K Perf -48, 1080p Q -21; same bytes). build_dx12.py's VGPR check (vgpr_limits.json) holds # them at 192. +# The two pm kernels build with SW_GELU_LATE0=3 (every group's GELU table loads after W1 of the next group, as +# swin_c64w2.hlsl): with swin_block.hlsl's c96 default (0) RADV's compiler (Mesa ACO, the players' Proton route) needs +# 194 registers before scheduling, 2 over the 192 that keep two 256-lane workgroups per CU, so it schedules for one +# workgroup and ends at 256 VGPRs; with 3 it stays at 192 (RX 9070 XT, 1440p Q: swin_block_c96_pm_rev_nc 260 -> 194 us, +# same bytes). HDR = """\ // GENERATED by fused_swin_gen.py from swin_block.hlsl: do not edit (edit swin_block.hlsl or the generator). // FUSION track: Swin blocks of the 96-channel stages with the neighbouring patch merge / expand fused in (swin_block.hlsl @@ -52,8 +57,8 @@ // 32 tokens per workgroup; ported to D3D12 they measured +41 / +0 / +8 us per frame: the per-workgroup weight traffic // and, for c64_mg, 134 VGPRs = one workgroup less per CU.) // -// @wg-kernel swin_block_c96_pm_nc co=swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_PM=1 -// @wg-kernel swin_block_c96_pm_rev_nc co=swin.co entry=swin_block_c96_pm_nc block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_PM=1,FZ_REV=1 +// @wg-kernel swin_block_c96_pm_nc co=swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_PM=1,SW_GELU_LATE0=3 +// @wg-kernel swin_block_c96_pm_rev_nc co=swin.co entry=swin_block_c96_pm_nc block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_PM=1,FZ_REV=1,SW_GELU_LATE0=3 // @wg-kernel swin_block_c96_add_xe_rev_nc co=swin.co entry=swin_block_c96_add_xe_nc block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=1,SW_SAFE=1,SW_ECL=128,SW_XE=1,FZ_REV=1,SW_HEAD_PAIRS=0 // @wg-kernel swin_block_c96_add_xe_nc co=swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=1,SW_SAFE=1,SW_ECL=128,SW_XE=1,SW_HEAD_PAIRS=0 // @wg-kernel swin_block_c96_add_xe_xo_nc co=swin.co entry=swin_block_c96_add_xe_nc block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p,p,p,u,u,u defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=1,SW_SAFE=1,SW_ECL=128,SW_XE=1,FZ_XO=1,SW_HEAD_PAIRS=0 diff --git a/clean/dx12/kernels/post.hlsl b/clean/dx12/kernels/post.hlsl index bc5bc7de..ef349ecd 100644 --- a/clean/dx12/kernels/post.hlsl +++ b/clean/dx12/kernels/post.hlsl @@ -286,6 +286,13 @@ TexF tex_features(uint dw) // r2_features of the texel at dword dw return f; } +// 8-byte arena loads with their alignment made visible to the driver: RWByteAddressBuffer.Load2 / Load4 at a computed +// offset is known to be 4-byte aligned only, so each became two / four global_load_b32. The history (hist_prev) and the +// A8 features are the runtime's own allocations (256-byte aligned) and every offset formed on them here is a multiple of +// 8 bytes, so clearing the address's low 3 bits changes no address; the driver then issues global_load_b64. +uint2 ld64_a8(uint a) { return wg_mem.Load2(a & ~7u); } +uint4 ld128_a8(uint a) { return wg_mem.Load4(a & ~7u); } + // Raw inputs of one staged render pixel (TfRaw) struct TfRaw { float4 c; uint d0, d1, c01, m0, m1, m2; }; @@ -300,7 +307,7 @@ TfRaw tf_fetch(int idx, int col, int row PD_P) #else t.c = wg_ld4f(color, ui * 16u); #endif - uint2 d = wg_ld64(A8, ui * 8u); + uint2 d = ld64_a8(A8 + ui * 8u); t.d0 = d.x; t.d1 = d.y; t.c01 = wg_ld32(C4, ui * 4u); @@ -468,8 +475,8 @@ float3 history_cr_span(float qx, float qy) b0 += 8u; #endif const uint b1 = b0 + pitch, b2 = b0 + 2u * pitch, b3 = b0 + 3u * pitch; - const uint4 a0 = wg_mem.Load4(b0 + 8u), a10 = wg_mem.Load4(b1), a11 = wg_mem.Load4(b1 + 16u); - const uint4 a20 = wg_mem.Load4(b2), a21 = wg_mem.Load4(b2 + 16u), a3 = wg_mem.Load4(b3 + 8u); + const uint4 a0 = ld128_a8(b0 + 8u), a10 = ld128_a8(b1), a11 = ld128_a8(b1 + 16u); + const uint4 a20 = ld128_a8(b2), a21 = ld128_a8(b2 + 16u), a3 = ld128_a8(b3 + 8u); return cr_sum(a0.xy, a0.zw, a10.xy, a10.zw, a11.xy, a11.zw, a20.xy, a20.zw, a21.xy, a21.zw, a3.xy, a3.zw, cr_weights(tx - fx0), cr_weights(ty - fy0)); } @@ -477,9 +484,9 @@ float3 history_cr_span(float qx, float qy) const uint ro2 = mul24((uint)clampi(iy + 1, 0, P_dh - 1), pitch), ro3 = mul24((uint)clampi(iy + 2, 0, P_dh - 1), pitch); const uint c0 = hist_prev + (uint)clampi(ix - 1, 0, P_dw - 1) * 8u, c1 = hist_prev + (uint)clampi(ix, 0, P_dw - 1) * 8u; const uint c2 = hist_prev + (uint)clampi(ix + 1, 0, P_dw - 1) * 8u, c3 = hist_prev + (uint)clampi(ix + 2, 0, P_dw - 1) * 8u; - return cr_sum(wg_mem.Load2(ro0 + c1), wg_mem.Load2(ro0 + c2), wg_mem.Load2(ro1 + c0), wg_mem.Load2(ro1 + c1), - wg_mem.Load2(ro1 + c2), wg_mem.Load2(ro1 + c3), wg_mem.Load2(ro2 + c0), wg_mem.Load2(ro2 + c1), - wg_mem.Load2(ro2 + c2), wg_mem.Load2(ro2 + c3), wg_mem.Load2(ro3 + c1), wg_mem.Load2(ro3 + c2), + return cr_sum(ld64_a8(ro0 + c1), ld64_a8(ro0 + c2), ld64_a8(ro1 + c0), ld64_a8(ro1 + c1), + ld64_a8(ro1 + c2), ld64_a8(ro1 + c3), ld64_a8(ro2 + c0), ld64_a8(ro2 + c1), + ld64_a8(ro2 + c2), ld64_a8(ro2 + c3), ld64_a8(ro3 + c1), ld64_a8(ro3 + c2), cr_weights(tx - fx0), cr_weights(ty - fy0)); } diff --git a/clean/runtime/post_tf_align.h b/clean/runtime/post_tf_align.h new file mode 100644 index 00000000..c30118ef --- /dev/null +++ b/clean/runtime/post_tf_align.h @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: MIT +// Developer builds (WG_DEV_DIAGNOSTICS): the 8-byte alignment premise of the D3D12 post_tf ports. WrongGPU's own code. +// dx12/kernels/post.hlsl reads the history (hist_prev) and the A8 features with 8-byte loads whose address has its low 3 +// bits cleared (ld64_a8 / ld128_a8), which is the requested address only when both pointers are 8-byte aligned. The +// runtime passes its own allocations (256-byte aligned); an interior pointer such as A + 4 or hist[cur] + 4 would make the +// kernel read 4 bytes earlier, silently. runtime.cpp checks both pointers before every post_tf launch (the ordinary and +// the motion-at-output form) and fails naming the pointer. Tested by clean/runtime/tests/test_post_tf_align.cpp. +#pragma once + +#include + +// The name of the first non-null pointer that is not a multiple of 8 ("A8": the A8 features, "hist_prev": the history the +// kernel samples), or nullptr when both are. +inline const char* post_tf_misaligned(uint64_t a8, uint64_t hist_prev) { + const struct { uint64_t v; const char* name; } f[] = {{a8, "A8"}, {hist_prev, "hist_prev"}}; + for (const auto& e : f) + if (e.v & 7u) return e.name; + return nullptr; +} diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index 95c6e460..c693e191 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -29,6 +29,9 @@ #include "model_select.h" #include "runtime_core.h" #include "wg_runtime.h" +#ifdef WG_DEV_DIAGNOSTICS +#include "post_tf_align.h" +#endif namespace wgrt { @@ -2951,6 +2954,11 @@ struct Runtime { post_last = tf_mo4 ? "post_tf4_dl_f32_p12_mo" : hp > 0 ? "post_tf_p12_mo" : "post_tf_f32_p12_mo"; if (native_direct_io) post_last += "_direct"; // typed colour input and output texture (dx12/kernels/post.hlsl) filter_last = "fused"; +#ifdef WG_DEV_DIAGNOSTICS + if (const char* bad = post_tf_misaligned(A, hist[cur])) + return fail(std::string("post_tf: the ") + bad + + " pointer is not 8-byte aligned (the D3D12 port reads it with 8-byte loads, dx12/kernels/post.hlsl)"); +#endif if (!launch(("pmo/" + post_last).c_str(), cdiv(ow, 40) * nch, tf_mo4 ? 256 : 512, with_taps({&cnet, &A, &B, &C, &hist[cur], &mnear[nxt], &mhi, &w, &hist[nxt], &reduced, &outp, &p, &expo, &ow_i, &oh_i, &dec_scale, &p_h, &hpitch, &cp, &k1, &c2, &mop}))) @@ -2972,6 +2980,11 @@ struct Runtime { float dec_scale = (float)(0.01 / (fr.exposure == 0.0f ? 1.0 : (double)fr.exposure)), p_h = hp > 0 ? pi : 1.0f; post_last = tf_kernel; filter_last = "fused"; +#ifdef WG_DEV_DIAGNOSTICS + if (const char* bad = post_tf_misaligned(A, hist[cur])) + return fail(std::string("post_tf: the ") + bad + + " pointer is not 8-byte aligned (the D3D12 port reads it with 8-byte loads, dx12/kernels/post.hlsl)"); +#endif if (!launch((post_pre + tf_kernel).c_str(), cdiv(ow, 40) * nch, tf_block, with_taps({&cnet, &A, &B, &C, &hist[cur], &mnear[nxt], &mcenter, &mfar, &w, &hist[nxt], &reduced, &outp, &p, &expo, &ow_i, &oh_i, &dec_scale, &p_h, &hpitch, &cp, &k1, &c2}))) diff --git a/clean/runtime/tests/test_post_tf_align.cpp b/clean/runtime/tests/test_post_tf_align.cpp new file mode 100644 index 00000000..88bf9b9c --- /dev/null +++ b/clean/runtime/tests/test_post_tf_align.cpp @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: MIT +// clean/runtime/post_tf_align.h: an A8 or history pointer that is not 8-byte aligned is refused naming it; null, 8- and +// 256-byte aligned pointers pass. Built and run by clean/tests/test_post_tf_align.py with the host compiler (no GPU). +#include +#include + +#include "clean/runtime/post_tf_align.h" + +int main() { + int bad = 0; + const unsigned long long base = 0x100000ull; // an arena allocation (256-byte aligned) + const unsigned long long ok_vals[] = {0ull, base, base + 8u, base + 256u, base + 4096u}; + for (unsigned long long a : ok_vals) + for (unsigned long long h : ok_vals) + if (const char* n = post_tf_misaligned(a, h)) { + std::printf("FAIL aligned pair %llx %llx refused as %s\n", a, h, n); + ++bad; + } + const unsigned offs[] = {1u, 2u, 4u, 6u}; + for (unsigned o : offs) { + const char* n = post_tf_misaligned(base + o, base); + if (n && std::strcmp(n, "A8") == 0) std::printf("A8 + %u refused naming it\n", o); + else { std::printf("FAIL A8 + %u\n", o); ++bad; } + n = post_tf_misaligned(base, base + o); + if (n && std::strcmp(n, "hist_prev") == 0) std::printf("hist_prev + %u refused naming it\n", o); + else { std::printf("FAIL hist_prev + %u\n", o); ++bad; } + } + return bad ? 1 : 0; +} diff --git a/clean/tests/test_post_tf_align.py b/clean/tests/test_post_tf_align.py new file mode 100644 index 00000000..eb527835 --- /dev/null +++ b/clean/tests/test_post_tf_align.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: MIT +"""Developer builds refuse a post_tf launch whose A8 or history pointer is not 8-byte aligned. + +The D3D12 post_tf ports (clean/dx12/kernels/post.hlsl ld64_a8 / ld128_a8) clear the low 3 bits of every A8 and history +address, which is the requested address only when both pointers are 8-byte aligned (the runtime's own allocations are +256-byte aligned; an interior pointer such as A + 4 would be read 4 bytes early, silently). In builds with +WG_DEV_DIAGNOSTICS runtime.cpp checks both before every post_tf launch and fails naming the pointer. No GPU: +- clean/runtime/tests/test_post_tf_align.cpp, built with the host C++ compiler: each pointer at base + 1, 2, 4 or 6 is + refused naming it; null, 8- and 256-byte aligned pointers pass. Negative controls: the check removed from + post_tf_align.h (red), and a pointer left out of its list (red). +- source pins: one check before each post_tf launch (the ordinary and the motion-at-output form), with the pointers the + launch passes (A, hist[cur]), ending in fail(); the header and both checks inside WG_DEV_DIAGNOSTICS only, so a release + build has none of it. + +Run: python -m unittest clean.tests.test_post_tf_align +""" +import os +import re +import shutil +import subprocess +import tempfile +import unittest + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +RT = os.path.join(ROOT, "clean", "runtime", "runtime.cpp") +HDR = os.path.join(ROOT, "clean", "runtime", "post_tf_align.h") +SRC = os.path.join(ROOT, "clean", "runtime", "tests", "test_post_tf_align.cpp") +DEV = "#ifdef WG_DEV_DIAGNOSTICS" + + +def read(p): + with open(p, encoding="utf-8") as f: + return f.read() + + +class TestPostTfAlign(unittest.TestCase): + def _run(self, header_text=None): + cxx = os.environ.get("CXX") or shutil.which("clang++") or shutil.which("g++") + if not cxx: + self.skipTest("no host C++ compiler (CXX, clang++ or g++)") + with tempfile.TemporaryDirectory(dir=os.environ.get("WG_TEST_TMP")) as d: + inc = ROOT + if header_text is not None: # a mutated post_tf_align.h in a copy of the tree layout the source includes + inc = d + os.makedirs(os.path.join(d, "clean", "runtime")) + with open(os.path.join(d, "clean", "runtime", "post_tf_align.h"), "w") as f: + f.write(header_text) + exe = os.path.join(d, "t.exe") + r = subprocess.run([cxx, "-std=c++17", "-O1", "-I", inc, SRC, "-o", exe], capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + return subprocess.run([exe], capture_output=True, text=True) + + def test_misaligned_pointers_are_refused(self): + r = self._run() + self.assertEqual(r.returncode, 0, r.stdout) + self.assertEqual(r.stdout.count("refused naming it"), 2 * 4) + + def test_negative_control_check_removed(self): + h = read(HDR) + self.assertIn("if (e.v & 7u) return e.name;", h) + r = self._run(h.replace("if (e.v & 7u) return e.name;", "(void)e;")) + self.assertNotEqual(r.returncode, 0, "the test passed with the check removed") + + def test_negative_control_pointer_dropped(self): + h = read(HDR) + self.assertIn('{a8, "A8"}, ', h) + r = self._run(h.replace('{a8, "A8"}, ', "")) + self.assertNotEqual(r.returncode, 0, "the test passed with a pointer left out of the check") + self.assertIn("FAIL A8 + 4", r.stdout) + + def test_runtime_wiring(self): + src = read(RT) + blocks = [m for m in re.finditer(re.escape(DEV) + r"\n(.*?)\n#endif", src, re.S)] + calls = [m.start() for m in re.finditer(r"post_tf_misaligned\(", src)] + self.assertEqual(len(calls), 2, "one check before each post_tf launch (ordinary, motion at output resolution)") + for c in calls: + self.assertTrue(any(b.start(1) <= c < b.end(1) for b in blocks), "a check outside WG_DEV_DIAGNOSTICS") + self.assertTrue(any('#include "post_tf_align.h"' in b.group(1) for b in blocks), "the header outside the macro") + self.assertNotIn('#include "post_tf_align.h"', re.sub(re.escape(DEV) + r"\n.*?\n#endif", "", src, flags=re.S)) + for launch in (r'launch\(\("pmo/" \+ post_last\)', r"launch\(\(post_pre \+ tf_kernel\)"): + m = re.search(launch, src) + self.assertIsNotNone(m, launch) + before = src[max(0, m.start() - 600):m.start()] + self.assertIn("post_tf_misaligned(A, hist[cur])", before, "no check right before " + launch) + self.assertRegex(before, r"return fail\(.*\n?.*is not 8-byte aligned") + call = src[m.start():m.start() + 300] + self.assertIn("&cnet, &A, &B, &C, &hist[cur]", call, "the launch no longer passes A and hist[cur] as checked") + + +if __name__ == "__main__": + unittest.main()