Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions clean/dx12/kernels/fused_swin.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@
// 32 tokens per workgroup; ported to D3D12 they measured +41 / +0 / +8 us per frame: the per-workgroup weight traffic
// and, for c64_mg, 134 VGPRs = one workgroup less per CU.)
//
// @wg-kernel swin_block_c96_pm_nc co=swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_PM=1
// @wg-kernel swin_block_c96_pm_rev_nc co=swin.co entry=swin_block_c96_pm_nc block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_PM=1,FZ_REV=1
// @wg-kernel swin_block_c96_pm_nc co=swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_PM=1,SW_GELU_LATE0=3
// @wg-kernel swin_block_c96_pm_rev_nc co=swin.co entry=swin_block_c96_pm_nc block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_PM=1,FZ_REV=1,SW_GELU_LATE0=3
// @wg-kernel swin_block_c96_add_xe_rev_nc co=swin.co entry=swin_block_c96_add_xe_nc block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=1,SW_SAFE=1,SW_ECL=128,SW_XE=1,FZ_REV=1,SW_HEAD_PAIRS=0
// @wg-kernel swin_block_c96_add_xe_nc co=swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=1,SW_SAFE=1,SW_ECL=128,SW_XE=1,SW_HEAD_PAIRS=0
// @wg-kernel swin_block_c96_add_xe_xo_nc co=swin.co entry=swin_block_c96_add_xe_nc block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p,p,p,u,u,u defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=1,SW_SAFE=1,SW_ECL=128,SW_XE=1,FZ_XO=1,SW_HEAD_PAIRS=0
Expand Down
9 changes: 7 additions & 2 deletions clean/dx12/kernels/fused_swin_gen.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,11 @@
# the pairs and the fused expand epilogue they took 215 VGPRs = one 256-lane workgroup per CU; without, 192 = two
# (1440p Q -38 us per frame, 4K Perf -48, 1080p Q -21; same bytes). build_dx12.py's VGPR check (vgpr_limits.json) holds
# them at 192.
# The two pm kernels build with SW_GELU_LATE0=3 (every group's GELU table loads after W1 of the next group, as
# swin_c64w2.hlsl): with swin_block.hlsl's c96 default (0) RADV's compiler (Mesa ACO, the players' Proton route) needs
# 194 registers before scheduling, 2 over the 192 that keep two 256-lane workgroups per CU, so it schedules for one
# workgroup and ends at 256 VGPRs; with 3 it stays at 192 (RX 9070 XT, 1440p Q: swin_block_c96_pm_rev_nc 260 -> 194 us,
# same bytes).
HDR = """\
// GENERATED by fused_swin_gen.py from swin_block.hlsl: do not edit (edit swin_block.hlsl or the generator).
// FUSION track: Swin blocks of the 96-channel stages with the neighbouring patch merge / expand fused in (swin_block.hlsl
Expand Down Expand Up @@ -52,8 +57,8 @@
// 32 tokens per workgroup; ported to D3D12 they measured +41 / +0 / +8 us per frame: the per-workgroup weight traffic
// and, for c64_mg, 134 VGPRs = one workgroup less per CU.)
//
// @wg-kernel swin_block_c96_pm_nc co=swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_PM=1
// @wg-kernel swin_block_c96_pm_rev_nc co=swin.co entry=swin_block_c96_pm_nc block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_PM=1,FZ_REV=1
// @wg-kernel swin_block_c96_pm_nc co=swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_PM=1,SW_GELU_LATE0=3
// @wg-kernel swin_block_c96_pm_rev_nc co=swin.co entry=swin_block_c96_pm_nc block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_PM=1,FZ_REV=1,SW_GELU_LATE0=3
// @wg-kernel swin_block_c96_add_xe_rev_nc co=swin.co entry=swin_block_c96_add_xe_nc block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=1,SW_SAFE=1,SW_ECL=128,SW_XE=1,FZ_REV=1,SW_HEAD_PAIRS=0
// @wg-kernel swin_block_c96_add_xe_nc co=swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=1,SW_SAFE=1,SW_ECL=128,SW_XE=1,SW_HEAD_PAIRS=0
// @wg-kernel swin_block_c96_add_xe_xo_nc co=swin.co entry=swin_block_c96_add_xe_nc block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p,p,p,u,u,u defines=SW_C=96,SW_HEADS=4,SW_MLP=384,SW_ADD=1,SW_SAFE=1,SW_ECL=128,SW_XE=1,FZ_XO=1,SW_HEAD_PAIRS=0
Expand Down
19 changes: 13 additions & 6 deletions clean/dx12/kernels/post.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,13 @@ TexF tex_features(uint dw) // r2_features of the texel at dword dw
return f;
}

// 8-byte arena loads with their alignment made visible to the driver: RWByteAddressBuffer.Load2 / Load4 at a computed
// offset is known to be 4-byte aligned only, so each became two / four global_load_b32. The history (hist_prev) and the
// A8 features are the runtime's own allocations (256-byte aligned) and every offset formed on them here is a multiple of
// 8 bytes, so clearing the address's low 3 bits changes no address; the driver then issues global_load_b64.
uint2 ld64_a8(uint a) { return wg_mem.Load2(a & ~7u); }
uint4 ld128_a8(uint a) { return wg_mem.Load4(a & ~7u); }

// Raw inputs of one staged render pixel (TfRaw)
struct TfRaw { float4 c; uint d0, d1, c01, m0, m1, m2; };

Expand All @@ -300,7 +307,7 @@ TfRaw tf_fetch(int idx, int col, int row PD_P)
#else
t.c = wg_ld4f(color, ui * 16u);
#endif
uint2 d = wg_ld64(A8, ui * 8u);
uint2 d = ld64_a8(A8 + ui * 8u);
t.d0 = d.x;
t.d1 = d.y;
t.c01 = wg_ld32(C4, ui * 4u);
Expand Down Expand Up @@ -468,18 +475,18 @@ float3 history_cr_span(float qx, float qy)
b0 += 8u;
#endif
const uint b1 = b0 + pitch, b2 = b0 + 2u * pitch, b3 = b0 + 3u * pitch;
const uint4 a0 = wg_mem.Load4(b0 + 8u), a10 = wg_mem.Load4(b1), a11 = wg_mem.Load4(b1 + 16u);
const uint4 a20 = wg_mem.Load4(b2), a21 = wg_mem.Load4(b2 + 16u), a3 = wg_mem.Load4(b3 + 8u);
const uint4 a0 = ld128_a8(b0 + 8u), a10 = ld128_a8(b1), a11 = ld128_a8(b1 + 16u);
const uint4 a20 = ld128_a8(b2), a21 = ld128_a8(b2 + 16u), a3 = ld128_a8(b3 + 8u);
return cr_sum(a0.xy, a0.zw, a10.xy, a10.zw, a11.xy, a11.zw, a20.xy, a20.zw, a21.xy, a21.zw, a3.xy, a3.zw,
cr_weights(tx - fx0), cr_weights(ty - fy0));
}
const uint ro0 = mul24((uint)clampi(iy - 1, 0, P_dh - 1), pitch), ro1 = mul24((uint)clampi(iy, 0, P_dh - 1), pitch);
const uint ro2 = mul24((uint)clampi(iy + 1, 0, P_dh - 1), pitch), ro3 = mul24((uint)clampi(iy + 2, 0, P_dh - 1), pitch);
const uint c0 = hist_prev + (uint)clampi(ix - 1, 0, P_dw - 1) * 8u, c1 = hist_prev + (uint)clampi(ix, 0, P_dw - 1) * 8u;
const uint c2 = hist_prev + (uint)clampi(ix + 1, 0, P_dw - 1) * 8u, c3 = hist_prev + (uint)clampi(ix + 2, 0, P_dw - 1) * 8u;
return cr_sum(wg_mem.Load2(ro0 + c1), wg_mem.Load2(ro0 + c2), wg_mem.Load2(ro1 + c0), wg_mem.Load2(ro1 + c1),
wg_mem.Load2(ro1 + c2), wg_mem.Load2(ro1 + c3), wg_mem.Load2(ro2 + c0), wg_mem.Load2(ro2 + c1),
wg_mem.Load2(ro2 + c2), wg_mem.Load2(ro2 + c3), wg_mem.Load2(ro3 + c1), wg_mem.Load2(ro3 + c2),
return cr_sum(ld64_a8(ro0 + c1), ld64_a8(ro0 + c2), ld64_a8(ro1 + c0), ld64_a8(ro1 + c1),
ld64_a8(ro1 + c2), ld64_a8(ro1 + c3), ld64_a8(ro2 + c0), ld64_a8(ro2 + c1),
ld64_a8(ro2 + c2), ld64_a8(ro2 + c3), ld64_a8(ro3 + c1), ld64_a8(ro3 + c2),
cr_weights(tx - fx0), cr_weights(ty - fy0));
}

Expand Down
19 changes: 19 additions & 0 deletions clean/runtime/post_tf_align.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
// SPDX-License-Identifier: MIT
// Developer builds (WG_DEV_DIAGNOSTICS): the 8-byte alignment premise of the D3D12 post_tf ports. WrongGPU's own code.
// dx12/kernels/post.hlsl reads the history (hist_prev) and the A8 features with 8-byte loads whose address has its low 3
// bits cleared (ld64_a8 / ld128_a8), which is the requested address only when both pointers are 8-byte aligned. The
// runtime passes its own allocations (256-byte aligned); an interior pointer such as A + 4 or hist[cur] + 4 would make the
// kernel read 4 bytes earlier, silently. runtime.cpp checks both pointers before every post_tf launch (the ordinary and
// the motion-at-output form) and fails naming the pointer. Tested by clean/runtime/tests/test_post_tf_align.cpp.
#pragma once

#include <cstdint>

// The name of the first non-null pointer that is not a multiple of 8 ("A8": the A8 features, "hist_prev": the history the
// kernel samples), or nullptr when both are.
inline const char* post_tf_misaligned(uint64_t a8, uint64_t hist_prev) {
const struct { uint64_t v; const char* name; } f[] = {{a8, "A8"}, {hist_prev, "hist_prev"}};
for (const auto& e : f)
if (e.v & 7u) return e.name;
return nullptr;
}
13 changes: 13 additions & 0 deletions clean/runtime/runtime.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@
#include "model_select.h"
#include "runtime_core.h"
#include "wg_runtime.h"
#ifdef WG_DEV_DIAGNOSTICS
#include "post_tf_align.h"
#endif

namespace wgrt {

Expand Down Expand Up @@ -2951,6 +2954,11 @@ struct Runtime {
post_last = tf_mo4 ? "post_tf4_dl_f32_p12_mo" : hp > 0 ? "post_tf_p12_mo" : "post_tf_f32_p12_mo";
if (native_direct_io) post_last += "_direct"; // typed colour input and output texture (dx12/kernels/post.hlsl)
filter_last = "fused";
#ifdef WG_DEV_DIAGNOSTICS
if (const char* bad = post_tf_misaligned(A, hist[cur]))
return fail(std::string("post_tf: the ") + bad +
" pointer is not 8-byte aligned (the D3D12 port reads it with 8-byte loads, dx12/kernels/post.hlsl)");
#endif
if (!launch(("pmo/" + post_last).c_str(), cdiv(ow, 40) * nch, tf_mo4 ? 256 : 512,
with_taps({&cnet, &A, &B, &C, &hist[cur], &mnear[nxt], &mhi, &w, &hist[nxt], &reduced, &outp, &p,
&expo, &ow_i, &oh_i, &dec_scale, &p_h, &hpitch, &cp, &k1, &c2, &mop})))
Expand All @@ -2972,6 +2980,11 @@ struct Runtime {
float dec_scale = (float)(0.01 / (fr.exposure == 0.0f ? 1.0 : (double)fr.exposure)), p_h = hp > 0 ? pi : 1.0f;
post_last = tf_kernel;
filter_last = "fused";
#ifdef WG_DEV_DIAGNOSTICS
if (const char* bad = post_tf_misaligned(A, hist[cur]))
return fail(std::string("post_tf: the ") + bad +
" pointer is not 8-byte aligned (the D3D12 port reads it with 8-byte loads, dx12/kernels/post.hlsl)");
#endif
if (!launch((post_pre + tf_kernel).c_str(), cdiv(ow, 40) * nch, tf_block,
with_taps({&cnet, &A, &B, &C, &hist[cur], &mnear[nxt], &mcenter, &mfar, &w, &hist[nxt], &reduced,
&outp, &p, &expo, &ow_i, &oh_i, &dec_scale, &p_h, &hpitch, &cp, &k1, &c2})))
Expand Down
29 changes: 29 additions & 0 deletions clean/runtime/tests/test_post_tf_align.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
// SPDX-License-Identifier: MIT
// clean/runtime/post_tf_align.h: an A8 or history pointer that is not 8-byte aligned is refused naming it; null, 8- and
// 256-byte aligned pointers pass. Built and run by clean/tests/test_post_tf_align.py with the host compiler (no GPU).
#include <cstdio>
#include <cstring>

#include "clean/runtime/post_tf_align.h"

int main() {
int bad = 0;
const unsigned long long base = 0x100000ull; // an arena allocation (256-byte aligned)
const unsigned long long ok_vals[] = {0ull, base, base + 8u, base + 256u, base + 4096u};
for (unsigned long long a : ok_vals)
for (unsigned long long h : ok_vals)
if (const char* n = post_tf_misaligned(a, h)) {
std::printf("FAIL aligned pair %llx %llx refused as %s\n", a, h, n);
++bad;
}
const unsigned offs[] = {1u, 2u, 4u, 6u};
for (unsigned o : offs) {
const char* n = post_tf_misaligned(base + o, base);
if (n && std::strcmp(n, "A8") == 0) std::printf("A8 + %u refused naming it\n", o);
else { std::printf("FAIL A8 + %u\n", o); ++bad; }
n = post_tf_misaligned(base, base + o);
if (n && std::strcmp(n, "hist_prev") == 0) std::printf("hist_prev + %u refused naming it\n", o);
else { std::printf("FAIL hist_prev + %u\n", o); ++bad; }
}
return bad ? 1 : 0;
}
91 changes: 91 additions & 0 deletions clean/tests/test_post_tf_align.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
# SPDX-License-Identifier: MIT
"""Developer builds refuse a post_tf launch whose A8 or history pointer is not 8-byte aligned.

The D3D12 post_tf ports (clean/dx12/kernels/post.hlsl ld64_a8 / ld128_a8) clear the low 3 bits of every A8 and history
address, which is the requested address only when both pointers are 8-byte aligned (the runtime's own allocations are
256-byte aligned; an interior pointer such as A + 4 would be read 4 bytes early, silently). In builds with
WG_DEV_DIAGNOSTICS runtime.cpp checks both before every post_tf launch and fails naming the pointer. No GPU:
- clean/runtime/tests/test_post_tf_align.cpp, built with the host C++ compiler: each pointer at base + 1, 2, 4 or 6 is
refused naming it; null, 8- and 256-byte aligned pointers pass. Negative controls: the check removed from
post_tf_align.h (red), and a pointer left out of its list (red).
- source pins: one check before each post_tf launch (the ordinary and the motion-at-output form), with the pointers the
launch passes (A, hist[cur]), ending in fail(); the header and both checks inside WG_DEV_DIAGNOSTICS only, so a release
build has none of it.

Run: python -m unittest clean.tests.test_post_tf_align
"""
import os
import re
import shutil
import subprocess
import tempfile
import unittest

ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
RT = os.path.join(ROOT, "clean", "runtime", "runtime.cpp")
HDR = os.path.join(ROOT, "clean", "runtime", "post_tf_align.h")
SRC = os.path.join(ROOT, "clean", "runtime", "tests", "test_post_tf_align.cpp")
DEV = "#ifdef WG_DEV_DIAGNOSTICS"


def read(p):
with open(p, encoding="utf-8") as f:
return f.read()


class TestPostTfAlign(unittest.TestCase):
def _run(self, header_text=None):
cxx = os.environ.get("CXX") or shutil.which("clang++") or shutil.which("g++")
if not cxx:
self.skipTest("no host C++ compiler (CXX, clang++ or g++)")
with tempfile.TemporaryDirectory(dir=os.environ.get("WG_TEST_TMP")) as d:
inc = ROOT
if header_text is not None: # a mutated post_tf_align.h in a copy of the tree layout the source includes
inc = d
os.makedirs(os.path.join(d, "clean", "runtime"))
with open(os.path.join(d, "clean", "runtime", "post_tf_align.h"), "w") as f:
f.write(header_text)
exe = os.path.join(d, "t.exe")
r = subprocess.run([cxx, "-std=c++17", "-O1", "-I", inc, SRC, "-o", exe], capture_output=True, text=True)
self.assertEqual(r.returncode, 0, r.stderr)
return subprocess.run([exe], capture_output=True, text=True)

def test_misaligned_pointers_are_refused(self):
r = self._run()
self.assertEqual(r.returncode, 0, r.stdout)
self.assertEqual(r.stdout.count("refused naming it"), 2 * 4)

def test_negative_control_check_removed(self):
h = read(HDR)
self.assertIn("if (e.v & 7u) return e.name;", h)
r = self._run(h.replace("if (e.v & 7u) return e.name;", "(void)e;"))
self.assertNotEqual(r.returncode, 0, "the test passed with the check removed")

def test_negative_control_pointer_dropped(self):
h = read(HDR)
self.assertIn('{a8, "A8"}, ', h)
r = self._run(h.replace('{a8, "A8"}, ', ""))
self.assertNotEqual(r.returncode, 0, "the test passed with a pointer left out of the check")
self.assertIn("FAIL A8 + 4", r.stdout)

def test_runtime_wiring(self):
src = read(RT)
blocks = [m for m in re.finditer(re.escape(DEV) + r"\n(.*?)\n#endif", src, re.S)]
calls = [m.start() for m in re.finditer(r"post_tf_misaligned\(", src)]
self.assertEqual(len(calls), 2, "one check before each post_tf launch (ordinary, motion at output resolution)")
for c in calls:
self.assertTrue(any(b.start(1) <= c < b.end(1) for b in blocks), "a check outside WG_DEV_DIAGNOSTICS")
self.assertTrue(any('#include "post_tf_align.h"' in b.group(1) for b in blocks), "the header outside the macro")
self.assertNotIn('#include "post_tf_align.h"', re.sub(re.escape(DEV) + r"\n.*?\n#endif", "", src, flags=re.S))
for launch in (r'launch\(\("pmo/" \+ post_last\)', r"launch\(\(post_pre \+ tf_kernel\)"):
m = re.search(launch, src)
self.assertIsNotNone(m, launch)
before = src[max(0, m.start() - 600):m.start()]
self.assertIn("post_tf_misaligned(A, hist[cur])", before, "no check right before " + launch)
self.assertRegex(before, r"return fail\(.*\n?.*is not 8-byte aligned")
call = src[m.start():m.start() + 300]
self.assertIn("&cnet, &A, &B, &C, &hist[cur]", call, "the launch no longer passes A and hist[cur] as checked")


if __name__ == "__main__":
unittest.main()
Loading