Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion clean/kernels/swin.hip
Original file line number Diff line number Diff line change
Expand Up @@ -483,7 +483,7 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8*
const u32 tid = wg_lane_id();
const u32 lane = tid & 31u, t = lane & 15u, hh = lane >> 4;
const int nwx = (W + ox + 3) >> 2, nwy = (H + oy + 3) >> 2, nwin = nwx * nwy;
const u32 wave_id = (WG_WAVE_SCALAR && ECL == 0) ? (u32)__builtin_amdgcn_readfirstlane((int)(tid >> 5)) : (tid >> 5);
const u32 wave_id = (WG_WAVE_SCALAR && ECL == 0) ? (u32)wg_uniform((int)(tid >> 5)) : (tid >> 5);
int win0 = (int)(wg_block_id() * (THREADS / WG_WAVE) + wave_id) * WPW;
static_assert(MCO == 0 && ECL == 0 || (THREADS == 256 && WPW == 1), "fused merge / expand: 8 windows per workgroup");
static_assert(ECL == 0 || 128 * C <= MAXU, "fused expand: the 8 windows' expanded tokens must fit one weight buffer");
Expand All @@ -501,7 +501,9 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8*
if constexpr (RESIDENT) { if (win0 >= nwin) break; }
// re-read the header vectors (norm gains, biases) per window: otherwise they are hoisted out of the window loop and
// held in registers (235 VGPRs instead of about 165, half the occupancy)
#ifndef WG_EMU // the "s" register constraint is AMDGCN only
if constexpr (RESIDENT) __asm__ volatile("" : "+s"(w));
#endif

// Gather: token t of the window, mirrored at the borders.
int dst[WPW];
Expand Down
16 changes: 11 additions & 5 deletions clean/tests/check_no_dev_diagnostics.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,14 @@ def dev_only_strings_in(data):
return [t.decode("ascii") for t in DEV_ONLY_STRINGS if t in data]


def _base(name):
"""Last component of a payload path: the packager writes Windows paths (backslash), which os.path.basename
does not split on Linux."""
return name.replace("\\", "/").rsplit("/", 1)[-1]


def is_runtime_name(name):
n = os.path.basename(name).lower()
n = _base(name).lower()
return n.startswith("wronggpu_runtime") and n.endswith(".dll")


Expand Down Expand Up @@ -108,15 +114,15 @@ def check_runtime_with_manifest(name, data, manifest_bytes):
if not ok:
return False, text
if manifest_bytes is None:
return False, "no build manifest (%s.build.json next to the library): build it with clean/runner/build.py" % os.path.splitext(os.path.basename(name))[0]
return False, "no build manifest (%s.build.json next to the library): build it with clean/runner/build.py" % os.path.splitext(_base(name))[0]
try:
m = json.loads(manifest_bytes.decode("utf-8"))
assert isinstance(m, dict)
except Exception:
return False, "the build manifest is not readable JSON"
if m.get("format") != 1:
return False, "the build manifest has an unknown format"
if str(m.get("file", "")).lower() != os.path.basename(name).lower():
if str(m.get("file", "")).lower() != _base(name).lower():
return False, "the build manifest is for another file (%s)" % m.get("file")
if str(m.get("sha256", "")).lower() != hashlib.sha256(data).hexdigest():
return False, "the library does not match its build manifest (SHA-256): changed after its build, or another build's file"
Expand All @@ -135,7 +141,7 @@ def dx12_manifest_problem(name, data):
"""None, or why a file named manifest.json that is a D3D12 kernel manifest (clean/dx12/build_dx12.py: format, kernels)
must not ship: test-only (or older than the flag), or an entry built past the softmax self-check reference, with
extra defines, or without provenance."""
if os.path.basename(name).lower() != "manifest.json":
if _base(name).lower() != "manifest.json":
return None
try:
doc = json.loads(data.decode("utf-8"))
Expand All @@ -160,7 +166,7 @@ def check_artifact(name, data):
if is_runtime_name(name):
ok, text = check_bytes(data)
return ok, text, ok
if PAYLOAD_CHECK_NAMES.search(os.path.basename(name)):
if PAYLOAD_CHECK_NAMES.search(_base(name)):
return False, "a check-only code object (mutant, probe or HIP twin) in a release payload", False
why = dx12_manifest_problem(name, data)
if why:
Expand Down
4 changes: 2 additions & 2 deletions clean/tests/test_model_wgcm.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@

ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
CONV = os.path.join(ROOT, "clean", "converter")
CONV_SOURCES = ["sha256.cpp", "json.cpp", "numeric.cpp", "map.cpp", "locate.cpp", "convert.cpp", "output.cpp",
"pe_version.cpp", "main.cpp"]
CONV_SOURCES = ["sha256.cpp", "json.cpp", "numeric.cpp", "map.cpp", "locate.cpp", "lz4block.cpp", "convert.cpp",
"convert5.cpp", "ops5.cpp", "output.cpp", "pe_version.cpp", "api.cpp", "main.cpp"]
_CASE = None


Expand Down
7 changes: 5 additions & 2 deletions clean/tests/test_post_emu.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

import numpy as np

from clean.reference import output_quality as OQ
from clean.reference import post as P
from clean.runner import post
from clean.tests.test_kernels_emu import EmuTest
Expand Down Expand Up @@ -144,7 +145,8 @@ class TestFilterEmu(EmuTest):
SIZES = [(8, 6), (7, 5), (1, 1), (2, 3), (130, 37), (260, 70), (515, 33)]

def check(self, disp, e, O, stage="filter"):
ref = P.output_filter(disp, e, O)
# quality9 is the separate Lanczos-3 profile (clean/reference/output_quality.py), not the default filter
ref = OQ.output_filter(disp, e, O) if stage.endswith("quality9") else P.output_filter(disp, e, O)
got = self.runner.run(stage, b"", {"display": disp, "exposure": e, "output_size": O})["out"]
self.assertEqual(got.shape, ref.shape)
self.assertTrue(np.all(np.isfinite(got)))
Expand All @@ -164,7 +166,8 @@ def test_random(self):
outs[k] = self.check(disp, e, O, stage="filter" if k == post.KERNEL_FILTER else f"filter:{k}")
with self.subTest(O=O, variants="identical"):
for k, v in outs.items():
np.testing.assert_array_equal(v, outs[post.KERNEL_FILTER], err_msg=k)
if not k.endswith("quality9"):
np.testing.assert_array_equal(v, outs[post.KERNEL_FILTER], err_msg=k)

def test_impulse_and_edges(self):
O = (40, 34)
Expand Down
2 changes: 1 addition & 1 deletion clean/tests/test_post_fused3_emu.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ def test_exact_variants(self):
g = PS.run_fused(self.runner, w, x, k)
np.testing.assert_array_equal(g["history"], a["history"])
np.testing.assert_array_equal(g["reduced"], a["reduced"])
np.testing.assert_array_equal(g["out"], out.astype(np.float16).astype(np.float32) if o16 else out)
np.testing.assert_array_equal(g["out"], np.clip(out, -65504.0, 65504.0).astype(np.float16).astype(np.float32) if o16 else out)
checked += 1
self.assertGreaterEqual(checked, 2 * 8)

Expand Down
2 changes: 1 addition & 1 deletion clean/tests/test_post_fused_emu.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def test_identical_to_two_kernel_path(self):
g = PS.run_fused(self.runner, w, x, k)
np.testing.assert_array_equal(g["history"], a["history"])
np.testing.assert_array_equal(g["reduced"], a["reduced"])
np.testing.assert_array_equal(g["out"], out.astype(np.float16).astype(np.float32) if o16 else out)
np.testing.assert_array_equal(g["out"], np.clip(out, -65504.0, 65504.0).astype(np.float16).astype(np.float32) if o16 else out)
checked += 1
self.assertGreaterEqual(checked, 4 * 9)

Expand Down
2 changes: 1 addition & 1 deletion clean/tests/test_post_out_emu.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def test_decode_matches_reference(self):
e = 1.7
got = PS.run_plan(self.runner, PS.prepare_decode(self.runner.dev, {"display": disp, "exposure": e}))["out"]
q = np.clip(disp.astype(np.float64), 1e-4, 3.0)
want = ((10.0 ** (q / 0.36) - 1.0) / 100.0 / e).astype(np.float16)
want = np.clip((10.0 ** (q / 0.36) - 1.0) / 100.0 / e, -65504.0, 65504.0).astype(np.float16)
self.assertLessEqual(int(PE.f16_steps(got, want.astype(np.float32)).max()), 1)

def test_against_reference(self):
Expand Down
2 changes: 1 addition & 1 deletion clean/tests/test_post_recon2_emu.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,5 +102,5 @@ def test_filter_o16(self):
plan["decoders"]["out"] = lambda raw, shape: np.frombuffer(raw, np.float16).reshape(
shape[0], shape[1], 4).astype(np.float32)
got = PS.run_plan(self.runner, plan)["out"]
np.testing.assert_array_equal(got[..., :3], base.astype(np.float16).astype(np.float32))
np.testing.assert_array_equal(got[..., :3], np.clip(base, -65504.0, 65504.0).astype(np.float16).astype(np.float32))
np.testing.assert_array_equal(got[..., 3], 1.0)
11 changes: 6 additions & 5 deletions clean/tests/test_runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ class RuntimeFusedPostTest(EmuTest):
def test_fused_post_whole_frame(self):
"""post_fused3_h6 (one launch for reconstruction + filter, 6-byte history) gives exactly the frames of the
default post_recon2_r + post_filter_c1_s4 path (output, history values, reduced history), also with pre-exposure
!= 1; post_fused3_h6_o16 gives the same output rounded to FP16; the 4-byte history variant runs; RGBA16F output
!= 1; post_fused3_h6_o16 gives the same output in saturating FP16 storage; the 4-byte history variant runs; RGBA16F output
is refused with pre-exposure != 1."""
mdir = model_dir(14)
size = dict(max_size=(20, 18), out_size=(30, 27))
Expand All @@ -314,7 +314,7 @@ def test_fused_post_whole_frame(self):
o16 = make_pipeline(mdir, kernels={"post": "post_fused3_h6_o16"}, **size)
self.assertEqual(o16.kernels()["out"], "rgba16f")
for f, w in zip(frs1, want1):
np.testing.assert_array_equal(o16.frame(f)["out"], w.astype(np.float16).astype(np.float32))
np.testing.assert_array_equal(o16.frame(f)["out"], np.clip(w, -65504.0, 65504.0).astype(np.float16).astype(np.float32))
# two-kernel path with fewer bytes (clean/post 6834679): 6-byte history (exact), RGBA16F filter output
r6 = make_pipeline(mdir, kernels={"post": "post_recon2_r_h6"}, **size)
self.assertEqual((r6.kernels()["history_bytes"], r6.kernels()["post_wave"]), ("6", "32")) # emulator: wave32
Expand All @@ -325,7 +325,7 @@ def test_fused_post_whole_frame(self):
f16 = make_pipeline(mdir, kernels={"filter": "post_filter_c1_s4_o16"}, **size)
self.assertEqual(f16.kernels()["out"], "rgba16f")
for f, w in zip(frs1, want1):
np.testing.assert_array_equal(f16.frame(f)["out"], w.astype(np.float16).astype(np.float32))
np.testing.assert_array_equal(f16.frame(f)["out"], np.clip(w, -65504.0, 65504.0).astype(np.float16).astype(np.float32))
h4 = make_pipeline(mdir, kernels={"post": "post_fused3_h4_o16"}, **size)
self.assertEqual(h4.kernels()["history_bytes"], "4")
h4.frame(frs1[0]) # runs (an approximation: its values are not checked here)
Expand Down Expand Up @@ -694,7 +694,7 @@ def test_static_scene_settles_like_spec(self):
def test_direct_fp16_output(self):
"""D = O with pi = 1 and the runtime's own output buffer: post_recon2_r_out writes the output (RGBA16F) in one
launch; equal to the post_recon2 + post_copy path (3 x f32, used with pi != 1 or a 3 x f32 caller buffer)
rounded to FP16, and the full history is the same."""
in saturating FP16 storage (finite values beyond +-65504 clamp, no infinity), and the full history is the same."""
rng = np.random.default_rng(13)
frs = [dict(frame_inputs(rng, 20, 18), pre_exposure=1.0) for _ in range(3)]
p16 = make_pipeline(self.mdir, out_size=(30, 27), flags={"depth_inverted": False, "auto_exposure": True,
Expand All @@ -707,7 +707,8 @@ def test_direct_fp16_output(self):
b = pf.frame(f, debug=True) # not post_recon2_r: the post_copy path
self.assertEqual(pf.kernels()["out"], "f32")
np.testing.assert_array_equal(a["post"]["history"], b["post"]["history"], err_msg=f"frame {i}")
np.testing.assert_allclose(a["out"], b["out"].astype(np.float16).astype(np.float32), rtol=1e-3, atol=1e-6,
np.testing.assert_allclose(a["out"], np.clip(b["out"], -65504.0, 65504.0).astype(np.float16).astype(np.float32),
rtol=1e-3, atol=1e-6,
err_msg=f"frame {i}")

def test_default_is_spec(self):
Expand Down
6 changes: 3 additions & 3 deletions clean/tests/test_runtime_external.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,8 +187,8 @@ def test_raw_inputs_equal_dense(self):

def check_half_output(self, flags, out_size, kernels=None, dense=False, pi=None, filter_last=None, bdir=None,
post_last=None):
"""Byte identity of the output copy: the RGBA16F rows equal the FP16 rounding (to nearest even) of the f32
output of the default path, which the output pass writes into the game's RGBA16F texture."""
"""Byte identity of the output copy: the RGBA16F rows equal the saturating FP16 storage (round to nearest even,
finite values beyond +-65504 clamp) of the f32 output of the default path, which the output pass writes into the game's RGBA16F texture."""
def mk(kern):
if bdir is None:
return External(make_pipeline(self.mdir, out_size=out_size, flags=flags, check_packing=False,
Expand Down Expand Up @@ -216,7 +216,7 @@ def mk(kern):
exposure_mode=2, out_half_pitch=op)
self.frame_io(b, x, io)
rows = self.download(b, half, op * oh // 2, np.float16).reshape(oh, op // 2)[:, :4 * ow].reshape(oh, ow, 4)
np.testing.assert_array_equal(rows[..., :3].view(np.uint16), want.astype(np.float16).view(np.uint16),
np.testing.assert_array_equal(rows[..., :3].view(np.uint16), np.clip(want, -65504.0, 65504.0).astype(np.float16).view(np.uint16),
err_msg=f"frame {t}")
self.assertTrue(np.all(rows[..., 3] == 1))
k = dict(kv.split("=", 1) for kv in b.lib.wg_rt_kernels(b.rt).decode().split())
Expand Down
Loading