diff --git a/clean/kernels/swin.hip b/clean/kernels/swin.hip index 7bf8c77a..27a4275f 100644 --- a/clean/kernels/swin.hip +++ b/clean/kernels/swin.hip @@ -483,7 +483,7 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8* const u32 tid = wg_lane_id(); const u32 lane = tid & 31u, t = lane & 15u, hh = lane >> 4; const int nwx = (W + ox + 3) >> 2, nwy = (H + oy + 3) >> 2, nwin = nwx * nwy; - const u32 wave_id = (WG_WAVE_SCALAR && ECL == 0) ? (u32)__builtin_amdgcn_readfirstlane((int)(tid >> 5)) : (tid >> 5); + const u32 wave_id = (WG_WAVE_SCALAR && ECL == 0) ? (u32)wg_uniform((int)(tid >> 5)) : (tid >> 5); int win0 = (int)(wg_block_id() * (THREADS / WG_WAVE) + wave_id) * WPW; static_assert(MCO == 0 && ECL == 0 || (THREADS == 256 && WPW == 1), "fused merge / expand: 8 windows per workgroup"); static_assert(ECL == 0 || 128 * C <= MAXU, "fused expand: the 8 windows' expanded tokens must fit one weight buffer"); @@ -501,7 +501,9 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8* if constexpr (RESIDENT) { if (win0 >= nwin) break; } // re-read the header vectors (norm gains, biases) per window: otherwise they are hoisted out of the window loop and // held in registers (235 VGPRs instead of about 165, half the occupancy) +#ifndef WG_EMU // the "s" register constraint is AMDGCN only if constexpr (RESIDENT) __asm__ volatile("" : "+s"(w)); +#endif // Gather: token t of the window, mirrored at the borders. int dst[WPW]; diff --git a/clean/tests/check_no_dev_diagnostics.py b/clean/tests/check_no_dev_diagnostics.py index a6a468af..30b01605 100644 --- a/clean/tests/check_no_dev_diagnostics.py +++ b/clean/tests/check_no_dev_diagnostics.py @@ -75,8 +75,14 @@ def dev_only_strings_in(data): return [t.decode("ascii") for t in DEV_ONLY_STRINGS if t in data] +def _base(name): + """Last component of a payload path: the packager writes Windows paths (backslash), which os.path.basename + does not split on Linux.""" + return name.replace("\\", "/").rsplit("/", 1)[-1] + + def is_runtime_name(name): - n = os.path.basename(name).lower() + n = _base(name).lower() return n.startswith("wronggpu_runtime") and n.endswith(".dll") @@ -108,7 +114,7 @@ def check_runtime_with_manifest(name, data, manifest_bytes): if not ok: return False, text if manifest_bytes is None: - return False, "no build manifest (%s.build.json next to the library): build it with clean/runner/build.py" % os.path.splitext(os.path.basename(name))[0] + return False, "no build manifest (%s.build.json next to the library): build it with clean/runner/build.py" % os.path.splitext(_base(name))[0] try: m = json.loads(manifest_bytes.decode("utf-8")) assert isinstance(m, dict) @@ -116,7 +122,7 @@ def check_runtime_with_manifest(name, data, manifest_bytes): return False, "the build manifest is not readable JSON" if m.get("format") != 1: return False, "the build manifest has an unknown format" - if str(m.get("file", "")).lower() != os.path.basename(name).lower(): + if str(m.get("file", "")).lower() != _base(name).lower(): return False, "the build manifest is for another file (%s)" % m.get("file") if str(m.get("sha256", "")).lower() != hashlib.sha256(data).hexdigest(): return False, "the library does not match its build manifest (SHA-256): changed after its build, or another build's file" @@ -135,7 +141,7 @@ def dx12_manifest_problem(name, data): """None, or why a file named manifest.json that is a D3D12 kernel manifest (clean/dx12/build_dx12.py: format, kernels) must not ship: test-only (or older than the flag), or an entry built past the softmax self-check reference, with extra defines, or without provenance.""" - if os.path.basename(name).lower() != "manifest.json": + if _base(name).lower() != "manifest.json": return None try: doc = json.loads(data.decode("utf-8")) @@ -160,7 +166,7 @@ def check_artifact(name, data): if is_runtime_name(name): ok, text = check_bytes(data) return ok, text, ok - if PAYLOAD_CHECK_NAMES.search(os.path.basename(name)): + if PAYLOAD_CHECK_NAMES.search(_base(name)): return False, "a check-only code object (mutant, probe or HIP twin) in a release payload", False why = dx12_manifest_problem(name, data) if why: diff --git a/clean/tests/test_model_wgcm.py b/clean/tests/test_model_wgcm.py index 37493543..896a527d 100644 --- a/clean/tests/test_model_wgcm.py +++ b/clean/tests/test_model_wgcm.py @@ -24,8 +24,8 @@ ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) CONV = os.path.join(ROOT, "clean", "converter") -CONV_SOURCES = ["sha256.cpp", "json.cpp", "numeric.cpp", "map.cpp", "locate.cpp", "convert.cpp", "output.cpp", - "pe_version.cpp", "main.cpp"] +CONV_SOURCES = ["sha256.cpp", "json.cpp", "numeric.cpp", "map.cpp", "locate.cpp", "lz4block.cpp", "convert.cpp", + "convert5.cpp", "ops5.cpp", "output.cpp", "pe_version.cpp", "api.cpp", "main.cpp"] _CASE = None diff --git a/clean/tests/test_post_emu.py b/clean/tests/test_post_emu.py index 6f22b263..f91f10e8 100644 --- a/clean/tests/test_post_emu.py +++ b/clean/tests/test_post_emu.py @@ -8,6 +8,7 @@ import numpy as np +from clean.reference import output_quality as OQ from clean.reference import post as P from clean.runner import post from clean.tests.test_kernels_emu import EmuTest @@ -144,7 +145,8 @@ class TestFilterEmu(EmuTest): SIZES = [(8, 6), (7, 5), (1, 1), (2, 3), (130, 37), (260, 70), (515, 33)] def check(self, disp, e, O, stage="filter"): - ref = P.output_filter(disp, e, O) + # quality9 is the separate Lanczos-3 profile (clean/reference/output_quality.py), not the default filter + ref = OQ.output_filter(disp, e, O) if stage.endswith("quality9") else P.output_filter(disp, e, O) got = self.runner.run(stage, b"", {"display": disp, "exposure": e, "output_size": O})["out"] self.assertEqual(got.shape, ref.shape) self.assertTrue(np.all(np.isfinite(got))) @@ -164,7 +166,8 @@ def test_random(self): outs[k] = self.check(disp, e, O, stage="filter" if k == post.KERNEL_FILTER else f"filter:{k}") with self.subTest(O=O, variants="identical"): for k, v in outs.items(): - np.testing.assert_array_equal(v, outs[post.KERNEL_FILTER], err_msg=k) + if not k.endswith("quality9"): + np.testing.assert_array_equal(v, outs[post.KERNEL_FILTER], err_msg=k) def test_impulse_and_edges(self): O = (40, 34) diff --git a/clean/tests/test_post_fused3_emu.py b/clean/tests/test_post_fused3_emu.py index ae897397..d9c787d2 100644 --- a/clean/tests/test_post_fused3_emu.py +++ b/clean/tests/test_post_fused3_emu.py @@ -56,7 +56,7 @@ def test_exact_variants(self): g = PS.run_fused(self.runner, w, x, k) np.testing.assert_array_equal(g["history"], a["history"]) np.testing.assert_array_equal(g["reduced"], a["reduced"]) - np.testing.assert_array_equal(g["out"], out.astype(np.float16).astype(np.float32) if o16 else out) + np.testing.assert_array_equal(g["out"], np.clip(out, -65504.0, 65504.0).astype(np.float16).astype(np.float32) if o16 else out) checked += 1 self.assertGreaterEqual(checked, 2 * 8) diff --git a/clean/tests/test_post_fused_emu.py b/clean/tests/test_post_fused_emu.py index 5f60c460..22342c3c 100644 --- a/clean/tests/test_post_fused_emu.py +++ b/clean/tests/test_post_fused_emu.py @@ -43,7 +43,7 @@ def test_identical_to_two_kernel_path(self): g = PS.run_fused(self.runner, w, x, k) np.testing.assert_array_equal(g["history"], a["history"]) np.testing.assert_array_equal(g["reduced"], a["reduced"]) - np.testing.assert_array_equal(g["out"], out.astype(np.float16).astype(np.float32) if o16 else out) + np.testing.assert_array_equal(g["out"], np.clip(out, -65504.0, 65504.0).astype(np.float16).astype(np.float32) if o16 else out) checked += 1 self.assertGreaterEqual(checked, 4 * 9) diff --git a/clean/tests/test_post_out_emu.py b/clean/tests/test_post_out_emu.py index 380a708f..155b4fbd 100644 --- a/clean/tests/test_post_out_emu.py +++ b/clean/tests/test_post_out_emu.py @@ -52,7 +52,7 @@ def test_decode_matches_reference(self): e = 1.7 got = PS.run_plan(self.runner, PS.prepare_decode(self.runner.dev, {"display": disp, "exposure": e}))["out"] q = np.clip(disp.astype(np.float64), 1e-4, 3.0) - want = ((10.0 ** (q / 0.36) - 1.0) / 100.0 / e).astype(np.float16) + want = np.clip((10.0 ** (q / 0.36) - 1.0) / 100.0 / e, -65504.0, 65504.0).astype(np.float16) self.assertLessEqual(int(PE.f16_steps(got, want.astype(np.float32)).max()), 1) def test_against_reference(self): diff --git a/clean/tests/test_post_recon2_emu.py b/clean/tests/test_post_recon2_emu.py index ff189f2d..f2ecc33b 100644 --- a/clean/tests/test_post_recon2_emu.py +++ b/clean/tests/test_post_recon2_emu.py @@ -102,5 +102,5 @@ def test_filter_o16(self): plan["decoders"]["out"] = lambda raw, shape: np.frombuffer(raw, np.float16).reshape( shape[0], shape[1], 4).astype(np.float32) got = PS.run_plan(self.runner, plan)["out"] - np.testing.assert_array_equal(got[..., :3], base.astype(np.float16).astype(np.float32)) + np.testing.assert_array_equal(got[..., :3], np.clip(base, -65504.0, 65504.0).astype(np.float16).astype(np.float32)) np.testing.assert_array_equal(got[..., 3], 1.0) diff --git a/clean/tests/test_runtime.py b/clean/tests/test_runtime.py index d92b8b6a..25984d3a 100644 --- a/clean/tests/test_runtime.py +++ b/clean/tests/test_runtime.py @@ -292,7 +292,7 @@ class RuntimeFusedPostTest(EmuTest): def test_fused_post_whole_frame(self): """post_fused3_h6 (one launch for reconstruction + filter, 6-byte history) gives exactly the frames of the default post_recon2_r + post_filter_c1_s4 path (output, history values, reduced history), also with pre-exposure - != 1; post_fused3_h6_o16 gives the same output rounded to FP16; the 4-byte history variant runs; RGBA16F output + != 1; post_fused3_h6_o16 gives the same output in saturating FP16 storage; the 4-byte history variant runs; RGBA16F output is refused with pre-exposure != 1.""" mdir = model_dir(14) size = dict(max_size=(20, 18), out_size=(30, 27)) @@ -314,7 +314,7 @@ def test_fused_post_whole_frame(self): o16 = make_pipeline(mdir, kernels={"post": "post_fused3_h6_o16"}, **size) self.assertEqual(o16.kernels()["out"], "rgba16f") for f, w in zip(frs1, want1): - np.testing.assert_array_equal(o16.frame(f)["out"], w.astype(np.float16).astype(np.float32)) + np.testing.assert_array_equal(o16.frame(f)["out"], np.clip(w, -65504.0, 65504.0).astype(np.float16).astype(np.float32)) # two-kernel path with fewer bytes (clean/post 6834679): 6-byte history (exact), RGBA16F filter output r6 = make_pipeline(mdir, kernels={"post": "post_recon2_r_h6"}, **size) self.assertEqual((r6.kernels()["history_bytes"], r6.kernels()["post_wave"]), ("6", "32")) # emulator: wave32 @@ -325,7 +325,7 @@ def test_fused_post_whole_frame(self): f16 = make_pipeline(mdir, kernels={"filter": "post_filter_c1_s4_o16"}, **size) self.assertEqual(f16.kernels()["out"], "rgba16f") for f, w in zip(frs1, want1): - np.testing.assert_array_equal(f16.frame(f)["out"], w.astype(np.float16).astype(np.float32)) + np.testing.assert_array_equal(f16.frame(f)["out"], np.clip(w, -65504.0, 65504.0).astype(np.float16).astype(np.float32)) h4 = make_pipeline(mdir, kernels={"post": "post_fused3_h4_o16"}, **size) self.assertEqual(h4.kernels()["history_bytes"], "4") h4.frame(frs1[0]) # runs (an approximation: its values are not checked here) @@ -694,7 +694,7 @@ def test_static_scene_settles_like_spec(self): def test_direct_fp16_output(self): """D = O with pi = 1 and the runtime's own output buffer: post_recon2_r_out writes the output (RGBA16F) in one launch; equal to the post_recon2 + post_copy path (3 x f32, used with pi != 1 or a 3 x f32 caller buffer) - rounded to FP16, and the full history is the same.""" + in saturating FP16 storage (finite values beyond +-65504 clamp, no infinity), and the full history is the same.""" rng = np.random.default_rng(13) frs = [dict(frame_inputs(rng, 20, 18), pre_exposure=1.0) for _ in range(3)] p16 = make_pipeline(self.mdir, out_size=(30, 27), flags={"depth_inverted": False, "auto_exposure": True, @@ -707,7 +707,8 @@ def test_direct_fp16_output(self): b = pf.frame(f, debug=True) # not post_recon2_r: the post_copy path self.assertEqual(pf.kernels()["out"], "f32") np.testing.assert_array_equal(a["post"]["history"], b["post"]["history"], err_msg=f"frame {i}") - np.testing.assert_allclose(a["out"], b["out"].astype(np.float16).astype(np.float32), rtol=1e-3, atol=1e-6, + np.testing.assert_allclose(a["out"], np.clip(b["out"], -65504.0, 65504.0).astype(np.float16).astype(np.float32), + rtol=1e-3, atol=1e-6, err_msg=f"frame {i}") def test_default_is_spec(self): diff --git a/clean/tests/test_runtime_external.py b/clean/tests/test_runtime_external.py index e0f70ca2..79896b32 100644 --- a/clean/tests/test_runtime_external.py +++ b/clean/tests/test_runtime_external.py @@ -187,8 +187,8 @@ def test_raw_inputs_equal_dense(self): def check_half_output(self, flags, out_size, kernels=None, dense=False, pi=None, filter_last=None, bdir=None, post_last=None): - """Byte identity of the output copy: the RGBA16F rows equal the FP16 rounding (to nearest even) of the f32 - output of the default path, which the output pass writes into the game's RGBA16F texture.""" + """Byte identity of the output copy: the RGBA16F rows equal the saturating FP16 storage (round to nearest even, + finite values beyond +-65504 clamp) of the f32 output of the default path, which the output pass writes into the game's RGBA16F texture.""" def mk(kern): if bdir is None: return External(make_pipeline(self.mdir, out_size=out_size, flags=flags, check_packing=False, @@ -216,7 +216,7 @@ def mk(kern): exposure_mode=2, out_half_pitch=op) self.frame_io(b, x, io) rows = self.download(b, half, op * oh // 2, np.float16).reshape(oh, op // 2)[:, :4 * ow].reshape(oh, ow, 4) - np.testing.assert_array_equal(rows[..., :3].view(np.uint16), want.astype(np.float16).view(np.uint16), + np.testing.assert_array_equal(rows[..., :3].view(np.uint16), np.clip(want, -65504.0, 65504.0).astype(np.float16).view(np.uint16), err_msg=f"frame {t}") self.assertTrue(np.all(rows[..., 3] == 1)) k = dict(kv.split("=", 1) for kv in b.lib.wg_rt_kernels(b.rt).decode().split())