diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index 0ac9e2aa..0c5bbccb 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -358,6 +358,9 @@ def main(argv=None): ap.add_argument("--model-k", action="store_true", help="also build Model K's kernels (clean/k/dx12/kernels; off by default: Model M's and Model L's " "kernel sets unchanged)") + ap.add_argument("--define", action="append", default=[], metavar="NAME=VALUE", + help="an extra preprocessor define for every kernel built (test variants: negative-control mutants, " + "test counters; never for a shipped build)") ap.add_argument("--list", action="store_true", help="print the kernels a build with these options compiles (name and code object, JSON) and exit: " "the cheap check before a build (kernel_sets.json, tests/test_kernel_sets.py)") @@ -367,6 +370,11 @@ def main(argv=None): return 0 if not a.out: ap.error("--out is required") + extra_defs = {} + for kv in a.define: + if not re.fullmatch(r"[A-Z][A-Z0-9_]*=-?[0-9A-Za-z_]+", kv): + ap.error(f"--define {kv!r} (NAME=VALUE)") + extra_defs[kv.split("=", 1)[0]] = kv.split("=", 1)[1] dst = os.path.join(a.out, "dx12") os.makedirs(dst, exist_ok=True) asm = build_asm_tool(a.out) @@ -412,7 +420,7 @@ def main(argv=None): def build_one(job): src, d, out, k_source = job - defs = {"WG_BLOCK": d["group"], "WG_WAVE": d["wave"], **d["defines"]} + defs = {"WG_BLOCK": d["group"], "WG_WAVE": d["wave"], **d["defines"], **extra_defs} if d["group"] != d["block"]: defs["WG_LAUNCH_BLOCK"] = d["block"] info = {} diff --git a/clean/dx12/kernels/l_post.hlsl b/clean/dx12/kernels/l_post.hlsl index 93451d89..d214e775 100644 --- a/clean/dx12/kernels/l_post.hlsl +++ b/clean/dx12/kernels/l_post.hlsl @@ -81,6 +81,29 @@ static float g_e, g_ep; // exposures (0 replaced by 1) static float g_rp; // 1 / e_prev (IEEE division, round to nearest even) float opq(float x) { return asfloat(asuint(x) ^ g_z); } half opqh(half x) { return asfloat16((uint16_t)(asuint16(x) ^ (uint16_t)g_z)); } +float mul_nz(float a, float b) { return mad(a, b, asfloat(g_z | 0x80000000u)); } +#ifndef LP_MUT +#define LP_MUT 0 // 1, 2, 4 ... 9: negative controls of the byte checks (never shipped) +#endif +#ifndef LP_DP +#define LP_DP 1 +#endif +#ifndef LP_FMANZ +#define LP_FMANZ 1 +#endif +#ifndef LP_HSCOUNT +#define LP_HSCOUNT 0 // 1: test builds only (clean/l/runner/l_post_edge_check.py): the history fast path's wave counter +#endif +// h(a b) as its own rounding that the driver cannot fuse into a following FP16 add: fma(a, b, -0) with an opaque -0 +// (RN16(a b + -0) = RN16(a b) for every non-NaN pair, zeros included); LP_FMANZ 0: the product through the xor guard +half mulh(half a, half b) +{ +#if LP_FMANZ + return mad(a, b, asfloat16((uint16_t)(0x8000u | g_z))); +#else + return opqh(a * b); +#endif +} half h_bits(uint b) { return asfloat16((uint16_t)b); } uint bits_h(half h) { return (uint)asuint16(h); } @@ -109,8 +132,20 @@ float dec_m1_poly(float qc) float l_dec(float q) { const float qc = clampf(q, k1em4, 3.0f); +#if LP_DP + // Model M's wave-level skip (post.hlsl post_core_s): the exponential for every lane, the polynomial only when some + // active lane of the wave is below kDecPolyMax; per lane the same select as below, so the same value + float v = exp2(qc * kDecK) - 1.0f; + const bool sm = qc < kDecPolyMax; +#if LP_MUT == 6 + [branch] if (WaveActiveAllTrue(sm)) v = sm ? dec_m1_poly(qc) : v; // mutant 6: all lanes instead of any lane +#else + [branch] if (WaveActiveAnyTrue(sm)) v = sm ? dec_m1_poly(qc) : v; +#endif +#else const float v = qc < kDecPolyMax ? dec_m1_poly(qc) : exp2(qc * kDecK) - 1.0f; - return isnan(q) ? q : opq(v) * k0p01; +#endif + return isnan(q) ? q : mul_nz(v, k0p01); } // (_Float16)((float)k / 255.0f) for the code k = 0 ... 255 (an IEEE FP32 division, then round to nearest even: HIP's @@ -216,7 +251,11 @@ LGather gather_zero() float tap_weight(uint t[kTexDw], half uu, half vv, half uv) { const half kA = h_bits(t[4] >> 16), kB = h_bits(t[5]), kC = h_bits(t[5] >> 16); - const half ta = opqh(uu * kA), tb = opqh(vv * kB); +#if LP_MUT == 1 + const half ta = mulh(uu, kB), tb = mulh(vv, kB); // mutant 1: wrong factor +#else + const half ta = mulh(uu, kA), tb = mulh(vv, kB); +#endif const half s = tb + ta; const half eps = mad(uv, kC, s); const float w = exp2((float)opqh(eps)); @@ -251,6 +290,36 @@ float3 history_cr(float qx, float qy) const float4 wx = cr_weights(tx - fx0), wy = cr_weights(ty - fy0); const int ix = (int)fx0, iy = (int)fy0; const uint pitch = (uint)P_dw * 8u; + // interior fast path (Model M's history_cr_span): every lane's 4 x 4 footprint inside the image, so the clamps are + // the identity; six 16-byte span loads from one base instead of twelve 8-byte loads, the same texels in the same + // sums (same mad order). Footprint inside: 1 <= ix <= P_dw - 3 and 1 <= iy <= P_dh - 3, the unsigned tests valid only + // for an image of at least 4 x 4 (below, P_d* - 4 would wrap to a huge bound): smaller images take the clamped path. +#if LP_MUT == 8 // mutant 8: the predicate always false (the fast path never taken; same bytes, found by the counter) + const bool interior = false; +#elif LP_MUT == 9 // mutant 9: without the size guard (the span reads past a history narrower or shorter than 4) + const bool interior = and((uint)(ix - 1) <= (uint)(P_dw - 4), (uint)(iy - 1) <= (uint)(P_dh - 4)); +#else + const bool interior = and(and(P_dw >= 4, P_dh >= 4), + and((uint)(ix - 1) <= (uint)(P_dw - 4), (uint)(iy - 1) <= (uint)(P_dh - 4))); +#endif + [branch] if (WaveActiveAllTrue(interior)) { +#if LP_HSCOUNT // test builds only: count the waves taking the fast path, in the first dword after the full history + if (WaveIsFirstLane()) wg_atomic_add(hist_out, (uint)(P_dw * P_dh) * 8u, 1u); +#endif + const uint b0 = hist_prev + (uint)(ix - 1) * 8u + (uint)(iy - 1) * pitch; + const uint b1 = b0 + pitch, b2 = b0 + 2u * pitch, b3 = b0 + 3u * pitch; + const uint4 a0 = wg_mem.Load4(b0 + 8u), a10 = wg_mem.Load4(b1), a11 = wg_mem.Load4(b1 + 16u); + const uint4 a20 = wg_mem.Load4(b2), a21 = wg_mem.Load4(b2 + 16u), a3 = wg_mem.Load4(b3 + (LP_MUT == 4 ? 16u : 8u)); // mutant 4: wrong row span + const float3 zi = float3(0.0f, 0.0f, 0.0f); + const float3 q0 = mad(wx.z, h3(a0.zw), mad(wx.y, h3(a0.xy), zi)); + const float3 q1 = mad(wx.w, h3(a11.zw), mad(wx.z, h3(a11.xy), mad(wx.y, h3(a10.zw), mad(wx.x, h3(a10.xy), zi)))); + const float3 q2 = mad(wx.w, h3(a21.zw), mad(wx.z, h3(a21.xy), mad(wx.y, h3(a20.zw), mad(wx.x, h3(a20.xy), zi)))); + const float3 q3 = mad(wx.z, h3(a3.zw), mad(wx.y, h3(a3.xy), zi)); + const float3 acci = mad(wy.w, q3, mad(wy.z, q2, mad(wy.y, q1, mad(wy.x, q0, zi)))); + const float sxi = ((wx.x + wx.y) + wx.z) + wx.w; + const float normi = (wx.y + wx.z) * (wy.x + wy.w) + sxi * (wy.y + wy.z); + return acci * rcp(normi); + } uint ro[4], co[4]; [unroll] for (int k = 0; k < 4; ++k) { co[k] = hist_prev + (uint)clampi(ix - 1 + k, 0, P_dw - 1) * 8u; @@ -277,15 +346,22 @@ void l_post_core(LGather g, int x, int y, int fx, int fy, float mx[3], float my[ float fbar[9]; [unroll] for (int k = 0; k < 9; ++k) fbar[k] = inv * (float)opqh(g.F[k]); const float sig0 = fbar[3], sig1 = fbar[4], sig2 = fbar[5]; - int first = 0; - float best = sig0; - if (sig1 > best) { first = 1; best = sig1; } - if (sig2 > best) first = 2; - const int o1 = first == 0 ? 1 : 0, o2 = first == 2 ? 1 : 2; - const float so1 = o1 == 1 ? sig1 : sig0, so2 = o2 == 1 ? sig1 : sig2; - const int second = so2 > so1 ? o2 : o1; - const float s1 = first == 0 ? sig0 : (first == 1 ? sig1 : sig2); - const float s2 = second == 0 ? sig0 : (second == 1 ? sig1 : sig2); + const bool sb1 = sig1 > sig0; + const float best = sb1 ? sig1 : sig0; + const bool sb2 = sig2 > best; + const float s1 = sb2 ? sig2 : best; + const float m1x = sb2 ? mx[2] : (sb1 ? mx[1] : mx[0]), m1y = sb2 ? my[2] : (sb1 ? my[1] : my[0]); + const bool sf0 = !or(sb1, sb2); + const float so1 = sf0 ? sig1 : sig0, so2 = sb2 ? sig1 : sig2; + const float mo1x = sf0 ? mx[1] : mx[0], mo1y = sf0 ? my[1] : my[0]; +#if LP_MUT == 7 + const float mo2x = sb2 ? mx[2] : mx[1], mo2y = sb2 ? my[1] : my[2]; // mutant 7: wrong candidate motion +#else + const float mo2x = sb2 ? mx[1] : mx[2], mo2y = sb2 ? my[1] : my[2]; +#endif + const bool scc = so2 > so1; + const float s2 = scc ? so2 : so1; + const float m2x = scc ? mo2x : mo1x, m2y = scc ? mo2y : mo1y; const float q1 = clampf(k1p1 * s1 - k0p05, 0.0f, 1.0f); const float q2 = clampf(k1p1 * s2 - k0p05, 0.0f, 1.0f); const float rden = rcp(max(q1 + q2, k1em10)); @@ -302,10 +378,13 @@ void l_post_core(LGather g, int x, int y, int fx, int fy, float mx[3], float my[ [unroll] for (int n = 0; n < 2; ++n) { L[n][0] = 0.0f; L[n][1] = 0.0f; L[n][2] = 0.0f; if (n < ncand) { - const int k = n == 0 ? first : second; - const float qx = X + (k == 0 ? mx[0] : (k == 1 ? mx[1] : mx[2])); - const float qy = Y + (k == 0 ? my[0] : (k == 1 ? my[1] : my[2])); - const bool valid = P_reset == 0 && qx >= 0.0f && qx <= (float)P_dw && qy >= 0.0f && qy <= (float)P_dh; + const float qx = X + (n == 0 ? m1x : m2x); + const float qy = Y + (n == 0 ? m1y : m2y); +#if LP_MUT == 5 + const bool valid = and(and(and(and(P_reset == 0, qx >= 0.0f), qx <= (float)P_dh), qy >= 0.0f), qy <= (float)P_dh); // mutant 5 +#else + const bool valid = and(and(and(and(P_reset == 0, qx >= 0.0f), qx <= (float)P_dw), qy >= 0.0f), qy <= (float)P_dh); +#endif if (valid) { const float3 h = history_cr(qx, qy); #if LP_LDR @@ -329,9 +408,9 @@ void l_post_core(LGather g, int x, int y, int fx, int fy, float mx[3], float my[ #if LP_LDR Hl[i] = opq(comb); #else - Hl[i] = opq(comb) * g_e; + Hl[i] = mul_nz(comb, g_e); #endif - hl_inf = hl_inf || abs(Hl[i]) == asfloat(0x7f800000u); + hl_inf = or(hl_inf, abs(Hl[i]) == asfloat(0x7f800000u)); #if LP_LDR Cl[i] = fbar[i]; #else @@ -346,10 +425,10 @@ void l_post_core(LGather g, int x, int y, int fx, int fy, float mx[3], float my[ oh[i] = mad(Cl[i] - Hl[i], bo2, Hl[i]); } const float sh = hh[0] + hh[1] + hh[2], so = oh[0] + oh[1] + oh[2]; - if (isnan(sh) || hl_inf) { + if (or(isnan(sh), hl_inf)) { [unroll] for (int i = 0; i < 3; ++i) hh[i] = Cl[i]; } - if (isnan(so) || hl_inf) { + if (or(isnan(so), hl_inf)) { [unroll] for (int i = 0; i < 3; ++i) oh[i] = Cl[i]; } } @@ -441,7 +520,7 @@ WG_KERNEL(l_post_recon) } GroupMemoryBarrierWithGroupSync(); - const bool active = x0 + (lane & 31) < P_dw && k0 + (lane >> 5) < npairs; + const bool active = and(x0 + (lane & 31) < P_dw, k0 + (lane >> 5) < npairs); const int x = mini(x0 + (lane & 31), P_dw - 1); const int y0 = 2 * mini(k0 + (lane >> 5), npairs - 1); const bool has1 = y0 + 1 < P_dh; @@ -464,26 +543,33 @@ WG_KERNEL(l_post_recon) [unroll] for (int jj = 0; jj < kLWin; ++jj) { v0[jj] = dy0 + (half)(float)(jj - 1); v1[jj] = dy1 + (half)(float)(jj - 1); - vv0[jj] = opqh(v0[jj] * v0[jj]); - vv1[jj] = opqh(v1[jj] * v1[jj]); + vv0[jj] = mulh(v0[jj], v0[jj]); + vv1[jj] = mulh(v1[jj], v1[jj]); } // taps column by column (i outer, j inner, each gather in that order as before); per column the eight weights (four // rows, two pixels) are rounded to FP16 by one driver conversion LGather g0 = gather_zero(), g1 = gather_zero(); + // the staged texel's LDS index without per-tap multiplies: one base per lane, the row stride and the column + // step wave-uniform (integer identity: ((b + j - r0) ncol + (ax + i - c0)) kTexDw) + const int tb0 = ((b - 1 - r0) * ncol + (ax - 1 - c0)) * kTexDw, rstride = ncol * kTexDw; [loop] for (int i = -1; i <= kLWin - 2; ++i) { const half u = dx + (half)(float)i; - const half uu = opqh(u * u); + const half uu = mulh(u, u); uint t[kLWin][kTexDw]; v8f wv; [unroll] for (int j = -1; j <= kLWin - 2; ++j) { if (staged) { - const int base = ((b + j - r0) * ncol + (ax + i - c0)) * kTexDw; + const int base = tb0 + (j + 1) * rstride + (i + 1) * kTexDw +#if LP_MUT == 2 + + (j == 2 && i == 2 ? kTexDw : 0) // mutant 2: one tap reads the wrong texel +#endif + ; [unroll] for (int d = 0; d < kTexDw; ++d) t[j + 1][d] = sh_tex[base + d]; } else { load_texel_hw(clampi(b + j, 0, P_rh - 1) * P_rw + clampi(ax + i, 0, P_rw - 1), t[j + 1]); } - wv.v[2 * (j + 1)] = tap_weight(t[j + 1], uu, vv0[j + 1], opqh(u * v0[j + 1])); - wv.v[2 * (j + 1) + 1] = tap_weight(t[j + 1], uu, vv1[j + 1], opqh(u * v1[j + 1])); + wv.v[2 * (j + 1)] = tap_weight(t[j + 1], uu, vv0[j + 1], mulh(u, v0[j + 1])); + wv.v[2 * (j + 1) + 1] = tap_weight(t[j + 1], uu, vv1[j + 1], mulh(u, v1[j + 1])); } const uint4 hw = wg_to_f16_ags(wv); [unroll] for (int jj = 0; jj < kLWin; ++jj) { @@ -521,7 +607,7 @@ WG_KERNEL(l_post_recon) cv.v[7] = 0.0f; const uint4 hc = wg_to_f16_ags(cv); const int y = y0 + row; - if (active && (!second || has1)) { + if (and(active, or(!second, has1))) { #if LP_LDR const uint q0 = hc.x & 0xffffu, q1 = hc.x >> 16, q2 = hc.y & 0xffffu; #else diff --git a/clean/dx12/tests/test_post_edge_cases.py b/clean/dx12/tests/test_post_edge_cases.py new file mode 100644 index 00000000..910dc017 --- /dev/null +++ b/clean/dx12/tests/test_post_edge_cases.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: MIT +"""CPU checks of Model L's post history fast path (clean/dx12/kernels/l_post.hlsl history_cr) and of the inputs of +its GPU edge check (clean/l/runner/l_post_edge_check.py): +- the fast path's predicate as written (unsigned tests with the size guard) holds exactly when the 4 x 4 footprint + lies inside the image, at every size from 1 x 1; without the guard (the old predicate, mutant LP_MUT 9) it also + holds outside an image narrower or shorter than 4; +- each edge-check kind puts the history positions where it says (inside the footprint, outside it, or both in one + wave), so the counter expectations of the GPU check follow.""" +import os +import sys +import unittest + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.runner import l_post_edge_check as E # noqa: E402 + + +def u32(v): + return np.int64(v) & 0xFFFFFFFF + + +def predicate(ix, iy, dw, dh, guard=True): + """l_post.hlsl's interior predicate of one lane (int32 arithmetic, unsigned compares).""" + p = u32(ix - 1) <= u32(dw - 4) and u32(iy - 1) <= u32(dh - 4) + return bool(p and (not guard or (dw >= 4 and dh >= 4))) + + +def inside(ix, iy, dw, dh): + """Rows iy - 1 ... iy + 2 and columns ix - 1 ... ix + 2 all in the image.""" + return ix - 1 >= 0 and ix + 2 <= dw - 1 and iy - 1 >= 0 and iy + 2 <= dh - 1 + + +class PredicateTest(unittest.TestCase): + def test_guarded_predicate_is_the_footprint(self): + for dw in range(1, 10): + for dh in range(1, 10): + for ix in range(-3, 12): + for iy in range(-3, 12): + self.assertEqual(predicate(ix, iy, dw, dh), inside(ix, iy, dw, dh), (dw, dh, ix, iy)) + + def test_old_predicate_reads_outside(self): + bad = [(dw, dh, ix, iy) for dw in range(1, 10) for dh in range(1, 10) for ix in range(-3, 12) + for iy in range(-3, 12) if predicate(ix, iy, dw, dh, guard=False) and not inside(ix, iy, dw, dh)] + self.assertIn((5, 3, 2, 1), bad) # height 3: the span reads row 3 + self.assertTrue(all(dw < 4 or dh < 4 for dw, dh, _, _ in bad)) + + +class EdgeCasesTest(unittest.TestCase): + def footprints(self, x, dw, dh): + X, Y = np.meshgrid(np.arange(dw) + 0.5, np.arange(dh) + 0.5) + qx = X + x["m_near"][..., 0].astype(np.float64) + qy = Y + x["m_near"][..., 1].astype(np.float64) + ix, iy = np.floor(qx - 0.5).astype(int), np.floor(qy - 0.5).astype(int) + return np.vectorize(lambda a, b: inside(a, b, dw, dh))(ix, iy) + + def test_kinds(self): + rng = np.random.default_rng(1) + mixed = set() + for dw, dh in E.SIZES: + for kind in E.KINDS: + if kind == "random": + continue + x = E.edge_case(rng, dw, dh, kind) + for k in ("m_center", "m_far"): + self.assertTrue(np.array_equal(x[k], x["m_near"])) + case = {"x": x, "dw": dw, "dh": dh, "kind": kind} + want = E.counter_expected(case) + rows = E.wave_rows(x, dw, dh) + if dw < 4 or dh < 4: + self.assertIs(want, False) + elif kind in ("center", "in_lo", "in_hi"): + self.assertTrue(want, (dw, dh, kind)) + self.assertTrue(self.footprints(x, dw, dh).all(), (dw, dh, kind)) + elif kind in ("left", "right", "out_lo", "out_hi"): + self.assertIs(want, False, (dw, dh, kind)) + self.assertFalse(self.footprints(x, dw, dh).any(), (dw, dh, kind)) + elif kind in ("split", "split_half"): + if any(i > 0 and o > 0 for i, o in rows): + mixed.add((dw, dh, kind)) + # the mixed waves (both inside and outside lanes) appear at every size of at least 4 x 4, for both split kinds + for dw, dh in E.SIZES: + if dw >= 4 and dh >= 4: + self.assertIn((dw, dh, "split"), mixed) + self.assertIn((dw, dh, "split_half"), mixed) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/l/runner/l_d3d12_own_post_check.py b/clean/l/runner/l_d3d12_own_post_check.py index 6adf3b57..90bfb23d 100644 --- a/clean/l/runner/l_d3d12_own_post_check.py +++ b/clean/l/runner/l_d3d12_own_post_check.py @@ -61,7 +61,7 @@ def run(build, special=True, stages=False): rng = np.random.default_rng(F.SEED) frames = [] for i, ((rw, rh), reset) in enumerate(F.SEQUENCE): - fr = F.frame_inputs(rng, rw, rh, reset, special) + fr = F.frame_inputs(rng, rw, rh, reset, special, index=i) F.fill_sentinels(p, SENTINELS[config], i == 0) out = p.frame(fr, debug=True) frames.append({"inputs": fr, "out": out, "raw": F.raw_outputs(p), "post_last": p.kernels().get("post_last")}) diff --git a/clean/l/runner/l_default_post_check.py b/clean/l/runner/l_default_post_check.py index b63d7cde..be765ea7 100644 --- a/clean/l/runner/l_default_post_check.py +++ b/clean/l/runner/l_default_post_check.py @@ -54,7 +54,7 @@ def run_one(build, config): rng = np.random.default_rng(F.SEED) frames = [] for i, ((rw, rh), reset) in enumerate(F.SEQUENCE): - fr = F.frame_inputs(rng, rw, rh, reset, True) + fr = F.frame_inputs(rng, rw, rh, reset, True, index=i) F.fill_sentinels(p, sentinel, i == 0) out = p.frame(fr, debug=True) frames.append({"inputs": fr, "out": out, "raw": F.raw_outputs(p), "post_last": p.kernels().get("post_last")}) diff --git a/clean/l/runner/l_frame_check.py b/clean/l/runner/l_frame_check.py index b0215c84..5febcf53 100644 --- a/clean/l/runner/l_frame_check.py +++ b/clean/l/runner/l_frame_check.py @@ -92,7 +92,11 @@ # and on the first frame; both parities on every size SEQUENCE = [((96, 64), False), ((96, 64), False), ((96, 64), True), ((96, 64), False), ((96, 64), False), ((80, 48), False), ((80, 48), False), ((80, 48), False), ((77, 45), False), ((77, 45), True), - ((77, 45), False)] + ((77, 45), False), ((77, 45), False), ((77, 45), False)] +# frames whose colour alternates dark and bright blocks of 8 columns (the dark ones scaled by 2^-10; blocks, not single +# columns, so that the 4 x 4 filters keep both levels), so that one wave of the post kernels holds lanes on both sides +# of a per-lane branch (e.g. l_dec's polynomial / exponential split); same size as the frame before (no reset) +MIXED_FRAMES = (11, 12) PRE_EXPOSURE = 1.25 DEFAULT_L_MAP = os.path.join(ROOT, "data", "preset-L.v1.map.json") @@ -188,9 +192,14 @@ def prepare(build, rocm, model=None, map_path=None): # ---- --run (GPU) ---------------------------------------------------------------------------------------------------- -def frame_inputs(rng, rw, rh, reset, special=True): +def frame_inputs(rng, rw, rh, reset, special=True, index=None): from clean.reference import encoder as En x = En.random_inputs(rng, rw=rw, rh=rh, fw=-(-rw // 32) * 32, fh=-(-rh // 32) * 32, special=special) + if index in MIXED_FRAMES: # dark blocks of 8 columns (exact power-of-two scaling of the colour channels) + c = np.array(x["color"], copy=True) + dark = (np.arange(c.shape[1]) // 8) % 2 == 1 + c[:, dark, :3] = c[:, dark, :3] * c.dtype.type(2.0 ** -10) + x["color"] = c return {"color": x["color"], "depth": x["depth"], "motion": x["motion"], "mv_scale": x["mv_scale"], "jitter": x["jitter"], "exposure": 1.0, "pre_exposure": PRE_EXPOSURE, "reset": reset} @@ -348,7 +357,7 @@ def run_d3d12(build, special=True, stages=False): rng = np.random.default_rng(SEED) frames = [] for i, ((rw, rh), reset) in enumerate(SEQUENCE): - fr = frame_inputs(rng, rw, rh, reset, special) + fr = frame_inputs(rng, rw, rh, reset, special, index=i) fill_sentinels(p, SENTINELS[config], i == 0) out = p.frame(fr, debug=True) frames.append({"inputs": fr, "out": out, "raw": raw_outputs(p)}) @@ -553,7 +562,7 @@ def run(build, rocm, special=True): rng = np.random.default_rng(SEED) frames = [] for i, ((rw, rh), reset) in enumerate(SEQUENCE): - fr = frame_inputs(rng, rw, rh, reset, special) + fr = frame_inputs(rng, rw, rh, reset, special, index=i) fill_sentinels(p, SENTINELS[config], i == 0) out = p.frame(fr, debug=True) frames.append({"inputs": fr, "out": out, "raw": raw_outputs(p)}) diff --git a/clean/l/runner/l_post_edge_check.py b/clean/l/runner/l_post_edge_check.py new file mode 100644 index 00000000..35d989f0 --- /dev/null +++ b/clean/l/runner/l_post_edge_check.py @@ -0,0 +1,386 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): the reconstruction kernels' history sample at the image edges and at small, odd image sizes, +D3D12 against HIP byte for byte, HIP against the oracle, with a counter that proves the D3D12 interior fast path runs. + + python clean/l/runner/l_post_edge_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --prepare + python clean/l/runner/l_post_edge_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --run --json out.json + +--prepare (CPU): l_post.co (HIP), kernel_replay.exe and four D3D12 builds of the post ports (clean/dx12/build_dx12.py +--define): /dx12 as shipped; /hscount (LP_HSCOUNT=1: each wave taking the history fast path adds 1 to the +first dword after the full history); /mut8 (counter + LP_MUT=8: the fast path's predicate always false) and +/mut9 (LP_MUT=9: the predicate without its size guard, the bug of images narrower or shorter than 4). +--run (GPU, one GPU slot). The kernels l_post_recon, l_post_recon_p12, l_post_recon_ldr and l_post_recon_p12_ldr (the +history sample, history_cr, is shared by all four), on every size of SIZES (1 x 1 ... 97 x 35: below 4, odd, a tile's interior boundary in the +middle of a wave) and every kind of KINDS, with a valid history (no reset): +- random: Model M's random inputs (random motions around a random base); +- center / left / right / top / bottom: every motion points at one history position: the centre, or just inside + one edge (the clamped sample on every lane); +- in_lo / in_hi / out_lo / out_hi: the position at the fast path's boundary, ix (and iy) = 1 or size - 3 (inside: the + 4 x 4 footprint fits) and 0 or size - 2 (outside); +- split: even columns point inside, odd columns at an edge (every wave mixes both); split_half: in each 32-column + tile the first 16 columns (half the width when narrower) inside, the rest at an edge (the boundary in the middle of + a wave). +Every replay leg (the shipped ports, hscount, mut8, mut9) starts from an emptied output folder and counts only when +kernel_replay exits 0 and every case wrote every output at its full size; otherwise the leg is invalid and every check +that uses it fails (a failed mutant run is never taken as caught). +Checks: +- raw bytes, D3D12 == HIP, every output (full history, reduced, display; the l_d3d12_ops_check rules: each back end's + own sentinel, the pitch padding and 64 slack bytes untouched), and the full history per edge band (the two columns + or rows at each of the four edges) reported apart; +- the counter (hscount build, same bytes otherwise): 0 at every size below 4; otherwise, for the non-random kinds, > 0 + exactly when some wave's sampling lanes all have their footprint inside (wave_rows: e.g. center, in_lo, in_hi) and + 0 when none has (left, right, out_lo, out_hi, the mixed waves of split); +- negative controls: mut8 must break the counter expectation (the fast path gone is noticed), mut9 must differ from + HIP (the old predicate reads past a history of height or width 3), and the run's first multi-tile case with half its + workgroups must differ; +- the oracle: l_post_recon and l_post_recon_ldr (HIP) against Model L's reconstruction (clean/l/runner/l_post_check.py + check_case: Model L's post acceptance) on every case. +Output: counts only, no timing. +""" +import argparse +import json +import os +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.runner import l_c32_check as C32 # noqa: E402 +from clean.l.runner import l_d3d12_ops_check as OPS # noqa: E402 +from clean.l.runner.l_d3d12_check import D3D12_SENTINEL, HIP_SENTINEL, prepare_replay # noqa: E402 + +KERNELS = ["l_post_recon", "l_post_recon_p12", "l_post_recon_ldr", "l_post_recon_p12_ldr"] +SIZES = [(1, 1), (2, 3), (3, 2), (3, 3), (3, 5), (5, 3), (4, 4), (5, 4), (4, 5), (7, 7), (33, 9), (37, 6), (67, 13), + (97, 35)] +KINDS = ["random", "center", "left", "right", "top", "bottom", "in_lo", "in_hi", "out_lo", "out_hi", "split", + "split_half"] +VARIANTS = {"hscount": ["LP_HSCOUNT=1"], "mut8": ["LP_HSCOUNT=1", "LP_MUT=8"], "mut9": ["LP_MUT=9"]} +SEED = 231 + + +def _pos(kind, n, lo): + """History coordinate q (q - 1/2 = t, ix = floor(t)) of one axis of size n for a kind; lo: the low side.""" + if kind == "center": + return n // 2 + 0.25 + if kind in ("in_lo", "in_hi"): # ix = 1 or n - 3 + return 2.25 if (kind == "in_lo") == lo else n - 2.25 + if kind in ("out_lo", "out_hi"): # ix = 0 or n - 2 + return 1.25 if (kind == "out_lo") == lo else n - 1.25 + return n // 2 + 0.25 + + +def _point(kind, dw, dh): + if kind == "left": + return 0.75, dh // 2 + 0.25 + if kind == "right": + return dw - 0.25, dh // 2 + 0.25 + if kind == "top": + return dw // 2 + 0.25, 0.75 + if kind == "bottom": + return dw // 2 + 0.25, dh - 0.25 + return _pos(kind, dw, True), _pos(kind, dh, True) + + +def edge_case(rng, dw, dh, kind): + """Model M's random post inputs at render size = output size (ratio 1, jitter 0: render pixel x is output pixel + x, so every candidate motion is the pixel's own) with the motions of `kind` (FP16-exact).""" + from clean.l.runner import l_post_check as LPC + if kind == "random": + rw, rh = max(1, (2 * dw) // 3), max(1, (2 * dh) // 3) + return LPC.make_case(rng, rw, rh, dw, dh, False, "random") + x = LPC.make_case(rng, dw, dh, dw, dh, False, "random") + x["jitter"] = np.zeros(2) + X, Y = np.meshgrid(np.arange(dw) + 0.5, np.arange(dh) + 0.5) + if kind in ("split", "split_half"): + inside = (np.arange(dw) % 2 == 0) if kind == "split" else (np.arange(dw) % 32 < min(16, dw // 2)) + qi, qe = _point("center", dw, dh), _point("left", dw, dh) + qx = np.where(inside[None, :], qi[0], qe[0]) + 0 * X + qy = np.where(inside[None, :], qi[1], qe[1]) + 0 * Y + else: + q = _point(kind, dw, dh) + qx, qy = q[0] + 0 * X, q[1] + 0 * Y + m = np.stack([qx - X, qy - Y], -1) + assert np.array_equal(m.astype(np.float16).astype(np.float64), m), "motions must be FP16-exact" + for k in ("m_near", "m_center", "m_far"): + x[k] = m.astype(np.float16) + return x + + +def wave_rows(x, dw, dh): + """The kernel's history samples of the non-random kinds, per wave and row: [(inside, outside)] counts of the lanes + that sample the history (valid position), as l_post.hlsl's grid runs them: 32-column tiles, wave k = row pair + k0 + k (pairs past the end repeat the last), both rows (a second row past the image is computed too: its motion is + the last row's), lanes past the width repeat the last column. Render size = output size, jitter 0.""" + m = x["m_near"].astype(np.float64) + npairs = (dh + 1) // 2 + out = [] + for x0 in range(0, dw, 32): + for k0 in range(0, npairs, 8): + for k in range(8): + y0 = 2 * min(k0 + k, npairs - 1) + for row in (0, 1): + y = y0 + row + ins = outs = 0 + for lane in range(32): + xx = min(x0 + lane, dw - 1) + fy = min(max(y, 0), dh - 1) + qx, qy = xx + 0.5 + m[fy, xx, 0], y + 0.5 + m[fy, xx, 1] + if not (0.0 <= qx <= dw and 0.0 <= qy <= dh): + continue + ix, iy = int(np.floor(qx - 0.5)), int(np.floor(qy - 0.5)) + fits = ix - 1 >= 0 and ix + 2 <= dw - 1 and iy - 1 >= 0 and iy + 2 <= dh - 1 + ins, outs = ins + fits, outs + (not fits) + out.append((ins, outs)) + return out + + +def counter_expected(case): + """True: the fast path must run (counter > 0: some wave's sampling lanes all fit); False: it must not (0); None: + either (random motions and jitter, at least 4 x 4).""" + dw, dh = case["dw"], case["dh"] + if dw < 4 or dh < 4: + return False + if case["kind"] == "random": + return None + return any(i > 0 and o == 0 for i, o in wave_rows(case["x"], dw, dh)) + + +def cases(kernel, rng): + """The post cases of l_d3d12_ops_check (same argument layout, sentinels, slack) on the edge inputs.""" + from clean.runner import post as RP + from clean.l.runner import l_post_check as LPC + out = [] + for (dw, dh) in SIZES: + for kind in KINDS: + x = edge_case(rng, dw, dh, kind) + prm = RP.post_params(x) + data = RP.post_device_inputs(x) + n_hist, n_red = dw * dh * 8, max(4, (dw // 2) * (dh // 2) * 4) + n_disp, pitch = RP.display_bytes(dw, dh), RP.display_pitch(dw) + disp_def = np.zeros((3, dh, pitch, 2), bool) + disp_def[:, :, :dw, :] = True + red_def = np.zeros(n_red, bool) + red_def[:(dw // 2) * (dh // 2) * 4] = True + defined = [np.concatenate([np.ones(n_hist, bool), np.zeros(OPS.POST_SLACK, bool)]), + np.concatenate([red_def, np.zeros(OPS.POST_SLACK, bool)]), + np.concatenate([disp_def.reshape(-1), np.zeros(OPS.POST_SLACK, bool)])] + if kernel.startswith("l_post_recon_p12"): + data = dict(data, **OPS._pack12(data)) + args = [("null",) if data[k] == b"" else ("buf", data[k]) + for k in ("color", "A", "B", "C", "hist_prev", "m_near", "m_center", "m_far")] + args += [("buf", RP.pack({"s": np.array(LPC.S)})), ("out", n_hist + OPS.POST_SLACK), + ("out", n_red + OPS.POST_SLACK), ("out", n_disp + OPS.POST_SLACK), ("bytes", bytes(prm)), ("null",)] + out.append({"kernel": kernel, "co": "l_post.co", "grid": RP.recon_grid(dw, dh), "block": 256, "args": args, + "defined": defined, "x": x, "dw": dw, "dh": dh, "kind": kind, + "label": f"{dw}x{dh} {kind}"}) + return out + + +def prepare(build, rocm): + C32.compile_kernels(build, rocm) + names = KERNELS + ["native_fill"] + prepare_replay(build, rocm, names) + from clean.runner.hip import hip_child_environment + env, _ = hip_child_environment(rocm_dir=rocm) + for tag, defs in VARIANTS.items(): + cmd = [sys.executable, os.path.join(ROOT, "clean", "dx12", "build_dx12.py"), "--out", os.path.join(build, tag), + "--only", ",".join(names), "-j", "1"] + for d in defs: + cmd += ["--define", d] + subprocess.check_call(cmd, env=env, cwd=ROOT) + + +def edge_bands(dw, dh): + """Pixel masks [dh][dw] of the two columns / rows at each edge.""" + ys, xs = np.mgrid[0:dh, 0:dw] + return {"left": xs < 2, "right": xs >= dw - 2, "top": ys < 2, "bottom": ys >= dh - 2} + + +def replay(build_dir, exe, cdir, lines, env_extra=None): + with open(os.path.join(cdir, "cases.txt"), "w", newline="\n") as f: + f.write("\n".join(lines)) + env = {k: v for k, v in os.environ.items() + if not any(s in k.upper() for s in ("HIP", "LLVM", "ROCM", "HSA")) or k.upper() in ("HIP_PLATFORM",)} + env["WG_D3D12_ARENA_MB"] = "512" + env.update(env_extra or {}) + r = subprocess.run([exe, build_dir, os.path.join(cdir, "cases.txt")], env=env, capture_output=True, text=True, + timeout=150, creationflags=0x08000000) + return {"exit": r.returncode, "tail": r.stdout[-400:] + r.stderr[-200:]} + + +def empty_dir(d): + """The leg's output folder, emptied of the files of an earlier run (files only, this folder only).""" + os.makedirs(d, exist_ok=True) + for f in os.listdir(d): + p = os.path.join(d, f) + if os.path.isfile(p): + os.remove(p) + + +def compare_case(cdir, name, case, wants, counter=False): + """Raw comparison of one case's three outputs (l_d3d12_ops_check rules); counter: the hscount build, whose first + dword after the full history holds the counter (excluded from the slack rule, returned).""" + dw, dh = case["dw"], case["dh"] + n_hist = dw * dh * 8 + res = {"case": name, "label": case["label"], "outputs": [], "complete": True} + count = None + for oi, want in enumerate(wants): + path = os.path.join(cdir, f"{name}.o{oi}") + got = open(path, "rb").read() if os.path.exists(path) else b"" + res["complete"] = res["complete"] and len(got) == len(want) + dmask = case["defined"][oi].copy() + if counter and oi == 0 and len(got) == len(want): + count = int(np.frombuffer(got[n_hist:n_hist + 4], np.uint32)[0]) - 0x5A5A5A5A + g, w = bytearray(got), bytearray(want) + g[n_hist:n_hist + 4] = bytes([D3D12_SENTINEL]) * 4 + w[n_hist:n_hist + 4] = bytes([HIP_SENTINEL]) * 4 + got, want = bytes(g), bytes(w) + a, b = np.frombuffer(got, np.uint8), np.frombuffer(want, np.uint8) + o = {"bytes": len(want)} + if a.size != b.size: + o.update(identical=False, differ=-1) + else: + o["differ"] = int(np.sum((a != b) & dmask)) + o["unwritten_bad"] = int(np.sum(~dmask & ((b != HIP_SENTINEL) | (a != D3D12_SENTINEL)))) + o["identical"] = o["differ"] == 0 and o["unwritten_bad"] == 0 + if oi == 0: # the full history per edge band + d = (a[:n_hist] != b[:n_hist]).reshape(dh, dw, 8).any(-1) + o["edges"] = {k: {"pixels": int(m.sum()), "differ": int((d & m).sum())} + for k, m in edge_bands(dw, dh).items()} + res["outputs"].append(o) + res["identical"] = all(o["identical"] for o in res["outputs"]) + if counter: + res["counter"] = count + return res + + +def run(build, rocm, part="all"): + """part: all, replay (D3D12 legs only) or oracle (the oracle leg only): two GPU slots when one is too short.""" + from clean.l.runner import l_post_check as LPC + g = C32.Gpu(build, rocm) + with open(os.path.join(build, "l_post.co"), "rb") as f: + mod = g.hip.load_module(f.read()) + exe = os.path.join(build, "kernel_replay.exe") + rng = np.random.default_rng(SEED) + allc, hip_out, lines = {}, {}, [] + neg = None + for kernel in KERNELS: + for ci, case in enumerate(cases(kernel, rng)): + name = f"{kernel}_{ci}" + allc[name] = case + if part != "oracle": + hip_out[name] = OPS.hip_run(g, mod, case) + res = {"sizes": SIZES, "kinds": KINDS, "part": part, "legs": {}} + cdir_base = os.path.join(build, "replay-edge") + for leg, bdir, env_extra, counter in () if part == "oracle" else (("d3d12", build, None, False), + ("hscount", os.path.join(build, "hscount"), None, True), + ("mut8", os.path.join(build, "mut8"), None, True), + ("mut9", os.path.join(build, "mut9"), None, False)): + cdir = os.path.join(cdir_base, leg) + empty_dir(cdir) + names = list(allc) + lines = [] + for n in names: + lines += OPS.replay_lines(n, allc[n], cdir) + if leg == "d3d12" and neg is None and allc[n]["grid"] >= 4: + neg = n + lines += OPS.replay_lines(n, allc[n], cdir, grid=allc[n]["grid"] // 2, prefix="negative_") + r = replay(bdir, exe, cdir, lines, env_extra) + out = [compare_case(cdir, n, allc[n], hip_out[n], counter) for n in names] + complete = sum(c["complete"] for c in out) + leg_res = {"replay_exit": r["exit"], "complete_cases": complete, "valid": r["exit"] == 0 and complete == len(out), + "replay_tail": r["tail"], "cases": len(out), + "identical": sum(c["identical"] for c in out), + "edges": {k: {"pixels": sum(c["outputs"][0].get("edges", {}).get(k, {}).get("pixels", 0) for c in out), + "differ": sum(c["outputs"][0].get("edges", {}).get(k, {}).get("differ", 0) for c in out)} + for k in ("left", "right", "top", "bottom")}, + "differing": [c["label"] + " " + c["case"] for c in out if not c["identical"]][:40]} + if counter: + bad, entered = [], 0 + for c in out: + case = allc[c["case"]] + want = counter_expected(case) + entered += c["counter"] > 0 if c["counter"] is not None else 0 + if c["counter"] is None or (want is True and c["counter"] <= 0) or (want is False and c["counter"] != 0): + bad.append(f"{c['case']} {c['label']}: counter {c['counter']}") + leg_res["counter"] = {"cases_entering_fast_path": entered, "expectation_failures": len(bad), + "first_failures": bad[:20], + "must_enter": sum(counter_expected(allc[c["case"]]) is True for c in out), + "must_not_enter": sum(counter_expected(allc[c["case"]]) is False for c in out)} + if leg == "d3d12" and neg: + wants = hip_out[neg] + got = [open(os.path.join(cdir, f"negative_{neg}.o{oi}"), "rb").read() + if os.path.exists(os.path.join(cdir, f"negative_{neg}.o{oi}")) else b"" for oi in range(len(wants))] + full = all(len(gv) == len(wv) for gv, wv in zip(got, wants)) + leg_res["negative_control"] = {"case": neg, "half_grid": True, "complete": full, + "caught": full and got != list(wants)} + res["legs"][leg] = leg_res + # the oracle: HIP's l_post_recon / l_post_recon_ldr against Model L's reconstruction on every case's inputs + # (cases under 100 pixels: Model L's post acceptance over all of them together, where one value is not 1 %) + orc, small = [], {"history": [], "display": [], "reduced_max": 0} + for name, case in allc.items() if part != "replay" else (): + if case["kernel"] not in ("l_post_recon", "l_post_recon_ldr"): + continue + m = LPC.check_case(g.hip, mod, case["x"], "random", ldr=case["kernel"].endswith("_ldr")) + tiny = case["dw"] * case["dh"] < 100 + ok = m["pass"] + if tiny: + for k in ("history", "display"): + small[k].append(m[k]) + small["reduced_max"] = max(small["reduced_max"], m["reduced"]["max"]) + ok = m["slack_untouched"] and m["alpha_zero"] and m["reduced_top_zero"] + orc.append({"case": name, "label": case["label"], "kernel": case["kernel"], "pass": bool(ok), "tiny": tiny}) + pooled = {k: {f: sum(i[f] for i in small[k]) for f in ("values", "nonfinite", "over_1", "differ")} + for k in ("history", "display")} + for k, v in pooled.items(): + v["pass"] = LPC.l_post_rule(v["values"], v["nonfinite"], v["over_1"], v["differ"], + max((i["mean_rel_rms"] for i in small[k]), default=0.0)) + pooled["reduced_max"] = small["reduced_max"] + res["oracle"] = {"cases": len(orc), "passed": sum(o["pass"] for o in orc), "tiny_pooled": pooled, + "failing": [o["kernel"] + " " + o["label"] for o in orc if not o["pass"]][:40]} + L = res["legs"] + checks = {} + if part != "oracle": + checks.update({ + "legs_valid": all(v["valid"] for v in L.values()), + "d3d12_identical": L["d3d12"]["valid"] and L["d3d12"]["identical"] == L["d3d12"]["cases"], + "hscount_identical": L["hscount"]["valid"] and L["hscount"]["identical"] == L["hscount"]["cases"], + "hscount_counter_ok": L["hscount"]["valid"] and L["hscount"]["counter"]["expectation_failures"] == 0, + "mut8_caught": L["mut8"]["valid"] and L["mut8"]["counter"]["expectation_failures"] > 0, + "mut9_caught": L["mut9"]["valid"] and L["mut9"]["identical"] < L["mut9"]["cases"], + "negative_control_caught": bool(L["d3d12"].get("negative_control", {}).get("caught"))}) + if part != "replay": + checks["oracle_pass"] = (res["oracle"]["passed"] == res["oracle"]["cases"] and pooled["history"]["pass"] and + pooled["display"]["pass"] and pooled["reduced_max"] <= 1) + res["checks"] = checks + res["pass"] = all(checks.values()) + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--prepare", action="store_true") + ap.add_argument("--run", action="store_true") + ap.add_argument("--part", choices=("all", "replay", "oracle"), default="all") + ap.add_argument("--json") + a = ap.parse_args(argv) + if a.prepare: + prepare(a.build, a.rocm) + if a.run: + res = run(a.build, a.rocm, a.part) + print(json.dumps(res["checks"], indent=1), "pass", res["pass"]) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + return 0 if res["pass"] else 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main())