Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion clean/dx12/build_dx12.py
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,9 @@ def main(argv=None):
ap.add_argument("--model-k", action="store_true",
help="also build Model K's kernels (clean/k/dx12/kernels; off by default: Model M's and Model L's "
"kernel sets unchanged)")
ap.add_argument("--define", action="append", default=[], metavar="NAME=VALUE",
help="an extra preprocessor define for every kernel built (test variants: negative-control mutants, "
"test counters; never for a shipped build)")
ap.add_argument("--list", action="store_true",
help="print the kernels a build with these options compiles (name and code object, JSON) and exit: "
"the cheap check before a build (kernel_sets.json, tests/test_kernel_sets.py)")
Expand All @@ -367,6 +370,11 @@ def main(argv=None):
return 0
if not a.out:
ap.error("--out is required")
extra_defs = {}
for kv in a.define:
if not re.fullmatch(r"[A-Z][A-Z0-9_]*=-?[0-9A-Za-z_]+", kv):
ap.error(f"--define {kv!r} (NAME=VALUE)")
extra_defs[kv.split("=", 1)[0]] = kv.split("=", 1)[1]
dst = os.path.join(a.out, "dx12")
os.makedirs(dst, exist_ok=True)
asm = build_asm_tool(a.out)
Expand Down Expand Up @@ -412,7 +420,7 @@ def main(argv=None):

def build_one(job):
src, d, out, k_source = job
defs = {"WG_BLOCK": d["group"], "WG_WAVE": d["wave"], **d["defines"]}
defs = {"WG_BLOCK": d["group"], "WG_WAVE": d["wave"], **d["defines"], **extra_defs}
if d["group"] != d["block"]:
defs["WG_LAUNCH_BLOCK"] = d["block"]
info = {}
Expand Down
140 changes: 113 additions & 27 deletions clean/dx12/kernels/l_post.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,29 @@ static float g_e, g_ep; // exposures (0 replaced by 1)
static float g_rp; // 1 / e_prev (IEEE division, round to nearest even)
float opq(float x) { return asfloat(asuint(x) ^ g_z); }
half opqh(half x) { return asfloat16((uint16_t)(asuint16(x) ^ (uint16_t)g_z)); }
float mul_nz(float a, float b) { return mad(a, b, asfloat(g_z | 0x80000000u)); }
#ifndef LP_MUT
#define LP_MUT 0 // 1, 2, 4 ... 9: negative controls of the byte checks (never shipped)
#endif
#ifndef LP_DP
#define LP_DP 1
#endif
#ifndef LP_FMANZ
#define LP_FMANZ 1
#endif
#ifndef LP_HSCOUNT
#define LP_HSCOUNT 0 // 1: test builds only (clean/l/runner/l_post_edge_check.py): the history fast path's wave counter
#endif
// h(a b) as its own rounding that the driver cannot fuse into a following FP16 add: fma(a, b, -0) with an opaque -0
// (RN16(a b + -0) = RN16(a b) for every non-NaN pair, zeros included); LP_FMANZ 0: the product through the xor guard
half mulh(half a, half b)
{
#if LP_FMANZ
return mad(a, b, asfloat16((uint16_t)(0x8000u | g_z)));
#else
return opqh(a * b);
#endif
}
half h_bits(uint b) { return asfloat16((uint16_t)b); }
uint bits_h(half h) { return (uint)asuint16(h); }

Expand Down Expand Up @@ -109,8 +132,20 @@ float dec_m1_poly(float qc)
float l_dec(float q)
{
const float qc = clampf(q, k1em4, 3.0f);
#if LP_DP
// Model M's wave-level skip (post.hlsl post_core_s): the exponential for every lane, the polynomial only when some
// active lane of the wave is below kDecPolyMax; per lane the same select as below, so the same value
float v = exp2(qc * kDecK) - 1.0f;
const bool sm = qc < kDecPolyMax;
#if LP_MUT == 6
[branch] if (WaveActiveAllTrue(sm)) v = sm ? dec_m1_poly(qc) : v; // mutant 6: all lanes instead of any lane
#else
[branch] if (WaveActiveAnyTrue(sm)) v = sm ? dec_m1_poly(qc) : v;
#endif
#else
const float v = qc < kDecPolyMax ? dec_m1_poly(qc) : exp2(qc * kDecK) - 1.0f;
return isnan(q) ? q : opq(v) * k0p01;
#endif
return isnan(q) ? q : mul_nz(v, k0p01);
}

// (_Float16)((float)k / 255.0f) for the code k = 0 ... 255 (an IEEE FP32 division, then round to nearest even: HIP's
Expand Down Expand Up @@ -216,7 +251,11 @@ LGather gather_zero()
float tap_weight(uint t[kTexDw], half uu, half vv, half uv)
{
const half kA = h_bits(t[4] >> 16), kB = h_bits(t[5]), kC = h_bits(t[5] >> 16);
const half ta = opqh(uu * kA), tb = opqh(vv * kB);
#if LP_MUT == 1
const half ta = mulh(uu, kB), tb = mulh(vv, kB); // mutant 1: wrong factor
#else
const half ta = mulh(uu, kA), tb = mulh(vv, kB);
#endif
const half s = tb + ta;
const half eps = mad(uv, kC, s);
const float w = exp2((float)opqh(eps));
Expand Down Expand Up @@ -251,6 +290,36 @@ float3 history_cr(float qx, float qy)
const float4 wx = cr_weights(tx - fx0), wy = cr_weights(ty - fy0);
const int ix = (int)fx0, iy = (int)fy0;
const uint pitch = (uint)P_dw * 8u;
// interior fast path (Model M's history_cr_span): every lane's 4 x 4 footprint inside the image, so the clamps are
// the identity; six 16-byte span loads from one base instead of twelve 8-byte loads, the same texels in the same
// sums (same mad order). Footprint inside: 1 <= ix <= P_dw - 3 and 1 <= iy <= P_dh - 3, the unsigned tests valid only
// for an image of at least 4 x 4 (below, P_d* - 4 would wrap to a huge bound): smaller images take the clamped path.
#if LP_MUT == 8 // mutant 8: the predicate always false (the fast path never taken; same bytes, found by the counter)
const bool interior = false;
#elif LP_MUT == 9 // mutant 9: without the size guard (the span reads past a history narrower or shorter than 4)
const bool interior = and((uint)(ix - 1) <= (uint)(P_dw - 4), (uint)(iy - 1) <= (uint)(P_dh - 4));
#else
const bool interior = and(and(P_dw >= 4, P_dh >= 4),
and((uint)(ix - 1) <= (uint)(P_dw - 4), (uint)(iy - 1) <= (uint)(P_dh - 4)));
#endif
[branch] if (WaveActiveAllTrue(interior)) {
#if LP_HSCOUNT // test builds only: count the waves taking the fast path, in the first dword after the full history
if (WaveIsFirstLane()) wg_atomic_add(hist_out, (uint)(P_dw * P_dh) * 8u, 1u);
#endif
const uint b0 = hist_prev + (uint)(ix - 1) * 8u + (uint)(iy - 1) * pitch;
const uint b1 = b0 + pitch, b2 = b0 + 2u * pitch, b3 = b0 + 3u * pitch;
const uint4 a0 = wg_mem.Load4(b0 + 8u), a10 = wg_mem.Load4(b1), a11 = wg_mem.Load4(b1 + 16u);
const uint4 a20 = wg_mem.Load4(b2), a21 = wg_mem.Load4(b2 + 16u), a3 = wg_mem.Load4(b3 + (LP_MUT == 4 ? 16u : 8u)); // mutant 4: wrong row span
const float3 zi = float3(0.0f, 0.0f, 0.0f);
const float3 q0 = mad(wx.z, h3(a0.zw), mad(wx.y, h3(a0.xy), zi));
const float3 q1 = mad(wx.w, h3(a11.zw), mad(wx.z, h3(a11.xy), mad(wx.y, h3(a10.zw), mad(wx.x, h3(a10.xy), zi))));
const float3 q2 = mad(wx.w, h3(a21.zw), mad(wx.z, h3(a21.xy), mad(wx.y, h3(a20.zw), mad(wx.x, h3(a20.xy), zi))));
const float3 q3 = mad(wx.z, h3(a3.zw), mad(wx.y, h3(a3.xy), zi));
const float3 acci = mad(wy.w, q3, mad(wy.z, q2, mad(wy.y, q1, mad(wy.x, q0, zi))));
const float sxi = ((wx.x + wx.y) + wx.z) + wx.w;
const float normi = (wx.y + wx.z) * (wy.x + wy.w) + sxi * (wy.y + wy.z);
return acci * rcp(normi);
}
uint ro[4], co[4];
[unroll] for (int k = 0; k < 4; ++k) {
co[k] = hist_prev + (uint)clampi(ix - 1 + k, 0, P_dw - 1) * 8u;
Expand All @@ -277,15 +346,22 @@ void l_post_core(LGather g, int x, int y, int fx, int fy, float mx[3], float my[
float fbar[9];
[unroll] for (int k = 0; k < 9; ++k) fbar[k] = inv * (float)opqh(g.F[k]);
const float sig0 = fbar[3], sig1 = fbar[4], sig2 = fbar[5];
int first = 0;
float best = sig0;
if (sig1 > best) { first = 1; best = sig1; }
if (sig2 > best) first = 2;
const int o1 = first == 0 ? 1 : 0, o2 = first == 2 ? 1 : 2;
const float so1 = o1 == 1 ? sig1 : sig0, so2 = o2 == 1 ? sig1 : sig2;
const int second = so2 > so1 ? o2 : o1;
const float s1 = first == 0 ? sig0 : (first == 1 ? sig1 : sig2);
const float s2 = second == 0 ? sig0 : (second == 1 ? sig1 : sig2);
const bool sb1 = sig1 > sig0;
const float best = sb1 ? sig1 : sig0;
const bool sb2 = sig2 > best;
const float s1 = sb2 ? sig2 : best;
const float m1x = sb2 ? mx[2] : (sb1 ? mx[1] : mx[0]), m1y = sb2 ? my[2] : (sb1 ? my[1] : my[0]);
const bool sf0 = !or(sb1, sb2);
const float so1 = sf0 ? sig1 : sig0, so2 = sb2 ? sig1 : sig2;
const float mo1x = sf0 ? mx[1] : mx[0], mo1y = sf0 ? my[1] : my[0];
#if LP_MUT == 7
const float mo2x = sb2 ? mx[2] : mx[1], mo2y = sb2 ? my[1] : my[2]; // mutant 7: wrong candidate motion
#else
const float mo2x = sb2 ? mx[1] : mx[2], mo2y = sb2 ? my[1] : my[2];
#endif
const bool scc = so2 > so1;
const float s2 = scc ? so2 : so1;
const float m2x = scc ? mo2x : mo1x, m2y = scc ? mo2y : mo1y;
const float q1 = clampf(k1p1 * s1 - k0p05, 0.0f, 1.0f);
const float q2 = clampf(k1p1 * s2 - k0p05, 0.0f, 1.0f);
const float rden = rcp(max(q1 + q2, k1em10));
Expand All @@ -302,10 +378,13 @@ void l_post_core(LGather g, int x, int y, int fx, int fy, float mx[3], float my[
[unroll] for (int n = 0; n < 2; ++n) {
L[n][0] = 0.0f; L[n][1] = 0.0f; L[n][2] = 0.0f;
if (n < ncand) {
const int k = n == 0 ? first : second;
const float qx = X + (k == 0 ? mx[0] : (k == 1 ? mx[1] : mx[2]));
const float qy = Y + (k == 0 ? my[0] : (k == 1 ? my[1] : my[2]));
const bool valid = P_reset == 0 && qx >= 0.0f && qx <= (float)P_dw && qy >= 0.0f && qy <= (float)P_dh;
const float qx = X + (n == 0 ? m1x : m2x);
const float qy = Y + (n == 0 ? m1y : m2y);
#if LP_MUT == 5
const bool valid = and(and(and(and(P_reset == 0, qx >= 0.0f), qx <= (float)P_dh), qy >= 0.0f), qy <= (float)P_dh); // mutant 5
#else
const bool valid = and(and(and(and(P_reset == 0, qx >= 0.0f), qx <= (float)P_dw), qy >= 0.0f), qy <= (float)P_dh);
#endif
if (valid) {
const float3 h = history_cr(qx, qy);
#if LP_LDR
Expand All @@ -329,9 +408,9 @@ void l_post_core(LGather g, int x, int y, int fx, int fy, float mx[3], float my[
#if LP_LDR
Hl[i] = opq(comb);
#else
Hl[i] = opq(comb) * g_e;
Hl[i] = mul_nz(comb, g_e);
#endif
hl_inf = hl_inf || abs(Hl[i]) == asfloat(0x7f800000u);
hl_inf = or(hl_inf, abs(Hl[i]) == asfloat(0x7f800000u));
#if LP_LDR
Cl[i] = fbar[i];
#else
Expand All @@ -346,10 +425,10 @@ void l_post_core(LGather g, int x, int y, int fx, int fy, float mx[3], float my[
oh[i] = mad(Cl[i] - Hl[i], bo2, Hl[i]);
}
const float sh = hh[0] + hh[1] + hh[2], so = oh[0] + oh[1] + oh[2];
if (isnan(sh) || hl_inf) {
if (or(isnan(sh), hl_inf)) {
[unroll] for (int i = 0; i < 3; ++i) hh[i] = Cl[i];
}
if (isnan(so) || hl_inf) {
if (or(isnan(so), hl_inf)) {
[unroll] for (int i = 0; i < 3; ++i) oh[i] = Cl[i];
}
}
Expand Down Expand Up @@ -441,7 +520,7 @@ WG_KERNEL(l_post_recon)
}
GroupMemoryBarrierWithGroupSync();

const bool active = x0 + (lane & 31) < P_dw && k0 + (lane >> 5) < npairs;
const bool active = and(x0 + (lane & 31) < P_dw, k0 + (lane >> 5) < npairs);
const int x = mini(x0 + (lane & 31), P_dw - 1);
const int y0 = 2 * mini(k0 + (lane >> 5), npairs - 1);
const bool has1 = y0 + 1 < P_dh;
Expand All @@ -464,26 +543,33 @@ WG_KERNEL(l_post_recon)
[unroll] for (int jj = 0; jj < kLWin; ++jj) {
v0[jj] = dy0 + (half)(float)(jj - 1);
v1[jj] = dy1 + (half)(float)(jj - 1);
vv0[jj] = opqh(v0[jj] * v0[jj]);
vv1[jj] = opqh(v1[jj] * v1[jj]);
vv0[jj] = mulh(v0[jj], v0[jj]);
vv1[jj] = mulh(v1[jj], v1[jj]);
}
// taps column by column (i outer, j inner, each gather in that order as before); per column the eight weights (four
// rows, two pixels) are rounded to FP16 by one driver conversion
LGather g0 = gather_zero(), g1 = gather_zero();
// the staged texel's LDS index without per-tap multiplies: one base per lane, the row stride and the column
// step wave-uniform (integer identity: ((b + j - r0) ncol + (ax + i - c0)) kTexDw)
const int tb0 = ((b - 1 - r0) * ncol + (ax - 1 - c0)) * kTexDw, rstride = ncol * kTexDw;
[loop] for (int i = -1; i <= kLWin - 2; ++i) {
const half u = dx + (half)(float)i;
const half uu = opqh(u * u);
const half uu = mulh(u, u);
uint t[kLWin][kTexDw];
v8f wv;
[unroll] for (int j = -1; j <= kLWin - 2; ++j) {
if (staged) {
const int base = ((b + j - r0) * ncol + (ax + i - c0)) * kTexDw;
const int base = tb0 + (j + 1) * rstride + (i + 1) * kTexDw
#if LP_MUT == 2
+ (j == 2 && i == 2 ? kTexDw : 0) // mutant 2: one tap reads the wrong texel
#endif
;
[unroll] for (int d = 0; d < kTexDw; ++d) t[j + 1][d] = sh_tex[base + d];
} else {
load_texel_hw(clampi(b + j, 0, P_rh - 1) * P_rw + clampi(ax + i, 0, P_rw - 1), t[j + 1]);
}
wv.v[2 * (j + 1)] = tap_weight(t[j + 1], uu, vv0[j + 1], opqh(u * v0[j + 1]));
wv.v[2 * (j + 1) + 1] = tap_weight(t[j + 1], uu, vv1[j + 1], opqh(u * v1[j + 1]));
wv.v[2 * (j + 1)] = tap_weight(t[j + 1], uu, vv0[j + 1], mulh(u, v0[j + 1]));
wv.v[2 * (j + 1) + 1] = tap_weight(t[j + 1], uu, vv1[j + 1], mulh(u, v1[j + 1]));
}
const uint4 hw = wg_to_f16_ags(wv);
[unroll] for (int jj = 0; jj < kLWin; ++jj) {
Expand Down Expand Up @@ -521,7 +607,7 @@ WG_KERNEL(l_post_recon)
cv.v[7] = 0.0f;
const uint4 hc = wg_to_f16_ags(cv);
const int y = y0 + row;
if (active && (!second || has1)) {
if (and(active, or(!second, has1))) {
#if LP_LDR
const uint q0 = hc.x & 0xffffu, q1 = hc.x >> 16, q2 = hc.y & 0xffffu;
#else
Expand Down
92 changes: 92 additions & 0 deletions clean/dx12/tests/test_post_edge_cases.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: MIT
"""CPU checks of Model L's post history fast path (clean/dx12/kernels/l_post.hlsl history_cr) and of the inputs of
its GPU edge check (clean/l/runner/l_post_edge_check.py):
- the fast path's predicate as written (unsigned tests with the size guard) holds exactly when the 4 x 4 footprint
lies inside the image, at every size from 1 x 1; without the guard (the old predicate, mutant LP_MUT 9) it also
holds outside an image narrower or shorter than 4;
- each edge-check kind puts the history positions where it says (inside the footprint, outside it, or both in one
wave), so the counter expectations of the GPU check follow."""
import os
import sys
import unittest

import numpy as np

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE)))
sys.path.insert(0, ROOT)

from clean.l.runner import l_post_edge_check as E # noqa: E402


def u32(v):
return np.int64(v) & 0xFFFFFFFF


def predicate(ix, iy, dw, dh, guard=True):
"""l_post.hlsl's interior predicate of one lane (int32 arithmetic, unsigned compares)."""
p = u32(ix - 1) <= u32(dw - 4) and u32(iy - 1) <= u32(dh - 4)
return bool(p and (not guard or (dw >= 4 and dh >= 4)))


def inside(ix, iy, dw, dh):
"""Rows iy - 1 ... iy + 2 and columns ix - 1 ... ix + 2 all in the image."""
return ix - 1 >= 0 and ix + 2 <= dw - 1 and iy - 1 >= 0 and iy + 2 <= dh - 1


class PredicateTest(unittest.TestCase):
def test_guarded_predicate_is_the_footprint(self):
for dw in range(1, 10):
for dh in range(1, 10):
for ix in range(-3, 12):
for iy in range(-3, 12):
self.assertEqual(predicate(ix, iy, dw, dh), inside(ix, iy, dw, dh), (dw, dh, ix, iy))

def test_old_predicate_reads_outside(self):
bad = [(dw, dh, ix, iy) for dw in range(1, 10) for dh in range(1, 10) for ix in range(-3, 12)
for iy in range(-3, 12) if predicate(ix, iy, dw, dh, guard=False) and not inside(ix, iy, dw, dh)]
self.assertIn((5, 3, 2, 1), bad) # height 3: the span reads row 3
self.assertTrue(all(dw < 4 or dh < 4 for dw, dh, _, _ in bad))


class EdgeCasesTest(unittest.TestCase):
def footprints(self, x, dw, dh):
X, Y = np.meshgrid(np.arange(dw) + 0.5, np.arange(dh) + 0.5)
qx = X + x["m_near"][..., 0].astype(np.float64)
qy = Y + x["m_near"][..., 1].astype(np.float64)
ix, iy = np.floor(qx - 0.5).astype(int), np.floor(qy - 0.5).astype(int)
return np.vectorize(lambda a, b: inside(a, b, dw, dh))(ix, iy)

def test_kinds(self):
rng = np.random.default_rng(1)
mixed = set()
for dw, dh in E.SIZES:
for kind in E.KINDS:
if kind == "random":
continue
x = E.edge_case(rng, dw, dh, kind)
for k in ("m_center", "m_far"):
self.assertTrue(np.array_equal(x[k], x["m_near"]))
case = {"x": x, "dw": dw, "dh": dh, "kind": kind}
want = E.counter_expected(case)
rows = E.wave_rows(x, dw, dh)
if dw < 4 or dh < 4:
self.assertIs(want, False)
elif kind in ("center", "in_lo", "in_hi"):
self.assertTrue(want, (dw, dh, kind))
self.assertTrue(self.footprints(x, dw, dh).all(), (dw, dh, kind))
elif kind in ("left", "right", "out_lo", "out_hi"):
self.assertIs(want, False, (dw, dh, kind))
self.assertFalse(self.footprints(x, dw, dh).any(), (dw, dh, kind))
elif kind in ("split", "split_half"):
if any(i > 0 and o > 0 for i, o in rows):
mixed.add((dw, dh, kind))
# the mixed waves (both inside and outside lanes) appear at every size of at least 4 x 4, for both split kinds
for dw, dh in E.SIZES:
if dw >= 4 and dh >= 4:
self.assertIn((dw, dh, "split"), mixed)
self.assertIn((dw, dh, "split_half"), mixed)


if __name__ == "__main__":
unittest.main()
2 changes: 1 addition & 1 deletion clean/l/runner/l_d3d12_own_post_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ def run(build, special=True, stages=False):
rng = np.random.default_rng(F.SEED)
frames = []
for i, ((rw, rh), reset) in enumerate(F.SEQUENCE):
fr = F.frame_inputs(rng, rw, rh, reset, special)
fr = F.frame_inputs(rng, rw, rh, reset, special, index=i)
F.fill_sentinels(p, SENTINELS[config], i == 0)
out = p.frame(fr, debug=True)
frames.append({"inputs": fr, "out": out, "raw": F.raw_outputs(p), "post_last": p.kernels().get("post_last")})
Expand Down
2 changes: 1 addition & 1 deletion clean/l/runner/l_default_post_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ def run_one(build, config):
rng = np.random.default_rng(F.SEED)
frames = []
for i, ((rw, rh), reset) in enumerate(F.SEQUENCE):
fr = F.frame_inputs(rng, rw, rh, reset, True)
fr = F.frame_inputs(rng, rw, rh, reset, True, index=i)
F.fill_sentinels(p, sentinel, i == 0)
out = p.frame(fr, debug=True)
frames.append({"inputs": fr, "out": out, "raw": F.raw_outputs(p), "post_last": p.kernels().get("post_last")})
Expand Down
Loading
Loading