Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions clean/dx12/kernels/post.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,21 @@
// @wg-kernel w64/post_tf4_dl_f32_p12_diet_vsit_direct co=post_w64__orig.co rs=direct entry=post_tf_f32_p12_diet_vsit_direct block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u,u,u,u,u,u defines=WG_DIRECT_TEXTURES=1,POST_OUT=3,WG_TF_NW=4,WG_TF_CAP=448
// @wg-kernel pmo/post_tf4_dl_f32_p12_mo co=post_mo_w64__orig.co entry=post_tf_f32_p12_diet_vsit block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u,b40,u,u,u,u,u defines=WG_POST_MO=1,WG_TF_NW=4,WG_TF_CAP=448
// @wg-kernel pmo/post_tf4_dl_f32_p12_mo_direct co=post_mo_w64__orig.co rs=direct entry=post_tf_f32_p12_diet_vsit_direct block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u,b40,u,u,u,u,u defines=WG_DIRECT_TEXTURES=1,POST_OUT=3,WG_POST_MO=1,WG_TF_NW=4,WG_TF_CAP=448
//
// Analytic output filter (WG_LZ_ANALYTIC = 1; HIP: post_w64.co, post.hip built with -mwavefrontsize64 and the default
// WG_LZ_PROFILE = 0, the build the runtime loads under "w64" when the model file has no output-filter weights, e.g.
// Model L's): the same kernels with spec 06's own Lanczos weights as FP32 constants (kLz0 ... kLz4): v = kLz0 t0, then
// one fused multiply-add per tap, clamp to the taps' range, every value FP32 (the horizontal results go to the ring as
// FP32, the vertical result straight into Dec). No weight arguments. Buffer forms 0 only (POST_OUT = 0).
// @wg-kernel w64/post_tf_f32_p12_diet_vsit co=post_w64.co file=w64__post_tf_f32_p12_diet_vsit__analytic entry=post_tf_f32_p12_diet_vsit block=512 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u defines=WG_LZ_ANALYTIC=1
// @wg-kernel w64/post_tf4_dl_f32_p12_diet_vsit co=post_w64.co file=w64__post_tf4_dl_f32_p12_diet_vsit__analytic entry=post_tf_f32_p12_diet_vsit block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u defines=WG_TF_NW=4,WG_TF_CAP=448,WG_LZ_ANALYTIC=1
#include "wg_platform.hlsli"
#ifndef WG_LZ_ANALYTIC
#define WG_LZ_ANALYTIC 0
#endif
#if WG_LZ_ANALYTIC && (POST_OUT != 0 || defined(WG_POST_MO))
#error "analytic output filter: buffer form 0 only"
#endif
#if defined(WG_DIRECT_TEXTURES)
#include "native_direct.hlsli"
#include "wg_f16lut.hlsli"
Expand Down Expand Up @@ -85,7 +99,9 @@ WG_ARGS_BEGIN
int mo_mw, mo_mh, mo_pitch, mo_wide; // WgMoParams (40 bytes)
float mo_sx, mo_sy, mo_bx, mo_by, mo_idw, mo_idh;
#endif
#if !WG_LZ_ANALYTIC
uint lz_w0, lz_w1, lz_w2, lz_w3, lz_w4; // FP16 bit patterns of the filter weights of taps -2 ... +2
#endif
WG_ARGS_END

// ---- constants (FP32 bit patterns of post.hip's constexpr values) ---------------------------------------------------
Expand Down Expand Up @@ -697,6 +713,26 @@ void mo_core(Gather g, float2 mv, int x, int y, int fx, int fy, float s, out flo
}
#endif

#if WG_LZ_ANALYTIC
// ---- output filter, profile 0 ("analytic", post_w64.co): FP32 constant weights of taps -2 ... +2 for an even output
// index (post.hip kLz0 ... kLz4; mirrored for an odd one), v = w0 t0, one fma per further tap, clamp to the taps' range
static const float kLz0 = asfloat(0xbdba3b4bu), kLz1 = asfloat(0x3de71ca9u), kLz2 = asfloat(0x3f23b70fu);
static const float kLz3 = asfloat(0x3ed182b5u), kLz4 = asfloat(0xbd90a4acu);
float lz5a(float t0, float t1, float t2, float t3, float t4, float w0, float w1, float w3, float w4)
{
float v = w0 * t0;
v = mad(w1, t1, v);
v = mad(kLz2, t2, v);
v = mad(w3, t3, v);
v = mad(w4, t4, v);
const float mid_lo = min(min(t1, t2), t3), mid_hi = max(max(t1, t2), t3);
const float lo = min(min(mid_lo, t0), t4), hi = max(max(mid_hi, t0), t4);
return min(max(v, lo), hi); // med3 (the taps are finite, lo <= hi)
}
// lz5<kEven> / lz5<false> on the FP32 taps t[o ... o + 4], as FP32 bits
uint lz5a_even(float t[10], int o) { return asuint(lz5a(t[o], t[o + 1], t[o + 2], t[o + 3], t[o + 4], kLz0, kLz1, kLz3, kLz4)); }
uint lz5a_odd(float t[10], int o) { return asuint(lz5a(t[o], t[o + 1], t[o + 2], t[o + 3], t[o + 4], kLz4, kLz3, kLz1, kLz0)); }
#else
// ---- output filter, profile 2 ("original"): FP16 weights, one FP16 fma per tap, clamp to the taps' range -------------
// Values are "boxes": FP16 bits in the low half of a dword.
half hbox(uint v) { return asfloat16((uint16_t)(v & 0xffffu)); }
Expand Down Expand Up @@ -726,6 +762,7 @@ uint lz5_odd(uint t[10], int o)
return lz5_original(t[o], t[o + 1], t[o + 2], t[o + 3], t[o + 4], hbox(lz_w4), hbox(lz_w3), hbox(lz_w2), hbox(lz_w1),
hbox(lz_w0));
}
#endif

// The reduced history's 10-bit code of one channel from its 2 x 2 sum (the chain of post.hip's reduced store)
uint rh_code(float sum)
Expand Down Expand Up @@ -1173,6 +1210,18 @@ WG_KERNEL(post_tf_p12_diet_vsit)
#else
const uint src = scr + (uint)hk * kRowDw + (uint)hc * 32u + (uint)hg * 3u;
#endif
#if WG_LZ_ANALYTIC
float t[10]; // the FP16 display taps as FP32 values (exact); results FP32 bits
[unroll] for (int i = 0; i < 5; ++i) {
const uint q = lds[src + (uint)i];
t[2 * i] = f16tof32(q);
t[2 * i + 1] = f16tof32(q >> 16);
}
hv[0] = lz5a_even(t, 0);
hv[1] = lz5a_odd(t, 2);
hv[2] = lz5a_even(t, 3);
hv[3] = lz5a_odd(t, 5);
#else
uint t[10];
[unroll] for (int i = 0; i < 5; ++i) {
const uint q = lds[src + (uint)i];
Expand All @@ -1183,6 +1232,7 @@ WG_KERNEL(post_tf_p12_diet_vsit)
hv[1] = lz5_odd(t, 2);
hv[2] = lz5_even(t, 3);
hv[3] = lz5_odd(t, 5);
#endif
}
tf_row_sync();
if (hact) {
Expand Down Expand Up @@ -1255,13 +1305,22 @@ WG_KERNEL(post_tf_p12_diet_vsit)
}
// emit (n = 1): weights of the lane's phase by selects (lz_wsel), lz5w, Dec, store 3 x f32
const bool ev = dy == 0;
#if WG_LZ_ANALYTIC
const float w0 = ev ? kLz0 : kLz4, w4 = ev ? kLz4 : kLz0, w1 = ev ? kLz1 : kLz3, w3 = ev ? kLz3 : kLz1;
#else
const half w0 = hbox(ev ? lz_w0 : lz_w4), w4 = hbox(ev ? lz_w4 : lz_w0);
const half w1 = hbox(ev ? lz_w1 : lz_w3), w3 = hbox(ev ? lz_w3 : lz_w1);
#endif
float vo[3], qv[3];
bool vsmall = false;
[unroll] for (int c = 0; c < 3; ++c) {
#if WG_LZ_ANALYTIC
qv[c] = clampf(lz5a(asfloat(t[0][c]), asfloat(t[1][c]), asfloat(t[2][c]), asfloat(t[3][c]), asfloat(t[4][c]),
w0, w1, w3, w4), k1em4, 3.0f);
#else
const uint r = lz5_original(t[0][c], t[1][c], t[2][c], t[3][c], t[4][c], w0, w1, hbox(lz_w2), w3, w4);
qv[c] = clampf(f16tof32(r), k1em4, 3.0f);
#endif
vo[c] = exp2(qv[c] * kDecK) - 1.0f;
vsmall = or(vsmall, qv[c] < kDecPolyMax);
}
Expand Down
138 changes: 138 additions & 0 deletions clean/dx12/tests/post_tf_d3d12_check.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
# SPDX-License-Identifier: MIT
"""The analytic-filter D3D12 ports of Model M's fused output kernels (clean/dx12/kernels/post.hlsl, WG_LZ_ANALYTIC)
against HIP's post_w64.co on the same synthetic inputs, byte for byte; the original-filter port (post_w64__orig.co,
already proven on whole frames by clean/dx12/compare.py) as the Model M control on the same harness.

python clean/dx12/tests/post_tf_d3d12_check.py --build <dir> --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --run
[--seeds N] --json out.json

<dir> holds post_w64.co and post_w64__orig.co (clean/runner/build.py's builds of kernels/post.hip), the D3D12 ports
(clean/dx12/build_dx12.py --only w64/post_tf_f32_p12_diet_vsit,w64/post_tf4_dl_f32_p12_diet_vsit,native_fill: both
builds of each name) and kernel_replay.exe (clean/l/runner/l_d3d12_check.py prepare_replay). --run runs in one GPU slot.
The engine and rules are clean/l/runner/l_d3d12_ops_check.py's: the RAW downloaded bytes of every output (the full
history with its alpha, the reduced codes, the FP32 output image) are compared; each output buffer starts filled with
its back end's own sentinel (HIP 0xA5, D3D12 0x5A) and 64 slack bytes follow it, which no back end may write; a negative
control (one case again on D3D12 with half its workgroups) must be caught. Output: counts only, no timing.

Kernels:
- w64/post_tf_f32_p12_diet_vsit (analytic port) vs HIP post_tf_f32_p12_diet_vsit of post_w64.co;
- w64/post_tf4_dl_f32_p12_diet_vsit (analytic port; D3D12 only, NW = 4 row pairs per step, 256 lanes) vs HIP
post_tf_f32_p12_diet_vsit (frames that its cap of 352 takes) or post_tf_dl_f32_p12 (native AA: D = 1.5 R), whose
every output pixel is the same function (the exposure pair holds the ratio and dec_scale that the non-diet kernel
computes itself);
- control: w64/post_tf_f32_p12_diet_vsit (original-filter port, five FP16 weights as arguments) vs post_w64__orig.co.
The same name as the first: the D3D12 device runs the port of the code object the case loads under "w64".
"""
import argparse
import json
import os
import sys

import numpy as np

HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE)))
sys.path.insert(0, ROOT)

from clean.l.runner import l_d3d12_ops_check as OPS # noqa: E402
from clean.reference import post as MP # noqa: E402
from clean.runner import post as RP # noqa: E402
from clean.runner import post_sizes as PZ # noqa: E402

SLACK = 64
CPAIRS = 68 # runtime.cpp kTfCpairs
# (rw, rh, ow, oh, reset): Quality (D = 2.25 R), Performance (D = 3 R), odd sizes, reset, a wide frame of 64 strips whose
# chunk plan has k1 > 0 chunks of 68 row pairs and then chunks of 36
TF_FRAMES = [(64, 36, 96, 54, False), (48, 27, 96, 54, False), (50, 29, 99, 57, True), (70, 41, 105, 61, False),
(1280, 187, 2560, 280, False)]
# post_tf4: Quality frames (HIP post_tf_f32_p12_diet_vsit takes them too) and native AA (D = 1.5 R: HIP post_tf_dl)
TF4_FRAMES = [(64, 36, 96, 54, False), (50, 29, 99, 57, True), (64, 36, 64, 36, False), (61, 33, 61, 33, False),
(96, 54, 96, 54, True)]
# FP16 bits of the original filter's five weights for the control (spec 06 draft 4 gives them from the model file; any
# FP16 set exercises the port)
ORIG_W = [np.float16(v).view(np.uint16) for v in (-0.0908, 0.1129, 0.6400, 0.4092, -0.0706)]


def _f32_bits(v):
return int(np.float32(v).view(np.uint32))


def tf_case(kernel, rng, frame, co, hip_kernel, block, hip_block, taps, expo_pair):
rw, rh, ow, oh, reset = frame
dw, dh = MP.recon_size_for(ow), MP.recon_size_for(oh)
x = MP.random_inputs(rng, rw, rh, dw, dh, reset=reset)
prm = RP.post_params(x)
data = RP.pack12_inputs(RP.post_device_inputs(x))
k1, c2, nch = PZ.tf_chunks(ow, oh, CPAIRS)
e, ep = prm.e, prm.e_prev
dec_scale = float(np.float32(0.01 / e))
ratio = float(np.float32(np.float64(e) / np.float64(ep)))
n_hist, n_red, n_out = dw * dh * 8, max(4, (dw // 2) * (dh // 2) * 4), ow * oh * 12
red_def = np.zeros(n_red, bool)
red_def[:(dw // 2) * (dh // 2) * 4] = True
slack = np.zeros(SLACK, bool)
defined = [np.concatenate([np.ones(n_hist, bool), slack]), np.concatenate([red_def, slack]),
np.concatenate([np.ones(n_out, bool), slack])]
args = [("buf", data[k]) for k in ("color", "A")] + [("null",)]
args += [("buf", data[k]) for k in ("C", "hist_prev", "m_near", "m_center", "m_far")]
args += [("buf", RP.pack({"s": np.array(rng.uniform(0.5, 4.0))})), ("out", n_hist + SLACK), ("out", n_red + SLACK),
("out", n_out + SLACK), ("bytes", bytes(prm))]
args += [("buf", np.array([e, ep, 0.0, ratio, dec_scale], np.float32).tobytes())] if expo_pair else [("null",)]
args += [("u32", ow), ("u32", oh), ("u32", _f32_bits(dec_scale)), ("u32", _f32_bits(1.0)), ("u32", 12 * ow),
("u32", CPAIRS), ("u32", k1), ("u32", c2)]
if taps:
args += [("u32", int(w)) for w in ORIG_W]
return {"kernel": kernel, "co": co, "tag": "w64", "hip_kernel": hip_kernel, "grid": PZ.PS._cdiv(ow, 40) * nch,
"block": block, "hip_block": hip_block, "args": args, "defined": defined,
"label": f"{rw}x{rh}->{ow}x{oh} (D {dw}x{dh})" + (" reset" if reset else "") + (" expo" if expo_pair else "")}


def tf_cases(kernel, rng):
return [tf_case(kernel, rng, f, "post_w64.co", "post_tf_f32_p12_diet_vsit", 512, 512, False, i % 2 == 0)
for i, f in enumerate(TF_FRAMES)]


def tf4_cases(kernel, rng):
out = []
for i, f in enumerate(TF4_FRAMES):
rw, rh, ow, oh, _ = f
dw, dh = MP.recon_size_for(ow), MP.recon_size_for(oh)
hk = "post_tf_f32_p12_diet_vsit" if RP.tf_supported(rw, rh, dw, dh) else "post_tf_dl_f32_p12"
assert RP.tf_supported(rw, rh, dw, dh, 4, RP.TF4_DL_CAP), f
# the non-diet HIP kernel computes the ratio and takes dec_scale from its argument: the exposure pair is given
out.append(tf_case(kernel, rng, f, "post_w64.co", hk, 256, 512, False, True if hk == "post_tf_dl_f32_p12"
else i % 2 == 0))
return out


def control_cases(kernel, rng):
return [dict(tf_case(kernel, rng, f, "post_w64__orig.co", "post_tf_f32_p12_diet_vsit", 512, 512, True, i % 2 == 0),
kernel="w64/post_tf_f32_p12_diet_vsit") for i, f in enumerate(TF_FRAMES[:4])]


KINDS = {"w64/post_tf_f32_p12_diet_vsit": tf_cases, "w64/post_tf4_dl_f32_p12_diet_vsit": tf4_cases,
"orig_w64/post_tf_f32_p12_diet_vsit": control_cases}


def main(argv=None):
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
ap.add_argument("--build", required=True)
ap.add_argument("--rocm", required=True)
ap.add_argument("--run", action="store_true")
ap.add_argument("--seeds", type=int, default=2)
ap.add_argument("--kernels", default=",".join(KINDS))
ap.add_argument("--json")
a = ap.parse_args(argv)
if not a.run:
return 0
kernels = [k for k in a.kernels.split(",") if k]
res = OPS.run(a.build, a.rocm, tuple(range(89, 89 + a.seeds)), kernels, kinds=KINDS)
if a.json:
with open(a.json, "w") as f:
json.dump(res, f, indent=1)
print({k: v for k, v in res.items() if k not in ("cases", "replay_tail")})
return 0 if res["pass"] else 1


if __name__ == "__main__":
sys.exit(main())
9 changes: 6 additions & 3 deletions clean/l/runner/l_d3d12_ops_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,8 @@ def hip_run(g, mod, case):
elif a[0] == "bytes":
args.append((ctypes.c_ubyte * len(a[1])).from_buffer_copy(a[1]))
try:
h.launch(h.get_function(mod, case.get("hip_kernel", case["kernel"])), case["grid"], case["block"], args)
h.launch(h.get_function(mod, case.get("hip_kernel", case["kernel"])), case["grid"],
case.get("hip_block", case["block"]), args)
h.sync()
return [bytes(h.download(p, n)) for p, n in outs]
finally:
Expand Down Expand Up @@ -371,7 +372,9 @@ def replay_lines(name, case, cdir, grid=None, prefix=""):
return lines + [""]


def run(build, rocm, seeds, kernels):
def run(build, rocm, seeds, kernels, kinds=None):
"""kinds: {kernel: case generator} (default this file's KINDS and CONTROL_KINDS). A case may name its HIP kernel
(hip_kernel) and HIP block size (hip_block) when the D3D12 port has none of its own."""
g = C32.Gpu(build, rocm)
mods = {}

Expand All @@ -381,7 +384,7 @@ def module(co):
mods[co] = g.hip.load_module(f.read())
return mods[co]

kinds = dict(KINDS, **CONTROL_KINDS)
kinds = kinds or dict(KINDS, **CONTROL_KINDS)
cdir = os.path.join(build, "replay-ops")
os.makedirs(cdir, exist_ok=True)
lines, hip_out, neg = [], {}, None
Expand Down
6 changes: 4 additions & 2 deletions clean/tests/test_post_tf_geometry.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,15 +161,17 @@ def test_nw4_takes_every_ratio_from_1_to_1_5(self):

def test_nw4_declarations_agree(self):
"""runtime.cpp's tf4_supported (cap, NW = 4 footprint), runner/post.py's TF4_DL_CAP and the four post_tf4_dl_*
kernels of dx12/kernels/post.hlsl (NW = 4, the same cap, 256 lanes) say the same."""
kernels of dx12/kernels/post.hlsl (NW = 4, the same cap, 256 lanes) say the same. A name may be declared once per
build it ports (the original filter's post_w64__orig.co, the analytic filter's post_w64.co): every declaration
line is checked."""
import re
clean = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
rt = open(os.path.join(clean, "runtime", "runtime.cpp"), encoding="utf-8").read()
self.assertIn("constexpr int kTf4Cap = %d;" % PS.TF4_DL_CAP, rt)
self.assertIn("((long long)(63.0 * rw / dw) + 4) * ((long long)(6.0 * rh / dh) + 5) <= kTf4Cap", rt)
hlsl = open(os.path.join(clean, "dx12", "kernels", "post.hlsl"), encoding="utf-8").read()
decl = re.findall(r"(?m)^// @wg-kernel (\S*post_tf4_dl\S*) .*$", hlsl)
self.assertEqual(sorted(decl), ["pmo/post_tf4_dl_f32_p12_mo", "pmo/post_tf4_dl_f32_p12_mo_direct",
self.assertEqual(sorted(set(decl)), ["pmo/post_tf4_dl_f32_p12_mo", "pmo/post_tf4_dl_f32_p12_mo_direct",
"w64/post_tf4_dl_f32_p12_diet_vsit", "w64/post_tf4_dl_f32_p12_diet_vsit_direct"])
for line in re.findall(r"(?m)^// @wg-kernel \S*post_tf4_dl\S* .*$", hlsl):
self.assertIn(" block=256 ", line)
Expand Down
Loading