Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions clean/dx12/kernel_sets.json
Original file line number Diff line number Diff line change
Expand Up @@ -218,14 +218,30 @@
"l_post_filter_direct",
"-"
],
[
"l_post_filter_direct_ldr",
"-"
],
[
"l_post_filter_ldr",
"l_post.co"
],
[
"l_post_recon",
"l_post.co"
],
[
"l_post_recon_ldr",
"l_post.co"
],
[
"l_post_recon_p12",
"l_post.co"
],
[
"l_post_recon_p12_ldr",
"l_post.co"
],
[
"l_project_gather",
"l_ops.co"
Expand Down
58 changes: 57 additions & 1 deletion clean/dx12/kernels/l_post.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,23 @@
// l_post_recon_p12 (LP_P12 = 1): the same kernel on the pack12 coefficients of l_coef_pack12: A = image A8 (8 bytes per
// render pixel: codes A0 A1 A2 B0 | B1 B2, C2 FP16), C = image C4 (4 bytes per render pixel: C0, C1 FP16), B unused.
//
// LDR forms (LP_LDR = 1; l_post.hip's l_post_recon_ldr / l_post_recon_p12_ldr, spec preset L part 3 item 6, Q1, Q2):
// identity transfer (the colour feature is the driver's FP16 conversion of the colour, a NaN kept; no decode of the
// history candidates or of C), no exposure (not read: e = e_prev = 1), the full history the driver's FP16 conversion of
// H^, the display image h(code RN32(1 / N)) of the 11 / 11 / 10-bit code floor(clamp(fused(O^ N + 1/2), 0, N)) (a NaN
// gives 0), the reduced codes rint(clamp(mean, 0, 1) 1023).
//
// @wg-kernel l_post_recon co=l_post.co block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p
// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1
// @wg-kernel l_post_recon_ldr co=l_post.co entry=l_post_recon_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_LDR=1
// @wg-kernel l_post_recon_p12_ldr co=l_post.co entry=l_post_recon_p12_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_LDR=1
#include "wg_platform.hlsli"
#ifndef LP_P12
#define LP_P12 0
#endif
#ifndef LP_LDR
#define LP_LDR 0
#endif

WG_ARGS_BEGIN
uint color, A, B, C, hist_prev, m_near, m_center, m_far, w_p, hist_out, reduced, display; // pointers
Expand Down Expand Up @@ -168,9 +179,15 @@ void load_texel_hw(int idx, out uint t[kTexDw])
texel_inputs(idx, c, a, b, k);
const float k0 = f16tof32(k.x), k1 = f16tof32(k.x >> 16), k2 = f16tof32(k.y);
v8f cv;
#if LP_LDR
cv.v[0] = opq(c.x);
cv.v[1] = opq(c.y);
cv.v[2] = opq(c.z);
#else
cv.v[0] = opq(enc_in(opq(g_e * c.x)));
cv.v[1] = opq(enc_in(opq(g_e * c.y)));
cv.v[2] = opq(enc_in(opq(g_e * c.z)));
#endif
cv.v[3] = mul_odd(k0, kShapeAB);
cv.v[4] = mul_odd(k1, kShapeAB);
cv.v[5] = mul_odd(k2, kShapeC);
Expand Down Expand Up @@ -291,7 +308,11 @@ void l_post_core(LGather g, int x, int y, int fx, int fy, float mx[3], float my[
const bool valid = P_reset == 0 && qx >= 0.0f && qx <= (float)P_dw && qy >= 0.0f && qy <= (float)P_dh;
if (valid) {
const float3 h = history_cr(qx, qy);
#if LP_LDR
const float3 lq = h;
#else
const float3 lq = float3(l_dec(h.x), l_dec(h.y), l_dec(h.z)) * g_rp;
#endif
L[n][0] = lq.x;
L[n][1] = lq.y;
L[n][2] = lq.z;
Expand All @@ -305,9 +326,17 @@ void l_post_core(LGather g, int x, int y, int fx, int fy, float mx[3], float my[
bool hl_inf = false;
[unroll] for (int i = 0; i < 3; ++i) {
const float comb = ncand == 2 ? mad(w1, L[0][i], w2 * L[1][i]) : w1 * L[0][i];
#if LP_LDR
Hl[i] = opq(comb);
#else
Hl[i] = opq(comb) * g_e;
#endif
hl_inf = hl_inf || abs(Hl[i]) == asfloat(0x7f800000u);
#if LP_LDR
Cl[i] = fbar[i];
#else
Cl[i] = l_dec(fbar[i]);
#endif
}
const float alpha = clampf(opq(s * g.W) * fbar[6], 0.0f, 1.0f);
const float bh2 = mad(alpha, 1.0f - fbar[7], fbar[7]);
Expand Down Expand Up @@ -354,14 +383,23 @@ void finish_pixel(LGather g, int x, int y, float px, float py, float s, out floa
float lpx(int xx) { return mad(opq((float)xx + 0.5f), P_rx, P_jx); } // fused(x + 1/2, rx, jx)
float lpy(int yy) { return mad(P_ry, opq((float)yy + 0.5f), P_jy); } // fused(ry, y + 1/2, jy)

#if LP_P12
#if LP_P12 && LP_LDR
WG_KERNEL(l_post_recon_p12_ldr)
#elif LP_LDR
WG_KERNEL(l_post_recon_ldr)
#elif LP_P12
WG_KERNEL(l_post_recon_p12)
#else
WG_KERNEL(l_post_recon)
#endif
{
WG_SETUP();
g_z = wg_grid_ >> 31;
#if LP_LDR
g_e = 1.0f; // LDR: no exposure (the exposure pair is not read)
g_ep = 1.0f;
g_rp = 1.0f;
#else
g_e = P_e;
g_ep = P_e_prev;
if (expo != 0u) {
Expand All @@ -370,6 +408,7 @@ WG_KERNEL(l_post_recon)
g_ep = e1 == 0.0f ? 1.0f : e1;
}
g_rp = wg_div(1.0f, g_ep);
#endif
const int lane = (int)wg_lane_id();
const int tiles_x = (P_dw + kTileX - 1) / kTileX;
const int tile = (int)wg_block_id();
Expand Down Expand Up @@ -466,17 +505,30 @@ WG_KERNEL(l_post_recon)
finish_pixel(g, x, y0 + row, px, second ? py1 : py0, s, hh, oo);
v8f cv;
[unroll] for (int ci = 0; ci < 3; ++ci) {
#if LP_LDR
// h(H^); the display code floor(clamp(fused(O^ N + 1/2), 0, N)), then h(code RN32(1 / N)) (one rounding or
// two give the same FP16 value for every code 0 ... N)
const float n = ci == 2 ? 1023.0f : 2047.0f;
const float k = asfloat(ci == 2 ? 0x3a802008u : 0x3a001002u);
cv.v[ci] = opq(hh[ci]);
cv.v[3 + ci] = opq(floor(clampf(mad(oo[ci], n, 0.5f), 0.0f, n)) * k);
#else
cv.v[ci] = mul_odd(kEncK, enc_log(hh[ci]));
cv.v[3 + ci] = opq(clampf(enc_e(oo[ci]), 0.0f, kDisplayMax));
#endif
}
cv.v[6] = 0.0f;
cv.v[7] = 0.0f;
const uint4 hc = wg_to_f16_ags(cv);
const int y = y0 + row;
if (active && (!second || has1)) {
#if LP_LDR
const uint q0 = hc.x & 0xffffu, q1 = hc.x >> 16, q2 = hc.y & 0xffffu;
#else
const uint q0 = mul_zero(kEncK, enc_log(hh[0])) ? 0u : (hc.x & 0xffffu);
const uint q1 = mul_zero(kEncK, enc_log(hh[1])) ? 0u : (hc.x >> 16);
const uint q2 = mul_zero(kEncK, enc_log(hh[2])) ? 0u : (hc.y & 0xffffu);
#endif
wg_st64(hist_out, (uint)(y * P_dw + x) * 8u, uint2(q0 | (q1 << 16), q2));
const int di = y * pitch + x;
wg_st16(display, (uint)di * 2u, hc.y >> 16);
Expand All @@ -494,7 +546,11 @@ WG_KERNEL(l_post_recon)
uint code = 0u;
[unroll] for (int i = 0; i < 3; ++i) {
const float m = opq((r[i] + o[i]) * 0.25f);
#if LP_LDR
const float v = clampf(m, 0.0f, 1.0f);
#else
const float v = clampf(opq(enc_e(m)) * kInv245, 0.0f, 1.0f);
#endif
code |= wg_f2u(round(v * 1023.0f)) << (10 * i);
}
wg_st32(reduced, (uint)((y0 / 2) * (P_dw / 2) + x / 2) * 4u, code);
Expand Down
20 changes: 20 additions & 0 deletions clean/dx12/kernels/post_filter.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,14 @@
// framebuffer_half's bytes (+-65504 clamp, round to nearest even), alpha 1. The lane of channel 0 of each output pixel
// gathers the other two channels from its lanes 3 g + 1, 3 g + 2 and stores the texel.
//
// l_post_filter_ldr (WG_FILTER_LDR = 1): l_post.hip's LDR output stage l_post_filter_ldr (spec preset L part 3 Q2), the
// same filter with the filtered value written as it is (no clamp to [1e-4, 3], no decode, no exposure: dec_scale and
// expo are not read); l_post_filter_direct_ldr: its direct form, stored as l_post_filter_direct stores.
//
// @wg-kernel w64/post_filter_c1_s4 co=post_w64.co entry=post_filter_c1_s4 block=64 wave=64 args=p,p,u,u,u,u,u,p
// @wg-kernel l_post_filter_direct co=- entry=post_filter_c1_s4 block=64 wave=64 args=p,p,u,u,u,u,u,p rs=direct defines=WG_DIRECT_TEXTURES=1
// @wg-kernel l_post_filter_ldr co=l_post.co entry=post_filter_c1_s4 block=64 wave=64 args=p,p,u,u,u,u,u,p defines=WG_FILTER_LDR=1
// @wg-kernel l_post_filter_direct_ldr co=- entry=post_filter_c1_s4 block=64 wave=64 args=p,p,u,u,u,u,u,p rs=direct defines=WG_DIRECT_TEXTURES=1,WG_FILTER_LDR=1
#include "wg_platform.hlsli"
#if defined(WG_DIRECT_TEXTURES)
#include "native_direct.hlsli"
Expand Down Expand Up @@ -123,11 +129,13 @@ WG_KERNEL(post_filter_c1_s4)
{
WG_SETUP();
g_z = (uint)ow >> 31;
#if !defined(WG_FILTER_LDR)
float dec_scale = dec_scale_arg;
if (expo != 0u) {
const float e = wg_ldf(expo, 0u);
dec_scale = (float)(0.01 / (double)(e == 0.0f ? 1.0f : e));
}
#endif
const int lane = (int)wg_lane_id();
const int wl = lane & 31;
if (wl >= 3 * kGroups) return;
Expand Down Expand Up @@ -163,11 +171,19 @@ WG_KERNEL(post_filter_c1_s4)
const uint o0 = (uint)(y * ow + 4 * m);
[unroll] for (int i = 0; i < 4; ++i) {
const float r = dy == 0 ? lz5e(r0[i], r1[i], r2[i], r3[i], r4[i]) : lz5o(r2[i], r3[i], r4[i], r5[i], r6[i]);
#if defined(WG_FILTER_LDR)
const float v = r;
#else
const float qc = clampf(r, k1em4, 3.0f);
const float v = qc < kDecPolyMax ? dec_m1_poly(qc) : exp2(qc * kDecK) - 1.0f;
#endif
#if defined(WG_DIRECT_TEXTURES)
// channels 1, 2 of this pixel from the lanes after channel 0's (the same pixel, so the same loop exits)
#if defined(WG_FILTER_LDR)
const float val = v;
#else
const float val = opq(v) * dec_scale;
#endif
const uint l0 = (uint)(lane - c);
const float v1 = WaveReadLaneAt(val, l0 + 1u), v2 = WaveReadLaneAt(val, l0 + 2u);
if (c == 0 && i < ncols) {
Expand All @@ -178,8 +194,12 @@ WG_KERNEL(post_filter_c1_s4)
if (nd_output_rne16 != 0u) o = float3(fd_half(o.x), fd_half(o.y), fd_half(o.z));
native_direct_output[uint2((uint)(4 * m + i), (uint)y)] = float4(o, 1.0f);
}
#else
#if defined(WG_FILTER_LDR)
if (i < ncols) wg_stf(out_p, (o0 + (uint)i) * 12u + (uint)c * 4u, v);
#else
if (i < ncols) wg_stf(out_p, (o0 + (uint)i) * 12u + (uint)c * 4u, opq(v) * dec_scale);
#endif
#endif
}
}
Expand Down
67 changes: 60 additions & 7 deletions clean/dx12/tests/post_tf_d3d12_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,52 @@ def store_cases(rng):
return out


# --store --ldr: Model L's LDR output stage on the direct route, l_post_filter_direct_ldr, against HIP's
# l_post_filter_ldr (FP32, l_post.co) and l_post_filter_ldr_h (RGBA16F rows, framebuffer_half(v pi), pitch 8 O) on the
# same display image: the LDR display (unpacked 11 / 11 / 10-bit codes), and on frames marked wide also values beyond
# the codes' range (large ones, so the stored value passes 65504 and is clamped, and tiny ones, FP16 subnormal
# results), which the store path must take as the HDR form does. (ow, oh, pi, wide[, "f32"])
LDR_STORE_FRAMES = [(96, 54, 1.0, False), (99, 57, 1.375, False), (161, 83, 1.0, True), (161, 83, 1.375, True),
(1281, 140, 1.25, False), (40, 16, 1.0, True), (7, 5, 1.0, False), (161, 83, 30000.0, True),
(96, 54, 1.0, False, "f32"), (1281, 140, 1.0, True, "f32")]


def _display_ldr(rng, dw, dh, wide):
from clean.l.reference import recon as R
from clean.runner import post as RP
pitch = RP.display_pitch(dw)
codes = np.stack([rng.integers(0, int(n) + 1, (dh, pitch)) for n in R.LDR_CODE_MAX], -1).astype(np.float64)
v = R.ldr_unpack(codes).transpose(2, 0, 1).astype(np.float64)
if wide:
pick = rng.random(v.shape)
v = np.where(pick < 0.05, rng.uniform(30000.0, 65504.0, v.shape), v)
v = np.where((pick >= 0.05) & (pick < 0.15), rng.uniform(0.0, 2.0 ** -20, v.shape), v)
return v.astype(np.float16).tobytes()


def store_cases_ldr(rng):
"""[(HIP FP32 filter case, HIP RGBA16F-rows case or None, pi, ow, oh, label)] of LDR_STORE_FRAMES."""
out = []
for fr in LDR_STORE_FRAMES:
ow, oh, pi, wide = fr[:4]
f32tex = len(fr) > 4 and fr[4] == "f32"
dw, dh = MP.recon_size_for(ow), MP.recon_size_for(oh)
disp = _display_ldr(rng, dw, dh, wide)
n32, n16 = ow * oh * 12, ow * oh * 8
base = [("buf", disp), None, ("u32", dw), ("u32", dh), ("u32", ow), ("u32", oh), ("u32", 0), ("null",)]
grid = PZ.PS._cdiv(ow, 40) * PZ.PS._cdiv(PZ.PS._cdiv(oh, 2), 8)
f32 = {"kernel": "l_post_filter_ldr", "co": "l_post.co", "hip_kernel": "l_post_filter_ldr", "grid": grid,
"block": 64, "args": [base[0], ("out", n32 + SLACK)] + base[2:],
"defined": [np.concatenate([np.ones(n32, bool), np.zeros(SLACK, bool)])]}
h = dict(f32, kernel="l_post_filter_ldr_h", hip_kernel="l_post_filter_ldr_h",
args=[base[0], ("out", n16 + SLACK)] + base[2:] +
[("u32", _s32(_f32_bits(np.float32(pi)))), ("u32", 8 * ow)])
label = f"LDR O {ow}x{oh} pi {pi}" + (" wide" if wide else "") + \
(" (RGBA32F: l_post_filter_ldr)" if f32tex else " (l_post_filter_ldr_h)")
out.append((f32, None if f32tex else h, pi, ow, oh, label))
return out


def _store_classes(v32, pi):
"""Counts of the values stored (FP32 filter output times pi, numpy's FP32 product): NaN, beyond +-65504 (clamped;
infinities included), FP16 subnormal range, exact ties of the FP16 rounding (normal and subnormal range)."""
Expand All @@ -339,24 +385,26 @@ def _store_classes(v32, pi):
"ties": int(np.sum(normal & ((bits & 0x1FFF) == 0x1000)) + np.sum(sub & (t - np.floor(t) == 0.5)))}


def store_run(build, rocm, seeds):
def store_run(build, rocm, seeds, ldr=False):
"""ldr: Model L's LDR direct filter (LDR_STORE_FRAMES) instead of the HDR one (STORE_FRAMES)."""
import subprocess
from clean.l.runner import l_c32_check as C32
g = C32.Gpu(build, rocm)
with open(os.path.join(build, "post_w64.co"), "rb") as f:
with open(os.path.join(build, "l_post.co" if ldr else "post_w64.co"), "rb") as f:
mod = g.hip.load_module(f.read())
cdir = os.path.join(build, "replay-store")
cdir = os.path.join(build, "replay-store-ldr" if ldr else "replay-store")
os.makedirs(cdir, exist_ok=True)
lines, todo, neg = [], [], None
for si, seed in enumerate(seeds):
rng = np.random.default_rng(seed)
for ci, (f32, o16, pi, ow, oh, label) in enumerate(store_cases(rng)):
for ci, (f32, o16, pi, ow, oh, label) in enumerate(store_cases_ldr(rng) if ldr else store_cases(rng)):
name = f"store_s{si}_{ci}"
w32 = OPS.hip_run(g, mod, f32)[0]
w16 = OPS.hip_run(g, mod, o16)[0] if o16 is not None else None
lines += OPS.replay_lines(f"{name}_filter", f32, cdir)
f16 = o16 is not None
st = dict(f32, kernel="l_post_filter_direct", hip_kernel=None, args=f32["args"],
st = dict(f32, kernel="l_post_filter_direct_ldr" if ldr else "l_post_filter_direct", hip_kernel=None,
args=f32["args"],
direct=[("color", OPS.DXGI_R32G32B32A32_FLOAT, 8, 8, bytes(8 * 8 * 16)),
("depth", OPS.DXGI_R32_FLOAT, 8, 8, bytes(8 * 8 * 4)),
("motion", OPS.DXGI_R32G32_FLOAT, 8, 8, bytes(8 * 8 * 8)),
Expand Down Expand Up @@ -430,9 +478,12 @@ def tex_judge(path, w16, ow, oh, w32=None, outp_path=None):
res["summary"] = {"cases": len(res["cases"]), "identical": sum(c["identical"] for c in res["cases"]),
"texture_bytes": sum(c["store_rgba16f"].get("bytes", 0) for c in res["cases"]),
"filter_f32_bytes": sum(c["filter_f32"]["bytes"] for c in res["cases"])}
# (LDR: no exposure scale, so no NaN or infinite stored values; the clamp past 65504, subnormals and ties)
need = ("beyond_65504", "f16_subnormal_range", "ties") if ldr else \
("nan", "beyond_65504", "infinite", "f16_subnormal_range", "ties")
res["pass"] = (r.returncode == 0 and bool(neg) and res["negative_control"]["caught"] and
res["summary"]["identical"] == res["summary"]["cases"] > 0 and
all(classes.get(k, 0) > 0 for k in ("nan", "beyond_65504", "infinite", "f16_subnormal_range", "ties")))
all(classes.get(k, 0) > 0 for k in need))
return res


Expand All @@ -445,13 +496,15 @@ def main(argv=None):
ap.add_argument("--kernels", default=",".join(KINDS))
ap.add_argument("--direct", action="store_true", help="the direct forms instead (see DIRECT)")
ap.add_argument("--store", action="store_true", help="Model L's direct filter instead (see STORE_FRAMES)")
ap.add_argument("--ldr", action="store_true", help="with --store: Model L's LDR direct filter (LDR_STORE_FRAMES)")
ap.add_argument("--json")
a = ap.parse_args(argv)
if not a.run:
return 0
kernels = [k for k in a.kernels.split(",") if k]
seeds = tuple(range(89, 89 + a.seeds))
res = store_run(a.build, a.rocm, seeds) if a.store else direct_run(a.build, a.rocm, seeds) if a.direct else \
res = store_run(a.build, a.rocm, seeds, ldr=a.ldr) if a.store else \
direct_run(a.build, a.rocm, seeds) if a.direct else \
OPS.run(a.build, a.rocm, seeds, kernels, kinds=KINDS)
if a.json:
with open(a.json, "w") as f:
Expand Down
Loading
Loading