From a5cbb01beadf443f5b63f144443b88fe777e1afe Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:50:44 -0500 Subject: [PATCH] D3D12: the analytic standalone output filter w64/post_filter_c1_s4 (post_w64.co), byte for byte against HIP post_filter.hlsl ports filter_strip_c1<4> of post_w64.co (the runtime's default unbanded filter, run after a reconstruction kernel that writes the display image, e.g. Model L's l_post_recon_p12, when the model file has no output-filter weights): the analytic filter of post.hlsl, the 7-row window as named vectors (a shifted local array compiled to a block copy that broke the overlapping shift). post_tf_d3d12_check.py: its cases against HIP post_filter_c1_s4. Refs #89 Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/kernels/post_filter.hlsl | 153 ++++++++++++++++++++++++ clean/dx12/tests/post_tf_d3d12_check.py | 42 ++++++- 2 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 clean/dx12/kernels/post_filter.hlsl diff --git a/clean/dx12/kernels/post_filter.hlsl b/clean/dx12/kernels/post_filter.hlsl new file mode 100644 index 00000000..119d3939 --- /dev/null +++ b/clean/dx12/kernels/post_filter.hlsl @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of the standalone output filter w64/post_filter_c1_s4 of post_w64.co (clean/kernels/post.hip +// filter_strip_c1<4>, built with -mwavefrontsize64 and the analytic filter, WG_LZ_PROFILE = 0), byte for byte. The +// runtime's default unbanded filter (kDefaultFilter): it runs after a reconstruction kernel that writes the display image +// (for example Model L's l_post_recon_p12) when the model file has no output-filter weights. +// display: three FP16 planes of D (E-encoded), row pitch display_pitch(dw) = (dw + 63) & ~63 halves; out: O, 3 x f32 +// (row-major RGB); dec_scale = 0.01 / e, or (float)(0.01 / (double)e) from the exposure pair when expo is not null. +// Workgroup of 64 lanes (one wave64): lane 3 g + c of each 32-lane half (g < 10; lanes 30, 31 idle) computes channel c of +// output columns 4m ... 4m + 3, m = 10 (tile column) + g, for 4 row pairs from p0 = (2 (tile row) + half) 4. Grid +// ceil(ow / 40) * ceil(ceil(oh / 2) / 8). +// Per value: the horizontal pass of five display taps per output column (v = w0 t0, one fused multiply-add per further +// tap, clamp to the taps' range; spec 06's Lanczos weights kLz0 ... kLz4 as FP32 constants, mirrored for an odd output +// index), a 7-row sliding window of the horizontal results, the vertical pass of the same form, clamp to [1e-4, 3], +// Dec (2^(kDecK q) - 1, the polynomial below kDecPolyMax), times dec_scale. The arithmetic of post.hlsl's analytic +// filter (WG_LZ_ANALYTIC), which is HIP's ISA. +// +// @wg-kernel w64/post_filter_c1_s4 co=post_w64.co entry=post_filter_c1_s4 block=64 wave=64 args=p,p,u,u,u,u,u,p +#include "wg_platform.hlsli" + +WG_ARGS_BEGIN + uint display, out_p; // pointers + int dw, dh, ow, oh; + float dec_scale_arg; + uint expo; // pointer (0 = null) +WG_ARGS_END + +static const float kLz0 = asfloat(0xbdba3b4bu), kLz1 = asfloat(0x3de71ca9u), kLz2 = asfloat(0x3f23b70fu); +static const float kLz3 = asfloat(0x3ed182b5u), kLz4 = asfloat(0xbd90a4acu); +static const float kDecK = asfloat(0x4113a428u); // 3.32192809488736235f / 0.36f +static const float kDecLn = asfloat(0x40ccac9au); // 2.30258509299404568f / 0.36f +static const float kDecPolyMax = asfloat(0x3d60233du); // 0.35f / kDecLn +static const float kP5040 = asfloat(0x39500d01u), kP720 = asfloat(0x3ab60b61u), kP120 = asfloat(0x3c088889u); +static const float kP24 = asfloat(0x3d2aaaabu), kP6 = asfloat(0x3e2aaaabu); +static const float k1em4 = asfloat(0x38d1b717u); // 1e-4f + +static uint g_z; // 0, opaque to the compiler (from ow > 0) +float opq(float x) { return asfloat(asuint(x) ^ g_z); } +int clampi(int v, int lo, int hi) { int t = v > hi ? hi : v; return t < lo ? lo : t; } +float clampf(float v, float lo, float hi) { return min(max(v, lo), hi); } + +float dec_m1_poly(float qc) +{ + float y = qc * kDecLn; + float t = mad(y, kP5040, kP720); + t = mad(t, y, kP120); + t = mad(t, y, kP24); + t = mad(t, y, kP6); + t = mad(t, y, 0.5f); + t = mad(t, y, 1.0f); + return t * y; +} + +// lz5 of post.hip, profile 0: v = w0 t0, then fma per tap; med3 with the taps' range (the taps are finite) +float lz5a(float t0, float t1, float t2, float t3, float t4, float w0, float w1, float w3, float w4) +{ + float v = w0 * t0; + v = mad(w1, t1, v); + v = mad(kLz2, t2, v); + v = mad(w3, t3, v); + v = mad(w4, t4, v); + const float mid_lo = min(min(t1, t2), t3), mid_hi = max(max(t1, t2), t3); + const float lo = min(min(mid_lo, t0), t4), hi = max(max(mid_hi, t0), t4); + return min(max(v, lo), hi); +} +float lz5e(float t0, float t1, float t2, float t3, float t4) { return lz5a(t0, t1, t2, t3, t4, kLz0, kLz1, kLz3, kLz4); } +float lz5o(float t0, float t1, float t2, float t3, float t4) { return lz5a(t0, t1, t2, t3, t4, kLz4, kLz3, kLz1, kLz0); } + +// filter_row_c1: horizontal results of display row r, channel c, for output columns 4m ... 4m + 3 (taps 6m - 2 ... 6m + 7) +void row_c1(int pitch, int plane, int r, int m, bool fast, int c, out float h[4]) +{ + const uint row = (uint)(r * pitch + c * plane) * 2u; + const int c0 = 6 * m - 2; + uint q[5]; + [unroll] for (int i = 0; i < 5; ++i) { + if (fast) { // c0 even, pitch and plane even: 4-byte aligned + q[i] = wg_ld32(display, row + (uint)(c0 + 2 * i) * 2u); + } else { + q[i] = wg_ld16(display, row + (uint)clampi(c0 + 2 * i, 0, dw - 1) * 2u) | + (wg_ld16(display, row + (uint)clampi(c0 + 2 * i + 1, 0, dw - 1) * 2u) << 16); + } + } + float t[10]; + [unroll] for (int j = 0; j < 5; ++j) { + t[2 * j] = f16tof32(q[j]); + t[2 * j + 1] = f16tof32(q[j] >> 16); + } + h[0] = lz5e(t[0], t[1], t[2], t[3], t[4]); + h[1] = lz5o(t[2], t[3], t[4], t[5], t[6]); + h[2] = lz5e(t[3], t[4], t[5], t[6], t[7]); + h[3] = lz5o(t[5], t[6], t[7], t[8], t[9]); +} + +float4 row4(int pitch, int plane, int r, int m, bool fast, int c) +{ + float h[4]; + row_c1(pitch, plane, r, m, fast, c, h); + return float4(h[0], h[1], h[2], h[3]); +} + +#define kS 4 // row pairs per lane (post_filter_c1_s4) +#define kGroups 10 // column groups per 32-lane half + +WG_KERNEL(post_filter_c1_s4) +{ + WG_SETUP(); + g_z = (uint)ow >> 31; + float dec_scale = dec_scale_arg; + if (expo != 0u) { + const float e = wg_ldf(expo, 0u); + dec_scale = (float)(0.01 / (double)(e == 0.0f ? 1.0f : e)); + } + const int lane = (int)wg_lane_id(); + const int wl = lane & 31; + if (wl >= 3 * kGroups) return; + const int g = wl / 3, c = wl - 3 * g; + const int tiles_x = (ow + 4 * kGroups - 1) / (4 * kGroups); + const int tile = (int)wg_block_id(); + const int m = (tile % tiles_x) * kGroups + g; + const int p0 = ((tile / tiles_x) * 2 + (lane >> 5)) * kS; + if (4 * m >= ow || 2 * p0 >= oh) return; + const int pitch = (dw + 63) & ~63, plane = pitch * dh; + const bool fast = m >= 1 && 6 * m + 7 <= dw - 1; + const int ncols = ow - 4 * m < 4 ? ow - 4 * m : 4; + + // the sliding window: horizontal results of rows 3p - 2 + k in r0 ... r6 (named vectors, shifted one by one: a + // shifted local array was compiled into a block copy that does not keep an overlapping shift's order) + float4 r0 = 0.0f, r1 = 0.0f, r2 = 0.0f, r3 = 0.0f, r4 = 0.0f, r5 = 0.0f, r6 = row4(pitch, plane, + clampi(3 * p0 - 2, 0, dh - 1), m, + fast, c); + [loop] for (int st = -1; st < kS; ++st) { + const int p = p0 + st; + if (2 * p >= oh) break; + r0 = r3; + r1 = r4; + r2 = r5; + r3 = r6; + r4 = row4(pitch, plane, clampi(3 * p + 2, 0, dh - 1), m, fast, c); + r5 = row4(pitch, plane, clampi(3 * p + 3, 0, dh - 1), m, fast, c); + r6 = row4(pitch, plane, clampi(3 * p + 4, 0, dh - 1), m, fast, c); + if (st < 0) continue; + [unroll] for (int dy = 0; dy < 2; ++dy) { + const int y = 2 * p + dy; + if (y >= oh) break; + const uint o0 = (uint)(y * ow + 4 * m); + [unroll] for (int i = 0; i < 4; ++i) { + const float r = dy == 0 ? lz5e(r0[i], r1[i], r2[i], r3[i], r4[i]) : lz5o(r2[i], r3[i], r4[i], r5[i], r6[i]); + const float qc = clampf(r, k1em4, 3.0f); + const float v = qc < kDecPolyMax ? dec_m1_poly(qc) : exp2(qc * kDecK) - 1.0f; + if (i < ncols) wg_stf(out_p, (o0 + (uint)i) * 12u + (uint)c * 4u, opq(v) * dec_scale); + } + } + } +} diff --git a/clean/dx12/tests/post_tf_d3d12_check.py b/clean/dx12/tests/post_tf_d3d12_check.py index af96264e..610a137a 100644 --- a/clean/dx12/tests/post_tf_d3d12_check.py +++ b/clean/dx12/tests/post_tf_d3d12_check.py @@ -20,6 +20,8 @@ post_tf_f32_p12_diet_vsit (frames that its cap of 352 takes) or post_tf_dl_f32_p12 (native AA: D = 1.5 R), whose every output pixel is the same function (the exposure pair holds the ratio and dec_scale that the non-diet kernel computes itself); +- w64/post_filter_c1_s4 (the analytic standalone filter, clean/dx12/kernels/post_filter.hlsl) vs HIP post_filter_c1_s4 + of post_w64.co: display image -> FP32 output, dec_scale from the argument or the exposure pair; - control: w64/post_tf_f32_p12_diet_vsit (original-filter port, five FP16 weights as arguments) vs post_w64__orig.co. The same name as the first: the D3D12 device runs the port of the code object the case loads under "w64". """ @@ -110,8 +112,46 @@ def control_cases(kernel, rng): kernel="w64/post_tf_f32_p12_diet_vsit") for i, f in enumerate(TF_FRAMES[:4])] +# (ow, oh, expo) of the standalone filter: D = recon_size_for(O); widths with partly filled tile columns (ow % 40), one +# column group of 4 cut short (ow % 4), odd heights (a last row pair of one row), several tile rows, and a wide frame +FILTER_FRAMES = [(96, 54, False), (99, 57, True), (40, 16, False), (161, 83, True), (7, 5, False), (1281, 140, True)] + + +def _display(rng, dw, dh): + """Three FP16 planes of display_pitch(dw) x dh: E values in [0, 2.45] (clamp(E(o), 0, 2.45)), with exact zeros, the + top value and values below Dec's polynomial bound (q < 0.0547) mixed in; the pitch padding random as well (never a + tap).""" + from clean.runner import post as RP + pitch = RP.display_pitch(dw) + v = rng.uniform(0.0, 2.45, (3, dh, pitch)) + pick = rng.random((3, dh, pitch)) + v = np.where(pick < 0.08, 0.0, v) + v = np.where((pick >= 0.08) & (pick < 0.12), 2.45, v) + v = np.where((pick >= 0.12) & (pick < 0.3), rng.uniform(0.0, 0.06, v.shape), v) + return v.astype(np.float16).tobytes() + + +def filter_cases(kernel, rng): + """w64/post_filter_c1_s4 (analytic, post_w64.co) vs HIP post_filter_c1_s4: display -> out (O, 3 x f32), with + dec_scale from the argument or from the exposure pair (expo[0]; 0 means 1).""" + out = [] + for ow, oh, expo in FILTER_FRAMES: + dw, dh = MP.recon_size_for(ow), MP.recon_size_for(oh) + e = float(np.float32(rng.uniform(0.5, 2.0))) + n_out = ow * oh * 12 + args = [("buf", _display(rng, dw, dh)), ("out", n_out + SLACK), ("u32", dw), ("u32", dh), ("u32", ow), + ("u32", oh), ("u32", _f32_bits(np.float32(0.01 / e)))] + args += [("buf", np.array([0.0 if rng.random() < 0.3 else e, 1.0, 0.0, 0.0, 0.0], np.float32).tobytes())] \ + if expo else [("null",)] + out.append({"kernel": kernel, "co": "post_w64.co", "tag": "w64", "hip_kernel": "post_filter_c1_s4", + "grid": PZ.PS._cdiv(ow, 40) * PZ.PS._cdiv(PZ.PS._cdiv(oh, 2), 8), "block": 64, "args": args, + "defined": [np.concatenate([np.ones(n_out, bool), np.zeros(SLACK, bool)])], + "label": f"O {ow}x{oh} (D {dw}x{dh})" + (" expo" if expo else "")}) + return out + + KINDS = {"w64/post_tf_f32_p12_diet_vsit": tf_cases, "w64/post_tf4_dl_f32_p12_diet_vsit": tf4_cases, - "orig_w64/post_tf_f32_p12_diet_vsit": control_cases} + "orig_w64/post_tf_f32_p12_diet_vsit": control_cases, "w64/post_filter_c1_s4": filter_cases} # ---- the direct forms (rs=direct: the colour from a texture, the output into a texture) ------------------------------- # No HIP kernel of their own: form 3 is form 0 followed by the output store (post.hlsl). With color_pi 1, an RGBA32F