Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions clean/dx12/kernels/l_post.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,15 @@
// value is clamp(E(o), 0, 2.45) rounded by v_cvt_f16_f32 (wg_f2h);
// - D(h) / e_prev is an IEEE division (wg_div).
// Model M's DXILs are untouched: this file only adds the kernel (bound to l_post.co).
// l_post_recon_p12 (LP_P12 = 1): the same kernel on the pack12 coefficients of l_coef_pack12: A = image A8 (8 bytes per
// render pixel: codes A0 A1 A2 B0 | B1 B2, C2 FP16), C = image C4 (4 bytes per render pixel: C0, C1 FP16), B unused.
//
// @wg-kernel l_post_recon co=l_post.co block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p
// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1
#include "wg_platform.hlsli"
#ifndef LP_P12
#define LP_P12 0
#endif

WG_ARGS_BEGIN
uint color, A, B, C, hist_prev, m_near, m_center, m_far, w_p, hist_out, reduced, display; // pointers
Expand Down Expand Up @@ -100,8 +106,15 @@ void load_texel(int idx, out uint t[kTexDw])
{
const uint ui = (uint)idx;
const float4 c = wg_ld4f(color, ui * 16u);
#if LP_P12
const uint2 d = wg_ld64(A, ui * 8u);
const uint c01 = wg_ld32(C, ui * 4u);
const uint a = d.x & 0xffffffu, b = (d.x >> 24) | ((d.y & 0xffffu) << 8);
const uint2 k = uint2(c01, d.y >> 16); // C0 | C1 << 16, C2 (l_coef's C dword 0 and the low half of dword 1)
#else
const uint b = wg_ld32(B, ui * 4u), a = wg_ld32(A, ui * 4u);
const uint2 k = wg_ld64(C, ui * 8u);
#endif
uint f[9];
f[0] = wg_f2h(enc_in(opq(g_e * c.x)));
f[1] = wg_f2h(enc_in(opq(g_e * c.y)));
Expand Down Expand Up @@ -297,7 +310,11 @@ void finish_pixel(LGather g, int x, int y, float px, float py, bool store, float
float lpx(int xx) { return mad(opq((float)xx + 0.5f), P_rx, P_jx); } // fused(x + 1/2, rx, jx)
float lpy(int yy) { return mad(P_ry, opq((float)yy + 0.5f), P_jy); } // fused(ry, y + 1/2, jy)

#if LP_P12
WG_KERNEL(l_post_recon_p12)
#else
WG_KERNEL(l_post_recon)
#endif
{
WG_SETUP();
g_z = wg_grid_ >> 31;
Expand Down
7 changes: 6 additions & 1 deletion clean/l/FORMATS.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ No kernel works in place.
|---|---|---|
| `l_enc_values.co` (loaded under the tag `lev`) | `clean/kernels/encoder.hip` with `-DWG_ENC_VALUES_OUT=1` | `input_encoder` |
| `l_ops.co` | `clean/l/kernels/l_ops.hip` | `l_linear_32_32_relu`, `l_linear_32_32_f16`, `l_project_gather`, `l_dec_project`, `l_coef`, `l_coef_pack12` |
| `l_post.co` | `clean/l/kernels/l_post.hip` (includes Model M's `clean/kernels/post.hip` for its functions; Model M's kernels come along unused) | `l_post_recon` (HDR) |
| `l_post.co` | `clean/l/kernels/l_post.hip` (includes Model M's `clean/kernels/post.hip` for its functions; Model M's kernels come along unused) | `l_post_recon`, `l_post_recon_p12` (HDR) |
| `l_swin.co` | `clean/l/kernels/l_swin.hip` | `l_swin_block_c32[_nc]` (exists, #102: FP8 in / FP8 out, mirror border; block checks only), `l_swin_block_c32_enc[_nc]`, `l_swin_block_c32_dec[_nc]` |

All are built with `clean/runner/build.py`'s `device_command`. The build's `-cuid` comes from the output file name,
Expand Down Expand Up @@ -141,6 +141,11 @@ the frame parity, as Model M's.
- its blend is the fused form beta' = fused(alpha (1 - beta-bar) + beta-bar), H^ = fused((C - Hl) beta' + Hl);
- the decode passes a NaN through.

`l_post_recon_p12` (`l_post.co`) is the same kernel on the pack12 coefficients of `l_coef_pack12`: `A` = image A8
(8 bytes per render pixel), `C` = image C4 (4 bytes per render pixel), `B` unused (may be null); the other arguments,
the grid and the outputs as `l_post_recon`, byte for byte (`clean/l/runner/l_d3d12_ops_check.py` checks it on HIP).
The D3D12 ports of both are `clean/dx12/kernels/l_post.hlsl`.

The LDR kernel is planned, with the same arguments. Still open for the kernel-order model: the hardware exp2,
reciprocal and divisions, and the Catmull-Rom weight forms, which the kernel takes from Model M's `post.hip`
(part of the Model M fidelity decision).
Expand Down
69 changes: 59 additions & 10 deletions clean/l/kernels/l_post.hip
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
// 2^-126, FP16 weights into half accumulators, a float32 weight sum. The blend is item 3's fused form. The decode
// passes a NaN through (a zero total gather weight gives C = D(NaN) = NaN; the NaN guard then substitutes it, as the
// oracle). Arguments, layouts and grid are post_recon's; w[0] = Model L's post scale s.
// l_post_recon_p12: the same kernel on the pack12 coefficients of l_coef_pack12 (A8 / C4 images, B unused), which the
// D3D12 frame reads; the same texel values, so the same outputs byte for byte.
#include "post.hip"

namespace {
Expand Down Expand Up @@ -48,6 +50,35 @@ WG_FN LTexel l_load_texel(const v4f* color, const u32* A, const u32* B, const v4
return t;
}

// l_load_texel on the pack12 coefficients (l_coef_pack12, clean/l/FORMATS.md): image A8, 8 bytes per render pixel =
// codes A0 A1 A2 B0 | B1 B2, C2 (FP16); image C4, 4 bytes per render pixel = C0, C1 (FP16). The same values as
// l_load_texel on l_coef's images (only padding bytes are left out), so the same texel.
WG_FN LTexel l_load_texel12(const v4f* color, const u32* A8, const u32* C4, int idx, float e) {
LTexel t;
const v4f c = color[idx];
t.f[0] = (h16)enc_in(e * c[0]);
t.f[1] = (h16)enc_in(e * c[1]);
t.f[2] = (h16)enc_in(e * c[2]);
const u32 d0 = A8[2 * idx], d1 = A8[2 * idx + 1], c01 = C4[idx];
const u32 a = d0 & 0xffffffu, b = (d0 >> 24) | ((d1 & 0xffffu) << 8);
#pragma unroll
for (int i = 0; i < 3; ++i) {
t.f[3 + i] = (h16)(byte_f(b, i) / 255.0f);
t.f[6 + i] = (h16)(byte_f(a, i) / 255.0f);
}
t.kA = (h16)((float)__builtin_bit_cast(_Float16, (unsigned short)(c01 & 0xffffu)) * -0.721325f);
t.kB = (h16)((float)__builtin_bit_cast(_Float16, (unsigned short)(c01 >> 16)) * -0.721325f);
t.kC = (h16)((float)__builtin_bit_cast(_Float16, (unsigned short)(d1 >> 16)) * -1.44265f);
return t;
}

// The texel of the kernel's coefficient form: l_coef's images (A, B, C) or pack12's (A8 in A, C4 in C, B unused).
template <bool kP12>
WG_FN LTexel l_texel(const v4f* color, const u32* A, const u32* B, const void* C, int idx, float e) {
if (kP12) return l_load_texel12(color, A, (const u32*)C, idx, e);
return l_load_texel(color, A, B, (const v4h*)C, idx, e);
}

// Gathered sums of one D pixel: W = float32 sum of the unrounded weights; F = the half accumulators.
struct LGather {
float W;
Expand Down Expand Up @@ -186,14 +217,11 @@ WG_FN void l_finish_pixel(const LGather& g, int x, int y, float px, float py, bo
}
}

} // namespace

// Stage "post" for Model L, HDR. Arguments, layouts and grid as post_recon (clean/kernels/post.hip): grid
// ceil(dw / 32) * ceil(ceil(dh / 2) / 8) workgroups of 256 lanes; w = the post blob (w[0] = s).
WG_KERNEL_N(l_post_recon, kPostThreads)(const v4f* color, const u32* A, const u32* B, const v4h* C,
const v4h* hist_prev, const u32* m_near, const u32* m_center,
const u32* m_far, const float* w, v4h* hist_out, u32* reduced,
_Float16* display, WgPostParams p, const float* expo) {
// The kernel body; kP12 selects the coefficient form (l_texel).
template <bool kP12>
WG_FN void l_post_recon_body(const v4f* color, const u32* A, const u32* B, const void* C, const v4h* hist_prev,
const u32* m_near, const u32* m_center, const u32* m_far, const float* w, v4h* hist_out,
u32* reduced, _Float16* display, WgPostParams p, const float* expo) {
if (expo) {
p.e = expo[0] == 0.0f ? 1.0f : expo[0];
p.e_prev = expo[1] == 0.0f ? 1.0f : expo[1];
Expand Down Expand Up @@ -224,7 +252,7 @@ WG_KERNEL_N(l_post_recon, kPostThreads)(const v4f* color, const u32* A, const u3
for (int i = lane; i < ncol * nrow; i += kPostThreads) {
const int r = i / ncol, c = i - r * ncol;
const int idx = clampi(r0 + r, 0, p.rh - 1) * p.rw + clampi(c0 + c, 0, p.rw - 1);
sh_tex[i] = l_load_texel(color, A, B, C, idx, p.e);
sh_tex[i] = l_texel<kP12>(color, A, B, C, idx, p.e);
}
}
wg_barrier();
Expand Down Expand Up @@ -259,7 +287,7 @@ WG_KERNEL_N(l_post_recon, kPostThreads)(const v4f* color, const u32* A, const u3
t = sh_tex[(b + j - r0) * ncol + (ax + i - c0)];
} else {
const int idx = clampi(b + j, 0, p.rh - 1) * p.rw + clampi(ax + i, 0, p.rw - 1);
t = l_load_texel(color, A, B, C, idx, p.e);
t = l_texel<kP12>(color, A, B, C, idx, p.e);
}
l_gather_add(g0, t, u, v0);
l_gather_add(g1, t, u, v1);
Expand Down Expand Up @@ -287,3 +315,24 @@ WG_KERNEL_N(l_post_recon, kPostThreads)(const v4f* color, const u32* A, const u3
reduced[(y0 / 2) * (p.dw / 2) + x / 2] = code;
}
}

} // namespace

// Stage "post" for Model L, HDR. Arguments, layouts and grid as post_recon (clean/kernels/post.hip): grid
// ceil(dw / 32) * ceil(ceil(dh / 2) / 8) workgroups of 256 lanes; w = the post blob (w[0] = s).
WG_KERNEL_N(l_post_recon, kPostThreads)(const v4f* color, const u32* A, const u32* B, const v4h* C,
const v4h* hist_prev, const u32* m_near, const u32* m_center,
const u32* m_far, const float* w, v4h* hist_out, u32* reduced,
_Float16* display, WgPostParams p, const float* expo) {
l_post_recon_body<false>(color, A, B, C, hist_prev, m_near, m_center, m_far, w, hist_out, reduced, display, p, expo);
}

// l_post_recon on the pack12 coefficients of l_coef_pack12: A = image A8 (8 bytes per render pixel), C = image C4
// (4 bytes per render pixel), B unused (may be null). Everything else as l_post_recon; the same outputs.
WG_KERNEL_N(l_post_recon_p12, kPostThreads)(const v4f* color, const u32* A8, const u32* B, const u32* C4,
const v4h* hist_prev, const u32* m_near, const u32* m_center,
const u32* m_far, const float* w, v4h* hist_out, u32* reduced,
_Float16* display, WgPostParams p, const float* expo) {
l_post_recon_body<true>(color, A8, B, C4, hist_prev, m_near, m_center, m_far, w, hist_out, reduced, display, p,
expo);
}
26 changes: 24 additions & 2 deletions clean/l/runner/l_d3d12_ops_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,16 @@ def _post_specials(rng, x):
return x


def _pack12(data):
"""l_coef's images A, B (4 bytes per pixel, codes in bytes 0 ... 2) and C (4 x FP16) in l_coef_pack12's layout:
A8 = A0 A1 A2 B0 | B1 B2, C2 (8 bytes per pixel); C4 = C0, C1 (4 bytes per pixel); B unused (passed as null)."""
a = np.frombuffer(data["A"], np.uint32) & 0xFFFFFF
b = np.frombuffer(data["B"], np.uint32) & 0xFFFFFF
c = np.frombuffer(data["C"], np.uint16).reshape(-1, 4).astype(np.uint32)
a8 = np.stack([a | ((b & 0xFF) << 24), (b >> 8) | (c[:, 2] << 16)], -1).astype(np.uint32)
return {"A": a8.tobytes(), "B": b"", "C": (c[:, 0] | (c[:, 1] << 16)).astype(np.uint32).tobytes()}


def post_cases(kernel, rng):
"""l_post_recon (l_post.co): color, A, B, C, hist_prev, m_near, m_center, m_far, w (post blob, w[0] = s) ->
full history [dh][dw] (RGBA16F), reduced codes [dh / 2][dw / 2], display (3 FP16 planes of pitch
Expand Down Expand Up @@ -261,7 +271,10 @@ def post_cases(kernel, rng):
defined = [np.concatenate([np.ones(n_hist, bool), np.zeros(POST_SLACK, bool)]),
np.concatenate([red_def, np.zeros(POST_SLACK, bool)]),
np.concatenate([disp_def.reshape(-1), np.zeros(POST_SLACK, bool)])]
args = [("buf", data[k]) for k in ("color", "A", "B", "C", "hist_prev", "m_near", "m_center", "m_far")]
if kernel == "l_post_recon_p12":
data = dict(data, **_pack12(data))
args = [("null",) if data[k] == b"" else ("buf", data[k])
for k in ("color", "A", "B", "C", "hist_prev", "m_near", "m_center", "m_far")]
args += [("buf", RP.pack({"s": np.array(LPC.S)})), ("out", n_hist + POST_SLACK), ("out", n_red + POST_SLACK),
("out", n_disp + POST_SLACK), ("bytes", bytes(prm))]
args += [("buf", ev.tobytes())] if expo else [("null",)]
Expand Down Expand Up @@ -301,7 +314,8 @@ def enc_direct_cases(kernel, rng):
KINDS = {"l_linear_32_32_relu": linear_cases, "l_linear_32_32_f16": linear_cases,
"l_project_gather": project_cases, "l_dec_project": project_cases, "l_coef": coef_cases,
"l_coef_pack12": coef_cases, "lev/input_encoder": enc_cases, "l_post_recon": post_cases,
"lev/input_encoder_direct": enc_direct_cases, "swin_block_c64_add_ex_nc": dec1_ex_cases}
"l_post_recon_p12": post_cases, "lev/input_encoder_direct": enc_direct_cases,
"swin_block_c64_add_ex_nc": dec1_ex_cases}
CONTROL_KINDS = {"swin_block_c96_nc": control_cases}
CO = "l_ops.co"

Expand Down Expand Up @@ -451,6 +465,14 @@ def compare(got, want, dmask):
for k in kernels}
res["pass"] = r.returncode == 0 and caught is True and all(v["cases"] > 0 and v["identical"] == v["cases"]
for v in res["per_kernel"].values())
if "l_post_recon" in kernels and "l_post_recon_p12" in kernels:
# The same cases (same seeds) on both coefficient forms: on HIP, the pack12 kernel's raw output bytes must be
# l_post_recon's.
pairs = [(n, n.replace("l_post_recon_", "l_post_recon_p12_", 1)) for n in hip_out
if hip_out[n][0] == "l_post_recon"]
same = sum(hip_out[a][1] == hip_out[b][1] for a, b in pairs if b in hip_out)
res["p12_equals_base_on_hip"] = {"cases": len(pairs), "identical": same}
res["pass"] = res["pass"] and same == len(pairs) > 0
return res


Expand Down
Loading