diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index f11f5ac6..082c79b8 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -218,14 +218,30 @@ "l_post_filter_direct", "-" ], + [ + "l_post_filter_direct_ldr", + "-" + ], + [ + "l_post_filter_ldr", + "l_post.co" + ], [ "l_post_recon", "l_post.co" ], + [ + "l_post_recon_ldr", + "l_post.co" + ], [ "l_post_recon_p12", "l_post.co" ], + [ + "l_post_recon_p12_ldr", + "l_post.co" + ], [ "l_project_gather", "l_ops.co" diff --git a/clean/dx12/kernels/l_post.hlsl b/clean/dx12/kernels/l_post.hlsl index 652304fa..93451d89 100644 --- a/clean/dx12/kernels/l_post.hlsl +++ b/clean/dx12/kernels/l_post.hlsl @@ -24,12 +24,23 @@ // l_post_recon_p12 (LP_P12 = 1): the same kernel on the pack12 coefficients of l_coef_pack12: A = image A8 (8 bytes per // render pixel: codes A0 A1 A2 B0 | B1 B2, C2 FP16), C = image C4 (4 bytes per render pixel: C0, C1 FP16), B unused. // +// LDR forms (LP_LDR = 1; l_post.hip's l_post_recon_ldr / l_post_recon_p12_ldr, spec preset L part 3 item 6, Q1, Q2): +// identity transfer (the colour feature is the driver's FP16 conversion of the colour, a NaN kept; no decode of the +// history candidates or of C), no exposure (not read: e = e_prev = 1), the full history the driver's FP16 conversion of +// H^, the display image h(code RN32(1 / N)) of the 11 / 11 / 10-bit code floor(clamp(fused(O^ N + 1/2), 0, N)) (a NaN +// gives 0), the reduced codes rint(clamp(mean, 0, 1) 1023). +// // @wg-kernel l_post_recon co=l_post.co block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p // @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1 +// @wg-kernel l_post_recon_ldr co=l_post.co entry=l_post_recon_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_LDR=1 +// @wg-kernel l_post_recon_p12_ldr co=l_post.co entry=l_post_recon_p12_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_LDR=1 #include "wg_platform.hlsli" #ifndef LP_P12 #define LP_P12 0 #endif +#ifndef LP_LDR +#define LP_LDR 0 +#endif WG_ARGS_BEGIN uint color, A, B, C, hist_prev, m_near, m_center, m_far, w_p, hist_out, reduced, display; // pointers @@ -168,9 +179,15 @@ void load_texel_hw(int idx, out uint t[kTexDw]) texel_inputs(idx, c, a, b, k); const float k0 = f16tof32(k.x), k1 = f16tof32(k.x >> 16), k2 = f16tof32(k.y); v8f cv; +#if LP_LDR + cv.v[0] = opq(c.x); + cv.v[1] = opq(c.y); + cv.v[2] = opq(c.z); +#else cv.v[0] = opq(enc_in(opq(g_e * c.x))); cv.v[1] = opq(enc_in(opq(g_e * c.y))); cv.v[2] = opq(enc_in(opq(g_e * c.z))); +#endif cv.v[3] = mul_odd(k0, kShapeAB); cv.v[4] = mul_odd(k1, kShapeAB); cv.v[5] = mul_odd(k2, kShapeC); @@ -291,7 +308,11 @@ void l_post_core(LGather g, int x, int y, int fx, int fy, float mx[3], float my[ const bool valid = P_reset == 0 && qx >= 0.0f && qx <= (float)P_dw && qy >= 0.0f && qy <= (float)P_dh; if (valid) { const float3 h = history_cr(qx, qy); +#if LP_LDR + const float3 lq = h; +#else const float3 lq = float3(l_dec(h.x), l_dec(h.y), l_dec(h.z)) * g_rp; +#endif L[n][0] = lq.x; L[n][1] = lq.y; L[n][2] = lq.z; @@ -305,9 +326,17 @@ void l_post_core(LGather g, int x, int y, int fx, int fy, float mx[3], float my[ bool hl_inf = false; [unroll] for (int i = 0; i < 3; ++i) { const float comb = ncand == 2 ? mad(w1, L[0][i], w2 * L[1][i]) : w1 * L[0][i]; +#if LP_LDR + Hl[i] = opq(comb); +#else Hl[i] = opq(comb) * g_e; +#endif hl_inf = hl_inf || abs(Hl[i]) == asfloat(0x7f800000u); +#if LP_LDR + Cl[i] = fbar[i]; +#else Cl[i] = l_dec(fbar[i]); +#endif } const float alpha = clampf(opq(s * g.W) * fbar[6], 0.0f, 1.0f); const float bh2 = mad(alpha, 1.0f - fbar[7], fbar[7]); @@ -354,7 +383,11 @@ void finish_pixel(LGather g, int x, int y, float px, float py, float s, out floa float lpx(int xx) { return mad(opq((float)xx + 0.5f), P_rx, P_jx); } // fused(x + 1/2, rx, jx) float lpy(int yy) { return mad(P_ry, opq((float)yy + 0.5f), P_jy); } // fused(ry, y + 1/2, jy) -#if LP_P12 +#if LP_P12 && LP_LDR +WG_KERNEL(l_post_recon_p12_ldr) +#elif LP_LDR +WG_KERNEL(l_post_recon_ldr) +#elif LP_P12 WG_KERNEL(l_post_recon_p12) #else WG_KERNEL(l_post_recon) @@ -362,6 +395,11 @@ WG_KERNEL(l_post_recon) { WG_SETUP(); g_z = wg_grid_ >> 31; +#if LP_LDR + g_e = 1.0f; // LDR: no exposure (the exposure pair is not read) + g_ep = 1.0f; + g_rp = 1.0f; +#else g_e = P_e; g_ep = P_e_prev; if (expo != 0u) { @@ -370,6 +408,7 @@ WG_KERNEL(l_post_recon) g_ep = e1 == 0.0f ? 1.0f : e1; } g_rp = wg_div(1.0f, g_ep); +#endif const int lane = (int)wg_lane_id(); const int tiles_x = (P_dw + kTileX - 1) / kTileX; const int tile = (int)wg_block_id(); @@ -466,17 +505,30 @@ WG_KERNEL(l_post_recon) finish_pixel(g, x, y0 + row, px, second ? py1 : py0, s, hh, oo); v8f cv; [unroll] for (int ci = 0; ci < 3; ++ci) { +#if LP_LDR + // h(H^); the display code floor(clamp(fused(O^ N + 1/2), 0, N)), then h(code RN32(1 / N)) (one rounding or + // two give the same FP16 value for every code 0 ... N) + const float n = ci == 2 ? 1023.0f : 2047.0f; + const float k = asfloat(ci == 2 ? 0x3a802008u : 0x3a001002u); + cv.v[ci] = opq(hh[ci]); + cv.v[3 + ci] = opq(floor(clampf(mad(oo[ci], n, 0.5f), 0.0f, n)) * k); +#else cv.v[ci] = mul_odd(kEncK, enc_log(hh[ci])); cv.v[3 + ci] = opq(clampf(enc_e(oo[ci]), 0.0f, kDisplayMax)); +#endif } cv.v[6] = 0.0f; cv.v[7] = 0.0f; const uint4 hc = wg_to_f16_ags(cv); const int y = y0 + row; if (active && (!second || has1)) { +#if LP_LDR + const uint q0 = hc.x & 0xffffu, q1 = hc.x >> 16, q2 = hc.y & 0xffffu; +#else const uint q0 = mul_zero(kEncK, enc_log(hh[0])) ? 0u : (hc.x & 0xffffu); const uint q1 = mul_zero(kEncK, enc_log(hh[1])) ? 0u : (hc.x >> 16); const uint q2 = mul_zero(kEncK, enc_log(hh[2])) ? 0u : (hc.y & 0xffffu); +#endif wg_st64(hist_out, (uint)(y * P_dw + x) * 8u, uint2(q0 | (q1 << 16), q2)); const int di = y * pitch + x; wg_st16(display, (uint)di * 2u, hc.y >> 16); @@ -494,7 +546,11 @@ WG_KERNEL(l_post_recon) uint code = 0u; [unroll] for (int i = 0; i < 3; ++i) { const float m = opq((r[i] + o[i]) * 0.25f); +#if LP_LDR + const float v = clampf(m, 0.0f, 1.0f); +#else const float v = clampf(opq(enc_e(m)) * kInv245, 0.0f, 1.0f); +#endif code |= wg_f2u(round(v * 1023.0f)) << (10 * i); } wg_st32(reduced, (uint)((y0 / 2) * (P_dw / 2) + x / 2) * 4u, code); diff --git a/clean/dx12/kernels/post_filter.hlsl b/clean/dx12/kernels/post_filter.hlsl index 534a826d..8127a8e8 100644 --- a/clean/dx12/kernels/post_filter.hlsl +++ b/clean/dx12/kernels/post_filter.hlsl @@ -20,8 +20,14 @@ // framebuffer_half's bytes (+-65504 clamp, round to nearest even), alpha 1. The lane of channel 0 of each output pixel // gathers the other two channels from its lanes 3 g + 1, 3 g + 2 and stores the texel. // +// l_post_filter_ldr (WG_FILTER_LDR = 1): l_post.hip's LDR output stage l_post_filter_ldr (spec preset L part 3 Q2), the +// same filter with the filtered value written as it is (no clamp to [1e-4, 3], no decode, no exposure: dec_scale and +// expo are not read); l_post_filter_direct_ldr: its direct form, stored as l_post_filter_direct stores. +// // @wg-kernel w64/post_filter_c1_s4 co=post_w64.co entry=post_filter_c1_s4 block=64 wave=64 args=p,p,u,u,u,u,u,p // @wg-kernel l_post_filter_direct co=- entry=post_filter_c1_s4 block=64 wave=64 args=p,p,u,u,u,u,u,p rs=direct defines=WG_DIRECT_TEXTURES=1 +// @wg-kernel l_post_filter_ldr co=l_post.co entry=post_filter_c1_s4 block=64 wave=64 args=p,p,u,u,u,u,u,p defines=WG_FILTER_LDR=1 +// @wg-kernel l_post_filter_direct_ldr co=- entry=post_filter_c1_s4 block=64 wave=64 args=p,p,u,u,u,u,u,p rs=direct defines=WG_DIRECT_TEXTURES=1,WG_FILTER_LDR=1 #include "wg_platform.hlsli" #if defined(WG_DIRECT_TEXTURES) #include "native_direct.hlsli" @@ -123,11 +129,13 @@ WG_KERNEL(post_filter_c1_s4) { WG_SETUP(); g_z = (uint)ow >> 31; +#if !defined(WG_FILTER_LDR) float dec_scale = dec_scale_arg; if (expo != 0u) { const float e = wg_ldf(expo, 0u); dec_scale = (float)(0.01 / (double)(e == 0.0f ? 1.0f : e)); } +#endif const int lane = (int)wg_lane_id(); const int wl = lane & 31; if (wl >= 3 * kGroups) return; @@ -163,11 +171,19 @@ WG_KERNEL(post_filter_c1_s4) const uint o0 = (uint)(y * ow + 4 * m); [unroll] for (int i = 0; i < 4; ++i) { const float r = dy == 0 ? lz5e(r0[i], r1[i], r2[i], r3[i], r4[i]) : lz5o(r2[i], r3[i], r4[i], r5[i], r6[i]); +#if defined(WG_FILTER_LDR) + const float v = r; +#else const float qc = clampf(r, k1em4, 3.0f); const float v = qc < kDecPolyMax ? dec_m1_poly(qc) : exp2(qc * kDecK) - 1.0f; +#endif #if defined(WG_DIRECT_TEXTURES) // channels 1, 2 of this pixel from the lanes after channel 0's (the same pixel, so the same loop exits) +#if defined(WG_FILTER_LDR) + const float val = v; +#else const float val = opq(v) * dec_scale; +#endif const uint l0 = (uint)(lane - c); const float v1 = WaveReadLaneAt(val, l0 + 1u), v2 = WaveReadLaneAt(val, l0 + 2u); if (c == 0 && i < ncols) { @@ -178,8 +194,12 @@ WG_KERNEL(post_filter_c1_s4) if (nd_output_rne16 != 0u) o = float3(fd_half(o.x), fd_half(o.y), fd_half(o.z)); native_direct_output[uint2((uint)(4 * m + i), (uint)y)] = float4(o, 1.0f); } +#else +#if defined(WG_FILTER_LDR) + if (i < ncols) wg_stf(out_p, (o0 + (uint)i) * 12u + (uint)c * 4u, v); #else if (i < ncols) wg_stf(out_p, (o0 + (uint)i) * 12u + (uint)c * 4u, opq(v) * dec_scale); +#endif #endif } } diff --git a/clean/dx12/tests/post_tf_d3d12_check.py b/clean/dx12/tests/post_tf_d3d12_check.py index f515e09c..34edc0ac 100644 --- a/clean/dx12/tests/post_tf_d3d12_check.py +++ b/clean/dx12/tests/post_tf_d3d12_check.py @@ -324,6 +324,52 @@ def store_cases(rng): return out +# --store --ldr: Model L's LDR output stage on the direct route, l_post_filter_direct_ldr, against HIP's +# l_post_filter_ldr (FP32, l_post.co) and l_post_filter_ldr_h (RGBA16F rows, framebuffer_half(v pi), pitch 8 O) on the +# same display image: the LDR display (unpacked 11 / 11 / 10-bit codes), and on frames marked wide also values beyond +# the codes' range (large ones, so the stored value passes 65504 and is clamped, and tiny ones, FP16 subnormal +# results), which the store path must take as the HDR form does. (ow, oh, pi, wide[, "f32"]) +LDR_STORE_FRAMES = [(96, 54, 1.0, False), (99, 57, 1.375, False), (161, 83, 1.0, True), (161, 83, 1.375, True), + (1281, 140, 1.25, False), (40, 16, 1.0, True), (7, 5, 1.0, False), (161, 83, 30000.0, True), + (96, 54, 1.0, False, "f32"), (1281, 140, 1.0, True, "f32")] + + +def _display_ldr(rng, dw, dh, wide): + from clean.l.reference import recon as R + from clean.runner import post as RP + pitch = RP.display_pitch(dw) + codes = np.stack([rng.integers(0, int(n) + 1, (dh, pitch)) for n in R.LDR_CODE_MAX], -1).astype(np.float64) + v = R.ldr_unpack(codes).transpose(2, 0, 1).astype(np.float64) + if wide: + pick = rng.random(v.shape) + v = np.where(pick < 0.05, rng.uniform(30000.0, 65504.0, v.shape), v) + v = np.where((pick >= 0.05) & (pick < 0.15), rng.uniform(0.0, 2.0 ** -20, v.shape), v) + return v.astype(np.float16).tobytes() + + +def store_cases_ldr(rng): + """[(HIP FP32 filter case, HIP RGBA16F-rows case or None, pi, ow, oh, label)] of LDR_STORE_FRAMES.""" + out = [] + for fr in LDR_STORE_FRAMES: + ow, oh, pi, wide = fr[:4] + f32tex = len(fr) > 4 and fr[4] == "f32" + dw, dh = MP.recon_size_for(ow), MP.recon_size_for(oh) + disp = _display_ldr(rng, dw, dh, wide) + n32, n16 = ow * oh * 12, ow * oh * 8 + base = [("buf", disp), None, ("u32", dw), ("u32", dh), ("u32", ow), ("u32", oh), ("u32", 0), ("null",)] + grid = PZ.PS._cdiv(ow, 40) * PZ.PS._cdiv(PZ.PS._cdiv(oh, 2), 8) + f32 = {"kernel": "l_post_filter_ldr", "co": "l_post.co", "hip_kernel": "l_post_filter_ldr", "grid": grid, + "block": 64, "args": [base[0], ("out", n32 + SLACK)] + base[2:], + "defined": [np.concatenate([np.ones(n32, bool), np.zeros(SLACK, bool)])]} + h = dict(f32, kernel="l_post_filter_ldr_h", hip_kernel="l_post_filter_ldr_h", + args=[base[0], ("out", n16 + SLACK)] + base[2:] + + [("u32", _s32(_f32_bits(np.float32(pi)))), ("u32", 8 * ow)]) + label = f"LDR O {ow}x{oh} pi {pi}" + (" wide" if wide else "") + \ + (" (RGBA32F: l_post_filter_ldr)" if f32tex else " (l_post_filter_ldr_h)") + out.append((f32, None if f32tex else h, pi, ow, oh, label)) + return out + + def _store_classes(v32, pi): """Counts of the values stored (FP32 filter output times pi, numpy's FP32 product): NaN, beyond +-65504 (clamped; infinities included), FP16 subnormal range, exact ties of the FP16 rounding (normal and subnormal range).""" @@ -339,24 +385,26 @@ def _store_classes(v32, pi): "ties": int(np.sum(normal & ((bits & 0x1FFF) == 0x1000)) + np.sum(sub & (t - np.floor(t) == 0.5)))} -def store_run(build, rocm, seeds): +def store_run(build, rocm, seeds, ldr=False): + """ldr: Model L's LDR direct filter (LDR_STORE_FRAMES) instead of the HDR one (STORE_FRAMES).""" import subprocess from clean.l.runner import l_c32_check as C32 g = C32.Gpu(build, rocm) - with open(os.path.join(build, "post_w64.co"), "rb") as f: + with open(os.path.join(build, "l_post.co" if ldr else "post_w64.co"), "rb") as f: mod = g.hip.load_module(f.read()) - cdir = os.path.join(build, "replay-store") + cdir = os.path.join(build, "replay-store-ldr" if ldr else "replay-store") os.makedirs(cdir, exist_ok=True) lines, todo, neg = [], [], None for si, seed in enumerate(seeds): rng = np.random.default_rng(seed) - for ci, (f32, o16, pi, ow, oh, label) in enumerate(store_cases(rng)): + for ci, (f32, o16, pi, ow, oh, label) in enumerate(store_cases_ldr(rng) if ldr else store_cases(rng)): name = f"store_s{si}_{ci}" w32 = OPS.hip_run(g, mod, f32)[0] w16 = OPS.hip_run(g, mod, o16)[0] if o16 is not None else None lines += OPS.replay_lines(f"{name}_filter", f32, cdir) f16 = o16 is not None - st = dict(f32, kernel="l_post_filter_direct", hip_kernel=None, args=f32["args"], + st = dict(f32, kernel="l_post_filter_direct_ldr" if ldr else "l_post_filter_direct", hip_kernel=None, + args=f32["args"], direct=[("color", OPS.DXGI_R32G32B32A32_FLOAT, 8, 8, bytes(8 * 8 * 16)), ("depth", OPS.DXGI_R32_FLOAT, 8, 8, bytes(8 * 8 * 4)), ("motion", OPS.DXGI_R32G32_FLOAT, 8, 8, bytes(8 * 8 * 8)), @@ -430,9 +478,12 @@ def tex_judge(path, w16, ow, oh, w32=None, outp_path=None): res["summary"] = {"cases": len(res["cases"]), "identical": sum(c["identical"] for c in res["cases"]), "texture_bytes": sum(c["store_rgba16f"].get("bytes", 0) for c in res["cases"]), "filter_f32_bytes": sum(c["filter_f32"]["bytes"] for c in res["cases"])} + # (LDR: no exposure scale, so no NaN or infinite stored values; the clamp past 65504, subnormals and ties) + need = ("beyond_65504", "f16_subnormal_range", "ties") if ldr else \ + ("nan", "beyond_65504", "infinite", "f16_subnormal_range", "ties") res["pass"] = (r.returncode == 0 and bool(neg) and res["negative_control"]["caught"] and res["summary"]["identical"] == res["summary"]["cases"] > 0 and - all(classes.get(k, 0) > 0 for k in ("nan", "beyond_65504", "infinite", "f16_subnormal_range", "ties"))) + all(classes.get(k, 0) > 0 for k in need)) return res @@ -445,13 +496,15 @@ def main(argv=None): ap.add_argument("--kernels", default=",".join(KINDS)) ap.add_argument("--direct", action="store_true", help="the direct forms instead (see DIRECT)") ap.add_argument("--store", action="store_true", help="Model L's direct filter instead (see STORE_FRAMES)") + ap.add_argument("--ldr", action="store_true", help="with --store: Model L's LDR direct filter (LDR_STORE_FRAMES)") ap.add_argument("--json") a = ap.parse_args(argv) if not a.run: return 0 kernels = [k for k in a.kernels.split(",") if k] seeds = tuple(range(89, 89 + a.seeds)) - res = store_run(a.build, a.rocm, seeds) if a.store else direct_run(a.build, a.rocm, seeds) if a.direct else \ + res = store_run(a.build, a.rocm, seeds, ldr=a.ldr) if a.store else \ + direct_run(a.build, a.rocm, seeds) if a.direct else \ OPS.run(a.build, a.rocm, seeds, kernels, kinds=KINDS) if a.json: with open(a.json, "w") as f: diff --git a/clean/l/kernels/l_post.hip b/clean/l/kernels/l_post.hip index 97fb4d60..9b198442 100644 --- a/clean/l/kernels/l_post.hip +++ b/clean/l/kernels/l_post.hip @@ -14,6 +14,14 @@ // oracle). Arguments, layouts and grid are post_recon's; w[0] = Model L's post scale s. // l_post_recon_p12: the same kernel on the pack12 coefficients of l_coef_pack12 (A8 / C4 images, B unused), which the // D3D12 frame reads; the same texel values, so the same outputs byte for byte. +// LDR forms (l_post_recon_ldr, l_post_recon_p12_ldr; spec preset L part 3 item 6, Q1, Q2): the same function with the +// identity transfer (E(v) = v, D(e) = e: the colour feature is h(colour), a NaN kept; no decode of the history +// candidates or of C), no exposure (the exposure pair is not read: e = e_prev = 1), the full history h(H^), the display +// image the output stage's unpacked 11 / 11 / 10-bit codes of O^ (code = floor(clamp(fused(O^ N + 1/2), 0, N)), +// N = 2047, 2047, 1023, NaN -> 0; stored h(code RN32(1 / N))) and the reduced codes rint(clamp(mean, 0, 1) 1023). +// l_post_filter_ldr / l_post_filter_ldr_h: the output stage for that display image (Q2): Model M's 5-tap filter +// (post.hip filter_row_c1, lz5) with no clamp to [1e-4, 3], no decode and no exposure; 3 x f32, or RGBA16F rows with pi +// folded in (framebuffer_half(v pi), alpha 1). #include "post.hip" namespace { @@ -32,12 +40,20 @@ struct LTexel { h16 kA, kB, kC; }; +// The colour feature: h(E_in(e c)) (HDR) or h(c) (LDR: identity transfer, no exposure; a NaN is kept). +template +WG_FN h16 l_colour_feature(float c, float e) { + if (kLdr) return (h16)c; + return (h16)enc_in(e * c); +} + +template WG_FN LTexel l_load_texel(const v4f* color, const u32* A, const u32* B, const v4h* C, int idx, float e) { LTexel t; const v4f c = color[idx]; - t.f[0] = (h16)enc_in(e * c[0]); - t.f[1] = (h16)enc_in(e * c[1]); - t.f[2] = (h16)enc_in(e * c[2]); + t.f[0] = l_colour_feature(c[0], e); + t.f[1] = l_colour_feature(c[1], e); + t.f[2] = l_colour_feature(c[2], e); const u32 b = B[idx], a = A[idx]; #pragma unroll for (int i = 0; i < 3; ++i) { @@ -54,12 +70,13 @@ WG_FN LTexel l_load_texel(const v4f* color, const u32* A, const u32* B, const v4 // l_load_texel on the pack12 coefficients (l_coef_pack12, clean/l/FORMATS.md): image A8, 8 bytes per render pixel = // codes A0 A1 A2 B0 | B1 B2, C2 (FP16); image C4, 4 bytes per render pixel = C0, C1 (FP16). The same values as // l_load_texel on l_coef's images (only padding bytes are left out), so the same texel. +template WG_FN LTexel l_load_texel12(const v4f* color, const u32* A8, const u32* C4, int idx, float e) { LTexel t; const v4f c = color[idx]; - t.f[0] = (h16)enc_in(e * c[0]); - t.f[1] = (h16)enc_in(e * c[1]); - t.f[2] = (h16)enc_in(e * c[2]); + t.f[0] = l_colour_feature(c[0], e); + t.f[1] = l_colour_feature(c[1], e); + t.f[2] = l_colour_feature(c[2], e); const u32 d0 = A8[2 * idx], d1 = A8[2 * idx + 1], c01 = C4[idx]; const u32 a = d0 & 0xffffffu, b = (d0 >> 24) | ((d1 & 0xffffu) << 8); #pragma unroll @@ -74,10 +91,10 @@ WG_FN LTexel l_load_texel12(const v4f* color, const u32* A8, const u32* C4, int } // The texel of the kernel's coefficient form: l_coef's images (A, B, C) or pack12's (A8 in A, C4 in C, B unused). -template +template WG_FN LTexel l_texel(const v4f* color, const u32* A, const u32* B, const void* C, int idx, float e) { - if (kP12) return l_load_texel12(color, A, (const u32*)C, idx, e); - return l_load_texel(color, A, B, (const v4h*)C, idx, e); + if (kP12) return l_load_texel12(color, A, (const u32*)C, idx, e); + return l_load_texel(color, A, B, (const v4h*)C, idx, e); } // Gathered sums of one D pixel: W = float32 sum of the unrounded weights; F = the half accumulators. @@ -144,13 +161,15 @@ WG_FN LSel l_select(const LGather& g) { } // Steps 3-5 for one D pixel (Model L). Returns the linear new history H^ in hh and the linear display value O^ in oh. +// LDR: no decode (the candidates and C are the values themselves) and no exposure. +template WG_FN void l_post_core(const LGather& g, int x, int y, int fx, int fy, const float mx[3], const float my[3], const WgPostParams& p, float s, const v4f* color, const void* hist_prev, float hh[3], float oh[3]) { const LSel sl = l_select(g); const float X = (float)x + 0.5f, Y = (float)y + 0.5f; const int ncand = sl.w2 != 0.0f ? 2 : 1; - const float rp = 1.0f / p.e_prev; // IEEE division + const float rp = kLdr ? 1.0f : 1.0f / p.e_prev; // IEEE division float L[2][3]; for (int n = 0; n < ncand; ++n) { const int k = n == 0 ? sl.first : sl.second; @@ -160,7 +179,7 @@ WG_FN void l_post_core(const LGather& g, int x, int y, int fx, int fy, const flo float h[3]; history_cr(hist_prev, p.dw, p.dh, qx, qy, h); #pragma unroll - for (int i = 0; i < 3; ++i) L[n][i] = l_dec(h[i]) * rp; + for (int i = 0; i < 3; ++i) L[n][i] = kLdr ? h[i] : l_dec(h[i]) * rp; } else { const v4f c = color[fy * p.rw + fx]; #pragma unroll @@ -172,9 +191,9 @@ WG_FN void l_post_core(const LGather& g, int x, int y, int fx, int fy, const flo #pragma unroll for (int i = 0; i < 3; ++i) { const float comb = ncand == 2 ? __builtin_fmaf(sl.w1, L[0][i], sl.w2 * L[1][i]) : sl.w1 * L[0][i]; - Hl[i] = comb * p.e; + Hl[i] = kLdr ? comb : comb * p.e; hl_inf |= __builtin_fabsf(Hl[i]) == __builtin_inff(); - Cl[i] = l_dec(sl.fbar[i]); + Cl[i] = kLdr ? sl.fbar[i] : l_dec(sl.fbar[i]); } // 5. Blend: alpha = sat((s W) g-bar), beta' = fused(alpha (1 - beta-bar) + beta-bar), H^ = fused((C - Hl) beta' + Hl), // O^ likewise (item 3). sat(NaN) = 0 (zero total weight). @@ -197,7 +216,18 @@ WG_FN void l_post_core(const LGather& g, int x, int y, int fx, int fy, const flo } } +// LDR display value of channel i: the output stage's unpack of the 11 / 11 / 10-bit code of O^ (Q2): code = +// floor(clamp(fused(o N + 1/2), 0, N)) (the clamp maps NaN to 0), then h(code RN32(1 / N)) (one float32 product, then +// the FP16 rounding). +WG_FN _Float16 l_ldr_display_value(float o, int i) { + const float n = i == 2 ? 1023.0f : 2047.0f; + const float k = __builtin_bit_cast(float, i == 2 ? 0x3a802008u : 0x3a001002u); // RN32(1/1023), RN32(1/2047) + const float code = __builtin_floorf(clampf(__builtin_fmaf(o, n, 0.5f), 0.0f, n)); + return (_Float16)(code * k); +} + // post_recon's finish_pixel with l_post_core. +template WG_FN void l_finish_pixel(const LGather& g, int x, int y, float px, float py, bool store, const WgPostParams& p, float s, const v4f* color, const v4h* hist_prev, const u32* m_near, const u32* m_center, const u32* m_far, v4h* hist_out, _Float16* display, float hh[3]) { @@ -209,7 +239,17 @@ WG_FN void l_finish_pixel(const LGather& g, int x, int y, float px, float py, bo [&](int c, int r) { return m_center[clampi(r, 0, p.rh - 1) * p.rw + clampi(c, 0, p.rw - 1)]; }, mx[1], my[1]); float oh[3]; - l_post_core(g, x, y, fx, fy, mx, my, p, s, color, hist_prev, hh, oh); + l_post_core(g, x, y, fx, fy, mx, my, p, s, color, hist_prev, hh, oh); + if (kLdr) { + if (store) hist_store(hist_out, y * p.dw + x, hh[0], hh[1], hh[2]); + if (store) { + const int pitch = display_pitch(p.dw), plane = pitch * p.dh, di = y * pitch + x; + display[di] = l_ldr_display_value(oh[0], 0); + display[plane + di] = l_ldr_display_value(oh[1], 1); + display[2 * plane + di] = l_ldr_display_value(oh[2], 2); + } + return; + } if (store) hist_store(hist_out, y * p.dw + x, enc_e(hh[0]), enc_e(hh[1]), enc_e(hh[2])); if (store) { const int pitch = display_pitch(p.dw), plane = pitch * p.dh, di = y * pitch + x; @@ -219,12 +259,16 @@ WG_FN void l_finish_pixel(const LGather& g, int x, int y, float px, float py, bo } } -// The kernel body; kP12 selects the coefficient form (l_texel). -template +// The kernel body; kP12 selects the coefficient form (l_texel), kLdr the LDR form (no exposure read). +template WG_FN void l_post_recon_body(const v4f* color, const u32* A, const u32* B, const void* C, const v4h* hist_prev, const u32* m_near, const u32* m_center, const u32* m_far, const float* w, v4h* hist_out, u32* reduced, _Float16* display, WgPostParams p, const float* expo) { - if (expo) { + if (kLdr) { + p.e = 1.0f; + p.e_prev = 1.0f; + p.ratio = 1.0f; + } else if (expo) { p.e = expo[0] == 0.0f ? 1.0f : expo[0]; p.e_prev = expo[1] == 0.0f ? 1.0f : expo[1]; p.ratio = (float)((double)p.e / (double)p.e_prev); @@ -254,7 +298,7 @@ WG_FN void l_post_recon_body(const v4f* color, const u32* A, const u32* B, const for (int i = lane; i < ncol * nrow; i += kPostThreads) { const int r = i / ncol, c = i - r * ncol; const int idx = clampi(r0 + r, 0, p.rh - 1) * p.rw + clampi(c0 + c, 0, p.rw - 1); - sh_tex[i] = l_texel(color, A, B, C, idx, p.e); + sh_tex[i] = l_texel(color, A, B, C, idx, p.e); } } wg_barrier(); @@ -289,7 +333,7 @@ WG_FN void l_post_recon_body(const v4f* color, const u32* A, const u32* B, const t = sh_tex[(b + j - r0) * ncol + (ax + i - c0)]; } else { const int idx = clampi(b + j, 0, p.rh - 1) * p.rw + clampi(ax + i, 0, p.rw - 1); - t = l_texel(color, A, B, C, idx, p.e); + t = l_texel(color, A, B, C, idx, p.e); } l_gather_add(g0, t, u, v0); l_gather_add(g1, t, u, v1); @@ -297,9 +341,10 @@ WG_FN void l_post_recon_body(const v4f* color, const u32* A, const u32* B, const } float h0[3], h1[3] = {0.0f, 0.0f, 0.0f}; - l_finish_pixel(g0, x, y0, px, py0, active, p, s, color, hist_prev, m_near, m_center, m_far, hist_out, display, h0); + l_finish_pixel(g0, x, y0, px, py0, active, p, s, color, hist_prev, m_near, m_center, m_far, hist_out, display, + h0); if (has1) - l_finish_pixel(g1, x, y0 + 1, px, py1, active, p, s, color, hist_prev, m_near, m_center, m_far, hist_out, + l_finish_pixel(g1, x, y0 + 1, px, py1, active, p, s, color, hist_prev, m_near, m_center, m_far, hist_out, display, h1); // Reduced history: mean of H^ over the 2 x 2 block (x, x + 1) x (y0, y0 + 1); lane x + 1 is lane ^ 1. @@ -311,13 +356,70 @@ WG_FN void l_post_recon_body(const v4f* color, const u32* A, const u32* B, const #pragma unroll for (int i = 0; i < 3; ++i) { const float m = (sh_red[lane * 3 + i] + sh_red[(lane ^ 1) * 3 + i]) * 0.25f; - const float v = clampf(enc_e(m) * (1.0f / 2.45f), 0.0f, 1.0f); + const float v = kLdr ? clampf(m, 0.0f, 1.0f) : clampf(enc_e(m) * (1.0f / 2.45f), 0.0f, 1.0f); code |= (u32)wg_rint(v * 1023.0f) << (10 * i); } reduced[(y0 / 2) * (p.dw / 2) + x / 2] = code; } } +// The LDR output stage (Q2): filter_strip_c1 of post.hip (the same lanes, tiles, sliding window and 5-tap passes) +// with the filtered value written as it is: no clamp to [1e-4, 3], no decode, no exposure. kH: RGBA16F rows of hpitch +// bytes, framebuffer_half(v pi), the lane of channel 2 also writes alpha 1; else 3 x f32 (row-major RGB). +template +WG_FN void l_filter_strip_ldr(const _Float16* display, void* out, int dw, int dh, int ow, int oh, float pi, + int hpitch) { + const int lane = (int)wg_lane_id(); + const int wl = lane & 31; + if (wl >= 3 * kFc1Groups) return; + const int g = wl / 3, c = wl - 3 * g; + const int tiles_x = (ow + 4 * kFc1Groups - 1) / (4 * kFc1Groups); + const int tile = (int)wg_block_id(); + const int m = (tile % tiles_x) * kFc1Groups + g; + const int p0 = ((tile / tiles_x) * kFiltStrips + (lane >> 5)) * S; + if (4 * m >= ow || 2 * p0 >= oh) return; + const int pitch = display_pitch(dw), plane = pitch * dh; + const bool fast = m >= 1 && 6 * m + 7 <= dw - 1; + const int ncols = ow - 4 * m < 4 ? ow - 4 * m : 4; + + float hr[7][4]; + filter_row_c1(display, pitch, plane, dw, clampi(3 * p0 - 2, 0, dh - 1), m, fast, c, hr[6]); +#pragma nounroll + for (int st = -1; st < S; ++st) { + const int p = p0 + st; + if (2 * p >= oh) break; +#pragma unroll + for (int k = 0; k < 4; ++k) +#pragma unroll + for (int i = 0; i < 4; ++i) hr[k][i] = hr[k + 3][i]; +#pragma unroll + for (int k = 4; k < 7; ++k) + filter_row_c1(display, pitch, plane, dw, clampi(3 * p - 2 + k, 0, dh - 1), m, fast, c, hr[k]); + if (st < 0) continue; +#pragma unroll + for (int dy = 0; dy < 2; ++dy) { + const int y = 2 * p + dy; + if (y >= oh) break; + const u32 o0 = mul24((u32)y, (u32)ow) + (u32)(4 * m); +#pragma unroll + for (int i = 0; i < 4; ++i) { + float t[5]; +#pragma unroll + for (int k = 0; k < 5; ++k) t[k] = hr[2 * dy + k][i]; + const float v = dy == 0 ? lz5(t) : lz5(t); + if (i >= ncols) continue; + if (kH) { + _Float16* o = (_Float16*)((char*)out + (size_t)y * (u32)hpitch + (u32)(4 * m + i) * 8u); + o[c] = framebuffer_half(v * pi); + if (c == 2) o[3] = (_Float16)1.0f; + } else { + ((float*)((char*)out + (o0 + (u32)i) * 12u))[c] = v; + } + } + } + } +} + } // namespace // Stage "post" for Model L, HDR. Arguments, layouts and grid as post_recon (clean/kernels/post.hip): grid @@ -326,7 +428,8 @@ WG_KERNEL_N(l_post_recon, kPostThreads)(const v4f* color, const u32* A, const u3 const v4h* hist_prev, const u32* m_near, const u32* m_center, const u32* m_far, const float* w, v4h* hist_out, u32* reduced, _Float16* display, WgPostParams p, const float* expo) { - l_post_recon_body(color, A, B, C, hist_prev, m_near, m_center, m_far, w, hist_out, reduced, display, p, expo); + l_post_recon_body(color, A, B, C, hist_prev, m_near, m_center, m_far, w, hist_out, reduced, display, p, + expo); } // l_post_recon on the pack12 coefficients of l_coef_pack12: A = image A8 (8 bytes per render pixel), C = image C4 @@ -335,6 +438,34 @@ WG_KERNEL_N(l_post_recon_p12, kPostThreads)(const v4f* color, const u32* A8, con const v4h* hist_prev, const u32* m_near, const u32* m_center, const u32* m_far, const float* w, v4h* hist_out, u32* reduced, _Float16* display, WgPostParams p, const float* expo) { - l_post_recon_body(color, A8, B, C4, hist_prev, m_near, m_center, m_far, w, hist_out, reduced, display, p, - expo); + l_post_recon_body(color, A8, B, C4, hist_prev, m_near, m_center, m_far, w, hist_out, reduced, display, + p, expo); +} + +// The LDR forms (the arguments of the HDR kernels; expo is not read). +WG_KERNEL_N(l_post_recon_ldr, kPostThreads)(const v4f* color, const u32* A, const u32* B, const v4h* C, + const v4h* hist_prev, const u32* m_near, const u32* m_center, + const u32* m_far, const float* w, v4h* hist_out, u32* reduced, + _Float16* display, WgPostParams p, const float* expo) { + l_post_recon_body(color, A, B, C, hist_prev, m_near, m_center, m_far, w, hist_out, reduced, display, p, + expo); +} +WG_KERNEL_N(l_post_recon_p12_ldr, kPostThreads)(const v4f* color, const u32* A8, const u32* B, const u32* C4, + const v4h* hist_prev, const u32* m_near, const u32* m_center, + const u32* m_far, const float* w, v4h* hist_out, u32* reduced, + _Float16* display, WgPostParams p, const float* expo) { + l_post_recon_body(color, A8, B, C4, hist_prev, m_near, m_center, m_far, w, hist_out, reduced, display, p, + expo); +} + +// The LDR output stage. Arguments as post_filter_c1_s4 (dec_scale and expo are not read), grid ceil(ow / 40) * +// ceil(ceil(oh / 2) / 8) workgroups of 64 lanes; _h as post_filter_h's arguments (pi folded in, RGBA16F rows of hpitch +// bytes) on the same grid. +WG_KERNEL_N(l_post_filter_ldr, kFiltThreads)(const _Float16* display, void* out, int dw, int dh, int ow, int oh, + float dec_scale, const float* expo) { + l_filter_strip_ldr<4, false>(display, out, dw, dh, ow, oh, 1.0f, 0); +} +WG_KERNEL_N(l_post_filter_ldr_h, kFiltThreads)(const _Float16* display, void* out, int dw, int dh, int ow, int oh, + float dec_scale, const float* expo, float pi, int hpitch) { + l_filter_strip_ldr<4, true>(display, out, dw, dh, ow, oh, pi, hpitch); } diff --git a/clean/l/records/l_post_ldr_record.json b/clean/l/records/l_post_ldr_record.json new file mode 100644 index 00000000..01754c26 --- /dev/null +++ b/clean/l/records/l_post_ldr_record.json @@ -0,0 +1,562 @@ +{ + "what": "Model L's reconstruction in LDR (l_post_recon_ldr, l_post_recon_p12_ldr; output stage l_post_filter_ldr, l_post_filter_ldr_h, l_post_filter_direct_ldr): HIP against Model L's LDR oracle (clean/l/reference/recon.py, ldr=True), the D3D12 ports against HIP byte for byte; spec preset L part 3 item 6, Q1 (LDR column), Q2", + "rule": "Model L's post acceptance: FP16 values more than 1 ulp off at most 0.05 % per frame / case and 0.01 % over the run (history and display), no non-finite value the reference does not hold too, at most 1 % differing, mean error at most 0.1 % of the reference RMS; reduced codes within 1 and 99 % exact", + "oracle_check_kernel_level": { + "tool": "clean/l/runner/l_post_check.py --ldr --seeds 6 (synthetic cases: random, extreme kernel shapes, special colours, total weight 0 everywhere, every FP16 weight 0)", + "pass": true, + "run": { + "history": { + "values": 431784, + "over_1": 0, + "frac_over_1": 0.0, + "pass": true + }, + "display": { + "values": 431784, + "over_1": 4, + "frac_over_1": 9.26389120486169e-06, + "pass": true + } + }, + "negative_control_fails": true, + "cases": 60, + "cases_passed": 60, + "known_answers": [ + { + "case": "offset_0.1", + "known_answer": true + }, + { + "case": "offset_0.05", + "known_answer": true + }, + { + "case": "offset_0.1", + "known_answer": true + }, + { + "case": "offset_0.05", + "known_answer": true + } + ], + "hdr_unchanged_run": { + "history": { + "values": 431784, + "over_1": 6, + "frac_over_1": 1.3895836807292535e-05, + "pass": true + }, + "display": { + "values": 431784, + "over_1": 6, + "frac_over_1": 1.3895836807292535e-05, + "pass": true + } + } + }, + "frames_real_weights": { + "model_source": { + "kind": "converter", + "model_sha256": "d3c040191ec7ab0164efc03ee76413277e7fb54e6e6d2b168f2c0202aa1ad806", + "map_sha256": "58e6d0e7cd6926817cfea28a0b4a65c056e9bfa5ca62aa05b968b62590c9e857" + }, + "frames": "l_frame_check's 11 synthetic frames, real-weight Model L, LDR colour (--ldr)", + "hip_against_ldr_reference": { + "L_pass": true, + "M_pass": true, + "post_run": { + "history": { + "values": 641520, + "over_1": 0, + "frac_over_1": 0.0, + "pass": true + }, + "display": { + "values": 641520, + "over_1": 6, + "frac_over_1": 9.352787130564908e-06, + "pass": true + } + }, + "per_frame": [ + { + "frame": 0, + "pass": true, + "history_over_1": [ + 0, + 58320 + ], + "display_over_1": [ + 0, + 58320 + ], + "history_nonfinite_agreeing": 9652, + "reduced_max_code_diff": 0, + "filter_outside": 0 + }, + { + "frame": 1, + "pass": true, + "history_over_1": [ + 0, + 58320 + ], + "display_over_1": [ + 0, + 58320 + ], + "history_nonfinite_agreeing": 9652, + "reduced_max_code_diff": 0, + "filter_outside": 0 + }, + { + "frame": 2, + "pass": true, + "history_over_1": [ + 0, + 58320 + ], + "display_over_1": [ + 0, + 58320 + ], + "history_nonfinite_agreeing": 9152, + "reduced_max_code_diff": 1, + "filter_outside": 0 + }, + { + "frame": 3, + "pass": true, + "history_over_1": [ + 0, + 58320 + ], + "display_over_1": [ + 0, + 58320 + ], + "history_nonfinite_agreeing": 9394, + "reduced_max_code_diff": 1, + "filter_outside": 0 + }, + { + "frame": 4, + "pass": true, + "history_over_1": [ + 0, + 58320 + ], + "display_over_1": [ + 1, + 58320 + ], + "history_nonfinite_agreeing": 10140, + "reduced_max_code_diff": 0, + "filter_outside": 0 + }, + { + "frame": 5, + "pass": true, + "history_over_1": [ + 0, + 58320 + ], + "display_over_1": [ + 1, + 58320 + ], + "history_nonfinite_agreeing": 8842, + "reduced_max_code_diff": 0, + "filter_outside": 0 + }, + { + "frame": 6, + "pass": true, + "history_over_1": [ + 0, + 58320 + ], + "display_over_1": [ + 2, + 58320 + ], + "history_nonfinite_agreeing": 8660, + "reduced_max_code_diff": 0, + "filter_outside": 0 + }, + { + "frame": 7, + "pass": true, + "history_over_1": [ + 0, + 58320 + ], + "display_over_1": [ + 0, + 58320 + ], + "history_nonfinite_agreeing": 8790, + "reduced_max_code_diff": 0, + "filter_outside": 0 + }, + { + "frame": 8, + "pass": true, + "history_over_1": [ + 0, + 58320 + ], + "display_over_1": [ + 1, + 58320 + ], + "history_nonfinite_agreeing": 8802, + "reduced_max_code_diff": 0, + "filter_outside": 0 + }, + { + "frame": 9, + "pass": true, + "history_over_1": [ + 0, + 58320 + ], + "display_over_1": [ + 1, + 58320 + ], + "history_nonfinite_agreeing": 9178, + "reduced_max_code_diff": 0, + "filter_outside": 0 + }, + { + "frame": 10, + "pass": true, + "history_over_1": [ + 0, + 58320 + ], + "display_over_1": [ + 0, + 58320 + ], + "history_nonfinite_agreeing": 9576, + "reduced_max_code_diff": 0, + "filter_outside": 0 + } + ] + }, + "d3d12_against_hip_raw_bytes": { + "L": { + "identical_frames": 11, + "frames": 11, + "outputs_compared": 28, + "differing": [], + "negative_controls_caught": { + "history": true, + "reduced": true, + "reduced_frame": true, + "missing": true + }, + "l_post_d3d12": "l_post_recon_ldr" + }, + "M": { + "identical_frames": 11, + "frames": 11, + "outputs_compared": 28, + "differing": [], + "negative_controls_caught": { + "history": true, + "reduced": true, + "reduced_frame": true, + "missing": true + }, + "l_post_d3d12": null + } + }, + "hdr_same_build": { + "L_pass": true, + "post_run": { + "history": { + "values": 641520, + "over_1": 2, + "frac_over_1": 3.117595710188303e-06, + "pass": true + }, + "display": { + "values": 641520, + "over_1": 4, + "frac_over_1": 6.235191420376606e-06, + "pass": true + } + }, + "d3d12_against_hip": { + "identical_frames": 11, + "frames": 11, + "outputs_compared": 28, + "differing": [], + "negative_controls_caught": { + "history": true, + "reduced": true, + "reduced_frame": true, + "missing": true + }, + "l_post_d3d12": "l_post_recon" + } + } + }, + "frames_synthetic_weights": { + "hip_against_ldr_reference": { + "L_pass": true, + "M_pass": true, + "post_run": { + "history": { + "values": 641520, + "over_1": 0, + "frac_over_1": 0.0, + "pass": true + }, + "display": { + "values": 641520, + "over_1": 2, + "frac_over_1": 3.117595710188303e-06, + "pass": true + } + } + }, + "d3d12_against_hip_raw_bytes": { + "L": { + "identical_frames": 11, + "frames": 11, + "outputs_compared": 28, + "differing": [], + "negative_controls_caught": { + "history": true, + "reduced": true, + "reduced_frame": true, + "missing": true + }, + "l_post_d3d12": "l_post_recon_ldr" + }, + "M": { + "identical_frames": 11, + "frames": 11, + "outputs_compared": 28, + "differing": [], + "negative_controls_caught": { + "history": true, + "reduced": true, + "reduced_frame": true, + "missing": true + }, + "l_post_d3d12": null + } + } + }, + "d3d12_kernels_against_hip": { + "tool": "clean/l/runner/l_d3d12_ops_check.py --seeds 6 (sentinels HIP 0xA5 / D3D12 0x5A, 64 slack bytes)", + "per_kernel": { + "l_post_recon": { + "cases": 72, + "identical": 72, + "bytes": 2271840, + "differ": 0 + }, + "l_post_recon_p12": { + "cases": 72, + "identical": 72, + "bytes": 2271840, + "differ": 0 + }, + "l_post_recon_ldr": { + "cases": 72, + "identical": 72, + "bytes": 2271840, + "differ": 0 + }, + "l_post_recon_p12_ldr": { + "cases": 72, + "identical": 72, + "bytes": 2271840, + "differ": 0 + }, + "l_post_filter_ldr": { + "cases": 30, + "identical": 30, + "bytes": 1061760, + "differ": 0 + }, + "swin_block_c96_nc": { + "cases": 36, + "identical": 36, + "bytes": 3167424, + "differ": 0 + } + }, + "negative_control": { + "case": "l_post_recon_s0_0", + "half_grid": true, + "caught": true, + "differ": 8498 + }, + "negative_controls_per_kernel": { + "l_post_recon": { + "case": "l_post_recon_s0_0", + "half_grid": true, + "caught": true, + "differ": 8498 + }, + "l_post_recon_p12": { + "case": "l_post_recon_p12_s0_0", + "half_grid": true, + "caught": true, + "differ": 8498 + }, + "l_post_recon_ldr": { + "case": "l_post_recon_ldr_s0_0", + "half_grid": true, + "caught": true, + "differ": 8509 + }, + "l_post_recon_p12_ldr": { + "case": "l_post_recon_p12_ldr_s0_0", + "half_grid": true, + "caught": true, + "differ": 8509 + }, + "l_post_filter_ldr": { + "case": "l_post_filter_ldr_s0_3", + "half_grid": true, + "caught": true, + "differ": 20461 + } + }, + "p12_equals_base_on_hip": { + "cases": 72, + "identical": 72 + }, + "p12_equals_base_on_hip_ldr": { + "cases": 72, + "identical": 72 + }, + "pass": true + }, + "fallback_route": { + "tool": "clean/l/runner/l_d3d12_fallback_check.py --ldr --seeds 2", + "per_step": { + "control swin_block_c96_nc": { + "cases": 12, + "identical": 12, + "bytes": 1055808, + "differ": 0 + }, + "l_coef": { + "cases": 10, + "identical": 10, + "bytes": 1380480, + "differ": 0 + }, + "l_post_filter_ldr": { + "cases": 10, + "identical": 10, + "bytes": 493504, + "differ": 0 + }, + "l_post_recon_ldr": { + "cases": 10, + "identical": 10, + "bytes": 1494328, + "differ": 0 + } + }, + "negative_control": { + "case": "l_post_recon_ldr_s0_f0", + "half_grid": true, + "caught": true, + "differ": 145998 + }, + "pass": true, + "hdr_per_step": { + "control swin_block_c96_nc": { + "cases": 12, + "identical": 12, + "bytes": 1055808, + "differ": 0 + }, + "l_coef": { + "cases": 10, + "identical": 10, + "bytes": 1380480, + "differ": 0 + }, + "l_post_recon": { + "cases": 10, + "identical": 10, + "bytes": 1494328, + "differ": 0 + }, + "w64/post_filter_c1_s4": { + "cases": 10, + "identical": 10, + "bytes": 493504, + "differ": 0 + } + } + }, + "direct_filter_store": { + "tool": "clean/dx12/tests/post_tf_d3d12_check.py --store --ldr --seeds 2", + "summary": { + "cases": 20, + "identical": 20, + "texture_bytes": 9599664, + "filter_f32_bytes": 9970920 + }, + "stored_value_classes": { + "values": 2492730, + "nan": 0, + "beyond_65504": 28176, + "infinite": 0, + "f16_subnormal_range": 44042, + "ties": 1098 + }, + "negative_control": { + "case": "store_s0_0", + "half_grid": true, + "caught": true, + "differ": 16874 + }, + "pass": true + }, + "native_direct_route": { + "tool": "native_direct_test matrix (direct textures against the staged route, synthetic Model L), 4 frames x 24 texture format combinations", + "ldr": "PASS, 96 frames compared, mutation caught", + "hdr": "PASS, 96 frames compared, mutation caught", + "ldr_kernels_launched": [ + "l_direct_color", + "l_post_recon_p12_ldr", + "l_post_filter_direct_ldr", + "l_post_filter_ldr", + "scale_rgb" + ] + }, + "ldr_filter_selection": { + "rule": "Model L with LDR colour runs only the output stage its kernels implement (l_post_filter_ldr: the default filter's geometry, 3 x f32; l_post_filter_ldr_h for the output copy; the direct form on native D3D12); another filter kernel or geometry, an RGBA16F filter output or output_filter=quality5 / quality9 / original is refused at create with the reason (runtime.cpp l_ldr_filter_unfit), on every backend and for the output copy too", + "tests": "test_l_runtime_frame.D3D12.test_ldr_filter_selection: post_filter, post_filter_c1_s16, post_filter_c1_s4_o16 and output_filter=quality5 refused on HIP (HDR keeps the filter kernels), quality9 refused, native D3D12 refused; the default launches l_post_filter_ldr on the default grid" + }, + "model_m_unchanged": { + "m_dxils_rebuilt_identical": [ + 16, + 16 + ], + "hip_l_post_hdr_isa_identical": [ + "l_post_recon", + "l_post_recon_p12" + ], + "hdr_dxils_identical": [ + "l_post_recon", + "l_post_recon_p12", + "w64/post_filter_c1_s4", + "l_post_filter_direct" + ] + } +} diff --git a/clean/l/runner/l_d3d12_fallback_check.py b/clean/l/runner/l_d3d12_fallback_check.py index adfdb0d6..cb75b155 100644 --- a/clean/l/runner/l_d3d12_fallback_check.py +++ b/clean/l/runner/l_d3d12_fallback_check.py @@ -24,6 +24,8 @@ beforehand with its back end's own sentinel (HIP 0xA5, D3D12 0x5A) and followed by 64 slack bytes that no back end may write (the display's pitch padding too); a negative control (one case again with half its workgroups) must be caught; Model M's swin_block_c96_nc through the same engine as the control. Output: counts only, no timing. +--ldr: the LDR route (LDR colour): steps 2 and 3 are l_post_recon_ldr and l_post_filter_ldr (both l_post.co; the +exposures are not read: the exposure-pair frames pass it all the same). """ import argparse import json @@ -78,7 +80,7 @@ def coef_case(rng, rw, rh): "defined": [_defined(n) for n in sizes], "sizes": sizes} -def post_case(rng, x, coef_out, coef_sizes, expo): +def post_case(rng, x, coef_out, coef_sizes, expo, ldr=False): prm = RP.post_params(x) dw, dh = prm.dw, prm.dh data = RP.post_device_inputs(x) @@ -99,19 +101,24 @@ def post_case(rng, x, coef_out, coef_sizes, expo): args += [("buf", RP.pack({"s": np.array(LPC.S)})), ("out", n_hist + SLACK), ("out", n_red + SLACK), ("out", n_disp + SLACK), ("bytes", bytes(prm))] args += [("buf", ev.tobytes())] if expo else [("null",)] - return {"kernel": "l_post_recon", "co": "l_post.co", "grid": RP.recon_grid(dw, dh), "block": 256, "args": args, + return {"kernel": "l_post_recon_ldr" if ldr else "l_post_recon", "co": "l_post.co", "grid": RP.recon_grid(dw, dh), + "block": 256, "args": args, "defined": [_defined(n_hist), np.concatenate([red_def, np.zeros(SLACK, bool)]), np.concatenate([disp_def.reshape(-1), np.zeros(SLACK, bool)])], "n_disp": n_disp, "e": float(ev[0]) if expo else float(prm.e)} -def filter_case(rng, display, dw, dh, ow, oh, e, expo): +def filter_case(rng, display, dw, dh, ow, oh, e, expo, ldr=False): n_out = ow * oh * 12 args = [("buf", display), ("out", n_out + SLACK), ("u32", dw), ("u32", dh), ("u32", ow), ("u32", oh), ("u32", _f32_bits(np.float32(0.01 / (e if e != 0.0 else 1.0))))] args += [("buf", np.array([e, 1.0, 0.0, 0.0, 0.0], np.float32).tobytes())] if expo else [("null",)] + grid = _cdiv(ow, 40) * _cdiv(_cdiv(oh, 2), 8) + if ldr: # the LDR output stage: the same arguments, dec_scale and the exposure pair not read + return {"kernel": "l_post_filter_ldr", "co": "l_post.co", "grid": grid, "block": 64, "args": args, + "defined": [_defined(n_out)]} return {"kernel": "w64/post_filter_c1_s4", "co": "post_w64.co", "tag": "w64", "hip_kernel": "post_filter_c1_s4", - "grid": _cdiv(ow, 40) * _cdiv(_cdiv(oh, 2), 8), "block": 64, "args": args, "defined": [_defined(n_out)]} + "grid": grid, "block": 64, "args": args, "defined": [_defined(n_out)]} def compare(got, want, dmask): @@ -126,7 +133,7 @@ def compare(got, want, dmask): return o -def run(build, rocm, seeds): +def run(build, rocm, seeds, ldr=False): g = C32.Gpu(build, rocm) mods = {} @@ -136,7 +143,7 @@ def module(co): mods[co] = g.hip.load_module(f.read()) return mods[co] - cdir = os.path.join(build, "replay-fallback") + cdir = os.path.join(build, "replay-fallback-ldr" if ldr else "replay-fallback") os.makedirs(cdir, exist_ok=True) lines, todo, neg, frames = [], [], None, [] @@ -145,7 +152,7 @@ def add(name, case, step, label): want = OPS.hip_run(g, module(case["co"]), case) lines += OPS.replay_lines(name, case, cdir) todo.append((name, case, want, step, label)) - if neg is None and step == "l_post_recon" and case["grid"] >= 4: + if neg is None and step.startswith("l_post_recon") and case["grid"] >= 4: neg = (name, case, want) lines += OPS.replay_lines(name, case, cdir, grid=case["grid"] // 2, prefix="negative_") return want @@ -166,10 +173,10 @@ def add(name, case, step, label): x = LPC.make_case(rng, rw, rh, dw, dh, reset, "random" if kind == "specials" else kind) if kind == "specials": x = OPS._post_specials(rng, x) - post = post_case(rng, x, coef_out, coef["sizes"], expo) - post_out = add(f"l_post_recon_{base}", post, "l_post_recon", label) - filt = filter_case(rng, post_out[2][:post["n_disp"]], dw, dh, ow, oh, post["e"], expo) - add(f"w64_post_filter_c1_s4_{base}", filt, "w64/post_filter_c1_s4", label) + post = post_case(rng, x, coef_out, coef["sizes"], expo, ldr) + post_out = add(f"{post['kernel']}_{base}", post, post["kernel"], label) + filt = filter_case(rng, post_out[2][:post["n_disp"]], dw, dh, ow, oh, post["e"], expo, ldr) + add(f"{filt['kernel'].replace('/', '_')}_{base}", filt, filt["kernel"], label) for ci, case in enumerate(OPS.control_cases("swin_block_c96_nc", rng)): add(f"control_swin_block_c96_nc_s{si}_{ci}", case, "control swin_block_c96_nc", "Model M control") with open(os.path.join(cdir, "cases.txt"), "w", newline="\n") as f: @@ -216,11 +223,12 @@ def main(argv=None): ap.add_argument("--rocm", required=True) ap.add_argument("--run", action="store_true") ap.add_argument("--seeds", type=int, default=2) + ap.add_argument("--ldr", action="store_true", help="the LDR route: l_post_recon_ldr, l_post_filter_ldr") ap.add_argument("--json") a = ap.parse_args(argv) if not a.run: return 0 - res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds))) + res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds)), ldr=a.ldr) if a.json: with open(a.json, "w") as f: json.dump(res, f, indent=1) diff --git a/clean/l/runner/l_d3d12_ops_check.py b/clean/l/runner/l_d3d12_ops_check.py index d8f8e40d..0d897cfa 100644 --- a/clean/l/runner/l_d3d12_ops_check.py +++ b/clean/l/runner/l_d3d12_ops_check.py @@ -301,7 +301,7 @@ def post_cases(kernel, rng): defined = [np.concatenate([np.ones(n_hist, bool), np.zeros(POST_SLACK, bool)]), np.concatenate([red_def, np.zeros(POST_SLACK, bool)]), np.concatenate([disp_def.reshape(-1), np.zeros(POST_SLACK, bool)])] - if kernel == "l_post_recon_p12": + if kernel.startswith("l_post_recon_p12"): # l_post_recon_p12, l_post_recon_p12_ldr data = dict(data, **_pack12(data)) args = [("null",) if data[k] == b"" else ("buf", data[k]) for k in ("color", "A", "B", "C", "hist_prev", "m_near", "m_center", "m_far")] @@ -315,6 +315,34 @@ def post_cases(kernel, rng): return cases +def _cdiv(a, b): + return -(-a // b) + + +def filter_ldr_cases(kernel, rng): + """l_post_filter_ldr (l_post.co): display (3 FP16 planes of D, row pitch display_pitch(dw): the LDR display image, + unpacked 11 / 11 / 10-bit codes h(code RN32(1 / N))), out (O, 3 x f32), dw, dh, ow, oh, dec_scale, expo (neither + read; passed as 0 and null). D = recon_size(O) for odd and even O, small (every tap clamped) to wide; every output + byte defined.""" + from clean.l.reference import recon as R + from clean.l.reference import table as T + from clean.runner import post as RP + cases = [] + for (ow, oh) in ((40, 24), (37, 23), (8, 6), (81, 45), (128, 72)): + dw, dh = T.recon_size(ow, oh) + pitch = RP.display_pitch(dw) + codes = np.stack([rng.integers(0, int(n) + 1, (dh, dw)) for n in R.LDR_CODE_MAX], 0).astype(np.float64) + planes = np.zeros((3, dh, pitch), np.float16) + planes[:, :, :dw] = R.ldr_unpack(codes.transpose(1, 2, 0)).transpose(2, 0, 1).astype(np.float16) + n_out = 3 * ow * oh * 4 + cases.append({"kernel": kernel, "co": "l_post.co", "grid": _cdiv(ow, 40) * _cdiv(_cdiv(oh, 2), 8), "block": 64, + "args": [("buf", planes.tobytes()), ("out", n_out + POST_SLACK), ("u32", dw), ("u32", dh), + ("u32", ow), ("u32", oh), ("u32", 0), ("null",)], + "defined": [np.concatenate([np.ones(n_out, bool), np.zeros(POST_SLACK, bool)])], + "label": f"D {dw}x{dh} -> O {ow}x{oh}"}) + return cases + + # DXGI formats of the direct cases' textures: the exact FP32 values of the HIP kernel's buffers (color RGBA, depth, # motion RG); the output texture (written by no encoder) as the game's RGBA16F DXGI_R32G32B32A32_FLOAT, DXGI_R32G32_FLOAT, DXGI_R16G16B16A16_FLOAT, DXGI_R32_FLOAT = 2, 16, 10, 41 @@ -344,9 +372,12 @@ def enc_direct_cases(kernel, rng): KINDS = {"l_linear_32_32_relu": linear_cases, "l_linear_32_32_f16": linear_cases, "l_project_gather": project_cases, "l_dec_project": project_cases, "l_coef": coef_cases, "l_coef_pack12": coef_cases, "lev/input_encoder": enc_cases, "l_post_recon": post_cases, - "l_post_recon_p12": post_cases, "lev/input_encoder_direct": enc_direct_cases, + "l_post_recon_p12": post_cases, "l_post_recon_ldr": post_cases, "l_post_recon_p12_ldr": post_cases, + "l_post_filter_ldr": filter_ldr_cases, "lev/input_encoder_direct": enc_direct_cases, "swin_block_c64_add_ex_nc": dec1_ex_cases} CONTROL_KINDS = {"swin_block_c96_nc": control_cases} +# kernels that get a negative control of their own (besides the run's first case) +NEG_KERNELS = ("l_post_recon", "l_post_recon_p12", "l_post_recon_ldr", "l_post_recon_p12_ldr", "l_post_filter_ldr") CO = "l_ops.co" @@ -431,7 +462,9 @@ def module(co): kinds = kinds or dict(KINDS, **CONTROL_KINDS) cdir = os.path.join(build, "replay-ops") os.makedirs(cdir, exist_ok=True) - lines, hip_out, neg = [], {}, None + # negative controls: the first case (with a grid of 4 or more) again with half its workgroups, for the first kernel + # and for every kernel of NEG_KERNELS (each LDR form its own) + lines, hip_out, neg, negs = [], {}, None, {} for kernel in kernels: for si, seed in enumerate(seeds): rng = np.random.default_rng(seed) @@ -443,8 +476,9 @@ def module(co): with open(os.path.join(cdir, f"{name}.hip.o{oi}"), "wb") as fh: fh.write(data) lines += replay_lines(name, case, cdir) - if neg is None and case["grid"] >= 4: - neg = name + if case["grid"] >= 4 and (neg is None or (kernel in NEG_KERNELS and kernel not in negs)): + neg = neg or name + negs.setdefault(kernel, name) lines += replay_lines(name, case, cdir, grid=case["grid"] // 2, prefix="negative_") with open(os.path.join(cdir, "cases.txt"), "w", newline="\n") as f: f.write("\n".join(lines)) @@ -482,14 +516,18 @@ def compare(got, want, dmask): c["outputs"].append(compare(got, want, defined[oi] if defined else None)) c["identical"] = all(o["identical"] for o in c["outputs"]) res["cases"].append(c) - caught = None - if neg: - wants, defined = hip_out[neg][1], hip_out[neg][2] - outs = [compare(read(os.path.join(cdir, f"negative_{neg}.o{oi}")), w, defined[oi] if defined else None) + def negative(nm): + wants, defined = hip_out[nm][1], hip_out[nm][2] + outs = [compare(read(os.path.join(cdir, f"negative_{nm}.o{oi}")), w, defined[oi] if defined else None) for oi, w in enumerate(wants)] caught = all(o["differ"] >= 0 for o in outs) and any(not o["identical"] for o in outs) - res["negative_control"] = {"case": neg, "half_grid": True, "caught": caught, - "differ": sum(max(o["differ"], 0) for o in outs)} + return {"case": nm, "half_grid": True, "caught": caught, "differ": sum(max(o["differ"], 0) for o in outs)} + + caught = None + if neg: + res["negative_control"] = negative(neg) + res["negative_controls"] = {k: negative(nm) for k, nm in negs.items()} + caught = res["negative_control"]["caught"] and all(v["caught"] for v in res["negative_controls"].values()) res["per_kernel"] = {k: {"cases": sum(c["kernel"] == k for c in res["cases"]), "identical": sum(c["identical"] for c in res["cases"] if c["kernel"] == k), "bytes": sum(o["bytes"] for c in res["cases"] if c["kernel"] == k for o in c["outputs"]), @@ -498,14 +536,16 @@ def compare(got, want, dmask): for k in kernels} res["pass"] = r.returncode == 0 and caught is True and all(v["cases"] > 0 and v["identical"] == v["cases"] for v in res["per_kernel"].values()) - if "l_post_recon" in kernels and "l_post_recon_p12" in kernels: - # The same cases (same seeds) on both coefficient forms: on HIP, the pack12 kernel's raw output bytes must be - # l_post_recon's. - pairs = [(n, n.replace("l_post_recon_", "l_post_recon_p12_", 1)) for n in hip_out - if hip_out[n][0] == "l_post_recon"] - same = sum(hip_out[a][1] == hip_out[b][1] for a, b in pairs if b in hip_out) - res["p12_equals_base_on_hip"] = {"cases": len(pairs), "identical": same} - res["pass"] = res["pass"] and same == len(pairs) > 0 + for sfx in ("", "_ldr"): + if "l_post_recon" + sfx in kernels and "l_post_recon_p12" + sfx in kernels: + # The same cases (same seeds) on both coefficient forms: on HIP, the pack12 kernel's raw output bytes must + # be l_post_recon's (HDR and LDR). + base = "l_post_recon" + sfx + pairs = [(n, n.replace(base + "_", "l_post_recon_p12" + sfx + "_", 1)) for n in hip_out + if hip_out[n][0] == base] + same = sum(hip_out[a][1] == hip_out[b][1] for a, b in pairs if b in hip_out) + res["p12_equals_base_on_hip" + sfx] = {"cases": len(pairs), "identical": same} + res["pass"] = res["pass"] and same == len(pairs) > 0 return res diff --git a/clean/l/runner/l_default_post_check.py b/clean/l/runner/l_default_post_check.py index aa7f5cda..4db982a7 100644 --- a/clean/l/runner/l_default_post_check.py +++ b/clean/l/runner/l_default_post_check.py @@ -7,14 +7,15 @@ python clean/l/runner/l_default_post_check.py --build --run # GPU, inside a GPU slot python clean/l/runner/l_default_post_check.py --build --compare --json out.json # CPU -/dx12 holds the D3D12 ports (python clean/dx12/build_dx12.py --out --model-l). --run runs l_frame_check's -frame sequence for Model L on D3D12 twice, each in its own process: -- "m_default" (A): runtime option l_post=m, today's default (Model M's fused pack12 post, w64/post_tf_f32_p12_diet_vsit, - with the runtime's default kernels). It writes the full and reduced history but no display image (the banded form, - which would, has no D3D12 port), so A is judged on the full history and the reduced codes; -- "own_stages" (B): l_post=own (l_post_recon_p12, then the analytic filter) with the D3D12 fusions off +/dx12 holds the D3D12 ports (python clean/dx12/build_dx12.py --out --model-l). The two candidates were: +- "m_default" (A): Model M's fused pack12 post (w64/post_tf_f32_p12_diet_vsit, then the runtime's default for Model L on + native D3D12, runtime option l_post=m). It writes the full and reduced history but no display image, so A is judged on + the full history and the reduced codes. It failed every frame (clean/l/records/l_default_post_real_weights.json), and + the runtime no longer has that route for Model L (l_post=m is refused): --run no longer runs it; --compare still + judges an A record a run before that change left in (lframe_L_dp_m_default.pkl); +- "own_stages" (B): Model L's own post (l_post_recon_p12, then the analytic filter) with the D3D12 fusions off (WG_D3D12_FUSION=none, every stage output written), judged on the full history, the display image and the reduced - codes. (l_d3d12_own_post_check.py shows the default kernels give the same bytes.) + codes. (l_d3d12_own_post_check.py shows the default kernels give the same bytes.) --run runs it in its own process. --compare checks each run's post per frame on the frame's own inputs (l_frame_check compare_model: the unpacked pack12 coefficients, the run's own previous history) against recon.reconstruct, with Model L's post acceptance (clean/l/runner/l_post_check.py): FP16 values more than one ulp off at most 0.05 % per frame and 0.01 % over the run @@ -38,7 +39,7 @@ from clean.l.runner import l_frame_check as F # noqa: E402 from clean.l.runner import l_post_check as LP # noqa: E402 -CONFIGS = {"m_default": ("m", False, 0x5A), "own_stages": ("own", True, 0x3C)} +CONFIGS = {"own_stages": ("own", True, 0x3C)} # (A, "m_default", ran with l_post=m: no longer a runtime route) def run_one(build, config): @@ -106,14 +107,19 @@ def compare(build, json_path): src = os.path.join(F.model_dir(build), "source.json") res = {"model_source": json.load(open(src)) if os.path.exists(src) else None, "rule": {"over_1_per_frame": LP.L_OVER1_CASE, "over_1_run": LP.L_OVER1_RUN}} - md, own = _load(build, "m_default"), _load(build, "own_stages") - res["kernels"] = {c: r["kernels"] for c, r in (("m_default", md), ("own_stages", own))} - res["A_m_post"] = post_frames(md, params) + own = _load(build, "own_stages") + res["kernels"] = {"own_stages": own["kernels"]} + keys = ["B_own_post"] + if os.path.exists(os.path.join(build, "lframe_L_dp_m_default.pkl")): # an A record from before the route went + md = _load(build, "m_default") + res["kernels"]["m_default"] = md["kernels"] + res["A_m_post"] = post_frames(md, params) + keys.insert(0, "A_m_post") res["B_own_post"] = post_frames(own, params) with open(json_path, "w") as f: json.dump(res, f, indent=1, default=lambda x: x.item() if hasattr(x, "item") else str(x)) print("model source:", json.dumps(res["model_source"])) - for key in ("A_m_post", "B_own_post"): + for key in keys: r = res[key] print(key, "pass" if r["pass"] else "FAIL", "frames passed", r["frames_passed"], "of", len(r["frames"]), "(over-1 limit only:", r["frames_passed_over_1"], ") run:", diff --git a/clean/l/runner/l_frame_check.py b/clean/l/runner/l_frame_check.py index 122c1672..0b8d4902 100644 --- a/clean/l/runner/l_frame_check.py +++ b/clean/l/runner/l_frame_check.py @@ -49,10 +49,11 @@ /dx12 holds the D3D12 ports (python clean/dx12/build_dx12.py --out ). The same frames run on D3D12 with the runtime's default kernels there (Model L: its encoder and decoder ports, the pack12 store l_coef_pack12, its own -reconstruction l_post_recon_p12 and the analytic filter; with l_post=m Model M's pack12 post path instead; Model M: -its native frame), and every stage output both backends write is compared with the HIP run's ("default"), byte for -byte: the D3D12 ports reproduce their HIP kernels byte for byte, and Model M's pack12 post path the banded one bit for -bit. (Where the D3D12 run's reconstruction is not the HIP run's, l_post=m for Model L, its post is checked on its own.) The pack12 images (A: A0 A1 A2 B0 | B1 B2 C2, C: C0 C1; clean/FORMATS.md) are +reconstruction l_post_recon_p12[_ldr] and the output filter; Model M: its native frame), and every stage output both +backends write is compared with the HIP run's ("default"), byte for byte: the D3D12 ports reproduce their HIP kernels +byte for byte, and Model M's pack12 post path the banded one bit for bit. (A record of a Model L D3D12 run with Model +M's post, from before the runtime dropped that route, is checked on its own.) +The pack12 images (A: A0 A1 A2 B0 | B1 B2 C2, C: C0 C1; clean/FORMATS.md) are rearranged into the base layout's bytes and compared with the HIP run's A, B and C. Byte identity (every "byte for byte" above): the RAW defined bytes of every output (RAW_OUTPUTS, the sizes wg_rt_buffer @@ -100,6 +101,16 @@ def model_dir(build): return os.path.join(build, "lframe_model") +# --ldr: Model L with LDR colour (WgRtConfig.hdr = 0: l_post_recon_ldr / l_post_recon_p12_ldr and l_post_filter_ldr), +# checked against the LDR reference (recon.reconstruct / output_filter with ldr=True); the records get an "_ldr" suffix. +# Model M's runs are the same either way (Model M does not read the flag). +LDR = False + + +def rec_path(build, model, config): + return os.path.join(build, f"lframe_{model}_{config}{'_ldr' if LDR else ''}.pkl") + + def l_map(build): return os.path.join(model_dir(build), "maps", "preset-L.v1.map.json") @@ -329,7 +340,7 @@ def run_d3d12(build, special=True, stages=False): d = model_dir(build) setups = {"L": dict(map_path=l_map(build), check_packing=False, kernels={"options": "model=L"}), "M": dict(check_packing=True, kernels={})} - flags = {"L": {"depth_inverted": False, "auto_exposure": True, "hdr": True}, # as the HIP run's + flags = {"L": {"depth_inverted": False, "auto_exposure": True, "hdr": not LDR}, # as the HIP run's "M": {"depth_inverted": False, "auto_exposure": True}} for model, kw in setups.items(): for config in ("d3d12_stages",) if stages else ("d3d12",): @@ -342,10 +353,11 @@ def run_d3d12(build, special=True, stages=False): out = p.frame(fr, debug=True) frames.append({"inputs": fr, "out": out, "raw": raw_outputs(p)}) rec = {"model": model, "config": config, "sentinel": SENTINELS[config], "kernels": p.kernels(), + "ldr": LDR and model == "L", "notes": p.notes(), "model_source": p.model_source, "dw": p.dw, "dh": p.dh, "ow": p.ow, "oh": p.oh, "taps": p.output_taps(), "frames": frames} - with open(os.path.join(build, f"lframe_{model}_{config}.pkl"), "wb") as f: + with open(rec_path(build, model, config), "wb") as f: pickle.dump(rec, f) print(model, config, "frames", len(frames), "post", rec["kernels"].get("post"), "pack", rec["kernels"].get("post_pack"), "bneck", rec["kernels"].get("swin_bneck")) @@ -453,11 +465,11 @@ def compare_d3d12(build, json_path): for model in ("L", "M"): recs = {} for config in ("default", "d3d12"): - with open(os.path.join(build, f"lframe_{model}_{config}.pkl"), "rb") as f: + with open(rec_path(build, model, config), "rb") as f: recs[config] = pickle.load(f) # the HIP run's reconstruction: Model M's (bit-identical to the D3D12 fused pack12 post) or Model L's own; the # D3D12 run's: for Model L its own (l_post_recon: the runtime's default where its route applies) or Model M's - # fused pack12 post (l_post=m: m_pack12) + # fused pack12 post (a record from before the runtime dropped that route for Model L: m_pack12) hip_post = recs["default"]["kernels"].get("l_post") or recs["default"]["kernels"].get("post_last") same_post = hip_post != "l_post_recon" or recs["d3d12"]["kernels"].get("l_post") == "l_post_recon" no_display = "post_tf" in str(recs["d3d12"]["kernels"].get("post_last", "")) @@ -480,7 +492,7 @@ def compare_d3d12(build, json_path): # previous frame's reconstruction, so the two runs part after the first frame. The D3D12 frame is checked # on its own: every stage of the run with all outputs written against the reference on that run's inputs # (as --compare), and the default kernels against that run byte for byte. - with open(os.path.join(build, f"lframe_{model}_d3d12_stages.pkl"), "rb") as f: + with open(rec_path(build, model, "d3d12_stages"), "rb") as f: st = fill_pack12(pickle.load(f)) per_stage = compare_model(st, params[model], post_ref="M", bneck_unit=True) fused, unw = compare_d3d12_configs(st, recs["d3d12"]) @@ -527,10 +539,10 @@ def run(build, rocm, special=True): clean_process_environment(rocm) from clean.runner.pipeline import Pipeline d = model_dir(build) - # Model L: HDR colour (its reconstruction kernel l_post_recon is HDR; the runtime refuses Model L without it) + # Model L: HDR colour, or with --ldr LDR colour (l_post_recon_ldr) setups = {"L": dict(map_path=l_map(build), check_packing=False, kernels={"options": "model=L"}), "M": dict(check_packing=True, kernels={"pack": "none"})} - flags = {"L": {"depth_inverted": False, "auto_exposure": True, "hdr": True}, + flags = {"L": {"depth_inverted": False, "auto_exposure": True, "hdr": not LDR}, "M": {"depth_inverted": False, "auto_exposure": True}} for model, kw in setups.items(): # "stages": the bottleneck as six block launches, so that b0 ... b4 are written (the default megakernel writes @@ -546,10 +558,11 @@ def run(build, rocm, special=True): out = p.frame(fr, debug=True) frames.append({"inputs": fr, "out": out, "raw": raw_outputs(p)}) rec = {"model": model, "config": config, "sentinel": SENTINELS[config], "kernels": p.kernels(), + "ldr": LDR and model == "L", "notes": p.notes(), "model_source": p.model_source, "dw": p.dw, "dh": p.dh, "ow": p.ow, "oh": p.oh, "taps": p.output_taps(), "frames": frames} - with open(os.path.join(build, f"lframe_{model}_{config}.pkl"), "wb") as f: + with open(rec_path(build, model, config), "wb") as f: pickle.dump(rec, f) print(model, config, "frames", len(frames), "bneck", rec["kernels"].get("swin_bneck"), "post", rec["kernels"].get("post")) @@ -604,13 +617,20 @@ def same_values(got, ref): def post_fp16(got, ref, model="M"): - finite = np.isfinite(got) - s = _f16_steps(got, ref) - d = np.abs(np.where(finite, got, 0.0).astype(np.float64) - ref) + """A non-finite value counts as agreeing only where the reference holds the same one (both NaN, or the same + infinity: Model L's LDR form stores H^ as it is, so a NaN reaches the history on both sides); FP16 steps, + differences and the RMS are over the finite pairs. (With a finite reference, as in HDR, this is the plain rule.)""" + got, ref = np.asarray(got, np.float32), np.asarray(ref, np.float32) + fin_g, fin_r = np.isfinite(got), np.isfinite(ref) + both = fin_g & fin_r + same_nf = ~fin_g & ~fin_r & ((got == ref) | (np.isnan(got) & np.isnan(ref))) + s = _f16_steps(np.where(both, got, 0.0), np.where(both, ref, 0.0)) + d = np.abs(np.where(both, got, 0.0).astype(np.float64) - np.where(both, ref, 0.0)) + rms = np.sqrt(np.mean(np.square(ref[fin_r], dtype=np.float64))) if fin_r.any() else 0.0 m = {"values": int(s.size), "differ": int(np.sum(s > 0)), "max_f16_steps": int(s.max()) if s.size else 0, - "nonfinite": int(np.sum(~finite)), "frac_over_1": float(np.mean(s > 1)), "over_1": int(np.sum(s > 1)), - "frac_diff": float(np.mean(s > 0)), - "mean_rel_rms": float(d.mean() / max(np.sqrt(np.mean(np.square(ref, dtype=np.float64))), 1e-30))} + "nonfinite": int(np.sum(~(both | same_nf))), "nonfinite_agreeing": int(np.sum(same_nf)), + "frac_over_1": float(np.mean(s > 1)), "over_1": int(np.sum(s > 1)), "frac_diff": float(np.mean(s > 0)), + "mean_rel_rms": float(d.mean() / max(rms, 1e-30))} m.update(_magnitudes(got, ref)) if model == "L": # Model L's post acceptance (clean/l/runner/l_post_check.py, provisional): per frame at most 0.05 % more than @@ -623,8 +643,8 @@ def post_fp16(got, ref, model="M"): def compare_model(rec, params, post_ref=None, bneck_unit=False): - """post_ref "M": the post against Model M's reference whatever the model (Model L's native D3D12 frame runs Model - M's fused pack12 post for now). bneck_unit: b0 ... b5 as one unit (a bottleneck megakernel writes only b5): the + """post_ref "M": the post against Model M's reference whatever the model (a record of a run with Model M's post). + bneck_unit: b0 ... b5 as one unit (a bottleneck megakernel writes only b5): the reference chained from the runtime's enc2.merged, b5 under the per-stage rule.""" from clean.l.reference import oracle as L from clean.l.reference import table as T @@ -720,7 +740,8 @@ def compare_model(rec, params, post_ref=None, bneck_unit=False): "jitter": fr["jitter"], "reset": eff_reset, "recon_size": D} s_post = float(np.asarray(params["post"]["s"]).reshape(-1)[0]) l_recon = model == "L" and post_ref != "M" - ref = R.reconstruct(xp, s_post) if l_recon else Po.reconstruct(xp, s_post) + ldr = bool(rec.get("ldr", False)) and l_recon + ref = R.reconstruct(xp, s_post, ldr=ldr) if l_recon else Po.reconstruct(xp, s_post) m["post"] = {"history": post_fp16(out["post"]["history"], ref["history"], model)} if out["post"]["display"] is not None: m["post"]["display"] = post_fp16(out["post"]["display"], ref["display"], model) @@ -732,11 +753,14 @@ def compare_model(rec, params, post_ref=None, bneck_unit=False): if rec["taps"] is not None: fo = Po.output_filter_original(disp, e, (rec["ow"], rec["oh"]), rec["taps"]) elif l_recon: - fo = R.output_filter(disp, e, (rec["ow"], rec["oh"])) + fo = R.output_filter(disp, e, (rec["ow"], rec["oh"]), ldr=ldr) else: fo = Po.output_filter(disp, e, (rec["ow"], rec["oh"])) fo = (np.asarray(fo, np.float32) * pi).astype(np.float32) - close = np.isclose(out["out"], fo, rtol=1e-5, atol=2e-9) + # (LDR: plain colours, about 1 at most, so the float32 rounding of the filter's two 5-tap passes is an + # absolute error, about 2^-24 per operation: a result near 0 from cancelling taps is off by up to some + # 1e-8; atol 1e-6 pi bounds the ten operations with the weights' sum of magnitudes) + close = np.isclose(out["out"], fo, rtol=1e-5, atol=1e-6 * float(pi) if ldr else 2e-9) m["filter"] = {"values": int(close.size), "outside": int(np.sum(~close)), "pass": bool(close.all())} stage_pass = {s: (all(v["pass"] for v in m[s].values() if isinstance(v, dict) and "pass" in v) @@ -792,7 +816,7 @@ def compare(build, json_path): for model in ("L", "M"): recs = {} for config in ("stages", "default"): - with open(os.path.join(build, f"lframe_{model}_{config}.pkl"), "rb") as f: + with open(rec_path(build, model, config), "rb") as f: recs[config] = pickle.load(f) frames = compare_model(recs["stages"], params[model]) # the default kernels against the six-launch bottleneck, raw defined bytes: b0 ... b4 are not written by the @@ -853,7 +877,11 @@ def main(argv=None): ap.add_argument("--model", help="--prepare: a converter-written Model L model.wgcm in place of the synthetic model") ap.add_argument("--map", help="--prepare with --model: the map it was converted with (default: the shipped " "data/preset-L.v1.map.json)") + ap.add_argument("--ldr", action="store_true", + help="--run / --compare / --compare-d3d12: Model L with LDR colour (records *_ldr.pkl)") a = ap.parse_args(argv) + global LDR + LDR = a.ldr if (a.model or a.map) and not a.prepare: ap.error("--model and --map go with --prepare") if a.map and not a.model: diff --git a/clean/l/runner/l_post_check.py b/clean/l/runner/l_post_check.py index fbd7b2e7..ef68a443 100644 --- a/clean/l/runner/l_post_check.py +++ b/clean/l/runner/l_post_check.py @@ -85,8 +85,9 @@ def make_case(rng, rw, rh, dw, dh, reset, kind): return x -def run_kernel(hip, mod, x, s): - """Launch l_post_recon with sentinel-filled outputs; returns the raw output buffers (with slack) and sizes.""" +def run_kernel(hip, mod, x, s, kernel="l_post_recon"): + """Launch l_post_recon (or kernel: its LDR form l_post_recon_ldr) with sentinel-filled outputs; returns the raw + output buffers (with slack) and sizes.""" prm = RP.post_params(x) data = RP.post_device_inputs(x) dw, dh = prm.dw, prm.dh @@ -105,7 +106,7 @@ def run_kernel(hip, mod, x, s): args = [ctypes.c_void_p(bufs[k]) for k in ("color", "A", "B", "C", "hist_prev", "m_near", "m_center", "m_far")] args += [ctypes.c_void_p(pw), ctypes.c_void_p(outs["history"]), ctypes.c_void_p(outs["reduced"]), ctypes.c_void_p(outs["display"]), prm, ctypes.c_void_p(0)] - hip.launch(hip.get_function(mod, "l_post_recon"), RP.recon_grid(dw, dh), 256, args) + hip.launch(hip.get_function(mod, kernel), RP.recon_grid(dw, dh), 256, args) hip.sync() raw = {k: np.frombuffer(hip.download(outs[k], n + SLACK), np.uint8) for k, n in sizes.items()} for ptr in list(bufs.values()) + [pw] + list(outs.values()): @@ -146,16 +147,23 @@ def l_post_run(items): def m_rule(got, ref): + """A non-finite value counts as agreeing only where the reference has the same one (both NaN, or the same + infinity: LDR stores H^ as it is, so a zero total gather weight gives NaN on both sides, spec part 3 Q1); every + other non-finite value fails the rule. FP16 steps, differences and the reference RMS are over the finite pairs (in + HDR the reference is always finite).""" res = {} for k in ("history", "display"): g, r = np.asarray(got[k], np.float32), np.asarray(ref[k], np.float32) - fin = np.isfinite(g) - s = C32.f16_steps(np.where(fin, g, 0), r) - d = np.abs(np.where(fin, g, 0.0).astype(np.float64) - r) - res[k] = {"values": int(s.size), "nonfinite": int(np.sum(~fin)), "frac_over_1": float(np.mean(s > 1)), + fin_g, fin_r = np.isfinite(g), np.isfinite(r) + both = fin_g & fin_r + same_nf = ~fin_g & ~fin_r & ((g == r) | (np.isnan(g) & np.isnan(r))) + s = C32.f16_steps(np.where(both, g, 0), np.where(both, r, 0)) + d = np.abs(np.where(both, g, 0.0).astype(np.float64) - np.where(both, r, 0.0)) + rms = np.sqrt(np.mean(np.square(r[fin_r], dtype=np.float64))) if fin_r.any() else 0.0 + res[k] = {"values": int(s.size), "nonfinite": int(np.sum(~(both | same_nf))), + "nonfinite_agreeing": int(np.sum(same_nf)), "frac_over_1": float(np.mean(s > 1)), "over_1": int(np.sum(s > 1)), "differ": int(np.sum(s > 0)), "frac_diff": float(np.mean(s > 0)), - "max_steps": int(s.max()), - "mean_rel_rms": float(d.mean() / max(np.sqrt(np.mean(np.square(r, dtype=np.float64))), 1e-30))} + "max_steps": int(s.max()), "mean_rel_rms": float(d.mean() / max(rms, 1e-30))} res[k]["pass"] = l_post_rule(res[k]["values"], res[k]["nonfinite"], res[k]["over_1"], res[k]["differ"], res[k]["mean_rel_rms"]) rc = np.rint(np.asarray(ref["reduced"], np.float64) * 1023.0).astype(np.int64) @@ -166,10 +174,10 @@ def m_rule(got, ref): return res -def check_case(hip, mod, x, kind, s=S): - raw, sizes, dims = run_kernel(hip, mod, x, s) +def check_case(hip, mod, x, kind, s=S, ldr=False): + raw, sizes, dims = run_kernel(hip, mod, x, s, "l_post_recon_ldr" if ldr else "l_post_recon") got = decode(raw, sizes, dims) - ref = R.reconstruct(x, s) + ref = R.reconstruct(x, s, ldr=ldr) m = m_rule(got, ref) m.update({"alpha_zero": got["alpha_zero"], "reduced_top_zero": got["reduced_top_zero"], "slack_untouched": got["slack_untouched"]}) @@ -189,38 +197,43 @@ def check_case(hip, mod, x, kind, s=S): "over_1": int((st[ill] > 1).sum())} m["well_conditioned"] = {"max_steps": int(st[~ill].max(initial=0)), "over_1": int((st[~ill] > 1).sum()), "values": int(st[~ill].size)} - if kind == "offset_0.1": - known = ((got["history"] == np.float32(-1.4404296875)).all() and (got["display"] == 0).all() - and (got["reduced_codes"] == 0).all() and m["weight_zero_pixels"] == W.size) + if kind == "offset_0.1": # total weight 0 everywhere (Q1): history -1.4404296875 (HDR) or NaN (LDR), code 0 + hist_ok = np.isnan(got["history"]).all() if ldr else (got["history"] == np.float32(-1.4404296875)).all() + known = (hist_ok and (got["display"] == 0).all() and (got["reduced_codes"] == 0).all() + and m["weight_zero_pixels"] == W.size) m["known_answer"] = bool(known) ok = ok and known if kind == "offset_0.05": - Hl = np.asarray(x["color"], np.float64) * float(np.asarray(x["exposure"])) - known = bool((W > 0).all()) and np.array_equal(got["history"], MP.f16(MP.enc(Hl))) + if ldr: # LDR: the linear colour as it is (no exposure, identity transfer) + want = MP.f16(np.asarray(x["color"], np.float64)) + else: + want = MP.f16(MP.enc(np.asarray(x["color"], np.float64) * float(np.asarray(x["exposure"])))) + known = bool((W > 0).all()) and np.array_equal(got["history"], want, equal_nan=True) m["known_answer"] = bool(known) ok = ok and known m["pass"] = bool(ok) return m -def run(build, rocm, seeds=(89,)): +def run(build, rocm, seeds=(89,), ldr=False): from clean.runner.hip import Hip hip = Hip(rocm) with open(os.path.join(build, "l_post.co"), "rb") as f: mod = hip.load_module(f.read()) - res = {"cases": []} + res = {"cases": [], "kernel": "l_post_recon_ldr" if ldr else "l_post_recon", "ldr": ldr} for seed in seeds: rng = np.random.default_rng(seed) for rw, rh, dw, dh, reset, kind in CASES: x = make_case(rng, rw, rh, dw, dh, reset, kind) - m = check_case(hip, mod, x, kind) + m = check_case(hip, mod, x, kind, ldr=ldr) m["case"] = [rw, rh, dw, dh, reset, kind] res["cases"].append(m) # Negative control: the first random case with the jitter moved by half a render pixel must fail. rng = np.random.default_rng(seeds[0]) x = make_case(rng, *CASES[0]) - raw, sizes, dims = run_kernel(hip, mod, dict(x, jitter=np.asarray(x["jitter"], np.float32) + 0.5), S) - neg = m_rule(decode(raw, sizes, dims), R.reconstruct(x, S)) + raw, sizes, dims = run_kernel(hip, mod, dict(x, jitter=np.asarray(x["jitter"], np.float32) + 0.5), S, + "l_post_recon_ldr" if ldr else "l_post_recon") + neg = m_rule(decode(raw, sizes, dims), R.reconstruct(x, S, ldr=ldr)) res["negative_control_fails"] = not neg["pass"] res["run"] = {k: l_post_run([c[k] for c in res["cases"]]) for k in ("history", "display")} res["pass"] = (all(c["pass"] for c in res["cases"]) and res["negative_control_fails"] @@ -233,9 +246,10 @@ def main(argv=None): ap.add_argument("--build", required=True) ap.add_argument("--rocm", required=True) ap.add_argument("--seeds", type=int, default=1, help="run the case list with this many seeds (89, 90, ...)") + ap.add_argument("--ldr", action="store_true", help="the LDR form l_post_recon_ldr against the LDR oracle") ap.add_argument("--json", required=True) a = ap.parse_args(argv) - res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds))) + res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds)), ldr=a.ldr) with open(a.json, "w") as f: json.dump(res, f, indent=1) for c in res["cases"]: diff --git a/clean/l/runtime/tests/test_l_runtime_frame.py b/clean/l/runtime/tests/test_l_runtime_frame.py index 6608135b..8f4ef723 100644 --- a/clean/l/runtime/tests/test_l_runtime_frame.py +++ b/clean/l/runtime/tests/test_l_runtime_frame.py @@ -185,7 +185,15 @@ def base(kernel): """A Model L launch's kernel without its clamp-free suffix; the pack12 coefficient store (native D3D12) as l_coef, whose arguments it shares.""" k = kernel.replace("_nc", "") - return "l_coef" if k == "l_coef_pack12" else "l_post_recon" if k == "l_post_recon_p12" else k + post = ("l_post_recon_p12", "l_post_recon_ldr", "l_post_recon_p12_ldr") # pack12 and LDR forms + return "l_coef" if k == "l_coef_pack12" else "l_post_recon" if k in post else k + + +def model_m_post(names): + """The launches of Model M's post route among a frame's kernel names: its reconstruction kernels (post_recon*, + post_tf*, post_fused*, the pmo/ motion-at-output forms and the banded forms), which Model L never runs.""" + return [n for n in names if n.split("/")[-1].startswith(("post_recon", "post_tf", "post_fused", "post_prep")) + or n.startswith("pmo/") or n.endswith("_band")] def check_roles(t, f, digests, l_post=True): @@ -408,11 +416,12 @@ def check_frame(self, t, f, roles=True, l_post=True): else: self.assertEqual(coef.params(6), [rw, rh, fw, fh, info["reset"]]) post = [ln for ln in ls if base(ln.kernel) == "l_post_recon"] - self.assertEqual(len(post), 1 if l_post else 0) # (native D3D12: Model M's fused pack12 post by default) + self.assertEqual(len(post), 1 if l_post else 0) if l_post: self.assertEqual(post[0].params(12), [rw, rh, (3 * 120 + 1) // 2, (3 * 72 + 1) // 2, info["reset"]]) self.assertEqual(post[0].grid, cdiv((3 * 120 + 1) // 2, 32) * cdiv(cdiv((3 * 72 + 1) // 2, 2), 8)) - self.assertTrue(ls[ls.index(post[0]) + 1].kernel.endswith("post_filter_c1_s4")) # Model M's filter next + nxt = ls[ls.index(post[0]) + 1].kernel # the output filter next (LDR: Model L's LDR output stage) + self.assertTrue(nxt.endswith("post_filter_c1_s4") or nxt == "l_post_filter_ldr", nxt) self.assertTrue(lev.args[11].startswith("b")) # the values blob: an allocation if roles: # every argument against the named buffers and blobs check_roles(t, f, blob_digests(), l_post) @@ -531,13 +540,12 @@ def test_swapped_weights_in_the_runtime(self): class D3D12(unittest.TestCase): - """Model L on native D3D12 (backend 3): Model M's native frame with Model L's encoder and decoder, the pack12 - coefficient store for Model M's pack12 post path; each missing D3D12 port refused by name, never replaced.""" + """Model L on native D3D12 (backend 3): Model M's native frame with Model L's encoder, decoder, pack12 coefficient + store and its own reconstruction (no Model M post route); each missing D3D12 port refused by name, never + replaced.""" def l_run(self, ports, frames=3, flags=(), **kw): - # Model M's fused pack12 post (l_post=m): the route these tests trace; Model L's own post (the default where - # its ports are present) is test_own_post's - return run("model=L;l_post=m", frames=frames, build_files=FT.SHIPPING, flags=("--backend=3",) + tuple(flags), + return run("model=L", frames=frames, build_files=FT.SHIPPING, flags=("--backend=3",) + tuple(flags), manifest=ports, **kw) def test_missing_ports_named(self): @@ -556,8 +564,9 @@ def test_each_port_named(self): self.assertEqual(got, ["ERR create: Model L on native D3D12: not yet ported to D3D12: " + k]) def check_d3d12_frame(self, t, f): - """The #111 checks (map chain, extents, offsets, parameters, every argument's buffer or blob), then what the - native frame adds: the pack12 store's images, dec1's own block, Model M's pack12 post path after it.""" + """The #111 checks (map chain, extents, offsets, parameters, every argument's buffer or blob, Model L's + reconstruction and the filter after it), then what the native frame adds: the pack12 store's images, dec1's own + block, l_post_recon_p12 right after the store, and no Model M post kernel.""" info = f["info"] rw, rh = info["rw"], info["rh"] names = [ln.kernel for ln in f["launches"]] @@ -578,12 +587,13 @@ def check_d3d12_frame(self, t, f): self.assertTrue(all(not n.startswith(("d1p12/", "d0p12/")) and n != "input_encoder" for n in names), names) dec1 = [ln for ln in f["launches"] if ln.kernel.startswith("swin_block_c64_add")] self.assertEqual([ln.kernel for ln in dec1], ["swin_block_c64_add_ex_nc"]) # dec1's own block, not fused - lev, coef = Arguments.check_frame(self, t, f, l_post=False) + lev, coef = Arguments.check_frame(self, t, f) a, c = (coef.args[8], coef.args[9]) if fold else (coef.args[1], coef.args[3]) self.assertGreaterEqual(t.extent(a), 8 * rw * rh) # pack12 image A: 8 bytes per pixel self.assertGreaterEqual(t.extent(c), 4 * rw * rh) # pack12 image C: 4 bytes per pixel i = names.index(FOLD if fold else "l_coef_pack12") - self.assertTrue(names[i + 1].startswith("w64/post_tf") and "_p12" in names[i + 1], names[i + 1:]) + self.assertEqual(names[i + 1], "l_post_recon_p12", names[i + 1:]) + self.assertEqual(model_m_post(names), []) return lev, coef def test_frame(self): @@ -644,79 +654,151 @@ def test_hip_unchanged(self): self.assertTrue(run("model=L", build_files=FT.SHIPPING, env={"WG_POST_PACK": "p12"})[0].startswith( "ERR create: Model L: post_pack=p12")) - # runtime option l_post=own: Model L's own reconstruction on native D3D12 (its D3D12 ports, then the analytic filter) - OWN = [("l_post_recon_p12", "l_post.co"), ("l_post_recon", "l_post.co"), ("l_coef", "l_ops.co"), - ("w64/post_filter_c1_s4", "post_w64.co")] + # Model L's reconstruction on native D3D12, HDR and LDR colour: its D3D12 ports, then the output filter + OWN = {1: [("l_post_recon_p12", "l_post.co"), ("l_post_recon", "l_post.co"), ("l_coef", "l_ops.co"), + ("w64/post_filter_c1_s4", "post_w64.co")], + 0: [("l_post_recon_p12_ldr", "l_post.co"), ("l_post_recon_ldr", "l_post.co"), ("l_coef", "l_ops.co"), + ("l_post_filter_ldr", "l_post.co")]} def own_ports(self): have = full_ports() - return have + [kc for kc in self.OWN if kc not in have] + return have + [kc for hdr in self.OWN for kc in self.OWN[hdr] if kc not in have] def test_own_post(self): - """l_post=own: l_post_recon_p12 on the frames l_coef_pack12 stores, with post_recon's arguments (the #111 role - checks), then the unbanded filter w64/post_filter_c1_s4 from its display image into the output; no fused post. - A frame the pack12 kernels do not take (render 180 x 108 = D) runs l_coef and l_post_recon.""" + """HDR and LDR colour: l_post_recon_p12[_ldr] on the frames l_coef_pack12 stores, with post_recon's arguments + (the #111 role checks), then the unbanded filter (HDR w64/post_filter_c1_s4 with the exposure pair; LDR + l_post_filter_ldr) from its display image into the output; no Model M post kernel. A frame the pack12 kernels + do not take (render 180 x 108 = D) runs l_coef and l_post_recon[_ldr] (the fallback route).""" sizes = "--sizes=96x64,96x64,180x108,80x48,77x45,180x108" - t = FT.Trace(run("model=L;l_post=own", frames=6, build_files=FT.SHIPPING, rw=180, rh=108, - flags=("--backend=3", sizes, "--reset=4"), manifest=self.own_ports())) - self.assertEqual(t.create, "create ok model L") - self.assertEqual(t.unknown(), []) - self.assertEqual(t.kernels.get("l_post"), "l_post_recon") - prev = None - for f in t.frames: - self.assertIsNone(f["error"]) - info, ls = f["info"], f["launches"] - names = [ln.kernel for ln in ls] - p12 = (info["rw"], info["rh"]) != (180, 108) # pack12 frames: the fold (fusion lc, the default) - self.assertIn(FOLD if p12 else "l_coef", names) - self.assertFalse({"l_coef_pack12", "l_coef" if p12 else FOLD} & set(names), names) - self.assertFalse([n for n in names if "post_tf" in n or "post_recon2" in n or n.endswith("_band")], names) - post = [ln for ln in ls if ln.kernel == ("l_post_recon_p12" if p12 else "l_post_recon")] - self.assertEqual(len(post), 1, names) - lev, coef = Arguments.check_frame(self, t, f) # the roles of every argument, the filter after the post - flt = ls[ls.index(post[0]) + 1] - self.assertEqual(flt.kernel, "w64/post_filter_c1_s4") - self.assertEqual((flt.grid, flt.block), (cdiv(120, 40) * cdiv(cdiv(72, 2), 8), 64)) - self.assertEqual(flt.args[:2], [f["names"]["post.display"], f["names"]["filter.out"]]) - self.assertEqual(flt.args[2:6], [str((3 * 120 + 1) // 2), str((3 * 72 + 1) // 2), "120", "72"]) - self.assertEqual(flt.args[7], f["names"]["exposure.pair"]) - if p12: # pack12 image A (the fold's argument 8) is what the reconstruction reads - self.assertEqual(coef.args[8], post[0].args[1]) - self.assertGreaterEqual(t.extent(coef.args[8]), 8 * info["rw"] * info["rh"]) - if prev: - self.assertEqual(lev.args[4], latent_out(prev[1])) # the latent the last frame's store wrote - prev = (lev, coef) + for hdr in (1, 0): + sfx = "" if hdr else "_ldr" + with self.subTest(hdr=hdr): + t = FT.Trace(run("model=L", frames=6, build_files=FT.SHIPPING, rw=180, rh=108, hdr=hdr, + flags=("--backend=3", sizes, "--reset=4"), manifest=self.own_ports())) + self.assertEqual(t.create, "create ok model L") + self.assertEqual(t.unknown(), []) + self.assertEqual(t.kernels.get("l_post"), "l_post_recon" + sfx) + prev = None + for f in t.frames: + self.assertIsNone(f["error"]) + info, ls = f["info"], f["launches"] + names = [ln.kernel for ln in ls] + p12 = (info["rw"], info["rh"]) != (180, 108) # pack12 frames: the fold (fusion lc, the default) + self.assertIn(FOLD if p12 else "l_coef", names) + self.assertFalse({"l_coef_pack12", "l_coef" if p12 else FOLD} & set(names), names) + self.assertEqual(model_m_post(names), []) + post = [ln for ln in ls if ln.kernel == ("l_post_recon_p12" if p12 else "l_post_recon") + sfx] + self.assertEqual(len(post), 1, names) + lev, coef = Arguments.check_frame(self, t, f) # every argument's role, the filter after the post + flt = ls[ls.index(post[0]) + 1] + self.assertEqual(flt.kernel, "w64/post_filter_c1_s4" if hdr else "l_post_filter_ldr") + self.assertEqual((flt.grid, flt.block), (cdiv(120, 40) * cdiv(cdiv(72, 2), 8), 64)) + self.assertEqual(flt.args[:2], [f["names"]["post.display"], f["names"]["filter.out"]]) + self.assertEqual(flt.args[2:6], [str((3 * 120 + 1) // 2), str((3 * 72 + 1) // 2), "120", "72"]) + if hdr: + self.assertEqual(flt.args[7], f["names"]["exposure.pair"]) + if p12: # pack12 image A (the fold's argument 8) is what the reconstruction reads + self.assertEqual(coef.args[8], post[0].args[1]) + self.assertGreaterEqual(t.extent(coef.args[8]), 8 * info["rw"] * info["rh"]) + if prev: + self.assertEqual(lev.args[4], latent_out(prev[1])) # the latent the last frame's store wrote + prev = (lev, coef) - def test_own_post_refusals(self): + def test_no_model_m_post_route(self): + """Model L never runs Model M's post: every route traced (HDR and LDR colour; HIP and native D3D12; pack12 and + fallback frames) launches Model L's reconstruction and no Model M post kernel; l_post=m is refused; where Model + L's own route cannot run (a port missing, another post / filter route asked for) creation is refused with the + reason, never a different algorithm.""" def first(options, **kw): return run(options, frames=1, build_files=FT.SHIPPING, **kw)[0] d3d12 = dict(flags=("--backend=3",), manifest=self.own_ports()) + # the runtime option: m refused, own a no-op, anything else refused + self.assertEqual(first("model=L;l_post=m", **d3d12), "ERR create: l_post=m: Model L has no Model M post route " + "(its reconstruction is l_post_recon[_ldr] only)") self.assertEqual(first("model=L;l_post=yes", **d3d12), - "ERR create: l_post must be m or own (Model L on native D3D12)") - self.assertTrue(first("model=L;l_post=own", hdr=0, **d3d12).startswith( - "ERR create: Model L: l_post=own needs HDR colour")) + "ERR create: l_post must be own (Model L's reconstruction; there is no other)") self.assertEqual(first("l_post=own", **d3d12), "ERR create: l_post=own is a Model L option (its own " "reconstruction)") - for k, _ in self.OWN: - with self.subTest(port=k): - got = first("model=L;l_post=own", flags=("--backend=3",), - manifest=[kc for kc in self.own_ports() if kc[0] != k]) - self.assertEqual(got, "ERR create: Model L on native D3D12 (l_post=own): not yet ported to D3D12: " + k) - # the default: Model L's own post where its route applies (HDR, its ports present), else Model M's fused pack12 - # post; l_post=m chooses Model M's; on HIP Model L runs its own post either way - def default(options="model=L", **kw): - return FT.Trace(run(options, frames=1, build_files=FT.SHIPPING, **dict(d3d12, **kw))).kernels.get("l_post") - self.assertEqual(default(), "l_post_recon") - self.assertEqual(default("model=L;l_post=m"), "m_pack12") - self.assertEqual(default(hdr=0), "m_pack12") - for k, _ in self.OWN: - with self.subTest(default_without=k): - self.assertEqual(default(manifest=[kc for kc in self.own_ports() if kc[0] != k]), "m_pack12") + # a missing port of Model L's route: refused by name (HDR and LDR), never another post + for hdr, kcs in self.OWN.items(): + for k, _ in kcs: + with self.subTest(hdr=hdr, port=k): + got = first("model=L", hdr=hdr, flags=("--backend=3",), + manifest=[kc for kc in self.own_ports() if kc[0] != k]) + self.assertEqual(got, "ERR create: Model L on native D3D12 (%s): not yet ported to D3D12: %s; " + "Model L has no other reconstruction (no Model M post route)" + % ("HDR" if hdr else "LDR", k)) + # another of Model M's filter routes asked for: refused (the RGBA16F filter output) + got = first("model=L", env={"WG_FILTER_KERNEL": "post_filter_c1_s4_o16"}, **d3d12) + self.assertTrue(got.startswith("ERR create: "), got) + self.assertNotIn("create ok", got) + # every route: Model L's reconstruction, no Model M post kernel (HIP: base coefficients and l_post_recon[_ldr]; + # native D3D12: pack12 frames and the fallback frame) + sizes = "--sizes=96x64,180x108,80x48" + for hdr in (1, 0): + sfx = "" if hdr else "_ldr" + for backend in ("hip", "d3d12"): + with self.subTest(hdr=hdr, backend=backend): + kw = dict(d3d12) if backend == "d3d12" else {} + kw["flags"] = kw.get("flags", ()) + (sizes,) + t = FT.Trace(run("model=L", frames=3, build_files=FT.SHIPPING, rw=180, rh=108, hdr=hdr, **kw)) + self.assertEqual(t.create, "create ok model L") + self.assertEqual(t.kernels.get("l_post"), "l_post_recon" + sfx) + for f in t.frames: + self.assertIsNone(f["error"]) + names = [ln.kernel for ln in f["launches"]] + self.assertEqual(model_m_post(names), [], names) + own = [n for n in names if base(n) == "l_post_recon"] + self.assertEqual(len(own), 1, names) + self.assertTrue(own[0].endswith(sfx), own) + if not hdr: # LDR: Model L's LDR output stage, no exposure-decoding filter + self.assertIn("l_post_filter_ldr", names) + self.assertFalse([n for n in names if n.endswith("post_filter_c1_s4")], names) def hip(options): return [re.sub(r"wg_lfb_\w+", "B", x) for x in run(options, frames=2, build_files=FT.SHIPPING)] self.assertEqual(hip("model=L;l_post=own"), hip("model=L")) + def test_ldr_filter_selection(self): + """LDR colour: the output stage is l_post_filter_ldr (the default filter's geometry, 3 x f32). Another filter + selection has no LDR form and is refused at create with the reason, on HIP (an alternate geometry, an RGBA16F + filter output, output_filter=quality9) and on native D3D12, never run with another filter's grid or format (the + output copy's l_post_filter_ldr_h is behind the same check). HDR keeps those selections. The default LDR frame + launches l_post_filter_ldr on the default filter's grid, from the display image into the output.""" + def first(options="model=L", **kw): + return run(options, frames=1, build_files=FT.SHIPPING, **kw)[0] + why = ("ERR create: Model L, LDR colour: its output stage is l_post_filter_ldr (the default filter's geometry, " + "40 columns x 4 row pairs, 3 x f32 RGB; RGBA16F rows for the output copy); the selected filter (") + for env, opts, tail in (({"WG_FILTER_KERNEL": "post_filter"}, "model=L", "post_filter) has no LDR form"), + ({"WG_FILTER_KERNEL": "post_filter_c1_s16"}, "model=L", + "post_filter_c1_s16) has no LDR form"), + ({"WG_FILTER_KERNEL": "post_filter_c1_s4_o16"}, "model=L", + "post_filter_c1_s4_o16, RGBA16F output) has no LDR form"), + ({}, "model=L;output_filter=quality5", + "post_filter_c1_s4, output_filter=quality5) has no LDR form")): + with self.subTest(env=env, options=opts): + self.assertEqual(first(opts, hdr=0, env=env), why + tail) + hdr_got = first(opts, hdr=1, env=env) # HDR: Model M's output stage takes the filter kernels + if env: + self.assertEqual(hdr_got, "create ok model L") + else: # (the output_filter options need Model M's own post paths: refused for Model L before) + self.assertTrue(hdr_got.startswith("ERR create: output_filter="), hdr_got) + self.assertTrue(first("model=L;output_filter=quality9", hdr=0).startswith("ERR create: ")) # refused before + got = first("model=L;output_filter=quality9", hdr=0, flags=("--backend=3",), manifest=self.own_ports()) + self.assertTrue(got.startswith("ERR create: Model L"), got) + # the default: l_post_filter_ldr on the default filter's grid (O 120 x 72: 3 x 5 workgroups of 64 lanes) + for backend in ("hip", "d3d12"): + with self.subTest(default=backend): + kw = dict(flags=("--backend=3",), manifest=self.own_ports()) if backend == "d3d12" else {} + t = FT.Trace(run("model=L", frames=2, build_files=FT.SHIPPING, hdr=0, **kw)) + self.assertEqual(t.create, "create ok model L") + for f in t.frames: + flt = [ln for ln in f["launches"] if ln.kernel == "l_post_filter_ldr"] + self.assertEqual(len(flt), 1) + self.assertEqual((flt[0].grid, flt[0].block), (cdiv(120, 40) * cdiv(cdiv(72, 2), 8), 64)) + self.assertEqual(flt[0].args[:2], [f["names"]["post.display"], f["names"]["filter.out"]]) + self.assertEqual(flt[0].args[2:6], [str((3 * 120 + 1) // 2), str((3 * 72 + 1) // 2), "120", "72"]) + class Refusals(unittest.TestCase): def test_paths(self): @@ -731,9 +813,8 @@ def test_paths(self): # Model L's reconstruction is its own kernel: Model M's post kernels are not chosen for it self.assertTrue(run("model=L", env={"WG_POST_KERNEL": "post_recon2_r"})[0].startswith( "ERR create: Model L: its reconstruction is l_post_recon")) - # LDR colour: Model L's LDR reconstruction kernel is not written yet (refused, never Model M's post instead) - self.assertTrue(run("model=L", hdr=0)[0].startswith( - "ERR create: Model L: LDR colour (WgRtConfig.hdr = 0) needs Model L's LDR reconstruction kernel")) + # LDR colour: Model L's LDR reconstruction (test_no_model_m_post_route traces it) + self.assertEqual(run("model=L", hdr=0)[0], "create ok model L") def test_model_data(self): d = tempfile.mkdtemp(prefix="wg_lf_") diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index 9517b6f2..df14987c 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -240,17 +240,18 @@ struct Runtime { // Model L (runtime option model=L, builds with WG_MODEL_L): Model M's frame with Model L's encoder in step 2 and its // decoder in step 4 (clean/l/FORMATS.md); the ten Swin stages, exposure and post are Model M's. bool model_l = false; - // runtime option l_post=own (native D3D12): Model L's own reconstruction l_post_recon_p12 (l_post.co's D3D12 port), - // then the unbanded analytic filter w64/post_filter_c1_s4, in place of Model M's fused pack12 post (l_post=m). On the - // direct-resource route the colour is staged into color_net (l_direct_color) and the filter writes the game's texture - // itself (l_post_filter_direct, dx12/kernels/post_filter.hlsl). - // Default (no l_post option): own wherever its route applies (HDR colour, the default pack12 post route, its D3D12 - // ports present: l_own_post_fits), else m. Why: Model M's post is another algorithm (3 x 3 gather, encoded-domain - // history); against Model L's post oracle it fails every frame (about 76 % of the full history's FP16 values more - // than one ulp off), Model L's own post passes Model L's post acceptance on every frame (real weights: - // clean/l/runner/l_default_post_check.py). LDR keeps m: l_post_recon's LDR form is not written yet. + // Model L reconstructs with its own kernel only, on every backend and route: l_post_recon (HDR) or l_post_recon_ldr + // (LDR colour, WgRtConfig.hdr = 0; spec preset L part 3 item 6), on native D3D12 their pack12 forms + // l_post_recon_p12[_ldr] on the frames l_coef_pack12 stores, then the output filter (HDR: the unbanded analytic + // w64/post_filter_c1_s4; LDR: l_post_filter_ldr, no decode and no exposure). On the direct-resource route the colour + // is staged into color_net (l_direct_color) and the filter's direct form writes the game's texture + // (l_post_filter_direct[_ldr], dx12/kernels/post_filter.hlsl). Model L has no Model M post route: where its own + // route cannot run (l_own_post_fits) the runtime refuses with the reason, never a different algorithm. Model M's + // post is another algorithm (3 x 3 gather, encoded-domain history): against Model L's post oracle it failed every + // frame (clean/l/records/l_default_post_real_weights.json). l_own_post: native D3D12 with Model L (its pack12 route). bool l_own_post = false; - int l_post_choice = 0; // the l_post option: 0 not given (the default above), 1 m, 2 own + bool l_ldr = false; // Model L with LDR colour: the _ldr kernels + bool l_post_own_given = false; // the runtime option l_post=own (accepted: Model L's only route) #ifdef WG_MODEL_L // The maps of clean/l/FORMATS.md in three buffers (no kernel works in place): // l_wide [fh + 4][fw + 4] x 64 bytes: q (FP8 halo, 32 bytes a pixel), then x2 (FP16 halo; q is dead after the @@ -580,17 +581,13 @@ struct Runtime { if (c.wpw128) wpw[2] = c.wpw128; if (!choose_kernels(c)) return false; if (model_l && recon_output) return fail("Model L: recon=output (D = O) is Model M's experimental mode"); - // Model L's reconstruction kernel l_post_recon is HDR; its LDR form (11/11/10-bit display codes) is not written yet. - // (Native D3D12 runs Model M's pack12 post for Model L, which takes both.) - if (model_l && !c.hdr && c.backend != 3) - return fail("Model L: LDR colour (WgRtConfig.hdr = 0) needs Model L's LDR reconstruction kernel, which is not " - "written yet (l_post_recon is HDR)"); + // (Model L's reconstruction has HDR and LDR forms: l_post_recon / l_post_recon_ldr; the frame chooses by hdr.) if (model_l && enc0_co != "encoder.co") return fail("Model L: enc0 variants are Model M's encoder; Model L's runs from l_enc_values.co"); // Model L's coefficient kernel (l_coef) writes Model M's coef_decoder formats, not pack12: the _p12 post kernels // are not for it (choose_kernels does not choose pack12 by default for Model L; asked for, it is refused here) - // On native D3D12 Model L stores its coefficients in pack12 (l_coef_pack12, lossless) for Model M's pack12 post - // path, the only post path ported there; on HIP it writes the base format (l_coef) for the post_recon path. + // On native D3D12 Model L stores its coefficients in pack12 (l_coef_pack12, lossless) for its + // l_post_recon_p12[_ldr]; on HIP it writes the base format (l_coef) for l_post_recon[_ldr]. if (model_l && p12 && c.backend != 3) return fail("Model L: post_pack=p12 (the pack12 decoder) is Model M's; Model L's l_coef writes the base " "coefficient format (pack12 is Model L's native D3D12 form only)"); @@ -713,11 +710,16 @@ struct Runtime { if (motion_outres == 1) return fail("Model L: motion_outres=original needs Model M's encoder (encoder_mo.co)"); d1d0_p12_opt = false; #ifdef WG_MODEL_L - if (c.backend == 3 && l_post_choice == 0) l_own_post = l_own_post_fits(c); - if (l_own_post && c.backend == 3 && !l_own_post_setup(c)) return false; + l_ldr = !c.hdr; + if (l_ldr) { // only the output stage the LDR kernels implement: any other filter selection is refused + const std::string why = l_ldr_filter_unfit(c); + if (!why.empty()) return fail("Model L, LDR colour: " + why); + } + l_own_post = c.backend == 3; + if (l_own_post && !l_own_post_setup(c)) return false; #endif } - if (l_post_choice == 2 && !model_l) return fail("l_post=own is a Model L option (its own reconstruction)"); + if (l_post_own_given && !model_l) return fail("l_post=own is a Model L option (its own reconstruction)"); if (!mid_setup(c)) return false; if (!bneck_setup(c)) return false; // D = O (fast mode, recon=output) runs without pack12, so dec1 + dec0 fused is the default there (bit-identical; @@ -924,11 +926,10 @@ struct Runtime { if (!post_given && !filter_unbanded && !recon_output) post_kernel = kDefaultBandPost; } #ifdef WG_MODEL_L - // Model L reconstructs with its own kernel, l_post_recon (l_post.co; clean/l/FORMATS.md step 5), which takes Model - // M's post_recon arguments and buffers: the unbanded post_recon route (8-byte history, base coefficient format, - // no pack12), then Model M's output filter (unbanded). Model M's post kernels are not for Model L. On native D3D12 - // Model L's frames run Model M's pack12 post path for now (l_coef_pack12 feeds it); frames that path does not take - // run l_post_recon there too (its D3D12 port, then the analytic filter). + // Model L reconstructs with its own kernel, l_post_recon[_ldr] (l_post.co; clean/l/FORMATS.md step 5), which takes + // Model M's post_recon arguments and buffers: the unbanded post_recon route (8-byte history, base coefficient + // format, no pack12), then the output filter (unbanded; LDR: l_post_filter_ldr). Model M's post kernels are not for + // Model L. On native D3D12 its pack12 forms run (l_own_post_setup); there is no Model M post route for Model L. if (model_l && c.backend != 3) { if ((c.post_kernel && *c.post_kernel) || choose(nullptr, "WG_POST_KERNEL", "") != "") return fail("Model L: its reconstruction is l_post_recon; a post kernel (post_kernel / WG_POST_KERNEL) " @@ -1111,10 +1112,11 @@ struct Runtime { if (v != "auto" && v != "force") return fail("post_tf4 must be auto or force"); post_tf4_force = v == "force"; } - if (k == "l_post") { - if (v != "m" && v != "own") return fail("l_post must be m or own (Model L on native D3D12)"); - l_own_post = v == "own"; - l_post_choice = l_own_post ? 2 : 1; + if (k == "l_post") { // Model L runs its own reconstruction only (no Model M post route): l_post=own is a no-op + if (v == "m") + return fail("l_post=m: Model L has no Model M post route (its reconstruction is l_post_recon[_ldr] only)"); + if (v != "own") return fail("l_post must be own (Model L's reconstruction; there is no other)"); + l_post_own_given = true; } if (k == "post_diet") post_diet = !(v == "0" || v == "off" || v == "false"); if (k == "tf_vdiet") tf_vdiet = !(v == "0" || v == "off" || v == "false"); @@ -1421,10 +1423,10 @@ struct Runtime { kernels_text += std::string(" model_map=") + (model_source == "raw" ? "none" : model_other_map ? "other" : "same"); if (native_direct_io) kernels_text += " native_io=direct_textures"; // Model L: its encoder and decoder (enc0= / dec0= above name Model M's, not loaded for Model L) - // (l_post: Model L's own reconstruction, or on native D3D12 by default Model M's fused pack12 post) + // (l_post: Model L's own reconstruction, HDR or LDR, on every backend) if (model_l) kernels_text += std::string(" model=L l_kernels=l_enc_values.co,l_ops.co,l_swin.co,l_post.co l_post=") + - (d3d12_native && !l_own_post ? "m_pack12" : "l_post_recon"); + (l_ldr ? "l_post_recon_ldr" : "l_post_recon"); if (l_fold_built) kernels_text += std::string(" l_dec_fold=") + (fz_lc ? "on" : "off"); // (native D3D12 only) if (l_lin_built) kernels_text += std::string(" l_lin_fused=") + (fz_ll ? "on" : "off"); return kernels_text.c_str(); @@ -1647,35 +1649,56 @@ struct Runtime { return missing; } - // l_post=own on native D3D12 (after runtime_options): the unbanded pack12 route with Model L's reconstruction in - // place of Model M's fused post. p12 stays (l_coef_pack12 feeds l_post_recon_p12; frames the pack12 kernels do not - // take run l_coef and l_post_recon); post_recon_stage launches Model L's kernel, step 6 the unbanded default filter - // (w64/post_filter_c1_s4, the analytic build: Model L's file has no output-filter weights). - // The default's test: l_own_post_setup's conditions, without failing - bool l_own_post_fits(const WgRtConfig& c) { - if (!c.hdr || !p12 || post_kernel != "post_recon2_r_band_p12" || filter_kernel != "post_filter_c1_s4_band" || out_f16) - return false; - std::vector need = {"l_post_recon_p12", "l_post_recon", "l_coef", "w64/post_filter_c1_s4"}; + // Model L with LDR colour: its output stage is l_post_filter_ldr (the default filter's geometry, post_filter_c1_s4: + // 40 output columns and 4 row pairs a lane group, 3 x f32 RGB), on native D3D12 also its direct form, and for the + // HIP output copy (RGBA16F rows) l_post_filter_ldr_h on the same geometry. Any other filter selection (another + // filter kernel or geometry, an RGBA16F filter output, output_filter=quality5 / quality9 / original) has no LDR form + // and is refused at create, with the reason, on every backend and for the output copy too (never ignored, never run + // with another filter's grid or output format). "" = the selection is the one the LDR kernels implement. + std::string l_ldr_filter_unfit(const WgRtConfig& c) const { + const bool default_filter = + c.backend == 3 ? filter_kernel == "post_filter_c1_s4_band" || filter_kernel == kDefaultFilter + : filter_kernel == kDefaultFilter; + if (default_filter && !out_f16 && !output_quality5 && !output_quality9 && output_filter_opt != "original") + return ""; + return "its output stage is l_post_filter_ldr (the default filter's geometry, 40 columns x 4 row pairs, 3 x f32 " + "RGB; RGBA16F rows for the output copy); the selected filter (" + filter_kernel + + (out_f16 ? ", RGBA16F output" : "") + + (output_filter_opt.empty() ? std::string() : ", output_filter=" + output_filter_opt) + + ") has no LDR form"; + } + + // Model L on native D3D12 (after runtime_options): the unbanded pack12 route with Model L's reconstruction. p12 stays + // (l_coef_pack12 feeds l_post_recon_p12[_ldr]; frames the pack12 kernels do not take run l_coef and + // l_post_recon[_ldr]); post_recon_stage launches Model L's kernel, step 6 the unbanded filter (HDR: + // w64/post_filter_c1_s4, the analytic build, as Model L's file has no output-filter weights; LDR: l_post_filter_ldr). + // The kernels Model L's route needs on native D3D12 (HDR or LDR colour; the direct route's two more). + std::vector l_own_post_kernels(bool ldr) const { + std::vector need = ldr ? std::vector{"l_post_recon_p12_ldr", "l_post_recon_ldr", "l_coef", + "l_post_filter_ldr"} + : std::vector{"l_post_recon_p12", "l_post_recon", "l_coef", + "w64/post_filter_c1_s4"}; if (native_direct_io) { need.push_back("l_direct_color"); - need.push_back("l_post_filter_direct"); + need.push_back(ldr ? "l_post_filter_direct_ldr" : "l_post_filter_direct"); } - return manifest_missing(need).empty(); + return need; } - bool l_own_post_setup(const WgRtConfig& c) { - if (!c.hdr) - return fail("Model L: l_post=own needs HDR colour (WgRtConfig.hdr = 1): l_post_recon is HDR, its LDR form is " - "not written yet"); + // Whether Model L's own route runs here, HDR and LDR alike: the default pack12 post route (not one of Model M's other + // post / filter routes, no RGBA16F filter output) and its D3D12 ports present. "" = it runs; otherwise why not. + std::string l_own_post_fits(const WgRtConfig& c) const { if (!p12 || post_kernel != "post_recon2_r_band_p12" || filter_kernel != "post_filter_c1_s4_band" || out_f16) - return fail("Model L: l_post=own replaces the default pack12 post route (post " + post_kernel + ", filter " + - filter_kernel + ")"); - std::vector need = {"l_post_recon_p12", "l_post_recon", "l_coef", "w64/post_filter_c1_s4"}; - if (native_direct_io) { - need.push_back("l_direct_color"); - need.push_back("l_post_filter_direct"); - } - const std::string missing = manifest_missing(need); - if (!missing.empty()) return fail("Model L on native D3D12 (l_post=own): not yet ported to D3D12: " + missing); + return "Model L's reconstruction runs on the default pack12 post route only (asked: post " + post_kernel + + ", filter " + filter_kernel + (out_f16 ? ", RGBA16F output" : "") + ")"; + const std::string missing = manifest_missing(l_own_post_kernels(!c.hdr)); + if (!missing.empty()) return "not yet ported to D3D12: " + missing; + return ""; + } + bool l_own_post_setup(const WgRtConfig& c) { + const std::string why = l_own_post_fits(c); + if (!why.empty()) // no other post route for Model L: refused, with the reason + return fail(std::string("Model L on native D3D12 (") + (c.hdr ? "HDR" : "LDR") + "): " + why + + "; Model L has no other reconstruction (no Model M post route)"); post_kernel = "post_recon2_r_p12"; // unbanded pack12 route; the reconstruction itself is Model L's (post_recon_stage) post_band = false; filter_kernel = kDefaultFilter; @@ -1848,8 +1871,9 @@ struct Runtime { "-byte history) cannot fall back to post_recon here"); #ifdef WG_MODEL_L if (model_l) { // Model L: l_post_recon (l_post.co, untagged) with post_recon's arguments - // (native D3D12, l_post=own: l_post_recon_p12 on the frames whose coefficients l_coef_pack12 stored) - post_last = p12_frame ? "l_post_recon_p12" : "l_post_recon"; + // (native D3D12: l_post_recon_p12 on the frames whose coefficients l_coef_pack12 stored; LDR colour: the _ldr + // form) + post_last = std::string(p12_frame ? "l_post_recon_p12" : "l_post_recon") + (l_ldr ? "_ldr" : ""); return launch(post_last.c_str(), grid, 256, {&cnet, &A, &B, &C, &hist[cur], &mnear[nxt], &mcenter, &mfar, &w, &hist[nxt], &reduced, &display, &p, &expo}); @@ -2447,6 +2471,13 @@ struct Runtime { {&display, &outp, &c, &d, &dec_scale, &expo, &p_h, &hpitch})) return false; filter_last = "post_copy_h"; + } else if (hp > 0 && model_l && l_ldr) { // Model L, LDR colour: its LDR output stage, RGBA16F rows, pi folded in + int tx = 0, fs = 0; + filter_shape(kDefaultFilter, &tx, &fs); + if (!launch("l_post_filter_ldr_h", cdiv(ow, tx) * cdiv(cdiv(oh, 2), 2 * fs), 64, + {&display, &outp, &a, &b, &c, &d, &dec_scale, &expo, &p_h, &hpitch})) + return false; + filter_last = "l_post_filter_ldr_h"; } else if (hp > 0) { // interop output copy: RGBA16F rows. With the default filter, pi = 1 and dense rows (pitch 8 ow): // post_filter_c1_s4_o16 (the FP16 rounding of post_filter_c1_s4); otherwise post_filter_h (pi folded in, @@ -2485,10 +2516,15 @@ struct Runtime { // (post_filter_quality9 is its own filter and never runs with output_filter=original) // l_post=own on the direct route: the filter's direct form writes the game's output texture (times pi as // scale_rgb, the Converter's typed store or the output-copy bytes); out_p is not written - const std::string fname = own_direct ? std::string("l_post_filter_direct") : post_pre + fk; - if (!launch(fname.c_str(), grid, 64, with_taps({&display, &outp, &a, &b, &c, &d, &dec_scale, &expo}))) + // (Model L with LDR colour: its LDR output stage l_post_filter_ldr[_direct], no decode, no exposure, Q2) + const std::string fname = own_direct ? std::string(l_ldr ? "l_post_filter_direct_ldr" : "l_post_filter_direct") + : model_l && l_ldr ? std::string("l_post_filter_ldr") + : post_pre + fk; + if (!launch(fname.c_str(), grid, 64, + model_l && l_ldr ? std::vector{&display, &outp, &a, &b, &c, &d, &dec_scale, &expo} + : with_taps({&display, &outp, &a, &b, &c, &d, &dec_scale, &expo}))) return false; - filter_last = own_direct ? "l_post_filter_direct" : fk; + filter_last = own_direct || (model_l && l_ldr) ? fname : fk; } if (!own_direct && hp <= 0 && pi != 1.0f) { // spec 10 draft 2: the filter's result times pi int n3 = 3 * ow * oh; diff --git a/clean/tests/runtime_frame_trace.cpp b/clean/tests/runtime_frame_trace.cpp index 5ab70621..f3fefa39 100644 --- a/clean/tests/runtime_frame_trace.cpp +++ b/clean/tests/runtime_frame_trace.cpp @@ -86,6 +86,10 @@ const Signature kSignatures[] = { {"l_coef_pack12", "ppppppD"}, // the same arguments (B unused) {"l_post_recon_p12", "ppppppppppppPp"}, // the same arguments (A, C: pack12 images; B unused) {"post_filter_c1_s4", "ppiiiiip"}, // display, out, dw, dh, ow, oh, dec_scale (its bits), exposure pair + {"l_post_recon_ldr", "ppppppppppppPp"}, // the LDR forms: the same arguments (the exposure pair not read) + {"l_post_recon_p12_ldr", "ppppppppppppPp"}, + {"l_post_filter_ldr", "ppiiiiip"}, // post_filter_c1_s4's arguments (dec_scale, exposure pair not read) + {"l_post_filter_ldr_h", "ppiiiiipii"}, // plus pi (its bits) and the row pitch in bytes }; const char* kinds_of(const std::string& kernel) {